1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3 #include <crypto/akcipher.h>
4 #include <crypto/curve25519.h>
6 #include <crypto/ecc_curve.h>
7 #include <crypto/ecdh.h>
8 #include <crypto/rng.h>
9 #include <crypto/internal/akcipher.h>
10 #include <crypto/internal/kpp.h>
11 #include <crypto/internal/rsa.h>
12 #include <crypto/kpp.h>
13 #include <crypto/scatterwalk.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/fips.h>
16 #include <linux/module.h>
17 #include <linux/time.h>
22 #define HPRE_CRYPTO_ALG_PRI 1000
23 #define HPRE_ALIGN_SZ 64
24 #define HPRE_BITS_2_BYTES_SHIFT 3
25 #define HPRE_RSA_512BITS_KSZ 64
26 #define HPRE_RSA_1536BITS_KSZ 192
27 #define HPRE_CRT_PRMS 5
30 #define HPRE_CRT_INV 4
31 #define HPRE_DH_G_FLAG 0x02
32 #define HPRE_TRY_SEND_TIMES 100
33 #define HPRE_INVLD_REQ_ID (-1)
35 #define HPRE_SQE_ALG_BITS 5
36 #define HPRE_SQE_DONE_SHIFT 30
37 #define HPRE_DH_MAX_P_SZ 512
39 #define HPRE_DFX_SEC_TO_US 1000000
40 #define HPRE_DFX_US_TO_NS 1000
42 /* due to nist p521 */
43 #define HPRE_ECC_MAX_KSZ 66
45 /* size in bytes of the n prime */
46 #define HPRE_ECC_NIST_P192_N_SIZE 24
47 #define HPRE_ECC_NIST_P256_N_SIZE 32
48 #define HPRE_ECC_NIST_P384_N_SIZE 48
51 #define HPRE_ECC_HW256_KSZ_B 32
52 #define HPRE_ECC_HW384_KSZ_B 48
54 /* capability register mask of driver */
55 #define HPRE_DRV_RSA_MASK_CAP BIT(0)
56 #define HPRE_DRV_DH_MASK_CAP BIT(1)
57 #define HPRE_DRV_ECDH_MASK_CAP BIT(2)
58 #define HPRE_DRV_X25519_MASK_CAP BIT(5)
60 static DEFINE_MUTEX(hpre_algs_lock);
61 static unsigned int hpre_available_devs;
63 typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe);
66 /* low address: e--->n */
68 dma_addr_t dma_pubkey;
70 /* low address: d--->n */
72 dma_addr_t dma_prikey;
74 /* low address: dq->dp->q->p->qinv */
76 dma_addr_t dma_crt_prikey;
78 struct crypto_akcipher *soft_tfm;
83 * If base is g we compute the public key
84 * ya = g^xa mod p; [RFC2631 sec 2.1.1]
85 * else if base if the counterpart public key we
86 * compute the shared secret
87 * ZZ = yb^xa mod p; [RFC2631 sec 2.1.1]
88 * low address: d--->n, please refer to Hisilicon HPRE UM
97 struct hpre_ecdh_ctx {
98 /* low address: p->a->k->b */
102 /* low address: x->y */
107 struct hpre_curve25519_ctx {
108 /* low address: p->a->k */
120 struct hpre_asym_request **req_list;
127 struct hpre_rsa_ctx rsa;
128 struct hpre_dh_ctx dh;
129 struct hpre_ecdh_ctx ecdh;
130 struct hpre_curve25519_ctx curve25519;
132 /* for ecc algorithms */
133 unsigned int curve_id;
136 struct hpre_asym_request {
140 struct hpre_ctx *ctx;
142 struct akcipher_request *rsa;
143 struct kpp_request *dh;
144 struct kpp_request *ecdh;
145 struct kpp_request *curve25519;
150 struct timespec64 req_time;
153 static inline unsigned int hpre_align_sz(void)
155 return ((crypto_dma_align() - 1) | (HPRE_ALIGN_SZ - 1)) + 1;
158 static inline unsigned int hpre_align_pd(void)
160 return (hpre_align_sz() - 1) & ~(crypto_tfm_ctx_alignment() - 1);
163 static int hpre_alloc_req_id(struct hpre_ctx *ctx)
168 spin_lock_irqsave(&ctx->req_lock, flags);
169 id = idr_alloc(&ctx->req_idr, NULL, 0, ctx->qp->sq_depth, GFP_ATOMIC);
170 spin_unlock_irqrestore(&ctx->req_lock, flags);
175 static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id)
179 spin_lock_irqsave(&ctx->req_lock, flags);
180 idr_remove(&ctx->req_idr, req_id);
181 spin_unlock_irqrestore(&ctx->req_lock, flags);
184 static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req)
186 struct hpre_ctx *ctx;
187 struct hpre_dfx *dfx;
191 id = hpre_alloc_req_id(ctx);
192 if (unlikely(id < 0))
195 ctx->req_list[id] = hpre_req;
196 hpre_req->req_id = id;
198 dfx = ctx->hpre->debug.dfx;
199 if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value))
200 ktime_get_ts64(&hpre_req->req_time);
205 static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req)
207 struct hpre_ctx *ctx = hpre_req->ctx;
208 int id = hpre_req->req_id;
210 if (hpre_req->req_id >= 0) {
211 hpre_req->req_id = HPRE_INVLD_REQ_ID;
212 ctx->req_list[id] = NULL;
213 hpre_free_req_id(ctx, id);
217 static struct hisi_qp *hpre_get_qp_and_start(u8 type)
222 qp = hpre_create_qp(type);
224 pr_err("Can not create hpre qp!\n");
225 return ERR_PTR(-ENODEV);
228 ret = hisi_qm_start_qp(qp, 0);
230 hisi_qm_free_qps(&qp, 1);
231 pci_err(qp->qm->pdev, "Can not start qp!\n");
232 return ERR_PTR(-EINVAL);
238 static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req,
239 struct scatterlist *data, unsigned int len,
240 int is_src, dma_addr_t *tmp)
242 struct device *dev = hpre_req->ctx->dev;
243 enum dma_data_direction dma_dir;
246 hpre_req->src = NULL;
247 dma_dir = DMA_TO_DEVICE;
249 hpre_req->dst = NULL;
250 dma_dir = DMA_FROM_DEVICE;
252 *tmp = dma_map_single(dev, sg_virt(data), len, dma_dir);
253 if (unlikely(dma_mapping_error(dev, *tmp))) {
254 dev_err(dev, "dma map data err!\n");
261 static int hpre_prepare_dma_buf(struct hpre_asym_request *hpre_req,
262 struct scatterlist *data, unsigned int len,
263 int is_src, dma_addr_t *tmp)
265 struct hpre_ctx *ctx = hpre_req->ctx;
266 struct device *dev = ctx->dev;
270 shift = ctx->key_sz - len;
271 if (unlikely(shift < 0))
274 ptr = dma_alloc_coherent(dev, ctx->key_sz, tmp, GFP_ATOMIC);
279 scatterwalk_map_and_copy(ptr + shift, data, 0, len, 0);
288 static int hpre_hw_data_init(struct hpre_asym_request *hpre_req,
289 struct scatterlist *data, unsigned int len,
290 int is_src, int is_dh)
292 struct hpre_sqe *msg = &hpre_req->req;
293 struct hpre_ctx *ctx = hpre_req->ctx;
297 /* when the data is dh's source, we should format it */
298 if ((sg_is_last(data) && len == ctx->key_sz) &&
299 ((is_dh && !is_src) || !is_dh))
300 ret = hpre_get_data_dma_addr(hpre_req, data, len, is_src, &tmp);
302 ret = hpre_prepare_dma_buf(hpre_req, data, len, is_src, &tmp);
308 msg->in = cpu_to_le64(tmp);
310 msg->out = cpu_to_le64(tmp);
315 static void hpre_hw_data_clr_all(struct hpre_ctx *ctx,
316 struct hpre_asym_request *req,
317 struct scatterlist *dst,
318 struct scatterlist *src)
320 struct device *dev = ctx->dev;
321 struct hpre_sqe *sqe = &req->req;
324 tmp = le64_to_cpu(sqe->in);
325 if (unlikely(dma_mapping_error(dev, tmp)))
330 dma_free_coherent(dev, ctx->key_sz, req->src, tmp);
332 dma_unmap_single(dev, tmp, ctx->key_sz, DMA_TO_DEVICE);
335 tmp = le64_to_cpu(sqe->out);
336 if (unlikely(dma_mapping_error(dev, tmp)))
341 scatterwalk_map_and_copy(req->dst, dst, 0,
343 dma_free_coherent(dev, ctx->key_sz, req->dst, tmp);
345 dma_unmap_single(dev, tmp, ctx->key_sz, DMA_FROM_DEVICE);
349 static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe,
352 struct hpre_asym_request *req;
353 unsigned int err, done, alg;
356 #define HPRE_NO_HW_ERR 0
357 #define HPRE_HW_TASK_DONE 3
358 #define HREE_HW_ERR_MASK GENMASK(10, 0)
359 #define HREE_SQE_DONE_MASK GENMASK(1, 0)
360 #define HREE_ALG_TYPE_MASK GENMASK(4, 0)
361 id = (int)le16_to_cpu(sqe->tag);
362 req = ctx->req_list[id];
363 hpre_rm_req_from_ctx(req);
366 err = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_ALG_BITS) &
369 done = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_DONE_SHIFT) &
372 if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE))
375 alg = le32_to_cpu(sqe->dw0) & HREE_ALG_TYPE_MASK;
376 dev_err_ratelimited(ctx->dev, "alg[0x%x] error: done[0x%x], etype[0x%x]\n",
382 static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen)
386 if (!ctx || !qp || qlen < 0)
389 spin_lock_init(&ctx->req_lock);
391 ctx->dev = &qp->qm->pdev->dev;
393 hpre = container_of(ctx->qp->qm, struct hpre, qm);
395 ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL);
399 ctx->crt_g2_mode = false;
400 idr_init(&ctx->req_idr);
405 static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all)
408 idr_destroy(&ctx->req_idr);
409 kfree(ctx->req_list);
410 hisi_qm_free_qps(&ctx->qp, 1);
413 ctx->crt_g2_mode = false;
417 static bool hpre_is_bd_timeout(struct hpre_asym_request *req,
420 struct timespec64 reply_time;
423 ktime_get_ts64(&reply_time);
424 time_use_us = (reply_time.tv_sec - req->req_time.tv_sec) *
426 (reply_time.tv_nsec - req->req_time.tv_nsec) /
429 if (time_use_us <= overtime_thrhld)
435 static void hpre_dh_cb(struct hpre_ctx *ctx, void *resp)
437 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
438 struct hpre_asym_request *req;
439 struct kpp_request *areq;
443 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
445 areq->dst_len = ctx->key_sz;
447 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
448 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
449 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
451 hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);
452 kpp_request_complete(areq, ret);
453 atomic64_inc(&dfx[HPRE_RECV_CNT].value);
456 static void hpre_rsa_cb(struct hpre_ctx *ctx, void *resp)
458 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
459 struct hpre_asym_request *req;
460 struct akcipher_request *areq;
464 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
466 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
467 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
468 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
470 areq = req->areq.rsa;
471 areq->dst_len = ctx->key_sz;
472 hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);
473 akcipher_request_complete(areq, ret);
474 atomic64_inc(&dfx[HPRE_RECV_CNT].value);
477 static void hpre_alg_cb(struct hisi_qp *qp, void *resp)
479 struct hpre_ctx *ctx = qp->qp_ctx;
480 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
481 struct hpre_sqe *sqe = resp;
482 struct hpre_asym_request *req = ctx->req_list[le16_to_cpu(sqe->tag)];
484 if (unlikely(!req)) {
485 atomic64_inc(&dfx[HPRE_INVALID_REQ_CNT].value);
492 static void hpre_stop_qp_and_put(struct hisi_qp *qp)
495 hisi_qm_free_qps(&qp, 1);
498 static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type)
503 qp = hpre_get_qp_and_start(type);
508 qp->req_cb = hpre_alg_cb;
510 ret = hpre_ctx_set(ctx, qp, qp->sq_depth);
512 hpre_stop_qp_and_put(qp);
517 static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa)
519 struct hpre_asym_request *h_req;
520 struct hpre_sqe *msg;
525 struct akcipher_request *akreq = req;
527 if (akreq->dst_len < ctx->key_sz) {
528 akreq->dst_len = ctx->key_sz;
532 tmp = akcipher_request_ctx(akreq);
533 h_req = PTR_ALIGN(tmp, hpre_align_sz());
534 h_req->cb = hpre_rsa_cb;
535 h_req->areq.rsa = akreq;
537 memset(msg, 0, sizeof(*msg));
539 struct kpp_request *kreq = req;
541 if (kreq->dst_len < ctx->key_sz) {
542 kreq->dst_len = ctx->key_sz;
546 tmp = kpp_request_ctx(kreq);
547 h_req = PTR_ALIGN(tmp, hpre_align_sz());
548 h_req->cb = hpre_dh_cb;
549 h_req->areq.dh = kreq;
551 memset(msg, 0, sizeof(*msg));
552 msg->key = cpu_to_le64(ctx->dh.dma_xa_p);
555 msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
556 msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
557 msg->dw0 |= cpu_to_le32(0x1 << HPRE_SQE_DONE_SHIFT);
558 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
561 req_id = hpre_add_req_to_ctx(h_req);
565 msg->tag = cpu_to_le16((u16)req_id);
570 static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg)
572 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
577 atomic64_inc(&dfx[HPRE_SEND_CNT].value);
578 ret = hisi_qp_send(ctx->qp, msg);
581 atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value);
582 } while (ctr++ < HPRE_TRY_SEND_TIMES);
588 atomic64_inc(&dfx[HPRE_SEND_FAIL_CNT].value);
593 static int hpre_dh_compute_value(struct kpp_request *req)
595 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
596 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
597 void *tmp = kpp_request_ctx(req);
598 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
599 struct hpre_sqe *msg = &hpre_req->req;
602 ret = hpre_msg_request_set(ctx, req, false);
607 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 1);
611 msg->in = cpu_to_le64(ctx->dh.dma_g);
614 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 1);
618 if (ctx->crt_g2_mode && !req->src)
619 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH_G2);
621 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH);
624 ret = hpre_send(ctx, msg);
629 hpre_rm_req_from_ctx(hpre_req);
630 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
635 static int hpre_is_dh_params_length_valid(unsigned int key_sz)
637 #define _HPRE_DH_GRP1 768
638 #define _HPRE_DH_GRP2 1024
639 #define _HPRE_DH_GRP5 1536
640 #define _HPRE_DH_GRP14 2048
641 #define _HPRE_DH_GRP15 3072
642 #define _HPRE_DH_GRP16 4096
656 static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params)
658 struct device *dev = ctx->dev;
661 if (params->p_size > HPRE_DH_MAX_P_SZ)
664 if (hpre_is_dh_params_length_valid(params->p_size <<
665 HPRE_BITS_2_BYTES_SHIFT))
668 sz = ctx->key_sz = params->p_size;
669 ctx->dh.xa_p = dma_alloc_coherent(dev, sz << 1,
670 &ctx->dh.dma_xa_p, GFP_KERNEL);
674 memcpy(ctx->dh.xa_p + sz, params->p, sz);
676 /* If g equals 2 don't copy it */
677 if (params->g_size == 1 && *(char *)params->g == HPRE_DH_G_FLAG) {
678 ctx->crt_g2_mode = true;
682 ctx->dh.g = dma_alloc_coherent(dev, sz, &ctx->dh.dma_g, GFP_KERNEL);
684 dma_free_coherent(dev, sz << 1, ctx->dh.xa_p,
690 memcpy(ctx->dh.g + (sz - params->g_size), params->g, params->g_size);
695 static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all)
697 struct device *dev = ctx->dev;
698 unsigned int sz = ctx->key_sz;
701 hisi_qm_stop_qp(ctx->qp);
704 dma_free_coherent(dev, sz, ctx->dh.g, ctx->dh.dma_g);
709 memzero_explicit(ctx->dh.xa_p, sz);
710 dma_free_coherent(dev, sz << 1, ctx->dh.xa_p,
715 hpre_ctx_clear(ctx, is_clear_all);
718 static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf,
721 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
725 if (crypto_dh_decode_key(buf, len, ¶ms) < 0)
728 /* Free old secret if any */
729 hpre_dh_clear_ctx(ctx, false);
731 ret = hpre_dh_set_params(ctx, ¶ms);
735 memcpy(ctx->dh.xa_p + (ctx->key_sz - params.key_size), params.key,
741 hpre_dh_clear_ctx(ctx, false);
745 static unsigned int hpre_dh_max_size(struct crypto_kpp *tfm)
747 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
752 static int hpre_dh_init_tfm(struct crypto_kpp *tfm)
754 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
756 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
758 return hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE);
761 static void hpre_dh_exit_tfm(struct crypto_kpp *tfm)
763 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
765 hpre_dh_clear_ctx(ctx, true);
768 static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len)
770 while (!**ptr && *len) {
776 static bool hpre_rsa_key_size_is_support(unsigned int len)
778 unsigned int bits = len << HPRE_BITS_2_BYTES_SHIFT;
780 #define _RSA_1024BITS_KEY_WDTH 1024
781 #define _RSA_2048BITS_KEY_WDTH 2048
782 #define _RSA_3072BITS_KEY_WDTH 3072
783 #define _RSA_4096BITS_KEY_WDTH 4096
786 case _RSA_1024BITS_KEY_WDTH:
787 case _RSA_2048BITS_KEY_WDTH:
788 case _RSA_3072BITS_KEY_WDTH:
789 case _RSA_4096BITS_KEY_WDTH:
796 static int hpre_rsa_enc(struct akcipher_request *req)
798 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
799 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
800 void *tmp = akcipher_request_ctx(req);
801 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
802 struct hpre_sqe *msg = &hpre_req->req;
805 /* For 512 and 1536 bits key size, use soft tfm instead */
806 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
807 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) {
808 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);
809 ret = crypto_akcipher_encrypt(req);
810 akcipher_request_set_tfm(req, tfm);
814 if (unlikely(!ctx->rsa.pubkey))
817 ret = hpre_msg_request_set(ctx, req, true);
821 msg->dw0 |= cpu_to_le32(HPRE_ALG_NC_NCRT);
822 msg->key = cpu_to_le64(ctx->rsa.dma_pubkey);
824 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0);
828 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0);
833 ret = hpre_send(ctx, msg);
838 hpre_rm_req_from_ctx(hpre_req);
839 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
844 static int hpre_rsa_dec(struct akcipher_request *req)
846 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
847 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
848 void *tmp = akcipher_request_ctx(req);
849 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
850 struct hpre_sqe *msg = &hpre_req->req;
853 /* For 512 and 1536 bits key size, use soft tfm instead */
854 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
855 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) {
856 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);
857 ret = crypto_akcipher_decrypt(req);
858 akcipher_request_set_tfm(req, tfm);
862 if (unlikely(!ctx->rsa.prikey))
865 ret = hpre_msg_request_set(ctx, req, true);
869 if (ctx->crt_g2_mode) {
870 msg->key = cpu_to_le64(ctx->rsa.dma_crt_prikey);
871 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) |
874 msg->key = cpu_to_le64(ctx->rsa.dma_prikey);
875 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) |
879 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0);
883 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0);
888 ret = hpre_send(ctx, msg);
893 hpre_rm_req_from_ctx(hpre_req);
894 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
899 static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value,
900 size_t vlen, bool private)
902 const char *ptr = value;
904 hpre_rsa_drop_leading_zeros(&ptr, &vlen);
908 /* if invalid key size provided, we use software tfm */
909 if (!hpre_rsa_key_size_is_support(ctx->key_sz))
912 ctx->rsa.pubkey = dma_alloc_coherent(ctx->dev, vlen << 1,
913 &ctx->rsa.dma_pubkey,
915 if (!ctx->rsa.pubkey)
919 ctx->rsa.prikey = dma_alloc_coherent(ctx->dev, vlen << 1,
920 &ctx->rsa.dma_prikey,
922 if (!ctx->rsa.prikey) {
923 dma_free_coherent(ctx->dev, vlen << 1,
925 ctx->rsa.dma_pubkey);
926 ctx->rsa.pubkey = NULL;
929 memcpy(ctx->rsa.prikey + vlen, ptr, vlen);
931 memcpy(ctx->rsa.pubkey + vlen, ptr, vlen);
933 /* Using hardware HPRE to do RSA */
937 static int hpre_rsa_set_e(struct hpre_ctx *ctx, const char *value,
940 const char *ptr = value;
942 hpre_rsa_drop_leading_zeros(&ptr, &vlen);
944 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz)
947 memcpy(ctx->rsa.pubkey + ctx->key_sz - vlen, ptr, vlen);
952 static int hpre_rsa_set_d(struct hpre_ctx *ctx, const char *value,
955 const char *ptr = value;
957 hpre_rsa_drop_leading_zeros(&ptr, &vlen);
959 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz)
962 memcpy(ctx->rsa.prikey + ctx->key_sz - vlen, ptr, vlen);
967 static int hpre_crt_para_get(char *para, size_t para_sz,
968 const char *raw, size_t raw_sz)
970 const char *ptr = raw;
973 hpre_rsa_drop_leading_zeros(&ptr, &len);
974 if (!len || len > para_sz)
977 memcpy(para + para_sz - len, ptr, len);
982 static int hpre_rsa_setkey_crt(struct hpre_ctx *ctx, struct rsa_key *rsa_key)
984 unsigned int hlf_ksz = ctx->key_sz >> 1;
985 struct device *dev = ctx->dev;
989 ctx->rsa.crt_prikey = dma_alloc_coherent(dev, hlf_ksz * HPRE_CRT_PRMS,
990 &ctx->rsa.dma_crt_prikey,
992 if (!ctx->rsa.crt_prikey)
995 ret = hpre_crt_para_get(ctx->rsa.crt_prikey, hlf_ksz,
996 rsa_key->dq, rsa_key->dq_sz);
1001 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
1002 rsa_key->dp, rsa_key->dp_sz);
1006 offset = hlf_ksz * HPRE_CRT_Q;
1007 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
1008 rsa_key->q, rsa_key->q_sz);
1012 offset = hlf_ksz * HPRE_CRT_P;
1013 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
1014 rsa_key->p, rsa_key->p_sz);
1018 offset = hlf_ksz * HPRE_CRT_INV;
1019 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
1020 rsa_key->qinv, rsa_key->qinv_sz);
1024 ctx->crt_g2_mode = true;
1029 offset = hlf_ksz * HPRE_CRT_PRMS;
1030 memzero_explicit(ctx->rsa.crt_prikey, offset);
1031 dma_free_coherent(dev, hlf_ksz * HPRE_CRT_PRMS, ctx->rsa.crt_prikey,
1032 ctx->rsa.dma_crt_prikey);
1033 ctx->rsa.crt_prikey = NULL;
1034 ctx->crt_g2_mode = false;
1039 /* If it is clear all, all the resources of the QP will be cleaned. */
1040 static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all)
1042 unsigned int half_key_sz = ctx->key_sz >> 1;
1043 struct device *dev = ctx->dev;
1046 hisi_qm_stop_qp(ctx->qp);
1048 if (ctx->rsa.pubkey) {
1049 dma_free_coherent(dev, ctx->key_sz << 1,
1050 ctx->rsa.pubkey, ctx->rsa.dma_pubkey);
1051 ctx->rsa.pubkey = NULL;
1054 if (ctx->rsa.crt_prikey) {
1055 memzero_explicit(ctx->rsa.crt_prikey,
1056 half_key_sz * HPRE_CRT_PRMS);
1057 dma_free_coherent(dev, half_key_sz * HPRE_CRT_PRMS,
1058 ctx->rsa.crt_prikey, ctx->rsa.dma_crt_prikey);
1059 ctx->rsa.crt_prikey = NULL;
1062 if (ctx->rsa.prikey) {
1063 memzero_explicit(ctx->rsa.prikey, ctx->key_sz);
1064 dma_free_coherent(dev, ctx->key_sz << 1, ctx->rsa.prikey,
1065 ctx->rsa.dma_prikey);
1066 ctx->rsa.prikey = NULL;
1069 hpre_ctx_clear(ctx, is_clear_all);
1073 * we should judge if it is CRT or not,
1074 * CRT: return true, N-CRT: return false .
1076 static bool hpre_is_crt_key(struct rsa_key *key)
1078 u16 len = key->p_sz + key->q_sz + key->dp_sz + key->dq_sz +
1081 #define LEN_OF_NCRT_PARA 5
1083 /* N-CRT less than 5 parameters */
1084 return len > LEN_OF_NCRT_PARA;
1087 static int hpre_rsa_setkey(struct hpre_ctx *ctx, const void *key,
1088 unsigned int keylen, bool private)
1090 struct rsa_key rsa_key;
1093 hpre_rsa_clear_ctx(ctx, false);
1096 ret = rsa_parse_priv_key(&rsa_key, key, keylen);
1098 ret = rsa_parse_pub_key(&rsa_key, key, keylen);
1102 ret = hpre_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz, private);
1107 ret = hpre_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz);
1111 if (hpre_is_crt_key(&rsa_key)) {
1112 ret = hpre_rsa_setkey_crt(ctx, &rsa_key);
1118 ret = hpre_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz);
1122 if ((private && !ctx->rsa.prikey) || !ctx->rsa.pubkey) {
1130 hpre_rsa_clear_ctx(ctx, false);
1134 static int hpre_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key,
1135 unsigned int keylen)
1137 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1140 ret = crypto_akcipher_set_pub_key(ctx->rsa.soft_tfm, key, keylen);
1144 return hpre_rsa_setkey(ctx, key, keylen, false);
1147 static int hpre_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key,
1148 unsigned int keylen)
1150 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1153 ret = crypto_akcipher_set_priv_key(ctx->rsa.soft_tfm, key, keylen);
1157 return hpre_rsa_setkey(ctx, key, keylen, true);
1160 static unsigned int hpre_rsa_max_size(struct crypto_akcipher *tfm)
1162 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1164 /* For 512 and 1536 bits key size, use soft tfm instead */
1165 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
1166 ctx->key_sz == HPRE_RSA_1536BITS_KSZ)
1167 return crypto_akcipher_maxsize(ctx->rsa.soft_tfm);
1172 static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm)
1174 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1177 ctx->rsa.soft_tfm = crypto_alloc_akcipher("rsa-generic", 0, 0);
1178 if (IS_ERR(ctx->rsa.soft_tfm)) {
1179 pr_err("Can not alloc_akcipher!\n");
1180 return PTR_ERR(ctx->rsa.soft_tfm);
1183 akcipher_set_reqsize(tfm, sizeof(struct hpre_asym_request) +
1186 ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE);
1188 crypto_free_akcipher(ctx->rsa.soft_tfm);
1193 static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm)
1195 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1197 hpre_rsa_clear_ctx(ctx, true);
1198 crypto_free_akcipher(ctx->rsa.soft_tfm);
1201 static void hpre_key_to_big_end(u8 *data, int len)
1205 for (i = 0; i < len / 2; i++) {
1207 swap(data[j], data[i]);
1211 static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all,
1214 struct device *dev = ctx->dev;
1215 unsigned int sz = ctx->key_sz;
1216 unsigned int shift = sz << 1;
1219 hisi_qm_stop_qp(ctx->qp);
1221 if (is_ecdh && ctx->ecdh.p) {
1222 /* ecdh: p->a->k->b */
1223 memzero_explicit(ctx->ecdh.p + shift, sz);
1224 dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p);
1226 } else if (!is_ecdh && ctx->curve25519.p) {
1227 /* curve25519: p->a->k */
1228 memzero_explicit(ctx->curve25519.p + shift, sz);
1229 dma_free_coherent(dev, sz << 2, ctx->curve25519.p,
1230 ctx->curve25519.dma_p);
1231 ctx->curve25519.p = NULL;
1234 hpre_ctx_clear(ctx, is_clear_all);
1238 * The bits of 192/224/256/384/521 are supported by HPRE,
1239 * and convert the bits like:
1240 * bits<=256, bits=256; 256<bits<=384, bits=384; 384<bits<=576, bits=576;
1241 * If the parameter bit width is insufficient, then we fill in the
1242 * high-order zeros by soft, so TASK_LENGTH1 is 0x3/0x5/0x8;
1244 static unsigned int hpre_ecdh_supported_curve(unsigned short id)
1247 case ECC_CURVE_NIST_P192:
1248 case ECC_CURVE_NIST_P256:
1249 return HPRE_ECC_HW256_KSZ_B;
1250 case ECC_CURVE_NIST_P384:
1251 return HPRE_ECC_HW384_KSZ_B;
1259 static void fill_curve_param(void *addr, u64 *param, unsigned int cur_sz, u8 ndigits)
1261 unsigned int sz = cur_sz - (ndigits - 1) * sizeof(u64);
1264 while (i < ndigits - 1) {
1265 memcpy(addr + sizeof(u64) * i, ¶m[i], sizeof(u64));
1269 memcpy(addr + sizeof(u64) * i, ¶m[ndigits - 1], sz);
1270 hpre_key_to_big_end((u8 *)addr, cur_sz);
1273 static int hpre_ecdh_fill_curve(struct hpre_ctx *ctx, struct ecdh *params,
1274 unsigned int cur_sz)
1276 unsigned int shifta = ctx->key_sz << 1;
1277 unsigned int shiftb = ctx->key_sz << 2;
1278 void *p = ctx->ecdh.p + ctx->key_sz - cur_sz;
1279 void *a = ctx->ecdh.p + shifta - cur_sz;
1280 void *b = ctx->ecdh.p + shiftb - cur_sz;
1281 void *x = ctx->ecdh.g + ctx->key_sz - cur_sz;
1282 void *y = ctx->ecdh.g + shifta - cur_sz;
1283 const struct ecc_curve *curve = ecc_get_curve(ctx->curve_id);
1286 if (unlikely(!curve))
1289 n = kzalloc(ctx->key_sz, GFP_KERNEL);
1293 fill_curve_param(p, curve->p, cur_sz, curve->g.ndigits);
1294 fill_curve_param(a, curve->a, cur_sz, curve->g.ndigits);
1295 fill_curve_param(b, curve->b, cur_sz, curve->g.ndigits);
1296 fill_curve_param(x, curve->g.x, cur_sz, curve->g.ndigits);
1297 fill_curve_param(y, curve->g.y, cur_sz, curve->g.ndigits);
1298 fill_curve_param(n, curve->n, cur_sz, curve->g.ndigits);
1300 if (params->key_size == cur_sz && memcmp(params->key, n, cur_sz) >= 0) {
1309 static unsigned int hpre_ecdh_get_curvesz(unsigned short id)
1312 case ECC_CURVE_NIST_P192:
1313 return HPRE_ECC_NIST_P192_N_SIZE;
1314 case ECC_CURVE_NIST_P256:
1315 return HPRE_ECC_NIST_P256_N_SIZE;
1316 case ECC_CURVE_NIST_P384:
1317 return HPRE_ECC_NIST_P384_N_SIZE;
1325 static int hpre_ecdh_set_param(struct hpre_ctx *ctx, struct ecdh *params)
1327 struct device *dev = ctx->dev;
1328 unsigned int sz, shift, curve_sz;
1331 ctx->key_sz = hpre_ecdh_supported_curve(ctx->curve_id);
1335 curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
1336 if (!curve_sz || params->key_size > curve_sz)
1342 ctx->ecdh.p = dma_alloc_coherent(dev, sz << 3, &ctx->ecdh.dma_p,
1349 ctx->ecdh.g = ctx->ecdh.p + shift;
1350 ctx->ecdh.dma_g = ctx->ecdh.dma_p + shift;
1352 ret = hpre_ecdh_fill_curve(ctx, params, curve_sz);
1354 dev_err(dev, "failed to fill curve_param, ret = %d!\n", ret);
1355 dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p);
1363 static bool hpre_key_is_zero(char *key, unsigned short key_sz)
1367 for (i = 0; i < key_sz; i++)
1374 static int ecdh_gen_privkey(struct hpre_ctx *ctx, struct ecdh *params)
1376 struct device *dev = ctx->dev;
1379 ret = crypto_get_default_rng();
1381 dev_err(dev, "failed to get default rng, ret = %d!\n", ret);
1385 ret = crypto_rng_get_bytes(crypto_default_rng, (u8 *)params->key,
1387 crypto_put_default_rng();
1389 dev_err(dev, "failed to get rng, ret = %d!\n", ret);
1394 static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
1397 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1398 unsigned int sz, sz_shift, curve_sz;
1399 struct device *dev = ctx->dev;
1400 char key[HPRE_ECC_MAX_KSZ];
1404 if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0) {
1405 dev_err(dev, "failed to decode ecdh key!\n");
1409 /* Use stdrng to generate private key */
1410 if (!params.key || !params.key_size) {
1412 curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
1414 dev_err(dev, "Invalid curve size!\n");
1418 params.key_size = curve_sz - 1;
1419 ret = ecdh_gen_privkey(ctx, ¶ms);
1424 if (hpre_key_is_zero(params.key, params.key_size)) {
1425 dev_err(dev, "Invalid hpre key!\n");
1429 hpre_ecc_clear_ctx(ctx, false, true);
1431 ret = hpre_ecdh_set_param(ctx, ¶ms);
1433 dev_err(dev, "failed to set hpre param, ret = %d!\n", ret);
1438 sz_shift = (sz << 1) + sz - params.key_size;
1439 memcpy(ctx->ecdh.p + sz_shift, params.key, params.key_size);
1444 static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx,
1445 struct hpre_asym_request *req,
1446 struct scatterlist *dst,
1447 struct scatterlist *src)
1449 struct device *dev = ctx->dev;
1450 struct hpre_sqe *sqe = &req->req;
1453 dma = le64_to_cpu(sqe->in);
1454 if (unlikely(dma_mapping_error(dev, dma)))
1457 if (src && req->src)
1458 dma_free_coherent(dev, ctx->key_sz << 2, req->src, dma);
1460 dma = le64_to_cpu(sqe->out);
1461 if (unlikely(dma_mapping_error(dev, dma)))
1465 dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma);
1467 dma_unmap_single(dev, dma, ctx->key_sz << 1, DMA_FROM_DEVICE);
1470 static void hpre_ecdh_cb(struct hpre_ctx *ctx, void *resp)
1472 unsigned int curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
1473 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
1474 struct hpre_asym_request *req = NULL;
1475 struct kpp_request *areq;
1476 u64 overtime_thrhld;
1480 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
1481 areq = req->areq.ecdh;
1482 areq->dst_len = ctx->key_sz << 1;
1484 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
1485 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
1486 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
1488 p = sg_virt(areq->dst);
1489 memmove(p, p + ctx->key_sz - curve_sz, curve_sz);
1490 memmove(p + curve_sz, p + areq->dst_len - curve_sz, curve_sz);
1492 hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src);
1493 kpp_request_complete(areq, ret);
1495 atomic64_inc(&dfx[HPRE_RECV_CNT].value);
1498 static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx,
1499 struct kpp_request *req)
1501 struct hpre_asym_request *h_req;
1502 struct hpre_sqe *msg;
1506 if (req->dst_len < ctx->key_sz << 1) {
1507 req->dst_len = ctx->key_sz << 1;
1511 tmp = kpp_request_ctx(req);
1512 h_req = PTR_ALIGN(tmp, hpre_align_sz());
1513 h_req->cb = hpre_ecdh_cb;
1514 h_req->areq.ecdh = req;
1516 memset(msg, 0, sizeof(*msg));
1517 msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
1518 msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
1519 msg->key = cpu_to_le64(ctx->ecdh.dma_p);
1521 msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT);
1522 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
1525 req_id = hpre_add_req_to_ctx(h_req);
1529 msg->tag = cpu_to_le16((u16)req_id);
1533 static int hpre_ecdh_src_data_init(struct hpre_asym_request *hpre_req,
1534 struct scatterlist *data, unsigned int len)
1536 struct hpre_sqe *msg = &hpre_req->req;
1537 struct hpre_ctx *ctx = hpre_req->ctx;
1538 struct device *dev = ctx->dev;
1539 unsigned int tmpshift;
1544 /* Src_data include gx and gy. */
1545 shift = ctx->key_sz - (len >> 1);
1546 if (unlikely(shift < 0))
1549 ptr = dma_alloc_coherent(dev, ctx->key_sz << 2, &dma, GFP_KERNEL);
1553 tmpshift = ctx->key_sz << 1;
1554 scatterwalk_map_and_copy(ptr + tmpshift, data, 0, len, 0);
1555 memcpy(ptr + shift, ptr + tmpshift, len >> 1);
1556 memcpy(ptr + ctx->key_sz + shift, ptr + tmpshift + (len >> 1), len >> 1);
1558 hpre_req->src = ptr;
1559 msg->in = cpu_to_le64(dma);
1563 static int hpre_ecdh_dst_data_init(struct hpre_asym_request *hpre_req,
1564 struct scatterlist *data, unsigned int len)
1566 struct hpre_sqe *msg = &hpre_req->req;
1567 struct hpre_ctx *ctx = hpre_req->ctx;
1568 struct device *dev = ctx->dev;
1571 if (unlikely(!data || !sg_is_last(data) || len != ctx->key_sz << 1)) {
1572 dev_err(dev, "data or data length is illegal!\n");
1576 hpre_req->dst = NULL;
1577 dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE);
1578 if (unlikely(dma_mapping_error(dev, dma))) {
1579 dev_err(dev, "dma map data err!\n");
1583 msg->out = cpu_to_le64(dma);
1587 static int hpre_ecdh_compute_value(struct kpp_request *req)
1589 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
1590 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1591 struct device *dev = ctx->dev;
1592 void *tmp = kpp_request_ctx(req);
1593 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
1594 struct hpre_sqe *msg = &hpre_req->req;
1597 ret = hpre_ecdh_msg_request_set(ctx, req);
1598 if (unlikely(ret)) {
1599 dev_err(dev, "failed to set ecdh request, ret = %d!\n", ret);
1604 ret = hpre_ecdh_src_data_init(hpre_req, req->src, req->src_len);
1605 if (unlikely(ret)) {
1606 dev_err(dev, "failed to init src data, ret = %d!\n", ret);
1610 msg->in = cpu_to_le64(ctx->ecdh.dma_g);
1613 ret = hpre_ecdh_dst_data_init(hpre_req, req->dst, req->dst_len);
1614 if (unlikely(ret)) {
1615 dev_err(dev, "failed to init dst data, ret = %d!\n", ret);
1619 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_ECC_MUL);
1620 ret = hpre_send(ctx, msg);
1622 return -EINPROGRESS;
1625 hpre_rm_req_from_ctx(hpre_req);
1626 hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
1630 static unsigned int hpre_ecdh_max_size(struct crypto_kpp *tfm)
1632 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1634 /* max size is the pub_key_size, include x and y */
1635 return ctx->key_sz << 1;
1638 static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm)
1640 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1642 ctx->curve_id = ECC_CURVE_NIST_P192;
1644 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
1646 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1649 static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm)
1651 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1653 ctx->curve_id = ECC_CURVE_NIST_P256;
1655 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
1657 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1660 static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm)
1662 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1664 ctx->curve_id = ECC_CURVE_NIST_P384;
1666 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
1668 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1671 static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm)
1673 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1675 hpre_ecc_clear_ctx(ctx, true, true);
1678 static void hpre_curve25519_fill_curve(struct hpre_ctx *ctx, const void *buf,
1681 u8 secret[CURVE25519_KEY_SIZE] = { 0 };
1682 unsigned int sz = ctx->key_sz;
1683 const struct ecc_curve *curve;
1684 unsigned int shift = sz << 1;
1688 * The key from 'buf' is in little-endian, we should preprocess it as
1689 * the description in rfc7748: "k[0] &= 248, k[31] &= 127, k[31] |= 64",
1690 * then convert it to big endian. Only in this way, the result can be
1691 * the same as the software curve-25519 that exists in crypto.
1693 memcpy(secret, buf, len);
1694 curve25519_clamp_secret(secret);
1695 hpre_key_to_big_end(secret, CURVE25519_KEY_SIZE);
1697 p = ctx->curve25519.p + sz - len;
1699 curve = ecc_get_curve25519();
1701 /* fill curve parameters */
1702 fill_curve_param(p, curve->p, len, curve->g.ndigits);
1703 fill_curve_param(p + sz, curve->a, len, curve->g.ndigits);
1704 memcpy(p + shift, secret, len);
1705 fill_curve_param(p + shift + sz, curve->g.x, len, curve->g.ndigits);
1706 memzero_explicit(secret, CURVE25519_KEY_SIZE);
1709 static int hpre_curve25519_set_param(struct hpre_ctx *ctx, const void *buf,
1712 struct device *dev = ctx->dev;
1713 unsigned int sz = ctx->key_sz;
1714 unsigned int shift = sz << 1;
1717 if (!ctx->curve25519.p) {
1718 ctx->curve25519.p = dma_alloc_coherent(dev, sz << 2,
1719 &ctx->curve25519.dma_p,
1721 if (!ctx->curve25519.p)
1725 ctx->curve25519.g = ctx->curve25519.p + shift + sz;
1726 ctx->curve25519.dma_g = ctx->curve25519.dma_p + shift + sz;
1728 hpre_curve25519_fill_curve(ctx, buf, len);
1733 static int hpre_curve25519_set_secret(struct crypto_kpp *tfm, const void *buf,
1736 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1737 struct device *dev = ctx->dev;
1740 if (len != CURVE25519_KEY_SIZE ||
1741 !crypto_memneq(buf, curve25519_null_point, CURVE25519_KEY_SIZE)) {
1742 dev_err(dev, "key is null or key len is not 32bytes!\n");
1746 /* Free old secret if any */
1747 hpre_ecc_clear_ctx(ctx, false, false);
1749 ctx->key_sz = CURVE25519_KEY_SIZE;
1750 ret = hpre_curve25519_set_param(ctx, buf, CURVE25519_KEY_SIZE);
1752 dev_err(dev, "failed to set curve25519 param, ret = %d!\n", ret);
1753 hpre_ecc_clear_ctx(ctx, false, false);
1760 static void hpre_curve25519_hw_data_clr_all(struct hpre_ctx *ctx,
1761 struct hpre_asym_request *req,
1762 struct scatterlist *dst,
1763 struct scatterlist *src)
1765 struct device *dev = ctx->dev;
1766 struct hpre_sqe *sqe = &req->req;
1769 dma = le64_to_cpu(sqe->in);
1770 if (unlikely(dma_mapping_error(dev, dma)))
1773 if (src && req->src)
1774 dma_free_coherent(dev, ctx->key_sz, req->src, dma);
1776 dma = le64_to_cpu(sqe->out);
1777 if (unlikely(dma_mapping_error(dev, dma)))
1781 dma_free_coherent(dev, ctx->key_sz, req->dst, dma);
1783 dma_unmap_single(dev, dma, ctx->key_sz, DMA_FROM_DEVICE);
1786 static void hpre_curve25519_cb(struct hpre_ctx *ctx, void *resp)
1788 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
1789 struct hpre_asym_request *req = NULL;
1790 struct kpp_request *areq;
1791 u64 overtime_thrhld;
1794 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
1795 areq = req->areq.curve25519;
1796 areq->dst_len = ctx->key_sz;
1798 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
1799 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
1800 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
1802 hpre_key_to_big_end(sg_virt(areq->dst), CURVE25519_KEY_SIZE);
1804 hpre_curve25519_hw_data_clr_all(ctx, req, areq->dst, areq->src);
1805 kpp_request_complete(areq, ret);
1807 atomic64_inc(&dfx[HPRE_RECV_CNT].value);
1810 static int hpre_curve25519_msg_request_set(struct hpre_ctx *ctx,
1811 struct kpp_request *req)
1813 struct hpre_asym_request *h_req;
1814 struct hpre_sqe *msg;
1818 if (unlikely(req->dst_len < ctx->key_sz)) {
1819 req->dst_len = ctx->key_sz;
1823 tmp = kpp_request_ctx(req);
1824 h_req = PTR_ALIGN(tmp, hpre_align_sz());
1825 h_req->cb = hpre_curve25519_cb;
1826 h_req->areq.curve25519 = req;
1828 memset(msg, 0, sizeof(*msg));
1829 msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
1830 msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
1831 msg->key = cpu_to_le64(ctx->curve25519.dma_p);
1833 msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT);
1834 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
1837 req_id = hpre_add_req_to_ctx(h_req);
1841 msg->tag = cpu_to_le16((u16)req_id);
1845 static void hpre_curve25519_src_modulo_p(u8 *ptr)
1849 for (i = 0; i < CURVE25519_KEY_SIZE - 1; i++)
1852 /* The modulus is ptr's last byte minus '0xed'(last byte of p) */
1856 static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req,
1857 struct scatterlist *data, unsigned int len)
1859 struct hpre_sqe *msg = &hpre_req->req;
1860 struct hpre_ctx *ctx = hpre_req->ctx;
1861 struct device *dev = ctx->dev;
1862 u8 p[CURVE25519_KEY_SIZE] = { 0 };
1863 const struct ecc_curve *curve;
1867 if (len != CURVE25519_KEY_SIZE) {
1868 dev_err(dev, "sourc_data len is not 32bytes, len = %u!\n", len);
1872 ptr = dma_alloc_coherent(dev, ctx->key_sz, &dma, GFP_KERNEL);
1876 scatterwalk_map_and_copy(ptr, data, 0, len, 0);
1878 if (!crypto_memneq(ptr, curve25519_null_point, CURVE25519_KEY_SIZE)) {
1879 dev_err(dev, "gx is null!\n");
1884 * Src_data(gx) is in little-endian order, MSB in the final byte should
1885 * be masked as described in RFC7748, then transform it to big-endian
1886 * form, then hisi_hpre can use the data.
1889 hpre_key_to_big_end(ptr, CURVE25519_KEY_SIZE);
1891 curve = ecc_get_curve25519();
1893 fill_curve_param(p, curve->p, CURVE25519_KEY_SIZE, curve->g.ndigits);
1896 * When src_data equals (2^255 - 19) ~ (2^255 - 1), it is out of p,
1897 * we get its modulus to p, and then use it.
1899 if (memcmp(ptr, p, ctx->key_sz) == 0) {
1900 dev_err(dev, "gx is p!\n");
1902 } else if (memcmp(ptr, p, ctx->key_sz) > 0) {
1903 hpre_curve25519_src_modulo_p(ptr);
1906 hpre_req->src = ptr;
1907 msg->in = cpu_to_le64(dma);
1911 dma_free_coherent(dev, ctx->key_sz, ptr, dma);
1915 static int hpre_curve25519_dst_init(struct hpre_asym_request *hpre_req,
1916 struct scatterlist *data, unsigned int len)
1918 struct hpre_sqe *msg = &hpre_req->req;
1919 struct hpre_ctx *ctx = hpre_req->ctx;
1920 struct device *dev = ctx->dev;
1923 if (!data || !sg_is_last(data) || len != ctx->key_sz) {
1924 dev_err(dev, "data or data length is illegal!\n");
1928 hpre_req->dst = NULL;
1929 dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE);
1930 if (unlikely(dma_mapping_error(dev, dma))) {
1931 dev_err(dev, "dma map data err!\n");
1935 msg->out = cpu_to_le64(dma);
1939 static int hpre_curve25519_compute_value(struct kpp_request *req)
1941 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
1942 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1943 struct device *dev = ctx->dev;
1944 void *tmp = kpp_request_ctx(req);
1945 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
1946 struct hpre_sqe *msg = &hpre_req->req;
1949 ret = hpre_curve25519_msg_request_set(ctx, req);
1950 if (unlikely(ret)) {
1951 dev_err(dev, "failed to set curve25519 request, ret = %d!\n", ret);
1956 ret = hpre_curve25519_src_init(hpre_req, req->src, req->src_len);
1957 if (unlikely(ret)) {
1958 dev_err(dev, "failed to init src data, ret = %d!\n",
1963 msg->in = cpu_to_le64(ctx->curve25519.dma_g);
1966 ret = hpre_curve25519_dst_init(hpre_req, req->dst, req->dst_len);
1967 if (unlikely(ret)) {
1968 dev_err(dev, "failed to init dst data, ret = %d!\n", ret);
1972 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_CURVE25519_MUL);
1973 ret = hpre_send(ctx, msg);
1975 return -EINPROGRESS;
1978 hpre_rm_req_from_ctx(hpre_req);
1979 hpre_curve25519_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
1983 static unsigned int hpre_curve25519_max_size(struct crypto_kpp *tfm)
1985 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1990 static int hpre_curve25519_init_tfm(struct crypto_kpp *tfm)
1992 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1994 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
1996 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1999 static void hpre_curve25519_exit_tfm(struct crypto_kpp *tfm)
2001 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
2003 hpre_ecc_clear_ctx(ctx, true, false);
2006 static struct akcipher_alg rsa = {
2007 .sign = hpre_rsa_dec,
2008 .verify = hpre_rsa_enc,
2009 .encrypt = hpre_rsa_enc,
2010 .decrypt = hpre_rsa_dec,
2011 .set_pub_key = hpre_rsa_setpubkey,
2012 .set_priv_key = hpre_rsa_setprivkey,
2013 .max_size = hpre_rsa_max_size,
2014 .init = hpre_rsa_init_tfm,
2015 .exit = hpre_rsa_exit_tfm,
2017 .cra_ctxsize = sizeof(struct hpre_ctx),
2018 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2020 .cra_driver_name = "hpre-rsa",
2021 .cra_module = THIS_MODULE,
2025 static struct kpp_alg dh = {
2026 .set_secret = hpre_dh_set_secret,
2027 .generate_public_key = hpre_dh_compute_value,
2028 .compute_shared_secret = hpre_dh_compute_value,
2029 .max_size = hpre_dh_max_size,
2030 .init = hpre_dh_init_tfm,
2031 .exit = hpre_dh_exit_tfm,
2033 .cra_ctxsize = sizeof(struct hpre_ctx),
2034 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2036 .cra_driver_name = "hpre-dh",
2037 .cra_module = THIS_MODULE,
2041 static struct kpp_alg ecdh_curves[] = {
2043 .set_secret = hpre_ecdh_set_secret,
2044 .generate_public_key = hpre_ecdh_compute_value,
2045 .compute_shared_secret = hpre_ecdh_compute_value,
2046 .max_size = hpre_ecdh_max_size,
2047 .init = hpre_ecdh_nist_p192_init_tfm,
2048 .exit = hpre_ecdh_exit_tfm,
2050 .cra_ctxsize = sizeof(struct hpre_ctx),
2051 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2052 .cra_name = "ecdh-nist-p192",
2053 .cra_driver_name = "hpre-ecdh-nist-p192",
2054 .cra_module = THIS_MODULE,
2057 .set_secret = hpre_ecdh_set_secret,
2058 .generate_public_key = hpre_ecdh_compute_value,
2059 .compute_shared_secret = hpre_ecdh_compute_value,
2060 .max_size = hpre_ecdh_max_size,
2061 .init = hpre_ecdh_nist_p256_init_tfm,
2062 .exit = hpre_ecdh_exit_tfm,
2064 .cra_ctxsize = sizeof(struct hpre_ctx),
2065 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2066 .cra_name = "ecdh-nist-p256",
2067 .cra_driver_name = "hpre-ecdh-nist-p256",
2068 .cra_module = THIS_MODULE,
2071 .set_secret = hpre_ecdh_set_secret,
2072 .generate_public_key = hpre_ecdh_compute_value,
2073 .compute_shared_secret = hpre_ecdh_compute_value,
2074 .max_size = hpre_ecdh_max_size,
2075 .init = hpre_ecdh_nist_p384_init_tfm,
2076 .exit = hpre_ecdh_exit_tfm,
2078 .cra_ctxsize = sizeof(struct hpre_ctx),
2079 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2080 .cra_name = "ecdh-nist-p384",
2081 .cra_driver_name = "hpre-ecdh-nist-p384",
2082 .cra_module = THIS_MODULE,
2087 static struct kpp_alg curve25519_alg = {
2088 .set_secret = hpre_curve25519_set_secret,
2089 .generate_public_key = hpre_curve25519_compute_value,
2090 .compute_shared_secret = hpre_curve25519_compute_value,
2091 .max_size = hpre_curve25519_max_size,
2092 .init = hpre_curve25519_init_tfm,
2093 .exit = hpre_curve25519_exit_tfm,
2095 .cra_ctxsize = sizeof(struct hpre_ctx),
2096 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2097 .cra_name = "curve25519",
2098 .cra_driver_name = "hpre-curve25519",
2099 .cra_module = THIS_MODULE,
2103 static int hpre_register_rsa(struct hisi_qm *qm)
2107 if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP))
2110 rsa.base.cra_flags = 0;
2111 ret = crypto_register_akcipher(&rsa);
2113 dev_err(&qm->pdev->dev, "failed to register rsa (%d)!\n", ret);
2118 static void hpre_unregister_rsa(struct hisi_qm *qm)
2120 if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP))
2123 crypto_unregister_akcipher(&rsa);
2126 static int hpre_register_dh(struct hisi_qm *qm)
2130 if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP))
2133 ret = crypto_register_kpp(&dh);
2135 dev_err(&qm->pdev->dev, "failed to register dh (%d)!\n", ret);
2140 static void hpre_unregister_dh(struct hisi_qm *qm)
2142 if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP))
2145 crypto_unregister_kpp(&dh);
2148 static int hpre_register_ecdh(struct hisi_qm *qm)
2152 if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP))
2155 for (i = 0; i < ARRAY_SIZE(ecdh_curves); i++) {
2156 ret = crypto_register_kpp(&ecdh_curves[i]);
2158 dev_err(&qm->pdev->dev, "failed to register %s (%d)!\n",
2159 ecdh_curves[i].base.cra_name, ret);
2167 for (--i; i >= 0; --i)
2168 crypto_unregister_kpp(&ecdh_curves[i]);
2173 static void hpre_unregister_ecdh(struct hisi_qm *qm)
2177 if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP))
2180 for (i = ARRAY_SIZE(ecdh_curves) - 1; i >= 0; --i)
2181 crypto_unregister_kpp(&ecdh_curves[i]);
2184 static int hpre_register_x25519(struct hisi_qm *qm)
2188 if (!hpre_check_alg_support(qm, HPRE_DRV_X25519_MASK_CAP))
2191 ret = crypto_register_kpp(&curve25519_alg);
2193 dev_err(&qm->pdev->dev, "failed to register x25519 (%d)!\n", ret);
2198 static void hpre_unregister_x25519(struct hisi_qm *qm)
2200 if (!hpre_check_alg_support(qm, HPRE_DRV_X25519_MASK_CAP))
2203 crypto_unregister_kpp(&curve25519_alg);
2206 int hpre_algs_register(struct hisi_qm *qm)
2210 mutex_lock(&hpre_algs_lock);
2211 if (hpre_available_devs) {
2212 hpre_available_devs++;
2216 ret = hpre_register_rsa(qm);
2220 ret = hpre_register_dh(qm);
2224 ret = hpre_register_ecdh(qm);
2228 ret = hpre_register_x25519(qm);
2232 hpre_available_devs++;
2233 mutex_unlock(&hpre_algs_lock);
2238 hpre_unregister_ecdh(qm);
2240 hpre_unregister_dh(qm);
2242 hpre_unregister_rsa(qm);
2244 mutex_unlock(&hpre_algs_lock);
2248 void hpre_algs_unregister(struct hisi_qm *qm)
2250 mutex_lock(&hpre_algs_lock);
2251 if (--hpre_available_devs)
2254 hpre_unregister_x25519(qm);
2255 hpre_unregister_ecdh(qm);
2256 hpre_unregister_dh(qm);
2257 hpre_unregister_rsa(qm);
2260 mutex_unlock(&hpre_algs_lock);