1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Asymmetric algorithms supported by virtio crypto device
4 * Authors: zhenwei pi <pizhenwei@bytedance.com>
5 * lei he <helei.sig11@bytedance.com>
7 * Copyright 2022 Bytedance CO., LTD.
10 #include <linux/mpi.h>
11 #include <linux/scatterlist.h>
12 #include <crypto/algapi.h>
13 #include <crypto/internal/akcipher.h>
14 #include <crypto/internal/rsa.h>
15 #include <linux/err.h>
16 #include <crypto/scatterwalk.h>
17 #include <linux/atomic.h>
19 #include <uapi/linux/virtio_crypto.h>
20 #include "virtio_crypto_common.h"
22 struct virtio_crypto_rsa_ctx {
26 struct virtio_crypto_akcipher_ctx {
27 struct crypto_engine_ctx enginectx;
28 struct virtio_crypto *vcrypto;
29 struct crypto_akcipher *tfm;
33 struct virtio_crypto_rsa_ctx rsa_ctx;
37 struct virtio_crypto_akcipher_request {
38 struct virtio_crypto_request base;
39 struct virtio_crypto_akcipher_ctx *akcipher_ctx;
40 struct akcipher_request *akcipher_req;
46 struct virtio_crypto_akcipher_algo {
49 unsigned int active_devs;
50 struct akcipher_alg algo;
53 static DEFINE_MUTEX(algs_lock);
55 static void virtio_crypto_akcipher_finalize_req(
56 struct virtio_crypto_akcipher_request *vc_akcipher_req,
57 struct akcipher_request *req, int err)
59 kfree(vc_akcipher_req->src_buf);
60 kfree(vc_akcipher_req->dst_buf);
61 vc_akcipher_req->src_buf = NULL;
62 vc_akcipher_req->dst_buf = NULL;
63 virtcrypto_clear_request(&vc_akcipher_req->base);
65 crypto_finalize_akcipher_request(vc_akcipher_req->base.dataq->engine, req, err);
68 static void virtio_crypto_dataq_akcipher_callback(struct virtio_crypto_request *vc_req, int len)
70 struct virtio_crypto_akcipher_request *vc_akcipher_req =
71 container_of(vc_req, struct virtio_crypto_akcipher_request, base);
72 struct akcipher_request *akcipher_req;
75 switch (vc_req->status) {
76 case VIRTIO_CRYPTO_OK:
79 case VIRTIO_CRYPTO_INVSESS:
80 case VIRTIO_CRYPTO_ERR:
83 case VIRTIO_CRYPTO_BADMSG:
87 case VIRTIO_CRYPTO_KEY_REJECTED:
88 error = -EKEYREJECTED;
96 akcipher_req = vc_akcipher_req->akcipher_req;
97 if (vc_akcipher_req->opcode != VIRTIO_CRYPTO_AKCIPHER_VERIFY)
98 sg_copy_from_buffer(akcipher_req->dst, sg_nents(akcipher_req->dst),
99 vc_akcipher_req->dst_buf, akcipher_req->dst_len);
100 virtio_crypto_akcipher_finalize_req(vc_akcipher_req, akcipher_req, error);
103 static int virtio_crypto_alg_akcipher_init_session(struct virtio_crypto_akcipher_ctx *ctx,
104 struct virtio_crypto_ctrl_header *header,
105 struct virtio_crypto_akcipher_session_para *para,
106 const uint8_t *key, unsigned int keylen)
108 struct scatterlist outhdr_sg, key_sg, inhdr_sg, *sgs[3];
109 struct virtio_crypto *vcrypto = ctx->vcrypto;
112 unsigned int num_out = 0, num_in = 0;
113 struct virtio_crypto_op_ctrl_req *ctrl;
114 struct virtio_crypto_session_input *input;
115 struct virtio_crypto_ctrl_request *vc_ctrl_req;
117 pkey = kmemdup(key, keylen, GFP_ATOMIC);
121 vc_ctrl_req = kzalloc(sizeof(*vc_ctrl_req), GFP_KERNEL);
127 ctrl = &vc_ctrl_req->ctrl;
128 memcpy(&ctrl->header, header, sizeof(ctrl->header));
129 memcpy(&ctrl->u.akcipher_create_session.para, para, sizeof(*para));
130 input = &vc_ctrl_req->input;
131 input->status = cpu_to_le32(VIRTIO_CRYPTO_ERR);
133 sg_init_one(&outhdr_sg, ctrl, sizeof(*ctrl));
134 sgs[num_out++] = &outhdr_sg;
136 sg_init_one(&key_sg, pkey, keylen);
137 sgs[num_out++] = &key_sg;
139 sg_init_one(&inhdr_sg, input, sizeof(*input));
140 sgs[num_out + num_in++] = &inhdr_sg;
142 err = virtio_crypto_ctrl_vq_request(vcrypto, sgs, num_out, num_in, vc_ctrl_req);
146 if (le32_to_cpu(input->status) != VIRTIO_CRYPTO_OK) {
147 pr_err("virtio_crypto: Create session failed status: %u\n",
148 le32_to_cpu(input->status));
153 ctx->session_id = le64_to_cpu(input->session_id);
154 ctx->session_valid = true;
159 kfree_sensitive(pkey);
164 static int virtio_crypto_alg_akcipher_close_session(struct virtio_crypto_akcipher_ctx *ctx)
166 struct scatterlist outhdr_sg, inhdr_sg, *sgs[2];
167 struct virtio_crypto_destroy_session_req *destroy_session;
168 struct virtio_crypto *vcrypto = ctx->vcrypto;
169 unsigned int num_out = 0, num_in = 0;
171 struct virtio_crypto_op_ctrl_req *ctrl;
172 struct virtio_crypto_inhdr *ctrl_status;
173 struct virtio_crypto_ctrl_request *vc_ctrl_req;
175 if (!ctx->session_valid)
178 vc_ctrl_req = kzalloc(sizeof(*vc_ctrl_req), GFP_KERNEL);
182 ctrl_status = &vc_ctrl_req->ctrl_status;
183 ctrl_status->status = VIRTIO_CRYPTO_ERR;
184 ctrl = &vc_ctrl_req->ctrl;
185 ctrl->header.opcode = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_DESTROY_SESSION);
186 ctrl->header.queue_id = 0;
188 destroy_session = &ctrl->u.destroy_session;
189 destroy_session->session_id = cpu_to_le64(ctx->session_id);
191 sg_init_one(&outhdr_sg, ctrl, sizeof(*ctrl));
192 sgs[num_out++] = &outhdr_sg;
194 sg_init_one(&inhdr_sg, &ctrl_status->status, sizeof(ctrl_status->status));
195 sgs[num_out + num_in++] = &inhdr_sg;
197 err = virtio_crypto_ctrl_vq_request(vcrypto, sgs, num_out, num_in, vc_ctrl_req);
201 if (ctrl_status->status != VIRTIO_CRYPTO_OK) {
202 pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n",
203 ctrl_status->status, destroy_session->session_id);
209 ctx->session_valid = false;
217 static int __virtio_crypto_akcipher_do_req(struct virtio_crypto_akcipher_request *vc_akcipher_req,
218 struct akcipher_request *req, struct data_queue *data_vq)
220 struct virtio_crypto_akcipher_ctx *ctx = vc_akcipher_req->akcipher_ctx;
221 struct virtio_crypto_request *vc_req = &vc_akcipher_req->base;
222 struct virtio_crypto *vcrypto = ctx->vcrypto;
223 struct virtio_crypto_op_data_req *req_data = vc_req->req_data;
224 struct scatterlist *sgs[4], outhdr_sg, inhdr_sg, srcdata_sg, dstdata_sg;
225 void *src_buf = NULL, *dst_buf = NULL;
226 unsigned int num_out = 0, num_in = 0;
227 int node = dev_to_node(&vcrypto->vdev->dev);
230 bool verify = vc_akcipher_req->opcode == VIRTIO_CRYPTO_AKCIPHER_VERIFY;
231 unsigned int src_len = verify ? req->src_len + req->dst_len : req->src_len;
234 sg_init_one(&outhdr_sg, req_data, sizeof(*req_data));
235 sgs[num_out++] = &outhdr_sg;
238 src_buf = kcalloc_node(src_len, 1, GFP_KERNEL, node);
243 /* for verify operation, both src and dst data work as OUT direction */
244 sg_copy_to_buffer(req->src, sg_nents(req->src), src_buf, src_len);
245 sg_init_one(&srcdata_sg, src_buf, src_len);
246 sgs[num_out++] = &srcdata_sg;
248 sg_copy_to_buffer(req->src, sg_nents(req->src), src_buf, src_len);
249 sg_init_one(&srcdata_sg, src_buf, src_len);
250 sgs[num_out++] = &srcdata_sg;
253 dst_buf = kcalloc_node(req->dst_len, 1, GFP_KERNEL, node);
257 sg_init_one(&dstdata_sg, dst_buf, req->dst_len);
258 sgs[num_out + num_in++] = &dstdata_sg;
261 vc_akcipher_req->src_buf = src_buf;
262 vc_akcipher_req->dst_buf = dst_buf;
265 sg_init_one(&inhdr_sg, &vc_req->status, sizeof(vc_req->status));
266 sgs[num_out + num_in++] = &inhdr_sg;
268 spin_lock_irqsave(&data_vq->lock, flags);
269 ret = virtqueue_add_sgs(data_vq->vq, sgs, num_out, num_in, vc_req, GFP_ATOMIC);
270 virtqueue_kick(data_vq->vq);
271 spin_unlock_irqrestore(&data_vq->lock, flags);
284 static int virtio_crypto_rsa_do_req(struct crypto_engine *engine, void *vreq)
286 struct akcipher_request *req = container_of(vreq, struct akcipher_request, base);
287 struct virtio_crypto_akcipher_request *vc_akcipher_req = akcipher_request_ctx(req);
288 struct virtio_crypto_request *vc_req = &vc_akcipher_req->base;
289 struct virtio_crypto_akcipher_ctx *ctx = vc_akcipher_req->akcipher_ctx;
290 struct virtio_crypto *vcrypto = ctx->vcrypto;
291 struct data_queue *data_vq = vc_req->dataq;
292 struct virtio_crypto_op_header *header;
293 struct virtio_crypto_akcipher_data_req *akcipher_req;
297 vc_req->req_data = kzalloc_node(sizeof(*vc_req->req_data),
298 GFP_KERNEL, dev_to_node(&vcrypto->vdev->dev));
299 if (!vc_req->req_data)
302 /* build request header */
303 header = &vc_req->req_data->header;
304 header->opcode = cpu_to_le32(vc_akcipher_req->opcode);
305 header->algo = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_RSA);
306 header->session_id = cpu_to_le64(ctx->session_id);
308 /* build request akcipher data */
309 akcipher_req = &vc_req->req_data->u.akcipher_req;
310 akcipher_req->para.src_data_len = cpu_to_le32(req->src_len);
311 akcipher_req->para.dst_data_len = cpu_to_le32(req->dst_len);
313 ret = __virtio_crypto_akcipher_do_req(vc_akcipher_req, req, data_vq);
315 kfree_sensitive(vc_req->req_data);
316 vc_req->req_data = NULL;
323 static int virtio_crypto_rsa_req(struct akcipher_request *req, uint32_t opcode)
325 struct crypto_akcipher *atfm = crypto_akcipher_reqtfm(req);
326 struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(atfm);
327 struct virtio_crypto_akcipher_request *vc_akcipher_req = akcipher_request_ctx(req);
328 struct virtio_crypto_request *vc_req = &vc_akcipher_req->base;
329 struct virtio_crypto *vcrypto = ctx->vcrypto;
330 /* Use the first data virtqueue as default */
331 struct data_queue *data_vq = &vcrypto->data_vq[0];
333 vc_req->dataq = data_vq;
334 vc_req->alg_cb = virtio_crypto_dataq_akcipher_callback;
335 vc_akcipher_req->akcipher_ctx = ctx;
336 vc_akcipher_req->akcipher_req = req;
337 vc_akcipher_req->opcode = opcode;
339 return crypto_transfer_akcipher_request_to_engine(data_vq->engine, req);
342 static int virtio_crypto_rsa_encrypt(struct akcipher_request *req)
344 return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_ENCRYPT);
347 static int virtio_crypto_rsa_decrypt(struct akcipher_request *req)
349 return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_DECRYPT);
352 static int virtio_crypto_rsa_sign(struct akcipher_request *req)
354 return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_SIGN);
357 static int virtio_crypto_rsa_verify(struct akcipher_request *req)
359 return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_VERIFY);
362 static int virtio_crypto_rsa_set_key(struct crypto_akcipher *tfm,
369 struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm);
370 struct virtio_crypto_rsa_ctx *rsa_ctx = &ctx->rsa_ctx;
371 struct virtio_crypto *vcrypto;
372 struct virtio_crypto_ctrl_header header;
373 struct virtio_crypto_akcipher_session_para para;
374 struct rsa_key rsa_key = {0};
375 int node = virtio_crypto_get_current_node();
379 /* mpi_free will test n, just free it. */
380 mpi_free(rsa_ctx->n);
384 keytype = VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PRIVATE;
385 ret = rsa_parse_priv_key(&rsa_key, key, keylen);
387 keytype = VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PUBLIC;
388 ret = rsa_parse_pub_key(&rsa_key, key, keylen);
394 rsa_ctx->n = mpi_read_raw_data(rsa_key.n, rsa_key.n_sz);
399 vcrypto = virtcrypto_get_dev_node(node, VIRTIO_CRYPTO_SERVICE_AKCIPHER,
400 VIRTIO_CRYPTO_AKCIPHER_RSA);
402 pr_err("virtio_crypto: Could not find a virtio device in the system or unsupported algo\n");
406 ctx->vcrypto = vcrypto;
408 virtio_crypto_alg_akcipher_close_session(ctx);
411 /* set ctrl header */
412 header.opcode = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_CREATE_SESSION);
413 header.algo = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_RSA);
417 para.algo = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_RSA);
418 para.keytype = cpu_to_le32(keytype);
419 para.keylen = cpu_to_le32(keylen);
420 para.u.rsa.padding_algo = cpu_to_le32(padding_algo);
421 para.u.rsa.hash_algo = cpu_to_le32(hash_algo);
423 return virtio_crypto_alg_akcipher_init_session(ctx, &header, ¶, key, keylen);
426 static int virtio_crypto_rsa_raw_set_priv_key(struct crypto_akcipher *tfm,
430 return virtio_crypto_rsa_set_key(tfm, key, keylen, 1,
431 VIRTIO_CRYPTO_RSA_RAW_PADDING,
432 VIRTIO_CRYPTO_RSA_NO_HASH);
436 static int virtio_crypto_p1pad_rsa_sha1_set_priv_key(struct crypto_akcipher *tfm,
440 return virtio_crypto_rsa_set_key(tfm, key, keylen, 1,
441 VIRTIO_CRYPTO_RSA_PKCS1_PADDING,
442 VIRTIO_CRYPTO_RSA_SHA1);
445 static int virtio_crypto_rsa_raw_set_pub_key(struct crypto_akcipher *tfm,
449 return virtio_crypto_rsa_set_key(tfm, key, keylen, 0,
450 VIRTIO_CRYPTO_RSA_RAW_PADDING,
451 VIRTIO_CRYPTO_RSA_NO_HASH);
454 static int virtio_crypto_p1pad_rsa_sha1_set_pub_key(struct crypto_akcipher *tfm,
458 return virtio_crypto_rsa_set_key(tfm, key, keylen, 0,
459 VIRTIO_CRYPTO_RSA_PKCS1_PADDING,
460 VIRTIO_CRYPTO_RSA_SHA1);
463 static unsigned int virtio_crypto_rsa_max_size(struct crypto_akcipher *tfm)
465 struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm);
466 struct virtio_crypto_rsa_ctx *rsa_ctx = &ctx->rsa_ctx;
468 return mpi_get_size(rsa_ctx->n);
471 static int virtio_crypto_rsa_init_tfm(struct crypto_akcipher *tfm)
473 struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm);
476 ctx->enginectx.op.do_one_request = virtio_crypto_rsa_do_req;
477 ctx->enginectx.op.prepare_request = NULL;
478 ctx->enginectx.op.unprepare_request = NULL;
483 static void virtio_crypto_rsa_exit_tfm(struct crypto_akcipher *tfm)
485 struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm);
486 struct virtio_crypto_rsa_ctx *rsa_ctx = &ctx->rsa_ctx;
488 virtio_crypto_alg_akcipher_close_session(ctx);
489 virtcrypto_dev_put(ctx->vcrypto);
490 mpi_free(rsa_ctx->n);
494 static struct virtio_crypto_akcipher_algo virtio_crypto_akcipher_algs[] = {
496 .algonum = VIRTIO_CRYPTO_AKCIPHER_RSA,
497 .service = VIRTIO_CRYPTO_SERVICE_AKCIPHER,
499 .encrypt = virtio_crypto_rsa_encrypt,
500 .decrypt = virtio_crypto_rsa_decrypt,
501 .set_pub_key = virtio_crypto_rsa_raw_set_pub_key,
502 .set_priv_key = virtio_crypto_rsa_raw_set_priv_key,
503 .max_size = virtio_crypto_rsa_max_size,
504 .init = virtio_crypto_rsa_init_tfm,
505 .exit = virtio_crypto_rsa_exit_tfm,
506 .reqsize = sizeof(struct virtio_crypto_akcipher_request),
509 .cra_driver_name = "virtio-crypto-rsa",
511 .cra_module = THIS_MODULE,
512 .cra_ctxsize = sizeof(struct virtio_crypto_akcipher_ctx),
517 .algonum = VIRTIO_CRYPTO_AKCIPHER_RSA,
518 .service = VIRTIO_CRYPTO_SERVICE_AKCIPHER,
520 .encrypt = virtio_crypto_rsa_encrypt,
521 .decrypt = virtio_crypto_rsa_decrypt,
522 .sign = virtio_crypto_rsa_sign,
523 .verify = virtio_crypto_rsa_verify,
524 .set_pub_key = virtio_crypto_p1pad_rsa_sha1_set_pub_key,
525 .set_priv_key = virtio_crypto_p1pad_rsa_sha1_set_priv_key,
526 .max_size = virtio_crypto_rsa_max_size,
527 .init = virtio_crypto_rsa_init_tfm,
528 .exit = virtio_crypto_rsa_exit_tfm,
529 .reqsize = sizeof(struct virtio_crypto_akcipher_request),
531 .cra_name = "pkcs1pad(rsa,sha1)",
532 .cra_driver_name = "virtio-pkcs1-rsa-with-sha1",
534 .cra_module = THIS_MODULE,
535 .cra_ctxsize = sizeof(struct virtio_crypto_akcipher_ctx),
541 int virtio_crypto_akcipher_algs_register(struct virtio_crypto *vcrypto)
546 mutex_lock(&algs_lock);
548 for (i = 0; i < ARRAY_SIZE(virtio_crypto_akcipher_algs); i++) {
549 uint32_t service = virtio_crypto_akcipher_algs[i].service;
550 uint32_t algonum = virtio_crypto_akcipher_algs[i].algonum;
552 if (!virtcrypto_algo_is_supported(vcrypto, service, algonum))
555 if (virtio_crypto_akcipher_algs[i].active_devs == 0) {
556 ret = crypto_register_akcipher(&virtio_crypto_akcipher_algs[i].algo);
561 virtio_crypto_akcipher_algs[i].active_devs++;
562 dev_info(&vcrypto->vdev->dev, "Registered akcipher algo %s\n",
563 virtio_crypto_akcipher_algs[i].algo.base.cra_name);
567 mutex_unlock(&algs_lock);
571 void virtio_crypto_akcipher_algs_unregister(struct virtio_crypto *vcrypto)
575 mutex_lock(&algs_lock);
577 for (i = 0; i < ARRAY_SIZE(virtio_crypto_akcipher_algs); i++) {
578 uint32_t service = virtio_crypto_akcipher_algs[i].service;
579 uint32_t algonum = virtio_crypto_akcipher_algs[i].algonum;
581 if (virtio_crypto_akcipher_algs[i].active_devs == 0 ||
582 !virtcrypto_algo_is_supported(vcrypto, service, algonum))
585 if (virtio_crypto_akcipher_algs[i].active_devs == 1)
586 crypto_unregister_akcipher(&virtio_crypto_akcipher_algs[i].algo);
588 virtio_crypto_akcipher_algs[i].active_devs--;
591 mutex_unlock(&algs_lock);