1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Driver for Virtio crypto device.
4 * Copyright 2016 HUAWEI TECHNOLOGIES CO., LTD.
8 #include <linux/module.h>
9 #include <linux/virtio_config.h>
10 #include <linux/cpu.h>
12 #include <uapi/linux/virtio_crypto.h>
13 #include "virtio_crypto_common.h"
17 virtcrypto_clear_request(struct virtio_crypto_request *vc_req)
20 kfree_sensitive(vc_req->req_data);
25 static void virtio_crypto_ctrlq_callback(struct virtio_crypto_ctrl_request *vc_ctrl_req)
27 complete(&vc_ctrl_req->compl);
30 static void virtcrypto_ctrlq_callback(struct virtqueue *vq)
32 struct virtio_crypto *vcrypto = vq->vdev->priv;
33 struct virtio_crypto_ctrl_request *vc_ctrl_req;
37 spin_lock_irqsave(&vcrypto->ctrl_lock, flags);
39 virtqueue_disable_cb(vq);
40 while ((vc_ctrl_req = virtqueue_get_buf(vq, &len)) != NULL) {
41 spin_unlock_irqrestore(&vcrypto->ctrl_lock, flags);
42 virtio_crypto_ctrlq_callback(vc_ctrl_req);
43 spin_lock_irqsave(&vcrypto->ctrl_lock, flags);
45 if (unlikely(virtqueue_is_broken(vq)))
47 } while (!virtqueue_enable_cb(vq));
48 spin_unlock_irqrestore(&vcrypto->ctrl_lock, flags);
51 int virtio_crypto_ctrl_vq_request(struct virtio_crypto *vcrypto, struct scatterlist *sgs[],
52 unsigned int out_sgs, unsigned int in_sgs,
53 struct virtio_crypto_ctrl_request *vc_ctrl_req)
58 init_completion(&vc_ctrl_req->compl);
60 spin_lock_irqsave(&vcrypto->ctrl_lock, flags);
61 err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, out_sgs, in_sgs, vc_ctrl_req, GFP_ATOMIC);
63 spin_unlock_irqrestore(&vcrypto->ctrl_lock, flags);
67 virtqueue_kick(vcrypto->ctrl_vq);
68 spin_unlock_irqrestore(&vcrypto->ctrl_lock, flags);
70 wait_for_completion(&vc_ctrl_req->compl);
75 static void virtcrypto_done_task(unsigned long data)
77 struct data_queue *data_vq = (struct data_queue *)data;
78 struct virtqueue *vq = data_vq->vq;
79 struct virtio_crypto_request *vc_req;
83 virtqueue_disable_cb(vq);
84 while ((vc_req = virtqueue_get_buf(vq, &len)) != NULL) {
86 vc_req->alg_cb(vc_req, len);
88 } while (!virtqueue_enable_cb(vq));
91 static void virtcrypto_dataq_callback(struct virtqueue *vq)
93 struct virtio_crypto *vcrypto = vq->vdev->priv;
94 struct data_queue *dq = &vcrypto->data_vq[vq->index];
96 tasklet_schedule(&dq->done_task);
99 static int virtcrypto_find_vqs(struct virtio_crypto *vi)
101 vq_callback_t **callbacks;
102 struct virtqueue **vqs;
106 struct device *dev = &vi->vdev->dev;
109 * We expect 1 data virtqueue, followed by
110 * possible N-1 data queues used in multiqueue mode,
111 * followed by control vq.
113 total_vqs = vi->max_data_queues + 1;
115 /* Allocate space for find_vqs parameters */
116 vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL);
119 callbacks = kcalloc(total_vqs, sizeof(*callbacks), GFP_KERNEL);
122 names = kcalloc(total_vqs, sizeof(*names), GFP_KERNEL);
126 /* Parameters for control virtqueue */
127 callbacks[total_vqs - 1] = virtcrypto_ctrlq_callback;
128 names[total_vqs - 1] = "controlq";
130 /* Allocate/initialize parameters for data virtqueues */
131 for (i = 0; i < vi->max_data_queues; i++) {
132 callbacks[i] = virtcrypto_dataq_callback;
133 snprintf(vi->data_vq[i].name, sizeof(vi->data_vq[i].name),
135 names[i] = vi->data_vq[i].name;
138 ret = virtio_find_vqs(vi->vdev, total_vqs, vqs, callbacks, names, NULL);
142 vi->ctrl_vq = vqs[total_vqs - 1];
144 for (i = 0; i < vi->max_data_queues; i++) {
145 spin_lock_init(&vi->data_vq[i].lock);
146 vi->data_vq[i].vq = vqs[i];
147 /* Initialize crypto engine */
148 vi->data_vq[i].engine = crypto_engine_alloc_init(dev, 1);
149 if (!vi->data_vq[i].engine) {
153 tasklet_init(&vi->data_vq[i].done_task, virtcrypto_done_task,
154 (unsigned long)&vi->data_vq[i]);
174 static int virtcrypto_alloc_queues(struct virtio_crypto *vi)
176 vi->data_vq = kcalloc(vi->max_data_queues, sizeof(*vi->data_vq),
184 static void virtcrypto_clean_affinity(struct virtio_crypto *vi, long hcpu)
188 if (vi->affinity_hint_set) {
189 for (i = 0; i < vi->max_data_queues; i++)
190 virtqueue_set_affinity(vi->data_vq[i].vq, NULL);
192 vi->affinity_hint_set = false;
196 static void virtcrypto_set_affinity(struct virtio_crypto *vcrypto)
202 * In single queue mode, we don't set the cpu affinity.
204 if (vcrypto->curr_queue == 1 || vcrypto->max_data_queues == 1) {
205 virtcrypto_clean_affinity(vcrypto, -1);
210 * In multiqueue mode, we let the queue to be private to one cpu
211 * by setting the affinity hint to eliminate the contention.
213 * TODO: adds cpu hotplug support by register cpu notifier.
216 for_each_online_cpu(cpu) {
217 virtqueue_set_affinity(vcrypto->data_vq[i].vq, cpumask_of(cpu));
218 if (++i >= vcrypto->max_data_queues)
222 vcrypto->affinity_hint_set = true;
225 static void virtcrypto_free_queues(struct virtio_crypto *vi)
230 static int virtcrypto_init_vqs(struct virtio_crypto *vi)
234 /* Allocate send & receive queues */
235 ret = virtcrypto_alloc_queues(vi);
239 ret = virtcrypto_find_vqs(vi);
244 virtcrypto_set_affinity(vi);
250 virtcrypto_free_queues(vi);
255 static int virtcrypto_update_status(struct virtio_crypto *vcrypto)
260 virtio_cread_le(vcrypto->vdev,
261 struct virtio_crypto_config, status, &status);
264 * Unknown status bits would be a host error and the driver
265 * should consider the device to be broken.
267 if (status & (~VIRTIO_CRYPTO_S_HW_READY)) {
268 dev_warn(&vcrypto->vdev->dev,
269 "Unknown status bits: 0x%x\n", status);
271 virtio_break_device(vcrypto->vdev);
275 if (vcrypto->status == status)
278 vcrypto->status = status;
280 if (vcrypto->status & VIRTIO_CRYPTO_S_HW_READY) {
281 err = virtcrypto_dev_start(vcrypto);
283 dev_err(&vcrypto->vdev->dev,
284 "Failed to start virtio crypto device.\n");
288 dev_info(&vcrypto->vdev->dev, "Accelerator device is ready\n");
290 virtcrypto_dev_stop(vcrypto);
291 dev_info(&vcrypto->vdev->dev, "Accelerator is not ready\n");
297 static int virtcrypto_start_crypto_engines(struct virtio_crypto *vcrypto)
302 for (i = 0; i < vcrypto->max_data_queues; i++) {
303 if (vcrypto->data_vq[i].engine) {
304 ret = crypto_engine_start(vcrypto->data_vq[i].engine);
314 if (vcrypto->data_vq[i].engine)
315 crypto_engine_exit(vcrypto->data_vq[i].engine);
320 static void virtcrypto_clear_crypto_engines(struct virtio_crypto *vcrypto)
324 for (i = 0; i < vcrypto->max_data_queues; i++)
325 if (vcrypto->data_vq[i].engine)
326 crypto_engine_exit(vcrypto->data_vq[i].engine);
329 static void virtcrypto_del_vqs(struct virtio_crypto *vcrypto)
331 struct virtio_device *vdev = vcrypto->vdev;
333 virtcrypto_clean_affinity(vcrypto, -1);
335 vdev->config->del_vqs(vdev);
337 virtcrypto_free_queues(vcrypto);
340 static int virtcrypto_probe(struct virtio_device *vdev)
343 struct virtio_crypto *vcrypto;
344 u32 max_data_queues = 0, max_cipher_key_len = 0;
345 u32 max_auth_key_len = 0;
347 u32 cipher_algo_l = 0;
348 u32 cipher_algo_h = 0;
353 u32 akcipher_algo = 0;
354 u32 crypto_services = 0;
356 if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
359 if (!vdev->config->get) {
360 dev_err(&vdev->dev, "%s failure: config access disabled\n",
365 if (num_possible_nodes() > 1 && dev_to_node(&vdev->dev) < 0) {
367 * If the accelerator is connected to a node with no memory
368 * there is no point in using the accelerator since the remote
369 * memory transaction will be very slow.
371 dev_err(&vdev->dev, "Invalid NUMA configuration.\n");
375 vcrypto = kzalloc_node(sizeof(*vcrypto), GFP_KERNEL,
376 dev_to_node(&vdev->dev));
380 virtio_cread_le(vdev, struct virtio_crypto_config,
381 max_dataqueues, &max_data_queues);
382 if (max_data_queues < 1)
385 virtio_cread_le(vdev, struct virtio_crypto_config,
386 max_cipher_key_len, &max_cipher_key_len);
387 virtio_cread_le(vdev, struct virtio_crypto_config,
388 max_auth_key_len, &max_auth_key_len);
389 virtio_cread_le(vdev, struct virtio_crypto_config,
390 max_size, &max_size);
391 virtio_cread_le(vdev, struct virtio_crypto_config,
392 crypto_services, &crypto_services);
393 virtio_cread_le(vdev, struct virtio_crypto_config,
394 cipher_algo_l, &cipher_algo_l);
395 virtio_cread_le(vdev, struct virtio_crypto_config,
396 cipher_algo_h, &cipher_algo_h);
397 virtio_cread_le(vdev, struct virtio_crypto_config,
398 hash_algo, &hash_algo);
399 virtio_cread_le(vdev, struct virtio_crypto_config,
400 mac_algo_l, &mac_algo_l);
401 virtio_cread_le(vdev, struct virtio_crypto_config,
402 mac_algo_h, &mac_algo_h);
403 virtio_cread_le(vdev, struct virtio_crypto_config,
404 aead_algo, &aead_algo);
405 if (crypto_services & (1 << VIRTIO_CRYPTO_SERVICE_AKCIPHER))
406 virtio_cread_le(vdev, struct virtio_crypto_config,
407 akcipher_algo, &akcipher_algo);
409 /* Add virtio crypto device to global table */
410 err = virtcrypto_devmgr_add_dev(vcrypto);
412 dev_err(&vdev->dev, "Failed to add new virtio crypto device.\n");
415 vcrypto->owner = THIS_MODULE;
416 vcrypto = vdev->priv = vcrypto;
417 vcrypto->vdev = vdev;
419 spin_lock_init(&vcrypto->ctrl_lock);
421 /* Use single data queue as default */
422 vcrypto->curr_queue = 1;
423 vcrypto->max_data_queues = max_data_queues;
424 vcrypto->max_cipher_key_len = max_cipher_key_len;
425 vcrypto->max_auth_key_len = max_auth_key_len;
426 vcrypto->max_size = max_size;
427 vcrypto->crypto_services = crypto_services;
428 vcrypto->cipher_algo_l = cipher_algo_l;
429 vcrypto->cipher_algo_h = cipher_algo_h;
430 vcrypto->mac_algo_l = mac_algo_l;
431 vcrypto->mac_algo_h = mac_algo_h;
432 vcrypto->hash_algo = hash_algo;
433 vcrypto->aead_algo = aead_algo;
434 vcrypto->akcipher_algo = akcipher_algo;
437 "max_queues: %u, max_cipher_key_len: %u, max_auth_key_len: %u, max_size 0x%llx\n",
438 vcrypto->max_data_queues,
439 vcrypto->max_cipher_key_len,
440 vcrypto->max_auth_key_len,
443 err = virtcrypto_init_vqs(vcrypto);
445 dev_err(&vdev->dev, "Failed to initialize vqs.\n");
449 err = virtcrypto_start_crypto_engines(vcrypto);
453 virtio_device_ready(vdev);
455 err = virtcrypto_update_status(vcrypto);
462 virtcrypto_clear_crypto_engines(vcrypto);
464 vcrypto->vdev->config->reset(vdev);
465 virtcrypto_del_vqs(vcrypto);
467 virtcrypto_devmgr_rm_dev(vcrypto);
473 static void virtcrypto_free_unused_reqs(struct virtio_crypto *vcrypto)
475 struct virtio_crypto_request *vc_req;
477 struct virtqueue *vq;
479 for (i = 0; i < vcrypto->max_data_queues; i++) {
480 vq = vcrypto->data_vq[i].vq;
481 while ((vc_req = virtqueue_detach_unused_buf(vq)) != NULL) {
482 kfree(vc_req->req_data);
488 static void virtcrypto_remove(struct virtio_device *vdev)
490 struct virtio_crypto *vcrypto = vdev->priv;
493 dev_info(&vdev->dev, "Start virtcrypto_remove.\n");
495 if (virtcrypto_dev_started(vcrypto))
496 virtcrypto_dev_stop(vcrypto);
497 for (i = 0; i < vcrypto->max_data_queues; i++)
498 tasklet_kill(&vcrypto->data_vq[i].done_task);
499 vdev->config->reset(vdev);
500 virtcrypto_free_unused_reqs(vcrypto);
501 virtcrypto_clear_crypto_engines(vcrypto);
502 virtcrypto_del_vqs(vcrypto);
503 virtcrypto_devmgr_rm_dev(vcrypto);
507 static void virtcrypto_config_changed(struct virtio_device *vdev)
509 struct virtio_crypto *vcrypto = vdev->priv;
511 virtcrypto_update_status(vcrypto);
514 #ifdef CONFIG_PM_SLEEP
515 static int virtcrypto_freeze(struct virtio_device *vdev)
517 struct virtio_crypto *vcrypto = vdev->priv;
519 vdev->config->reset(vdev);
520 virtcrypto_free_unused_reqs(vcrypto);
521 if (virtcrypto_dev_started(vcrypto))
522 virtcrypto_dev_stop(vcrypto);
524 virtcrypto_clear_crypto_engines(vcrypto);
525 virtcrypto_del_vqs(vcrypto);
529 static int virtcrypto_restore(struct virtio_device *vdev)
531 struct virtio_crypto *vcrypto = vdev->priv;
534 err = virtcrypto_init_vqs(vcrypto);
538 err = virtcrypto_start_crypto_engines(vcrypto);
542 virtio_device_ready(vdev);
544 err = virtcrypto_dev_start(vcrypto);
546 dev_err(&vdev->dev, "Failed to start virtio crypto device.\n");
553 virtcrypto_clear_crypto_engines(vcrypto);
555 vcrypto->vdev->config->reset(vdev);
556 virtcrypto_del_vqs(vcrypto);
561 static const unsigned int features[] = {
565 static const struct virtio_device_id id_table[] = {
566 { VIRTIO_ID_CRYPTO, VIRTIO_DEV_ANY_ID },
570 static struct virtio_driver virtio_crypto_driver = {
571 .driver.name = KBUILD_MODNAME,
572 .driver.owner = THIS_MODULE,
573 .feature_table = features,
574 .feature_table_size = ARRAY_SIZE(features),
575 .id_table = id_table,
576 .probe = virtcrypto_probe,
577 .remove = virtcrypto_remove,
578 .config_changed = virtcrypto_config_changed,
579 #ifdef CONFIG_PM_SLEEP
580 .freeze = virtcrypto_freeze,
581 .restore = virtcrypto_restore,
585 module_virtio_driver(virtio_crypto_driver);
587 MODULE_DEVICE_TABLE(virtio, id_table);
588 MODULE_DESCRIPTION("virtio crypto device driver");
589 MODULE_LICENSE("GPL");
590 MODULE_AUTHOR("Gonglei <arei.gonglei@huawei.com>");