1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
7 * DOC: Nitro Enclaves (NE) PCI device driver.
10 #include <linux/delay.h>
11 #include <linux/device.h>
12 #include <linux/list.h>
13 #include <linux/module.h>
14 #include <linux/mutex.h>
15 #include <linux/nitro_enclaves.h>
16 #include <linux/pci.h>
17 #include <linux/types.h>
18 #include <linux/wait.h>
20 #include "ne_misc_dev.h"
21 #include "ne_pci_dev.h"
24 * NE_DEFAULT_TIMEOUT_MSECS - Default timeout to wait for a reply from
27 #define NE_DEFAULT_TIMEOUT_MSECS (120000) /* 120 sec */
29 static const struct pci_device_id ne_pci_ids[] = {
30 { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_NE) },
34 MODULE_DEVICE_TABLE(pci, ne_pci_ids);
37 * ne_submit_request() - Submit command request to the PCI device based on the
39 * @pdev: PCI device to send the command to.
40 * @cmd_type: Command type of the request sent to the PCI device.
41 * @cmd_request: Command request payload.
42 * @cmd_request_size: Size of the command request payload.
44 * Context: Process context. This function is called with the ne_pci_dev mutex held.
46 static void ne_submit_request(struct pci_dev *pdev, enum ne_pci_dev_cmd_type cmd_type,
47 void *cmd_request, size_t cmd_request_size)
49 struct ne_pci_dev *ne_pci_dev = pci_get_drvdata(pdev);
51 memcpy_toio(ne_pci_dev->iomem_base + NE_SEND_DATA, cmd_request, cmd_request_size);
53 iowrite32(cmd_type, ne_pci_dev->iomem_base + NE_COMMAND);
57 * ne_retrieve_reply() - Retrieve reply from the PCI device.
58 * @pdev: PCI device to receive the reply from.
59 * @cmd_reply: Command reply payload.
60 * @cmd_reply_size: Size of the command reply payload.
62 * Context: Process context. This function is called with the ne_pci_dev mutex held.
64 static void ne_retrieve_reply(struct pci_dev *pdev, struct ne_pci_dev_cmd_reply *cmd_reply,
65 size_t cmd_reply_size)
67 struct ne_pci_dev *ne_pci_dev = pci_get_drvdata(pdev);
69 memcpy_fromio(cmd_reply, ne_pci_dev->iomem_base + NE_RECV_DATA, cmd_reply_size);
73 * ne_wait_for_reply() - Wait for a reply of a PCI device command.
74 * @pdev: PCI device for which a reply is waited.
76 * Context: Process context. This function is called with the ne_pci_dev mutex held.
79 * * Negative return value on failure.
81 static int ne_wait_for_reply(struct pci_dev *pdev)
83 struct ne_pci_dev *ne_pci_dev = pci_get_drvdata(pdev);
87 * TODO: Update to _interruptible and handle interrupted wait event
88 * e.g. -ERESTARTSYS, incoming signals + update timeout, if needed.
90 rc = wait_event_timeout(ne_pci_dev->cmd_reply_wait_q,
91 atomic_read(&ne_pci_dev->cmd_reply_avail) != 0,
92 msecs_to_jiffies(NE_DEFAULT_TIMEOUT_MSECS));
99 int ne_do_request(struct pci_dev *pdev, enum ne_pci_dev_cmd_type cmd_type,
100 void *cmd_request, size_t cmd_request_size,
101 struct ne_pci_dev_cmd_reply *cmd_reply, size_t cmd_reply_size)
103 struct ne_pci_dev *ne_pci_dev = pci_get_drvdata(pdev);
106 if (cmd_type <= INVALID_CMD || cmd_type >= MAX_CMD) {
107 dev_err_ratelimited(&pdev->dev, "Invalid cmd type=%u\n", cmd_type);
113 dev_err_ratelimited(&pdev->dev, "Null cmd request for cmd type=%u\n",
119 if (cmd_request_size > NE_SEND_DATA_SIZE) {
120 dev_err_ratelimited(&pdev->dev, "Invalid req size=%zu for cmd type=%u\n",
121 cmd_request_size, cmd_type);
127 dev_err_ratelimited(&pdev->dev, "Null cmd reply for cmd type=%u\n",
133 if (cmd_reply_size > NE_RECV_DATA_SIZE) {
134 dev_err_ratelimited(&pdev->dev, "Invalid reply size=%zu for cmd type=%u\n",
135 cmd_reply_size, cmd_type);
141 * Use this mutex so that the PCI device handles one command request at
144 mutex_lock(&ne_pci_dev->pci_dev_mutex);
146 atomic_set(&ne_pci_dev->cmd_reply_avail, 0);
148 ne_submit_request(pdev, cmd_type, cmd_request, cmd_request_size);
150 rc = ne_wait_for_reply(pdev);
152 dev_err_ratelimited(&pdev->dev, "Error in wait for reply for cmd type=%u [rc=%d]\n",
158 ne_retrieve_reply(pdev, cmd_reply, cmd_reply_size);
160 atomic_set(&ne_pci_dev->cmd_reply_avail, 0);
162 if (cmd_reply->rc < 0) {
165 dev_err_ratelimited(&pdev->dev, "Error in cmd process logic, cmd type=%u [rc=%d]\n",
174 mutex_unlock(&ne_pci_dev->pci_dev_mutex);
180 * ne_reply_handler() - Interrupt handler for retrieving a reply matching a
181 * request sent to the PCI device for enclave lifetime
183 * @irq: Received interrupt for a reply sent by the PCI device.
184 * @args: PCI device private data structure.
186 * Context: Interrupt context.
188 * * IRQ_HANDLED on handled interrupt.
190 static irqreturn_t ne_reply_handler(int irq, void *args)
192 struct ne_pci_dev *ne_pci_dev = (struct ne_pci_dev *)args;
194 atomic_set(&ne_pci_dev->cmd_reply_avail, 1);
196 /* TODO: Update to _interruptible. */
197 wake_up(&ne_pci_dev->cmd_reply_wait_q);
203 * ne_event_work_handler() - Work queue handler for notifying enclaves on a
204 * state change received by the event interrupt
206 * @work: Item containing the NE PCI device for which an out-of-band event
209 * An out-of-band event is being issued by the Nitro Hypervisor when at least
210 * one enclave is changing state without client interaction.
212 * Context: Work queue context.
214 static void ne_event_work_handler(struct work_struct *work)
216 struct ne_pci_dev_cmd_reply cmd_reply = {};
217 struct ne_enclave *ne_enclave = NULL;
218 struct ne_pci_dev *ne_pci_dev =
219 container_of(work, struct ne_pci_dev, notify_work);
220 struct pci_dev *pdev = ne_pci_dev->pdev;
222 struct slot_info_req slot_info_req = {};
224 mutex_lock(&ne_pci_dev->enclaves_list_mutex);
227 * Iterate over all enclaves registered for the Nitro Enclaves
228 * PCI device and determine for which enclave(s) the out-of-band event
229 * is corresponding to.
231 list_for_each_entry(ne_enclave, &ne_pci_dev->enclaves_list, enclave_list_entry) {
232 mutex_lock(&ne_enclave->enclave_info_mutex);
235 * Enclaves that were never started cannot receive out-of-band
238 if (ne_enclave->state != NE_STATE_RUNNING)
241 slot_info_req.slot_uid = ne_enclave->slot_uid;
243 rc = ne_do_request(pdev, SLOT_INFO,
244 &slot_info_req, sizeof(slot_info_req),
245 &cmd_reply, sizeof(cmd_reply));
247 dev_err(&pdev->dev, "Error in slot info [rc=%d]\n", rc);
249 /* Notify enclave process that the enclave state changed. */
250 if (ne_enclave->state != cmd_reply.state) {
251 ne_enclave->state = cmd_reply.state;
253 ne_enclave->has_event = true;
255 wake_up_interruptible(&ne_enclave->eventq);
259 mutex_unlock(&ne_enclave->enclave_info_mutex);
262 mutex_unlock(&ne_pci_dev->enclaves_list_mutex);
266 * ne_event_handler() - Interrupt handler for PCI device out-of-band events.
267 * This interrupt does not supply any data in the MMIO
268 * region. It notifies a change in the state of any of
269 * the launched enclaves.
270 * @irq: Received interrupt for an out-of-band event.
271 * @args: PCI device private data structure.
273 * Context: Interrupt context.
275 * * IRQ_HANDLED on handled interrupt.
277 static irqreturn_t ne_event_handler(int irq, void *args)
279 struct ne_pci_dev *ne_pci_dev = (struct ne_pci_dev *)args;
281 queue_work(ne_pci_dev->event_wq, &ne_pci_dev->notify_work);
287 * ne_setup_msix() - Setup MSI-X vectors for the PCI device.
288 * @pdev: PCI device to setup the MSI-X for.
290 * Context: Process context.
293 * * Negative return value on failure.
295 static int ne_setup_msix(struct pci_dev *pdev)
297 struct ne_pci_dev *ne_pci_dev = pci_get_drvdata(pdev);
301 nr_vecs = pci_msix_vec_count(pdev);
305 dev_err(&pdev->dev, "Error in getting vec count [rc=%d]\n", rc);
310 rc = pci_alloc_irq_vectors(pdev, nr_vecs, nr_vecs, PCI_IRQ_MSIX);
312 dev_err(&pdev->dev, "Error in alloc MSI-X vecs [rc=%d]\n", rc);
318 * This IRQ gets triggered every time the PCI device responds to a
319 * command request. The reply is then retrieved, reading from the MMIO
320 * space of the PCI device.
322 rc = request_irq(pci_irq_vector(pdev, NE_VEC_REPLY), ne_reply_handler,
323 0, "enclave_cmd", ne_pci_dev);
325 dev_err(&pdev->dev, "Error in request irq reply [rc=%d]\n", rc);
327 goto free_irq_vectors;
330 ne_pci_dev->event_wq = create_singlethread_workqueue("ne_pci_dev_wq");
331 if (!ne_pci_dev->event_wq) {
334 dev_err(&pdev->dev, "Cannot get wq for dev events [rc=%d]\n", rc);
336 goto free_reply_irq_vec;
339 INIT_WORK(&ne_pci_dev->notify_work, ne_event_work_handler);
342 * This IRQ gets triggered every time any enclave's state changes. Its
343 * handler then scans for the changes and propagates them to the user
346 rc = request_irq(pci_irq_vector(pdev, NE_VEC_EVENT), ne_event_handler,
347 0, "enclave_evt", ne_pci_dev);
349 dev_err(&pdev->dev, "Error in request irq event [rc=%d]\n", rc);
357 destroy_workqueue(ne_pci_dev->event_wq);
359 free_irq(pci_irq_vector(pdev, NE_VEC_REPLY), ne_pci_dev);
361 pci_free_irq_vectors(pdev);
367 * ne_teardown_msix() - Teardown MSI-X vectors for the PCI device.
368 * @pdev: PCI device to teardown the MSI-X for.
370 * Context: Process context.
372 static void ne_teardown_msix(struct pci_dev *pdev)
374 struct ne_pci_dev *ne_pci_dev = pci_get_drvdata(pdev);
376 free_irq(pci_irq_vector(pdev, NE_VEC_EVENT), ne_pci_dev);
378 flush_work(&ne_pci_dev->notify_work);
379 flush_workqueue(ne_pci_dev->event_wq);
380 destroy_workqueue(ne_pci_dev->event_wq);
382 free_irq(pci_irq_vector(pdev, NE_VEC_REPLY), ne_pci_dev);
384 pci_free_irq_vectors(pdev);
388 * ne_pci_dev_enable() - Select the PCI device version and enable it.
389 * @pdev: PCI device to select version for and then enable.
391 * Context: Process context.
394 * * Negative return value on failure.
396 static int ne_pci_dev_enable(struct pci_dev *pdev)
398 u8 dev_enable_reply = 0;
399 u16 dev_version_reply = 0;
400 struct ne_pci_dev *ne_pci_dev = pci_get_drvdata(pdev);
402 iowrite16(NE_VERSION_MAX, ne_pci_dev->iomem_base + NE_VERSION);
404 dev_version_reply = ioread16(ne_pci_dev->iomem_base + NE_VERSION);
405 if (dev_version_reply != NE_VERSION_MAX) {
406 dev_err(&pdev->dev, "Error in pci dev version cmd\n");
411 iowrite8(NE_ENABLE_ON, ne_pci_dev->iomem_base + NE_ENABLE);
413 dev_enable_reply = ioread8(ne_pci_dev->iomem_base + NE_ENABLE);
414 if (dev_enable_reply != NE_ENABLE_ON) {
415 dev_err(&pdev->dev, "Error in pci dev enable cmd\n");
424 * ne_pci_dev_disable() - Disable the PCI device.
425 * @pdev: PCI device to disable.
427 * Context: Process context.
429 static void ne_pci_dev_disable(struct pci_dev *pdev)
431 u8 dev_disable_reply = 0;
432 struct ne_pci_dev *ne_pci_dev = pci_get_drvdata(pdev);
433 const unsigned int sleep_time = 10; /* 10 ms */
434 unsigned int sleep_time_count = 0;
436 iowrite8(NE_ENABLE_OFF, ne_pci_dev->iomem_base + NE_ENABLE);
439 * Check for NE_ENABLE_OFF in a loop, to handle cases when the device
440 * state is not immediately set to disabled and going through a
441 * transitory state of disabling.
443 while (sleep_time_count < NE_DEFAULT_TIMEOUT_MSECS) {
444 dev_disable_reply = ioread8(ne_pci_dev->iomem_base + NE_ENABLE);
445 if (dev_disable_reply == NE_ENABLE_OFF)
448 msleep_interruptible(sleep_time);
449 sleep_time_count += sleep_time;
452 dev_disable_reply = ioread8(ne_pci_dev->iomem_base + NE_ENABLE);
453 if (dev_disable_reply != NE_ENABLE_OFF)
454 dev_err(&pdev->dev, "Error in pci dev disable cmd\n");
458 * ne_pci_probe() - Probe function for the NE PCI device.
459 * @pdev: PCI device to match with the NE PCI driver.
460 * @id : PCI device id table associated with the NE PCI driver.
462 * Context: Process context.
465 * * Negative return value on failure.
467 static int ne_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
469 struct ne_pci_dev *ne_pci_dev = NULL;
472 ne_pci_dev = kzalloc(sizeof(*ne_pci_dev), GFP_KERNEL);
476 rc = pci_enable_device(pdev);
478 dev_err(&pdev->dev, "Error in pci dev enable [rc=%d]\n", rc);
480 goto free_ne_pci_dev;
483 rc = pci_request_regions_exclusive(pdev, "nitro_enclaves");
485 dev_err(&pdev->dev, "Error in pci request regions [rc=%d]\n", rc);
487 goto disable_pci_dev;
490 ne_pci_dev->iomem_base = pci_iomap(pdev, PCI_BAR_NE, 0);
491 if (!ne_pci_dev->iomem_base) {
494 dev_err(&pdev->dev, "Error in pci iomap [rc=%d]\n", rc);
496 goto release_pci_regions;
499 pci_set_drvdata(pdev, ne_pci_dev);
501 rc = ne_setup_msix(pdev);
503 dev_err(&pdev->dev, "Error in pci dev msix setup [rc=%d]\n", rc);
505 goto iounmap_pci_bar;
508 ne_pci_dev_disable(pdev);
510 rc = ne_pci_dev_enable(pdev);
512 dev_err(&pdev->dev, "Error in ne_pci_dev enable [rc=%d]\n", rc);
517 atomic_set(&ne_pci_dev->cmd_reply_avail, 0);
518 init_waitqueue_head(&ne_pci_dev->cmd_reply_wait_q);
519 INIT_LIST_HEAD(&ne_pci_dev->enclaves_list);
520 mutex_init(&ne_pci_dev->enclaves_list_mutex);
521 mutex_init(&ne_pci_dev->pci_dev_mutex);
522 ne_pci_dev->pdev = pdev;
524 ne_devs.ne_pci_dev = ne_pci_dev;
526 rc = misc_register(ne_devs.ne_misc_dev);
528 dev_err(&pdev->dev, "Error in misc dev register [rc=%d]\n", rc);
530 goto disable_ne_pci_dev;
536 ne_devs.ne_pci_dev = NULL;
537 ne_pci_dev_disable(pdev);
539 ne_teardown_msix(pdev);
541 pci_set_drvdata(pdev, NULL);
542 pci_iounmap(pdev, ne_pci_dev->iomem_base);
544 pci_release_regions(pdev);
546 pci_disable_device(pdev);
554 * ne_pci_remove() - Remove function for the NE PCI device.
555 * @pdev: PCI device associated with the NE PCI driver.
557 * Context: Process context.
559 static void ne_pci_remove(struct pci_dev *pdev)
561 struct ne_pci_dev *ne_pci_dev = pci_get_drvdata(pdev);
563 misc_deregister(ne_devs.ne_misc_dev);
565 ne_devs.ne_pci_dev = NULL;
567 ne_pci_dev_disable(pdev);
569 ne_teardown_msix(pdev);
571 pci_set_drvdata(pdev, NULL);
573 pci_iounmap(pdev, ne_pci_dev->iomem_base);
575 pci_release_regions(pdev);
577 pci_disable_device(pdev);
583 * ne_pci_shutdown() - Shutdown function for the NE PCI device.
584 * @pdev: PCI device associated with the NE PCI driver.
586 * Context: Process context.
588 static void ne_pci_shutdown(struct pci_dev *pdev)
590 struct ne_pci_dev *ne_pci_dev = pci_get_drvdata(pdev);
595 misc_deregister(ne_devs.ne_misc_dev);
597 ne_devs.ne_pci_dev = NULL;
599 ne_pci_dev_disable(pdev);
601 ne_teardown_msix(pdev);
603 pci_set_drvdata(pdev, NULL);
605 pci_iounmap(pdev, ne_pci_dev->iomem_base);
607 pci_release_regions(pdev);
609 pci_disable_device(pdev);
615 * TODO: Add suspend / resume functions for power management w/ CONFIG_PM, if
618 /* NE PCI device driver. */
619 struct pci_driver ne_pci_driver = {
620 .name = "nitro_enclaves",
621 .id_table = ne_pci_ids,
622 .probe = ne_pci_probe,
623 .remove = ne_pci_remove,
624 .shutdown = ne_pci_shutdown,