drivers/dma/idxd/submit.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
   3 #include <linux/init.h>
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <linux/pci.h>
   7 #include <uapi/linux/idxd.h>
   8 #include "idxd.h"
   9 #include "registers.h"
  10
  11 static struct idxd_desc *__get_desc(struct idxd_wq *wq, int idx, int cpu)
  12 {
  13         struct idxd_desc *desc;
  14
  15         desc = wq->descs[idx];
  16         memset(desc->hw, 0, sizeof(struct dsa_hw_desc));
  17         memset(desc->completion, 0, sizeof(struct dsa_completion_record));
  18         desc->cpu = cpu;
  19         return desc;
  20 }
  21
  22 struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype)
  23 {
  24         int cpu, idx;
  25         struct idxd_device *idxd = wq->idxd;
  26         DEFINE_SBQ_WAIT(wait);
  27         struct sbq_wait_state *ws;
  28         struct sbitmap_queue *sbq;
  29
  30         if (idxd->state != IDXD_DEV_ENABLED)
  31                 return ERR_PTR(-EIO);
  32
  33         sbq = &wq->sbq;
  34         idx = sbitmap_queue_get(sbq, &cpu);
  35         if (idx < 0) {
  36                 if (optype == IDXD_OP_NONBLOCK)
  37                         return ERR_PTR(-EAGAIN);
  38         } else {
  39                 return __get_desc(wq, idx, cpu);
  40         }
  41
  42         ws = &sbq->ws[0];
  43         for (;;) {
  44                 sbitmap_prepare_to_wait(sbq, ws, &wait, TASK_INTERRUPTIBLE);
  45                 if (signal_pending_state(TASK_INTERRUPTIBLE, current))
  46                         break;
  47                 idx = sbitmap_queue_get(sbq, &cpu);
  48                 if (idx >= 0)
  49                         break;
  50                 schedule();
  51         }
  52
  53         sbitmap_finish_wait(sbq, ws, &wait);
  54         if (idx < 0)
  55                 return ERR_PTR(-EAGAIN);
  56
  57         return __get_desc(wq, idx, cpu);
  58 }
  59
  60 void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc)
  61 {
  62         int cpu = desc->cpu;
  63
  64         desc->cpu = -1;
  65         sbitmap_queue_clear(&wq->sbq, desc->id, cpu);
  66 }
  67
  68 int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc)
  69 {
  70         struct idxd_device *idxd = wq->idxd;
  71         int vec = desc->hw->int_handle;
  72         void __iomem *portal;
  73
  74         if (idxd->state != IDXD_DEV_ENABLED)
  75                 return -EIO;
  76
  77         portal = wq->dportal;
  78         /*
  79          * The wmb() flushes writes to coherent DMA data before possibly
  80          * triggering a DMA read. The wmb() is necessary even on UP because
  81          * the recipient is a device.
  82          */
  83         wmb();
  84         iosubmit_cmds512(portal, desc->hw, 1);
  85
  86         /*
  87          * Pending the descriptor to the lockless list for the irq_entry
  88          * that we designated the descriptor to.
  89          */
  90         if (desc->hw->flags & IDXD_OP_FLAG_RCI)
  91                 llist_add(&desc->llnode,
  92                           &idxd->irq_entries[vec].pending_llist);
  93
  94         return 0;
  95 }