1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3 #include <linux/init.h>
4 #include <linux/kernel.h>
5 #include <linux/module.h>
7 #include <linux/device.h>
8 #include <linux/io-64-nonatomic-lo-hi.h>
9 #include <linux/dmaengine.h>
10 #include <uapi/linux/idxd.h>
11 #include "../dmaengine.h"
12 #include "registers.h"
15 static inline struct idxd_wq *to_idxd_wq(struct dma_chan *c)
17 struct idxd_dma_chan *idxd_chan;
19 idxd_chan = container_of(c, struct idxd_dma_chan, chan);
23 void idxd_dma_complete_txd(struct idxd_desc *desc,
24 enum idxd_complete_type comp_type)
26 struct dma_async_tx_descriptor *tx;
27 struct dmaengine_result res;
30 if (desc->completion->status == DSA_COMP_SUCCESS)
31 res.result = DMA_TRANS_NOERROR;
32 else if (desc->completion->status)
33 res.result = DMA_TRANS_WRITE_FAILED;
34 else if (comp_type == IDXD_COMPLETE_ABORT)
35 res.result = DMA_TRANS_ABORTED;
40 if (complete && tx->cookie) {
41 dma_cookie_complete(tx);
42 dma_descriptor_unmap(tx);
43 dmaengine_desc_get_callback_invoke(tx, &res);
45 tx->callback_result = NULL;
49 static void op_flag_setup(unsigned long flags, u32 *desc_flags)
51 *desc_flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR;
52 if (flags & DMA_PREP_INTERRUPT)
53 *desc_flags |= IDXD_OP_FLAG_RCI;
56 static inline void set_completion_address(struct idxd_desc *desc,
59 *compl_addr = desc->compl_dma;
62 static inline void idxd_prep_desc_common(struct idxd_wq *wq,
63 struct dsa_hw_desc *hw, char opcode,
64 u64 addr_f1, u64 addr_f2, u64 len,
67 struct idxd_device *idxd = wq->idxd;
71 hw->src_addr = addr_f1;
72 hw->dst_addr = addr_f2;
74 hw->priv = !!(wq->type == IDXD_WQT_KERNEL);
75 hw->completion_addr = compl;
78 * Descriptor completion vectors are 1-8 for MSIX. We will round
79 * robin through the 8 vectors.
81 wq->vec_ptr = (wq->vec_ptr % idxd->num_wq_irqs) + 1;
82 hw->int_handle = wq->vec_ptr;
85 static struct dma_async_tx_descriptor *
86 idxd_dma_prep_interrupt(struct dma_chan *c, unsigned long flags)
88 struct idxd_wq *wq = to_idxd_wq(c);
90 struct idxd_desc *desc;
92 if (wq->state != IDXD_WQ_ENABLED)
95 op_flag_setup(flags, &desc_flags);
96 desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
100 idxd_prep_desc_common(wq, desc->hw, DSA_OPCODE_NOOP,
101 0, 0, 0, desc->compl_dma, desc_flags);
102 desc->txd.flags = flags;
106 static struct dma_async_tx_descriptor *
107 idxd_dma_submit_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
108 dma_addr_t dma_src, size_t len, unsigned long flags)
110 struct idxd_wq *wq = to_idxd_wq(c);
112 struct idxd_device *idxd = wq->idxd;
113 struct idxd_desc *desc;
115 if (wq->state != IDXD_WQ_ENABLED)
118 if (len > idxd->max_xfer_bytes)
121 op_flag_setup(flags, &desc_flags);
122 desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
126 idxd_prep_desc_common(wq, desc->hw, DSA_OPCODE_MEMMOVE,
127 dma_src, dma_dest, len, desc->compl_dma,
130 desc->txd.flags = flags;
135 static int idxd_dma_alloc_chan_resources(struct dma_chan *chan)
137 struct idxd_wq *wq = to_idxd_wq(chan);
138 struct device *dev = &wq->idxd->pdev->dev;
141 dev_dbg(dev, "%s: client_count: %d\n", __func__,
142 idxd_wq_refcount(wq));
146 static void idxd_dma_free_chan_resources(struct dma_chan *chan)
148 struct idxd_wq *wq = to_idxd_wq(chan);
149 struct device *dev = &wq->idxd->pdev->dev;
152 dev_dbg(dev, "%s: client_count: %d\n", __func__,
153 idxd_wq_refcount(wq));
156 static enum dma_status idxd_dma_tx_status(struct dma_chan *dma_chan,
158 struct dma_tx_state *txstate)
160 return DMA_OUT_OF_ORDER;
164 * issue_pending() does not need to do anything since tx_submit() does the job
167 static void idxd_dma_issue_pending(struct dma_chan *dma_chan)
171 static dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx)
173 struct dma_chan *c = tx->chan;
174 struct idxd_wq *wq = to_idxd_wq(c);
177 struct idxd_desc *desc = container_of(tx, struct idxd_desc, txd);
179 cookie = dma_cookie_assign(tx);
181 rc = idxd_submit_desc(wq, desc);
183 idxd_free_desc(wq, desc);
190 static void idxd_dma_release(struct dma_device *device)
192 struct idxd_dma_dev *idxd_dma = container_of(device, struct idxd_dma_dev, dma);
197 int idxd_register_dma_device(struct idxd_device *idxd)
199 struct idxd_dma_dev *idxd_dma;
200 struct dma_device *dma;
201 struct device *dev = &idxd->pdev->dev;
204 idxd_dma = kzalloc_node(sizeof(*idxd_dma), GFP_KERNEL, dev_to_node(dev));
208 dma = &idxd_dma->dma;
209 INIT_LIST_HEAD(&dma->channels);
212 dma_cap_set(DMA_INTERRUPT, dma->cap_mask);
213 dma_cap_set(DMA_PRIVATE, dma->cap_mask);
214 dma_cap_set(DMA_COMPLETION_NO_ORDER, dma->cap_mask);
215 dma->device_release = idxd_dma_release;
217 dma->device_prep_dma_interrupt = idxd_dma_prep_interrupt;
218 if (idxd->hw.opcap.bits[0] & IDXD_OPCAP_MEMMOVE) {
219 dma_cap_set(DMA_MEMCPY, dma->cap_mask);
220 dma->device_prep_dma_memcpy = idxd_dma_submit_memcpy;
223 dma->device_tx_status = idxd_dma_tx_status;
224 dma->device_issue_pending = idxd_dma_issue_pending;
225 dma->device_alloc_chan_resources = idxd_dma_alloc_chan_resources;
226 dma->device_free_chan_resources = idxd_dma_free_chan_resources;
228 rc = dma_async_device_register(dma);
234 idxd_dma->idxd = idxd;
236 * This pointer is protected by the refs taken by the dma_chan. It will remain valid
237 * as long as there are outstanding channels.
239 idxd->idxd_dma = idxd_dma;
243 void idxd_unregister_dma_device(struct idxd_device *idxd)
245 dma_async_device_unregister(&idxd->idxd_dma->dma);
248 int idxd_register_dma_channel(struct idxd_wq *wq)
250 struct idxd_device *idxd = wq->idxd;
251 struct dma_device *dma = &idxd->idxd_dma->dma;
252 struct device *dev = &idxd->pdev->dev;
253 struct idxd_dma_chan *idxd_chan;
254 struct dma_chan *chan;
257 idxd_chan = kzalloc_node(sizeof(*idxd_chan), GFP_KERNEL, dev_to_node(dev));
261 chan = &idxd_chan->chan;
263 list_add_tail(&chan->device_node, &dma->channels);
265 for (i = 0; i < wq->num_descs; i++) {
266 struct idxd_desc *desc = wq->descs[i];
268 dma_async_tx_descriptor_init(&desc->txd, chan);
269 desc->txd.tx_submit = idxd_dma_tx_submit;
272 rc = dma_async_device_channel_register(dma, chan);
278 wq->idxd_chan = idxd_chan;
280 get_device(&wq->conf_dev);
285 void idxd_unregister_dma_channel(struct idxd_wq *wq)
287 struct idxd_dma_chan *idxd_chan = wq->idxd_chan;
288 struct dma_chan *chan = &idxd_chan->chan;
289 struct idxd_dma_dev *idxd_dma = wq->idxd->idxd_dma;
291 dma_async_device_channel_unregister(&idxd_dma->dma, chan);
292 list_del(&chan->device_node);
293 kfree(wq->idxd_chan);
294 wq->idxd_chan = NULL;
295 put_device(&wq->conf_dev);