1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
6 #include <linux/sbitmap.h>
7 #include <linux/dmaengine.h>
8 #include <linux/percpu-rwsem.h>
9 #include <linux/wait.h>
10 #include <linux/cdev.h>
11 #include "registers.h"
13 #define IDXD_DRIVER_VERSION "1.00"
15 extern struct kmem_cache *idxd_desc_pool;
20 #define IDXD_REG_TIMEOUT 50
21 #define IDXD_DRAIN_TIMEOUT 5000
24 IDXD_TYPE_UNKNOWN = -1,
29 #define IDXD_NAME_SIZE 128
31 struct idxd_device_driver {
32 struct device_driver drv;
35 struct idxd_irq_entry {
36 struct idxd_device *idxd;
38 struct llist_head pending_llist;
39 struct list_head work_list;
43 struct device conf_dev;
44 struct idxd_device *idxd;
56 #define IDXD_MAX_PRIORITY 0xf
64 WQ_FLAG_DEDICATED = 0,
80 #define IDXD_ALLOCATED_BATCH_SIZE 128U
81 #define WQ_NAME_SIZE 1024
82 #define WQ_TYPE_SIZE 10
89 enum idxd_complete_type {
90 IDXD_COMPLETE_NORMAL = 0,
94 struct idxd_dma_chan {
100 void __iomem *dportal;
101 struct device conf_dev;
102 struct idxd_cdev *idxd_cdev;
103 struct wait_queue_head err_queue;
104 struct idxd_device *idxd;
106 enum idxd_wq_type type;
107 struct idxd_group *group;
109 struct mutex wq_lock; /* mutex for workqueue */
113 enum idxd_wq_state state;
116 u32 vec_ptr; /* interrupt steering */
117 struct dsa_hw_desc **hw_descs;
119 struct dsa_completion_record *compls;
120 dma_addr_t compls_addr;
122 struct idxd_desc **descs;
123 struct sbitmap_queue sbq;
124 struct idxd_dma_chan *idxd_chan;
125 char name[WQ_NAME_SIZE + 1];
131 struct device conf_dev;
133 struct idxd_group *group;
134 struct idxd_device *idxd;
137 /* shadow registers */
140 union gen_cap_reg gen_cap;
141 union wq_cap_reg wq_cap;
142 union group_cap_reg group_cap;
143 union engine_cap_reg engine_cap;
147 enum idxd_device_state {
148 IDXD_DEV_HALTED = -1,
149 IDXD_DEV_DISABLED = 0,
154 enum idxd_device_flag {
155 IDXD_FLAG_CONFIGURABLE = 0,
156 IDXD_FLAG_CMD_RUNNING,
159 struct idxd_dma_dev {
160 struct idxd_device *idxd;
161 struct dma_device dma;
166 struct device conf_dev;
167 struct list_head list;
169 enum idxd_device_state state;
175 struct pci_dev *pdev;
176 void __iomem *reg_base;
178 spinlock_t dev_lock; /* spinlock for device */
179 struct completion *cmd_done;
180 struct idxd_group *groups;
182 struct idxd_engine *engines;
186 u32 msix_perm_offset;
199 int nr_tokens; /* non-reserved tokens */
200 unsigned int wqcfg_size;
202 union sw_err_reg sw_err;
203 wait_queue_head_t cmd_waitq;
204 struct msix_entry *msix_entries;
206 struct idxd_irq_entry *irq_entries;
208 struct idxd_dma_dev *idxd_dma;
209 struct workqueue_struct *wq;
210 struct work_struct work;
213 /* IDXD software descriptor */
215 struct dsa_hw_desc *hw;
217 struct dsa_completion_record *completion;
218 dma_addr_t compl_dma;
219 struct dma_async_tx_descriptor txd;
220 struct llist_node llnode;
221 struct list_head list;
227 #define confdev_to_idxd(dev) container_of(dev, struct idxd_device, conf_dev)
228 #define confdev_to_wq(dev) container_of(dev, struct idxd_wq, conf_dev)
230 extern struct bus_type dsa_bus_type;
232 static inline bool wq_dedicated(struct idxd_wq *wq)
234 return test_bit(WQ_FLAG_DEDICATED, &wq->flags);
237 enum idxd_portal_prot {
238 IDXD_PORTAL_UNLIMITED = 0,
242 static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot)
244 return prot * 0x1000;
247 static inline int idxd_get_wq_portal_full_offset(int wq_id,
248 enum idxd_portal_prot prot)
250 return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot);
253 static inline void idxd_set_type(struct idxd_device *idxd)
255 struct pci_dev *pdev = idxd->pdev;
257 if (pdev->device == PCI_DEVICE_ID_INTEL_DSA_SPR0)
258 idxd->type = IDXD_TYPE_DSA;
260 idxd->type = IDXD_TYPE_UNKNOWN;
263 static inline void idxd_wq_get(struct idxd_wq *wq)
268 static inline void idxd_wq_put(struct idxd_wq *wq)
273 static inline int idxd_wq_refcount(struct idxd_wq *wq)
275 return wq->client_count;
278 const char *idxd_get_dev_name(struct idxd_device *idxd);
279 int idxd_register_bus_type(void);
280 void idxd_unregister_bus_type(void);
281 int idxd_setup_sysfs(struct idxd_device *idxd);
282 void idxd_cleanup_sysfs(struct idxd_device *idxd);
283 int idxd_register_driver(void);
284 void idxd_unregister_driver(void);
285 struct bus_type *idxd_get_bus_type(struct idxd_device *idxd);
287 /* device interrupt control */
288 irqreturn_t idxd_irq_handler(int vec, void *data);
289 irqreturn_t idxd_misc_thread(int vec, void *data);
290 irqreturn_t idxd_wq_thread(int irq, void *data);
291 void idxd_mask_error_interrupts(struct idxd_device *idxd);
292 void idxd_unmask_error_interrupts(struct idxd_device *idxd);
293 void idxd_mask_msix_vectors(struct idxd_device *idxd);
294 void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id);
295 void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id);
298 int idxd_device_init_reset(struct idxd_device *idxd);
299 int idxd_device_enable(struct idxd_device *idxd);
300 int idxd_device_disable(struct idxd_device *idxd);
301 void idxd_device_reset(struct idxd_device *idxd);
302 void idxd_device_cleanup(struct idxd_device *idxd);
303 int idxd_device_config(struct idxd_device *idxd);
304 void idxd_device_wqs_clear_state(struct idxd_device *idxd);
306 /* work queue control */
307 int idxd_wq_alloc_resources(struct idxd_wq *wq);
308 void idxd_wq_free_resources(struct idxd_wq *wq);
309 int idxd_wq_enable(struct idxd_wq *wq);
310 int idxd_wq_disable(struct idxd_wq *wq);
311 void idxd_wq_drain(struct idxd_wq *wq);
312 void idxd_wq_reset(struct idxd_wq *wq);
313 int idxd_wq_map_portal(struct idxd_wq *wq);
314 void idxd_wq_unmap_portal(struct idxd_wq *wq);
315 void idxd_wq_disable_cleanup(struct idxd_wq *wq);
318 int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);
319 struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype);
320 void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc);
323 int idxd_register_dma_device(struct idxd_device *idxd);
324 void idxd_unregister_dma_device(struct idxd_device *idxd);
325 int idxd_register_dma_channel(struct idxd_wq *wq);
326 void idxd_unregister_dma_channel(struct idxd_wq *wq);
327 void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res);
328 void idxd_dma_complete_txd(struct idxd_desc *desc,
329 enum idxd_complete_type comp_type);
332 int idxd_cdev_register(void);
333 void idxd_cdev_remove(void);
334 int idxd_cdev_get_major(struct idxd_device *idxd);
335 int idxd_wq_add_cdev(struct idxd_wq *wq);
336 void idxd_wq_del_cdev(struct idxd_wq *wq);