drivers/dma/idxd/cdev.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
   3 #include <linux/init.h>
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <linux/pci.h>
   7 #include <linux/device.h>
   8 #include <linux/sched/task.h>
   9 #include <linux/intel-svm.h>
  10 #include <linux/io-64-nonatomic-lo-hi.h>
  11 #include <linux/cdev.h>
  12 #include <linux/fs.h>
  13 #include <linux/poll.h>
  14 #include <uapi/linux/idxd.h>
  15 #include "registers.h"
  16 #include "idxd.h"
  17
  18 struct idxd_cdev_context {
  19         const char *name;
  20         dev_t devt;
  21         struct ida minor_ida;
  22 };
  23
  24 /*
  25  * ictx is an array based off of accelerator types. enum idxd_type
  26  * is used as index
  27  */
  28 static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {
  29         { .name = "dsa" },
  30 };
  31
  32 struct idxd_user_context {
  33         struct idxd_wq *wq;
  34         struct task_struct *task;
  35         unsigned int flags;
  36 };
  37
  38 static void idxd_cdev_dev_release(struct device *dev)
  39 {
  40         struct idxd_cdev *idxd_cdev = container_of(dev, struct idxd_cdev, dev);
  41         struct idxd_cdev_context *cdev_ctx;
  42         struct idxd_wq *wq = idxd_cdev->wq;
  43
  44         cdev_ctx = &ictx[wq->idxd->type];
  45         ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor);
  46         kfree(idxd_cdev);
  47 }
  48
  49 static struct device_type idxd_cdev_device_type = {
  50         .name = "idxd_cdev",
  51         .release = idxd_cdev_dev_release,
  52 };
  53
  54 static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode)
  55 {
  56         struct cdev *cdev = inode->i_cdev;
  57
  58         return container_of(cdev, struct idxd_cdev, cdev);
  59 }
  60
  61 static inline struct idxd_wq *inode_wq(struct inode *inode)
  62 {
  63         struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode);
  64
  65         return idxd_cdev->wq;
  66 }
  67
  68 static int idxd_cdev_open(struct inode *inode, struct file *filp)
  69 {
  70         struct idxd_user_context *ctx;
  71         struct idxd_device *idxd;
  72         struct idxd_wq *wq;
  73         struct device *dev;
  74         int rc = 0;
  75
  76         wq = inode_wq(inode);
  77         idxd = wq->idxd;
  78         dev = &idxd->pdev->dev;
  79
  80         dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq));
  81
  82         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
  83         if (!ctx)
  84                 return -ENOMEM;
  85
  86         mutex_lock(&wq->wq_lock);
  87
  88         if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) {
  89                 rc = -EBUSY;
  90                 goto failed;
  91         }
  92
  93         ctx->wq = wq;
  94         filp->private_data = ctx;
  95         idxd_wq_get(wq);
  96         mutex_unlock(&wq->wq_lock);
  97         return 0;
  98
  99  failed:
 100         mutex_unlock(&wq->wq_lock);
 101         kfree(ctx);
 102         return rc;
 103 }
 104
 105 static int idxd_cdev_release(struct inode *node, struct file *filep)
 106 {
 107         struct idxd_user_context *ctx = filep->private_data;
 108         struct idxd_wq *wq = ctx->wq;
 109         struct idxd_device *idxd = wq->idxd;
 110         struct device *dev = &idxd->pdev->dev;
 111
 112         dev_dbg(dev, "%s called\n", __func__);
 113         filep->private_data = NULL;
 114
 115         /* Wait for in-flight operations to complete. */
 116         idxd_wq_drain(wq);
 117
 118         kfree(ctx);
 119         mutex_lock(&wq->wq_lock);
 120         idxd_wq_put(wq);
 121         mutex_unlock(&wq->wq_lock);
 122         return 0;
 123 }
 124
 125 static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma,
 126                      const char *func)
 127 {
 128         struct device *dev = &wq->idxd->pdev->dev;
 129
 130         if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
 131                 dev_info_ratelimited(dev,
 132                                      "%s: %s: mapping too large: %lu\n",
 133                                      current->comm, func,
 134                                      vma->vm_end - vma->vm_start);
 135                 return -EINVAL;
 136         }
 137
 138         return 0;
 139 }
 140
 141 static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma)
 142 {
 143         struct idxd_user_context *ctx = filp->private_data;
 144         struct idxd_wq *wq = ctx->wq;
 145         struct idxd_device *idxd = wq->idxd;
 146         struct pci_dev *pdev = idxd->pdev;
 147         phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR);
 148         unsigned long pfn;
 149         int rc;
 150
 151         dev_dbg(&pdev->dev, "%s called\n", __func__);
 152         rc = check_vma(wq, vma, __func__);
 153         if (rc < 0)
 154                 return rc;
 155
 156         vma->vm_flags |= VM_DONTCOPY;
 157         pfn = (base + idxd_get_wq_portal_full_offset(wq->id,
 158                                 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT;
 159         vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 160         vma->vm_private_data = ctx;
 161
 162         return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,
 163                         vma->vm_page_prot);
 164 }
 165
 166 static __poll_t idxd_cdev_poll(struct file *filp,
 167                                struct poll_table_struct *wait)
 168 {
 169         struct idxd_user_context *ctx = filp->private_data;
 170         struct idxd_wq *wq = ctx->wq;
 171         struct idxd_device *idxd = wq->idxd;
 172         unsigned long flags;
 173         __poll_t out = 0;
 174
 175         poll_wait(filp, &wq->err_queue, wait);
 176         spin_lock_irqsave(&idxd->dev_lock, flags);
 177         if (idxd->sw_err.valid)
 178                 out = EPOLLIN | EPOLLRDNORM;
 179         spin_unlock_irqrestore(&idxd->dev_lock, flags);
 180
 181         return out;
 182 }
 183
 184 static const struct file_operations idxd_cdev_fops = {
 185         .owner = THIS_MODULE,
 186         .open = idxd_cdev_open,
 187         .release = idxd_cdev_release,
 188         .mmap = idxd_cdev_mmap,
 189         .poll = idxd_cdev_poll,
 190 };
 191
 192 int idxd_cdev_get_major(struct idxd_device *idxd)
 193 {
 194         return MAJOR(ictx[idxd->type].devt);
 195 }
 196
 197 int idxd_wq_add_cdev(struct idxd_wq *wq)
 198 {
 199         struct idxd_device *idxd = wq->idxd;
 200         struct idxd_cdev *idxd_cdev;
 201         struct cdev *cdev;
 202         struct device *dev;
 203         struct idxd_cdev_context *cdev_ctx;
 204         int rc, minor;
 205
 206         idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL);
 207         if (!idxd_cdev)
 208                 return -ENOMEM;
 209
 210         idxd_cdev->wq = wq;
 211         cdev = &idxd_cdev->cdev;
 212         dev = &idxd_cdev->dev;
 213         cdev_ctx = &ictx[wq->idxd->type];
 214         minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL);
 215         if (minor < 0) {
 216                 kfree(idxd_cdev);
 217                 return minor;
 218         }
 219         idxd_cdev->minor = minor;
 220
 221         device_initialize(dev);
 222         dev->parent = &wq->conf_dev;
 223         dev->bus = idxd_get_bus_type(idxd);
 224         dev->type = &idxd_cdev_device_type;
 225         dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
 226
 227         rc = dev_set_name(dev, "%s/wq%u.%u", idxd_get_dev_name(idxd),
 228                           idxd->id, wq->id);
 229         if (rc < 0)
 230                 goto err;
 231
 232         wq->idxd_cdev = idxd_cdev;
 233         cdev_init(cdev, &idxd_cdev_fops);
 234         rc = cdev_device_add(cdev, dev);
 235         if (rc) {
 236                 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);
 237                 goto err;
 238         }
 239
 240         return 0;
 241
 242  err:
 243         put_device(dev);
 244         wq->idxd_cdev = NULL;
 245         return rc;
 246 }
 247
 248 void idxd_wq_del_cdev(struct idxd_wq *wq)
 249 {
 250         struct idxd_cdev *idxd_cdev;
 251         struct idxd_cdev_context *cdev_ctx;
 252
 253         cdev_ctx = &ictx[wq->idxd->type];
 254         idxd_cdev = wq->idxd_cdev;
 255         wq->idxd_cdev = NULL;
 256         cdev_device_del(&idxd_cdev->cdev, &idxd_cdev->dev);
 257         put_device(&idxd_cdev->dev);
 258 }
 259
 260 int idxd_cdev_register(void)
 261 {
 262         int rc, i;
 263
 264         for (i = 0; i < IDXD_TYPE_MAX; i++) {
 265                 ida_init(&ictx[i].minor_ida);
 266                 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK,
 267                                          ictx[i].name);
 268                 if (rc)
 269                         goto err_free_chrdev_region;
 270         }
 271
 272         return 0;
 273
 274 err_free_chrdev_region:
 275         for (i--; i >= 0; i--)
 276                 unregister_chrdev_region(ictx[i].devt, MINORMASK);
 277
 278         return rc;
 279 }
 280
 281 void idxd_cdev_remove(void)
 282 {
 283         int i;
 284
 285         for (i = 0; i < IDXD_TYPE_MAX; i++) {
 286                 unregister_chrdev_region(ictx[i].devt, MINORMASK);
 287                 ida_destroy(&ictx[i].minor_ida);
 288         }
 289 }