drivers/dma/idxd/init.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
   3 #include <linux/init.h>
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <linux/slab.h>
   7 #include <linux/pci.h>
   8 #include <linux/interrupt.h>
   9 #include <linux/delay.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/workqueue.h>
  12 #include <linux/aer.h>
  13 #include <linux/fs.h>
  14 #include <linux/io-64-nonatomic-lo-hi.h>
  15 #include <linux/device.h>
  16 #include <linux/idr.h>
  17 #include <linux/intel-svm.h>
  18 #include <linux/iommu.h>
  19 #include <uapi/linux/idxd.h>
  20 #include <linux/dmaengine.h>
  21 #include "../dmaengine.h"
  22 #include "registers.h"
  23 #include "idxd.h"
  24 #include "perfmon.h"
  25
  26 MODULE_VERSION(IDXD_DRIVER_VERSION);
  27 MODULE_LICENSE("GPL v2");
  28 MODULE_AUTHOR("Intel Corporation");
  29
  30 static bool sva = true;
  31 module_param(sva, bool, 0644);
  32 MODULE_PARM_DESC(sva, "Toggle SVA support on/off");
  33
  34 #define DRV_NAME "idxd"
  35
  36 bool support_enqcmd;
  37 DEFINE_IDA(idxd_ida);
  38
  39 static struct idxd_driver_data idxd_driver_data[] = {
  40         [IDXD_TYPE_DSA] = {
  41                 .name_prefix = "dsa",
  42                 .type = IDXD_TYPE_DSA,
  43                 .compl_size = sizeof(struct dsa_completion_record),
  44                 .align = 32,
  45                 .dev_type = &dsa_device_type,
  46         },
  47         [IDXD_TYPE_IAX] = {
  48                 .name_prefix = "iax",
  49                 .type = IDXD_TYPE_IAX,
  50                 .compl_size = sizeof(struct iax_completion_record),
  51                 .align = 64,
  52                 .dev_type = &iax_device_type,
  53         },
  54 };
  55
  56 static struct pci_device_id idxd_pci_tbl[] = {
  57         /* DSA ver 1.0 platforms */
  58         { PCI_DEVICE_DATA(INTEL, DSA_SPR0, &idxd_driver_data[IDXD_TYPE_DSA]) },
  59
  60         /* IAX ver 1.0 platforms */
  61         { PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) },
  62         { 0, }
  63 };
  64 MODULE_DEVICE_TABLE(pci, idxd_pci_tbl);
  65
  66 static int idxd_setup_interrupts(struct idxd_device *idxd)
  67 {
  68         struct pci_dev *pdev = idxd->pdev;
  69         struct device *dev = &pdev->dev;
  70         struct idxd_irq_entry *irq_entry;
  71         int i, msixcnt;
  72         int rc = 0;
  73
  74         msixcnt = pci_msix_vec_count(pdev);
  75         if (msixcnt < 0) {
  76                 dev_err(dev, "Not MSI-X interrupt capable.\n");
  77                 return -ENOSPC;
  78         }
  79
  80         rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX);
  81         if (rc != msixcnt) {
  82                 dev_err(dev, "Failed enabling %d MSIX entries: %d\n", msixcnt, rc);
  83                 return -ENOSPC;
  84         }
  85         dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt);
  86
  87         /*
  88          * We implement 1 completion list per MSI-X entry except for
  89          * entry 0, which is for errors and others.
  90          */
  91         idxd->irq_entries = kcalloc_node(msixcnt, sizeof(struct idxd_irq_entry),
  92                                          GFP_KERNEL, dev_to_node(dev));
  93         if (!idxd->irq_entries) {
  94                 rc = -ENOMEM;
  95                 goto err_irq_entries;
  96         }
  97
  98         for (i = 0; i < msixcnt; i++) {
  99                 idxd->irq_entries[i].id = i;
 100                 idxd->irq_entries[i].idxd = idxd;
 101                 idxd->irq_entries[i].vector = pci_irq_vector(pdev, i);
 102                 spin_lock_init(&idxd->irq_entries[i].list_lock);
 103         }
 104
 105         idxd_msix_perm_setup(idxd);
 106
 107         irq_entry = &idxd->irq_entries[0];
 108         rc = request_threaded_irq(irq_entry->vector, NULL, idxd_misc_thread,
 109                                   0, "idxd-misc", irq_entry);
 110         if (rc < 0) {
 111                 dev_err(dev, "Failed to allocate misc interrupt.\n");
 112                 goto err_misc_irq;
 113         }
 114
 115         dev_dbg(dev, "Allocated idxd-misc handler on msix vector %d\n", irq_entry->vector);
 116
 117         /* first MSI-X entry is not for wq interrupts */
 118         idxd->num_wq_irqs = msixcnt - 1;
 119
 120         for (i = 1; i < msixcnt; i++) {
 121                 irq_entry = &idxd->irq_entries[i];
 122
 123                 init_llist_head(&idxd->irq_entries[i].pending_llist);
 124                 INIT_LIST_HEAD(&idxd->irq_entries[i].work_list);
 125                 rc = request_threaded_irq(irq_entry->vector, NULL,
 126                                           idxd_wq_thread, 0, "idxd-portal", irq_entry);
 127                 if (rc < 0) {
 128                         dev_err(dev, "Failed to allocate irq %d.\n", irq_entry->vector);
 129                         goto err_wq_irqs;
 130                 }
 131
 132                 dev_dbg(dev, "Allocated idxd-msix %d for vector %d\n", i, irq_entry->vector);
 133                 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) {
 134                         /*
 135                          * The MSIX vector enumeration starts at 1 with vector 0 being the
 136                          * misc interrupt that handles non I/O completion events. The
 137                          * interrupt handles are for IMS enumeration on guest. The misc
 138                          * interrupt vector does not require a handle and therefore we start
 139                          * the int_handles at index 0. Since 'i' starts at 1, the first
 140                          * int_handles index will be 0.
 141                          */
 142                         rc = idxd_device_request_int_handle(idxd, i, &idxd->int_handles[i - 1],
 143                                                             IDXD_IRQ_MSIX);
 144                         if (rc < 0) {
 145                                 free_irq(irq_entry->vector, irq_entry);
 146                                 goto err_wq_irqs;
 147                         }
 148                         dev_dbg(dev, "int handle requested: %u\n", idxd->int_handles[i - 1]);
 149                 }
 150         }
 151
 152         idxd_unmask_error_interrupts(idxd);
 153         return 0;
 154
 155  err_wq_irqs:
 156         while (--i >= 0) {
 157                 irq_entry = &idxd->irq_entries[i];
 158                 free_irq(irq_entry->vector, irq_entry);
 159                 if (i != 0)
 160                         idxd_device_release_int_handle(idxd,
 161                                                        idxd->int_handles[i], IDXD_IRQ_MSIX);
 162         }
 163  err_misc_irq:
 164         /* Disable error interrupt generation */
 165         idxd_mask_error_interrupts(idxd);
 166         idxd_msix_perm_clear(idxd);
 167  err_irq_entries:
 168         pci_free_irq_vectors(pdev);
 169         dev_err(dev, "No usable interrupts\n");
 170         return rc;
 171 }
 172
 173 static void idxd_cleanup_interrupts(struct idxd_device *idxd)
 174 {
 175         struct pci_dev *pdev = idxd->pdev;
 176         struct idxd_irq_entry *irq_entry;
 177         int i, msixcnt;
 178
 179         msixcnt = pci_msix_vec_count(pdev);
 180         if (msixcnt <= 0)
 181                 return;
 182
 183         irq_entry = &idxd->irq_entries[0];
 184         free_irq(irq_entry->vector, irq_entry);
 185
 186         for (i = 1; i < msixcnt; i++) {
 187
 188                 irq_entry = &idxd->irq_entries[i];
 189                 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE))
 190                         idxd_device_release_int_handle(idxd, idxd->int_handles[i],
 191                                                        IDXD_IRQ_MSIX);
 192                 free_irq(irq_entry->vector, irq_entry);
 193         }
 194
 195         idxd_mask_error_interrupts(idxd);
 196         pci_free_irq_vectors(pdev);
 197 }
 198
 199 static int idxd_setup_wqs(struct idxd_device *idxd)
 200 {
 201         struct device *dev = &idxd->pdev->dev;
 202         struct idxd_wq *wq;
 203         int i, rc;
 204
 205         idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *),
 206                                  GFP_KERNEL, dev_to_node(dev));
 207         if (!idxd->wqs)
 208                 return -ENOMEM;
 209
 210         for (i = 0; i < idxd->max_wqs; i++) {
 211                 wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev));
 212                 if (!wq) {
 213                         rc = -ENOMEM;
 214                         goto err;
 215                 }
 216
 217                 wq->id = i;
 218                 wq->idxd = idxd;
 219                 device_initialize(&wq->conf_dev);
 220                 wq->conf_dev.parent = &idxd->conf_dev;
 221                 wq->conf_dev.bus = &dsa_bus_type;
 222                 wq->conf_dev.type = &idxd_wq_device_type;
 223                 rc = dev_set_name(&wq->conf_dev, "wq%d.%d", idxd->id, wq->id);
 224                 if (rc < 0) {
 225                         put_device(&wq->conf_dev);
 226                         goto err;
 227                 }
 228
 229                 mutex_init(&wq->wq_lock);
 230                 init_waitqueue_head(&wq->err_queue);
 231                 init_completion(&wq->wq_dead);
 232                 wq->max_xfer_bytes = idxd->max_xfer_bytes;
 233                 wq->max_batch_size = idxd->max_batch_size;
 234                 wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev));
 235                 if (!wq->wqcfg) {
 236                         put_device(&wq->conf_dev);
 237                         rc = -ENOMEM;
 238                         goto err;
 239                 }
 240                 idxd->wqs[i] = wq;
 241         }
 242
 243         return 0;
 244
 245  err:
 246         while (--i >= 0)
 247                 put_device(&idxd->wqs[i]->conf_dev);
 248         return rc;
 249 }
 250
 251 static int idxd_setup_engines(struct idxd_device *idxd)
 252 {
 253         struct idxd_engine *engine;
 254         struct device *dev = &idxd->pdev->dev;
 255         int i, rc;
 256
 257         idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *),
 258                                      GFP_KERNEL, dev_to_node(dev));
 259         if (!idxd->engines)
 260                 return -ENOMEM;
 261
 262         for (i = 0; i < idxd->max_engines; i++) {
 263                 engine = kzalloc_node(sizeof(*engine), GFP_KERNEL, dev_to_node(dev));
 264                 if (!engine) {
 265                         rc = -ENOMEM;
 266                         goto err;
 267                 }
 268
 269                 engine->id = i;
 270                 engine->idxd = idxd;
 271                 device_initialize(&engine->conf_dev);
 272                 engine->conf_dev.parent = &idxd->conf_dev;
 273                 engine->conf_dev.bus = &dsa_bus_type;
 274                 engine->conf_dev.type = &idxd_engine_device_type;
 275                 rc = dev_set_name(&engine->conf_dev, "engine%d.%d", idxd->id, engine->id);
 276                 if (rc < 0) {
 277                         put_device(&engine->conf_dev);
 278                         goto err;
 279                 }
 280
 281                 idxd->engines[i] = engine;
 282         }
 283
 284         return 0;
 285
 286  err:
 287         while (--i >= 0)
 288                 put_device(&idxd->engines[i]->conf_dev);
 289         return rc;
 290 }
 291
 292 static int idxd_setup_groups(struct idxd_device *idxd)
 293 {
 294         struct device *dev = &idxd->pdev->dev;
 295         struct idxd_group *group;
 296         int i, rc;
 297
 298         idxd->groups = kcalloc_node(idxd->max_groups, sizeof(struct idxd_group *),
 299                                     GFP_KERNEL, dev_to_node(dev));
 300         if (!idxd->groups)
 301                 return -ENOMEM;
 302
 303         for (i = 0; i < idxd->max_groups; i++) {
 304                 group = kzalloc_node(sizeof(*group), GFP_KERNEL, dev_to_node(dev));
 305                 if (!group) {
 306                         rc = -ENOMEM;
 307                         goto err;
 308                 }
 309
 310                 group->id = i;
 311                 group->idxd = idxd;
 312                 device_initialize(&group->conf_dev);
 313                 group->conf_dev.parent = &idxd->conf_dev;
 314                 group->conf_dev.bus = &dsa_bus_type;
 315                 group->conf_dev.type = &idxd_group_device_type;
 316                 rc = dev_set_name(&group->conf_dev, "group%d.%d", idxd->id, group->id);
 317                 if (rc < 0) {
 318                         put_device(&group->conf_dev);
 319                         goto err;
 320                 }
 321
 322                 idxd->groups[i] = group;
 323                 group->tc_a = -1;
 324                 group->tc_b = -1;
 325         }
 326
 327         return 0;
 328
 329  err:
 330         while (--i >= 0)
 331                 put_device(&idxd->groups[i]->conf_dev);
 332         return rc;
 333 }
 334
 335 static void idxd_cleanup_internals(struct idxd_device *idxd)
 336 {
 337         int i;
 338
 339         for (i = 0; i < idxd->max_groups; i++)
 340                 put_device(&idxd->groups[i]->conf_dev);
 341         for (i = 0; i < idxd->max_engines; i++)
 342                 put_device(&idxd->engines[i]->conf_dev);
 343         for (i = 0; i < idxd->max_wqs; i++)
 344                 put_device(&idxd->wqs[i]->conf_dev);
 345         destroy_workqueue(idxd->wq);
 346 }
 347
 348 static int idxd_setup_internals(struct idxd_device *idxd)
 349 {
 350         struct device *dev = &idxd->pdev->dev;
 351         int rc, i;
 352
 353         init_waitqueue_head(&idxd->cmd_waitq);
 354
 355         if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) {
 356                 idxd->int_handles = devm_kcalloc(dev, idxd->max_wqs, sizeof(int), GFP_KERNEL);
 357                 if (!idxd->int_handles)
 358                         return -ENOMEM;
 359         }
 360
 361         rc = idxd_setup_wqs(idxd);
 362         if (rc < 0)
 363                 goto err_wqs;
 364
 365         rc = idxd_setup_engines(idxd);
 366         if (rc < 0)
 367                 goto err_engine;
 368
 369         rc = idxd_setup_groups(idxd);
 370         if (rc < 0)
 371                 goto err_group;
 372
 373         idxd->wq = create_workqueue(dev_name(dev));
 374         if (!idxd->wq) {
 375                 rc = -ENOMEM;
 376                 goto err_wkq_create;
 377         }
 378
 379         return 0;
 380
 381  err_wkq_create:
 382         for (i = 0; i < idxd->max_groups; i++)
 383                 put_device(&idxd->groups[i]->conf_dev);
 384  err_group:
 385         for (i = 0; i < idxd->max_engines; i++)
 386                 put_device(&idxd->engines[i]->conf_dev);
 387  err_engine:
 388         for (i = 0; i < idxd->max_wqs; i++)
 389                 put_device(&idxd->wqs[i]->conf_dev);
 390  err_wqs:
 391         kfree(idxd->int_handles);
 392         return rc;
 393 }
 394
 395 static void idxd_read_table_offsets(struct idxd_device *idxd)
 396 {
 397         union offsets_reg offsets;
 398         struct device *dev = &idxd->pdev->dev;
 399
 400         offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET);
 401         offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64));
 402         idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT;
 403         dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset);
 404         idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT;
 405         dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset);
 406         idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT;
 407         dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset);
 408         idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT;
 409         dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset);
 410 }
 411
 412 static void idxd_read_caps(struct idxd_device *idxd)
 413 {
 414         struct device *dev = &idxd->pdev->dev;
 415         int i;
 416
 417         /* reading generic capabilities */
 418         idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET);
 419         dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits);
 420
 421         if (idxd->hw.gen_cap.cmd_cap) {
 422                 idxd->hw.cmd_cap = ioread32(idxd->reg_base + IDXD_CMDCAP_OFFSET);
 423                 dev_dbg(dev, "cmd_cap: %#x\n", idxd->hw.cmd_cap);
 424         }
 425
 426         idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift;
 427         dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes);
 428         idxd->max_batch_size = 1U << idxd->hw.gen_cap.max_batch_shift;
 429         dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size);
 430         if (idxd->hw.gen_cap.config_en)
 431                 set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags);
 432
 433         /* reading group capabilities */
 434         idxd->hw.group_cap.bits =
 435                 ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET);
 436         dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits);
 437         idxd->max_groups = idxd->hw.group_cap.num_groups;
 438         dev_dbg(dev, "max groups: %u\n", idxd->max_groups);
 439         idxd->max_tokens = idxd->hw.group_cap.total_tokens;
 440         dev_dbg(dev, "max tokens: %u\n", idxd->max_tokens);
 441         idxd->nr_tokens = idxd->max_tokens;
 442
 443         /* read engine capabilities */
 444         idxd->hw.engine_cap.bits =
 445                 ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET);
 446         dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits);
 447         idxd->max_engines = idxd->hw.engine_cap.num_engines;
 448         dev_dbg(dev, "max engines: %u\n", idxd->max_engines);
 449
 450         /* read workqueue capabilities */
 451         idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET);
 452         dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits);
 453         idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size;
 454         dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size);
 455         idxd->max_wqs = idxd->hw.wq_cap.num_wqs;
 456         dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs);
 457         idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN);
 458         dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size);
 459
 460         /* reading operation capabilities */
 461         for (i = 0; i < 4; i++) {
 462                 idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base +
 463                                 IDXD_OPCAP_OFFSET + i * sizeof(u64));
 464                 dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]);
 465         }
 466 }
 467
 468 static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data)
 469 {
 470         struct device *dev = &pdev->dev;
 471         struct idxd_device *idxd;
 472         int rc;
 473
 474         idxd = kzalloc_node(sizeof(*idxd), GFP_KERNEL, dev_to_node(dev));
 475         if (!idxd)
 476                 return NULL;
 477
 478         idxd->pdev = pdev;
 479         idxd->data = data;
 480         idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL);
 481         if (idxd->id < 0)
 482                 return NULL;
 483
 484         device_initialize(&idxd->conf_dev);
 485         idxd->conf_dev.parent = dev;
 486         idxd->conf_dev.bus = &dsa_bus_type;
 487         idxd->conf_dev.type = idxd->data->dev_type;
 488         rc = dev_set_name(&idxd->conf_dev, "%s%d", idxd->data->name_prefix, idxd->id);
 489         if (rc < 0) {
 490                 put_device(&idxd->conf_dev);
 491                 return NULL;
 492         }
 493
 494         spin_lock_init(&idxd->dev_lock);
 495         spin_lock_init(&idxd->cmd_lock);
 496
 497         return idxd;
 498 }
 499
 500 static int idxd_enable_system_pasid(struct idxd_device *idxd)
 501 {
 502         int flags;
 503         unsigned int pasid;
 504         struct iommu_sva *sva;
 505
 506         flags = SVM_FLAG_SUPERVISOR_MODE;
 507
 508         sva = iommu_sva_bind_device(&idxd->pdev->dev, NULL, &flags);
 509         if (IS_ERR(sva)) {
 510                 dev_warn(&idxd->pdev->dev,
 511                          "iommu sva bind failed: %ld\n", PTR_ERR(sva));
 512                 return PTR_ERR(sva);
 513         }
 514
 515         pasid = iommu_sva_get_pasid(sva);
 516         if (pasid == IOMMU_PASID_INVALID) {
 517                 iommu_sva_unbind_device(sva);
 518                 return -ENODEV;
 519         }
 520
 521         idxd->sva = sva;
 522         idxd->pasid = pasid;
 523         dev_dbg(&idxd->pdev->dev, "system pasid: %u\n", pasid);
 524         return 0;
 525 }
 526
 527 static void idxd_disable_system_pasid(struct idxd_device *idxd)
 528 {
 529
 530         iommu_sva_unbind_device(idxd->sva);
 531         idxd->sva = NULL;
 532 }
 533
 534 static int idxd_probe(struct idxd_device *idxd)
 535 {
 536         struct pci_dev *pdev = idxd->pdev;
 537         struct device *dev = &pdev->dev;
 538         int rc;
 539
 540         dev_dbg(dev, "%s entered and resetting device\n", __func__);
 541         rc = idxd_device_init_reset(idxd);
 542         if (rc < 0)
 543                 return rc;
 544
 545         dev_dbg(dev, "IDXD reset complete\n");
 546
 547         if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) {
 548                 rc = iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA);
 549                 if (rc == 0) {
 550                         rc = idxd_enable_system_pasid(idxd);
 551                         if (rc < 0) {
 552                                 iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA);
 553                                 dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc);
 554                         } else {
 555                                 set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
 556                         }
 557                 } else {
 558                         dev_warn(dev, "Unable to turn on SVA feature.\n");
 559                 }
 560         } else if (!sva) {
 561                 dev_warn(dev, "User forced SVA off via module param.\n");
 562         }
 563
 564         idxd_read_caps(idxd);
 565         idxd_read_table_offsets(idxd);
 566
 567         rc = idxd_setup_internals(idxd);
 568         if (rc)
 569                 goto err;
 570
 571         /* If the configs are readonly, then load them from device */
 572         if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) {
 573                 dev_dbg(dev, "Loading RO device config\n");
 574                 rc = idxd_device_load_config(idxd);
 575                 if (rc < 0)
 576                         goto err_config;
 577         }
 578
 579         rc = idxd_setup_interrupts(idxd);
 580         if (rc)
 581                 goto err_config;
 582
 583         dev_dbg(dev, "IDXD interrupt setup complete.\n");
 584
 585         idxd->major = idxd_cdev_get_major(idxd);
 586
 587         rc = perfmon_pmu_init(idxd);
 588         if (rc < 0)
 589                 dev_warn(dev, "Failed to initialize perfmon. No PMU support: %d\n", rc);
 590
 591         dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id);
 592         return 0;
 593
 594  err_config:
 595         idxd_cleanup_internals(idxd);
 596  err:
 597         if (device_pasid_enabled(idxd))
 598                 idxd_disable_system_pasid(idxd);
 599         iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA);
 600         return rc;
 601 }
 602
 603 static void idxd_cleanup(struct idxd_device *idxd)
 604 {
 605         struct device *dev = &idxd->pdev->dev;
 606
 607         perfmon_pmu_remove(idxd);
 608         idxd_cleanup_interrupts(idxd);
 609         idxd_cleanup_internals(idxd);
 610         if (device_pasid_enabled(idxd))
 611                 idxd_disable_system_pasid(idxd);
 612         iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA);
 613 }
 614
 615 static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 616 {
 617         struct device *dev = &pdev->dev;
 618         struct idxd_device *idxd;
 619         struct idxd_driver_data *data = (struct idxd_driver_data *)id->driver_data;
 620         int rc;
 621
 622         rc = pci_enable_device(pdev);
 623         if (rc)
 624                 return rc;
 625
 626         dev_dbg(dev, "Alloc IDXD context\n");
 627         idxd = idxd_alloc(pdev, data);
 628         if (!idxd) {
 629                 rc = -ENOMEM;
 630                 goto err_idxd_alloc;
 631         }
 632
 633         dev_dbg(dev, "Mapping BARs\n");
 634         idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0);
 635         if (!idxd->reg_base) {
 636                 rc = -ENOMEM;
 637                 goto err_iomap;
 638         }
 639
 640         dev_dbg(dev, "Set DMA masks\n");
 641         rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
 642         if (rc)
 643                 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
 644         if (rc)
 645                 goto err;
 646
 647         rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
 648         if (rc)
 649                 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
 650         if (rc)
 651                 goto err;
 652
 653         dev_dbg(dev, "Set PCI master\n");
 654         pci_set_master(pdev);
 655         pci_set_drvdata(pdev, idxd);
 656
 657         idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET);
 658         rc = idxd_probe(idxd);
 659         if (rc) {
 660                 dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n");
 661                 goto err;
 662         }
 663
 664         rc = idxd_register_devices(idxd);
 665         if (rc) {
 666                 dev_err(dev, "IDXD sysfs setup failed\n");
 667                 goto err_dev_register;
 668         }
 669
 670         idxd->state = IDXD_DEV_CONF_READY;
 671
 672         dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n",
 673                  idxd->hw.version);
 674
 675         return 0;
 676
 677  err_dev_register:
 678         idxd_cleanup(idxd);
 679  err:
 680         pci_iounmap(pdev, idxd->reg_base);
 681  err_iomap:
 682         put_device(&idxd->conf_dev);
 683  err_idxd_alloc:
 684         pci_disable_device(pdev);
 685         return rc;
 686 }
 687
 688 static void idxd_flush_pending_llist(struct idxd_irq_entry *ie)
 689 {
 690         struct idxd_desc *desc, *itr;
 691         struct llist_node *head;
 692
 693         head = llist_del_all(&ie->pending_llist);
 694         if (!head)
 695                 return;
 696
 697         llist_for_each_entry_safe(desc, itr, head, llnode) {
 698                 idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT);
 699                 idxd_free_desc(desc->wq, desc);
 700         }
 701 }
 702
 703 static void idxd_flush_work_list(struct idxd_irq_entry *ie)
 704 {
 705         struct idxd_desc *desc, *iter;
 706
 707         list_for_each_entry_safe(desc, iter, &ie->work_list, list) {
 708                 list_del(&desc->list);
 709                 idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT);
 710                 idxd_free_desc(desc->wq, desc);
 711         }
 712 }
 713
 714 void idxd_wqs_quiesce(struct idxd_device *idxd)
 715 {
 716         struct idxd_wq *wq;
 717         int i;
 718
 719         for (i = 0; i < idxd->max_wqs; i++) {
 720                 wq = idxd->wqs[i];
 721                 if (wq->state == IDXD_WQ_ENABLED && wq->type == IDXD_WQT_KERNEL)
 722                         idxd_wq_quiesce(wq);
 723         }
 724 }
 725
 726 static void idxd_release_int_handles(struct idxd_device *idxd)
 727 {
 728         struct device *dev = &idxd->pdev->dev;
 729         int i, rc;
 730
 731         for (i = 0; i < idxd->num_wq_irqs; i++) {
 732                 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE)) {
 733                         rc = idxd_device_release_int_handle(idxd, idxd->int_handles[i],
 734                                                             IDXD_IRQ_MSIX);
 735                         if (rc < 0)
 736                                 dev_warn(dev, "irq handle %d release failed\n",
 737                                          idxd->int_handles[i]);
 738                         else
 739                                 dev_dbg(dev, "int handle requested: %u\n", idxd->int_handles[i]);
 740                 }
 741         }
 742 }
 743
 744 static void idxd_shutdown(struct pci_dev *pdev)
 745 {
 746         struct idxd_device *idxd = pci_get_drvdata(pdev);
 747         int rc, i;
 748         struct idxd_irq_entry *irq_entry;
 749         int msixcnt = pci_msix_vec_count(pdev);
 750
 751         rc = idxd_device_disable(idxd);
 752         if (rc)
 753                 dev_err(&pdev->dev, "Disabling device failed\n");
 754
 755         dev_dbg(&pdev->dev, "%s called\n", __func__);
 756         idxd_mask_msix_vectors(idxd);
 757         idxd_mask_error_interrupts(idxd);
 758
 759         for (i = 0; i < msixcnt; i++) {
 760                 irq_entry = &idxd->irq_entries[i];
 761                 synchronize_irq(irq_entry->vector);
 762                 if (i == 0)
 763                         continue;
 764                 idxd_flush_pending_llist(irq_entry);
 765                 idxd_flush_work_list(irq_entry);
 766         }
 767         flush_workqueue(idxd->wq);
 768 }
 769
 770 static void idxd_remove(struct pci_dev *pdev)
 771 {
 772         struct idxd_device *idxd = pci_get_drvdata(pdev);
 773         struct idxd_irq_entry *irq_entry;
 774         int msixcnt = pci_msix_vec_count(pdev);
 775         int i;
 776
 777         dev_dbg(&pdev->dev, "%s called\n", __func__);
 778         idxd_shutdown(pdev);
 779         if (device_pasid_enabled(idxd))
 780                 idxd_disable_system_pasid(idxd);
 781         idxd_unregister_devices(idxd);
 782
 783         for (i = 0; i < msixcnt; i++) {
 784                 irq_entry = &idxd->irq_entries[i];
 785                 free_irq(irq_entry->vector, irq_entry);
 786         }
 787         idxd_msix_perm_clear(idxd);
 788         idxd_release_int_handles(idxd);
 789         pci_free_irq_vectors(pdev);
 790         pci_iounmap(pdev, idxd->reg_base);
 791         iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA);
 792         pci_disable_device(pdev);
 793         destroy_workqueue(idxd->wq);
 794         perfmon_pmu_remove(idxd);
 795         device_unregister(&idxd->conf_dev);
 796 }
 797
 798 static struct pci_driver idxd_pci_driver = {
 799         .name           = DRV_NAME,
 800         .id_table       = idxd_pci_tbl,
 801         .probe          = idxd_pci_probe,
 802         .remove         = idxd_remove,
 803         .shutdown       = idxd_shutdown,
 804 };
 805
 806 static int __init idxd_init_module(void)
 807 {
 808         int err;
 809
 810         /*
 811          * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in
 812          * enumerating the device. We can not utilize it.
 813          */
 814         if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) {
 815                 pr_warn("idxd driver failed to load without MOVDIR64B.\n");
 816                 return -ENODEV;
 817         }
 818
 819         if (!cpu_feature_enabled(X86_FEATURE_ENQCMD))
 820                 pr_warn("Platform does not have ENQCMD(S) support.\n");
 821         else
 822                 support_enqcmd = true;
 823
 824         perfmon_init();
 825
 826         err = idxd_register_bus_type();
 827         if (err < 0)
 828                 return err;
 829
 830         err = idxd_register_driver();
 831         if (err < 0)
 832                 goto err_idxd_driver_register;
 833
 834         err = idxd_cdev_register();
 835         if (err)
 836                 goto err_cdev_register;
 837
 838         err = pci_register_driver(&idxd_pci_driver);
 839         if (err)
 840                 goto err_pci_register;
 841
 842         return 0;
 843
 844 err_pci_register:
 845         idxd_cdev_remove();
 846 err_cdev_register:
 847         idxd_unregister_driver();
 848 err_idxd_driver_register:
 849         idxd_unregister_bus_type();
 850         return err;
 851 }
 852 module_init(idxd_init_module);
 853
 854 static void __exit idxd_exit_module(void)
 855 {
 856         idxd_unregister_driver();
 857         pci_unregister_driver(&idxd_pci_driver);
 858         idxd_cdev_remove();
 859         idxd_unregister_bus_type();
 860         perfmon_exit();
 861 }
 862 module_exit(idxd_exit_module);