drivers/iommu/amd_iommu_v2.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2010-2012 Advanced Micro Devices, Inc.
   4  * Author: Joerg Roedel <jroedel@suse.de>
   5  */
   6
   7 #define pr_fmt(fmt)     "AMD-Vi: " fmt
   8
   9 #include <linux/mmu_notifier.h>
  10 #include <linux/amd-iommu.h>
  11 #include <linux/mm_types.h>
  12 #include <linux/profile.h>
  13 #include <linux/module.h>
  14 #include <linux/sched.h>
  15 #include <linux/sched/mm.h>
  16 #include <linux/iommu.h>
  17 #include <linux/wait.h>
  18 #include <linux/pci.h>
  19 #include <linux/gfp.h>
  20
  21 #include "amd_iommu_types.h"
  22 #include "amd_iommu_proto.h"
  23
  24 MODULE_LICENSE("GPL v2");
  25 MODULE_AUTHOR("Joerg Roedel <jroedel@suse.de>");
  26
  27 #define MAX_DEVICES             0x10000
  28 #define PRI_QUEUE_SIZE          512
  29
  30 struct pri_queue {
  31         atomic_t inflight;
  32         bool finish;
  33         int status;
  34 };
  35
  36 struct pasid_state {
  37         struct list_head list;                  /* For global state-list */
  38         atomic_t count;                         /* Reference count */
  39         unsigned mmu_notifier_count;            /* Counting nested mmu_notifier
  40                                                    calls */
  41         struct mm_struct *mm;                   /* mm_struct for the faults */
  42         struct mmu_notifier mn;                 /* mmu_notifier handle */
  43         struct pri_queue pri[PRI_QUEUE_SIZE];   /* PRI tag states */
  44         struct device_state *device_state;      /* Link to our device_state */
  45         int pasid;                              /* PASID index */
  46         bool invalid;                           /* Used during setup and
  47                                                    teardown of the pasid */
  48         spinlock_t lock;                        /* Protect pri_queues and
  49                                                    mmu_notifer_count */
  50         wait_queue_head_t wq;                   /* To wait for count == 0 */
  51 };
  52
  53 struct device_state {
  54         struct list_head list;
  55         u16 devid;
  56         atomic_t count;
  57         struct pci_dev *pdev;
  58         struct pasid_state **states;
  59         struct iommu_domain *domain;
  60         int pasid_levels;
  61         int max_pasids;
  62         amd_iommu_invalid_ppr_cb inv_ppr_cb;
  63         amd_iommu_invalidate_ctx inv_ctx_cb;
  64         spinlock_t lock;
  65         wait_queue_head_t wq;
  66 };
  67
  68 struct fault {
  69         struct work_struct work;
  70         struct device_state *dev_state;
  71         struct pasid_state *state;
  72         struct mm_struct *mm;
  73         u64 address;
  74         u16 devid;
  75         u16 pasid;
  76         u16 tag;
  77         u16 finish;
  78         u16 flags;
  79 };
  80
  81 static LIST_HEAD(state_list);
  82 static spinlock_t state_lock;
  83
  84 static struct workqueue_struct *iommu_wq;
  85
  86 static void free_pasid_states(struct device_state *dev_state);
  87
  88 static u16 device_id(struct pci_dev *pdev)
  89 {
  90         u16 devid;
  91
  92         devid = pdev->bus->number;
  93         devid = (devid << 8) | pdev->devfn;
  94
  95         return devid;
  96 }
  97
  98 static struct device_state *__get_device_state(u16 devid)
  99 {
 100         struct device_state *dev_state;
 101
 102         list_for_each_entry(dev_state, &state_list, list) {
 103                 if (dev_state->devid == devid)
 104                         return dev_state;
 105         }
 106
 107         return NULL;
 108 }
 109
 110 static struct device_state *get_device_state(u16 devid)
 111 {
 112         struct device_state *dev_state;
 113         unsigned long flags;
 114
 115         spin_lock_irqsave(&state_lock, flags);
 116         dev_state = __get_device_state(devid);
 117         if (dev_state != NULL)
 118                 atomic_inc(&dev_state->count);
 119         spin_unlock_irqrestore(&state_lock, flags);
 120
 121         return dev_state;
 122 }
 123
 124 static void free_device_state(struct device_state *dev_state)
 125 {
 126         struct iommu_group *group;
 127
 128         /*
 129          * First detach device from domain - No more PRI requests will arrive
 130          * from that device after it is unbound from the IOMMUv2 domain.
 131          */
 132         group = iommu_group_get(&dev_state->pdev->dev);
 133         if (WARN_ON(!group))
 134                 return;
 135
 136         iommu_detach_group(dev_state->domain, group);
 137
 138         iommu_group_put(group);
 139
 140         /* Everything is down now, free the IOMMUv2 domain */
 141         iommu_domain_free(dev_state->domain);
 142
 143         /* Finally get rid of the device-state */
 144         kfree(dev_state);
 145 }
 146
 147 static void put_device_state(struct device_state *dev_state)
 148 {
 149         if (atomic_dec_and_test(&dev_state->count))
 150                 wake_up(&dev_state->wq);
 151 }
 152
 153 /* Must be called under dev_state->lock */
 154 static struct pasid_state **__get_pasid_state_ptr(struct device_state *dev_state,
 155                                                   int pasid, bool alloc)
 156 {
 157         struct pasid_state **root, **ptr;
 158         int level, index;
 159
 160         level = dev_state->pasid_levels;
 161         root  = dev_state->states;
 162
 163         while (true) {
 164
 165                 index = (pasid >> (9 * level)) & 0x1ff;
 166                 ptr   = &root[index];
 167
 168                 if (level == 0)
 169                         break;
 170
 171                 if (*ptr == NULL) {
 172                         if (!alloc)
 173                                 return NULL;
 174
 175                         *ptr = (void *)get_zeroed_page(GFP_ATOMIC);
 176                         if (*ptr == NULL)
 177                                 return NULL;
 178                 }
 179
 180                 root   = (struct pasid_state **)*ptr;
 181                 level -= 1;
 182         }
 183
 184         return ptr;
 185 }
 186
 187 static int set_pasid_state(struct device_state *dev_state,
 188                            struct pasid_state *pasid_state,
 189                            int pasid)
 190 {
 191         struct pasid_state **ptr;
 192         unsigned long flags;
 193         int ret;
 194
 195         spin_lock_irqsave(&dev_state->lock, flags);
 196         ptr = __get_pasid_state_ptr(dev_state, pasid, true);
 197
 198         ret = -ENOMEM;
 199         if (ptr == NULL)
 200                 goto out_unlock;
 201
 202         ret = -ENOMEM;
 203         if (*ptr != NULL)
 204                 goto out_unlock;
 205
 206         *ptr = pasid_state;
 207
 208         ret = 0;
 209
 210 out_unlock:
 211         spin_unlock_irqrestore(&dev_state->lock, flags);
 212
 213         return ret;
 214 }
 215
 216 static void clear_pasid_state(struct device_state *dev_state, int pasid)
 217 {
 218         struct pasid_state **ptr;
 219         unsigned long flags;
 220
 221         spin_lock_irqsave(&dev_state->lock, flags);
 222         ptr = __get_pasid_state_ptr(dev_state, pasid, true);
 223
 224         if (ptr == NULL)
 225                 goto out_unlock;
 226
 227         *ptr = NULL;
 228
 229 out_unlock:
 230         spin_unlock_irqrestore(&dev_state->lock, flags);
 231 }
 232
 233 static struct pasid_state *get_pasid_state(struct device_state *dev_state,
 234                                            int pasid)
 235 {
 236         struct pasid_state **ptr, *ret = NULL;
 237         unsigned long flags;
 238
 239         spin_lock_irqsave(&dev_state->lock, flags);
 240         ptr = __get_pasid_state_ptr(dev_state, pasid, false);
 241
 242         if (ptr == NULL)
 243                 goto out_unlock;
 244
 245         ret = *ptr;
 246         if (ret)
 247                 atomic_inc(&ret->count);
 248
 249 out_unlock:
 250         spin_unlock_irqrestore(&dev_state->lock, flags);
 251
 252         return ret;
 253 }
 254
 255 static void free_pasid_state(struct pasid_state *pasid_state)
 256 {
 257         kfree(pasid_state);
 258 }
 259
 260 static void put_pasid_state(struct pasid_state *pasid_state)
 261 {
 262         if (atomic_dec_and_test(&pasid_state->count))
 263                 wake_up(&pasid_state->wq);
 264 }
 265
 266 static void put_pasid_state_wait(struct pasid_state *pasid_state)
 267 {
 268         atomic_dec(&pasid_state->count);
 269         wait_event(pasid_state->wq, !atomic_read(&pasid_state->count));
 270         free_pasid_state(pasid_state);
 271 }
 272
 273 static void unbind_pasid(struct pasid_state *pasid_state)
 274 {
 275         struct iommu_domain *domain;
 276
 277         domain = pasid_state->device_state->domain;
 278
 279         /*
 280          * Mark pasid_state as invalid, no more faults will we added to the
 281          * work queue after this is visible everywhere.
 282          */
 283         pasid_state->invalid = true;
 284
 285         /* Make sure this is visible */
 286         smp_wmb();
 287
 288         /* After this the device/pasid can't access the mm anymore */
 289         amd_iommu_domain_clear_gcr3(domain, pasid_state->pasid);
 290
 291         /* Make sure no more pending faults are in the queue */
 292         flush_workqueue(iommu_wq);
 293 }
 294
 295 static void free_pasid_states_level1(struct pasid_state **tbl)
 296 {
 297         int i;
 298
 299         for (i = 0; i < 512; ++i) {
 300                 if (tbl[i] == NULL)
 301                         continue;
 302
 303                 free_page((unsigned long)tbl[i]);
 304         }
 305 }
 306
 307 static void free_pasid_states_level2(struct pasid_state **tbl)
 308 {
 309         struct pasid_state **ptr;
 310         int i;
 311
 312         for (i = 0; i < 512; ++i) {
 313                 if (tbl[i] == NULL)
 314                         continue;
 315
 316                 ptr = (struct pasid_state **)tbl[i];
 317                 free_pasid_states_level1(ptr);
 318         }
 319 }
 320
 321 static void free_pasid_states(struct device_state *dev_state)
 322 {
 323         struct pasid_state *pasid_state;
 324         int i;
 325
 326         for (i = 0; i < dev_state->max_pasids; ++i) {
 327                 pasid_state = get_pasid_state(dev_state, i);
 328                 if (pasid_state == NULL)
 329                         continue;
 330
 331                 put_pasid_state(pasid_state);
 332
 333                 /*
 334                  * This will call the mn_release function and
 335                  * unbind the PASID
 336                  */
 337                 mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);
 338
 339                 put_pasid_state_wait(pasid_state); /* Reference taken in
 340                                                       amd_iommu_bind_pasid */
 341
 342                 /* Drop reference taken in amd_iommu_bind_pasid */
 343                 put_device_state(dev_state);
 344         }
 345
 346         if (dev_state->pasid_levels == 2)
 347                 free_pasid_states_level2(dev_state->states);
 348         else if (dev_state->pasid_levels == 1)
 349                 free_pasid_states_level1(dev_state->states);
 350         else
 351                 BUG_ON(dev_state->pasid_levels != 0);
 352
 353         free_page((unsigned long)dev_state->states);
 354 }
 355
 356 static struct pasid_state *mn_to_state(struct mmu_notifier *mn)
 357 {
 358         return container_of(mn, struct pasid_state, mn);
 359 }
 360
 361 static void mn_invalidate_range(struct mmu_notifier *mn,
 362                                 struct mm_struct *mm,
 363                                 unsigned long start, unsigned long end)
 364 {
 365         struct pasid_state *pasid_state;
 366         struct device_state *dev_state;
 367
 368         pasid_state = mn_to_state(mn);
 369         dev_state   = pasid_state->device_state;
 370
 371         if ((start ^ (end - 1)) < PAGE_SIZE)
 372                 amd_iommu_flush_page(dev_state->domain, pasid_state->pasid,
 373                                      start);
 374         else
 375                 amd_iommu_flush_tlb(dev_state->domain, pasid_state->pasid);
 376 }
 377
 378 static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm)
 379 {
 380         struct pasid_state *pasid_state;
 381         struct device_state *dev_state;
 382         bool run_inv_ctx_cb;
 383
 384         might_sleep();
 385
 386         pasid_state    = mn_to_state(mn);
 387         dev_state      = pasid_state->device_state;
 388         run_inv_ctx_cb = !pasid_state->invalid;
 389
 390         if (run_inv_ctx_cb && dev_state->inv_ctx_cb)
 391                 dev_state->inv_ctx_cb(dev_state->pdev, pasid_state->pasid);
 392
 393         unbind_pasid(pasid_state);
 394 }
 395
 396 static const struct mmu_notifier_ops iommu_mn = {
 397         .release                = mn_release,
 398         .invalidate_range       = mn_invalidate_range,
 399 };
 400
 401 static void set_pri_tag_status(struct pasid_state *pasid_state,
 402                                u16 tag, int status)
 403 {
 404         unsigned long flags;
 405
 406         spin_lock_irqsave(&pasid_state->lock, flags);
 407         pasid_state->pri[tag].status = status;
 408         spin_unlock_irqrestore(&pasid_state->lock, flags);
 409 }
 410
 411 static void finish_pri_tag(struct device_state *dev_state,
 412                            struct pasid_state *pasid_state,
 413                            u16 tag)
 414 {
 415         unsigned long flags;
 416
 417         spin_lock_irqsave(&pasid_state->lock, flags);
 418         if (atomic_dec_and_test(&pasid_state->pri[tag].inflight) &&
 419             pasid_state->pri[tag].finish) {
 420                 amd_iommu_complete_ppr(dev_state->pdev, pasid_state->pasid,
 421                                        pasid_state->pri[tag].status, tag);
 422                 pasid_state->pri[tag].finish = false;
 423                 pasid_state->pri[tag].status = PPR_SUCCESS;
 424         }
 425         spin_unlock_irqrestore(&pasid_state->lock, flags);
 426 }
 427
 428 static void handle_fault_error(struct fault *fault)
 429 {
 430         int status;
 431
 432         if (!fault->dev_state->inv_ppr_cb) {
 433                 set_pri_tag_status(fault->state, fault->tag, PPR_INVALID);
 434                 return;
 435         }
 436
 437         status = fault->dev_state->inv_ppr_cb(fault->dev_state->pdev,
 438                                               fault->pasid,
 439                                               fault->address,
 440                                               fault->flags);
 441         switch (status) {
 442         case AMD_IOMMU_INV_PRI_RSP_SUCCESS:
 443                 set_pri_tag_status(fault->state, fault->tag, PPR_SUCCESS);
 444                 break;
 445         case AMD_IOMMU_INV_PRI_RSP_INVALID:
 446                 set_pri_tag_status(fault->state, fault->tag, PPR_INVALID);
 447                 break;
 448         case AMD_IOMMU_INV_PRI_RSP_FAIL:
 449                 set_pri_tag_status(fault->state, fault->tag, PPR_FAILURE);
 450                 break;
 451         default:
 452                 BUG();
 453         }
 454 }
 455
 456 static bool access_error(struct vm_area_struct *vma, struct fault *fault)
 457 {
 458         unsigned long requested = 0;
 459
 460         if (fault->flags & PPR_FAULT_EXEC)
 461                 requested |= VM_EXEC;
 462
 463         if (fault->flags & PPR_FAULT_READ)
 464                 requested |= VM_READ;
 465
 466         if (fault->flags & PPR_FAULT_WRITE)
 467                 requested |= VM_WRITE;
 468
 469         return (requested & ~vma->vm_flags) != 0;
 470 }
 471
 472 static void do_fault(struct work_struct *work)
 473 {
 474         struct fault *fault = container_of(work, struct fault, work);
 475         struct vm_area_struct *vma;
 476         vm_fault_t ret = VM_FAULT_ERROR;
 477         unsigned int flags = 0;
 478         struct mm_struct *mm;
 479         u64 address;
 480
 481         mm = fault->state->mm;
 482         address = fault->address;
 483
 484         if (fault->flags & PPR_FAULT_USER)
 485                 flags |= FAULT_FLAG_USER;
 486         if (fault->flags & PPR_FAULT_WRITE)
 487                 flags |= FAULT_FLAG_WRITE;
 488         flags |= FAULT_FLAG_REMOTE;
 489
 490         down_read(&mm->mmap_sem);
 491         vma = find_extend_vma(mm, address);
 492         if (!vma || address < vma->vm_start)
 493                 /* failed to get a vma in the right range */
 494                 goto out;
 495
 496         /* Check if we have the right permissions on the vma */
 497         if (access_error(vma, fault))
 498                 goto out;
 499
 500         ret = handle_mm_fault(vma, address, flags);
 501 out:
 502         up_read(&mm->mmap_sem);
 503
 504         if (ret & VM_FAULT_ERROR)
 505                 /* failed to service fault */
 506                 handle_fault_error(fault);
 507
 508         finish_pri_tag(fault->dev_state, fault->state, fault->tag);
 509
 510         put_pasid_state(fault->state);
 511
 512         kfree(fault);
 513 }
 514
 515 static int ppr_notifier(struct notifier_block *nb, unsigned long e, void *data)
 516 {
 517         struct amd_iommu_fault *iommu_fault;
 518         struct pasid_state *pasid_state;
 519         struct device_state *dev_state;
 520         unsigned long flags;
 521         struct fault *fault;
 522         bool finish;
 523         u16 tag, devid;
 524         int ret;
 525         struct iommu_dev_data *dev_data;
 526         struct pci_dev *pdev = NULL;
 527
 528         iommu_fault = data;
 529         tag         = iommu_fault->tag & 0x1ff;
 530         finish      = (iommu_fault->tag >> 9) & 1;
 531
 532         devid = iommu_fault->device_id;
 533         pdev = pci_get_domain_bus_and_slot(0, PCI_BUS_NUM(devid),
 534                                            devid & 0xff);
 535         if (!pdev)
 536                 return -ENODEV;
 537         dev_data = get_dev_data(&pdev->dev);
 538
 539         /* In kdump kernel pci dev is not initialized yet -> send INVALID */
 540         ret = NOTIFY_DONE;
 541         if (translation_pre_enabled(amd_iommu_rlookup_table[devid])
 542                 && dev_data->defer_attach) {
 543                 amd_iommu_complete_ppr(pdev, iommu_fault->pasid,
 544                                        PPR_INVALID, tag);
 545                 goto out;
 546         }
 547
 548         dev_state = get_device_state(iommu_fault->device_id);
 549         if (dev_state == NULL)
 550                 goto out;
 551
 552         pasid_state = get_pasid_state(dev_state, iommu_fault->pasid);
 553         if (pasid_state == NULL || pasid_state->invalid) {
 554                 /* We know the device but not the PASID -> send INVALID */
 555                 amd_iommu_complete_ppr(dev_state->pdev, iommu_fault->pasid,
 556                                        PPR_INVALID, tag);
 557                 goto out_drop_state;
 558         }
 559
 560         spin_lock_irqsave(&pasid_state->lock, flags);
 561         atomic_inc(&pasid_state->pri[tag].inflight);
 562         if (finish)
 563                 pasid_state->pri[tag].finish = true;
 564         spin_unlock_irqrestore(&pasid_state->lock, flags);
 565
 566         fault = kzalloc(sizeof(*fault), GFP_ATOMIC);
 567         if (fault == NULL) {
 568                 /* We are OOM - send success and let the device re-fault */
 569                 finish_pri_tag(dev_state, pasid_state, tag);
 570                 goto out_drop_state;
 571         }
 572
 573         fault->dev_state = dev_state;
 574         fault->address   = iommu_fault->address;
 575         fault->state     = pasid_state;
 576         fault->tag       = tag;
 577         fault->finish    = finish;
 578         fault->pasid     = iommu_fault->pasid;
 579         fault->flags     = iommu_fault->flags;
 580         INIT_WORK(&fault->work, do_fault);
 581
 582         queue_work(iommu_wq, &fault->work);
 583
 584         ret = NOTIFY_OK;
 585
 586 out_drop_state:
 587
 588         if (ret != NOTIFY_OK && pasid_state)
 589                 put_pasid_state(pasid_state);
 590
 591         put_device_state(dev_state);
 592
 593 out:
 594         pci_dev_put(pdev);
 595         return ret;
 596 }
 597
 598 static struct notifier_block ppr_nb = {
 599         .notifier_call = ppr_notifier,
 600 };
 601
 602 int amd_iommu_bind_pasid(struct pci_dev *pdev, int pasid,
 603                          struct task_struct *task)
 604 {
 605         struct pasid_state *pasid_state;
 606         struct device_state *dev_state;
 607         struct mm_struct *mm;
 608         u16 devid;
 609         int ret;
 610
 611         might_sleep();
 612
 613         if (!amd_iommu_v2_supported())
 614                 return -ENODEV;
 615
 616         devid     = device_id(pdev);
 617         dev_state = get_device_state(devid);
 618
 619         if (dev_state == NULL)
 620                 return -EINVAL;
 621
 622         ret = -EINVAL;
 623         if (pasid < 0 || pasid >= dev_state->max_pasids)
 624                 goto out;
 625
 626         ret = -ENOMEM;
 627         pasid_state = kzalloc(sizeof(*pasid_state), GFP_KERNEL);
 628         if (pasid_state == NULL)
 629                 goto out;
 630
 631
 632         atomic_set(&pasid_state->count, 1);
 633         init_waitqueue_head(&pasid_state->wq);
 634         spin_lock_init(&pasid_state->lock);
 635
 636         mm                        = get_task_mm(task);
 637         pasid_state->mm           = mm;
 638         pasid_state->device_state = dev_state;
 639         pasid_state->pasid        = pasid;
 640         pasid_state->invalid      = true; /* Mark as valid only if we are
 641                                              done with setting up the pasid */
 642         pasid_state->mn.ops       = &iommu_mn;
 643
 644         if (pasid_state->mm == NULL)
 645                 goto out_free;
 646
 647         mmu_notifier_register(&pasid_state->mn, mm);
 648
 649         ret = set_pasid_state(dev_state, pasid_state, pasid);
 650         if (ret)
 651                 goto out_unregister;
 652
 653         ret = amd_iommu_domain_set_gcr3(dev_state->domain, pasid,
 654                                         __pa(pasid_state->mm->pgd));
 655         if (ret)
 656                 goto out_clear_state;
 657
 658         /* Now we are ready to handle faults */
 659         pasid_state->invalid = false;
 660
 661         /*
 662          * Drop the reference to the mm_struct here. We rely on the
 663          * mmu_notifier release call-back to inform us when the mm
 664          * is going away.
 665          */
 666         mmput(mm);
 667
 668         return 0;
 669
 670 out_clear_state:
 671         clear_pasid_state(dev_state, pasid);
 672
 673 out_unregister:
 674         mmu_notifier_unregister(&pasid_state->mn, mm);
 675         mmput(mm);
 676
 677 out_free:
 678         free_pasid_state(pasid_state);
 679
 680 out:
 681         put_device_state(dev_state);
 682
 683         return ret;
 684 }
 685 EXPORT_SYMBOL(amd_iommu_bind_pasid);
 686
 687 void amd_iommu_unbind_pasid(struct pci_dev *pdev, int pasid)
 688 {
 689         struct pasid_state *pasid_state;
 690         struct device_state *dev_state;
 691         u16 devid;
 692
 693         might_sleep();
 694
 695         if (!amd_iommu_v2_supported())
 696                 return;
 697
 698         devid = device_id(pdev);
 699         dev_state = get_device_state(devid);
 700         if (dev_state == NULL)
 701                 return;
 702
 703         if (pasid < 0 || pasid >= dev_state->max_pasids)
 704                 goto out;
 705
 706         pasid_state = get_pasid_state(dev_state, pasid);
 707         if (pasid_state == NULL)
 708                 goto out;
 709         /*
 710          * Drop reference taken here. We are safe because we still hold
 711          * the reference taken in the amd_iommu_bind_pasid function.
 712          */
 713         put_pasid_state(pasid_state);
 714
 715         /* Clear the pasid state so that the pasid can be re-used */
 716         clear_pasid_state(dev_state, pasid_state->pasid);
 717
 718         /*
 719          * Call mmu_notifier_unregister to drop our reference
 720          * to pasid_state->mm
 721          */
 722         mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);
 723
 724         put_pasid_state_wait(pasid_state); /* Reference taken in
 725                                               amd_iommu_bind_pasid */
 726 out:
 727         /* Drop reference taken in this function */
 728         put_device_state(dev_state);
 729
 730         /* Drop reference taken in amd_iommu_bind_pasid */
 731         put_device_state(dev_state);
 732 }
 733 EXPORT_SYMBOL(amd_iommu_unbind_pasid);
 734
 735 int amd_iommu_init_device(struct pci_dev *pdev, int pasids)
 736 {
 737         struct device_state *dev_state;
 738         struct iommu_group *group;
 739         unsigned long flags;
 740         int ret, tmp;
 741         u16 devid;
 742
 743         might_sleep();
 744
 745         /*
 746          * When memory encryption is active the device is likely not in a
 747          * direct-mapped domain. Forbid using IOMMUv2 functionality for now.
 748          */
 749         if (mem_encrypt_active())
 750                 return -ENODEV;
 751
 752         if (!amd_iommu_v2_supported())
 753                 return -ENODEV;
 754
 755         if (pasids <= 0 || pasids > (PASID_MASK + 1))
 756                 return -EINVAL;
 757
 758         devid = device_id(pdev);
 759
 760         dev_state = kzalloc(sizeof(*dev_state), GFP_KERNEL);
 761         if (dev_state == NULL)
 762                 return -ENOMEM;
 763
 764         spin_lock_init(&dev_state->lock);
 765         init_waitqueue_head(&dev_state->wq);
 766         dev_state->pdev  = pdev;
 767         dev_state->devid = devid;
 768
 769         tmp = pasids;
 770         for (dev_state->pasid_levels = 0; (tmp - 1) & ~0x1ff; tmp >>= 9)
 771                 dev_state->pasid_levels += 1;
 772
 773         atomic_set(&dev_state->count, 1);
 774         dev_state->max_pasids = pasids;
 775
 776         ret = -ENOMEM;
 777         dev_state->states = (void *)get_zeroed_page(GFP_KERNEL);
 778         if (dev_state->states == NULL)
 779                 goto out_free_dev_state;
 780
 781         dev_state->domain = iommu_domain_alloc(&pci_bus_type);
 782         if (dev_state->domain == NULL)
 783                 goto out_free_states;
 784
 785         amd_iommu_domain_direct_map(dev_state->domain);
 786
 787         ret = amd_iommu_domain_enable_v2(dev_state->domain, pasids);
 788         if (ret)
 789                 goto out_free_domain;
 790
 791         group = iommu_group_get(&pdev->dev);
 792         if (!group) {
 793                 ret = -EINVAL;
 794                 goto out_free_domain;
 795         }
 796
 797         ret = iommu_attach_group(dev_state->domain, group);
 798         if (ret != 0)
 799                 goto out_drop_group;
 800
 801         iommu_group_put(group);
 802
 803         spin_lock_irqsave(&state_lock, flags);
 804
 805         if (__get_device_state(devid) != NULL) {
 806                 spin_unlock_irqrestore(&state_lock, flags);
 807                 ret = -EBUSY;
 808                 goto out_free_domain;
 809         }
 810
 811         list_add_tail(&dev_state->list, &state_list);
 812
 813         spin_unlock_irqrestore(&state_lock, flags);
 814
 815         return 0;
 816
 817 out_drop_group:
 818         iommu_group_put(group);
 819
 820 out_free_domain:
 821         iommu_domain_free(dev_state->domain);
 822
 823 out_free_states:
 824         free_page((unsigned long)dev_state->states);
 825
 826 out_free_dev_state:
 827         kfree(dev_state);
 828
 829         return ret;
 830 }
 831 EXPORT_SYMBOL(amd_iommu_init_device);
 832
 833 void amd_iommu_free_device(struct pci_dev *pdev)
 834 {
 835         struct device_state *dev_state;
 836         unsigned long flags;
 837         u16 devid;
 838
 839         if (!amd_iommu_v2_supported())
 840                 return;
 841
 842         devid = device_id(pdev);
 843
 844         spin_lock_irqsave(&state_lock, flags);
 845
 846         dev_state = __get_device_state(devid);
 847         if (dev_state == NULL) {
 848                 spin_unlock_irqrestore(&state_lock, flags);
 849                 return;
 850         }
 851
 852         list_del(&dev_state->list);
 853
 854         spin_unlock_irqrestore(&state_lock, flags);
 855
 856         /* Get rid of any remaining pasid states */
 857         free_pasid_states(dev_state);
 858
 859         put_device_state(dev_state);
 860         /*
 861          * Wait until the last reference is dropped before freeing
 862          * the device state.
 863          */
 864         wait_event(dev_state->wq, !atomic_read(&dev_state->count));
 865         free_device_state(dev_state);
 866 }
 867 EXPORT_SYMBOL(amd_iommu_free_device);
 868
 869 int amd_iommu_set_invalid_ppr_cb(struct pci_dev *pdev,
 870                                  amd_iommu_invalid_ppr_cb cb)
 871 {
 872         struct device_state *dev_state;
 873         unsigned long flags;
 874         u16 devid;
 875         int ret;
 876
 877         if (!amd_iommu_v2_supported())
 878                 return -ENODEV;
 879
 880         devid = device_id(pdev);
 881
 882         spin_lock_irqsave(&state_lock, flags);
 883
 884         ret = -EINVAL;
 885         dev_state = __get_device_state(devid);
 886         if (dev_state == NULL)
 887                 goto out_unlock;
 888
 889         dev_state->inv_ppr_cb = cb;
 890
 891         ret = 0;
 892
 893 out_unlock:
 894         spin_unlock_irqrestore(&state_lock, flags);
 895
 896         return ret;
 897 }
 898 EXPORT_SYMBOL(amd_iommu_set_invalid_ppr_cb);
 899
 900 int amd_iommu_set_invalidate_ctx_cb(struct pci_dev *pdev,
 901                                     amd_iommu_invalidate_ctx cb)
 902 {
 903         struct device_state *dev_state;
 904         unsigned long flags;
 905         u16 devid;
 906         int ret;
 907
 908         if (!amd_iommu_v2_supported())
 909                 return -ENODEV;
 910
 911         devid = device_id(pdev);
 912
 913         spin_lock_irqsave(&state_lock, flags);
 914
 915         ret = -EINVAL;
 916         dev_state = __get_device_state(devid);
 917         if (dev_state == NULL)
 918                 goto out_unlock;
 919
 920         dev_state->inv_ctx_cb = cb;
 921
 922         ret = 0;
 923
 924 out_unlock:
 925         spin_unlock_irqrestore(&state_lock, flags);
 926
 927         return ret;
 928 }
 929 EXPORT_SYMBOL(amd_iommu_set_invalidate_ctx_cb);
 930
 931 static int __init amd_iommu_v2_init(void)
 932 {
 933         int ret;
 934
 935         pr_info("AMD IOMMUv2 driver by Joerg Roedel <jroedel@suse.de>\n");
 936
 937         if (!amd_iommu_v2_supported()) {
 938                 pr_info("AMD IOMMUv2 functionality not available on this system\n");
 939                 /*
 940                  * Load anyway to provide the symbols to other modules
 941                  * which may use AMD IOMMUv2 optionally.
 942                  */
 943                 return 0;
 944         }
 945
 946         spin_lock_init(&state_lock);
 947
 948         ret = -ENOMEM;
 949         iommu_wq = alloc_workqueue("amd_iommu_v2", WQ_MEM_RECLAIM, 0);
 950         if (iommu_wq == NULL)
 951                 goto out;
 952
 953         amd_iommu_register_ppr_notifier(&ppr_nb);
 954
 955         return 0;
 956
 957 out:
 958         return ret;
 959 }
 960
 961 static void __exit amd_iommu_v2_exit(void)
 962 {
 963         struct device_state *dev_state;
 964         int i;
 965
 966         if (!amd_iommu_v2_supported())
 967                 return;
 968
 969         amd_iommu_unregister_ppr_notifier(&ppr_nb);
 970
 971         flush_workqueue(iommu_wq);
 972
 973         /*
 974          * The loop below might call flush_workqueue(), so call
 975          * destroy_workqueue() after it
 976          */
 977         for (i = 0; i < MAX_DEVICES; ++i) {
 978                 dev_state = get_device_state(i);
 979
 980                 if (dev_state == NULL)
 981                         continue;
 982
 983                 WARN_ON_ONCE(1);
 984
 985                 put_device_state(dev_state);
 986                 amd_iommu_free_device(dev_state->pdev);
 987         }
 988
 989         destroy_workqueue(iommu_wq);
 990 }
 991
 992 module_init(amd_iommu_v2_init);
 993 module_exit(amd_iommu_v2_exit);