drivers/iommu/intel/dmar.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2006, Intel Corporation.
   4  *
   5  * Copyright (C) 2006-2008 Intel Corporation
   6  * Author: Ashok Raj <ashok.raj@intel.com>
   7  * Author: Shaohua Li <shaohua.li@intel.com>
   8  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
   9  *
  10  * This file implements early detection/parsing of Remapping Devices
  11  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  12  * tables.
  13  *
  14  * These routines are used by both DMA-remapping and Interrupt-remapping
  15  */
  16
  17 #define pr_fmt(fmt)     "DMAR: " fmt
  18
  19 #include <linux/pci.h>
  20 #include <linux/dmar.h>
  21 #include <linux/iova.h>
  22 #include <linux/intel-iommu.h>
  23 #include <linux/timer.h>
  24 #include <linux/irq.h>
  25 #include <linux/interrupt.h>
  26 #include <linux/tboot.h>
  27 #include <linux/dmi.h>
  28 #include <linux/slab.h>
  29 #include <linux/iommu.h>
  30 #include <linux/numa.h>
  31 #include <linux/limits.h>
  32 #include <asm/irq_remapping.h>
  33 #include <asm/iommu_table.h>
  34
  35 #include "../irq_remapping.h"
  36
  37 typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *);
  38 struct dmar_res_callback {
  39         dmar_res_handler_t      cb[ACPI_DMAR_TYPE_RESERVED];
  40         void                    *arg[ACPI_DMAR_TYPE_RESERVED];
  41         bool                    ignore_unhandled;
  42         bool                    print_entry;
  43 };
  44
  45 /*
  46  * Assumptions:
  47  * 1) The hotplug framework guarentees that DMAR unit will be hot-added
  48  *    before IO devices managed by that unit.
  49  * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
  50  *    after IO devices managed by that unit.
  51  * 3) Hotplug events are rare.
  52  *
  53  * Locking rules for DMA and interrupt remapping related global data structures:
  54  * 1) Use dmar_global_lock in process context
  55  * 2) Use RCU in interrupt context
  56  */
  57 DECLARE_RWSEM(dmar_global_lock);
  58 LIST_HEAD(dmar_drhd_units);
  59
  60 struct acpi_table_header * __initdata dmar_tbl;
  61 static int dmar_dev_scope_status = 1;
  62 static unsigned long dmar_seq_ids[BITS_TO_LONGS(DMAR_UNITS_SUPPORTED)];
  63
  64 static int alloc_iommu(struct dmar_drhd_unit *drhd);
  65 static void free_iommu(struct intel_iommu *iommu);
  66
  67 extern const struct iommu_ops intel_iommu_ops;
  68
  69 static void dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  70 {
  71         /*
  72          * add INCLUDE_ALL at the tail, so scan the list will find it at
  73          * the very end.
  74          */
  75         if (drhd->include_all)
  76                 list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
  77         else
  78                 list_add_rcu(&drhd->list, &dmar_drhd_units);
  79 }
  80
  81 void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
  82 {
  83         struct acpi_dmar_device_scope *scope;
  84
  85         *cnt = 0;
  86         while (start < end) {
  87                 scope = start;
  88                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
  89                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
  90                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
  91                         (*cnt)++;
  92                 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
  93                         scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
  94                         pr_warn("Unsupported device scope\n");
  95                 }
  96                 start += scope->length;
  97         }
  98         if (*cnt == 0)
  99                 return NULL;
 100
 101         return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
 102 }
 103
 104 void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
 105 {
 106         int i;
 107         struct device *tmp_dev;
 108
 109         if (*devices && *cnt) {
 110                 for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
 111                         put_device(tmp_dev);
 112                 kfree(*devices);
 113         }
 114
 115         *devices = NULL;
 116         *cnt = 0;
 117 }
 118
 119 /* Optimize out kzalloc()/kfree() for normal cases */
 120 static char dmar_pci_notify_info_buf[64];
 121
 122 static struct dmar_pci_notify_info *
 123 dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
 124 {
 125         int level = 0;
 126         size_t size;
 127         struct pci_dev *tmp;
 128         struct dmar_pci_notify_info *info;
 129
 130         BUG_ON(dev->is_virtfn);
 131
 132         /*
 133          * Ignore devices that have a domain number higher than what can
 134          * be looked up in DMAR, e.g. VMD subdevices with domain 0x10000
 135          */
 136         if (pci_domain_nr(dev->bus) > U16_MAX)
 137                 return NULL;
 138
 139         /* Only generate path[] for device addition event */
 140         if (event == BUS_NOTIFY_ADD_DEVICE)
 141                 for (tmp = dev; tmp; tmp = tmp->bus->self)
 142                         level++;
 143
 144         size = struct_size(info, path, level);
 145         if (size <= sizeof(dmar_pci_notify_info_buf)) {
 146                 info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
 147         } else {
 148                 info = kzalloc(size, GFP_KERNEL);
 149                 if (!info) {
 150                         pr_warn("Out of memory when allocating notify_info "
 151                                 "for %s.\n", pci_name(dev));
 152                         if (dmar_dev_scope_status == 0)
 153                                 dmar_dev_scope_status = -ENOMEM;
 154                         return NULL;
 155                 }
 156         }
 157
 158         info->event = event;
 159         info->dev = dev;
 160         info->seg = pci_domain_nr(dev->bus);
 161         info->level = level;
 162         if (event == BUS_NOTIFY_ADD_DEVICE) {
 163                 for (tmp = dev; tmp; tmp = tmp->bus->self) {
 164                         level--;
 165                         info->path[level].bus = tmp->bus->number;
 166                         info->path[level].device = PCI_SLOT(tmp->devfn);
 167                         info->path[level].function = PCI_FUNC(tmp->devfn);
 168                         if (pci_is_root_bus(tmp->bus))
 169                                 info->bus = tmp->bus->number;
 170                 }
 171         }
 172
 173         return info;
 174 }
 175
 176 static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
 177 {
 178         if ((void *)info != dmar_pci_notify_info_buf)
 179                 kfree(info);
 180 }
 181
 182 static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
 183                                 struct acpi_dmar_pci_path *path, int count)
 184 {
 185         int i;
 186
 187         if (info->bus != bus)
 188                 goto fallback;
 189         if (info->level != count)
 190                 goto fallback;
 191
 192         for (i = 0; i < count; i++) {
 193                 if (path[i].device != info->path[i].device ||
 194                     path[i].function != info->path[i].function)
 195                         goto fallback;
 196         }
 197
 198         return true;
 199
 200 fallback:
 201
 202         if (count != 1)
 203                 return false;
 204
 205         i = info->level - 1;
 206         if (bus              == info->path[i].bus &&
 207             path[0].device   == info->path[i].device &&
 208             path[0].function == info->path[i].function) {
 209                 pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
 210                         bus, path[0].device, path[0].function);
 211                 return true;
 212         }
 213
 214         return false;
 215 }
 216
 217 /* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
 218 int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
 219                           void *start, void*end, u16 segment,
 220                           struct dmar_dev_scope *devices,
 221                           int devices_cnt)
 222 {
 223         int i, level;
 224         struct device *tmp, *dev = &info->dev->dev;
 225         struct acpi_dmar_device_scope *scope;
 226         struct acpi_dmar_pci_path *path;
 227
 228         if (segment != info->seg)
 229                 return 0;
 230
 231         for (; start < end; start += scope->length) {
 232                 scope = start;
 233                 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
 234                     scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 235                         continue;
 236
 237                 path = (struct acpi_dmar_pci_path *)(scope + 1);
 238                 level = (scope->length - sizeof(*scope)) / sizeof(*path);
 239                 if (!dmar_match_pci_path(info, scope->bus, path, level))
 240                         continue;
 241
 242                 /*
 243                  * We expect devices with endpoint scope to have normal PCI
 244                  * headers, and devices with bridge scope to have bridge PCI
 245                  * headers.  However PCI NTB devices may be listed in the
 246                  * DMAR table with bridge scope, even though they have a
 247                  * normal PCI header.  NTB devices are identified by class
 248                  * "BRIDGE_OTHER" (0680h) - we don't declare a socpe mismatch
 249                  * for this special case.
 250                  */
 251                 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
 252                      info->dev->hdr_type != PCI_HEADER_TYPE_NORMAL) ||
 253                     (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE &&
 254                      (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
 255                       info->dev->class >> 16 != PCI_BASE_CLASS_BRIDGE))) {
 256                         pr_warn("Device scope type does not match for %s\n",
 257                                 pci_name(info->dev));
 258                         return -EINVAL;
 259                 }
 260
 261                 for_each_dev_scope(devices, devices_cnt, i, tmp)
 262                         if (tmp == NULL) {
 263                                 devices[i].bus = info->dev->bus->number;
 264                                 devices[i].devfn = info->dev->devfn;
 265                                 rcu_assign_pointer(devices[i].dev,
 266                                                    get_device(dev));
 267                                 return 1;
 268                         }
 269                 BUG_ON(i >= devices_cnt);
 270         }
 271
 272         return 0;
 273 }
 274
 275 int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
 276                           struct dmar_dev_scope *devices, int count)
 277 {
 278         int index;
 279         struct device *tmp;
 280
 281         if (info->seg != segment)
 282                 return 0;
 283
 284         for_each_active_dev_scope(devices, count, index, tmp)
 285                 if (tmp == &info->dev->dev) {
 286                         RCU_INIT_POINTER(devices[index].dev, NULL);
 287                         synchronize_rcu();
 288                         put_device(tmp);
 289                         return 1;
 290                 }
 291
 292         return 0;
 293 }
 294
 295 static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
 296 {
 297         int ret = 0;
 298         struct dmar_drhd_unit *dmaru;
 299         struct acpi_dmar_hardware_unit *drhd;
 300
 301         for_each_drhd_unit(dmaru) {
 302                 if (dmaru->include_all)
 303                         continue;
 304
 305                 drhd = container_of(dmaru->hdr,
 306                                     struct acpi_dmar_hardware_unit, header);
 307                 ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
 308                                 ((void *)drhd) + drhd->header.length,
 309                                 dmaru->segment,
 310                                 dmaru->devices, dmaru->devices_cnt);
 311                 if (ret)
 312                         break;
 313         }
 314         if (ret >= 0)
 315                 ret = dmar_iommu_notify_scope_dev(info);
 316         if (ret < 0 && dmar_dev_scope_status == 0)
 317                 dmar_dev_scope_status = ret;
 318
 319         if (ret >= 0)
 320                 intel_irq_remap_add_device(info);
 321
 322         return ret;
 323 }
 324
 325 static void  dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
 326 {
 327         struct dmar_drhd_unit *dmaru;
 328
 329         for_each_drhd_unit(dmaru)
 330                 if (dmar_remove_dev_scope(info, dmaru->segment,
 331                         dmaru->devices, dmaru->devices_cnt))
 332                         break;
 333         dmar_iommu_notify_scope_dev(info);
 334 }
 335
 336 static inline void vf_inherit_msi_domain(struct pci_dev *pdev)
 337 {
 338         struct pci_dev *physfn = pci_physfn(pdev);
 339
 340         dev_set_msi_domain(&pdev->dev, dev_get_msi_domain(&physfn->dev));
 341 }
 342
 343 static int dmar_pci_bus_notifier(struct notifier_block *nb,
 344                                  unsigned long action, void *data)
 345 {
 346         struct pci_dev *pdev = to_pci_dev(data);
 347         struct dmar_pci_notify_info *info;
 348
 349         /* Only care about add/remove events for physical functions.
 350          * For VFs we actually do the lookup based on the corresponding
 351          * PF in device_to_iommu() anyway. */
 352         if (pdev->is_virtfn) {
 353                 /*
 354                  * Ensure that the VF device inherits the irq domain of the
 355                  * PF device. Ideally the device would inherit the domain
 356                  * from the bus, but DMAR can have multiple units per bus
 357                  * which makes this impossible. The VF 'bus' could inherit
 358                  * from the PF device, but that's yet another x86'sism to
 359                  * inflict on everybody else.
 360                  */
 361                 if (action == BUS_NOTIFY_ADD_DEVICE)
 362                         vf_inherit_msi_domain(pdev);
 363                 return NOTIFY_DONE;
 364         }
 365
 366         if (action != BUS_NOTIFY_ADD_DEVICE &&
 367             action != BUS_NOTIFY_REMOVED_DEVICE)
 368                 return NOTIFY_DONE;
 369
 370         info = dmar_alloc_pci_notify_info(pdev, action);
 371         if (!info)
 372                 return NOTIFY_DONE;
 373
 374         down_write(&dmar_global_lock);
 375         if (action == BUS_NOTIFY_ADD_DEVICE)
 376                 dmar_pci_bus_add_dev(info);
 377         else if (action == BUS_NOTIFY_REMOVED_DEVICE)
 378                 dmar_pci_bus_del_dev(info);
 379         up_write(&dmar_global_lock);
 380
 381         dmar_free_pci_notify_info(info);
 382
 383         return NOTIFY_OK;
 384 }
 385
 386 static struct notifier_block dmar_pci_bus_nb = {
 387         .notifier_call = dmar_pci_bus_notifier,
 388         .priority = 1,
 389 };
 390
 391 static struct dmar_drhd_unit *
 392 dmar_find_dmaru(struct acpi_dmar_hardware_unit *drhd)
 393 {
 394         struct dmar_drhd_unit *dmaru;
 395
 396         list_for_each_entry_rcu(dmaru, &dmar_drhd_units, list,
 397                                 dmar_rcu_check())
 398                 if (dmaru->segment == drhd->segment &&
 399                     dmaru->reg_base_addr == drhd->address)
 400                         return dmaru;
 401
 402         return NULL;
 403 }
 404
 405 /*
 406  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 407  * structure which uniquely represent one DMA remapping hardware unit
 408  * present in the platform
 409  */
 410 static int dmar_parse_one_drhd(struct acpi_dmar_header *header, void *arg)
 411 {
 412         struct acpi_dmar_hardware_unit *drhd;
 413         struct dmar_drhd_unit *dmaru;
 414         int ret;
 415
 416         drhd = (struct acpi_dmar_hardware_unit *)header;
 417         dmaru = dmar_find_dmaru(drhd);
 418         if (dmaru)
 419                 goto out;
 420
 421         dmaru = kzalloc(sizeof(*dmaru) + header->length, GFP_KERNEL);
 422         if (!dmaru)
 423                 return -ENOMEM;
 424
 425         /*
 426          * If header is allocated from slab by ACPI _DSM method, we need to
 427          * copy the content because the memory buffer will be freed on return.
 428          */
 429         dmaru->hdr = (void *)(dmaru + 1);
 430         memcpy(dmaru->hdr, header, header->length);
 431         dmaru->reg_base_addr = drhd->address;
 432         dmaru->segment = drhd->segment;
 433         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 434         dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
 435                                               ((void *)drhd) + drhd->header.length,
 436                                               &dmaru->devices_cnt);
 437         if (dmaru->devices_cnt && dmaru->devices == NULL) {
 438                 kfree(dmaru);
 439                 return -ENOMEM;
 440         }
 441
 442         ret = alloc_iommu(dmaru);
 443         if (ret) {
 444                 dmar_free_dev_scope(&dmaru->devices,
 445                                     &dmaru->devices_cnt);
 446                 kfree(dmaru);
 447                 return ret;
 448         }
 449         dmar_register_drhd_unit(dmaru);
 450
 451 out:
 452         if (arg)
 453                 (*(int *)arg)++;
 454
 455         return 0;
 456 }
 457
 458 static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
 459 {
 460         if (dmaru->devices && dmaru->devices_cnt)
 461                 dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
 462         if (dmaru->iommu)
 463                 free_iommu(dmaru->iommu);
 464         kfree(dmaru);
 465 }
 466
 467 static int __init dmar_parse_one_andd(struct acpi_dmar_header *header,
 468                                       void *arg)
 469 {
 470         struct acpi_dmar_andd *andd = (void *)header;
 471
 472         /* Check for NUL termination within the designated length */
 473         if (strnlen(andd->device_name, header->length - 8) == header->length - 8) {
 474                 pr_warn(FW_BUG
 475                            "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
 476                            "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 477                            dmi_get_system_info(DMI_BIOS_VENDOR),
 478                            dmi_get_system_info(DMI_BIOS_VERSION),
 479                            dmi_get_system_info(DMI_PRODUCT_VERSION));
 480                 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 481                 return -EINVAL;
 482         }
 483         pr_info("ANDD device: %x name: %s\n", andd->device_number,
 484                 andd->device_name);
 485
 486         return 0;
 487 }
 488
 489 #ifdef CONFIG_ACPI_NUMA
 490 static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg)
 491 {
 492         struct acpi_dmar_rhsa *rhsa;
 493         struct dmar_drhd_unit *drhd;
 494
 495         rhsa = (struct acpi_dmar_rhsa *)header;
 496         for_each_drhd_unit(drhd) {
 497                 if (drhd->reg_base_addr == rhsa->base_address) {
 498                         int node = pxm_to_node(rhsa->proximity_domain);
 499
 500                         if (node != NUMA_NO_NODE && !node_online(node))
 501                                 node = NUMA_NO_NODE;
 502                         drhd->iommu->node = node;
 503                         return 0;
 504                 }
 505         }
 506         pr_warn(FW_BUG
 507                 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
 508                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 509                 rhsa->base_address,
 510                 dmi_get_system_info(DMI_BIOS_VENDOR),
 511                 dmi_get_system_info(DMI_BIOS_VERSION),
 512                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 513         add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 514
 515         return 0;
 516 }
 517 #else
 518 #define dmar_parse_one_rhsa             dmar_res_noop
 519 #endif
 520
 521 static void
 522 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 523 {
 524         struct acpi_dmar_hardware_unit *drhd;
 525         struct acpi_dmar_reserved_memory *rmrr;
 526         struct acpi_dmar_atsr *atsr;
 527         struct acpi_dmar_rhsa *rhsa;
 528
 529         switch (header->type) {
 530         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 531                 drhd = container_of(header, struct acpi_dmar_hardware_unit,
 532                                     header);
 533                 pr_info("DRHD base: %#016Lx flags: %#x\n",
 534                         (unsigned long long)drhd->address, drhd->flags);
 535                 break;
 536         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 537                 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
 538                                     header);
 539                 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
 540                         (unsigned long long)rmrr->base_address,
 541                         (unsigned long long)rmrr->end_address);
 542                 break;
 543         case ACPI_DMAR_TYPE_ROOT_ATS:
 544                 atsr = container_of(header, struct acpi_dmar_atsr, header);
 545                 pr_info("ATSR flags: %#x\n", atsr->flags);
 546                 break;
 547         case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
 548                 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
 549                 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
 550                        (unsigned long long)rhsa->base_address,
 551                        rhsa->proximity_domain);
 552                 break;
 553         case ACPI_DMAR_TYPE_NAMESPACE:
 554                 /* We don't print this here because we need to sanity-check
 555                    it first. So print it in dmar_parse_one_andd() instead. */
 556                 break;
 557         }
 558 }
 559
 560 /**
 561  * dmar_table_detect - checks to see if the platform supports DMAR devices
 562  */
 563 static int __init dmar_table_detect(void)
 564 {
 565         acpi_status status = AE_OK;
 566
 567         /* if we could find DMAR table, then there are DMAR devices */
 568         status = acpi_get_table(ACPI_SIG_DMAR, 0, &dmar_tbl);
 569
 570         if (ACPI_SUCCESS(status) && !dmar_tbl) {
 571                 pr_warn("Unable to map DMAR\n");
 572                 status = AE_NOT_FOUND;
 573         }
 574
 575         return ACPI_SUCCESS(status) ? 0 : -ENOENT;
 576 }
 577
 578 static int dmar_walk_remapping_entries(struct acpi_dmar_header *start,
 579                                        size_t len, struct dmar_res_callback *cb)
 580 {
 581         struct acpi_dmar_header *iter, *next;
 582         struct acpi_dmar_header *end = ((void *)start) + len;
 583
 584         for (iter = start; iter < end; iter = next) {
 585                 next = (void *)iter + iter->length;
 586                 if (iter->length == 0) {
 587                         /* Avoid looping forever on bad ACPI tables */
 588                         pr_debug(FW_BUG "Invalid 0-length structure\n");
 589                         break;
 590                 } else if (next > end) {
 591                         /* Avoid passing table end */
 592                         pr_warn(FW_BUG "Record passes table end\n");
 593                         return -EINVAL;
 594                 }
 595
 596                 if (cb->print_entry)
 597                         dmar_table_print_dmar_entry(iter);
 598
 599                 if (iter->type >= ACPI_DMAR_TYPE_RESERVED) {
 600                         /* continue for forward compatibility */
 601                         pr_debug("Unknown DMAR structure type %d\n",
 602                                  iter->type);
 603                 } else if (cb->cb[iter->type]) {
 604                         int ret;
 605
 606                         ret = cb->cb[iter->type](iter, cb->arg[iter->type]);
 607                         if (ret)
 608                                 return ret;
 609                 } else if (!cb->ignore_unhandled) {
 610                         pr_warn("No handler for DMAR structure type %d\n",
 611                                 iter->type);
 612                         return -EINVAL;
 613                 }
 614         }
 615
 616         return 0;
 617 }
 618
 619 static inline int dmar_walk_dmar_table(struct acpi_table_dmar *dmar,
 620                                        struct dmar_res_callback *cb)
 621 {
 622         return dmar_walk_remapping_entries((void *)(dmar + 1),
 623                         dmar->header.length - sizeof(*dmar), cb);
 624 }
 625
 626 /**
 627  * parse_dmar_table - parses the DMA reporting table
 628  */
 629 static int __init
 630 parse_dmar_table(void)
 631 {
 632         struct acpi_table_dmar *dmar;
 633         int drhd_count = 0;
 634         int ret;
 635         struct dmar_res_callback cb = {
 636                 .print_entry = true,
 637                 .ignore_unhandled = true,
 638                 .arg[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &drhd_count,
 639                 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_parse_one_drhd,
 640                 .cb[ACPI_DMAR_TYPE_RESERVED_MEMORY] = &dmar_parse_one_rmrr,
 641                 .cb[ACPI_DMAR_TYPE_ROOT_ATS] = &dmar_parse_one_atsr,
 642                 .cb[ACPI_DMAR_TYPE_HARDWARE_AFFINITY] = &dmar_parse_one_rhsa,
 643                 .cb[ACPI_DMAR_TYPE_NAMESPACE] = &dmar_parse_one_andd,
 644         };
 645
 646         /*
 647          * Do it again, earlier dmar_tbl mapping could be mapped with
 648          * fixed map.
 649          */
 650         dmar_table_detect();
 651
 652         /*
 653          * ACPI tables may not be DMA protected by tboot, so use DMAR copy
 654          * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
 655          */
 656         dmar_tbl = tboot_get_dmar_table(dmar_tbl);
 657
 658         dmar = (struct acpi_table_dmar *)dmar_tbl;
 659         if (!dmar)
 660                 return -ENODEV;
 661
 662         if (dmar->width < PAGE_SHIFT - 1) {
 663                 pr_warn("Invalid DMAR haw\n");
 664                 return -EINVAL;
 665         }
 666
 667         pr_info("Host address width %d\n", dmar->width + 1);
 668         ret = dmar_walk_dmar_table(dmar, &cb);
 669         if (ret == 0 && drhd_count == 0)
 670                 pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
 671
 672         return ret;
 673 }
 674
 675 static int dmar_pci_device_match(struct dmar_dev_scope devices[],
 676                                  int cnt, struct pci_dev *dev)
 677 {
 678         int index;
 679         struct device *tmp;
 680
 681         while (dev) {
 682                 for_each_active_dev_scope(devices, cnt, index, tmp)
 683                         if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
 684                                 return 1;
 685
 686                 /* Check our parent */
 687                 dev = dev->bus->self;
 688         }
 689
 690         return 0;
 691 }
 692
 693 struct dmar_drhd_unit *
 694 dmar_find_matched_drhd_unit(struct pci_dev *dev)
 695 {
 696         struct dmar_drhd_unit *dmaru;
 697         struct acpi_dmar_hardware_unit *drhd;
 698
 699         dev = pci_physfn(dev);
 700
 701         rcu_read_lock();
 702         for_each_drhd_unit(dmaru) {
 703                 drhd = container_of(dmaru->hdr,
 704                                     struct acpi_dmar_hardware_unit,
 705                                     header);
 706
 707                 if (dmaru->include_all &&
 708                     drhd->segment == pci_domain_nr(dev->bus))
 709                         goto out;
 710
 711                 if (dmar_pci_device_match(dmaru->devices,
 712                                           dmaru->devices_cnt, dev))
 713                         goto out;
 714         }
 715         dmaru = NULL;
 716 out:
 717         rcu_read_unlock();
 718
 719         return dmaru;
 720 }
 721
 722 static void __init dmar_acpi_insert_dev_scope(u8 device_number,
 723                                               struct acpi_device *adev)
 724 {
 725         struct dmar_drhd_unit *dmaru;
 726         struct acpi_dmar_hardware_unit *drhd;
 727         struct acpi_dmar_device_scope *scope;
 728         struct device *tmp;
 729         int i;
 730         struct acpi_dmar_pci_path *path;
 731
 732         for_each_drhd_unit(dmaru) {
 733                 drhd = container_of(dmaru->hdr,
 734                                     struct acpi_dmar_hardware_unit,
 735                                     header);
 736
 737                 for (scope = (void *)(drhd + 1);
 738                      (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
 739                      scope = ((void *)scope) + scope->length) {
 740                         if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
 741                                 continue;
 742                         if (scope->enumeration_id != device_number)
 743                                 continue;
 744
 745                         path = (void *)(scope + 1);
 746                         pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
 747                                 dev_name(&adev->dev), dmaru->reg_base_addr,
 748                                 scope->bus, path->device, path->function);
 749                         for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
 750                                 if (tmp == NULL) {
 751                                         dmaru->devices[i].bus = scope->bus;
 752                                         dmaru->devices[i].devfn = PCI_DEVFN(path->device,
 753                                                                             path->function);
 754                                         rcu_assign_pointer(dmaru->devices[i].dev,
 755                                                            get_device(&adev->dev));
 756                                         return;
 757                                 }
 758                         BUG_ON(i >= dmaru->devices_cnt);
 759                 }
 760         }
 761         pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
 762                 device_number, dev_name(&adev->dev));
 763 }
 764
 765 static int __init dmar_acpi_dev_scope_init(void)
 766 {
 767         struct acpi_dmar_andd *andd;
 768
 769         if (dmar_tbl == NULL)
 770                 return -ENODEV;
 771
 772         for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
 773              ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
 774              andd = ((void *)andd) + andd->header.length) {
 775                 if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
 776                         acpi_handle h;
 777                         struct acpi_device *adev;
 778
 779                         if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
 780                                                           andd->device_name,
 781                                                           &h))) {
 782                                 pr_err("Failed to find handle for ACPI object %s\n",
 783                                        andd->device_name);
 784                                 continue;
 785                         }
 786                         if (acpi_bus_get_device(h, &adev)) {
 787                                 pr_err("Failed to get device for ACPI object %s\n",
 788                                        andd->device_name);
 789                                 continue;
 790                         }
 791                         dmar_acpi_insert_dev_scope(andd->device_number, adev);
 792                 }
 793         }
 794         return 0;
 795 }
 796
 797 int __init dmar_dev_scope_init(void)
 798 {
 799         struct pci_dev *dev = NULL;
 800         struct dmar_pci_notify_info *info;
 801
 802         if (dmar_dev_scope_status != 1)
 803                 return dmar_dev_scope_status;
 804
 805         if (list_empty(&dmar_drhd_units)) {
 806                 dmar_dev_scope_status = -ENODEV;
 807         } else {
 808                 dmar_dev_scope_status = 0;
 809
 810                 dmar_acpi_dev_scope_init();
 811
 812                 for_each_pci_dev(dev) {
 813                         if (dev->is_virtfn)
 814                                 continue;
 815
 816                         info = dmar_alloc_pci_notify_info(dev,
 817                                         BUS_NOTIFY_ADD_DEVICE);
 818                         if (!info) {
 819                                 return dmar_dev_scope_status;
 820                         } else {
 821                                 dmar_pci_bus_add_dev(info);
 822                                 dmar_free_pci_notify_info(info);
 823                         }
 824                 }
 825         }
 826
 827         return dmar_dev_scope_status;
 828 }
 829
 830 void __init dmar_register_bus_notifier(void)
 831 {
 832         bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
 833 }
 834
 835
 836 int __init dmar_table_init(void)
 837 {
 838         static int dmar_table_initialized;
 839         int ret;
 840
 841         if (dmar_table_initialized == 0) {
 842                 ret = parse_dmar_table();
 843                 if (ret < 0) {
 844                         if (ret != -ENODEV)
 845                                 pr_info("Parse DMAR table failure.\n");
 846                 } else  if (list_empty(&dmar_drhd_units)) {
 847                         pr_info("No DMAR devices found\n");
 848                         ret = -ENODEV;
 849                 }
 850
 851                 if (ret < 0)
 852                         dmar_table_initialized = ret;
 853                 else
 854                         dmar_table_initialized = 1;
 855         }
 856
 857         return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
 858 }
 859
 860 static void warn_invalid_dmar(u64 addr, const char *message)
 861 {
 862         pr_warn_once(FW_BUG
 863                 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
 864                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 865                 addr, message,
 866                 dmi_get_system_info(DMI_BIOS_VENDOR),
 867                 dmi_get_system_info(DMI_BIOS_VERSION),
 868                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 869         add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 870 }
 871
 872 static int __ref
 873 dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
 874 {
 875         struct acpi_dmar_hardware_unit *drhd;
 876         void __iomem *addr;
 877         u64 cap, ecap;
 878
 879         drhd = (void *)entry;
 880         if (!drhd->address) {
 881                 warn_invalid_dmar(0, "");
 882                 return -EINVAL;
 883         }
 884
 885         if (arg)
 886                 addr = ioremap(drhd->address, VTD_PAGE_SIZE);
 887         else
 888                 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
 889         if (!addr) {
 890                 pr_warn("Can't validate DRHD address: %llx\n", drhd->address);
 891                 return -EINVAL;
 892         }
 893
 894         cap = dmar_readq(addr + DMAR_CAP_REG);
 895         ecap = dmar_readq(addr + DMAR_ECAP_REG);
 896
 897         if (arg)
 898                 iounmap(addr);
 899         else
 900                 early_iounmap(addr, VTD_PAGE_SIZE);
 901
 902         if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
 903                 warn_invalid_dmar(drhd->address, " returns all ones");
 904                 return -EINVAL;
 905         }
 906
 907         return 0;
 908 }
 909
 910 int __init detect_intel_iommu(void)
 911 {
 912         int ret;
 913         struct dmar_res_callback validate_drhd_cb = {
 914                 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_validate_one_drhd,
 915                 .ignore_unhandled = true,
 916         };
 917
 918         down_write(&dmar_global_lock);
 919         ret = dmar_table_detect();
 920         if (!ret)
 921                 ret = dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl,
 922                                            &validate_drhd_cb);
 923         if (!ret && !no_iommu && !iommu_detected &&
 924             (!dmar_disabled || dmar_platform_optin())) {
 925                 iommu_detected = 1;
 926                 /* Make sure ACS will be enabled */
 927                 pci_request_acs();
 928         }
 929
 930 #ifdef CONFIG_X86
 931         if (!ret) {
 932                 x86_init.iommu.iommu_init = intel_iommu_init;
 933                 x86_platform.iommu_shutdown = intel_iommu_shutdown;
 934         }
 935
 936 #endif
 937
 938         if (dmar_tbl) {
 939                 acpi_put_table(dmar_tbl);
 940                 dmar_tbl = NULL;
 941         }
 942         up_write(&dmar_global_lock);
 943
 944         return ret ? ret : 1;
 945 }
 946
 947 static void unmap_iommu(struct intel_iommu *iommu)
 948 {
 949         iounmap(iommu->reg);
 950         release_mem_region(iommu->reg_phys, iommu->reg_size);
 951 }
 952
 953 /**
 954  * map_iommu: map the iommu's registers
 955  * @iommu: the iommu to map
 956  * @phys_addr: the physical address of the base resgister
 957  *
 958  * Memory map the iommu's registers.  Start w/ a single page, and
 959  * possibly expand if that turns out to be insufficent.
 960  */
 961 static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
 962 {
 963         int map_size, err=0;
 964
 965         iommu->reg_phys = phys_addr;
 966         iommu->reg_size = VTD_PAGE_SIZE;
 967
 968         if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
 969                 pr_err("Can't reserve memory\n");
 970                 err = -EBUSY;
 971                 goto out;
 972         }
 973
 974         iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 975         if (!iommu->reg) {
 976                 pr_err("Can't map the region\n");
 977                 err = -ENOMEM;
 978                 goto release;
 979         }
 980
 981         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 982         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 983
 984         if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
 985                 err = -EINVAL;
 986                 warn_invalid_dmar(phys_addr, " returns all ones");
 987                 goto unmap;
 988         }
 989         if (ecap_vcs(iommu->ecap))
 990                 iommu->vccap = dmar_readq(iommu->reg + DMAR_VCCAP_REG);
 991
 992         /* the registers might be more than one page */
 993         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 994                          cap_max_fault_reg_offset(iommu->cap));
 995         map_size = VTD_PAGE_ALIGN(map_size);
 996         if (map_size > iommu->reg_size) {
 997                 iounmap(iommu->reg);
 998                 release_mem_region(iommu->reg_phys, iommu->reg_size);
 999                 iommu->reg_size = map_size;
1000                 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
1001                                         iommu->name)) {
1002                         pr_err("Can't reserve memory\n");
1003                         err = -EBUSY;
1004                         goto out;
1005                 }
1006                 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
1007                 if (!iommu->reg) {
1008                         pr_err("Can't map the region\n");
1009                         err = -ENOMEM;
1010                         goto release;
1011                 }
1012         }
1013         err = 0;
1014         goto out;
1015
1016 unmap:
1017         iounmap(iommu->reg);
1018 release:
1019         release_mem_region(iommu->reg_phys, iommu->reg_size);
1020 out:
1021         return err;
1022 }
1023
1024 static int dmar_alloc_seq_id(struct intel_iommu *iommu)
1025 {
1026         iommu->seq_id = find_first_zero_bit(dmar_seq_ids,
1027                                             DMAR_UNITS_SUPPORTED);
1028         if (iommu->seq_id >= DMAR_UNITS_SUPPORTED) {
1029                 iommu->seq_id = -1;
1030         } else {
1031                 set_bit(iommu->seq_id, dmar_seq_ids);
1032                 sprintf(iommu->name, "dmar%d", iommu->seq_id);
1033         }
1034
1035         return iommu->seq_id;
1036 }
1037
1038 static void dmar_free_seq_id(struct intel_iommu *iommu)
1039 {
1040         if (iommu->seq_id >= 0) {
1041                 clear_bit(iommu->seq_id, dmar_seq_ids);
1042                 iommu->seq_id = -1;
1043         }
1044 }
1045
1046 static int alloc_iommu(struct dmar_drhd_unit *drhd)
1047 {
1048         struct intel_iommu *iommu;
1049         u32 ver, sts;
1050         int agaw = -1;
1051         int msagaw = -1;
1052         int err;
1053
1054         if (!drhd->reg_base_addr) {
1055                 warn_invalid_dmar(0, "");
1056                 return -EINVAL;
1057         }
1058
1059         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
1060         if (!iommu)
1061                 return -ENOMEM;
1062
1063         if (dmar_alloc_seq_id(iommu) < 0) {
1064                 pr_err("Failed to allocate seq_id\n");
1065                 err = -ENOSPC;
1066                 goto error;
1067         }
1068
1069         err = map_iommu(iommu, drhd->reg_base_addr);
1070         if (err) {
1071                 pr_err("Failed to map %s\n", iommu->name);
1072                 goto error_free_seq_id;
1073         }
1074
1075         err = -EINVAL;
1076         if (cap_sagaw(iommu->cap) == 0) {
1077                 pr_info("%s: No supported address widths. Not attempting DMA translation.\n",
1078                         iommu->name);
1079                 drhd->ignored = 1;
1080         }
1081
1082         if (!drhd->ignored) {
1083                 agaw = iommu_calculate_agaw(iommu);
1084                 if (agaw < 0) {
1085                         pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
1086                                iommu->seq_id);
1087                         drhd->ignored = 1;
1088                 }
1089         }
1090         if (!drhd->ignored) {
1091                 msagaw = iommu_calculate_max_sagaw(iommu);
1092                 if (msagaw < 0) {
1093                         pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
1094                                iommu->seq_id);
1095                         drhd->ignored = 1;
1096                         agaw = -1;
1097                 }
1098         }
1099         iommu->agaw = agaw;
1100         iommu->msagaw = msagaw;
1101         iommu->segment = drhd->segment;
1102
1103         iommu->node = NUMA_NO_NODE;
1104
1105         ver = readl(iommu->reg + DMAR_VER_REG);
1106         pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
1107                 iommu->name,
1108                 (unsigned long long)drhd->reg_base_addr,
1109                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
1110                 (unsigned long long)iommu->cap,
1111                 (unsigned long long)iommu->ecap);
1112
1113         /* Reflect status in gcmd */
1114         sts = readl(iommu->reg + DMAR_GSTS_REG);
1115         if (sts & DMA_GSTS_IRES)
1116                 iommu->gcmd |= DMA_GCMD_IRE;
1117         if (sts & DMA_GSTS_TES)
1118                 iommu->gcmd |= DMA_GCMD_TE;
1119         if (sts & DMA_GSTS_QIES)
1120                 iommu->gcmd |= DMA_GCMD_QIE;
1121
1122         raw_spin_lock_init(&iommu->register_lock);
1123
1124         /*
1125          * This is only for hotplug; at boot time intel_iommu_enabled won't
1126          * be set yet. When intel_iommu_init() runs, it registers the units
1127          * present at boot time, then sets intel_iommu_enabled.
1128          */
1129         if (intel_iommu_enabled && !drhd->ignored) {
1130                 err = iommu_device_sysfs_add(&iommu->iommu, NULL,
1131                                              intel_iommu_groups,
1132                                              "%s", iommu->name);
1133                 if (err)
1134                         goto err_unmap;
1135
1136                 iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);
1137
1138                 err = iommu_device_register(&iommu->iommu);
1139                 if (err)
1140                         goto err_sysfs;
1141         }
1142
1143         drhd->iommu = iommu;
1144         iommu->drhd = drhd;
1145
1146         return 0;
1147
1148 err_sysfs:
1149         iommu_device_sysfs_remove(&iommu->iommu);
1150 err_unmap:
1151         unmap_iommu(iommu);
1152 error_free_seq_id:
1153         dmar_free_seq_id(iommu);
1154 error:
1155         kfree(iommu);
1156         return err;
1157 }
1158
1159 static void free_iommu(struct intel_iommu *iommu)
1160 {
1161         if (intel_iommu_enabled && !iommu->drhd->ignored) {
1162                 iommu_device_unregister(&iommu->iommu);
1163                 iommu_device_sysfs_remove(&iommu->iommu);
1164         }
1165
1166         if (iommu->irq) {
1167                 if (iommu->pr_irq) {
1168                         free_irq(iommu->pr_irq, iommu);
1169                         dmar_free_hwirq(iommu->pr_irq);
1170                         iommu->pr_irq = 0;
1171                 }
1172                 free_irq(iommu->irq, iommu);
1173                 dmar_free_hwirq(iommu->irq);
1174                 iommu->irq = 0;
1175         }
1176
1177         if (iommu->qi) {
1178                 free_page((unsigned long)iommu->qi->desc);
1179                 kfree(iommu->qi->desc_status);
1180                 kfree(iommu->qi);
1181         }
1182
1183         if (iommu->reg)
1184                 unmap_iommu(iommu);
1185
1186         dmar_free_seq_id(iommu);
1187         kfree(iommu);
1188 }
1189
1190 /*
1191  * Reclaim all the submitted descriptors which have completed its work.
1192  */
1193 static inline void reclaim_free_desc(struct q_inval *qi)
1194 {
1195         while (qi->desc_status[qi->free_tail] == QI_DONE ||
1196                qi->desc_status[qi->free_tail] == QI_ABORT) {
1197                 qi->desc_status[qi->free_tail] = QI_FREE;
1198                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1199                 qi->free_cnt++;
1200         }
1201 }
1202
1203 static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index)
1204 {
1205         u32 fault;
1206         int head, tail;
1207         struct q_inval *qi = iommu->qi;
1208         int shift = qi_shift(iommu);
1209
1210         if (qi->desc_status[wait_index] == QI_ABORT)
1211                 return -EAGAIN;
1212
1213         fault = readl(iommu->reg + DMAR_FSTS_REG);
1214
1215         /*
1216          * If IQE happens, the head points to the descriptor associated
1217          * with the error. No new descriptors are fetched until the IQE
1218          * is cleared.
1219          */
1220         if (fault & DMA_FSTS_IQE) {
1221                 head = readl(iommu->reg + DMAR_IQH_REG);
1222                 if ((head >> shift) == index) {
1223                         struct qi_desc *desc = qi->desc + head;
1224
1225                         /*
1226                          * desc->qw2 and desc->qw3 are either reserved or
1227                          * used by software as private data. We won't print
1228                          * out these two qw's for security consideration.
1229                          */
1230                         pr_err("VT-d detected invalid descriptor: qw0 = %llx, qw1 = %llx\n",
1231                                (unsigned long long)desc->qw0,
1232                                (unsigned long long)desc->qw1);
1233                         memcpy(desc, qi->desc + (wait_index << shift),
1234                                1 << shift);
1235                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1236                         return -EINVAL;
1237                 }
1238         }
1239
1240         /*
1241          * If ITE happens, all pending wait_desc commands are aborted.
1242          * No new descriptors are fetched until the ITE is cleared.
1243          */
1244         if (fault & DMA_FSTS_ITE) {
1245                 head = readl(iommu->reg + DMAR_IQH_REG);
1246                 head = ((head >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
1247                 head |= 1;
1248                 tail = readl(iommu->reg + DMAR_IQT_REG);
1249                 tail = ((tail >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
1250
1251                 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
1252
1253                 do {
1254                         if (qi->desc_status[head] == QI_IN_USE)
1255                                 qi->desc_status[head] = QI_ABORT;
1256                         head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1257                 } while (head != tail);
1258
1259                 if (qi->desc_status[wait_index] == QI_ABORT)
1260                         return -EAGAIN;
1261         }
1262
1263         if (fault & DMA_FSTS_ICE)
1264                 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
1265
1266         return 0;
1267 }
1268
1269 /*
1270  * Function to submit invalidation descriptors of all types to the queued
1271  * invalidation interface(QI). Multiple descriptors can be submitted at a
1272  * time, a wait descriptor will be appended to each submission to ensure
1273  * hardware has completed the invalidation before return. Wait descriptors
1274  * can be part of the submission but it will not be polled for completion.
1275  */
1276 int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc,
1277                    unsigned int count, unsigned long options)
1278 {
1279         struct q_inval *qi = iommu->qi;
1280         struct qi_desc wait_desc;
1281         int wait_index, index;
1282         unsigned long flags;
1283         int offset, shift;
1284         int rc, i;
1285
1286         if (!qi)
1287                 return 0;
1288
1289 restart:
1290         rc = 0;
1291
1292         raw_spin_lock_irqsave(&qi->q_lock, flags);
1293         /*
1294          * Check if we have enough empty slots in the queue to submit,
1295          * the calculation is based on:
1296          * # of desc + 1 wait desc + 1 space between head and tail
1297          */
1298         while (qi->free_cnt < count + 2) {
1299                 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1300                 cpu_relax();
1301                 raw_spin_lock_irqsave(&qi->q_lock, flags);
1302         }
1303
1304         index = qi->free_head;
1305         wait_index = (index + count) % QI_LENGTH;
1306         shift = qi_shift(iommu);
1307
1308         for (i = 0; i < count; i++) {
1309                 offset = ((index + i) % QI_LENGTH) << shift;
1310                 memcpy(qi->desc + offset, &desc[i], 1 << shift);
1311                 qi->desc_status[(index + i) % QI_LENGTH] = QI_IN_USE;
1312         }
1313         qi->desc_status[wait_index] = QI_IN_USE;
1314
1315         wait_desc.qw0 = QI_IWD_STATUS_DATA(QI_DONE) |
1316                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1317         if (options & QI_OPT_WAIT_DRAIN)
1318                 wait_desc.qw0 |= QI_IWD_PRQ_DRAIN;
1319         wait_desc.qw1 = virt_to_phys(&qi->desc_status[wait_index]);
1320         wait_desc.qw2 = 0;
1321         wait_desc.qw3 = 0;
1322
1323         offset = wait_index << shift;
1324         memcpy(qi->desc + offset, &wait_desc, 1 << shift);
1325
1326         qi->free_head = (qi->free_head + count + 1) % QI_LENGTH;
1327         qi->free_cnt -= count + 1;
1328
1329         /*
1330          * update the HW tail register indicating the presence of
1331          * new descriptors.
1332          */
1333         writel(qi->free_head << shift, iommu->reg + DMAR_IQT_REG);
1334
1335         while (qi->desc_status[wait_index] != QI_DONE) {
1336                 /*
1337                  * We will leave the interrupts disabled, to prevent interrupt
1338                  * context to queue another cmd while a cmd is already submitted
1339                  * and waiting for completion on this cpu. This is to avoid
1340                  * a deadlock where the interrupt context can wait indefinitely
1341                  * for free slots in the queue.
1342                  */
1343                 rc = qi_check_fault(iommu, index, wait_index);
1344                 if (rc)
1345                         break;
1346
1347                 raw_spin_unlock(&qi->q_lock);
1348                 cpu_relax();
1349                 raw_spin_lock(&qi->q_lock);
1350         }
1351
1352         for (i = 0; i < count; i++)
1353                 qi->desc_status[(index + i) % QI_LENGTH] = QI_DONE;
1354
1355         reclaim_free_desc(qi);
1356         raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1357
1358         if (rc == -EAGAIN)
1359                 goto restart;
1360
1361         return rc;
1362 }
1363
1364 /*
1365  * Flush the global interrupt entry cache.
1366  */
1367 void qi_global_iec(struct intel_iommu *iommu)
1368 {
1369         struct qi_desc desc;
1370
1371         desc.qw0 = QI_IEC_TYPE;
1372         desc.qw1 = 0;
1373         desc.qw2 = 0;
1374         desc.qw3 = 0;
1375
1376         /* should never fail */
1377         qi_submit_sync(iommu, &desc, 1, 0);
1378 }
1379
1380 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1381                       u64 type)
1382 {
1383         struct qi_desc desc;
1384
1385         desc.qw0 = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1386                         | QI_CC_GRAN(type) | QI_CC_TYPE;
1387         desc.qw1 = 0;
1388         desc.qw2 = 0;
1389         desc.qw3 = 0;
1390
1391         qi_submit_sync(iommu, &desc, 1, 0);
1392 }
1393
1394 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1395                     unsigned int size_order, u64 type)
1396 {
1397         u8 dw = 0, dr = 0;
1398
1399         struct qi_desc desc;
1400         int ih = 0;
1401
1402         if (cap_write_drain(iommu->cap))
1403                 dw = 1;
1404
1405         if (cap_read_drain(iommu->cap))
1406                 dr = 1;
1407
1408         desc.qw0 = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1409                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1410         desc.qw1 = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1411                 | QI_IOTLB_AM(size_order);
1412         desc.qw2 = 0;
1413         desc.qw3 = 0;
1414
1415         qi_submit_sync(iommu, &desc, 1, 0);
1416 }
1417
1418 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
1419                         u16 qdep, u64 addr, unsigned mask)
1420 {
1421         struct qi_desc desc;
1422
1423         if (mask) {
1424                 addr |= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1425                 desc.qw1 = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1426         } else
1427                 desc.qw1 = QI_DEV_IOTLB_ADDR(addr);
1428
1429         if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1430                 qdep = 0;
1431
1432         desc.qw0 = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1433                    QI_DIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid);
1434         desc.qw2 = 0;
1435         desc.qw3 = 0;
1436
1437         qi_submit_sync(iommu, &desc, 1, 0);
1438 }
1439
1440 /* PASID-based IOTLB invalidation */
1441 void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
1442                      unsigned long npages, bool ih)
1443 {
1444         struct qi_desc desc = {.qw2 = 0, .qw3 = 0};
1445
1446         /*
1447          * npages == -1 means a PASID-selective invalidation, otherwise,
1448          * a positive value for Page-selective-within-PASID invalidation.
1449          * 0 is not a valid input.
1450          */
1451         if (WARN_ON(!npages)) {
1452                 pr_err("Invalid input npages = %ld\n", npages);
1453                 return;
1454         }
1455
1456         if (npages == -1) {
1457                 desc.qw0 = QI_EIOTLB_PASID(pasid) |
1458                                 QI_EIOTLB_DID(did) |
1459                                 QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |
1460                                 QI_EIOTLB_TYPE;
1461                 desc.qw1 = 0;
1462         } else {
1463                 int mask = ilog2(__roundup_pow_of_two(npages));
1464                 unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask));
1465
1466                 if (WARN_ON_ONCE(!IS_ALIGNED(addr, align)))
1467                         addr = ALIGN_DOWN(addr, align);
1468
1469                 desc.qw0 = QI_EIOTLB_PASID(pasid) |
1470                                 QI_EIOTLB_DID(did) |
1471                                 QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) |
1472                                 QI_EIOTLB_TYPE;
1473                 desc.qw1 = QI_EIOTLB_ADDR(addr) |
1474                                 QI_EIOTLB_IH(ih) |
1475                                 QI_EIOTLB_AM(mask);
1476         }
1477
1478         qi_submit_sync(iommu, &desc, 1, 0);
1479 }
1480
1481 /* PASID-based device IOTLB Invalidate */
1482 void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,
1483                               u32 pasid,  u16 qdep, u64 addr, unsigned int size_order)
1484 {
1485         unsigned long mask = 1UL << (VTD_PAGE_SHIFT + size_order - 1);
1486         struct qi_desc desc = {.qw1 = 0, .qw2 = 0, .qw3 = 0};
1487
1488         desc.qw0 = QI_DEV_EIOTLB_PASID(pasid) | QI_DEV_EIOTLB_SID(sid) |
1489                 QI_DEV_EIOTLB_QDEP(qdep) | QI_DEIOTLB_TYPE |
1490                 QI_DEV_IOTLB_PFSID(pfsid);
1491
1492         /*
1493          * If S bit is 0, we only flush a single page. If S bit is set,
1494          * The least significant zero bit indicates the invalidation address
1495          * range. VT-d spec 6.5.2.6.
1496          * e.g. address bit 12[0] indicates 8KB, 13[0] indicates 16KB.
1497          * size order = 0 is PAGE_SIZE 4KB
1498          * Max Invs Pending (MIP) is set to 0 for now until we have DIT in
1499          * ECAP.
1500          */
1501         if (!IS_ALIGNED(addr, VTD_PAGE_SIZE << size_order))
1502                 pr_warn_ratelimited("Invalidate non-aligned address %llx, order %d\n",
1503                                     addr, size_order);
1504
1505         /* Take page address */
1506         desc.qw1 = QI_DEV_EIOTLB_ADDR(addr);
1507
1508         if (size_order) {
1509                 /*
1510                  * Existing 0s in address below size_order may be the least
1511                  * significant bit, we must set them to 1s to avoid having
1512                  * smaller size than desired.
1513                  */
1514                 desc.qw1 |= GENMASK_ULL(size_order + VTD_PAGE_SHIFT - 1,
1515                                         VTD_PAGE_SHIFT);
1516                 /* Clear size_order bit to indicate size */
1517                 desc.qw1 &= ~mask;
1518                 /* Set the S bit to indicate flushing more than 1 page */
1519                 desc.qw1 |= QI_DEV_EIOTLB_SIZE;
1520         }
1521
1522         qi_submit_sync(iommu, &desc, 1, 0);
1523 }
1524
1525 void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did,
1526                           u64 granu, u32 pasid)
1527 {
1528         struct qi_desc desc = {.qw1 = 0, .qw2 = 0, .qw3 = 0};
1529
1530         desc.qw0 = QI_PC_PASID(pasid) | QI_PC_DID(did) |
1531                         QI_PC_GRAN(granu) | QI_PC_TYPE;
1532         qi_submit_sync(iommu, &desc, 1, 0);
1533 }
1534
1535 /*
1536  * Disable Queued Invalidation interface.
1537  */
1538 void dmar_disable_qi(struct intel_iommu *iommu)
1539 {
1540         unsigned long flags;
1541         u32 sts;
1542         cycles_t start_time = get_cycles();
1543
1544         if (!ecap_qis(iommu->ecap))
1545                 return;
1546
1547         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1548
1549         sts =  readl(iommu->reg + DMAR_GSTS_REG);
1550         if (!(sts & DMA_GSTS_QIES))
1551                 goto end;
1552
1553         /*
1554          * Give a chance to HW to complete the pending invalidation requests.
1555          */
1556         while ((readl(iommu->reg + DMAR_IQT_REG) !=
1557                 readl(iommu->reg + DMAR_IQH_REG)) &&
1558                 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1559                 cpu_relax();
1560
1561         iommu->gcmd &= ~DMA_GCMD_QIE;
1562         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1563
1564         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1565                       !(sts & DMA_GSTS_QIES), sts);
1566 end:
1567         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1568 }
1569
1570 /*
1571  * Enable queued invalidation.
1572  */
1573 static void __dmar_enable_qi(struct intel_iommu *iommu)
1574 {
1575         u32 sts;
1576         unsigned long flags;
1577         struct q_inval *qi = iommu->qi;
1578         u64 val = virt_to_phys(qi->desc);
1579
1580         qi->free_head = qi->free_tail = 0;
1581         qi->free_cnt = QI_LENGTH;
1582
1583         /*
1584          * Set DW=1 and QS=1 in IQA_REG when Scalable Mode capability
1585          * is present.
1586          */
1587         if (ecap_smts(iommu->ecap))
1588                 val |= (1 << 11) | 1;
1589
1590         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1591
1592         /* write zero to the tail reg */
1593         writel(0, iommu->reg + DMAR_IQT_REG);
1594
1595         dmar_writeq(iommu->reg + DMAR_IQA_REG, val);
1596
1597         iommu->gcmd |= DMA_GCMD_QIE;
1598         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1599
1600         /* Make sure hardware complete it */
1601         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1602
1603         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1604 }
1605
1606 /*
1607  * Enable Queued Invalidation interface. This is a must to support
1608  * interrupt-remapping. Also used by DMA-remapping, which replaces
1609  * register based IOTLB invalidation.
1610  */
1611 int dmar_enable_qi(struct intel_iommu *iommu)
1612 {
1613         struct q_inval *qi;
1614         struct page *desc_page;
1615
1616         if (!ecap_qis(iommu->ecap))
1617                 return -ENOENT;
1618
1619         /*
1620          * queued invalidation is already setup and enabled.
1621          */
1622         if (iommu->qi)
1623                 return 0;
1624
1625         iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1626         if (!iommu->qi)
1627                 return -ENOMEM;
1628
1629         qi = iommu->qi;
1630
1631         /*
1632          * Need two pages to accommodate 256 descriptors of 256 bits each
1633          * if the remapping hardware supports scalable mode translation.
1634          */
1635         desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO,
1636                                      !!ecap_smts(iommu->ecap));
1637         if (!desc_page) {
1638                 kfree(qi);
1639                 iommu->qi = NULL;
1640                 return -ENOMEM;
1641         }
1642
1643         qi->desc = page_address(desc_page);
1644
1645         qi->desc_status = kcalloc(QI_LENGTH, sizeof(int), GFP_ATOMIC);
1646         if (!qi->desc_status) {
1647                 free_page((unsigned long) qi->desc);
1648                 kfree(qi);
1649                 iommu->qi = NULL;
1650                 return -ENOMEM;
1651         }
1652
1653         raw_spin_lock_init(&qi->q_lock);
1654
1655         __dmar_enable_qi(iommu);
1656
1657         return 0;
1658 }
1659
1660 /* iommu interrupt handling. Most stuff are MSI-like. */
1661
1662 enum faulttype {
1663         DMA_REMAP,
1664         INTR_REMAP,
1665         UNKNOWN,
1666 };
1667
1668 static const char *dma_remap_fault_reasons[] =
1669 {
1670         "Software",
1671         "Present bit in root entry is clear",
1672         "Present bit in context entry is clear",
1673         "Invalid context entry",
1674         "Access beyond MGAW",
1675         "PTE Write access is not set",
1676         "PTE Read access is not set",
1677         "Next page table ptr is invalid",
1678         "Root table address invalid",
1679         "Context table ptr is invalid",
1680         "non-zero reserved fields in RTP",
1681         "non-zero reserved fields in CTP",
1682         "non-zero reserved fields in PTE",
1683         "PCE for translation request specifies blocking",
1684 };
1685
1686 static const char * const dma_remap_sm_fault_reasons[] = {
1687         "SM: Invalid Root Table Address",
1688         "SM: TTM 0 for request with PASID",
1689         "SM: TTM 0 for page group request",
1690         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x33-0x37 */
1691         "SM: Error attempting to access Root Entry",
1692         "SM: Present bit in Root Entry is clear",
1693         "SM: Non-zero reserved field set in Root Entry",
1694         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x3B-0x3F */
1695         "SM: Error attempting to access Context Entry",
1696         "SM: Present bit in Context Entry is clear",
1697         "SM: Non-zero reserved field set in the Context Entry",
1698         "SM: Invalid Context Entry",
1699         "SM: DTE field in Context Entry is clear",
1700         "SM: PASID Enable field in Context Entry is clear",
1701         "SM: PASID is larger than the max in Context Entry",
1702         "SM: PRE field in Context-Entry is clear",
1703         "SM: RID_PASID field error in Context-Entry",
1704         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x49-0x4F */
1705         "SM: Error attempting to access the PASID Directory Entry",
1706         "SM: Present bit in Directory Entry is clear",
1707         "SM: Non-zero reserved field set in PASID Directory Entry",
1708         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x53-0x57 */
1709         "SM: Error attempting to access PASID Table Entry",
1710         "SM: Present bit in PASID Table Entry is clear",
1711         "SM: Non-zero reserved field set in PASID Table Entry",
1712         "SM: Invalid Scalable-Mode PASID Table Entry",
1713         "SM: ERE field is clear in PASID Table Entry",
1714         "SM: SRE field is clear in PASID Table Entry",
1715         "Unknown", "Unknown",/* 0x5E-0x5F */
1716         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x60-0x67 */
1717         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x68-0x6F */
1718         "SM: Error attempting to access first-level paging entry",
1719         "SM: Present bit in first-level paging entry is clear",
1720         "SM: Non-zero reserved field set in first-level paging entry",
1721         "SM: Error attempting to access FL-PML4 entry",
1722         "SM: First-level entry address beyond MGAW in Nested translation",
1723         "SM: Read permission error in FL-PML4 entry in Nested translation",
1724         "SM: Read permission error in first-level paging entry in Nested translation",
1725         "SM: Write permission error in first-level paging entry in Nested translation",
1726         "SM: Error attempting to access second-level paging entry",
1727         "SM: Read/Write permission error in second-level paging entry",
1728         "SM: Non-zero reserved field set in second-level paging entry",
1729         "SM: Invalid second-level page table pointer",
1730         "SM: A/D bit update needed in second-level entry when set up in no snoop",
1731         "Unknown", "Unknown", "Unknown", /* 0x7D-0x7F */
1732         "SM: Address in first-level translation is not canonical",
1733         "SM: U/S set 0 for first-level translation with user privilege",
1734         "SM: No execute permission for request with PASID and ER=1",
1735         "SM: Address beyond the DMA hardware max",
1736         "SM: Second-level entry address beyond the max",
1737         "SM: No write permission for Write/AtomicOp request",
1738         "SM: No read permission for Read/AtomicOp request",
1739         "SM: Invalid address-interrupt address",
1740         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x88-0x8F */
1741         "SM: A/D bit update needed in first-level entry when set up in no snoop",
1742 };
1743
1744 static const char *irq_remap_fault_reasons[] =
1745 {
1746         "Detected reserved fields in the decoded interrupt-remapped request",
1747         "Interrupt index exceeded the interrupt-remapping table size",
1748         "Present field in the IRTE entry is clear",
1749         "Error accessing interrupt-remapping table pointed by IRTA_REG",
1750         "Detected reserved fields in the IRTE entry",
1751         "Blocked a compatibility format interrupt request",
1752         "Blocked an interrupt request due to source-id verification failure",
1753 };
1754
1755 static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1756 {
1757         if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1758                                         ARRAY_SIZE(irq_remap_fault_reasons))) {
1759                 *fault_type = INTR_REMAP;
1760                 return irq_remap_fault_reasons[fault_reason - 0x20];
1761         } else if (fault_reason >= 0x30 && (fault_reason - 0x30 <
1762                         ARRAY_SIZE(dma_remap_sm_fault_reasons))) {
1763                 *fault_type = DMA_REMAP;
1764                 return dma_remap_sm_fault_reasons[fault_reason - 0x30];
1765         } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1766                 *fault_type = DMA_REMAP;
1767                 return dma_remap_fault_reasons[fault_reason];
1768         } else {
1769                 *fault_type = UNKNOWN;
1770                 return "Unknown";
1771         }
1772 }
1773
1774
1775 static inline int dmar_msi_reg(struct intel_iommu *iommu, int irq)
1776 {
1777         if (iommu->irq == irq)
1778                 return DMAR_FECTL_REG;
1779         else if (iommu->pr_irq == irq)
1780                 return DMAR_PECTL_REG;
1781         else
1782                 BUG();
1783 }
1784
1785 void dmar_msi_unmask(struct irq_data *data)
1786 {
1787         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1788         int reg = dmar_msi_reg(iommu, data->irq);
1789         unsigned long flag;
1790
1791         /* unmask it */
1792         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1793         writel(0, iommu->reg + reg);
1794         /* Read a reg to force flush the post write */
1795         readl(iommu->reg + reg);
1796         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1797 }
1798
1799 void dmar_msi_mask(struct irq_data *data)
1800 {
1801         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1802         int reg = dmar_msi_reg(iommu, data->irq);
1803         unsigned long flag;
1804
1805         /* mask it */
1806         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1807         writel(DMA_FECTL_IM, iommu->reg + reg);
1808         /* Read a reg to force flush the post write */
1809         readl(iommu->reg + reg);
1810         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1811 }
1812
1813 void dmar_msi_write(int irq, struct msi_msg *msg)
1814 {
1815         struct intel_iommu *iommu = irq_get_handler_data(irq);
1816         int reg = dmar_msi_reg(iommu, irq);
1817         unsigned long flag;
1818
1819         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1820         writel(msg->data, iommu->reg + reg + 4);
1821         writel(msg->address_lo, iommu->reg + reg + 8);
1822         writel(msg->address_hi, iommu->reg + reg + 12);
1823         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1824 }
1825
1826 void dmar_msi_read(int irq, struct msi_msg *msg)
1827 {
1828         struct intel_iommu *iommu = irq_get_handler_data(irq);
1829         int reg = dmar_msi_reg(iommu, irq);
1830         unsigned long flag;
1831
1832         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1833         msg->data = readl(iommu->reg + reg + 4);
1834         msg->address_lo = readl(iommu->reg + reg + 8);
1835         msg->address_hi = readl(iommu->reg + reg + 12);
1836         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1837 }
1838
1839 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1840                 u8 fault_reason, u32 pasid, u16 source_id,
1841                 unsigned long long addr)
1842 {
1843         const char *reason;
1844         int fault_type;
1845
1846         reason = dmar_get_fault_reason(fault_reason, &fault_type);
1847
1848         if (fault_type == INTR_REMAP)
1849                 pr_err("[INTR-REMAP] Request device [%02x:%02x.%d] fault index %llx [fault reason %02d] %s\n",
1850                         source_id >> 8, PCI_SLOT(source_id & 0xFF),
1851                         PCI_FUNC(source_id & 0xFF), addr >> 48,
1852                         fault_reason, reason);
1853         else
1854                 pr_err("[%s] Request device [%02x:%02x.%d] PASID %x fault addr %llx [fault reason %02d] %s\n",
1855                        type ? "DMA Read" : "DMA Write",
1856                        source_id >> 8, PCI_SLOT(source_id & 0xFF),
1857                        PCI_FUNC(source_id & 0xFF), pasid, addr,
1858                        fault_reason, reason);
1859         return 0;
1860 }
1861
1862 #define PRIMARY_FAULT_REG_LEN (16)
1863 irqreturn_t dmar_fault(int irq, void *dev_id)
1864 {
1865         struct intel_iommu *iommu = dev_id;
1866         int reg, fault_index;
1867         u32 fault_status;
1868         unsigned long flag;
1869         static DEFINE_RATELIMIT_STATE(rs,
1870                                       DEFAULT_RATELIMIT_INTERVAL,
1871                                       DEFAULT_RATELIMIT_BURST);
1872
1873         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1874         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1875         if (fault_status && __ratelimit(&rs))
1876                 pr_err("DRHD: handling fault status reg %x\n", fault_status);
1877
1878         /* TBD: ignore advanced fault log currently */
1879         if (!(fault_status & DMA_FSTS_PPF))
1880                 goto unlock_exit;
1881
1882         fault_index = dma_fsts_fault_record_index(fault_status);
1883         reg = cap_fault_reg_offset(iommu->cap);
1884         while (1) {
1885                 /* Disable printing, simply clear the fault when ratelimited */
1886                 bool ratelimited = !__ratelimit(&rs);
1887                 u8 fault_reason;
1888                 u16 source_id;
1889                 u64 guest_addr;
1890                 u32 pasid;
1891                 int type;
1892                 u32 data;
1893                 bool pasid_present;
1894
1895                 /* highest 32 bits */
1896                 data = readl(iommu->reg + reg +
1897                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1898                 if (!(data & DMA_FRCD_F))
1899                         break;
1900
1901                 if (!ratelimited) {
1902                         fault_reason = dma_frcd_fault_reason(data);
1903                         type = dma_frcd_type(data);
1904
1905                         pasid = dma_frcd_pasid_value(data);
1906                         data = readl(iommu->reg + reg +
1907                                      fault_index * PRIMARY_FAULT_REG_LEN + 8);
1908                         source_id = dma_frcd_source_id(data);
1909
1910                         pasid_present = dma_frcd_pasid_present(data);
1911                         guest_addr = dmar_readq(iommu->reg + reg +
1912                                         fault_index * PRIMARY_FAULT_REG_LEN);
1913                         guest_addr = dma_frcd_page_addr(guest_addr);
1914                 }
1915
1916                 /* clear the fault */
1917                 writel(DMA_FRCD_F, iommu->reg + reg +
1918                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
1919
1920                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1921
1922                 if (!ratelimited)
1923                         /* Using pasid -1 if pasid is not present */
1924                         dmar_fault_do_one(iommu, type, fault_reason,
1925                                           pasid_present ? pasid : -1,
1926                                           source_id, guest_addr);
1927
1928                 fault_index++;
1929                 if (fault_index >= cap_num_fault_regs(iommu->cap))
1930                         fault_index = 0;
1931                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1932         }
1933
1934         writel(DMA_FSTS_PFO | DMA_FSTS_PPF | DMA_FSTS_PRO,
1935                iommu->reg + DMAR_FSTS_REG);
1936
1937 unlock_exit:
1938         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1939         return IRQ_HANDLED;
1940 }
1941
1942 int dmar_set_interrupt(struct intel_iommu *iommu)
1943 {
1944         int irq, ret;
1945
1946         /*
1947          * Check if the fault interrupt is already initialized.
1948          */
1949         if (iommu->irq)
1950                 return 0;
1951
1952         irq = dmar_alloc_hwirq(iommu->seq_id, iommu->node, iommu);
1953         if (irq > 0) {
1954                 iommu->irq = irq;
1955         } else {
1956                 pr_err("No free IRQ vectors\n");
1957                 return -EINVAL;
1958         }
1959
1960         ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1961         if (ret)
1962                 pr_err("Can't request irq\n");
1963         return ret;
1964 }
1965
1966 int __init enable_drhd_fault_handling(void)
1967 {
1968         struct dmar_drhd_unit *drhd;
1969         struct intel_iommu *iommu;
1970
1971         /*
1972          * Enable fault control interrupt.
1973          */
1974         for_each_iommu(iommu, drhd) {
1975                 u32 fault_status;
1976                 int ret = dmar_set_interrupt(iommu);
1977
1978                 if (ret) {
1979                         pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1980                                (unsigned long long)drhd->reg_base_addr, ret);
1981                         return -1;
1982                 }
1983
1984                 /*
1985                  * Clear any previous faults.
1986                  */
1987                 dmar_fault(iommu->irq, iommu);
1988                 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1989                 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1990         }
1991
1992         return 0;
1993 }
1994
1995 /*
1996  * Re-enable Queued Invalidation interface.
1997  */
1998 int dmar_reenable_qi(struct intel_iommu *iommu)
1999 {
2000         if (!ecap_qis(iommu->ecap))
2001                 return -ENOENT;
2002
2003         if (!iommu->qi)
2004                 return -ENOENT;
2005
2006         /*
2007          * First disable queued invalidation.
2008          */
2009         dmar_disable_qi(iommu);
2010         /*
2011          * Then enable queued invalidation again. Since there is no pending
2012          * invalidation requests now, it's safe to re-enable queued
2013          * invalidation.
2014          */
2015         __dmar_enable_qi(iommu);
2016
2017         return 0;
2018 }
2019
2020 /*
2021  * Check interrupt remapping support in DMAR table description.
2022  */
2023 int __init dmar_ir_support(void)
2024 {
2025         struct acpi_table_dmar *dmar;
2026         dmar = (struct acpi_table_dmar *)dmar_tbl;
2027         if (!dmar)
2028                 return 0;
2029         return dmar->flags & 0x1;
2030 }
2031
2032 /* Check whether DMAR units are in use */
2033 static inline bool dmar_in_use(void)
2034 {
2035         return irq_remapping_enabled || intel_iommu_enabled;
2036 }
2037
2038 static int __init dmar_free_unused_resources(void)
2039 {
2040         struct dmar_drhd_unit *dmaru, *dmaru_n;
2041
2042         if (dmar_in_use())
2043                 return 0;
2044
2045         if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
2046                 bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
2047
2048         down_write(&dmar_global_lock);
2049         list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
2050                 list_del(&dmaru->list);
2051                 dmar_free_drhd(dmaru);
2052         }
2053         up_write(&dmar_global_lock);
2054
2055         return 0;
2056 }
2057
2058 late_initcall(dmar_free_unused_resources);
2059 IOMMU_INIT_POST(detect_intel_iommu);
2060
2061 /*
2062  * DMAR Hotplug Support
2063  * For more details, please refer to Intel(R) Virtualization Technology
2064  * for Directed-IO Architecture Specifiction, Rev 2.2, Section 8.8
2065  * "Remapping Hardware Unit Hot Plug".
2066  */
2067 static guid_t dmar_hp_guid =
2068         GUID_INIT(0xD8C1A3A6, 0xBE9B, 0x4C9B,
2069                   0x91, 0xBF, 0xC3, 0xCB, 0x81, 0xFC, 0x5D, 0xAF);
2070
2071 /*
2072  * Currently there's only one revision and BIOS will not check the revision id,
2073  * so use 0 for safety.
2074  */
2075 #define DMAR_DSM_REV_ID                 0
2076 #define DMAR_DSM_FUNC_DRHD              1
2077 #define DMAR_DSM_FUNC_ATSR              2
2078 #define DMAR_DSM_FUNC_RHSA              3
2079
2080 static inline bool dmar_detect_dsm(acpi_handle handle, int func)
2081 {
2082         return acpi_check_dsm(handle, &dmar_hp_guid, DMAR_DSM_REV_ID, 1 << func);
2083 }
2084
2085 static int dmar_walk_dsm_resource(acpi_handle handle, int func,
2086                                   dmar_res_handler_t handler, void *arg)
2087 {
2088         int ret = -ENODEV;
2089         union acpi_object *obj;
2090         struct acpi_dmar_header *start;
2091         struct dmar_res_callback callback;
2092         static int res_type[] = {
2093                 [DMAR_DSM_FUNC_DRHD] = ACPI_DMAR_TYPE_HARDWARE_UNIT,
2094                 [DMAR_DSM_FUNC_ATSR] = ACPI_DMAR_TYPE_ROOT_ATS,
2095                 [DMAR_DSM_FUNC_RHSA] = ACPI_DMAR_TYPE_HARDWARE_AFFINITY,
2096         };
2097
2098         if (!dmar_detect_dsm(handle, func))
2099                 return 0;
2100
2101         obj = acpi_evaluate_dsm_typed(handle, &dmar_hp_guid, DMAR_DSM_REV_ID,
2102                                       func, NULL, ACPI_TYPE_BUFFER);
2103         if (!obj)
2104                 return -ENODEV;
2105
2106         memset(&callback, 0, sizeof(callback));
2107         callback.cb[res_type[func]] = handler;
2108         callback.arg[res_type[func]] = arg;
2109         start = (struct acpi_dmar_header *)obj->buffer.pointer;
2110         ret = dmar_walk_remapping_entries(start, obj->buffer.length, &callback);
2111
2112         ACPI_FREE(obj);
2113
2114         return ret;
2115 }
2116
2117 static int dmar_hp_add_drhd(struct acpi_dmar_header *header, void *arg)
2118 {
2119         int ret;
2120         struct dmar_drhd_unit *dmaru;
2121
2122         dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
2123         if (!dmaru)
2124                 return -ENODEV;
2125
2126         ret = dmar_ir_hotplug(dmaru, true);
2127         if (ret == 0)
2128                 ret = dmar_iommu_hotplug(dmaru, true);
2129
2130         return ret;
2131 }
2132
2133 static int dmar_hp_remove_drhd(struct acpi_dmar_header *header, void *arg)
2134 {
2135         int i, ret;
2136         struct device *dev;
2137         struct dmar_drhd_unit *dmaru;
2138
2139         dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
2140         if (!dmaru)
2141                 return 0;
2142
2143         /*
2144          * All PCI devices managed by this unit should have been destroyed.
2145          */
2146         if (!dmaru->include_all && dmaru->devices && dmaru->devices_cnt) {
2147                 for_each_active_dev_scope(dmaru->devices,
2148                                           dmaru->devices_cnt, i, dev)
2149                         return -EBUSY;
2150         }
2151
2152         ret = dmar_ir_hotplug(dmaru, false);
2153         if (ret == 0)
2154                 ret = dmar_iommu_hotplug(dmaru, false);
2155
2156         return ret;
2157 }
2158
2159 static int dmar_hp_release_drhd(struct acpi_dmar_header *header, void *arg)
2160 {
2161         struct dmar_drhd_unit *dmaru;
2162
2163         dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
2164         if (dmaru) {
2165                 list_del_rcu(&dmaru->list);
2166                 synchronize_rcu();
2167                 dmar_free_drhd(dmaru);
2168         }
2169
2170         return 0;
2171 }
2172
2173 static int dmar_hotplug_insert(acpi_handle handle)
2174 {
2175         int ret;
2176         int drhd_count = 0;
2177
2178         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2179                                      &dmar_validate_one_drhd, (void *)1);
2180         if (ret)
2181                 goto out;
2182
2183         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2184                                      &dmar_parse_one_drhd, (void *)&drhd_count);
2185         if (ret == 0 && drhd_count == 0) {
2186                 pr_warn(FW_BUG "No DRHD structures in buffer returned by _DSM method\n");
2187                 goto out;
2188         } else if (ret) {
2189                 goto release_drhd;
2190         }
2191
2192         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_RHSA,
2193                                      &dmar_parse_one_rhsa, NULL);
2194         if (ret)
2195                 goto release_drhd;
2196
2197         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2198                                      &dmar_parse_one_atsr, NULL);
2199         if (ret)
2200                 goto release_atsr;
2201
2202         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2203                                      &dmar_hp_add_drhd, NULL);
2204         if (!ret)
2205                 return 0;
2206
2207         dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2208                                &dmar_hp_remove_drhd, NULL);
2209 release_atsr:
2210         dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2211                                &dmar_release_one_atsr, NULL);
2212 release_drhd:
2213         dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2214                                &dmar_hp_release_drhd, NULL);
2215 out:
2216         return ret;
2217 }
2218
2219 static int dmar_hotplug_remove(acpi_handle handle)
2220 {
2221         int ret;
2222
2223         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2224                                      &dmar_check_one_atsr, NULL);
2225         if (ret)
2226                 return ret;
2227
2228         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2229                                      &dmar_hp_remove_drhd, NULL);
2230         if (ret == 0) {
2231                 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2232                                                &dmar_release_one_atsr, NULL));
2233                 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2234                                                &dmar_hp_release_drhd, NULL));
2235         } else {
2236                 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2237                                        &dmar_hp_add_drhd, NULL);
2238         }
2239
2240         return ret;
2241 }
2242
2243 static acpi_status dmar_get_dsm_handle(acpi_handle handle, u32 lvl,
2244                                        void *context, void **retval)
2245 {
2246         acpi_handle *phdl = retval;
2247
2248         if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
2249                 *phdl = handle;
2250                 return AE_CTRL_TERMINATE;
2251         }
2252
2253         return AE_OK;
2254 }
2255
2256 static int dmar_device_hotplug(acpi_handle handle, bool insert)
2257 {
2258         int ret;
2259         acpi_handle tmp = NULL;
2260         acpi_status status;
2261
2262         if (!dmar_in_use())
2263                 return 0;
2264
2265         if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
2266                 tmp = handle;
2267         } else {
2268                 status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle,
2269                                              ACPI_UINT32_MAX,
2270                                              dmar_get_dsm_handle,
2271                                              NULL, NULL, &tmp);
2272                 if (ACPI_FAILURE(status)) {
2273                         pr_warn("Failed to locate _DSM method.\n");
2274                         return -ENXIO;
2275                 }
2276         }
2277         if (tmp == NULL)
2278                 return 0;
2279
2280         down_write(&dmar_global_lock);
2281         if (insert)
2282                 ret = dmar_hotplug_insert(tmp);
2283         else
2284                 ret = dmar_hotplug_remove(tmp);
2285         up_write(&dmar_global_lock);
2286
2287         return ret;
2288 }
2289
2290 int dmar_device_add(acpi_handle handle)
2291 {
2292         return dmar_device_hotplug(handle, true);
2293 }
2294
2295 int dmar_device_remove(acpi_handle handle)
2296 {
2297         return dmar_device_hotplug(handle, false);
2298 }
2299
2300 /*
2301  * dmar_platform_optin - Is %DMA_CTRL_PLATFORM_OPT_IN_FLAG set in DMAR table
2302  *
2303  * Returns true if the platform has %DMA_CTRL_PLATFORM_OPT_IN_FLAG set in
2304  * the ACPI DMAR table. This means that the platform boot firmware has made
2305  * sure no device can issue DMA outside of RMRR regions.
2306  */
2307 bool dmar_platform_optin(void)
2308 {
2309         struct acpi_table_dmar *dmar;
2310         acpi_status status;
2311         bool ret;
2312
2313         status = acpi_get_table(ACPI_SIG_DMAR, 0,
2314                                 (struct acpi_table_header **)&dmar);
2315         if (ACPI_FAILURE(status))
2316                 return false;
2317
2318         ret = !!(dmar->flags & DMAR_PLATFORM_OPT_IN);
2319         acpi_put_table((struct acpi_table_header *)dmar);
2320
2321         return ret;
2322 }
2323 EXPORT_SYMBOL_GPL(dmar_platform_optin);