drivers/iommu/dmar.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  *
  22  * This file implements early detection/parsing of Remapping Devices
  23  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  24  * tables.
  25  *
  26  * These routines are used by both DMA-remapping and Interrupt-remapping
  27  */
  28
  29 #define pr_fmt(fmt)     "DMAR: " fmt
  30
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/iova.h>
  34 #include <linux/intel-iommu.h>
  35 #include <linux/timer.h>
  36 #include <linux/irq.h>
  37 #include <linux/interrupt.h>
  38 #include <linux/tboot.h>
  39 #include <linux/dmi.h>
  40 #include <linux/slab.h>
  41 #include <linux/iommu.h>
  42 #include <linux/limits.h>
  43 #include <asm/irq_remapping.h>
  44 #include <asm/iommu_table.h>
  45
  46 #include "irq_remapping.h"
  47
  48 typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *);
  49 struct dmar_res_callback {
  50         dmar_res_handler_t      cb[ACPI_DMAR_TYPE_RESERVED];
  51         void                    *arg[ACPI_DMAR_TYPE_RESERVED];
  52         bool                    ignore_unhandled;
  53         bool                    print_entry;
  54 };
  55
  56 /*
  57  * Assumptions:
  58  * 1) The hotplug framework guarentees that DMAR unit will be hot-added
  59  *    before IO devices managed by that unit.
  60  * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
  61  *    after IO devices managed by that unit.
  62  * 3) Hotplug events are rare.
  63  *
  64  * Locking rules for DMA and interrupt remapping related global data structures:
  65  * 1) Use dmar_global_lock in process context
  66  * 2) Use RCU in interrupt context
  67  */
  68 DECLARE_RWSEM(dmar_global_lock);
  69 LIST_HEAD(dmar_drhd_units);
  70
  71 struct acpi_table_header * __initdata dmar_tbl;
  72 static acpi_size dmar_tbl_size;
  73 static int dmar_dev_scope_status = 1;
  74 static unsigned long dmar_seq_ids[BITS_TO_LONGS(DMAR_UNITS_SUPPORTED)];
  75
  76 static int alloc_iommu(struct dmar_drhd_unit *drhd);
  77 static void free_iommu(struct intel_iommu *iommu);
  78
  79 static void dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  80 {
  81         /*
  82          * add INCLUDE_ALL at the tail, so scan the list will find it at
  83          * the very end.
  84          */
  85         if (drhd->include_all)
  86                 list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
  87         else
  88                 list_add_rcu(&drhd->list, &dmar_drhd_units);
  89 }
  90
  91 void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
  92 {
  93         struct acpi_dmar_device_scope *scope;
  94
  95         *cnt = 0;
  96         while (start < end) {
  97                 scope = start;
  98                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
  99                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 100                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 101                         (*cnt)++;
 102                 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
 103                         scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
 104                         pr_warn("Unsupported device scope\n");
 105                 }
 106                 start += scope->length;
 107         }
 108         if (*cnt == 0)
 109                 return NULL;
 110
 111         return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
 112 }
 113
 114 void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
 115 {
 116         int i;
 117         struct device *tmp_dev;
 118
 119         if (*devices && *cnt) {
 120                 for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
 121                         put_device(tmp_dev);
 122                 kfree(*devices);
 123         }
 124
 125         *devices = NULL;
 126         *cnt = 0;
 127 }
 128
 129 /* Optimize out kzalloc()/kfree() for normal cases */
 130 static char dmar_pci_notify_info_buf[64];
 131
 132 static struct dmar_pci_notify_info *
 133 dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
 134 {
 135         int level = 0;
 136         size_t size;
 137         struct pci_dev *tmp;
 138         struct dmar_pci_notify_info *info;
 139
 140         BUG_ON(dev->is_virtfn);
 141
 142         /*
 143          * Ignore devices that have a domain number higher than what can
 144          * be looked up in DMAR, e.g. VMD subdevices with domain 0x10000
 145          */
 146         if (pci_domain_nr(dev->bus) > U16_MAX)
 147                 return NULL;
 148
 149         /* Only generate path[] for device addition event */
 150         if (event == BUS_NOTIFY_ADD_DEVICE)
 151                 for (tmp = dev; tmp; tmp = tmp->bus->self)
 152                         level++;
 153
 154         size = sizeof(*info) + level * sizeof(info->path[0]);
 155         if (size <= sizeof(dmar_pci_notify_info_buf)) {
 156                 info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
 157         } else {
 158                 info = kzalloc(size, GFP_KERNEL);
 159                 if (!info) {
 160                         pr_warn("Out of memory when allocating notify_info "
 161                                 "for %s.\n", pci_name(dev));
 162                         if (dmar_dev_scope_status == 0)
 163                                 dmar_dev_scope_status = -ENOMEM;
 164                         return NULL;
 165                 }
 166         }
 167
 168         info->event = event;
 169         info->dev = dev;
 170         info->seg = pci_domain_nr(dev->bus);
 171         info->level = level;
 172         if (event == BUS_NOTIFY_ADD_DEVICE) {
 173                 for (tmp = dev; tmp; tmp = tmp->bus->self) {
 174                         level--;
 175                         info->path[level].bus = tmp->bus->number;
 176                         info->path[level].device = PCI_SLOT(tmp->devfn);
 177                         info->path[level].function = PCI_FUNC(tmp->devfn);
 178                         if (pci_is_root_bus(tmp->bus))
 179                                 info->bus = tmp->bus->number;
 180                 }
 181         }
 182
 183         return info;
 184 }
 185
 186 static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
 187 {
 188         if ((void *)info != dmar_pci_notify_info_buf)
 189                 kfree(info);
 190 }
 191
 192 static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
 193                                 struct acpi_dmar_pci_path *path, int count)
 194 {
 195         int i;
 196
 197         if (info->bus != bus)
 198                 goto fallback;
 199         if (info->level != count)
 200                 goto fallback;
 201
 202         for (i = 0; i < count; i++) {
 203                 if (path[i].device != info->path[i].device ||
 204                     path[i].function != info->path[i].function)
 205                         goto fallback;
 206         }
 207
 208         return true;
 209
 210 fallback:
 211
 212         if (count != 1)
 213                 return false;
 214
 215         i = info->level - 1;
 216         if (bus              == info->path[i].bus &&
 217             path[0].device   == info->path[i].device &&
 218             path[0].function == info->path[i].function) {
 219                 pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
 220                         bus, path[0].device, path[0].function);
 221                 return true;
 222         }
 223
 224         return false;
 225 }
 226
 227 /* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
 228 int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
 229                           void *start, void*end, u16 segment,
 230                           struct dmar_dev_scope *devices,
 231                           int devices_cnt)
 232 {
 233         int i, level;
 234         struct device *tmp, *dev = &info->dev->dev;
 235         struct acpi_dmar_device_scope *scope;
 236         struct acpi_dmar_pci_path *path;
 237
 238         if (segment != info->seg)
 239                 return 0;
 240
 241         for (; start < end; start += scope->length) {
 242                 scope = start;
 243                 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
 244                     scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 245                         continue;
 246
 247                 path = (struct acpi_dmar_pci_path *)(scope + 1);
 248                 level = (scope->length - sizeof(*scope)) / sizeof(*path);
 249                 if (!dmar_match_pci_path(info, scope->bus, path, level))
 250                         continue;
 251
 252                 /*
 253                  * We expect devices with endpoint scope to have normal PCI
 254                  * headers, and devices with bridge scope to have bridge PCI
 255                  * headers.  However PCI NTB devices may be listed in the
 256                  * DMAR table with bridge scope, even though they have a
 257                  * normal PCI header.  NTB devices are identified by class
 258                  * "BRIDGE_OTHER" (0680h) - we don't declare a socpe mismatch
 259                  * for this special case.
 260                  */
 261                 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
 262                      info->dev->hdr_type != PCI_HEADER_TYPE_NORMAL) ||
 263                     (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE &&
 264                      (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
 265                       info->dev->class >> 8 != PCI_CLASS_BRIDGE_OTHER))) {
 266                         pr_warn("Device scope type does not match for %s\n",
 267                                 pci_name(info->dev));
 268                         return -EINVAL;
 269                 }
 270
 271                 for_each_dev_scope(devices, devices_cnt, i, tmp)
 272                         if (tmp == NULL) {
 273                                 devices[i].bus = info->dev->bus->number;
 274                                 devices[i].devfn = info->dev->devfn;
 275                                 rcu_assign_pointer(devices[i].dev,
 276                                                    get_device(dev));
 277                                 return 1;
 278                         }
 279                 BUG_ON(i >= devices_cnt);
 280         }
 281
 282         return 0;
 283 }
 284
 285 int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
 286                           struct dmar_dev_scope *devices, int count)
 287 {
 288         int index;
 289         struct device *tmp;
 290
 291         if (info->seg != segment)
 292                 return 0;
 293
 294         for_each_active_dev_scope(devices, count, index, tmp)
 295                 if (tmp == &info->dev->dev) {
 296                         RCU_INIT_POINTER(devices[index].dev, NULL);
 297                         synchronize_rcu();
 298                         put_device(tmp);
 299                         return 1;
 300                 }
 301
 302         return 0;
 303 }
 304
 305 static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
 306 {
 307         int ret = 0;
 308         struct dmar_drhd_unit *dmaru;
 309         struct acpi_dmar_hardware_unit *drhd;
 310
 311         for_each_drhd_unit(dmaru) {
 312                 if (dmaru->include_all)
 313                         continue;
 314
 315                 drhd = container_of(dmaru->hdr,
 316                                     struct acpi_dmar_hardware_unit, header);
 317                 ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
 318                                 ((void *)drhd) + drhd->header.length,
 319                                 dmaru->segment,
 320                                 dmaru->devices, dmaru->devices_cnt);
 321                 if (ret != 0)
 322                         break;
 323         }
 324         if (ret >= 0)
 325                 ret = dmar_iommu_notify_scope_dev(info);
 326         if (ret < 0 && dmar_dev_scope_status == 0)
 327                 dmar_dev_scope_status = ret;
 328
 329         return ret;
 330 }
 331
 332 static void  dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
 333 {
 334         struct dmar_drhd_unit *dmaru;
 335
 336         for_each_drhd_unit(dmaru)
 337                 if (dmar_remove_dev_scope(info, dmaru->segment,
 338                         dmaru->devices, dmaru->devices_cnt))
 339                         break;
 340         dmar_iommu_notify_scope_dev(info);
 341 }
 342
 343 static int dmar_pci_bus_notifier(struct notifier_block *nb,
 344                                  unsigned long action, void *data)
 345 {
 346         struct pci_dev *pdev = to_pci_dev(data);
 347         struct dmar_pci_notify_info *info;
 348
 349         /* Only care about add/remove events for physical functions.
 350          * For VFs we actually do the lookup based on the corresponding
 351          * PF in device_to_iommu() anyway. */
 352         if (pdev->is_virtfn)
 353                 return NOTIFY_DONE;
 354         if (action != BUS_NOTIFY_ADD_DEVICE &&
 355             action != BUS_NOTIFY_REMOVED_DEVICE)
 356                 return NOTIFY_DONE;
 357
 358         info = dmar_alloc_pci_notify_info(pdev, action);
 359         if (!info)
 360                 return NOTIFY_DONE;
 361
 362         down_write(&dmar_global_lock);
 363         if (action == BUS_NOTIFY_ADD_DEVICE)
 364                 dmar_pci_bus_add_dev(info);
 365         else if (action == BUS_NOTIFY_REMOVED_DEVICE)
 366                 dmar_pci_bus_del_dev(info);
 367         up_write(&dmar_global_lock);
 368
 369         dmar_free_pci_notify_info(info);
 370
 371         return NOTIFY_OK;
 372 }
 373
 374 static struct notifier_block dmar_pci_bus_nb = {
 375         .notifier_call = dmar_pci_bus_notifier,
 376         .priority = INT_MIN,
 377 };
 378
 379 static struct dmar_drhd_unit *
 380 dmar_find_dmaru(struct acpi_dmar_hardware_unit *drhd)
 381 {
 382         struct dmar_drhd_unit *dmaru;
 383
 384         list_for_each_entry_rcu(dmaru, &dmar_drhd_units, list)
 385                 if (dmaru->segment == drhd->segment &&
 386                     dmaru->reg_base_addr == drhd->address)
 387                         return dmaru;
 388
 389         return NULL;
 390 }
 391
 392 /**
 393  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 394  * structure which uniquely represent one DMA remapping hardware unit
 395  * present in the platform
 396  */
 397 static int dmar_parse_one_drhd(struct acpi_dmar_header *header, void *arg)
 398 {
 399         struct acpi_dmar_hardware_unit *drhd;
 400         struct dmar_drhd_unit *dmaru;
 401         int ret = 0;
 402
 403         drhd = (struct acpi_dmar_hardware_unit *)header;
 404         dmaru = dmar_find_dmaru(drhd);
 405         if (dmaru)
 406                 goto out;
 407
 408         dmaru = kzalloc(sizeof(*dmaru) + header->length, GFP_KERNEL);
 409         if (!dmaru)
 410                 return -ENOMEM;
 411
 412         /*
 413          * If header is allocated from slab by ACPI _DSM method, we need to
 414          * copy the content because the memory buffer will be freed on return.
 415          */
 416         dmaru->hdr = (void *)(dmaru + 1);
 417         memcpy(dmaru->hdr, header, header->length);
 418         dmaru->reg_base_addr = drhd->address;
 419         dmaru->segment = drhd->segment;
 420         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 421         dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
 422                                               ((void *)drhd) + drhd->header.length,
 423                                               &dmaru->devices_cnt);
 424         if (dmaru->devices_cnt && dmaru->devices == NULL) {
 425                 kfree(dmaru);
 426                 return -ENOMEM;
 427         }
 428
 429         ret = alloc_iommu(dmaru);
 430         if (ret) {
 431                 dmar_free_dev_scope(&dmaru->devices,
 432                                     &dmaru->devices_cnt);
 433                 kfree(dmaru);
 434                 return ret;
 435         }
 436         dmar_register_drhd_unit(dmaru);
 437
 438 out:
 439         if (arg)
 440                 (*(int *)arg)++;
 441
 442         return 0;
 443 }
 444
 445 static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
 446 {
 447         if (dmaru->devices && dmaru->devices_cnt)
 448                 dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
 449         if (dmaru->iommu)
 450                 free_iommu(dmaru->iommu);
 451         kfree(dmaru);
 452 }
 453
 454 static int __init dmar_parse_one_andd(struct acpi_dmar_header *header,
 455                                       void *arg)
 456 {
 457         struct acpi_dmar_andd *andd = (void *)header;
 458
 459         /* Check for NUL termination within the designated length */
 460         if (strnlen(andd->device_name, header->length - 8) == header->length - 8) {
 461                 pr_warn(FW_BUG
 462                            "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
 463                            "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 464                            dmi_get_system_info(DMI_BIOS_VENDOR),
 465                            dmi_get_system_info(DMI_BIOS_VERSION),
 466                            dmi_get_system_info(DMI_PRODUCT_VERSION));
 467                 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 468                 return -EINVAL;
 469         }
 470         pr_info("ANDD device: %x name: %s\n", andd->device_number,
 471                 andd->device_name);
 472
 473         return 0;
 474 }
 475
 476 #ifdef CONFIG_ACPI_NUMA
 477 static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg)
 478 {
 479         struct acpi_dmar_rhsa *rhsa;
 480         struct dmar_drhd_unit *drhd;
 481
 482         rhsa = (struct acpi_dmar_rhsa *)header;
 483         for_each_drhd_unit(drhd) {
 484                 if (drhd->reg_base_addr == rhsa->base_address) {
 485                         int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
 486
 487                         if (!node_online(node))
 488                                 node = -1;
 489                         drhd->iommu->node = node;
 490                         return 0;
 491                 }
 492         }
 493         pr_warn(FW_BUG
 494                 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
 495                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 496                 rhsa->base_address,
 497                 dmi_get_system_info(DMI_BIOS_VENDOR),
 498                 dmi_get_system_info(DMI_BIOS_VERSION),
 499                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 500         add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 501
 502         return 0;
 503 }
 504 #else
 505 #define dmar_parse_one_rhsa             dmar_res_noop
 506 #endif
 507
 508 static void __init
 509 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 510 {
 511         struct acpi_dmar_hardware_unit *drhd;
 512         struct acpi_dmar_reserved_memory *rmrr;
 513         struct acpi_dmar_atsr *atsr;
 514         struct acpi_dmar_rhsa *rhsa;
 515
 516         switch (header->type) {
 517         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 518                 drhd = container_of(header, struct acpi_dmar_hardware_unit,
 519                                     header);
 520                 pr_info("DRHD base: %#016Lx flags: %#x\n",
 521                         (unsigned long long)drhd->address, drhd->flags);
 522                 break;
 523         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 524                 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
 525                                     header);
 526                 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
 527                         (unsigned long long)rmrr->base_address,
 528                         (unsigned long long)rmrr->end_address);
 529                 break;
 530         case ACPI_DMAR_TYPE_ROOT_ATS:
 531                 atsr = container_of(header, struct acpi_dmar_atsr, header);
 532                 pr_info("ATSR flags: %#x\n", atsr->flags);
 533                 break;
 534         case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
 535                 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
 536                 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
 537                        (unsigned long long)rhsa->base_address,
 538                        rhsa->proximity_domain);
 539                 break;
 540         case ACPI_DMAR_TYPE_NAMESPACE:
 541                 /* We don't print this here because we need to sanity-check
 542                    it first. So print it in dmar_parse_one_andd() instead. */
 543                 break;
 544         }
 545 }
 546
 547 /**
 548  * dmar_table_detect - checks to see if the platform supports DMAR devices
 549  */
 550 static int __init dmar_table_detect(void)
 551 {
 552         acpi_status status = AE_OK;
 553
 554         /* if we could find DMAR table, then there are DMAR devices */
 555         status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
 556                                 (struct acpi_table_header **)&dmar_tbl,
 557                                 &dmar_tbl_size);
 558
 559         if (ACPI_SUCCESS(status) && !dmar_tbl) {
 560                 pr_warn("Unable to map DMAR\n");
 561                 status = AE_NOT_FOUND;
 562         }
 563
 564         return (ACPI_SUCCESS(status) ? 1 : 0);
 565 }
 566
 567 static int dmar_walk_remapping_entries(struct acpi_dmar_header *start,
 568                                        size_t len, struct dmar_res_callback *cb)
 569 {
 570         int ret = 0;
 571         struct acpi_dmar_header *iter, *next;
 572         struct acpi_dmar_header *end = ((void *)start) + len;
 573
 574         for (iter = start; iter < end && ret == 0; iter = next) {
 575                 next = (void *)iter + iter->length;
 576                 if (iter->length == 0) {
 577                         /* Avoid looping forever on bad ACPI tables */
 578                         pr_debug(FW_BUG "Invalid 0-length structure\n");
 579                         break;
 580                 } else if (next > end) {
 581                         /* Avoid passing table end */
 582                         pr_warn(FW_BUG "Record passes table end\n");
 583                         ret = -EINVAL;
 584                         break;
 585                 }
 586
 587                 if (cb->print_entry)
 588                         dmar_table_print_dmar_entry(iter);
 589
 590                 if (iter->type >= ACPI_DMAR_TYPE_RESERVED) {
 591                         /* continue for forward compatibility */
 592                         pr_debug("Unknown DMAR structure type %d\n",
 593                                  iter->type);
 594                 } else if (cb->cb[iter->type]) {
 595                         ret = cb->cb[iter->type](iter, cb->arg[iter->type]);
 596                 } else if (!cb->ignore_unhandled) {
 597                         pr_warn("No handler for DMAR structure type %d\n",
 598                                 iter->type);
 599                         ret = -EINVAL;
 600                 }
 601         }
 602
 603         return ret;
 604 }
 605
 606 static inline int dmar_walk_dmar_table(struct acpi_table_dmar *dmar,
 607                                        struct dmar_res_callback *cb)
 608 {
 609         return dmar_walk_remapping_entries((void *)(dmar + 1),
 610                         dmar->header.length - sizeof(*dmar), cb);
 611 }
 612
 613 /**
 614  * parse_dmar_table - parses the DMA reporting table
 615  */
 616 static int __init
 617 parse_dmar_table(void)
 618 {
 619         struct acpi_table_dmar *dmar;
 620         int ret = 0;
 621         int drhd_count = 0;
 622         struct dmar_res_callback cb = {
 623                 .print_entry = true,
 624                 .ignore_unhandled = true,
 625                 .arg[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &drhd_count,
 626                 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_parse_one_drhd,
 627                 .cb[ACPI_DMAR_TYPE_RESERVED_MEMORY] = &dmar_parse_one_rmrr,
 628                 .cb[ACPI_DMAR_TYPE_ROOT_ATS] = &dmar_parse_one_atsr,
 629                 .cb[ACPI_DMAR_TYPE_HARDWARE_AFFINITY] = &dmar_parse_one_rhsa,
 630                 .cb[ACPI_DMAR_TYPE_NAMESPACE] = &dmar_parse_one_andd,
 631         };
 632
 633         /*
 634          * Do it again, earlier dmar_tbl mapping could be mapped with
 635          * fixed map.
 636          */
 637         dmar_table_detect();
 638
 639         /*
 640          * ACPI tables may not be DMA protected by tboot, so use DMAR copy
 641          * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
 642          */
 643         dmar_tbl = tboot_get_dmar_table(dmar_tbl);
 644
 645         dmar = (struct acpi_table_dmar *)dmar_tbl;
 646         if (!dmar)
 647                 return -ENODEV;
 648
 649         if (dmar->width < PAGE_SHIFT - 1) {
 650                 pr_warn("Invalid DMAR haw\n");
 651                 return -EINVAL;
 652         }
 653
 654         pr_info("Host address width %d\n", dmar->width + 1);
 655         ret = dmar_walk_dmar_table(dmar, &cb);
 656         if (ret == 0 && drhd_count == 0)
 657                 pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
 658
 659         return ret;
 660 }
 661
 662 static int dmar_pci_device_match(struct dmar_dev_scope devices[],
 663                                  int cnt, struct pci_dev *dev)
 664 {
 665         int index;
 666         struct device *tmp;
 667
 668         while (dev) {
 669                 for_each_active_dev_scope(devices, cnt, index, tmp)
 670                         if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
 671                                 return 1;
 672
 673                 /* Check our parent */
 674                 dev = dev->bus->self;
 675         }
 676
 677         return 0;
 678 }
 679
 680 struct dmar_drhd_unit *
 681 dmar_find_matched_drhd_unit(struct pci_dev *dev)
 682 {
 683         struct dmar_drhd_unit *dmaru;
 684         struct acpi_dmar_hardware_unit *drhd;
 685
 686         dev = pci_physfn(dev);
 687
 688         rcu_read_lock();
 689         for_each_drhd_unit(dmaru) {
 690                 drhd = container_of(dmaru->hdr,
 691                                     struct acpi_dmar_hardware_unit,
 692                                     header);
 693
 694                 if (dmaru->include_all &&
 695                     drhd->segment == pci_domain_nr(dev->bus))
 696                         goto out;
 697
 698                 if (dmar_pci_device_match(dmaru->devices,
 699                                           dmaru->devices_cnt, dev))
 700                         goto out;
 701         }
 702         dmaru = NULL;
 703 out:
 704         rcu_read_unlock();
 705
 706         return dmaru;
 707 }
 708
 709 static void __init dmar_acpi_insert_dev_scope(u8 device_number,
 710                                               struct acpi_device *adev)
 711 {
 712         struct dmar_drhd_unit *dmaru;
 713         struct acpi_dmar_hardware_unit *drhd;
 714         struct acpi_dmar_device_scope *scope;
 715         struct device *tmp;
 716         int i;
 717         struct acpi_dmar_pci_path *path;
 718
 719         for_each_drhd_unit(dmaru) {
 720                 drhd = container_of(dmaru->hdr,
 721                                     struct acpi_dmar_hardware_unit,
 722                                     header);
 723
 724                 for (scope = (void *)(drhd + 1);
 725                      (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
 726                      scope = ((void *)scope) + scope->length) {
 727                         if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
 728                                 continue;
 729                         if (scope->enumeration_id != device_number)
 730                                 continue;
 731
 732                         path = (void *)(scope + 1);
 733                         pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
 734                                 dev_name(&adev->dev), dmaru->reg_base_addr,
 735                                 scope->bus, path->device, path->function);
 736                         for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
 737                                 if (tmp == NULL) {
 738                                         dmaru->devices[i].bus = scope->bus;
 739                                         dmaru->devices[i].devfn = PCI_DEVFN(path->device,
 740                                                                             path->function);
 741                                         rcu_assign_pointer(dmaru->devices[i].dev,
 742                                                            get_device(&adev->dev));
 743                                         return;
 744                                 }
 745                         BUG_ON(i >= dmaru->devices_cnt);
 746                 }
 747         }
 748         pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
 749                 device_number, dev_name(&adev->dev));
 750 }
 751
 752 static int __init dmar_acpi_dev_scope_init(void)
 753 {
 754         struct acpi_dmar_andd *andd;
 755
 756         if (dmar_tbl == NULL)
 757                 return -ENODEV;
 758
 759         for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
 760              ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
 761              andd = ((void *)andd) + andd->header.length) {
 762                 if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
 763                         acpi_handle h;
 764                         struct acpi_device *adev;
 765
 766                         if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
 767                                                           andd->device_name,
 768                                                           &h))) {
 769                                 pr_err("Failed to find handle for ACPI object %s\n",
 770                                        andd->device_name);
 771                                 continue;
 772                         }
 773                         if (acpi_bus_get_device(h, &adev)) {
 774                                 pr_err("Failed to get device for ACPI object %s\n",
 775                                        andd->device_name);
 776                                 continue;
 777                         }
 778                         dmar_acpi_insert_dev_scope(andd->device_number, adev);
 779                 }
 780         }
 781         return 0;
 782 }
 783
 784 int __init dmar_dev_scope_init(void)
 785 {
 786         struct pci_dev *dev = NULL;
 787         struct dmar_pci_notify_info *info;
 788
 789         if (dmar_dev_scope_status != 1)
 790                 return dmar_dev_scope_status;
 791
 792         if (list_empty(&dmar_drhd_units)) {
 793                 dmar_dev_scope_status = -ENODEV;
 794         } else {
 795                 dmar_dev_scope_status = 0;
 796
 797                 dmar_acpi_dev_scope_init();
 798
 799                 for_each_pci_dev(dev) {
 800                         if (dev->is_virtfn)
 801                                 continue;
 802
 803                         info = dmar_alloc_pci_notify_info(dev,
 804                                         BUS_NOTIFY_ADD_DEVICE);
 805                         if (!info) {
 806                                 return dmar_dev_scope_status;
 807                         } else {
 808                                 dmar_pci_bus_add_dev(info);
 809                                 dmar_free_pci_notify_info(info);
 810                         }
 811                 }
 812
 813                 bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
 814         }
 815
 816         return dmar_dev_scope_status;
 817 }
 818
 819
 820 int __init dmar_table_init(void)
 821 {
 822         static int dmar_table_initialized;
 823         int ret;
 824
 825         if (dmar_table_initialized == 0) {
 826                 ret = parse_dmar_table();
 827                 if (ret < 0) {
 828                         if (ret != -ENODEV)
 829                                 pr_info("Parse DMAR table failure.\n");
 830                 } else  if (list_empty(&dmar_drhd_units)) {
 831                         pr_info("No DMAR devices found\n");
 832                         ret = -ENODEV;
 833                 }
 834
 835                 if (ret < 0)
 836                         dmar_table_initialized = ret;
 837                 else
 838                         dmar_table_initialized = 1;
 839         }
 840
 841         return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
 842 }
 843
 844 static void warn_invalid_dmar(u64 addr, const char *message)
 845 {
 846         pr_warn_once(FW_BUG
 847                 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
 848                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 849                 addr, message,
 850                 dmi_get_system_info(DMI_BIOS_VENDOR),
 851                 dmi_get_system_info(DMI_BIOS_VERSION),
 852                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 853         add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 854 }
 855
 856 static int __ref
 857 dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
 858 {
 859         struct acpi_dmar_hardware_unit *drhd;
 860         void __iomem *addr;
 861         u64 cap, ecap;
 862
 863         drhd = (void *)entry;
 864         if (!drhd->address) {
 865                 warn_invalid_dmar(0, "");
 866                 return -EINVAL;
 867         }
 868
 869         if (arg)
 870                 addr = ioremap(drhd->address, VTD_PAGE_SIZE);
 871         else
 872                 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
 873         if (!addr) {
 874                 pr_warn("Can't validate DRHD address: %llx\n", drhd->address);
 875                 return -EINVAL;
 876         }
 877
 878         cap = dmar_readq(addr + DMAR_CAP_REG);
 879         ecap = dmar_readq(addr + DMAR_ECAP_REG);
 880
 881         if (arg)
 882                 iounmap(addr);
 883         else
 884                 early_iounmap(addr, VTD_PAGE_SIZE);
 885
 886         if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
 887                 warn_invalid_dmar(drhd->address, " returns all ones");
 888                 return -EINVAL;
 889         }
 890
 891         return 0;
 892 }
 893
 894 int __init detect_intel_iommu(void)
 895 {
 896         int ret;
 897         struct dmar_res_callback validate_drhd_cb = {
 898                 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_validate_one_drhd,
 899                 .ignore_unhandled = true,
 900         };
 901
 902         down_write(&dmar_global_lock);
 903         ret = dmar_table_detect();
 904         if (ret)
 905                 ret = !dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl,
 906                                             &validate_drhd_cb);
 907         if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
 908                 iommu_detected = 1;
 909                 /* Make sure ACS will be enabled */
 910                 pci_request_acs();
 911         }
 912
 913 #ifdef CONFIG_X86
 914         if (ret)
 915                 x86_init.iommu.iommu_init = intel_iommu_init;
 916 #endif
 917
 918         early_acpi_os_unmap_memory((void __iomem *)dmar_tbl, dmar_tbl_size);
 919         dmar_tbl = NULL;
 920         up_write(&dmar_global_lock);
 921
 922         return ret ? 1 : -ENODEV;
 923 }
 924
 925
 926 static void unmap_iommu(struct intel_iommu *iommu)
 927 {
 928         iounmap(iommu->reg);
 929         release_mem_region(iommu->reg_phys, iommu->reg_size);
 930 }
 931
 932 /**
 933  * map_iommu: map the iommu's registers
 934  * @iommu: the iommu to map
 935  * @phys_addr: the physical address of the base resgister
 936  *
 937  * Memory map the iommu's registers.  Start w/ a single page, and
 938  * possibly expand if that turns out to be insufficent.
 939  */
 940 static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
 941 {
 942         int map_size, err=0;
 943
 944         iommu->reg_phys = phys_addr;
 945         iommu->reg_size = VTD_PAGE_SIZE;
 946
 947         if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
 948                 pr_err("Can't reserve memory\n");
 949                 err = -EBUSY;
 950                 goto out;
 951         }
 952
 953         iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 954         if (!iommu->reg) {
 955                 pr_err("Can't map the region\n");
 956                 err = -ENOMEM;
 957                 goto release;
 958         }
 959
 960         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 961         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 962
 963         if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
 964                 err = -EINVAL;
 965                 warn_invalid_dmar(phys_addr, " returns all ones");
 966                 goto unmap;
 967         }
 968
 969         /* the registers might be more than one page */
 970         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 971                          cap_max_fault_reg_offset(iommu->cap));
 972         map_size = VTD_PAGE_ALIGN(map_size);
 973         if (map_size > iommu->reg_size) {
 974                 iounmap(iommu->reg);
 975                 release_mem_region(iommu->reg_phys, iommu->reg_size);
 976                 iommu->reg_size = map_size;
 977                 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
 978                                         iommu->name)) {
 979                         pr_err("Can't reserve memory\n");
 980                         err = -EBUSY;
 981                         goto out;
 982                 }
 983                 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 984                 if (!iommu->reg) {
 985                         pr_err("Can't map the region\n");
 986                         err = -ENOMEM;
 987                         goto release;
 988                 }
 989         }
 990         err = 0;
 991         goto out;
 992
 993 unmap:
 994         iounmap(iommu->reg);
 995 release:
 996         release_mem_region(iommu->reg_phys, iommu->reg_size);
 997 out:
 998         return err;
 999 }
1000
1001 static int dmar_alloc_seq_id(struct intel_iommu *iommu)
1002 {
1003         iommu->seq_id = find_first_zero_bit(dmar_seq_ids,
1004                                             DMAR_UNITS_SUPPORTED);
1005         if (iommu->seq_id >= DMAR_UNITS_SUPPORTED) {
1006                 iommu->seq_id = -1;
1007         } else {
1008                 set_bit(iommu->seq_id, dmar_seq_ids);
1009                 sprintf(iommu->name, "dmar%d", iommu->seq_id);
1010         }
1011
1012         return iommu->seq_id;
1013 }
1014
1015 static void dmar_free_seq_id(struct intel_iommu *iommu)
1016 {
1017         if (iommu->seq_id >= 0) {
1018                 clear_bit(iommu->seq_id, dmar_seq_ids);
1019                 iommu->seq_id = -1;
1020         }
1021 }
1022
1023 static int alloc_iommu(struct dmar_drhd_unit *drhd)
1024 {
1025         struct intel_iommu *iommu;
1026         u32 ver, sts;
1027         int agaw = -1;
1028         int msagaw = -1;
1029         int err;
1030
1031         if (!drhd->reg_base_addr) {
1032                 warn_invalid_dmar(0, "");
1033                 return -EINVAL;
1034         }
1035
1036         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
1037         if (!iommu)
1038                 return -ENOMEM;
1039
1040         if (dmar_alloc_seq_id(iommu) < 0) {
1041                 pr_err("Failed to allocate seq_id\n");
1042                 err = -ENOSPC;
1043                 goto error;
1044         }
1045
1046         err = map_iommu(iommu, drhd->reg_base_addr);
1047         if (err) {
1048                 pr_err("Failed to map %s\n", iommu->name);
1049                 goto error_free_seq_id;
1050         }
1051
1052         err = -EINVAL;
1053         if (cap_sagaw(iommu->cap) == 0) {
1054                 pr_info("%s: No supported address widths. Not attempting DMA translation.\n",
1055                         iommu->name);
1056                 drhd->ignored = 1;
1057         }
1058
1059         if (!drhd->ignored) {
1060                 agaw = iommu_calculate_agaw(iommu);
1061                 if (agaw < 0) {
1062                         pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
1063                                iommu->seq_id);
1064                         drhd->ignored = 1;
1065                 }
1066         }
1067         if (!drhd->ignored) {
1068                 msagaw = iommu_calculate_max_sagaw(iommu);
1069                 if (msagaw < 0) {
1070                         pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
1071                                iommu->seq_id);
1072                         drhd->ignored = 1;
1073                         agaw = -1;
1074                 }
1075         }
1076         iommu->agaw = agaw;
1077         iommu->msagaw = msagaw;
1078         iommu->segment = drhd->segment;
1079
1080         iommu->node = -1;
1081
1082         ver = readl(iommu->reg + DMAR_VER_REG);
1083         pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
1084                 iommu->name,
1085                 (unsigned long long)drhd->reg_base_addr,
1086                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
1087                 (unsigned long long)iommu->cap,
1088                 (unsigned long long)iommu->ecap);
1089
1090         /* Reflect status in gcmd */
1091         sts = readl(iommu->reg + DMAR_GSTS_REG);
1092         if (sts & DMA_GSTS_IRES)
1093                 iommu->gcmd |= DMA_GCMD_IRE;
1094         if (sts & DMA_GSTS_TES)
1095                 iommu->gcmd |= DMA_GCMD_TE;
1096         if (sts & DMA_GSTS_QIES)
1097                 iommu->gcmd |= DMA_GCMD_QIE;
1098
1099         raw_spin_lock_init(&iommu->register_lock);
1100
1101         if (intel_iommu_enabled && !drhd->ignored) {
1102                 iommu->iommu_dev = iommu_device_create(NULL, iommu,
1103                                                        intel_iommu_groups,
1104                                                        "%s", iommu->name);
1105
1106                 if (IS_ERR(iommu->iommu_dev)) {
1107                         err = PTR_ERR(iommu->iommu_dev);
1108                         goto err_unmap;
1109                 }
1110         }
1111
1112         drhd->iommu = iommu;
1113         iommu->drhd = drhd;
1114
1115         return 0;
1116
1117 err_unmap:
1118         unmap_iommu(iommu);
1119 error_free_seq_id:
1120         dmar_free_seq_id(iommu);
1121 error:
1122         kfree(iommu);
1123         return err;
1124 }
1125
1126 static void free_iommu(struct intel_iommu *iommu)
1127 {
1128         if (intel_iommu_enabled && !iommu->drhd->ignored)
1129                 iommu_device_destroy(iommu->iommu_dev);
1130
1131         if (iommu->irq) {
1132                 if (iommu->pr_irq) {
1133                         free_irq(iommu->pr_irq, iommu);
1134                         dmar_free_hwirq(iommu->pr_irq);
1135                         iommu->pr_irq = 0;
1136                 }
1137                 free_irq(iommu->irq, iommu);
1138                 dmar_free_hwirq(iommu->irq);
1139                 iommu->irq = 0;
1140         }
1141
1142         if (iommu->qi) {
1143                 free_page((unsigned long)iommu->qi->desc);
1144                 kfree(iommu->qi->desc_status);
1145                 kfree(iommu->qi);
1146         }
1147
1148         if (iommu->reg)
1149                 unmap_iommu(iommu);
1150
1151         dmar_free_seq_id(iommu);
1152         kfree(iommu);
1153 }
1154
1155 /*
1156  * Reclaim all the submitted descriptors which have completed its work.
1157  */
1158 static inline void reclaim_free_desc(struct q_inval *qi)
1159 {
1160         while (qi->desc_status[qi->free_tail] == QI_DONE ||
1161                qi->desc_status[qi->free_tail] == QI_ABORT) {
1162                 qi->desc_status[qi->free_tail] = QI_FREE;
1163                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1164                 qi->free_cnt++;
1165         }
1166 }
1167
1168 static int qi_check_fault(struct intel_iommu *iommu, int index)
1169 {
1170         u32 fault;
1171         int head, tail;
1172         struct q_inval *qi = iommu->qi;
1173         int wait_index = (index + 1) % QI_LENGTH;
1174
1175         if (qi->desc_status[wait_index] == QI_ABORT)
1176                 return -EAGAIN;
1177
1178         fault = readl(iommu->reg + DMAR_FSTS_REG);
1179
1180         /*
1181          * If IQE happens, the head points to the descriptor associated
1182          * with the error. No new descriptors are fetched until the IQE
1183          * is cleared.
1184          */
1185         if (fault & DMA_FSTS_IQE) {
1186                 head = readl(iommu->reg + DMAR_IQH_REG);
1187                 if ((head >> DMAR_IQ_SHIFT) == index) {
1188                         pr_err("VT-d detected invalid descriptor: "
1189                                 "low=%llx, high=%llx\n",
1190                                 (unsigned long long)qi->desc[index].low,
1191                                 (unsigned long long)qi->desc[index].high);
1192                         memcpy(&qi->desc[index], &qi->desc[wait_index],
1193                                         sizeof(struct qi_desc));
1194                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1195                         return -EINVAL;
1196                 }
1197         }
1198
1199         /*
1200          * If ITE happens, all pending wait_desc commands are aborted.
1201          * No new descriptors are fetched until the ITE is cleared.
1202          */
1203         if (fault & DMA_FSTS_ITE) {
1204                 head = readl(iommu->reg + DMAR_IQH_REG);
1205                 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1206                 head |= 1;
1207                 tail = readl(iommu->reg + DMAR_IQT_REG);
1208                 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1209
1210                 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
1211
1212                 do {
1213                         if (qi->desc_status[head] == QI_IN_USE)
1214                                 qi->desc_status[head] = QI_ABORT;
1215                         head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1216                 } while (head != tail);
1217
1218                 if (qi->desc_status[wait_index] == QI_ABORT)
1219                         return -EAGAIN;
1220         }
1221
1222         if (fault & DMA_FSTS_ICE)
1223                 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
1224
1225         return 0;
1226 }
1227
1228 /*
1229  * Submit the queued invalidation descriptor to the remapping
1230  * hardware unit and wait for its completion.
1231  */
1232 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
1233 {
1234         int rc;
1235         struct q_inval *qi = iommu->qi;
1236         struct qi_desc *hw, wait_desc;
1237         int wait_index, index;
1238         unsigned long flags;
1239
1240         if (!qi)
1241                 return 0;
1242
1243         hw = qi->desc;
1244
1245 restart:
1246         rc = 0;
1247
1248         raw_spin_lock_irqsave(&qi->q_lock, flags);
1249         while (qi->free_cnt < 3) {
1250                 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1251                 cpu_relax();
1252                 raw_spin_lock_irqsave(&qi->q_lock, flags);
1253         }
1254
1255         index = qi->free_head;
1256         wait_index = (index + 1) % QI_LENGTH;
1257
1258         qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
1259
1260         hw[index] = *desc;
1261
1262         wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
1263                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1264         wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
1265
1266         hw[wait_index] = wait_desc;
1267
1268         qi->free_head = (qi->free_head + 2) % QI_LENGTH;
1269         qi->free_cnt -= 2;
1270
1271         /*
1272          * update the HW tail register indicating the presence of
1273          * new descriptors.
1274          */
1275         writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
1276
1277         while (qi->desc_status[wait_index] != QI_DONE) {
1278                 /*
1279                  * We will leave the interrupts disabled, to prevent interrupt
1280                  * context to queue another cmd while a cmd is already submitted
1281                  * and waiting for completion on this cpu. This is to avoid
1282                  * a deadlock where the interrupt context can wait indefinitely
1283                  * for free slots in the queue.
1284                  */
1285                 rc = qi_check_fault(iommu, index);
1286                 if (rc)
1287                         break;
1288
1289                 raw_spin_unlock(&qi->q_lock);
1290                 cpu_relax();
1291                 raw_spin_lock(&qi->q_lock);
1292         }
1293
1294         qi->desc_status[index] = QI_DONE;
1295
1296         reclaim_free_desc(qi);
1297         raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1298
1299         if (rc == -EAGAIN)
1300                 goto restart;
1301
1302         return rc;
1303 }
1304
1305 /*
1306  * Flush the global interrupt entry cache.
1307  */
1308 void qi_global_iec(struct intel_iommu *iommu)
1309 {
1310         struct qi_desc desc;
1311
1312         desc.low = QI_IEC_TYPE;
1313         desc.high = 0;
1314
1315         /* should never fail */
1316         qi_submit_sync(&desc, iommu);
1317 }
1318
1319 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1320                       u64 type)
1321 {
1322         struct qi_desc desc;
1323
1324         desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1325                         | QI_CC_GRAN(type) | QI_CC_TYPE;
1326         desc.high = 0;
1327
1328         qi_submit_sync(&desc, iommu);
1329 }
1330
1331 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1332                     unsigned int size_order, u64 type)
1333 {
1334         u8 dw = 0, dr = 0;
1335
1336         struct qi_desc desc;
1337         int ih = 0;
1338
1339         if (cap_write_drain(iommu->cap))
1340                 dw = 1;
1341
1342         if (cap_read_drain(iommu->cap))
1343                 dr = 1;
1344
1345         desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1346                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1347         desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1348                 | QI_IOTLB_AM(size_order);
1349
1350         qi_submit_sync(&desc, iommu);
1351 }
1352
1353 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
1354                         u16 qdep, u64 addr, unsigned mask)
1355 {
1356         struct qi_desc desc;
1357
1358         if (mask) {
1359                 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
1360                 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1361                 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1362         } else
1363                 desc.high = QI_DEV_IOTLB_ADDR(addr);
1364
1365         if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1366                 qdep = 0;
1367
1368         desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1369                    QI_DIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid);
1370
1371         qi_submit_sync(&desc, iommu);
1372 }
1373
1374 /*
1375  * Disable Queued Invalidation interface.
1376  */
1377 void dmar_disable_qi(struct intel_iommu *iommu)
1378 {
1379         unsigned long flags;
1380         u32 sts;
1381         cycles_t start_time = get_cycles();
1382
1383         if (!ecap_qis(iommu->ecap))
1384                 return;
1385
1386         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1387
1388         sts =  readl(iommu->reg + DMAR_GSTS_REG);
1389         if (!(sts & DMA_GSTS_QIES))
1390                 goto end;
1391
1392         /*
1393          * Give a chance to HW to complete the pending invalidation requests.
1394          */
1395         while ((readl(iommu->reg + DMAR_IQT_REG) !=
1396                 readl(iommu->reg + DMAR_IQH_REG)) &&
1397                 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1398                 cpu_relax();
1399
1400         iommu->gcmd &= ~DMA_GCMD_QIE;
1401         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1402
1403         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1404                       !(sts & DMA_GSTS_QIES), sts);
1405 end:
1406         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1407 }
1408
1409 /*
1410  * Enable queued invalidation.
1411  */
1412 static void __dmar_enable_qi(struct intel_iommu *iommu)
1413 {
1414         u32 sts;
1415         unsigned long flags;
1416         struct q_inval *qi = iommu->qi;
1417
1418         qi->free_head = qi->free_tail = 0;
1419         qi->free_cnt = QI_LENGTH;
1420
1421         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1422
1423         /* write zero to the tail reg */
1424         writel(0, iommu->reg + DMAR_IQT_REG);
1425
1426         dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1427
1428         iommu->gcmd |= DMA_GCMD_QIE;
1429         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1430
1431         /* Make sure hardware complete it */
1432         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1433
1434         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1435 }
1436
1437 /*
1438  * Enable Queued Invalidation interface. This is a must to support
1439  * interrupt-remapping. Also used by DMA-remapping, which replaces
1440  * register based IOTLB invalidation.
1441  */
1442 int dmar_enable_qi(struct intel_iommu *iommu)
1443 {
1444         struct q_inval *qi;
1445         struct page *desc_page;
1446
1447         if (!ecap_qis(iommu->ecap))
1448                 return -ENOENT;
1449
1450         /*
1451          * queued invalidation is already setup and enabled.
1452          */
1453         if (iommu->qi)
1454                 return 0;
1455
1456         iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1457         if (!iommu->qi)
1458                 return -ENOMEM;
1459
1460         qi = iommu->qi;
1461
1462
1463         desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1464         if (!desc_page) {
1465                 kfree(qi);
1466                 iommu->qi = NULL;
1467                 return -ENOMEM;
1468         }
1469
1470         qi->desc = page_address(desc_page);
1471
1472         qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1473         if (!qi->desc_status) {
1474                 free_page((unsigned long) qi->desc);
1475                 kfree(qi);
1476                 iommu->qi = NULL;
1477                 return -ENOMEM;
1478         }
1479
1480         raw_spin_lock_init(&qi->q_lock);
1481
1482         __dmar_enable_qi(iommu);
1483
1484         return 0;
1485 }
1486
1487 /* iommu interrupt handling. Most stuff are MSI-like. */
1488
1489 enum faulttype {
1490         DMA_REMAP,
1491         INTR_REMAP,
1492         UNKNOWN,
1493 };
1494
1495 static const char *dma_remap_fault_reasons[] =
1496 {
1497         "Software",
1498         "Present bit in root entry is clear",
1499         "Present bit in context entry is clear",
1500         "Invalid context entry",
1501         "Access beyond MGAW",
1502         "PTE Write access is not set",
1503         "PTE Read access is not set",
1504         "Next page table ptr is invalid",
1505         "Root table address invalid",
1506         "Context table ptr is invalid",
1507         "non-zero reserved fields in RTP",
1508         "non-zero reserved fields in CTP",
1509         "non-zero reserved fields in PTE",
1510         "PCE for translation request specifies blocking",
1511 };
1512
1513 static const char *irq_remap_fault_reasons[] =
1514 {
1515         "Detected reserved fields in the decoded interrupt-remapped request",
1516         "Interrupt index exceeded the interrupt-remapping table size",
1517         "Present field in the IRTE entry is clear",
1518         "Error accessing interrupt-remapping table pointed by IRTA_REG",
1519         "Detected reserved fields in the IRTE entry",
1520         "Blocked a compatibility format interrupt request",
1521         "Blocked an interrupt request due to source-id verification failure",
1522 };
1523
1524 static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1525 {
1526         if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1527                                         ARRAY_SIZE(irq_remap_fault_reasons))) {
1528                 *fault_type = INTR_REMAP;
1529                 return irq_remap_fault_reasons[fault_reason - 0x20];
1530         } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1531                 *fault_type = DMA_REMAP;
1532                 return dma_remap_fault_reasons[fault_reason];
1533         } else {
1534                 *fault_type = UNKNOWN;
1535                 return "Unknown";
1536         }
1537 }
1538
1539
1540 static inline int dmar_msi_reg(struct intel_iommu *iommu, int irq)
1541 {
1542         if (iommu->irq == irq)
1543                 return DMAR_FECTL_REG;
1544         else if (iommu->pr_irq == irq)
1545                 return DMAR_PECTL_REG;
1546         else
1547                 BUG();
1548 }
1549
1550 void dmar_msi_unmask(struct irq_data *data)
1551 {
1552         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1553         int reg = dmar_msi_reg(iommu, data->irq);
1554         unsigned long flag;
1555
1556         /* unmask it */
1557         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1558         writel(0, iommu->reg + reg);
1559         /* Read a reg to force flush the post write */
1560         readl(iommu->reg + reg);
1561         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1562 }
1563
1564 void dmar_msi_mask(struct irq_data *data)
1565 {
1566         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1567         int reg = dmar_msi_reg(iommu, data->irq);
1568         unsigned long flag;
1569
1570         /* mask it */
1571         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1572         writel(DMA_FECTL_IM, iommu->reg + reg);
1573         /* Read a reg to force flush the post write */
1574         readl(iommu->reg + reg);
1575         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1576 }
1577
1578 void dmar_msi_write(int irq, struct msi_msg *msg)
1579 {
1580         struct intel_iommu *iommu = irq_get_handler_data(irq);
1581         int reg = dmar_msi_reg(iommu, irq);
1582         unsigned long flag;
1583
1584         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1585         writel(msg->data, iommu->reg + reg + 4);
1586         writel(msg->address_lo, iommu->reg + reg + 8);
1587         writel(msg->address_hi, iommu->reg + reg + 12);
1588         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1589 }
1590
1591 void dmar_msi_read(int irq, struct msi_msg *msg)
1592 {
1593         struct intel_iommu *iommu = irq_get_handler_data(irq);
1594         int reg = dmar_msi_reg(iommu, irq);
1595         unsigned long flag;
1596
1597         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1598         msg->data = readl(iommu->reg + reg + 4);
1599         msg->address_lo = readl(iommu->reg + reg + 8);
1600         msg->address_hi = readl(iommu->reg + reg + 12);
1601         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1602 }
1603
1604 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1605                 u8 fault_reason, u16 source_id, unsigned long long addr)
1606 {
1607         const char *reason;
1608         int fault_type;
1609
1610         reason = dmar_get_fault_reason(fault_reason, &fault_type);
1611
1612         if (fault_type == INTR_REMAP)
1613                 pr_err("[INTR-REMAP] Request device [%02x:%02x.%d] fault index %llx [fault reason %02d] %s\n",
1614                         source_id >> 8, PCI_SLOT(source_id & 0xFF),
1615                         PCI_FUNC(source_id & 0xFF), addr >> 48,
1616                         fault_reason, reason);
1617         else
1618                 pr_err("[%s] Request device [%02x:%02x.%d] fault addr %llx [fault reason %02d] %s\n",
1619                        type ? "DMA Read" : "DMA Write",
1620                        source_id >> 8, PCI_SLOT(source_id & 0xFF),
1621                        PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1622         return 0;
1623 }
1624
1625 #define PRIMARY_FAULT_REG_LEN (16)
1626 irqreturn_t dmar_fault(int irq, void *dev_id)
1627 {
1628         struct intel_iommu *iommu = dev_id;
1629         int reg, fault_index;
1630         u32 fault_status;
1631         unsigned long flag;
1632         bool ratelimited;
1633         static DEFINE_RATELIMIT_STATE(rs,
1634                                       DEFAULT_RATELIMIT_INTERVAL,
1635                                       DEFAULT_RATELIMIT_BURST);
1636
1637         /* Disable printing, simply clear the fault when ratelimited */
1638         ratelimited = !__ratelimit(&rs);
1639
1640         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1641         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1642         if (fault_status && !ratelimited)
1643                 pr_err("DRHD: handling fault status reg %x\n", fault_status);
1644
1645         /* TBD: ignore advanced fault log currently */
1646         if (!(fault_status & DMA_FSTS_PPF))
1647                 goto unlock_exit;
1648
1649         fault_index = dma_fsts_fault_record_index(fault_status);
1650         reg = cap_fault_reg_offset(iommu->cap);
1651         while (1) {
1652                 u8 fault_reason;
1653                 u16 source_id;
1654                 u64 guest_addr;
1655                 int type;
1656                 u32 data;
1657
1658                 /* highest 32 bits */
1659                 data = readl(iommu->reg + reg +
1660                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1661                 if (!(data & DMA_FRCD_F))
1662                         break;
1663
1664                 if (!ratelimited) {
1665                         fault_reason = dma_frcd_fault_reason(data);
1666                         type = dma_frcd_type(data);
1667
1668                         data = readl(iommu->reg + reg +
1669                                      fault_index * PRIMARY_FAULT_REG_LEN + 8);
1670                         source_id = dma_frcd_source_id(data);
1671
1672                         guest_addr = dmar_readq(iommu->reg + reg +
1673                                         fault_index * PRIMARY_FAULT_REG_LEN);
1674                         guest_addr = dma_frcd_page_addr(guest_addr);
1675                 }
1676
1677                 /* clear the fault */
1678                 writel(DMA_FRCD_F, iommu->reg + reg +
1679                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
1680
1681                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1682
1683                 if (!ratelimited)
1684                         dmar_fault_do_one(iommu, type, fault_reason,
1685                                           source_id, guest_addr);
1686
1687                 fault_index++;
1688                 if (fault_index >= cap_num_fault_regs(iommu->cap))
1689                         fault_index = 0;
1690                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1691         }
1692
1693         writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG);
1694
1695 unlock_exit:
1696         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1697         return IRQ_HANDLED;
1698 }
1699
1700 int dmar_set_interrupt(struct intel_iommu *iommu)
1701 {
1702         int irq, ret;
1703
1704         /*
1705          * Check if the fault interrupt is already initialized.
1706          */
1707         if (iommu->irq)
1708                 return 0;
1709
1710         irq = dmar_alloc_hwirq(iommu->seq_id, iommu->node, iommu);
1711         if (irq > 0) {
1712                 iommu->irq = irq;
1713         } else {
1714                 pr_err("No free IRQ vectors\n");
1715                 return -EINVAL;
1716         }
1717
1718         ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1719         if (ret)
1720                 pr_err("Can't request irq\n");
1721         return ret;
1722 }
1723
1724 int __init enable_drhd_fault_handling(void)
1725 {
1726         struct dmar_drhd_unit *drhd;
1727         struct intel_iommu *iommu;
1728
1729         /*
1730          * Enable fault control interrupt.
1731          */
1732         for_each_iommu(iommu, drhd) {
1733                 u32 fault_status;
1734                 int ret = dmar_set_interrupt(iommu);
1735
1736                 if (ret) {
1737                         pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1738                                (unsigned long long)drhd->reg_base_addr, ret);
1739                         return -1;
1740                 }
1741
1742                 /*
1743                  * Clear any previous faults.
1744                  */
1745                 dmar_fault(iommu->irq, iommu);
1746                 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1747                 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1748         }
1749
1750         return 0;
1751 }
1752
1753 /*
1754  * Re-enable Queued Invalidation interface.
1755  */
1756 int dmar_reenable_qi(struct intel_iommu *iommu)
1757 {
1758         if (!ecap_qis(iommu->ecap))
1759                 return -ENOENT;
1760
1761         if (!iommu->qi)
1762                 return -ENOENT;
1763
1764         /*
1765          * First disable queued invalidation.
1766          */
1767         dmar_disable_qi(iommu);
1768         /*
1769          * Then enable queued invalidation again. Since there is no pending
1770          * invalidation requests now, it's safe to re-enable queued
1771          * invalidation.
1772          */
1773         __dmar_enable_qi(iommu);
1774
1775         return 0;
1776 }
1777
1778 /*
1779  * Check interrupt remapping support in DMAR table description.
1780  */
1781 int __init dmar_ir_support(void)
1782 {
1783         struct acpi_table_dmar *dmar;
1784         dmar = (struct acpi_table_dmar *)dmar_tbl;
1785         if (!dmar)
1786                 return 0;
1787         return dmar->flags & 0x1;
1788 }
1789
1790 /* Check whether DMAR units are in use */
1791 static inline bool dmar_in_use(void)
1792 {
1793         return irq_remapping_enabled || intel_iommu_enabled;
1794 }
1795
1796 static int __init dmar_free_unused_resources(void)
1797 {
1798         struct dmar_drhd_unit *dmaru, *dmaru_n;
1799
1800         if (dmar_in_use())
1801                 return 0;
1802
1803         if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
1804                 bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
1805
1806         down_write(&dmar_global_lock);
1807         list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
1808                 list_del(&dmaru->list);
1809                 dmar_free_drhd(dmaru);
1810         }
1811         up_write(&dmar_global_lock);
1812
1813         return 0;
1814 }
1815
1816 late_initcall(dmar_free_unused_resources);
1817 IOMMU_INIT_POST(detect_intel_iommu);
1818
1819 /*
1820  * DMAR Hotplug Support
1821  * For more details, please refer to Intel(R) Virtualization Technology
1822  * for Directed-IO Architecture Specifiction, Rev 2.2, Section 8.8
1823  * "Remapping Hardware Unit Hot Plug".
1824  */
1825 static u8 dmar_hp_uuid[] = {
1826         /* 0000 */    0xA6, 0xA3, 0xC1, 0xD8, 0x9B, 0xBE, 0x9B, 0x4C,
1827         /* 0008 */    0x91, 0xBF, 0xC3, 0xCB, 0x81, 0xFC, 0x5D, 0xAF
1828 };
1829
1830 /*
1831  * Currently there's only one revision and BIOS will not check the revision id,
1832  * so use 0 for safety.
1833  */
1834 #define DMAR_DSM_REV_ID                 0
1835 #define DMAR_DSM_FUNC_DRHD              1
1836 #define DMAR_DSM_FUNC_ATSR              2
1837 #define DMAR_DSM_FUNC_RHSA              3
1838
1839 static inline bool dmar_detect_dsm(acpi_handle handle, int func)
1840 {
1841         return acpi_check_dsm(handle, dmar_hp_uuid, DMAR_DSM_REV_ID, 1 << func);
1842 }
1843
1844 static int dmar_walk_dsm_resource(acpi_handle handle, int func,
1845                                   dmar_res_handler_t handler, void *arg)
1846 {
1847         int ret = -ENODEV;
1848         union acpi_object *obj;
1849         struct acpi_dmar_header *start;
1850         struct dmar_res_callback callback;
1851         static int res_type[] = {
1852                 [DMAR_DSM_FUNC_DRHD] = ACPI_DMAR_TYPE_HARDWARE_UNIT,
1853                 [DMAR_DSM_FUNC_ATSR] = ACPI_DMAR_TYPE_ROOT_ATS,
1854                 [DMAR_DSM_FUNC_RHSA] = ACPI_DMAR_TYPE_HARDWARE_AFFINITY,
1855         };
1856
1857         if (!dmar_detect_dsm(handle, func))
1858                 return 0;
1859
1860         obj = acpi_evaluate_dsm_typed(handle, dmar_hp_uuid, DMAR_DSM_REV_ID,
1861                                       func, NULL, ACPI_TYPE_BUFFER);
1862         if (!obj)
1863                 return -ENODEV;
1864
1865         memset(&callback, 0, sizeof(callback));
1866         callback.cb[res_type[func]] = handler;
1867         callback.arg[res_type[func]] = arg;
1868         start = (struct acpi_dmar_header *)obj->buffer.pointer;
1869         ret = dmar_walk_remapping_entries(start, obj->buffer.length, &callback);
1870
1871         ACPI_FREE(obj);
1872
1873         return ret;
1874 }
1875
1876 static int dmar_hp_add_drhd(struct acpi_dmar_header *header, void *arg)
1877 {
1878         int ret;
1879         struct dmar_drhd_unit *dmaru;
1880
1881         dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1882         if (!dmaru)
1883                 return -ENODEV;
1884
1885         ret = dmar_ir_hotplug(dmaru, true);
1886         if (ret == 0)
1887                 ret = dmar_iommu_hotplug(dmaru, true);
1888
1889         return ret;
1890 }
1891
1892 static int dmar_hp_remove_drhd(struct acpi_dmar_header *header, void *arg)
1893 {
1894         int i, ret;
1895         struct device *dev;
1896         struct dmar_drhd_unit *dmaru;
1897
1898         dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1899         if (!dmaru)
1900                 return 0;
1901
1902         /*
1903          * All PCI devices managed by this unit should have been destroyed.
1904          */
1905         if (!dmaru->include_all && dmaru->devices && dmaru->devices_cnt) {
1906                 for_each_active_dev_scope(dmaru->devices,
1907                                           dmaru->devices_cnt, i, dev)
1908                         return -EBUSY;
1909         }
1910
1911         ret = dmar_ir_hotplug(dmaru, false);
1912         if (ret == 0)
1913                 ret = dmar_iommu_hotplug(dmaru, false);
1914
1915         return ret;
1916 }
1917
1918 static int dmar_hp_release_drhd(struct acpi_dmar_header *header, void *arg)
1919 {
1920         struct dmar_drhd_unit *dmaru;
1921
1922         dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1923         if (dmaru) {
1924                 list_del_rcu(&dmaru->list);
1925                 synchronize_rcu();
1926                 dmar_free_drhd(dmaru);
1927         }
1928
1929         return 0;
1930 }
1931
1932 static int dmar_hotplug_insert(acpi_handle handle)
1933 {
1934         int ret;
1935         int drhd_count = 0;
1936
1937         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1938                                      &dmar_validate_one_drhd, (void *)1);
1939         if (ret)
1940                 goto out;
1941
1942         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1943                                      &dmar_parse_one_drhd, (void *)&drhd_count);
1944         if (ret == 0 && drhd_count == 0) {
1945                 pr_warn(FW_BUG "No DRHD structures in buffer returned by _DSM method\n");
1946                 goto out;
1947         } else if (ret) {
1948                 goto release_drhd;
1949         }
1950
1951         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_RHSA,
1952                                      &dmar_parse_one_rhsa, NULL);
1953         if (ret)
1954                 goto release_drhd;
1955
1956         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1957                                      &dmar_parse_one_atsr, NULL);
1958         if (ret)
1959                 goto release_atsr;
1960
1961         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1962                                      &dmar_hp_add_drhd, NULL);
1963         if (!ret)
1964                 return 0;
1965
1966         dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1967                                &dmar_hp_remove_drhd, NULL);
1968 release_atsr:
1969         dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1970                                &dmar_release_one_atsr, NULL);
1971 release_drhd:
1972         dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1973                                &dmar_hp_release_drhd, NULL);
1974 out:
1975         return ret;
1976 }
1977
1978 static int dmar_hotplug_remove(acpi_handle handle)
1979 {
1980         int ret;
1981
1982         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1983                                      &dmar_check_one_atsr, NULL);
1984         if (ret)
1985                 return ret;
1986
1987         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1988                                      &dmar_hp_remove_drhd, NULL);
1989         if (ret == 0) {
1990                 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1991                                                &dmar_release_one_atsr, NULL));
1992                 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1993                                                &dmar_hp_release_drhd, NULL));
1994         } else {
1995                 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1996                                        &dmar_hp_add_drhd, NULL);
1997         }
1998
1999         return ret;
2000 }
2001
2002 static acpi_status dmar_get_dsm_handle(acpi_handle handle, u32 lvl,
2003                                        void *context, void **retval)
2004 {
2005         acpi_handle *phdl = retval;
2006
2007         if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
2008                 *phdl = handle;
2009                 return AE_CTRL_TERMINATE;
2010         }
2011
2012         return AE_OK;
2013 }
2014
2015 static int dmar_device_hotplug(acpi_handle handle, bool insert)
2016 {
2017         int ret;
2018         acpi_handle tmp = NULL;
2019         acpi_status status;
2020
2021         if (!dmar_in_use())
2022                 return 0;
2023
2024         if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
2025                 tmp = handle;
2026         } else {
2027                 status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle,
2028                                              ACPI_UINT32_MAX,
2029                                              dmar_get_dsm_handle,
2030                                              NULL, NULL, &tmp);
2031                 if (ACPI_FAILURE(status)) {
2032                         pr_warn("Failed to locate _DSM method.\n");
2033                         return -ENXIO;
2034                 }
2035         }
2036         if (tmp == NULL)
2037                 return 0;
2038
2039         down_write(&dmar_global_lock);
2040         if (insert)
2041                 ret = dmar_hotplug_insert(tmp);
2042         else
2043                 ret = dmar_hotplug_remove(tmp);
2044         up_write(&dmar_global_lock);
2045
2046         return ret;
2047 }
2048
2049 int dmar_device_add(acpi_handle handle)
2050 {
2051         return dmar_device_hotplug(handle, true);
2052 }
2053
2054 int dmar_device_remove(acpi_handle handle)
2055 {
2056         return dmar_device_hotplug(handle, false);
2057 }