2 * Copyright © 2006-2014 Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
18 * Joerg Roedel <jroedel@suse.de>
21 #define pr_fmt(fmt) "DMAR: " fmt
23 #include <linux/init.h>
24 #include <linux/bitmap.h>
25 #include <linux/debugfs.h>
26 #include <linux/export.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/memory.h>
36 #include <linux/cpu.h>
37 #include <linux/timer.h>
39 #include <linux/iova.h>
40 #include <linux/iommu.h>
41 #include <linux/intel-iommu.h>
42 #include <linux/syscore_ops.h>
43 #include <linux/tboot.h>
44 #include <linux/dmi.h>
45 #include <linux/pci-ats.h>
46 #include <linux/memblock.h>
47 #include <linux/dma-contiguous.h>
48 #include <linux/dma-direct.h>
49 #include <linux/crash_dump.h>
50 #include <asm/irq_remapping.h>
51 #include <asm/cacheflush.h>
52 #include <asm/iommu.h>
54 #include "irq_remapping.h"
55 #include "intel-pasid.h"
57 #define ROOT_SIZE VTD_PAGE_SIZE
58 #define CONTEXT_SIZE VTD_PAGE_SIZE
60 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
61 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
62 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
63 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
65 #define IOAPIC_RANGE_START (0xfee00000)
66 #define IOAPIC_RANGE_END (0xfeefffff)
67 #define IOVA_START_ADDR (0x1000)
69 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
71 #define MAX_AGAW_WIDTH 64
72 #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
74 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
75 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
77 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
78 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
79 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
80 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
81 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
83 /* IO virtual address start page frame number */
84 #define IOVA_START_PFN (1)
86 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
88 /* page table handling */
89 #define LEVEL_STRIDE (9)
90 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
93 * This bitmap is used to advertise the page sizes our hardware support
94 * to the IOMMU core, which will then use this information to split
95 * physically contiguous memory regions it is mapping into page sizes
98 * Traditionally the IOMMU core just handed us the mappings directly,
99 * after making sure the size is an order of a 4KiB page and that the
100 * mapping has natural alignment.
102 * To retain this behavior, we currently advertise that we support
103 * all page sizes that are an order of 4KiB.
105 * If at some point we'd like to utilize the IOMMU core's new behavior,
106 * we could change this to advertise the real page sizes we support.
108 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
110 static inline int agaw_to_level(int agaw)
115 static inline int agaw_to_width(int agaw)
117 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
120 static inline int width_to_agaw(int width)
122 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
125 static inline unsigned int level_to_offset_bits(int level)
127 return (level - 1) * LEVEL_STRIDE;
130 static inline int pfn_level_offset(unsigned long pfn, int level)
132 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
135 static inline unsigned long level_mask(int level)
137 return -1UL << level_to_offset_bits(level);
140 static inline unsigned long level_size(int level)
142 return 1UL << level_to_offset_bits(level);
145 static inline unsigned long align_to_level(unsigned long pfn, int level)
147 return (pfn + level_size(level) - 1) & level_mask(level);
150 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
152 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
155 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
156 are never going to work. */
157 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
159 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
162 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
164 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
166 static inline unsigned long page_to_dma_pfn(struct page *pg)
168 return mm_to_dma_pfn(page_to_pfn(pg));
170 static inline unsigned long virt_to_dma_pfn(void *p)
172 return page_to_dma_pfn(virt_to_page(p));
175 /* global iommu list, set NULL for ignored DMAR units */
176 static struct intel_iommu **g_iommus;
178 static void __init check_tylersburg_isoch(void);
179 static int rwbf_quirk;
182 * set to 1 to panic kernel if can't successfully enable VT-d
183 * (used when kernel is launched w/ TXT)
185 static int force_on = 0;
186 int intel_iommu_tboot_noforce;
191 * 12-63: Context Ptr (12 - (haw-1))
198 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
201 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
204 static phys_addr_t root_entry_lctp(struct root_entry *re)
209 return re->lo & VTD_PAGE_MASK;
213 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
216 static phys_addr_t root_entry_uctp(struct root_entry *re)
221 return re->hi & VTD_PAGE_MASK;
226 * 1: fault processing disable
227 * 2-3: translation type
228 * 12-63: address space root
234 struct context_entry {
239 static inline void context_clear_pasid_enable(struct context_entry *context)
241 context->lo &= ~(1ULL << 11);
244 static inline bool context_pasid_enabled(struct context_entry *context)
246 return !!(context->lo & (1ULL << 11));
249 static inline void context_set_copied(struct context_entry *context)
251 context->hi |= (1ull << 3);
254 static inline bool context_copied(struct context_entry *context)
256 return !!(context->hi & (1ULL << 3));
259 static inline bool __context_present(struct context_entry *context)
261 return (context->lo & 1);
264 static inline bool context_present(struct context_entry *context)
266 return context_pasid_enabled(context) ?
267 __context_present(context) :
268 __context_present(context) && !context_copied(context);
271 static inline void context_set_present(struct context_entry *context)
276 static inline void context_set_fault_enable(struct context_entry *context)
278 context->lo &= (((u64)-1) << 2) | 1;
281 static inline void context_set_translation_type(struct context_entry *context,
284 context->lo &= (((u64)-1) << 4) | 3;
285 context->lo |= (value & 3) << 2;
288 static inline void context_set_address_root(struct context_entry *context,
291 context->lo &= ~VTD_PAGE_MASK;
292 context->lo |= value & VTD_PAGE_MASK;
295 static inline void context_set_address_width(struct context_entry *context,
298 context->hi |= value & 7;
301 static inline void context_set_domain_id(struct context_entry *context,
304 context->hi |= (value & ((1 << 16) - 1)) << 8;
307 static inline int context_domain_id(struct context_entry *c)
309 return((c->hi >> 8) & 0xffff);
312 static inline void context_clear_entry(struct context_entry *context)
325 * 12-63: Host physcial address
331 static inline void dma_clear_pte(struct dma_pte *pte)
336 static inline u64 dma_pte_addr(struct dma_pte *pte)
339 return pte->val & VTD_PAGE_MASK;
341 /* Must have a full atomic 64-bit read */
342 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
346 static inline bool dma_pte_present(struct dma_pte *pte)
348 return (pte->val & 3) != 0;
351 static inline bool dma_pte_superpage(struct dma_pte *pte)
353 return (pte->val & DMA_PTE_LARGE_PAGE);
356 static inline int first_pte_in_page(struct dma_pte *pte)
358 return !((unsigned long)pte & ~VTD_PAGE_MASK);
362 * This domain is a statically identity mapping domain.
363 * 1. This domain creats a static 1:1 mapping to all usable memory.
364 * 2. It maps to each iommu if successful.
365 * 3. Each iommu mapps to this domain if successful.
367 static struct dmar_domain *si_domain;
368 static int hw_pass_through = 1;
371 * Domain represents a virtual machine, more than one devices
372 * across iommus may be owned in one domain, e.g. kvm guest.
374 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
376 /* si_domain contains mulitple devices */
377 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
379 #define for_each_domain_iommu(idx, domain) \
380 for (idx = 0; idx < g_num_of_iommus; idx++) \
381 if (domain->iommu_refcnt[idx])
383 struct dmar_rmrr_unit {
384 struct list_head list; /* list of rmrr units */
385 struct acpi_dmar_header *hdr; /* ACPI header */
386 u64 base_address; /* reserved base address*/
387 u64 end_address; /* reserved end address */
388 struct dmar_dev_scope *devices; /* target devices */
389 int devices_cnt; /* target device count */
392 struct dmar_atsr_unit {
393 struct list_head list; /* list of ATSR units */
394 struct acpi_dmar_header *hdr; /* ACPI header */
395 struct dmar_dev_scope *devices; /* target devices */
396 int devices_cnt; /* target device count */
397 u8 include_all:1; /* include all ports */
400 static LIST_HEAD(dmar_atsr_units);
401 static LIST_HEAD(dmar_rmrr_units);
403 #define for_each_rmrr_units(rmrr) \
404 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
406 /* bitmap for indexing intel_iommus */
407 static int g_num_of_iommus;
409 static void domain_exit(struct dmar_domain *domain);
410 static void domain_remove_dev_info(struct dmar_domain *domain);
411 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
413 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
414 static void domain_context_clear(struct intel_iommu *iommu,
416 static int domain_detach_iommu(struct dmar_domain *domain,
417 struct intel_iommu *iommu);
419 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
420 int dmar_disabled = 0;
422 int dmar_disabled = 1;
423 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
425 int intel_iommu_enabled = 0;
426 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
428 static int dmar_map_gfx = 1;
429 static int dmar_forcedac;
430 static int intel_iommu_strict;
431 static int intel_iommu_superpage = 1;
432 static int intel_iommu_ecs = 1;
433 static int intel_iommu_pasid28;
434 static int iommu_identity_mapping;
436 #define IDENTMAP_ALL 1
437 #define IDENTMAP_GFX 2
438 #define IDENTMAP_AZALIA 4
440 /* Broadwell and Skylake have broken ECS support — normal so-called "second
441 * level" translation of DMA requests-without-PASID doesn't actually happen
442 * unless you also set the NESTE bit in an extended context-entry. Which of
443 * course means that SVM doesn't work because it's trying to do nested
444 * translation of the physical addresses it finds in the process page tables,
445 * through the IOVA->phys mapping found in the "second level" page tables.
447 * The VT-d specification was retroactively changed to change the definition
448 * of the capability bits and pretend that Broadwell/Skylake never happened...
449 * but unfortunately the wrong bit was changed. It's ECS which is broken, but
450 * for some reason it was the PASID capability bit which was redefined (from
451 * bit 28 on BDW/SKL to bit 40 in future).
453 * So our test for ECS needs to eschew those implementations which set the old
454 * PASID capabiity bit 28, since those are the ones on which ECS is broken.
455 * Unless we are working around the 'pasid28' limitations, that is, by putting
456 * the device into passthrough mode for normal DMA and thus masking the bug.
458 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
459 (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
460 /* PASID support is thus enabled if ECS is enabled and *either* of the old
461 * or new capability bits are set. */
462 #define pasid_enabled(iommu) (ecs_enabled(iommu) && \
463 (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
465 int intel_iommu_gfx_mapped;
466 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
468 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
469 static DEFINE_SPINLOCK(device_domain_lock);
470 static LIST_HEAD(device_domain_list);
473 * Iterate over elements in device_domain_list and call the specified
474 * callback @fn against each element. This helper should only be used
475 * in the context where the device_domain_lock has already been holden.
477 int for_each_device_domain(int (*fn)(struct device_domain_info *info,
478 void *data), void *data)
481 struct device_domain_info *info;
483 assert_spin_locked(&device_domain_lock);
484 list_for_each_entry(info, &device_domain_list, global) {
485 ret = fn(info, data);
493 const struct iommu_ops intel_iommu_ops;
495 static bool translation_pre_enabled(struct intel_iommu *iommu)
497 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
500 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
502 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
505 static void init_translation_status(struct intel_iommu *iommu)
509 gsts = readl(iommu->reg + DMAR_GSTS_REG);
510 if (gsts & DMA_GSTS_TES)
511 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
514 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
515 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
517 return container_of(dom, struct dmar_domain, domain);
520 static int __init intel_iommu_setup(char *str)
525 if (!strncmp(str, "on", 2)) {
527 pr_info("IOMMU enabled\n");
528 } else if (!strncmp(str, "off", 3)) {
530 pr_info("IOMMU disabled\n");
531 } else if (!strncmp(str, "igfx_off", 8)) {
533 pr_info("Disable GFX device mapping\n");
534 } else if (!strncmp(str, "forcedac", 8)) {
535 pr_info("Forcing DAC for PCI devices\n");
537 } else if (!strncmp(str, "strict", 6)) {
538 pr_info("Disable batched IOTLB flush\n");
539 intel_iommu_strict = 1;
540 } else if (!strncmp(str, "sp_off", 6)) {
541 pr_info("Disable supported super page\n");
542 intel_iommu_superpage = 0;
543 } else if (!strncmp(str, "ecs_off", 7)) {
545 "Intel-IOMMU: disable extended context table support\n");
547 } else if (!strncmp(str, "pasid28", 7)) {
549 "Intel-IOMMU: enable pre-production PASID support\n");
550 intel_iommu_pasid28 = 1;
551 iommu_identity_mapping |= IDENTMAP_GFX;
552 } else if (!strncmp(str, "tboot_noforce", 13)) {
554 "Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
555 intel_iommu_tboot_noforce = 1;
558 str += strcspn(str, ",");
564 __setup("intel_iommu=", intel_iommu_setup);
566 static struct kmem_cache *iommu_domain_cache;
567 static struct kmem_cache *iommu_devinfo_cache;
569 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
571 struct dmar_domain **domains;
574 domains = iommu->domains[idx];
578 return domains[did & 0xff];
581 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
582 struct dmar_domain *domain)
584 struct dmar_domain **domains;
587 if (!iommu->domains[idx]) {
588 size_t size = 256 * sizeof(struct dmar_domain *);
589 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
592 domains = iommu->domains[idx];
593 if (WARN_ON(!domains))
596 domains[did & 0xff] = domain;
599 void *alloc_pgtable_page(int node)
604 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
606 vaddr = page_address(page);
610 void free_pgtable_page(void *vaddr)
612 free_page((unsigned long)vaddr);
615 static inline void *alloc_domain_mem(void)
617 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
620 static void free_domain_mem(void *vaddr)
622 kmem_cache_free(iommu_domain_cache, vaddr);
625 static inline void * alloc_devinfo_mem(void)
627 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
630 static inline void free_devinfo_mem(void *vaddr)
632 kmem_cache_free(iommu_devinfo_cache, vaddr);
635 static inline int domain_type_is_vm(struct dmar_domain *domain)
637 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
640 static inline int domain_type_is_si(struct dmar_domain *domain)
642 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
645 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
647 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
648 DOMAIN_FLAG_STATIC_IDENTITY);
651 static inline int domain_pfn_supported(struct dmar_domain *domain,
654 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
656 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
659 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
664 sagaw = cap_sagaw(iommu->cap);
665 for (agaw = width_to_agaw(max_gaw);
667 if (test_bit(agaw, &sagaw))
675 * Calculate max SAGAW for each iommu.
677 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
679 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
683 * calculate agaw for each iommu.
684 * "SAGAW" may be different across iommus, use a default agaw, and
685 * get a supported less agaw for iommus that don't support the default agaw.
687 int iommu_calculate_agaw(struct intel_iommu *iommu)
689 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
692 /* This functionin only returns single iommu in a domain */
693 struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
697 /* si_domain and vm domain should not get here. */
698 BUG_ON(domain_type_is_vm_or_si(domain));
699 for_each_domain_iommu(iommu_id, domain)
702 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
705 return g_iommus[iommu_id];
708 static void domain_update_iommu_coherency(struct dmar_domain *domain)
710 struct dmar_drhd_unit *drhd;
711 struct intel_iommu *iommu;
715 domain->iommu_coherency = 1;
717 for_each_domain_iommu(i, domain) {
719 if (!ecap_coherent(g_iommus[i]->ecap)) {
720 domain->iommu_coherency = 0;
727 /* No hardware attached; use lowest common denominator */
729 for_each_active_iommu(iommu, drhd) {
730 if (!ecap_coherent(iommu->ecap)) {
731 domain->iommu_coherency = 0;
738 static int domain_update_iommu_snooping(struct intel_iommu *skip)
740 struct dmar_drhd_unit *drhd;
741 struct intel_iommu *iommu;
745 for_each_active_iommu(iommu, drhd) {
747 if (!ecap_sc_support(iommu->ecap)) {
758 static int domain_update_iommu_superpage(struct intel_iommu *skip)
760 struct dmar_drhd_unit *drhd;
761 struct intel_iommu *iommu;
764 if (!intel_iommu_superpage) {
768 /* set iommu_superpage to the smallest common denominator */
770 for_each_active_iommu(iommu, drhd) {
772 mask &= cap_super_page_val(iommu->cap);
782 /* Some capabilities may be different across iommus */
783 static void domain_update_iommu_cap(struct dmar_domain *domain)
785 domain_update_iommu_coherency(domain);
786 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
787 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
790 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
791 u8 bus, u8 devfn, int alloc)
793 struct root_entry *root = &iommu->root_entry[bus];
794 struct context_entry *context;
798 if (ecs_enabled(iommu)) {
806 context = phys_to_virt(*entry & VTD_PAGE_MASK);
808 unsigned long phy_addr;
812 context = alloc_pgtable_page(iommu->node);
816 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
817 phy_addr = virt_to_phys((void *)context);
818 *entry = phy_addr | 1;
819 __iommu_flush_cache(iommu, entry, sizeof(*entry));
821 return &context[devfn];
824 static int iommu_dummy(struct device *dev)
826 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
829 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
831 struct dmar_drhd_unit *drhd = NULL;
832 struct intel_iommu *iommu;
834 struct pci_dev *ptmp, *pdev = NULL;
838 if (iommu_dummy(dev))
841 if (dev_is_pci(dev)) {
842 struct pci_dev *pf_pdev;
844 pdev = to_pci_dev(dev);
847 /* VMD child devices currently cannot be handled individually */
848 if (is_vmd(pdev->bus))
852 /* VFs aren't listed in scope tables; we need to look up
853 * the PF instead to find the IOMMU. */
854 pf_pdev = pci_physfn(pdev);
856 segment = pci_domain_nr(pdev->bus);
857 } else if (has_acpi_companion(dev))
858 dev = &ACPI_COMPANION(dev)->dev;
861 for_each_active_iommu(iommu, drhd) {
862 if (pdev && segment != drhd->segment)
865 for_each_active_dev_scope(drhd->devices,
866 drhd->devices_cnt, i, tmp) {
868 /* For a VF use its original BDF# not that of the PF
869 * which we used for the IOMMU lookup. Strictly speaking
870 * we could do this for all PCI devices; we only need to
871 * get the BDF# from the scope table for ACPI matches. */
872 if (pdev && pdev->is_virtfn)
875 *bus = drhd->devices[i].bus;
876 *devfn = drhd->devices[i].devfn;
880 if (!pdev || !dev_is_pci(tmp))
883 ptmp = to_pci_dev(tmp);
884 if (ptmp->subordinate &&
885 ptmp->subordinate->number <= pdev->bus->number &&
886 ptmp->subordinate->busn_res.end >= pdev->bus->number)
890 if (pdev && drhd->include_all) {
892 *bus = pdev->bus->number;
893 *devfn = pdev->devfn;
904 static void domain_flush_cache(struct dmar_domain *domain,
905 void *addr, int size)
907 if (!domain->iommu_coherency)
908 clflush_cache_range(addr, size);
911 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
913 struct context_entry *context;
917 spin_lock_irqsave(&iommu->lock, flags);
918 context = iommu_context_addr(iommu, bus, devfn, 0);
920 ret = context_present(context);
921 spin_unlock_irqrestore(&iommu->lock, flags);
925 static void free_context_table(struct intel_iommu *iommu)
929 struct context_entry *context;
931 spin_lock_irqsave(&iommu->lock, flags);
932 if (!iommu->root_entry) {
935 for (i = 0; i < ROOT_ENTRY_NR; i++) {
936 context = iommu_context_addr(iommu, i, 0, 0);
938 free_pgtable_page(context);
940 if (!ecs_enabled(iommu))
943 context = iommu_context_addr(iommu, i, 0x80, 0);
945 free_pgtable_page(context);
948 free_pgtable_page(iommu->root_entry);
949 iommu->root_entry = NULL;
951 spin_unlock_irqrestore(&iommu->lock, flags);
954 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
955 unsigned long pfn, int *target_level)
957 struct dma_pte *parent, *pte = NULL;
958 int level = agaw_to_level(domain->agaw);
961 BUG_ON(!domain->pgd);
963 if (!domain_pfn_supported(domain, pfn))
964 /* Address beyond IOMMU's addressing capabilities. */
967 parent = domain->pgd;
972 offset = pfn_level_offset(pfn, level);
973 pte = &parent[offset];
974 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
976 if (level == *target_level)
979 if (!dma_pte_present(pte)) {
982 tmp_page = alloc_pgtable_page(domain->nid);
987 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
988 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
989 if (cmpxchg64(&pte->val, 0ULL, pteval))
990 /* Someone else set it while we were thinking; use theirs. */
991 free_pgtable_page(tmp_page);
993 domain_flush_cache(domain, pte, sizeof(*pte));
998 parent = phys_to_virt(dma_pte_addr(pte));
1003 *target_level = level;
1009 /* return address's pte at specific level */
1010 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1012 int level, int *large_page)
1014 struct dma_pte *parent, *pte = NULL;
1015 int total = agaw_to_level(domain->agaw);
1018 parent = domain->pgd;
1019 while (level <= total) {
1020 offset = pfn_level_offset(pfn, total);
1021 pte = &parent[offset];
1025 if (!dma_pte_present(pte)) {
1026 *large_page = total;
1030 if (dma_pte_superpage(pte)) {
1031 *large_page = total;
1035 parent = phys_to_virt(dma_pte_addr(pte));
1041 /* clear last level pte, a tlb flush should be followed */
1042 static void dma_pte_clear_range(struct dmar_domain *domain,
1043 unsigned long start_pfn,
1044 unsigned long last_pfn)
1046 unsigned int large_page = 1;
1047 struct dma_pte *first_pte, *pte;
1049 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1050 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1051 BUG_ON(start_pfn > last_pfn);
1053 /* we don't need lock here; nobody else touches the iova range */
1056 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1058 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1063 start_pfn += lvl_to_nr_pages(large_page);
1065 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1067 domain_flush_cache(domain, first_pte,
1068 (void *)pte - (void *)first_pte);
1070 } while (start_pfn && start_pfn <= last_pfn);
1073 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1074 int retain_level, struct dma_pte *pte,
1075 unsigned long pfn, unsigned long start_pfn,
1076 unsigned long last_pfn)
1078 pfn = max(start_pfn, pfn);
1079 pte = &pte[pfn_level_offset(pfn, level)];
1082 unsigned long level_pfn;
1083 struct dma_pte *level_pte;
1085 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1088 level_pfn = pfn & level_mask(level);
1089 level_pte = phys_to_virt(dma_pte_addr(pte));
1092 dma_pte_free_level(domain, level - 1, retain_level,
1093 level_pte, level_pfn, start_pfn,
1098 * Free the page table if we're below the level we want to
1099 * retain and the range covers the entire table.
1101 if (level < retain_level && !(start_pfn > level_pfn ||
1102 last_pfn < level_pfn + level_size(level) - 1)) {
1104 domain_flush_cache(domain, pte, sizeof(*pte));
1105 free_pgtable_page(level_pte);
1108 pfn += level_size(level);
1109 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1113 * clear last level (leaf) ptes and free page table pages below the
1114 * level we wish to keep intact.
1116 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1117 unsigned long start_pfn,
1118 unsigned long last_pfn,
1121 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1122 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1123 BUG_ON(start_pfn > last_pfn);
1125 dma_pte_clear_range(domain, start_pfn, last_pfn);
1127 /* We don't need lock here; nobody else touches the iova range */
1128 dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
1129 domain->pgd, 0, start_pfn, last_pfn);
1132 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1133 free_pgtable_page(domain->pgd);
1138 /* When a page at a given level is being unlinked from its parent, we don't
1139 need to *modify* it at all. All we need to do is make a list of all the
1140 pages which can be freed just as soon as we've flushed the IOTLB and we
1141 know the hardware page-walk will no longer touch them.
1142 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1144 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1145 int level, struct dma_pte *pte,
1146 struct page *freelist)
1150 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1151 pg->freelist = freelist;
1157 pte = page_address(pg);
1159 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1160 freelist = dma_pte_list_pagetables(domain, level - 1,
1163 } while (!first_pte_in_page(pte));
1168 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1169 struct dma_pte *pte, unsigned long pfn,
1170 unsigned long start_pfn,
1171 unsigned long last_pfn,
1172 struct page *freelist)
1174 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1176 pfn = max(start_pfn, pfn);
1177 pte = &pte[pfn_level_offset(pfn, level)];
1180 unsigned long level_pfn;
1182 if (!dma_pte_present(pte))
1185 level_pfn = pfn & level_mask(level);
1187 /* If range covers entire pagetable, free it */
1188 if (start_pfn <= level_pfn &&
1189 last_pfn >= level_pfn + level_size(level) - 1) {
1190 /* These suborbinate page tables are going away entirely. Don't
1191 bother to clear them; we're just going to *free* them. */
1192 if (level > 1 && !dma_pte_superpage(pte))
1193 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1199 } else if (level > 1) {
1200 /* Recurse down into a level that isn't *entirely* obsolete */
1201 freelist = dma_pte_clear_level(domain, level - 1,
1202 phys_to_virt(dma_pte_addr(pte)),
1203 level_pfn, start_pfn, last_pfn,
1207 pfn += level_size(level);
1208 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1211 domain_flush_cache(domain, first_pte,
1212 (void *)++last_pte - (void *)first_pte);
1217 /* We can't just free the pages because the IOMMU may still be walking
1218 the page tables, and may have cached the intermediate levels. The
1219 pages can only be freed after the IOTLB flush has been done. */
1220 static struct page *domain_unmap(struct dmar_domain *domain,
1221 unsigned long start_pfn,
1222 unsigned long last_pfn)
1224 struct page *freelist = NULL;
1226 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1227 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1228 BUG_ON(start_pfn > last_pfn);
1230 /* we don't need lock here; nobody else touches the iova range */
1231 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1232 domain->pgd, 0, start_pfn, last_pfn, NULL);
1235 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1236 struct page *pgd_page = virt_to_page(domain->pgd);
1237 pgd_page->freelist = freelist;
1238 freelist = pgd_page;
1246 static void dma_free_pagelist(struct page *freelist)
1250 while ((pg = freelist)) {
1251 freelist = pg->freelist;
1252 free_pgtable_page(page_address(pg));
1256 static void iova_entry_free(unsigned long data)
1258 struct page *freelist = (struct page *)data;
1260 dma_free_pagelist(freelist);
1263 /* iommu handling */
1264 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1266 struct root_entry *root;
1267 unsigned long flags;
1269 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1271 pr_err("Allocating root entry for %s failed\n",
1276 __iommu_flush_cache(iommu, root, ROOT_SIZE);
1278 spin_lock_irqsave(&iommu->lock, flags);
1279 iommu->root_entry = root;
1280 spin_unlock_irqrestore(&iommu->lock, flags);
1285 static void iommu_set_root_entry(struct intel_iommu *iommu)
1291 addr = virt_to_phys(iommu->root_entry);
1292 if (ecs_enabled(iommu))
1293 addr |= DMA_RTADDR_RTT;
1295 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1296 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1298 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1300 /* Make sure hardware complete it */
1301 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1302 readl, (sts & DMA_GSTS_RTPS), sts);
1304 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1307 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1312 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1315 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1316 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1318 /* Make sure hardware complete it */
1319 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1320 readl, (!(val & DMA_GSTS_WBFS)), val);
1322 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1325 /* return value determine if we need a write buffer flush */
1326 static void __iommu_flush_context(struct intel_iommu *iommu,
1327 u16 did, u16 source_id, u8 function_mask,
1334 case DMA_CCMD_GLOBAL_INVL:
1335 val = DMA_CCMD_GLOBAL_INVL;
1337 case DMA_CCMD_DOMAIN_INVL:
1338 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1340 case DMA_CCMD_DEVICE_INVL:
1341 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1342 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1347 val |= DMA_CCMD_ICC;
1349 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1350 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1352 /* Make sure hardware complete it */
1353 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1354 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1356 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1359 /* return value determine if we need a write buffer flush */
1360 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1361 u64 addr, unsigned int size_order, u64 type)
1363 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1364 u64 val = 0, val_iva = 0;
1368 case DMA_TLB_GLOBAL_FLUSH:
1369 /* global flush doesn't need set IVA_REG */
1370 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1372 case DMA_TLB_DSI_FLUSH:
1373 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1375 case DMA_TLB_PSI_FLUSH:
1376 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1377 /* IH bit is passed in as part of address */
1378 val_iva = size_order | addr;
1383 /* Note: set drain read/write */
1386 * This is probably to be super secure.. Looks like we can
1387 * ignore it without any impact.
1389 if (cap_read_drain(iommu->cap))
1390 val |= DMA_TLB_READ_DRAIN;
1392 if (cap_write_drain(iommu->cap))
1393 val |= DMA_TLB_WRITE_DRAIN;
1395 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1396 /* Note: Only uses first TLB reg currently */
1398 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1399 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1401 /* Make sure hardware complete it */
1402 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1403 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1405 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1407 /* check IOTLB invalidation granularity */
1408 if (DMA_TLB_IAIG(val) == 0)
1409 pr_err("Flush IOTLB failed\n");
1410 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1411 pr_debug("TLB flush request %Lx, actual %Lx\n",
1412 (unsigned long long)DMA_TLB_IIRG(type),
1413 (unsigned long long)DMA_TLB_IAIG(val));
1416 static struct device_domain_info *
1417 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1420 struct device_domain_info *info;
1422 assert_spin_locked(&device_domain_lock);
1427 list_for_each_entry(info, &domain->devices, link)
1428 if (info->iommu == iommu && info->bus == bus &&
1429 info->devfn == devfn) {
1430 if (info->ats_supported && info->dev)
1438 static void domain_update_iotlb(struct dmar_domain *domain)
1440 struct device_domain_info *info;
1441 bool has_iotlb_device = false;
1443 assert_spin_locked(&device_domain_lock);
1445 list_for_each_entry(info, &domain->devices, link) {
1446 struct pci_dev *pdev;
1448 if (!info->dev || !dev_is_pci(info->dev))
1451 pdev = to_pci_dev(info->dev);
1452 if (pdev->ats_enabled) {
1453 has_iotlb_device = true;
1458 domain->has_iotlb_device = has_iotlb_device;
1461 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1463 struct pci_dev *pdev;
1465 assert_spin_locked(&device_domain_lock);
1467 if (!info || !dev_is_pci(info->dev))
1470 pdev = to_pci_dev(info->dev);
1471 /* For IOMMU that supports device IOTLB throttling (DIT), we assign
1472 * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
1473 * queue depth at PF level. If DIT is not set, PFSID will be treated as
1474 * reserved, which should be set to 0.
1476 if (!ecap_dit(info->iommu->ecap))
1479 struct pci_dev *pf_pdev;
1481 /* pdev will be returned if device is not a vf */
1482 pf_pdev = pci_physfn(pdev);
1483 info->pfsid = PCI_DEVID(pf_pdev->bus->number, pf_pdev->devfn);
1486 #ifdef CONFIG_INTEL_IOMMU_SVM
1487 /* The PCIe spec, in its wisdom, declares that the behaviour of
1488 the device if you enable PASID support after ATS support is
1489 undefined. So always enable PASID support on devices which
1490 have it, even if we can't yet know if we're ever going to
1492 if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1493 info->pasid_enabled = 1;
1495 if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1496 info->pri_enabled = 1;
1498 if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1499 info->ats_enabled = 1;
1500 domain_update_iotlb(info->domain);
1501 info->ats_qdep = pci_ats_queue_depth(pdev);
1505 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1507 struct pci_dev *pdev;
1509 assert_spin_locked(&device_domain_lock);
1511 if (!dev_is_pci(info->dev))
1514 pdev = to_pci_dev(info->dev);
1516 if (info->ats_enabled) {
1517 pci_disable_ats(pdev);
1518 info->ats_enabled = 0;
1519 domain_update_iotlb(info->domain);
1521 #ifdef CONFIG_INTEL_IOMMU_SVM
1522 if (info->pri_enabled) {
1523 pci_disable_pri(pdev);
1524 info->pri_enabled = 0;
1526 if (info->pasid_enabled) {
1527 pci_disable_pasid(pdev);
1528 info->pasid_enabled = 0;
1533 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1534 u64 addr, unsigned mask)
1537 unsigned long flags;
1538 struct device_domain_info *info;
1540 if (!domain->has_iotlb_device)
1543 spin_lock_irqsave(&device_domain_lock, flags);
1544 list_for_each_entry(info, &domain->devices, link) {
1545 if (!info->ats_enabled)
1548 sid = info->bus << 8 | info->devfn;
1549 qdep = info->ats_qdep;
1550 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1553 spin_unlock_irqrestore(&device_domain_lock, flags);
1556 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1557 struct dmar_domain *domain,
1558 unsigned long pfn, unsigned int pages,
1561 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1562 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1563 u16 did = domain->iommu_did[iommu->seq_id];
1570 * Fallback to domain selective flush if no PSI support or the size is
1572 * PSI requires page size to be 2 ^ x, and the base address is naturally
1573 * aligned to the size
1575 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1576 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1579 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1583 * In caching mode, changes of pages from non-present to present require
1584 * flush. However, device IOTLB doesn't need to be flushed in this case.
1586 if (!cap_caching_mode(iommu->cap) || !map)
1587 iommu_flush_dev_iotlb(domain, addr, mask);
1590 /* Notification for newly created mappings */
1591 static inline void __mapping_notify_one(struct intel_iommu *iommu,
1592 struct dmar_domain *domain,
1593 unsigned long pfn, unsigned int pages)
1595 /* It's a non-present to present mapping. Only flush if caching mode */
1596 if (cap_caching_mode(iommu->cap))
1597 iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
1599 iommu_flush_write_buffer(iommu);
1602 static void iommu_flush_iova(struct iova_domain *iovad)
1604 struct dmar_domain *domain;
1607 domain = container_of(iovad, struct dmar_domain, iovad);
1609 for_each_domain_iommu(idx, domain) {
1610 struct intel_iommu *iommu = g_iommus[idx];
1611 u16 did = domain->iommu_did[iommu->seq_id];
1613 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1615 if (!cap_caching_mode(iommu->cap))
1616 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1617 0, MAX_AGAW_PFN_WIDTH);
1621 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1624 unsigned long flags;
1626 if (!cap_plmr(iommu->cap) && !cap_phmr(iommu->cap))
1629 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1630 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1631 pmen &= ~DMA_PMEN_EPM;
1632 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1634 /* wait for the protected region status bit to clear */
1635 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1636 readl, !(pmen & DMA_PMEN_PRS), pmen);
1638 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1641 static void iommu_enable_translation(struct intel_iommu *iommu)
1644 unsigned long flags;
1646 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1647 iommu->gcmd |= DMA_GCMD_TE;
1648 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1650 /* Make sure hardware complete it */
1651 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1652 readl, (sts & DMA_GSTS_TES), sts);
1654 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1657 static void iommu_disable_translation(struct intel_iommu *iommu)
1662 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1663 iommu->gcmd &= ~DMA_GCMD_TE;
1664 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1666 /* Make sure hardware complete it */
1667 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1668 readl, (!(sts & DMA_GSTS_TES)), sts);
1670 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1674 static int iommu_init_domains(struct intel_iommu *iommu)
1676 u32 ndomains, nlongs;
1679 ndomains = cap_ndoms(iommu->cap);
1680 pr_debug("%s: Number of Domains supported <%d>\n",
1681 iommu->name, ndomains);
1682 nlongs = BITS_TO_LONGS(ndomains);
1684 spin_lock_init(&iommu->lock);
1686 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1687 if (!iommu->domain_ids) {
1688 pr_err("%s: Allocating domain id array failed\n",
1693 size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1694 iommu->domains = kzalloc(size, GFP_KERNEL);
1696 if (iommu->domains) {
1697 size = 256 * sizeof(struct dmar_domain *);
1698 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1701 if (!iommu->domains || !iommu->domains[0]) {
1702 pr_err("%s: Allocating domain array failed\n",
1704 kfree(iommu->domain_ids);
1705 kfree(iommu->domains);
1706 iommu->domain_ids = NULL;
1707 iommu->domains = NULL;
1714 * If Caching mode is set, then invalid translations are tagged
1715 * with domain-id 0, hence we need to pre-allocate it. We also
1716 * use domain-id 0 as a marker for non-allocated domain-id, so
1717 * make sure it is not used for a real domain.
1719 set_bit(0, iommu->domain_ids);
1724 static void disable_dmar_iommu(struct intel_iommu *iommu)
1726 struct device_domain_info *info, *tmp;
1727 unsigned long flags;
1729 if (!iommu->domains || !iommu->domain_ids)
1733 spin_lock_irqsave(&device_domain_lock, flags);
1734 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1735 struct dmar_domain *domain;
1737 if (info->iommu != iommu)
1740 if (!info->dev || !info->domain)
1743 domain = info->domain;
1745 __dmar_remove_one_dev_info(info);
1747 if (!domain_type_is_vm_or_si(domain)) {
1749 * The domain_exit() function can't be called under
1750 * device_domain_lock, as it takes this lock itself.
1751 * So release the lock here and re-run the loop
1754 spin_unlock_irqrestore(&device_domain_lock, flags);
1755 domain_exit(domain);
1759 spin_unlock_irqrestore(&device_domain_lock, flags);
1761 if (iommu->gcmd & DMA_GCMD_TE)
1762 iommu_disable_translation(iommu);
1765 static void free_dmar_iommu(struct intel_iommu *iommu)
1767 if ((iommu->domains) && (iommu->domain_ids)) {
1768 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1771 for (i = 0; i < elems; i++)
1772 kfree(iommu->domains[i]);
1773 kfree(iommu->domains);
1774 kfree(iommu->domain_ids);
1775 iommu->domains = NULL;
1776 iommu->domain_ids = NULL;
1779 g_iommus[iommu->seq_id] = NULL;
1781 /* free context mapping */
1782 free_context_table(iommu);
1784 #ifdef CONFIG_INTEL_IOMMU_SVM
1785 if (pasid_enabled(iommu)) {
1786 if (ecap_prs(iommu->ecap))
1787 intel_svm_finish_prq(iommu);
1788 intel_svm_exit(iommu);
1793 static struct dmar_domain *alloc_domain(int flags)
1795 struct dmar_domain *domain;
1797 domain = alloc_domain_mem();
1801 memset(domain, 0, sizeof(*domain));
1803 domain->flags = flags;
1804 domain->has_iotlb_device = false;
1805 INIT_LIST_HEAD(&domain->devices);
1810 /* Must be called with iommu->lock */
1811 static int domain_attach_iommu(struct dmar_domain *domain,
1812 struct intel_iommu *iommu)
1814 unsigned long ndomains;
1817 assert_spin_locked(&device_domain_lock);
1818 assert_spin_locked(&iommu->lock);
1820 domain->iommu_refcnt[iommu->seq_id] += 1;
1821 domain->iommu_count += 1;
1822 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1823 ndomains = cap_ndoms(iommu->cap);
1824 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1826 if (num >= ndomains) {
1827 pr_err("%s: No free domain ids\n", iommu->name);
1828 domain->iommu_refcnt[iommu->seq_id] -= 1;
1829 domain->iommu_count -= 1;
1833 set_bit(num, iommu->domain_ids);
1834 set_iommu_domain(iommu, num, domain);
1836 domain->iommu_did[iommu->seq_id] = num;
1837 domain->nid = iommu->node;
1839 domain_update_iommu_cap(domain);
1845 static int domain_detach_iommu(struct dmar_domain *domain,
1846 struct intel_iommu *iommu)
1848 int num, count = INT_MAX;
1850 assert_spin_locked(&device_domain_lock);
1851 assert_spin_locked(&iommu->lock);
1853 domain->iommu_refcnt[iommu->seq_id] -= 1;
1854 count = --domain->iommu_count;
1855 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1856 num = domain->iommu_did[iommu->seq_id];
1857 clear_bit(num, iommu->domain_ids);
1858 set_iommu_domain(iommu, num, NULL);
1860 domain_update_iommu_cap(domain);
1861 domain->iommu_did[iommu->seq_id] = 0;
1867 static struct iova_domain reserved_iova_list;
1868 static struct lock_class_key reserved_rbtree_key;
1870 static int dmar_init_reserved_ranges(void)
1872 struct pci_dev *pdev = NULL;
1876 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN);
1878 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1879 &reserved_rbtree_key);
1881 /* IOAPIC ranges shouldn't be accessed by DMA */
1882 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1883 IOVA_PFN(IOAPIC_RANGE_END));
1885 pr_err("Reserve IOAPIC range failed\n");
1889 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1890 for_each_pci_dev(pdev) {
1893 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1894 r = &pdev->resource[i];
1895 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1897 iova = reserve_iova(&reserved_iova_list,
1901 pr_err("Reserve iova failed\n");
1909 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1911 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1914 static inline int guestwidth_to_adjustwidth(int gaw)
1917 int r = (gaw - 12) % 9;
1928 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1931 int adjust_width, agaw, cap_width;
1932 unsigned long sagaw;
1935 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
1937 err = init_iova_flush_queue(&domain->iovad,
1938 iommu_flush_iova, iova_entry_free);
1942 domain_reserve_special_ranges(domain);
1944 /* calculate AGAW */
1945 cap_width = min_t(int, cap_mgaw(iommu->cap), agaw_to_width(iommu->agaw));
1946 if (guest_width > cap_width)
1947 guest_width = cap_width;
1948 domain->gaw = guest_width;
1949 adjust_width = guestwidth_to_adjustwidth(guest_width);
1950 agaw = width_to_agaw(adjust_width);
1951 sagaw = cap_sagaw(iommu->cap);
1952 if (!test_bit(agaw, &sagaw)) {
1953 /* hardware doesn't support it, choose a bigger one */
1954 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1955 agaw = find_next_bit(&sagaw, 5, agaw);
1959 domain->agaw = agaw;
1961 if (ecap_coherent(iommu->ecap))
1962 domain->iommu_coherency = 1;
1964 domain->iommu_coherency = 0;
1966 if (ecap_sc_support(iommu->ecap))
1967 domain->iommu_snooping = 1;
1969 domain->iommu_snooping = 0;
1971 if (intel_iommu_superpage)
1972 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1974 domain->iommu_superpage = 0;
1976 domain->nid = iommu->node;
1978 /* always allocate the top pgd */
1979 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1982 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1986 static void domain_exit(struct dmar_domain *domain)
1988 struct page *freelist = NULL;
1990 /* Domain 0 is reserved, so dont process it */
1994 /* Remove associated devices and clear attached or cached domains */
1996 domain_remove_dev_info(domain);
2000 put_iova_domain(&domain->iovad);
2002 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
2004 dma_free_pagelist(freelist);
2006 free_domain_mem(domain);
2009 static int domain_context_mapping_one(struct dmar_domain *domain,
2010 struct intel_iommu *iommu,
2013 u16 did = domain->iommu_did[iommu->seq_id];
2014 int translation = CONTEXT_TT_MULTI_LEVEL;
2015 struct device_domain_info *info = NULL;
2016 struct context_entry *context;
2017 unsigned long flags;
2018 struct dma_pte *pgd;
2023 if (hw_pass_through && domain_type_is_si(domain))
2024 translation = CONTEXT_TT_PASS_THROUGH;
2026 pr_debug("Set context mapping for %02x:%02x.%d\n",
2027 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
2029 BUG_ON(!domain->pgd);
2031 spin_lock_irqsave(&device_domain_lock, flags);
2032 spin_lock(&iommu->lock);
2035 context = iommu_context_addr(iommu, bus, devfn, 1);
2040 if (context_present(context))
2044 * For kdump cases, old valid entries may be cached due to the
2045 * in-flight DMA and copied pgtable, but there is no unmapping
2046 * behaviour for them, thus we need an explicit cache flush for
2047 * the newly-mapped device. For kdump, at this point, the device
2048 * is supposed to finish reset at its driver probe stage, so no
2049 * in-flight DMA will exist, and we don't need to worry anymore
2052 if (context_copied(context)) {
2053 u16 did_old = context_domain_id(context);
2055 if (did_old < cap_ndoms(iommu->cap)) {
2056 iommu->flush.flush_context(iommu, did_old,
2057 (((u16)bus) << 8) | devfn,
2058 DMA_CCMD_MASK_NOBIT,
2059 DMA_CCMD_DEVICE_INVL);
2060 iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
2067 context_clear_entry(context);
2068 context_set_domain_id(context, did);
2071 * Skip top levels of page tables for iommu which has less agaw
2072 * than default. Unnecessary for PT mode.
2074 if (translation != CONTEXT_TT_PASS_THROUGH) {
2075 for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
2077 pgd = phys_to_virt(dma_pte_addr(pgd));
2078 if (!dma_pte_present(pgd))
2082 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2083 if (info && info->ats_supported)
2084 translation = CONTEXT_TT_DEV_IOTLB;
2086 translation = CONTEXT_TT_MULTI_LEVEL;
2088 context_set_address_root(context, virt_to_phys(pgd));
2089 context_set_address_width(context, agaw);
2092 * In pass through mode, AW must be programmed to
2093 * indicate the largest AGAW value supported by
2094 * hardware. And ASR is ignored by hardware.
2096 context_set_address_width(context, iommu->msagaw);
2099 context_set_translation_type(context, translation);
2100 context_set_fault_enable(context);
2101 context_set_present(context);
2102 domain_flush_cache(domain, context, sizeof(*context));
2105 * It's a non-present to present mapping. If hardware doesn't cache
2106 * non-present entry we only need to flush the write-buffer. If the
2107 * _does_ cache non-present entries, then it does so in the special
2108 * domain #0, which we have to flush:
2110 if (cap_caching_mode(iommu->cap)) {
2111 iommu->flush.flush_context(iommu, 0,
2112 (((u16)bus) << 8) | devfn,
2113 DMA_CCMD_MASK_NOBIT,
2114 DMA_CCMD_DEVICE_INVL);
2115 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2117 iommu_flush_write_buffer(iommu);
2119 iommu_enable_dev_iotlb(info);
2124 spin_unlock(&iommu->lock);
2125 spin_unlock_irqrestore(&device_domain_lock, flags);
2130 struct domain_context_mapping_data {
2131 struct dmar_domain *domain;
2132 struct intel_iommu *iommu;
2135 static int domain_context_mapping_cb(struct pci_dev *pdev,
2136 u16 alias, void *opaque)
2138 struct domain_context_mapping_data *data = opaque;
2140 return domain_context_mapping_one(data->domain, data->iommu,
2141 PCI_BUS_NUM(alias), alias & 0xff);
2145 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2147 struct intel_iommu *iommu;
2149 struct domain_context_mapping_data data;
2151 iommu = device_to_iommu(dev, &bus, &devfn);
2155 if (!dev_is_pci(dev))
2156 return domain_context_mapping_one(domain, iommu, bus, devfn);
2158 data.domain = domain;
2161 return pci_for_each_dma_alias(to_pci_dev(dev),
2162 &domain_context_mapping_cb, &data);
2165 static int domain_context_mapped_cb(struct pci_dev *pdev,
2166 u16 alias, void *opaque)
2168 struct intel_iommu *iommu = opaque;
2170 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2173 static int domain_context_mapped(struct device *dev)
2175 struct intel_iommu *iommu;
2178 iommu = device_to_iommu(dev, &bus, &devfn);
2182 if (!dev_is_pci(dev))
2183 return device_context_mapped(iommu, bus, devfn);
2185 return !pci_for_each_dma_alias(to_pci_dev(dev),
2186 domain_context_mapped_cb, iommu);
2189 /* Returns a number of VTD pages, but aligned to MM page size */
2190 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2193 host_addr &= ~PAGE_MASK;
2194 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2197 /* Return largest possible superpage level for a given mapping */
2198 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2199 unsigned long iov_pfn,
2200 unsigned long phy_pfn,
2201 unsigned long pages)
2203 int support, level = 1;
2204 unsigned long pfnmerge;
2206 support = domain->iommu_superpage;
2208 /* To use a large page, the virtual *and* physical addresses
2209 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2210 of them will mean we have to use smaller pages. So just
2211 merge them and check both at once. */
2212 pfnmerge = iov_pfn | phy_pfn;
2214 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2215 pages >>= VTD_STRIDE_SHIFT;
2218 pfnmerge >>= VTD_STRIDE_SHIFT;
2225 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2226 struct scatterlist *sg, unsigned long phys_pfn,
2227 unsigned long nr_pages, int prot)
2229 struct dma_pte *first_pte = NULL, *pte = NULL;
2230 phys_addr_t uninitialized_var(pteval);
2231 unsigned long sg_res = 0;
2232 unsigned int largepage_lvl = 0;
2233 unsigned long lvl_pages = 0;
2235 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2237 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2240 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2244 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2247 while (nr_pages > 0) {
2251 unsigned int pgoff = sg->offset & ~PAGE_MASK;
2253 sg_res = aligned_nrpages(sg->offset, sg->length);
2254 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
2255 sg->dma_length = sg->length;
2256 pteval = (sg_phys(sg) - pgoff) | prot;
2257 phys_pfn = pteval >> VTD_PAGE_SHIFT;
2261 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2263 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2266 /* It is large page*/
2267 if (largepage_lvl > 1) {
2268 unsigned long nr_superpages, end_pfn;
2270 pteval |= DMA_PTE_LARGE_PAGE;
2271 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2273 nr_superpages = sg_res / lvl_pages;
2274 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2277 * Ensure that old small page tables are
2278 * removed to make room for superpage(s).
2279 * We're adding new large pages, so make sure
2280 * we don't remove their parent tables.
2282 dma_pte_free_pagetable(domain, iov_pfn, end_pfn,
2285 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2289 /* We don't need lock here, nobody else
2290 * touches the iova range
2292 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2294 static int dumps = 5;
2295 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2296 iov_pfn, tmp, (unsigned long long)pteval);
2299 debug_dma_dump_mappings(NULL);
2304 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2306 BUG_ON(nr_pages < lvl_pages);
2307 BUG_ON(sg_res < lvl_pages);
2309 nr_pages -= lvl_pages;
2310 iov_pfn += lvl_pages;
2311 phys_pfn += lvl_pages;
2312 pteval += lvl_pages * VTD_PAGE_SIZE;
2313 sg_res -= lvl_pages;
2315 /* If the next PTE would be the first in a new page, then we
2316 need to flush the cache on the entries we've just written.
2317 And then we'll need to recalculate 'pte', so clear it and
2318 let it get set again in the if (!pte) block above.
2320 If we're done (!nr_pages) we need to flush the cache too.
2322 Also if we've been setting superpages, we may need to
2323 recalculate 'pte' and switch back to smaller pages for the
2324 end of the mapping, if the trailing size is not enough to
2325 use another superpage (i.e. sg_res < lvl_pages). */
2327 if (!nr_pages || first_pte_in_page(pte) ||
2328 (largepage_lvl > 1 && sg_res < lvl_pages)) {
2329 domain_flush_cache(domain, first_pte,
2330 (void *)pte - (void *)first_pte);
2334 if (!sg_res && nr_pages)
2340 static int domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2341 struct scatterlist *sg, unsigned long phys_pfn,
2342 unsigned long nr_pages, int prot)
2345 struct intel_iommu *iommu;
2347 /* Do the real mapping first */
2348 ret = __domain_mapping(domain, iov_pfn, sg, phys_pfn, nr_pages, prot);
2352 /* Notify about the new mapping */
2353 if (domain_type_is_vm(domain)) {
2354 /* VM typed domains can have more than one IOMMUs */
2356 for_each_domain_iommu(iommu_id, domain) {
2357 iommu = g_iommus[iommu_id];
2358 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2361 /* General domains only have one IOMMU */
2362 iommu = domain_get_iommu(domain);
2363 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2369 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2370 struct scatterlist *sg, unsigned long nr_pages,
2373 return domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2376 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2377 unsigned long phys_pfn, unsigned long nr_pages,
2380 return domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2383 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2385 unsigned long flags;
2386 struct context_entry *context;
2392 spin_lock_irqsave(&iommu->lock, flags);
2393 context = iommu_context_addr(iommu, bus, devfn, 0);
2395 spin_unlock_irqrestore(&iommu->lock, flags);
2398 did_old = context_domain_id(context);
2399 context_clear_entry(context);
2400 __iommu_flush_cache(iommu, context, sizeof(*context));
2401 spin_unlock_irqrestore(&iommu->lock, flags);
2402 iommu->flush.flush_context(iommu,
2404 (((u16)bus) << 8) | devfn,
2405 DMA_CCMD_MASK_NOBIT,
2406 DMA_CCMD_DEVICE_INVL);
2407 iommu->flush.flush_iotlb(iommu,
2414 static inline void unlink_domain_info(struct device_domain_info *info)
2416 assert_spin_locked(&device_domain_lock);
2417 list_del(&info->link);
2418 list_del(&info->global);
2420 info->dev->archdata.iommu = NULL;
2423 static void domain_remove_dev_info(struct dmar_domain *domain)
2425 struct device_domain_info *info, *tmp;
2426 unsigned long flags;
2428 spin_lock_irqsave(&device_domain_lock, flags);
2429 list_for_each_entry_safe(info, tmp, &domain->devices, link)
2430 __dmar_remove_one_dev_info(info);
2431 spin_unlock_irqrestore(&device_domain_lock, flags);
2436 * Note: we use struct device->archdata.iommu stores the info
2438 static struct dmar_domain *find_domain(struct device *dev)
2440 struct device_domain_info *info;
2442 /* No lock here, assumes no domain exit in normal case */
2443 info = dev->archdata.iommu;
2445 return info->domain;
2449 static inline struct device_domain_info *
2450 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2452 struct device_domain_info *info;
2454 list_for_each_entry(info, &device_domain_list, global)
2455 if (info->iommu->segment == segment && info->bus == bus &&
2456 info->devfn == devfn)
2462 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2465 struct dmar_domain *domain)
2467 struct dmar_domain *found = NULL;
2468 struct device_domain_info *info;
2469 unsigned long flags;
2472 info = alloc_devinfo_mem();
2477 info->devfn = devfn;
2478 info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2479 info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2482 info->domain = domain;
2483 info->iommu = iommu;
2484 info->pasid_table = NULL;
2486 if (dev && dev_is_pci(dev)) {
2487 struct pci_dev *pdev = to_pci_dev(info->dev);
2489 if (!pci_ats_disabled() &&
2490 ecap_dev_iotlb_support(iommu->ecap) &&
2491 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2492 dmar_find_matched_atsr_unit(pdev))
2493 info->ats_supported = 1;
2495 if (ecs_enabled(iommu)) {
2496 if (pasid_enabled(iommu)) {
2497 int features = pci_pasid_features(pdev);
2499 info->pasid_supported = features | 1;
2502 if (info->ats_supported && ecap_prs(iommu->ecap) &&
2503 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2504 info->pri_supported = 1;
2508 spin_lock_irqsave(&device_domain_lock, flags);
2510 found = find_domain(dev);
2513 struct device_domain_info *info2;
2514 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2516 found = info2->domain;
2522 spin_unlock_irqrestore(&device_domain_lock, flags);
2523 free_devinfo_mem(info);
2524 /* Caller must free the original domain */
2528 spin_lock(&iommu->lock);
2529 ret = domain_attach_iommu(domain, iommu);
2530 spin_unlock(&iommu->lock);
2533 spin_unlock_irqrestore(&device_domain_lock, flags);
2534 free_devinfo_mem(info);
2538 list_add(&info->link, &domain->devices);
2539 list_add(&info->global, &device_domain_list);
2541 dev->archdata.iommu = info;
2543 if (dev && dev_is_pci(dev) && info->pasid_supported) {
2544 ret = intel_pasid_alloc_table(dev);
2546 pr_warn("No pasid table for %s, pasid disabled\n",
2548 info->pasid_supported = 0;
2551 spin_unlock_irqrestore(&device_domain_lock, flags);
2553 if (dev && domain_context_mapping(domain, dev)) {
2554 pr_err("Domain context map for %s failed\n", dev_name(dev));
2555 dmar_remove_one_dev_info(domain, dev);
2562 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2564 *(u16 *)opaque = alias;
2568 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2570 struct device_domain_info *info = NULL;
2571 struct dmar_domain *domain = NULL;
2572 struct intel_iommu *iommu;
2574 unsigned long flags;
2577 iommu = device_to_iommu(dev, &bus, &devfn);
2581 if (dev_is_pci(dev)) {
2582 struct pci_dev *pdev = to_pci_dev(dev);
2584 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2586 spin_lock_irqsave(&device_domain_lock, flags);
2587 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2588 PCI_BUS_NUM(dma_alias),
2591 iommu = info->iommu;
2592 domain = info->domain;
2594 spin_unlock_irqrestore(&device_domain_lock, flags);
2596 /* DMA alias already has a domain, use it */
2601 /* Allocate and initialize new domain for the device */
2602 domain = alloc_domain(0);
2605 if (domain_init(domain, iommu, gaw)) {
2606 domain_exit(domain);
2615 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2616 struct dmar_domain *domain)
2618 struct intel_iommu *iommu;
2619 struct dmar_domain *tmp;
2620 u16 req_id, dma_alias;
2623 iommu = device_to_iommu(dev, &bus, &devfn);
2627 req_id = ((u16)bus << 8) | devfn;
2629 if (dev_is_pci(dev)) {
2630 struct pci_dev *pdev = to_pci_dev(dev);
2632 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2634 /* register PCI DMA alias device */
2635 if (req_id != dma_alias) {
2636 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2637 dma_alias & 0xff, NULL, domain);
2639 if (!tmp || tmp != domain)
2644 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2645 if (!tmp || tmp != domain)
2651 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2653 struct dmar_domain *domain, *tmp;
2655 domain = find_domain(dev);
2659 domain = find_or_alloc_domain(dev, gaw);
2663 tmp = set_domain_for_dev(dev, domain);
2664 if (!tmp || domain != tmp) {
2665 domain_exit(domain);
2674 static int iommu_domain_identity_map(struct dmar_domain *domain,
2675 unsigned long long start,
2676 unsigned long long end)
2678 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2679 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2681 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2682 dma_to_mm_pfn(last_vpfn))) {
2683 pr_err("Reserving iova failed\n");
2687 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2689 * RMRR range might have overlap with physical memory range,
2692 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2694 return __domain_mapping(domain, first_vpfn, NULL,
2695 first_vpfn, last_vpfn - first_vpfn + 1,
2696 DMA_PTE_READ|DMA_PTE_WRITE);
2699 static int domain_prepare_identity_map(struct device *dev,
2700 struct dmar_domain *domain,
2701 unsigned long long start,
2702 unsigned long long end)
2704 /* For _hardware_ passthrough, don't bother. But for software
2705 passthrough, we do it anyway -- it may indicate a memory
2706 range which is reserved in E820, so which didn't get set
2707 up to start with in si_domain */
2708 if (domain == si_domain && hw_pass_through) {
2709 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2710 dev_name(dev), start, end);
2714 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2715 dev_name(dev), start, end);
2718 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2719 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2720 dmi_get_system_info(DMI_BIOS_VENDOR),
2721 dmi_get_system_info(DMI_BIOS_VERSION),
2722 dmi_get_system_info(DMI_PRODUCT_VERSION));
2726 if (end >> agaw_to_width(domain->agaw)) {
2727 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2728 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2729 agaw_to_width(domain->agaw),
2730 dmi_get_system_info(DMI_BIOS_VENDOR),
2731 dmi_get_system_info(DMI_BIOS_VERSION),
2732 dmi_get_system_info(DMI_PRODUCT_VERSION));
2736 return iommu_domain_identity_map(domain, start, end);
2739 static int iommu_prepare_identity_map(struct device *dev,
2740 unsigned long long start,
2741 unsigned long long end)
2743 struct dmar_domain *domain;
2746 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2750 ret = domain_prepare_identity_map(dev, domain, start, end);
2752 domain_exit(domain);
2757 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2760 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2762 return iommu_prepare_identity_map(dev, rmrr->base_address,
2766 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2767 static inline void iommu_prepare_isa(void)
2769 struct pci_dev *pdev;
2772 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2776 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2777 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2780 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2785 static inline void iommu_prepare_isa(void)
2789 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2791 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2793 static int __init si_domain_init(int hw)
2797 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2801 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2802 domain_exit(si_domain);
2807 pr_debug("Identity mapping domain allocated\n");
2812 for_each_online_node(nid) {
2813 unsigned long start_pfn, end_pfn;
2816 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2817 ret = iommu_domain_identity_map(si_domain,
2818 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2827 static int identity_mapping(struct device *dev)
2829 struct device_domain_info *info;
2831 if (likely(!iommu_identity_mapping))
2834 info = dev->archdata.iommu;
2835 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2836 return (info->domain == si_domain);
2841 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2843 struct dmar_domain *ndomain;
2844 struct intel_iommu *iommu;
2847 iommu = device_to_iommu(dev, &bus, &devfn);
2851 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2852 if (ndomain != domain)
2858 static bool device_has_rmrr(struct device *dev)
2860 struct dmar_rmrr_unit *rmrr;
2865 for_each_rmrr_units(rmrr) {
2867 * Return TRUE if this RMRR contains the device that
2870 for_each_active_dev_scope(rmrr->devices,
2871 rmrr->devices_cnt, i, tmp)
2882 * There are a couple cases where we need to restrict the functionality of
2883 * devices associated with RMRRs. The first is when evaluating a device for
2884 * identity mapping because problems exist when devices are moved in and out
2885 * of domains and their respective RMRR information is lost. This means that
2886 * a device with associated RMRRs will never be in a "passthrough" domain.
2887 * The second is use of the device through the IOMMU API. This interface
2888 * expects to have full control of the IOVA space for the device. We cannot
2889 * satisfy both the requirement that RMRR access is maintained and have an
2890 * unencumbered IOVA space. We also have no ability to quiesce the device's
2891 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2892 * We therefore prevent devices associated with an RMRR from participating in
2893 * the IOMMU API, which eliminates them from device assignment.
2895 * In both cases we assume that PCI USB devices with RMRRs have them largely
2896 * for historical reasons and that the RMRR space is not actively used post
2897 * boot. This exclusion may change if vendors begin to abuse it.
2899 * The same exception is made for graphics devices, with the requirement that
2900 * any use of the RMRR regions will be torn down before assigning the device
2903 static bool device_is_rmrr_locked(struct device *dev)
2905 if (!device_has_rmrr(dev))
2908 if (dev_is_pci(dev)) {
2909 struct pci_dev *pdev = to_pci_dev(dev);
2911 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2918 static int iommu_should_identity_map(struct device *dev, int startup)
2921 if (dev_is_pci(dev)) {
2922 struct pci_dev *pdev = to_pci_dev(dev);
2924 if (device_is_rmrr_locked(dev))
2927 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2930 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2933 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2937 * We want to start off with all devices in the 1:1 domain, and
2938 * take them out later if we find they can't access all of memory.
2940 * However, we can't do this for PCI devices behind bridges,
2941 * because all PCI devices behind the same bridge will end up
2942 * with the same source-id on their transactions.
2944 * Practically speaking, we can't change things around for these
2945 * devices at run-time, because we can't be sure there'll be no
2946 * DMA transactions in flight for any of their siblings.
2948 * So PCI devices (unless they're on the root bus) as well as
2949 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2950 * the 1:1 domain, just in _case_ one of their siblings turns out
2951 * not to be able to map all of memory.
2953 if (!pci_is_pcie(pdev)) {
2954 if (!pci_is_root_bus(pdev->bus))
2956 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2958 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2961 if (device_has_rmrr(dev))
2966 * At boot time, we don't yet know if devices will be 64-bit capable.
2967 * Assume that they will — if they turn out not to be, then we can
2968 * take them out of the 1:1 domain later.
2972 * If the device's dma_mask is less than the system's memory
2973 * size then this is not a candidate for identity mapping.
2975 u64 dma_mask = *dev->dma_mask;
2977 if (dev->coherent_dma_mask &&
2978 dev->coherent_dma_mask < dma_mask)
2979 dma_mask = dev->coherent_dma_mask;
2981 return dma_mask >= dma_get_required_mask(dev);
2987 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2991 if (!iommu_should_identity_map(dev, 1))
2994 ret = domain_add_dev_info(si_domain, dev);
2996 pr_info("%s identity mapping for device %s\n",
2997 hw ? "Hardware" : "Software", dev_name(dev));
2998 else if (ret == -ENODEV)
2999 /* device not associated with an iommu */
3006 static int __init iommu_prepare_static_identity_mapping(int hw)
3008 struct pci_dev *pdev = NULL;
3009 struct dmar_drhd_unit *drhd;
3010 struct intel_iommu *iommu;
3015 for_each_pci_dev(pdev) {
3016 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
3021 for_each_active_iommu(iommu, drhd)
3022 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
3023 struct acpi_device_physical_node *pn;
3024 struct acpi_device *adev;
3026 if (dev->bus != &acpi_bus_type)
3029 adev= to_acpi_device(dev);
3030 mutex_lock(&adev->physical_node_lock);
3031 list_for_each_entry(pn, &adev->physical_node_list, node) {
3032 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
3036 mutex_unlock(&adev->physical_node_lock);
3044 static void intel_iommu_init_qi(struct intel_iommu *iommu)
3047 * Start from the sane iommu hardware state.
3048 * If the queued invalidation is already initialized by us
3049 * (for example, while enabling interrupt-remapping) then
3050 * we got the things already rolling from a sane state.
3054 * Clear any previous faults.
3056 dmar_fault(-1, iommu);
3058 * Disable queued invalidation if supported and already enabled
3059 * before OS handover.
3061 dmar_disable_qi(iommu);
3064 if (dmar_enable_qi(iommu)) {
3066 * Queued Invalidate not enabled, use Register Based Invalidate
3068 iommu->flush.flush_context = __iommu_flush_context;
3069 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
3070 pr_info("%s: Using Register based invalidation\n",
3073 iommu->flush.flush_context = qi_flush_context;
3074 iommu->flush.flush_iotlb = qi_flush_iotlb;
3075 pr_info("%s: Using Queued invalidation\n", iommu->name);
3079 static int copy_context_table(struct intel_iommu *iommu,
3080 struct root_entry *old_re,
3081 struct context_entry **tbl,
3084 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
3085 struct context_entry *new_ce = NULL, ce;
3086 struct context_entry *old_ce = NULL;
3087 struct root_entry re;
3088 phys_addr_t old_ce_phys;
3090 tbl_idx = ext ? bus * 2 : bus;
3091 memcpy(&re, old_re, sizeof(re));
3093 for (devfn = 0; devfn < 256; devfn++) {
3094 /* First calculate the correct index */
3095 idx = (ext ? devfn * 2 : devfn) % 256;
3098 /* First save what we may have and clean up */
3100 tbl[tbl_idx] = new_ce;
3101 __iommu_flush_cache(iommu, new_ce,
3111 old_ce_phys = root_entry_lctp(&re);
3113 old_ce_phys = root_entry_uctp(&re);
3116 if (ext && devfn == 0) {
3117 /* No LCTP, try UCTP */
3126 old_ce = memremap(old_ce_phys, PAGE_SIZE,
3131 new_ce = alloc_pgtable_page(iommu->node);
3138 /* Now copy the context entry */
3139 memcpy(&ce, old_ce + idx, sizeof(ce));
3141 if (!__context_present(&ce))
3144 did = context_domain_id(&ce);
3145 if (did >= 0 && did < cap_ndoms(iommu->cap))
3146 set_bit(did, iommu->domain_ids);
3149 * We need a marker for copied context entries. This
3150 * marker needs to work for the old format as well as
3151 * for extended context entries.
3153 * Bit 67 of the context entry is used. In the old
3154 * format this bit is available to software, in the
3155 * extended format it is the PGE bit, but PGE is ignored
3156 * by HW if PASIDs are disabled (and thus still
3159 * So disable PASIDs first and then mark the entry
3160 * copied. This means that we don't copy PASID
3161 * translations from the old kernel, but this is fine as
3162 * faults there are not fatal.
3164 context_clear_pasid_enable(&ce);
3165 context_set_copied(&ce);
3170 tbl[tbl_idx + pos] = new_ce;
3172 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3181 static int copy_translation_tables(struct intel_iommu *iommu)
3183 struct context_entry **ctxt_tbls;
3184 struct root_entry *old_rt;
3185 phys_addr_t old_rt_phys;
3186 int ctxt_table_entries;
3187 unsigned long flags;
3192 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3193 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
3194 new_ext = !!ecap_ecs(iommu->ecap);
3197 * The RTT bit can only be changed when translation is disabled,
3198 * but disabling translation means to open a window for data
3199 * corruption. So bail out and don't copy anything if we would
3200 * have to change the bit.
3205 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3209 old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3213 /* This is too big for the stack - allocate it from slab */
3214 ctxt_table_entries = ext ? 512 : 256;
3216 ctxt_tbls = kcalloc(ctxt_table_entries, sizeof(void *), GFP_KERNEL);
3220 for (bus = 0; bus < 256; bus++) {
3221 ret = copy_context_table(iommu, &old_rt[bus],
3222 ctxt_tbls, bus, ext);
3224 pr_err("%s: Failed to copy context table for bus %d\n",
3230 spin_lock_irqsave(&iommu->lock, flags);
3232 /* Context tables are copied, now write them to the root_entry table */
3233 for (bus = 0; bus < 256; bus++) {
3234 int idx = ext ? bus * 2 : bus;
3237 if (ctxt_tbls[idx]) {
3238 val = virt_to_phys(ctxt_tbls[idx]) | 1;
3239 iommu->root_entry[bus].lo = val;
3242 if (!ext || !ctxt_tbls[idx + 1])
3245 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3246 iommu->root_entry[bus].hi = val;
3249 spin_unlock_irqrestore(&iommu->lock, flags);
3253 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3263 static int __init init_dmars(void)
3265 struct dmar_drhd_unit *drhd;
3266 struct dmar_rmrr_unit *rmrr;
3267 bool copied_tables = false;
3269 struct intel_iommu *iommu;
3275 * initialize and program root entry to not present
3278 for_each_drhd_unit(drhd) {
3280 * lock not needed as this is only incremented in the single
3281 * threaded kernel __init code path all other access are read
3284 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3288 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3291 /* Preallocate enough resources for IOMMU hot-addition */
3292 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3293 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3295 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3298 pr_err("Allocating global iommu array failed\n");
3303 for_each_active_iommu(iommu, drhd) {
3305 * Find the max pasid size of all IOMMU's in the system.
3306 * We need to ensure the system pasid table is no bigger
3307 * than the smallest supported.
3309 if (pasid_enabled(iommu)) {
3310 u32 temp = 2 << ecap_pss(iommu->ecap);
3312 intel_pasid_max_id = min_t(u32, temp,
3313 intel_pasid_max_id);
3316 g_iommus[iommu->seq_id] = iommu;
3318 intel_iommu_init_qi(iommu);
3320 ret = iommu_init_domains(iommu);
3324 init_translation_status(iommu);
3326 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3327 iommu_disable_translation(iommu);
3328 clear_translation_pre_enabled(iommu);
3329 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3335 * we could share the same root & context tables
3336 * among all IOMMU's. Need to Split it later.
3338 ret = iommu_alloc_root_entry(iommu);
3342 if (translation_pre_enabled(iommu)) {
3343 pr_info("Translation already enabled - trying to copy translation structures\n");
3345 ret = copy_translation_tables(iommu);
3348 * We found the IOMMU with translation
3349 * enabled - but failed to copy over the
3350 * old root-entry table. Try to proceed
3351 * by disabling translation now and
3352 * allocating a clean root-entry table.
3353 * This might cause DMAR faults, but
3354 * probably the dump will still succeed.
3356 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3358 iommu_disable_translation(iommu);
3359 clear_translation_pre_enabled(iommu);
3361 pr_info("Copied translation tables from previous kernel for %s\n",
3363 copied_tables = true;
3367 if (!ecap_pass_through(iommu->ecap))
3368 hw_pass_through = 0;
3370 if (!intel_iommu_strict && cap_caching_mode(iommu->cap)) {
3371 pr_info("Disable batched IOTLB flush due to virtualization");
3372 intel_iommu_strict = 1;
3375 #ifdef CONFIG_INTEL_IOMMU_SVM
3376 if (pasid_enabled(iommu))
3377 intel_svm_init(iommu);
3382 * Now that qi is enabled on all iommus, set the root entry and flush
3383 * caches. This is required on some Intel X58 chipsets, otherwise the
3384 * flush_context function will loop forever and the boot hangs.
3386 for_each_active_iommu(iommu, drhd) {
3387 iommu_flush_write_buffer(iommu);
3388 iommu_set_root_entry(iommu);
3389 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3390 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3393 if (iommu_pass_through)
3394 iommu_identity_mapping |= IDENTMAP_ALL;
3396 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3401 iommu_identity_mapping |= IDENTMAP_GFX;
3403 check_tylersburg_isoch();
3405 if (iommu_identity_mapping) {
3406 ret = si_domain_init(hw_pass_through);
3413 * If we copied translations from a previous kernel in the kdump
3414 * case, we can not assign the devices to domains now, as that
3415 * would eliminate the old mappings. So skip this part and defer
3416 * the assignment to device driver initialization time.
3422 * If pass through is not set or not enabled, setup context entries for
3423 * identity mappings for rmrr, gfx, and isa and may fall back to static
3424 * identity mapping if iommu_identity_mapping is set.
3426 if (iommu_identity_mapping) {
3427 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3429 pr_crit("Failed to setup IOMMU pass-through\n");
3435 * for each dev attached to rmrr
3437 * locate drhd for dev, alloc domain for dev
3438 * allocate free domain
3439 * allocate page table entries for rmrr
3440 * if context not allocated for bus
3441 * allocate and init context
3442 * set present in root table for this bus
3443 * init context with domain, translation etc
3447 pr_info("Setting RMRR:\n");
3448 for_each_rmrr_units(rmrr) {
3449 /* some BIOS lists non-exist devices in DMAR table. */
3450 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3452 ret = iommu_prepare_rmrr_dev(rmrr, dev);
3454 pr_err("Mapping reserved region failed\n");
3458 iommu_prepare_isa();
3465 * global invalidate context cache
3466 * global invalidate iotlb
3467 * enable translation
3469 for_each_iommu(iommu, drhd) {
3470 if (drhd->ignored) {
3472 * we always have to disable PMRs or DMA may fail on
3476 iommu_disable_protect_mem_regions(iommu);
3480 iommu_flush_write_buffer(iommu);
3482 #ifdef CONFIG_INTEL_IOMMU_SVM
3483 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3484 ret = intel_svm_enable_prq(iommu);
3489 ret = dmar_set_interrupt(iommu);
3493 if (!translation_pre_enabled(iommu))
3494 iommu_enable_translation(iommu);
3496 iommu_disable_protect_mem_regions(iommu);
3502 for_each_active_iommu(iommu, drhd) {
3503 disable_dmar_iommu(iommu);
3504 free_dmar_iommu(iommu);
3507 domain_exit(si_domain);
3517 /* This takes a number of _MM_ pages, not VTD pages */
3518 static unsigned long intel_alloc_iova(struct device *dev,
3519 struct dmar_domain *domain,
3520 unsigned long nrpages, uint64_t dma_mask)
3522 unsigned long iova_pfn = 0;
3524 /* Restrict dma_mask to the width that the iommu can handle */
3525 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3526 /* Ensure we reserve the whole size-aligned region */
3527 nrpages = __roundup_pow_of_two(nrpages);
3529 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3531 * First try to allocate an io virtual address in
3532 * DMA_BIT_MASK(32) and if that fails then try allocating
3535 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3536 IOVA_PFN(DMA_BIT_MASK(32)), false);
3540 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3541 IOVA_PFN(dma_mask), true);
3542 if (unlikely(!iova_pfn)) {
3543 pr_err("Allocating %ld-page iova for %s failed",
3544 nrpages, dev_name(dev));
3551 struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3553 struct dmar_domain *domain, *tmp;
3554 struct dmar_rmrr_unit *rmrr;
3555 struct device *i_dev;
3558 domain = find_domain(dev);
3562 domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3566 /* We have a new domain - setup possible RMRRs for the device */
3568 for_each_rmrr_units(rmrr) {
3569 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3574 ret = domain_prepare_identity_map(dev, domain,
3578 dev_err(dev, "Mapping reserved region failed\n");
3583 tmp = set_domain_for_dev(dev, domain);
3584 if (!tmp || domain != tmp) {
3585 domain_exit(domain);
3592 pr_err("Allocating domain for %s failed\n", dev_name(dev));
3598 /* Check if the dev needs to go through non-identity map and unmap process.*/
3599 static int iommu_no_mapping(struct device *dev)
3603 if (iommu_dummy(dev))
3606 if (!iommu_identity_mapping)
3609 found = identity_mapping(dev);
3611 if (iommu_should_identity_map(dev, 0))
3615 * 32 bit DMA is removed from si_domain and fall back
3616 * to non-identity mapping.
3618 dmar_remove_one_dev_info(si_domain, dev);
3619 pr_info("32bit %s uses non-identity mapping\n",
3625 * In case of a detached 64 bit DMA device from vm, the device
3626 * is put into si_domain for identity mapping.
3628 if (iommu_should_identity_map(dev, 0)) {
3630 ret = domain_add_dev_info(si_domain, dev);
3632 pr_info("64bit %s uses identity mapping\n",
3642 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3643 size_t size, int dir, u64 dma_mask)
3645 struct dmar_domain *domain;
3646 phys_addr_t start_paddr;
3647 unsigned long iova_pfn;
3650 struct intel_iommu *iommu;
3651 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3653 BUG_ON(dir == DMA_NONE);
3655 if (iommu_no_mapping(dev))
3658 domain = get_valid_domain_for_dev(dev);
3662 iommu = domain_get_iommu(domain);
3663 size = aligned_nrpages(paddr, size);
3665 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3670 * Check if DMAR supports zero-length reads on write only
3673 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3674 !cap_zlr(iommu->cap))
3675 prot |= DMA_PTE_READ;
3676 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3677 prot |= DMA_PTE_WRITE;
3679 * paddr - (paddr + size) might be partial page, we should map the whole
3680 * page. Note: if two part of one page are separately mapped, we
3681 * might have two guest_addr mapping to the same host paddr, but this
3682 * is not a big problem
3684 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3685 mm_to_dma_pfn(paddr_pfn), size, prot);
3689 start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3690 start_paddr += paddr & ~PAGE_MASK;
3695 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3696 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3697 dev_name(dev), size, (unsigned long long)paddr, dir);
3701 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3702 unsigned long offset, size_t size,
3703 enum dma_data_direction dir,
3704 unsigned long attrs)
3706 return __intel_map_single(dev, page_to_phys(page) + offset, size,
3707 dir, *dev->dma_mask);
3710 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3712 struct dmar_domain *domain;
3713 unsigned long start_pfn, last_pfn;
3714 unsigned long nrpages;
3715 unsigned long iova_pfn;
3716 struct intel_iommu *iommu;
3717 struct page *freelist;
3719 if (iommu_no_mapping(dev))
3722 domain = find_domain(dev);
3725 iommu = domain_get_iommu(domain);
3727 iova_pfn = IOVA_PFN(dev_addr);
3729 nrpages = aligned_nrpages(dev_addr, size);
3730 start_pfn = mm_to_dma_pfn(iova_pfn);
3731 last_pfn = start_pfn + nrpages - 1;
3733 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3734 dev_name(dev), start_pfn, last_pfn);
3736 freelist = domain_unmap(domain, start_pfn, last_pfn);
3738 if (intel_iommu_strict || !has_iova_flush_queue(&domain->iovad)) {
3739 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3740 nrpages, !freelist, 0);
3742 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3743 dma_free_pagelist(freelist);
3745 queue_iova(&domain->iovad, iova_pfn, nrpages,
3746 (unsigned long)freelist);
3748 * queue up the release of the unmap to save the 1/6th of the
3749 * cpu used up by the iotlb flush operation...
3754 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3755 size_t size, enum dma_data_direction dir,
3756 unsigned long attrs)
3758 intel_unmap(dev, dev_addr, size);
3761 static void *intel_alloc_coherent(struct device *dev, size_t size,
3762 dma_addr_t *dma_handle, gfp_t flags,
3763 unsigned long attrs)
3765 struct page *page = NULL;
3768 size = PAGE_ALIGN(size);
3769 order = get_order(size);
3771 if (!iommu_no_mapping(dev))
3772 flags &= ~(GFP_DMA | GFP_DMA32);
3773 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3774 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3780 if (gfpflags_allow_blocking(flags)) {
3781 unsigned int count = size >> PAGE_SHIFT;
3783 page = dma_alloc_from_contiguous(dev, count, order,
3784 flags & __GFP_NOWARN);
3785 if (page && iommu_no_mapping(dev) &&
3786 page_to_phys(page) + size > dev->coherent_dma_mask) {
3787 dma_release_from_contiguous(dev, page, count);
3793 page = alloc_pages(flags, order);
3796 memset(page_address(page), 0, size);
3798 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3800 dev->coherent_dma_mask);
3802 return page_address(page);
3803 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3804 __free_pages(page, order);
3809 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3810 dma_addr_t dma_handle, unsigned long attrs)
3813 struct page *page = virt_to_page(vaddr);
3815 size = PAGE_ALIGN(size);
3816 order = get_order(size);
3818 intel_unmap(dev, dma_handle, size);
3819 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3820 __free_pages(page, order);
3823 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3824 int nelems, enum dma_data_direction dir,
3825 unsigned long attrs)
3827 dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3828 unsigned long nrpages = 0;
3829 struct scatterlist *sg;
3832 for_each_sg(sglist, sg, nelems, i) {
3833 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3836 intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3839 static int intel_nontranslate_map_sg(struct device *hddev,
3840 struct scatterlist *sglist, int nelems, int dir)
3843 struct scatterlist *sg;
3845 for_each_sg(sglist, sg, nelems, i) {
3846 BUG_ON(!sg_page(sg));
3847 sg->dma_address = sg_phys(sg);
3848 sg->dma_length = sg->length;
3853 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3854 enum dma_data_direction dir, unsigned long attrs)
3857 struct dmar_domain *domain;
3860 unsigned long iova_pfn;
3862 struct scatterlist *sg;
3863 unsigned long start_vpfn;
3864 struct intel_iommu *iommu;
3866 BUG_ON(dir == DMA_NONE);
3867 if (iommu_no_mapping(dev))
3868 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3870 domain = get_valid_domain_for_dev(dev);
3874 iommu = domain_get_iommu(domain);
3876 for_each_sg(sglist, sg, nelems, i)
3877 size += aligned_nrpages(sg->offset, sg->length);
3879 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3882 sglist->dma_length = 0;
3887 * Check if DMAR supports zero-length reads on write only
3890 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3891 !cap_zlr(iommu->cap))
3892 prot |= DMA_PTE_READ;
3893 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3894 prot |= DMA_PTE_WRITE;
3896 start_vpfn = mm_to_dma_pfn(iova_pfn);
3898 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3899 if (unlikely(ret)) {
3900 dma_pte_free_pagetable(domain, start_vpfn,
3901 start_vpfn + size - 1,
3902 agaw_to_level(domain->agaw) + 1);
3903 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3910 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3915 const struct dma_map_ops intel_dma_ops = {
3916 .alloc = intel_alloc_coherent,
3917 .free = intel_free_coherent,
3918 .map_sg = intel_map_sg,
3919 .unmap_sg = intel_unmap_sg,
3920 .map_page = intel_map_page,
3921 .unmap_page = intel_unmap_page,
3922 .mapping_error = intel_mapping_error,
3924 .dma_supported = dma_direct_supported,
3928 static inline int iommu_domain_cache_init(void)
3932 iommu_domain_cache = kmem_cache_create("iommu_domain",
3933 sizeof(struct dmar_domain),
3938 if (!iommu_domain_cache) {
3939 pr_err("Couldn't create iommu_domain cache\n");
3946 static inline int iommu_devinfo_cache_init(void)
3950 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3951 sizeof(struct device_domain_info),
3955 if (!iommu_devinfo_cache) {
3956 pr_err("Couldn't create devinfo cache\n");
3963 static int __init iommu_init_mempool(void)
3966 ret = iova_cache_get();
3970 ret = iommu_domain_cache_init();
3974 ret = iommu_devinfo_cache_init();
3978 kmem_cache_destroy(iommu_domain_cache);
3985 static void __init iommu_exit_mempool(void)
3987 kmem_cache_destroy(iommu_devinfo_cache);
3988 kmem_cache_destroy(iommu_domain_cache);
3992 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3994 struct dmar_drhd_unit *drhd;
3998 /* We know that this device on this chipset has its own IOMMU.
3999 * If we find it under a different IOMMU, then the BIOS is lying
4000 * to us. Hope that the IOMMU for this device is actually
4001 * disabled, and it needs no translation...
4003 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
4005 /* "can't" happen */
4006 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
4009 vtbar &= 0xffff0000;
4011 /* we know that the this iommu should be at offset 0xa000 from vtbar */
4012 drhd = dmar_find_matched_drhd_unit(pdev);
4013 if (!drhd || drhd->reg_base_addr - vtbar != 0xa000) {
4014 pr_warn_once(FW_BUG "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n");
4015 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
4016 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4019 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
4021 static void __init init_no_remapping_devices(void)
4023 struct dmar_drhd_unit *drhd;
4027 for_each_drhd_unit(drhd) {
4028 if (!drhd->include_all) {
4029 for_each_active_dev_scope(drhd->devices,
4030 drhd->devices_cnt, i, dev)
4032 /* ignore DMAR unit if no devices exist */
4033 if (i == drhd->devices_cnt)
4038 for_each_active_drhd_unit(drhd) {
4039 if (drhd->include_all)
4042 for_each_active_dev_scope(drhd->devices,
4043 drhd->devices_cnt, i, dev)
4044 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
4046 if (i < drhd->devices_cnt)
4049 /* This IOMMU has *only* gfx devices. Either bypass it or
4050 set the gfx_mapped flag, as appropriate */
4051 if (!dmar_map_gfx) {
4053 for_each_active_dev_scope(drhd->devices,
4054 drhd->devices_cnt, i, dev)
4055 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4060 #ifdef CONFIG_SUSPEND
4061 static int init_iommu_hw(void)
4063 struct dmar_drhd_unit *drhd;
4064 struct intel_iommu *iommu = NULL;
4066 for_each_active_iommu(iommu, drhd)
4068 dmar_reenable_qi(iommu);
4070 for_each_iommu(iommu, drhd) {
4071 if (drhd->ignored) {
4073 * we always have to disable PMRs or DMA may fail on
4077 iommu_disable_protect_mem_regions(iommu);
4081 iommu_flush_write_buffer(iommu);
4083 iommu_set_root_entry(iommu);
4085 iommu->flush.flush_context(iommu, 0, 0, 0,
4086 DMA_CCMD_GLOBAL_INVL);
4087 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4088 iommu_enable_translation(iommu);
4089 iommu_disable_protect_mem_regions(iommu);
4095 static void iommu_flush_all(void)
4097 struct dmar_drhd_unit *drhd;
4098 struct intel_iommu *iommu;
4100 for_each_active_iommu(iommu, drhd) {
4101 iommu->flush.flush_context(iommu, 0, 0, 0,
4102 DMA_CCMD_GLOBAL_INVL);
4103 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4104 DMA_TLB_GLOBAL_FLUSH);
4108 static int iommu_suspend(void)
4110 struct dmar_drhd_unit *drhd;
4111 struct intel_iommu *iommu = NULL;
4114 for_each_active_iommu(iommu, drhd) {
4115 iommu->iommu_state = kcalloc(MAX_SR_DMAR_REGS, sizeof(u32),
4117 if (!iommu->iommu_state)
4123 for_each_active_iommu(iommu, drhd) {
4124 iommu_disable_translation(iommu);
4126 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4128 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4129 readl(iommu->reg + DMAR_FECTL_REG);
4130 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4131 readl(iommu->reg + DMAR_FEDATA_REG);
4132 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4133 readl(iommu->reg + DMAR_FEADDR_REG);
4134 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4135 readl(iommu->reg + DMAR_FEUADDR_REG);
4137 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4142 for_each_active_iommu(iommu, drhd)
4143 kfree(iommu->iommu_state);
4148 static void iommu_resume(void)
4150 struct dmar_drhd_unit *drhd;
4151 struct intel_iommu *iommu = NULL;
4154 if (init_iommu_hw()) {
4156 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4158 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4162 for_each_active_iommu(iommu, drhd) {
4164 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4166 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4167 iommu->reg + DMAR_FECTL_REG);
4168 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4169 iommu->reg + DMAR_FEDATA_REG);
4170 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4171 iommu->reg + DMAR_FEADDR_REG);
4172 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4173 iommu->reg + DMAR_FEUADDR_REG);
4175 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4178 for_each_active_iommu(iommu, drhd)
4179 kfree(iommu->iommu_state);
4182 static struct syscore_ops iommu_syscore_ops = {
4183 .resume = iommu_resume,
4184 .suspend = iommu_suspend,
4187 static void __init init_iommu_pm_ops(void)
4189 register_syscore_ops(&iommu_syscore_ops);
4193 static inline void init_iommu_pm_ops(void) {}
4194 #endif /* CONFIG_PM */
4197 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4199 struct acpi_dmar_reserved_memory *rmrr;
4200 struct dmar_rmrr_unit *rmrru;
4203 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4207 rmrru->hdr = header;
4208 rmrr = (struct acpi_dmar_reserved_memory *)header;
4209 rmrru->base_address = rmrr->base_address;
4210 rmrru->end_address = rmrr->end_address;
4212 length = rmrr->end_address - rmrr->base_address + 1;
4214 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4215 ((void *)rmrr) + rmrr->header.length,
4216 &rmrru->devices_cnt);
4217 if (rmrru->devices_cnt && rmrru->devices == NULL)
4220 list_add(&rmrru->list, &dmar_rmrr_units);
4229 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4231 struct dmar_atsr_unit *atsru;
4232 struct acpi_dmar_atsr *tmp;
4234 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4235 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4236 if (atsr->segment != tmp->segment)
4238 if (atsr->header.length != tmp->header.length)
4240 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4247 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4249 struct acpi_dmar_atsr *atsr;
4250 struct dmar_atsr_unit *atsru;
4252 if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
4255 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4256 atsru = dmar_find_atsr(atsr);
4260 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4265 * If memory is allocated from slab by ACPI _DSM method, we need to
4266 * copy the memory content because the memory buffer will be freed
4269 atsru->hdr = (void *)(atsru + 1);
4270 memcpy(atsru->hdr, hdr, hdr->length);
4271 atsru->include_all = atsr->flags & 0x1;
4272 if (!atsru->include_all) {
4273 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4274 (void *)atsr + atsr->header.length,
4275 &atsru->devices_cnt);
4276 if (atsru->devices_cnt && atsru->devices == NULL) {
4282 list_add_rcu(&atsru->list, &dmar_atsr_units);
4287 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4289 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4293 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4295 struct acpi_dmar_atsr *atsr;
4296 struct dmar_atsr_unit *atsru;
4298 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4299 atsru = dmar_find_atsr(atsr);
4301 list_del_rcu(&atsru->list);
4303 intel_iommu_free_atsr(atsru);
4309 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4313 struct acpi_dmar_atsr *atsr;
4314 struct dmar_atsr_unit *atsru;
4316 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4317 atsru = dmar_find_atsr(atsr);
4321 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4322 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4330 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4333 struct intel_iommu *iommu = dmaru->iommu;
4335 if (g_iommus[iommu->seq_id])
4338 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4339 pr_warn("%s: Doesn't support hardware pass through.\n",
4343 if (!ecap_sc_support(iommu->ecap) &&
4344 domain_update_iommu_snooping(iommu)) {
4345 pr_warn("%s: Doesn't support snooping.\n",
4349 sp = domain_update_iommu_superpage(iommu) - 1;
4350 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4351 pr_warn("%s: Doesn't support large page.\n",
4357 * Disable translation if already enabled prior to OS handover.
4359 if (iommu->gcmd & DMA_GCMD_TE)
4360 iommu_disable_translation(iommu);
4362 g_iommus[iommu->seq_id] = iommu;
4363 ret = iommu_init_domains(iommu);
4365 ret = iommu_alloc_root_entry(iommu);
4369 #ifdef CONFIG_INTEL_IOMMU_SVM
4370 if (pasid_enabled(iommu))
4371 intel_svm_init(iommu);
4374 if (dmaru->ignored) {
4376 * we always have to disable PMRs or DMA may fail on this device
4379 iommu_disable_protect_mem_regions(iommu);
4383 intel_iommu_init_qi(iommu);
4384 iommu_flush_write_buffer(iommu);
4386 #ifdef CONFIG_INTEL_IOMMU_SVM
4387 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4388 ret = intel_svm_enable_prq(iommu);
4393 ret = dmar_set_interrupt(iommu);
4397 iommu_set_root_entry(iommu);
4398 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4399 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4400 iommu_enable_translation(iommu);
4402 iommu_disable_protect_mem_regions(iommu);
4406 disable_dmar_iommu(iommu);
4408 free_dmar_iommu(iommu);
4412 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4415 struct intel_iommu *iommu = dmaru->iommu;
4417 if (!intel_iommu_enabled)
4423 ret = intel_iommu_add(dmaru);
4425 disable_dmar_iommu(iommu);
4426 free_dmar_iommu(iommu);
4432 static void intel_iommu_free_dmars(void)
4434 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4435 struct dmar_atsr_unit *atsru, *atsr_n;
4437 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4438 list_del(&rmrru->list);
4439 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4443 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4444 list_del(&atsru->list);
4445 intel_iommu_free_atsr(atsru);
4449 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4452 struct pci_bus *bus;
4453 struct pci_dev *bridge = NULL;
4455 struct acpi_dmar_atsr *atsr;
4456 struct dmar_atsr_unit *atsru;
4458 dev = pci_physfn(dev);
4459 for (bus = dev->bus; bus; bus = bus->parent) {
4461 /* If it's an integrated device, allow ATS */
4464 /* Connected via non-PCIe: no ATS */
4465 if (!pci_is_pcie(bridge) ||
4466 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4468 /* If we found the root port, look it up in the ATSR */
4469 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4474 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4475 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4476 if (atsr->segment != pci_domain_nr(dev->bus))
4479 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4480 if (tmp == &bridge->dev)
4483 if (atsru->include_all)
4493 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4496 struct dmar_rmrr_unit *rmrru;
4497 struct dmar_atsr_unit *atsru;
4498 struct acpi_dmar_atsr *atsr;
4499 struct acpi_dmar_reserved_memory *rmrr;
4501 if (!intel_iommu_enabled && system_state >= SYSTEM_RUNNING)
4504 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4505 rmrr = container_of(rmrru->hdr,
4506 struct acpi_dmar_reserved_memory, header);
4507 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4508 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4509 ((void *)rmrr) + rmrr->header.length,
4510 rmrr->segment, rmrru->devices,
4511 rmrru->devices_cnt);
4514 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4515 dmar_remove_dev_scope(info, rmrr->segment,
4516 rmrru->devices, rmrru->devices_cnt);
4520 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4521 if (atsru->include_all)
4524 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4525 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4526 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4527 (void *)atsr + atsr->header.length,
4528 atsr->segment, atsru->devices,
4529 atsru->devices_cnt);
4534 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4535 if (dmar_remove_dev_scope(info, atsr->segment,
4536 atsru->devices, atsru->devices_cnt))
4545 * Here we only respond to action of unbound device from driver.
4547 * Added device is not attached to its DMAR domain here yet. That will happen
4548 * when mapping the device to iova.
4550 static int device_notifier(struct notifier_block *nb,
4551 unsigned long action, void *data)
4553 struct device *dev = data;
4554 struct dmar_domain *domain;
4556 if (iommu_dummy(dev))
4559 if (action != BUS_NOTIFY_REMOVED_DEVICE)
4562 domain = find_domain(dev);
4566 dmar_remove_one_dev_info(domain, dev);
4567 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4568 domain_exit(domain);
4573 static struct notifier_block device_nb = {
4574 .notifier_call = device_notifier,
4577 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4578 unsigned long val, void *v)
4580 struct memory_notify *mhp = v;
4581 unsigned long long start, end;
4582 unsigned long start_vpfn, last_vpfn;
4585 case MEM_GOING_ONLINE:
4586 start = mhp->start_pfn << PAGE_SHIFT;
4587 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4588 if (iommu_domain_identity_map(si_domain, start, end)) {
4589 pr_warn("Failed to build identity map for [%llx-%llx]\n",
4596 case MEM_CANCEL_ONLINE:
4597 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4598 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4599 while (start_vpfn <= last_vpfn) {
4601 struct dmar_drhd_unit *drhd;
4602 struct intel_iommu *iommu;
4603 struct page *freelist;
4605 iova = find_iova(&si_domain->iovad, start_vpfn);
4607 pr_debug("Failed get IOVA for PFN %lx\n",
4612 iova = split_and_remove_iova(&si_domain->iovad, iova,
4613 start_vpfn, last_vpfn);
4615 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4616 start_vpfn, last_vpfn);
4620 freelist = domain_unmap(si_domain, iova->pfn_lo,
4624 for_each_active_iommu(iommu, drhd)
4625 iommu_flush_iotlb_psi(iommu, si_domain,
4626 iova->pfn_lo, iova_size(iova),
4629 dma_free_pagelist(freelist);
4631 start_vpfn = iova->pfn_hi + 1;
4632 free_iova_mem(iova);
4640 static struct notifier_block intel_iommu_memory_nb = {
4641 .notifier_call = intel_iommu_memory_notifier,
4645 static void free_all_cpu_cached_iovas(unsigned int cpu)
4649 for (i = 0; i < g_num_of_iommus; i++) {
4650 struct intel_iommu *iommu = g_iommus[i];
4651 struct dmar_domain *domain;
4657 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
4658 domain = get_iommu_domain(iommu, (u16)did);
4662 free_cpu_cached_iovas(cpu, &domain->iovad);
4667 static int intel_iommu_cpu_dead(unsigned int cpu)
4669 free_all_cpu_cached_iovas(cpu);
4673 static void intel_disable_iommus(void)
4675 struct intel_iommu *iommu = NULL;
4676 struct dmar_drhd_unit *drhd;
4678 for_each_iommu(iommu, drhd)
4679 iommu_disable_translation(iommu);
4682 static inline struct intel_iommu *dev_to_intel_iommu(struct device *dev)
4684 struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
4686 return container_of(iommu_dev, struct intel_iommu, iommu);
4689 static ssize_t intel_iommu_show_version(struct device *dev,
4690 struct device_attribute *attr,
4693 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4694 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4695 return sprintf(buf, "%d:%d\n",
4696 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4698 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4700 static ssize_t intel_iommu_show_address(struct device *dev,
4701 struct device_attribute *attr,
4704 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4705 return sprintf(buf, "%llx\n", iommu->reg_phys);
4707 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4709 static ssize_t intel_iommu_show_cap(struct device *dev,
4710 struct device_attribute *attr,
4713 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4714 return sprintf(buf, "%llx\n", iommu->cap);
4716 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4718 static ssize_t intel_iommu_show_ecap(struct device *dev,
4719 struct device_attribute *attr,
4722 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4723 return sprintf(buf, "%llx\n", iommu->ecap);
4725 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4727 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4728 struct device_attribute *attr,
4731 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4732 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4734 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4736 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4737 struct device_attribute *attr,
4740 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4741 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4742 cap_ndoms(iommu->cap)));
4744 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4746 static struct attribute *intel_iommu_attrs[] = {
4747 &dev_attr_version.attr,
4748 &dev_attr_address.attr,
4750 &dev_attr_ecap.attr,
4751 &dev_attr_domains_supported.attr,
4752 &dev_attr_domains_used.attr,
4756 static struct attribute_group intel_iommu_group = {
4757 .name = "intel-iommu",
4758 .attrs = intel_iommu_attrs,
4761 const struct attribute_group *intel_iommu_groups[] = {
4766 int __init intel_iommu_init(void)
4769 struct dmar_drhd_unit *drhd;
4770 struct intel_iommu *iommu;
4772 /* VT-d is required for a TXT/tboot launch, so enforce that */
4773 force_on = tboot_force_iommu();
4775 if (iommu_init_mempool()) {
4777 panic("tboot: Failed to initialize iommu memory\n");
4781 down_write(&dmar_global_lock);
4782 if (dmar_table_init()) {
4784 panic("tboot: Failed to initialize DMAR table\n");
4788 if (dmar_dev_scope_init() < 0) {
4790 panic("tboot: Failed to initialize DMAR device scope\n");
4794 up_write(&dmar_global_lock);
4797 * The bus notifier takes the dmar_global_lock, so lockdep will
4798 * complain later when we register it under the lock.
4800 dmar_register_bus_notifier();
4802 down_write(&dmar_global_lock);
4804 if (no_iommu || dmar_disabled) {
4806 * We exit the function here to ensure IOMMU's remapping and
4807 * mempool aren't setup, which means that the IOMMU's PMRs
4808 * won't be disabled via the call to init_dmars(). So disable
4809 * it explicitly here. The PMRs were setup by tboot prior to
4810 * calling SENTER, but the kernel is expected to reset/tear
4813 if (intel_iommu_tboot_noforce) {
4814 for_each_iommu(iommu, drhd)
4815 iommu_disable_protect_mem_regions(iommu);
4819 * Make sure the IOMMUs are switched off, even when we
4820 * boot into a kexec kernel and the previous kernel left
4823 intel_disable_iommus();
4827 if (list_empty(&dmar_rmrr_units))
4828 pr_info("No RMRR found\n");
4830 if (list_empty(&dmar_atsr_units))
4831 pr_info("No ATSR found\n");
4833 if (dmar_init_reserved_ranges()) {
4835 panic("tboot: Failed to reserve iommu ranges\n");
4836 goto out_free_reserved_range;
4840 intel_iommu_gfx_mapped = 1;
4842 init_no_remapping_devices();
4847 panic("tboot: Failed to initialize DMARs\n");
4848 pr_err("Initialization failed\n");
4849 goto out_free_reserved_range;
4851 up_write(&dmar_global_lock);
4852 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4854 #if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
4857 dma_ops = &intel_dma_ops;
4859 init_iommu_pm_ops();
4861 for_each_active_iommu(iommu, drhd) {
4862 iommu_device_sysfs_add(&iommu->iommu, NULL,
4865 iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);
4866 iommu_device_register(&iommu->iommu);
4869 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4870 bus_register_notifier(&pci_bus_type, &device_nb);
4871 if (si_domain && !hw_pass_through)
4872 register_memory_notifier(&intel_iommu_memory_nb);
4873 cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL,
4874 intel_iommu_cpu_dead);
4875 intel_iommu_enabled = 1;
4879 out_free_reserved_range:
4880 put_iova_domain(&reserved_iova_list);
4882 intel_iommu_free_dmars();
4883 up_write(&dmar_global_lock);
4884 iommu_exit_mempool();
4888 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4890 struct intel_iommu *iommu = opaque;
4892 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4897 * NB - intel-iommu lacks any sort of reference counting for the users of
4898 * dependent devices. If multiple endpoints have intersecting dependent
4899 * devices, unbinding the driver from any one of them will possibly leave
4900 * the others unable to operate.
4902 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4904 if (!iommu || !dev || !dev_is_pci(dev))
4907 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4910 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4912 struct intel_iommu *iommu;
4913 unsigned long flags;
4915 assert_spin_locked(&device_domain_lock);
4920 iommu = info->iommu;
4923 iommu_disable_dev_iotlb(info);
4924 domain_context_clear(iommu, info->dev);
4925 intel_pasid_free_table(info->dev);
4928 unlink_domain_info(info);
4930 spin_lock_irqsave(&iommu->lock, flags);
4931 domain_detach_iommu(info->domain, iommu);
4932 spin_unlock_irqrestore(&iommu->lock, flags);
4934 free_devinfo_mem(info);
4937 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4940 struct device_domain_info *info;
4941 unsigned long flags;
4943 spin_lock_irqsave(&device_domain_lock, flags);
4944 info = dev->archdata.iommu;
4945 __dmar_remove_one_dev_info(info);
4946 spin_unlock_irqrestore(&device_domain_lock, flags);
4949 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4953 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
4954 domain_reserve_special_ranges(domain);
4956 /* calculate AGAW */
4957 domain->gaw = guest_width;
4958 adjust_width = guestwidth_to_adjustwidth(guest_width);
4959 domain->agaw = width_to_agaw(adjust_width);
4961 domain->iommu_coherency = 0;
4962 domain->iommu_snooping = 0;
4963 domain->iommu_superpage = 0;
4964 domain->max_addr = 0;
4966 /* always allocate the top pgd */
4967 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4970 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4974 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4976 struct dmar_domain *dmar_domain;
4977 struct iommu_domain *domain;
4979 if (type != IOMMU_DOMAIN_UNMANAGED)
4982 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4984 pr_err("Can't allocate dmar_domain\n");
4987 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4988 pr_err("Domain initialization failed\n");
4989 domain_exit(dmar_domain);
4992 domain_update_iommu_cap(dmar_domain);
4994 domain = &dmar_domain->domain;
4995 domain->geometry.aperture_start = 0;
4996 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4997 domain->geometry.force_aperture = true;
5002 static void intel_iommu_domain_free(struct iommu_domain *domain)
5004 domain_exit(to_dmar_domain(domain));
5007 static int intel_iommu_attach_device(struct iommu_domain *domain,
5010 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5011 struct intel_iommu *iommu;
5015 if (device_is_rmrr_locked(dev)) {
5016 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
5020 /* normally dev is not mapped */
5021 if (unlikely(domain_context_mapped(dev))) {
5022 struct dmar_domain *old_domain;
5024 old_domain = find_domain(dev);
5027 dmar_remove_one_dev_info(old_domain, dev);
5030 if (!domain_type_is_vm_or_si(old_domain) &&
5031 list_empty(&old_domain->devices))
5032 domain_exit(old_domain);
5036 iommu = device_to_iommu(dev, &bus, &devfn);
5040 /* check if this iommu agaw is sufficient for max mapped address */
5041 addr_width = agaw_to_width(iommu->agaw);
5042 if (addr_width > cap_mgaw(iommu->cap))
5043 addr_width = cap_mgaw(iommu->cap);
5045 if (dmar_domain->max_addr > (1LL << addr_width)) {
5046 pr_err("%s: iommu width (%d) is not "
5047 "sufficient for the mapped address (%llx)\n",
5048 __func__, addr_width, dmar_domain->max_addr);
5051 dmar_domain->gaw = addr_width;
5054 * Knock out extra levels of page tables if necessary
5056 while (iommu->agaw < dmar_domain->agaw) {
5057 struct dma_pte *pte;
5059 pte = dmar_domain->pgd;
5060 if (dma_pte_present(pte)) {
5061 dmar_domain->pgd = (struct dma_pte *)
5062 phys_to_virt(dma_pte_addr(pte));
5063 free_pgtable_page(pte);
5065 dmar_domain->agaw--;
5068 return domain_add_dev_info(dmar_domain, dev);
5071 static void intel_iommu_detach_device(struct iommu_domain *domain,
5074 dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
5077 static int intel_iommu_map(struct iommu_domain *domain,
5078 unsigned long iova, phys_addr_t hpa,
5079 size_t size, int iommu_prot)
5081 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5086 if (iommu_prot & IOMMU_READ)
5087 prot |= DMA_PTE_READ;
5088 if (iommu_prot & IOMMU_WRITE)
5089 prot |= DMA_PTE_WRITE;
5090 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5091 prot |= DMA_PTE_SNP;
5093 max_addr = iova + size;
5094 if (dmar_domain->max_addr < max_addr) {
5097 /* check if minimum agaw is sufficient for mapped address */
5098 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5099 if (end < max_addr) {
5100 pr_err("%s: iommu width (%d) is not "
5101 "sufficient for the mapped address (%llx)\n",
5102 __func__, dmar_domain->gaw, max_addr);
5105 dmar_domain->max_addr = max_addr;
5107 /* Round up size to next multiple of PAGE_SIZE, if it and
5108 the low bits of hpa would take us onto the next page */
5109 size = aligned_nrpages(hpa, size);
5110 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5111 hpa >> VTD_PAGE_SHIFT, size, prot);
5115 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5116 unsigned long iova, size_t size)
5118 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5119 struct page *freelist = NULL;
5120 unsigned long start_pfn, last_pfn;
5121 unsigned int npages;
5122 int iommu_id, level = 0;
5124 /* Cope with horrid API which requires us to unmap more than the
5125 size argument if it happens to be a large-page mapping. */
5126 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5128 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5129 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5131 start_pfn = iova >> VTD_PAGE_SHIFT;
5132 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5134 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5136 npages = last_pfn - start_pfn + 1;
5138 for_each_domain_iommu(iommu_id, dmar_domain)
5139 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5140 start_pfn, npages, !freelist, 0);
5142 dma_free_pagelist(freelist);
5144 if (dmar_domain->max_addr == iova + size)
5145 dmar_domain->max_addr = iova;
5150 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5153 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5154 struct dma_pte *pte;
5158 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5159 if (pte && dma_pte_present(pte))
5160 phys = dma_pte_addr(pte) +
5161 (iova & (BIT_MASK(level_to_offset_bits(level) +
5162 VTD_PAGE_SHIFT) - 1));
5167 static bool intel_iommu_capable(enum iommu_cap cap)
5169 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5170 return domain_update_iommu_snooping(NULL) == 1;
5171 if (cap == IOMMU_CAP_INTR_REMAP)
5172 return irq_remapping_enabled == 1;
5177 static int intel_iommu_add_device(struct device *dev)
5179 struct intel_iommu *iommu;
5180 struct iommu_group *group;
5183 iommu = device_to_iommu(dev, &bus, &devfn);
5187 iommu_device_link(&iommu->iommu, dev);
5189 group = iommu_group_get_for_dev(dev);
5192 return PTR_ERR(group);
5194 iommu_group_put(group);
5198 static void intel_iommu_remove_device(struct device *dev)
5200 struct intel_iommu *iommu;
5203 iommu = device_to_iommu(dev, &bus, &devfn);
5207 iommu_group_remove_device(dev);
5209 iommu_device_unlink(&iommu->iommu, dev);
5212 static void intel_iommu_get_resv_regions(struct device *device,
5213 struct list_head *head)
5215 int prot = DMA_PTE_READ | DMA_PTE_WRITE;
5216 struct iommu_resv_region *reg;
5217 struct dmar_rmrr_unit *rmrr;
5218 struct device *i_dev;
5221 down_read(&dmar_global_lock);
5222 for_each_rmrr_units(rmrr) {
5223 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
5225 struct iommu_resv_region *resv;
5228 if (i_dev != device)
5231 length = rmrr->end_address - rmrr->base_address + 1;
5232 resv = iommu_alloc_resv_region(rmrr->base_address,
5238 list_add_tail(&resv->list, head);
5241 up_read(&dmar_global_lock);
5243 reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
5244 IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
5248 list_add_tail(®->list, head);
5251 static void intel_iommu_put_resv_regions(struct device *dev,
5252 struct list_head *head)
5254 struct iommu_resv_region *entry, *next;
5256 list_for_each_entry_safe(entry, next, head, list)
5260 #ifdef CONFIG_INTEL_IOMMU_SVM
5261 #define MAX_NR_PASID_BITS (20)
5262 static inline unsigned long intel_iommu_get_pts(struct device *dev)
5266 max_pasid = intel_pasid_get_dev_max_id(dev);
5267 pts = find_first_bit((unsigned long *)&max_pasid, MAX_NR_PASID_BITS);
5274 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5276 struct device_domain_info *info;
5277 struct context_entry *context;
5278 struct dmar_domain *domain;
5279 unsigned long flags;
5283 domain = get_valid_domain_for_dev(sdev->dev);
5287 spin_lock_irqsave(&device_domain_lock, flags);
5288 spin_lock(&iommu->lock);
5291 info = sdev->dev->archdata.iommu;
5292 if (!info || !info->pasid_supported)
5295 context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5296 if (WARN_ON(!context))
5299 ctx_lo = context[0].lo;
5301 sdev->did = domain->iommu_did[iommu->seq_id];
5302 sdev->sid = PCI_DEVID(info->bus, info->devfn);
5304 if (!(ctx_lo & CONTEXT_PASIDE)) {
5305 if (iommu->pasid_state_table)
5306 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
5307 context[1].lo = (u64)virt_to_phys(info->pasid_table->table) |
5308 intel_iommu_get_pts(sdev->dev);
5311 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5312 * extended to permit requests-with-PASID if the PASIDE bit
5313 * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5314 * however, the PASIDE bit is ignored and requests-with-PASID
5315 * are unconditionally blocked. Which makes less sense.
5316 * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5317 * "guest mode" translation types depending on whether ATS
5318 * is available or not. Annoyingly, we can't use the new
5319 * modes *unless* PASIDE is set. */
5320 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5321 ctx_lo &= ~CONTEXT_TT_MASK;
5322 if (info->ats_supported)
5323 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5325 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5327 ctx_lo |= CONTEXT_PASIDE;
5328 if (iommu->pasid_state_table)
5329 ctx_lo |= CONTEXT_DINVE;
5330 if (info->pri_supported)
5331 ctx_lo |= CONTEXT_PRS;
5332 context[0].lo = ctx_lo;
5334 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5335 DMA_CCMD_MASK_NOBIT,
5336 DMA_CCMD_DEVICE_INVL);
5339 /* Enable PASID support in the device, if it wasn't already */
5340 if (!info->pasid_enabled)
5341 iommu_enable_dev_iotlb(info);
5343 if (info->ats_enabled) {
5344 sdev->dev_iotlb = 1;
5345 sdev->qdep = info->ats_qdep;
5346 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5352 spin_unlock(&iommu->lock);
5353 spin_unlock_irqrestore(&device_domain_lock, flags);
5358 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5360 struct intel_iommu *iommu;
5363 if (iommu_dummy(dev)) {
5365 "No IOMMU translation for device; cannot enable SVM\n");
5369 iommu = device_to_iommu(dev, &bus, &devfn);
5371 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5377 #endif /* CONFIG_INTEL_IOMMU_SVM */
5379 const struct iommu_ops intel_iommu_ops = {
5380 .capable = intel_iommu_capable,
5381 .domain_alloc = intel_iommu_domain_alloc,
5382 .domain_free = intel_iommu_domain_free,
5383 .attach_dev = intel_iommu_attach_device,
5384 .detach_dev = intel_iommu_detach_device,
5385 .map = intel_iommu_map,
5386 .unmap = intel_iommu_unmap,
5387 .iova_to_phys = intel_iommu_iova_to_phys,
5388 .add_device = intel_iommu_add_device,
5389 .remove_device = intel_iommu_remove_device,
5390 .get_resv_regions = intel_iommu_get_resv_regions,
5391 .put_resv_regions = intel_iommu_put_resv_regions,
5392 .device_group = pci_device_group,
5393 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
5396 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5398 /* G4x/GM45 integrated gfx dmar support is totally busted. */
5399 pr_info("Disabling IOMMU for graphics on this chipset\n");
5403 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5404 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5405 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5406 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5407 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5408 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5409 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5411 static void quirk_iommu_rwbf(struct pci_dev *dev)
5414 * Mobile 4 Series Chipset neglects to set RWBF capability,
5415 * but needs it. Same seems to hold for the desktop versions.
5417 pr_info("Forcing write-buffer flush capability\n");
5421 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5422 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5423 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5424 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5425 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5426 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5427 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5430 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
5431 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
5432 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
5433 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
5434 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
5435 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
5436 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
5437 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
5439 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5443 if (pci_read_config_word(dev, GGC, &ggc))
5446 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5447 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5449 } else if (dmar_map_gfx) {
5450 /* we have to ensure the gfx device is idle before we flush */
5451 pr_info("Disabling batched IOTLB flush on Ironlake\n");
5452 intel_iommu_strict = 1;
5455 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5456 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5457 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5458 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5460 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5461 ISOCH DMAR unit for the Azalia sound device, but not give it any
5462 TLB entries, which causes it to deadlock. Check for that. We do
5463 this in a function called from init_dmars(), instead of in a PCI
5464 quirk, because we don't want to print the obnoxious "BIOS broken"
5465 message if VT-d is actually disabled.
5467 static void __init check_tylersburg_isoch(void)
5469 struct pci_dev *pdev;
5470 uint32_t vtisochctrl;
5472 /* If there's no Azalia in the system anyway, forget it. */
5473 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5478 /* System Management Registers. Might be hidden, in which case
5479 we can't do the sanity check. But that's OK, because the
5480 known-broken BIOSes _don't_ actually hide it, so far. */
5481 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5485 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5492 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5493 if (vtisochctrl & 1)
5496 /* Drop all bits other than the number of TLB entries */
5497 vtisochctrl &= 0x1c;
5499 /* If we have the recommended number of TLB entries (16), fine. */
5500 if (vtisochctrl == 0x10)
5503 /* Zero TLB entries? You get to ride the short bus to school. */
5505 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5506 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5507 dmi_get_system_info(DMI_BIOS_VENDOR),
5508 dmi_get_system_info(DMI_BIOS_VERSION),
5509 dmi_get_system_info(DMI_PRODUCT_VERSION));
5510 iommu_identity_mapping |= IDENTMAP_AZALIA;
5514 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",