2 * Copyright © 2006-2014 Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
18 * Joerg Roedel <jroedel@suse.de>
21 #define pr_fmt(fmt) "DMAR: " fmt
23 #include <linux/init.h>
24 #include <linux/bitmap.h>
25 #include <linux/debugfs.h>
26 #include <linux/export.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/memory.h>
36 #include <linux/timer.h>
38 #include <linux/iova.h>
39 #include <linux/iommu.h>
40 #include <linux/intel-iommu.h>
41 #include <linux/syscore_ops.h>
42 #include <linux/tboot.h>
43 #include <linux/dmi.h>
44 #include <linux/pci-ats.h>
45 #include <linux/memblock.h>
46 #include <linux/dma-contiguous.h>
47 #include <linux/crash_dump.h>
48 #include <asm/irq_remapping.h>
49 #include <asm/cacheflush.h>
50 #include <asm/iommu.h>
52 #include "irq_remapping.h"
54 #define ROOT_SIZE VTD_PAGE_SIZE
55 #define CONTEXT_SIZE VTD_PAGE_SIZE
57 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
58 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
59 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
60 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
62 #define IOAPIC_RANGE_START (0xfee00000)
63 #define IOAPIC_RANGE_END (0xfeefffff)
64 #define IOVA_START_ADDR (0x1000)
66 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
68 #define MAX_AGAW_WIDTH 64
69 #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
71 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
72 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
74 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
75 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
76 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
77 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
78 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
80 /* IO virtual address start page frame number */
81 #define IOVA_START_PFN (1)
83 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
84 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
85 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
87 /* page table handling */
88 #define LEVEL_STRIDE (9)
89 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
92 * This bitmap is used to advertise the page sizes our hardware support
93 * to the IOMMU core, which will then use this information to split
94 * physically contiguous memory regions it is mapping into page sizes
97 * Traditionally the IOMMU core just handed us the mappings directly,
98 * after making sure the size is an order of a 4KiB page and that the
99 * mapping has natural alignment.
101 * To retain this behavior, we currently advertise that we support
102 * all page sizes that are an order of 4KiB.
104 * If at some point we'd like to utilize the IOMMU core's new behavior,
105 * we could change this to advertise the real page sizes we support.
107 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
109 static inline int agaw_to_level(int agaw)
114 static inline int agaw_to_width(int agaw)
116 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
119 static inline int width_to_agaw(int width)
121 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
124 static inline unsigned int level_to_offset_bits(int level)
126 return (level - 1) * LEVEL_STRIDE;
129 static inline int pfn_level_offset(unsigned long pfn, int level)
131 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
134 static inline unsigned long level_mask(int level)
136 return -1UL << level_to_offset_bits(level);
139 static inline unsigned long level_size(int level)
141 return 1UL << level_to_offset_bits(level);
144 static inline unsigned long align_to_level(unsigned long pfn, int level)
146 return (pfn + level_size(level) - 1) & level_mask(level);
149 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
151 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
154 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
155 are never going to work. */
156 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
158 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
161 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
163 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
165 static inline unsigned long page_to_dma_pfn(struct page *pg)
167 return mm_to_dma_pfn(page_to_pfn(pg));
169 static inline unsigned long virt_to_dma_pfn(void *p)
171 return page_to_dma_pfn(virt_to_page(p));
174 /* global iommu list, set NULL for ignored DMAR units */
175 static struct intel_iommu **g_iommus;
177 static void __init check_tylersburg_isoch(void);
178 static int rwbf_quirk;
181 * set to 1 to panic kernel if can't successfully enable VT-d
182 * (used when kernel is launched w/ TXT)
184 static int force_on = 0;
189 * 12-63: Context Ptr (12 - (haw-1))
196 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
199 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
202 static phys_addr_t root_entry_lctp(struct root_entry *re)
207 return re->lo & VTD_PAGE_MASK;
211 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
214 static phys_addr_t root_entry_uctp(struct root_entry *re)
219 return re->hi & VTD_PAGE_MASK;
224 * 1: fault processing disable
225 * 2-3: translation type
226 * 12-63: address space root
232 struct context_entry {
237 static inline void context_clear_pasid_enable(struct context_entry *context)
239 context->lo &= ~(1ULL << 11);
242 static inline bool context_pasid_enabled(struct context_entry *context)
244 return !!(context->lo & (1ULL << 11));
247 static inline void context_set_copied(struct context_entry *context)
249 context->hi |= (1ull << 3);
252 static inline bool context_copied(struct context_entry *context)
254 return !!(context->hi & (1ULL << 3));
257 static inline bool __context_present(struct context_entry *context)
259 return (context->lo & 1);
262 static inline bool context_present(struct context_entry *context)
264 return context_pasid_enabled(context) ?
265 __context_present(context) :
266 __context_present(context) && !context_copied(context);
269 static inline void context_set_present(struct context_entry *context)
274 static inline void context_set_fault_enable(struct context_entry *context)
276 context->lo &= (((u64)-1) << 2) | 1;
279 static inline void context_set_translation_type(struct context_entry *context,
282 context->lo &= (((u64)-1) << 4) | 3;
283 context->lo |= (value & 3) << 2;
286 static inline void context_set_address_root(struct context_entry *context,
289 context->lo &= ~VTD_PAGE_MASK;
290 context->lo |= value & VTD_PAGE_MASK;
293 static inline void context_set_address_width(struct context_entry *context,
296 context->hi |= value & 7;
299 static inline void context_set_domain_id(struct context_entry *context,
302 context->hi |= (value & ((1 << 16) - 1)) << 8;
305 static inline int context_domain_id(struct context_entry *c)
307 return((c->hi >> 8) & 0xffff);
310 static inline void context_clear_entry(struct context_entry *context)
323 * 12-63: Host physcial address
329 static inline void dma_clear_pte(struct dma_pte *pte)
334 static inline u64 dma_pte_addr(struct dma_pte *pte)
337 return pte->val & VTD_PAGE_MASK;
339 /* Must have a full atomic 64-bit read */
340 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
344 static inline bool dma_pte_present(struct dma_pte *pte)
346 return (pte->val & 3) != 0;
349 static inline bool dma_pte_superpage(struct dma_pte *pte)
351 return (pte->val & DMA_PTE_LARGE_PAGE);
354 static inline int first_pte_in_page(struct dma_pte *pte)
356 return !((unsigned long)pte & ~VTD_PAGE_MASK);
360 * This domain is a statically identity mapping domain.
361 * 1. This domain creats a static 1:1 mapping to all usable memory.
362 * 2. It maps to each iommu if successful.
363 * 3. Each iommu mapps to this domain if successful.
365 static struct dmar_domain *si_domain;
366 static int hw_pass_through = 1;
369 * Domain represents a virtual machine, more than one devices
370 * across iommus may be owned in one domain, e.g. kvm guest.
372 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
374 /* si_domain contains mulitple devices */
375 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
377 #define for_each_domain_iommu(idx, domain) \
378 for (idx = 0; idx < g_num_of_iommus; idx++) \
379 if (domain->iommu_refcnt[idx])
382 int nid; /* node id */
384 unsigned iommu_refcnt[DMAR_UNITS_SUPPORTED];
385 /* Refcount of devices per iommu */
388 u16 iommu_did[DMAR_UNITS_SUPPORTED];
389 /* Domain ids per IOMMU. Use u16 since
390 * domain ids are 16 bit wide according
391 * to VT-d spec, section 9.3 */
393 struct list_head devices; /* all devices' list */
394 struct iova_domain iovad; /* iova's that belong to this domain */
396 struct dma_pte *pgd; /* virtual address */
397 int gaw; /* max guest address width */
399 /* adjusted guest address width, 0 is level 2 30-bit */
402 int flags; /* flags to find out type of domain */
404 int iommu_coherency;/* indicate coherency of iommu access */
405 int iommu_snooping; /* indicate snooping control feature*/
406 int iommu_count; /* reference count of iommu */
407 int iommu_superpage;/* Level of superpages supported:
408 0 == 4KiB (no superpages), 1 == 2MiB,
409 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
410 u64 max_addr; /* maximum mapped address */
412 struct iommu_domain domain; /* generic domain data structure for
416 /* PCI domain-device relationship */
417 struct device_domain_info {
418 struct list_head link; /* link to domain siblings */
419 struct list_head global; /* link to global list */
420 u8 bus; /* PCI bus number */
421 u8 devfn; /* PCI devfn number */
422 u16 pfsid; /* SRIOV physical function source ID */
423 u8 pasid_supported:3;
430 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
431 struct intel_iommu *iommu; /* IOMMU used by this device */
432 struct dmar_domain *domain; /* pointer to domain */
435 struct dmar_rmrr_unit {
436 struct list_head list; /* list of rmrr units */
437 struct acpi_dmar_header *hdr; /* ACPI header */
438 u64 base_address; /* reserved base address*/
439 u64 end_address; /* reserved end address */
440 struct dmar_dev_scope *devices; /* target devices */
441 int devices_cnt; /* target device count */
444 struct dmar_atsr_unit {
445 struct list_head list; /* list of ATSR units */
446 struct acpi_dmar_header *hdr; /* ACPI header */
447 struct dmar_dev_scope *devices; /* target devices */
448 int devices_cnt; /* target device count */
449 u8 include_all:1; /* include all ports */
452 static LIST_HEAD(dmar_atsr_units);
453 static LIST_HEAD(dmar_rmrr_units);
455 #define for_each_rmrr_units(rmrr) \
456 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
458 static void flush_unmaps_timeout(unsigned long data);
460 static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
462 #define HIGH_WATER_MARK 250
463 struct deferred_flush_tables {
465 struct iova *iova[HIGH_WATER_MARK];
466 struct dmar_domain *domain[HIGH_WATER_MARK];
467 struct page *freelist[HIGH_WATER_MARK];
470 static struct deferred_flush_tables *deferred_flush;
472 /* bitmap for indexing intel_iommus */
473 static int g_num_of_iommus;
475 static DEFINE_SPINLOCK(async_umap_flush_lock);
476 static LIST_HEAD(unmaps_to_do);
479 static long list_size;
481 static void domain_exit(struct dmar_domain *domain);
482 static void domain_remove_dev_info(struct dmar_domain *domain);
483 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
485 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
486 static void domain_context_clear(struct intel_iommu *iommu,
488 static int domain_detach_iommu(struct dmar_domain *domain,
489 struct intel_iommu *iommu);
491 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
492 int dmar_disabled = 0;
494 int dmar_disabled = 1;
495 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
497 int intel_iommu_enabled = 0;
498 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
500 static int dmar_map_gfx = 1;
501 static int dmar_forcedac;
502 static int intel_iommu_strict;
503 static int intel_iommu_superpage = 1;
504 static int intel_iommu_ecs = 1;
505 static int intel_iommu_pasid28;
506 static int iommu_identity_mapping;
508 #define IDENTMAP_ALL 1
509 #define IDENTMAP_GFX 2
510 #define IDENTMAP_AZALIA 4
512 /* Broadwell and Skylake have broken ECS support — normal so-called "second
513 * level" translation of DMA requests-without-PASID doesn't actually happen
514 * unless you also set the NESTE bit in an extended context-entry. Which of
515 * course means that SVM doesn't work because it's trying to do nested
516 * translation of the physical addresses it finds in the process page tables,
517 * through the IOVA->phys mapping found in the "second level" page tables.
519 * The VT-d specification was retroactively changed to change the definition
520 * of the capability bits and pretend that Broadwell/Skylake never happened...
521 * but unfortunately the wrong bit was changed. It's ECS which is broken, but
522 * for some reason it was the PASID capability bit which was redefined (from
523 * bit 28 on BDW/SKL to bit 40 in future).
525 * So our test for ECS needs to eschew those implementations which set the old
526 * PASID capabiity bit 28, since those are the ones on which ECS is broken.
527 * Unless we are working around the 'pasid28' limitations, that is, by putting
528 * the device into passthrough mode for normal DMA and thus masking the bug.
530 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
531 (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
532 /* PASID support is thus enabled if ECS is enabled and *either* of the old
533 * or new capability bits are set. */
534 #define pasid_enabled(iommu) (ecs_enabled(iommu) && \
535 (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
537 int intel_iommu_gfx_mapped;
538 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
540 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
541 static DEFINE_SPINLOCK(device_domain_lock);
542 static LIST_HEAD(device_domain_list);
544 static const struct iommu_ops intel_iommu_ops;
546 static bool translation_pre_enabled(struct intel_iommu *iommu)
548 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
551 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
553 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
556 static void init_translation_status(struct intel_iommu *iommu)
560 gsts = readl(iommu->reg + DMAR_GSTS_REG);
561 if (gsts & DMA_GSTS_TES)
562 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
565 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
566 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
568 return container_of(dom, struct dmar_domain, domain);
571 static int __init intel_iommu_setup(char *str)
576 if (!strncmp(str, "on", 2)) {
578 pr_info("IOMMU enabled\n");
579 } else if (!strncmp(str, "off", 3)) {
581 pr_info("IOMMU disabled\n");
582 } else if (!strncmp(str, "igfx_off", 8)) {
584 pr_info("Disable GFX device mapping\n");
585 } else if (!strncmp(str, "forcedac", 8)) {
586 pr_info("Forcing DAC for PCI devices\n");
588 } else if (!strncmp(str, "strict", 6)) {
589 pr_info("Disable batched IOTLB flush\n");
590 intel_iommu_strict = 1;
591 } else if (!strncmp(str, "sp_off", 6)) {
592 pr_info("Disable supported super page\n");
593 intel_iommu_superpage = 0;
594 } else if (!strncmp(str, "ecs_off", 7)) {
596 "Intel-IOMMU: disable extended context table support\n");
598 } else if (!strncmp(str, "pasid28", 7)) {
600 "Intel-IOMMU: enable pre-production PASID support\n");
601 intel_iommu_pasid28 = 1;
602 iommu_identity_mapping |= IDENTMAP_GFX;
605 str += strcspn(str, ",");
611 __setup("intel_iommu=", intel_iommu_setup);
613 static struct kmem_cache *iommu_domain_cache;
614 static struct kmem_cache *iommu_devinfo_cache;
616 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
618 struct dmar_domain **domains;
621 domains = iommu->domains[idx];
625 return domains[did & 0xff];
628 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
629 struct dmar_domain *domain)
631 struct dmar_domain **domains;
634 if (!iommu->domains[idx]) {
635 size_t size = 256 * sizeof(struct dmar_domain *);
636 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
639 domains = iommu->domains[idx];
640 if (WARN_ON(!domains))
643 domains[did & 0xff] = domain;
646 static inline void *alloc_pgtable_page(int node)
651 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
653 vaddr = page_address(page);
657 static inline void free_pgtable_page(void *vaddr)
659 free_page((unsigned long)vaddr);
662 static inline void *alloc_domain_mem(void)
664 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
667 static void free_domain_mem(void *vaddr)
669 kmem_cache_free(iommu_domain_cache, vaddr);
672 static inline void * alloc_devinfo_mem(void)
674 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
677 static inline void free_devinfo_mem(void *vaddr)
679 kmem_cache_free(iommu_devinfo_cache, vaddr);
682 static inline int domain_type_is_vm(struct dmar_domain *domain)
684 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
687 static inline int domain_type_is_si(struct dmar_domain *domain)
689 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
692 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
694 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
695 DOMAIN_FLAG_STATIC_IDENTITY);
698 static inline int domain_pfn_supported(struct dmar_domain *domain,
701 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
703 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
706 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
711 sagaw = cap_sagaw(iommu->cap);
712 for (agaw = width_to_agaw(max_gaw);
714 if (test_bit(agaw, &sagaw))
722 * Calculate max SAGAW for each iommu.
724 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
726 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
730 * calculate agaw for each iommu.
731 * "SAGAW" may be different across iommus, use a default agaw, and
732 * get a supported less agaw for iommus that don't support the default agaw.
734 int iommu_calculate_agaw(struct intel_iommu *iommu)
736 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
739 /* This functionin only returns single iommu in a domain */
740 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
744 /* si_domain and vm domain should not get here. */
745 BUG_ON(domain_type_is_vm_or_si(domain));
746 for_each_domain_iommu(iommu_id, domain)
749 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
752 return g_iommus[iommu_id];
755 static void domain_update_iommu_coherency(struct dmar_domain *domain)
757 struct dmar_drhd_unit *drhd;
758 struct intel_iommu *iommu;
762 domain->iommu_coherency = 1;
764 for_each_domain_iommu(i, domain) {
766 if (!ecap_coherent(g_iommus[i]->ecap)) {
767 domain->iommu_coherency = 0;
774 /* No hardware attached; use lowest common denominator */
776 for_each_active_iommu(iommu, drhd) {
777 if (!ecap_coherent(iommu->ecap)) {
778 domain->iommu_coherency = 0;
785 static int domain_update_iommu_snooping(struct intel_iommu *skip)
787 struct dmar_drhd_unit *drhd;
788 struct intel_iommu *iommu;
792 for_each_active_iommu(iommu, drhd) {
794 if (!ecap_sc_support(iommu->ecap)) {
805 static int domain_update_iommu_superpage(struct intel_iommu *skip)
807 struct dmar_drhd_unit *drhd;
808 struct intel_iommu *iommu;
811 if (!intel_iommu_superpage) {
815 /* set iommu_superpage to the smallest common denominator */
817 for_each_active_iommu(iommu, drhd) {
819 mask &= cap_super_page_val(iommu->cap);
829 /* Some capabilities may be different across iommus */
830 static void domain_update_iommu_cap(struct dmar_domain *domain)
832 domain_update_iommu_coherency(domain);
833 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
834 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
837 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
838 u8 bus, u8 devfn, int alloc)
840 struct root_entry *root = &iommu->root_entry[bus];
841 struct context_entry *context;
845 if (ecs_enabled(iommu)) {
853 context = phys_to_virt(*entry & VTD_PAGE_MASK);
855 unsigned long phy_addr;
859 context = alloc_pgtable_page(iommu->node);
863 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
864 phy_addr = virt_to_phys((void *)context);
865 *entry = phy_addr | 1;
866 __iommu_flush_cache(iommu, entry, sizeof(*entry));
868 return &context[devfn];
871 static int iommu_dummy(struct device *dev)
873 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
876 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
878 struct dmar_drhd_unit *drhd = NULL;
879 struct intel_iommu *iommu;
881 struct pci_dev *ptmp, *pdev = NULL;
885 if (iommu_dummy(dev))
888 if (dev_is_pci(dev)) {
889 struct pci_dev *pf_pdev;
891 pdev = to_pci_dev(dev);
892 /* VFs aren't listed in scope tables; we need to look up
893 * the PF instead to find the IOMMU. */
894 pf_pdev = pci_physfn(pdev);
896 segment = pci_domain_nr(pdev->bus);
897 } else if (has_acpi_companion(dev))
898 dev = &ACPI_COMPANION(dev)->dev;
901 for_each_active_iommu(iommu, drhd) {
902 if (pdev && segment != drhd->segment)
905 for_each_active_dev_scope(drhd->devices,
906 drhd->devices_cnt, i, tmp) {
908 /* For a VF use its original BDF# not that of the PF
909 * which we used for the IOMMU lookup. Strictly speaking
910 * we could do this for all PCI devices; we only need to
911 * get the BDF# from the scope table for ACPI matches. */
912 if (pdev && pdev->is_virtfn)
915 *bus = drhd->devices[i].bus;
916 *devfn = drhd->devices[i].devfn;
920 if (!pdev || !dev_is_pci(tmp))
923 ptmp = to_pci_dev(tmp);
924 if (ptmp->subordinate &&
925 ptmp->subordinate->number <= pdev->bus->number &&
926 ptmp->subordinate->busn_res.end >= pdev->bus->number)
930 if (pdev && drhd->include_all) {
932 *bus = pdev->bus->number;
933 *devfn = pdev->devfn;
944 static void domain_flush_cache(struct dmar_domain *domain,
945 void *addr, int size)
947 if (!domain->iommu_coherency)
948 clflush_cache_range(addr, size);
951 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
953 struct context_entry *context;
957 spin_lock_irqsave(&iommu->lock, flags);
958 context = iommu_context_addr(iommu, bus, devfn, 0);
960 ret = context_present(context);
961 spin_unlock_irqrestore(&iommu->lock, flags);
965 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
967 struct context_entry *context;
970 spin_lock_irqsave(&iommu->lock, flags);
971 context = iommu_context_addr(iommu, bus, devfn, 0);
973 context_clear_entry(context);
974 __iommu_flush_cache(iommu, context, sizeof(*context));
976 spin_unlock_irqrestore(&iommu->lock, flags);
979 static void free_context_table(struct intel_iommu *iommu)
983 struct context_entry *context;
985 spin_lock_irqsave(&iommu->lock, flags);
986 if (!iommu->root_entry) {
989 for (i = 0; i < ROOT_ENTRY_NR; i++) {
990 context = iommu_context_addr(iommu, i, 0, 0);
992 free_pgtable_page(context);
994 if (!ecs_enabled(iommu))
997 context = iommu_context_addr(iommu, i, 0x80, 0);
999 free_pgtable_page(context);
1002 free_pgtable_page(iommu->root_entry);
1003 iommu->root_entry = NULL;
1005 spin_unlock_irqrestore(&iommu->lock, flags);
1008 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
1009 unsigned long pfn, int *target_level)
1011 struct dma_pte *parent, *pte = NULL;
1012 int level = agaw_to_level(domain->agaw);
1015 BUG_ON(!domain->pgd);
1017 if (!domain_pfn_supported(domain, pfn))
1018 /* Address beyond IOMMU's addressing capabilities. */
1021 parent = domain->pgd;
1026 offset = pfn_level_offset(pfn, level);
1027 pte = &parent[offset];
1028 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
1030 if (level == *target_level)
1033 if (!dma_pte_present(pte)) {
1036 tmp_page = alloc_pgtable_page(domain->nid);
1041 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
1042 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
1043 if (cmpxchg64(&pte->val, 0ULL, pteval))
1044 /* Someone else set it while we were thinking; use theirs. */
1045 free_pgtable_page(tmp_page);
1047 domain_flush_cache(domain, pte, sizeof(*pte));
1052 parent = phys_to_virt(dma_pte_addr(pte));
1057 *target_level = level;
1063 /* return address's pte at specific level */
1064 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1066 int level, int *large_page)
1068 struct dma_pte *parent, *pte = NULL;
1069 int total = agaw_to_level(domain->agaw);
1072 parent = domain->pgd;
1073 while (level <= total) {
1074 offset = pfn_level_offset(pfn, total);
1075 pte = &parent[offset];
1079 if (!dma_pte_present(pte)) {
1080 *large_page = total;
1084 if (dma_pte_superpage(pte)) {
1085 *large_page = total;
1089 parent = phys_to_virt(dma_pte_addr(pte));
1095 /* clear last level pte, a tlb flush should be followed */
1096 static void dma_pte_clear_range(struct dmar_domain *domain,
1097 unsigned long start_pfn,
1098 unsigned long last_pfn)
1100 unsigned int large_page = 1;
1101 struct dma_pte *first_pte, *pte;
1103 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1104 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1105 BUG_ON(start_pfn > last_pfn);
1107 /* we don't need lock here; nobody else touches the iova range */
1110 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1112 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1117 start_pfn += lvl_to_nr_pages(large_page);
1119 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1121 domain_flush_cache(domain, first_pte,
1122 (void *)pte - (void *)first_pte);
1124 } while (start_pfn && start_pfn <= last_pfn);
1127 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1128 struct dma_pte *pte, unsigned long pfn,
1129 unsigned long start_pfn, unsigned long last_pfn)
1131 pfn = max(start_pfn, pfn);
1132 pte = &pte[pfn_level_offset(pfn, level)];
1135 unsigned long level_pfn;
1136 struct dma_pte *level_pte;
1138 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1141 level_pfn = pfn & level_mask(level);
1142 level_pte = phys_to_virt(dma_pte_addr(pte));
1145 dma_pte_free_level(domain, level - 1, level_pte,
1146 level_pfn, start_pfn, last_pfn);
1148 /* If range covers entire pagetable, free it */
1149 if (!(start_pfn > level_pfn ||
1150 last_pfn < level_pfn + level_size(level) - 1)) {
1152 domain_flush_cache(domain, pte, sizeof(*pte));
1153 free_pgtable_page(level_pte);
1156 pfn += level_size(level);
1157 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1160 /* free page table pages. last level pte should already be cleared */
1161 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1162 unsigned long start_pfn,
1163 unsigned long last_pfn)
1165 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1166 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1167 BUG_ON(start_pfn > last_pfn);
1169 dma_pte_clear_range(domain, start_pfn, last_pfn);
1171 /* We don't need lock here; nobody else touches the iova range */
1172 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1173 domain->pgd, 0, start_pfn, last_pfn);
1176 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1177 free_pgtable_page(domain->pgd);
1182 /* When a page at a given level is being unlinked from its parent, we don't
1183 need to *modify* it at all. All we need to do is make a list of all the
1184 pages which can be freed just as soon as we've flushed the IOTLB and we
1185 know the hardware page-walk will no longer touch them.
1186 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1188 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1189 int level, struct dma_pte *pte,
1190 struct page *freelist)
1194 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1195 pg->freelist = freelist;
1201 pte = page_address(pg);
1203 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1204 freelist = dma_pte_list_pagetables(domain, level - 1,
1207 } while (!first_pte_in_page(pte));
1212 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1213 struct dma_pte *pte, unsigned long pfn,
1214 unsigned long start_pfn,
1215 unsigned long last_pfn,
1216 struct page *freelist)
1218 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1220 pfn = max(start_pfn, pfn);
1221 pte = &pte[pfn_level_offset(pfn, level)];
1224 unsigned long level_pfn;
1226 if (!dma_pte_present(pte))
1229 level_pfn = pfn & level_mask(level);
1231 /* If range covers entire pagetable, free it */
1232 if (start_pfn <= level_pfn &&
1233 last_pfn >= level_pfn + level_size(level) - 1) {
1234 /* These suborbinate page tables are going away entirely. Don't
1235 bother to clear them; we're just going to *free* them. */
1236 if (level > 1 && !dma_pte_superpage(pte))
1237 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1243 } else if (level > 1) {
1244 /* Recurse down into a level that isn't *entirely* obsolete */
1245 freelist = dma_pte_clear_level(domain, level - 1,
1246 phys_to_virt(dma_pte_addr(pte)),
1247 level_pfn, start_pfn, last_pfn,
1251 pfn += level_size(level);
1252 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1255 domain_flush_cache(domain, first_pte,
1256 (void *)++last_pte - (void *)first_pte);
1261 /* We can't just free the pages because the IOMMU may still be walking
1262 the page tables, and may have cached the intermediate levels. The
1263 pages can only be freed after the IOTLB flush has been done. */
1264 static struct page *domain_unmap(struct dmar_domain *domain,
1265 unsigned long start_pfn,
1266 unsigned long last_pfn)
1268 struct page *freelist = NULL;
1270 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1271 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1272 BUG_ON(start_pfn > last_pfn);
1274 /* we don't need lock here; nobody else touches the iova range */
1275 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1276 domain->pgd, 0, start_pfn, last_pfn, NULL);
1279 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1280 struct page *pgd_page = virt_to_page(domain->pgd);
1281 pgd_page->freelist = freelist;
1282 freelist = pgd_page;
1290 static void dma_free_pagelist(struct page *freelist)
1294 while ((pg = freelist)) {
1295 freelist = pg->freelist;
1296 free_pgtable_page(page_address(pg));
1300 /* iommu handling */
1301 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1303 struct root_entry *root;
1304 unsigned long flags;
1306 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1308 pr_err("Allocating root entry for %s failed\n",
1313 __iommu_flush_cache(iommu, root, ROOT_SIZE);
1315 spin_lock_irqsave(&iommu->lock, flags);
1316 iommu->root_entry = root;
1317 spin_unlock_irqrestore(&iommu->lock, flags);
1322 static void iommu_set_root_entry(struct intel_iommu *iommu)
1328 addr = virt_to_phys(iommu->root_entry);
1329 if (ecs_enabled(iommu))
1330 addr |= DMA_RTADDR_RTT;
1332 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1333 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1335 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1337 /* Make sure hardware complete it */
1338 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1339 readl, (sts & DMA_GSTS_RTPS), sts);
1341 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1344 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1349 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1352 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1353 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1355 /* Make sure hardware complete it */
1356 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1357 readl, (!(val & DMA_GSTS_WBFS)), val);
1359 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1362 /* return value determine if we need a write buffer flush */
1363 static void __iommu_flush_context(struct intel_iommu *iommu,
1364 u16 did, u16 source_id, u8 function_mask,
1371 case DMA_CCMD_GLOBAL_INVL:
1372 val = DMA_CCMD_GLOBAL_INVL;
1374 case DMA_CCMD_DOMAIN_INVL:
1375 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1377 case DMA_CCMD_DEVICE_INVL:
1378 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1379 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1384 val |= DMA_CCMD_ICC;
1386 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1387 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1389 /* Make sure hardware complete it */
1390 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1391 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1393 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1396 /* return value determine if we need a write buffer flush */
1397 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1398 u64 addr, unsigned int size_order, u64 type)
1400 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1401 u64 val = 0, val_iva = 0;
1405 case DMA_TLB_GLOBAL_FLUSH:
1406 /* global flush doesn't need set IVA_REG */
1407 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1409 case DMA_TLB_DSI_FLUSH:
1410 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1412 case DMA_TLB_PSI_FLUSH:
1413 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1414 /* IH bit is passed in as part of address */
1415 val_iva = size_order | addr;
1420 /* Note: set drain read/write */
1423 * This is probably to be super secure.. Looks like we can
1424 * ignore it without any impact.
1426 if (cap_read_drain(iommu->cap))
1427 val |= DMA_TLB_READ_DRAIN;
1429 if (cap_write_drain(iommu->cap))
1430 val |= DMA_TLB_WRITE_DRAIN;
1432 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1433 /* Note: Only uses first TLB reg currently */
1435 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1436 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1438 /* Make sure hardware complete it */
1439 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1440 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1442 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1444 /* check IOTLB invalidation granularity */
1445 if (DMA_TLB_IAIG(val) == 0)
1446 pr_err("Flush IOTLB failed\n");
1447 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1448 pr_debug("TLB flush request %Lx, actual %Lx\n",
1449 (unsigned long long)DMA_TLB_IIRG(type),
1450 (unsigned long long)DMA_TLB_IAIG(val));
1453 static struct device_domain_info *
1454 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1457 struct device_domain_info *info;
1459 assert_spin_locked(&device_domain_lock);
1464 list_for_each_entry(info, &domain->devices, link)
1465 if (info->iommu == iommu && info->bus == bus &&
1466 info->devfn == devfn) {
1467 if (info->ats_supported && info->dev)
1475 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1477 struct pci_dev *pdev;
1479 if (!info || !dev_is_pci(info->dev))
1482 pdev = to_pci_dev(info->dev);
1483 /* For IOMMU that supports device IOTLB throttling (DIT), we assign
1484 * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
1485 * queue depth at PF level. If DIT is not set, PFSID will be treated as
1486 * reserved, which should be set to 0.
1488 if (!ecap_dit(info->iommu->ecap))
1491 struct pci_dev *pf_pdev;
1493 /* pdev will be returned if device is not a vf */
1494 pf_pdev = pci_physfn(pdev);
1495 info->pfsid = PCI_DEVID(pf_pdev->bus->number, pf_pdev->devfn);
1498 #ifdef CONFIG_INTEL_IOMMU_SVM
1499 /* The PCIe spec, in its wisdom, declares that the behaviour of
1500 the device if you enable PASID support after ATS support is
1501 undefined. So always enable PASID support on devices which
1502 have it, even if we can't yet know if we're ever going to
1504 if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1505 info->pasid_enabled = 1;
1507 if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1508 info->pri_enabled = 1;
1510 if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1511 info->ats_enabled = 1;
1512 info->ats_qdep = pci_ats_queue_depth(pdev);
1516 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1518 struct pci_dev *pdev;
1520 if (!dev_is_pci(info->dev))
1523 pdev = to_pci_dev(info->dev);
1525 if (info->ats_enabled) {
1526 pci_disable_ats(pdev);
1527 info->ats_enabled = 0;
1529 #ifdef CONFIG_INTEL_IOMMU_SVM
1530 if (info->pri_enabled) {
1531 pci_disable_pri(pdev);
1532 info->pri_enabled = 0;
1534 if (info->pasid_enabled) {
1535 pci_disable_pasid(pdev);
1536 info->pasid_enabled = 0;
1541 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1542 u64 addr, unsigned mask)
1545 unsigned long flags;
1546 struct device_domain_info *info;
1548 spin_lock_irqsave(&device_domain_lock, flags);
1549 list_for_each_entry(info, &domain->devices, link) {
1550 if (!info->ats_enabled)
1553 sid = info->bus << 8 | info->devfn;
1554 qdep = info->ats_qdep;
1555 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1558 spin_unlock_irqrestore(&device_domain_lock, flags);
1561 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1562 struct dmar_domain *domain,
1563 unsigned long pfn, unsigned int pages,
1566 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1567 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1568 u16 did = domain->iommu_did[iommu->seq_id];
1575 * Fallback to domain selective flush if no PSI support or the size is
1577 * PSI requires page size to be 2 ^ x, and the base address is naturally
1578 * aligned to the size
1580 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1581 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1584 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1588 * In caching mode, changes of pages from non-present to present require
1589 * flush. However, device IOTLB doesn't need to be flushed in this case.
1591 if (!cap_caching_mode(iommu->cap) || !map)
1592 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1596 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1599 unsigned long flags;
1601 if (!cap_plmr(iommu->cap) && !cap_phmr(iommu->cap))
1604 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1605 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1606 pmen &= ~DMA_PMEN_EPM;
1607 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1609 /* wait for the protected region status bit to clear */
1610 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1611 readl, !(pmen & DMA_PMEN_PRS), pmen);
1613 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1616 static void iommu_enable_translation(struct intel_iommu *iommu)
1619 unsigned long flags;
1621 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1622 iommu->gcmd |= DMA_GCMD_TE;
1623 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1625 /* Make sure hardware complete it */
1626 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1627 readl, (sts & DMA_GSTS_TES), sts);
1629 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1632 static void iommu_disable_translation(struct intel_iommu *iommu)
1637 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1638 iommu->gcmd &= ~DMA_GCMD_TE;
1639 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1641 /* Make sure hardware complete it */
1642 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1643 readl, (!(sts & DMA_GSTS_TES)), sts);
1645 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1649 static int iommu_init_domains(struct intel_iommu *iommu)
1651 u32 ndomains, nlongs;
1654 ndomains = cap_ndoms(iommu->cap);
1655 pr_debug("%s: Number of Domains supported <%d>\n",
1656 iommu->name, ndomains);
1657 nlongs = BITS_TO_LONGS(ndomains);
1659 spin_lock_init(&iommu->lock);
1661 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1662 if (!iommu->domain_ids) {
1663 pr_err("%s: Allocating domain id array failed\n",
1668 size = ((ndomains >> 8) + 1) * sizeof(struct dmar_domain **);
1669 iommu->domains = kzalloc(size, GFP_KERNEL);
1671 if (iommu->domains) {
1672 size = 256 * sizeof(struct dmar_domain *);
1673 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1676 if (!iommu->domains || !iommu->domains[0]) {
1677 pr_err("%s: Allocating domain array failed\n",
1679 kfree(iommu->domain_ids);
1680 kfree(iommu->domains);
1681 iommu->domain_ids = NULL;
1682 iommu->domains = NULL;
1689 * If Caching mode is set, then invalid translations are tagged
1690 * with domain-id 0, hence we need to pre-allocate it. We also
1691 * use domain-id 0 as a marker for non-allocated domain-id, so
1692 * make sure it is not used for a real domain.
1694 set_bit(0, iommu->domain_ids);
1699 static void disable_dmar_iommu(struct intel_iommu *iommu)
1701 struct device_domain_info *info, *tmp;
1702 unsigned long flags;
1704 if (!iommu->domains || !iommu->domain_ids)
1708 spin_lock_irqsave(&device_domain_lock, flags);
1709 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1710 struct dmar_domain *domain;
1712 if (info->iommu != iommu)
1715 if (!info->dev || !info->domain)
1718 domain = info->domain;
1720 __dmar_remove_one_dev_info(info);
1722 if (!domain_type_is_vm_or_si(domain)) {
1724 * The domain_exit() function can't be called under
1725 * device_domain_lock, as it takes this lock itself.
1726 * So release the lock here and re-run the loop
1729 spin_unlock_irqrestore(&device_domain_lock, flags);
1730 domain_exit(domain);
1734 spin_unlock_irqrestore(&device_domain_lock, flags);
1736 if (iommu->gcmd & DMA_GCMD_TE)
1737 iommu_disable_translation(iommu);
1740 static void free_dmar_iommu(struct intel_iommu *iommu)
1742 if ((iommu->domains) && (iommu->domain_ids)) {
1743 int elems = (cap_ndoms(iommu->cap) >> 8) + 1;
1746 for (i = 0; i < elems; i++)
1747 kfree(iommu->domains[i]);
1748 kfree(iommu->domains);
1749 kfree(iommu->domain_ids);
1750 iommu->domains = NULL;
1751 iommu->domain_ids = NULL;
1754 g_iommus[iommu->seq_id] = NULL;
1756 /* free context mapping */
1757 free_context_table(iommu);
1759 #ifdef CONFIG_INTEL_IOMMU_SVM
1760 if (pasid_enabled(iommu)) {
1761 if (ecap_prs(iommu->ecap))
1762 intel_svm_finish_prq(iommu);
1763 intel_svm_free_pasid_tables(iommu);
1768 static struct dmar_domain *alloc_domain(int flags)
1770 struct dmar_domain *domain;
1772 domain = alloc_domain_mem();
1776 memset(domain, 0, sizeof(*domain));
1778 domain->flags = flags;
1779 INIT_LIST_HEAD(&domain->devices);
1784 /* Must be called with iommu->lock */
1785 static int domain_attach_iommu(struct dmar_domain *domain,
1786 struct intel_iommu *iommu)
1788 unsigned long ndomains;
1791 assert_spin_locked(&device_domain_lock);
1792 assert_spin_locked(&iommu->lock);
1794 domain->iommu_refcnt[iommu->seq_id] += 1;
1795 domain->iommu_count += 1;
1796 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1797 ndomains = cap_ndoms(iommu->cap);
1798 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1800 if (num >= ndomains) {
1801 pr_err("%s: No free domain ids\n", iommu->name);
1802 domain->iommu_refcnt[iommu->seq_id] -= 1;
1803 domain->iommu_count -= 1;
1807 set_bit(num, iommu->domain_ids);
1808 set_iommu_domain(iommu, num, domain);
1810 domain->iommu_did[iommu->seq_id] = num;
1811 domain->nid = iommu->node;
1813 domain_update_iommu_cap(domain);
1819 static int domain_detach_iommu(struct dmar_domain *domain,
1820 struct intel_iommu *iommu)
1822 int num, count = INT_MAX;
1824 assert_spin_locked(&device_domain_lock);
1825 assert_spin_locked(&iommu->lock);
1827 domain->iommu_refcnt[iommu->seq_id] -= 1;
1828 count = --domain->iommu_count;
1829 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1830 num = domain->iommu_did[iommu->seq_id];
1831 clear_bit(num, iommu->domain_ids);
1832 set_iommu_domain(iommu, num, NULL);
1834 domain_update_iommu_cap(domain);
1835 domain->iommu_did[iommu->seq_id] = 0;
1841 static struct iova_domain reserved_iova_list;
1842 static struct lock_class_key reserved_rbtree_key;
1844 static int dmar_init_reserved_ranges(void)
1846 struct pci_dev *pdev = NULL;
1850 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1853 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1854 &reserved_rbtree_key);
1856 /* IOAPIC ranges shouldn't be accessed by DMA */
1857 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1858 IOVA_PFN(IOAPIC_RANGE_END));
1860 pr_err("Reserve IOAPIC range failed\n");
1864 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1865 for_each_pci_dev(pdev) {
1868 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1869 r = &pdev->resource[i];
1870 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1872 iova = reserve_iova(&reserved_iova_list,
1876 pr_err("Reserve iova failed\n");
1884 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1886 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1889 static inline int guestwidth_to_adjustwidth(int gaw)
1892 int r = (gaw - 12) % 9;
1903 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1906 int adjust_width, agaw;
1907 unsigned long sagaw;
1909 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1911 domain_reserve_special_ranges(domain);
1913 /* calculate AGAW */
1914 if (guest_width > cap_mgaw(iommu->cap))
1915 guest_width = cap_mgaw(iommu->cap);
1916 domain->gaw = guest_width;
1917 adjust_width = guestwidth_to_adjustwidth(guest_width);
1918 agaw = width_to_agaw(adjust_width);
1919 sagaw = cap_sagaw(iommu->cap);
1920 if (!test_bit(agaw, &sagaw)) {
1921 /* hardware doesn't support it, choose a bigger one */
1922 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1923 agaw = find_next_bit(&sagaw, 5, agaw);
1927 domain->agaw = agaw;
1929 if (ecap_coherent(iommu->ecap))
1930 domain->iommu_coherency = 1;
1932 domain->iommu_coherency = 0;
1934 if (ecap_sc_support(iommu->ecap))
1935 domain->iommu_snooping = 1;
1937 domain->iommu_snooping = 0;
1939 if (intel_iommu_superpage)
1940 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1942 domain->iommu_superpage = 0;
1944 domain->nid = iommu->node;
1946 /* always allocate the top pgd */
1947 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1950 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1954 static void domain_exit(struct dmar_domain *domain)
1956 struct page *freelist = NULL;
1958 /* Domain 0 is reserved, so dont process it */
1962 /* Flush any lazy unmaps that may reference this domain */
1963 if (!intel_iommu_strict)
1964 flush_unmaps_timeout(0);
1966 /* Remove associated devices and clear attached or cached domains */
1968 domain_remove_dev_info(domain);
1972 put_iova_domain(&domain->iovad);
1974 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1976 dma_free_pagelist(freelist);
1978 free_domain_mem(domain);
1981 static int domain_context_mapping_one(struct dmar_domain *domain,
1982 struct intel_iommu *iommu,
1985 u16 did = domain->iommu_did[iommu->seq_id];
1986 int translation = CONTEXT_TT_MULTI_LEVEL;
1987 struct device_domain_info *info = NULL;
1988 struct context_entry *context;
1989 unsigned long flags;
1990 struct dma_pte *pgd;
1995 if (hw_pass_through && domain_type_is_si(domain))
1996 translation = CONTEXT_TT_PASS_THROUGH;
1998 pr_debug("Set context mapping for %02x:%02x.%d\n",
1999 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
2001 BUG_ON(!domain->pgd);
2003 spin_lock_irqsave(&device_domain_lock, flags);
2004 spin_lock(&iommu->lock);
2007 context = iommu_context_addr(iommu, bus, devfn, 1);
2012 if (context_present(context))
2016 * For kdump cases, old valid entries may be cached due to the
2017 * in-flight DMA and copied pgtable, but there is no unmapping
2018 * behaviour for them, thus we need an explicit cache flush for
2019 * the newly-mapped device. For kdump, at this point, the device
2020 * is supposed to finish reset at its driver probe stage, so no
2021 * in-flight DMA will exist, and we don't need to worry anymore
2024 if (context_copied(context)) {
2025 u16 did_old = context_domain_id(context);
2027 if (did_old >= 0 && did_old < cap_ndoms(iommu->cap)) {
2028 iommu->flush.flush_context(iommu, did_old,
2029 (((u16)bus) << 8) | devfn,
2030 DMA_CCMD_MASK_NOBIT,
2031 DMA_CCMD_DEVICE_INVL);
2032 iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
2039 context_clear_entry(context);
2040 context_set_domain_id(context, did);
2043 * Skip top levels of page tables for iommu which has less agaw
2044 * than default. Unnecessary for PT mode.
2046 if (translation != CONTEXT_TT_PASS_THROUGH) {
2047 for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
2049 pgd = phys_to_virt(dma_pte_addr(pgd));
2050 if (!dma_pte_present(pgd))
2054 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2055 if (info && info->ats_supported)
2056 translation = CONTEXT_TT_DEV_IOTLB;
2058 translation = CONTEXT_TT_MULTI_LEVEL;
2060 context_set_address_root(context, virt_to_phys(pgd));
2061 context_set_address_width(context, agaw);
2064 * In pass through mode, AW must be programmed to
2065 * indicate the largest AGAW value supported by
2066 * hardware. And ASR is ignored by hardware.
2068 context_set_address_width(context, iommu->msagaw);
2071 context_set_translation_type(context, translation);
2072 context_set_fault_enable(context);
2073 context_set_present(context);
2074 domain_flush_cache(domain, context, sizeof(*context));
2077 * It's a non-present to present mapping. If hardware doesn't cache
2078 * non-present entry we only need to flush the write-buffer. If the
2079 * _does_ cache non-present entries, then it does so in the special
2080 * domain #0, which we have to flush:
2082 if (cap_caching_mode(iommu->cap)) {
2083 iommu->flush.flush_context(iommu, 0,
2084 (((u16)bus) << 8) | devfn,
2085 DMA_CCMD_MASK_NOBIT,
2086 DMA_CCMD_DEVICE_INVL);
2087 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2089 iommu_flush_write_buffer(iommu);
2091 iommu_enable_dev_iotlb(info);
2096 spin_unlock(&iommu->lock);
2097 spin_unlock_irqrestore(&device_domain_lock, flags);
2102 struct domain_context_mapping_data {
2103 struct dmar_domain *domain;
2104 struct intel_iommu *iommu;
2107 static int domain_context_mapping_cb(struct pci_dev *pdev,
2108 u16 alias, void *opaque)
2110 struct domain_context_mapping_data *data = opaque;
2112 return domain_context_mapping_one(data->domain, data->iommu,
2113 PCI_BUS_NUM(alias), alias & 0xff);
2117 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2119 struct intel_iommu *iommu;
2121 struct domain_context_mapping_data data;
2123 iommu = device_to_iommu(dev, &bus, &devfn);
2127 if (!dev_is_pci(dev))
2128 return domain_context_mapping_one(domain, iommu, bus, devfn);
2130 data.domain = domain;
2133 return pci_for_each_dma_alias(to_pci_dev(dev),
2134 &domain_context_mapping_cb, &data);
2137 static int domain_context_mapped_cb(struct pci_dev *pdev,
2138 u16 alias, void *opaque)
2140 struct intel_iommu *iommu = opaque;
2142 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2145 static int domain_context_mapped(struct device *dev)
2147 struct intel_iommu *iommu;
2150 iommu = device_to_iommu(dev, &bus, &devfn);
2154 if (!dev_is_pci(dev))
2155 return device_context_mapped(iommu, bus, devfn);
2157 return !pci_for_each_dma_alias(to_pci_dev(dev),
2158 domain_context_mapped_cb, iommu);
2161 /* Returns a number of VTD pages, but aligned to MM page size */
2162 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2165 host_addr &= ~PAGE_MASK;
2166 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2169 /* Return largest possible superpage level for a given mapping */
2170 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2171 unsigned long iov_pfn,
2172 unsigned long phy_pfn,
2173 unsigned long pages)
2175 int support, level = 1;
2176 unsigned long pfnmerge;
2178 support = domain->iommu_superpage;
2180 /* To use a large page, the virtual *and* physical addresses
2181 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2182 of them will mean we have to use smaller pages. So just
2183 merge them and check both at once. */
2184 pfnmerge = iov_pfn | phy_pfn;
2186 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2187 pages >>= VTD_STRIDE_SHIFT;
2190 pfnmerge >>= VTD_STRIDE_SHIFT;
2197 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2198 struct scatterlist *sg, unsigned long phys_pfn,
2199 unsigned long nr_pages, int prot)
2201 struct dma_pte *first_pte = NULL, *pte = NULL;
2202 phys_addr_t uninitialized_var(pteval);
2203 unsigned long sg_res = 0;
2204 unsigned int largepage_lvl = 0;
2205 unsigned long lvl_pages = 0;
2207 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2209 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2212 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2216 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2219 while (nr_pages > 0) {
2223 unsigned int pgoff = sg->offset & ~PAGE_MASK;
2225 sg_res = aligned_nrpages(sg->offset, sg->length);
2226 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
2227 sg->dma_length = sg->length;
2228 pteval = (sg_phys(sg) - pgoff) | prot;
2229 phys_pfn = pteval >> VTD_PAGE_SHIFT;
2233 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2235 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2238 /* It is large page*/
2239 if (largepage_lvl > 1) {
2240 unsigned long nr_superpages, end_pfn;
2242 pteval |= DMA_PTE_LARGE_PAGE;
2243 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2245 nr_superpages = sg_res / lvl_pages;
2246 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2249 * Ensure that old small page tables are
2250 * removed to make room for superpage(s).
2252 dma_pte_free_pagetable(domain, iov_pfn, end_pfn);
2254 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2258 /* We don't need lock here, nobody else
2259 * touches the iova range
2261 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2263 static int dumps = 5;
2264 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2265 iov_pfn, tmp, (unsigned long long)pteval);
2268 debug_dma_dump_mappings(NULL);
2273 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2275 BUG_ON(nr_pages < lvl_pages);
2276 BUG_ON(sg_res < lvl_pages);
2278 nr_pages -= lvl_pages;
2279 iov_pfn += lvl_pages;
2280 phys_pfn += lvl_pages;
2281 pteval += lvl_pages * VTD_PAGE_SIZE;
2282 sg_res -= lvl_pages;
2284 /* If the next PTE would be the first in a new page, then we
2285 need to flush the cache on the entries we've just written.
2286 And then we'll need to recalculate 'pte', so clear it and
2287 let it get set again in the if (!pte) block above.
2289 If we're done (!nr_pages) we need to flush the cache too.
2291 Also if we've been setting superpages, we may need to
2292 recalculate 'pte' and switch back to smaller pages for the
2293 end of the mapping, if the trailing size is not enough to
2294 use another superpage (i.e. sg_res < lvl_pages). */
2296 if (!nr_pages || first_pte_in_page(pte) ||
2297 (largepage_lvl > 1 && sg_res < lvl_pages)) {
2298 domain_flush_cache(domain, first_pte,
2299 (void *)pte - (void *)first_pte);
2303 if (!sg_res && nr_pages)
2309 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2310 struct scatterlist *sg, unsigned long nr_pages,
2313 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2316 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2317 unsigned long phys_pfn, unsigned long nr_pages,
2320 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2323 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2328 clear_context_table(iommu, bus, devfn);
2329 iommu->flush.flush_context(iommu, 0, 0, 0,
2330 DMA_CCMD_GLOBAL_INVL);
2331 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2334 static inline void unlink_domain_info(struct device_domain_info *info)
2336 assert_spin_locked(&device_domain_lock);
2337 list_del(&info->link);
2338 list_del(&info->global);
2340 info->dev->archdata.iommu = NULL;
2343 static void domain_remove_dev_info(struct dmar_domain *domain)
2345 struct device_domain_info *info, *tmp;
2346 unsigned long flags;
2348 spin_lock_irqsave(&device_domain_lock, flags);
2349 list_for_each_entry_safe(info, tmp, &domain->devices, link)
2350 __dmar_remove_one_dev_info(info);
2351 spin_unlock_irqrestore(&device_domain_lock, flags);
2356 * Note: we use struct device->archdata.iommu stores the info
2358 static struct dmar_domain *find_domain(struct device *dev)
2360 struct device_domain_info *info;
2362 /* No lock here, assumes no domain exit in normal case */
2363 info = dev->archdata.iommu;
2365 return info->domain;
2369 static inline struct device_domain_info *
2370 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2372 struct device_domain_info *info;
2374 list_for_each_entry(info, &device_domain_list, global)
2375 if (info->iommu->segment == segment && info->bus == bus &&
2376 info->devfn == devfn)
2382 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2385 struct dmar_domain *domain)
2387 struct dmar_domain *found = NULL;
2388 struct device_domain_info *info;
2389 unsigned long flags;
2392 info = alloc_devinfo_mem();
2397 info->devfn = devfn;
2398 info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2399 info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2402 info->domain = domain;
2403 info->iommu = iommu;
2405 if (dev && dev_is_pci(dev)) {
2406 struct pci_dev *pdev = to_pci_dev(info->dev);
2408 if (ecap_dev_iotlb_support(iommu->ecap) &&
2409 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2410 dmar_find_matched_atsr_unit(pdev))
2411 info->ats_supported = 1;
2413 if (ecs_enabled(iommu)) {
2414 if (pasid_enabled(iommu)) {
2415 int features = pci_pasid_features(pdev);
2417 info->pasid_supported = features | 1;
2420 if (info->ats_supported && ecap_prs(iommu->ecap) &&
2421 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2422 info->pri_supported = 1;
2426 spin_lock_irqsave(&device_domain_lock, flags);
2428 found = find_domain(dev);
2431 struct device_domain_info *info2;
2432 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2434 found = info2->domain;
2440 spin_unlock_irqrestore(&device_domain_lock, flags);
2441 free_devinfo_mem(info);
2442 /* Caller must free the original domain */
2446 spin_lock(&iommu->lock);
2447 ret = domain_attach_iommu(domain, iommu);
2448 spin_unlock(&iommu->lock);
2451 spin_unlock_irqrestore(&device_domain_lock, flags);
2452 free_devinfo_mem(info);
2456 list_add(&info->link, &domain->devices);
2457 list_add(&info->global, &device_domain_list);
2459 dev->archdata.iommu = info;
2460 spin_unlock_irqrestore(&device_domain_lock, flags);
2462 if (dev && domain_context_mapping(domain, dev)) {
2463 pr_err("Domain context map for %s failed\n", dev_name(dev));
2464 dmar_remove_one_dev_info(domain, dev);
2471 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2473 *(u16 *)opaque = alias;
2477 /* domain is initialized */
2478 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2480 struct device_domain_info *info = NULL;
2481 struct dmar_domain *domain, *tmp;
2482 struct intel_iommu *iommu;
2483 u16 req_id, dma_alias;
2484 unsigned long flags;
2487 domain = find_domain(dev);
2491 iommu = device_to_iommu(dev, &bus, &devfn);
2495 req_id = ((u16)bus << 8) | devfn;
2497 if (dev_is_pci(dev)) {
2498 struct pci_dev *pdev = to_pci_dev(dev);
2500 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2502 spin_lock_irqsave(&device_domain_lock, flags);
2503 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2504 PCI_BUS_NUM(dma_alias),
2507 iommu = info->iommu;
2508 domain = info->domain;
2510 spin_unlock_irqrestore(&device_domain_lock, flags);
2512 /* DMA alias already has a domain, uses it */
2517 /* Allocate and initialize new domain for the device */
2518 domain = alloc_domain(0);
2521 if (domain_init(domain, iommu, gaw)) {
2522 domain_exit(domain);
2526 /* register PCI DMA alias device */
2527 if (req_id != dma_alias && dev_is_pci(dev)) {
2528 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2529 dma_alias & 0xff, NULL, domain);
2531 if (!tmp || tmp != domain) {
2532 domain_exit(domain);
2541 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2543 if (!tmp || tmp != domain) {
2544 domain_exit(domain);
2551 static int iommu_domain_identity_map(struct dmar_domain *domain,
2552 unsigned long long start,
2553 unsigned long long end)
2555 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2556 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2558 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2559 dma_to_mm_pfn(last_vpfn))) {
2560 pr_err("Reserving iova failed\n");
2564 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2566 * RMRR range might have overlap with physical memory range,
2569 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2571 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2572 last_vpfn - first_vpfn + 1,
2573 DMA_PTE_READ|DMA_PTE_WRITE);
2576 static int domain_prepare_identity_map(struct device *dev,
2577 struct dmar_domain *domain,
2578 unsigned long long start,
2579 unsigned long long end)
2581 /* For _hardware_ passthrough, don't bother. But for software
2582 passthrough, we do it anyway -- it may indicate a memory
2583 range which is reserved in E820, so which didn't get set
2584 up to start with in si_domain */
2585 if (domain == si_domain && hw_pass_through) {
2586 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2587 dev_name(dev), start, end);
2591 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2592 dev_name(dev), start, end);
2595 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2596 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2597 dmi_get_system_info(DMI_BIOS_VENDOR),
2598 dmi_get_system_info(DMI_BIOS_VERSION),
2599 dmi_get_system_info(DMI_PRODUCT_VERSION));
2603 if (end >> agaw_to_width(domain->agaw)) {
2604 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2605 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2606 agaw_to_width(domain->agaw),
2607 dmi_get_system_info(DMI_BIOS_VENDOR),
2608 dmi_get_system_info(DMI_BIOS_VERSION),
2609 dmi_get_system_info(DMI_PRODUCT_VERSION));
2613 return iommu_domain_identity_map(domain, start, end);
2616 static int iommu_prepare_identity_map(struct device *dev,
2617 unsigned long long start,
2618 unsigned long long end)
2620 struct dmar_domain *domain;
2623 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2627 ret = domain_prepare_identity_map(dev, domain, start, end);
2629 domain_exit(domain);
2634 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2637 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2639 return iommu_prepare_identity_map(dev, rmrr->base_address,
2643 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2644 static inline void iommu_prepare_isa(void)
2646 struct pci_dev *pdev;
2649 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2653 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2654 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2657 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2662 static inline void iommu_prepare_isa(void)
2666 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2668 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2670 static int __init si_domain_init(int hw)
2674 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2678 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2679 domain_exit(si_domain);
2683 pr_debug("Identity mapping domain allocated\n");
2688 for_each_online_node(nid) {
2689 unsigned long start_pfn, end_pfn;
2692 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2693 ret = iommu_domain_identity_map(si_domain,
2694 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2703 static int identity_mapping(struct device *dev)
2705 struct device_domain_info *info;
2707 if (likely(!iommu_identity_mapping))
2710 info = dev->archdata.iommu;
2711 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2712 return (info->domain == si_domain);
2717 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2719 struct dmar_domain *ndomain;
2720 struct intel_iommu *iommu;
2723 iommu = device_to_iommu(dev, &bus, &devfn);
2727 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2728 if (ndomain != domain)
2734 static bool device_has_rmrr(struct device *dev)
2736 struct dmar_rmrr_unit *rmrr;
2741 for_each_rmrr_units(rmrr) {
2743 * Return TRUE if this RMRR contains the device that
2746 for_each_active_dev_scope(rmrr->devices,
2747 rmrr->devices_cnt, i, tmp)
2758 * There are a couple cases where we need to restrict the functionality of
2759 * devices associated with RMRRs. The first is when evaluating a device for
2760 * identity mapping because problems exist when devices are moved in and out
2761 * of domains and their respective RMRR information is lost. This means that
2762 * a device with associated RMRRs will never be in a "passthrough" domain.
2763 * The second is use of the device through the IOMMU API. This interface
2764 * expects to have full control of the IOVA space for the device. We cannot
2765 * satisfy both the requirement that RMRR access is maintained and have an
2766 * unencumbered IOVA space. We also have no ability to quiesce the device's
2767 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2768 * We therefore prevent devices associated with an RMRR from participating in
2769 * the IOMMU API, which eliminates them from device assignment.
2771 * In both cases we assume that PCI USB devices with RMRRs have them largely
2772 * for historical reasons and that the RMRR space is not actively used post
2773 * boot. This exclusion may change if vendors begin to abuse it.
2775 * The same exception is made for graphics devices, with the requirement that
2776 * any use of the RMRR regions will be torn down before assigning the device
2779 static bool device_is_rmrr_locked(struct device *dev)
2781 if (!device_has_rmrr(dev))
2784 if (dev_is_pci(dev)) {
2785 struct pci_dev *pdev = to_pci_dev(dev);
2787 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2794 static int iommu_should_identity_map(struct device *dev, int startup)
2797 if (dev_is_pci(dev)) {
2798 struct pci_dev *pdev = to_pci_dev(dev);
2800 if (device_is_rmrr_locked(dev))
2803 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2806 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2809 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2813 * We want to start off with all devices in the 1:1 domain, and
2814 * take them out later if we find they can't access all of memory.
2816 * However, we can't do this for PCI devices behind bridges,
2817 * because all PCI devices behind the same bridge will end up
2818 * with the same source-id on their transactions.
2820 * Practically speaking, we can't change things around for these
2821 * devices at run-time, because we can't be sure there'll be no
2822 * DMA transactions in flight for any of their siblings.
2824 * So PCI devices (unless they're on the root bus) as well as
2825 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2826 * the 1:1 domain, just in _case_ one of their siblings turns out
2827 * not to be able to map all of memory.
2829 if (!pci_is_pcie(pdev)) {
2830 if (!pci_is_root_bus(pdev->bus))
2832 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2834 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2837 if (device_has_rmrr(dev))
2842 * At boot time, we don't yet know if devices will be 64-bit capable.
2843 * Assume that they will — if they turn out not to be, then we can
2844 * take them out of the 1:1 domain later.
2848 * If the device's dma_mask is less than the system's memory
2849 * size then this is not a candidate for identity mapping.
2851 u64 dma_mask = *dev->dma_mask;
2853 if (dev->coherent_dma_mask &&
2854 dev->coherent_dma_mask < dma_mask)
2855 dma_mask = dev->coherent_dma_mask;
2857 return dma_mask >= dma_get_required_mask(dev);
2863 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2867 if (!iommu_should_identity_map(dev, 1))
2870 ret = domain_add_dev_info(si_domain, dev);
2872 pr_info("%s identity mapping for device %s\n",
2873 hw ? "Hardware" : "Software", dev_name(dev));
2874 else if (ret == -ENODEV)
2875 /* device not associated with an iommu */
2882 static int __init iommu_prepare_static_identity_mapping(int hw)
2884 struct pci_dev *pdev = NULL;
2885 struct dmar_drhd_unit *drhd;
2886 struct intel_iommu *iommu;
2891 for_each_pci_dev(pdev) {
2892 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2897 for_each_active_iommu(iommu, drhd)
2898 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2899 struct acpi_device_physical_node *pn;
2900 struct acpi_device *adev;
2902 if (dev->bus != &acpi_bus_type)
2905 adev= to_acpi_device(dev);
2906 mutex_lock(&adev->physical_node_lock);
2907 list_for_each_entry(pn, &adev->physical_node_list, node) {
2908 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2912 mutex_unlock(&adev->physical_node_lock);
2920 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2923 * Start from the sane iommu hardware state.
2924 * If the queued invalidation is already initialized by us
2925 * (for example, while enabling interrupt-remapping) then
2926 * we got the things already rolling from a sane state.
2930 * Clear any previous faults.
2932 dmar_fault(-1, iommu);
2934 * Disable queued invalidation if supported and already enabled
2935 * before OS handover.
2937 dmar_disable_qi(iommu);
2940 if (dmar_enable_qi(iommu)) {
2942 * Queued Invalidate not enabled, use Register Based Invalidate
2944 iommu->flush.flush_context = __iommu_flush_context;
2945 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2946 pr_info("%s: Using Register based invalidation\n",
2949 iommu->flush.flush_context = qi_flush_context;
2950 iommu->flush.flush_iotlb = qi_flush_iotlb;
2951 pr_info("%s: Using Queued invalidation\n", iommu->name);
2955 static int copy_context_table(struct intel_iommu *iommu,
2956 struct root_entry *old_re,
2957 struct context_entry **tbl,
2960 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
2961 struct context_entry *new_ce = NULL, ce;
2962 struct context_entry *old_ce = NULL;
2963 struct root_entry re;
2964 phys_addr_t old_ce_phys;
2966 tbl_idx = ext ? bus * 2 : bus;
2967 memcpy(&re, old_re, sizeof(re));
2969 for (devfn = 0; devfn < 256; devfn++) {
2970 /* First calculate the correct index */
2971 idx = (ext ? devfn * 2 : devfn) % 256;
2974 /* First save what we may have and clean up */
2976 tbl[tbl_idx] = new_ce;
2977 __iommu_flush_cache(iommu, new_ce,
2987 old_ce_phys = root_entry_lctp(&re);
2989 old_ce_phys = root_entry_uctp(&re);
2992 if (ext && devfn == 0) {
2993 /* No LCTP, try UCTP */
3002 old_ce = memremap(old_ce_phys, PAGE_SIZE,
3007 new_ce = alloc_pgtable_page(iommu->node);
3014 /* Now copy the context entry */
3015 memcpy(&ce, old_ce + idx, sizeof(ce));
3017 if (!__context_present(&ce))
3020 did = context_domain_id(&ce);
3021 if (did >= 0 && did < cap_ndoms(iommu->cap))
3022 set_bit(did, iommu->domain_ids);
3025 * We need a marker for copied context entries. This
3026 * marker needs to work for the old format as well as
3027 * for extended context entries.
3029 * Bit 67 of the context entry is used. In the old
3030 * format this bit is available to software, in the
3031 * extended format it is the PGE bit, but PGE is ignored
3032 * by HW if PASIDs are disabled (and thus still
3035 * So disable PASIDs first and then mark the entry
3036 * copied. This means that we don't copy PASID
3037 * translations from the old kernel, but this is fine as
3038 * faults there are not fatal.
3040 context_clear_pasid_enable(&ce);
3041 context_set_copied(&ce);
3046 tbl[tbl_idx + pos] = new_ce;
3048 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3057 static int copy_translation_tables(struct intel_iommu *iommu)
3059 struct context_entry **ctxt_tbls;
3060 struct root_entry *old_rt;
3061 phys_addr_t old_rt_phys;
3062 int ctxt_table_entries;
3063 unsigned long flags;
3068 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3069 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
3070 new_ext = !!ecap_ecs(iommu->ecap);
3073 * The RTT bit can only be changed when translation is disabled,
3074 * but disabling translation means to open a window for data
3075 * corruption. So bail out and don't copy anything if we would
3076 * have to change the bit.
3081 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3085 old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3089 /* This is too big for the stack - allocate it from slab */
3090 ctxt_table_entries = ext ? 512 : 256;
3092 ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
3096 for (bus = 0; bus < 256; bus++) {
3097 ret = copy_context_table(iommu, &old_rt[bus],
3098 ctxt_tbls, bus, ext);
3100 pr_err("%s: Failed to copy context table for bus %d\n",
3106 spin_lock_irqsave(&iommu->lock, flags);
3108 /* Context tables are copied, now write them to the root_entry table */
3109 for (bus = 0; bus < 256; bus++) {
3110 int idx = ext ? bus * 2 : bus;
3113 if (ctxt_tbls[idx]) {
3114 val = virt_to_phys(ctxt_tbls[idx]) | 1;
3115 iommu->root_entry[bus].lo = val;
3118 if (!ext || !ctxt_tbls[idx + 1])
3121 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3122 iommu->root_entry[bus].hi = val;
3125 spin_unlock_irqrestore(&iommu->lock, flags);
3129 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3139 static int __init init_dmars(void)
3141 struct dmar_drhd_unit *drhd;
3142 struct dmar_rmrr_unit *rmrr;
3143 bool copied_tables = false;
3145 struct intel_iommu *iommu;
3151 * initialize and program root entry to not present
3154 for_each_drhd_unit(drhd) {
3156 * lock not needed as this is only incremented in the single
3157 * threaded kernel __init code path all other access are read
3160 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3164 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3167 /* Preallocate enough resources for IOMMU hot-addition */
3168 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3169 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3171 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3174 pr_err("Allocating global iommu array failed\n");
3179 deferred_flush = kzalloc(g_num_of_iommus *
3180 sizeof(struct deferred_flush_tables), GFP_KERNEL);
3181 if (!deferred_flush) {
3186 for_each_active_iommu(iommu, drhd) {
3187 g_iommus[iommu->seq_id] = iommu;
3189 intel_iommu_init_qi(iommu);
3191 ret = iommu_init_domains(iommu);
3195 init_translation_status(iommu);
3197 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3198 iommu_disable_translation(iommu);
3199 clear_translation_pre_enabled(iommu);
3200 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3206 * we could share the same root & context tables
3207 * among all IOMMU's. Need to Split it later.
3209 ret = iommu_alloc_root_entry(iommu);
3213 if (translation_pre_enabled(iommu)) {
3214 pr_info("Translation already enabled - trying to copy translation structures\n");
3216 ret = copy_translation_tables(iommu);
3219 * We found the IOMMU with translation
3220 * enabled - but failed to copy over the
3221 * old root-entry table. Try to proceed
3222 * by disabling translation now and
3223 * allocating a clean root-entry table.
3224 * This might cause DMAR faults, but
3225 * probably the dump will still succeed.
3227 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3229 iommu_disable_translation(iommu);
3230 clear_translation_pre_enabled(iommu);
3232 pr_info("Copied translation tables from previous kernel for %s\n",
3234 copied_tables = true;
3238 if (!ecap_pass_through(iommu->ecap))
3239 hw_pass_through = 0;
3240 #ifdef CONFIG_INTEL_IOMMU_SVM
3241 if (pasid_enabled(iommu))
3242 intel_svm_alloc_pasid_tables(iommu);
3247 * Now that qi is enabled on all iommus, set the root entry and flush
3248 * caches. This is required on some Intel X58 chipsets, otherwise the
3249 * flush_context function will loop forever and the boot hangs.
3251 for_each_active_iommu(iommu, drhd) {
3252 iommu_flush_write_buffer(iommu);
3253 iommu_set_root_entry(iommu);
3254 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3255 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3258 if (iommu_pass_through)
3259 iommu_identity_mapping |= IDENTMAP_ALL;
3261 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3266 iommu_identity_mapping |= IDENTMAP_GFX;
3268 check_tylersburg_isoch();
3270 if (iommu_identity_mapping) {
3271 ret = si_domain_init(hw_pass_through);
3278 * If we copied translations from a previous kernel in the kdump
3279 * case, we can not assign the devices to domains now, as that
3280 * would eliminate the old mappings. So skip this part and defer
3281 * the assignment to device driver initialization time.
3287 * If pass through is not set or not enabled, setup context entries for
3288 * identity mappings for rmrr, gfx, and isa and may fall back to static
3289 * identity mapping if iommu_identity_mapping is set.
3291 if (iommu_identity_mapping) {
3292 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3294 pr_crit("Failed to setup IOMMU pass-through\n");
3300 * for each dev attached to rmrr
3302 * locate drhd for dev, alloc domain for dev
3303 * allocate free domain
3304 * allocate page table entries for rmrr
3305 * if context not allocated for bus
3306 * allocate and init context
3307 * set present in root table for this bus
3308 * init context with domain, translation etc
3312 pr_info("Setting RMRR:\n");
3313 for_each_rmrr_units(rmrr) {
3314 /* some BIOS lists non-exist devices in DMAR table. */
3315 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3317 ret = iommu_prepare_rmrr_dev(rmrr, dev);
3319 pr_err("Mapping reserved region failed\n");
3323 iommu_prepare_isa();
3330 * global invalidate context cache
3331 * global invalidate iotlb
3332 * enable translation
3334 for_each_iommu(iommu, drhd) {
3335 if (drhd->ignored) {
3337 * we always have to disable PMRs or DMA may fail on
3341 iommu_disable_protect_mem_regions(iommu);
3345 iommu_flush_write_buffer(iommu);
3347 #ifdef CONFIG_INTEL_IOMMU_SVM
3348 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3349 ret = intel_svm_enable_prq(iommu);
3354 ret = dmar_set_interrupt(iommu);
3358 if (!translation_pre_enabled(iommu))
3359 iommu_enable_translation(iommu);
3361 iommu_disable_protect_mem_regions(iommu);
3367 for_each_active_iommu(iommu, drhd) {
3368 disable_dmar_iommu(iommu);
3369 free_dmar_iommu(iommu);
3371 kfree(deferred_flush);
3378 /* This takes a number of _MM_ pages, not VTD pages */
3379 static struct iova *intel_alloc_iova(struct device *dev,
3380 struct dmar_domain *domain,
3381 unsigned long nrpages, uint64_t dma_mask)
3383 struct iova *iova = NULL;
3385 /* Restrict dma_mask to the width that the iommu can handle */
3386 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3387 /* Ensure we reserve the whole size-aligned region */
3388 nrpages = __roundup_pow_of_two(nrpages);
3390 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3392 * First try to allocate an io virtual address in
3393 * DMA_BIT_MASK(32) and if that fails then try allocating
3396 iova = alloc_iova(&domain->iovad, nrpages,
3397 IOVA_PFN(DMA_BIT_MASK(32)), 1);
3401 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
3402 if (unlikely(!iova)) {
3403 pr_err("Allocating %ld-page iova for %s failed",
3404 nrpages, dev_name(dev));
3411 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
3413 struct dmar_rmrr_unit *rmrr;
3414 struct dmar_domain *domain;
3415 struct device *i_dev;
3418 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3420 pr_err("Allocating domain for %s failed\n",
3425 /* We have a new domain - setup possible RMRRs for the device */
3427 for_each_rmrr_units(rmrr) {
3428 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3433 ret = domain_prepare_identity_map(dev, domain,
3437 dev_err(dev, "Mapping reserved region failed\n");
3445 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3447 struct device_domain_info *info;
3449 /* No lock here, assumes no domain exit in normal case */
3450 info = dev->archdata.iommu;
3452 return info->domain;
3454 return __get_valid_domain_for_dev(dev);
3457 /* Check if the dev needs to go through non-identity map and unmap process.*/
3458 static int iommu_no_mapping(struct device *dev)
3462 if (iommu_dummy(dev))
3465 if (!iommu_identity_mapping)
3468 found = identity_mapping(dev);
3470 if (iommu_should_identity_map(dev, 0))
3474 * 32 bit DMA is removed from si_domain and fall back
3475 * to non-identity mapping.
3477 dmar_remove_one_dev_info(si_domain, dev);
3478 pr_info("32bit %s uses non-identity mapping\n",
3484 * In case of a detached 64 bit DMA device from vm, the device
3485 * is put into si_domain for identity mapping.
3487 if (iommu_should_identity_map(dev, 0)) {
3489 ret = domain_add_dev_info(si_domain, dev);
3491 pr_info("64bit %s uses identity mapping\n",
3501 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3502 size_t size, int dir, u64 dma_mask)
3504 struct dmar_domain *domain;
3505 phys_addr_t start_paddr;
3509 struct intel_iommu *iommu;
3510 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3512 BUG_ON(dir == DMA_NONE);
3514 if (iommu_no_mapping(dev))
3517 domain = get_valid_domain_for_dev(dev);
3521 iommu = domain_get_iommu(domain);
3522 size = aligned_nrpages(paddr, size);
3524 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3529 * Check if DMAR supports zero-length reads on write only
3532 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3533 !cap_zlr(iommu->cap))
3534 prot |= DMA_PTE_READ;
3535 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3536 prot |= DMA_PTE_WRITE;
3538 * paddr - (paddr + size) might be partial page, we should map the whole
3539 * page. Note: if two part of one page are separately mapped, we
3540 * might have two guest_addr mapping to the same host paddr, but this
3541 * is not a big problem
3543 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
3544 mm_to_dma_pfn(paddr_pfn), size, prot);
3548 /* it's a non-present to present mapping. Only flush if caching mode */
3549 if (cap_caching_mode(iommu->cap))
3550 iommu_flush_iotlb_psi(iommu, domain,
3551 mm_to_dma_pfn(iova->pfn_lo),
3554 iommu_flush_write_buffer(iommu);
3556 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3557 start_paddr += paddr & ~PAGE_MASK;
3562 __free_iova(&domain->iovad, iova);
3563 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3564 dev_name(dev), size, (unsigned long long)paddr, dir);
3568 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3569 unsigned long offset, size_t size,
3570 enum dma_data_direction dir,
3571 struct dma_attrs *attrs)
3573 return __intel_map_single(dev, page_to_phys(page) + offset, size,
3574 dir, *dev->dma_mask);
3577 static void flush_unmaps(void)
3583 /* just flush them all */
3584 for (i = 0; i < g_num_of_iommus; i++) {
3585 struct intel_iommu *iommu = g_iommus[i];
3589 if (!deferred_flush[i].next)
3592 /* In caching mode, global flushes turn emulation expensive */
3593 if (!cap_caching_mode(iommu->cap))
3594 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3595 DMA_TLB_GLOBAL_FLUSH);
3596 for (j = 0; j < deferred_flush[i].next; j++) {
3598 struct iova *iova = deferred_flush[i].iova[j];
3599 struct dmar_domain *domain = deferred_flush[i].domain[j];
3601 /* On real hardware multiple invalidations are expensive */
3602 if (cap_caching_mode(iommu->cap))
3603 iommu_flush_iotlb_psi(iommu, domain,
3604 iova->pfn_lo, iova_size(iova),
3605 !deferred_flush[i].freelist[j], 0);
3607 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
3608 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3609 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3611 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
3612 if (deferred_flush[i].freelist[j])
3613 dma_free_pagelist(deferred_flush[i].freelist[j]);
3615 deferred_flush[i].next = 0;
3621 static void flush_unmaps_timeout(unsigned long data)
3623 unsigned long flags;
3625 spin_lock_irqsave(&async_umap_flush_lock, flags);
3627 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3630 static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
3632 unsigned long flags;
3634 struct intel_iommu *iommu;
3636 spin_lock_irqsave(&async_umap_flush_lock, flags);
3637 if (list_size == HIGH_WATER_MARK)
3640 iommu = domain_get_iommu(dom);
3641 iommu_id = iommu->seq_id;
3643 next = deferred_flush[iommu_id].next;
3644 deferred_flush[iommu_id].domain[next] = dom;
3645 deferred_flush[iommu_id].iova[next] = iova;
3646 deferred_flush[iommu_id].freelist[next] = freelist;
3647 deferred_flush[iommu_id].next++;
3650 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3654 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3657 static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
3659 struct dmar_domain *domain;
3660 unsigned long start_pfn, last_pfn;
3662 struct intel_iommu *iommu;
3663 struct page *freelist;
3665 if (iommu_no_mapping(dev))
3668 domain = find_domain(dev);
3671 iommu = domain_get_iommu(domain);
3673 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
3674 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3675 (unsigned long long)dev_addr))
3678 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3679 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
3681 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3682 dev_name(dev), start_pfn, last_pfn);
3684 freelist = domain_unmap(domain, start_pfn, last_pfn);
3686 if (intel_iommu_strict) {
3687 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3688 last_pfn - start_pfn + 1, !freelist, 0);
3690 __free_iova(&domain->iovad, iova);
3691 dma_free_pagelist(freelist);
3693 add_unmap(domain, iova, freelist);
3695 * queue up the release of the unmap to save the 1/6th of the
3696 * cpu used up by the iotlb flush operation...
3701 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3702 size_t size, enum dma_data_direction dir,
3703 struct dma_attrs *attrs)
3705 intel_unmap(dev, dev_addr);
3708 static void *intel_alloc_coherent(struct device *dev, size_t size,
3709 dma_addr_t *dma_handle, gfp_t flags,
3710 struct dma_attrs *attrs)
3712 struct page *page = NULL;
3715 size = PAGE_ALIGN(size);
3716 order = get_order(size);
3718 if (!iommu_no_mapping(dev))
3719 flags &= ~(GFP_DMA | GFP_DMA32);
3720 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3721 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3727 if (gfpflags_allow_blocking(flags)) {
3728 unsigned int count = size >> PAGE_SHIFT;
3730 page = dma_alloc_from_contiguous(dev, count, order);
3731 if (page && iommu_no_mapping(dev) &&
3732 page_to_phys(page) + size > dev->coherent_dma_mask) {
3733 dma_release_from_contiguous(dev, page, count);
3739 page = alloc_pages(flags, order);
3742 memset(page_address(page), 0, size);
3744 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3746 dev->coherent_dma_mask);
3748 return page_address(page);
3749 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3750 __free_pages(page, order);
3755 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3756 dma_addr_t dma_handle, struct dma_attrs *attrs)
3759 struct page *page = virt_to_page(vaddr);
3761 size = PAGE_ALIGN(size);
3762 order = get_order(size);
3764 intel_unmap(dev, dma_handle);
3765 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3766 __free_pages(page, order);
3769 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3770 int nelems, enum dma_data_direction dir,
3771 struct dma_attrs *attrs)
3773 intel_unmap(dev, sglist[0].dma_address);
3776 static int intel_nontranslate_map_sg(struct device *hddev,
3777 struct scatterlist *sglist, int nelems, int dir)
3780 struct scatterlist *sg;
3782 for_each_sg(sglist, sg, nelems, i) {
3783 BUG_ON(!sg_page(sg));
3784 sg->dma_address = sg_phys(sg);
3785 sg->dma_length = sg->length;
3790 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3791 enum dma_data_direction dir, struct dma_attrs *attrs)
3794 struct dmar_domain *domain;
3797 struct iova *iova = NULL;
3799 struct scatterlist *sg;
3800 unsigned long start_vpfn;
3801 struct intel_iommu *iommu;
3803 BUG_ON(dir == DMA_NONE);
3804 if (iommu_no_mapping(dev))
3805 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3807 domain = get_valid_domain_for_dev(dev);
3811 iommu = domain_get_iommu(domain);
3813 for_each_sg(sglist, sg, nelems, i)
3814 size += aligned_nrpages(sg->offset, sg->length);
3816 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3819 sglist->dma_length = 0;
3824 * Check if DMAR supports zero-length reads on write only
3827 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3828 !cap_zlr(iommu->cap))
3829 prot |= DMA_PTE_READ;
3830 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3831 prot |= DMA_PTE_WRITE;
3833 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
3835 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3836 if (unlikely(ret)) {
3837 dma_pte_free_pagetable(domain, start_vpfn,
3838 start_vpfn + size - 1);
3839 __free_iova(&domain->iovad, iova);
3843 /* it's a non-present to present mapping. Only flush if caching mode */
3844 if (cap_caching_mode(iommu->cap))
3845 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
3847 iommu_flush_write_buffer(iommu);
3852 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3857 struct dma_map_ops intel_dma_ops = {
3858 .alloc = intel_alloc_coherent,
3859 .free = intel_free_coherent,
3860 .map_sg = intel_map_sg,
3861 .unmap_sg = intel_unmap_sg,
3862 .map_page = intel_map_page,
3863 .unmap_page = intel_unmap_page,
3864 .mapping_error = intel_mapping_error,
3867 static inline int iommu_domain_cache_init(void)
3871 iommu_domain_cache = kmem_cache_create("iommu_domain",
3872 sizeof(struct dmar_domain),
3877 if (!iommu_domain_cache) {
3878 pr_err("Couldn't create iommu_domain cache\n");
3885 static inline int iommu_devinfo_cache_init(void)
3889 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3890 sizeof(struct device_domain_info),
3894 if (!iommu_devinfo_cache) {
3895 pr_err("Couldn't create devinfo cache\n");
3902 static int __init iommu_init_mempool(void)
3905 ret = iova_cache_get();
3909 ret = iommu_domain_cache_init();
3913 ret = iommu_devinfo_cache_init();
3917 kmem_cache_destroy(iommu_domain_cache);
3924 static void __init iommu_exit_mempool(void)
3926 kmem_cache_destroy(iommu_devinfo_cache);
3927 kmem_cache_destroy(iommu_domain_cache);
3931 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3933 struct dmar_drhd_unit *drhd;
3937 /* We know that this device on this chipset has its own IOMMU.
3938 * If we find it under a different IOMMU, then the BIOS is lying
3939 * to us. Hope that the IOMMU for this device is actually
3940 * disabled, and it needs no translation...
3942 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3944 /* "can't" happen */
3945 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3948 vtbar &= 0xffff0000;
3950 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3951 drhd = dmar_find_matched_drhd_unit(pdev);
3952 if (!drhd || drhd->reg_base_addr - vtbar != 0xa000) {
3953 pr_warn_once(FW_BUG "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n");
3954 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
3955 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3958 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3960 static void __init init_no_remapping_devices(void)
3962 struct dmar_drhd_unit *drhd;
3966 for_each_drhd_unit(drhd) {
3967 if (!drhd->include_all) {
3968 for_each_active_dev_scope(drhd->devices,
3969 drhd->devices_cnt, i, dev)
3971 /* ignore DMAR unit if no devices exist */
3972 if (i == drhd->devices_cnt)
3977 for_each_active_drhd_unit(drhd) {
3978 if (drhd->include_all)
3981 for_each_active_dev_scope(drhd->devices,
3982 drhd->devices_cnt, i, dev)
3983 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
3985 if (i < drhd->devices_cnt)
3988 /* This IOMMU has *only* gfx devices. Either bypass it or
3989 set the gfx_mapped flag, as appropriate */
3990 if (!dmar_map_gfx) {
3992 for_each_active_dev_scope(drhd->devices,
3993 drhd->devices_cnt, i, dev)
3994 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3999 #ifdef CONFIG_SUSPEND
4000 static int init_iommu_hw(void)
4002 struct dmar_drhd_unit *drhd;
4003 struct intel_iommu *iommu = NULL;
4005 for_each_active_iommu(iommu, drhd)
4007 dmar_reenable_qi(iommu);
4009 for_each_iommu(iommu, drhd) {
4010 if (drhd->ignored) {
4012 * we always have to disable PMRs or DMA may fail on
4016 iommu_disable_protect_mem_regions(iommu);
4020 iommu_flush_write_buffer(iommu);
4022 iommu_set_root_entry(iommu);
4024 iommu->flush.flush_context(iommu, 0, 0, 0,
4025 DMA_CCMD_GLOBAL_INVL);
4026 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4027 iommu_enable_translation(iommu);
4028 iommu_disable_protect_mem_regions(iommu);
4034 static void iommu_flush_all(void)
4036 struct dmar_drhd_unit *drhd;
4037 struct intel_iommu *iommu;
4039 for_each_active_iommu(iommu, drhd) {
4040 iommu->flush.flush_context(iommu, 0, 0, 0,
4041 DMA_CCMD_GLOBAL_INVL);
4042 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4043 DMA_TLB_GLOBAL_FLUSH);
4047 static int iommu_suspend(void)
4049 struct dmar_drhd_unit *drhd;
4050 struct intel_iommu *iommu = NULL;
4053 for_each_active_iommu(iommu, drhd) {
4054 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
4056 if (!iommu->iommu_state)
4062 for_each_active_iommu(iommu, drhd) {
4063 iommu_disable_translation(iommu);
4065 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4067 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4068 readl(iommu->reg + DMAR_FECTL_REG);
4069 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4070 readl(iommu->reg + DMAR_FEDATA_REG);
4071 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4072 readl(iommu->reg + DMAR_FEADDR_REG);
4073 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4074 readl(iommu->reg + DMAR_FEUADDR_REG);
4076 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4081 for_each_active_iommu(iommu, drhd)
4082 kfree(iommu->iommu_state);
4087 static void iommu_resume(void)
4089 struct dmar_drhd_unit *drhd;
4090 struct intel_iommu *iommu = NULL;
4093 if (init_iommu_hw()) {
4095 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4097 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4101 for_each_active_iommu(iommu, drhd) {
4103 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4105 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4106 iommu->reg + DMAR_FECTL_REG);
4107 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4108 iommu->reg + DMAR_FEDATA_REG);
4109 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4110 iommu->reg + DMAR_FEADDR_REG);
4111 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4112 iommu->reg + DMAR_FEUADDR_REG);
4114 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4117 for_each_active_iommu(iommu, drhd)
4118 kfree(iommu->iommu_state);
4121 static struct syscore_ops iommu_syscore_ops = {
4122 .resume = iommu_resume,
4123 .suspend = iommu_suspend,
4126 static void __init init_iommu_pm_ops(void)
4128 register_syscore_ops(&iommu_syscore_ops);
4132 static inline void init_iommu_pm_ops(void) {}
4133 #endif /* CONFIG_PM */
4136 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4138 struct acpi_dmar_reserved_memory *rmrr;
4139 struct dmar_rmrr_unit *rmrru;
4141 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4145 rmrru->hdr = header;
4146 rmrr = (struct acpi_dmar_reserved_memory *)header;
4147 rmrru->base_address = rmrr->base_address;
4148 rmrru->end_address = rmrr->end_address;
4149 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4150 ((void *)rmrr) + rmrr->header.length,
4151 &rmrru->devices_cnt);
4152 if (rmrru->devices_cnt && rmrru->devices == NULL) {
4157 list_add(&rmrru->list, &dmar_rmrr_units);
4162 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4164 struct dmar_atsr_unit *atsru;
4165 struct acpi_dmar_atsr *tmp;
4167 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4168 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4169 if (atsr->segment != tmp->segment)
4171 if (atsr->header.length != tmp->header.length)
4173 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4180 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4182 struct acpi_dmar_atsr *atsr;
4183 struct dmar_atsr_unit *atsru;
4185 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4188 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4189 atsru = dmar_find_atsr(atsr);
4193 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4198 * If memory is allocated from slab by ACPI _DSM method, we need to
4199 * copy the memory content because the memory buffer will be freed
4202 atsru->hdr = (void *)(atsru + 1);
4203 memcpy(atsru->hdr, hdr, hdr->length);
4204 atsru->include_all = atsr->flags & 0x1;
4205 if (!atsru->include_all) {
4206 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4207 (void *)atsr + atsr->header.length,
4208 &atsru->devices_cnt);
4209 if (atsru->devices_cnt && atsru->devices == NULL) {
4215 list_add_rcu(&atsru->list, &dmar_atsr_units);
4220 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4222 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4226 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4228 struct acpi_dmar_atsr *atsr;
4229 struct dmar_atsr_unit *atsru;
4231 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4232 atsru = dmar_find_atsr(atsr);
4234 list_del_rcu(&atsru->list);
4236 intel_iommu_free_atsr(atsru);
4242 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4246 struct acpi_dmar_atsr *atsr;
4247 struct dmar_atsr_unit *atsru;
4249 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4250 atsru = dmar_find_atsr(atsr);
4254 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4255 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4263 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4266 struct intel_iommu *iommu = dmaru->iommu;
4268 if (g_iommus[iommu->seq_id])
4271 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4272 pr_warn("%s: Doesn't support hardware pass through.\n",
4276 if (!ecap_sc_support(iommu->ecap) &&
4277 domain_update_iommu_snooping(iommu)) {
4278 pr_warn("%s: Doesn't support snooping.\n",
4282 sp = domain_update_iommu_superpage(iommu) - 1;
4283 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4284 pr_warn("%s: Doesn't support large page.\n",
4290 * Disable translation if already enabled prior to OS handover.
4292 if (iommu->gcmd & DMA_GCMD_TE)
4293 iommu_disable_translation(iommu);
4295 g_iommus[iommu->seq_id] = iommu;
4296 ret = iommu_init_domains(iommu);
4298 ret = iommu_alloc_root_entry(iommu);
4302 #ifdef CONFIG_INTEL_IOMMU_SVM
4303 if (pasid_enabled(iommu))
4304 intel_svm_alloc_pasid_tables(iommu);
4307 if (dmaru->ignored) {
4309 * we always have to disable PMRs or DMA may fail on this device
4312 iommu_disable_protect_mem_regions(iommu);
4316 intel_iommu_init_qi(iommu);
4317 iommu_flush_write_buffer(iommu);
4319 #ifdef CONFIG_INTEL_IOMMU_SVM
4320 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4321 ret = intel_svm_enable_prq(iommu);
4326 ret = dmar_set_interrupt(iommu);
4330 iommu_set_root_entry(iommu);
4331 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4332 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4333 iommu_enable_translation(iommu);
4335 iommu_disable_protect_mem_regions(iommu);
4339 disable_dmar_iommu(iommu);
4341 free_dmar_iommu(iommu);
4345 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4348 struct intel_iommu *iommu = dmaru->iommu;
4350 if (!intel_iommu_enabled)
4356 ret = intel_iommu_add(dmaru);
4358 disable_dmar_iommu(iommu);
4359 free_dmar_iommu(iommu);
4365 static void intel_iommu_free_dmars(void)
4367 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4368 struct dmar_atsr_unit *atsru, *atsr_n;
4370 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4371 list_del(&rmrru->list);
4372 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4376 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4377 list_del(&atsru->list);
4378 intel_iommu_free_atsr(atsru);
4382 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4385 struct pci_bus *bus;
4386 struct pci_dev *bridge = NULL;
4388 struct acpi_dmar_atsr *atsr;
4389 struct dmar_atsr_unit *atsru;
4391 dev = pci_physfn(dev);
4392 for (bus = dev->bus; bus; bus = bus->parent) {
4394 /* If it's an integrated device, allow ATS */
4397 /* Connected via non-PCIe: no ATS */
4398 if (!pci_is_pcie(bridge) ||
4399 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4401 /* If we found the root port, look it up in the ATSR */
4402 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4407 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4408 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4409 if (atsr->segment != pci_domain_nr(dev->bus))
4412 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4413 if (tmp == &bridge->dev)
4416 if (atsru->include_all)
4426 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4429 struct dmar_rmrr_unit *rmrru;
4430 struct dmar_atsr_unit *atsru;
4431 struct acpi_dmar_atsr *atsr;
4432 struct acpi_dmar_reserved_memory *rmrr;
4434 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4437 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4438 rmrr = container_of(rmrru->hdr,
4439 struct acpi_dmar_reserved_memory, header);
4440 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4441 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4442 ((void *)rmrr) + rmrr->header.length,
4443 rmrr->segment, rmrru->devices,
4444 rmrru->devices_cnt);
4447 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4448 dmar_remove_dev_scope(info, rmrr->segment,
4449 rmrru->devices, rmrru->devices_cnt);
4453 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4454 if (atsru->include_all)
4457 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4458 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4459 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4460 (void *)atsr + atsr->header.length,
4461 atsr->segment, atsru->devices,
4462 atsru->devices_cnt);
4467 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4468 if (dmar_remove_dev_scope(info, atsr->segment,
4469 atsru->devices, atsru->devices_cnt))
4478 * Here we only respond to action of unbound device from driver.
4480 * Added device is not attached to its DMAR domain here yet. That will happen
4481 * when mapping the device to iova.
4483 static int device_notifier(struct notifier_block *nb,
4484 unsigned long action, void *data)
4486 struct device *dev = data;
4487 struct dmar_domain *domain;
4489 if (iommu_dummy(dev))
4492 if (action != BUS_NOTIFY_REMOVED_DEVICE)
4495 domain = find_domain(dev);
4499 dmar_remove_one_dev_info(domain, dev);
4500 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4501 domain_exit(domain);
4506 static struct notifier_block device_nb = {
4507 .notifier_call = device_notifier,
4510 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4511 unsigned long val, void *v)
4513 struct memory_notify *mhp = v;
4514 unsigned long long start, end;
4515 unsigned long start_vpfn, last_vpfn;
4518 case MEM_GOING_ONLINE:
4519 start = mhp->start_pfn << PAGE_SHIFT;
4520 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4521 if (iommu_domain_identity_map(si_domain, start, end)) {
4522 pr_warn("Failed to build identity map for [%llx-%llx]\n",
4529 case MEM_CANCEL_ONLINE:
4530 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4531 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4532 while (start_vpfn <= last_vpfn) {
4534 struct dmar_drhd_unit *drhd;
4535 struct intel_iommu *iommu;
4536 struct page *freelist;
4538 iova = find_iova(&si_domain->iovad, start_vpfn);
4540 pr_debug("Failed get IOVA for PFN %lx\n",
4545 iova = split_and_remove_iova(&si_domain->iovad, iova,
4546 start_vpfn, last_vpfn);
4548 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4549 start_vpfn, last_vpfn);
4553 freelist = domain_unmap(si_domain, iova->pfn_lo,
4557 for_each_active_iommu(iommu, drhd)
4558 iommu_flush_iotlb_psi(iommu, si_domain,
4559 iova->pfn_lo, iova_size(iova),
4562 dma_free_pagelist(freelist);
4564 start_vpfn = iova->pfn_hi + 1;
4565 free_iova_mem(iova);
4573 static struct notifier_block intel_iommu_memory_nb = {
4574 .notifier_call = intel_iommu_memory_notifier,
4579 static ssize_t intel_iommu_show_version(struct device *dev,
4580 struct device_attribute *attr,
4583 struct intel_iommu *iommu = dev_get_drvdata(dev);
4584 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4585 return sprintf(buf, "%d:%d\n",
4586 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4588 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4590 static ssize_t intel_iommu_show_address(struct device *dev,
4591 struct device_attribute *attr,
4594 struct intel_iommu *iommu = dev_get_drvdata(dev);
4595 return sprintf(buf, "%llx\n", iommu->reg_phys);
4597 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4599 static ssize_t intel_iommu_show_cap(struct device *dev,
4600 struct device_attribute *attr,
4603 struct intel_iommu *iommu = dev_get_drvdata(dev);
4604 return sprintf(buf, "%llx\n", iommu->cap);
4606 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4608 static ssize_t intel_iommu_show_ecap(struct device *dev,
4609 struct device_attribute *attr,
4612 struct intel_iommu *iommu = dev_get_drvdata(dev);
4613 return sprintf(buf, "%llx\n", iommu->ecap);
4615 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4617 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4618 struct device_attribute *attr,
4621 struct intel_iommu *iommu = dev_get_drvdata(dev);
4622 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4624 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4626 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4627 struct device_attribute *attr,
4630 struct intel_iommu *iommu = dev_get_drvdata(dev);
4631 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4632 cap_ndoms(iommu->cap)));
4634 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4636 static struct attribute *intel_iommu_attrs[] = {
4637 &dev_attr_version.attr,
4638 &dev_attr_address.attr,
4640 &dev_attr_ecap.attr,
4641 &dev_attr_domains_supported.attr,
4642 &dev_attr_domains_used.attr,
4646 static struct attribute_group intel_iommu_group = {
4647 .name = "intel-iommu",
4648 .attrs = intel_iommu_attrs,
4651 const struct attribute_group *intel_iommu_groups[] = {
4656 int __init intel_iommu_init(void)
4659 struct dmar_drhd_unit *drhd;
4660 struct intel_iommu *iommu;
4662 /* VT-d is required for a TXT/tboot launch, so enforce that */
4663 force_on = tboot_force_iommu();
4665 if (iommu_init_mempool()) {
4667 panic("tboot: Failed to initialize iommu memory\n");
4671 down_write(&dmar_global_lock);
4672 if (dmar_table_init()) {
4674 panic("tboot: Failed to initialize DMAR table\n");
4678 if (dmar_dev_scope_init() < 0) {
4680 panic("tboot: Failed to initialize DMAR device scope\n");
4684 if (no_iommu || dmar_disabled)
4687 if (list_empty(&dmar_rmrr_units))
4688 pr_info("No RMRR found\n");
4690 if (list_empty(&dmar_atsr_units))
4691 pr_info("No ATSR found\n");
4693 if (dmar_init_reserved_ranges()) {
4695 panic("tboot: Failed to reserve iommu ranges\n");
4696 goto out_free_reserved_range;
4700 intel_iommu_gfx_mapped = 1;
4702 init_no_remapping_devices();
4707 panic("tboot: Failed to initialize DMARs\n");
4708 pr_err("Initialization failed\n");
4709 goto out_free_reserved_range;
4711 up_write(&dmar_global_lock);
4712 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4714 init_timer(&unmap_timer);
4715 #ifdef CONFIG_SWIOTLB
4718 dma_ops = &intel_dma_ops;
4720 init_iommu_pm_ops();
4722 for_each_active_iommu(iommu, drhd)
4723 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4727 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4728 bus_register_notifier(&pci_bus_type, &device_nb);
4729 if (si_domain && !hw_pass_through)
4730 register_memory_notifier(&intel_iommu_memory_nb);
4732 intel_iommu_enabled = 1;
4736 out_free_reserved_range:
4737 put_iova_domain(&reserved_iova_list);
4739 intel_iommu_free_dmars();
4740 up_write(&dmar_global_lock);
4741 iommu_exit_mempool();
4745 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4747 struct intel_iommu *iommu = opaque;
4749 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4754 * NB - intel-iommu lacks any sort of reference counting for the users of
4755 * dependent devices. If multiple endpoints have intersecting dependent
4756 * devices, unbinding the driver from any one of them will possibly leave
4757 * the others unable to operate.
4759 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4761 if (!iommu || !dev || !dev_is_pci(dev))
4764 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4767 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4769 struct intel_iommu *iommu;
4770 unsigned long flags;
4772 assert_spin_locked(&device_domain_lock);
4777 iommu = info->iommu;
4780 iommu_disable_dev_iotlb(info);
4781 domain_context_clear(iommu, info->dev);
4784 unlink_domain_info(info);
4786 spin_lock_irqsave(&iommu->lock, flags);
4787 domain_detach_iommu(info->domain, iommu);
4788 spin_unlock_irqrestore(&iommu->lock, flags);
4790 free_devinfo_mem(info);
4793 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4796 struct device_domain_info *info;
4797 unsigned long flags;
4799 spin_lock_irqsave(&device_domain_lock, flags);
4800 info = dev->archdata.iommu;
4801 __dmar_remove_one_dev_info(info);
4802 spin_unlock_irqrestore(&device_domain_lock, flags);
4805 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4809 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4811 domain_reserve_special_ranges(domain);
4813 /* calculate AGAW */
4814 domain->gaw = guest_width;
4815 adjust_width = guestwidth_to_adjustwidth(guest_width);
4816 domain->agaw = width_to_agaw(adjust_width);
4818 domain->iommu_coherency = 0;
4819 domain->iommu_snooping = 0;
4820 domain->iommu_superpage = 0;
4821 domain->max_addr = 0;
4823 /* always allocate the top pgd */
4824 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4827 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4831 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4833 struct dmar_domain *dmar_domain;
4834 struct iommu_domain *domain;
4836 if (type != IOMMU_DOMAIN_UNMANAGED)
4839 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4841 pr_err("Can't allocate dmar_domain\n");
4844 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4845 pr_err("Domain initialization failed\n");
4846 domain_exit(dmar_domain);
4849 domain_update_iommu_cap(dmar_domain);
4851 domain = &dmar_domain->domain;
4852 domain->geometry.aperture_start = 0;
4853 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4854 domain->geometry.force_aperture = true;
4859 static void intel_iommu_domain_free(struct iommu_domain *domain)
4861 domain_exit(to_dmar_domain(domain));
4864 static int intel_iommu_attach_device(struct iommu_domain *domain,
4867 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4868 struct intel_iommu *iommu;
4872 if (device_is_rmrr_locked(dev)) {
4873 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4877 /* normally dev is not mapped */
4878 if (unlikely(domain_context_mapped(dev))) {
4879 struct dmar_domain *old_domain;
4881 old_domain = find_domain(dev);
4884 dmar_remove_one_dev_info(old_domain, dev);
4887 if (!domain_type_is_vm_or_si(old_domain) &&
4888 list_empty(&old_domain->devices))
4889 domain_exit(old_domain);
4893 iommu = device_to_iommu(dev, &bus, &devfn);
4897 /* check if this iommu agaw is sufficient for max mapped address */
4898 addr_width = agaw_to_width(iommu->agaw);
4899 if (addr_width > cap_mgaw(iommu->cap))
4900 addr_width = cap_mgaw(iommu->cap);
4902 if (dmar_domain->max_addr > (1LL << addr_width)) {
4903 pr_err("%s: iommu width (%d) is not "
4904 "sufficient for the mapped address (%llx)\n",
4905 __func__, addr_width, dmar_domain->max_addr);
4908 dmar_domain->gaw = addr_width;
4911 * Knock out extra levels of page tables if necessary
4913 while (iommu->agaw < dmar_domain->agaw) {
4914 struct dma_pte *pte;
4916 pte = dmar_domain->pgd;
4917 if (dma_pte_present(pte)) {
4918 dmar_domain->pgd = (struct dma_pte *)
4919 phys_to_virt(dma_pte_addr(pte));
4920 free_pgtable_page(pte);
4922 dmar_domain->agaw--;
4925 return domain_add_dev_info(dmar_domain, dev);
4928 static void intel_iommu_detach_device(struct iommu_domain *domain,
4931 dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
4934 static int intel_iommu_map(struct iommu_domain *domain,
4935 unsigned long iova, phys_addr_t hpa,
4936 size_t size, int iommu_prot)
4938 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4943 if (iommu_prot & IOMMU_READ)
4944 prot |= DMA_PTE_READ;
4945 if (iommu_prot & IOMMU_WRITE)
4946 prot |= DMA_PTE_WRITE;
4947 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4948 prot |= DMA_PTE_SNP;
4950 max_addr = iova + size;
4951 if (dmar_domain->max_addr < max_addr) {
4954 /* check if minimum agaw is sufficient for mapped address */
4955 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
4956 if (end < max_addr) {
4957 pr_err("%s: iommu width (%d) is not "
4958 "sufficient for the mapped address (%llx)\n",
4959 __func__, dmar_domain->gaw, max_addr);
4962 dmar_domain->max_addr = max_addr;
4964 /* Round up size to next multiple of PAGE_SIZE, if it and
4965 the low bits of hpa would take us onto the next page */
4966 size = aligned_nrpages(hpa, size);
4967 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4968 hpa >> VTD_PAGE_SHIFT, size, prot);
4972 static size_t intel_iommu_unmap(struct iommu_domain *domain,
4973 unsigned long iova, size_t size)
4975 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4976 struct page *freelist = NULL;
4977 struct intel_iommu *iommu;
4978 unsigned long start_pfn, last_pfn;
4979 unsigned int npages;
4980 int iommu_id, level = 0;
4982 /* Cope with horrid API which requires us to unmap more than the
4983 size argument if it happens to be a large-page mapping. */
4984 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
4986 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4987 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4989 start_pfn = iova >> VTD_PAGE_SHIFT;
4990 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4992 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4994 npages = last_pfn - start_pfn + 1;
4996 for_each_domain_iommu(iommu_id, dmar_domain) {
4997 iommu = g_iommus[iommu_id];
4999 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5000 start_pfn, npages, !freelist, 0);
5003 dma_free_pagelist(freelist);
5005 if (dmar_domain->max_addr == iova + size)
5006 dmar_domain->max_addr = iova;
5011 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5014 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5015 struct dma_pte *pte;
5019 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5020 if (pte && dma_pte_present(pte))
5021 phys = dma_pte_addr(pte) +
5022 (iova & (BIT_MASK(level_to_offset_bits(level) +
5023 VTD_PAGE_SHIFT) - 1));
5028 static bool intel_iommu_capable(enum iommu_cap cap)
5030 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5031 return domain_update_iommu_snooping(NULL) == 1;
5032 if (cap == IOMMU_CAP_INTR_REMAP)
5033 return irq_remapping_enabled == 1;
5038 static int intel_iommu_add_device(struct device *dev)
5040 struct intel_iommu *iommu;
5041 struct iommu_group *group;
5044 iommu = device_to_iommu(dev, &bus, &devfn);
5048 iommu_device_link(iommu->iommu_dev, dev);
5050 group = iommu_group_get_for_dev(dev);
5053 return PTR_ERR(group);
5055 iommu_group_put(group);
5059 static void intel_iommu_remove_device(struct device *dev)
5061 struct intel_iommu *iommu;
5064 iommu = device_to_iommu(dev, &bus, &devfn);
5068 iommu_group_remove_device(dev);
5070 iommu_device_unlink(iommu->iommu_dev, dev);
5073 #ifdef CONFIG_INTEL_IOMMU_SVM
5074 #define MAX_NR_PASID_BITS (20)
5075 static inline unsigned long intel_iommu_get_pts(struct intel_iommu *iommu)
5078 * Convert ecap_pss to extend context entry pts encoding, also
5079 * respect the soft pasid_max value set by the iommu.
5080 * - number of PASID bits = ecap_pss + 1
5081 * - number of PASID table entries = 2^(pts + 5)
5082 * Therefore, pts = ecap_pss - 4
5083 * e.g. KBL ecap_pss = 0x13, PASID has 20 bits, pts = 15
5085 if (ecap_pss(iommu->ecap) < 5)
5088 /* pasid_max is encoded as actual number of entries not the bits */
5089 return find_first_bit((unsigned long *)&iommu->pasid_max,
5090 MAX_NR_PASID_BITS) - 5;
5093 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5095 struct device_domain_info *info;
5096 struct context_entry *context;
5097 struct dmar_domain *domain;
5098 unsigned long flags;
5102 domain = get_valid_domain_for_dev(sdev->dev);
5106 spin_lock_irqsave(&device_domain_lock, flags);
5107 spin_lock(&iommu->lock);
5110 info = sdev->dev->archdata.iommu;
5111 if (!info || !info->pasid_supported)
5114 context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5115 if (WARN_ON(!context))
5118 ctx_lo = context[0].lo;
5120 sdev->did = domain->iommu_did[iommu->seq_id];
5121 sdev->sid = PCI_DEVID(info->bus, info->devfn);
5123 if (!(ctx_lo & CONTEXT_PASIDE)) {
5124 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
5125 context[1].lo = (u64)virt_to_phys(iommu->pasid_table) |
5126 intel_iommu_get_pts(iommu);
5129 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5130 * extended to permit requests-with-PASID if the PASIDE bit
5131 * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5132 * however, the PASIDE bit is ignored and requests-with-PASID
5133 * are unconditionally blocked. Which makes less sense.
5134 * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5135 * "guest mode" translation types depending on whether ATS
5136 * is available or not. Annoyingly, we can't use the new
5137 * modes *unless* PASIDE is set. */
5138 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5139 ctx_lo &= ~CONTEXT_TT_MASK;
5140 if (info->ats_supported)
5141 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5143 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5145 ctx_lo |= CONTEXT_PASIDE;
5146 if (iommu->pasid_state_table)
5147 ctx_lo |= CONTEXT_DINVE;
5148 if (info->pri_supported)
5149 ctx_lo |= CONTEXT_PRS;
5150 context[0].lo = ctx_lo;
5152 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5153 DMA_CCMD_MASK_NOBIT,
5154 DMA_CCMD_DEVICE_INVL);
5157 /* Enable PASID support in the device, if it wasn't already */
5158 if (!info->pasid_enabled)
5159 iommu_enable_dev_iotlb(info);
5161 if (info->ats_enabled) {
5162 sdev->dev_iotlb = 1;
5163 sdev->qdep = info->ats_qdep;
5164 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5170 spin_unlock(&iommu->lock);
5171 spin_unlock_irqrestore(&device_domain_lock, flags);
5176 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5178 struct intel_iommu *iommu;
5181 if (iommu_dummy(dev)) {
5183 "No IOMMU translation for device; cannot enable SVM\n");
5187 iommu = device_to_iommu(dev, &bus, &devfn);
5189 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5193 if (!iommu->pasid_table) {
5194 dev_err(dev, "PASID not enabled on IOMMU; cannot enable SVM\n");
5200 #endif /* CONFIG_INTEL_IOMMU_SVM */
5202 static const struct iommu_ops intel_iommu_ops = {
5203 .capable = intel_iommu_capable,
5204 .domain_alloc = intel_iommu_domain_alloc,
5205 .domain_free = intel_iommu_domain_free,
5206 .attach_dev = intel_iommu_attach_device,
5207 .detach_dev = intel_iommu_detach_device,
5208 .map = intel_iommu_map,
5209 .unmap = intel_iommu_unmap,
5210 .map_sg = default_iommu_map_sg,
5211 .iova_to_phys = intel_iommu_iova_to_phys,
5212 .add_device = intel_iommu_add_device,
5213 .remove_device = intel_iommu_remove_device,
5214 .device_group = pci_device_group,
5215 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
5218 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5220 /* G4x/GM45 integrated gfx dmar support is totally busted. */
5221 pr_info("Disabling IOMMU for graphics on this chipset\n");
5225 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5226 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5227 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5228 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5229 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5230 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5231 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5233 static void quirk_iommu_rwbf(struct pci_dev *dev)
5236 * Mobile 4 Series Chipset neglects to set RWBF capability,
5237 * but needs it. Same seems to hold for the desktop versions.
5239 pr_info("Forcing write-buffer flush capability\n");
5243 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5244 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5245 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5246 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5247 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5248 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5249 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5252 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
5253 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
5254 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
5255 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
5256 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
5257 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
5258 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
5259 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
5261 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5265 if (pci_read_config_word(dev, GGC, &ggc))
5268 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5269 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5271 } else if (dmar_map_gfx) {
5272 /* we have to ensure the gfx device is idle before we flush */
5273 pr_info("Disabling batched IOTLB flush on Ironlake\n");
5274 intel_iommu_strict = 1;
5277 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5278 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5279 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5280 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5282 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5283 ISOCH DMAR unit for the Azalia sound device, but not give it any
5284 TLB entries, which causes it to deadlock. Check for that. We do
5285 this in a function called from init_dmars(), instead of in a PCI
5286 quirk, because we don't want to print the obnoxious "BIOS broken"
5287 message if VT-d is actually disabled.
5289 static void __init check_tylersburg_isoch(void)
5291 struct pci_dev *pdev;
5292 uint32_t vtisochctrl;
5294 /* If there's no Azalia in the system anyway, forget it. */
5295 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5300 /* System Management Registers. Might be hidden, in which case
5301 we can't do the sanity check. But that's OK, because the
5302 known-broken BIOSes _don't_ actually hide it, so far. */
5303 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5307 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5314 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5315 if (vtisochctrl & 1)
5318 /* Drop all bits other than the number of TLB entries */
5319 vtisochctrl &= 0x1c;
5321 /* If we have the recommended number of TLB entries (16), fine. */
5322 if (vtisochctrl == 0x10)
5325 /* Zero TLB entries? You get to ride the short bus to school. */
5327 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5328 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5329 dmi_get_system_info(DMI_BIOS_VENDOR),
5330 dmi_get_system_info(DMI_BIOS_VERSION),
5331 dmi_get_system_info(DMI_PRODUCT_VERSION));
5332 iommu_identity_mapping |= IDENTMAP_AZALIA;
5336 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",