2 * Copyright © 2006-2014 Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
18 * Joerg Roedel <jroedel@suse.de>
21 #define pr_fmt(fmt) "DMAR: " fmt
23 #include <linux/init.h>
24 #include <linux/bitmap.h>
25 #include <linux/debugfs.h>
26 #include <linux/export.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/memory.h>
36 #include <linux/cpu.h>
37 #include <linux/timer.h>
39 #include <linux/iova.h>
40 #include <linux/iommu.h>
41 #include <linux/intel-iommu.h>
42 #include <linux/syscore_ops.h>
43 #include <linux/tboot.h>
44 #include <linux/dmi.h>
45 #include <linux/pci-ats.h>
46 #include <linux/memblock.h>
47 #include <linux/dma-contiguous.h>
48 #include <linux/crash_dump.h>
49 #include <asm/irq_remapping.h>
50 #include <asm/cacheflush.h>
51 #include <asm/iommu.h>
53 #include "irq_remapping.h"
55 #define ROOT_SIZE VTD_PAGE_SIZE
56 #define CONTEXT_SIZE VTD_PAGE_SIZE
58 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
59 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
60 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
61 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
63 #define IOAPIC_RANGE_START (0xfee00000)
64 #define IOAPIC_RANGE_END (0xfeefffff)
65 #define IOVA_START_ADDR (0x1000)
67 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
69 #define MAX_AGAW_WIDTH 64
70 #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
72 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
73 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
75 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
76 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
77 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
78 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
79 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
81 /* IO virtual address start page frame number */
82 #define IOVA_START_PFN (1)
84 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
85 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
86 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
88 /* page table handling */
89 #define LEVEL_STRIDE (9)
90 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
93 * This bitmap is used to advertise the page sizes our hardware support
94 * to the IOMMU core, which will then use this information to split
95 * physically contiguous memory regions it is mapping into page sizes
98 * Traditionally the IOMMU core just handed us the mappings directly,
99 * after making sure the size is an order of a 4KiB page and that the
100 * mapping has natural alignment.
102 * To retain this behavior, we currently advertise that we support
103 * all page sizes that are an order of 4KiB.
105 * If at some point we'd like to utilize the IOMMU core's new behavior,
106 * we could change this to advertise the real page sizes we support.
108 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
110 static inline int agaw_to_level(int agaw)
115 static inline int agaw_to_width(int agaw)
117 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
120 static inline int width_to_agaw(int width)
122 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
125 static inline unsigned int level_to_offset_bits(int level)
127 return (level - 1) * LEVEL_STRIDE;
130 static inline int pfn_level_offset(unsigned long pfn, int level)
132 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
135 static inline unsigned long level_mask(int level)
137 return -1UL << level_to_offset_bits(level);
140 static inline unsigned long level_size(int level)
142 return 1UL << level_to_offset_bits(level);
145 static inline unsigned long align_to_level(unsigned long pfn, int level)
147 return (pfn + level_size(level) - 1) & level_mask(level);
150 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
152 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
155 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
156 are never going to work. */
157 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
159 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
162 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
164 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
166 static inline unsigned long page_to_dma_pfn(struct page *pg)
168 return mm_to_dma_pfn(page_to_pfn(pg));
170 static inline unsigned long virt_to_dma_pfn(void *p)
172 return page_to_dma_pfn(virt_to_page(p));
175 /* global iommu list, set NULL for ignored DMAR units */
176 static struct intel_iommu **g_iommus;
178 static void __init check_tylersburg_isoch(void);
179 static int rwbf_quirk;
182 * set to 1 to panic kernel if can't successfully enable VT-d
183 * (used when kernel is launched w/ TXT)
185 static int force_on = 0;
190 * 12-63: Context Ptr (12 - (haw-1))
197 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
200 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
203 static phys_addr_t root_entry_lctp(struct root_entry *re)
208 return re->lo & VTD_PAGE_MASK;
212 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
215 static phys_addr_t root_entry_uctp(struct root_entry *re)
220 return re->hi & VTD_PAGE_MASK;
225 * 1: fault processing disable
226 * 2-3: translation type
227 * 12-63: address space root
233 struct context_entry {
238 static inline void context_clear_pasid_enable(struct context_entry *context)
240 context->lo &= ~(1ULL << 11);
243 static inline bool context_pasid_enabled(struct context_entry *context)
245 return !!(context->lo & (1ULL << 11));
248 static inline void context_set_copied(struct context_entry *context)
250 context->hi |= (1ull << 3);
253 static inline bool context_copied(struct context_entry *context)
255 return !!(context->hi & (1ULL << 3));
258 static inline bool __context_present(struct context_entry *context)
260 return (context->lo & 1);
263 static inline bool context_present(struct context_entry *context)
265 return context_pasid_enabled(context) ?
266 __context_present(context) :
267 __context_present(context) && !context_copied(context);
270 static inline void context_set_present(struct context_entry *context)
275 static inline void context_set_fault_enable(struct context_entry *context)
277 context->lo &= (((u64)-1) << 2) | 1;
280 static inline void context_set_translation_type(struct context_entry *context,
283 context->lo &= (((u64)-1) << 4) | 3;
284 context->lo |= (value & 3) << 2;
287 static inline void context_set_address_root(struct context_entry *context,
290 context->lo &= ~VTD_PAGE_MASK;
291 context->lo |= value & VTD_PAGE_MASK;
294 static inline void context_set_address_width(struct context_entry *context,
297 context->hi |= value & 7;
300 static inline void context_set_domain_id(struct context_entry *context,
303 context->hi |= (value & ((1 << 16) - 1)) << 8;
306 static inline int context_domain_id(struct context_entry *c)
308 return((c->hi >> 8) & 0xffff);
311 static inline void context_clear_entry(struct context_entry *context)
324 * 12-63: Host physcial address
330 static inline void dma_clear_pte(struct dma_pte *pte)
335 static inline u64 dma_pte_addr(struct dma_pte *pte)
338 return pte->val & VTD_PAGE_MASK;
340 /* Must have a full atomic 64-bit read */
341 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
345 static inline bool dma_pte_present(struct dma_pte *pte)
347 return (pte->val & 3) != 0;
350 static inline bool dma_pte_superpage(struct dma_pte *pte)
352 return (pte->val & DMA_PTE_LARGE_PAGE);
355 static inline int first_pte_in_page(struct dma_pte *pte)
357 return !((unsigned long)pte & ~VTD_PAGE_MASK);
361 * This domain is a statically identity mapping domain.
362 * 1. This domain creats a static 1:1 mapping to all usable memory.
363 * 2. It maps to each iommu if successful.
364 * 3. Each iommu mapps to this domain if successful.
366 static struct dmar_domain *si_domain;
367 static int hw_pass_through = 1;
370 * Domain represents a virtual machine, more than one devices
371 * across iommus may be owned in one domain, e.g. kvm guest.
373 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
375 /* si_domain contains mulitple devices */
376 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
378 #define for_each_domain_iommu(idx, domain) \
379 for (idx = 0; idx < g_num_of_iommus; idx++) \
380 if (domain->iommu_refcnt[idx])
383 int nid; /* node id */
385 unsigned iommu_refcnt[DMAR_UNITS_SUPPORTED];
386 /* Refcount of devices per iommu */
389 u16 iommu_did[DMAR_UNITS_SUPPORTED];
390 /* Domain ids per IOMMU. Use u16 since
391 * domain ids are 16 bit wide according
392 * to VT-d spec, section 9.3 */
394 bool has_iotlb_device;
395 struct list_head devices; /* all devices' list */
396 struct iova_domain iovad; /* iova's that belong to this domain */
398 struct dma_pte *pgd; /* virtual address */
399 int gaw; /* max guest address width */
401 /* adjusted guest address width, 0 is level 2 30-bit */
404 int flags; /* flags to find out type of domain */
406 int iommu_coherency;/* indicate coherency of iommu access */
407 int iommu_snooping; /* indicate snooping control feature*/
408 int iommu_count; /* reference count of iommu */
409 int iommu_superpage;/* Level of superpages supported:
410 0 == 4KiB (no superpages), 1 == 2MiB,
411 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
412 u64 max_addr; /* maximum mapped address */
414 struct iommu_domain domain; /* generic domain data structure for
418 /* PCI domain-device relationship */
419 struct device_domain_info {
420 struct list_head link; /* link to domain siblings */
421 struct list_head global; /* link to global list */
422 u8 bus; /* PCI bus number */
423 u8 devfn; /* PCI devfn number */
424 u16 pfsid; /* SRIOV physical function source ID */
425 u8 pasid_supported:3;
432 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
433 struct intel_iommu *iommu; /* IOMMU used by this device */
434 struct dmar_domain *domain; /* pointer to domain */
437 struct dmar_rmrr_unit {
438 struct list_head list; /* list of rmrr units */
439 struct acpi_dmar_header *hdr; /* ACPI header */
440 u64 base_address; /* reserved base address*/
441 u64 end_address; /* reserved end address */
442 struct dmar_dev_scope *devices; /* target devices */
443 int devices_cnt; /* target device count */
446 struct dmar_atsr_unit {
447 struct list_head list; /* list of ATSR units */
448 struct acpi_dmar_header *hdr; /* ACPI header */
449 struct dmar_dev_scope *devices; /* target devices */
450 int devices_cnt; /* target device count */
451 u8 include_all:1; /* include all ports */
454 static LIST_HEAD(dmar_atsr_units);
455 static LIST_HEAD(dmar_rmrr_units);
457 #define for_each_rmrr_units(rmrr) \
458 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
460 static void flush_unmaps_timeout(unsigned long data);
462 struct deferred_flush_entry {
463 unsigned long iova_pfn;
464 unsigned long nrpages;
465 struct dmar_domain *domain;
466 struct page *freelist;
469 #define HIGH_WATER_MARK 250
470 struct deferred_flush_table {
472 struct deferred_flush_entry entries[HIGH_WATER_MARK];
475 struct deferred_flush_data {
478 struct timer_list timer;
480 struct deferred_flush_table *tables;
483 DEFINE_PER_CPU(struct deferred_flush_data, deferred_flush);
485 /* bitmap for indexing intel_iommus */
486 static int g_num_of_iommus;
488 static void domain_exit(struct dmar_domain *domain);
489 static void domain_remove_dev_info(struct dmar_domain *domain);
490 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
492 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
493 static void domain_context_clear(struct intel_iommu *iommu,
495 static int domain_detach_iommu(struct dmar_domain *domain,
496 struct intel_iommu *iommu);
498 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
499 int dmar_disabled = 0;
501 int dmar_disabled = 1;
502 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
504 int intel_iommu_enabled = 0;
505 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
507 static int dmar_map_gfx = 1;
508 static int dmar_forcedac;
509 static int intel_iommu_strict;
510 static int intel_iommu_superpage = 1;
511 static int intel_iommu_ecs = 1;
512 static int intel_iommu_pasid28;
513 static int iommu_identity_mapping;
515 #define IDENTMAP_ALL 1
516 #define IDENTMAP_GFX 2
517 #define IDENTMAP_AZALIA 4
519 /* Broadwell and Skylake have broken ECS support — normal so-called "second
520 * level" translation of DMA requests-without-PASID doesn't actually happen
521 * unless you also set the NESTE bit in an extended context-entry. Which of
522 * course means that SVM doesn't work because it's trying to do nested
523 * translation of the physical addresses it finds in the process page tables,
524 * through the IOVA->phys mapping found in the "second level" page tables.
526 * The VT-d specification was retroactively changed to change the definition
527 * of the capability bits and pretend that Broadwell/Skylake never happened...
528 * but unfortunately the wrong bit was changed. It's ECS which is broken, but
529 * for some reason it was the PASID capability bit which was redefined (from
530 * bit 28 on BDW/SKL to bit 40 in future).
532 * So our test for ECS needs to eschew those implementations which set the old
533 * PASID capabiity bit 28, since those are the ones on which ECS is broken.
534 * Unless we are working around the 'pasid28' limitations, that is, by putting
535 * the device into passthrough mode for normal DMA and thus masking the bug.
537 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
538 (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
539 /* PASID support is thus enabled if ECS is enabled and *either* of the old
540 * or new capability bits are set. */
541 #define pasid_enabled(iommu) (ecs_enabled(iommu) && \
542 (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
544 int intel_iommu_gfx_mapped;
545 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
547 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
548 static DEFINE_SPINLOCK(device_domain_lock);
549 static LIST_HEAD(device_domain_list);
551 static const struct iommu_ops intel_iommu_ops;
553 static bool translation_pre_enabled(struct intel_iommu *iommu)
555 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
558 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
560 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
563 static void init_translation_status(struct intel_iommu *iommu)
567 gsts = readl(iommu->reg + DMAR_GSTS_REG);
568 if (gsts & DMA_GSTS_TES)
569 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
572 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
573 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
575 return container_of(dom, struct dmar_domain, domain);
578 static int __init intel_iommu_setup(char *str)
583 if (!strncmp(str, "on", 2)) {
585 pr_info("IOMMU enabled\n");
586 } else if (!strncmp(str, "off", 3)) {
588 pr_info("IOMMU disabled\n");
589 } else if (!strncmp(str, "igfx_off", 8)) {
591 pr_info("Disable GFX device mapping\n");
592 } else if (!strncmp(str, "forcedac", 8)) {
593 pr_info("Forcing DAC for PCI devices\n");
595 } else if (!strncmp(str, "strict", 6)) {
596 pr_info("Disable batched IOTLB flush\n");
597 intel_iommu_strict = 1;
598 } else if (!strncmp(str, "sp_off", 6)) {
599 pr_info("Disable supported super page\n");
600 intel_iommu_superpage = 0;
601 } else if (!strncmp(str, "ecs_off", 7)) {
603 "Intel-IOMMU: disable extended context table support\n");
605 } else if (!strncmp(str, "pasid28", 7)) {
607 "Intel-IOMMU: enable pre-production PASID support\n");
608 intel_iommu_pasid28 = 1;
609 iommu_identity_mapping |= IDENTMAP_GFX;
612 str += strcspn(str, ",");
618 __setup("intel_iommu=", intel_iommu_setup);
620 static struct kmem_cache *iommu_domain_cache;
621 static struct kmem_cache *iommu_devinfo_cache;
623 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
625 struct dmar_domain **domains;
628 domains = iommu->domains[idx];
632 return domains[did & 0xff];
635 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
636 struct dmar_domain *domain)
638 struct dmar_domain **domains;
641 if (!iommu->domains[idx]) {
642 size_t size = 256 * sizeof(struct dmar_domain *);
643 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
646 domains = iommu->domains[idx];
647 if (WARN_ON(!domains))
650 domains[did & 0xff] = domain;
653 static inline void *alloc_pgtable_page(int node)
658 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
660 vaddr = page_address(page);
664 static inline void free_pgtable_page(void *vaddr)
666 free_page((unsigned long)vaddr);
669 static inline void *alloc_domain_mem(void)
671 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
674 static void free_domain_mem(void *vaddr)
676 kmem_cache_free(iommu_domain_cache, vaddr);
679 static inline void * alloc_devinfo_mem(void)
681 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
684 static inline void free_devinfo_mem(void *vaddr)
686 kmem_cache_free(iommu_devinfo_cache, vaddr);
689 static inline int domain_type_is_vm(struct dmar_domain *domain)
691 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
694 static inline int domain_type_is_si(struct dmar_domain *domain)
696 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
699 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
701 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
702 DOMAIN_FLAG_STATIC_IDENTITY);
705 static inline int domain_pfn_supported(struct dmar_domain *domain,
708 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
710 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
713 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
718 sagaw = cap_sagaw(iommu->cap);
719 for (agaw = width_to_agaw(max_gaw);
721 if (test_bit(agaw, &sagaw))
729 * Calculate max SAGAW for each iommu.
731 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
733 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
737 * calculate agaw for each iommu.
738 * "SAGAW" may be different across iommus, use a default agaw, and
739 * get a supported less agaw for iommus that don't support the default agaw.
741 int iommu_calculate_agaw(struct intel_iommu *iommu)
743 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
746 /* This functionin only returns single iommu in a domain */
747 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
751 /* si_domain and vm domain should not get here. */
752 BUG_ON(domain_type_is_vm_or_si(domain));
753 for_each_domain_iommu(iommu_id, domain)
756 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
759 return g_iommus[iommu_id];
762 static void domain_update_iommu_coherency(struct dmar_domain *domain)
764 struct dmar_drhd_unit *drhd;
765 struct intel_iommu *iommu;
769 domain->iommu_coherency = 1;
771 for_each_domain_iommu(i, domain) {
773 if (!ecap_coherent(g_iommus[i]->ecap)) {
774 domain->iommu_coherency = 0;
781 /* No hardware attached; use lowest common denominator */
783 for_each_active_iommu(iommu, drhd) {
784 if (!ecap_coherent(iommu->ecap)) {
785 domain->iommu_coherency = 0;
792 static int domain_update_iommu_snooping(struct intel_iommu *skip)
794 struct dmar_drhd_unit *drhd;
795 struct intel_iommu *iommu;
799 for_each_active_iommu(iommu, drhd) {
801 if (!ecap_sc_support(iommu->ecap)) {
812 static int domain_update_iommu_superpage(struct intel_iommu *skip)
814 struct dmar_drhd_unit *drhd;
815 struct intel_iommu *iommu;
818 if (!intel_iommu_superpage) {
822 /* set iommu_superpage to the smallest common denominator */
824 for_each_active_iommu(iommu, drhd) {
826 mask &= cap_super_page_val(iommu->cap);
836 /* Some capabilities may be different across iommus */
837 static void domain_update_iommu_cap(struct dmar_domain *domain)
839 domain_update_iommu_coherency(domain);
840 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
841 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
844 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
845 u8 bus, u8 devfn, int alloc)
847 struct root_entry *root = &iommu->root_entry[bus];
848 struct context_entry *context;
852 if (ecs_enabled(iommu)) {
860 context = phys_to_virt(*entry & VTD_PAGE_MASK);
862 unsigned long phy_addr;
866 context = alloc_pgtable_page(iommu->node);
870 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
871 phy_addr = virt_to_phys((void *)context);
872 *entry = phy_addr | 1;
873 __iommu_flush_cache(iommu, entry, sizeof(*entry));
875 return &context[devfn];
878 static int iommu_dummy(struct device *dev)
880 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
883 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
885 struct dmar_drhd_unit *drhd = NULL;
886 struct intel_iommu *iommu;
888 struct pci_dev *ptmp, *pdev = NULL;
892 if (iommu_dummy(dev))
895 if (dev_is_pci(dev)) {
896 struct pci_dev *pf_pdev;
898 pdev = to_pci_dev(dev);
899 /* VFs aren't listed in scope tables; we need to look up
900 * the PF instead to find the IOMMU. */
901 pf_pdev = pci_physfn(pdev);
903 segment = pci_domain_nr(pdev->bus);
904 } else if (has_acpi_companion(dev))
905 dev = &ACPI_COMPANION(dev)->dev;
908 for_each_active_iommu(iommu, drhd) {
909 if (pdev && segment != drhd->segment)
912 for_each_active_dev_scope(drhd->devices,
913 drhd->devices_cnt, i, tmp) {
915 /* For a VF use its original BDF# not that of the PF
916 * which we used for the IOMMU lookup. Strictly speaking
917 * we could do this for all PCI devices; we only need to
918 * get the BDF# from the scope table for ACPI matches. */
919 if (pdev && pdev->is_virtfn)
922 *bus = drhd->devices[i].bus;
923 *devfn = drhd->devices[i].devfn;
927 if (!pdev || !dev_is_pci(tmp))
930 ptmp = to_pci_dev(tmp);
931 if (ptmp->subordinate &&
932 ptmp->subordinate->number <= pdev->bus->number &&
933 ptmp->subordinate->busn_res.end >= pdev->bus->number)
937 if (pdev && drhd->include_all) {
939 *bus = pdev->bus->number;
940 *devfn = pdev->devfn;
951 static void domain_flush_cache(struct dmar_domain *domain,
952 void *addr, int size)
954 if (!domain->iommu_coherency)
955 clflush_cache_range(addr, size);
958 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
960 struct context_entry *context;
964 spin_lock_irqsave(&iommu->lock, flags);
965 context = iommu_context_addr(iommu, bus, devfn, 0);
967 ret = context_present(context);
968 spin_unlock_irqrestore(&iommu->lock, flags);
972 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
974 struct context_entry *context;
977 spin_lock_irqsave(&iommu->lock, flags);
978 context = iommu_context_addr(iommu, bus, devfn, 0);
980 context_clear_entry(context);
981 __iommu_flush_cache(iommu, context, sizeof(*context));
983 spin_unlock_irqrestore(&iommu->lock, flags);
986 static void free_context_table(struct intel_iommu *iommu)
990 struct context_entry *context;
992 spin_lock_irqsave(&iommu->lock, flags);
993 if (!iommu->root_entry) {
996 for (i = 0; i < ROOT_ENTRY_NR; i++) {
997 context = iommu_context_addr(iommu, i, 0, 0);
999 free_pgtable_page(context);
1001 if (!ecs_enabled(iommu))
1004 context = iommu_context_addr(iommu, i, 0x80, 0);
1006 free_pgtable_page(context);
1009 free_pgtable_page(iommu->root_entry);
1010 iommu->root_entry = NULL;
1012 spin_unlock_irqrestore(&iommu->lock, flags);
1015 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
1016 unsigned long pfn, int *target_level)
1018 struct dma_pte *parent, *pte = NULL;
1019 int level = agaw_to_level(domain->agaw);
1022 BUG_ON(!domain->pgd);
1024 if (!domain_pfn_supported(domain, pfn))
1025 /* Address beyond IOMMU's addressing capabilities. */
1028 parent = domain->pgd;
1033 offset = pfn_level_offset(pfn, level);
1034 pte = &parent[offset];
1035 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
1037 if (level == *target_level)
1040 if (!dma_pte_present(pte)) {
1043 tmp_page = alloc_pgtable_page(domain->nid);
1048 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
1049 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
1050 if (cmpxchg64(&pte->val, 0ULL, pteval))
1051 /* Someone else set it while we were thinking; use theirs. */
1052 free_pgtable_page(tmp_page);
1054 domain_flush_cache(domain, pte, sizeof(*pte));
1059 parent = phys_to_virt(dma_pte_addr(pte));
1064 *target_level = level;
1070 /* return address's pte at specific level */
1071 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1073 int level, int *large_page)
1075 struct dma_pte *parent, *pte = NULL;
1076 int total = agaw_to_level(domain->agaw);
1079 parent = domain->pgd;
1080 while (level <= total) {
1081 offset = pfn_level_offset(pfn, total);
1082 pte = &parent[offset];
1086 if (!dma_pte_present(pte)) {
1087 *large_page = total;
1091 if (dma_pte_superpage(pte)) {
1092 *large_page = total;
1096 parent = phys_to_virt(dma_pte_addr(pte));
1102 /* clear last level pte, a tlb flush should be followed */
1103 static void dma_pte_clear_range(struct dmar_domain *domain,
1104 unsigned long start_pfn,
1105 unsigned long last_pfn)
1107 unsigned int large_page = 1;
1108 struct dma_pte *first_pte, *pte;
1110 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1111 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1112 BUG_ON(start_pfn > last_pfn);
1114 /* we don't need lock here; nobody else touches the iova range */
1117 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1119 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1124 start_pfn += lvl_to_nr_pages(large_page);
1126 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1128 domain_flush_cache(domain, first_pte,
1129 (void *)pte - (void *)first_pte);
1131 } while (start_pfn && start_pfn <= last_pfn);
1134 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1135 struct dma_pte *pte, unsigned long pfn,
1136 unsigned long start_pfn, unsigned long last_pfn)
1138 pfn = max(start_pfn, pfn);
1139 pte = &pte[pfn_level_offset(pfn, level)];
1142 unsigned long level_pfn;
1143 struct dma_pte *level_pte;
1145 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1148 level_pfn = pfn & level_mask(level);
1149 level_pte = phys_to_virt(dma_pte_addr(pte));
1152 dma_pte_free_level(domain, level - 1, level_pte,
1153 level_pfn, start_pfn, last_pfn);
1155 /* If range covers entire pagetable, free it */
1156 if (!(start_pfn > level_pfn ||
1157 last_pfn < level_pfn + level_size(level) - 1)) {
1159 domain_flush_cache(domain, pte, sizeof(*pte));
1160 free_pgtable_page(level_pte);
1163 pfn += level_size(level);
1164 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1167 /* clear last level (leaf) ptes and free page table pages. */
1168 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1169 unsigned long start_pfn,
1170 unsigned long last_pfn)
1172 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1173 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1174 BUG_ON(start_pfn > last_pfn);
1176 dma_pte_clear_range(domain, start_pfn, last_pfn);
1178 /* We don't need lock here; nobody else touches the iova range */
1179 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1180 domain->pgd, 0, start_pfn, last_pfn);
1183 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1184 free_pgtable_page(domain->pgd);
1189 /* When a page at a given level is being unlinked from its parent, we don't
1190 need to *modify* it at all. All we need to do is make a list of all the
1191 pages which can be freed just as soon as we've flushed the IOTLB and we
1192 know the hardware page-walk will no longer touch them.
1193 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1195 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1196 int level, struct dma_pte *pte,
1197 struct page *freelist)
1201 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1202 pg->freelist = freelist;
1208 pte = page_address(pg);
1210 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1211 freelist = dma_pte_list_pagetables(domain, level - 1,
1214 } while (!first_pte_in_page(pte));
1219 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1220 struct dma_pte *pte, unsigned long pfn,
1221 unsigned long start_pfn,
1222 unsigned long last_pfn,
1223 struct page *freelist)
1225 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1227 pfn = max(start_pfn, pfn);
1228 pte = &pte[pfn_level_offset(pfn, level)];
1231 unsigned long level_pfn;
1233 if (!dma_pte_present(pte))
1236 level_pfn = pfn & level_mask(level);
1238 /* If range covers entire pagetable, free it */
1239 if (start_pfn <= level_pfn &&
1240 last_pfn >= level_pfn + level_size(level) - 1) {
1241 /* These suborbinate page tables are going away entirely. Don't
1242 bother to clear them; we're just going to *free* them. */
1243 if (level > 1 && !dma_pte_superpage(pte))
1244 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1250 } else if (level > 1) {
1251 /* Recurse down into a level that isn't *entirely* obsolete */
1252 freelist = dma_pte_clear_level(domain, level - 1,
1253 phys_to_virt(dma_pte_addr(pte)),
1254 level_pfn, start_pfn, last_pfn,
1258 pfn += level_size(level);
1259 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1262 domain_flush_cache(domain, first_pte,
1263 (void *)++last_pte - (void *)first_pte);
1268 /* We can't just free the pages because the IOMMU may still be walking
1269 the page tables, and may have cached the intermediate levels. The
1270 pages can only be freed after the IOTLB flush has been done. */
1271 static struct page *domain_unmap(struct dmar_domain *domain,
1272 unsigned long start_pfn,
1273 unsigned long last_pfn)
1275 struct page *freelist = NULL;
1277 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1278 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1279 BUG_ON(start_pfn > last_pfn);
1281 /* we don't need lock here; nobody else touches the iova range */
1282 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1283 domain->pgd, 0, start_pfn, last_pfn, NULL);
1286 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1287 struct page *pgd_page = virt_to_page(domain->pgd);
1288 pgd_page->freelist = freelist;
1289 freelist = pgd_page;
1297 static void dma_free_pagelist(struct page *freelist)
1301 while ((pg = freelist)) {
1302 freelist = pg->freelist;
1303 free_pgtable_page(page_address(pg));
1307 /* iommu handling */
1308 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1310 struct root_entry *root;
1311 unsigned long flags;
1313 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1315 pr_err("Allocating root entry for %s failed\n",
1320 __iommu_flush_cache(iommu, root, ROOT_SIZE);
1322 spin_lock_irqsave(&iommu->lock, flags);
1323 iommu->root_entry = root;
1324 spin_unlock_irqrestore(&iommu->lock, flags);
1329 static void iommu_set_root_entry(struct intel_iommu *iommu)
1335 addr = virt_to_phys(iommu->root_entry);
1336 if (ecs_enabled(iommu))
1337 addr |= DMA_RTADDR_RTT;
1339 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1340 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1342 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1344 /* Make sure hardware complete it */
1345 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1346 readl, (sts & DMA_GSTS_RTPS), sts);
1348 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1351 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1356 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1359 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1360 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1362 /* Make sure hardware complete it */
1363 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1364 readl, (!(val & DMA_GSTS_WBFS)), val);
1366 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1369 /* return value determine if we need a write buffer flush */
1370 static void __iommu_flush_context(struct intel_iommu *iommu,
1371 u16 did, u16 source_id, u8 function_mask,
1378 case DMA_CCMD_GLOBAL_INVL:
1379 val = DMA_CCMD_GLOBAL_INVL;
1381 case DMA_CCMD_DOMAIN_INVL:
1382 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1384 case DMA_CCMD_DEVICE_INVL:
1385 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1386 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1391 val |= DMA_CCMD_ICC;
1393 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1394 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1396 /* Make sure hardware complete it */
1397 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1398 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1400 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1403 /* return value determine if we need a write buffer flush */
1404 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1405 u64 addr, unsigned int size_order, u64 type)
1407 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1408 u64 val = 0, val_iva = 0;
1412 case DMA_TLB_GLOBAL_FLUSH:
1413 /* global flush doesn't need set IVA_REG */
1414 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1416 case DMA_TLB_DSI_FLUSH:
1417 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1419 case DMA_TLB_PSI_FLUSH:
1420 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1421 /* IH bit is passed in as part of address */
1422 val_iva = size_order | addr;
1427 /* Note: set drain read/write */
1430 * This is probably to be super secure.. Looks like we can
1431 * ignore it without any impact.
1433 if (cap_read_drain(iommu->cap))
1434 val |= DMA_TLB_READ_DRAIN;
1436 if (cap_write_drain(iommu->cap))
1437 val |= DMA_TLB_WRITE_DRAIN;
1439 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1440 /* Note: Only uses first TLB reg currently */
1442 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1443 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1445 /* Make sure hardware complete it */
1446 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1447 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1449 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1451 /* check IOTLB invalidation granularity */
1452 if (DMA_TLB_IAIG(val) == 0)
1453 pr_err("Flush IOTLB failed\n");
1454 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1455 pr_debug("TLB flush request %Lx, actual %Lx\n",
1456 (unsigned long long)DMA_TLB_IIRG(type),
1457 (unsigned long long)DMA_TLB_IAIG(val));
1460 static struct device_domain_info *
1461 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1464 struct device_domain_info *info;
1466 assert_spin_locked(&device_domain_lock);
1471 list_for_each_entry(info, &domain->devices, link)
1472 if (info->iommu == iommu && info->bus == bus &&
1473 info->devfn == devfn) {
1474 if (info->ats_supported && info->dev)
1482 static void domain_update_iotlb(struct dmar_domain *domain)
1484 struct device_domain_info *info;
1485 bool has_iotlb_device = false;
1487 assert_spin_locked(&device_domain_lock);
1489 list_for_each_entry(info, &domain->devices, link) {
1490 struct pci_dev *pdev;
1492 if (!info->dev || !dev_is_pci(info->dev))
1495 pdev = to_pci_dev(info->dev);
1496 if (pdev->ats_enabled) {
1497 has_iotlb_device = true;
1502 domain->has_iotlb_device = has_iotlb_device;
1505 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1507 struct pci_dev *pdev;
1509 assert_spin_locked(&device_domain_lock);
1511 if (!info || !dev_is_pci(info->dev))
1514 pdev = to_pci_dev(info->dev);
1515 /* For IOMMU that supports device IOTLB throttling (DIT), we assign
1516 * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
1517 * queue depth at PF level. If DIT is not set, PFSID will be treated as
1518 * reserved, which should be set to 0.
1520 if (!ecap_dit(info->iommu->ecap))
1523 struct pci_dev *pf_pdev;
1525 /* pdev will be returned if device is not a vf */
1526 pf_pdev = pci_physfn(pdev);
1527 info->pfsid = PCI_DEVID(pf_pdev->bus->number, pf_pdev->devfn);
1530 #ifdef CONFIG_INTEL_IOMMU_SVM
1531 /* The PCIe spec, in its wisdom, declares that the behaviour of
1532 the device if you enable PASID support after ATS support is
1533 undefined. So always enable PASID support on devices which
1534 have it, even if we can't yet know if we're ever going to
1536 if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1537 info->pasid_enabled = 1;
1539 if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1540 info->pri_enabled = 1;
1542 if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1543 info->ats_enabled = 1;
1544 domain_update_iotlb(info->domain);
1545 info->ats_qdep = pci_ats_queue_depth(pdev);
1549 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1551 struct pci_dev *pdev;
1553 assert_spin_locked(&device_domain_lock);
1555 if (!dev_is_pci(info->dev))
1558 pdev = to_pci_dev(info->dev);
1560 if (info->ats_enabled) {
1561 pci_disable_ats(pdev);
1562 info->ats_enabled = 0;
1563 domain_update_iotlb(info->domain);
1565 #ifdef CONFIG_INTEL_IOMMU_SVM
1566 if (info->pri_enabled) {
1567 pci_disable_pri(pdev);
1568 info->pri_enabled = 0;
1570 if (info->pasid_enabled) {
1571 pci_disable_pasid(pdev);
1572 info->pasid_enabled = 0;
1577 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1578 u64 addr, unsigned mask)
1581 unsigned long flags;
1582 struct device_domain_info *info;
1584 if (!domain->has_iotlb_device)
1587 spin_lock_irqsave(&device_domain_lock, flags);
1588 list_for_each_entry(info, &domain->devices, link) {
1589 if (!info->ats_enabled)
1592 sid = info->bus << 8 | info->devfn;
1593 qdep = info->ats_qdep;
1594 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1597 spin_unlock_irqrestore(&device_domain_lock, flags);
1600 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1601 struct dmar_domain *domain,
1602 unsigned long pfn, unsigned int pages,
1605 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1606 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1607 u16 did = domain->iommu_did[iommu->seq_id];
1614 * Fallback to domain selective flush if no PSI support or the size is
1616 * PSI requires page size to be 2 ^ x, and the base address is naturally
1617 * aligned to the size
1619 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1620 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1623 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1627 * In caching mode, changes of pages from non-present to present require
1628 * flush. However, device IOTLB doesn't need to be flushed in this case.
1630 if (!cap_caching_mode(iommu->cap) || !map)
1631 iommu_flush_dev_iotlb(domain, addr, mask);
1634 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1637 unsigned long flags;
1639 if (!cap_plmr(iommu->cap) && !cap_phmr(iommu->cap))
1642 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1643 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1644 pmen &= ~DMA_PMEN_EPM;
1645 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1647 /* wait for the protected region status bit to clear */
1648 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1649 readl, !(pmen & DMA_PMEN_PRS), pmen);
1651 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1654 static void iommu_enable_translation(struct intel_iommu *iommu)
1657 unsigned long flags;
1659 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1660 iommu->gcmd |= DMA_GCMD_TE;
1661 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1663 /* Make sure hardware complete it */
1664 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1665 readl, (sts & DMA_GSTS_TES), sts);
1667 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1670 static void iommu_disable_translation(struct intel_iommu *iommu)
1675 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1676 iommu->gcmd &= ~DMA_GCMD_TE;
1677 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1679 /* Make sure hardware complete it */
1680 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1681 readl, (!(sts & DMA_GSTS_TES)), sts);
1683 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1687 static int iommu_init_domains(struct intel_iommu *iommu)
1689 u32 ndomains, nlongs;
1692 ndomains = cap_ndoms(iommu->cap);
1693 pr_debug("%s: Number of Domains supported <%d>\n",
1694 iommu->name, ndomains);
1695 nlongs = BITS_TO_LONGS(ndomains);
1697 spin_lock_init(&iommu->lock);
1699 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1700 if (!iommu->domain_ids) {
1701 pr_err("%s: Allocating domain id array failed\n",
1706 size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1707 iommu->domains = kzalloc(size, GFP_KERNEL);
1709 if (iommu->domains) {
1710 size = 256 * sizeof(struct dmar_domain *);
1711 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1714 if (!iommu->domains || !iommu->domains[0]) {
1715 pr_err("%s: Allocating domain array failed\n",
1717 kfree(iommu->domain_ids);
1718 kfree(iommu->domains);
1719 iommu->domain_ids = NULL;
1720 iommu->domains = NULL;
1727 * If Caching mode is set, then invalid translations are tagged
1728 * with domain-id 0, hence we need to pre-allocate it. We also
1729 * use domain-id 0 as a marker for non-allocated domain-id, so
1730 * make sure it is not used for a real domain.
1732 set_bit(0, iommu->domain_ids);
1737 static void disable_dmar_iommu(struct intel_iommu *iommu)
1739 struct device_domain_info *info, *tmp;
1740 unsigned long flags;
1742 if (!iommu->domains || !iommu->domain_ids)
1746 spin_lock_irqsave(&device_domain_lock, flags);
1747 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1748 struct dmar_domain *domain;
1750 if (info->iommu != iommu)
1753 if (!info->dev || !info->domain)
1756 domain = info->domain;
1758 __dmar_remove_one_dev_info(info);
1760 if (!domain_type_is_vm_or_si(domain)) {
1762 * The domain_exit() function can't be called under
1763 * device_domain_lock, as it takes this lock itself.
1764 * So release the lock here and re-run the loop
1767 spin_unlock_irqrestore(&device_domain_lock, flags);
1768 domain_exit(domain);
1772 spin_unlock_irqrestore(&device_domain_lock, flags);
1774 if (iommu->gcmd & DMA_GCMD_TE)
1775 iommu_disable_translation(iommu);
1778 static void free_dmar_iommu(struct intel_iommu *iommu)
1780 if ((iommu->domains) && (iommu->domain_ids)) {
1781 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1784 for (i = 0; i < elems; i++)
1785 kfree(iommu->domains[i]);
1786 kfree(iommu->domains);
1787 kfree(iommu->domain_ids);
1788 iommu->domains = NULL;
1789 iommu->domain_ids = NULL;
1792 g_iommus[iommu->seq_id] = NULL;
1794 /* free context mapping */
1795 free_context_table(iommu);
1797 #ifdef CONFIG_INTEL_IOMMU_SVM
1798 if (pasid_enabled(iommu)) {
1799 if (ecap_prs(iommu->ecap))
1800 intel_svm_finish_prq(iommu);
1801 intel_svm_free_pasid_tables(iommu);
1806 static struct dmar_domain *alloc_domain(int flags)
1808 struct dmar_domain *domain;
1810 domain = alloc_domain_mem();
1814 memset(domain, 0, sizeof(*domain));
1816 domain->flags = flags;
1817 domain->has_iotlb_device = false;
1818 INIT_LIST_HEAD(&domain->devices);
1823 /* Must be called with iommu->lock */
1824 static int domain_attach_iommu(struct dmar_domain *domain,
1825 struct intel_iommu *iommu)
1827 unsigned long ndomains;
1830 assert_spin_locked(&device_domain_lock);
1831 assert_spin_locked(&iommu->lock);
1833 domain->iommu_refcnt[iommu->seq_id] += 1;
1834 domain->iommu_count += 1;
1835 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1836 ndomains = cap_ndoms(iommu->cap);
1837 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1839 if (num >= ndomains) {
1840 pr_err("%s: No free domain ids\n", iommu->name);
1841 domain->iommu_refcnt[iommu->seq_id] -= 1;
1842 domain->iommu_count -= 1;
1846 set_bit(num, iommu->domain_ids);
1847 set_iommu_domain(iommu, num, domain);
1849 domain->iommu_did[iommu->seq_id] = num;
1850 domain->nid = iommu->node;
1852 domain_update_iommu_cap(domain);
1858 static int domain_detach_iommu(struct dmar_domain *domain,
1859 struct intel_iommu *iommu)
1861 int num, count = INT_MAX;
1863 assert_spin_locked(&device_domain_lock);
1864 assert_spin_locked(&iommu->lock);
1866 domain->iommu_refcnt[iommu->seq_id] -= 1;
1867 count = --domain->iommu_count;
1868 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1869 num = domain->iommu_did[iommu->seq_id];
1870 clear_bit(num, iommu->domain_ids);
1871 set_iommu_domain(iommu, num, NULL);
1873 domain_update_iommu_cap(domain);
1874 domain->iommu_did[iommu->seq_id] = 0;
1880 static struct iova_domain reserved_iova_list;
1881 static struct lock_class_key reserved_rbtree_key;
1883 static int dmar_init_reserved_ranges(void)
1885 struct pci_dev *pdev = NULL;
1889 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1892 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1893 &reserved_rbtree_key);
1895 /* IOAPIC ranges shouldn't be accessed by DMA */
1896 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1897 IOVA_PFN(IOAPIC_RANGE_END));
1899 pr_err("Reserve IOAPIC range failed\n");
1903 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1904 for_each_pci_dev(pdev) {
1907 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1908 r = &pdev->resource[i];
1909 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1911 iova = reserve_iova(&reserved_iova_list,
1915 pr_err("Reserve iova failed\n");
1923 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1925 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1928 static inline int guestwidth_to_adjustwidth(int gaw)
1931 int r = (gaw - 12) % 9;
1942 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1945 int adjust_width, agaw;
1946 unsigned long sagaw;
1948 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1950 domain_reserve_special_ranges(domain);
1952 /* calculate AGAW */
1953 if (guest_width > cap_mgaw(iommu->cap))
1954 guest_width = cap_mgaw(iommu->cap);
1955 domain->gaw = guest_width;
1956 adjust_width = guestwidth_to_adjustwidth(guest_width);
1957 agaw = width_to_agaw(adjust_width);
1958 sagaw = cap_sagaw(iommu->cap);
1959 if (!test_bit(agaw, &sagaw)) {
1960 /* hardware doesn't support it, choose a bigger one */
1961 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1962 agaw = find_next_bit(&sagaw, 5, agaw);
1966 domain->agaw = agaw;
1968 if (ecap_coherent(iommu->ecap))
1969 domain->iommu_coherency = 1;
1971 domain->iommu_coherency = 0;
1973 if (ecap_sc_support(iommu->ecap))
1974 domain->iommu_snooping = 1;
1976 domain->iommu_snooping = 0;
1978 if (intel_iommu_superpage)
1979 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1981 domain->iommu_superpage = 0;
1983 domain->nid = iommu->node;
1985 /* always allocate the top pgd */
1986 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1989 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1993 static void domain_exit(struct dmar_domain *domain)
1995 struct page *freelist = NULL;
1997 /* Domain 0 is reserved, so dont process it */
2001 /* Flush any lazy unmaps that may reference this domain */
2002 if (!intel_iommu_strict) {
2005 for_each_possible_cpu(cpu)
2006 flush_unmaps_timeout(cpu);
2009 /* Remove associated devices and clear attached or cached domains */
2011 domain_remove_dev_info(domain);
2015 put_iova_domain(&domain->iovad);
2017 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
2019 dma_free_pagelist(freelist);
2021 free_domain_mem(domain);
2024 static int domain_context_mapping_one(struct dmar_domain *domain,
2025 struct intel_iommu *iommu,
2028 u16 did = domain->iommu_did[iommu->seq_id];
2029 int translation = CONTEXT_TT_MULTI_LEVEL;
2030 struct device_domain_info *info = NULL;
2031 struct context_entry *context;
2032 unsigned long flags;
2033 struct dma_pte *pgd;
2038 if (hw_pass_through && domain_type_is_si(domain))
2039 translation = CONTEXT_TT_PASS_THROUGH;
2041 pr_debug("Set context mapping for %02x:%02x.%d\n",
2042 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
2044 BUG_ON(!domain->pgd);
2046 spin_lock_irqsave(&device_domain_lock, flags);
2047 spin_lock(&iommu->lock);
2050 context = iommu_context_addr(iommu, bus, devfn, 1);
2055 if (context_present(context))
2059 * For kdump cases, old valid entries may be cached due to the
2060 * in-flight DMA and copied pgtable, but there is no unmapping
2061 * behaviour for them, thus we need an explicit cache flush for
2062 * the newly-mapped device. For kdump, at this point, the device
2063 * is supposed to finish reset at its driver probe stage, so no
2064 * in-flight DMA will exist, and we don't need to worry anymore
2067 if (context_copied(context)) {
2068 u16 did_old = context_domain_id(context);
2070 if (did_old >= 0 && did_old < cap_ndoms(iommu->cap)) {
2071 iommu->flush.flush_context(iommu, did_old,
2072 (((u16)bus) << 8) | devfn,
2073 DMA_CCMD_MASK_NOBIT,
2074 DMA_CCMD_DEVICE_INVL);
2075 iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
2082 context_clear_entry(context);
2083 context_set_domain_id(context, did);
2086 * Skip top levels of page tables for iommu which has less agaw
2087 * than default. Unnecessary for PT mode.
2089 if (translation != CONTEXT_TT_PASS_THROUGH) {
2090 for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
2092 pgd = phys_to_virt(dma_pte_addr(pgd));
2093 if (!dma_pte_present(pgd))
2097 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2098 if (info && info->ats_supported)
2099 translation = CONTEXT_TT_DEV_IOTLB;
2101 translation = CONTEXT_TT_MULTI_LEVEL;
2103 context_set_address_root(context, virt_to_phys(pgd));
2104 context_set_address_width(context, agaw);
2107 * In pass through mode, AW must be programmed to
2108 * indicate the largest AGAW value supported by
2109 * hardware. And ASR is ignored by hardware.
2111 context_set_address_width(context, iommu->msagaw);
2114 context_set_translation_type(context, translation);
2115 context_set_fault_enable(context);
2116 context_set_present(context);
2117 domain_flush_cache(domain, context, sizeof(*context));
2120 * It's a non-present to present mapping. If hardware doesn't cache
2121 * non-present entry we only need to flush the write-buffer. If the
2122 * _does_ cache non-present entries, then it does so in the special
2123 * domain #0, which we have to flush:
2125 if (cap_caching_mode(iommu->cap)) {
2126 iommu->flush.flush_context(iommu, 0,
2127 (((u16)bus) << 8) | devfn,
2128 DMA_CCMD_MASK_NOBIT,
2129 DMA_CCMD_DEVICE_INVL);
2130 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2132 iommu_flush_write_buffer(iommu);
2134 iommu_enable_dev_iotlb(info);
2139 spin_unlock(&iommu->lock);
2140 spin_unlock_irqrestore(&device_domain_lock, flags);
2145 struct domain_context_mapping_data {
2146 struct dmar_domain *domain;
2147 struct intel_iommu *iommu;
2150 static int domain_context_mapping_cb(struct pci_dev *pdev,
2151 u16 alias, void *opaque)
2153 struct domain_context_mapping_data *data = opaque;
2155 return domain_context_mapping_one(data->domain, data->iommu,
2156 PCI_BUS_NUM(alias), alias & 0xff);
2160 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2162 struct intel_iommu *iommu;
2164 struct domain_context_mapping_data data;
2166 iommu = device_to_iommu(dev, &bus, &devfn);
2170 if (!dev_is_pci(dev))
2171 return domain_context_mapping_one(domain, iommu, bus, devfn);
2173 data.domain = domain;
2176 return pci_for_each_dma_alias(to_pci_dev(dev),
2177 &domain_context_mapping_cb, &data);
2180 static int domain_context_mapped_cb(struct pci_dev *pdev,
2181 u16 alias, void *opaque)
2183 struct intel_iommu *iommu = opaque;
2185 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2188 static int domain_context_mapped(struct device *dev)
2190 struct intel_iommu *iommu;
2193 iommu = device_to_iommu(dev, &bus, &devfn);
2197 if (!dev_is_pci(dev))
2198 return device_context_mapped(iommu, bus, devfn);
2200 return !pci_for_each_dma_alias(to_pci_dev(dev),
2201 domain_context_mapped_cb, iommu);
2204 /* Returns a number of VTD pages, but aligned to MM page size */
2205 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2208 host_addr &= ~PAGE_MASK;
2209 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2212 /* Return largest possible superpage level for a given mapping */
2213 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2214 unsigned long iov_pfn,
2215 unsigned long phy_pfn,
2216 unsigned long pages)
2218 int support, level = 1;
2219 unsigned long pfnmerge;
2221 support = domain->iommu_superpage;
2223 /* To use a large page, the virtual *and* physical addresses
2224 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2225 of them will mean we have to use smaller pages. So just
2226 merge them and check both at once. */
2227 pfnmerge = iov_pfn | phy_pfn;
2229 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2230 pages >>= VTD_STRIDE_SHIFT;
2233 pfnmerge >>= VTD_STRIDE_SHIFT;
2240 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2241 struct scatterlist *sg, unsigned long phys_pfn,
2242 unsigned long nr_pages, int prot)
2244 struct dma_pte *first_pte = NULL, *pte = NULL;
2245 phys_addr_t uninitialized_var(pteval);
2246 unsigned long sg_res = 0;
2247 unsigned int largepage_lvl = 0;
2248 unsigned long lvl_pages = 0;
2250 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2252 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2255 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2259 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2262 while (nr_pages > 0) {
2266 unsigned int pgoff = sg->offset & ~PAGE_MASK;
2268 sg_res = aligned_nrpages(sg->offset, sg->length);
2269 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
2270 sg->dma_length = sg->length;
2271 pteval = (sg_phys(sg) - pgoff) | prot;
2272 phys_pfn = pteval >> VTD_PAGE_SHIFT;
2276 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2278 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2281 /* It is large page*/
2282 if (largepage_lvl > 1) {
2283 unsigned long nr_superpages, end_pfn;
2285 pteval |= DMA_PTE_LARGE_PAGE;
2286 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2288 nr_superpages = sg_res / lvl_pages;
2289 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2292 * Ensure that old small page tables are
2293 * removed to make room for superpage(s).
2295 dma_pte_free_pagetable(domain, iov_pfn, end_pfn);
2297 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2301 /* We don't need lock here, nobody else
2302 * touches the iova range
2304 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2306 static int dumps = 5;
2307 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2308 iov_pfn, tmp, (unsigned long long)pteval);
2311 debug_dma_dump_mappings(NULL);
2316 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2318 BUG_ON(nr_pages < lvl_pages);
2319 BUG_ON(sg_res < lvl_pages);
2321 nr_pages -= lvl_pages;
2322 iov_pfn += lvl_pages;
2323 phys_pfn += lvl_pages;
2324 pteval += lvl_pages * VTD_PAGE_SIZE;
2325 sg_res -= lvl_pages;
2327 /* If the next PTE would be the first in a new page, then we
2328 need to flush the cache on the entries we've just written.
2329 And then we'll need to recalculate 'pte', so clear it and
2330 let it get set again in the if (!pte) block above.
2332 If we're done (!nr_pages) we need to flush the cache too.
2334 Also if we've been setting superpages, we may need to
2335 recalculate 'pte' and switch back to smaller pages for the
2336 end of the mapping, if the trailing size is not enough to
2337 use another superpage (i.e. sg_res < lvl_pages). */
2339 if (!nr_pages || first_pte_in_page(pte) ||
2340 (largepage_lvl > 1 && sg_res < lvl_pages)) {
2341 domain_flush_cache(domain, first_pte,
2342 (void *)pte - (void *)first_pte);
2346 if (!sg_res && nr_pages)
2352 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2353 struct scatterlist *sg, unsigned long nr_pages,
2356 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2359 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2360 unsigned long phys_pfn, unsigned long nr_pages,
2363 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2366 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2371 clear_context_table(iommu, bus, devfn);
2372 iommu->flush.flush_context(iommu, 0, 0, 0,
2373 DMA_CCMD_GLOBAL_INVL);
2374 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2377 static inline void unlink_domain_info(struct device_domain_info *info)
2379 assert_spin_locked(&device_domain_lock);
2380 list_del(&info->link);
2381 list_del(&info->global);
2383 info->dev->archdata.iommu = NULL;
2386 static void domain_remove_dev_info(struct dmar_domain *domain)
2388 struct device_domain_info *info, *tmp;
2389 unsigned long flags;
2391 spin_lock_irqsave(&device_domain_lock, flags);
2392 list_for_each_entry_safe(info, tmp, &domain->devices, link)
2393 __dmar_remove_one_dev_info(info);
2394 spin_unlock_irqrestore(&device_domain_lock, flags);
2399 * Note: we use struct device->archdata.iommu stores the info
2401 static struct dmar_domain *find_domain(struct device *dev)
2403 struct device_domain_info *info;
2405 /* No lock here, assumes no domain exit in normal case */
2406 info = dev->archdata.iommu;
2408 return info->domain;
2412 static inline struct device_domain_info *
2413 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2415 struct device_domain_info *info;
2417 list_for_each_entry(info, &device_domain_list, global)
2418 if (info->iommu->segment == segment && info->bus == bus &&
2419 info->devfn == devfn)
2425 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2428 struct dmar_domain *domain)
2430 struct dmar_domain *found = NULL;
2431 struct device_domain_info *info;
2432 unsigned long flags;
2435 info = alloc_devinfo_mem();
2440 info->devfn = devfn;
2441 info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2442 info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2445 info->domain = domain;
2446 info->iommu = iommu;
2448 if (dev && dev_is_pci(dev)) {
2449 struct pci_dev *pdev = to_pci_dev(info->dev);
2451 if (ecap_dev_iotlb_support(iommu->ecap) &&
2452 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2453 dmar_find_matched_atsr_unit(pdev))
2454 info->ats_supported = 1;
2456 if (ecs_enabled(iommu)) {
2457 if (pasid_enabled(iommu)) {
2458 int features = pci_pasid_features(pdev);
2460 info->pasid_supported = features | 1;
2463 if (info->ats_supported && ecap_prs(iommu->ecap) &&
2464 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2465 info->pri_supported = 1;
2469 spin_lock_irqsave(&device_domain_lock, flags);
2471 found = find_domain(dev);
2474 struct device_domain_info *info2;
2475 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2477 found = info2->domain;
2483 spin_unlock_irqrestore(&device_domain_lock, flags);
2484 free_devinfo_mem(info);
2485 /* Caller must free the original domain */
2489 spin_lock(&iommu->lock);
2490 ret = domain_attach_iommu(domain, iommu);
2491 spin_unlock(&iommu->lock);
2494 spin_unlock_irqrestore(&device_domain_lock, flags);
2495 free_devinfo_mem(info);
2499 list_add(&info->link, &domain->devices);
2500 list_add(&info->global, &device_domain_list);
2502 dev->archdata.iommu = info;
2503 spin_unlock_irqrestore(&device_domain_lock, flags);
2505 if (dev && domain_context_mapping(domain, dev)) {
2506 pr_err("Domain context map for %s failed\n", dev_name(dev));
2507 dmar_remove_one_dev_info(domain, dev);
2514 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2516 *(u16 *)opaque = alias;
2520 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2522 struct device_domain_info *info = NULL;
2523 struct dmar_domain *domain = NULL;
2524 struct intel_iommu *iommu;
2525 u16 req_id, dma_alias;
2526 unsigned long flags;
2529 iommu = device_to_iommu(dev, &bus, &devfn);
2533 req_id = ((u16)bus << 8) | devfn;
2535 if (dev_is_pci(dev)) {
2536 struct pci_dev *pdev = to_pci_dev(dev);
2538 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2540 spin_lock_irqsave(&device_domain_lock, flags);
2541 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2542 PCI_BUS_NUM(dma_alias),
2545 iommu = info->iommu;
2546 domain = info->domain;
2548 spin_unlock_irqrestore(&device_domain_lock, flags);
2550 /* DMA alias already has a domain, use it */
2555 /* Allocate and initialize new domain for the device */
2556 domain = alloc_domain(0);
2559 if (domain_init(domain, iommu, gaw)) {
2560 domain_exit(domain);
2569 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2570 struct dmar_domain *domain)
2572 struct intel_iommu *iommu;
2573 struct dmar_domain *tmp;
2574 u16 req_id, dma_alias;
2577 iommu = device_to_iommu(dev, &bus, &devfn);
2581 req_id = ((u16)bus << 8) | devfn;
2583 if (dev_is_pci(dev)) {
2584 struct pci_dev *pdev = to_pci_dev(dev);
2586 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2588 /* register PCI DMA alias device */
2589 if (req_id != dma_alias) {
2590 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2591 dma_alias & 0xff, NULL, domain);
2593 if (!tmp || tmp != domain)
2598 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2599 if (!tmp || tmp != domain)
2605 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2607 struct dmar_domain *domain, *tmp;
2609 domain = find_domain(dev);
2613 domain = find_or_alloc_domain(dev, gaw);
2617 tmp = set_domain_for_dev(dev, domain);
2618 if (!tmp || domain != tmp) {
2619 domain_exit(domain);
2628 static int iommu_domain_identity_map(struct dmar_domain *domain,
2629 unsigned long long start,
2630 unsigned long long end)
2632 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2633 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2635 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2636 dma_to_mm_pfn(last_vpfn))) {
2637 pr_err("Reserving iova failed\n");
2641 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2643 * RMRR range might have overlap with physical memory range,
2646 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2648 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2649 last_vpfn - first_vpfn + 1,
2650 DMA_PTE_READ|DMA_PTE_WRITE);
2653 static int domain_prepare_identity_map(struct device *dev,
2654 struct dmar_domain *domain,
2655 unsigned long long start,
2656 unsigned long long end)
2658 /* For _hardware_ passthrough, don't bother. But for software
2659 passthrough, we do it anyway -- it may indicate a memory
2660 range which is reserved in E820, so which didn't get set
2661 up to start with in si_domain */
2662 if (domain == si_domain && hw_pass_through) {
2663 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2664 dev_name(dev), start, end);
2668 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2669 dev_name(dev), start, end);
2672 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2673 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2674 dmi_get_system_info(DMI_BIOS_VENDOR),
2675 dmi_get_system_info(DMI_BIOS_VERSION),
2676 dmi_get_system_info(DMI_PRODUCT_VERSION));
2680 if (end >> agaw_to_width(domain->agaw)) {
2681 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2682 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2683 agaw_to_width(domain->agaw),
2684 dmi_get_system_info(DMI_BIOS_VENDOR),
2685 dmi_get_system_info(DMI_BIOS_VERSION),
2686 dmi_get_system_info(DMI_PRODUCT_VERSION));
2690 return iommu_domain_identity_map(domain, start, end);
2693 static int iommu_prepare_identity_map(struct device *dev,
2694 unsigned long long start,
2695 unsigned long long end)
2697 struct dmar_domain *domain;
2700 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2704 ret = domain_prepare_identity_map(dev, domain, start, end);
2706 domain_exit(domain);
2711 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2714 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2716 return iommu_prepare_identity_map(dev, rmrr->base_address,
2720 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2721 static inline void iommu_prepare_isa(void)
2723 struct pci_dev *pdev;
2726 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2730 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2731 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2734 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2739 static inline void iommu_prepare_isa(void)
2743 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2745 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2747 static int __init si_domain_init(int hw)
2751 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2755 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2756 domain_exit(si_domain);
2760 pr_debug("Identity mapping domain allocated\n");
2765 for_each_online_node(nid) {
2766 unsigned long start_pfn, end_pfn;
2769 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2770 ret = iommu_domain_identity_map(si_domain,
2771 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2780 static int identity_mapping(struct device *dev)
2782 struct device_domain_info *info;
2784 if (likely(!iommu_identity_mapping))
2787 info = dev->archdata.iommu;
2788 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2789 return (info->domain == si_domain);
2794 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2796 struct dmar_domain *ndomain;
2797 struct intel_iommu *iommu;
2800 iommu = device_to_iommu(dev, &bus, &devfn);
2804 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2805 if (ndomain != domain)
2811 static bool device_has_rmrr(struct device *dev)
2813 struct dmar_rmrr_unit *rmrr;
2818 for_each_rmrr_units(rmrr) {
2820 * Return TRUE if this RMRR contains the device that
2823 for_each_active_dev_scope(rmrr->devices,
2824 rmrr->devices_cnt, i, tmp)
2835 * There are a couple cases where we need to restrict the functionality of
2836 * devices associated with RMRRs. The first is when evaluating a device for
2837 * identity mapping because problems exist when devices are moved in and out
2838 * of domains and their respective RMRR information is lost. This means that
2839 * a device with associated RMRRs will never be in a "passthrough" domain.
2840 * The second is use of the device through the IOMMU API. This interface
2841 * expects to have full control of the IOVA space for the device. We cannot
2842 * satisfy both the requirement that RMRR access is maintained and have an
2843 * unencumbered IOVA space. We also have no ability to quiesce the device's
2844 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2845 * We therefore prevent devices associated with an RMRR from participating in
2846 * the IOMMU API, which eliminates them from device assignment.
2848 * In both cases we assume that PCI USB devices with RMRRs have them largely
2849 * for historical reasons and that the RMRR space is not actively used post
2850 * boot. This exclusion may change if vendors begin to abuse it.
2852 * The same exception is made for graphics devices, with the requirement that
2853 * any use of the RMRR regions will be torn down before assigning the device
2856 static bool device_is_rmrr_locked(struct device *dev)
2858 if (!device_has_rmrr(dev))
2861 if (dev_is_pci(dev)) {
2862 struct pci_dev *pdev = to_pci_dev(dev);
2864 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2871 static int iommu_should_identity_map(struct device *dev, int startup)
2874 if (dev_is_pci(dev)) {
2875 struct pci_dev *pdev = to_pci_dev(dev);
2877 if (device_is_rmrr_locked(dev))
2880 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2883 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2886 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2890 * We want to start off with all devices in the 1:1 domain, and
2891 * take them out later if we find they can't access all of memory.
2893 * However, we can't do this for PCI devices behind bridges,
2894 * because all PCI devices behind the same bridge will end up
2895 * with the same source-id on their transactions.
2897 * Practically speaking, we can't change things around for these
2898 * devices at run-time, because we can't be sure there'll be no
2899 * DMA transactions in flight for any of their siblings.
2901 * So PCI devices (unless they're on the root bus) as well as
2902 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2903 * the 1:1 domain, just in _case_ one of their siblings turns out
2904 * not to be able to map all of memory.
2906 if (!pci_is_pcie(pdev)) {
2907 if (!pci_is_root_bus(pdev->bus))
2909 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2911 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2914 if (device_has_rmrr(dev))
2919 * At boot time, we don't yet know if devices will be 64-bit capable.
2920 * Assume that they will — if they turn out not to be, then we can
2921 * take them out of the 1:1 domain later.
2925 * If the device's dma_mask is less than the system's memory
2926 * size then this is not a candidate for identity mapping.
2928 u64 dma_mask = *dev->dma_mask;
2930 if (dev->coherent_dma_mask &&
2931 dev->coherent_dma_mask < dma_mask)
2932 dma_mask = dev->coherent_dma_mask;
2934 return dma_mask >= dma_get_required_mask(dev);
2940 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2944 if (!iommu_should_identity_map(dev, 1))
2947 ret = domain_add_dev_info(si_domain, dev);
2949 pr_info("%s identity mapping for device %s\n",
2950 hw ? "Hardware" : "Software", dev_name(dev));
2951 else if (ret == -ENODEV)
2952 /* device not associated with an iommu */
2959 static int __init iommu_prepare_static_identity_mapping(int hw)
2961 struct pci_dev *pdev = NULL;
2962 struct dmar_drhd_unit *drhd;
2963 struct intel_iommu *iommu;
2968 for_each_pci_dev(pdev) {
2969 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2974 for_each_active_iommu(iommu, drhd)
2975 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2976 struct acpi_device_physical_node *pn;
2977 struct acpi_device *adev;
2979 if (dev->bus != &acpi_bus_type)
2982 adev= to_acpi_device(dev);
2983 mutex_lock(&adev->physical_node_lock);
2984 list_for_each_entry(pn, &adev->physical_node_list, node) {
2985 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2989 mutex_unlock(&adev->physical_node_lock);
2997 static void intel_iommu_init_qi(struct intel_iommu *iommu)
3000 * Start from the sane iommu hardware state.
3001 * If the queued invalidation is already initialized by us
3002 * (for example, while enabling interrupt-remapping) then
3003 * we got the things already rolling from a sane state.
3007 * Clear any previous faults.
3009 dmar_fault(-1, iommu);
3011 * Disable queued invalidation if supported and already enabled
3012 * before OS handover.
3014 dmar_disable_qi(iommu);
3017 if (dmar_enable_qi(iommu)) {
3019 * Queued Invalidate not enabled, use Register Based Invalidate
3021 iommu->flush.flush_context = __iommu_flush_context;
3022 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
3023 pr_info("%s: Using Register based invalidation\n",
3026 iommu->flush.flush_context = qi_flush_context;
3027 iommu->flush.flush_iotlb = qi_flush_iotlb;
3028 pr_info("%s: Using Queued invalidation\n", iommu->name);
3032 static int copy_context_table(struct intel_iommu *iommu,
3033 struct root_entry *old_re,
3034 struct context_entry **tbl,
3037 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
3038 struct context_entry *new_ce = NULL, ce;
3039 struct context_entry *old_ce = NULL;
3040 struct root_entry re;
3041 phys_addr_t old_ce_phys;
3043 tbl_idx = ext ? bus * 2 : bus;
3044 memcpy(&re, old_re, sizeof(re));
3046 for (devfn = 0; devfn < 256; devfn++) {
3047 /* First calculate the correct index */
3048 idx = (ext ? devfn * 2 : devfn) % 256;
3051 /* First save what we may have and clean up */
3053 tbl[tbl_idx] = new_ce;
3054 __iommu_flush_cache(iommu, new_ce,
3064 old_ce_phys = root_entry_lctp(&re);
3066 old_ce_phys = root_entry_uctp(&re);
3069 if (ext && devfn == 0) {
3070 /* No LCTP, try UCTP */
3079 old_ce = memremap(old_ce_phys, PAGE_SIZE,
3084 new_ce = alloc_pgtable_page(iommu->node);
3091 /* Now copy the context entry */
3092 memcpy(&ce, old_ce + idx, sizeof(ce));
3094 if (!__context_present(&ce))
3097 did = context_domain_id(&ce);
3098 if (did >= 0 && did < cap_ndoms(iommu->cap))
3099 set_bit(did, iommu->domain_ids);
3102 * We need a marker for copied context entries. This
3103 * marker needs to work for the old format as well as
3104 * for extended context entries.
3106 * Bit 67 of the context entry is used. In the old
3107 * format this bit is available to software, in the
3108 * extended format it is the PGE bit, but PGE is ignored
3109 * by HW if PASIDs are disabled (and thus still
3112 * So disable PASIDs first and then mark the entry
3113 * copied. This means that we don't copy PASID
3114 * translations from the old kernel, but this is fine as
3115 * faults there are not fatal.
3117 context_clear_pasid_enable(&ce);
3118 context_set_copied(&ce);
3123 tbl[tbl_idx + pos] = new_ce;
3125 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3134 static int copy_translation_tables(struct intel_iommu *iommu)
3136 struct context_entry **ctxt_tbls;
3137 struct root_entry *old_rt;
3138 phys_addr_t old_rt_phys;
3139 int ctxt_table_entries;
3140 unsigned long flags;
3145 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3146 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
3147 new_ext = !!ecap_ecs(iommu->ecap);
3150 * The RTT bit can only be changed when translation is disabled,
3151 * but disabling translation means to open a window for data
3152 * corruption. So bail out and don't copy anything if we would
3153 * have to change the bit.
3158 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3162 old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3166 /* This is too big for the stack - allocate it from slab */
3167 ctxt_table_entries = ext ? 512 : 256;
3169 ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
3173 for (bus = 0; bus < 256; bus++) {
3174 ret = copy_context_table(iommu, &old_rt[bus],
3175 ctxt_tbls, bus, ext);
3177 pr_err("%s: Failed to copy context table for bus %d\n",
3183 spin_lock_irqsave(&iommu->lock, flags);
3185 /* Context tables are copied, now write them to the root_entry table */
3186 for (bus = 0; bus < 256; bus++) {
3187 int idx = ext ? bus * 2 : bus;
3190 if (ctxt_tbls[idx]) {
3191 val = virt_to_phys(ctxt_tbls[idx]) | 1;
3192 iommu->root_entry[bus].lo = val;
3195 if (!ext || !ctxt_tbls[idx + 1])
3198 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3199 iommu->root_entry[bus].hi = val;
3202 spin_unlock_irqrestore(&iommu->lock, flags);
3206 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3216 static int __init init_dmars(void)
3218 struct dmar_drhd_unit *drhd;
3219 struct dmar_rmrr_unit *rmrr;
3220 bool copied_tables = false;
3222 struct intel_iommu *iommu;
3228 * initialize and program root entry to not present
3231 for_each_drhd_unit(drhd) {
3233 * lock not needed as this is only incremented in the single
3234 * threaded kernel __init code path all other access are read
3237 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3241 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3244 /* Preallocate enough resources for IOMMU hot-addition */
3245 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3246 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3248 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3251 pr_err("Allocating global iommu array failed\n");
3256 for_each_possible_cpu(cpu) {
3257 struct deferred_flush_data *dfd = per_cpu_ptr(&deferred_flush,
3260 dfd->tables = kzalloc(g_num_of_iommus *
3261 sizeof(struct deferred_flush_table),
3268 spin_lock_init(&dfd->lock);
3269 setup_timer(&dfd->timer, flush_unmaps_timeout, cpu);
3272 for_each_active_iommu(iommu, drhd) {
3273 g_iommus[iommu->seq_id] = iommu;
3275 intel_iommu_init_qi(iommu);
3277 ret = iommu_init_domains(iommu);
3281 init_translation_status(iommu);
3283 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3284 iommu_disable_translation(iommu);
3285 clear_translation_pre_enabled(iommu);
3286 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3292 * we could share the same root & context tables
3293 * among all IOMMU's. Need to Split it later.
3295 ret = iommu_alloc_root_entry(iommu);
3299 if (translation_pre_enabled(iommu)) {
3300 pr_info("Translation already enabled - trying to copy translation structures\n");
3302 ret = copy_translation_tables(iommu);
3305 * We found the IOMMU with translation
3306 * enabled - but failed to copy over the
3307 * old root-entry table. Try to proceed
3308 * by disabling translation now and
3309 * allocating a clean root-entry table.
3310 * This might cause DMAR faults, but
3311 * probably the dump will still succeed.
3313 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3315 iommu_disable_translation(iommu);
3316 clear_translation_pre_enabled(iommu);
3318 pr_info("Copied translation tables from previous kernel for %s\n",
3320 copied_tables = true;
3324 if (!ecap_pass_through(iommu->ecap))
3325 hw_pass_through = 0;
3327 if (!intel_iommu_strict && cap_caching_mode(iommu->cap)) {
3328 pr_info("Disable batched IOTLB flush due to virtualization");
3329 intel_iommu_strict = 1;
3332 #ifdef CONFIG_INTEL_IOMMU_SVM
3333 if (pasid_enabled(iommu))
3334 intel_svm_alloc_pasid_tables(iommu);
3339 * Now that qi is enabled on all iommus, set the root entry and flush
3340 * caches. This is required on some Intel X58 chipsets, otherwise the
3341 * flush_context function will loop forever and the boot hangs.
3343 for_each_active_iommu(iommu, drhd) {
3344 iommu_flush_write_buffer(iommu);
3345 iommu_set_root_entry(iommu);
3346 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3347 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3350 if (iommu_pass_through)
3351 iommu_identity_mapping |= IDENTMAP_ALL;
3353 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3358 iommu_identity_mapping |= IDENTMAP_GFX;
3360 check_tylersburg_isoch();
3362 if (iommu_identity_mapping) {
3363 ret = si_domain_init(hw_pass_through);
3370 * If we copied translations from a previous kernel in the kdump
3371 * case, we can not assign the devices to domains now, as that
3372 * would eliminate the old mappings. So skip this part and defer
3373 * the assignment to device driver initialization time.
3379 * If pass through is not set or not enabled, setup context entries for
3380 * identity mappings for rmrr, gfx, and isa and may fall back to static
3381 * identity mapping if iommu_identity_mapping is set.
3383 if (iommu_identity_mapping) {
3384 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3386 pr_crit("Failed to setup IOMMU pass-through\n");
3392 * for each dev attached to rmrr
3394 * locate drhd for dev, alloc domain for dev
3395 * allocate free domain
3396 * allocate page table entries for rmrr
3397 * if context not allocated for bus
3398 * allocate and init context
3399 * set present in root table for this bus
3400 * init context with domain, translation etc
3404 pr_info("Setting RMRR:\n");
3405 for_each_rmrr_units(rmrr) {
3406 /* some BIOS lists non-exist devices in DMAR table. */
3407 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3409 ret = iommu_prepare_rmrr_dev(rmrr, dev);
3411 pr_err("Mapping reserved region failed\n");
3415 iommu_prepare_isa();
3422 * global invalidate context cache
3423 * global invalidate iotlb
3424 * enable translation
3426 for_each_iommu(iommu, drhd) {
3427 if (drhd->ignored) {
3429 * we always have to disable PMRs or DMA may fail on
3433 iommu_disable_protect_mem_regions(iommu);
3437 iommu_flush_write_buffer(iommu);
3439 #ifdef CONFIG_INTEL_IOMMU_SVM
3440 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3441 ret = intel_svm_enable_prq(iommu);
3446 ret = dmar_set_interrupt(iommu);
3450 if (!translation_pre_enabled(iommu))
3451 iommu_enable_translation(iommu);
3453 iommu_disable_protect_mem_regions(iommu);
3459 for_each_active_iommu(iommu, drhd) {
3460 disable_dmar_iommu(iommu);
3461 free_dmar_iommu(iommu);
3464 for_each_possible_cpu(cpu)
3465 kfree(per_cpu_ptr(&deferred_flush, cpu)->tables);
3471 /* This takes a number of _MM_ pages, not VTD pages */
3472 static unsigned long intel_alloc_iova(struct device *dev,
3473 struct dmar_domain *domain,
3474 unsigned long nrpages, uint64_t dma_mask)
3476 unsigned long iova_pfn = 0;
3478 /* Restrict dma_mask to the width that the iommu can handle */
3479 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3480 /* Ensure we reserve the whole size-aligned region */
3481 nrpages = __roundup_pow_of_two(nrpages);
3483 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3485 * First try to allocate an io virtual address in
3486 * DMA_BIT_MASK(32) and if that fails then try allocating
3489 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3490 IOVA_PFN(DMA_BIT_MASK(32)));
3494 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages, IOVA_PFN(dma_mask));
3495 if (unlikely(!iova_pfn)) {
3496 pr_err("Allocating %ld-page iova for %s failed",
3497 nrpages, dev_name(dev));
3504 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
3506 struct dmar_domain *domain, *tmp;
3507 struct dmar_rmrr_unit *rmrr;
3508 struct device *i_dev;
3511 domain = find_domain(dev);
3515 domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3519 /* We have a new domain - setup possible RMRRs for the device */
3521 for_each_rmrr_units(rmrr) {
3522 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3527 ret = domain_prepare_identity_map(dev, domain,
3531 dev_err(dev, "Mapping reserved region failed\n");
3536 tmp = set_domain_for_dev(dev, domain);
3537 if (!tmp || domain != tmp) {
3538 domain_exit(domain);
3545 pr_err("Allocating domain for %s failed\n", dev_name(dev));
3551 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3553 struct device_domain_info *info;
3555 /* No lock here, assumes no domain exit in normal case */
3556 info = dev->archdata.iommu;
3558 return info->domain;
3560 return __get_valid_domain_for_dev(dev);
3563 /* Check if the dev needs to go through non-identity map and unmap process.*/
3564 static int iommu_no_mapping(struct device *dev)
3568 if (iommu_dummy(dev))
3571 if (!iommu_identity_mapping)
3574 found = identity_mapping(dev);
3576 if (iommu_should_identity_map(dev, 0))
3580 * 32 bit DMA is removed from si_domain and fall back
3581 * to non-identity mapping.
3583 dmar_remove_one_dev_info(si_domain, dev);
3584 pr_info("32bit %s uses non-identity mapping\n",
3590 * In case of a detached 64 bit DMA device from vm, the device
3591 * is put into si_domain for identity mapping.
3593 if (iommu_should_identity_map(dev, 0)) {
3595 ret = domain_add_dev_info(si_domain, dev);
3597 pr_info("64bit %s uses identity mapping\n",
3607 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3608 size_t size, int dir, u64 dma_mask)
3610 struct dmar_domain *domain;
3611 phys_addr_t start_paddr;
3612 unsigned long iova_pfn;
3615 struct intel_iommu *iommu;
3616 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3618 BUG_ON(dir == DMA_NONE);
3620 if (iommu_no_mapping(dev))
3623 domain = get_valid_domain_for_dev(dev);
3627 iommu = domain_get_iommu(domain);
3628 size = aligned_nrpages(paddr, size);
3630 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3635 * Check if DMAR supports zero-length reads on write only
3638 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3639 !cap_zlr(iommu->cap))
3640 prot |= DMA_PTE_READ;
3641 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3642 prot |= DMA_PTE_WRITE;
3644 * paddr - (paddr + size) might be partial page, we should map the whole
3645 * page. Note: if two part of one page are separately mapped, we
3646 * might have two guest_addr mapping to the same host paddr, but this
3647 * is not a big problem
3649 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3650 mm_to_dma_pfn(paddr_pfn), size, prot);
3654 /* it's a non-present to present mapping. Only flush if caching mode */
3655 if (cap_caching_mode(iommu->cap))
3656 iommu_flush_iotlb_psi(iommu, domain,
3657 mm_to_dma_pfn(iova_pfn),
3660 iommu_flush_write_buffer(iommu);
3662 start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3663 start_paddr += paddr & ~PAGE_MASK;
3668 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3669 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3670 dev_name(dev), size, (unsigned long long)paddr, dir);
3674 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3675 unsigned long offset, size_t size,
3676 enum dma_data_direction dir,
3677 unsigned long attrs)
3679 return __intel_map_single(dev, page_to_phys(page) + offset, size,
3680 dir, *dev->dma_mask);
3683 static void flush_unmaps(struct deferred_flush_data *flush_data)
3687 flush_data->timer_on = 0;
3689 /* just flush them all */
3690 for (i = 0; i < g_num_of_iommus; i++) {
3691 struct intel_iommu *iommu = g_iommus[i];
3692 struct deferred_flush_table *flush_table =
3693 &flush_data->tables[i];
3697 if (!flush_table->next)
3700 /* In caching mode, global flushes turn emulation expensive */
3701 if (!cap_caching_mode(iommu->cap))
3702 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3703 DMA_TLB_GLOBAL_FLUSH);
3704 for (j = 0; j < flush_table->next; j++) {
3706 struct deferred_flush_entry *entry =
3707 &flush_table->entries[j];
3708 unsigned long iova_pfn = entry->iova_pfn;
3709 unsigned long nrpages = entry->nrpages;
3710 struct dmar_domain *domain = entry->domain;
3711 struct page *freelist = entry->freelist;
3713 /* On real hardware multiple invalidations are expensive */
3714 if (cap_caching_mode(iommu->cap))
3715 iommu_flush_iotlb_psi(iommu, domain,
3716 mm_to_dma_pfn(iova_pfn),
3717 nrpages, !freelist, 0);
3719 mask = ilog2(nrpages);
3720 iommu_flush_dev_iotlb(domain,
3721 (uint64_t)iova_pfn << PAGE_SHIFT, mask);
3723 free_iova_fast(&domain->iovad, iova_pfn, nrpages);
3725 dma_free_pagelist(freelist);
3727 flush_table->next = 0;
3730 flush_data->size = 0;
3733 static void flush_unmaps_timeout(unsigned long cpuid)
3735 struct deferred_flush_data *flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3736 unsigned long flags;
3738 spin_lock_irqsave(&flush_data->lock, flags);
3739 flush_unmaps(flush_data);
3740 spin_unlock_irqrestore(&flush_data->lock, flags);
3743 static void add_unmap(struct dmar_domain *dom, unsigned long iova_pfn,
3744 unsigned long nrpages, struct page *freelist)
3746 unsigned long flags;
3747 int entry_id, iommu_id;
3748 struct intel_iommu *iommu;
3749 struct deferred_flush_entry *entry;
3750 struct deferred_flush_data *flush_data;
3754 flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3756 /* Flush all CPUs' entries to avoid deferring too much. If
3757 * this becomes a bottleneck, can just flush us, and rely on
3758 * flush timer for the rest.
3760 if (flush_data->size == HIGH_WATER_MARK) {
3763 for_each_online_cpu(cpu)
3764 flush_unmaps_timeout(cpu);
3767 spin_lock_irqsave(&flush_data->lock, flags);
3769 iommu = domain_get_iommu(dom);
3770 iommu_id = iommu->seq_id;
3772 entry_id = flush_data->tables[iommu_id].next;
3773 ++(flush_data->tables[iommu_id].next);
3775 entry = &flush_data->tables[iommu_id].entries[entry_id];
3776 entry->domain = dom;
3777 entry->iova_pfn = iova_pfn;
3778 entry->nrpages = nrpages;
3779 entry->freelist = freelist;
3781 if (!flush_data->timer_on) {
3782 mod_timer(&flush_data->timer, jiffies + msecs_to_jiffies(10));
3783 flush_data->timer_on = 1;
3786 spin_unlock_irqrestore(&flush_data->lock, flags);
3791 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3793 struct dmar_domain *domain;
3794 unsigned long start_pfn, last_pfn;
3795 unsigned long nrpages;
3796 unsigned long iova_pfn;
3797 struct intel_iommu *iommu;
3798 struct page *freelist;
3800 if (iommu_no_mapping(dev))
3803 domain = find_domain(dev);
3806 iommu = domain_get_iommu(domain);
3808 iova_pfn = IOVA_PFN(dev_addr);
3810 nrpages = aligned_nrpages(dev_addr, size);
3811 start_pfn = mm_to_dma_pfn(iova_pfn);
3812 last_pfn = start_pfn + nrpages - 1;
3814 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3815 dev_name(dev), start_pfn, last_pfn);
3817 freelist = domain_unmap(domain, start_pfn, last_pfn);
3819 if (intel_iommu_strict) {
3820 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3821 nrpages, !freelist, 0);
3823 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3824 dma_free_pagelist(freelist);
3826 add_unmap(domain, iova_pfn, nrpages, freelist);
3828 * queue up the release of the unmap to save the 1/6th of the
3829 * cpu used up by the iotlb flush operation...
3834 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3835 size_t size, enum dma_data_direction dir,
3836 unsigned long attrs)
3838 intel_unmap(dev, dev_addr, size);
3841 static void *intel_alloc_coherent(struct device *dev, size_t size,
3842 dma_addr_t *dma_handle, gfp_t flags,
3843 unsigned long attrs)
3845 struct page *page = NULL;
3848 size = PAGE_ALIGN(size);
3849 order = get_order(size);
3851 if (!iommu_no_mapping(dev))
3852 flags &= ~(GFP_DMA | GFP_DMA32);
3853 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3854 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3860 if (gfpflags_allow_blocking(flags)) {
3861 unsigned int count = size >> PAGE_SHIFT;
3863 page = dma_alloc_from_contiguous(dev, count, order);
3864 if (page && iommu_no_mapping(dev) &&
3865 page_to_phys(page) + size > dev->coherent_dma_mask) {
3866 dma_release_from_contiguous(dev, page, count);
3872 page = alloc_pages(flags, order);
3875 memset(page_address(page), 0, size);
3877 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3879 dev->coherent_dma_mask);
3881 return page_address(page);
3882 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3883 __free_pages(page, order);
3888 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3889 dma_addr_t dma_handle, unsigned long attrs)
3892 struct page *page = virt_to_page(vaddr);
3894 size = PAGE_ALIGN(size);
3895 order = get_order(size);
3897 intel_unmap(dev, dma_handle, size);
3898 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3899 __free_pages(page, order);
3902 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3903 int nelems, enum dma_data_direction dir,
3904 unsigned long attrs)
3906 dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3907 unsigned long nrpages = 0;
3908 struct scatterlist *sg;
3911 for_each_sg(sglist, sg, nelems, i) {
3912 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3915 intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3918 static int intel_nontranslate_map_sg(struct device *hddev,
3919 struct scatterlist *sglist, int nelems, int dir)
3922 struct scatterlist *sg;
3924 for_each_sg(sglist, sg, nelems, i) {
3925 BUG_ON(!sg_page(sg));
3926 sg->dma_address = sg_phys(sg);
3927 sg->dma_length = sg->length;
3932 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3933 enum dma_data_direction dir, unsigned long attrs)
3936 struct dmar_domain *domain;
3939 unsigned long iova_pfn;
3941 struct scatterlist *sg;
3942 unsigned long start_vpfn;
3943 struct intel_iommu *iommu;
3945 BUG_ON(dir == DMA_NONE);
3946 if (iommu_no_mapping(dev))
3947 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3949 domain = get_valid_domain_for_dev(dev);
3953 iommu = domain_get_iommu(domain);
3955 for_each_sg(sglist, sg, nelems, i)
3956 size += aligned_nrpages(sg->offset, sg->length);
3958 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3961 sglist->dma_length = 0;
3966 * Check if DMAR supports zero-length reads on write only
3969 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3970 !cap_zlr(iommu->cap))
3971 prot |= DMA_PTE_READ;
3972 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3973 prot |= DMA_PTE_WRITE;
3975 start_vpfn = mm_to_dma_pfn(iova_pfn);
3977 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3978 if (unlikely(ret)) {
3979 dma_pte_free_pagetable(domain, start_vpfn,
3980 start_vpfn + size - 1);
3981 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3985 /* it's a non-present to present mapping. Only flush if caching mode */
3986 if (cap_caching_mode(iommu->cap))
3987 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
3989 iommu_flush_write_buffer(iommu);
3994 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3999 struct dma_map_ops intel_dma_ops = {
4000 .alloc = intel_alloc_coherent,
4001 .free = intel_free_coherent,
4002 .map_sg = intel_map_sg,
4003 .unmap_sg = intel_unmap_sg,
4004 .map_page = intel_map_page,
4005 .unmap_page = intel_unmap_page,
4006 .mapping_error = intel_mapping_error,
4009 static inline int iommu_domain_cache_init(void)
4013 iommu_domain_cache = kmem_cache_create("iommu_domain",
4014 sizeof(struct dmar_domain),
4019 if (!iommu_domain_cache) {
4020 pr_err("Couldn't create iommu_domain cache\n");
4027 static inline int iommu_devinfo_cache_init(void)
4031 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
4032 sizeof(struct device_domain_info),
4036 if (!iommu_devinfo_cache) {
4037 pr_err("Couldn't create devinfo cache\n");
4044 static int __init iommu_init_mempool(void)
4047 ret = iova_cache_get();
4051 ret = iommu_domain_cache_init();
4055 ret = iommu_devinfo_cache_init();
4059 kmem_cache_destroy(iommu_domain_cache);
4066 static void __init iommu_exit_mempool(void)
4068 kmem_cache_destroy(iommu_devinfo_cache);
4069 kmem_cache_destroy(iommu_domain_cache);
4073 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
4075 struct dmar_drhd_unit *drhd;
4079 /* We know that this device on this chipset has its own IOMMU.
4080 * If we find it under a different IOMMU, then the BIOS is lying
4081 * to us. Hope that the IOMMU for this device is actually
4082 * disabled, and it needs no translation...
4084 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
4086 /* "can't" happen */
4087 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
4090 vtbar &= 0xffff0000;
4092 /* we know that the this iommu should be at offset 0xa000 from vtbar */
4093 drhd = dmar_find_matched_drhd_unit(pdev);
4094 if (!drhd || drhd->reg_base_addr - vtbar != 0xa000) {
4095 pr_warn_once(FW_BUG "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n");
4096 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
4097 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4100 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
4102 static void __init init_no_remapping_devices(void)
4104 struct dmar_drhd_unit *drhd;
4108 for_each_drhd_unit(drhd) {
4109 if (!drhd->include_all) {
4110 for_each_active_dev_scope(drhd->devices,
4111 drhd->devices_cnt, i, dev)
4113 /* ignore DMAR unit if no devices exist */
4114 if (i == drhd->devices_cnt)
4119 for_each_active_drhd_unit(drhd) {
4120 if (drhd->include_all)
4123 for_each_active_dev_scope(drhd->devices,
4124 drhd->devices_cnt, i, dev)
4125 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
4127 if (i < drhd->devices_cnt)
4130 /* This IOMMU has *only* gfx devices. Either bypass it or
4131 set the gfx_mapped flag, as appropriate */
4132 if (!dmar_map_gfx) {
4134 for_each_active_dev_scope(drhd->devices,
4135 drhd->devices_cnt, i, dev)
4136 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4141 #ifdef CONFIG_SUSPEND
4142 static int init_iommu_hw(void)
4144 struct dmar_drhd_unit *drhd;
4145 struct intel_iommu *iommu = NULL;
4147 for_each_active_iommu(iommu, drhd)
4149 dmar_reenable_qi(iommu);
4151 for_each_iommu(iommu, drhd) {
4152 if (drhd->ignored) {
4154 * we always have to disable PMRs or DMA may fail on
4158 iommu_disable_protect_mem_regions(iommu);
4162 iommu_flush_write_buffer(iommu);
4164 iommu_set_root_entry(iommu);
4166 iommu->flush.flush_context(iommu, 0, 0, 0,
4167 DMA_CCMD_GLOBAL_INVL);
4168 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4169 iommu_enable_translation(iommu);
4170 iommu_disable_protect_mem_regions(iommu);
4176 static void iommu_flush_all(void)
4178 struct dmar_drhd_unit *drhd;
4179 struct intel_iommu *iommu;
4181 for_each_active_iommu(iommu, drhd) {
4182 iommu->flush.flush_context(iommu, 0, 0, 0,
4183 DMA_CCMD_GLOBAL_INVL);
4184 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4185 DMA_TLB_GLOBAL_FLUSH);
4189 static int iommu_suspend(void)
4191 struct dmar_drhd_unit *drhd;
4192 struct intel_iommu *iommu = NULL;
4195 for_each_active_iommu(iommu, drhd) {
4196 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
4198 if (!iommu->iommu_state)
4204 for_each_active_iommu(iommu, drhd) {
4205 iommu_disable_translation(iommu);
4207 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4209 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4210 readl(iommu->reg + DMAR_FECTL_REG);
4211 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4212 readl(iommu->reg + DMAR_FEDATA_REG);
4213 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4214 readl(iommu->reg + DMAR_FEADDR_REG);
4215 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4216 readl(iommu->reg + DMAR_FEUADDR_REG);
4218 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4223 for_each_active_iommu(iommu, drhd)
4224 kfree(iommu->iommu_state);
4229 static void iommu_resume(void)
4231 struct dmar_drhd_unit *drhd;
4232 struct intel_iommu *iommu = NULL;
4235 if (init_iommu_hw()) {
4237 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4239 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4243 for_each_active_iommu(iommu, drhd) {
4245 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4247 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4248 iommu->reg + DMAR_FECTL_REG);
4249 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4250 iommu->reg + DMAR_FEDATA_REG);
4251 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4252 iommu->reg + DMAR_FEADDR_REG);
4253 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4254 iommu->reg + DMAR_FEUADDR_REG);
4256 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4259 for_each_active_iommu(iommu, drhd)
4260 kfree(iommu->iommu_state);
4263 static struct syscore_ops iommu_syscore_ops = {
4264 .resume = iommu_resume,
4265 .suspend = iommu_suspend,
4268 static void __init init_iommu_pm_ops(void)
4270 register_syscore_ops(&iommu_syscore_ops);
4274 static inline void init_iommu_pm_ops(void) {}
4275 #endif /* CONFIG_PM */
4278 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4280 struct acpi_dmar_reserved_memory *rmrr;
4281 struct dmar_rmrr_unit *rmrru;
4283 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4287 rmrru->hdr = header;
4288 rmrr = (struct acpi_dmar_reserved_memory *)header;
4289 rmrru->base_address = rmrr->base_address;
4290 rmrru->end_address = rmrr->end_address;
4291 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4292 ((void *)rmrr) + rmrr->header.length,
4293 &rmrru->devices_cnt);
4294 if (rmrru->devices_cnt && rmrru->devices == NULL) {
4299 list_add(&rmrru->list, &dmar_rmrr_units);
4304 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4306 struct dmar_atsr_unit *atsru;
4307 struct acpi_dmar_atsr *tmp;
4309 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4310 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4311 if (atsr->segment != tmp->segment)
4313 if (atsr->header.length != tmp->header.length)
4315 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4322 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4324 struct acpi_dmar_atsr *atsr;
4325 struct dmar_atsr_unit *atsru;
4327 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4330 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4331 atsru = dmar_find_atsr(atsr);
4335 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4340 * If memory is allocated from slab by ACPI _DSM method, we need to
4341 * copy the memory content because the memory buffer will be freed
4344 atsru->hdr = (void *)(atsru + 1);
4345 memcpy(atsru->hdr, hdr, hdr->length);
4346 atsru->include_all = atsr->flags & 0x1;
4347 if (!atsru->include_all) {
4348 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4349 (void *)atsr + atsr->header.length,
4350 &atsru->devices_cnt);
4351 if (atsru->devices_cnt && atsru->devices == NULL) {
4357 list_add_rcu(&atsru->list, &dmar_atsr_units);
4362 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4364 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4368 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4370 struct acpi_dmar_atsr *atsr;
4371 struct dmar_atsr_unit *atsru;
4373 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4374 atsru = dmar_find_atsr(atsr);
4376 list_del_rcu(&atsru->list);
4378 intel_iommu_free_atsr(atsru);
4384 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4388 struct acpi_dmar_atsr *atsr;
4389 struct dmar_atsr_unit *atsru;
4391 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4392 atsru = dmar_find_atsr(atsr);
4396 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4397 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4405 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4408 struct intel_iommu *iommu = dmaru->iommu;
4410 if (g_iommus[iommu->seq_id])
4413 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4414 pr_warn("%s: Doesn't support hardware pass through.\n",
4418 if (!ecap_sc_support(iommu->ecap) &&
4419 domain_update_iommu_snooping(iommu)) {
4420 pr_warn("%s: Doesn't support snooping.\n",
4424 sp = domain_update_iommu_superpage(iommu) - 1;
4425 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4426 pr_warn("%s: Doesn't support large page.\n",
4432 * Disable translation if already enabled prior to OS handover.
4434 if (iommu->gcmd & DMA_GCMD_TE)
4435 iommu_disable_translation(iommu);
4437 g_iommus[iommu->seq_id] = iommu;
4438 ret = iommu_init_domains(iommu);
4440 ret = iommu_alloc_root_entry(iommu);
4444 #ifdef CONFIG_INTEL_IOMMU_SVM
4445 if (pasid_enabled(iommu))
4446 intel_svm_alloc_pasid_tables(iommu);
4449 if (dmaru->ignored) {
4451 * we always have to disable PMRs or DMA may fail on this device
4454 iommu_disable_protect_mem_regions(iommu);
4458 intel_iommu_init_qi(iommu);
4459 iommu_flush_write_buffer(iommu);
4461 #ifdef CONFIG_INTEL_IOMMU_SVM
4462 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4463 ret = intel_svm_enable_prq(iommu);
4468 ret = dmar_set_interrupt(iommu);
4472 iommu_set_root_entry(iommu);
4473 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4474 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4475 iommu_enable_translation(iommu);
4477 iommu_disable_protect_mem_regions(iommu);
4481 disable_dmar_iommu(iommu);
4483 free_dmar_iommu(iommu);
4487 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4490 struct intel_iommu *iommu = dmaru->iommu;
4492 if (!intel_iommu_enabled)
4498 ret = intel_iommu_add(dmaru);
4500 disable_dmar_iommu(iommu);
4501 free_dmar_iommu(iommu);
4507 static void intel_iommu_free_dmars(void)
4509 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4510 struct dmar_atsr_unit *atsru, *atsr_n;
4512 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4513 list_del(&rmrru->list);
4514 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4518 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4519 list_del(&atsru->list);
4520 intel_iommu_free_atsr(atsru);
4524 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4527 struct pci_bus *bus;
4528 struct pci_dev *bridge = NULL;
4530 struct acpi_dmar_atsr *atsr;
4531 struct dmar_atsr_unit *atsru;
4533 dev = pci_physfn(dev);
4534 for (bus = dev->bus; bus; bus = bus->parent) {
4536 /* If it's an integrated device, allow ATS */
4539 /* Connected via non-PCIe: no ATS */
4540 if (!pci_is_pcie(bridge) ||
4541 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4543 /* If we found the root port, look it up in the ATSR */
4544 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4549 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4550 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4551 if (atsr->segment != pci_domain_nr(dev->bus))
4554 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4555 if (tmp == &bridge->dev)
4558 if (atsru->include_all)
4568 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4571 struct dmar_rmrr_unit *rmrru;
4572 struct dmar_atsr_unit *atsru;
4573 struct acpi_dmar_atsr *atsr;
4574 struct acpi_dmar_reserved_memory *rmrr;
4576 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4579 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4580 rmrr = container_of(rmrru->hdr,
4581 struct acpi_dmar_reserved_memory, header);
4582 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4583 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4584 ((void *)rmrr) + rmrr->header.length,
4585 rmrr->segment, rmrru->devices,
4586 rmrru->devices_cnt);
4589 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4590 dmar_remove_dev_scope(info, rmrr->segment,
4591 rmrru->devices, rmrru->devices_cnt);
4595 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4596 if (atsru->include_all)
4599 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4600 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4601 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4602 (void *)atsr + atsr->header.length,
4603 atsr->segment, atsru->devices,
4604 atsru->devices_cnt);
4609 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4610 if (dmar_remove_dev_scope(info, atsr->segment,
4611 atsru->devices, atsru->devices_cnt))
4620 * Here we only respond to action of unbound device from driver.
4622 * Added device is not attached to its DMAR domain here yet. That will happen
4623 * when mapping the device to iova.
4625 static int device_notifier(struct notifier_block *nb,
4626 unsigned long action, void *data)
4628 struct device *dev = data;
4629 struct dmar_domain *domain;
4631 if (iommu_dummy(dev))
4634 if (action != BUS_NOTIFY_REMOVED_DEVICE)
4637 domain = find_domain(dev);
4641 dmar_remove_one_dev_info(domain, dev);
4642 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4643 domain_exit(domain);
4648 static struct notifier_block device_nb = {
4649 .notifier_call = device_notifier,
4652 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4653 unsigned long val, void *v)
4655 struct memory_notify *mhp = v;
4656 unsigned long long start, end;
4657 unsigned long start_vpfn, last_vpfn;
4660 case MEM_GOING_ONLINE:
4661 start = mhp->start_pfn << PAGE_SHIFT;
4662 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4663 if (iommu_domain_identity_map(si_domain, start, end)) {
4664 pr_warn("Failed to build identity map for [%llx-%llx]\n",
4671 case MEM_CANCEL_ONLINE:
4672 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4673 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4674 while (start_vpfn <= last_vpfn) {
4676 struct dmar_drhd_unit *drhd;
4677 struct intel_iommu *iommu;
4678 struct page *freelist;
4680 iova = find_iova(&si_domain->iovad, start_vpfn);
4682 pr_debug("Failed get IOVA for PFN %lx\n",
4687 iova = split_and_remove_iova(&si_domain->iovad, iova,
4688 start_vpfn, last_vpfn);
4690 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4691 start_vpfn, last_vpfn);
4695 freelist = domain_unmap(si_domain, iova->pfn_lo,
4699 for_each_active_iommu(iommu, drhd)
4700 iommu_flush_iotlb_psi(iommu, si_domain,
4701 iova->pfn_lo, iova_size(iova),
4704 dma_free_pagelist(freelist);
4706 start_vpfn = iova->pfn_hi + 1;
4707 free_iova_mem(iova);
4715 static struct notifier_block intel_iommu_memory_nb = {
4716 .notifier_call = intel_iommu_memory_notifier,
4720 static void free_all_cpu_cached_iovas(unsigned int cpu)
4724 for (i = 0; i < g_num_of_iommus; i++) {
4725 struct intel_iommu *iommu = g_iommus[i];
4726 struct dmar_domain *domain;
4732 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
4733 domain = get_iommu_domain(iommu, (u16)did);
4737 free_cpu_cached_iovas(cpu, &domain->iovad);
4742 static int intel_iommu_cpu_notifier(struct notifier_block *nfb,
4743 unsigned long action, void *v)
4745 unsigned int cpu = (unsigned long)v;
4749 case CPU_DEAD_FROZEN:
4750 free_all_cpu_cached_iovas(cpu);
4751 flush_unmaps_timeout(cpu);
4757 static struct notifier_block intel_iommu_cpu_nb = {
4758 .notifier_call = intel_iommu_cpu_notifier,
4761 static ssize_t intel_iommu_show_version(struct device *dev,
4762 struct device_attribute *attr,
4765 struct intel_iommu *iommu = dev_get_drvdata(dev);
4766 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4767 return sprintf(buf, "%d:%d\n",
4768 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4770 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4772 static ssize_t intel_iommu_show_address(struct device *dev,
4773 struct device_attribute *attr,
4776 struct intel_iommu *iommu = dev_get_drvdata(dev);
4777 return sprintf(buf, "%llx\n", iommu->reg_phys);
4779 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4781 static ssize_t intel_iommu_show_cap(struct device *dev,
4782 struct device_attribute *attr,
4785 struct intel_iommu *iommu = dev_get_drvdata(dev);
4786 return sprintf(buf, "%llx\n", iommu->cap);
4788 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4790 static ssize_t intel_iommu_show_ecap(struct device *dev,
4791 struct device_attribute *attr,
4794 struct intel_iommu *iommu = dev_get_drvdata(dev);
4795 return sprintf(buf, "%llx\n", iommu->ecap);
4797 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4799 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4800 struct device_attribute *attr,
4803 struct intel_iommu *iommu = dev_get_drvdata(dev);
4804 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4806 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4808 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4809 struct device_attribute *attr,
4812 struct intel_iommu *iommu = dev_get_drvdata(dev);
4813 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4814 cap_ndoms(iommu->cap)));
4816 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4818 static struct attribute *intel_iommu_attrs[] = {
4819 &dev_attr_version.attr,
4820 &dev_attr_address.attr,
4822 &dev_attr_ecap.attr,
4823 &dev_attr_domains_supported.attr,
4824 &dev_attr_domains_used.attr,
4828 static struct attribute_group intel_iommu_group = {
4829 .name = "intel-iommu",
4830 .attrs = intel_iommu_attrs,
4833 const struct attribute_group *intel_iommu_groups[] = {
4838 int __init intel_iommu_init(void)
4841 struct dmar_drhd_unit *drhd;
4842 struct intel_iommu *iommu;
4844 /* VT-d is required for a TXT/tboot launch, so enforce that */
4845 force_on = tboot_force_iommu();
4847 if (iommu_init_mempool()) {
4849 panic("tboot: Failed to initialize iommu memory\n");
4853 down_write(&dmar_global_lock);
4854 if (dmar_table_init()) {
4856 panic("tboot: Failed to initialize DMAR table\n");
4860 if (dmar_dev_scope_init() < 0) {
4862 panic("tboot: Failed to initialize DMAR device scope\n");
4866 if (no_iommu || dmar_disabled)
4869 if (list_empty(&dmar_rmrr_units))
4870 pr_info("No RMRR found\n");
4872 if (list_empty(&dmar_atsr_units))
4873 pr_info("No ATSR found\n");
4875 if (dmar_init_reserved_ranges()) {
4877 panic("tboot: Failed to reserve iommu ranges\n");
4878 goto out_free_reserved_range;
4882 intel_iommu_gfx_mapped = 1;
4884 init_no_remapping_devices();
4889 panic("tboot: Failed to initialize DMARs\n");
4890 pr_err("Initialization failed\n");
4891 goto out_free_reserved_range;
4893 up_write(&dmar_global_lock);
4894 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4896 #ifdef CONFIG_SWIOTLB
4899 dma_ops = &intel_dma_ops;
4901 init_iommu_pm_ops();
4903 for_each_active_iommu(iommu, drhd)
4904 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4908 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4909 bus_register_notifier(&pci_bus_type, &device_nb);
4910 if (si_domain && !hw_pass_through)
4911 register_memory_notifier(&intel_iommu_memory_nb);
4912 register_hotcpu_notifier(&intel_iommu_cpu_nb);
4914 intel_iommu_enabled = 1;
4918 out_free_reserved_range:
4919 put_iova_domain(&reserved_iova_list);
4921 intel_iommu_free_dmars();
4922 up_write(&dmar_global_lock);
4923 iommu_exit_mempool();
4927 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4929 struct intel_iommu *iommu = opaque;
4931 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4936 * NB - intel-iommu lacks any sort of reference counting for the users of
4937 * dependent devices. If multiple endpoints have intersecting dependent
4938 * devices, unbinding the driver from any one of them will possibly leave
4939 * the others unable to operate.
4941 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4943 if (!iommu || !dev || !dev_is_pci(dev))
4946 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4949 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4951 struct intel_iommu *iommu;
4952 unsigned long flags;
4954 assert_spin_locked(&device_domain_lock);
4959 iommu = info->iommu;
4962 iommu_disable_dev_iotlb(info);
4963 domain_context_clear(iommu, info->dev);
4966 unlink_domain_info(info);
4968 spin_lock_irqsave(&iommu->lock, flags);
4969 domain_detach_iommu(info->domain, iommu);
4970 spin_unlock_irqrestore(&iommu->lock, flags);
4972 free_devinfo_mem(info);
4975 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4978 struct device_domain_info *info;
4979 unsigned long flags;
4981 spin_lock_irqsave(&device_domain_lock, flags);
4982 info = dev->archdata.iommu;
4983 __dmar_remove_one_dev_info(info);
4984 spin_unlock_irqrestore(&device_domain_lock, flags);
4987 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4991 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4993 domain_reserve_special_ranges(domain);
4995 /* calculate AGAW */
4996 domain->gaw = guest_width;
4997 adjust_width = guestwidth_to_adjustwidth(guest_width);
4998 domain->agaw = width_to_agaw(adjust_width);
5000 domain->iommu_coherency = 0;
5001 domain->iommu_snooping = 0;
5002 domain->iommu_superpage = 0;
5003 domain->max_addr = 0;
5005 /* always allocate the top pgd */
5006 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5009 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
5013 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
5015 struct dmar_domain *dmar_domain;
5016 struct iommu_domain *domain;
5018 if (type != IOMMU_DOMAIN_UNMANAGED)
5021 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5023 pr_err("Can't allocate dmar_domain\n");
5026 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
5027 pr_err("Domain initialization failed\n");
5028 domain_exit(dmar_domain);
5031 domain_update_iommu_cap(dmar_domain);
5033 domain = &dmar_domain->domain;
5034 domain->geometry.aperture_start = 0;
5035 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
5036 domain->geometry.force_aperture = true;
5041 static void intel_iommu_domain_free(struct iommu_domain *domain)
5043 domain_exit(to_dmar_domain(domain));
5046 static int intel_iommu_attach_device(struct iommu_domain *domain,
5049 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5050 struct intel_iommu *iommu;
5054 if (device_is_rmrr_locked(dev)) {
5055 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
5059 /* normally dev is not mapped */
5060 if (unlikely(domain_context_mapped(dev))) {
5061 struct dmar_domain *old_domain;
5063 old_domain = find_domain(dev);
5066 dmar_remove_one_dev_info(old_domain, dev);
5069 if (!domain_type_is_vm_or_si(old_domain) &&
5070 list_empty(&old_domain->devices))
5071 domain_exit(old_domain);
5075 iommu = device_to_iommu(dev, &bus, &devfn);
5079 /* check if this iommu agaw is sufficient for max mapped address */
5080 addr_width = agaw_to_width(iommu->agaw);
5081 if (addr_width > cap_mgaw(iommu->cap))
5082 addr_width = cap_mgaw(iommu->cap);
5084 if (dmar_domain->max_addr > (1LL << addr_width)) {
5085 pr_err("%s: iommu width (%d) is not "
5086 "sufficient for the mapped address (%llx)\n",
5087 __func__, addr_width, dmar_domain->max_addr);
5090 dmar_domain->gaw = addr_width;
5093 * Knock out extra levels of page tables if necessary
5095 while (iommu->agaw < dmar_domain->agaw) {
5096 struct dma_pte *pte;
5098 pte = dmar_domain->pgd;
5099 if (dma_pte_present(pte)) {
5100 dmar_domain->pgd = (struct dma_pte *)
5101 phys_to_virt(dma_pte_addr(pte));
5102 free_pgtable_page(pte);
5104 dmar_domain->agaw--;
5107 return domain_add_dev_info(dmar_domain, dev);
5110 static void intel_iommu_detach_device(struct iommu_domain *domain,
5113 dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
5116 static int intel_iommu_map(struct iommu_domain *domain,
5117 unsigned long iova, phys_addr_t hpa,
5118 size_t size, int iommu_prot)
5120 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5125 if (iommu_prot & IOMMU_READ)
5126 prot |= DMA_PTE_READ;
5127 if (iommu_prot & IOMMU_WRITE)
5128 prot |= DMA_PTE_WRITE;
5129 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5130 prot |= DMA_PTE_SNP;
5132 max_addr = iova + size;
5133 if (dmar_domain->max_addr < max_addr) {
5136 /* check if minimum agaw is sufficient for mapped address */
5137 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5138 if (end < max_addr) {
5139 pr_err("%s: iommu width (%d) is not "
5140 "sufficient for the mapped address (%llx)\n",
5141 __func__, dmar_domain->gaw, max_addr);
5144 dmar_domain->max_addr = max_addr;
5146 /* Round up size to next multiple of PAGE_SIZE, if it and
5147 the low bits of hpa would take us onto the next page */
5148 size = aligned_nrpages(hpa, size);
5149 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5150 hpa >> VTD_PAGE_SHIFT, size, prot);
5154 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5155 unsigned long iova, size_t size)
5157 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5158 struct page *freelist = NULL;
5159 struct intel_iommu *iommu;
5160 unsigned long start_pfn, last_pfn;
5161 unsigned int npages;
5162 int iommu_id, level = 0;
5164 /* Cope with horrid API which requires us to unmap more than the
5165 size argument if it happens to be a large-page mapping. */
5166 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5168 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5169 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5171 start_pfn = iova >> VTD_PAGE_SHIFT;
5172 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5174 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5176 npages = last_pfn - start_pfn + 1;
5178 for_each_domain_iommu(iommu_id, dmar_domain) {
5179 iommu = g_iommus[iommu_id];
5181 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5182 start_pfn, npages, !freelist, 0);
5185 dma_free_pagelist(freelist);
5187 if (dmar_domain->max_addr == iova + size)
5188 dmar_domain->max_addr = iova;
5193 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5196 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5197 struct dma_pte *pte;
5201 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5202 if (pte && dma_pte_present(pte))
5203 phys = dma_pte_addr(pte) +
5204 (iova & (BIT_MASK(level_to_offset_bits(level) +
5205 VTD_PAGE_SHIFT) - 1));
5210 static bool intel_iommu_capable(enum iommu_cap cap)
5212 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5213 return domain_update_iommu_snooping(NULL) == 1;
5214 if (cap == IOMMU_CAP_INTR_REMAP)
5215 return irq_remapping_enabled == 1;
5220 static int intel_iommu_add_device(struct device *dev)
5222 struct intel_iommu *iommu;
5223 struct iommu_group *group;
5226 iommu = device_to_iommu(dev, &bus, &devfn);
5230 iommu_device_link(iommu->iommu_dev, dev);
5232 group = iommu_group_get_for_dev(dev);
5235 return PTR_ERR(group);
5237 iommu_group_put(group);
5241 static void intel_iommu_remove_device(struct device *dev)
5243 struct intel_iommu *iommu;
5246 iommu = device_to_iommu(dev, &bus, &devfn);
5250 iommu_group_remove_device(dev);
5252 iommu_device_unlink(iommu->iommu_dev, dev);
5255 #ifdef CONFIG_INTEL_IOMMU_SVM
5256 #define MAX_NR_PASID_BITS (20)
5257 static inline unsigned long intel_iommu_get_pts(struct intel_iommu *iommu)
5260 * Convert ecap_pss to extend context entry pts encoding, also
5261 * respect the soft pasid_max value set by the iommu.
5262 * - number of PASID bits = ecap_pss + 1
5263 * - number of PASID table entries = 2^(pts + 5)
5264 * Therefore, pts = ecap_pss - 4
5265 * e.g. KBL ecap_pss = 0x13, PASID has 20 bits, pts = 15
5267 if (ecap_pss(iommu->ecap) < 5)
5270 /* pasid_max is encoded as actual number of entries not the bits */
5271 return find_first_bit((unsigned long *)&iommu->pasid_max,
5272 MAX_NR_PASID_BITS) - 5;
5275 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5277 struct device_domain_info *info;
5278 struct context_entry *context;
5279 struct dmar_domain *domain;
5280 unsigned long flags;
5284 domain = get_valid_domain_for_dev(sdev->dev);
5288 spin_lock_irqsave(&device_domain_lock, flags);
5289 spin_lock(&iommu->lock);
5292 info = sdev->dev->archdata.iommu;
5293 if (!info || !info->pasid_supported)
5296 context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5297 if (WARN_ON(!context))
5300 ctx_lo = context[0].lo;
5302 sdev->did = domain->iommu_did[iommu->seq_id];
5303 sdev->sid = PCI_DEVID(info->bus, info->devfn);
5305 if (!(ctx_lo & CONTEXT_PASIDE)) {
5306 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
5307 context[1].lo = (u64)virt_to_phys(iommu->pasid_table) |
5308 intel_iommu_get_pts(iommu);
5311 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5312 * extended to permit requests-with-PASID if the PASIDE bit
5313 * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5314 * however, the PASIDE bit is ignored and requests-with-PASID
5315 * are unconditionally blocked. Which makes less sense.
5316 * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5317 * "guest mode" translation types depending on whether ATS
5318 * is available or not. Annoyingly, we can't use the new
5319 * modes *unless* PASIDE is set. */
5320 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5321 ctx_lo &= ~CONTEXT_TT_MASK;
5322 if (info->ats_supported)
5323 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5325 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5327 ctx_lo |= CONTEXT_PASIDE;
5328 if (iommu->pasid_state_table)
5329 ctx_lo |= CONTEXT_DINVE;
5330 if (info->pri_supported)
5331 ctx_lo |= CONTEXT_PRS;
5332 context[0].lo = ctx_lo;
5334 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5335 DMA_CCMD_MASK_NOBIT,
5336 DMA_CCMD_DEVICE_INVL);
5339 /* Enable PASID support in the device, if it wasn't already */
5340 if (!info->pasid_enabled)
5341 iommu_enable_dev_iotlb(info);
5343 if (info->ats_enabled) {
5344 sdev->dev_iotlb = 1;
5345 sdev->qdep = info->ats_qdep;
5346 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5352 spin_unlock(&iommu->lock);
5353 spin_unlock_irqrestore(&device_domain_lock, flags);
5358 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5360 struct intel_iommu *iommu;
5363 if (iommu_dummy(dev)) {
5365 "No IOMMU translation for device; cannot enable SVM\n");
5369 iommu = device_to_iommu(dev, &bus, &devfn);
5371 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5375 if (!iommu->pasid_table) {
5376 dev_err(dev, "PASID not enabled on IOMMU; cannot enable SVM\n");
5382 #endif /* CONFIG_INTEL_IOMMU_SVM */
5384 static const struct iommu_ops intel_iommu_ops = {
5385 .capable = intel_iommu_capable,
5386 .domain_alloc = intel_iommu_domain_alloc,
5387 .domain_free = intel_iommu_domain_free,
5388 .attach_dev = intel_iommu_attach_device,
5389 .detach_dev = intel_iommu_detach_device,
5390 .map = intel_iommu_map,
5391 .unmap = intel_iommu_unmap,
5392 .map_sg = default_iommu_map_sg,
5393 .iova_to_phys = intel_iommu_iova_to_phys,
5394 .add_device = intel_iommu_add_device,
5395 .remove_device = intel_iommu_remove_device,
5396 .device_group = pci_device_group,
5397 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
5400 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5402 /* G4x/GM45 integrated gfx dmar support is totally busted. */
5403 pr_info("Disabling IOMMU for graphics on this chipset\n");
5407 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5408 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5409 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5410 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5411 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5412 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5413 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5415 static void quirk_iommu_rwbf(struct pci_dev *dev)
5418 * Mobile 4 Series Chipset neglects to set RWBF capability,
5419 * but needs it. Same seems to hold for the desktop versions.
5421 pr_info("Forcing write-buffer flush capability\n");
5425 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5426 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5427 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5428 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5429 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5430 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5431 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5434 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
5435 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
5436 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
5437 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
5438 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
5439 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
5440 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
5441 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
5443 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5447 if (pci_read_config_word(dev, GGC, &ggc))
5450 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5451 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5453 } else if (dmar_map_gfx) {
5454 /* we have to ensure the gfx device is idle before we flush */
5455 pr_info("Disabling batched IOTLB flush on Ironlake\n");
5456 intel_iommu_strict = 1;
5459 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5460 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5461 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5462 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5464 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5465 ISOCH DMAR unit for the Azalia sound device, but not give it any
5466 TLB entries, which causes it to deadlock. Check for that. We do
5467 this in a function called from init_dmars(), instead of in a PCI
5468 quirk, because we don't want to print the obnoxious "BIOS broken"
5469 message if VT-d is actually disabled.
5471 static void __init check_tylersburg_isoch(void)
5473 struct pci_dev *pdev;
5474 uint32_t vtisochctrl;
5476 /* If there's no Azalia in the system anyway, forget it. */
5477 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5482 /* System Management Registers. Might be hidden, in which case
5483 we can't do the sanity check. But that's OK, because the
5484 known-broken BIOSes _don't_ actually hide it, so far. */
5485 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5489 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5496 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5497 if (vtisochctrl & 1)
5500 /* Drop all bits other than the number of TLB entries */
5501 vtisochctrl &= 0x1c;
5503 /* If we have the recommended number of TLB entries (16), fine. */
5504 if (vtisochctrl == 0x10)
5507 /* Zero TLB entries? You get to ride the short bus to school. */
5509 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5510 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5511 dmi_get_system_info(DMI_BIOS_VENDOR),
5512 dmi_get_system_info(DMI_BIOS_VERSION),
5513 dmi_get_system_info(DMI_PRODUCT_VERSION));
5514 iommu_identity_mapping |= IDENTMAP_AZALIA;
5518 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",