1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2016, Semihalf
4 * Author: Tomasz Nowicki <tn@semihalf.com>
6 * This file implements early detection/parsing of I/O mapping
7 * reported to OS through firmware via I/O Remapping Table (IORT)
8 * IORT document number: ARM DEN 0049A
11 #define pr_fmt(fmt) "ACPI: IORT: " fmt
13 #include <linux/acpi_iort.h>
14 #include <linux/bitfield.h>
15 #include <linux/iommu.h>
16 #include <linux/kernel.h>
17 #include <linux/list.h>
18 #include <linux/pci.h>
19 #include <linux/platform_device.h>
20 #include <linux/slab.h>
21 #include <linux/dma-map-ops.h>
23 #define IORT_TYPE_MASK(type) (1 << (type))
24 #define IORT_MSI_TYPE (1 << ACPI_IORT_NODE_ITS_GROUP)
25 #define IORT_IOMMU_TYPE ((1 << ACPI_IORT_NODE_SMMU) | \
26 (1 << ACPI_IORT_NODE_SMMU_V3))
28 struct iort_its_msi_chip {
29 struct list_head list;
30 struct fwnode_handle *fw_node;
31 phys_addr_t base_addr;
36 struct list_head list;
37 struct acpi_iort_node *iort_node;
38 struct fwnode_handle *fwnode;
40 static LIST_HEAD(iort_fwnode_list);
41 static DEFINE_SPINLOCK(iort_fwnode_lock);
44 * iort_set_fwnode() - Create iort_fwnode and use it to register
45 * iommu data in the iort_fwnode_list
47 * @iort_node: IORT table node associated with the IOMMU
48 * @fwnode: fwnode associated with the IORT node
50 * Returns: 0 on success
53 static inline int iort_set_fwnode(struct acpi_iort_node *iort_node,
54 struct fwnode_handle *fwnode)
56 struct iort_fwnode *np;
58 np = kzalloc(sizeof(struct iort_fwnode), GFP_ATOMIC);
63 INIT_LIST_HEAD(&np->list);
64 np->iort_node = iort_node;
67 spin_lock(&iort_fwnode_lock);
68 list_add_tail(&np->list, &iort_fwnode_list);
69 spin_unlock(&iort_fwnode_lock);
75 * iort_get_fwnode() - Retrieve fwnode associated with an IORT node
77 * @node: IORT table node to be looked-up
79 * Returns: fwnode_handle pointer on success, NULL on failure
81 static inline struct fwnode_handle *iort_get_fwnode(
82 struct acpi_iort_node *node)
84 struct iort_fwnode *curr;
85 struct fwnode_handle *fwnode = NULL;
87 spin_lock(&iort_fwnode_lock);
88 list_for_each_entry(curr, &iort_fwnode_list, list) {
89 if (curr->iort_node == node) {
90 fwnode = curr->fwnode;
94 spin_unlock(&iort_fwnode_lock);
100 * iort_delete_fwnode() - Delete fwnode associated with an IORT node
102 * @node: IORT table node associated with fwnode to delete
104 static inline void iort_delete_fwnode(struct acpi_iort_node *node)
106 struct iort_fwnode *curr, *tmp;
108 spin_lock(&iort_fwnode_lock);
109 list_for_each_entry_safe(curr, tmp, &iort_fwnode_list, list) {
110 if (curr->iort_node == node) {
111 list_del(&curr->list);
116 spin_unlock(&iort_fwnode_lock);
120 * iort_get_iort_node() - Retrieve iort_node associated with an fwnode
122 * @fwnode: fwnode associated with device to be looked-up
124 * Returns: iort_node pointer on success, NULL on failure
126 static inline struct acpi_iort_node *iort_get_iort_node(
127 struct fwnode_handle *fwnode)
129 struct iort_fwnode *curr;
130 struct acpi_iort_node *iort_node = NULL;
132 spin_lock(&iort_fwnode_lock);
133 list_for_each_entry(curr, &iort_fwnode_list, list) {
134 if (curr->fwnode == fwnode) {
135 iort_node = curr->iort_node;
139 spin_unlock(&iort_fwnode_lock);
144 typedef acpi_status (*iort_find_node_callback)
145 (struct acpi_iort_node *node, void *context);
147 /* Root pointer to the mapped IORT table */
148 static struct acpi_table_header *iort_table;
150 static LIST_HEAD(iort_msi_chip_list);
151 static DEFINE_SPINLOCK(iort_msi_chip_lock);
154 * iort_register_domain_token() - register domain token along with related
155 * ITS ID and base address to the list from where we can get it back later on.
157 * @base: ITS base address.
158 * @fw_node: Domain token.
160 * Returns: 0 on success, -ENOMEM if no memory when allocating list element
162 int iort_register_domain_token(int trans_id, phys_addr_t base,
163 struct fwnode_handle *fw_node)
165 struct iort_its_msi_chip *its_msi_chip;
167 its_msi_chip = kzalloc(sizeof(*its_msi_chip), GFP_KERNEL);
171 its_msi_chip->fw_node = fw_node;
172 its_msi_chip->translation_id = trans_id;
173 its_msi_chip->base_addr = base;
175 spin_lock(&iort_msi_chip_lock);
176 list_add(&its_msi_chip->list, &iort_msi_chip_list);
177 spin_unlock(&iort_msi_chip_lock);
183 * iort_deregister_domain_token() - Deregister domain token based on ITS ID
188 void iort_deregister_domain_token(int trans_id)
190 struct iort_its_msi_chip *its_msi_chip, *t;
192 spin_lock(&iort_msi_chip_lock);
193 list_for_each_entry_safe(its_msi_chip, t, &iort_msi_chip_list, list) {
194 if (its_msi_chip->translation_id == trans_id) {
195 list_del(&its_msi_chip->list);
200 spin_unlock(&iort_msi_chip_lock);
204 * iort_find_domain_token() - Find domain token based on given ITS ID
207 * Returns: domain token when find on the list, NULL otherwise
209 struct fwnode_handle *iort_find_domain_token(int trans_id)
211 struct fwnode_handle *fw_node = NULL;
212 struct iort_its_msi_chip *its_msi_chip;
214 spin_lock(&iort_msi_chip_lock);
215 list_for_each_entry(its_msi_chip, &iort_msi_chip_list, list) {
216 if (its_msi_chip->translation_id == trans_id) {
217 fw_node = its_msi_chip->fw_node;
221 spin_unlock(&iort_msi_chip_lock);
226 static struct acpi_iort_node *iort_scan_node(enum acpi_iort_node_type type,
227 iort_find_node_callback callback,
230 struct acpi_iort_node *iort_node, *iort_end;
231 struct acpi_table_iort *iort;
237 /* Get the first IORT node */
238 iort = (struct acpi_table_iort *)iort_table;
239 iort_node = ACPI_ADD_PTR(struct acpi_iort_node, iort,
241 iort_end = ACPI_ADD_PTR(struct acpi_iort_node, iort_table,
244 for (i = 0; i < iort->node_count; i++) {
245 if (WARN_TAINT(iort_node >= iort_end, TAINT_FIRMWARE_WORKAROUND,
246 "IORT node pointer overflows, bad table!\n"))
249 if (iort_node->type == type &&
250 ACPI_SUCCESS(callback(iort_node, context)))
253 iort_node = ACPI_ADD_PTR(struct acpi_iort_node, iort_node,
260 static acpi_status iort_match_node_callback(struct acpi_iort_node *node,
263 struct device *dev = context;
264 acpi_status status = AE_NOT_FOUND;
266 if (node->type == ACPI_IORT_NODE_NAMED_COMPONENT) {
267 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
268 struct acpi_device *adev;
269 struct acpi_iort_named_component *ncomp;
270 struct device *nc_dev = dev;
273 * Walk the device tree to find a device with an
274 * ACPI companion; there is no point in scanning
275 * IORT for a device matching a named component if
276 * the device does not have an ACPI companion to
280 adev = ACPI_COMPANION(nc_dev);
284 nc_dev = nc_dev->parent;
290 status = acpi_get_name(adev->handle, ACPI_FULL_PATHNAME, &buf);
291 if (ACPI_FAILURE(status)) {
292 dev_warn(nc_dev, "Can't get device full path name\n");
296 ncomp = (struct acpi_iort_named_component *)node->node_data;
297 status = !strcmp(ncomp->device_name, buf.pointer) ?
298 AE_OK : AE_NOT_FOUND;
299 acpi_os_free(buf.pointer);
300 } else if (node->type == ACPI_IORT_NODE_PCI_ROOT_COMPLEX) {
301 struct acpi_iort_root_complex *pci_rc;
304 bus = to_pci_bus(dev);
305 pci_rc = (struct acpi_iort_root_complex *)node->node_data;
308 * It is assumed that PCI segment numbers maps one-to-one
309 * with root complexes. Each segment number can represent only
312 status = pci_rc->pci_segment_number == pci_domain_nr(bus) ?
313 AE_OK : AE_NOT_FOUND;
319 static int iort_id_map(struct acpi_iort_id_mapping *map, u8 type, u32 rid_in,
320 u32 *rid_out, bool check_overlap)
322 /* Single mapping does not care for input id */
323 if (map->flags & ACPI_IORT_ID_SINGLE_MAPPING) {
324 if (type == ACPI_IORT_NODE_NAMED_COMPONENT ||
325 type == ACPI_IORT_NODE_PCI_ROOT_COMPLEX) {
326 *rid_out = map->output_base;
330 pr_warn(FW_BUG "[map %p] SINGLE MAPPING flag not allowed for node type %d, skipping ID map\n",
335 if (rid_in < map->input_base ||
336 (rid_in > map->input_base + map->id_count))
341 * We already found a mapping for this input ID at the end of
342 * another region. If it coincides with the start of this
343 * region, we assume the prior match was due to the off-by-1
344 * issue mentioned below, and allow it to be superseded.
345 * Otherwise, things are *really* broken, and we just disregard
346 * duplicate matches entirely to retain compatibility.
348 pr_err(FW_BUG "[map %p] conflicting mapping for input ID 0x%x\n",
350 if (rid_in != map->input_base)
353 pr_err(FW_BUG "applying workaround.\n");
356 *rid_out = map->output_base + (rid_in - map->input_base);
359 * Due to confusion regarding the meaning of the id_count field (which
360 * carries the number of IDs *minus 1*), we may have to disregard this
361 * match if it is at the end of the range, and overlaps with the start
364 if (map->id_count > 0 && rid_in == map->input_base + map->id_count)
369 static struct acpi_iort_node *iort_node_get_id(struct acpi_iort_node *node,
370 u32 *id_out, int index)
372 struct acpi_iort_node *parent;
373 struct acpi_iort_id_mapping *map;
375 if (!node->mapping_offset || !node->mapping_count ||
376 index >= node->mapping_count)
379 map = ACPI_ADD_PTR(struct acpi_iort_id_mapping, node,
380 node->mapping_offset + index * sizeof(*map));
383 if (!map->output_reference) {
384 pr_err(FW_BUG "[node %p type %d] ID map has NULL parent reference\n",
389 parent = ACPI_ADD_PTR(struct acpi_iort_node, iort_table,
390 map->output_reference);
392 if (map->flags & ACPI_IORT_ID_SINGLE_MAPPING) {
393 if (node->type == ACPI_IORT_NODE_NAMED_COMPONENT ||
394 node->type == ACPI_IORT_NODE_PCI_ROOT_COMPLEX ||
395 node->type == ACPI_IORT_NODE_SMMU_V3 ||
396 node->type == ACPI_IORT_NODE_PMCG) {
397 *id_out = map->output_base;
405 static int iort_get_id_mapping_index(struct acpi_iort_node *node)
407 struct acpi_iort_smmu_v3 *smmu;
408 struct acpi_iort_pmcg *pmcg;
410 switch (node->type) {
411 case ACPI_IORT_NODE_SMMU_V3:
413 * SMMUv3 dev ID mapping index was introduced in revision 1
414 * table, not available in revision 0
416 if (node->revision < 1)
419 smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
421 * ID mapping index is only ignored if all interrupts are
424 if (smmu->event_gsiv && smmu->pri_gsiv && smmu->gerr_gsiv
428 if (smmu->id_mapping_index >= node->mapping_count) {
429 pr_err(FW_BUG "[node %p type %d] ID mapping index overflows valid mappings\n",
434 return smmu->id_mapping_index;
435 case ACPI_IORT_NODE_PMCG:
436 pmcg = (struct acpi_iort_pmcg *)node->node_data;
437 if (pmcg->overflow_gsiv || node->mapping_count == 0)
446 static struct acpi_iort_node *iort_node_map_id(struct acpi_iort_node *node,
447 u32 id_in, u32 *id_out,
452 /* Parse the ID mapping tree to find specified node type */
454 struct acpi_iort_id_mapping *map;
455 int i, index, rc = 0;
456 u32 out_ref = 0, map_id = id;
458 if (IORT_TYPE_MASK(node->type) & type_mask) {
464 if (!node->mapping_offset || !node->mapping_count)
467 map = ACPI_ADD_PTR(struct acpi_iort_id_mapping, node,
468 node->mapping_offset);
471 if (!map->output_reference) {
472 pr_err(FW_BUG "[node %p type %d] ID map has NULL parent reference\n",
478 * Get the special ID mapping index (if any) and skip its
479 * associated ID map to prevent erroneous multi-stage
480 * IORT ID translations.
482 index = iort_get_id_mapping_index(node);
484 /* Do the ID translation */
485 for (i = 0; i < node->mapping_count; i++, map++) {
486 /* if it is special mapping index, skip it */
490 rc = iort_id_map(map, node->type, map_id, &id, out_ref);
494 out_ref = map->output_reference;
497 if (i == node->mapping_count && !out_ref)
500 node = ACPI_ADD_PTR(struct acpi_iort_node, iort_table,
501 rc ? out_ref : map->output_reference);
505 /* Map input ID to output ID unchanged on mapping failure */
512 static struct acpi_iort_node *iort_node_map_platform_id(
513 struct acpi_iort_node *node, u32 *id_out, u8 type_mask,
516 struct acpi_iort_node *parent;
519 /* step 1: retrieve the initial dev id */
520 parent = iort_node_get_id(node, &id, index);
525 * optional step 2: map the initial dev id if its parent is not
526 * the target type we want, map it again for the use cases such
527 * as NC (named component) -> SMMU -> ITS. If the type is matched,
528 * return the initial dev id and its parent pointer directly.
530 if (!(IORT_TYPE_MASK(parent->type) & type_mask))
531 parent = iort_node_map_id(parent, id, id_out, type_mask);
539 static struct acpi_iort_node *iort_find_dev_node(struct device *dev)
541 struct pci_bus *pbus;
543 if (!dev_is_pci(dev)) {
544 struct acpi_iort_node *node;
546 * scan iort_fwnode_list to see if it's an iort platform
547 * device (such as SMMU, PMCG),its iort node already cached
548 * and associated with fwnode when iort platform devices
551 node = iort_get_iort_node(dev->fwnode);
555 * if not, then it should be a platform device defined in
556 * DSDT/SSDT (with Named Component node in IORT)
558 return iort_scan_node(ACPI_IORT_NODE_NAMED_COMPONENT,
559 iort_match_node_callback, dev);
562 pbus = to_pci_dev(dev)->bus;
564 return iort_scan_node(ACPI_IORT_NODE_PCI_ROOT_COMPLEX,
565 iort_match_node_callback, &pbus->dev);
569 * iort_msi_map_id() - Map a MSI input ID for a device
570 * @dev: The device for which the mapping is to be done.
571 * @input_id: The device input ID.
573 * Returns: mapped MSI ID on success, input ID otherwise
575 u32 iort_msi_map_id(struct device *dev, u32 input_id)
577 struct acpi_iort_node *node;
580 node = iort_find_dev_node(dev);
584 iort_node_map_id(node, input_id, &dev_id, IORT_MSI_TYPE);
589 * iort_pmsi_get_dev_id() - Get the device id for a device
590 * @dev: The device for which the mapping is to be done.
591 * @dev_id: The device ID found.
593 * Returns: 0 for successful find a dev id, -ENODEV on error
595 int iort_pmsi_get_dev_id(struct device *dev, u32 *dev_id)
598 struct acpi_iort_node *node;
600 node = iort_find_dev_node(dev);
604 index = iort_get_id_mapping_index(node);
605 /* if there is a valid index, go get the dev_id directly */
607 if (iort_node_get_id(node, dev_id, index))
610 for (i = 0; i < node->mapping_count; i++) {
611 if (iort_node_map_platform_id(node, dev_id,
620 static int __maybe_unused iort_find_its_base(u32 its_id, phys_addr_t *base)
622 struct iort_its_msi_chip *its_msi_chip;
625 spin_lock(&iort_msi_chip_lock);
626 list_for_each_entry(its_msi_chip, &iort_msi_chip_list, list) {
627 if (its_msi_chip->translation_id == its_id) {
628 *base = its_msi_chip->base_addr;
633 spin_unlock(&iort_msi_chip_lock);
639 * iort_dev_find_its_id() - Find the ITS identifier for a device
642 * @idx: Index of the ITS identifier list.
643 * @its_id: ITS identifier.
645 * Returns: 0 on success, appropriate error value otherwise
647 static int iort_dev_find_its_id(struct device *dev, u32 id,
648 unsigned int idx, int *its_id)
650 struct acpi_iort_its_group *its;
651 struct acpi_iort_node *node;
653 node = iort_find_dev_node(dev);
657 node = iort_node_map_id(node, id, NULL, IORT_MSI_TYPE);
661 /* Move to ITS specific data */
662 its = (struct acpi_iort_its_group *)node->node_data;
663 if (idx >= its->its_count) {
664 dev_err(dev, "requested ITS ID index [%d] overruns ITS entries [%d]\n",
665 idx, its->its_count);
669 *its_id = its->identifiers[idx];
674 * iort_get_device_domain() - Find MSI domain related to a device
676 * @id: Requester ID for the device.
677 * @bus_token: irq domain bus token.
679 * Returns: the MSI domain for this device, NULL otherwise
681 struct irq_domain *iort_get_device_domain(struct device *dev, u32 id,
682 enum irq_domain_bus_token bus_token)
684 struct fwnode_handle *handle;
687 if (iort_dev_find_its_id(dev, id, 0, &its_id))
690 handle = iort_find_domain_token(its_id);
694 return irq_find_matching_fwnode(handle, bus_token);
697 static void iort_set_device_domain(struct device *dev,
698 struct acpi_iort_node *node)
700 struct acpi_iort_its_group *its;
701 struct acpi_iort_node *msi_parent;
702 struct acpi_iort_id_mapping *map;
703 struct fwnode_handle *iort_fwnode;
704 struct irq_domain *domain;
707 index = iort_get_id_mapping_index(node);
711 map = ACPI_ADD_PTR(struct acpi_iort_id_mapping, node,
712 node->mapping_offset + index * sizeof(*map));
715 if (!map->output_reference ||
716 !(map->flags & ACPI_IORT_ID_SINGLE_MAPPING)) {
717 pr_err(FW_BUG "[node %p type %d] Invalid MSI mapping\n",
722 msi_parent = ACPI_ADD_PTR(struct acpi_iort_node, iort_table,
723 map->output_reference);
725 if (!msi_parent || msi_parent->type != ACPI_IORT_NODE_ITS_GROUP)
728 /* Move to ITS specific data */
729 its = (struct acpi_iort_its_group *)msi_parent->node_data;
731 iort_fwnode = iort_find_domain_token(its->identifiers[0]);
735 domain = irq_find_matching_fwnode(iort_fwnode, DOMAIN_BUS_PLATFORM_MSI);
737 dev_set_msi_domain(dev, domain);
741 * iort_get_platform_device_domain() - Find MSI domain related to a
743 * @dev: the dev pointer associated with the platform device
745 * Returns: the MSI domain for this device, NULL otherwise
747 static struct irq_domain *iort_get_platform_device_domain(struct device *dev)
749 struct acpi_iort_node *node, *msi_parent = NULL;
750 struct fwnode_handle *iort_fwnode;
751 struct acpi_iort_its_group *its;
754 /* find its associated iort node */
755 node = iort_scan_node(ACPI_IORT_NODE_NAMED_COMPONENT,
756 iort_match_node_callback, dev);
760 /* then find its msi parent node */
761 for (i = 0; i < node->mapping_count; i++) {
762 msi_parent = iort_node_map_platform_id(node, NULL,
771 /* Move to ITS specific data */
772 its = (struct acpi_iort_its_group *)msi_parent->node_data;
774 iort_fwnode = iort_find_domain_token(its->identifiers[0]);
778 return irq_find_matching_fwnode(iort_fwnode, DOMAIN_BUS_PLATFORM_MSI);
781 void acpi_configure_pmsi_domain(struct device *dev)
783 struct irq_domain *msi_domain;
785 msi_domain = iort_get_platform_device_domain(dev);
787 dev_set_msi_domain(dev, msi_domain);
790 #ifdef CONFIG_IOMMU_API
791 static struct acpi_iort_node *iort_get_msi_resv_iommu(struct device *dev)
793 struct acpi_iort_node *iommu;
794 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
796 iommu = iort_get_iort_node(fwspec->iommu_fwnode);
798 if (iommu && (iommu->type == ACPI_IORT_NODE_SMMU_V3)) {
799 struct acpi_iort_smmu_v3 *smmu;
801 smmu = (struct acpi_iort_smmu_v3 *)iommu->node_data;
802 if (smmu->model == ACPI_IORT_SMMU_V3_HISILICON_HI161X)
810 * iort_iommu_msi_get_resv_regions - Reserved region driver helper
811 * @dev: Device from iommu_get_resv_regions()
812 * @head: Reserved region list from iommu_get_resv_regions()
814 * Returns: Number of msi reserved regions on success (0 if platform
815 * doesn't require the reservation or no associated msi regions),
816 * appropriate error value otherwise. The ITS interrupt translation
817 * spaces (ITS_base + SZ_64K, SZ_64K) associated with the device
818 * are the msi reserved regions.
820 int iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head)
822 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
823 struct acpi_iort_its_group *its;
824 struct acpi_iort_node *iommu_node, *its_node = NULL;
827 iommu_node = iort_get_msi_resv_iommu(dev);
832 * Current logic to reserve ITS regions relies on HW topologies
833 * where a given PCI or named component maps its IDs to only one
834 * ITS group; if a PCI or named component can map its IDs to
835 * different ITS groups through IORT mappings this function has
836 * to be reworked to ensure we reserve regions for all ITS groups
837 * a given PCI or named component may map IDs to.
840 for (i = 0; i < fwspec->num_ids; i++) {
841 its_node = iort_node_map_id(iommu_node,
843 NULL, IORT_MSI_TYPE);
851 /* Move to ITS specific data */
852 its = (struct acpi_iort_its_group *)its_node->node_data;
854 for (i = 0; i < its->its_count; i++) {
857 if (!iort_find_its_base(its->identifiers[i], &base)) {
858 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
859 struct iommu_resv_region *region;
861 region = iommu_alloc_resv_region(base + SZ_64K, SZ_64K,
862 prot, IOMMU_RESV_MSI);
864 list_add_tail(®ion->list, head);
870 return (resv == its->its_count) ? resv : -ENODEV;
873 static inline bool iort_iommu_driver_enabled(u8 type)
876 case ACPI_IORT_NODE_SMMU_V3:
877 return IS_ENABLED(CONFIG_ARM_SMMU_V3);
878 case ACPI_IORT_NODE_SMMU:
879 return IS_ENABLED(CONFIG_ARM_SMMU);
881 pr_warn("IORT node type %u does not describe an SMMU\n", type);
886 static bool iort_pci_rc_supports_ats(struct acpi_iort_node *node)
888 struct acpi_iort_root_complex *pci_rc;
890 pci_rc = (struct acpi_iort_root_complex *)node->node_data;
891 return pci_rc->ats_attribute & ACPI_IORT_ATS_SUPPORTED;
894 static int iort_iommu_xlate(struct device *dev, struct acpi_iort_node *node,
897 const struct iommu_ops *ops;
898 struct fwnode_handle *iort_fwnode;
903 iort_fwnode = iort_get_fwnode(node);
908 * If the ops look-up fails, this means that either
909 * the SMMU drivers have not been probed yet or that
910 * the SMMU drivers are not built in the kernel;
911 * Depending on whether the SMMU drivers are built-in
912 * in the kernel or not, defer the IOMMU configuration
915 ops = iommu_ops_from_fwnode(iort_fwnode);
917 return iort_iommu_driver_enabled(node->type) ?
918 -EPROBE_DEFER : -ENODEV;
920 return acpi_iommu_fwspec_init(dev, streamid, iort_fwnode, ops);
923 struct iort_pci_alias_info {
925 struct acpi_iort_node *node;
928 static int iort_pci_iommu_init(struct pci_dev *pdev, u16 alias, void *data)
930 struct iort_pci_alias_info *info = data;
931 struct acpi_iort_node *parent;
934 parent = iort_node_map_id(info->node, alias, &streamid,
936 return iort_iommu_xlate(info->dev, parent, streamid);
939 static void iort_named_component_init(struct device *dev,
940 struct acpi_iort_node *node)
942 struct property_entry props[3] = {};
943 struct acpi_iort_named_component *nc;
945 nc = (struct acpi_iort_named_component *)node->node_data;
946 props[0] = PROPERTY_ENTRY_U32("pasid-num-bits",
947 FIELD_GET(ACPI_IORT_NC_PASID_BITS,
949 if (nc->node_flags & ACPI_IORT_NC_STALL_SUPPORTED)
950 props[1] = PROPERTY_ENTRY_BOOL("dma-can-stall");
952 if (device_create_managed_software_node(dev, props, NULL))
953 dev_warn(dev, "Could not add device properties\n");
956 static int iort_nc_iommu_map(struct device *dev, struct acpi_iort_node *node)
958 struct acpi_iort_node *parent;
959 int err = -ENODEV, i = 0;
964 parent = iort_node_map_platform_id(node, &streamid,
969 err = iort_iommu_xlate(dev, parent, streamid);
970 } while (parent && !err);
975 static int iort_nc_iommu_map_id(struct device *dev,
976 struct acpi_iort_node *node,
979 struct acpi_iort_node *parent;
982 parent = iort_node_map_id(node, *in_id, &streamid, IORT_IOMMU_TYPE);
984 return iort_iommu_xlate(dev, parent, streamid);
991 * iort_iommu_configure_id - Set-up IOMMU configuration for a device.
993 * @dev: device to configure
994 * @id_in: optional input id const value pointer
996 * Returns: 0 on success, <0 on failure
998 int iort_iommu_configure_id(struct device *dev, const u32 *id_in)
1000 struct acpi_iort_node *node;
1003 if (dev_is_pci(dev)) {
1004 struct iommu_fwspec *fwspec;
1005 struct pci_bus *bus = to_pci_dev(dev)->bus;
1006 struct iort_pci_alias_info info = { .dev = dev };
1008 node = iort_scan_node(ACPI_IORT_NODE_PCI_ROOT_COMPLEX,
1009 iort_match_node_callback, &bus->dev);
1014 err = pci_for_each_dma_alias(to_pci_dev(dev),
1015 iort_pci_iommu_init, &info);
1017 fwspec = dev_iommu_fwspec_get(dev);
1018 if (fwspec && iort_pci_rc_supports_ats(node))
1019 fwspec->flags |= IOMMU_FWSPEC_PCI_RC_ATS;
1021 node = iort_scan_node(ACPI_IORT_NODE_NAMED_COMPONENT,
1022 iort_match_node_callback, dev);
1026 err = id_in ? iort_nc_iommu_map_id(dev, node, id_in) :
1027 iort_nc_iommu_map(dev, node);
1030 iort_named_component_init(dev, node);
1037 int iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head)
1039 int iort_iommu_configure_id(struct device *dev, const u32 *input_id)
1043 static int nc_dma_get_range(struct device *dev, u64 *size)
1045 struct acpi_iort_node *node;
1046 struct acpi_iort_named_component *ncomp;
1048 node = iort_scan_node(ACPI_IORT_NODE_NAMED_COMPONENT,
1049 iort_match_node_callback, dev);
1053 ncomp = (struct acpi_iort_named_component *)node->node_data;
1055 if (!ncomp->memory_address_limit) {
1056 pr_warn(FW_BUG "Named component missing memory address limit\n");
1060 *size = ncomp->memory_address_limit >= 64 ? U64_MAX :
1061 1ULL<<ncomp->memory_address_limit;
1066 static int rc_dma_get_range(struct device *dev, u64 *size)
1068 struct acpi_iort_node *node;
1069 struct acpi_iort_root_complex *rc;
1070 struct pci_bus *pbus = to_pci_dev(dev)->bus;
1072 node = iort_scan_node(ACPI_IORT_NODE_PCI_ROOT_COMPLEX,
1073 iort_match_node_callback, &pbus->dev);
1074 if (!node || node->revision < 1)
1077 rc = (struct acpi_iort_root_complex *)node->node_data;
1079 if (!rc->memory_address_limit) {
1080 pr_warn(FW_BUG "Root complex missing memory address limit\n");
1084 *size = rc->memory_address_limit >= 64 ? U64_MAX :
1085 1ULL<<rc->memory_address_limit;
1091 * iort_dma_get_ranges() - Look up DMA addressing limit for the device
1092 * @dev: device to lookup
1093 * @size: DMA range size result pointer
1095 * Return: 0 on success, an error otherwise.
1097 int iort_dma_get_ranges(struct device *dev, u64 *size)
1099 if (dev_is_pci(dev))
1100 return rc_dma_get_range(dev, size);
1102 return nc_dma_get_range(dev, size);
1105 static void __init acpi_iort_register_irq(int hwirq, const char *name,
1107 struct resource *res)
1109 int irq = acpi_register_gsi(NULL, hwirq, trigger,
1113 pr_err("could not register gsi hwirq %d name [%s]\n", hwirq,
1120 res->flags = IORESOURCE_IRQ;
1124 static int __init arm_smmu_v3_count_resources(struct acpi_iort_node *node)
1126 struct acpi_iort_smmu_v3 *smmu;
1127 /* Always present mem resource */
1130 /* Retrieve SMMUv3 specific data */
1131 smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
1133 if (smmu->event_gsiv)
1139 if (smmu->gerr_gsiv)
1142 if (smmu->sync_gsiv)
1148 static bool arm_smmu_v3_is_combined_irq(struct acpi_iort_smmu_v3 *smmu)
1151 * Cavium ThunderX2 implementation doesn't not support unique
1152 * irq line. Use single irq line for all the SMMUv3 interrupts.
1154 if (smmu->model != ACPI_IORT_SMMU_V3_CAVIUM_CN99XX)
1158 * ThunderX2 doesn't support MSIs from the SMMU, so we're checking
1161 return smmu->event_gsiv == smmu->pri_gsiv &&
1162 smmu->event_gsiv == smmu->gerr_gsiv &&
1163 smmu->event_gsiv == smmu->sync_gsiv;
1166 static unsigned long arm_smmu_v3_resource_size(struct acpi_iort_smmu_v3 *smmu)
1169 * Override the size, for Cavium ThunderX2 implementation
1170 * which doesn't support the page 1 SMMU register space.
1172 if (smmu->model == ACPI_IORT_SMMU_V3_CAVIUM_CN99XX)
1178 static void __init arm_smmu_v3_init_resources(struct resource *res,
1179 struct acpi_iort_node *node)
1181 struct acpi_iort_smmu_v3 *smmu;
1184 /* Retrieve SMMUv3 specific data */
1185 smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
1187 res[num_res].start = smmu->base_address;
1188 res[num_res].end = smmu->base_address +
1189 arm_smmu_v3_resource_size(smmu) - 1;
1190 res[num_res].flags = IORESOURCE_MEM;
1193 if (arm_smmu_v3_is_combined_irq(smmu)) {
1194 if (smmu->event_gsiv)
1195 acpi_iort_register_irq(smmu->event_gsiv, "combined",
1196 ACPI_EDGE_SENSITIVE,
1200 if (smmu->event_gsiv)
1201 acpi_iort_register_irq(smmu->event_gsiv, "eventq",
1202 ACPI_EDGE_SENSITIVE,
1206 acpi_iort_register_irq(smmu->pri_gsiv, "priq",
1207 ACPI_EDGE_SENSITIVE,
1210 if (smmu->gerr_gsiv)
1211 acpi_iort_register_irq(smmu->gerr_gsiv, "gerror",
1212 ACPI_EDGE_SENSITIVE,
1215 if (smmu->sync_gsiv)
1216 acpi_iort_register_irq(smmu->sync_gsiv, "cmdq-sync",
1217 ACPI_EDGE_SENSITIVE,
1222 static void __init arm_smmu_v3_dma_configure(struct device *dev,
1223 struct acpi_iort_node *node)
1225 struct acpi_iort_smmu_v3 *smmu;
1226 enum dev_dma_attr attr;
1228 /* Retrieve SMMUv3 specific data */
1229 smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
1231 attr = (smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE) ?
1232 DEV_DMA_COHERENT : DEV_DMA_NON_COHERENT;
1234 /* We expect the dma masks to be equivalent for all SMMUv3 set-ups */
1235 dev->dma_mask = &dev->coherent_dma_mask;
1237 /* Configure DMA for the page table walker */
1238 acpi_dma_configure(dev, attr);
1241 #if defined(CONFIG_ACPI_NUMA)
1243 * set numa proximity domain for smmuv3 device
1245 static int __init arm_smmu_v3_set_proximity(struct device *dev,
1246 struct acpi_iort_node *node)
1248 struct acpi_iort_smmu_v3 *smmu;
1250 smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
1251 if (smmu->flags & ACPI_IORT_SMMU_V3_PXM_VALID) {
1252 int dev_node = pxm_to_node(smmu->pxm);
1254 if (dev_node != NUMA_NO_NODE && !node_online(dev_node))
1257 set_dev_node(dev, dev_node);
1258 pr_info("SMMU-v3[%llx] Mapped to Proximity domain %d\n",
1265 #define arm_smmu_v3_set_proximity NULL
1268 static int __init arm_smmu_count_resources(struct acpi_iort_node *node)
1270 struct acpi_iort_smmu *smmu;
1272 /* Retrieve SMMU specific data */
1273 smmu = (struct acpi_iort_smmu *)node->node_data;
1276 * Only consider the global fault interrupt and ignore the
1277 * configuration access interrupt.
1279 * MMIO address and global fault interrupt resources are always
1280 * present so add them to the context interrupt count as a static
1283 return smmu->context_interrupt_count + 2;
1286 static void __init arm_smmu_init_resources(struct resource *res,
1287 struct acpi_iort_node *node)
1289 struct acpi_iort_smmu *smmu;
1290 int i, hw_irq, trigger, num_res = 0;
1291 u64 *ctx_irq, *glb_irq;
1293 /* Retrieve SMMU specific data */
1294 smmu = (struct acpi_iort_smmu *)node->node_data;
1296 res[num_res].start = smmu->base_address;
1297 res[num_res].end = smmu->base_address + smmu->span - 1;
1298 res[num_res].flags = IORESOURCE_MEM;
1301 glb_irq = ACPI_ADD_PTR(u64, node, smmu->global_interrupt_offset);
1303 hw_irq = IORT_IRQ_MASK(glb_irq[0]);
1304 trigger = IORT_IRQ_TRIGGER_MASK(glb_irq[0]);
1306 acpi_iort_register_irq(hw_irq, "arm-smmu-global", trigger,
1310 ctx_irq = ACPI_ADD_PTR(u64, node, smmu->context_interrupt_offset);
1311 for (i = 0; i < smmu->context_interrupt_count; i++) {
1312 hw_irq = IORT_IRQ_MASK(ctx_irq[i]);
1313 trigger = IORT_IRQ_TRIGGER_MASK(ctx_irq[i]);
1315 acpi_iort_register_irq(hw_irq, "arm-smmu-context", trigger,
1320 static void __init arm_smmu_dma_configure(struct device *dev,
1321 struct acpi_iort_node *node)
1323 struct acpi_iort_smmu *smmu;
1324 enum dev_dma_attr attr;
1326 /* Retrieve SMMU specific data */
1327 smmu = (struct acpi_iort_smmu *)node->node_data;
1329 attr = (smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK) ?
1330 DEV_DMA_COHERENT : DEV_DMA_NON_COHERENT;
1332 /* We expect the dma masks to be equivalent for SMMU set-ups */
1333 dev->dma_mask = &dev->coherent_dma_mask;
1335 /* Configure DMA for the page table walker */
1336 acpi_dma_configure(dev, attr);
1339 static int __init arm_smmu_v3_pmcg_count_resources(struct acpi_iort_node *node)
1341 struct acpi_iort_pmcg *pmcg;
1343 /* Retrieve PMCG specific data */
1344 pmcg = (struct acpi_iort_pmcg *)node->node_data;
1347 * There are always 2 memory resources.
1348 * If the overflow_gsiv is present then add that for a total of 3.
1350 return pmcg->overflow_gsiv ? 3 : 2;
1353 static void __init arm_smmu_v3_pmcg_init_resources(struct resource *res,
1354 struct acpi_iort_node *node)
1356 struct acpi_iort_pmcg *pmcg;
1358 /* Retrieve PMCG specific data */
1359 pmcg = (struct acpi_iort_pmcg *)node->node_data;
1361 res[0].start = pmcg->page0_base_address;
1362 res[0].end = pmcg->page0_base_address + SZ_4K - 1;
1363 res[0].flags = IORESOURCE_MEM;
1365 * The initial version in DEN0049C lacked a way to describe register
1366 * page 1, which makes it broken for most PMCG implementations; in
1367 * that case, just let the driver fail gracefully if it expects to
1368 * find a second memory resource.
1370 if (node->revision > 0) {
1371 res[1].start = pmcg->page1_base_address;
1372 res[1].end = pmcg->page1_base_address + SZ_4K - 1;
1373 res[1].flags = IORESOURCE_MEM;
1376 if (pmcg->overflow_gsiv)
1377 acpi_iort_register_irq(pmcg->overflow_gsiv, "overflow",
1378 ACPI_EDGE_SENSITIVE, &res[2]);
1381 static struct acpi_platform_list pmcg_plat_info[] __initdata = {
1382 /* HiSilicon Hip08 Platform */
1383 {"HISI ", "HIP08 ", 0, ACPI_SIG_IORT, greater_than_or_equal,
1384 "Erratum #162001800", IORT_SMMU_V3_PMCG_HISI_HIP08},
1388 static int __init arm_smmu_v3_pmcg_add_platdata(struct platform_device *pdev)
1393 idx = acpi_match_platform_list(pmcg_plat_info);
1395 model = pmcg_plat_info[idx].data;
1397 model = IORT_SMMU_V3_PMCG_GENERIC;
1399 return platform_device_add_data(pdev, &model, sizeof(model));
1402 struct iort_dev_config {
1404 int (*dev_init)(struct acpi_iort_node *node);
1405 void (*dev_dma_configure)(struct device *dev,
1406 struct acpi_iort_node *node);
1407 int (*dev_count_resources)(struct acpi_iort_node *node);
1408 void (*dev_init_resources)(struct resource *res,
1409 struct acpi_iort_node *node);
1410 int (*dev_set_proximity)(struct device *dev,
1411 struct acpi_iort_node *node);
1412 int (*dev_add_platdata)(struct platform_device *pdev);
1415 static const struct iort_dev_config iort_arm_smmu_v3_cfg __initconst = {
1416 .name = "arm-smmu-v3",
1417 .dev_dma_configure = arm_smmu_v3_dma_configure,
1418 .dev_count_resources = arm_smmu_v3_count_resources,
1419 .dev_init_resources = arm_smmu_v3_init_resources,
1420 .dev_set_proximity = arm_smmu_v3_set_proximity,
1423 static const struct iort_dev_config iort_arm_smmu_cfg __initconst = {
1425 .dev_dma_configure = arm_smmu_dma_configure,
1426 .dev_count_resources = arm_smmu_count_resources,
1427 .dev_init_resources = arm_smmu_init_resources,
1430 static const struct iort_dev_config iort_arm_smmu_v3_pmcg_cfg __initconst = {
1431 .name = "arm-smmu-v3-pmcg",
1432 .dev_count_resources = arm_smmu_v3_pmcg_count_resources,
1433 .dev_init_resources = arm_smmu_v3_pmcg_init_resources,
1434 .dev_add_platdata = arm_smmu_v3_pmcg_add_platdata,
1437 static __init const struct iort_dev_config *iort_get_dev_cfg(
1438 struct acpi_iort_node *node)
1440 switch (node->type) {
1441 case ACPI_IORT_NODE_SMMU_V3:
1442 return &iort_arm_smmu_v3_cfg;
1443 case ACPI_IORT_NODE_SMMU:
1444 return &iort_arm_smmu_cfg;
1445 case ACPI_IORT_NODE_PMCG:
1446 return &iort_arm_smmu_v3_pmcg_cfg;
1453 * iort_add_platform_device() - Allocate a platform device for IORT node
1454 * @node: Pointer to device ACPI IORT node
1455 * @ops: Pointer to IORT device config struct
1457 * Returns: 0 on success, <0 failure
1459 static int __init iort_add_platform_device(struct acpi_iort_node *node,
1460 const struct iort_dev_config *ops)
1462 struct fwnode_handle *fwnode;
1463 struct platform_device *pdev;
1467 pdev = platform_device_alloc(ops->name, PLATFORM_DEVID_AUTO);
1471 if (ops->dev_set_proximity) {
1472 ret = ops->dev_set_proximity(&pdev->dev, node);
1477 count = ops->dev_count_resources(node);
1479 r = kcalloc(count, sizeof(*r), GFP_KERNEL);
1485 ops->dev_init_resources(r, node);
1487 ret = platform_device_add_resources(pdev, r, count);
1489 * Resources are duplicated in platform_device_add_resources,
1490 * free their allocated memory
1498 * Platform devices based on PMCG nodes uses platform_data to
1499 * pass the hardware model info to the driver. For others, add
1500 * a copy of IORT node pointer to platform_data to be used to
1501 * retrieve IORT data information.
1503 if (ops->dev_add_platdata)
1504 ret = ops->dev_add_platdata(pdev);
1506 ret = platform_device_add_data(pdev, &node, sizeof(node));
1511 fwnode = iort_get_fwnode(node);
1518 pdev->dev.fwnode = fwnode;
1520 if (ops->dev_dma_configure)
1521 ops->dev_dma_configure(&pdev->dev, node);
1523 iort_set_device_domain(&pdev->dev, node);
1525 ret = platform_device_add(pdev);
1527 goto dma_deconfigure;
1532 arch_teardown_dma_ops(&pdev->dev);
1534 platform_device_put(pdev);
1540 static void __init iort_enable_acs(struct acpi_iort_node *iort_node)
1542 static bool acs_enabled __initdata;
1547 if (iort_node->type == ACPI_IORT_NODE_PCI_ROOT_COMPLEX) {
1548 struct acpi_iort_node *parent;
1549 struct acpi_iort_id_mapping *map;
1552 map = ACPI_ADD_PTR(struct acpi_iort_id_mapping, iort_node,
1553 iort_node->mapping_offset);
1555 for (i = 0; i < iort_node->mapping_count; i++, map++) {
1556 if (!map->output_reference)
1559 parent = ACPI_ADD_PTR(struct acpi_iort_node,
1560 iort_table, map->output_reference);
1562 * If we detect a RC->SMMU mapping, make sure
1563 * we enable ACS on the system.
1565 if ((parent->type == ACPI_IORT_NODE_SMMU) ||
1566 (parent->type == ACPI_IORT_NODE_SMMU_V3)) {
1575 static inline void iort_enable_acs(struct acpi_iort_node *iort_node) { }
1578 static void __init iort_init_platform_devices(void)
1580 struct acpi_iort_node *iort_node, *iort_end;
1581 struct acpi_table_iort *iort;
1582 struct fwnode_handle *fwnode;
1584 const struct iort_dev_config *ops;
1587 * iort_table and iort both point to the start of IORT table, but
1588 * have different struct types
1590 iort = (struct acpi_table_iort *)iort_table;
1592 /* Get the first IORT node */
1593 iort_node = ACPI_ADD_PTR(struct acpi_iort_node, iort,
1595 iort_end = ACPI_ADD_PTR(struct acpi_iort_node, iort,
1596 iort_table->length);
1598 for (i = 0; i < iort->node_count; i++) {
1599 if (iort_node >= iort_end) {
1600 pr_err("iort node pointer overflows, bad table\n");
1604 iort_enable_acs(iort_node);
1606 ops = iort_get_dev_cfg(iort_node);
1608 fwnode = acpi_alloc_fwnode_static();
1612 iort_set_fwnode(iort_node, fwnode);
1614 ret = iort_add_platform_device(iort_node, ops);
1616 iort_delete_fwnode(iort_node);
1617 acpi_free_fwnode_static(fwnode);
1622 iort_node = ACPI_ADD_PTR(struct acpi_iort_node, iort_node,
1627 void __init acpi_iort_init(void)
1631 /* iort_table will be used at runtime after the iort init,
1632 * so we don't need to call acpi_put_table() to release
1633 * the IORT table mapping.
1635 status = acpi_get_table(ACPI_SIG_IORT, 0, &iort_table);
1636 if (ACPI_FAILURE(status)) {
1637 if (status != AE_NOT_FOUND) {
1638 const char *msg = acpi_format_exception(status);
1640 pr_err("Failed to get table, %s\n", msg);
1646 iort_init_platform_devices();
1649 #ifdef CONFIG_ZONE_DMA
1651 * Extract the highest CPU physical address accessible to all DMA masters in
1652 * the system. PHYS_ADDR_MAX is returned when no constrained device is found.
1654 phys_addr_t __init acpi_iort_dma_get_max_cpu_address(void)
1656 phys_addr_t limit = PHYS_ADDR_MAX;
1657 struct acpi_iort_node *node, *end;
1658 struct acpi_table_iort *iort;
1665 status = acpi_get_table(ACPI_SIG_IORT, 0,
1666 (struct acpi_table_header **)&iort);
1667 if (ACPI_FAILURE(status))
1670 node = ACPI_ADD_PTR(struct acpi_iort_node, iort, iort->node_offset);
1671 end = ACPI_ADD_PTR(struct acpi_iort_node, iort, iort->header.length);
1673 for (i = 0; i < iort->node_count; i++) {
1677 switch (node->type) {
1678 struct acpi_iort_named_component *ncomp;
1679 struct acpi_iort_root_complex *rc;
1680 phys_addr_t local_limit;
1682 case ACPI_IORT_NODE_NAMED_COMPONENT:
1683 ncomp = (struct acpi_iort_named_component *)node->node_data;
1684 local_limit = DMA_BIT_MASK(ncomp->memory_address_limit);
1685 limit = min_not_zero(limit, local_limit);
1688 case ACPI_IORT_NODE_PCI_ROOT_COMPLEX:
1689 if (node->revision < 1)
1692 rc = (struct acpi_iort_root_complex *)node->node_data;
1693 local_limit = DMA_BIT_MASK(rc->memory_address_limit);
1694 limit = min_not_zero(limit, local_limit);
1697 node = ACPI_ADD_PTR(struct acpi_iort_node, node, node->length);
1699 acpi_put_table(&iort->header);