drivers/iommu/arm/arm-smmu/arm-smmu-impl.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 // Miscellaneous Arm SMMU implementation and integration quirks
   3 // Copyright (C) 2019 Arm Limited
   4
   5 #define pr_fmt(fmt) "arm-smmu: " fmt
   6
   7 #include <linux/bitfield.h>
   8 #include <linux/of.h>
   9
  10 #include "arm-smmu.h"
  11
  12
  13 static int arm_smmu_gr0_ns(int offset)
  14 {
  15         switch(offset) {
  16         case ARM_SMMU_GR0_sCR0:
  17         case ARM_SMMU_GR0_sACR:
  18         case ARM_SMMU_GR0_sGFSR:
  19         case ARM_SMMU_GR0_sGFSYNR0:
  20         case ARM_SMMU_GR0_sGFSYNR1:
  21         case ARM_SMMU_GR0_sGFSYNR2:
  22                 return offset + 0x400;
  23         default:
  24                 return offset;
  25         }
  26 }
  27
  28 static u32 arm_smmu_read_ns(struct arm_smmu_device *smmu, int page,
  29                             int offset)
  30 {
  31         if (page == ARM_SMMU_GR0)
  32                 offset = arm_smmu_gr0_ns(offset);
  33         return readl_relaxed(arm_smmu_page(smmu, page) + offset);
  34 }
  35
  36 static void arm_smmu_write_ns(struct arm_smmu_device *smmu, int page,
  37                               int offset, u32 val)
  38 {
  39         if (page == ARM_SMMU_GR0)
  40                 offset = arm_smmu_gr0_ns(offset);
  41         writel_relaxed(val, arm_smmu_page(smmu, page) + offset);
  42 }
  43
  44 /* Since we don't care for sGFAR, we can do without 64-bit accessors */
  45 static const struct arm_smmu_impl calxeda_impl = {
  46         .read_reg = arm_smmu_read_ns,
  47         .write_reg = arm_smmu_write_ns,
  48 };
  49
  50
  51 struct cavium_smmu {
  52         struct arm_smmu_device smmu;
  53         u32 id_base;
  54 };
  55
  56 static int cavium_cfg_probe(struct arm_smmu_device *smmu)
  57 {
  58         static atomic_t context_count = ATOMIC_INIT(0);
  59         struct cavium_smmu *cs = container_of(smmu, struct cavium_smmu, smmu);
  60         /*
  61          * Cavium CN88xx erratum #27704.
  62          * Ensure ASID and VMID allocation is unique across all SMMUs in
  63          * the system.
  64          */
  65         cs->id_base = atomic_fetch_add(smmu->num_context_banks, &context_count);
  66         dev_notice(smmu->dev, "\tenabling workaround for Cavium erratum 27704\n");
  67
  68         return 0;
  69 }
  70
  71 static int cavium_init_context(struct arm_smmu_domain *smmu_domain,
  72                 struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
  73 {
  74         struct cavium_smmu *cs = container_of(smmu_domain->smmu,
  75                                               struct cavium_smmu, smmu);
  76
  77         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
  78                 smmu_domain->cfg.vmid += cs->id_base;
  79         else
  80                 smmu_domain->cfg.asid += cs->id_base;
  81
  82         return 0;
  83 }
  84
  85 static const struct arm_smmu_impl cavium_impl = {
  86         .cfg_probe = cavium_cfg_probe,
  87         .init_context = cavium_init_context,
  88 };
  89
  90 static struct arm_smmu_device *cavium_smmu_impl_init(struct arm_smmu_device *smmu)
  91 {
  92         struct cavium_smmu *cs;
  93
  94         cs = devm_kzalloc(smmu->dev, sizeof(*cs), GFP_KERNEL);
  95         if (!cs)
  96                 return ERR_PTR(-ENOMEM);
  97
  98         cs->smmu = *smmu;
  99         cs->smmu.impl = &cavium_impl;
 100
 101         devm_kfree(smmu->dev, smmu);
 102
 103         return &cs->smmu;
 104 }
 105
 106
 107 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
 108
 109 #define ARM_MMU500_ACR_CACHE_LOCK       (1 << 26)
 110 #define ARM_MMU500_ACR_S2CRB_TLBEN      (1 << 10)
 111 #define ARM_MMU500_ACR_SMTNMB_TLBEN     (1 << 8)
 112
 113 int arm_mmu500_reset(struct arm_smmu_device *smmu)
 114 {
 115         u32 reg, major;
 116         int i;
 117         /*
 118          * On MMU-500 r2p0 onwards we need to clear ACR.CACHE_LOCK before
 119          * writes to the context bank ACTLRs will stick. And we just hope that
 120          * Secure has also cleared SACR.CACHE_LOCK for this to take effect...
 121          */
 122         reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID7);
 123         major = FIELD_GET(ARM_SMMU_ID7_MAJOR, reg);
 124         reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR);
 125         if (major >= 2)
 126                 reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
 127         /*
 128          * Allow unmatched Stream IDs to allocate bypass
 129          * TLB entries for reduced latency.
 130          */
 131         reg |= ARM_MMU500_ACR_SMTNMB_TLBEN | ARM_MMU500_ACR_S2CRB_TLBEN;
 132         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sACR, reg);
 133
 134         /*
 135          * Disable MMU-500's not-particularly-beneficial next-page
 136          * prefetcher for the sake of errata #841119 and #826419.
 137          */
 138         for (i = 0; i < smmu->num_context_banks; ++i) {
 139                 reg = arm_smmu_cb_read(smmu, i, ARM_SMMU_CB_ACTLR);
 140                 reg &= ~ARM_MMU500_ACTLR_CPRE;
 141                 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_ACTLR, reg);
 142         }
 143
 144         return 0;
 145 }
 146
 147 static const struct arm_smmu_impl arm_mmu500_impl = {
 148         .reset = arm_mmu500_reset,
 149 };
 150
 151 static u64 mrvl_mmu500_readq(struct arm_smmu_device *smmu, int page, int off)
 152 {
 153         /*
 154          * Marvell Armada-AP806 erratum #582743.
 155          * Split all the readq to double readl
 156          */
 157         return hi_lo_readq_relaxed(arm_smmu_page(smmu, page) + off);
 158 }
 159
 160 static void mrvl_mmu500_writeq(struct arm_smmu_device *smmu, int page, int off,
 161                                u64 val)
 162 {
 163         /*
 164          * Marvell Armada-AP806 erratum #582743.
 165          * Split all the writeq to double writel
 166          */
 167         hi_lo_writeq_relaxed(val, arm_smmu_page(smmu, page) + off);
 168 }
 169
 170 static int mrvl_mmu500_cfg_probe(struct arm_smmu_device *smmu)
 171 {
 172
 173         /*
 174          * Armada-AP806 erratum #582743.
 175          * Hide the SMMU_IDR2.PTFSv8 fields to sidestep the AArch64
 176          * formats altogether and allow using 32 bits access on the
 177          * interconnect.
 178          */
 179         smmu->features &= ~(ARM_SMMU_FEAT_FMT_AARCH64_4K |
 180                             ARM_SMMU_FEAT_FMT_AARCH64_16K |
 181                             ARM_SMMU_FEAT_FMT_AARCH64_64K);
 182
 183         return 0;
 184 }
 185
 186 static const struct arm_smmu_impl mrvl_mmu500_impl = {
 187         .read_reg64 = mrvl_mmu500_readq,
 188         .write_reg64 = mrvl_mmu500_writeq,
 189         .cfg_probe = mrvl_mmu500_cfg_probe,
 190         .reset = arm_mmu500_reset,
 191 };
 192
 193
 194 struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu)
 195 {
 196         const struct device_node *np = smmu->dev->of_node;
 197
 198         /*
 199          * Set the impl for model-specific implementation quirks first,
 200          * such that platform integration quirks can pick it up and
 201          * inherit from it if necessary.
 202          */
 203         switch (smmu->model) {
 204         case ARM_MMU500:
 205                 smmu->impl = &arm_mmu500_impl;
 206                 break;
 207         case CAVIUM_SMMUV2:
 208                 return cavium_smmu_impl_init(smmu);
 209         default:
 210                 break;
 211         }
 212
 213         /* This is implicitly MMU-400 */
 214         if (of_property_read_bool(np, "calxeda,smmu-secure-config-access"))
 215                 smmu->impl = &calxeda_impl;
 216
 217         if (of_device_is_compatible(np, "nvidia,tegra194-smmu"))
 218                 return nvidia_smmu_impl_init(smmu);
 219
 220         if (of_device_is_compatible(np, "qcom,sdm845-smmu-500") ||
 221             of_device_is_compatible(np, "qcom,sc7180-smmu-500") ||
 222             of_device_is_compatible(np, "qcom,sm8150-smmu-500") ||
 223             of_device_is_compatible(np, "qcom,sm8250-smmu-500"))
 224                 return qcom_smmu_impl_init(smmu);
 225
 226         if (of_device_is_compatible(np, "marvell,ap806-smmu-500"))
 227                 smmu->impl = &mrvl_mmu500_impl;
 228
 229         return smmu;
 230 }