drivers/cache/sifive_ccache.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * SiFive composable cache controller Driver
   4  *
   5  * Copyright (C) 2018-2022 SiFive, Inc.
   6  *
   7  */
   8
   9 #define pr_fmt(fmt) "CCACHE: " fmt
  10
  11 #include <linux/align.h>
  12 #include <linux/debugfs.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/of_irq.h>
  15 #include <linux/of_address.h>
  16 #include <linux/device.h>
  17 #include <linux/bitfield.h>
  18 #include <asm/cacheflush.h>
  19 #include <asm/cacheinfo.h>
  20 #include <asm/dma-noncoherent.h>
  21 #include <soc/sifive/sifive_ccache.h>
  22
  23 #define SIFIVE_CCACHE_DIRECCFIX_LOW 0x100
  24 #define SIFIVE_CCACHE_DIRECCFIX_HIGH 0x104
  25 #define SIFIVE_CCACHE_DIRECCFIX_COUNT 0x108
  26
  27 #define SIFIVE_CCACHE_DIRECCFAIL_LOW 0x120
  28 #define SIFIVE_CCACHE_DIRECCFAIL_HIGH 0x124
  29 #define SIFIVE_CCACHE_DIRECCFAIL_COUNT 0x128
  30
  31 #define SIFIVE_CCACHE_DATECCFIX_LOW 0x140
  32 #define SIFIVE_CCACHE_DATECCFIX_HIGH 0x144
  33 #define SIFIVE_CCACHE_DATECCFIX_COUNT 0x148
  34
  35 #define SIFIVE_CCACHE_DATECCFAIL_LOW 0x160
  36 #define SIFIVE_CCACHE_DATECCFAIL_HIGH 0x164
  37 #define SIFIVE_CCACHE_DATECCFAIL_COUNT 0x168
  38
  39 #define SIFIVE_CCACHE_CONFIG 0x00
  40 #define SIFIVE_CCACHE_CONFIG_BANK_MASK GENMASK_ULL(7, 0)
  41 #define SIFIVE_CCACHE_CONFIG_WAYS_MASK GENMASK_ULL(15, 8)
  42 #define SIFIVE_CCACHE_CONFIG_SETS_MASK GENMASK_ULL(23, 16)
  43 #define SIFIVE_CCACHE_CONFIG_BLKS_MASK GENMASK_ULL(31, 24)
  44
  45 #define SIFIVE_CCACHE_FLUSH64 0x200
  46 #define SIFIVE_CCACHE_FLUSH32 0x240
  47
  48 #define SIFIVE_CCACHE_WAYENABLE 0x08
  49 #define SIFIVE_CCACHE_ECCINJECTERR 0x40
  50
  51 #define SIFIVE_CCACHE_MAX_ECCINTR 4
  52 #define SIFIVE_CCACHE_LINE_SIZE 64
  53
  54 static void __iomem *ccache_base;
  55 static int g_irq[SIFIVE_CCACHE_MAX_ECCINTR];
  56 static struct riscv_cacheinfo_ops ccache_cache_ops;
  57 static int level;
  58
  59 enum {
  60         DIR_CORR = 0,
  61         DATA_CORR,
  62         DATA_UNCORR,
  63         DIR_UNCORR,
  64 };
  65
  66 enum {
  67         QUIRK_NONSTANDARD_CACHE_OPS     = BIT(0),
  68         QUIRK_BROKEN_DATA_UNCORR        = BIT(1),
  69 };
  70
  71 #ifdef CONFIG_DEBUG_FS
  72 static struct dentry *sifive_test;
  73
  74 static ssize_t ccache_write(struct file *file, const char __user *data,
  75                             size_t count, loff_t *ppos)
  76 {
  77         unsigned int val;
  78
  79         if (kstrtouint_from_user(data, count, 0, &val))
  80                 return -EINVAL;
  81         if ((val < 0xFF) || (val >= 0x10000 && val < 0x100FF))
  82                 writel(val, ccache_base + SIFIVE_CCACHE_ECCINJECTERR);
  83         else
  84                 return -EINVAL;
  85         return count;
  86 }
  87
  88 static const struct file_operations ccache_fops = {
  89         .owner = THIS_MODULE,
  90         .open = simple_open,
  91         .write = ccache_write
  92 };
  93
  94 static void setup_sifive_debug(void)
  95 {
  96         sifive_test = debugfs_create_dir("sifive_ccache_cache", NULL);
  97
  98         debugfs_create_file("sifive_debug_inject_error", 0200,
  99                             sifive_test, NULL, &ccache_fops);
 100 }
 101 #endif
 102
 103 static void ccache_config_read(void)
 104 {
 105         u32 cfg;
 106
 107         cfg = readl(ccache_base + SIFIVE_CCACHE_CONFIG);
 108         pr_info("%llu banks, %llu ways, sets/bank=%llu, bytes/block=%llu\n",
 109                 FIELD_GET(SIFIVE_CCACHE_CONFIG_BANK_MASK, cfg),
 110                 FIELD_GET(SIFIVE_CCACHE_CONFIG_WAYS_MASK, cfg),
 111                 BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_SETS_MASK, cfg)),
 112                 BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_BLKS_MASK, cfg)));
 113
 114         cfg = readl(ccache_base + SIFIVE_CCACHE_WAYENABLE);
 115         pr_info("Index of the largest way enabled: %u\n", cfg);
 116 }
 117
 118 static const struct of_device_id sifive_ccache_ids[] = {
 119         { .compatible = "sifive,fu540-c000-ccache" },
 120         { .compatible = "sifive,fu740-c000-ccache" },
 121         { .compatible = "starfive,jh7100-ccache",
 122           .data = (void *)(QUIRK_NONSTANDARD_CACHE_OPS | QUIRK_BROKEN_DATA_UNCORR) },
 123         { .compatible = "sifive,ccache0" },
 124         { /* end of table */ }
 125 };
 126
 127 static ATOMIC_NOTIFIER_HEAD(ccache_err_chain);
 128
 129 int register_sifive_ccache_error_notifier(struct notifier_block *nb)
 130 {
 131         return atomic_notifier_chain_register(&ccache_err_chain, nb);
 132 }
 133 EXPORT_SYMBOL_GPL(register_sifive_ccache_error_notifier);
 134
 135 int unregister_sifive_ccache_error_notifier(struct notifier_block *nb)
 136 {
 137         return atomic_notifier_chain_unregister(&ccache_err_chain, nb);
 138 }
 139 EXPORT_SYMBOL_GPL(unregister_sifive_ccache_error_notifier);
 140
 141 #ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
 142 static void ccache_flush_range(phys_addr_t start, size_t len)
 143 {
 144         phys_addr_t end = start + len;
 145         phys_addr_t line;
 146
 147         if (!len)
 148                 return;
 149
 150         mb();
 151         for (line = ALIGN_DOWN(start, SIFIVE_CCACHE_LINE_SIZE); line < end;
 152                         line += SIFIVE_CCACHE_LINE_SIZE) {
 153 #ifdef CONFIG_32BIT
 154                 writel(line >> 4, ccache_base + SIFIVE_CCACHE_FLUSH32);
 155 #else
 156                 writeq(line, ccache_base + SIFIVE_CCACHE_FLUSH64);
 157 #endif
 158                 mb();
 159         }
 160 }
 161
 162 static const struct riscv_nonstd_cache_ops ccache_mgmt_ops __initconst = {
 163         .wback = &ccache_flush_range,
 164         .inv = &ccache_flush_range,
 165         .wback_inv = &ccache_flush_range,
 166 };
 167 #endif /* CONFIG_RISCV_NONSTANDARD_CACHE_OPS */
 168
 169 static int ccache_largest_wayenabled(void)
 170 {
 171         return readl(ccache_base + SIFIVE_CCACHE_WAYENABLE) & 0xFF;
 172 }
 173
 174 static ssize_t number_of_ways_enabled_show(struct device *dev,
 175                                            struct device_attribute *attr,
 176                                            char *buf)
 177 {
 178         return sprintf(buf, "%u\n", ccache_largest_wayenabled());
 179 }
 180
 181 static DEVICE_ATTR_RO(number_of_ways_enabled);
 182
 183 static struct attribute *priv_attrs[] = {
 184         &dev_attr_number_of_ways_enabled.attr,
 185         NULL,
 186 };
 187
 188 static const struct attribute_group priv_attr_group = {
 189         .attrs = priv_attrs,
 190 };
 191
 192 static const struct attribute_group *ccache_get_priv_group(struct cacheinfo
 193                                                            *this_leaf)
 194 {
 195         /* We want to use private group for composable cache only */
 196         if (this_leaf->level == level)
 197                 return &priv_attr_group;
 198         else
 199                 return NULL;
 200 }
 201
 202 static irqreturn_t ccache_int_handler(int irq, void *device)
 203 {
 204         unsigned int add_h, add_l;
 205
 206         if (irq == g_irq[DIR_CORR]) {
 207                 add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_HIGH);
 208                 add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_LOW);
 209                 pr_err("DirError @ 0x%08X.%08X\n", add_h, add_l);
 210                 /* Reading this register clears the DirError interrupt sig */
 211                 readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_COUNT);
 212                 atomic_notifier_call_chain(&ccache_err_chain,
 213                                            SIFIVE_CCACHE_ERR_TYPE_CE,
 214                                            "DirECCFix");
 215         }
 216         if (irq == g_irq[DIR_UNCORR]) {
 217                 add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_HIGH);
 218                 add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_LOW);
 219                 /* Reading this register clears the DirFail interrupt sig */
 220                 readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_COUNT);
 221                 atomic_notifier_call_chain(&ccache_err_chain,
 222                                            SIFIVE_CCACHE_ERR_TYPE_UE,
 223                                            "DirECCFail");
 224                 panic("CCACHE: DirFail @ 0x%08X.%08X\n", add_h, add_l);
 225         }
 226         if (irq == g_irq[DATA_CORR]) {
 227                 add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_HIGH);
 228                 add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_LOW);
 229                 pr_err("DataError @ 0x%08X.%08X\n", add_h, add_l);
 230                 /* Reading this register clears the DataError interrupt sig */
 231                 readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_COUNT);
 232                 atomic_notifier_call_chain(&ccache_err_chain,
 233                                            SIFIVE_CCACHE_ERR_TYPE_CE,
 234                                            "DatECCFix");
 235         }
 236         if (irq == g_irq[DATA_UNCORR]) {
 237                 add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_HIGH);
 238                 add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_LOW);
 239                 pr_err("DataFail @ 0x%08X.%08X\n", add_h, add_l);
 240                 /* Reading this register clears the DataFail interrupt sig */
 241                 readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_COUNT);
 242                 atomic_notifier_call_chain(&ccache_err_chain,
 243                                            SIFIVE_CCACHE_ERR_TYPE_UE,
 244                                            "DatECCFail");
 245         }
 246
 247         return IRQ_HANDLED;
 248 }
 249
 250 static int __init sifive_ccache_init(void)
 251 {
 252         struct device_node *np;
 253         struct resource res;
 254         int i, rc, intr_num;
 255         const struct of_device_id *match;
 256         unsigned long quirks;
 257
 258         np = of_find_matching_node_and_match(NULL, sifive_ccache_ids, &match);
 259         if (!np)
 260                 return -ENODEV;
 261
 262         quirks = (uintptr_t)match->data;
 263
 264         if (of_address_to_resource(np, 0, &res)) {
 265                 rc = -ENODEV;
 266                 goto err_node_put;
 267         }
 268
 269         ccache_base = ioremap(res.start, resource_size(&res));
 270         if (!ccache_base) {
 271                 rc = -ENOMEM;
 272                 goto err_node_put;
 273         }
 274
 275         if (of_property_read_u32(np, "cache-level", &level)) {
 276                 rc = -ENOENT;
 277                 goto err_unmap;
 278         }
 279
 280         intr_num = of_property_count_u32_elems(np, "interrupts");
 281         if (!intr_num) {
 282                 pr_err("No interrupts property\n");
 283                 rc = -ENODEV;
 284                 goto err_unmap;
 285         }
 286
 287         for (i = 0; i < intr_num; i++) {
 288                 g_irq[i] = irq_of_parse_and_map(np, i);
 289
 290                 if (i == DATA_UNCORR && (quirks & QUIRK_BROKEN_DATA_UNCORR))
 291                         continue;
 292
 293                 rc = request_irq(g_irq[i], ccache_int_handler, 0, "ccache_ecc",
 294                                  NULL);
 295                 if (rc) {
 296                         pr_err("Could not request IRQ %d\n", g_irq[i]);
 297                         goto err_free_irq;
 298                 }
 299         }
 300         of_node_put(np);
 301
 302 #ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
 303         if (quirks & QUIRK_NONSTANDARD_CACHE_OPS) {
 304                 riscv_cbom_block_size = SIFIVE_CCACHE_LINE_SIZE;
 305                 riscv_noncoherent_supported();
 306                 riscv_noncoherent_register_cache_ops(&ccache_mgmt_ops);
 307         }
 308 #endif
 309
 310         ccache_config_read();
 311
 312         ccache_cache_ops.get_priv_group = ccache_get_priv_group;
 313         riscv_set_cacheinfo_ops(&ccache_cache_ops);
 314
 315 #ifdef CONFIG_DEBUG_FS
 316         setup_sifive_debug();
 317 #endif
 318         return 0;
 319
 320 err_free_irq:
 321         while (--i >= 0)
 322                 free_irq(g_irq[i], NULL);
 323 err_unmap:
 324         iounmap(ccache_base);
 325 err_node_put:
 326         of_node_put(np);
 327         return rc;
 328 }
 329
 330 arch_initcall(sifive_ccache_init);