riscv/kernel/cpufeature.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copied from arch/arm64/kernel/cpufeature.c
   4  *
   5  * Copyright (C) 2015 ARM Ltd.
   6  * Copyright (C) 2017 SiFive
   7  */
   8
   9 #include <linux/bitmap.h>
  10 #include <linux/ctype.h>
  11 #include <linux/libfdt.h>
  12 #include <linux/memory.h>
  13 #include <linux/module.h>
  14 #include <linux/of.h>
  15 #include <asm/alternative.h>
  16 #include <asm/cacheflush.h>
  17 #include <asm/errata_list.h>
  18 #include <asm/hwcap.h>
  19 #include <asm/patch.h>
  20 #include <asm/pgtable.h>
  21 #include <asm/processor.h>
  22 #include <asm/smp.h>
  23 #include <asm/switch_to.h>
  24
  25 #define NUM_ALPHA_EXTS ('z' - 'a' + 1)
  26
  27 unsigned long elf_hwcap __read_mostly;
  28
  29 /* Host ISA bitmap */
  30 static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly;
  31
  32 DEFINE_STATIC_KEY_ARRAY_FALSE(riscv_isa_ext_keys, RISCV_ISA_EXT_KEY_MAX);
  33 EXPORT_SYMBOL(riscv_isa_ext_keys);
  34
  35 /**
  36  * riscv_isa_extension_base() - Get base extension word
  37  *
  38  * @isa_bitmap: ISA bitmap to use
  39  * Return: base extension word as unsigned long value
  40  *
  41  * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used.
  42  */
  43 unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap)
  44 {
  45         if (!isa_bitmap)
  46                 return riscv_isa[0];
  47         return isa_bitmap[0];
  48 }
  49 EXPORT_SYMBOL_GPL(riscv_isa_extension_base);
  50
  51 /**
  52  * __riscv_isa_extension_available() - Check whether given extension
  53  * is available or not
  54  *
  55  * @isa_bitmap: ISA bitmap to use
  56  * @bit: bit position of the desired extension
  57  * Return: true or false
  58  *
  59  * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used.
  60  */
  61 bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, int bit)
  62 {
  63         const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa;
  64
  65         if (bit >= RISCV_ISA_EXT_MAX)
  66                 return false;
  67
  68         return test_bit(bit, bmap) ? true : false;
  69 }
  70 EXPORT_SYMBOL_GPL(__riscv_isa_extension_available);
  71
  72 void __init riscv_fill_hwcap(void)
  73 {
  74         struct device_node *node;
  75         const char *isa;
  76         char print_str[NUM_ALPHA_EXTS + 1];
  77         int i, j, rc;
  78         static unsigned long isa2hwcap[256] = {0};
  79         unsigned long hartid;
  80
  81         isa2hwcap['i'] = isa2hwcap['I'] = COMPAT_HWCAP_ISA_I;
  82         isa2hwcap['m'] = isa2hwcap['M'] = COMPAT_HWCAP_ISA_M;
  83         isa2hwcap['a'] = isa2hwcap['A'] = COMPAT_HWCAP_ISA_A;
  84         isa2hwcap['f'] = isa2hwcap['F'] = COMPAT_HWCAP_ISA_F;
  85         isa2hwcap['d'] = isa2hwcap['D'] = COMPAT_HWCAP_ISA_D;
  86         isa2hwcap['c'] = isa2hwcap['C'] = COMPAT_HWCAP_ISA_C;
  87
  88         elf_hwcap = 0;
  89
  90         bitmap_zero(riscv_isa, RISCV_ISA_EXT_MAX);
  91
  92         for_each_of_cpu_node(node) {
  93                 unsigned long this_hwcap = 0;
  94                 DECLARE_BITMAP(this_isa, RISCV_ISA_EXT_MAX);
  95                 const char *temp;
  96
  97                 rc = riscv_of_processor_hartid(node, &hartid);
  98                 if (rc < 0)
  99                         continue;
 100
 101                 if (of_property_read_string(node, "riscv,isa", &isa)) {
 102                         pr_warn("Unable to find \"riscv,isa\" devicetree entry\n");
 103                         continue;
 104                 }
 105
 106                 temp = isa;
 107 #if IS_ENABLED(CONFIG_32BIT)
 108                 if (!strncmp(isa, "rv32", 4))
 109                         isa += 4;
 110 #elif IS_ENABLED(CONFIG_64BIT)
 111                 if (!strncmp(isa, "rv64", 4))
 112                         isa += 4;
 113 #endif
 114                 /* The riscv,isa DT property must start with rv64 or rv32 */
 115                 if (temp == isa)
 116                         continue;
 117                 bitmap_zero(this_isa, RISCV_ISA_EXT_MAX);
 118                 for (; *isa; ++isa) {
 119                         const char *ext = isa++;
 120                         const char *ext_end = isa;
 121                         bool ext_long = false, ext_err = false;
 122
 123                         switch (*ext) {
 124                         case 's':
 125                                 /**
 126                                  * Workaround for invalid single-letter 's' & 'u'(QEMU).
 127                                  * No need to set the bit in riscv_isa as 's' & 'u' are
 128                                  * not valid ISA extensions. It works until multi-letter
 129                                  * extension starting with "Su" appears.
 130                                  */
 131                                 if (ext[-1] != '_' && ext[1] == 'u') {
 132                                         ++isa;
 133                                         ext_err = true;
 134                                         break;
 135                                 }
 136                                 fallthrough;
 137                         case 'x':
 138                         case 'z':
 139                                 ext_long = true;
 140                                 /* Multi-letter extension must be delimited */
 141                                 for (; *isa && *isa != '_'; ++isa)
 142                                         if (unlikely(!islower(*isa)
 143                                                      && !isdigit(*isa)))
 144                                                 ext_err = true;
 145                                 /* Parse backwards */
 146                                 ext_end = isa;
 147                                 if (unlikely(ext_err))
 148                                         break;
 149                                 if (!isdigit(ext_end[-1]))
 150                                         break;
 151                                 /* Skip the minor version */
 152                                 while (isdigit(*--ext_end))
 153                                         ;
 154                                 if (ext_end[0] != 'p'
 155                                     || !isdigit(ext_end[-1])) {
 156                                         /* Advance it to offset the pre-decrement */
 157                                         ++ext_end;
 158                                         break;
 159                                 }
 160                                 /* Skip the major version */
 161                                 while (isdigit(*--ext_end))
 162                                         ;
 163                                 ++ext_end;
 164                                 break;
 165                         default:
 166                                 if (unlikely(!islower(*ext))) {
 167                                         ext_err = true;
 168                                         break;
 169                                 }
 170                                 /* Find next extension */
 171                                 if (!isdigit(*isa))
 172                                         break;
 173                                 /* Skip the minor version */
 174                                 while (isdigit(*++isa))
 175                                         ;
 176                                 if (*isa != 'p')
 177                                         break;
 178                                 if (!isdigit(*++isa)) {
 179                                         --isa;
 180                                         break;
 181                                 }
 182                                 /* Skip the major version */
 183                                 while (isdigit(*++isa))
 184                                         ;
 185                                 break;
 186                         }
 187                         if (*isa != '_')
 188                                 --isa;
 189
 190 #define SET_ISA_EXT_MAP(name, bit)                                              \
 191                         do {                                                    \
 192                                 if ((ext_end - ext == sizeof(name) - 1) &&      \
 193                                      !memcmp(ext, name, sizeof(name) - 1))      \
 194                                         set_bit(bit, this_isa);                 \
 195                         } while (false)                                         \
 196
 197                         if (unlikely(ext_err))
 198                                 continue;
 199                         if (!ext_long) {
 200                                 this_hwcap |= isa2hwcap[(unsigned char)(*ext)];
 201                                 set_bit(*ext - 'a', this_isa);
 202                         } else {
 203                                 SET_ISA_EXT_MAP("sscofpmf", RISCV_ISA_EXT_SSCOFPMF);
 204                                 SET_ISA_EXT_MAP("svpbmt", RISCV_ISA_EXT_SVPBMT);
 205                                 SET_ISA_EXT_MAP("zicbom", RISCV_ISA_EXT_ZICBOM);
 206                                 SET_ISA_EXT_MAP("zihintpause", RISCV_ISA_EXT_ZIHINTPAUSE);
 207                                 SET_ISA_EXT_MAP("sstc", RISCV_ISA_EXT_SSTC);
 208                                 SET_ISA_EXT_MAP("svinval", RISCV_ISA_EXT_SVINVAL);
 209                         }
 210 #undef SET_ISA_EXT_MAP
 211                 }
 212
 213                 /*
 214                  * All "okay" hart should have same isa. Set HWCAP based on
 215                  * common capabilities of every "okay" hart, in case they don't
 216                  * have.
 217                  */
 218                 if (elf_hwcap)
 219                         elf_hwcap &= this_hwcap;
 220                 else
 221                         elf_hwcap = this_hwcap;
 222
 223                 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX))
 224                         bitmap_copy(riscv_isa, this_isa, RISCV_ISA_EXT_MAX);
 225                 else
 226                         bitmap_and(riscv_isa, riscv_isa, this_isa, RISCV_ISA_EXT_MAX);
 227         }
 228
 229         /* We don't support systems with F but without D, so mask those out
 230          * here. */
 231         if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) {
 232                 pr_info("This kernel does not support systems with F but not D\n");
 233                 elf_hwcap &= ~COMPAT_HWCAP_ISA_F;
 234         }
 235
 236         memset(print_str, 0, sizeof(print_str));
 237         for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++)
 238                 if (riscv_isa[0] & BIT_MASK(i))
 239                         print_str[j++] = (char)('a' + i);
 240         pr_info("riscv: base ISA extensions %s\n", print_str);
 241
 242         memset(print_str, 0, sizeof(print_str));
 243         for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++)
 244                 if (elf_hwcap & BIT_MASK(i))
 245                         print_str[j++] = (char)('a' + i);
 246         pr_info("riscv: ELF capabilities %s\n", print_str);
 247
 248         for_each_set_bit(i, riscv_isa, RISCV_ISA_EXT_MAX) {
 249                 j = riscv_isa_ext2key(i);
 250                 if (j >= 0)
 251                         static_branch_enable(&riscv_isa_ext_keys[j]);
 252         }
 253 }
 254
 255 #ifdef CONFIG_RISCV_ALTERNATIVE
 256 static bool __init_or_module cpufeature_probe_svpbmt(unsigned int stage)
 257 {
 258         if (!IS_ENABLED(CONFIG_RISCV_ISA_SVPBMT))
 259                 return false;
 260
 261         if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
 262                 return false;
 263
 264         return riscv_isa_extension_available(NULL, SVPBMT);
 265 }
 266
 267 static bool __init_or_module cpufeature_probe_zicbom(unsigned int stage)
 268 {
 269         if (!IS_ENABLED(CONFIG_RISCV_ISA_ZICBOM))
 270                 return false;
 271
 272         if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
 273                 return false;
 274
 275         if (!riscv_isa_extension_available(NULL, ZICBOM))
 276                 return false;
 277
 278         riscv_noncoherent_supported();
 279         return true;
 280 }
 281
 282 /*
 283  * Probe presence of individual extensions.
 284  *
 285  * This code may also be executed before kernel relocation, so we cannot use
 286  * addresses generated by the address-of operator as they won't be valid in
 287  * this context.
 288  */
 289 static u32 __init_or_module cpufeature_probe(unsigned int stage)
 290 {
 291         u32 cpu_req_feature = 0;
 292
 293         if (cpufeature_probe_svpbmt(stage))
 294                 cpu_req_feature |= BIT(CPUFEATURE_SVPBMT);
 295
 296         if (cpufeature_probe_zicbom(stage))
 297                 cpu_req_feature |= BIT(CPUFEATURE_ZICBOM);
 298
 299         return cpu_req_feature;
 300 }
 301
 302 void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin,
 303                                                   struct alt_entry *end,
 304                                                   unsigned int stage)
 305 {
 306         u32 cpu_req_feature = cpufeature_probe(stage);
 307         struct alt_entry *alt;
 308         u32 tmp;
 309
 310         for (alt = begin; alt < end; alt++) {
 311                 if (alt->vendor_id != 0)
 312                         continue;
 313                 if (alt->errata_id >= CPUFEATURE_NUMBER) {
 314                         WARN(1, "This feature id:%d is not in kernel cpufeature list",
 315                                 alt->errata_id);
 316                         continue;
 317                 }
 318
 319                 tmp = (1U << alt->errata_id);
 320                 if (cpu_req_feature & tmp) {
 321                         mutex_lock(&text_mutex);
 322                         patch_text_nosync(alt->old_ptr, alt->alt_ptr, alt->alt_len);
 323                         mutex_unlock(&text_mutex);
 324                 }
 325         }
 326 }
 327 #endif