arch/loongarch/kernel/module.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Author: Hanlu Li <lihanlu@loongson.cn>
   4  *         Huacai Chen <chenhuacai@loongson.cn>
   5  *
   6  * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
   7  */
   8
   9 #define pr_fmt(fmt) "kmod: " fmt
  10
  11 #include <linux/moduleloader.h>
  12 #include <linux/elf.h>
  13 #include <linux/mm.h>
  14 #include <linux/numa.h>
  15 #include <linux/vmalloc.h>
  16 #include <linux/slab.h>
  17 #include <linux/fs.h>
  18 #include <linux/string.h>
  19 #include <linux/kernel.h>
  20
  21 static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top)
  22 {
  23         if (*rela_stack_top >= RELA_STACK_DEPTH)
  24                 return -ENOEXEC;
  25
  26         rela_stack[(*rela_stack_top)++] = stack_value;
  27         pr_debug("%s stack_value = 0x%llx\n", __func__, stack_value);
  28
  29         return 0;
  30 }
  31
  32 static int rela_stack_pop(s64 *stack_value, s64 *rela_stack, size_t *rela_stack_top)
  33 {
  34         if (*rela_stack_top == 0)
  35                 return -ENOEXEC;
  36
  37         *stack_value = rela_stack[--(*rela_stack_top)];
  38         pr_debug("%s stack_value = 0x%llx\n", __func__, *stack_value);
  39
  40         return 0;
  41 }
  42
  43 static int apply_r_larch_none(struct module *mod, u32 *location, Elf_Addr v,
  44                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
  45 {
  46         return 0;
  47 }
  48
  49 static int apply_r_larch_error(struct module *me, u32 *location, Elf_Addr v,
  50                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
  51 {
  52         pr_err("%s: Unsupport relocation type %u, please add its support.\n", me->name, type);
  53         return -EINVAL;
  54 }
  55
  56 static int apply_r_larch_32(struct module *mod, u32 *location, Elf_Addr v,
  57                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
  58 {
  59         *location = v;
  60         return 0;
  61 }
  62
  63 static int apply_r_larch_64(struct module *mod, u32 *location, Elf_Addr v,
  64                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
  65 {
  66         *(Elf_Addr *)location = v;
  67         return 0;
  68 }
  69
  70 static int apply_r_larch_sop_push_pcrel(struct module *mod, u32 *location, Elf_Addr v,
  71                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
  72 {
  73         return rela_stack_push(v - (u64)location, rela_stack, rela_stack_top);
  74 }
  75
  76 static int apply_r_larch_sop_push_absolute(struct module *mod, u32 *location, Elf_Addr v,
  77                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
  78 {
  79         return rela_stack_push(v, rela_stack, rela_stack_top);
  80 }
  81
  82 static int apply_r_larch_sop_push_dup(struct module *mod, u32 *location, Elf_Addr v,
  83                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
  84 {
  85         int err = 0;
  86         s64 opr1;
  87
  88         err = rela_stack_pop(&opr1, rela_stack, rela_stack_top);
  89         if (err)
  90                 return err;
  91         err = rela_stack_push(opr1, rela_stack, rela_stack_top);
  92         if (err)
  93                 return err;
  94         err = rela_stack_push(opr1, rela_stack, rela_stack_top);
  95         if (err)
  96                 return err;
  97
  98         return 0;
  99 }
 100
 101 static int apply_r_larch_sop_push_plt_pcrel(struct module *mod, u32 *location, Elf_Addr v,
 102                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
 103 {
 104         ptrdiff_t offset = (void *)v - (void *)location;
 105
 106         if (offset >= SZ_128M)
 107                 v = module_emit_plt_entry(mod, v);
 108
 109         if (offset < -SZ_128M)
 110                 v = module_emit_plt_entry(mod, v);
 111
 112         return apply_r_larch_sop_push_pcrel(mod, location, v, rela_stack, rela_stack_top, type);
 113 }
 114
 115 static int apply_r_larch_sop(struct module *mod, u32 *location, Elf_Addr v,
 116                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
 117 {
 118         int err = 0;
 119         s64 opr1, opr2, opr3;
 120
 121         if (type == R_LARCH_SOP_IF_ELSE) {
 122                 err = rela_stack_pop(&opr3, rela_stack, rela_stack_top);
 123                 if (err)
 124                         return err;
 125         }
 126
 127         err = rela_stack_pop(&opr2, rela_stack, rela_stack_top);
 128         if (err)
 129                 return err;
 130         err = rela_stack_pop(&opr1, rela_stack, rela_stack_top);
 131         if (err)
 132                 return err;
 133
 134         switch (type) {
 135         case R_LARCH_SOP_AND:
 136                 err = rela_stack_push(opr1 & opr2, rela_stack, rela_stack_top);
 137                 break;
 138         case R_LARCH_SOP_ADD:
 139                 err = rela_stack_push(opr1 + opr2, rela_stack, rela_stack_top);
 140                 break;
 141         case R_LARCH_SOP_SUB:
 142                 err = rela_stack_push(opr1 - opr2, rela_stack, rela_stack_top);
 143                 break;
 144         case R_LARCH_SOP_SL:
 145                 err = rela_stack_push(opr1 << opr2, rela_stack, rela_stack_top);
 146                 break;
 147         case R_LARCH_SOP_SR:
 148                 err = rela_stack_push(opr1 >> opr2, rela_stack, rela_stack_top);
 149                 break;
 150         case R_LARCH_SOP_IF_ELSE:
 151                 err = rela_stack_push(opr1 ? opr2 : opr3, rela_stack, rela_stack_top);
 152                 break;
 153         default:
 154                 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
 155                 return -EINVAL;
 156         }
 157
 158         return err;
 159 }
 160
 161 static int apply_r_larch_sop_imm_field(struct module *mod, u32 *location, Elf_Addr v,
 162                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
 163 {
 164         int err = 0;
 165         s64 opr1;
 166         union loongarch_instruction *insn = (union loongarch_instruction *)location;
 167
 168         err = rela_stack_pop(&opr1, rela_stack, rela_stack_top);
 169         if (err)
 170                 return err;
 171
 172         switch (type) {
 173         case R_LARCH_SOP_POP_32_U_10_12:
 174                 if (!unsigned_imm_check(opr1, 12))
 175                         goto overflow;
 176
 177                 /* (*(uint32_t *) PC) [21 ... 10] = opr [11 ... 0] */
 178                 insn->reg2i12_format.immediate = opr1 & 0xfff;
 179                 return 0;
 180         case R_LARCH_SOP_POP_32_S_10_12:
 181                 if (!signed_imm_check(opr1, 12))
 182                         goto overflow;
 183
 184                 insn->reg2i12_format.immediate = opr1 & 0xfff;
 185                 return 0;
 186         case R_LARCH_SOP_POP_32_S_10_16:
 187                 if (!signed_imm_check(opr1, 16))
 188                         goto overflow;
 189
 190                 insn->reg2i16_format.immediate = opr1 & 0xffff;
 191                 return 0;
 192         case R_LARCH_SOP_POP_32_S_10_16_S2:
 193                 if (opr1 % 4)
 194                         goto unaligned;
 195
 196                 if (!signed_imm_check(opr1, 18))
 197                         goto overflow;
 198
 199                 insn->reg2i16_format.immediate = (opr1 >> 2) & 0xffff;
 200                 return 0;
 201         case R_LARCH_SOP_POP_32_S_5_20:
 202                 if (!signed_imm_check(opr1, 20))
 203                         goto overflow;
 204
 205                 insn->reg1i20_format.immediate = (opr1) & 0xfffff;
 206                 return 0;
 207         case R_LARCH_SOP_POP_32_S_0_5_10_16_S2:
 208                 if (opr1 % 4)
 209                         goto unaligned;
 210
 211                 if (!signed_imm_check(opr1, 23))
 212                         goto overflow;
 213
 214                 opr1 >>= 2;
 215                 insn->reg1i21_format.immediate_l = opr1 & 0xffff;
 216                 insn->reg1i21_format.immediate_h = (opr1 >> 16) & 0x1f;
 217                 return 0;
 218         case R_LARCH_SOP_POP_32_S_0_10_10_16_S2:
 219                 if (opr1 % 4)
 220                         goto unaligned;
 221
 222                 if (!signed_imm_check(opr1, 28))
 223                         goto overflow;
 224
 225                 opr1 >>= 2;
 226                 insn->reg0i26_format.immediate_l = opr1 & 0xffff;
 227                 insn->reg0i26_format.immediate_h = (opr1 >> 16) & 0x3ff;
 228                 return 0;
 229         case R_LARCH_SOP_POP_32_U:
 230                 if (!unsigned_imm_check(opr1, 32))
 231                         goto overflow;
 232
 233                 /* (*(uint32_t *) PC) = opr */
 234                 *location = (u32)opr1;
 235                 return 0;
 236         default:
 237                 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
 238                 return -EINVAL;
 239         }
 240
 241 overflow:
 242         pr_err("module %s: opr1 = 0x%llx overflow! dangerous %s (%u) relocation\n",
 243                 mod->name, opr1, __func__, type);
 244         return -ENOEXEC;
 245
 246 unaligned:
 247         pr_err("module %s: opr1 = 0x%llx unaligned! dangerous %s (%u) relocation\n",
 248                 mod->name, opr1, __func__, type);
 249         return -ENOEXEC;
 250 }
 251
 252 static int apply_r_larch_add_sub(struct module *mod, u32 *location, Elf_Addr v,
 253                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
 254 {
 255         switch (type) {
 256         case R_LARCH_ADD32:
 257                 *(s32 *)location += v;
 258                 return 0;
 259         case R_LARCH_ADD64:
 260                 *(s64 *)location += v;
 261                 return 0;
 262         case R_LARCH_SUB32:
 263                 *(s32 *)location -= v;
 264                 return 0;
 265         case R_LARCH_SUB64:
 266                 *(s64 *)location -= v;
 267                 return 0;
 268         default:
 269                 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
 270                 return -EINVAL;
 271         }
 272 }
 273
 274 static int apply_r_larch_b26(struct module *mod, u32 *location, Elf_Addr v,
 275                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
 276 {
 277         ptrdiff_t offset = (void *)v - (void *)location;
 278         union loongarch_instruction *insn = (union loongarch_instruction *)location;
 279
 280         if (offset >= SZ_128M)
 281                 v = module_emit_plt_entry(mod, v);
 282
 283         if (offset < -SZ_128M)
 284                 v = module_emit_plt_entry(mod, v);
 285
 286         offset = (void *)v - (void *)location;
 287
 288         if (offset & 3) {
 289                 pr_err("module %s: jump offset = 0x%llx unaligned! dangerous R_LARCH_B26 (%u) relocation\n",
 290                                 mod->name, (long long)offset, type);
 291                 return -ENOEXEC;
 292         }
 293
 294         if (!signed_imm_check(offset, 28)) {
 295                 pr_err("module %s: jump offset = 0x%llx overflow! dangerous R_LARCH_B26 (%u) relocation\n",
 296                                 mod->name, (long long)offset, type);
 297                 return -ENOEXEC;
 298         }
 299
 300         offset >>= 2;
 301         insn->reg0i26_format.immediate_l = offset & 0xffff;
 302         insn->reg0i26_format.immediate_h = (offset >> 16) & 0x3ff;
 303
 304         return 0;
 305 }
 306
 307 static int apply_r_larch_pcala(struct module *mod, u32 *location, Elf_Addr v,
 308                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
 309 {
 310         union loongarch_instruction *insn = (union loongarch_instruction *)location;
 311         /* Use s32 for a sign-extension deliberately. */
 312         s32 offset_hi20 = (void *)((v + 0x800) & ~0xfff) -
 313                           (void *)((Elf_Addr)location & ~0xfff);
 314         Elf_Addr anchor = (((Elf_Addr)location) & ~0xfff) + offset_hi20;
 315         ptrdiff_t offset_rem = (void *)v - (void *)anchor;
 316
 317         switch (type) {
 318         case R_LARCH_PCALA_LO12:
 319                 insn->reg2i12_format.immediate = v & 0xfff;
 320                 break;
 321         case R_LARCH_PCALA_HI20:
 322                 v = offset_hi20 >> 12;
 323                 insn->reg1i20_format.immediate = v & 0xfffff;
 324                 break;
 325         case R_LARCH_PCALA64_LO20:
 326                 v = offset_rem >> 32;
 327                 insn->reg1i20_format.immediate = v & 0xfffff;
 328                 break;
 329         case R_LARCH_PCALA64_HI12:
 330                 v = offset_rem >> 52;
 331                 insn->reg2i12_format.immediate = v & 0xfff;
 332                 break;
 333         default:
 334                 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
 335                 return -EINVAL;
 336         }
 337
 338         return 0;
 339 }
 340
 341 static int apply_r_larch_got_pc(struct module *mod, u32 *location, Elf_Addr v,
 342                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
 343 {
 344         Elf_Addr got = module_emit_got_entry(mod, v);
 345
 346         if (!got)
 347                 return -EINVAL;
 348
 349         switch (type) {
 350         case R_LARCH_GOT_PC_LO12:
 351                 type = R_LARCH_PCALA_LO12;
 352                 break;
 353         case R_LARCH_GOT_PC_HI20:
 354                 type = R_LARCH_PCALA_HI20;
 355                 break;
 356         default:
 357                 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
 358                 return -EINVAL;
 359         }
 360
 361         return apply_r_larch_pcala(mod, location, got, rela_stack, rela_stack_top, type);
 362 }
 363
 364 /*
 365  * reloc_handlers_rela() - Apply a particular relocation to a module
 366  * @mod: the module to apply the reloc to
 367  * @location: the address at which the reloc is to be applied
 368  * @v: the value of the reloc, with addend for RELA-style
 369  * @rela_stack: the stack used for store relocation info, LOCAL to THIS module
 370  * @rela_stac_top: where the stack operation(pop/push) applies to
 371  *
 372  * Return: 0 upon success, else -ERRNO
 373  */
 374 typedef int (*reloc_rela_handler)(struct module *mod, u32 *location, Elf_Addr v,
 375                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type);
 376
 377 /* The handlers for known reloc types */
 378 static reloc_rela_handler reloc_rela_handlers[] = {
 379         [R_LARCH_NONE ... R_LARCH_64_PCREL]                  = apply_r_larch_error,
 380
 381         [R_LARCH_NONE]                                       = apply_r_larch_none,
 382         [R_LARCH_32]                                         = apply_r_larch_32,
 383         [R_LARCH_64]                                         = apply_r_larch_64,
 384         [R_LARCH_MARK_LA]                                    = apply_r_larch_none,
 385         [R_LARCH_MARK_PCREL]                                 = apply_r_larch_none,
 386         [R_LARCH_SOP_PUSH_PCREL]                             = apply_r_larch_sop_push_pcrel,
 387         [R_LARCH_SOP_PUSH_ABSOLUTE]                          = apply_r_larch_sop_push_absolute,
 388         [R_LARCH_SOP_PUSH_DUP]                               = apply_r_larch_sop_push_dup,
 389         [R_LARCH_SOP_PUSH_PLT_PCREL]                         = apply_r_larch_sop_push_plt_pcrel,
 390         [R_LARCH_SOP_SUB ... R_LARCH_SOP_IF_ELSE]            = apply_r_larch_sop,
 391         [R_LARCH_SOP_POP_32_S_10_5 ... R_LARCH_SOP_POP_32_U] = apply_r_larch_sop_imm_field,
 392         [R_LARCH_ADD32 ... R_LARCH_SUB64]                    = apply_r_larch_add_sub,
 393         [R_LARCH_B26]                                        = apply_r_larch_b26,
 394         [R_LARCH_PCALA_HI20...R_LARCH_PCALA64_HI12]          = apply_r_larch_pcala,
 395         [R_LARCH_GOT_PC_HI20...R_LARCH_GOT_PC_LO12]          = apply_r_larch_got_pc,
 396 };
 397
 398 int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
 399                        unsigned int symindex, unsigned int relsec,
 400                        struct module *mod)
 401 {
 402         int i, err;
 403         unsigned int type;
 404         s64 rela_stack[RELA_STACK_DEPTH];
 405         size_t rela_stack_top = 0;
 406         reloc_rela_handler handler;
 407         void *location;
 408         Elf_Addr v;
 409         Elf_Sym *sym;
 410         Elf_Rela *rel = (void *) sechdrs[relsec].sh_addr;
 411
 412         pr_debug("%s: Applying relocate section %u to %u\n", __func__, relsec,
 413                sechdrs[relsec].sh_info);
 414
 415         rela_stack_top = 0;
 416         for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) {
 417                 /* This is where to make the change */
 418                 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr + rel[i].r_offset;
 419                 /* This is the symbol it is referring to */
 420                 sym = (Elf_Sym *)sechdrs[symindex].sh_addr + ELF_R_SYM(rel[i].r_info);
 421                 if (IS_ERR_VALUE(sym->st_value)) {
 422                         /* Ignore unresolved weak symbol */
 423                         if (ELF_ST_BIND(sym->st_info) == STB_WEAK)
 424                                 continue;
 425                         pr_warn("%s: Unknown symbol %s\n", mod->name, strtab + sym->st_name);
 426                         return -ENOENT;
 427                 }
 428
 429                 type = ELF_R_TYPE(rel[i].r_info);
 430
 431                 if (type < ARRAY_SIZE(reloc_rela_handlers))
 432                         handler = reloc_rela_handlers[type];
 433                 else
 434                         handler = NULL;
 435
 436                 if (!handler) {
 437                         pr_err("%s: Unknown relocation type %u\n", mod->name, type);
 438                         return -EINVAL;
 439                 }
 440
 441                 pr_debug("type %d st_value %llx r_addend %llx loc %llx\n",
 442                        (int)ELF_R_TYPE(rel[i].r_info),
 443                        sym->st_value, rel[i].r_addend, (u64)location);
 444
 445                 v = sym->st_value + rel[i].r_addend;
 446                 err = handler(mod, location, v, rela_stack, &rela_stack_top, type);
 447                 if (err)
 448                         return err;
 449         }
 450
 451         return 0;
 452 }
 453
 454 void *module_alloc(unsigned long size)
 455 {
 456         return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END,
 457                         GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0));
 458 }