arch/loongarch/kernel/module.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Author: Hanlu Li <lihanlu@loongson.cn>
   4  *         Huacai Chen <chenhuacai@loongson.cn>
   5  *
   6  * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
   7  */
   8
   9 #define pr_fmt(fmt) "kmod: " fmt
  10
  11 #include <linux/moduleloader.h>
  12 #include <linux/elf.h>
  13 #include <linux/mm.h>
  14 #include <linux/numa.h>
  15 #include <linux/vmalloc.h>
  16 #include <linux/slab.h>
  17 #include <linux/fs.h>
  18 #include <linux/ftrace.h>
  19 #include <linux/string.h>
  20 #include <linux/kernel.h>
  21 #include <asm/alternative.h>
  22 #include <asm/inst.h>
  23
  24 static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top)
  25 {
  26         if (*rela_stack_top >= RELA_STACK_DEPTH)
  27                 return -ENOEXEC;
  28
  29         rela_stack[(*rela_stack_top)++] = stack_value;
  30         pr_debug("%s stack_value = 0x%llx\n", __func__, stack_value);
  31
  32         return 0;
  33 }
  34
  35 static int rela_stack_pop(s64 *stack_value, s64 *rela_stack, size_t *rela_stack_top)
  36 {
  37         if (*rela_stack_top == 0)
  38                 return -ENOEXEC;
  39
  40         *stack_value = rela_stack[--(*rela_stack_top)];
  41         pr_debug("%s stack_value = 0x%llx\n", __func__, *stack_value);
  42
  43         return 0;
  44 }
  45
  46 static int apply_r_larch_none(struct module *mod, u32 *location, Elf_Addr v,
  47                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
  48 {
  49         return 0;
  50 }
  51
  52 static int apply_r_larch_error(struct module *me, u32 *location, Elf_Addr v,
  53                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
  54 {
  55         pr_err("%s: Unsupport relocation type %u, please add its support.\n", me->name, type);
  56         return -EINVAL;
  57 }
  58
  59 static int apply_r_larch_32(struct module *mod, u32 *location, Elf_Addr v,
  60                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
  61 {
  62         *location = v;
  63         return 0;
  64 }
  65
  66 static int apply_r_larch_64(struct module *mod, u32 *location, Elf_Addr v,
  67                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
  68 {
  69         *(Elf_Addr *)location = v;
  70         return 0;
  71 }
  72
  73 static int apply_r_larch_sop_push_pcrel(struct module *mod, u32 *location, Elf_Addr v,
  74                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
  75 {
  76         return rela_stack_push(v - (u64)location, rela_stack, rela_stack_top);
  77 }
  78
  79 static int apply_r_larch_sop_push_absolute(struct module *mod, u32 *location, Elf_Addr v,
  80                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
  81 {
  82         return rela_stack_push(v, rela_stack, rela_stack_top);
  83 }
  84
  85 static int apply_r_larch_sop_push_dup(struct module *mod, u32 *location, Elf_Addr v,
  86                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
  87 {
  88         int err = 0;
  89         s64 opr1;
  90
  91         err = rela_stack_pop(&opr1, rela_stack, rela_stack_top);
  92         if (err)
  93                 return err;
  94         err = rela_stack_push(opr1, rela_stack, rela_stack_top);
  95         if (err)
  96                 return err;
  97         err = rela_stack_push(opr1, rela_stack, rela_stack_top);
  98         if (err)
  99                 return err;
 100
 101         return 0;
 102 }
 103
 104 static int apply_r_larch_sop_push_plt_pcrel(struct module *mod,
 105                         Elf_Shdr *sechdrs, u32 *location, Elf_Addr v,
 106                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
 107 {
 108         ptrdiff_t offset = (void *)v - (void *)location;
 109
 110         if (offset >= SZ_128M)
 111                 v = module_emit_plt_entry(mod, sechdrs, v);
 112
 113         if (offset < -SZ_128M)
 114                 v = module_emit_plt_entry(mod, sechdrs, v);
 115
 116         return apply_r_larch_sop_push_pcrel(mod, location, v, rela_stack, rela_stack_top, type);
 117 }
 118
 119 static int apply_r_larch_sop(struct module *mod, u32 *location, Elf_Addr v,
 120                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
 121 {
 122         int err = 0;
 123         s64 opr1, opr2, opr3;
 124
 125         if (type == R_LARCH_SOP_IF_ELSE) {
 126                 err = rela_stack_pop(&opr3, rela_stack, rela_stack_top);
 127                 if (err)
 128                         return err;
 129         }
 130
 131         err = rela_stack_pop(&opr2, rela_stack, rela_stack_top);
 132         if (err)
 133                 return err;
 134         err = rela_stack_pop(&opr1, rela_stack, rela_stack_top);
 135         if (err)
 136                 return err;
 137
 138         switch (type) {
 139         case R_LARCH_SOP_AND:
 140                 err = rela_stack_push(opr1 & opr2, rela_stack, rela_stack_top);
 141                 break;
 142         case R_LARCH_SOP_ADD:
 143                 err = rela_stack_push(opr1 + opr2, rela_stack, rela_stack_top);
 144                 break;
 145         case R_LARCH_SOP_SUB:
 146                 err = rela_stack_push(opr1 - opr2, rela_stack, rela_stack_top);
 147                 break;
 148         case R_LARCH_SOP_SL:
 149                 err = rela_stack_push(opr1 << opr2, rela_stack, rela_stack_top);
 150                 break;
 151         case R_LARCH_SOP_SR:
 152                 err = rela_stack_push(opr1 >> opr2, rela_stack, rela_stack_top);
 153                 break;
 154         case R_LARCH_SOP_IF_ELSE:
 155                 err = rela_stack_push(opr1 ? opr2 : opr3, rela_stack, rela_stack_top);
 156                 break;
 157         default:
 158                 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
 159                 return -EINVAL;
 160         }
 161
 162         return err;
 163 }
 164
 165 static int apply_r_larch_sop_imm_field(struct module *mod, u32 *location, Elf_Addr v,
 166                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
 167 {
 168         int err = 0;
 169         s64 opr1;
 170         union loongarch_instruction *insn = (union loongarch_instruction *)location;
 171
 172         err = rela_stack_pop(&opr1, rela_stack, rela_stack_top);
 173         if (err)
 174                 return err;
 175
 176         switch (type) {
 177         case R_LARCH_SOP_POP_32_U_10_12:
 178                 if (!unsigned_imm_check(opr1, 12))
 179                         goto overflow;
 180
 181                 /* (*(uint32_t *) PC) [21 ... 10] = opr [11 ... 0] */
 182                 insn->reg2i12_format.immediate = opr1 & 0xfff;
 183                 return 0;
 184         case R_LARCH_SOP_POP_32_S_10_12:
 185                 if (!signed_imm_check(opr1, 12))
 186                         goto overflow;
 187
 188                 insn->reg2i12_format.immediate = opr1 & 0xfff;
 189                 return 0;
 190         case R_LARCH_SOP_POP_32_S_10_16:
 191                 if (!signed_imm_check(opr1, 16))
 192                         goto overflow;
 193
 194                 insn->reg2i16_format.immediate = opr1 & 0xffff;
 195                 return 0;
 196         case R_LARCH_SOP_POP_32_S_10_16_S2:
 197                 if (opr1 % 4)
 198                         goto unaligned;
 199
 200                 if (!signed_imm_check(opr1, 18))
 201                         goto overflow;
 202
 203                 insn->reg2i16_format.immediate = (opr1 >> 2) & 0xffff;
 204                 return 0;
 205         case R_LARCH_SOP_POP_32_S_5_20:
 206                 if (!signed_imm_check(opr1, 20))
 207                         goto overflow;
 208
 209                 insn->reg1i20_format.immediate = (opr1) & 0xfffff;
 210                 return 0;
 211         case R_LARCH_SOP_POP_32_S_0_5_10_16_S2:
 212                 if (opr1 % 4)
 213                         goto unaligned;
 214
 215                 if (!signed_imm_check(opr1, 23))
 216                         goto overflow;
 217
 218                 opr1 >>= 2;
 219                 insn->reg1i21_format.immediate_l = opr1 & 0xffff;
 220                 insn->reg1i21_format.immediate_h = (opr1 >> 16) & 0x1f;
 221                 return 0;
 222         case R_LARCH_SOP_POP_32_S_0_10_10_16_S2:
 223                 if (opr1 % 4)
 224                         goto unaligned;
 225
 226                 if (!signed_imm_check(opr1, 28))
 227                         goto overflow;
 228
 229                 opr1 >>= 2;
 230                 insn->reg0i26_format.immediate_l = opr1 & 0xffff;
 231                 insn->reg0i26_format.immediate_h = (opr1 >> 16) & 0x3ff;
 232                 return 0;
 233         case R_LARCH_SOP_POP_32_U:
 234                 if (!unsigned_imm_check(opr1, 32))
 235                         goto overflow;
 236
 237                 /* (*(uint32_t *) PC) = opr */
 238                 *location = (u32)opr1;
 239                 return 0;
 240         default:
 241                 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
 242                 return -EINVAL;
 243         }
 244
 245 overflow:
 246         pr_err("module %s: opr1 = 0x%llx overflow! dangerous %s (%u) relocation\n",
 247                 mod->name, opr1, __func__, type);
 248         return -ENOEXEC;
 249
 250 unaligned:
 251         pr_err("module %s: opr1 = 0x%llx unaligned! dangerous %s (%u) relocation\n",
 252                 mod->name, opr1, __func__, type);
 253         return -ENOEXEC;
 254 }
 255
 256 static int apply_r_larch_add_sub(struct module *mod, u32 *location, Elf_Addr v,
 257                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
 258 {
 259         switch (type) {
 260         case R_LARCH_ADD32:
 261                 *(s32 *)location += v;
 262                 return 0;
 263         case R_LARCH_ADD64:
 264                 *(s64 *)location += v;
 265                 return 0;
 266         case R_LARCH_SUB32:
 267                 *(s32 *)location -= v;
 268                 return 0;
 269         case R_LARCH_SUB64:
 270                 *(s64 *)location -= v;
 271                 return 0;
 272         default:
 273                 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
 274                 return -EINVAL;
 275         }
 276 }
 277
 278 static int apply_r_larch_b26(struct module *mod,
 279                         Elf_Shdr *sechdrs, u32 *location, Elf_Addr v,
 280                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
 281 {
 282         ptrdiff_t offset = (void *)v - (void *)location;
 283         union loongarch_instruction *insn = (union loongarch_instruction *)location;
 284
 285         if (offset >= SZ_128M)
 286                 v = module_emit_plt_entry(mod, sechdrs, v);
 287
 288         if (offset < -SZ_128M)
 289                 v = module_emit_plt_entry(mod, sechdrs, v);
 290
 291         offset = (void *)v - (void *)location;
 292
 293         if (offset & 3) {
 294                 pr_err("module %s: jump offset = 0x%llx unaligned! dangerous R_LARCH_B26 (%u) relocation\n",
 295                                 mod->name, (long long)offset, type);
 296                 return -ENOEXEC;
 297         }
 298
 299         if (!signed_imm_check(offset, 28)) {
 300                 pr_err("module %s: jump offset = 0x%llx overflow! dangerous R_LARCH_B26 (%u) relocation\n",
 301                                 mod->name, (long long)offset, type);
 302                 return -ENOEXEC;
 303         }
 304
 305         offset >>= 2;
 306         insn->reg0i26_format.immediate_l = offset & 0xffff;
 307         insn->reg0i26_format.immediate_h = (offset >> 16) & 0x3ff;
 308
 309         return 0;
 310 }
 311
 312 static int apply_r_larch_pcala(struct module *mod, u32 *location, Elf_Addr v,
 313                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
 314 {
 315         union loongarch_instruction *insn = (union loongarch_instruction *)location;
 316         /* Use s32 for a sign-extension deliberately. */
 317         s32 offset_hi20 = (void *)((v + 0x800) & ~0xfff) -
 318                           (void *)((Elf_Addr)location & ~0xfff);
 319         Elf_Addr anchor = (((Elf_Addr)location) & ~0xfff) + offset_hi20;
 320         ptrdiff_t offset_rem = (void *)v - (void *)anchor;
 321
 322         switch (type) {
 323         case R_LARCH_PCALA_LO12:
 324                 insn->reg2i12_format.immediate = v & 0xfff;
 325                 break;
 326         case R_LARCH_PCALA_HI20:
 327                 v = offset_hi20 >> 12;
 328                 insn->reg1i20_format.immediate = v & 0xfffff;
 329                 break;
 330         case R_LARCH_PCALA64_LO20:
 331                 v = offset_rem >> 32;
 332                 insn->reg1i20_format.immediate = v & 0xfffff;
 333                 break;
 334         case R_LARCH_PCALA64_HI12:
 335                 v = offset_rem >> 52;
 336                 insn->reg2i12_format.immediate = v & 0xfff;
 337                 break;
 338         default:
 339                 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
 340                 return -EINVAL;
 341         }
 342
 343         return 0;
 344 }
 345
 346 static int apply_r_larch_got_pc(struct module *mod,
 347                         Elf_Shdr *sechdrs, u32 *location, Elf_Addr v,
 348                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
 349 {
 350         Elf_Addr got = module_emit_got_entry(mod, sechdrs, v);
 351
 352         if (!got)
 353                 return -EINVAL;
 354
 355         switch (type) {
 356         case R_LARCH_GOT_PC_LO12:
 357                 type = R_LARCH_PCALA_LO12;
 358                 break;
 359         case R_LARCH_GOT_PC_HI20:
 360                 type = R_LARCH_PCALA_HI20;
 361                 break;
 362         default:
 363                 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
 364                 return -EINVAL;
 365         }
 366
 367         return apply_r_larch_pcala(mod, location, got, rela_stack, rela_stack_top, type);
 368 }
 369
 370 static int apply_r_larch_32_pcrel(struct module *mod, u32 *location, Elf_Addr v,
 371                                   s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
 372 {
 373         ptrdiff_t offset = (void *)v - (void *)location;
 374
 375         *(u32 *)location = offset;
 376         return 0;
 377 }
 378
 379 static int apply_r_larch_64_pcrel(struct module *mod, u32 *location, Elf_Addr v,
 380                                   s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
 381 {
 382         ptrdiff_t offset = (void *)v - (void *)location;
 383
 384         *(u64 *)location = offset;
 385         return 0;
 386 }
 387
 388 /*
 389  * reloc_handlers_rela() - Apply a particular relocation to a module
 390  * @mod: the module to apply the reloc to
 391  * @location: the address at which the reloc is to be applied
 392  * @v: the value of the reloc, with addend for RELA-style
 393  * @rela_stack: the stack used for store relocation info, LOCAL to THIS module
 394  * @rela_stac_top: where the stack operation(pop/push) applies to
 395  *
 396  * Return: 0 upon success, else -ERRNO
 397  */
 398 typedef int (*reloc_rela_handler)(struct module *mod, u32 *location, Elf_Addr v,
 399                         s64 *rela_stack, size_t *rela_stack_top, unsigned int type);
 400
 401 /* The handlers for known reloc types */
 402 static reloc_rela_handler reloc_rela_handlers[] = {
 403         [R_LARCH_NONE ... R_LARCH_64_PCREL]                  = apply_r_larch_error,
 404
 405         [R_LARCH_NONE]                                       = apply_r_larch_none,
 406         [R_LARCH_32]                                         = apply_r_larch_32,
 407         [R_LARCH_64]                                         = apply_r_larch_64,
 408         [R_LARCH_MARK_LA]                                    = apply_r_larch_none,
 409         [R_LARCH_MARK_PCREL]                                 = apply_r_larch_none,
 410         [R_LARCH_SOP_PUSH_PCREL]                             = apply_r_larch_sop_push_pcrel,
 411         [R_LARCH_SOP_PUSH_ABSOLUTE]                          = apply_r_larch_sop_push_absolute,
 412         [R_LARCH_SOP_PUSH_DUP]                               = apply_r_larch_sop_push_dup,
 413         [R_LARCH_SOP_SUB ... R_LARCH_SOP_IF_ELSE]            = apply_r_larch_sop,
 414         [R_LARCH_SOP_POP_32_S_10_5 ... R_LARCH_SOP_POP_32_U] = apply_r_larch_sop_imm_field,
 415         [R_LARCH_ADD32 ... R_LARCH_SUB64]                    = apply_r_larch_add_sub,
 416         [R_LARCH_PCALA_HI20...R_LARCH_PCALA64_HI12]          = apply_r_larch_pcala,
 417         [R_LARCH_32_PCREL]                                   = apply_r_larch_32_pcrel,
 418         [R_LARCH_64_PCREL]                                   = apply_r_larch_64_pcrel,
 419 };
 420
 421 int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
 422                        unsigned int symindex, unsigned int relsec,
 423                        struct module *mod)
 424 {
 425         int i, err;
 426         unsigned int type;
 427         s64 rela_stack[RELA_STACK_DEPTH];
 428         size_t rela_stack_top = 0;
 429         reloc_rela_handler handler;
 430         void *location;
 431         Elf_Addr v;
 432         Elf_Sym *sym;
 433         Elf_Rela *rel = (void *) sechdrs[relsec].sh_addr;
 434
 435         pr_debug("%s: Applying relocate section %u to %u\n", __func__, relsec,
 436                sechdrs[relsec].sh_info);
 437
 438         rela_stack_top = 0;
 439         for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) {
 440                 /* This is where to make the change */
 441                 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr + rel[i].r_offset;
 442                 /* This is the symbol it is referring to */
 443                 sym = (Elf_Sym *)sechdrs[symindex].sh_addr + ELF_R_SYM(rel[i].r_info);
 444                 if (IS_ERR_VALUE(sym->st_value)) {
 445                         /* Ignore unresolved weak symbol */
 446                         if (ELF_ST_BIND(sym->st_info) == STB_WEAK)
 447                                 continue;
 448                         pr_warn("%s: Unknown symbol %s\n", mod->name, strtab + sym->st_name);
 449                         return -ENOENT;
 450                 }
 451
 452                 type = ELF_R_TYPE(rel[i].r_info);
 453
 454                 if (type < ARRAY_SIZE(reloc_rela_handlers))
 455                         handler = reloc_rela_handlers[type];
 456                 else
 457                         handler = NULL;
 458
 459                 if (!handler) {
 460                         pr_err("%s: Unknown relocation type %u\n", mod->name, type);
 461                         return -EINVAL;
 462                 }
 463
 464                 pr_debug("type %d st_value %llx r_addend %llx loc %llx\n",
 465                        (int)ELF_R_TYPE(rel[i].r_info),
 466                        sym->st_value, rel[i].r_addend, (u64)location);
 467
 468                 v = sym->st_value + rel[i].r_addend;
 469                 switch (type) {
 470                 case R_LARCH_B26:
 471                         err = apply_r_larch_b26(mod, sechdrs, location,
 472                                                      v, rela_stack, &rela_stack_top, type);
 473                         break;
 474                 case R_LARCH_GOT_PC_HI20...R_LARCH_GOT_PC_LO12:
 475                         err = apply_r_larch_got_pc(mod, sechdrs, location,
 476                                                      v, rela_stack, &rela_stack_top, type);
 477                         break;
 478                 case R_LARCH_SOP_PUSH_PLT_PCREL:
 479                         err = apply_r_larch_sop_push_plt_pcrel(mod, sechdrs, location,
 480                                                      v, rela_stack, &rela_stack_top, type);
 481                         break;
 482                 default:
 483                         err = handler(mod, location, v, rela_stack, &rela_stack_top, type);
 484                 }
 485                 if (err)
 486                         return err;
 487         }
 488
 489         return 0;
 490 }
 491
 492 void *module_alloc(unsigned long size)
 493 {
 494         return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END,
 495                         GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0));
 496 }
 497
 498 static void module_init_ftrace_plt(const Elf_Ehdr *hdr,
 499                                    const Elf_Shdr *sechdrs, struct module *mod)
 500 {
 501 #ifdef CONFIG_DYNAMIC_FTRACE
 502         struct plt_entry *ftrace_plts;
 503
 504         ftrace_plts = (void *)sechdrs->sh_addr;
 505
 506         ftrace_plts[FTRACE_PLT_IDX] = emit_plt_entry(FTRACE_ADDR);
 507
 508         if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
 509                 ftrace_plts[FTRACE_REGS_PLT_IDX] = emit_plt_entry(FTRACE_REGS_ADDR);
 510
 511         mod->arch.ftrace_trampolines = ftrace_plts;
 512 #endif
 513 }
 514
 515 int module_finalize(const Elf_Ehdr *hdr,
 516                     const Elf_Shdr *sechdrs, struct module *mod)
 517 {
 518         const Elf_Shdr *s, *se;
 519         const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
 520
 521         for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) {
 522                 if (!strcmp(".altinstructions", secstrs + s->sh_name))
 523                         apply_alternatives((void *)s->sh_addr, (void *)s->sh_addr + s->sh_size);
 524                 if (!strcmp(".ftrace_trampoline", secstrs + s->sh_name))
 525                         module_init_ftrace_plt(hdr, s, mod);
 526         }
 527
 528         return 0;
 529 }