1 // SPDX-License-Identifier: GPL-2.0+
3 * Author: Hanlu Li <lihanlu@loongson.cn>
4 * Huacai Chen <chenhuacai@loongson.cn>
6 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
9 #define pr_fmt(fmt) "kmod: " fmt
11 #include <linux/moduleloader.h>
12 #include <linux/elf.h>
14 #include <linux/numa.h>
15 #include <linux/vmalloc.h>
16 #include <linux/slab.h>
18 #include <linux/ftrace.h>
19 #include <linux/string.h>
20 #include <linux/kernel.h>
21 #include <asm/alternative.h>
24 static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top)
26 if (*rela_stack_top >= RELA_STACK_DEPTH)
29 rela_stack[(*rela_stack_top)++] = stack_value;
30 pr_debug("%s stack_value = 0x%llx\n", __func__, stack_value);
35 static int rela_stack_pop(s64 *stack_value, s64 *rela_stack, size_t *rela_stack_top)
37 if (*rela_stack_top == 0)
40 *stack_value = rela_stack[--(*rela_stack_top)];
41 pr_debug("%s stack_value = 0x%llx\n", __func__, *stack_value);
46 static int apply_r_larch_none(struct module *mod, u32 *location, Elf_Addr v,
47 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
52 static int apply_r_larch_error(struct module *me, u32 *location, Elf_Addr v,
53 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
55 pr_err("%s: Unsupport relocation type %u, please add its support.\n", me->name, type);
59 static int apply_r_larch_32(struct module *mod, u32 *location, Elf_Addr v,
60 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
66 static int apply_r_larch_64(struct module *mod, u32 *location, Elf_Addr v,
67 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
69 *(Elf_Addr *)location = v;
73 static int apply_r_larch_sop_push_pcrel(struct module *mod, u32 *location, Elf_Addr v,
74 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
76 return rela_stack_push(v - (u64)location, rela_stack, rela_stack_top);
79 static int apply_r_larch_sop_push_absolute(struct module *mod, u32 *location, Elf_Addr v,
80 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
82 return rela_stack_push(v, rela_stack, rela_stack_top);
85 static int apply_r_larch_sop_push_dup(struct module *mod, u32 *location, Elf_Addr v,
86 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
91 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top);
94 err = rela_stack_push(opr1, rela_stack, rela_stack_top);
97 err = rela_stack_push(opr1, rela_stack, rela_stack_top);
104 static int apply_r_larch_sop_push_plt_pcrel(struct module *mod,
105 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v,
106 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
108 ptrdiff_t offset = (void *)v - (void *)location;
110 if (offset >= SZ_128M)
111 v = module_emit_plt_entry(mod, sechdrs, v);
113 if (offset < -SZ_128M)
114 v = module_emit_plt_entry(mod, sechdrs, v);
116 return apply_r_larch_sop_push_pcrel(mod, location, v, rela_stack, rela_stack_top, type);
119 static int apply_r_larch_sop(struct module *mod, u32 *location, Elf_Addr v,
120 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
123 s64 opr1, opr2, opr3;
125 if (type == R_LARCH_SOP_IF_ELSE) {
126 err = rela_stack_pop(&opr3, rela_stack, rela_stack_top);
131 err = rela_stack_pop(&opr2, rela_stack, rela_stack_top);
134 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top);
139 case R_LARCH_SOP_AND:
140 err = rela_stack_push(opr1 & opr2, rela_stack, rela_stack_top);
142 case R_LARCH_SOP_ADD:
143 err = rela_stack_push(opr1 + opr2, rela_stack, rela_stack_top);
145 case R_LARCH_SOP_SUB:
146 err = rela_stack_push(opr1 - opr2, rela_stack, rela_stack_top);
149 err = rela_stack_push(opr1 << opr2, rela_stack, rela_stack_top);
152 err = rela_stack_push(opr1 >> opr2, rela_stack, rela_stack_top);
154 case R_LARCH_SOP_IF_ELSE:
155 err = rela_stack_push(opr1 ? opr2 : opr3, rela_stack, rela_stack_top);
158 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
165 static int apply_r_larch_sop_imm_field(struct module *mod, u32 *location, Elf_Addr v,
166 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
170 union loongarch_instruction *insn = (union loongarch_instruction *)location;
172 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top);
177 case R_LARCH_SOP_POP_32_U_10_12:
178 if (!unsigned_imm_check(opr1, 12))
181 /* (*(uint32_t *) PC) [21 ... 10] = opr [11 ... 0] */
182 insn->reg2i12_format.immediate = opr1 & 0xfff;
184 case R_LARCH_SOP_POP_32_S_10_12:
185 if (!signed_imm_check(opr1, 12))
188 insn->reg2i12_format.immediate = opr1 & 0xfff;
190 case R_LARCH_SOP_POP_32_S_10_16:
191 if (!signed_imm_check(opr1, 16))
194 insn->reg2i16_format.immediate = opr1 & 0xffff;
196 case R_LARCH_SOP_POP_32_S_10_16_S2:
200 if (!signed_imm_check(opr1, 18))
203 insn->reg2i16_format.immediate = (opr1 >> 2) & 0xffff;
205 case R_LARCH_SOP_POP_32_S_5_20:
206 if (!signed_imm_check(opr1, 20))
209 insn->reg1i20_format.immediate = (opr1) & 0xfffff;
211 case R_LARCH_SOP_POP_32_S_0_5_10_16_S2:
215 if (!signed_imm_check(opr1, 23))
219 insn->reg1i21_format.immediate_l = opr1 & 0xffff;
220 insn->reg1i21_format.immediate_h = (opr1 >> 16) & 0x1f;
222 case R_LARCH_SOP_POP_32_S_0_10_10_16_S2:
226 if (!signed_imm_check(opr1, 28))
230 insn->reg0i26_format.immediate_l = opr1 & 0xffff;
231 insn->reg0i26_format.immediate_h = (opr1 >> 16) & 0x3ff;
233 case R_LARCH_SOP_POP_32_U:
234 if (!unsigned_imm_check(opr1, 32))
237 /* (*(uint32_t *) PC) = opr */
238 *location = (u32)opr1;
241 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
246 pr_err("module %s: opr1 = 0x%llx overflow! dangerous %s (%u) relocation\n",
247 mod->name, opr1, __func__, type);
251 pr_err("module %s: opr1 = 0x%llx unaligned! dangerous %s (%u) relocation\n",
252 mod->name, opr1, __func__, type);
256 static int apply_r_larch_add_sub(struct module *mod, u32 *location, Elf_Addr v,
257 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
261 *(s32 *)location += v;
264 *(s64 *)location += v;
267 *(s32 *)location -= v;
270 *(s64 *)location -= v;
273 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
278 static int apply_r_larch_b26(struct module *mod,
279 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v,
280 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
282 ptrdiff_t offset = (void *)v - (void *)location;
283 union loongarch_instruction *insn = (union loongarch_instruction *)location;
285 if (offset >= SZ_128M)
286 v = module_emit_plt_entry(mod, sechdrs, v);
288 if (offset < -SZ_128M)
289 v = module_emit_plt_entry(mod, sechdrs, v);
291 offset = (void *)v - (void *)location;
294 pr_err("module %s: jump offset = 0x%llx unaligned! dangerous R_LARCH_B26 (%u) relocation\n",
295 mod->name, (long long)offset, type);
299 if (!signed_imm_check(offset, 28)) {
300 pr_err("module %s: jump offset = 0x%llx overflow! dangerous R_LARCH_B26 (%u) relocation\n",
301 mod->name, (long long)offset, type);
306 insn->reg0i26_format.immediate_l = offset & 0xffff;
307 insn->reg0i26_format.immediate_h = (offset >> 16) & 0x3ff;
312 static int apply_r_larch_pcala(struct module *mod, u32 *location, Elf_Addr v,
313 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
315 union loongarch_instruction *insn = (union loongarch_instruction *)location;
316 /* Use s32 for a sign-extension deliberately. */
317 s32 offset_hi20 = (void *)((v + 0x800) & ~0xfff) -
318 (void *)((Elf_Addr)location & ~0xfff);
319 Elf_Addr anchor = (((Elf_Addr)location) & ~0xfff) + offset_hi20;
320 ptrdiff_t offset_rem = (void *)v - (void *)anchor;
323 case R_LARCH_PCALA_LO12:
324 insn->reg2i12_format.immediate = v & 0xfff;
326 case R_LARCH_PCALA_HI20:
327 v = offset_hi20 >> 12;
328 insn->reg1i20_format.immediate = v & 0xfffff;
330 case R_LARCH_PCALA64_LO20:
331 v = offset_rem >> 32;
332 insn->reg1i20_format.immediate = v & 0xfffff;
334 case R_LARCH_PCALA64_HI12:
335 v = offset_rem >> 52;
336 insn->reg2i12_format.immediate = v & 0xfff;
339 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
346 static int apply_r_larch_got_pc(struct module *mod,
347 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v,
348 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
350 Elf_Addr got = module_emit_got_entry(mod, sechdrs, v);
356 case R_LARCH_GOT_PC_LO12:
357 type = R_LARCH_PCALA_LO12;
359 case R_LARCH_GOT_PC_HI20:
360 type = R_LARCH_PCALA_HI20;
363 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
367 return apply_r_larch_pcala(mod, location, got, rela_stack, rela_stack_top, type);
370 static int apply_r_larch_32_pcrel(struct module *mod, u32 *location, Elf_Addr v,
371 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
373 ptrdiff_t offset = (void *)v - (void *)location;
375 *(u32 *)location = offset;
379 static int apply_r_larch_64_pcrel(struct module *mod, u32 *location, Elf_Addr v,
380 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
382 ptrdiff_t offset = (void *)v - (void *)location;
384 *(u64 *)location = offset;
389 * reloc_handlers_rela() - Apply a particular relocation to a module
390 * @mod: the module to apply the reloc to
391 * @location: the address at which the reloc is to be applied
392 * @v: the value of the reloc, with addend for RELA-style
393 * @rela_stack: the stack used for store relocation info, LOCAL to THIS module
394 * @rela_stac_top: where the stack operation(pop/push) applies to
396 * Return: 0 upon success, else -ERRNO
398 typedef int (*reloc_rela_handler)(struct module *mod, u32 *location, Elf_Addr v,
399 s64 *rela_stack, size_t *rela_stack_top, unsigned int type);
401 /* The handlers for known reloc types */
402 static reloc_rela_handler reloc_rela_handlers[] = {
403 [R_LARCH_NONE ... R_LARCH_64_PCREL] = apply_r_larch_error,
405 [R_LARCH_NONE] = apply_r_larch_none,
406 [R_LARCH_32] = apply_r_larch_32,
407 [R_LARCH_64] = apply_r_larch_64,
408 [R_LARCH_MARK_LA] = apply_r_larch_none,
409 [R_LARCH_MARK_PCREL] = apply_r_larch_none,
410 [R_LARCH_SOP_PUSH_PCREL] = apply_r_larch_sop_push_pcrel,
411 [R_LARCH_SOP_PUSH_ABSOLUTE] = apply_r_larch_sop_push_absolute,
412 [R_LARCH_SOP_PUSH_DUP] = apply_r_larch_sop_push_dup,
413 [R_LARCH_SOP_SUB ... R_LARCH_SOP_IF_ELSE] = apply_r_larch_sop,
414 [R_LARCH_SOP_POP_32_S_10_5 ... R_LARCH_SOP_POP_32_U] = apply_r_larch_sop_imm_field,
415 [R_LARCH_ADD32 ... R_LARCH_SUB64] = apply_r_larch_add_sub,
416 [R_LARCH_PCALA_HI20...R_LARCH_PCALA64_HI12] = apply_r_larch_pcala,
417 [R_LARCH_32_PCREL] = apply_r_larch_32_pcrel,
418 [R_LARCH_64_PCREL] = apply_r_larch_64_pcrel,
421 int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
422 unsigned int symindex, unsigned int relsec,
427 s64 rela_stack[RELA_STACK_DEPTH];
428 size_t rela_stack_top = 0;
429 reloc_rela_handler handler;
433 Elf_Rela *rel = (void *) sechdrs[relsec].sh_addr;
435 pr_debug("%s: Applying relocate section %u to %u\n", __func__, relsec,
436 sechdrs[relsec].sh_info);
439 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) {
440 /* This is where to make the change */
441 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr + rel[i].r_offset;
442 /* This is the symbol it is referring to */
443 sym = (Elf_Sym *)sechdrs[symindex].sh_addr + ELF_R_SYM(rel[i].r_info);
444 if (IS_ERR_VALUE(sym->st_value)) {
445 /* Ignore unresolved weak symbol */
446 if (ELF_ST_BIND(sym->st_info) == STB_WEAK)
448 pr_warn("%s: Unknown symbol %s\n", mod->name, strtab + sym->st_name);
452 type = ELF_R_TYPE(rel[i].r_info);
454 if (type < ARRAY_SIZE(reloc_rela_handlers))
455 handler = reloc_rela_handlers[type];
460 pr_err("%s: Unknown relocation type %u\n", mod->name, type);
464 pr_debug("type %d st_value %llx r_addend %llx loc %llx\n",
465 (int)ELF_R_TYPE(rel[i].r_info),
466 sym->st_value, rel[i].r_addend, (u64)location);
468 v = sym->st_value + rel[i].r_addend;
471 err = apply_r_larch_b26(mod, sechdrs, location,
472 v, rela_stack, &rela_stack_top, type);
474 case R_LARCH_GOT_PC_HI20...R_LARCH_GOT_PC_LO12:
475 err = apply_r_larch_got_pc(mod, sechdrs, location,
476 v, rela_stack, &rela_stack_top, type);
478 case R_LARCH_SOP_PUSH_PLT_PCREL:
479 err = apply_r_larch_sop_push_plt_pcrel(mod, sechdrs, location,
480 v, rela_stack, &rela_stack_top, type);
483 err = handler(mod, location, v, rela_stack, &rela_stack_top, type);
492 void *module_alloc(unsigned long size)
494 return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END,
495 GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0));
498 static void module_init_ftrace_plt(const Elf_Ehdr *hdr,
499 const Elf_Shdr *sechdrs, struct module *mod)
501 #ifdef CONFIG_DYNAMIC_FTRACE
502 struct plt_entry *ftrace_plts;
504 ftrace_plts = (void *)sechdrs->sh_addr;
506 ftrace_plts[FTRACE_PLT_IDX] = emit_plt_entry(FTRACE_ADDR);
508 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
509 ftrace_plts[FTRACE_REGS_PLT_IDX] = emit_plt_entry(FTRACE_REGS_ADDR);
511 mod->arch.ftrace_trampolines = ftrace_plts;
515 int module_finalize(const Elf_Ehdr *hdr,
516 const Elf_Shdr *sechdrs, struct module *mod)
518 const Elf_Shdr *s, *se;
519 const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
521 for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) {
522 if (!strcmp(".altinstructions", secstrs + s->sh_name))
523 apply_alternatives((void *)s->sh_addr, (void *)s->sh_addr + s->sh_size);
524 if (!strcmp(".ftrace_trampoline", secstrs + s->sh_name))
525 module_init_ftrace_plt(hdr, s, mod);