1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Copyright 2008 Michael Ellerman, IBM Corporation.
6 #include <linux/kprobes.h>
7 #include <linux/vmalloc.h>
8 #include <linux/init.h>
9 #include <linux/cpuhotplug.h>
10 #include <linux/uaccess.h>
11 #include <linux/jump_label.h>
13 #include <asm/tlbflush.h>
15 #include <asm/code-patching.h>
18 static int __patch_instruction(u32 *exec_addr, ppc_inst_t instr, u32 *patch_addr)
20 if (!ppc_inst_prefixed(instr)) {
21 u32 val = ppc_inst_val(instr);
23 __put_kernel_nofault(patch_addr, &val, u32, failed);
25 u64 val = ppc_inst_as_ulong(instr);
27 __put_kernel_nofault(patch_addr, &val, u64, failed);
30 asm ("dcbst 0, %0; sync; icbi 0,%1; sync; isync" :: "r" (patch_addr),
39 int raw_patch_instruction(u32 *addr, ppc_inst_t instr)
41 return __patch_instruction(addr, instr, addr);
44 #ifdef CONFIG_STRICT_KERNEL_RWX
45 static DEFINE_PER_CPU(struct vm_struct *, text_poke_area);
47 static int map_patch_area(void *addr, unsigned long text_poke_addr);
48 static void unmap_patch_area(unsigned long addr);
50 static int text_area_cpu_up(unsigned int cpu)
52 struct vm_struct *area;
56 area = get_vm_area(PAGE_SIZE, VM_ALLOC);
58 WARN_ONCE(1, "Failed to create text area for cpu %d\n",
63 // Map/unmap the area to ensure all page tables are pre-allocated
64 addr = (unsigned long)area->addr;
65 err = map_patch_area(empty_zero_page, addr);
69 unmap_patch_area(addr);
71 this_cpu_write(text_poke_area, area);
76 static int text_area_cpu_down(unsigned int cpu)
78 free_vm_area(this_cpu_read(text_poke_area));
82 static __ro_after_init DEFINE_STATIC_KEY_FALSE(poking_init_done);
85 * Although BUG_ON() is rude, in this case it should only happen if ENOMEM, and
86 * we judge it as being preferable to a kernel that will crash later when
87 * someone tries to use patch_instruction().
89 void __init poking_init(void)
91 BUG_ON(!cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
92 "powerpc/text_poke:online", text_area_cpu_up,
94 static_branch_enable(&poking_init_done);
98 * This can be called for kernel text or a module.
100 static int map_patch_area(void *addr, unsigned long text_poke_addr)
104 if (IS_ENABLED(CONFIG_MODULES) && is_vmalloc_or_module_addr(addr))
105 pfn = vmalloc_to_pfn(addr);
107 pfn = __pa_symbol(addr) >> PAGE_SHIFT;
109 return map_kernel_page(text_poke_addr, (pfn << PAGE_SHIFT), PAGE_KERNEL);
112 static void unmap_patch_area(unsigned long addr)
120 pgdp = pgd_offset_k(addr);
121 if (WARN_ON(pgd_none(*pgdp)))
124 p4dp = p4d_offset(pgdp, addr);
125 if (WARN_ON(p4d_none(*p4dp)))
128 pudp = pud_offset(p4dp, addr);
129 if (WARN_ON(pud_none(*pudp)))
132 pmdp = pmd_offset(pudp, addr);
133 if (WARN_ON(pmd_none(*pmdp)))
136 ptep = pte_offset_kernel(pmdp, addr);
137 if (WARN_ON(pte_none(*ptep)))
141 * In hash, pte_clear flushes the tlb, in radix, we have to
143 pte_clear(&init_mm, addr, ptep);
144 flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
147 static int __do_patch_instruction(u32 *addr, ppc_inst_t instr)
151 unsigned long text_poke_addr;
153 text_poke_addr = (unsigned long)__this_cpu_read(text_poke_area)->addr;
154 patch_addr = (u32 *)(text_poke_addr + offset_in_page(addr));
156 err = map_patch_area(addr, text_poke_addr);
160 err = __patch_instruction(addr, instr, patch_addr);
162 unmap_patch_area(text_poke_addr);
167 static int do_patch_instruction(u32 *addr, ppc_inst_t instr)
173 * During early early boot patch_instruction is called
174 * when text_poke_area is not ready, but we still need
175 * to allow patching. We just do the plain old patching
177 if (!static_branch_likely(&poking_init_done))
178 return raw_patch_instruction(addr, instr);
180 local_irq_save(flags);
181 err = __do_patch_instruction(addr, instr);
182 local_irq_restore(flags);
186 #else /* !CONFIG_STRICT_KERNEL_RWX */
188 static int do_patch_instruction(u32 *addr, ppc_inst_t instr)
190 return raw_patch_instruction(addr, instr);
193 #endif /* CONFIG_STRICT_KERNEL_RWX */
195 __ro_after_init DEFINE_STATIC_KEY_FALSE(init_mem_is_free);
197 int patch_instruction(u32 *addr, ppc_inst_t instr)
199 /* Make sure we aren't patching a freed init section */
200 if (static_branch_likely(&init_mem_is_free) && init_section_contains(addr, 4))
203 return do_patch_instruction(addr, instr);
205 NOKPROBE_SYMBOL(patch_instruction);
207 int patch_branch(u32 *addr, unsigned long target, int flags)
211 if (create_branch(&instr, addr, target, flags))
214 return patch_instruction(addr, instr);
218 * Helper to check if a given instruction is a conditional branch
219 * Derived from the conditional checks in analyse_instr()
221 bool is_conditional_branch(ppc_inst_t instr)
223 unsigned int opcode = ppc_inst_primary_opcode(instr);
225 if (opcode == 16) /* bc, bca, bcl, bcla */
228 switch ((ppc_inst_val(instr) >> 1) & 0x3ff) {
229 case 16: /* bclr, bclrl */
230 case 528: /* bcctr, bcctrl */
231 case 560: /* bctar, bctarl */
237 NOKPROBE_SYMBOL(is_conditional_branch);
239 int create_cond_branch(ppc_inst_t *instr, const u32 *addr,
240 unsigned long target, int flags)
245 if (! (flags & BRANCH_ABSOLUTE))
246 offset = offset - (unsigned long)addr;
248 /* Check we can represent the target in the instruction format */
249 if (!is_offset_in_cond_branch_range(offset))
252 /* Mask out the flags and target, so they don't step on each other. */
253 *instr = ppc_inst(0x40000000 | (flags & 0x3FF0003) | (offset & 0xFFFC));
258 int instr_is_relative_branch(ppc_inst_t instr)
260 if (ppc_inst_val(instr) & BRANCH_ABSOLUTE)
263 return instr_is_branch_iform(instr) || instr_is_branch_bform(instr);
266 int instr_is_relative_link_branch(ppc_inst_t instr)
268 return instr_is_relative_branch(instr) && (ppc_inst_val(instr) & BRANCH_SET_LINK);
271 static unsigned long branch_iform_target(const u32 *instr)
275 imm = ppc_inst_val(ppc_inst_read(instr)) & 0x3FFFFFC;
277 /* If the top bit of the immediate value is set this is negative */
281 if ((ppc_inst_val(ppc_inst_read(instr)) & BRANCH_ABSOLUTE) == 0)
282 imm += (unsigned long)instr;
284 return (unsigned long)imm;
287 static unsigned long branch_bform_target(const u32 *instr)
291 imm = ppc_inst_val(ppc_inst_read(instr)) & 0xFFFC;
293 /* If the top bit of the immediate value is set this is negative */
297 if ((ppc_inst_val(ppc_inst_read(instr)) & BRANCH_ABSOLUTE) == 0)
298 imm += (unsigned long)instr;
300 return (unsigned long)imm;
303 unsigned long branch_target(const u32 *instr)
305 if (instr_is_branch_iform(ppc_inst_read(instr)))
306 return branch_iform_target(instr);
307 else if (instr_is_branch_bform(ppc_inst_read(instr)))
308 return branch_bform_target(instr);
313 int translate_branch(ppc_inst_t *instr, const u32 *dest, const u32 *src)
315 unsigned long target;
316 target = branch_target(src);
318 if (instr_is_branch_iform(ppc_inst_read(src)))
319 return create_branch(instr, dest, target,
320 ppc_inst_val(ppc_inst_read(src)));
321 else if (instr_is_branch_bform(ppc_inst_read(src)))
322 return create_cond_branch(instr, dest, target,
323 ppc_inst_val(ppc_inst_read(src)));