1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Copyright 2008 Michael Ellerman, IBM Corporation.
6 #include <linux/kprobes.h>
7 #include <linux/vmalloc.h>
8 #include <linux/init.h>
9 #include <linux/cpuhotplug.h>
10 #include <linux/uaccess.h>
11 #include <linux/jump_label.h>
13 #include <asm/tlbflush.h>
15 #include <asm/code-patching.h>
18 static int __patch_instruction(u32 *exec_addr, ppc_inst_t instr, u32 *patch_addr)
20 if (!ppc_inst_prefixed(instr)) {
21 u32 val = ppc_inst_val(instr);
23 __put_kernel_nofault(patch_addr, &val, u32, failed);
25 u64 val = ppc_inst_as_ulong(instr);
27 __put_kernel_nofault(patch_addr, &val, u64, failed);
30 asm ("dcbst 0, %0; sync; icbi 0,%1; sync; isync" :: "r" (patch_addr),
39 int raw_patch_instruction(u32 *addr, ppc_inst_t instr)
41 return __patch_instruction(addr, instr, addr);
44 #ifdef CONFIG_STRICT_KERNEL_RWX
45 static DEFINE_PER_CPU(struct vm_struct *, text_poke_area);
47 static int map_patch_area(void *addr, unsigned long text_poke_addr);
48 static void unmap_patch_area(unsigned long addr);
50 static int text_area_cpu_up(unsigned int cpu)
52 struct vm_struct *area;
56 area = get_vm_area(PAGE_SIZE, VM_ALLOC);
58 WARN_ONCE(1, "Failed to create text area for cpu %d\n",
63 // Map/unmap the area to ensure all page tables are pre-allocated
64 addr = (unsigned long)area->addr;
65 err = map_patch_area(empty_zero_page, addr);
69 unmap_patch_area(addr);
71 this_cpu_write(text_poke_area, area);
76 static int text_area_cpu_down(unsigned int cpu)
78 free_vm_area(this_cpu_read(text_poke_area));
82 static __ro_after_init DEFINE_STATIC_KEY_FALSE(poking_init_done);
85 * Although BUG_ON() is rude, in this case it should only happen if ENOMEM, and
86 * we judge it as being preferable to a kernel that will crash later when
87 * someone tries to use patch_instruction().
89 void __init poking_init(void)
91 BUG_ON(!cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
92 "powerpc/text_poke:online", text_area_cpu_up,
94 static_branch_enable(&poking_init_done);
97 static unsigned long get_patch_pfn(void *addr)
99 if (IS_ENABLED(CONFIG_MODULES) && is_vmalloc_or_module_addr(addr))
100 return vmalloc_to_pfn(addr);
102 return __pa_symbol(addr) >> PAGE_SHIFT;
106 * This can be called for kernel text or a module.
108 static int map_patch_area(void *addr, unsigned long text_poke_addr)
110 unsigned long pfn = get_patch_pfn(addr);
112 return map_kernel_page(text_poke_addr, (pfn << PAGE_SHIFT), PAGE_KERNEL);
115 static void unmap_patch_area(unsigned long addr)
123 pgdp = pgd_offset_k(addr);
124 if (WARN_ON(pgd_none(*pgdp)))
127 p4dp = p4d_offset(pgdp, addr);
128 if (WARN_ON(p4d_none(*p4dp)))
131 pudp = pud_offset(p4dp, addr);
132 if (WARN_ON(pud_none(*pudp)))
135 pmdp = pmd_offset(pudp, addr);
136 if (WARN_ON(pmd_none(*pmdp)))
139 ptep = pte_offset_kernel(pmdp, addr);
140 if (WARN_ON(pte_none(*ptep)))
144 * In hash, pte_clear flushes the tlb, in radix, we have to
146 pte_clear(&init_mm, addr, ptep);
147 flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
150 static int __do_patch_instruction(u32 *addr, ppc_inst_t instr)
154 unsigned long text_poke_addr;
156 unsigned long pfn = get_patch_pfn(addr);
158 text_poke_addr = (unsigned long)__this_cpu_read(text_poke_area)->addr & PAGE_MASK;
159 patch_addr = (u32 *)(text_poke_addr + offset_in_page(addr));
161 pte = virt_to_kpte(text_poke_addr);
162 __set_pte_at(&init_mm, text_poke_addr, pte, pfn_pte(pfn, PAGE_KERNEL), 0);
163 /* See ptesync comment in radix__set_pte_at() */
165 asm volatile("ptesync": : :"memory");
167 err = __patch_instruction(addr, instr, patch_addr);
169 pte_clear(&init_mm, text_poke_addr, pte);
170 flush_tlb_kernel_range(text_poke_addr, text_poke_addr + PAGE_SIZE);
175 static int do_patch_instruction(u32 *addr, ppc_inst_t instr)
181 * During early early boot patch_instruction is called
182 * when text_poke_area is not ready, but we still need
183 * to allow patching. We just do the plain old patching
185 if (!static_branch_likely(&poking_init_done))
186 return raw_patch_instruction(addr, instr);
188 local_irq_save(flags);
189 err = __do_patch_instruction(addr, instr);
190 local_irq_restore(flags);
194 #else /* !CONFIG_STRICT_KERNEL_RWX */
196 static int do_patch_instruction(u32 *addr, ppc_inst_t instr)
198 return raw_patch_instruction(addr, instr);
201 #endif /* CONFIG_STRICT_KERNEL_RWX */
203 __ro_after_init DEFINE_STATIC_KEY_FALSE(init_mem_is_free);
205 int patch_instruction(u32 *addr, ppc_inst_t instr)
207 /* Make sure we aren't patching a freed init section */
208 if (static_branch_likely(&init_mem_is_free) && init_section_contains(addr, 4))
211 return do_patch_instruction(addr, instr);
213 NOKPROBE_SYMBOL(patch_instruction);
215 int patch_branch(u32 *addr, unsigned long target, int flags)
219 if (create_branch(&instr, addr, target, flags))
222 return patch_instruction(addr, instr);
226 * Helper to check if a given instruction is a conditional branch
227 * Derived from the conditional checks in analyse_instr()
229 bool is_conditional_branch(ppc_inst_t instr)
231 unsigned int opcode = ppc_inst_primary_opcode(instr);
233 if (opcode == 16) /* bc, bca, bcl, bcla */
236 switch ((ppc_inst_val(instr) >> 1) & 0x3ff) {
237 case 16: /* bclr, bclrl */
238 case 528: /* bcctr, bcctrl */
239 case 560: /* bctar, bctarl */
245 NOKPROBE_SYMBOL(is_conditional_branch);
247 int create_cond_branch(ppc_inst_t *instr, const u32 *addr,
248 unsigned long target, int flags)
253 if (! (flags & BRANCH_ABSOLUTE))
254 offset = offset - (unsigned long)addr;
256 /* Check we can represent the target in the instruction format */
257 if (!is_offset_in_cond_branch_range(offset))
260 /* Mask out the flags and target, so they don't step on each other. */
261 *instr = ppc_inst(0x40000000 | (flags & 0x3FF0003) | (offset & 0xFFFC));
266 int instr_is_relative_branch(ppc_inst_t instr)
268 if (ppc_inst_val(instr) & BRANCH_ABSOLUTE)
271 return instr_is_branch_iform(instr) || instr_is_branch_bform(instr);
274 int instr_is_relative_link_branch(ppc_inst_t instr)
276 return instr_is_relative_branch(instr) && (ppc_inst_val(instr) & BRANCH_SET_LINK);
279 static unsigned long branch_iform_target(const u32 *instr)
283 imm = ppc_inst_val(ppc_inst_read(instr)) & 0x3FFFFFC;
285 /* If the top bit of the immediate value is set this is negative */
289 if ((ppc_inst_val(ppc_inst_read(instr)) & BRANCH_ABSOLUTE) == 0)
290 imm += (unsigned long)instr;
292 return (unsigned long)imm;
295 static unsigned long branch_bform_target(const u32 *instr)
299 imm = ppc_inst_val(ppc_inst_read(instr)) & 0xFFFC;
301 /* If the top bit of the immediate value is set this is negative */
305 if ((ppc_inst_val(ppc_inst_read(instr)) & BRANCH_ABSOLUTE) == 0)
306 imm += (unsigned long)instr;
308 return (unsigned long)imm;
311 unsigned long branch_target(const u32 *instr)
313 if (instr_is_branch_iform(ppc_inst_read(instr)))
314 return branch_iform_target(instr);
315 else if (instr_is_branch_bform(ppc_inst_read(instr)))
316 return branch_bform_target(instr);
321 int translate_branch(ppc_inst_t *instr, const u32 *dest, const u32 *src)
323 unsigned long target;
324 target = branch_target(src);
326 if (instr_is_branch_iform(ppc_inst_read(src)))
327 return create_branch(instr, dest, target,
328 ppc_inst_val(ppc_inst_read(src)));
329 else if (instr_is_branch_bform(ppc_inst_read(src)))
330 return create_cond_branch(instr, dest, target,
331 ppc_inst_val(ppc_inst_read(src)));