1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2014 ARM Limited
7 #include <linux/init.h>
8 #include <linux/list.h>
9 #include <linux/perf_event.h>
10 #include <linux/sched.h>
11 #include <linux/slab.h>
12 #include <linux/sysctl.h>
13 #include <linux/uaccess.h>
15 #include <asm/cpufeature.h>
17 #include <asm/sysreg.h>
18 #include <asm/system_misc.h>
19 #include <asm/traps.h>
21 #define CREATE_TRACE_POINTS
22 #include "trace-events-emulation.h"
25 * The runtime support for deprecated instruction support can be in one of
26 * following three states -
29 * 1 = emulate (software emulation)
30 * 2 = hw (supported in hardware)
32 enum insn_emulation_mode {
38 enum legacy_insn_status {
44 struct insn_emulation {
46 enum legacy_insn_status status;
47 bool (*try_emulate)(struct pt_regs *regs,
49 int (*set_hw_mode)(bool enable);
56 * sysctl for this emulation + a sentinal entry.
58 struct ctl_table sysctl[2];
61 #define ARM_OPCODE_CONDTEST_FAIL 0
62 #define ARM_OPCODE_CONDTEST_PASS 1
63 #define ARM_OPCODE_CONDTEST_UNCOND 2
65 #define ARM_OPCODE_CONDITION_UNCOND 0xf
67 static unsigned int __maybe_unused aarch32_check_condition(u32 opcode, u32 psr)
69 u32 cc_bits = opcode >> 28;
71 if (cc_bits != ARM_OPCODE_CONDITION_UNCOND) {
72 if ((*aarch32_opcode_cond_checks[cc_bits])(psr))
73 return ARM_OPCODE_CONDTEST_PASS;
75 return ARM_OPCODE_CONDTEST_FAIL;
77 return ARM_OPCODE_CONDTEST_UNCOND;
80 #ifdef CONFIG_SWP_EMULATION
82 * Implement emulation of the SWP/SWPB instructions using load-exclusive and
85 * Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>]
86 * Where: Rt = destination
92 * Error-checking SWP macros implemented using ldxr{b}/stxr{b}
95 /* Arbitrary constant to ensure forward-progress of the LL/SC loop */
96 #define __SWP_LL_SC_LOOPS 4
98 #define __user_swpX_asm(data, addr, res, temp, temp2, B) \
100 uaccess_enable_privileged(); \
101 __asm__ __volatile__( \
103 "0: ldxr"B" %w2, [%4]\n" \
104 "1: stxr"B" %w0, %w1, [%4]\n" \
106 " sub %w3, %w3, #1\n" \
113 " .pushsection .fixup,\"ax\"\n" \
115 "4: mov %w0, %w6\n" \
118 _ASM_EXTABLE(0b, 4b) \
119 _ASM_EXTABLE(1b, 4b) \
120 : "=&r" (res), "+r" (data), "=&r" (temp), "=&r" (temp2) \
121 : "r" ((unsigned long)addr), "i" (-EAGAIN), \
123 "i" (__SWP_LL_SC_LOOPS) \
125 uaccess_disable_privileged(); \
128 #define __user_swp_asm(data, addr, res, temp, temp2) \
129 __user_swpX_asm(data, addr, res, temp, temp2, "")
130 #define __user_swpb_asm(data, addr, res, temp, temp2) \
131 __user_swpX_asm(data, addr, res, temp, temp2, "b")
134 * Bit 22 of the instruction encoding distinguishes between
135 * the SWP and SWPB variants (bit set means SWPB).
137 #define TYPE_SWPB (1 << 22)
139 static int emulate_swpX(unsigned int address, unsigned int *data,
142 unsigned int res = 0;
144 if ((type != TYPE_SWPB) && (address & 0x3)) {
145 /* SWP to unaligned address not permitted */
146 pr_debug("SWP instruction on unaligned pointer!\n");
151 unsigned long temp, temp2;
153 if (type == TYPE_SWPB)
154 __user_swpb_asm(*data, address, res, temp, temp2);
156 __user_swp_asm(*data, address, res, temp, temp2);
158 if (likely(res != -EAGAIN) || signal_pending(current))
168 * swp_handler logs the id of calling process, dissects the instruction, sanity
169 * checks the memory location, calls emulate_swpX for the actual operation and
170 * deals with fixup/error handling before returning
172 static int swp_handler(struct pt_regs *regs, u32 instr)
174 u32 destreg, data, type, address = 0;
175 const void __user *user_ptr;
176 int rn, rt2, res = 0;
178 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
180 type = instr & TYPE_SWPB;
182 switch (aarch32_check_condition(instr, regs->pstate)) {
183 case ARM_OPCODE_CONDTEST_PASS:
185 case ARM_OPCODE_CONDTEST_FAIL:
186 /* Condition failed - return to next instruction */
188 case ARM_OPCODE_CONDTEST_UNCOND:
189 /* If unconditional encoding - not a SWP, undef */
195 rn = aarch32_insn_extract_reg_num(instr, A32_RN_OFFSET);
196 rt2 = aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET);
198 address = (u32)regs->user_regs.regs[rn];
199 data = (u32)regs->user_regs.regs[rt2];
200 destreg = aarch32_insn_extract_reg_num(instr, A32_RT_OFFSET);
202 pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n",
203 rn, address, destreg,
204 aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET), data);
206 /* Check access in reasonable access range for both SWP and SWPB */
207 user_ptr = (const void __user *)(unsigned long)(address & ~3);
208 if (!access_ok(user_ptr, 4)) {
209 pr_debug("SWP{B} emulation: access to 0x%08x not allowed!\n",
214 res = emulate_swpX(address, &data, type);
218 regs->user_regs.regs[destreg] = data;
221 if (type == TYPE_SWPB)
222 trace_instruction_emulation("swpb", regs->pc);
224 trace_instruction_emulation("swp", regs->pc);
226 pr_warn_ratelimited("\"%s\" (%ld) uses obsolete SWP{B} instruction at 0x%llx\n",
227 current->comm, (unsigned long)current->pid, regs->pc);
229 arm64_skip_faulting_instruction(regs, 4);
233 pr_debug("SWP{B} emulation: access caused memory abort!\n");
234 arm64_notify_segfault(address);
239 static bool try_emulate_swp(struct pt_regs *regs, u32 insn)
241 /* SWP{B} only exists in ARM state and does not exist in Thumb */
242 if (!compat_user_mode(regs) || compat_thumb_mode(regs))
245 if ((insn & 0x0fb00ff0) != 0x01000090)
248 return swp_handler(regs, insn) == 0;
251 static struct insn_emulation insn_swp = {
253 .status = INSN_OBSOLETE,
254 .try_emulate = try_emulate_swp,
257 #endif /* CONFIG_SWP_EMULATION */
259 #ifdef CONFIG_CP15_BARRIER_EMULATION
260 static int cp15barrier_handler(struct pt_regs *regs, u32 instr)
262 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
264 switch (aarch32_check_condition(instr, regs->pstate)) {
265 case ARM_OPCODE_CONDTEST_PASS:
267 case ARM_OPCODE_CONDTEST_FAIL:
268 /* Condition failed - return to next instruction */
270 case ARM_OPCODE_CONDTEST_UNCOND:
271 /* If unconditional encoding - not a barrier instruction */
277 switch (aarch32_insn_mcr_extract_crm(instr)) {
280 * dmb - mcr p15, 0, Rt, c7, c10, 5
281 * dsb - mcr p15, 0, Rt, c7, c10, 4
283 if (aarch32_insn_mcr_extract_opc2(instr) == 5) {
285 trace_instruction_emulation(
286 "mcr p15, 0, Rt, c7, c10, 5 ; dmb", regs->pc);
289 trace_instruction_emulation(
290 "mcr p15, 0, Rt, c7, c10, 4 ; dsb", regs->pc);
295 * isb - mcr p15, 0, Rt, c7, c5, 4
297 * Taking an exception or returning from one acts as an
298 * instruction barrier. So no explicit barrier needed here.
300 trace_instruction_emulation(
301 "mcr p15, 0, Rt, c7, c5, 4 ; isb", regs->pc);
306 pr_warn_ratelimited("\"%s\" (%ld) uses deprecated CP15 Barrier instruction at 0x%llx\n",
307 current->comm, (unsigned long)current->pid, regs->pc);
309 arm64_skip_faulting_instruction(regs, 4);
313 static int cp15_barrier_set_hw_mode(bool enable)
316 sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_CP15BEN);
318 sysreg_clear_set(sctlr_el1, SCTLR_EL1_CP15BEN, 0);
322 static bool try_emulate_cp15_barrier(struct pt_regs *regs, u32 insn)
324 if (!compat_user_mode(regs) || compat_thumb_mode(regs))
327 if ((insn & 0x0fff0fdf) == 0x0e070f9a)
328 return cp15barrier_handler(regs, insn) == 0;
330 if ((insn & 0x0fff0fff) == 0x0e070f95)
331 return cp15barrier_handler(regs, insn) == 0;
336 static struct insn_emulation insn_cp15_barrier = {
337 .name = "cp15_barrier",
338 .status = INSN_DEPRECATED,
339 .try_emulate = try_emulate_cp15_barrier,
340 .set_hw_mode = cp15_barrier_set_hw_mode,
342 #endif /* CONFIG_CP15_BARRIER_EMULATION */
344 #ifdef CONFIG_SETEND_EMULATION
345 static int setend_set_hw_mode(bool enable)
347 if (!cpu_supports_mixed_endian_el0())
351 sysreg_clear_set(sctlr_el1, SCTLR_EL1_SED, 0);
353 sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_SED);
357 static int compat_setend_handler(struct pt_regs *regs, u32 big_endian)
361 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
365 regs->pstate |= PSR_AA32_E_BIT;
368 regs->pstate &= ~PSR_AA32_E_BIT;
371 trace_instruction_emulation(insn, regs->pc);
372 pr_warn_ratelimited("\"%s\" (%ld) uses deprecated setend instruction at 0x%llx\n",
373 current->comm, (unsigned long)current->pid, regs->pc);
378 static int a32_setend_handler(struct pt_regs *regs, u32 instr)
380 int rc = compat_setend_handler(regs, (instr >> 9) & 1);
381 arm64_skip_faulting_instruction(regs, 4);
385 static int t16_setend_handler(struct pt_regs *regs, u32 instr)
387 int rc = compat_setend_handler(regs, (instr >> 3) & 1);
388 arm64_skip_faulting_instruction(regs, 2);
392 static bool try_emulate_setend(struct pt_regs *regs, u32 insn)
394 if (compat_thumb_mode(regs) &&
395 (insn & 0xfffffff7) == 0x0000b650)
396 return t16_setend_handler(regs, insn) == 0;
398 if (compat_user_mode(regs) &&
399 (insn & 0xfffffdff) == 0xf1010000)
400 return a32_setend_handler(regs, insn) == 0;
405 static struct insn_emulation insn_setend = {
407 .status = INSN_DEPRECATED,
408 .try_emulate = try_emulate_setend,
409 .set_hw_mode = setend_set_hw_mode,
411 #endif /* CONFIG_SETEND_EMULATION */
413 static struct insn_emulation *insn_emulations[] = {
414 #ifdef CONFIG_SWP_EMULATION
417 #ifdef CONFIG_CP15_BARRIER_EMULATION
420 #ifdef CONFIG_SETEND_EMULATION
425 static DEFINE_MUTEX(insn_emulation_mutex);
427 static void enable_insn_hw_mode(void *data)
429 struct insn_emulation *insn = (struct insn_emulation *)data;
430 if (insn->set_hw_mode)
431 insn->set_hw_mode(true);
434 static void disable_insn_hw_mode(void *data)
436 struct insn_emulation *insn = (struct insn_emulation *)data;
437 if (insn->set_hw_mode)
438 insn->set_hw_mode(false);
441 /* Run set_hw_mode(mode) on all active CPUs */
442 static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable)
444 if (!insn->set_hw_mode)
447 on_each_cpu(enable_insn_hw_mode, (void *)insn, true);
449 on_each_cpu(disable_insn_hw_mode, (void *)insn, true);
454 * Run set_hw_mode for all insns on a starting CPU.
456 * 0 - If all the hooks ran successfully.
457 * -EINVAL - At least one hook is not supported by the CPU.
459 static int run_all_insn_set_hw_mode(unsigned int cpu)
466 * Disable IRQs to serialize against an IPI from
467 * run_all_cpu_set_hw_mode(), ensuring the HW is programmed to the most
468 * recent enablement state if the two race with one another.
470 local_irq_save(flags);
471 for (i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
472 struct insn_emulation *insn = insn_emulations[i];
473 bool enable = READ_ONCE(insn->current_mode) == INSN_HW;
474 if (insn->set_hw_mode && insn->set_hw_mode(enable)) {
475 pr_warn("CPU[%u] cannot support the emulation of %s",
480 local_irq_restore(flags);
485 static int update_insn_emulation_mode(struct insn_emulation *insn,
486 enum insn_emulation_mode prev)
491 case INSN_UNDEF: /* Nothing to be done */
496 if (!run_all_cpu_set_hw_mode(insn, false))
497 pr_notice("Disabled %s support\n", insn->name);
501 switch (insn->current_mode) {
507 ret = run_all_cpu_set_hw_mode(insn, true);
509 pr_notice("Enabled %s support\n", insn->name);
516 static int emulation_proc_handler(struct ctl_table *table, int write,
517 void *buffer, size_t *lenp,
521 struct insn_emulation *insn = container_of(table->data, struct insn_emulation, current_mode);
522 enum insn_emulation_mode prev_mode = insn->current_mode;
524 mutex_lock(&insn_emulation_mutex);
525 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
527 if (ret || !write || prev_mode == insn->current_mode)
530 ret = update_insn_emulation_mode(insn, prev_mode);
532 /* Mode change failed, revert to previous mode. */
533 WRITE_ONCE(insn->current_mode, prev_mode);
534 update_insn_emulation_mode(insn, INSN_UNDEF);
537 mutex_unlock(&insn_emulation_mutex);
541 static void __init register_insn_emulation(struct insn_emulation *insn)
543 struct ctl_table *sysctl;
545 insn->min = INSN_UNDEF;
547 switch (insn->status) {
548 case INSN_DEPRECATED:
549 insn->current_mode = INSN_EMULATE;
550 /* Disable the HW mode if it was turned on at early boot time */
551 run_all_cpu_set_hw_mode(insn, false);
555 insn->current_mode = INSN_UNDEF;
556 insn->max = INSN_EMULATE;
558 case INSN_UNAVAILABLE:
559 insn->current_mode = INSN_UNDEF;
560 insn->max = INSN_UNDEF;
564 /* Program the HW if required */
565 update_insn_emulation_mode(insn, INSN_UNDEF);
567 if (insn->status != INSN_UNAVAILABLE) {
568 sysctl = &insn->sysctl[0];
571 sysctl->maxlen = sizeof(int);
573 sysctl->procname = insn->name;
574 sysctl->data = &insn->current_mode;
575 sysctl->extra1 = &insn->min;
576 sysctl->extra2 = &insn->max;
577 sysctl->proc_handler = emulation_proc_handler;
579 register_sysctl("abi", sysctl);
583 bool try_emulate_armv8_deprecated(struct pt_regs *regs, u32 insn)
587 for (i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
588 struct insn_emulation *ie = insn_emulations[i];
590 if (ie->status == INSN_UNAVAILABLE)
594 * A trap may race with the mode being changed
595 * INSN_EMULATE<->INSN_HW. Try to emulate the instruction to
596 * avoid a spurious UNDEF.
598 if (READ_ONCE(ie->current_mode) == INSN_UNDEF)
601 if (ie->try_emulate(regs, insn))
609 * Invoked as core_initcall, which guarantees that the instruction
610 * emulation is ready for userspace.
612 static int __init armv8_deprecated_init(void)
616 #ifdef CONFIG_SETEND_EMULATION
617 if (!system_supports_mixed_endian_el0()) {
618 insn_setend.status = INSN_UNAVAILABLE;
619 pr_info("setend instruction emulation is not supported on this system\n");
623 for (i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
624 struct insn_emulation *ie = insn_emulations[i];
626 if (ie->status == INSN_UNAVAILABLE)
629 register_insn_emulation(ie);
632 cpuhp_setup_state_nocalls(CPUHP_AP_ARM64_ISNDEP_STARTING,
633 "arm64/isndep:starting",
634 run_all_insn_set_hw_mode, NULL);
638 core_initcall(armv8_deprecated_init);