2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
6 * Copyright (c) 2014 Imagination Technologies Ltd.
7 * Author: Leonid Yegoshin <Leonid.Yegoshin@imgtec.com>
8 * Author: Markos Chandras <markos.chandras@imgtec.com>
10 * MIPS R2 user space instruction emulator for MIPS R6
13 #include <linux/bug.h>
14 #include <linux/compiler.h>
15 #include <linux/debugfs.h>
16 #include <linux/init.h>
17 #include <linux/kernel.h>
18 #include <linux/ptrace.h>
19 #include <linux/seq_file.h>
22 #include <asm/branch.h>
23 #include <asm/break.h>
24 #include <asm/debug.h>
26 #include <asm/fpu_emulator.h>
28 #include <asm/mips-r2-to-r6-emul.h>
29 #include <asm/local.h>
30 #include <asm/mipsregs.h>
31 #include <asm/ptrace.h>
32 #include <asm/uaccess.h>
35 #define ADDIU "daddiu "
39 #define ADDIU "addiu "
42 #endif /* CONFIG_64BIT */
49 DEFINE_PER_CPU(struct mips_r2_emulator_stats, mipsr2emustats);
50 DEFINE_PER_CPU(struct mips_r2_emulator_stats, mipsr2bdemustats);
51 DEFINE_PER_CPU(struct mips_r2br_emulator_stats, mipsr2bremustats);
53 extern const unsigned int fpucondbit[8];
55 #define MIPS_R2_EMUL_TOTAL_PASS 10
57 int mipsr2_emulation = 0;
59 static int __init mipsr2emu_enable(char *s)
63 pr_info("MIPS R2-to-R6 Emulator Enabled!");
67 __setup("mipsr2emu", mipsr2emu_enable);
70 * mipsr6_emul - Emulate some frequent R2/R5/R6 instructions in delay slot
71 * for performance instead of the traditional way of using a stack trampoline
72 * which is rather slow.
73 * @regs: Process register set
76 static inline int mipsr6_emul(struct pt_regs *regs, u32 ir)
78 switch (MIPSInst_OPCODE(ir)) {
81 regs->regs[MIPSInst_RT(ir)] =
82 (s32)regs->regs[MIPSInst_RS(ir)] +
83 (s32)MIPSInst_SIMM(ir);
86 if (IS_ENABLED(CONFIG_32BIT))
90 regs->regs[MIPSInst_RT(ir)] =
91 (s64)regs->regs[MIPSInst_RS(ir)] +
92 (s64)MIPSInst_SIMM(ir);
98 /* FPU instructions in delay slot */
101 switch (MIPSInst_FUNC(ir)) {
104 regs->regs[MIPSInst_RD(ir)] =
105 regs->regs[MIPSInst_RS(ir)] |
106 regs->regs[MIPSInst_RT(ir)];
113 regs->regs[MIPSInst_RD(ir)] =
114 (s32)(((u32)regs->regs[MIPSInst_RT(ir)]) <<
122 regs->regs[MIPSInst_RD(ir)] =
123 (s32)(((u32)regs->regs[MIPSInst_RT(ir)]) >>
131 regs->regs[MIPSInst_RD(ir)] =
132 (s32)((u32)regs->regs[MIPSInst_RS(ir)] +
133 (u32)regs->regs[MIPSInst_RT(ir)]);
140 regs->regs[MIPSInst_RD(ir)] =
141 (s32)((u32)regs->regs[MIPSInst_RS(ir)] -
142 (u32)regs->regs[MIPSInst_RT(ir)]);
145 if (IS_ENABLED(CONFIG_32BIT) || MIPSInst_RS(ir))
149 regs->regs[MIPSInst_RD(ir)] =
150 (s64)(((u64)regs->regs[MIPSInst_RT(ir)]) <<
154 if (IS_ENABLED(CONFIG_32BIT) || MIPSInst_RS(ir))
158 regs->regs[MIPSInst_RD(ir)] =
159 (s64)(((u64)regs->regs[MIPSInst_RT(ir)]) >>
163 if (IS_ENABLED(CONFIG_32BIT) || MIPSInst_FD(ir))
167 regs->regs[MIPSInst_RD(ir)] =
168 (u64)regs->regs[MIPSInst_RS(ir)] +
169 (u64)regs->regs[MIPSInst_RT(ir)];
172 if (IS_ENABLED(CONFIG_32BIT) || MIPSInst_FD(ir))
176 regs->regs[MIPSInst_RD(ir)] =
177 (s64)((u64)regs->regs[MIPSInst_RS(ir)] -
178 (u64)regs->regs[MIPSInst_RT(ir)]);
183 pr_debug("No fastpath BD emulation for instruction 0x%08x (op: %02x)\n",
184 ir, MIPSInst_OPCODE(ir));
191 * movf_func - Emulate a MOVF instruction
192 * @regs: Process register set
195 * Returns 0 since it always succeeds.
197 static int movf_func(struct pt_regs *regs, u32 ir)
202 csr = current->thread.fpu.fcr31;
203 cond = fpucondbit[MIPSInst_RT(ir) >> 2];
205 if (((csr & cond) == 0) && MIPSInst_RD(ir))
206 regs->regs[MIPSInst_RD(ir)] = regs->regs[MIPSInst_RS(ir)];
214 * movt_func - Emulate a MOVT instruction
215 * @regs: Process register set
218 * Returns 0 since it always succeeds.
220 static int movt_func(struct pt_regs *regs, u32 ir)
225 csr = current->thread.fpu.fcr31;
226 cond = fpucondbit[MIPSInst_RT(ir) >> 2];
228 if (((csr & cond) != 0) && MIPSInst_RD(ir))
229 regs->regs[MIPSInst_RD(ir)] = regs->regs[MIPSInst_RS(ir)];
237 * jr_func - Emulate a JR instruction.
238 * @pt_regs: Process register set
241 * Returns SIGILL if JR was in delay slot, SIGEMT if we
242 * can't compute the EPC, SIGSEGV if we can't access the
243 * userland instruction or 0 on success.
245 static int jr_func(struct pt_regs *regs, u32 ir)
248 unsigned long cepc, epc, nepc;
251 if (delay_slot(regs))
254 /* EPC after the RI/JR instruction */
255 nepc = regs->cp0_epc;
256 /* Roll back to the reserved R2 JR instruction */
259 err = __compute_return_epc(regs);
266 cepc = regs->cp0_epc;
268 /* Get DS instruction */
269 err = __get_user(nir, (u32 __user *)nepc);
273 MIPS_R2BR_STATS(jrs);
275 /* If nir == 0(NOP), then nothing else to do */
278 * Negative err means FPU instruction in BD-slot,
279 * Zero err means 'BD-slot emulation done'
280 * For anything else we go back to trampoline emulation.
282 err = mipsr6_emul(regs, nir);
284 regs->cp0_epc = nepc;
285 err = mips_dsemul(regs, nir, epc, cepc);
288 MIPS_R2_STATS(dsemul);
296 * movz_func - Emulate a MOVZ instruction
297 * @regs: Process register set
300 * Returns 0 since it always succeeds.
302 static int movz_func(struct pt_regs *regs, u32 ir)
304 if (((regs->regs[MIPSInst_RT(ir)]) == 0) && MIPSInst_RD(ir))
305 regs->regs[MIPSInst_RD(ir)] = regs->regs[MIPSInst_RS(ir)];
312 * movn_func - Emulate a MOVZ instruction
313 * @regs: Process register set
316 * Returns 0 since it always succeeds.
318 static int movn_func(struct pt_regs *regs, u32 ir)
320 if (((regs->regs[MIPSInst_RT(ir)]) != 0) && MIPSInst_RD(ir))
321 regs->regs[MIPSInst_RD(ir)] = regs->regs[MIPSInst_RS(ir)];
328 * mfhi_func - Emulate a MFHI instruction
329 * @regs: Process register set
332 * Returns 0 since it always succeeds.
334 static int mfhi_func(struct pt_regs *regs, u32 ir)
337 regs->regs[MIPSInst_RD(ir)] = regs->hi;
345 * mthi_func - Emulate a MTHI instruction
346 * @regs: Process register set
349 * Returns 0 since it always succeeds.
351 static int mthi_func(struct pt_regs *regs, u32 ir)
353 regs->hi = regs->regs[MIPSInst_RS(ir)];
361 * mflo_func - Emulate a MFLO instruction
362 * @regs: Process register set
365 * Returns 0 since it always succeeds.
367 static int mflo_func(struct pt_regs *regs, u32 ir)
370 regs->regs[MIPSInst_RD(ir)] = regs->lo;
378 * mtlo_func - Emulate a MTLO instruction
379 * @regs: Process register set
382 * Returns 0 since it always succeeds.
384 static int mtlo_func(struct pt_regs *regs, u32 ir)
386 regs->lo = regs->regs[MIPSInst_RS(ir)];
394 * mult_func - Emulate a MULT instruction
395 * @regs: Process register set
398 * Returns 0 since it always succeeds.
400 static int mult_func(struct pt_regs *regs, u32 ir)
405 rt = regs->regs[MIPSInst_RT(ir)];
406 rs = regs->regs[MIPSInst_RS(ir)];
407 res = (s64)rt * (s64)rs;
421 * multu_func - Emulate a MULTU instruction
422 * @regs: Process register set
425 * Returns 0 since it always succeeds.
427 static int multu_func(struct pt_regs *regs, u32 ir)
432 rt = regs->regs[MIPSInst_RT(ir)];
433 rs = regs->regs[MIPSInst_RS(ir)];
434 res = (u64)rt * (u64)rs;
436 regs->lo = (s64)(s32)rt;
437 regs->hi = (s64)(s32)(res >> 32);
445 * div_func - Emulate a DIV instruction
446 * @regs: Process register set
449 * Returns 0 since it always succeeds.
451 static int div_func(struct pt_regs *regs, u32 ir)
455 rt = regs->regs[MIPSInst_RT(ir)];
456 rs = regs->regs[MIPSInst_RS(ir)];
458 regs->lo = (s64)(rs / rt);
459 regs->hi = (s64)(rs % rt);
467 * divu_func - Emulate a DIVU instruction
468 * @regs: Process register set
471 * Returns 0 since it always succeeds.
473 static int divu_func(struct pt_regs *regs, u32 ir)
477 rt = regs->regs[MIPSInst_RT(ir)];
478 rs = regs->regs[MIPSInst_RS(ir)];
480 regs->lo = (s64)(rs / rt);
481 regs->hi = (s64)(rs % rt);
489 * dmult_func - Emulate a DMULT instruction
490 * @regs: Process register set
493 * Returns 0 on success or SIGILL for 32-bit kernels.
495 static int dmult_func(struct pt_regs *regs, u32 ir)
500 if (IS_ENABLED(CONFIG_32BIT))
503 rt = regs->regs[MIPSInst_RT(ir)];
504 rs = regs->regs[MIPSInst_RS(ir)];
508 __asm__ __volatile__(
509 "dmuh %0, %1, %2\t\n"
521 * dmultu_func - Emulate a DMULTU instruction
522 * @regs: Process register set
525 * Returns 0 on success or SIGILL for 32-bit kernels.
527 static int dmultu_func(struct pt_regs *regs, u32 ir)
532 if (IS_ENABLED(CONFIG_32BIT))
535 rt = regs->regs[MIPSInst_RT(ir)];
536 rs = regs->regs[MIPSInst_RS(ir)];
540 __asm__ __volatile__(
541 "dmuhu %0, %1, %2\t\n"
553 * ddiv_func - Emulate a DDIV instruction
554 * @regs: Process register set
557 * Returns 0 on success or SIGILL for 32-bit kernels.
559 static int ddiv_func(struct pt_regs *regs, u32 ir)
563 if (IS_ENABLED(CONFIG_32BIT))
566 rt = regs->regs[MIPSInst_RT(ir)];
567 rs = regs->regs[MIPSInst_RS(ir)];
578 * ddivu_func - Emulate a DDIVU instruction
579 * @regs: Process register set
582 * Returns 0 on success or SIGILL for 32-bit kernels.
584 static int ddivu_func(struct pt_regs *regs, u32 ir)
588 if (IS_ENABLED(CONFIG_32BIT))
591 rt = regs->regs[MIPSInst_RT(ir)];
592 rs = regs->regs[MIPSInst_RS(ir)];
602 /* R6 removed instructions for the SPECIAL opcode */
603 static struct r2_decoder_table spec_op_table[] = {
604 { 0xfc1ff83f, 0x00000008, jr_func },
605 { 0xfc00ffff, 0x00000018, mult_func },
606 { 0xfc00ffff, 0x00000019, multu_func },
607 { 0xfc00ffff, 0x0000001c, dmult_func },
608 { 0xfc00ffff, 0x0000001d, dmultu_func },
609 { 0xffff07ff, 0x00000010, mfhi_func },
610 { 0xfc1fffff, 0x00000011, mthi_func },
611 { 0xffff07ff, 0x00000012, mflo_func },
612 { 0xfc1fffff, 0x00000013, mtlo_func },
613 { 0xfc0307ff, 0x00000001, movf_func },
614 { 0xfc0307ff, 0x00010001, movt_func },
615 { 0xfc0007ff, 0x0000000a, movz_func },
616 { 0xfc0007ff, 0x0000000b, movn_func },
617 { 0xfc00ffff, 0x0000001a, div_func },
618 { 0xfc00ffff, 0x0000001b, divu_func },
619 { 0xfc00ffff, 0x0000001e, ddiv_func },
620 { 0xfc00ffff, 0x0000001f, ddivu_func },
625 * madd_func - Emulate a MADD instruction
626 * @regs: Process register set
629 * Returns 0 since it always succeeds.
631 static int madd_func(struct pt_regs *regs, u32 ir)
636 rt = regs->regs[MIPSInst_RT(ir)];
637 rs = regs->regs[MIPSInst_RS(ir)];
638 res = (s64)rt * (s64)rs;
641 res += ((((s64)rt) << 32) | (u32)rs);
654 * maddu_func - Emulate a MADDU instruction
655 * @regs: Process register set
658 * Returns 0 since it always succeeds.
660 static int maddu_func(struct pt_regs *regs, u32 ir)
665 rt = regs->regs[MIPSInst_RT(ir)];
666 rs = regs->regs[MIPSInst_RS(ir)];
667 res = (u64)rt * (u64)rs;
670 res += ((((s64)rt) << 32) | (u32)rs);
673 regs->lo = (s64)(s32)rt;
675 regs->hi = (s64)(s32)rs;
683 * msub_func - Emulate a MSUB instruction
684 * @regs: Process register set
687 * Returns 0 since it always succeeds.
689 static int msub_func(struct pt_regs *regs, u32 ir)
694 rt = regs->regs[MIPSInst_RT(ir)];
695 rs = regs->regs[MIPSInst_RS(ir)];
696 res = (s64)rt * (s64)rs;
699 res = ((((s64)rt) << 32) | (u32)rs) - res;
712 * msubu_func - Emulate a MSUBU instruction
713 * @regs: Process register set
716 * Returns 0 since it always succeeds.
718 static int msubu_func(struct pt_regs *regs, u32 ir)
723 rt = regs->regs[MIPSInst_RT(ir)];
724 rs = regs->regs[MIPSInst_RS(ir)];
725 res = (u64)rt * (u64)rs;
728 res = ((((s64)rt) << 32) | (u32)rs) - res;
731 regs->lo = (s64)(s32)rt;
733 regs->hi = (s64)(s32)rs;
741 * mul_func - Emulate a MUL instruction
742 * @regs: Process register set
745 * Returns 0 since it always succeeds.
747 static int mul_func(struct pt_regs *regs, u32 ir)
752 if (!MIPSInst_RD(ir))
754 rt = regs->regs[MIPSInst_RT(ir)];
755 rs = regs->regs[MIPSInst_RS(ir)];
756 res = (s64)rt * (s64)rs;
759 regs->regs[MIPSInst_RD(ir)] = (s64)rs;
767 * clz_func - Emulate a CLZ instruction
768 * @regs: Process register set
771 * Returns 0 since it always succeeds.
773 static int clz_func(struct pt_regs *regs, u32 ir)
778 if (!MIPSInst_RD(ir))
781 rs = regs->regs[MIPSInst_RS(ir)];
782 __asm__ __volatile__("clz %0, %1" : "=r"(res) : "r"(rs));
783 regs->regs[MIPSInst_RD(ir)] = res;
791 * clo_func - Emulate a CLO instruction
792 * @regs: Process register set
795 * Returns 0 since it always succeeds.
798 static int clo_func(struct pt_regs *regs, u32 ir)
803 if (!MIPSInst_RD(ir))
806 rs = regs->regs[MIPSInst_RS(ir)];
807 __asm__ __volatile__("clo %0, %1" : "=r"(res) : "r"(rs));
808 regs->regs[MIPSInst_RD(ir)] = res;
816 * dclz_func - Emulate a DCLZ instruction
817 * @regs: Process register set
820 * Returns 0 since it always succeeds.
822 static int dclz_func(struct pt_regs *regs, u32 ir)
827 if (IS_ENABLED(CONFIG_32BIT))
830 if (!MIPSInst_RD(ir))
833 rs = regs->regs[MIPSInst_RS(ir)];
834 __asm__ __volatile__("dclz %0, %1" : "=r"(res) : "r"(rs));
835 regs->regs[MIPSInst_RD(ir)] = res;
843 * dclo_func - Emulate a DCLO instruction
844 * @regs: Process register set
847 * Returns 0 since it always succeeds.
849 static int dclo_func(struct pt_regs *regs, u32 ir)
854 if (IS_ENABLED(CONFIG_32BIT))
857 if (!MIPSInst_RD(ir))
860 rs = regs->regs[MIPSInst_RS(ir)];
861 __asm__ __volatile__("dclo %0, %1" : "=r"(res) : "r"(rs));
862 regs->regs[MIPSInst_RD(ir)] = res;
869 /* R6 removed instructions for the SPECIAL2 opcode */
870 static struct r2_decoder_table spec2_op_table[] = {
871 { 0xfc00ffff, 0x70000000, madd_func },
872 { 0xfc00ffff, 0x70000001, maddu_func },
873 { 0xfc0007ff, 0x70000002, mul_func },
874 { 0xfc00ffff, 0x70000004, msub_func },
875 { 0xfc00ffff, 0x70000005, msubu_func },
876 { 0xfc0007ff, 0x70000020, clz_func },
877 { 0xfc0007ff, 0x70000021, clo_func },
878 { 0xfc0007ff, 0x70000024, dclz_func },
879 { 0xfc0007ff, 0x70000025, dclo_func },
883 static inline int mipsr2_find_op_func(struct pt_regs *regs, u32 inst,
884 struct r2_decoder_table *table)
886 struct r2_decoder_table *p;
889 for (p = table; p->func; p++) {
890 if ((inst & p->mask) == p->code) {
891 err = (p->func)(regs, inst);
899 * mipsr2_decoder: Decode and emulate a MIPS R2 instruction
900 * @regs: Process register set
901 * @inst: Instruction to decode and emulate
902 * @fcr31: Floating Point Control and Status Register Cause bits returned
904 int mipsr2_decoder(struct pt_regs *regs, u32 inst, unsigned long *fcr31)
909 unsigned long cpc, epc, nepc, r31, res, rs, rt;
911 void __user *fault_addr = NULL;
915 r31 = regs->regs[31];
917 err = compute_return_epc(regs);
922 pr_debug("Emulating the 0x%08x R2 instruction @ 0x%08lx (pass=%d))\n",
925 switch (MIPSInst_OPCODE(inst)) {
927 err = mipsr2_find_op_func(regs, inst, spec_op_table);
929 /* FPU instruction under JR */
930 regs->cp0_cause |= CAUSEF_BD;
935 err = mipsr2_find_op_func(regs, inst, spec2_op_table);
938 rt = MIPSInst_RT(inst);
939 rs = MIPSInst_RS(inst);
942 if ((long)regs->regs[rs] >= MIPSInst_SIMM(inst))
943 do_trap_or_bp(regs, 0, 0, "TGEI");
945 MIPS_R2_STATS(traps);
949 if (regs->regs[rs] >= MIPSInst_UIMM(inst))
950 do_trap_or_bp(regs, 0, 0, "TGEIU");
952 MIPS_R2_STATS(traps);
956 if ((long)regs->regs[rs] < MIPSInst_SIMM(inst))
957 do_trap_or_bp(regs, 0, 0, "TLTI");
959 MIPS_R2_STATS(traps);
963 if (regs->regs[rs] < MIPSInst_UIMM(inst))
964 do_trap_or_bp(regs, 0, 0, "TLTIU");
966 MIPS_R2_STATS(traps);
970 if (regs->regs[rs] == MIPSInst_SIMM(inst))
971 do_trap_or_bp(regs, 0, 0, "TEQI");
973 MIPS_R2_STATS(traps);
977 if (regs->regs[rs] != MIPSInst_SIMM(inst))
978 do_trap_or_bp(regs, 0, 0, "TNEI");
980 MIPS_R2_STATS(traps);
987 if (delay_slot(regs)) {
991 regs->regs[31] = r31;
993 err = __compute_return_epc(regs);
996 if (err != BRANCH_LIKELY_TAKEN)
1000 err = __get_user(nir, (u32 __user *)nepc);
1006 * This will probably be optimized away when
1007 * CONFIG_DEBUG_FS is not enabled
1011 MIPS_R2BR_STATS(bltzl);
1014 MIPS_R2BR_STATS(bgezl);
1017 MIPS_R2BR_STATS(bltzall);
1020 MIPS_R2BR_STATS(bgezall);
1024 switch (MIPSInst_OPCODE(nir)) {
1029 regs->cp0_cause |= CAUSEF_BD;
1033 err = mipsr6_emul(regs, nir);
1035 err = mips_dsemul(regs, nir, epc, cpc);
1038 MIPS_R2_STATS(dsemul);
1044 if (delay_slot(regs)) {
1048 regs->regs[31] = r31;
1049 regs->cp0_epc = epc;
1050 err = __compute_return_epc(regs);
1053 cpc = regs->cp0_epc;
1055 err = __get_user(nir, (u32 __user *)nepc);
1061 * This will probably be optimized away when
1062 * CONFIG_DEBUG_FS is not enabled
1066 MIPS_R2BR_STATS(bltzal);
1069 MIPS_R2BR_STATS(bgezal);
1073 switch (MIPSInst_OPCODE(nir)) {
1078 regs->cp0_cause |= CAUSEF_BD;
1082 err = mipsr6_emul(regs, nir);
1084 err = mips_dsemul(regs, nir, epc, cpc);
1087 MIPS_R2_STATS(dsemul);
1092 regs->regs[31] = r31;
1093 regs->cp0_epc = epc;
1102 * For BLEZL and BGTZL, rt field must be set to 0. If this
1103 * is not the case, this may be an encoding of a MIPS R6
1104 * instruction, so return to CPU execution if this occurs
1106 if (MIPSInst_RT(inst)) {
1113 if (delay_slot(regs)) {
1117 regs->regs[31] = r31;
1118 regs->cp0_epc = epc;
1119 err = __compute_return_epc(regs);
1122 if (err != BRANCH_LIKELY_TAKEN)
1124 cpc = regs->cp0_epc;
1126 err = __get_user(nir, (u32 __user *)nepc);
1132 * This will probably be optimized away when
1133 * CONFIG_DEBUG_FS is not enabled
1135 switch (MIPSInst_OPCODE(inst)) {
1137 MIPS_R2BR_STATS(beql);
1140 MIPS_R2BR_STATS(bnel);
1143 MIPS_R2BR_STATS(blezl);
1146 MIPS_R2BR_STATS(bgtzl);
1150 switch (MIPSInst_OPCODE(nir)) {
1155 regs->cp0_cause |= CAUSEF_BD;
1159 err = mipsr6_emul(regs, nir);
1161 err = mips_dsemul(regs, nir, epc, cpc);
1164 MIPS_R2_STATS(dsemul);
1173 regs->regs[31] = r31;
1174 regs->cp0_epc = epc;
1175 if (!used_math()) { /* First time FPU user. */
1181 lose_fpu(1); /* Save FPU state for the emulator. */
1183 err = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 0,
1187 * We can't allow the emulated instruction to leave any
1188 * enabled Cause bits set in $fcr31.
1190 *fcr31 = res = mask_fcr31_x(current->thread.fpu.fcr31);
1191 current->thread.fpu.fcr31 &= ~res;
1194 * this is a tricky issue - lose_fpu() uses LL/SC atomics
1195 * if FPU is owned and effectively cancels user level LL/SC.
1196 * So, it could be logical to don't restore FPU ownership here.
1197 * But the sequence of multiple FPU instructions is much much
1198 * more often than LL-FPU-SC and I prefer loop here until
1199 * next scheduler cycle cancels FPU ownership
1201 own_fpu(1); /* Restore FPU state. */
1204 current->thread.cp0_baduaddr = (unsigned long)fault_addr;
1206 MIPS_R2_STATS(fpus);
1211 rt = regs->regs[MIPSInst_RT(inst)];
1212 vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst);
1213 if (!access_ok(VERIFY_READ, vaddr, 4)) {
1214 current->thread.cp0_baduaddr = vaddr;
1218 __asm__ __volatile__(
1221 #ifdef CONFIG_CPU_LITTLE_ENDIAN
1222 "1:" LB "%1, 0(%2)\n"
1223 INS "%0, %1, 24, 8\n"
1224 " andi %1, %2, 0x3\n"
1226 ADDIU "%2, %2, -1\n"
1227 "2:" LB "%1, 0(%2)\n"
1228 INS "%0, %1, 16, 8\n"
1229 " andi %1, %2, 0x3\n"
1231 ADDIU "%2, %2, -1\n"
1232 "3:" LB "%1, 0(%2)\n"
1233 INS "%0, %1, 8, 8\n"
1234 " andi %1, %2, 0x3\n"
1236 ADDIU "%2, %2, -1\n"
1237 "4:" LB "%1, 0(%2)\n"
1238 INS "%0, %1, 0, 8\n"
1239 #else /* !CONFIG_CPU_LITTLE_ENDIAN */
1240 "1:" LB "%1, 0(%2)\n"
1241 INS "%0, %1, 24, 8\n"
1243 " andi %1, %2, 0x3\n"
1245 "2:" LB "%1, 0(%2)\n"
1246 INS "%0, %1, 16, 8\n"
1248 " andi %1, %2, 0x3\n"
1250 "3:" LB "%1, 0(%2)\n"
1251 INS "%0, %1, 8, 8\n"
1253 " andi %1, %2, 0x3\n"
1255 "4:" LB "%1, 0(%2)\n"
1256 INS "%0, %1, 0, 8\n"
1257 #endif /* CONFIG_CPU_LITTLE_ENDIAN */
1258 "9: sll %0, %0, 0\n"
1261 " .section .fixup,\"ax\"\n"
1265 " .section __ex_table,\"a\"\n"
1272 : "+&r"(rt), "=&r"(rs),
1273 "+&r"(vaddr), "+&r"(err)
1276 if (MIPSInst_RT(inst) && !err)
1277 regs->regs[MIPSInst_RT(inst)] = rt;
1279 MIPS_R2_STATS(loads);
1284 rt = regs->regs[MIPSInst_RT(inst)];
1285 vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst);
1286 if (!access_ok(VERIFY_READ, vaddr, 4)) {
1287 current->thread.cp0_baduaddr = vaddr;
1291 __asm__ __volatile__(
1294 #ifdef CONFIG_CPU_LITTLE_ENDIAN
1295 "1:" LB "%1, 0(%2)\n"
1296 INS "%0, %1, 0, 8\n"
1298 " andi %1, %2, 0x3\n"
1300 "2:" LB "%1, 0(%2)\n"
1301 INS "%0, %1, 8, 8\n"
1303 " andi %1, %2, 0x3\n"
1305 "3:" LB "%1, 0(%2)\n"
1306 INS "%0, %1, 16, 8\n"
1308 " andi %1, %2, 0x3\n"
1310 "4:" LB "%1, 0(%2)\n"
1311 INS "%0, %1, 24, 8\n"
1313 #else /* !CONFIG_CPU_LITTLE_ENDIAN */
1314 "1:" LB "%1, 0(%2)\n"
1315 INS "%0, %1, 0, 8\n"
1316 " andi %1, %2, 0x3\n"
1318 ADDIU "%2, %2, -1\n"
1319 "2:" LB "%1, 0(%2)\n"
1320 INS "%0, %1, 8, 8\n"
1321 " andi %1, %2, 0x3\n"
1323 ADDIU "%2, %2, -1\n"
1324 "3:" LB "%1, 0(%2)\n"
1325 INS "%0, %1, 16, 8\n"
1326 " andi %1, %2, 0x3\n"
1328 ADDIU "%2, %2, -1\n"
1329 "4:" LB "%1, 0(%2)\n"
1330 INS "%0, %1, 24, 8\n"
1332 #endif /* CONFIG_CPU_LITTLE_ENDIAN */
1336 " .section .fixup,\"ax\"\n"
1340 " .section __ex_table,\"a\"\n"
1347 : "+&r"(rt), "=&r"(rs),
1348 "+&r"(vaddr), "+&r"(err)
1350 if (MIPSInst_RT(inst) && !err)
1351 regs->regs[MIPSInst_RT(inst)] = rt;
1353 MIPS_R2_STATS(loads);
1358 rt = regs->regs[MIPSInst_RT(inst)];
1359 vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst);
1360 if (!access_ok(VERIFY_WRITE, vaddr, 4)) {
1361 current->thread.cp0_baduaddr = vaddr;
1365 __asm__ __volatile__(
1368 #ifdef CONFIG_CPU_LITTLE_ENDIAN
1369 EXT "%1, %0, 24, 8\n"
1370 "1:" SB "%1, 0(%2)\n"
1371 " andi %1, %2, 0x3\n"
1373 ADDIU "%2, %2, -1\n"
1374 EXT "%1, %0, 16, 8\n"
1375 "2:" SB "%1, 0(%2)\n"
1376 " andi %1, %2, 0x3\n"
1378 ADDIU "%2, %2, -1\n"
1379 EXT "%1, %0, 8, 8\n"
1380 "3:" SB "%1, 0(%2)\n"
1381 " andi %1, %2, 0x3\n"
1383 ADDIU "%2, %2, -1\n"
1384 EXT "%1, %0, 0, 8\n"
1385 "4:" SB "%1, 0(%2)\n"
1386 #else /* !CONFIG_CPU_LITTLE_ENDIAN */
1387 EXT "%1, %0, 24, 8\n"
1388 "1:" SB "%1, 0(%2)\n"
1390 " andi %1, %2, 0x3\n"
1392 EXT "%1, %0, 16, 8\n"
1393 "2:" SB "%1, 0(%2)\n"
1395 " andi %1, %2, 0x3\n"
1397 EXT "%1, %0, 8, 8\n"
1398 "3:" SB "%1, 0(%2)\n"
1400 " andi %1, %2, 0x3\n"
1402 EXT "%1, %0, 0, 8\n"
1403 "4:" SB "%1, 0(%2)\n"
1404 #endif /* CONFIG_CPU_LITTLE_ENDIAN */
1407 " .section .fixup,\"ax\"\n"
1411 " .section __ex_table,\"a\"\n"
1418 : "+&r"(rt), "=&r"(rs),
1419 "+&r"(vaddr), "+&r"(err)
1423 MIPS_R2_STATS(stores);
1428 rt = regs->regs[MIPSInst_RT(inst)];
1429 vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst);
1430 if (!access_ok(VERIFY_WRITE, vaddr, 4)) {
1431 current->thread.cp0_baduaddr = vaddr;
1435 __asm__ __volatile__(
1438 #ifdef CONFIG_CPU_LITTLE_ENDIAN
1439 EXT "%1, %0, 0, 8\n"
1440 "1:" SB "%1, 0(%2)\n"
1442 " andi %1, %2, 0x3\n"
1444 EXT "%1, %0, 8, 8\n"
1445 "2:" SB "%1, 0(%2)\n"
1447 " andi %1, %2, 0x3\n"
1449 EXT "%1, %0, 16, 8\n"
1450 "3:" SB "%1, 0(%2)\n"
1452 " andi %1, %2, 0x3\n"
1454 EXT "%1, %0, 24, 8\n"
1455 "4:" SB "%1, 0(%2)\n"
1456 #else /* !CONFIG_CPU_LITTLE_ENDIAN */
1457 EXT "%1, %0, 0, 8\n"
1458 "1:" SB "%1, 0(%2)\n"
1459 " andi %1, %2, 0x3\n"
1461 ADDIU "%2, %2, -1\n"
1462 EXT "%1, %0, 8, 8\n"
1463 "2:" SB "%1, 0(%2)\n"
1464 " andi %1, %2, 0x3\n"
1466 ADDIU "%2, %2, -1\n"
1467 EXT "%1, %0, 16, 8\n"
1468 "3:" SB "%1, 0(%2)\n"
1469 " andi %1, %2, 0x3\n"
1471 ADDIU "%2, %2, -1\n"
1472 EXT "%1, %0, 24, 8\n"
1473 "4:" SB "%1, 0(%2)\n"
1474 #endif /* CONFIG_CPU_LITTLE_ENDIAN */
1477 " .section .fixup,\"ax\"\n"
1481 " .section __ex_table,\"a\"\n"
1488 : "+&r"(rt), "=&r"(rs),
1489 "+&r"(vaddr), "+&r"(err)
1493 MIPS_R2_STATS(stores);
1498 if (IS_ENABLED(CONFIG_32BIT)) {
1503 rt = regs->regs[MIPSInst_RT(inst)];
1504 vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst);
1505 if (!access_ok(VERIFY_READ, vaddr, 8)) {
1506 current->thread.cp0_baduaddr = vaddr;
1510 __asm__ __volatile__(
1513 #ifdef CONFIG_CPU_LITTLE_ENDIAN
1515 " dinsu %0, %1, 56, 8\n"
1516 " andi %1, %2, 0x7\n"
1518 " daddiu %2, %2, -1\n"
1520 " dinsu %0, %1, 48, 8\n"
1521 " andi %1, %2, 0x7\n"
1523 " daddiu %2, %2, -1\n"
1525 " dinsu %0, %1, 40, 8\n"
1526 " andi %1, %2, 0x7\n"
1528 " daddiu %2, %2, -1\n"
1530 " dinsu %0, %1, 32, 8\n"
1531 " andi %1, %2, 0x7\n"
1533 " daddiu %2, %2, -1\n"
1535 " dins %0, %1, 24, 8\n"
1536 " andi %1, %2, 0x7\n"
1538 " daddiu %2, %2, -1\n"
1540 " dins %0, %1, 16, 8\n"
1541 " andi %1, %2, 0x7\n"
1543 " daddiu %2, %2, -1\n"
1545 " dins %0, %1, 8, 8\n"
1546 " andi %1, %2, 0x7\n"
1548 " daddiu %2, %2, -1\n"
1550 " dins %0, %1, 0, 8\n"
1551 #else /* !CONFIG_CPU_LITTLE_ENDIAN */
1553 " dinsu %0, %1, 56, 8\n"
1554 " daddiu %2, %2, 1\n"
1555 " andi %1, %2, 0x7\n"
1558 " dinsu %0, %1, 48, 8\n"
1559 " daddiu %2, %2, 1\n"
1560 " andi %1, %2, 0x7\n"
1563 " dinsu %0, %1, 40, 8\n"
1564 " daddiu %2, %2, 1\n"
1565 " andi %1, %2, 0x7\n"
1568 " dinsu %0, %1, 32, 8\n"
1569 " daddiu %2, %2, 1\n"
1570 " andi %1, %2, 0x7\n"
1573 " dins %0, %1, 24, 8\n"
1574 " daddiu %2, %2, 1\n"
1575 " andi %1, %2, 0x7\n"
1578 " dins %0, %1, 16, 8\n"
1579 " daddiu %2, %2, 1\n"
1580 " andi %1, %2, 0x7\n"
1583 " dins %0, %1, 8, 8\n"
1584 " daddiu %2, %2, 1\n"
1585 " andi %1, %2, 0x7\n"
1588 " dins %0, %1, 0, 8\n"
1589 #endif /* CONFIG_CPU_LITTLE_ENDIAN */
1592 " .section .fixup,\"ax\"\n"
1596 " .section __ex_table,\"a\"\n"
1607 : "+&r"(rt), "=&r"(rs),
1608 "+&r"(vaddr), "+&r"(err)
1610 if (MIPSInst_RT(inst) && !err)
1611 regs->regs[MIPSInst_RT(inst)] = rt;
1613 MIPS_R2_STATS(loads);
1617 if (IS_ENABLED(CONFIG_32BIT)) {
1622 rt = regs->regs[MIPSInst_RT(inst)];
1623 vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst);
1624 if (!access_ok(VERIFY_READ, vaddr, 8)) {
1625 current->thread.cp0_baduaddr = vaddr;
1629 __asm__ __volatile__(
1632 #ifdef CONFIG_CPU_LITTLE_ENDIAN
1634 " dins %0, %1, 0, 8\n"
1635 " daddiu %2, %2, 1\n"
1636 " andi %1, %2, 0x7\n"
1639 " dins %0, %1, 8, 8\n"
1640 " daddiu %2, %2, 1\n"
1641 " andi %1, %2, 0x7\n"
1644 " dins %0, %1, 16, 8\n"
1645 " daddiu %2, %2, 1\n"
1646 " andi %1, %2, 0x7\n"
1649 " dins %0, %1, 24, 8\n"
1650 " daddiu %2, %2, 1\n"
1651 " andi %1, %2, 0x7\n"
1654 " dinsu %0, %1, 32, 8\n"
1655 " daddiu %2, %2, 1\n"
1656 " andi %1, %2, 0x7\n"
1659 " dinsu %0, %1, 40, 8\n"
1660 " daddiu %2, %2, 1\n"
1661 " andi %1, %2, 0x7\n"
1664 " dinsu %0, %1, 48, 8\n"
1665 " daddiu %2, %2, 1\n"
1666 " andi %1, %2, 0x7\n"
1669 " dinsu %0, %1, 56, 8\n"
1670 #else /* !CONFIG_CPU_LITTLE_ENDIAN */
1672 " dins %0, %1, 0, 8\n"
1673 " andi %1, %2, 0x7\n"
1675 " daddiu %2, %2, -1\n"
1677 " dins %0, %1, 8, 8\n"
1678 " andi %1, %2, 0x7\n"
1680 " daddiu %2, %2, -1\n"
1682 " dins %0, %1, 16, 8\n"
1683 " andi %1, %2, 0x7\n"
1685 " daddiu %2, %2, -1\n"
1687 " dins %0, %1, 24, 8\n"
1688 " andi %1, %2, 0x7\n"
1690 " daddiu %2, %2, -1\n"
1692 " dinsu %0, %1, 32, 8\n"
1693 " andi %1, %2, 0x7\n"
1695 " daddiu %2, %2, -1\n"
1697 " dinsu %0, %1, 40, 8\n"
1698 " andi %1, %2, 0x7\n"
1700 " daddiu %2, %2, -1\n"
1702 " dinsu %0, %1, 48, 8\n"
1703 " andi %1, %2, 0x7\n"
1705 " daddiu %2, %2, -1\n"
1707 " dinsu %0, %1, 56, 8\n"
1708 #endif /* CONFIG_CPU_LITTLE_ENDIAN */
1711 " .section .fixup,\"ax\"\n"
1715 " .section __ex_table,\"a\"\n"
1726 : "+&r"(rt), "=&r"(rs),
1727 "+&r"(vaddr), "+&r"(err)
1729 if (MIPSInst_RT(inst) && !err)
1730 regs->regs[MIPSInst_RT(inst)] = rt;
1732 MIPS_R2_STATS(loads);
1736 if (IS_ENABLED(CONFIG_32BIT)) {
1741 rt = regs->regs[MIPSInst_RT(inst)];
1742 vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst);
1743 if (!access_ok(VERIFY_WRITE, vaddr, 8)) {
1744 current->thread.cp0_baduaddr = vaddr;
1748 __asm__ __volatile__(
1751 #ifdef CONFIG_CPU_LITTLE_ENDIAN
1752 " dextu %1, %0, 56, 8\n"
1754 " andi %1, %2, 0x7\n"
1756 " daddiu %2, %2, -1\n"
1757 " dextu %1, %0, 48, 8\n"
1759 " andi %1, %2, 0x7\n"
1761 " daddiu %2, %2, -1\n"
1762 " dextu %1, %0, 40, 8\n"
1764 " andi %1, %2, 0x7\n"
1766 " daddiu %2, %2, -1\n"
1767 " dextu %1, %0, 32, 8\n"
1769 " andi %1, %2, 0x7\n"
1771 " daddiu %2, %2, -1\n"
1772 " dext %1, %0, 24, 8\n"
1774 " andi %1, %2, 0x7\n"
1776 " daddiu %2, %2, -1\n"
1777 " dext %1, %0, 16, 8\n"
1779 " andi %1, %2, 0x7\n"
1781 " daddiu %2, %2, -1\n"
1782 " dext %1, %0, 8, 8\n"
1784 " andi %1, %2, 0x7\n"
1786 " daddiu %2, %2, -1\n"
1787 " dext %1, %0, 0, 8\n"
1789 #else /* !CONFIG_CPU_LITTLE_ENDIAN */
1790 " dextu %1, %0, 56, 8\n"
1792 " daddiu %2, %2, 1\n"
1793 " andi %1, %2, 0x7\n"
1795 " dextu %1, %0, 48, 8\n"
1797 " daddiu %2, %2, 1\n"
1798 " andi %1, %2, 0x7\n"
1800 " dextu %1, %0, 40, 8\n"
1802 " daddiu %2, %2, 1\n"
1803 " andi %1, %2, 0x7\n"
1805 " dextu %1, %0, 32, 8\n"
1807 " daddiu %2, %2, 1\n"
1808 " andi %1, %2, 0x7\n"
1810 " dext %1, %0, 24, 8\n"
1812 " daddiu %2, %2, 1\n"
1813 " andi %1, %2, 0x7\n"
1815 " dext %1, %0, 16, 8\n"
1817 " daddiu %2, %2, 1\n"
1818 " andi %1, %2, 0x7\n"
1820 " dext %1, %0, 8, 8\n"
1822 " daddiu %2, %2, 1\n"
1823 " andi %1, %2, 0x7\n"
1825 " dext %1, %0, 0, 8\n"
1827 #endif /* CONFIG_CPU_LITTLE_ENDIAN */
1830 " .section .fixup,\"ax\"\n"
1834 " .section __ex_table,\"a\"\n"
1845 : "+&r"(rt), "=&r"(rs),
1846 "+&r"(vaddr), "+&r"(err)
1850 MIPS_R2_STATS(stores);
1854 if (IS_ENABLED(CONFIG_32BIT)) {
1859 rt = regs->regs[MIPSInst_RT(inst)];
1860 vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst);
1861 if (!access_ok(VERIFY_WRITE, vaddr, 8)) {
1862 current->thread.cp0_baduaddr = vaddr;
1866 __asm__ __volatile__(
1869 #ifdef CONFIG_CPU_LITTLE_ENDIAN
1870 " dext %1, %0, 0, 8\n"
1872 " daddiu %2, %2, 1\n"
1873 " andi %1, %2, 0x7\n"
1875 " dext %1, %0, 8, 8\n"
1877 " daddiu %2, %2, 1\n"
1878 " andi %1, %2, 0x7\n"
1880 " dext %1, %0, 16, 8\n"
1882 " daddiu %2, %2, 1\n"
1883 " andi %1, %2, 0x7\n"
1885 " dext %1, %0, 24, 8\n"
1887 " daddiu %2, %2, 1\n"
1888 " andi %1, %2, 0x7\n"
1890 " dextu %1, %0, 32, 8\n"
1892 " daddiu %2, %2, 1\n"
1893 " andi %1, %2, 0x7\n"
1895 " dextu %1, %0, 40, 8\n"
1897 " daddiu %2, %2, 1\n"
1898 " andi %1, %2, 0x7\n"
1900 " dextu %1, %0, 48, 8\n"
1902 " daddiu %2, %2, 1\n"
1903 " andi %1, %2, 0x7\n"
1905 " dextu %1, %0, 56, 8\n"
1907 #else /* !CONFIG_CPU_LITTLE_ENDIAN */
1908 " dext %1, %0, 0, 8\n"
1910 " andi %1, %2, 0x7\n"
1912 " daddiu %2, %2, -1\n"
1913 " dext %1, %0, 8, 8\n"
1915 " andi %1, %2, 0x7\n"
1917 " daddiu %2, %2, -1\n"
1918 " dext %1, %0, 16, 8\n"
1920 " andi %1, %2, 0x7\n"
1922 " daddiu %2, %2, -1\n"
1923 " dext %1, %0, 24, 8\n"
1925 " andi %1, %2, 0x7\n"
1927 " daddiu %2, %2, -1\n"
1928 " dextu %1, %0, 32, 8\n"
1930 " andi %1, %2, 0x7\n"
1932 " daddiu %2, %2, -1\n"
1933 " dextu %1, %0, 40, 8\n"
1935 " andi %1, %2, 0x7\n"
1937 " daddiu %2, %2, -1\n"
1938 " dextu %1, %0, 48, 8\n"
1940 " andi %1, %2, 0x7\n"
1942 " daddiu %2, %2, -1\n"
1943 " dextu %1, %0, 56, 8\n"
1945 #endif /* CONFIG_CPU_LITTLE_ENDIAN */
1948 " .section .fixup,\"ax\"\n"
1952 " .section __ex_table,\"a\"\n"
1963 : "+&r"(rt), "=&r"(rs),
1964 "+&r"(vaddr), "+&r"(err)
1968 MIPS_R2_STATS(stores);
1972 vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst);
1974 current->thread.cp0_baduaddr = vaddr;
1978 if (!access_ok(VERIFY_READ, vaddr, 4)) {
1979 current->thread.cp0_baduaddr = vaddr;
1984 if (!cpu_has_rw_llb) {
1986 * An LL/SC block can't be safely emulated without
1987 * a Config5/LLB availability. So it's probably time to
1988 * kill our process before things get any worse. This is
1989 * because Config5/LLB allows us to use ERETNC so that
1990 * the LLAddr/LLB bit is not cleared when we return from
1991 * an exception. MIPS R2 LL/SC instructions trap with an
1992 * RI exception so once we emulate them here, we return
1993 * back to userland with ERETNC. That preserves the
1994 * LLAddr/LLB so the subsequent SC instruction will
1995 * succeed preserving the atomic semantics of the LL/SC
1996 * block. Without that, there is no safe way to emulate
1997 * an LL/SC block in MIPSR2 userland.
1999 pr_err("Can't emulate MIPSR2 LL/SC without Config5/LLB\n");
2004 __asm__ __volatile__(
2009 ".section .fixup,\"ax\"\n"
2014 ".section __ex_table,\"a\"\n"
2017 : "=&r"(res), "+&r"(err)
2018 : "r"(vaddr), "i"(SIGSEGV)
2021 if (MIPSInst_RT(inst) && !err)
2022 regs->regs[MIPSInst_RT(inst)] = res;
2023 MIPS_R2_STATS(llsc);
2028 vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst);
2030 current->thread.cp0_baduaddr = vaddr;
2034 if (!access_ok(VERIFY_WRITE, vaddr, 4)) {
2035 current->thread.cp0_baduaddr = vaddr;
2040 if (!cpu_has_rw_llb) {
2042 * An LL/SC block can't be safely emulated without
2043 * a Config5/LLB availability. So it's probably time to
2044 * kill our process before things get any worse. This is
2045 * because Config5/LLB allows us to use ERETNC so that
2046 * the LLAddr/LLB bit is not cleared when we return from
2047 * an exception. MIPS R2 LL/SC instructions trap with an
2048 * RI exception so once we emulate them here, we return
2049 * back to userland with ERETNC. That preserves the
2050 * LLAddr/LLB so the subsequent SC instruction will
2051 * succeed preserving the atomic semantics of the LL/SC
2052 * block. Without that, there is no safe way to emulate
2053 * an LL/SC block in MIPSR2 userland.
2055 pr_err("Can't emulate MIPSR2 LL/SC without Config5/LLB\n");
2060 res = regs->regs[MIPSInst_RT(inst)];
2062 __asm__ __volatile__(
2067 ".section .fixup,\"ax\"\n"
2072 ".section __ex_table,\"a\"\n"
2075 : "+&r"(res), "+&r"(err)
2076 : "r"(vaddr), "i"(SIGSEGV));
2078 if (MIPSInst_RT(inst) && !err)
2079 regs->regs[MIPSInst_RT(inst)] = res;
2081 MIPS_R2_STATS(llsc);
2086 if (IS_ENABLED(CONFIG_32BIT)) {
2091 vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst);
2093 current->thread.cp0_baduaddr = vaddr;
2097 if (!access_ok(VERIFY_READ, vaddr, 8)) {
2098 current->thread.cp0_baduaddr = vaddr;
2103 if (!cpu_has_rw_llb) {
2105 * An LL/SC block can't be safely emulated without
2106 * a Config5/LLB availability. So it's probably time to
2107 * kill our process before things get any worse. This is
2108 * because Config5/LLB allows us to use ERETNC so that
2109 * the LLAddr/LLB bit is not cleared when we return from
2110 * an exception. MIPS R2 LL/SC instructions trap with an
2111 * RI exception so once we emulate them here, we return
2112 * back to userland with ERETNC. That preserves the
2113 * LLAddr/LLB so the subsequent SC instruction will
2114 * succeed preserving the atomic semantics of the LL/SC
2115 * block. Without that, there is no safe way to emulate
2116 * an LL/SC block in MIPSR2 userland.
2118 pr_err("Can't emulate MIPSR2 LL/SC without Config5/LLB\n");
2123 __asm__ __volatile__(
2128 ".section .fixup,\"ax\"\n"
2133 ".section __ex_table,\"a\"\n"
2136 : "=&r"(res), "+&r"(err)
2137 : "r"(vaddr), "i"(SIGSEGV)
2139 if (MIPSInst_RT(inst) && !err)
2140 regs->regs[MIPSInst_RT(inst)] = res;
2142 MIPS_R2_STATS(llsc);
2147 if (IS_ENABLED(CONFIG_32BIT)) {
2152 vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst);
2154 current->thread.cp0_baduaddr = vaddr;
2158 if (!access_ok(VERIFY_WRITE, vaddr, 8)) {
2159 current->thread.cp0_baduaddr = vaddr;
2164 if (!cpu_has_rw_llb) {
2166 * An LL/SC block can't be safely emulated without
2167 * a Config5/LLB availability. So it's probably time to
2168 * kill our process before things get any worse. This is
2169 * because Config5/LLB allows us to use ERETNC so that
2170 * the LLAddr/LLB bit is not cleared when we return from
2171 * an exception. MIPS R2 LL/SC instructions trap with an
2172 * RI exception so once we emulate them here, we return
2173 * back to userland with ERETNC. That preserves the
2174 * LLAddr/LLB so the subsequent SC instruction will
2175 * succeed preserving the atomic semantics of the LL/SC
2176 * block. Without that, there is no safe way to emulate
2177 * an LL/SC block in MIPSR2 userland.
2179 pr_err("Can't emulate MIPSR2 LL/SC without Config5/LLB\n");
2184 res = regs->regs[MIPSInst_RT(inst)];
2186 __asm__ __volatile__(
2191 ".section .fixup,\"ax\"\n"
2196 ".section __ex_table,\"a\"\n"
2199 : "+&r"(res), "+&r"(err)
2200 : "r"(vaddr), "i"(SIGSEGV));
2202 if (MIPSInst_RT(inst) && !err)
2203 regs->regs[MIPSInst_RT(inst)] = res;
2205 MIPS_R2_STATS(llsc);
2216 * Let's not return to userland just yet. It's costly and
2217 * it's likely we have more R2 instructions to emulate
2219 if (!err && (pass++ < MIPS_R2_EMUL_TOTAL_PASS)) {
2220 regs->cp0_cause &= ~CAUSEF_BD;
2221 err = get_user(inst, (u32 __user *)regs->cp0_epc);
2229 if (err && (err != SIGEMT)) {
2230 regs->regs[31] = r31;
2231 regs->cp0_epc = epc;
2234 /* Likely a MIPS R6 compatible instruction */
2235 if (pass && (err == SIGILL))
2241 #ifdef CONFIG_DEBUG_FS
2243 static int mipsr2_stats_show(struct seq_file *s, void *unused)
2246 seq_printf(s, "Instruction\tTotal\tBDslot\n------------------------------\n");
2247 seq_printf(s, "movs\t\t%ld\t%ld\n",
2248 (unsigned long)__this_cpu_read(mipsr2emustats.movs),
2249 (unsigned long)__this_cpu_read(mipsr2bdemustats.movs));
2250 seq_printf(s, "hilo\t\t%ld\t%ld\n",
2251 (unsigned long)__this_cpu_read(mipsr2emustats.hilo),
2252 (unsigned long)__this_cpu_read(mipsr2bdemustats.hilo));
2253 seq_printf(s, "muls\t\t%ld\t%ld\n",
2254 (unsigned long)__this_cpu_read(mipsr2emustats.muls),
2255 (unsigned long)__this_cpu_read(mipsr2bdemustats.muls));
2256 seq_printf(s, "divs\t\t%ld\t%ld\n",
2257 (unsigned long)__this_cpu_read(mipsr2emustats.divs),
2258 (unsigned long)__this_cpu_read(mipsr2bdemustats.divs));
2259 seq_printf(s, "dsps\t\t%ld\t%ld\n",
2260 (unsigned long)__this_cpu_read(mipsr2emustats.dsps),
2261 (unsigned long)__this_cpu_read(mipsr2bdemustats.dsps));
2262 seq_printf(s, "bops\t\t%ld\t%ld\n",
2263 (unsigned long)__this_cpu_read(mipsr2emustats.bops),
2264 (unsigned long)__this_cpu_read(mipsr2bdemustats.bops));
2265 seq_printf(s, "traps\t\t%ld\t%ld\n",
2266 (unsigned long)__this_cpu_read(mipsr2emustats.traps),
2267 (unsigned long)__this_cpu_read(mipsr2bdemustats.traps));
2268 seq_printf(s, "fpus\t\t%ld\t%ld\n",
2269 (unsigned long)__this_cpu_read(mipsr2emustats.fpus),
2270 (unsigned long)__this_cpu_read(mipsr2bdemustats.fpus));
2271 seq_printf(s, "loads\t\t%ld\t%ld\n",
2272 (unsigned long)__this_cpu_read(mipsr2emustats.loads),
2273 (unsigned long)__this_cpu_read(mipsr2bdemustats.loads));
2274 seq_printf(s, "stores\t\t%ld\t%ld\n",
2275 (unsigned long)__this_cpu_read(mipsr2emustats.stores),
2276 (unsigned long)__this_cpu_read(mipsr2bdemustats.stores));
2277 seq_printf(s, "llsc\t\t%ld\t%ld\n",
2278 (unsigned long)__this_cpu_read(mipsr2emustats.llsc),
2279 (unsigned long)__this_cpu_read(mipsr2bdemustats.llsc));
2280 seq_printf(s, "dsemul\t\t%ld\t%ld\n",
2281 (unsigned long)__this_cpu_read(mipsr2emustats.dsemul),
2282 (unsigned long)__this_cpu_read(mipsr2bdemustats.dsemul));
2283 seq_printf(s, "jr\t\t%ld\n",
2284 (unsigned long)__this_cpu_read(mipsr2bremustats.jrs));
2285 seq_printf(s, "bltzl\t\t%ld\n",
2286 (unsigned long)__this_cpu_read(mipsr2bremustats.bltzl));
2287 seq_printf(s, "bgezl\t\t%ld\n",
2288 (unsigned long)__this_cpu_read(mipsr2bremustats.bgezl));
2289 seq_printf(s, "bltzll\t\t%ld\n",
2290 (unsigned long)__this_cpu_read(mipsr2bremustats.bltzll));
2291 seq_printf(s, "bgezll\t\t%ld\n",
2292 (unsigned long)__this_cpu_read(mipsr2bremustats.bgezll));
2293 seq_printf(s, "bltzal\t\t%ld\n",
2294 (unsigned long)__this_cpu_read(mipsr2bremustats.bltzal));
2295 seq_printf(s, "bgezal\t\t%ld\n",
2296 (unsigned long)__this_cpu_read(mipsr2bremustats.bgezal));
2297 seq_printf(s, "beql\t\t%ld\n",
2298 (unsigned long)__this_cpu_read(mipsr2bremustats.beql));
2299 seq_printf(s, "bnel\t\t%ld\n",
2300 (unsigned long)__this_cpu_read(mipsr2bremustats.bnel));
2301 seq_printf(s, "blezl\t\t%ld\n",
2302 (unsigned long)__this_cpu_read(mipsr2bremustats.blezl));
2303 seq_printf(s, "bgtzl\t\t%ld\n",
2304 (unsigned long)__this_cpu_read(mipsr2bremustats.bgtzl));
2309 static int mipsr2_stats_clear_show(struct seq_file *s, void *unused)
2311 mipsr2_stats_show(s, unused);
2313 __this_cpu_write((mipsr2emustats).movs, 0);
2314 __this_cpu_write((mipsr2bdemustats).movs, 0);
2315 __this_cpu_write((mipsr2emustats).hilo, 0);
2316 __this_cpu_write((mipsr2bdemustats).hilo, 0);
2317 __this_cpu_write((mipsr2emustats).muls, 0);
2318 __this_cpu_write((mipsr2bdemustats).muls, 0);
2319 __this_cpu_write((mipsr2emustats).divs, 0);
2320 __this_cpu_write((mipsr2bdemustats).divs, 0);
2321 __this_cpu_write((mipsr2emustats).dsps, 0);
2322 __this_cpu_write((mipsr2bdemustats).dsps, 0);
2323 __this_cpu_write((mipsr2emustats).bops, 0);
2324 __this_cpu_write((mipsr2bdemustats).bops, 0);
2325 __this_cpu_write((mipsr2emustats).traps, 0);
2326 __this_cpu_write((mipsr2bdemustats).traps, 0);
2327 __this_cpu_write((mipsr2emustats).fpus, 0);
2328 __this_cpu_write((mipsr2bdemustats).fpus, 0);
2329 __this_cpu_write((mipsr2emustats).loads, 0);
2330 __this_cpu_write((mipsr2bdemustats).loads, 0);
2331 __this_cpu_write((mipsr2emustats).stores, 0);
2332 __this_cpu_write((mipsr2bdemustats).stores, 0);
2333 __this_cpu_write((mipsr2emustats).llsc, 0);
2334 __this_cpu_write((mipsr2bdemustats).llsc, 0);
2335 __this_cpu_write((mipsr2emustats).dsemul, 0);
2336 __this_cpu_write((mipsr2bdemustats).dsemul, 0);
2337 __this_cpu_write((mipsr2bremustats).jrs, 0);
2338 __this_cpu_write((mipsr2bremustats).bltzl, 0);
2339 __this_cpu_write((mipsr2bremustats).bgezl, 0);
2340 __this_cpu_write((mipsr2bremustats).bltzll, 0);
2341 __this_cpu_write((mipsr2bremustats).bgezll, 0);
2342 __this_cpu_write((mipsr2bremustats).bltzall, 0);
2343 __this_cpu_write((mipsr2bremustats).bgezall, 0);
2344 __this_cpu_write((mipsr2bremustats).bltzal, 0);
2345 __this_cpu_write((mipsr2bremustats).bgezal, 0);
2346 __this_cpu_write((mipsr2bremustats).beql, 0);
2347 __this_cpu_write((mipsr2bremustats).bnel, 0);
2348 __this_cpu_write((mipsr2bremustats).blezl, 0);
2349 __this_cpu_write((mipsr2bremustats).bgtzl, 0);
2354 static int mipsr2_stats_open(struct inode *inode, struct file *file)
2356 return single_open(file, mipsr2_stats_show, inode->i_private);
2359 static int mipsr2_stats_clear_open(struct inode *inode, struct file *file)
2361 return single_open(file, mipsr2_stats_clear_show, inode->i_private);
2364 static const struct file_operations mipsr2_emul_fops = {
2365 .open = mipsr2_stats_open,
2367 .llseek = seq_lseek,
2368 .release = single_release,
2371 static const struct file_operations mipsr2_clear_fops = {
2372 .open = mipsr2_stats_clear_open,
2374 .llseek = seq_lseek,
2375 .release = single_release,
2379 static int __init mipsr2_init_debugfs(void)
2381 struct dentry *mipsr2_emul;
2383 if (!mips_debugfs_dir)
2386 mipsr2_emul = debugfs_create_file("r2_emul_stats", S_IRUGO,
2387 mips_debugfs_dir, NULL,
2392 mipsr2_emul = debugfs_create_file("r2_emul_stats_clear", S_IRUGO,
2393 mips_debugfs_dir, NULL,
2394 &mipsr2_clear_fops);
2401 device_initcall(mipsr2_init_debugfs);
2403 #endif /* CONFIG_DEBUG_FS */