2 * arch/powerpc/math-emu/math_efp.c
4 * Copyright (C) 2006-2008, 2010 Freescale Semiconductor, Inc.
6 * Author: Ebony Zhu, <ebony.zhu@freescale.com>
7 * Yu Liu, <yu.liu@freescale.com>
9 * Derived from arch/alpha/math-emu/math.c
10 * arch/powerpc/math-emu/math.c
13 * This file is the exception handler to make E500 SPE instructions
14 * fully comply with IEEE-754 floating point standard.
16 * This program is free software; you can redistribute it and/or
17 * modify it under the terms of the GNU General Public License
18 * as published by the Free Software Foundation; either version
19 * 2 of the License, or (at your option) any later version.
22 #include <linux/types.h>
23 #include <linux/prctl.h>
24 #include <linux/module.h>
26 #include <linux/uaccess.h>
29 #define FP_EX_BOOKE_E500_SPE
30 #include <asm/sfp-machine.h>
32 #include <math-emu/soft-fp.h>
33 #include <math-emu/single.h>
34 #include <math-emu/double.h>
49 #define EFSCMPGT 0x2cc
50 #define EFSCMPLT 0x2cd
51 #define EFSCMPEQ 0x2ce
58 #define EFSCTUIZ 0x2d8
59 #define EFSCTSIZ 0x2da
64 #define EVFSNABS 0x285
68 #define EVFSCMPGT 0x28c
69 #define EVFSCMPLT 0x28d
70 #define EVFSCMPEQ 0x28e
71 #define EVFSCTUI 0x294
72 #define EVFSCTSI 0x295
73 #define EVFSCTUF 0x296
74 #define EVFSCTSF 0x297
75 #define EVFSCTUIZ 0x298
76 #define EVFSCTSIZ 0x29a
85 #define EFDCTUIDZ 0x2ea
86 #define EFDCTSIDZ 0x2eb
87 #define EFDCMPGT 0x2ec
88 #define EFDCMPLT 0x2ed
89 #define EFDCMPEQ 0x2ee
95 #define EFDCTUIZ 0x2f8
96 #define EFDCTSIZ 0x2fa
104 #define SIGN_BIT_S (1UL << 31)
105 #define SIGN_BIT_D (1ULL << 63)
106 #define FP_EX_MASK (FP_EX_INEXACT | FP_EX_INVALID | FP_EX_DIVZERO | \
107 FP_EX_UNDERFLOW | FP_EX_OVERFLOW)
109 static int have_e500_cpu_a005_erratum;
116 static unsigned long insn_type(unsigned long speinsn)
118 unsigned long ret = NOTYPE;
120 switch (speinsn & 0x7ff) {
121 case EFSABS: ret = XA; break;
122 case EFSADD: ret = AB; break;
123 case EFSCFD: ret = XB; break;
124 case EFSCMPEQ: ret = XCR; break;
125 case EFSCMPGT: ret = XCR; break;
126 case EFSCMPLT: ret = XCR; break;
127 case EFSCTSF: ret = XB; break;
128 case EFSCTSI: ret = XB; break;
129 case EFSCTSIZ: ret = XB; break;
130 case EFSCTUF: ret = XB; break;
131 case EFSCTUI: ret = XB; break;
132 case EFSCTUIZ: ret = XB; break;
133 case EFSDIV: ret = AB; break;
134 case EFSMUL: ret = AB; break;
135 case EFSNABS: ret = XA; break;
136 case EFSNEG: ret = XA; break;
137 case EFSSUB: ret = AB; break;
138 case EFSCFSI: ret = XB; break;
140 case EVFSABS: ret = XA; break;
141 case EVFSADD: ret = AB; break;
142 case EVFSCMPEQ: ret = XCR; break;
143 case EVFSCMPGT: ret = XCR; break;
144 case EVFSCMPLT: ret = XCR; break;
145 case EVFSCTSF: ret = XB; break;
146 case EVFSCTSI: ret = XB; break;
147 case EVFSCTSIZ: ret = XB; break;
148 case EVFSCTUF: ret = XB; break;
149 case EVFSCTUI: ret = XB; break;
150 case EVFSCTUIZ: ret = XB; break;
151 case EVFSDIV: ret = AB; break;
152 case EVFSMUL: ret = AB; break;
153 case EVFSNABS: ret = XA; break;
154 case EVFSNEG: ret = XA; break;
155 case EVFSSUB: ret = AB; break;
157 case EFDABS: ret = XA; break;
158 case EFDADD: ret = AB; break;
159 case EFDCFS: ret = XB; break;
160 case EFDCMPEQ: ret = XCR; break;
161 case EFDCMPGT: ret = XCR; break;
162 case EFDCMPLT: ret = XCR; break;
163 case EFDCTSF: ret = XB; break;
164 case EFDCTSI: ret = XB; break;
165 case EFDCTSIDZ: ret = XB; break;
166 case EFDCTSIZ: ret = XB; break;
167 case EFDCTUF: ret = XB; break;
168 case EFDCTUI: ret = XB; break;
169 case EFDCTUIDZ: ret = XB; break;
170 case EFDCTUIZ: ret = XB; break;
171 case EFDDIV: ret = AB; break;
172 case EFDMUL: ret = AB; break;
173 case EFDNABS: ret = XA; break;
174 case EFDNEG: ret = XA; break;
175 case EFDSUB: ret = AB; break;
181 int do_spe_mathemu(struct pt_regs *regs)
186 unsigned long type, func, fc, fa, fb, src, speinsn;
187 union dw_union vc, va, vb;
189 if (get_user(speinsn, (unsigned int __user *) regs->nip))
191 if ((speinsn >> 26) != EFAPU)
192 return -EINVAL; /* not an spe instruction */
194 type = insn_type(speinsn);
198 func = speinsn & 0x7ff;
199 fc = (speinsn >> 21) & 0x1f;
200 fa = (speinsn >> 16) & 0x1f;
201 fb = (speinsn >> 11) & 0x1f;
202 src = (speinsn >> 5) & 0x7;
204 vc.wp[0] = current->thread.evr[fc];
205 vc.wp[1] = regs->gpr[fc];
206 va.wp[0] = current->thread.evr[fa];
207 va.wp[1] = regs->gpr[fa];
208 vb.wp[0] = current->thread.evr[fb];
209 vb.wp[1] = regs->gpr[fb];
211 __FPU_FPSCR = mfspr(SPRN_SPEFSCR);
213 pr_debug("speinsn:%08lx spefscr:%08lx\n", speinsn, __FPU_FPSCR);
214 pr_debug("vc: %08x %08x\n", vc.wp[0], vc.wp[1]);
215 pr_debug("va: %08x %08x\n", va.wp[0], va.wp[1]);
216 pr_debug("vb: %08x %08x\n", vb.wp[0], vb.wp[1]);
220 FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SR);
225 FP_UNPACK_SP(SA, va.wp + 1);
227 FP_UNPACK_SP(SB, vb.wp + 1);
230 FP_UNPACK_SP(SA, va.wp + 1);
234 pr_debug("SA: %ld %08lx %ld (%ld)\n", SA_s, SA_f, SA_e, SA_c);
235 pr_debug("SB: %ld %08lx %ld (%ld)\n", SB_s, SB_f, SB_e, SB_c);
239 vc.wp[1] = va.wp[1] & ~SIGN_BIT_S;
243 vc.wp[1] = va.wp[1] | SIGN_BIT_S;
247 vc.wp[1] = va.wp[1] ^ SIGN_BIT_S;
251 FP_ADD_S(SR, SA, SB);
255 FP_SUB_S(SR, SA, SB);
259 FP_MUL_S(SR, SA, SB);
263 FP_DIV_S(SR, SA, SB);
280 if (SB_c == FP_CLS_NAN) {
282 FP_SET_EXCEPTION(FP_EX_INVALID);
284 SB_e += (func == EFSCTSF ? 31 : 32);
285 FP_TO_INT_ROUND_S(vc.wp[1], SB, 32,
293 FP_UNPACK_DP(DB, vb.dp);
295 pr_debug("DB: %ld %08lx %08lx %ld (%ld)\n",
296 DB_s, DB_f1, DB_f0, DB_e, DB_c);
298 FP_CONV(S, D, 1, 2, SR, DB);
304 if (SB_c == FP_CLS_NAN) {
306 FP_SET_EXCEPTION(FP_EX_INVALID);
308 FP_TO_INT_ROUND_S(vc.wp[1], SB, 32,
309 ((func & 0x3) != 0));
315 if (SB_c == FP_CLS_NAN) {
317 FP_SET_EXCEPTION(FP_EX_INVALID);
319 FP_TO_INT_S(vc.wp[1], SB, 32,
320 ((func & 0x3) != 0));
330 pr_debug("SR: %ld %08lx %ld (%ld)\n", SR_s, SR_f, SR_e, SR_c);
332 FP_PACK_SP(vc.wp + 1, SR);
336 FP_CMP_S(IR, SA, SB, 3);
337 if (IR == 3 && (FP_ISSIGNAN_S(SA) || FP_ISSIGNAN_S(SB)))
338 FP_SET_EXCEPTION(FP_EX_INVALID);
348 FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DR);
353 FP_UNPACK_DP(DA, va.dp);
355 FP_UNPACK_DP(DB, vb.dp);
358 FP_UNPACK_DP(DA, va.dp);
362 pr_debug("DA: %ld %08lx %08lx %ld (%ld)\n",
363 DA_s, DA_f1, DA_f0, DA_e, DA_c);
364 pr_debug("DB: %ld %08lx %08lx %ld (%ld)\n",
365 DB_s, DB_f1, DB_f0, DB_e, DB_c);
369 vc.dp[0] = va.dp[0] & ~SIGN_BIT_D;
373 vc.dp[0] = va.dp[0] | SIGN_BIT_D;
377 vc.dp[0] = va.dp[0] ^ SIGN_BIT_D;
381 FP_ADD_D(DR, DA, DB);
385 FP_SUB_D(DR, DA, DB);
389 FP_MUL_D(DR, DA, DB);
393 FP_DIV_D(DR, DA, DB);
410 if (DB_c == FP_CLS_NAN) {
412 FP_SET_EXCEPTION(FP_EX_INVALID);
414 DB_e += (func == EFDCTSF ? 31 : 32);
415 FP_TO_INT_ROUND_D(vc.wp[1], DB, 32,
423 FP_UNPACK_SP(SB, vb.wp + 1);
425 pr_debug("SB: %ld %08lx %ld (%ld)\n",
426 SB_s, SB_f, SB_e, SB_c);
428 FP_CONV(D, S, 2, 1, DR, SB);
434 if (DB_c == FP_CLS_NAN) {
436 FP_SET_EXCEPTION(FP_EX_INVALID);
438 FP_TO_INT_D(vc.dp[0], DB, 64,
439 ((func & 0x1) == 0));
445 if (DB_c == FP_CLS_NAN) {
447 FP_SET_EXCEPTION(FP_EX_INVALID);
449 FP_TO_INT_ROUND_D(vc.wp[1], DB, 32,
450 ((func & 0x3) != 0));
456 if (DB_c == FP_CLS_NAN) {
458 FP_SET_EXCEPTION(FP_EX_INVALID);
460 FP_TO_INT_D(vc.wp[1], DB, 32,
461 ((func & 0x3) != 0));
471 pr_debug("DR: %ld %08lx %08lx %ld (%ld)\n",
472 DR_s, DR_f1, DR_f0, DR_e, DR_c);
474 FP_PACK_DP(vc.dp, DR);
478 FP_CMP_D(IR, DA, DB, 3);
479 if (IR == 3 && (FP_ISSIGNAN_D(DA) || FP_ISSIGNAN_D(DB)))
480 FP_SET_EXCEPTION(FP_EX_INVALID);
491 FP_DECL_S(SA0); FP_DECL_S(SB0); FP_DECL_S(SR0);
492 FP_DECL_S(SA1); FP_DECL_S(SB1); FP_DECL_S(SR1);
498 FP_UNPACK_SP(SA0, va.wp);
499 FP_UNPACK_SP(SA1, va.wp + 1);
501 FP_UNPACK_SP(SB0, vb.wp);
502 FP_UNPACK_SP(SB1, vb.wp + 1);
505 FP_UNPACK_SP(SA0, va.wp);
506 FP_UNPACK_SP(SA1, va.wp + 1);
510 pr_debug("SA0: %ld %08lx %ld (%ld)\n",
511 SA0_s, SA0_f, SA0_e, SA0_c);
512 pr_debug("SA1: %ld %08lx %ld (%ld)\n",
513 SA1_s, SA1_f, SA1_e, SA1_c);
514 pr_debug("SB0: %ld %08lx %ld (%ld)\n",
515 SB0_s, SB0_f, SB0_e, SB0_c);
516 pr_debug("SB1: %ld %08lx %ld (%ld)\n",
517 SB1_s, SB1_f, SB1_e, SB1_c);
521 vc.wp[0] = va.wp[0] & ~SIGN_BIT_S;
522 vc.wp[1] = va.wp[1] & ~SIGN_BIT_S;
526 vc.wp[0] = va.wp[0] | SIGN_BIT_S;
527 vc.wp[1] = va.wp[1] | SIGN_BIT_S;
531 vc.wp[0] = va.wp[0] ^ SIGN_BIT_S;
532 vc.wp[1] = va.wp[1] ^ SIGN_BIT_S;
536 FP_ADD_S(SR0, SA0, SB0);
537 FP_ADD_S(SR1, SA1, SB1);
541 FP_SUB_S(SR0, SA0, SB0);
542 FP_SUB_S(SR1, SA1, SB1);
546 FP_MUL_S(SR0, SA0, SB0);
547 FP_MUL_S(SR1, SA1, SB1);
551 FP_DIV_S(SR0, SA0, SB0);
552 FP_DIV_S(SR1, SA1, SB1);
569 if (SB0_c == FP_CLS_NAN) {
571 FP_SET_EXCEPTION(FP_EX_INVALID);
573 SB0_e += (func == EVFSCTSF ? 31 : 32);
574 FP_TO_INT_ROUND_S(vc.wp[0], SB0, 32,
577 if (SB1_c == FP_CLS_NAN) {
579 FP_SET_EXCEPTION(FP_EX_INVALID);
581 SB1_e += (func == EVFSCTSF ? 31 : 32);
582 FP_TO_INT_ROUND_S(vc.wp[1], SB1, 32,
589 if (SB0_c == FP_CLS_NAN) {
591 FP_SET_EXCEPTION(FP_EX_INVALID);
593 FP_TO_INT_ROUND_S(vc.wp[0], SB0, 32,
594 ((func & 0x3) != 0));
596 if (SB1_c == FP_CLS_NAN) {
598 FP_SET_EXCEPTION(FP_EX_INVALID);
600 FP_TO_INT_ROUND_S(vc.wp[1], SB1, 32,
601 ((func & 0x3) != 0));
607 if (SB0_c == FP_CLS_NAN) {
609 FP_SET_EXCEPTION(FP_EX_INVALID);
611 FP_TO_INT_S(vc.wp[0], SB0, 32,
612 ((func & 0x3) != 0));
614 if (SB1_c == FP_CLS_NAN) {
616 FP_SET_EXCEPTION(FP_EX_INVALID);
618 FP_TO_INT_S(vc.wp[1], SB1, 32,
619 ((func & 0x3) != 0));
629 pr_debug("SR0: %ld %08lx %ld (%ld)\n",
630 SR0_s, SR0_f, SR0_e, SR0_c);
631 pr_debug("SR1: %ld %08lx %ld (%ld)\n",
632 SR1_s, SR1_f, SR1_e, SR1_c);
634 FP_PACK_SP(vc.wp, SR0);
635 FP_PACK_SP(vc.wp + 1, SR1);
642 FP_CMP_S(IR0, SA0, SB0, 3);
643 FP_CMP_S(IR1, SA1, SB1, 3);
644 if (IR0 == 3 && (FP_ISSIGNAN_S(SA0) || FP_ISSIGNAN_S(SB0)))
645 FP_SET_EXCEPTION(FP_EX_INVALID);
646 if (IR1 == 3 && (FP_ISSIGNAN_S(SA1) || FP_ISSIGNAN_S(SB1)))
647 FP_SET_EXCEPTION(FP_EX_INVALID);
648 ch = (IR0 == cmp) ? 1 : 0;
649 cl = (IR1 == cmp) ? 1 : 0;
650 IR = (ch << 3) | (cl << 2) | ((ch | cl) << 1) |
660 regs->ccr &= ~(15 << ((7 - ((speinsn >> 23) & 0x7)) << 2));
661 regs->ccr |= (IR << ((7 - ((speinsn >> 23) & 0x7)) << 2));
665 * If the "invalid" exception sticky bit was set by the
666 * processor for non-finite input, but was not set before the
667 * instruction being emulated, clear it. Likewise for the
668 * "underflow" bit, which may have been set by the processor
669 * for exact underflow, not just inexact underflow when the
670 * flag should be set for IEEE 754 semantics. Other sticky
671 * exceptions will only be set by the processor when they are
672 * correct according to IEEE 754 semantics, and we must not
673 * clear sticky bits that were already set before the emulated
674 * instruction as they represent the user-visible sticky
675 * exception status. "inexact" traps to kernel are not
676 * required for IEEE semantics and are not enabled by default,
677 * so the "inexact" sticky bit may have been set by a previous
678 * instruction without the kernel being aware of it.
681 &= ~(FP_EX_INVALID | FP_EX_UNDERFLOW) | current->thread.spefscr_last;
682 __FPU_FPSCR |= (FP_CUR_EXCEPTIONS & FP_EX_MASK);
683 mtspr(SPRN_SPEFSCR, __FPU_FPSCR);
684 current->thread.spefscr_last = __FPU_FPSCR;
686 current->thread.evr[fc] = vc.wp[0];
687 regs->gpr[fc] = vc.wp[1];
689 pr_debug("ccr = %08lx\n", regs->ccr);
690 pr_debug("cur exceptions = %08x spefscr = %08lx\n",
691 FP_CUR_EXCEPTIONS, __FPU_FPSCR);
692 pr_debug("vc: %08x %08x\n", vc.wp[0], vc.wp[1]);
693 pr_debug("va: %08x %08x\n", va.wp[0], va.wp[1]);
694 pr_debug("vb: %08x %08x\n", vb.wp[0], vb.wp[1]);
696 if (current->thread.fpexc_mode & PR_FP_EXC_SW_ENABLE) {
697 if ((FP_CUR_EXCEPTIONS & FP_EX_DIVZERO)
698 && (current->thread.fpexc_mode & PR_FP_EXC_DIV))
700 if ((FP_CUR_EXCEPTIONS & FP_EX_OVERFLOW)
701 && (current->thread.fpexc_mode & PR_FP_EXC_OVF))
703 if ((FP_CUR_EXCEPTIONS & FP_EX_UNDERFLOW)
704 && (current->thread.fpexc_mode & PR_FP_EXC_UND))
706 if ((FP_CUR_EXCEPTIONS & FP_EX_INEXACT)
707 && (current->thread.fpexc_mode & PR_FP_EXC_RES))
709 if ((FP_CUR_EXCEPTIONS & FP_EX_INVALID)
710 && (current->thread.fpexc_mode & PR_FP_EXC_INV))
716 if (have_e500_cpu_a005_erratum) {
717 /* according to e500 cpu a005 erratum, reissue efp inst */
719 pr_debug("re-issue efp inst: %08lx\n", speinsn);
723 printk(KERN_ERR "\nOoops! IEEE-754 compliance handler encountered un-supported instruction.\ninst code: %08lx\n", speinsn);
727 int speround_handler(struct pt_regs *regs)
731 int lo_inexact, hi_inexact;
733 unsigned long speinsn, type, fb, fc, fptype, func;
735 if (get_user(speinsn, (unsigned int __user *) regs->nip))
737 if ((speinsn >> 26) != 4)
738 return -EINVAL; /* not an spe instruction */
740 func = speinsn & 0x7ff;
741 type = insn_type(func);
742 if (type == XCR) return -ENOSYS;
744 __FPU_FPSCR = mfspr(SPRN_SPEFSCR);
745 pr_debug("speinsn:%08lx spefscr:%08lx\n", speinsn, __FPU_FPSCR);
747 fptype = (speinsn >> 5) & 0x7;
749 /* No need to round if the result is exact */
750 lo_inexact = __FPU_FPSCR & (SPEFSCR_FG | SPEFSCR_FX);
751 hi_inexact = __FPU_FPSCR & (SPEFSCR_FGH | SPEFSCR_FXH);
752 if (!(lo_inexact || (hi_inexact && fptype == VCT)))
755 fc = (speinsn >> 21) & 0x1f;
756 s_lo = regs->gpr[fc] & SIGN_BIT_S;
757 s_hi = current->thread.evr[fc] & SIGN_BIT_S;
758 fgpr.wp[0] = current->thread.evr[fc];
759 fgpr.wp[1] = regs->gpr[fc];
761 fb = (speinsn >> 11) & 0x1f;
772 * These instructions always round to zero,
773 * independent of the rounding mode.
791 /* Recover the sign of a zero result if possible. */
793 s_lo = regs->gpr[fb] & SIGN_BIT_S;
799 /* Recover the sign of a zero result if possible. */
801 s_lo = regs->gpr[fb] & SIGN_BIT_S;
803 s_hi = current->thread.evr[fb] & SIGN_BIT_S;
810 /* Recover the sign of a zero result if possible. */
812 s_hi = current->thread.evr[fb] & SIGN_BIT_S;
820 pr_debug("round fgpr: %08x %08x\n", fgpr.wp[0], fgpr.wp[1]);
823 /* Since SPE instructions on E500 core can handle round to nearest
824 * and round toward zero with IEEE-754 complied, we just need
825 * to handle round toward +Inf and round toward -Inf by software.
828 if ((FP_ROUNDMODE) == FP_RND_PINF) {
829 if (!s_lo) fgpr.wp[1]++; /* Z > 0, choose Z1 */
830 } else { /* round to -Inf */
833 fgpr.wp[1]++; /* Z < 0, choose Z2 */
835 fgpr.wp[1]--; /* Z < 0, choose Z2 */
841 if (FP_ROUNDMODE == FP_RND_PINF) {
844 fgpr.dp[0]++; /* Z > 0, choose Z1 */
846 fgpr.wp[1]++; /* Z > 0, choose Z1 */
848 } else { /* round to -Inf */
851 fgpr.dp[0]++; /* Z < 0, choose Z2 */
853 fgpr.wp[1]--; /* Z < 0, choose Z2 */
859 if (FP_ROUNDMODE == FP_RND_PINF) {
860 if (lo_inexact && !s_lo)
861 fgpr.wp[1]++; /* Z_low > 0, choose Z1 */
862 if (hi_inexact && !s_hi)
863 fgpr.wp[0]++; /* Z_high word > 0, choose Z1 */
864 } else { /* round to -Inf */
865 if (lo_inexact && s_lo) {
867 fgpr.wp[1]++; /* Z_low < 0, choose Z2 */
869 fgpr.wp[1]--; /* Z_low < 0, choose Z2 */
871 if (hi_inexact && s_hi) {
873 fgpr.wp[0]++; /* Z_high < 0, choose Z2 */
875 fgpr.wp[0]--; /* Z_high < 0, choose Z2 */
884 current->thread.evr[fc] = fgpr.wp[0];
885 regs->gpr[fc] = fgpr.wp[1];
887 pr_debug(" to fgpr: %08x %08x\n", fgpr.wp[0], fgpr.wp[1]);
889 if (current->thread.fpexc_mode & PR_FP_EXC_SW_ENABLE)
890 return (current->thread.fpexc_mode & PR_FP_EXC_RES) ? 1 : 0;
894 int __init spe_mathemu_init(void)
898 pvr = mfspr(SPRN_PVR);
900 if ((PVR_VER(pvr) == PVR_VER_E500V1) ||
901 (PVR_VER(pvr) == PVR_VER_E500V2)) {
906 * E500 revision below 1.1, 2.3, 3.1, 4.1, 5.1
907 * need cpu a005 errata workaround
912 have_e500_cpu_a005_erratum = 1;
916 have_e500_cpu_a005_erratum = 1;
922 have_e500_cpu_a005_erratum = 1;
932 module_init(spe_mathemu_init);