arch/arm/vfp/vfpdouble.c

   1 /*
   2  *  linux/arch/arm/vfp/vfpdouble.c
   3  *
   4  * This code is derived in part from John R. Housers softfloat library, which
   5  * carries the following notice:
   6  *
   7  * ===========================================================================
   8  * This C source file is part of the SoftFloat IEC/IEEE Floating-point
   9  * Arithmetic Package, Release 2.
  10  *
  11  * Written by John R. Hauser.  This work was made possible in part by the
  12  * International Computer Science Institute, located at Suite 600, 1947 Center
  13  * Street, Berkeley, California 94704.  Funding was partially provided by the
  14  * National Science Foundation under grant MIP-9311980.  The original version
  15  * of this code was written as part of a project to build a fixed-point vector
  16  * processor in collaboration with the University of California at Berkeley,
  17  * overseen by Profs. Nelson Morgan and John Wawrzynek.  More information
  18  * is available through the web page `http://HTTP.CS.Berkeley.EDU/~jhauser/
  19  * arithmetic/softfloat.html'.
  20  *
  21  * THIS SOFTWARE IS DISTRIBUTED AS IS, FOR FREE.  Although reasonable effort
  22  * has been made to avoid it, THIS SOFTWARE MAY CONTAIN FAULTS THAT WILL AT
  23  * TIMES RESULT IN INCORRECT BEHAVIOR.  USE OF THIS SOFTWARE IS RESTRICTED TO
  24  * PERSONS AND ORGANIZATIONS WHO CAN AND WILL TAKE FULL RESPONSIBILITY FOR ANY
  25  * AND ALL LOSSES, COSTS, OR OTHER PROBLEMS ARISING FROM ITS USE.
  26  *
  27  * Derivative works are acceptable, even for commercial purposes, so long as
  28  * (1) they include prominent notice that the work is derivative, and (2) they
  29  * include prominent notice akin to these three paragraphs for those parts of
  30  * this code that are retained.
  31  * ===========================================================================
  32  */
  33 #include <linux/kernel.h>
  34 #include <linux/bitops.h>
  35
  36 #include <asm/div64.h>
  37 #include <asm/vfp.h>
  38
  39 #include "vfpinstr.h"
  40 #include "vfp.h"
  41
  42 static struct vfp_double vfp_double_default_qnan = {
  43         .exponent       = 2047,
  44         .sign           = 0,
  45         .significand    = VFP_DOUBLE_SIGNIFICAND_QNAN,
  46 };
  47
  48 static void vfp_double_dump(const char *str, struct vfp_double *d)
  49 {
  50         pr_debug("VFP: %s: sign=%d exponent=%d significand=%016llx\n",
  51                  str, d->sign != 0, d->exponent, d->significand);
  52 }
  53
  54 static void vfp_double_normalise_denormal(struct vfp_double *vd)
  55 {
  56         int bits = 31 - fls(vd->significand >> 32);
  57         if (bits == 31)
  58                 bits = 63 - fls(vd->significand);
  59
  60         vfp_double_dump("normalise_denormal: in", vd);
  61
  62         if (bits) {
  63                 vd->exponent -= bits - 1;
  64                 vd->significand <<= bits;
  65         }
  66
  67         vfp_double_dump("normalise_denormal: out", vd);
  68 }
  69
  70 u32 vfp_double_normaliseround(int dd, struct vfp_double *vd, u32 fpscr, u32 exceptions, const char *func)
  71 {
  72         u64 significand, incr;
  73         int exponent, shift, underflow;
  74         u32 rmode;
  75
  76         vfp_double_dump("pack: in", vd);
  77
  78         /*
  79          * Infinities and NaNs are a special case.
  80          */
  81         if (vd->exponent == 2047 && (vd->significand == 0 || exceptions))
  82                 goto pack;
  83
  84         /*
  85          * Special-case zero.
  86          */
  87         if (vd->significand == 0) {
  88                 vd->exponent = 0;
  89                 goto pack;
  90         }
  91
  92         exponent = vd->exponent;
  93         significand = vd->significand;
  94
  95         shift = 32 - fls(significand >> 32);
  96         if (shift == 32)
  97                 shift = 64 - fls(significand);
  98         if (shift) {
  99                 exponent -= shift;
 100                 significand <<= shift;
 101         }
 102
 103 #ifdef DEBUG
 104         vd->exponent = exponent;
 105         vd->significand = significand;
 106         vfp_double_dump("pack: normalised", vd);
 107 #endif
 108
 109         /*
 110          * Tiny number?
 111          */
 112         underflow = exponent < 0;
 113         if (underflow) {
 114                 significand = vfp_shiftright64jamming(significand, -exponent);
 115                 exponent = 0;
 116 #ifdef DEBUG
 117                 vd->exponent = exponent;
 118                 vd->significand = significand;
 119                 vfp_double_dump("pack: tiny number", vd);
 120 #endif
 121                 if (!(significand & ((1ULL << (VFP_DOUBLE_LOW_BITS + 1)) - 1)))
 122                         underflow = 0;
 123         }
 124
 125         /*
 126          * Select rounding increment.
 127          */
 128         incr = 0;
 129         rmode = fpscr & FPSCR_RMODE_MASK;
 130
 131         if (rmode == FPSCR_ROUND_NEAREST) {
 132                 incr = 1ULL << VFP_DOUBLE_LOW_BITS;
 133                 if ((significand & (1ULL << (VFP_DOUBLE_LOW_BITS + 1))) == 0)
 134                         incr -= 1;
 135         } else if (rmode == FPSCR_ROUND_TOZERO) {
 136                 incr = 0;
 137         } else if ((rmode == FPSCR_ROUND_PLUSINF) ^ (vd->sign != 0))
 138                 incr = (1ULL << (VFP_DOUBLE_LOW_BITS + 1)) - 1;
 139
 140         pr_debug("VFP: rounding increment = 0x%08llx\n", incr);
 141
 142         /*
 143          * Is our rounding going to overflow?
 144          */
 145         if ((significand + incr) < significand) {
 146                 exponent += 1;
 147                 significand = (significand >> 1) | (significand & 1);
 148                 incr >>= 1;
 149 #ifdef DEBUG
 150                 vd->exponent = exponent;
 151                 vd->significand = significand;
 152                 vfp_double_dump("pack: overflow", vd);
 153 #endif
 154         }
 155
 156         /*
 157          * If any of the low bits (which will be shifted out of the
 158          * number) are non-zero, the result is inexact.
 159          */
 160         if (significand & ((1 << (VFP_DOUBLE_LOW_BITS + 1)) - 1))
 161                 exceptions |= FPSCR_IXC;
 162
 163         /*
 164          * Do our rounding.
 165          */
 166         significand += incr;
 167
 168         /*
 169          * Infinity?
 170          */
 171         if (exponent >= 2046) {
 172                 exceptions |= FPSCR_OFC | FPSCR_IXC;
 173                 if (incr == 0) {
 174                         vd->exponent = 2045;
 175                         vd->significand = 0x7fffffffffffffffULL;
 176                 } else {
 177                         vd->exponent = 2047;            /* infinity */
 178                         vd->significand = 0;
 179                 }
 180         } else {
 181                 if (significand >> (VFP_DOUBLE_LOW_BITS + 1) == 0)
 182                         exponent = 0;
 183                 if (exponent || significand > 0x8000000000000000ULL)
 184                         underflow = 0;
 185                 if (underflow)
 186                         exceptions |= FPSCR_UFC;
 187                 vd->exponent = exponent;
 188                 vd->significand = significand >> 1;
 189         }
 190
 191  pack:
 192         vfp_double_dump("pack: final", vd);
 193         {
 194                 s64 d = vfp_double_pack(vd);
 195                 pr_debug("VFP: %s: d(d%d)=%016llx exceptions=%08x\n", func,
 196                          dd, d, exceptions);
 197                 vfp_put_double(d, dd);
 198         }
 199         return exceptions;
 200 }
 201
 202 /*
 203  * Propagate the NaN, setting exceptions if it is signalling.
 204  * 'n' is always a NaN.  'm' may be a number, NaN or infinity.
 205  */
 206 static u32
 207 vfp_propagate_nan(struct vfp_double *vdd, struct vfp_double *vdn,
 208                   struct vfp_double *vdm, u32 fpscr)
 209 {
 210         struct vfp_double *nan;
 211         int tn, tm = 0;
 212
 213         tn = vfp_double_type(vdn);
 214
 215         if (vdm)
 216                 tm = vfp_double_type(vdm);
 217
 218         if (fpscr & FPSCR_DEFAULT_NAN)
 219                 /*
 220                  * Default NaN mode - always returns a quiet NaN
 221                  */
 222                 nan = &vfp_double_default_qnan;
 223         else {
 224                 /*
 225                  * Contemporary mode - select the first signalling
 226                  * NAN, or if neither are signalling, the first
 227                  * quiet NAN.
 228                  */
 229                 if (tn == VFP_SNAN || (tm != VFP_SNAN && tn == VFP_QNAN))
 230                         nan = vdn;
 231                 else
 232                         nan = vdm;
 233                 /*
 234                  * Make the NaN quiet.
 235                  */
 236                 nan->significand |= VFP_DOUBLE_SIGNIFICAND_QNAN;
 237         }
 238
 239         *vdd = *nan;
 240
 241         /*
 242          * If one was a signalling NAN, raise invalid operation.
 243          */
 244         return tn == VFP_SNAN || tm == VFP_SNAN ? FPSCR_IOC : VFP_NAN_FLAG;
 245 }
 246
 247 /*
 248  * Extended operations
 249  */
 250 static u32 vfp_double_fabs(int dd, int unused, int dm, u32 fpscr)
 251 {
 252         vfp_put_double(vfp_double_packed_abs(vfp_get_double(dm)), dd);
 253         return 0;
 254 }
 255
 256 static u32 vfp_double_fcpy(int dd, int unused, int dm, u32 fpscr)
 257 {
 258         vfp_put_double(vfp_get_double(dm), dd);
 259         return 0;
 260 }
 261
 262 static u32 vfp_double_fneg(int dd, int unused, int dm, u32 fpscr)
 263 {
 264         vfp_put_double(vfp_double_packed_negate(vfp_get_double(dm)), dd);
 265         return 0;
 266 }
 267
 268 static u32 vfp_double_fsqrt(int dd, int unused, int dm, u32 fpscr)
 269 {
 270         struct vfp_double vdm, vdd;
 271         int ret, tm;
 272
 273         vfp_double_unpack(&vdm, vfp_get_double(dm));
 274         tm = vfp_double_type(&vdm);
 275         if (tm & (VFP_NAN|VFP_INFINITY)) {
 276                 struct vfp_double *vdp = &vdd;
 277
 278                 if (tm & VFP_NAN)
 279                         ret = vfp_propagate_nan(vdp, &vdm, NULL, fpscr);
 280                 else if (vdm.sign == 0) {
 281  sqrt_copy:
 282                         vdp = &vdm;
 283                         ret = 0;
 284                 } else {
 285  sqrt_invalid:
 286                         vdp = &vfp_double_default_qnan;
 287                         ret = FPSCR_IOC;
 288                 }
 289                 vfp_put_double(vfp_double_pack(vdp), dd);
 290                 return ret;
 291         }
 292
 293         /*
 294          * sqrt(+/- 0) == +/- 0
 295          */
 296         if (tm & VFP_ZERO)
 297                 goto sqrt_copy;
 298
 299         /*
 300          * Normalise a denormalised number
 301          */
 302         if (tm & VFP_DENORMAL)
 303                 vfp_double_normalise_denormal(&vdm);
 304
 305         /*
 306          * sqrt(<0) = invalid
 307          */
 308         if (vdm.sign)
 309                 goto sqrt_invalid;
 310
 311         vfp_double_dump("sqrt", &vdm);
 312
 313         /*
 314          * Estimate the square root.
 315          */
 316         vdd.sign = 0;
 317         vdd.exponent = ((vdm.exponent - 1023) >> 1) + 1023;
 318         vdd.significand = (u64)vfp_estimate_sqrt_significand(vdm.exponent, vdm.significand >> 32) << 31;
 319
 320         vfp_double_dump("sqrt estimate1", &vdd);
 321
 322         vdm.significand >>= 1 + (vdm.exponent & 1);
 323         vdd.significand += 2 + vfp_estimate_div128to64(vdm.significand, 0, vdd.significand);
 324
 325         vfp_double_dump("sqrt estimate2", &vdd);
 326
 327         /*
 328          * And now adjust.
 329          */
 330         if ((vdd.significand & VFP_DOUBLE_LOW_BITS_MASK) <= 5) {
 331                 if (vdd.significand < 2) {
 332                         vdd.significand = ~0ULL;
 333                 } else {
 334                         u64 termh, terml, remh, reml;
 335                         vdm.significand <<= 2;
 336                         mul64to128(&termh, &terml, vdd.significand, vdd.significand);
 337                         sub128(&remh, &reml, vdm.significand, 0, termh, terml);
 338                         while ((s64)remh < 0) {
 339                                 vdd.significand -= 1;
 340                                 shift64left(&termh, &terml, vdd.significand);
 341                                 terml |= 1;
 342                                 add128(&remh, &reml, remh, reml, termh, terml);
 343                         }
 344                         vdd.significand |= (remh | reml) != 0;
 345                 }
 346         }
 347         vdd.significand = vfp_shiftright64jamming(vdd.significand, 1);
 348
 349         return vfp_double_normaliseround(dd, &vdd, fpscr, 0, "fsqrt");
 350 }
 351
 352 /*
 353  * Equal        := ZC
 354  * Less than    := N
 355  * Greater than := C
 356  * Unordered    := CV
 357  */
 358 static u32 vfp_compare(int dd, int signal_on_qnan, int dm, u32 fpscr)
 359 {
 360         s64 d, m;
 361         u32 ret = 0;
 362
 363         m = vfp_get_double(dm);
 364         if (vfp_double_packed_exponent(m) == 2047 && vfp_double_packed_mantissa(m)) {
 365                 ret |= FPSCR_C | FPSCR_V;
 366                 if (signal_on_qnan || !(vfp_double_packed_mantissa(m) & (1ULL << (VFP_DOUBLE_MANTISSA_BITS - 1))))
 367                         /*
 368                          * Signalling NaN, or signalling on quiet NaN
 369                          */
 370                         ret |= FPSCR_IOC;
 371         }
 372
 373         d = vfp_get_double(dd);
 374         if (vfp_double_packed_exponent(d) == 2047 && vfp_double_packed_mantissa(d)) {
 375                 ret |= FPSCR_C | FPSCR_V;
 376                 if (signal_on_qnan || !(vfp_double_packed_mantissa(d) & (1ULL << (VFP_DOUBLE_MANTISSA_BITS - 1))))
 377                         /*
 378                          * Signalling NaN, or signalling on quiet NaN
 379                          */
 380                         ret |= FPSCR_IOC;
 381         }
 382
 383         if (ret == 0) {
 384                 if (d == m || vfp_double_packed_abs(d | m) == 0) {
 385                         /*
 386                          * equal
 387                          */
 388                         ret |= FPSCR_Z | FPSCR_C;
 389                 } else if (vfp_double_packed_sign(d ^ m)) {
 390                         /*
 391                          * different signs
 392                          */
 393                         if (vfp_double_packed_sign(d))
 394                                 /*
 395                                  * d is negative, so d < m
 396                                  */
 397                                 ret |= FPSCR_N;
 398                         else
 399                                 /*
 400                                  * d is positive, so d > m
 401                                  */
 402                                 ret |= FPSCR_C;
 403                 } else if ((vfp_double_packed_sign(d) != 0) ^ (d < m)) {
 404                         /*
 405                          * d < m
 406                          */
 407                         ret |= FPSCR_N;
 408                 } else if ((vfp_double_packed_sign(d) != 0) ^ (d > m)) {
 409                         /*
 410                          * d > m
 411                          */
 412                         ret |= FPSCR_C;
 413                 }
 414         }
 415
 416         return ret;
 417 }
 418
 419 static u32 vfp_double_fcmp(int dd, int unused, int dm, u32 fpscr)
 420 {
 421         return vfp_compare(dd, 0, dm, fpscr);
 422 }
 423
 424 static u32 vfp_double_fcmpe(int dd, int unused, int dm, u32 fpscr)
 425 {
 426         return vfp_compare(dd, 1, dm, fpscr);
 427 }
 428
 429 static u32 vfp_double_fcmpz(int dd, int unused, int dm, u32 fpscr)
 430 {
 431         return vfp_compare(dd, 0, VFP_REG_ZERO, fpscr);
 432 }
 433
 434 static u32 vfp_double_fcmpez(int dd, int unused, int dm, u32 fpscr)
 435 {
 436         return vfp_compare(dd, 1, VFP_REG_ZERO, fpscr);
 437 }
 438
 439 static u32 vfp_double_fcvts(int sd, int unused, int dm, u32 fpscr)
 440 {
 441         struct vfp_double vdm;
 442         struct vfp_single vsd;
 443         int tm;
 444         u32 exceptions = 0;
 445
 446         vfp_double_unpack(&vdm, vfp_get_double(dm));
 447
 448         tm = vfp_double_type(&vdm);
 449
 450         /*
 451          * If we have a signalling NaN, signal invalid operation.
 452          */
 453         if (tm == VFP_SNAN)
 454                 exceptions = FPSCR_IOC;
 455
 456         if (tm & VFP_DENORMAL)
 457                 vfp_double_normalise_denormal(&vdm);
 458
 459         vsd.sign = vdm.sign;
 460         vsd.significand = vfp_hi64to32jamming(vdm.significand);
 461
 462         /*
 463          * If we have an infinity or a NaN, the exponent must be 255
 464          */
 465         if (tm & (VFP_INFINITY|VFP_NAN)) {
 466                 vsd.exponent = 255;
 467                 if (tm == VFP_QNAN)
 468                         vsd.significand |= VFP_SINGLE_SIGNIFICAND_QNAN;
 469                 goto pack_nan;
 470         } else if (tm & VFP_ZERO)
 471                 vsd.exponent = 0;
 472         else
 473                 vsd.exponent = vdm.exponent - (1023 - 127);
 474
 475         return vfp_single_normaliseround(sd, &vsd, fpscr, exceptions, "fcvts");
 476
 477  pack_nan:
 478         vfp_put_float(vfp_single_pack(&vsd), sd);
 479         return exceptions;
 480 }
 481
 482 static u32 vfp_double_fuito(int dd, int unused, int dm, u32 fpscr)
 483 {
 484         struct vfp_double vdm;
 485         u32 m = vfp_get_float(dm);
 486
 487         vdm.sign = 0;
 488         vdm.exponent = 1023 + 63 - 1;
 489         vdm.significand = (u64)m;
 490
 491         return vfp_double_normaliseround(dd, &vdm, fpscr, 0, "fuito");
 492 }
 493
 494 static u32 vfp_double_fsito(int dd, int unused, int dm, u32 fpscr)
 495 {
 496         struct vfp_double vdm;
 497         u32 m = vfp_get_float(dm);
 498
 499         vdm.sign = (m & 0x80000000) >> 16;
 500         vdm.exponent = 1023 + 63 - 1;
 501         vdm.significand = vdm.sign ? -m : m;
 502
 503         return vfp_double_normaliseround(dd, &vdm, fpscr, 0, "fsito");
 504 }
 505
 506 static u32 vfp_double_ftoui(int sd, int unused, int dm, u32 fpscr)
 507 {
 508         struct vfp_double vdm;
 509         u32 d, exceptions = 0;
 510         int rmode = fpscr & FPSCR_RMODE_MASK;
 511         int tm;
 512
 513         vfp_double_unpack(&vdm, vfp_get_double(dm));
 514
 515         /*
 516          * Do we have a denormalised number?
 517          */
 518         tm = vfp_double_type(&vdm);
 519         if (tm & VFP_DENORMAL)
 520                 exceptions |= FPSCR_IDC;
 521
 522         if (tm & VFP_NAN)
 523                 vdm.sign = 0;
 524
 525         if (vdm.exponent >= 1023 + 32) {
 526                 d = vdm.sign ? 0 : 0xffffffff;
 527                 exceptions = FPSCR_IOC;
 528         } else if (vdm.exponent >= 1023 - 1) {
 529                 int shift = 1023 + 63 - vdm.exponent;
 530                 u64 rem, incr = 0;
 531
 532                 /*
 533                  * 2^0 <= m < 2^32-2^8
 534                  */
 535                 d = (vdm.significand << 1) >> shift;
 536                 rem = vdm.significand << (65 - shift);
 537
 538                 if (rmode == FPSCR_ROUND_NEAREST) {
 539                         incr = 0x8000000000000000ULL;
 540                         if ((d & 1) == 0)
 541                                 incr -= 1;
 542                 } else if (rmode == FPSCR_ROUND_TOZERO) {
 543                         incr = 0;
 544                 } else if ((rmode == FPSCR_ROUND_PLUSINF) ^ (vdm.sign != 0)) {
 545                         incr = ~0ULL;
 546                 }
 547
 548                 if ((rem + incr) < rem) {
 549                         if (d < 0xffffffff)
 550                                 d += 1;
 551                         else
 552                                 exceptions |= FPSCR_IOC;
 553                 }
 554
 555                 if (d && vdm.sign) {
 556                         d = 0;
 557                         exceptions |= FPSCR_IOC;
 558                 } else if (rem)
 559                         exceptions |= FPSCR_IXC;
 560         } else {
 561                 d = 0;
 562                 if (vdm.exponent | vdm.significand) {
 563                         exceptions |= FPSCR_IXC;
 564                         if (rmode == FPSCR_ROUND_PLUSINF && vdm.sign == 0)
 565                                 d = 1;
 566                         else if (rmode == FPSCR_ROUND_MINUSINF && vdm.sign) {
 567                                 d = 0;
 568                                 exceptions |= FPSCR_IOC;
 569                         }
 570                 }
 571         }
 572
 573         pr_debug("VFP: ftoui: d(s%d)=%08x exceptions=%08x\n", sd, d, exceptions);
 574
 575         vfp_put_float(d, sd);
 576
 577         return exceptions;
 578 }
 579
 580 static u32 vfp_double_ftouiz(int sd, int unused, int dm, u32 fpscr)
 581 {
 582         return vfp_double_ftoui(sd, unused, dm, FPSCR_ROUND_TOZERO);
 583 }
 584
 585 static u32 vfp_double_ftosi(int sd, int unused, int dm, u32 fpscr)
 586 {
 587         struct vfp_double vdm;
 588         u32 d, exceptions = 0;
 589         int rmode = fpscr & FPSCR_RMODE_MASK;
 590         int tm;
 591
 592         vfp_double_unpack(&vdm, vfp_get_double(dm));
 593         vfp_double_dump("VDM", &vdm);
 594
 595         /*
 596          * Do we have denormalised number?
 597          */
 598         tm = vfp_double_type(&vdm);
 599         if (tm & VFP_DENORMAL)
 600                 exceptions |= FPSCR_IDC;
 601
 602         if (tm & VFP_NAN) {
 603                 d = 0;
 604                 exceptions |= FPSCR_IOC;
 605         } else if (vdm.exponent >= 1023 + 32) {
 606                 d = 0x7fffffff;
 607                 if (vdm.sign)
 608                         d = ~d;
 609                 exceptions |= FPSCR_IOC;
 610         } else if (vdm.exponent >= 1023 - 1) {
 611                 int shift = 1023 + 63 - vdm.exponent;   /* 58 */
 612                 u64 rem, incr = 0;
 613
 614                 d = (vdm.significand << 1) >> shift;
 615                 rem = vdm.significand << (65 - shift);
 616
 617                 if (rmode == FPSCR_ROUND_NEAREST) {
 618                         incr = 0x8000000000000000ULL;
 619                         if ((d & 1) == 0)
 620                                 incr -= 1;
 621                 } else if (rmode == FPSCR_ROUND_TOZERO) {
 622                         incr = 0;
 623                 } else if ((rmode == FPSCR_ROUND_PLUSINF) ^ (vdm.sign != 0)) {
 624                         incr = ~0ULL;
 625                 }
 626
 627                 if ((rem + incr) < rem && d < 0xffffffff)
 628                         d += 1;
 629                 if (d > 0x7fffffff + (vdm.sign != 0)) {
 630                         d = 0x7fffffff + (vdm.sign != 0);
 631                         exceptions |= FPSCR_IOC;
 632                 } else if (rem)
 633                         exceptions |= FPSCR_IXC;
 634
 635                 if (vdm.sign)
 636                         d = -d;
 637         } else {
 638                 d = 0;
 639                 if (vdm.exponent | vdm.significand) {
 640                         exceptions |= FPSCR_IXC;
 641                         if (rmode == FPSCR_ROUND_PLUSINF && vdm.sign == 0)
 642                                 d = 1;
 643                         else if (rmode == FPSCR_ROUND_MINUSINF && vdm.sign)
 644                                 d = -1;
 645                 }
 646         }
 647
 648         pr_debug("VFP: ftosi: d(s%d)=%08x exceptions=%08x\n", sd, d, exceptions);
 649
 650         vfp_put_float((s32)d, sd);
 651
 652         return exceptions;
 653 }
 654
 655 static u32 vfp_double_ftosiz(int dd, int unused, int dm, u32 fpscr)
 656 {
 657         return vfp_double_ftosi(dd, unused, dm, FPSCR_ROUND_TOZERO);
 658 }
 659
 660
 661 static struct op fops_ext[32] = {
 662         [FEXT_TO_IDX(FEXT_FCPY)]        = { vfp_double_fcpy,   0 },
 663         [FEXT_TO_IDX(FEXT_FABS)]        = { vfp_double_fabs,   0 },
 664         [FEXT_TO_IDX(FEXT_FNEG)]        = { vfp_double_fneg,   0 },
 665         [FEXT_TO_IDX(FEXT_FSQRT)]       = { vfp_double_fsqrt,  0 },
 666         [FEXT_TO_IDX(FEXT_FCMP)]        = { vfp_double_fcmp,   OP_SCALAR },
 667         [FEXT_TO_IDX(FEXT_FCMPE)]       = { vfp_double_fcmpe,  OP_SCALAR },
 668         [FEXT_TO_IDX(FEXT_FCMPZ)]       = { vfp_double_fcmpz,  OP_SCALAR },
 669         [FEXT_TO_IDX(FEXT_FCMPEZ)]      = { vfp_double_fcmpez, OP_SCALAR },
 670         [FEXT_TO_IDX(FEXT_FCVT)]        = { vfp_double_fcvts,  OP_SCALAR|OP_SD },
 671         [FEXT_TO_IDX(FEXT_FUITO)]       = { vfp_double_fuito,  OP_SCALAR|OP_SM },
 672         [FEXT_TO_IDX(FEXT_FSITO)]       = { vfp_double_fsito,  OP_SCALAR|OP_SM },
 673         [FEXT_TO_IDX(FEXT_FTOUI)]       = { vfp_double_ftoui,  OP_SCALAR|OP_SD },
 674         [FEXT_TO_IDX(FEXT_FTOUIZ)]      = { vfp_double_ftouiz, OP_SCALAR|OP_SD },
 675         [FEXT_TO_IDX(FEXT_FTOSI)]       = { vfp_double_ftosi,  OP_SCALAR|OP_SD },
 676         [FEXT_TO_IDX(FEXT_FTOSIZ)]      = { vfp_double_ftosiz, OP_SCALAR|OP_SD },
 677 };
 678
 679
 680
 681
 682 static u32
 683 vfp_double_fadd_nonnumber(struct vfp_double *vdd, struct vfp_double *vdn,
 684                           struct vfp_double *vdm, u32 fpscr)
 685 {
 686         struct vfp_double *vdp;
 687         u32 exceptions = 0;
 688         int tn, tm;
 689
 690         tn = vfp_double_type(vdn);
 691         tm = vfp_double_type(vdm);
 692
 693         if (tn & tm & VFP_INFINITY) {
 694                 /*
 695                  * Two infinities.  Are they different signs?
 696                  */
 697                 if (vdn->sign ^ vdm->sign) {
 698                         /*
 699                          * different signs -> invalid
 700                          */
 701                         exceptions = FPSCR_IOC;
 702                         vdp = &vfp_double_default_qnan;
 703                 } else {
 704                         /*
 705                          * same signs -> valid
 706                          */
 707                         vdp = vdn;
 708                 }
 709         } else if (tn & VFP_INFINITY && tm & VFP_NUMBER) {
 710                 /*
 711                  * One infinity and one number -> infinity
 712                  */
 713                 vdp = vdn;
 714         } else {
 715                 /*
 716                  * 'n' is a NaN of some type
 717                  */
 718                 return vfp_propagate_nan(vdd, vdn, vdm, fpscr);
 719         }
 720         *vdd = *vdp;
 721         return exceptions;
 722 }
 723
 724 static u32
 725 vfp_double_add(struct vfp_double *vdd, struct vfp_double *vdn,
 726                struct vfp_double *vdm, u32 fpscr)
 727 {
 728         u32 exp_diff;
 729         u64 m_sig;
 730
 731         if (vdn->significand & (1ULL << 63) ||
 732             vdm->significand & (1ULL << 63)) {
 733                 pr_info("VFP: bad FP values in %s\n", __func__);
 734                 vfp_double_dump("VDN", vdn);
 735                 vfp_double_dump("VDM", vdm);
 736         }
 737
 738         /*
 739          * Ensure that 'n' is the largest magnitude number.  Note that
 740          * if 'n' and 'm' have equal exponents, we do not swap them.
 741          * This ensures that NaN propagation works correctly.
 742          */
 743         if (vdn->exponent < vdm->exponent) {
 744                 struct vfp_double *t = vdn;
 745                 vdn = vdm;
 746                 vdm = t;
 747         }
 748
 749         /*
 750          * Is 'n' an infinity or a NaN?  Note that 'm' may be a number,
 751          * infinity or a NaN here.
 752          */
 753         if (vdn->exponent == 2047)
 754                 return vfp_double_fadd_nonnumber(vdd, vdn, vdm, fpscr);
 755
 756         /*
 757          * We have two proper numbers, where 'vdn' is the larger magnitude.
 758          *
 759          * Copy 'n' to 'd' before doing the arithmetic.
 760          */
 761         *vdd = *vdn;
 762
 763         /*
 764          * Align 'm' with the result.
 765          */
 766         exp_diff = vdn->exponent - vdm->exponent;
 767         m_sig = vfp_shiftright64jamming(vdm->significand, exp_diff);
 768
 769         /*
 770          * If the signs are different, we are really subtracting.
 771          */
 772         if (vdn->sign ^ vdm->sign) {
 773                 m_sig = vdn->significand - m_sig;
 774                 if ((s64)m_sig < 0) {
 775                         vdd->sign = vfp_sign_negate(vdd->sign);
 776                         m_sig = -m_sig;
 777                 } else if (m_sig == 0) {
 778                         vdd->sign = (fpscr & FPSCR_RMODE_MASK) ==
 779                                       FPSCR_ROUND_MINUSINF ? 0x8000 : 0;
 780                 }
 781         } else {
 782                 m_sig += vdn->significand;
 783         }
 784         vdd->significand = m_sig;
 785
 786         return 0;
 787 }
 788
 789 static u32
 790 vfp_double_multiply(struct vfp_double *vdd, struct vfp_double *vdn,
 791                     struct vfp_double *vdm, u32 fpscr)
 792 {
 793         vfp_double_dump("VDN", vdn);
 794         vfp_double_dump("VDM", vdm);
 795
 796         /*
 797          * Ensure that 'n' is the largest magnitude number.  Note that
 798          * if 'n' and 'm' have equal exponents, we do not swap them.
 799          * This ensures that NaN propagation works correctly.
 800          */
 801         if (vdn->exponent < vdm->exponent) {
 802                 struct vfp_double *t = vdn;
 803                 vdn = vdm;
 804                 vdm = t;
 805                 pr_debug("VFP: swapping M <-> N\n");
 806         }
 807
 808         vdd->sign = vdn->sign ^ vdm->sign;
 809
 810         /*
 811          * If 'n' is an infinity or NaN, handle it.  'm' may be anything.
 812          */
 813         if (vdn->exponent == 2047) {
 814                 if (vdn->significand || (vdm->exponent == 2047 && vdm->significand))
 815                         return vfp_propagate_nan(vdd, vdn, vdm, fpscr);
 816                 if ((vdm->exponent | vdm->significand) == 0) {
 817                         *vdd = vfp_double_default_qnan;
 818                         return FPSCR_IOC;
 819                 }
 820                 vdd->exponent = vdn->exponent;
 821                 vdd->significand = 0;
 822                 return 0;
 823         }
 824
 825         /*
 826          * If 'm' is zero, the result is always zero.  In this case,
 827          * 'n' may be zero or a number, but it doesn't matter which.
 828          */
 829         if ((vdm->exponent | vdm->significand) == 0) {
 830                 vdd->exponent = 0;
 831                 vdd->significand = 0;
 832                 return 0;
 833         }
 834
 835         /*
 836          * We add 2 to the destination exponent for the same reason
 837          * as the addition case - though this time we have +1 from
 838          * each input operand.
 839          */
 840         vdd->exponent = vdn->exponent + vdm->exponent - 1023 + 2;
 841         vdd->significand = vfp_hi64multiply64(vdn->significand, vdm->significand);
 842
 843         vfp_double_dump("VDD", vdd);
 844         return 0;
 845 }
 846
 847 #define NEG_MULTIPLY    (1 << 0)
 848 #define NEG_SUBTRACT    (1 << 1)
 849
 850 static u32
 851 vfp_double_multiply_accumulate(int dd, int dn, int dm, u32 fpscr, u32 negate, char *func)
 852 {
 853         struct vfp_double vdd, vdp, vdn, vdm;
 854         u32 exceptions;
 855
 856         vfp_double_unpack(&vdn, vfp_get_double(dn));
 857         if (vdn.exponent == 0 && vdn.significand)
 858                 vfp_double_normalise_denormal(&vdn);
 859
 860         vfp_double_unpack(&vdm, vfp_get_double(dm));
 861         if (vdm.exponent == 0 && vdm.significand)
 862                 vfp_double_normalise_denormal(&vdm);
 863
 864         exceptions = vfp_double_multiply(&vdp, &vdn, &vdm, fpscr);
 865         if (negate & NEG_MULTIPLY)
 866                 vdp.sign = vfp_sign_negate(vdp.sign);
 867
 868         vfp_double_unpack(&vdn, vfp_get_double(dd));
 869         if (vdn.exponent == 0 && vdn.significand)
 870                 vfp_double_normalise_denormal(&vdn);
 871         if (negate & NEG_SUBTRACT)
 872                 vdn.sign = vfp_sign_negate(vdn.sign);
 873
 874         exceptions |= vfp_double_add(&vdd, &vdn, &vdp, fpscr);
 875
 876         return vfp_double_normaliseround(dd, &vdd, fpscr, exceptions, func);
 877 }
 878
 879 /*
 880  * Standard operations
 881  */
 882
 883 /*
 884  * sd = sd + (sn * sm)
 885  */
 886 static u32 vfp_double_fmac(int dd, int dn, int dm, u32 fpscr)
 887 {
 888         return vfp_double_multiply_accumulate(dd, dn, dm, fpscr, 0, "fmac");
 889 }
 890
 891 /*
 892  * sd = sd - (sn * sm)
 893  */
 894 static u32 vfp_double_fnmac(int dd, int dn, int dm, u32 fpscr)
 895 {
 896         return vfp_double_multiply_accumulate(dd, dn, dm, fpscr, NEG_MULTIPLY, "fnmac");
 897 }
 898
 899 /*
 900  * sd = -sd + (sn * sm)
 901  */
 902 static u32 vfp_double_fmsc(int dd, int dn, int dm, u32 fpscr)
 903 {
 904         return vfp_double_multiply_accumulate(dd, dn, dm, fpscr, NEG_SUBTRACT, "fmsc");
 905 }
 906
 907 /*
 908  * sd = -sd - (sn * sm)
 909  */
 910 static u32 vfp_double_fnmsc(int dd, int dn, int dm, u32 fpscr)
 911 {
 912         return vfp_double_multiply_accumulate(dd, dn, dm, fpscr, NEG_SUBTRACT | NEG_MULTIPLY, "fnmsc");
 913 }
 914
 915 /*
 916  * sd = sn * sm
 917  */
 918 static u32 vfp_double_fmul(int dd, int dn, int dm, u32 fpscr)
 919 {
 920         struct vfp_double vdd, vdn, vdm;
 921         u32 exceptions;
 922
 923         vfp_double_unpack(&vdn, vfp_get_double(dn));
 924         if (vdn.exponent == 0 && vdn.significand)
 925                 vfp_double_normalise_denormal(&vdn);
 926
 927         vfp_double_unpack(&vdm, vfp_get_double(dm));
 928         if (vdm.exponent == 0 && vdm.significand)
 929                 vfp_double_normalise_denormal(&vdm);
 930
 931         exceptions = vfp_double_multiply(&vdd, &vdn, &vdm, fpscr);
 932         return vfp_double_normaliseround(dd, &vdd, fpscr, exceptions, "fmul");
 933 }
 934
 935 /*
 936  * sd = -(sn * sm)
 937  */
 938 static u32 vfp_double_fnmul(int dd, int dn, int dm, u32 fpscr)
 939 {
 940         struct vfp_double vdd, vdn, vdm;
 941         u32 exceptions;
 942
 943         vfp_double_unpack(&vdn, vfp_get_double(dn));
 944         if (vdn.exponent == 0 && vdn.significand)
 945                 vfp_double_normalise_denormal(&vdn);
 946
 947         vfp_double_unpack(&vdm, vfp_get_double(dm));
 948         if (vdm.exponent == 0 && vdm.significand)
 949                 vfp_double_normalise_denormal(&vdm);
 950
 951         exceptions = vfp_double_multiply(&vdd, &vdn, &vdm, fpscr);
 952         vdd.sign = vfp_sign_negate(vdd.sign);
 953
 954         return vfp_double_normaliseround(dd, &vdd, fpscr, exceptions, "fnmul");
 955 }
 956
 957 /*
 958  * sd = sn + sm
 959  */
 960 static u32 vfp_double_fadd(int dd, int dn, int dm, u32 fpscr)
 961 {
 962         struct vfp_double vdd, vdn, vdm;
 963         u32 exceptions;
 964
 965         vfp_double_unpack(&vdn, vfp_get_double(dn));
 966         if (vdn.exponent == 0 && vdn.significand)
 967                 vfp_double_normalise_denormal(&vdn);
 968
 969         vfp_double_unpack(&vdm, vfp_get_double(dm));
 970         if (vdm.exponent == 0 && vdm.significand)
 971                 vfp_double_normalise_denormal(&vdm);
 972
 973         exceptions = vfp_double_add(&vdd, &vdn, &vdm, fpscr);
 974
 975         return vfp_double_normaliseround(dd, &vdd, fpscr, exceptions, "fadd");
 976 }
 977
 978 /*
 979  * sd = sn - sm
 980  */
 981 static u32 vfp_double_fsub(int dd, int dn, int dm, u32 fpscr)
 982 {
 983         struct vfp_double vdd, vdn, vdm;
 984         u32 exceptions;
 985
 986         vfp_double_unpack(&vdn, vfp_get_double(dn));
 987         if (vdn.exponent == 0 && vdn.significand)
 988                 vfp_double_normalise_denormal(&vdn);
 989
 990         vfp_double_unpack(&vdm, vfp_get_double(dm));
 991         if (vdm.exponent == 0 && vdm.significand)
 992                 vfp_double_normalise_denormal(&vdm);
 993
 994         /*
 995          * Subtraction is like addition, but with a negated operand.
 996          */
 997         vdm.sign = vfp_sign_negate(vdm.sign);
 998
 999         exceptions = vfp_double_add(&vdd, &vdn, &vdm, fpscr);
1000
1001         return vfp_double_normaliseround(dd, &vdd, fpscr, exceptions, "fsub");
1002 }
1003
1004 /*
1005  * sd = sn / sm
1006  */
1007 static u32 vfp_double_fdiv(int dd, int dn, int dm, u32 fpscr)
1008 {
1009         struct vfp_double vdd, vdn, vdm;
1010         u32 exceptions = 0;
1011         int tm, tn;
1012
1013         vfp_double_unpack(&vdn, vfp_get_double(dn));
1014         vfp_double_unpack(&vdm, vfp_get_double(dm));
1015
1016         vdd.sign = vdn.sign ^ vdm.sign;
1017
1018         tn = vfp_double_type(&vdn);
1019         tm = vfp_double_type(&vdm);
1020
1021         /*
1022          * Is n a NAN?
1023          */
1024         if (tn & VFP_NAN)
1025                 goto vdn_nan;
1026
1027         /*
1028          * Is m a NAN?
1029          */
1030         if (tm & VFP_NAN)
1031                 goto vdm_nan;
1032
1033         /*
1034          * If n and m are infinity, the result is invalid
1035          * If n and m are zero, the result is invalid
1036          */
1037         if (tm & tn & (VFP_INFINITY|VFP_ZERO))
1038                 goto invalid;
1039
1040         /*
1041          * If n is infinity, the result is infinity
1042          */
1043         if (tn & VFP_INFINITY)
1044                 goto infinity;
1045
1046         /*
1047          * If m is zero, raise div0 exceptions
1048          */
1049         if (tm & VFP_ZERO)
1050                 goto divzero;
1051
1052         /*
1053          * If m is infinity, or n is zero, the result is zero
1054          */
1055         if (tm & VFP_INFINITY || tn & VFP_ZERO)
1056                 goto zero;
1057
1058         if (tn & VFP_DENORMAL)
1059                 vfp_double_normalise_denormal(&vdn);
1060         if (tm & VFP_DENORMAL)
1061                 vfp_double_normalise_denormal(&vdm);
1062
1063         /*
1064          * Ok, we have two numbers, we can perform division.
1065          */
1066         vdd.exponent = vdn.exponent - vdm.exponent + 1023 - 1;
1067         vdm.significand <<= 1;
1068         if (vdm.significand <= (2 * vdn.significand)) {
1069                 vdn.significand >>= 1;
1070                 vdd.exponent++;
1071         }
1072         vdd.significand = vfp_estimate_div128to64(vdn.significand, 0, vdm.significand);
1073         if ((vdd.significand & 0x1ff) <= 2) {
1074                 u64 termh, terml, remh, reml;
1075                 mul64to128(&termh, &terml, vdm.significand, vdd.significand);
1076                 sub128(&remh, &reml, vdn.significand, 0, termh, terml);
1077                 while ((s64)remh < 0) {
1078                         vdd.significand -= 1;
1079                         add128(&remh, &reml, remh, reml, 0, vdm.significand);
1080                 }
1081                 vdd.significand |= (reml != 0);
1082         }
1083         return vfp_double_normaliseround(dd, &vdd, fpscr, 0, "fdiv");
1084
1085  vdn_nan:
1086         exceptions = vfp_propagate_nan(&vdd, &vdn, &vdm, fpscr);
1087  pack:
1088         vfp_put_double(vfp_double_pack(&vdd), dd);
1089         return exceptions;
1090
1091  vdm_nan:
1092         exceptions = vfp_propagate_nan(&vdd, &vdm, &vdn, fpscr);
1093         goto pack;
1094
1095  zero:
1096         vdd.exponent = 0;
1097         vdd.significand = 0;
1098         goto pack;
1099
1100  divzero:
1101         exceptions = FPSCR_DZC;
1102  infinity:
1103         vdd.exponent = 2047;
1104         vdd.significand = 0;
1105         goto pack;
1106
1107  invalid:
1108         vfp_put_double(vfp_double_pack(&vfp_double_default_qnan), dd);
1109         return FPSCR_IOC;
1110 }
1111
1112 static struct op fops[16] = {
1113         [FOP_TO_IDX(FOP_FMAC)]  = { vfp_double_fmac,  0 },
1114         [FOP_TO_IDX(FOP_FNMAC)] = { vfp_double_fnmac, 0 },
1115         [FOP_TO_IDX(FOP_FMSC)]  = { vfp_double_fmsc,  0 },
1116         [FOP_TO_IDX(FOP_FNMSC)] = { vfp_double_fnmsc, 0 },
1117         [FOP_TO_IDX(FOP_FMUL)]  = { vfp_double_fmul,  0 },
1118         [FOP_TO_IDX(FOP_FNMUL)] = { vfp_double_fnmul, 0 },
1119         [FOP_TO_IDX(FOP_FADD)]  = { vfp_double_fadd,  0 },
1120         [FOP_TO_IDX(FOP_FSUB)]  = { vfp_double_fsub,  0 },
1121         [FOP_TO_IDX(FOP_FDIV)]  = { vfp_double_fdiv,  0 },
1122 };
1123
1124 #define FREG_BANK(x)    ((x) & 0x0c)
1125 #define FREG_IDX(x)     ((x) & 3)
1126
1127 u32 vfp_double_cpdo(u32 inst, u32 fpscr)
1128 {
1129         u32 op = inst & FOP_MASK;
1130         u32 exceptions = 0;
1131         unsigned int dest;
1132         unsigned int dn = vfp_get_dn(inst);
1133         unsigned int dm;
1134         unsigned int vecitr, veclen, vecstride;
1135         struct op *fop;
1136
1137         vecstride = (1 + ((fpscr & FPSCR_STRIDE_MASK) == FPSCR_STRIDE_MASK));
1138
1139         fop = (op == FOP_EXT) ? &fops_ext[FEXT_TO_IDX(inst)] : &fops[FOP_TO_IDX(op)];
1140
1141         /*
1142          * fcvtds takes an sN register number as destination, not dN.
1143          * It also always operates on scalars.
1144          */
1145         if (fop->flags & OP_SD)
1146                 dest = vfp_get_sd(inst);
1147         else
1148                 dest = vfp_get_dd(inst);
1149
1150         /*
1151          * f[us]ito takes a sN operand, not a dN operand.
1152          */
1153         if (fop->flags & OP_SM)
1154                 dm = vfp_get_sm(inst);
1155         else
1156                 dm = vfp_get_dm(inst);
1157
1158         /*
1159          * If destination bank is zero, vector length is always '1'.
1160          * ARM DDI0100F C5.1.3, C5.3.2.
1161          */
1162         if ((fop->flags & OP_SCALAR) || (FREG_BANK(dest) == 0))
1163                 veclen = 0;
1164         else
1165                 veclen = fpscr & FPSCR_LENGTH_MASK;
1166
1167         pr_debug("VFP: vecstride=%u veclen=%u\n", vecstride,
1168                  (veclen >> FPSCR_LENGTH_BIT) + 1);
1169
1170         if (!fop->fn)
1171                 goto invalid;
1172
1173         for (vecitr = 0; vecitr <= veclen; vecitr += 1 << FPSCR_LENGTH_BIT) {
1174                 u32 except;
1175                 char type;
1176
1177                 type = fop->flags & OP_SD ? 's' : 'd';
1178                 if (op == FOP_EXT)
1179                         pr_debug("VFP: itr%d (%c%u) = op[%u] (d%u)\n",
1180                                  vecitr >> FPSCR_LENGTH_BIT,
1181                                  type, dest, dn, dm);
1182                 else
1183                         pr_debug("VFP: itr%d (%c%u) = (d%u) op[%u] (d%u)\n",
1184                                  vecitr >> FPSCR_LENGTH_BIT,
1185                                  type, dest, dn, FOP_TO_IDX(op), dm);
1186
1187                 except = fop->fn(dest, dn, dm, fpscr);
1188                 pr_debug("VFP: itr%d: exceptions=%08x\n",
1189                          vecitr >> FPSCR_LENGTH_BIT, except);
1190
1191                 exceptions |= except;
1192
1193                 /*
1194                  * CHECK: It appears to be undefined whether we stop when
1195                  * we encounter an exception.  We continue.
1196                  */
1197                 dest = FREG_BANK(dest) + ((FREG_IDX(dest) + vecstride) & 3);
1198                 dn = FREG_BANK(dn) + ((FREG_IDX(dn) + vecstride) & 3);
1199                 if (FREG_BANK(dm) != 0)
1200                         dm = FREG_BANK(dm) + ((FREG_IDX(dm) + vecstride) & 3);
1201         }
1202         return exceptions;
1203
1204  invalid:
1205         return ~0;
1206 }