arch/x86/math-emu/reg_ld_str.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*---------------------------------------------------------------------------+
   3  |  reg_ld_str.c                                                             |
   4  |                                                                           |
   5  | All of the functions which transfer data between user memory and FPU_REGs.|
   6  |                                                                           |
   7  | Copyright (C) 1992,1993,1994,1996,1997                                    |
   8  |                  W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
   9  |                  E-mail   billm@suburbia.net                              |
  10  |                                                                           |
  11  |                                                                           |
  12  +---------------------------------------------------------------------------*/
  13
  14 /*---------------------------------------------------------------------------+
  15  | Note:                                                                     |
  16  |    The file contains code which accesses user memory.                     |
  17  |    Emulator static data may change when user memory is accessed, due to   |
  18  |    other processes using the emulator while swapping is in progress.      |
  19  +---------------------------------------------------------------------------*/
  20
  21 #include "fpu_emu.h"
  22
  23 #include <linux/uaccess.h>
  24
  25 #include "fpu_system.h"
  26 #include "exception.h"
  27 #include "reg_constant.h"
  28 #include "control_w.h"
  29 #include "status_w.h"
  30
  31 #define DOUBLE_Emax 1023        /* largest valid exponent */
  32 #define DOUBLE_Ebias 1023
  33 #define DOUBLE_Emin (-1022)     /* smallest valid exponent */
  34
  35 #define SINGLE_Emax 127         /* largest valid exponent */
  36 #define SINGLE_Ebias 127
  37 #define SINGLE_Emin (-126)      /* smallest valid exponent */
  38
  39 static u_char normalize_no_excep(FPU_REG *r, int exp, int sign)
  40 {
  41         u_char tag;
  42
  43         setexponent16(r, exp);
  44
  45         tag = FPU_normalize_nuo(r);
  46         stdexp(r);
  47         if (sign)
  48                 setnegative(r);
  49
  50         return tag;
  51 }
  52
  53 int FPU_tagof(FPU_REG *ptr)
  54 {
  55         int exp;
  56
  57         exp = exponent16(ptr) & 0x7fff;
  58         if (exp == 0) {
  59                 if (!(ptr->sigh | ptr->sigl)) {
  60                         return TAG_Zero;
  61                 }
  62                 /* The number is a de-normal or pseudodenormal. */
  63                 return TAG_Special;
  64         }
  65
  66         if (exp == 0x7fff) {
  67                 /* Is an Infinity, a NaN, or an unsupported data type. */
  68                 return TAG_Special;
  69         }
  70
  71         if (!(ptr->sigh & 0x80000000)) {
  72                 /* Unsupported data type. */
  73                 /* Valid numbers have the ms bit set to 1. */
  74                 /* Unnormal. */
  75                 return TAG_Special;
  76         }
  77
  78         return TAG_Valid;
  79 }
  80
  81 /* Get a long double from user memory */
  82 int FPU_load_extended(long double __user *s, int stnr)
  83 {
  84         FPU_REG *sti_ptr = &st(stnr);
  85
  86         RE_ENTRANT_CHECK_OFF;
  87         FPU_access_ok(VERIFY_READ, s, 10);
  88         __copy_from_user(sti_ptr, s, 10);
  89         RE_ENTRANT_CHECK_ON;
  90
  91         return FPU_tagof(sti_ptr);
  92 }
  93
  94 /* Get a double from user memory */
  95 int FPU_load_double(double __user *dfloat, FPU_REG *loaded_data)
  96 {
  97         int exp, tag, negative;
  98         unsigned m64, l64;
  99
 100         RE_ENTRANT_CHECK_OFF;
 101         FPU_access_ok(VERIFY_READ, dfloat, 8);
 102         FPU_get_user(m64, 1 + (unsigned long __user *)dfloat);
 103         FPU_get_user(l64, (unsigned long __user *)dfloat);
 104         RE_ENTRANT_CHECK_ON;
 105
 106         negative = (m64 & 0x80000000) ? SIGN_Negative : SIGN_Positive;
 107         exp = ((m64 & 0x7ff00000) >> 20) - DOUBLE_Ebias + EXTENDED_Ebias;
 108         m64 &= 0xfffff;
 109         if (exp > DOUBLE_Emax + EXTENDED_Ebias) {
 110                 /* Infinity or NaN */
 111                 if ((m64 == 0) && (l64 == 0)) {
 112                         /* +- infinity */
 113                         loaded_data->sigh = 0x80000000;
 114                         loaded_data->sigl = 0x00000000;
 115                         exp = EXP_Infinity + EXTENDED_Ebias;
 116                         tag = TAG_Special;
 117                 } else {
 118                         /* Must be a signaling or quiet NaN */
 119                         exp = EXP_NaN + EXTENDED_Ebias;
 120                         loaded_data->sigh = (m64 << 11) | 0x80000000;
 121                         loaded_data->sigh |= l64 >> 21;
 122                         loaded_data->sigl = l64 << 11;
 123                         tag = TAG_Special;      /* The calling function must look for NaNs */
 124                 }
 125         } else if (exp < DOUBLE_Emin + EXTENDED_Ebias) {
 126                 /* Zero or de-normal */
 127                 if ((m64 == 0) && (l64 == 0)) {
 128                         /* Zero */
 129                         reg_copy(&CONST_Z, loaded_data);
 130                         exp = 0;
 131                         tag = TAG_Zero;
 132                 } else {
 133                         /* De-normal */
 134                         loaded_data->sigh = m64 << 11;
 135                         loaded_data->sigh |= l64 >> 21;
 136                         loaded_data->sigl = l64 << 11;
 137
 138                         return normalize_no_excep(loaded_data, DOUBLE_Emin,
 139                                                   negative)
 140                             | (denormal_operand() < 0 ? FPU_Exception : 0);
 141                 }
 142         } else {
 143                 loaded_data->sigh = (m64 << 11) | 0x80000000;
 144                 loaded_data->sigh |= l64 >> 21;
 145                 loaded_data->sigl = l64 << 11;
 146
 147                 tag = TAG_Valid;
 148         }
 149
 150         setexponent16(loaded_data, exp | negative);
 151
 152         return tag;
 153 }
 154
 155 /* Get a float from user memory */
 156 int FPU_load_single(float __user *single, FPU_REG *loaded_data)
 157 {
 158         unsigned m32;
 159         int exp, tag, negative;
 160
 161         RE_ENTRANT_CHECK_OFF;
 162         FPU_access_ok(VERIFY_READ, single, 4);
 163         FPU_get_user(m32, (unsigned long __user *)single);
 164         RE_ENTRANT_CHECK_ON;
 165
 166         negative = (m32 & 0x80000000) ? SIGN_Negative : SIGN_Positive;
 167
 168         if (!(m32 & 0x7fffffff)) {
 169                 /* Zero */
 170                 reg_copy(&CONST_Z, loaded_data);
 171                 addexponent(loaded_data, negative);
 172                 return TAG_Zero;
 173         }
 174         exp = ((m32 & 0x7f800000) >> 23) - SINGLE_Ebias + EXTENDED_Ebias;
 175         m32 = (m32 & 0x7fffff) << 8;
 176         if (exp < SINGLE_Emin + EXTENDED_Ebias) {
 177                 /* De-normals */
 178                 loaded_data->sigh = m32;
 179                 loaded_data->sigl = 0;
 180
 181                 return normalize_no_excep(loaded_data, SINGLE_Emin, negative)
 182                     | (denormal_operand() < 0 ? FPU_Exception : 0);
 183         } else if (exp > SINGLE_Emax + EXTENDED_Ebias) {
 184                 /* Infinity or NaN */
 185                 if (m32 == 0) {
 186                         /* +- infinity */
 187                         loaded_data->sigh = 0x80000000;
 188                         loaded_data->sigl = 0x00000000;
 189                         exp = EXP_Infinity + EXTENDED_Ebias;
 190                         tag = TAG_Special;
 191                 } else {
 192                         /* Must be a signaling or quiet NaN */
 193                         exp = EXP_NaN + EXTENDED_Ebias;
 194                         loaded_data->sigh = m32 | 0x80000000;
 195                         loaded_data->sigl = 0;
 196                         tag = TAG_Special;      /* The calling function must look for NaNs */
 197                 }
 198         } else {
 199                 loaded_data->sigh = m32 | 0x80000000;
 200                 loaded_data->sigl = 0;
 201                 tag = TAG_Valid;
 202         }
 203
 204         setexponent16(loaded_data, exp | negative);     /* Set the sign. */
 205
 206         return tag;
 207 }
 208
 209 /* Get a long long from user memory */
 210 int FPU_load_int64(long long __user *_s)
 211 {
 212         long long s;
 213         int sign;
 214         FPU_REG *st0_ptr = &st(0);
 215
 216         RE_ENTRANT_CHECK_OFF;
 217         FPU_access_ok(VERIFY_READ, _s, 8);
 218         if (copy_from_user(&s, _s, 8))
 219                 FPU_abort;
 220         RE_ENTRANT_CHECK_ON;
 221
 222         if (s == 0) {
 223                 reg_copy(&CONST_Z, st0_ptr);
 224                 return TAG_Zero;
 225         }
 226
 227         if (s > 0)
 228                 sign = SIGN_Positive;
 229         else {
 230                 s = -s;
 231                 sign = SIGN_Negative;
 232         }
 233
 234         significand(st0_ptr) = s;
 235
 236         return normalize_no_excep(st0_ptr, 63, sign);
 237 }
 238
 239 /* Get a long from user memory */
 240 int FPU_load_int32(long __user *_s, FPU_REG *loaded_data)
 241 {
 242         long s;
 243         int negative;
 244
 245         RE_ENTRANT_CHECK_OFF;
 246         FPU_access_ok(VERIFY_READ, _s, 4);
 247         FPU_get_user(s, _s);
 248         RE_ENTRANT_CHECK_ON;
 249
 250         if (s == 0) {
 251                 reg_copy(&CONST_Z, loaded_data);
 252                 return TAG_Zero;
 253         }
 254
 255         if (s > 0)
 256                 negative = SIGN_Positive;
 257         else {
 258                 s = -s;
 259                 negative = SIGN_Negative;
 260         }
 261
 262         loaded_data->sigh = s;
 263         loaded_data->sigl = 0;
 264
 265         return normalize_no_excep(loaded_data, 31, negative);
 266 }
 267
 268 /* Get a short from user memory */
 269 int FPU_load_int16(short __user *_s, FPU_REG *loaded_data)
 270 {
 271         int s, negative;
 272
 273         RE_ENTRANT_CHECK_OFF;
 274         FPU_access_ok(VERIFY_READ, _s, 2);
 275         /* Cast as short to get the sign extended. */
 276         FPU_get_user(s, _s);
 277         RE_ENTRANT_CHECK_ON;
 278
 279         if (s == 0) {
 280                 reg_copy(&CONST_Z, loaded_data);
 281                 return TAG_Zero;
 282         }
 283
 284         if (s > 0)
 285                 negative = SIGN_Positive;
 286         else {
 287                 s = -s;
 288                 negative = SIGN_Negative;
 289         }
 290
 291         loaded_data->sigh = s << 16;
 292         loaded_data->sigl = 0;
 293
 294         return normalize_no_excep(loaded_data, 15, negative);
 295 }
 296
 297 /* Get a packed bcd array from user memory */
 298 int FPU_load_bcd(u_char __user *s)
 299 {
 300         FPU_REG *st0_ptr = &st(0);
 301         int pos;
 302         u_char bcd;
 303         long long l = 0;
 304         int sign;
 305
 306         RE_ENTRANT_CHECK_OFF;
 307         FPU_access_ok(VERIFY_READ, s, 10);
 308         RE_ENTRANT_CHECK_ON;
 309         for (pos = 8; pos >= 0; pos--) {
 310                 l *= 10;
 311                 RE_ENTRANT_CHECK_OFF;
 312                 FPU_get_user(bcd, s + pos);
 313                 RE_ENTRANT_CHECK_ON;
 314                 l += bcd >> 4;
 315                 l *= 10;
 316                 l += bcd & 0x0f;
 317         }
 318
 319         RE_ENTRANT_CHECK_OFF;
 320         FPU_get_user(sign, s + 9);
 321         sign = sign & 0x80 ? SIGN_Negative : SIGN_Positive;
 322         RE_ENTRANT_CHECK_ON;
 323
 324         if (l == 0) {
 325                 reg_copy(&CONST_Z, st0_ptr);
 326                 addexponent(st0_ptr, sign);     /* Set the sign. */
 327                 return TAG_Zero;
 328         } else {
 329                 significand(st0_ptr) = l;
 330                 return normalize_no_excep(st0_ptr, 63, sign);
 331         }
 332 }
 333
 334 /*===========================================================================*/
 335
 336 /* Put a long double into user memory */
 337 int FPU_store_extended(FPU_REG *st0_ptr, u_char st0_tag,
 338                        long double __user * d)
 339 {
 340         /*
 341            The only exception raised by an attempt to store to an
 342            extended format is the Invalid Stack exception, i.e.
 343            attempting to store from an empty register.
 344          */
 345
 346         if (st0_tag != TAG_Empty) {
 347                 RE_ENTRANT_CHECK_OFF;
 348                 FPU_access_ok(VERIFY_WRITE, d, 10);
 349
 350                 FPU_put_user(st0_ptr->sigl, (unsigned long __user *)d);
 351                 FPU_put_user(st0_ptr->sigh,
 352                              (unsigned long __user *)((u_char __user *) d + 4));
 353                 FPU_put_user(exponent16(st0_ptr),
 354                              (unsigned short __user *)((u_char __user *) d +
 355                                                        8));
 356                 RE_ENTRANT_CHECK_ON;
 357
 358                 return 1;
 359         }
 360
 361         /* Empty register (stack underflow) */
 362         EXCEPTION(EX_StackUnder);
 363         if (control_word & CW_Invalid) {
 364                 /* The masked response */
 365                 /* Put out the QNaN indefinite */
 366                 RE_ENTRANT_CHECK_OFF;
 367                 FPU_access_ok(VERIFY_WRITE, d, 10);
 368                 FPU_put_user(0, (unsigned long __user *)d);
 369                 FPU_put_user(0xc0000000, 1 + (unsigned long __user *)d);
 370                 FPU_put_user(0xffff, 4 + (short __user *)d);
 371                 RE_ENTRANT_CHECK_ON;
 372                 return 1;
 373         } else
 374                 return 0;
 375
 376 }
 377
 378 /* Put a double into user memory */
 379 int FPU_store_double(FPU_REG *st0_ptr, u_char st0_tag, double __user *dfloat)
 380 {
 381         unsigned long l[2];
 382         unsigned long increment = 0;    /* avoid gcc warnings */
 383         int precision_loss;
 384         int exp;
 385         FPU_REG tmp;
 386
 387         l[0] = 0;
 388         l[1] = 0;
 389         if (st0_tag == TAG_Valid) {
 390                 reg_copy(st0_ptr, &tmp);
 391                 exp = exponent(&tmp);
 392
 393                 if (exp < DOUBLE_Emin) {        /* It may be a denormal */
 394                         addexponent(&tmp, -DOUBLE_Emin + 52);   /* largest exp to be 51 */
 395 denormal_arg:
 396                         if ((precision_loss = FPU_round_to_int(&tmp, st0_tag))) {
 397 #ifdef PECULIAR_486
 398                                 /* Did it round to a non-denormal ? */
 399                                 /* This behaviour might be regarded as peculiar, it appears
 400                                    that the 80486 rounds to the dest precision, then
 401                                    converts to decide underflow. */
 402                                 if (!
 403                                     ((tmp.sigh == 0x00100000) && (tmp.sigl == 0)
 404                                      && (st0_ptr->sigl & 0x000007ff)))
 405 #endif /* PECULIAR_486 */
 406                                 {
 407                                         EXCEPTION(EX_Underflow);
 408                                         /* This is a special case: see sec 16.2.5.1 of
 409                                            the 80486 book */
 410                                         if (!(control_word & CW_Underflow))
 411                                                 return 0;
 412                                 }
 413                                 EXCEPTION(precision_loss);
 414                                 if (!(control_word & CW_Precision))
 415                                         return 0;
 416                         }
 417                         l[0] = tmp.sigl;
 418                         l[1] = tmp.sigh;
 419                 } else {
 420                         if (tmp.sigl & 0x000007ff) {
 421                                 precision_loss = 1;
 422                                 switch (control_word & CW_RC) {
 423                                 case RC_RND:
 424                                         /* Rounding can get a little messy.. */
 425                                         increment = ((tmp.sigl & 0x7ff) > 0x400) |      /* nearest */
 426                                             ((tmp.sigl & 0xc00) == 0xc00);      /* odd -> even */
 427                                         break;
 428                                 case RC_DOWN:   /* towards -infinity */
 429                                         increment =
 430                                             signpositive(&tmp) ? 0 : tmp.
 431                                             sigl & 0x7ff;
 432                                         break;
 433                                 case RC_UP:     /* towards +infinity */
 434                                         increment =
 435                                             signpositive(&tmp) ? tmp.
 436                                             sigl & 0x7ff : 0;
 437                                         break;
 438                                 case RC_CHOP:
 439                                         increment = 0;
 440                                         break;
 441                                 }
 442
 443                                 /* Truncate the mantissa */
 444                                 tmp.sigl &= 0xfffff800;
 445
 446                                 if (increment) {
 447                                         if (tmp.sigl >= 0xfffff800) {
 448                                                 /* the sigl part overflows */
 449                                                 if (tmp.sigh == 0xffffffff) {
 450                                                         /* The sigh part overflows */
 451                                                         tmp.sigh = 0x80000000;
 452                                                         exp++;
 453                                                         if (exp >= EXP_OVER)
 454                                                                 goto overflow;
 455                                                 } else {
 456                                                         tmp.sigh++;
 457                                                 }
 458                                                 tmp.sigl = 0x00000000;
 459                                         } else {
 460                                                 /* We only need to increment sigl */
 461                                                 tmp.sigl += 0x00000800;
 462                                         }
 463                                 }
 464                         } else
 465                                 precision_loss = 0;
 466
 467                         l[0] = (tmp.sigl >> 11) | (tmp.sigh << 21);
 468                         l[1] = ((tmp.sigh >> 11) & 0xfffff);
 469
 470                         if (exp > DOUBLE_Emax) {
 471                               overflow:
 472                                 EXCEPTION(EX_Overflow);
 473                                 if (!(control_word & CW_Overflow))
 474                                         return 0;
 475                                 set_precision_flag_up();
 476                                 if (!(control_word & CW_Precision))
 477                                         return 0;
 478
 479                                 /* This is a special case: see sec 16.2.5.1 of the 80486 book */
 480                                 /* Overflow to infinity */
 481                                 l[1] = 0x7ff00000;      /* Set to + INF */
 482                         } else {
 483                                 if (precision_loss) {
 484                                         if (increment)
 485                                                 set_precision_flag_up();
 486                                         else
 487                                                 set_precision_flag_down();
 488                                 }
 489                                 /* Add the exponent */
 490                                 l[1] |= (((exp + DOUBLE_Ebias) & 0x7ff) << 20);
 491                         }
 492                 }
 493         } else if (st0_tag == TAG_Zero) {
 494                 /* Number is zero */
 495         } else if (st0_tag == TAG_Special) {
 496                 st0_tag = FPU_Special(st0_ptr);
 497                 if (st0_tag == TW_Denormal) {
 498                         /* A denormal will always underflow. */
 499 #ifndef PECULIAR_486
 500                         /* An 80486 is supposed to be able to generate
 501                            a denormal exception here, but... */
 502                         /* Underflow has priority. */
 503                         if (control_word & CW_Underflow)
 504                                 denormal_operand();
 505 #endif /* PECULIAR_486 */
 506                         reg_copy(st0_ptr, &tmp);
 507                         goto denormal_arg;
 508                 } else if (st0_tag == TW_Infinity) {
 509                         l[1] = 0x7ff00000;
 510                 } else if (st0_tag == TW_NaN) {
 511                         /* Is it really a NaN ? */
 512                         if ((exponent(st0_ptr) == EXP_OVER)
 513                             && (st0_ptr->sigh & 0x80000000)) {
 514                                 /* See if we can get a valid NaN from the FPU_REG */
 515                                 l[0] =
 516                                     (st0_ptr->sigl >> 11) | (st0_ptr->
 517                                                              sigh << 21);
 518                                 l[1] = ((st0_ptr->sigh >> 11) & 0xfffff);
 519                                 if (!(st0_ptr->sigh & 0x40000000)) {
 520                                         /* It is a signalling NaN */
 521                                         EXCEPTION(EX_Invalid);
 522                                         if (!(control_word & CW_Invalid))
 523                                                 return 0;
 524                                         l[1] |= (0x40000000 >> 11);
 525                                 }
 526                                 l[1] |= 0x7ff00000;
 527                         } else {
 528                                 /* It is an unsupported data type */
 529                                 EXCEPTION(EX_Invalid);
 530                                 if (!(control_word & CW_Invalid))
 531                                         return 0;
 532                                 l[1] = 0xfff80000;
 533                         }
 534                 }
 535         } else if (st0_tag == TAG_Empty) {
 536                 /* Empty register (stack underflow) */
 537                 EXCEPTION(EX_StackUnder);
 538                 if (control_word & CW_Invalid) {
 539                         /* The masked response */
 540                         /* Put out the QNaN indefinite */
 541                         RE_ENTRANT_CHECK_OFF;
 542                         FPU_access_ok(VERIFY_WRITE, dfloat, 8);
 543                         FPU_put_user(0, (unsigned long __user *)dfloat);
 544                         FPU_put_user(0xfff80000,
 545                                      1 + (unsigned long __user *)dfloat);
 546                         RE_ENTRANT_CHECK_ON;
 547                         return 1;
 548                 } else
 549                         return 0;
 550         }
 551         if (getsign(st0_ptr))
 552                 l[1] |= 0x80000000;
 553
 554         RE_ENTRANT_CHECK_OFF;
 555         FPU_access_ok(VERIFY_WRITE, dfloat, 8);
 556         FPU_put_user(l[0], (unsigned long __user *)dfloat);
 557         FPU_put_user(l[1], 1 + (unsigned long __user *)dfloat);
 558         RE_ENTRANT_CHECK_ON;
 559
 560         return 1;
 561 }
 562
 563 /* Put a float into user memory */
 564 int FPU_store_single(FPU_REG *st0_ptr, u_char st0_tag, float __user *single)
 565 {
 566         long templ = 0;
 567         unsigned long increment = 0;    /* avoid gcc warnings */
 568         int precision_loss;
 569         int exp;
 570         FPU_REG tmp;
 571
 572         if (st0_tag == TAG_Valid) {
 573
 574                 reg_copy(st0_ptr, &tmp);
 575                 exp = exponent(&tmp);
 576
 577                 if (exp < SINGLE_Emin) {
 578                         addexponent(&tmp, -SINGLE_Emin + 23);   /* largest exp to be 22 */
 579
 580                       denormal_arg:
 581
 582                         if ((precision_loss = FPU_round_to_int(&tmp, st0_tag))) {
 583 #ifdef PECULIAR_486
 584                                 /* Did it round to a non-denormal ? */
 585                                 /* This behaviour might be regarded as peculiar, it appears
 586                                    that the 80486 rounds to the dest precision, then
 587                                    converts to decide underflow. */
 588                                 if (!((tmp.sigl == 0x00800000) &&
 589                                       ((st0_ptr->sigh & 0x000000ff)
 590                                        || st0_ptr->sigl)))
 591 #endif /* PECULIAR_486 */
 592                                 {
 593                                         EXCEPTION(EX_Underflow);
 594                                         /* This is a special case: see sec 16.2.5.1 of
 595                                            the 80486 book */
 596                                         if (!(control_word & CW_Underflow))
 597                                                 return 0;
 598                                 }
 599                                 EXCEPTION(precision_loss);
 600                                 if (!(control_word & CW_Precision))
 601                                         return 0;
 602                         }
 603                         templ = tmp.sigl;
 604                 } else {
 605                         if (tmp.sigl | (tmp.sigh & 0x000000ff)) {
 606                                 unsigned long sigh = tmp.sigh;
 607                                 unsigned long sigl = tmp.sigl;
 608
 609                                 precision_loss = 1;
 610                                 switch (control_word & CW_RC) {
 611                                 case RC_RND:
 612                                         increment = ((sigh & 0xff) > 0x80)      /* more than half */
 613                                             ||(((sigh & 0xff) == 0x80) && sigl) /* more than half */
 614                                             ||((sigh & 0x180) == 0x180);        /* round to even */
 615                                         break;
 616                                 case RC_DOWN:   /* towards -infinity */
 617                                         increment = signpositive(&tmp)
 618                                             ? 0 : (sigl | (sigh & 0xff));
 619                                         break;
 620                                 case RC_UP:     /* towards +infinity */
 621                                         increment = signpositive(&tmp)
 622                                             ? (sigl | (sigh & 0xff)) : 0;
 623                                         break;
 624                                 case RC_CHOP:
 625                                         increment = 0;
 626                                         break;
 627                                 }
 628
 629                                 /* Truncate part of the mantissa */
 630                                 tmp.sigl = 0;
 631
 632                                 if (increment) {
 633                                         if (sigh >= 0xffffff00) {
 634                                                 /* The sigh part overflows */
 635                                                 tmp.sigh = 0x80000000;
 636                                                 exp++;
 637                                                 if (exp >= EXP_OVER)
 638                                                         goto overflow;
 639                                         } else {
 640                                                 tmp.sigh &= 0xffffff00;
 641                                                 tmp.sigh += 0x100;
 642                                         }
 643                                 } else {
 644                                         tmp.sigh &= 0xffffff00; /* Finish the truncation */
 645                                 }
 646                         } else
 647                                 precision_loss = 0;
 648
 649                         templ = (tmp.sigh >> 8) & 0x007fffff;
 650
 651                         if (exp > SINGLE_Emax) {
 652                               overflow:
 653                                 EXCEPTION(EX_Overflow);
 654                                 if (!(control_word & CW_Overflow))
 655                                         return 0;
 656                                 set_precision_flag_up();
 657                                 if (!(control_word & CW_Precision))
 658                                         return 0;
 659
 660                                 /* This is a special case: see sec 16.2.5.1 of the 80486 book. */
 661                                 /* Masked response is overflow to infinity. */
 662                                 templ = 0x7f800000;
 663                         } else {
 664                                 if (precision_loss) {
 665                                         if (increment)
 666                                                 set_precision_flag_up();
 667                                         else
 668                                                 set_precision_flag_down();
 669                                 }
 670                                 /* Add the exponent */
 671                                 templ |= ((exp + SINGLE_Ebias) & 0xff) << 23;
 672                         }
 673                 }
 674         } else if (st0_tag == TAG_Zero) {
 675                 templ = 0;
 676         } else if (st0_tag == TAG_Special) {
 677                 st0_tag = FPU_Special(st0_ptr);
 678                 if (st0_tag == TW_Denormal) {
 679                         reg_copy(st0_ptr, &tmp);
 680
 681                         /* A denormal will always underflow. */
 682 #ifndef PECULIAR_486
 683                         /* An 80486 is supposed to be able to generate
 684                            a denormal exception here, but... */
 685                         /* Underflow has priority. */
 686                         if (control_word & CW_Underflow)
 687                                 denormal_operand();
 688 #endif /* PECULIAR_486 */
 689                         goto denormal_arg;
 690                 } else if (st0_tag == TW_Infinity) {
 691                         templ = 0x7f800000;
 692                 } else if (st0_tag == TW_NaN) {
 693                         /* Is it really a NaN ? */
 694                         if ((exponent(st0_ptr) == EXP_OVER)
 695                             && (st0_ptr->sigh & 0x80000000)) {
 696                                 /* See if we can get a valid NaN from the FPU_REG */
 697                                 templ = st0_ptr->sigh >> 8;
 698                                 if (!(st0_ptr->sigh & 0x40000000)) {
 699                                         /* It is a signalling NaN */
 700                                         EXCEPTION(EX_Invalid);
 701                                         if (!(control_word & CW_Invalid))
 702                                                 return 0;
 703                                         templ |= (0x40000000 >> 8);
 704                                 }
 705                                 templ |= 0x7f800000;
 706                         } else {
 707                                 /* It is an unsupported data type */
 708                                 EXCEPTION(EX_Invalid);
 709                                 if (!(control_word & CW_Invalid))
 710                                         return 0;
 711                                 templ = 0xffc00000;
 712                         }
 713                 }
 714 #ifdef PARANOID
 715                 else {
 716                         EXCEPTION(EX_INTERNAL | 0x164);
 717                         return 0;
 718                 }
 719 #endif
 720         } else if (st0_tag == TAG_Empty) {
 721                 /* Empty register (stack underflow) */
 722                 EXCEPTION(EX_StackUnder);
 723                 if (control_word & EX_Invalid) {
 724                         /* The masked response */
 725                         /* Put out the QNaN indefinite */
 726                         RE_ENTRANT_CHECK_OFF;
 727                         FPU_access_ok(VERIFY_WRITE, single, 4);
 728                         FPU_put_user(0xffc00000,
 729                                      (unsigned long __user *)single);
 730                         RE_ENTRANT_CHECK_ON;
 731                         return 1;
 732                 } else
 733                         return 0;
 734         }
 735 #ifdef PARANOID
 736         else {
 737                 EXCEPTION(EX_INTERNAL | 0x163);
 738                 return 0;
 739         }
 740 #endif
 741         if (getsign(st0_ptr))
 742                 templ |= 0x80000000;
 743
 744         RE_ENTRANT_CHECK_OFF;
 745         FPU_access_ok(VERIFY_WRITE, single, 4);
 746         FPU_put_user(templ, (unsigned long __user *)single);
 747         RE_ENTRANT_CHECK_ON;
 748
 749         return 1;
 750 }
 751
 752 /* Put a long long into user memory */
 753 int FPU_store_int64(FPU_REG *st0_ptr, u_char st0_tag, long long __user *d)
 754 {
 755         FPU_REG t;
 756         long long tll;
 757         int precision_loss;
 758
 759         if (st0_tag == TAG_Empty) {
 760                 /* Empty register (stack underflow) */
 761                 EXCEPTION(EX_StackUnder);
 762                 goto invalid_operand;
 763         } else if (st0_tag == TAG_Special) {
 764                 st0_tag = FPU_Special(st0_ptr);
 765                 if ((st0_tag == TW_Infinity) || (st0_tag == TW_NaN)) {
 766                         EXCEPTION(EX_Invalid);
 767                         goto invalid_operand;
 768                 }
 769         }
 770
 771         reg_copy(st0_ptr, &t);
 772         precision_loss = FPU_round_to_int(&t, st0_tag);
 773         ((long *)&tll)[0] = t.sigl;
 774         ((long *)&tll)[1] = t.sigh;
 775         if ((precision_loss == 1) ||
 776             ((t.sigh & 0x80000000) &&
 777              !((t.sigh == 0x80000000) && (t.sigl == 0) && signnegative(&t)))) {
 778                 EXCEPTION(EX_Invalid);
 779                 /* This is a special case: see sec 16.2.5.1 of the 80486 book */
 780               invalid_operand:
 781                 if (control_word & EX_Invalid) {
 782                         /* Produce something like QNaN "indefinite" */
 783                         tll = 0x8000000000000000LL;
 784                 } else
 785                         return 0;
 786         } else {
 787                 if (precision_loss)
 788                         set_precision_flag(precision_loss);
 789                 if (signnegative(&t))
 790                         tll = -tll;
 791         }
 792
 793         RE_ENTRANT_CHECK_OFF;
 794         FPU_access_ok(VERIFY_WRITE, d, 8);
 795         if (copy_to_user(d, &tll, 8))
 796                 FPU_abort;
 797         RE_ENTRANT_CHECK_ON;
 798
 799         return 1;
 800 }
 801
 802 /* Put a long into user memory */
 803 int FPU_store_int32(FPU_REG *st0_ptr, u_char st0_tag, long __user *d)
 804 {
 805         FPU_REG t;
 806         int precision_loss;
 807
 808         if (st0_tag == TAG_Empty) {
 809                 /* Empty register (stack underflow) */
 810                 EXCEPTION(EX_StackUnder);
 811                 goto invalid_operand;
 812         } else if (st0_tag == TAG_Special) {
 813                 st0_tag = FPU_Special(st0_ptr);
 814                 if ((st0_tag == TW_Infinity) || (st0_tag == TW_NaN)) {
 815                         EXCEPTION(EX_Invalid);
 816                         goto invalid_operand;
 817                 }
 818         }
 819
 820         reg_copy(st0_ptr, &t);
 821         precision_loss = FPU_round_to_int(&t, st0_tag);
 822         if (t.sigh ||
 823             ((t.sigl & 0x80000000) &&
 824              !((t.sigl == 0x80000000) && signnegative(&t)))) {
 825                 EXCEPTION(EX_Invalid);
 826                 /* This is a special case: see sec 16.2.5.1 of the 80486 book */
 827               invalid_operand:
 828                 if (control_word & EX_Invalid) {
 829                         /* Produce something like QNaN "indefinite" */
 830                         t.sigl = 0x80000000;
 831                 } else
 832                         return 0;
 833         } else {
 834                 if (precision_loss)
 835                         set_precision_flag(precision_loss);
 836                 if (signnegative(&t))
 837                         t.sigl = -(long)t.sigl;
 838         }
 839
 840         RE_ENTRANT_CHECK_OFF;
 841         FPU_access_ok(VERIFY_WRITE, d, 4);
 842         FPU_put_user(t.sigl, (unsigned long __user *)d);
 843         RE_ENTRANT_CHECK_ON;
 844
 845         return 1;
 846 }
 847
 848 /* Put a short into user memory */
 849 int FPU_store_int16(FPU_REG *st0_ptr, u_char st0_tag, short __user *d)
 850 {
 851         FPU_REG t;
 852         int precision_loss;
 853
 854         if (st0_tag == TAG_Empty) {
 855                 /* Empty register (stack underflow) */
 856                 EXCEPTION(EX_StackUnder);
 857                 goto invalid_operand;
 858         } else if (st0_tag == TAG_Special) {
 859                 st0_tag = FPU_Special(st0_ptr);
 860                 if ((st0_tag == TW_Infinity) || (st0_tag == TW_NaN)) {
 861                         EXCEPTION(EX_Invalid);
 862                         goto invalid_operand;
 863                 }
 864         }
 865
 866         reg_copy(st0_ptr, &t);
 867         precision_loss = FPU_round_to_int(&t, st0_tag);
 868         if (t.sigh ||
 869             ((t.sigl & 0xffff8000) &&
 870              !((t.sigl == 0x8000) && signnegative(&t)))) {
 871                 EXCEPTION(EX_Invalid);
 872                 /* This is a special case: see sec 16.2.5.1 of the 80486 book */
 873               invalid_operand:
 874                 if (control_word & EX_Invalid) {
 875                         /* Produce something like QNaN "indefinite" */
 876                         t.sigl = 0x8000;
 877                 } else
 878                         return 0;
 879         } else {
 880                 if (precision_loss)
 881                         set_precision_flag(precision_loss);
 882                 if (signnegative(&t))
 883                         t.sigl = -t.sigl;
 884         }
 885
 886         RE_ENTRANT_CHECK_OFF;
 887         FPU_access_ok(VERIFY_WRITE, d, 2);
 888         FPU_put_user((short)t.sigl, d);
 889         RE_ENTRANT_CHECK_ON;
 890
 891         return 1;
 892 }
 893
 894 /* Put a packed bcd array into user memory */
 895 int FPU_store_bcd(FPU_REG *st0_ptr, u_char st0_tag, u_char __user *d)
 896 {
 897         FPU_REG t;
 898         unsigned long long ll;
 899         u_char b;
 900         int i, precision_loss;
 901         u_char sign = (getsign(st0_ptr) == SIGN_NEG) ? 0x80 : 0;
 902
 903         if (st0_tag == TAG_Empty) {
 904                 /* Empty register (stack underflow) */
 905                 EXCEPTION(EX_StackUnder);
 906                 goto invalid_operand;
 907         } else if (st0_tag == TAG_Special) {
 908                 st0_tag = FPU_Special(st0_ptr);
 909                 if ((st0_tag == TW_Infinity) || (st0_tag == TW_NaN)) {
 910                         EXCEPTION(EX_Invalid);
 911                         goto invalid_operand;
 912                 }
 913         }
 914
 915         reg_copy(st0_ptr, &t);
 916         precision_loss = FPU_round_to_int(&t, st0_tag);
 917         ll = significand(&t);
 918
 919         /* Check for overflow, by comparing with 999999999999999999 decimal. */
 920         if ((t.sigh > 0x0de0b6b3) ||
 921             ((t.sigh == 0x0de0b6b3) && (t.sigl > 0xa763ffff))) {
 922                 EXCEPTION(EX_Invalid);
 923                 /* This is a special case: see sec 16.2.5.1 of the 80486 book */
 924               invalid_operand:
 925                 if (control_word & CW_Invalid) {
 926                         /* Produce the QNaN "indefinite" */
 927                         RE_ENTRANT_CHECK_OFF;
 928                         FPU_access_ok(VERIFY_WRITE, d, 10);
 929                         for (i = 0; i < 7; i++)
 930                                 FPU_put_user(0, d + i); /* These bytes "undefined" */
 931                         FPU_put_user(0xc0, d + 7);      /* This byte "undefined" */
 932                         FPU_put_user(0xff, d + 8);
 933                         FPU_put_user(0xff, d + 9);
 934                         RE_ENTRANT_CHECK_ON;
 935                         return 1;
 936                 } else
 937                         return 0;
 938         } else if (precision_loss) {
 939                 /* Precision loss doesn't stop the data transfer */
 940                 set_precision_flag(precision_loss);
 941         }
 942
 943         RE_ENTRANT_CHECK_OFF;
 944         FPU_access_ok(VERIFY_WRITE, d, 10);
 945         RE_ENTRANT_CHECK_ON;
 946         for (i = 0; i < 9; i++) {
 947                 b = FPU_div_small(&ll, 10);
 948                 b |= (FPU_div_small(&ll, 10)) << 4;
 949                 RE_ENTRANT_CHECK_OFF;
 950                 FPU_put_user(b, d + i);
 951                 RE_ENTRANT_CHECK_ON;
 952         }
 953         RE_ENTRANT_CHECK_OFF;
 954         FPU_put_user(sign, d + 9);
 955         RE_ENTRANT_CHECK_ON;
 956
 957         return 1;
 958 }
 959
 960 /*===========================================================================*/
 961
 962 /* r gets mangled such that sig is int, sign:
 963    it is NOT normalized */
 964 /* The return value (in eax) is zero if the result is exact,
 965    if bits are changed due to rounding, truncation, etc, then
 966    a non-zero value is returned */
 967 /* Overflow is signalled by a non-zero return value (in eax).
 968    In the case of overflow, the returned significand always has the
 969    largest possible value */
 970 int FPU_round_to_int(FPU_REG *r, u_char tag)
 971 {
 972         u_char very_big;
 973         unsigned eax;
 974
 975         if (tag == TAG_Zero) {
 976                 /* Make sure that zero is returned */
 977                 significand(r) = 0;
 978                 return 0;       /* o.k. */
 979         }
 980
 981         if (exponent(r) > 63) {
 982                 r->sigl = r->sigh = ~0; /* The largest representable number */
 983                 return 1;       /* overflow */
 984         }
 985
 986         eax = FPU_shrxs(&r->sigl, 63 - exponent(r));
 987         very_big = !(~(r->sigh) | ~(r->sigl));  /* test for 0xfff...fff */
 988 #define half_or_more    (eax & 0x80000000)
 989 #define frac_part       (eax)
 990 #define more_than_half  ((eax & 0x80000001) == 0x80000001)
 991         switch (control_word & CW_RC) {
 992         case RC_RND:
 993                 if (more_than_half      /* nearest */
 994                     || (half_or_more && (r->sigl & 1))) {       /* odd -> even */
 995                         if (very_big)
 996                                 return 1;       /* overflow */
 997                         significand(r)++;
 998                         return PRECISION_LOST_UP;
 999                 }
1000                 break;
1001         case RC_DOWN:
1002                 if (frac_part && getsign(r)) {
1003                         if (very_big)
1004                                 return 1;       /* overflow */
1005                         significand(r)++;
1006                         return PRECISION_LOST_UP;
1007                 }
1008                 break;
1009         case RC_UP:
1010                 if (frac_part && !getsign(r)) {
1011                         if (very_big)
1012                                 return 1;       /* overflow */
1013                         significand(r)++;
1014                         return PRECISION_LOST_UP;
1015                 }
1016                 break;
1017         case RC_CHOP:
1018                 break;
1019         }
1020
1021         return eax ? PRECISION_LOST_DOWN : 0;
1022
1023 }
1024
1025 /*===========================================================================*/
1026
1027 u_char __user *fldenv(fpu_addr_modes addr_modes, u_char __user *s)
1028 {
1029         unsigned short tag_word = 0;
1030         u_char tag;
1031         int i;
1032
1033         if ((addr_modes.default_mode == VM86) ||
1034             ((addr_modes.default_mode == PM16)
1035              ^ (addr_modes.override.operand_size == OP_SIZE_PREFIX))) {
1036                 RE_ENTRANT_CHECK_OFF;
1037                 FPU_access_ok(VERIFY_READ, s, 0x0e);
1038                 FPU_get_user(control_word, (unsigned short __user *)s);
1039                 FPU_get_user(partial_status, (unsigned short __user *)(s + 2));
1040                 FPU_get_user(tag_word, (unsigned short __user *)(s + 4));
1041                 FPU_get_user(instruction_address.offset,
1042                              (unsigned short __user *)(s + 6));
1043                 FPU_get_user(instruction_address.selector,
1044                              (unsigned short __user *)(s + 8));
1045                 FPU_get_user(operand_address.offset,
1046                              (unsigned short __user *)(s + 0x0a));
1047                 FPU_get_user(operand_address.selector,
1048                              (unsigned short __user *)(s + 0x0c));
1049                 RE_ENTRANT_CHECK_ON;
1050                 s += 0x0e;
1051                 if (addr_modes.default_mode == VM86) {
1052                         instruction_address.offset
1053                             += (instruction_address.selector & 0xf000) << 4;
1054                         operand_address.offset +=
1055                             (operand_address.selector & 0xf000) << 4;
1056                 }
1057         } else {
1058                 RE_ENTRANT_CHECK_OFF;
1059                 FPU_access_ok(VERIFY_READ, s, 0x1c);
1060                 FPU_get_user(control_word, (unsigned short __user *)s);
1061                 FPU_get_user(partial_status, (unsigned short __user *)(s + 4));
1062                 FPU_get_user(tag_word, (unsigned short __user *)(s + 8));
1063                 FPU_get_user(instruction_address.offset,
1064                              (unsigned long __user *)(s + 0x0c));
1065                 FPU_get_user(instruction_address.selector,
1066                              (unsigned short __user *)(s + 0x10));
1067                 FPU_get_user(instruction_address.opcode,
1068                              (unsigned short __user *)(s + 0x12));
1069                 FPU_get_user(operand_address.offset,
1070                              (unsigned long __user *)(s + 0x14));
1071                 FPU_get_user(operand_address.selector,
1072                              (unsigned long __user *)(s + 0x18));
1073                 RE_ENTRANT_CHECK_ON;
1074                 s += 0x1c;
1075         }
1076
1077 #ifdef PECULIAR_486
1078         control_word &= ~0xe080;
1079 #endif /* PECULIAR_486 */
1080
1081         top = (partial_status >> SW_Top_Shift) & 7;
1082
1083         if (partial_status & ~control_word & CW_Exceptions)
1084                 partial_status |= (SW_Summary | SW_Backward);
1085         else
1086                 partial_status &= ~(SW_Summary | SW_Backward);
1087
1088         for (i = 0; i < 8; i++) {
1089                 tag = tag_word & 3;
1090                 tag_word >>= 2;
1091
1092                 if (tag == TAG_Empty)
1093                         /* New tag is empty.  Accept it */
1094                         FPU_settag(i, TAG_Empty);
1095                 else if (FPU_gettag(i) == TAG_Empty) {
1096                         /* Old tag is empty and new tag is not empty.  New tag is determined
1097                            by old reg contents */
1098                         if (exponent(&fpu_register(i)) == -EXTENDED_Ebias) {
1099                                 if (!
1100                                     (fpu_register(i).sigl | fpu_register(i).
1101                                      sigh))
1102                                         FPU_settag(i, TAG_Zero);
1103                                 else
1104                                         FPU_settag(i, TAG_Special);
1105                         } else if (exponent(&fpu_register(i)) ==
1106                                    0x7fff - EXTENDED_Ebias) {
1107                                 FPU_settag(i, TAG_Special);
1108                         } else if (fpu_register(i).sigh & 0x80000000)
1109                                 FPU_settag(i, TAG_Valid);
1110                         else
1111                                 FPU_settag(i, TAG_Special);     /* An Un-normal */
1112                 }
1113                 /* Else old tag is not empty and new tag is not empty.  Old tag
1114                    remains correct */
1115         }
1116
1117         return s;
1118 }
1119
1120 void frstor(fpu_addr_modes addr_modes, u_char __user *data_address)
1121 {
1122         int i, regnr;
1123         u_char __user *s = fldenv(addr_modes, data_address);
1124         int offset = (top & 7) * 10, other = 80 - offset;
1125
1126         /* Copy all registers in stack order. */
1127         RE_ENTRANT_CHECK_OFF;
1128         FPU_access_ok(VERIFY_READ, s, 80);
1129         __copy_from_user(register_base + offset, s, other);
1130         if (offset)
1131                 __copy_from_user(register_base, s + other, offset);
1132         RE_ENTRANT_CHECK_ON;
1133
1134         for (i = 0; i < 8; i++) {
1135                 regnr = (i + top) & 7;
1136                 if (FPU_gettag(regnr) != TAG_Empty)
1137                         /* The loaded data over-rides all other cases. */
1138                         FPU_settag(regnr, FPU_tagof(&st(i)));
1139         }
1140
1141 }
1142
1143 u_char __user *fstenv(fpu_addr_modes addr_modes, u_char __user *d)
1144 {
1145         if ((addr_modes.default_mode == VM86) ||
1146             ((addr_modes.default_mode == PM16)
1147              ^ (addr_modes.override.operand_size == OP_SIZE_PREFIX))) {
1148                 RE_ENTRANT_CHECK_OFF;
1149                 FPU_access_ok(VERIFY_WRITE, d, 14);
1150 #ifdef PECULIAR_486
1151                 FPU_put_user(control_word & ~0xe080, (unsigned long __user *)d);
1152 #else
1153                 FPU_put_user(control_word, (unsigned short __user *)d);
1154 #endif /* PECULIAR_486 */
1155                 FPU_put_user(status_word(), (unsigned short __user *)(d + 2));
1156                 FPU_put_user(fpu_tag_word, (unsigned short __user *)(d + 4));
1157                 FPU_put_user(instruction_address.offset,
1158                              (unsigned short __user *)(d + 6));
1159                 FPU_put_user(operand_address.offset,
1160                              (unsigned short __user *)(d + 0x0a));
1161                 if (addr_modes.default_mode == VM86) {
1162                         FPU_put_user((instruction_address.
1163                                       offset & 0xf0000) >> 4,
1164                                      (unsigned short __user *)(d + 8));
1165                         FPU_put_user((operand_address.offset & 0xf0000) >> 4,
1166                                      (unsigned short __user *)(d + 0x0c));
1167                 } else {
1168                         FPU_put_user(instruction_address.selector,
1169                                      (unsigned short __user *)(d + 8));
1170                         FPU_put_user(operand_address.selector,
1171                                      (unsigned short __user *)(d + 0x0c));
1172                 }
1173                 RE_ENTRANT_CHECK_ON;
1174                 d += 0x0e;
1175         } else {
1176                 RE_ENTRANT_CHECK_OFF;
1177                 FPU_access_ok(VERIFY_WRITE, d, 7 * 4);
1178 #ifdef PECULIAR_486
1179                 control_word &= ~0xe080;
1180                 /* An 80486 sets nearly all of the reserved bits to 1. */
1181                 control_word |= 0xffff0040;
1182                 partial_status = status_word() | 0xffff0000;
1183                 fpu_tag_word |= 0xffff0000;
1184                 I387->soft.fcs &= ~0xf8000000;
1185                 I387->soft.fos |= 0xffff0000;
1186 #endif /* PECULIAR_486 */
1187                 if (__copy_to_user(d, &control_word, 7 * 4))
1188                         FPU_abort;
1189                 RE_ENTRANT_CHECK_ON;
1190                 d += 0x1c;
1191         }
1192
1193         control_word |= CW_Exceptions;
1194         partial_status &= ~(SW_Summary | SW_Backward);
1195
1196         return d;
1197 }
1198
1199 void fsave(fpu_addr_modes addr_modes, u_char __user *data_address)
1200 {
1201         u_char __user *d;
1202         int offset = (top & 7) * 10, other = 80 - offset;
1203
1204         d = fstenv(addr_modes, data_address);
1205
1206         RE_ENTRANT_CHECK_OFF;
1207         FPU_access_ok(VERIFY_WRITE, d, 80);
1208
1209         /* Copy all registers in stack order. */
1210         if (__copy_to_user(d, register_base + offset, other))
1211                 FPU_abort;
1212         if (offset)
1213                 if (__copy_to_user(d + other, register_base, offset))
1214                         FPU_abort;
1215         RE_ENTRANT_CHECK_ON;
1216
1217         finit();
1218 }
1219
1220 /*===========================================================================*/