x86/include/asm/uaccess.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ASM_X86_UACCESS_H
   3 #define _ASM_X86_UACCESS_H
   4 /*
   5  * User space memory access functions
   6  */
   7 #include <linux/compiler.h>
   8 #include <linux/instrumented.h>
   9 #include <linux/kasan-checks.h>
  10 #include <linux/string.h>
  11 #include <asm/asm.h>
  12 #include <asm/page.h>
  13 #include <asm/smap.h>
  14 #include <asm/extable.h>
  15
  16 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
  17 static inline bool pagefault_disabled(void);
  18 # define WARN_ON_IN_IRQ()       \
  19         WARN_ON_ONCE(!in_task() && !pagefault_disabled())
  20 #else
  21 # define WARN_ON_IN_IRQ()
  22 #endif
  23
  24 /**
  25  * access_ok - Checks if a user space pointer is valid
  26  * @addr: User space pointer to start of block to check
  27  * @size: Size of block to check
  28  *
  29  * Context: User context only. This function may sleep if pagefaults are
  30  *          enabled.
  31  *
  32  * Checks if a pointer to a block of memory in user space is valid.
  33  *
  34  * Note that, depending on architecture, this function probably just
  35  * checks that the pointer is in the user space range - after calling
  36  * this function, memory access functions may still return -EFAULT.
  37  *
  38  * Return: true (nonzero) if the memory block may be valid, false (zero)
  39  * if it is definitely invalid.
  40  */
  41 #define access_ok(addr, size)                                   \
  42 ({                                                                      \
  43         WARN_ON_IN_IRQ();                                               \
  44         likely(__access_ok(addr, size));                                \
  45 })
  46
  47 #include <asm-generic/access_ok.h>
  48
  49 extern int __get_user_1(void);
  50 extern int __get_user_2(void);
  51 extern int __get_user_4(void);
  52 extern int __get_user_8(void);
  53 extern int __get_user_nocheck_1(void);
  54 extern int __get_user_nocheck_2(void);
  55 extern int __get_user_nocheck_4(void);
  56 extern int __get_user_nocheck_8(void);
  57 extern int __get_user_bad(void);
  58
  59 #define __uaccess_begin() stac()
  60 #define __uaccess_end()   clac()
  61 #define __uaccess_begin_nospec()        \
  62 ({                                      \
  63         stac();                         \
  64         barrier_nospec();               \
  65 })
  66
  67 /*
  68  * This is the smallest unsigned integer type that can fit a value
  69  * (up to 'long long')
  70  */
  71 #define __inttype(x) __typeof__(                \
  72         __typefits(x,char,                      \
  73           __typefits(x,short,                   \
  74             __typefits(x,int,                   \
  75               __typefits(x,long,0ULL)))))
  76
  77 #define __typefits(x,type,not) \
  78         __builtin_choose_expr(sizeof(x)<=sizeof(type),(unsigned type)0,not)
  79
  80 /*
  81  * This is used for both get_user() and __get_user() to expand to
  82  * the proper special function call that has odd calling conventions
  83  * due to returning both a value and an error, and that depends on
  84  * the size of the pointer passed in.
  85  *
  86  * Careful: we have to cast the result to the type of the pointer
  87  * for sign reasons.
  88  *
  89  * The use of _ASM_DX as the register specifier is a bit of a
  90  * simplification, as gcc only cares about it as the starting point
  91  * and not size: for a 64-bit value it will use %ecx:%edx on 32 bits
  92  * (%ecx being the next register in gcc's x86 register sequence), and
  93  * %rdx on 64 bits.
  94  *
  95  * Clang/LLVM cares about the size of the register, but still wants
  96  * the base register for something that ends up being a pair.
  97  */
  98 #define do_get_user_call(fn,x,ptr)                                      \
  99 ({                                                                      \
 100         int __ret_gu;                                                   \
 101         register __inttype(*(ptr)) __val_gu asm("%"_ASM_DX);            \
 102         __chk_user_ptr(ptr);                                            \
 103         asm volatile("call __" #fn "_%P4"                               \
 104                      : "=a" (__ret_gu), "=r" (__val_gu),                \
 105                         ASM_CALL_CONSTRAINT                             \
 106                      : "0" (ptr), "i" (sizeof(*(ptr))));                \
 107         instrument_get_user(__val_gu);                                  \
 108         (x) = (__force __typeof__(*(ptr))) __val_gu;                    \
 109         __builtin_expect(__ret_gu, 0);                                  \
 110 })
 111
 112 /**
 113  * get_user - Get a simple variable from user space.
 114  * @x:   Variable to store result.
 115  * @ptr: Source address, in user space.
 116  *
 117  * Context: User context only. This function may sleep if pagefaults are
 118  *          enabled.
 119  *
 120  * This macro copies a single simple variable from user space to kernel
 121  * space.  It supports simple types like char and int, but not larger
 122  * data types like structures or arrays.
 123  *
 124  * @ptr must have pointer-to-simple-variable type, and the result of
 125  * dereferencing @ptr must be assignable to @x without a cast.
 126  *
 127  * Return: zero on success, or -EFAULT on error.
 128  * On error, the variable @x is set to zero.
 129  */
 130 #define get_user(x,ptr) ({ might_fault(); do_get_user_call(get_user,x,ptr); })
 131
 132 /**
 133  * __get_user - Get a simple variable from user space, with less checking.
 134  * @x:   Variable to store result.
 135  * @ptr: Source address, in user space.
 136  *
 137  * Context: User context only. This function may sleep if pagefaults are
 138  *          enabled.
 139  *
 140  * This macro copies a single simple variable from user space to kernel
 141  * space.  It supports simple types like char and int, but not larger
 142  * data types like structures or arrays.
 143  *
 144  * @ptr must have pointer-to-simple-variable type, and the result of
 145  * dereferencing @ptr must be assignable to @x without a cast.
 146  *
 147  * Caller must check the pointer with access_ok() before calling this
 148  * function.
 149  *
 150  * Return: zero on success, or -EFAULT on error.
 151  * On error, the variable @x is set to zero.
 152  */
 153 #define __get_user(x,ptr) do_get_user_call(get_user_nocheck,x,ptr)
 154
 155
 156 #ifdef CONFIG_X86_32
 157 #define __put_user_goto_u64(x, addr, label)                     \
 158         asm goto("\n"                                   \
 159                      "1:        movl %%eax,0(%1)\n"             \
 160                      "2:        movl %%edx,4(%1)\n"             \
 161                      _ASM_EXTABLE_UA(1b, %l2)                   \
 162                      _ASM_EXTABLE_UA(2b, %l2)                   \
 163                      : : "A" (x), "r" (addr)                    \
 164                      : : label)
 165
 166 #else
 167 #define __put_user_goto_u64(x, ptr, label) \
 168         __put_user_goto(x, ptr, "q", "er", label)
 169 #endif
 170
 171 extern void __put_user_bad(void);
 172
 173 /*
 174  * Strange magic calling convention: pointer in %ecx,
 175  * value in %eax(:%edx), return value in %ecx. clobbers %rbx
 176  */
 177 extern void __put_user_1(void);
 178 extern void __put_user_2(void);
 179 extern void __put_user_4(void);
 180 extern void __put_user_8(void);
 181 extern void __put_user_nocheck_1(void);
 182 extern void __put_user_nocheck_2(void);
 183 extern void __put_user_nocheck_4(void);
 184 extern void __put_user_nocheck_8(void);
 185
 186 /*
 187  * ptr must be evaluated and assigned to the temporary __ptr_pu before
 188  * the assignment of x to __val_pu, to avoid any function calls
 189  * involved in the ptr expression (possibly implicitly generated due
 190  * to KASAN) from clobbering %ax.
 191  */
 192 #define do_put_user_call(fn,x,ptr)                                      \
 193 ({                                                                      \
 194         int __ret_pu;                                                   \
 195         void __user *__ptr_pu;                                          \
 196         register __typeof__(*(ptr)) __val_pu asm("%"_ASM_AX);           \
 197         __typeof__(*(ptr)) __x = (x); /* eval x once */                 \
 198         __typeof__(ptr) __ptr = (ptr); /* eval ptr once */              \
 199         __chk_user_ptr(__ptr);                                          \
 200         __ptr_pu = __ptr;                                               \
 201         __val_pu = __x;                                                 \
 202         asm volatile("call __" #fn "_%P[size]"                          \
 203                      : "=c" (__ret_pu),                                 \
 204                         ASM_CALL_CONSTRAINT                             \
 205                      : "0" (__ptr_pu),                                  \
 206                        "r" (__val_pu),                                  \
 207                        [size] "i" (sizeof(*(ptr)))                      \
 208                      :"ebx");                                           \
 209         instrument_put_user(__x, __ptr, sizeof(*(ptr)));                \
 210         __builtin_expect(__ret_pu, 0);                                  \
 211 })
 212
 213 /**
 214  * put_user - Write a simple value into user space.
 215  * @x:   Value to copy to user space.
 216  * @ptr: Destination address, in user space.
 217  *
 218  * Context: User context only. This function may sleep if pagefaults are
 219  *          enabled.
 220  *
 221  * This macro copies a single simple value from kernel space to user
 222  * space.  It supports simple types like char and int, but not larger
 223  * data types like structures or arrays.
 224  *
 225  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 226  * to the result of dereferencing @ptr.
 227  *
 228  * Return: zero on success, or -EFAULT on error.
 229  */
 230 #define put_user(x, ptr) ({ might_fault(); do_put_user_call(put_user,x,ptr); })
 231
 232 /**
 233  * __put_user - Write a simple value into user space, with less checking.
 234  * @x:   Value to copy to user space.
 235  * @ptr: Destination address, in user space.
 236  *
 237  * Context: User context only. This function may sleep if pagefaults are
 238  *          enabled.
 239  *
 240  * This macro copies a single simple value from kernel space to user
 241  * space.  It supports simple types like char and int, but not larger
 242  * data types like structures or arrays.
 243  *
 244  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 245  * to the result of dereferencing @ptr.
 246  *
 247  * Caller must check the pointer with access_ok() before calling this
 248  * function.
 249  *
 250  * Return: zero on success, or -EFAULT on error.
 251  */
 252 #define __put_user(x, ptr) do_put_user_call(put_user_nocheck,x,ptr)
 253
 254 #define __put_user_size(x, ptr, size, label)                            \
 255 do {                                                                    \
 256         __typeof__(*(ptr)) __x = (x); /* eval x once */                 \
 257         __typeof__(ptr) __ptr = (ptr); /* eval ptr once */              \
 258         __chk_user_ptr(__ptr);                                          \
 259         switch (size) {                                                 \
 260         case 1:                                                         \
 261                 __put_user_goto(__x, __ptr, "b", "iq", label);          \
 262                 break;                                                  \
 263         case 2:                                                         \
 264                 __put_user_goto(__x, __ptr, "w", "ir", label);          \
 265                 break;                                                  \
 266         case 4:                                                         \
 267                 __put_user_goto(__x, __ptr, "l", "ir", label);          \
 268                 break;                                                  \
 269         case 8:                                                         \
 270                 __put_user_goto_u64(__x, __ptr, label);                 \
 271                 break;                                                  \
 272         default:                                                        \
 273                 __put_user_bad();                                       \
 274         }                                                               \
 275         instrument_put_user(__x, __ptr, size);                          \
 276 } while (0)
 277
 278 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
 279
 280 #ifdef CONFIG_X86_32
 281 #define __get_user_asm_u64(x, ptr, label) do {                          \
 282         unsigned int __gu_low, __gu_high;                               \
 283         const unsigned int __user *__gu_ptr;                            \
 284         __gu_ptr = (const void __user *)(ptr);                          \
 285         __get_user_asm(__gu_low, __gu_ptr, "l", "=r", label);           \
 286         __get_user_asm(__gu_high, __gu_ptr+1, "l", "=r", label);        \
 287         (x) = ((unsigned long long)__gu_high << 32) | __gu_low;         \
 288 } while (0)
 289 #else
 290 #define __get_user_asm_u64(x, ptr, label)                               \
 291         __get_user_asm(x, ptr, "q", "=r", label)
 292 #endif
 293
 294 #define __get_user_size(x, ptr, size, label)                            \
 295 do {                                                                    \
 296         __chk_user_ptr(ptr);                                            \
 297         switch (size) {                                                 \
 298         case 1: {                                                       \
 299                 unsigned char x_u8__;                                   \
 300                 __get_user_asm(x_u8__, ptr, "b", "=q", label);          \
 301                 (x) = x_u8__;                                           \
 302                 break;                                                  \
 303         }                                                               \
 304         case 2:                                                         \
 305                 __get_user_asm(x, ptr, "w", "=r", label);               \
 306                 break;                                                  \
 307         case 4:                                                         \
 308                 __get_user_asm(x, ptr, "l", "=r", label);               \
 309                 break;                                                  \
 310         case 8:                                                         \
 311                 __get_user_asm_u64(x, ptr, label);                      \
 312                 break;                                                  \
 313         default:                                                        \
 314                 (x) = __get_user_bad();                                 \
 315         }                                                               \
 316         instrument_get_user(x);                                         \
 317 } while (0)
 318
 319 #define __get_user_asm(x, addr, itype, ltype, label)                    \
 320         asm_goto_output("\n"                                            \
 321                      "1:        mov"itype" %[umem],%[output]\n"         \
 322                      _ASM_EXTABLE_UA(1b, %l2)                           \
 323                      : [output] ltype(x)                                \
 324                      : [umem] "m" (__m(addr))                           \
 325                      : : label)
 326
 327 #else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT
 328
 329 #ifdef CONFIG_X86_32
 330 #define __get_user_asm_u64(x, ptr, retval)                              \
 331 ({                                                                      \
 332         __typeof__(ptr) __ptr = (ptr);                                  \
 333         asm volatile("\n"                                               \
 334                      "1:        movl %[lowbits],%%eax\n"                \
 335                      "2:        movl %[highbits],%%edx\n"               \
 336                      "3:\n"                                             \
 337                      _ASM_EXTABLE_TYPE_REG(1b, 3b, EX_TYPE_EFAULT_REG | \
 338                                            EX_FLAG_CLEAR_AX_DX,         \
 339                                            %[errout])                   \
 340                      _ASM_EXTABLE_TYPE_REG(2b, 3b, EX_TYPE_EFAULT_REG | \
 341                                            EX_FLAG_CLEAR_AX_DX,         \
 342                                            %[errout])                   \
 343                      : [errout] "=r" (retval),                          \
 344                        [output] "=&A"(x)                                \
 345                      : [lowbits] "m" (__m(__ptr)),                      \
 346                        [highbits] "m" __m(((u32 __user *)(__ptr)) + 1), \
 347                        "0" (retval));                                   \
 348 })
 349
 350 #else
 351 #define __get_user_asm_u64(x, ptr, retval) \
 352          __get_user_asm(x, ptr, retval, "q")
 353 #endif
 354
 355 #define __get_user_size(x, ptr, size, retval)                           \
 356 do {                                                                    \
 357         unsigned char x_u8__;                                           \
 358                                                                         \
 359         retval = 0;                                                     \
 360         __chk_user_ptr(ptr);                                            \
 361         switch (size) {                                                 \
 362         case 1:                                                         \
 363                 __get_user_asm(x_u8__, ptr, retval, "b");               \
 364                 (x) = x_u8__;                                           \
 365                 break;                                                  \
 366         case 2:                                                         \
 367                 __get_user_asm(x, ptr, retval, "w");                    \
 368                 break;                                                  \
 369         case 4:                                                         \
 370                 __get_user_asm(x, ptr, retval, "l");                    \
 371                 break;                                                  \
 372         case 8:                                                         \
 373                 __get_user_asm_u64(x, ptr, retval);                     \
 374                 break;                                                  \
 375         default:                                                        \
 376                 (x) = __get_user_bad();                                 \
 377         }                                                               \
 378 } while (0)
 379
 380 #define __get_user_asm(x, addr, err, itype)                             \
 381         asm volatile("\n"                                               \
 382                      "1:        mov"itype" %[umem],%[output]\n"         \
 383                      "2:\n"                                             \
 384                      _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG | \
 385                                            EX_FLAG_CLEAR_AX,            \
 386                                            %[errout])                   \
 387                      : [errout] "=r" (err),                             \
 388                        [output] "=a" (x)                                \
 389                      : [umem] "m" (__m(addr)),                          \
 390                        "0" (err))
 391
 392 #endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT
 393
 394 #ifdef CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
 395 #define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label)  ({ \
 396         bool success;                                                   \
 397         __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);              \
 398         __typeof__(*(_ptr)) __old = *_old;                              \
 399         __typeof__(*(_ptr)) __new = (_new);                             \
 400         asm_goto_output("\n"                                            \
 401                      "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\
 402                      _ASM_EXTABLE_UA(1b, %l[label])                     \
 403                      : CC_OUT(z) (success),                             \
 404                        [ptr] "+m" (*_ptr),                              \
 405                        [old] "+a" (__old)                               \
 406                      : [new] ltype (__new)                              \
 407                      : "memory"                                         \
 408                      : label);                                          \
 409         if (unlikely(!success))                                         \
 410                 *_old = __old;                                          \
 411         likely(success);                                        })
 412
 413 #ifdef CONFIG_X86_32
 414 #define __try_cmpxchg64_user_asm(_ptr, _pold, _new, label)      ({      \
 415         bool success;                                                   \
 416         __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);              \
 417         __typeof__(*(_ptr)) __old = *_old;                              \
 418         __typeof__(*(_ptr)) __new = (_new);                             \
 419         asm_goto_output("\n"                                            \
 420                      "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n"             \
 421                      _ASM_EXTABLE_UA(1b, %l[label])                     \
 422                      : CC_OUT(z) (success),                             \
 423                        "+A" (__old),                                    \
 424                        [ptr] "+m" (*_ptr)                               \
 425                      : "b" ((u32)__new),                                \
 426                        "c" ((u32)((u64)__new >> 32))                    \
 427                      : "memory"                                         \
 428                      : label);                                          \
 429         if (unlikely(!success))                                         \
 430                 *_old = __old;                                          \
 431         likely(success);                                        })
 432 #endif // CONFIG_X86_32
 433 #else  // !CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
 434 #define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label)  ({ \
 435         int __err = 0;                                                  \
 436         bool success;                                                   \
 437         __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);              \
 438         __typeof__(*(_ptr)) __old = *_old;                              \
 439         __typeof__(*(_ptr)) __new = (_new);                             \
 440         asm volatile("\n"                                               \
 441                      "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\
 442                      CC_SET(z)                                          \
 443                      "2:\n"                                             \
 444                      _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG,  \
 445                                            %[errout])                   \
 446                      : CC_OUT(z) (success),                             \
 447                        [errout] "+r" (__err),                           \
 448                        [ptr] "+m" (*_ptr),                              \
 449                        [old] "+a" (__old)                               \
 450                      : [new] ltype (__new)                              \
 451                      : "memory");                                       \
 452         if (unlikely(__err))                                            \
 453                 goto label;                                             \
 454         if (unlikely(!success))                                         \
 455                 *_old = __old;                                          \
 456         likely(success);                                        })
 457
 458 #ifdef CONFIG_X86_32
 459 /*
 460  * Unlike the normal CMPXCHG, use output GPR for both success/fail and error.
 461  * There are only six GPRs available and four (EAX, EBX, ECX, and EDX) are
 462  * hardcoded by CMPXCHG8B, leaving only ESI and EDI.  If the compiler uses
 463  * both ESI and EDI for the memory operand, compilation will fail if the error
 464  * is an input+output as there will be no register available for input.
 465  */
 466 #define __try_cmpxchg64_user_asm(_ptr, _pold, _new, label)      ({      \
 467         int __result;                                                   \
 468         __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);              \
 469         __typeof__(*(_ptr)) __old = *_old;                              \
 470         __typeof__(*(_ptr)) __new = (_new);                             \
 471         asm volatile("\n"                                               \
 472                      "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n"             \
 473                      "mov $0, %[result]\n\t"                            \
 474                      "setz %b[result]\n"                                \
 475                      "2:\n"                                             \
 476                      _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG,  \
 477                                            %[result])                   \
 478                      : [result] "=q" (__result),                        \
 479                        "+A" (__old),                                    \
 480                        [ptr] "+m" (*_ptr)                               \
 481                      : "b" ((u32)__new),                                \
 482                        "c" ((u32)((u64)__new >> 32))                    \
 483                      : "memory", "cc");                                 \
 484         if (unlikely(__result < 0))                                     \
 485                 goto label;                                             \
 486         if (unlikely(!__result))                                        \
 487                 *_old = __old;                                          \
 488         likely(__result);                                       })
 489 #endif // CONFIG_X86_32
 490 #endif // CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
 491
 492 /* FIXME: this hack is definitely wrong -AK */
 493 struct __large_struct { unsigned long buf[100]; };
 494 #define __m(x) (*(struct __large_struct __user *)(x))
 495
 496 /*
 497  * Tell gcc we read from memory instead of writing: this is because
 498  * we do not write to any memory gcc knows about, so there are no
 499  * aliasing issues.
 500  */
 501 #define __put_user_goto(x, addr, itype, ltype, label)                   \
 502         asm goto("\n"                                                   \
 503                 "1:     mov"itype" %0,%1\n"                             \
 504                 _ASM_EXTABLE_UA(1b, %l2)                                \
 505                 : : ltype(x), "m" (__m(addr))                           \
 506                 : : label)
 507
 508 extern unsigned long
 509 copy_from_user_nmi(void *to, const void __user *from, unsigned long n);
 510 extern __must_check long
 511 strncpy_from_user(char *dst, const char __user *src, long count);
 512
 513 extern __must_check long strnlen_user(const char __user *str, long n);
 514
 515 #ifdef CONFIG_ARCH_HAS_COPY_MC
 516 unsigned long __must_check
 517 copy_mc_to_kernel(void *to, const void *from, unsigned len);
 518 #define copy_mc_to_kernel copy_mc_to_kernel
 519
 520 unsigned long __must_check
 521 copy_mc_to_user(void __user *to, const void *from, unsigned len);
 522 #endif
 523
 524 /*
 525  * movsl can be slow when source and dest are not both 8-byte aligned
 526  */
 527 #ifdef CONFIG_X86_INTEL_USERCOPY
 528 extern struct movsl_mask {
 529         int mask;
 530 } ____cacheline_aligned_in_smp movsl_mask;
 531 #endif
 532
 533 #define ARCH_HAS_NOCACHE_UACCESS 1
 534
 535 #ifdef CONFIG_X86_32
 536 unsigned long __must_check clear_user(void __user *mem, unsigned long len);
 537 unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
 538 # include <asm/uaccess_32.h>
 539 #else
 540 # include <asm/uaccess_64.h>
 541 #endif
 542
 543 /*
 544  * The "unsafe" user accesses aren't really "unsafe", but the naming
 545  * is a big fat warning: you have to not only do the access_ok()
 546  * checking before using them, but you have to surround them with the
 547  * user_access_begin/end() pair.
 548  */
 549 static __must_check __always_inline bool user_access_begin(const void __user *ptr, size_t len)
 550 {
 551         if (unlikely(!access_ok(ptr,len)))
 552                 return 0;
 553         __uaccess_begin_nospec();
 554         return 1;
 555 }
 556 #define user_access_begin(a,b)  user_access_begin(a,b)
 557 #define user_access_end()       __uaccess_end()
 558
 559 #define user_access_save()      smap_save()
 560 #define user_access_restore(x)  smap_restore(x)
 561
 562 #define unsafe_put_user(x, ptr, label)  \
 563         __put_user_size((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)), label)
 564
 565 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
 566 #define unsafe_get_user(x, ptr, err_label)                                      \
 567 do {                                                                            \
 568         __inttype(*(ptr)) __gu_val;                                             \
 569         __get_user_size(__gu_val, (ptr), sizeof(*(ptr)), err_label);            \
 570         (x) = (__force __typeof__(*(ptr)))__gu_val;                             \
 571 } while (0)
 572 #else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT
 573 #define unsafe_get_user(x, ptr, err_label)                                      \
 574 do {                                                                            \
 575         int __gu_err;                                                           \
 576         __inttype(*(ptr)) __gu_val;                                             \
 577         __get_user_size(__gu_val, (ptr), sizeof(*(ptr)), __gu_err);             \
 578         (x) = (__force __typeof__(*(ptr)))__gu_val;                             \
 579         if (unlikely(__gu_err)) goto err_label;                                 \
 580 } while (0)
 581 #endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT
 582
 583 extern void __try_cmpxchg_user_wrong_size(void);
 584
 585 #ifndef CONFIG_X86_32
 586 #define __try_cmpxchg64_user_asm(_ptr, _oldp, _nval, _label)            \
 587         __try_cmpxchg_user_asm("q", "r", (_ptr), (_oldp), (_nval), _label)
 588 #endif
 589
 590 /*
 591  * Force the pointer to u<size> to match the size expected by the asm helper.
 592  * clang/LLVM compiles all cases and only discards the unused paths after
 593  * processing errors, which breaks i386 if the pointer is an 8-byte value.
 594  */
 595 #define unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label) ({                  \
 596         bool __ret;                                                             \
 597         __chk_user_ptr(_ptr);                                                   \
 598         switch (sizeof(*(_ptr))) {                                              \
 599         case 1: __ret = __try_cmpxchg_user_asm("b", "q",                        \
 600                                                (__force u8 *)(_ptr), (_oldp),   \
 601                                                (_nval), _label);                \
 602                 break;                                                          \
 603         case 2: __ret = __try_cmpxchg_user_asm("w", "r",                        \
 604                                                (__force u16 *)(_ptr), (_oldp),  \
 605                                                (_nval), _label);                \
 606                 break;                                                          \
 607         case 4: __ret = __try_cmpxchg_user_asm("l", "r",                        \
 608                                                (__force u32 *)(_ptr), (_oldp),  \
 609                                                (_nval), _label);                \
 610                 break;                                                          \
 611         case 8: __ret = __try_cmpxchg64_user_asm((__force u64 *)(_ptr), (_oldp),\
 612                                                  (_nval), _label);              \
 613                 break;                                                          \
 614         default: __try_cmpxchg_user_wrong_size();                               \
 615         }                                                                       \
 616         __ret;                                          })
 617
 618 /* "Returns" 0 on success, 1 on failure, -EFAULT if the access faults. */
 619 #define __try_cmpxchg_user(_ptr, _oldp, _nval, _label)  ({              \
 620         int __ret = -EFAULT;                                            \
 621         __uaccess_begin_nospec();                                       \
 622         __ret = !unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label);   \
 623 _label:                                                                 \
 624         __uaccess_end();                                                \
 625         __ret;                                                          \
 626                                                         })
 627
 628 /*
 629  * We want the unsafe accessors to always be inlined and use
 630  * the error labels - thus the macro games.
 631  */
 632 #define unsafe_copy_loop(dst, src, len, type, label)                            \
 633         while (len >= sizeof(type)) {                                           \
 634                 unsafe_put_user(*(type *)(src),(type __user *)(dst),label);     \
 635                 dst += sizeof(type);                                            \
 636                 src += sizeof(type);                                            \
 637                 len -= sizeof(type);                                            \
 638         }
 639
 640 #define unsafe_copy_to_user(_dst,_src,_len,label)                       \
 641 do {                                                                    \
 642         char __user *__ucu_dst = (_dst);                                \
 643         const char *__ucu_src = (_src);                                 \
 644         size_t __ucu_len = (_len);                                      \
 645         unsafe_copy_loop(__ucu_dst, __ucu_src, __ucu_len, u64, label);  \
 646         unsafe_copy_loop(__ucu_dst, __ucu_src, __ucu_len, u32, label);  \
 647         unsafe_copy_loop(__ucu_dst, __ucu_src, __ucu_len, u16, label);  \
 648         unsafe_copy_loop(__ucu_dst, __ucu_src, __ucu_len, u8, label);   \
 649 } while (0)
 650
 651 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
 652 #define __get_kernel_nofault(dst, src, type, err_label)                 \
 653         __get_user_size(*((type *)(dst)), (__force type __user *)(src), \
 654                         sizeof(type), err_label)
 655 #else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT
 656 #define __get_kernel_nofault(dst, src, type, err_label)                 \
 657 do {                                                                    \
 658         int __kr_err;                                                   \
 659                                                                         \
 660         __get_user_size(*((type *)(dst)), (__force type __user *)(src), \
 661                         sizeof(type), __kr_err);                        \
 662         if (unlikely(__kr_err))                                         \
 663                 goto err_label;                                         \
 664 } while (0)
 665 #endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT
 666
 667 #define __put_kernel_nofault(dst, src, type, err_label)                 \
 668         __put_user_size(*((type *)(src)), (__force type __user *)(dst), \
 669                         sizeof(type), err_label)
 670
 671 #endif /* _ASM_X86_UACCESS_H */
 672