arch/x86/include/asm/nospec-branch.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2
   3 #ifndef _ASM_X86_NOSPEC_BRANCH_H_
   4 #define _ASM_X86_NOSPEC_BRANCH_H_
   5
   6 #include <linux/static_key.h>
   7 #include <linux/frame.h>
   8
   9 #include <asm/alternative.h>
  10 #include <asm/alternative-asm.h>
  11 #include <asm/cpufeatures.h>
  12 #include <asm/msr-index.h>
  13 #include <asm/unwind_hints.h>
  14 #include <asm/percpu.h>
  15
  16 #include <linux/frame.h>
  17 #include <asm/unwind_hints.h>
  18 /*
  19  * This should be used immediately before a retpoline alternative. It tells
  20  * objtool where the retpolines are so that it can make sense of the control
  21  * flow by just reading the original instruction(s) and ignoring the
  22  * alternatives.
  23  */
  24 #define ANNOTATE_NOSPEC_ALTERNATIVE \
  25         ANNOTATE_IGNORE_ALTERNATIVE
  26
  27 /*
  28  * Fill the CPU return stack buffer.
  29  *
  30  * Each entry in the RSB, if used for a speculative 'ret', contains an
  31  * infinite 'pause; lfence; jmp' loop to capture speculative execution.
  32  *
  33  * This is required in various cases for retpoline and IBRS-based
  34  * mitigations for the Spectre variant 2 vulnerability. Sometimes to
  35  * eliminate potentially bogus entries from the RSB, and sometimes
  36  * purely to ensure that it doesn't get empty, which on some CPUs would
  37  * allow predictions from other (unwanted!) sources to be used.
  38  *
  39  * We define a CPP macro such that it can be used from both .S files and
  40  * inline assembly. It's possible to do a .macro and then include that
  41  * from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
  42  */
  43
  44 #define RSB_CLEAR_LOOPS         32      /* To forcibly overwrite all entries */
  45 #define RSB_FILL_LOOPS          16      /* To avoid underflow */
  46
  47 /*
  48  * Google experimented with loop-unrolling and this turned out to be
  49  * the optimal version — two calls, each with their own speculation
  50  * trap should their return address end up getting used, in a loop.
  51  */
  52 #ifdef CONFIG_X86_64
  53 #define __FILL_RETURN_BUFFER(reg, nr, sp)       \
  54         mov     $(nr/2), reg;                   \
  55 771:                                            \
  56         ANNOTATE_INTRA_FUNCTION_CALL;           \
  57         call    772f;                           \
  58 773:    /* speculation trap */                  \
  59         UNWIND_HINT_EMPTY;              \
  60         pause;                                  \
  61         lfence;                                 \
  62         jmp     773b;                           \
  63 772:                                            \
  64         ANNOTATE_INTRA_FUNCTION_CALL;           \
  65         call    774f;                           \
  66 775:    /* speculation trap */                  \
  67         UNWIND_HINT_EMPTY;                      \
  68         pause;                                  \
  69         lfence;                                 \
  70         jmp     775b;                           \
  71 774:                                            \
  72         add     $(BITS_PER_LONG/8) * 2, sp;     \
  73         dec     reg;                            \
  74         jnz     771b;                           \
  75         /* barrier for jnz misprediction */     \
  76         lfence;
  77 #else
  78 /*
  79  * i386 doesn't unconditionally have LFENCE, as such it can't
  80  * do a loop.
  81  */
  82 #define __FILL_RETURN_BUFFER(reg, nr, sp)       \
  83         .rept nr;                               \
  84         call    772f;                           \
  85         int3;                                   \
  86 772:;                                           \
  87         .endr;                                  \
  88         add     $(BITS_PER_LONG/8) * nr, sp;
  89 #endif
  90
  91 #ifdef __ASSEMBLY__
  92
  93 /*
  94  * This should be used immediately before an indirect jump/call. It tells
  95  * objtool the subsequent indirect jump/call is vouched safe for retpoline
  96  * builds.
  97  */
  98 .macro ANNOTATE_RETPOLINE_SAFE
  99         .Lannotate_\@:
 100         .pushsection .discard.retpoline_safe
 101         _ASM_PTR .Lannotate_\@
 102         .popsection
 103 .endm
 104
 105 /*
 106  * These are the bare retpoline primitives for indirect jmp and call.
 107  * Do not use these directly; they only exist to make the ALTERNATIVE
 108  * invocation below less ugly.
 109  */
 110 .macro RETPOLINE_JMP reg:req
 111         call    .Ldo_rop_\@
 112 .Lspec_trap_\@:
 113         pause
 114         lfence
 115         jmp     .Lspec_trap_\@
 116 .Ldo_rop_\@:
 117         mov     \reg, (%_ASM_SP)
 118         ret
 119 .endm
 120
 121 /*
 122  * This is a wrapper around RETPOLINE_JMP so the called function in reg
 123  * returns to the instruction after the macro.
 124  */
 125 .macro RETPOLINE_CALL reg:req
 126         jmp     .Ldo_call_\@
 127 .Ldo_retpoline_jmp_\@:
 128         RETPOLINE_JMP \reg
 129 .Ldo_call_\@:
 130         call    .Ldo_retpoline_jmp_\@
 131 .endm
 132
 133 /*
 134  * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
 135  * indirect jmp/call which may be susceptible to the Spectre variant 2
 136  * attack.
 137  */
 138 .macro JMP_NOSPEC reg:req
 139 #ifdef CONFIG_RETPOLINE
 140         ANNOTATE_NOSPEC_ALTERNATIVE
 141         ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *\reg),  \
 142                 __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \
 143                 __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *\reg), X86_FEATURE_RETPOLINE_LFENCE
 144 #else
 145         jmp     *\reg
 146 #endif
 147 .endm
 148
 149 .macro CALL_NOSPEC reg:req
 150 #ifdef CONFIG_RETPOLINE
 151         ANNOTATE_NOSPEC_ALTERNATIVE
 152         ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *\reg), \
 153                 __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
 154                 __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *\reg), X86_FEATURE_RETPOLINE_LFENCE
 155 #else
 156         call    *\reg
 157 #endif
 158 .endm
 159
 160 .macro ISSUE_UNBALANCED_RET_GUARD
 161         ANNOTATE_INTRA_FUNCTION_CALL;
 162         call .Lunbalanced_ret_guard_\@
 163         int3
 164 .Lunbalanced_ret_guard_\@:
 165         add $(BITS_PER_LONG/8), %_ASM_SP
 166         lfence
 167 .endm
 168
 169  /*
 170   * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
 171   * monstrosity above, manually.
 172   */
 173 .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req ftr2
 174 .ifb \ftr2
 175         ALTERNATIVE "jmp .Lskip_rsb_\@", "", \ftr
 176 .else
 177         ALTERNATIVE_2 "jmp .Lskip_rsb_\@", "", \ftr, "jmp .Lunbalanced_\@", \ftr2
 178 .endif
 179         __FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)
 180 .Lunbalanced_\@:
 181         ISSUE_UNBALANCED_RET_GUARD
 182 .Lskip_rsb_\@:
 183 .endm
 184
 185 #else /* __ASSEMBLY__ */
 186
 187 #define ANNOTATE_RETPOLINE_SAFE                                 \
 188         "999:\n\t"                                              \
 189         ".pushsection .discard.retpoline_safe\n\t"              \
 190         _ASM_PTR " 999b\n\t"                                    \
 191         ".popsection\n\t"
 192
 193 #ifdef CONFIG_RETPOLINE
 194 #ifdef CONFIG_X86_64
 195
 196 /*
 197  * Inline asm uses the %V modifier which is only in newer GCC
 198  * which is ensured when CONFIG_RETPOLINE is defined.
 199  */
 200 # define CALL_NOSPEC                                            \
 201         ANNOTATE_NOSPEC_ALTERNATIVE                             \
 202         ALTERNATIVE_2(                                          \
 203         ANNOTATE_RETPOLINE_SAFE                                 \
 204         "call *%[thunk_target]\n",                              \
 205         "call __x86_indirect_thunk_%V[thunk_target]\n",         \
 206         X86_FEATURE_RETPOLINE,                                  \
 207         "lfence;\n"                                             \
 208         ANNOTATE_RETPOLINE_SAFE                                 \
 209         "call *%[thunk_target]\n",                              \
 210         X86_FEATURE_RETPOLINE_LFENCE)
 211 # define THUNK_TARGET(addr) [thunk_target] "r" (addr)
 212
 213 #else /* CONFIG_X86_32 */
 214 /*
 215  * For i386 we use the original ret-equivalent retpoline, because
 216  * otherwise we'll run out of registers. We don't care about CET
 217  * here, anyway.
 218  */
 219 # define CALL_NOSPEC                                            \
 220         ANNOTATE_NOSPEC_ALTERNATIVE                             \
 221         ALTERNATIVE_2(                                          \
 222         ANNOTATE_RETPOLINE_SAFE                                 \
 223         "call *%[thunk_target]\n",                              \
 224         "       jmp    904f;\n"                                 \
 225         "       .align 16\n"                                    \
 226         "901:   call   903f;\n"                                 \
 227         "902:   pause;\n"                                       \
 228         "       lfence;\n"                                      \
 229         "       jmp    902b;\n"                                 \
 230         "       .align 16\n"                                    \
 231         "903:   lea    4(%%esp), %%esp;\n"                      \
 232         "       pushl  %[thunk_target];\n"                      \
 233         "       ret;\n"                                         \
 234         "       .align 16\n"                                    \
 235         "904:   call   901b;\n",                                \
 236         X86_FEATURE_RETPOLINE,                                  \
 237         "lfence;\n"                                             \
 238         ANNOTATE_RETPOLINE_SAFE                                 \
 239         "call *%[thunk_target]\n",                              \
 240         X86_FEATURE_RETPOLINE_LFENCE)
 241
 242 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 243 #endif
 244 #else /* No retpoline for C / inline asm */
 245 # define CALL_NOSPEC "call *%[thunk_target]\n"
 246 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 247 #endif
 248
 249 /* The Spectre V2 mitigation variants */
 250 enum spectre_v2_mitigation {
 251         SPECTRE_V2_NONE,
 252         SPECTRE_V2_RETPOLINE,
 253         SPECTRE_V2_LFENCE,
 254         SPECTRE_V2_EIBRS,
 255         SPECTRE_V2_EIBRS_RETPOLINE,
 256         SPECTRE_V2_EIBRS_LFENCE,
 257         SPECTRE_V2_IBRS,
 258 };
 259
 260 /* The indirect branch speculation control variants */
 261 enum spectre_v2_user_mitigation {
 262         SPECTRE_V2_USER_NONE,
 263         SPECTRE_V2_USER_STRICT,
 264         SPECTRE_V2_USER_STRICT_PREFERRED,
 265         SPECTRE_V2_USER_PRCTL,
 266         SPECTRE_V2_USER_SECCOMP,
 267 };
 268
 269 /* The Speculative Store Bypass disable variants */
 270 enum ssb_mitigation {
 271         SPEC_STORE_BYPASS_NONE,
 272         SPEC_STORE_BYPASS_DISABLE,
 273         SPEC_STORE_BYPASS_PRCTL,
 274         SPEC_STORE_BYPASS_SECCOMP,
 275 };
 276
 277 extern char __indirect_thunk_start[];
 278 extern char __indirect_thunk_end[];
 279
 280 /*
 281  * On VMEXIT we must ensure that no RSB predictions learned in the guest
 282  * can be followed in the host, by overwriting the RSB completely. Both
 283  * retpoline and IBRS mitigations for Spectre v2 need this; only on future
 284  * CPUs with IBRS_ALL *might* it be avoided.
 285  */
 286 static inline void vmexit_fill_RSB(void)
 287 {
 288 #ifdef CONFIG_RETPOLINE
 289         unsigned long loops;
 290
 291         asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
 292                       ALTERNATIVE("jmp 910f",
 293                                   __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
 294                                   X86_FEATURE_RETPOLINE)
 295                       "910:"
 296                       : "=r" (loops), ASM_CALL_CONSTRAINT
 297                       : : "memory" );
 298 #endif
 299 }
 300
 301 static __always_inline
 302 void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature)
 303 {
 304         asm volatile(ALTERNATIVE("", "wrmsr", %c[feature])
 305                 : : "c" (msr),
 306                     "a" ((u32)val),
 307                     "d" ((u32)(val >> 32)),
 308                     [feature] "i" (feature)
 309                 : "memory");
 310 }
 311
 312 static inline void indirect_branch_prediction_barrier(void)
 313 {
 314         u64 val = PRED_CMD_IBPB;
 315
 316         alternative_msr_write(MSR_IA32_PRED_CMD, val, X86_FEATURE_USE_IBPB);
 317 }
 318
 319 /* The Intel SPEC CTRL MSR base value cache */
 320 extern u64 x86_spec_ctrl_base;
 321 DECLARE_PER_CPU(u64, x86_spec_ctrl_current);
 322 extern void update_spec_ctrl_cond(u64 val);
 323 extern u64 spec_ctrl_current(void);
 324
 325 /*
 326  * With retpoline, we must use IBRS to restrict branch prediction
 327  * before calling into firmware.
 328  *
 329  * (Implemented as CPP macros due to header hell.)
 330  */
 331 #define firmware_restrict_branch_speculation_start()                    \
 332 do {                                                                    \
 333         preempt_disable();                                              \
 334         alternative_msr_write(MSR_IA32_SPEC_CTRL,                       \
 335                               spec_ctrl_current() | SPEC_CTRL_IBRS,     \
 336                               X86_FEATURE_USE_IBRS_FW);                 \
 337 } while (0)
 338
 339 #define firmware_restrict_branch_speculation_end()                      \
 340 do {                                                                    \
 341         alternative_msr_write(MSR_IA32_SPEC_CTRL,                       \
 342                               spec_ctrl_current(),                      \
 343                               X86_FEATURE_USE_IBRS_FW);                 \
 344         preempt_enable();                                               \
 345 } while (0)
 346
 347 DECLARE_STATIC_KEY_FALSE(switch_to_cond_stibp);
 348 DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
 349 DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
 350
 351 DECLARE_STATIC_KEY_FALSE(mds_user_clear);
 352 DECLARE_STATIC_KEY_FALSE(mds_idle_clear);
 353
 354 DECLARE_STATIC_KEY_FALSE(mmio_stale_data_clear);
 355
 356 #include <asm/segment.h>
 357
 358 /**
 359  * mds_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability
 360  *
 361  * This uses the otherwise unused and obsolete VERW instruction in
 362  * combination with microcode which triggers a CPU buffer flush when the
 363  * instruction is executed.
 364  */
 365 static __always_inline void mds_clear_cpu_buffers(void)
 366 {
 367         static const u16 ds = __KERNEL_DS;
 368
 369         /*
 370          * Has to be the memory-operand variant because only that
 371          * guarantees the CPU buffer flush functionality according to
 372          * documentation. The register-operand variant does not.
 373          * Works with any segment selector, but a valid writable
 374          * data segment is the fastest variant.
 375          *
 376          * "cc" clobber is required because VERW modifies ZF.
 377          */
 378         asm volatile("verw %[ds]" : : [ds] "m" (ds) : "cc");
 379 }
 380
 381 /**
 382  * mds_user_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability
 383  *
 384  * Clear CPU buffers if the corresponding static key is enabled
 385  */
 386 static __always_inline void mds_user_clear_cpu_buffers(void)
 387 {
 388         if (static_branch_likely(&mds_user_clear))
 389                 mds_clear_cpu_buffers();
 390 }
 391
 392 /**
 393  * mds_idle_clear_cpu_buffers - Mitigation for MDS vulnerability
 394  *
 395  * Clear CPU buffers if the corresponding static key is enabled
 396  */
 397 static inline void mds_idle_clear_cpu_buffers(void)
 398 {
 399         if (static_branch_likely(&mds_idle_clear))
 400                 mds_clear_cpu_buffers();
 401 }
 402
 403 #endif /* __ASSEMBLY__ */
 404
 405 /*
 406  * Below is used in the eBPF JIT compiler and emits the byte sequence
 407  * for the following assembly:
 408  *
 409  * With retpolines configured:
 410  *
 411  *    callq do_rop
 412  *  spec_trap:
 413  *    pause
 414  *    lfence
 415  *    jmp spec_trap
 416  *  do_rop:
 417  *    mov %rax,(%rsp) for x86_64
 418  *    mov %edx,(%esp) for x86_32
 419  *    retq
 420  *
 421  * Without retpolines configured:
 422  *
 423  *    jmp *%rax for x86_64
 424  *    jmp *%edx for x86_32
 425  */
 426 #ifdef CONFIG_RETPOLINE
 427 # ifdef CONFIG_X86_64
 428 #  define RETPOLINE_RAX_BPF_JIT_SIZE    17
 429 #  define RETPOLINE_RAX_BPF_JIT()                               \
 430 do {                                                            \
 431         EMIT1_off32(0xE8, 7);    /* callq do_rop */             \
 432         /* spec_trap: */                                        \
 433         EMIT2(0xF3, 0x90);       /* pause */                    \
 434         EMIT3(0x0F, 0xAE, 0xE8); /* lfence */                   \
 435         EMIT2(0xEB, 0xF9);       /* jmp spec_trap */            \
 436         /* do_rop: */                                           \
 437         EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */    \
 438         EMIT1(0xC3);             /* retq */                     \
 439 } while (0)
 440 # else /* !CONFIG_X86_64 */
 441 #  define RETPOLINE_EDX_BPF_JIT()                               \
 442 do {                                                            \
 443         EMIT1_off32(0xE8, 7);    /* call do_rop */              \
 444         /* spec_trap: */                                        \
 445         EMIT2(0xF3, 0x90);       /* pause */                    \
 446         EMIT3(0x0F, 0xAE, 0xE8); /* lfence */                   \
 447         EMIT2(0xEB, 0xF9);       /* jmp spec_trap */            \
 448         /* do_rop: */                                           \
 449         EMIT3(0x89, 0x14, 0x24); /* mov %edx,(%esp) */          \
 450         EMIT1(0xC3);             /* ret */                      \
 451 } while (0)
 452 # endif
 453 #else /* !CONFIG_RETPOLINE */
 454 # ifdef CONFIG_X86_64
 455 #  define RETPOLINE_RAX_BPF_JIT_SIZE    2
 456 #  define RETPOLINE_RAX_BPF_JIT()                               \
 457         EMIT2(0xFF, 0xE0);       /* jmp *%rax */
 458 # else /* !CONFIG_X86_64 */
 459 #  define RETPOLINE_EDX_BPF_JIT()                               \
 460         EMIT2(0xFF, 0xE2)        /* jmp *%edx */
 461 # endif
 462 #endif
 463
 464 #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */