arch/x86/kernel/cpu/bugs.c

   1 /*
   2  *  Copyright (C) 1994  Linus Torvalds
   3  *
   4  *  Cyrix stuff, June 1998 by:
   5  *      - Rafael R. Reilova (moved everything from head.S),
   6  *        <rreilova@ececs.uc.edu>
   7  *      - Channing Corn (tests & fixes),
   8  *      - Andrew D. Balsa (code cleanup).
   9  */
  10 #include <linux/init.h>
  11 #include <linux/utsname.h>
  12 #include <linux/cpu.h>
  13 #include <linux/module.h>
  14 #include <linux/nospec.h>
  15 #include <linux/prctl.h>
  16 #include <linux/sched/smt.h>
  17
  18 #include <asm/spec-ctrl.h>
  19 #include <asm/cmdline.h>
  20 #include <asm/bugs.h>
  21 #include <asm/processor.h>
  22 #include <asm/processor-flags.h>
  23 #include <asm/fpu/internal.h>
  24 #include <asm/msr.h>
  25 #include <asm/vmx.h>
  26 #include <asm/paravirt.h>
  27 #include <asm/alternative.h>
  28 #include <asm/hypervisor.h>
  29 #include <asm/pgtable.h>
  30 #include <asm/cacheflush.h>
  31 #include <asm/intel-family.h>
  32 #include <asm/e820.h>
  33 #include <linux/bpf.h>
  34
  35 #include "cpu.h"
  36
  37 static void __init spectre_v1_select_mitigation(void);
  38 static void __init spectre_v2_select_mitigation(void);
  39 static void __init ssb_select_mitigation(void);
  40 static void __init l1tf_select_mitigation(void);
  41 static void __init mds_select_mitigation(void);
  42 static void __init mds_print_mitigation(void);
  43 static void __init taa_select_mitigation(void);
  44 static void __init srbds_select_mitigation(void);
  45
  46 /* The base value of the SPEC_CTRL MSR that always has to be preserved. */
  47 u64 x86_spec_ctrl_base;
  48 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
  49 static DEFINE_MUTEX(spec_ctrl_mutex);
  50
  51 /*
  52  * The vendor and possibly platform specific bits which can be modified in
  53  * x86_spec_ctrl_base.
  54  */
  55 static u64 __ro_after_init x86_spec_ctrl_mask = SPEC_CTRL_IBRS;
  56
  57 /*
  58  * AMD specific MSR info for Speculative Store Bypass control.
  59  * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
  60  */
  61 u64 __ro_after_init x86_amd_ls_cfg_base;
  62 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
  63
  64 /* Control conditional STIBP in switch_to() */
  65 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp);
  66 /* Control conditional IBPB in switch_mm() */
  67 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
  68 /* Control unconditional IBPB in switch_mm() */
  69 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
  70
  71 /* Control MDS CPU buffer clear before returning to user space */
  72 DEFINE_STATIC_KEY_FALSE(mds_user_clear);
  73 EXPORT_SYMBOL_GPL(mds_user_clear);
  74 /* Control MDS CPU buffer clear before idling (halt, mwait) */
  75 DEFINE_STATIC_KEY_FALSE(mds_idle_clear);
  76 EXPORT_SYMBOL_GPL(mds_idle_clear);
  77
  78 void __init check_bugs(void)
  79 {
  80         identify_boot_cpu();
  81
  82         /*
  83          * identify_boot_cpu() initialized SMT support information, let the
  84          * core code know.
  85          */
  86         cpu_smt_check_topology_early();
  87
  88         if (!IS_ENABLED(CONFIG_SMP)) {
  89                 pr_info("CPU: ");
  90                 print_cpu_info(&boot_cpu_data);
  91         }
  92
  93         /*
  94          * Read the SPEC_CTRL MSR to account for reserved bits which may
  95          * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
  96          * init code as it is not enumerated and depends on the family.
  97          */
  98         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
  99                 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 100
 101         /* Allow STIBP in MSR_SPEC_CTRL if supported */
 102         if (boot_cpu_has(X86_FEATURE_STIBP))
 103                 x86_spec_ctrl_mask |= SPEC_CTRL_STIBP;
 104
 105         /* Select the proper CPU mitigations before patching alternatives: */
 106         spectre_v1_select_mitigation();
 107         spectre_v2_select_mitigation();
 108         ssb_select_mitigation();
 109         l1tf_select_mitigation();
 110         mds_select_mitigation();
 111         taa_select_mitigation();
 112         srbds_select_mitigation();
 113
 114         /*
 115          * As MDS and TAA mitigations are inter-related, print MDS
 116          * mitigation until after TAA mitigation selection is done.
 117          */
 118         mds_print_mitigation();
 119
 120         arch_smt_update();
 121
 122 #ifdef CONFIG_X86_32
 123         /*
 124          * Check whether we are able to run this kernel safely on SMP.
 125          *
 126          * - i386 is no longer supported.
 127          * - In order to run on anything without a TSC, we need to be
 128          *   compiled for a i486.
 129          */
 130         if (boot_cpu_data.x86 < 4)
 131                 panic("Kernel requires i486+ for 'invlpg' and other features");
 132
 133         init_utsname()->machine[1] =
 134                 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
 135         alternative_instructions();
 136
 137         fpu__init_check_bugs();
 138 #else /* CONFIG_X86_64 */
 139         alternative_instructions();
 140
 141         /*
 142          * Make sure the first 2MB area is not mapped by huge pages
 143          * There are typically fixed size MTRRs in there and overlapping
 144          * MTRRs into large pages causes slow downs.
 145          *
 146          * Right now we don't do that with gbpages because there seems
 147          * very little benefit for that case.
 148          */
 149         if (!direct_gbpages)
 150                 set_memory_4k((unsigned long)__va(0), 1);
 151 #endif
 152 }
 153
 154 void
 155 x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
 156 {
 157         u64 msrval, guestval, hostval = x86_spec_ctrl_base;
 158         struct thread_info *ti = current_thread_info();
 159
 160         /* Is MSR_SPEC_CTRL implemented ? */
 161         if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) {
 162                 /*
 163                  * Restrict guest_spec_ctrl to supported values. Clear the
 164                  * modifiable bits in the host base value and or the
 165                  * modifiable bits from the guest value.
 166                  */
 167                 guestval = hostval & ~x86_spec_ctrl_mask;
 168                 guestval |= guest_spec_ctrl & x86_spec_ctrl_mask;
 169
 170                 /* SSBD controlled in MSR_SPEC_CTRL */
 171                 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
 172                     static_cpu_has(X86_FEATURE_AMD_SSBD))
 173                         hostval |= ssbd_tif_to_spec_ctrl(ti->flags);
 174
 175                 /* Conditional STIBP enabled? */
 176                 if (static_branch_unlikely(&switch_to_cond_stibp))
 177                         hostval |= stibp_tif_to_spec_ctrl(ti->flags);
 178
 179                 if (hostval != guestval) {
 180                         msrval = setguest ? guestval : hostval;
 181                         wrmsrl(MSR_IA32_SPEC_CTRL, msrval);
 182                 }
 183         }
 184
 185         /*
 186          * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
 187          * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
 188          */
 189         if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
 190             !static_cpu_has(X86_FEATURE_VIRT_SSBD))
 191                 return;
 192
 193         /*
 194          * If the host has SSBD mitigation enabled, force it in the host's
 195          * virtual MSR value. If its not permanently enabled, evaluate
 196          * current's TIF_SSBD thread flag.
 197          */
 198         if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
 199                 hostval = SPEC_CTRL_SSBD;
 200         else
 201                 hostval = ssbd_tif_to_spec_ctrl(ti->flags);
 202
 203         /* Sanitize the guest value */
 204         guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
 205
 206         if (hostval != guestval) {
 207                 unsigned long tif;
 208
 209                 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
 210                                  ssbd_spec_ctrl_to_tif(hostval);
 211
 212                 speculation_ctrl_update(tif);
 213         }
 214 }
 215 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
 216
 217 static void x86_amd_ssb_disable(void)
 218 {
 219         u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
 220
 221         if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
 222                 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
 223         else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
 224                 wrmsrl(MSR_AMD64_LS_CFG, msrval);
 225 }
 226
 227 #undef pr_fmt
 228 #define pr_fmt(fmt)     "MDS: " fmt
 229
 230 /* Default mitigation for MDS-affected CPUs */
 231 static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL;
 232 static bool mds_nosmt __ro_after_init = false;
 233
 234 static const char * const mds_strings[] = {
 235         [MDS_MITIGATION_OFF]    = "Vulnerable",
 236         [MDS_MITIGATION_FULL]   = "Mitigation: Clear CPU buffers",
 237         [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode",
 238 };
 239
 240 static void __init mds_select_mitigation(void)
 241 {
 242         if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) {
 243                 mds_mitigation = MDS_MITIGATION_OFF;
 244                 return;
 245         }
 246
 247         if (mds_mitigation == MDS_MITIGATION_FULL) {
 248                 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR))
 249                         mds_mitigation = MDS_MITIGATION_VMWERV;
 250
 251                 static_branch_enable(&mds_user_clear);
 252
 253                 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) &&
 254                     (mds_nosmt || cpu_mitigations_auto_nosmt()))
 255                         cpu_smt_disable(false);
 256         }
 257 }
 258
 259 static void __init mds_print_mitigation(void)
 260 {
 261         if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off())
 262                 return;
 263
 264         pr_info("%s\n", mds_strings[mds_mitigation]);
 265 }
 266
 267 static int __init mds_cmdline(char *str)
 268 {
 269         if (!boot_cpu_has_bug(X86_BUG_MDS))
 270                 return 0;
 271
 272         if (!str)
 273                 return -EINVAL;
 274
 275         if (!strcmp(str, "off"))
 276                 mds_mitigation = MDS_MITIGATION_OFF;
 277         else if (!strcmp(str, "full"))
 278                 mds_mitigation = MDS_MITIGATION_FULL;
 279         else if (!strcmp(str, "full,nosmt")) {
 280                 mds_mitigation = MDS_MITIGATION_FULL;
 281                 mds_nosmt = true;
 282         }
 283
 284         return 0;
 285 }
 286 early_param("mds", mds_cmdline);
 287
 288 #undef pr_fmt
 289 #define pr_fmt(fmt)     "TAA: " fmt
 290
 291 /* Default mitigation for TAA-affected CPUs */
 292 static enum taa_mitigations taa_mitigation __ro_after_init = TAA_MITIGATION_VERW;
 293 static bool taa_nosmt __ro_after_init;
 294
 295 static const char * const taa_strings[] = {
 296         [TAA_MITIGATION_OFF]            = "Vulnerable",
 297         [TAA_MITIGATION_UCODE_NEEDED]   = "Vulnerable: Clear CPU buffers attempted, no microcode",
 298         [TAA_MITIGATION_VERW]           = "Mitigation: Clear CPU buffers",
 299         [TAA_MITIGATION_TSX_DISABLED]   = "Mitigation: TSX disabled",
 300 };
 301
 302 static void __init taa_select_mitigation(void)
 303 {
 304         u64 ia32_cap;
 305
 306         if (!boot_cpu_has_bug(X86_BUG_TAA)) {
 307                 taa_mitigation = TAA_MITIGATION_OFF;
 308                 return;
 309         }
 310
 311         /* TSX previously disabled by tsx=off */
 312         if (!boot_cpu_has(X86_FEATURE_RTM)) {
 313                 taa_mitigation = TAA_MITIGATION_TSX_DISABLED;
 314                 goto out;
 315         }
 316
 317         if (cpu_mitigations_off()) {
 318                 taa_mitigation = TAA_MITIGATION_OFF;
 319                 return;
 320         }
 321
 322         /*
 323          * TAA mitigation via VERW is turned off if both
 324          * tsx_async_abort=off and mds=off are specified.
 325          */
 326         if (taa_mitigation == TAA_MITIGATION_OFF &&
 327             mds_mitigation == MDS_MITIGATION_OFF)
 328                 goto out;
 329
 330         if (boot_cpu_has(X86_FEATURE_MD_CLEAR))
 331                 taa_mitigation = TAA_MITIGATION_VERW;
 332         else
 333                 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
 334
 335         /*
 336          * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1.
 337          * A microcode update fixes this behavior to clear CPU buffers. It also
 338          * adds support for MSR_IA32_TSX_CTRL which is enumerated by the
 339          * ARCH_CAP_TSX_CTRL_MSR bit.
 340          *
 341          * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode
 342          * update is required.
 343          */
 344         ia32_cap = x86_read_arch_cap_msr();
 345         if ( (ia32_cap & ARCH_CAP_MDS_NO) &&
 346             !(ia32_cap & ARCH_CAP_TSX_CTRL_MSR))
 347                 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
 348
 349         /*
 350          * TSX is enabled, select alternate mitigation for TAA which is
 351          * the same as MDS. Enable MDS static branch to clear CPU buffers.
 352          *
 353          * For guests that can't determine whether the correct microcode is
 354          * present on host, enable the mitigation for UCODE_NEEDED as well.
 355          */
 356         static_branch_enable(&mds_user_clear);
 357
 358         if (taa_nosmt || cpu_mitigations_auto_nosmt())
 359                 cpu_smt_disable(false);
 360
 361         /*
 362          * Update MDS mitigation, if necessary, as the mds_user_clear is
 363          * now enabled for TAA mitigation.
 364          */
 365         if (mds_mitigation == MDS_MITIGATION_OFF &&
 366             boot_cpu_has_bug(X86_BUG_MDS)) {
 367                 mds_mitigation = MDS_MITIGATION_FULL;
 368                 mds_select_mitigation();
 369         }
 370 out:
 371         pr_info("%s\n", taa_strings[taa_mitigation]);
 372 }
 373
 374 static int __init tsx_async_abort_parse_cmdline(char *str)
 375 {
 376         if (!boot_cpu_has_bug(X86_BUG_TAA))
 377                 return 0;
 378
 379         if (!str)
 380                 return -EINVAL;
 381
 382         if (!strcmp(str, "off")) {
 383                 taa_mitigation = TAA_MITIGATION_OFF;
 384         } else if (!strcmp(str, "full")) {
 385                 taa_mitigation = TAA_MITIGATION_VERW;
 386         } else if (!strcmp(str, "full,nosmt")) {
 387                 taa_mitigation = TAA_MITIGATION_VERW;
 388                 taa_nosmt = true;
 389         }
 390
 391         return 0;
 392 }
 393 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline);
 394
 395 #undef pr_fmt
 396 #define pr_fmt(fmt)     "SRBDS: " fmt
 397
 398 enum srbds_mitigations {
 399         SRBDS_MITIGATION_OFF,
 400         SRBDS_MITIGATION_UCODE_NEEDED,
 401         SRBDS_MITIGATION_FULL,
 402         SRBDS_MITIGATION_TSX_OFF,
 403         SRBDS_MITIGATION_HYPERVISOR,
 404 };
 405
 406 static enum srbds_mitigations srbds_mitigation __ro_after_init = SRBDS_MITIGATION_FULL;
 407
 408 static const char * const srbds_strings[] = {
 409         [SRBDS_MITIGATION_OFF]          = "Vulnerable",
 410         [SRBDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode",
 411         [SRBDS_MITIGATION_FULL]         = "Mitigation: Microcode",
 412         [SRBDS_MITIGATION_TSX_OFF]      = "Mitigation: TSX disabled",
 413         [SRBDS_MITIGATION_HYPERVISOR]   = "Unknown: Dependent on hypervisor status",
 414 };
 415
 416 static bool srbds_off;
 417
 418 void update_srbds_msr(void)
 419 {
 420         u64 mcu_ctrl;
 421
 422         if (!boot_cpu_has_bug(X86_BUG_SRBDS))
 423                 return;
 424
 425         if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
 426                 return;
 427
 428         if (srbds_mitigation == SRBDS_MITIGATION_UCODE_NEEDED)
 429                 return;
 430
 431         rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 432
 433         switch (srbds_mitigation) {
 434         case SRBDS_MITIGATION_OFF:
 435         case SRBDS_MITIGATION_TSX_OFF:
 436                 mcu_ctrl |= RNGDS_MITG_DIS;
 437                 break;
 438         case SRBDS_MITIGATION_FULL:
 439                 mcu_ctrl &= ~RNGDS_MITG_DIS;
 440                 break;
 441         default:
 442                 break;
 443         }
 444
 445         wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 446 }
 447
 448 static void __init srbds_select_mitigation(void)
 449 {
 450         u64 ia32_cap;
 451
 452         if (!boot_cpu_has_bug(X86_BUG_SRBDS))
 453                 return;
 454
 455         /*
 456          * Check to see if this is one of the MDS_NO systems supporting
 457          * TSX that are only exposed to SRBDS when TSX is enabled.
 458          */
 459         ia32_cap = x86_read_arch_cap_msr();
 460         if ((ia32_cap & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM))
 461                 srbds_mitigation = SRBDS_MITIGATION_TSX_OFF;
 462         else if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
 463                 srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR;
 464         else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL))
 465                 srbds_mitigation = SRBDS_MITIGATION_UCODE_NEEDED;
 466         else if (cpu_mitigations_off() || srbds_off)
 467                 srbds_mitigation = SRBDS_MITIGATION_OFF;
 468
 469         update_srbds_msr();
 470         pr_info("%s\n", srbds_strings[srbds_mitigation]);
 471 }
 472
 473 static int __init srbds_parse_cmdline(char *str)
 474 {
 475         if (!str)
 476                 return -EINVAL;
 477
 478         if (!boot_cpu_has_bug(X86_BUG_SRBDS))
 479                 return 0;
 480
 481         srbds_off = !strcmp(str, "off");
 482         return 0;
 483 }
 484 early_param("srbds", srbds_parse_cmdline);
 485
 486 #undef pr_fmt
 487 #define pr_fmt(fmt)     "Spectre V1 : " fmt
 488
 489 enum spectre_v1_mitigation {
 490         SPECTRE_V1_MITIGATION_NONE,
 491         SPECTRE_V1_MITIGATION_AUTO,
 492 };
 493
 494 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init =
 495         SPECTRE_V1_MITIGATION_AUTO;
 496
 497 static const char * const spectre_v1_strings[] = {
 498         [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers",
 499         [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization",
 500 };
 501
 502 /*
 503  * Does SMAP provide full mitigation against speculative kernel access to
 504  * userspace?
 505  */
 506 static bool smap_works_speculatively(void)
 507 {
 508         if (!boot_cpu_has(X86_FEATURE_SMAP))
 509                 return false;
 510
 511         /*
 512          * On CPUs which are vulnerable to Meltdown, SMAP does not
 513          * prevent speculative access to user data in the L1 cache.
 514          * Consider SMAP to be non-functional as a mitigation on these
 515          * CPUs.
 516          */
 517         if (boot_cpu_has(X86_BUG_CPU_MELTDOWN))
 518                 return false;
 519
 520         return true;
 521 }
 522
 523 static void __init spectre_v1_select_mitigation(void)
 524 {
 525         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) {
 526                 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
 527                 return;
 528         }
 529
 530         if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) {
 531                 /*
 532                  * With Spectre v1, a user can speculatively control either
 533                  * path of a conditional swapgs with a user-controlled GS
 534                  * value.  The mitigation is to add lfences to both code paths.
 535                  *
 536                  * If FSGSBASE is enabled, the user can put a kernel address in
 537                  * GS, in which case SMAP provides no protection.
 538                  *
 539                  * [ NOTE: Don't check for X86_FEATURE_FSGSBASE until the
 540                  *         FSGSBASE enablement patches have been merged. ]
 541                  *
 542                  * If FSGSBASE is disabled, the user can only put a user space
 543                  * address in GS.  That makes an attack harder, but still
 544                  * possible if there's no SMAP protection.
 545                  */
 546                 if (!smap_works_speculatively()) {
 547                         /*
 548                          * Mitigation can be provided from SWAPGS itself or
 549                          * PTI as the CR3 write in the Meltdown mitigation
 550                          * is serializing.
 551                          *
 552                          * If neither is there, mitigate with an LFENCE to
 553                          * stop speculation through swapgs.
 554                          */
 555                         if (boot_cpu_has_bug(X86_BUG_SWAPGS) &&
 556                             !boot_cpu_has(X86_FEATURE_KAISER))
 557                                 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER);
 558
 559                         /*
 560                          * Enable lfences in the kernel entry (non-swapgs)
 561                          * paths, to prevent user entry from speculatively
 562                          * skipping swapgs.
 563                          */
 564                         setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL);
 565                 }
 566         }
 567
 568         pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]);
 569 }
 570
 571 static int __init nospectre_v1_cmdline(char *str)
 572 {
 573         spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
 574         return 0;
 575 }
 576 early_param("nospectre_v1", nospectre_v1_cmdline);
 577
 578 #undef pr_fmt
 579 #define pr_fmt(fmt)     "Spectre V2 : " fmt
 580
 581 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
 582         SPECTRE_V2_NONE;
 583
 584 static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init =
 585         SPECTRE_V2_USER_NONE;
 586 static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init =
 587         SPECTRE_V2_USER_NONE;
 588
 589 #ifdef CONFIG_RETPOLINE
 590 static bool spectre_v2_bad_module;
 591
 592 bool retpoline_module_ok(bool has_retpoline)
 593 {
 594         if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
 595                 return true;
 596
 597         pr_err("System may be vulnerable to spectre v2\n");
 598         spectre_v2_bad_module = true;
 599         return false;
 600 }
 601
 602 static inline const char *spectre_v2_module_string(void)
 603 {
 604         return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
 605 }
 606 #else
 607 static inline const char *spectre_v2_module_string(void) { return ""; }
 608 #endif
 609
 610 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n"
 611 #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n"
 612 #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n"
 613
 614 #ifdef CONFIG_BPF_SYSCALL
 615 void unpriv_ebpf_notify(int new_state)
 616 {
 617         if (new_state)
 618                 return;
 619
 620         /* Unprivileged eBPF is enabled */
 621
 622         switch (spectre_v2_enabled) {
 623         case SPECTRE_V2_EIBRS:
 624                 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
 625                 break;
 626         case SPECTRE_V2_EIBRS_LFENCE:
 627                 if (sched_smt_active())
 628                         pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
 629                 break;
 630         default:
 631                 break;
 632         }
 633 }
 634 #endif
 635
 636 static inline bool match_option(const char *arg, int arglen, const char *opt)
 637 {
 638         int len = strlen(opt);
 639
 640         return len == arglen && !strncmp(arg, opt, len);
 641 }
 642
 643 /* The kernel command line selection for spectre v2 */
 644 enum spectre_v2_mitigation_cmd {
 645         SPECTRE_V2_CMD_NONE,
 646         SPECTRE_V2_CMD_AUTO,
 647         SPECTRE_V2_CMD_FORCE,
 648         SPECTRE_V2_CMD_RETPOLINE,
 649         SPECTRE_V2_CMD_RETPOLINE_GENERIC,
 650         SPECTRE_V2_CMD_RETPOLINE_LFENCE,
 651         SPECTRE_V2_CMD_EIBRS,
 652         SPECTRE_V2_CMD_EIBRS_RETPOLINE,
 653         SPECTRE_V2_CMD_EIBRS_LFENCE,
 654 };
 655
 656 enum spectre_v2_user_cmd {
 657         SPECTRE_V2_USER_CMD_NONE,
 658         SPECTRE_V2_USER_CMD_AUTO,
 659         SPECTRE_V2_USER_CMD_FORCE,
 660         SPECTRE_V2_USER_CMD_PRCTL,
 661         SPECTRE_V2_USER_CMD_PRCTL_IBPB,
 662         SPECTRE_V2_USER_CMD_SECCOMP,
 663         SPECTRE_V2_USER_CMD_SECCOMP_IBPB,
 664 };
 665
 666 static const char * const spectre_v2_user_strings[] = {
 667         [SPECTRE_V2_USER_NONE]                  = "User space: Vulnerable",
 668         [SPECTRE_V2_USER_STRICT]                = "User space: Mitigation: STIBP protection",
 669         [SPECTRE_V2_USER_STRICT_PREFERRED]      = "User space: Mitigation: STIBP always-on protection",
 670         [SPECTRE_V2_USER_PRCTL]                 = "User space: Mitigation: STIBP via prctl",
 671         [SPECTRE_V2_USER_SECCOMP]               = "User space: Mitigation: STIBP via seccomp and prctl",
 672 };
 673
 674 static const struct {
 675         const char                      *option;
 676         enum spectre_v2_user_cmd        cmd;
 677         bool                            secure;
 678 } v2_user_options[] __initconst = {
 679         { "auto",               SPECTRE_V2_USER_CMD_AUTO,               false },
 680         { "off",                SPECTRE_V2_USER_CMD_NONE,               false },
 681         { "on",                 SPECTRE_V2_USER_CMD_FORCE,              true  },
 682         { "prctl",              SPECTRE_V2_USER_CMD_PRCTL,              false },
 683         { "prctl,ibpb",         SPECTRE_V2_USER_CMD_PRCTL_IBPB,         false },
 684         { "seccomp",            SPECTRE_V2_USER_CMD_SECCOMP,            false },
 685         { "seccomp,ibpb",       SPECTRE_V2_USER_CMD_SECCOMP_IBPB,       false },
 686 };
 687
 688 static void __init spec_v2_user_print_cond(const char *reason, bool secure)
 689 {
 690         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
 691                 pr_info("spectre_v2_user=%s forced on command line.\n", reason);
 692 }
 693
 694 static enum spectre_v2_user_cmd __init
 695 spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd)
 696 {
 697         char arg[20];
 698         int ret, i;
 699
 700         switch (v2_cmd) {
 701         case SPECTRE_V2_CMD_NONE:
 702                 return SPECTRE_V2_USER_CMD_NONE;
 703         case SPECTRE_V2_CMD_FORCE:
 704                 return SPECTRE_V2_USER_CMD_FORCE;
 705         default:
 706                 break;
 707         }
 708
 709         ret = cmdline_find_option(boot_command_line, "spectre_v2_user",
 710                                   arg, sizeof(arg));
 711         if (ret < 0)
 712                 return SPECTRE_V2_USER_CMD_AUTO;
 713
 714         for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) {
 715                 if (match_option(arg, ret, v2_user_options[i].option)) {
 716                         spec_v2_user_print_cond(v2_user_options[i].option,
 717                                                 v2_user_options[i].secure);
 718                         return v2_user_options[i].cmd;
 719                 }
 720         }
 721
 722         pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg);
 723         return SPECTRE_V2_USER_CMD_AUTO;
 724 }
 725
 726 static inline bool spectre_v2_in_eibrs_mode(enum spectre_v2_mitigation mode)
 727 {
 728         return (mode == SPECTRE_V2_EIBRS ||
 729                 mode == SPECTRE_V2_EIBRS_RETPOLINE ||
 730                 mode == SPECTRE_V2_EIBRS_LFENCE);
 731 }
 732
 733 static void __init
 734 spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd)
 735 {
 736         enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE;
 737         bool smt_possible = IS_ENABLED(CONFIG_SMP);
 738         enum spectre_v2_user_cmd cmd;
 739
 740         if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP))
 741                 return;
 742
 743         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
 744             cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
 745                 smt_possible = false;
 746
 747         cmd = spectre_v2_parse_user_cmdline(v2_cmd);
 748         switch (cmd) {
 749         case SPECTRE_V2_USER_CMD_NONE:
 750                 goto set_mode;
 751         case SPECTRE_V2_USER_CMD_FORCE:
 752                 mode = SPECTRE_V2_USER_STRICT;
 753                 break;
 754         case SPECTRE_V2_USER_CMD_PRCTL:
 755         case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
 756                 mode = SPECTRE_V2_USER_PRCTL;
 757                 break;
 758         case SPECTRE_V2_USER_CMD_AUTO:
 759         case SPECTRE_V2_USER_CMD_SECCOMP:
 760         case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
 761                 if (IS_ENABLED(CONFIG_SECCOMP))
 762                         mode = SPECTRE_V2_USER_SECCOMP;
 763                 else
 764                         mode = SPECTRE_V2_USER_PRCTL;
 765                 break;
 766         }
 767
 768         /* Initialize Indirect Branch Prediction Barrier */
 769         if (boot_cpu_has(X86_FEATURE_IBPB)) {
 770                 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
 771
 772                 spectre_v2_user_ibpb = mode;
 773                 switch (cmd) {
 774                 case SPECTRE_V2_USER_CMD_FORCE:
 775                 case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
 776                 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
 777                         static_branch_enable(&switch_mm_always_ibpb);
 778                         spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT;
 779                         break;
 780                 case SPECTRE_V2_USER_CMD_PRCTL:
 781                 case SPECTRE_V2_USER_CMD_AUTO:
 782                 case SPECTRE_V2_USER_CMD_SECCOMP:
 783                         static_branch_enable(&switch_mm_cond_ibpb);
 784                         break;
 785                 default:
 786                         break;
 787                 }
 788
 789                 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
 790                         static_key_enabled(&switch_mm_always_ibpb) ?
 791                         "always-on" : "conditional");
 792         }
 793
 794         /*
 795          * If no STIBP, enhanced IBRS is enabled or SMT impossible, STIBP is not
 796          * required.
 797          */
 798         if (!boot_cpu_has(X86_FEATURE_STIBP) ||
 799             !smt_possible ||
 800             spectre_v2_in_eibrs_mode(spectre_v2_enabled))
 801                 return;
 802
 803         /*
 804          * At this point, an STIBP mode other than "off" has been set.
 805          * If STIBP support is not being forced, check if STIBP always-on
 806          * is preferred.
 807          */
 808         if (mode != SPECTRE_V2_USER_STRICT &&
 809             boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON))
 810                 mode = SPECTRE_V2_USER_STRICT_PREFERRED;
 811
 812         spectre_v2_user_stibp = mode;
 813
 814 set_mode:
 815         pr_info("%s\n", spectre_v2_user_strings[mode]);
 816 }
 817
 818 static const char * const spectre_v2_strings[] = {
 819         [SPECTRE_V2_NONE]                       = "Vulnerable",
 820         [SPECTRE_V2_RETPOLINE]                  = "Mitigation: Retpolines",
 821         [SPECTRE_V2_LFENCE]                     = "Mitigation: LFENCE",
 822         [SPECTRE_V2_EIBRS]                      = "Mitigation: Enhanced IBRS",
 823         [SPECTRE_V2_EIBRS_LFENCE]               = "Mitigation: Enhanced IBRS + LFENCE",
 824         [SPECTRE_V2_EIBRS_RETPOLINE]            = "Mitigation: Enhanced IBRS + Retpolines",
 825 };
 826
 827 static const struct {
 828         const char *option;
 829         enum spectre_v2_mitigation_cmd cmd;
 830         bool secure;
 831 } mitigation_options[] __initconst = {
 832         { "off",                SPECTRE_V2_CMD_NONE,              false },
 833         { "on",                 SPECTRE_V2_CMD_FORCE,             true  },
 834         { "retpoline",          SPECTRE_V2_CMD_RETPOLINE,         false },
 835         { "retpoline,amd",      SPECTRE_V2_CMD_RETPOLINE_LFENCE,  false },
 836         { "retpoline,lfence",   SPECTRE_V2_CMD_RETPOLINE_LFENCE,  false },
 837         { "retpoline,generic",  SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
 838         { "eibrs",              SPECTRE_V2_CMD_EIBRS,             false },
 839         { "eibrs,lfence",       SPECTRE_V2_CMD_EIBRS_LFENCE,      false },
 840         { "eibrs,retpoline",    SPECTRE_V2_CMD_EIBRS_RETPOLINE,   false },
 841         { "auto",               SPECTRE_V2_CMD_AUTO,              false },
 842 };
 843
 844 static void __init spec_v2_print_cond(const char *reason, bool secure)
 845 {
 846         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
 847                 pr_info("%s selected on command line.\n", reason);
 848 }
 849
 850 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
 851 {
 852         enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
 853         char arg[20];
 854         int ret, i;
 855
 856         if (cmdline_find_option_bool(boot_command_line, "nospectre_v2") ||
 857             cpu_mitigations_off())
 858                 return SPECTRE_V2_CMD_NONE;
 859
 860         ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
 861         if (ret < 0)
 862                 return SPECTRE_V2_CMD_AUTO;
 863
 864         for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
 865                 if (!match_option(arg, ret, mitigation_options[i].option))
 866                         continue;
 867                 cmd = mitigation_options[i].cmd;
 868                 break;
 869         }
 870
 871         if (i >= ARRAY_SIZE(mitigation_options)) {
 872                 pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 873                 return SPECTRE_V2_CMD_AUTO;
 874         }
 875
 876         if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
 877              cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
 878              cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC ||
 879              cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
 880              cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
 881             !IS_ENABLED(CONFIG_RETPOLINE)) {
 882                 pr_err("%s selected but not compiled in. Switching to AUTO select\n",
 883                        mitigation_options[i].option);
 884                 return SPECTRE_V2_CMD_AUTO;
 885         }
 886
 887         if ((cmd == SPECTRE_V2_CMD_EIBRS ||
 888              cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
 889              cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
 890             !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
 891                 pr_err("%s selected but CPU doesn't have eIBRS. Switching to AUTO select\n",
 892                        mitigation_options[i].option);
 893                 return SPECTRE_V2_CMD_AUTO;
 894         }
 895
 896         if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
 897              cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) &&
 898             !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
 899                 pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n",
 900                        mitigation_options[i].option);
 901                 return SPECTRE_V2_CMD_AUTO;
 902         }
 903
 904         spec_v2_print_cond(mitigation_options[i].option,
 905                            mitigation_options[i].secure);
 906         return cmd;
 907 }
 908
 909 static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void)
 910 {
 911         if (!IS_ENABLED(CONFIG_RETPOLINE)) {
 912                 pr_err("Kernel not compiled with retpoline; no mitigation available!");
 913                 return SPECTRE_V2_NONE;
 914         }
 915
 916         return SPECTRE_V2_RETPOLINE;
 917 }
 918
 919 static void __init spectre_v2_select_mitigation(void)
 920 {
 921         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
 922         enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
 923
 924         /*
 925          * If the CPU is not affected and the command line mode is NONE or AUTO
 926          * then nothing to do.
 927          */
 928         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
 929             (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
 930                 return;
 931
 932         switch (cmd) {
 933         case SPECTRE_V2_CMD_NONE:
 934                 return;
 935
 936         case SPECTRE_V2_CMD_FORCE:
 937         case SPECTRE_V2_CMD_AUTO:
 938                 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
 939                         mode = SPECTRE_V2_EIBRS;
 940                         break;
 941                 }
 942
 943                 mode = spectre_v2_select_retpoline();
 944                 break;
 945
 946         case SPECTRE_V2_CMD_RETPOLINE_LFENCE:
 947                 pr_err(SPECTRE_V2_LFENCE_MSG);
 948                 mode = SPECTRE_V2_LFENCE;
 949                 break;
 950
 951         case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
 952                 mode = SPECTRE_V2_RETPOLINE;
 953                 break;
 954
 955         case SPECTRE_V2_CMD_RETPOLINE:
 956                 mode = spectre_v2_select_retpoline();
 957                 break;
 958
 959         case SPECTRE_V2_CMD_EIBRS:
 960                 mode = SPECTRE_V2_EIBRS;
 961                 break;
 962
 963         case SPECTRE_V2_CMD_EIBRS_LFENCE:
 964                 mode = SPECTRE_V2_EIBRS_LFENCE;
 965                 break;
 966
 967         case SPECTRE_V2_CMD_EIBRS_RETPOLINE:
 968                 mode = SPECTRE_V2_EIBRS_RETPOLINE;
 969                 break;
 970         }
 971
 972         if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
 973                 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
 974
 975         if (spectre_v2_in_eibrs_mode(mode)) {
 976                 /* Force it so VMEXIT will restore correctly */
 977                 x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
 978                 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 979         }
 980
 981         switch (mode) {
 982         case SPECTRE_V2_NONE:
 983         case SPECTRE_V2_EIBRS:
 984                 break;
 985
 986         case SPECTRE_V2_LFENCE:
 987         case SPECTRE_V2_EIBRS_LFENCE:
 988                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE);
 989                 /* fallthrough */
 990
 991         case SPECTRE_V2_RETPOLINE:
 992         case SPECTRE_V2_EIBRS_RETPOLINE:
 993                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 994                 break;
 995         }
 996
 997         spectre_v2_enabled = mode;
 998         pr_info("%s\n", spectre_v2_strings[mode]);
 999
1000         /*
1001          * If spectre v2 protection has been enabled, unconditionally fill
1002          * RSB during a context switch; this protects against two independent
1003          * issues:
1004          *
1005          *      - RSB underflow (and switch to BTB) on Skylake+
1006          *      - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs
1007          */
1008         setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
1009         pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
1010
1011         /*
1012          * Retpoline means the kernel is safe because it has no indirect
1013          * branches. Enhanced IBRS protects firmware too, so, enable restricted
1014          * speculation around firmware calls only when Enhanced IBRS isn't
1015          * supported.
1016          *
1017          * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
1018          * the user might select retpoline on the kernel command line and if
1019          * the CPU supports Enhanced IBRS, kernel might un-intentionally not
1020          * enable IBRS around firmware calls.
1021          */
1022         if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_eibrs_mode(mode)) {
1023                 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
1024                 pr_info("Enabling Restricted Speculation for firmware calls\n");
1025         }
1026
1027         /* Set up IBPB and STIBP depending on the general spectre V2 command */
1028         spectre_v2_user_select_mitigation(cmd);
1029 }
1030
1031 static void update_stibp_msr(void * __unused)
1032 {
1033         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
1034 }
1035
1036 /* Update x86_spec_ctrl_base in case SMT state changed. */
1037 static void update_stibp_strict(void)
1038 {
1039         u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP;
1040
1041         if (sched_smt_active())
1042                 mask |= SPEC_CTRL_STIBP;
1043
1044         if (mask == x86_spec_ctrl_base)
1045                 return;
1046
1047         pr_info("Update user space SMT mitigation: STIBP %s\n",
1048                 mask & SPEC_CTRL_STIBP ? "always-on" : "off");
1049         x86_spec_ctrl_base = mask;
1050         on_each_cpu(update_stibp_msr, NULL, 1);
1051 }
1052
1053 /* Update the static key controlling the evaluation of TIF_SPEC_IB */
1054 static void update_indir_branch_cond(void)
1055 {
1056         if (sched_smt_active())
1057                 static_branch_enable(&switch_to_cond_stibp);
1058         else
1059                 static_branch_disable(&switch_to_cond_stibp);
1060 }
1061
1062 #undef pr_fmt
1063 #define pr_fmt(fmt) fmt
1064
1065 /* Update the static key controlling the MDS CPU buffer clear in idle */
1066 static void update_mds_branch_idle(void)
1067 {
1068         /*
1069          * Enable the idle clearing if SMT is active on CPUs which are
1070          * affected only by MSBDS and not any other MDS variant.
1071          *
1072          * The other variants cannot be mitigated when SMT is enabled, so
1073          * clearing the buffers on idle just to prevent the Store Buffer
1074          * repartitioning leak would be a window dressing exercise.
1075          */
1076         if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY))
1077                 return;
1078
1079         if (sched_smt_active())
1080                 static_branch_enable(&mds_idle_clear);
1081         else
1082                 static_branch_disable(&mds_idle_clear);
1083 }
1084
1085 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n"
1086 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n"
1087
1088 void arch_smt_update(void)
1089 {
1090         mutex_lock(&spec_ctrl_mutex);
1091
1092         if (sched_smt_active() && unprivileged_ebpf_enabled() &&
1093             spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
1094                 pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
1095
1096         switch (spectre_v2_user_stibp) {
1097         case SPECTRE_V2_USER_NONE:
1098                 break;
1099         case SPECTRE_V2_USER_STRICT:
1100         case SPECTRE_V2_USER_STRICT_PREFERRED:
1101                 update_stibp_strict();
1102                 break;
1103         case SPECTRE_V2_USER_PRCTL:
1104         case SPECTRE_V2_USER_SECCOMP:
1105                 update_indir_branch_cond();
1106                 break;
1107         }
1108
1109         switch (mds_mitigation) {
1110         case MDS_MITIGATION_FULL:
1111         case MDS_MITIGATION_VMWERV:
1112                 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY))
1113                         pr_warn_once(MDS_MSG_SMT);
1114                 update_mds_branch_idle();
1115                 break;
1116         case MDS_MITIGATION_OFF:
1117                 break;
1118         }
1119
1120         switch (taa_mitigation) {
1121         case TAA_MITIGATION_VERW:
1122         case TAA_MITIGATION_UCODE_NEEDED:
1123                 if (sched_smt_active())
1124                         pr_warn_once(TAA_MSG_SMT);
1125                 break;
1126         case TAA_MITIGATION_TSX_DISABLED:
1127         case TAA_MITIGATION_OFF:
1128                 break;
1129         }
1130
1131         mutex_unlock(&spec_ctrl_mutex);
1132 }
1133
1134 #undef pr_fmt
1135 #define pr_fmt(fmt)     "Speculative Store Bypass: " fmt
1136
1137 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE;
1138
1139 /* The kernel command line selection */
1140 enum ssb_mitigation_cmd {
1141         SPEC_STORE_BYPASS_CMD_NONE,
1142         SPEC_STORE_BYPASS_CMD_AUTO,
1143         SPEC_STORE_BYPASS_CMD_ON,
1144         SPEC_STORE_BYPASS_CMD_PRCTL,
1145         SPEC_STORE_BYPASS_CMD_SECCOMP,
1146 };
1147
1148 static const char * const ssb_strings[] = {
1149         [SPEC_STORE_BYPASS_NONE]        = "Vulnerable",
1150         [SPEC_STORE_BYPASS_DISABLE]     = "Mitigation: Speculative Store Bypass disabled",
1151         [SPEC_STORE_BYPASS_PRCTL]       = "Mitigation: Speculative Store Bypass disabled via prctl",
1152         [SPEC_STORE_BYPASS_SECCOMP]     = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
1153 };
1154
1155 static const struct {
1156         const char *option;
1157         enum ssb_mitigation_cmd cmd;
1158 } ssb_mitigation_options[]  __initconst = {
1159         { "auto",       SPEC_STORE_BYPASS_CMD_AUTO },    /* Platform decides */
1160         { "on",         SPEC_STORE_BYPASS_CMD_ON },      /* Disable Speculative Store Bypass */
1161         { "off",        SPEC_STORE_BYPASS_CMD_NONE },    /* Don't touch Speculative Store Bypass */
1162         { "prctl",      SPEC_STORE_BYPASS_CMD_PRCTL },   /* Disable Speculative Store Bypass via prctl */
1163         { "seccomp",    SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */
1164 };
1165
1166 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
1167 {
1168         enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
1169         char arg[20];
1170         int ret, i;
1171
1172         if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable") ||
1173             cpu_mitigations_off()) {
1174                 return SPEC_STORE_BYPASS_CMD_NONE;
1175         } else {
1176                 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
1177                                           arg, sizeof(arg));
1178                 if (ret < 0)
1179                         return SPEC_STORE_BYPASS_CMD_AUTO;
1180
1181                 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
1182                         if (!match_option(arg, ret, ssb_mitigation_options[i].option))
1183                                 continue;
1184
1185                         cmd = ssb_mitigation_options[i].cmd;
1186                         break;
1187                 }
1188
1189                 if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
1190                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
1191                         return SPEC_STORE_BYPASS_CMD_AUTO;
1192                 }
1193         }
1194
1195         return cmd;
1196 }
1197
1198 static enum ssb_mitigation __init __ssb_select_mitigation(void)
1199 {
1200         enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
1201         enum ssb_mitigation_cmd cmd;
1202
1203         if (!boot_cpu_has(X86_FEATURE_SSBD))
1204                 return mode;
1205
1206         cmd = ssb_parse_cmdline();
1207         if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
1208             (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
1209              cmd == SPEC_STORE_BYPASS_CMD_AUTO))
1210                 return mode;
1211
1212         switch (cmd) {
1213         case SPEC_STORE_BYPASS_CMD_AUTO:
1214         case SPEC_STORE_BYPASS_CMD_SECCOMP:
1215                 /*
1216                  * Choose prctl+seccomp as the default mode if seccomp is
1217                  * enabled.
1218                  */
1219                 if (IS_ENABLED(CONFIG_SECCOMP))
1220                         mode = SPEC_STORE_BYPASS_SECCOMP;
1221                 else
1222                         mode = SPEC_STORE_BYPASS_PRCTL;
1223                 break;
1224         case SPEC_STORE_BYPASS_CMD_ON:
1225                 mode = SPEC_STORE_BYPASS_DISABLE;
1226                 break;
1227         case SPEC_STORE_BYPASS_CMD_PRCTL:
1228                 mode = SPEC_STORE_BYPASS_PRCTL;
1229                 break;
1230         case SPEC_STORE_BYPASS_CMD_NONE:
1231                 break;
1232         }
1233
1234         /*
1235          * If SSBD is controlled by the SPEC_CTRL MSR, then set the proper
1236          * bit in the mask to allow guests to use the mitigation even in the
1237          * case where the host does not enable it.
1238          */
1239         if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
1240             static_cpu_has(X86_FEATURE_AMD_SSBD)) {
1241                 x86_spec_ctrl_mask |= SPEC_CTRL_SSBD;
1242         }
1243
1244         /*
1245          * We have three CPU feature flags that are in play here:
1246          *  - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
1247          *  - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
1248          *  - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
1249          */
1250         if (mode == SPEC_STORE_BYPASS_DISABLE) {
1251                 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
1252                 /*
1253                  * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
1254                  * use a completely different MSR and bit dependent on family.
1255                  */
1256                 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
1257                     !static_cpu_has(X86_FEATURE_AMD_SSBD)) {
1258                         x86_amd_ssb_disable();
1259                 } else {
1260                         x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
1261                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
1262                 }
1263         }
1264
1265         return mode;
1266 }
1267
1268 static void ssb_select_mitigation(void)
1269 {
1270         ssb_mode = __ssb_select_mitigation();
1271
1272         if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
1273                 pr_info("%s\n", ssb_strings[ssb_mode]);
1274 }
1275
1276 #undef pr_fmt
1277 #define pr_fmt(fmt)     "Speculation prctl: " fmt
1278
1279 static void task_update_spec_tif(struct task_struct *tsk)
1280 {
1281         /* Force the update of the real TIF bits */
1282         set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE);
1283
1284         /*
1285          * Immediately update the speculation control MSRs for the current
1286          * task, but for a non-current task delay setting the CPU
1287          * mitigation until it is scheduled next.
1288          *
1289          * This can only happen for SECCOMP mitigation. For PRCTL it's
1290          * always the current task.
1291          */
1292         if (tsk == current)
1293                 speculation_ctrl_update_current();
1294 }
1295
1296 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
1297 {
1298         if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
1299             ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
1300                 return -ENXIO;
1301
1302         switch (ctrl) {
1303         case PR_SPEC_ENABLE:
1304                 /* If speculation is force disabled, enable is not allowed */
1305                 if (task_spec_ssb_force_disable(task))
1306                         return -EPERM;
1307                 task_clear_spec_ssb_disable(task);
1308                 task_update_spec_tif(task);
1309                 break;
1310         case PR_SPEC_DISABLE:
1311                 task_set_spec_ssb_disable(task);
1312                 task_update_spec_tif(task);
1313                 break;
1314         case PR_SPEC_FORCE_DISABLE:
1315                 task_set_spec_ssb_disable(task);
1316                 task_set_spec_ssb_force_disable(task);
1317                 task_update_spec_tif(task);
1318                 break;
1319         default:
1320                 return -ERANGE;
1321         }
1322         return 0;
1323 }
1324
1325 static bool is_spec_ib_user_controlled(void)
1326 {
1327         return spectre_v2_user_ibpb == SPECTRE_V2_USER_PRCTL ||
1328                 spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP ||
1329                 spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL ||
1330                 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP;
1331 }
1332
1333 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl)
1334 {
1335         switch (ctrl) {
1336         case PR_SPEC_ENABLE:
1337                 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
1338                     spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
1339                         return 0;
1340                 /*
1341                  * With strict mode for both IBPB and STIBP, the instruction
1342                  * code paths avoid checking this task flag and instead,
1343                  * unconditionally run the instruction. However, STIBP and IBPB
1344                  * are independent and either can be set to conditionally
1345                  * enabled regardless of the mode of the other.
1346                  *
1347                  * If either is set to conditional, allow the task flag to be
1348                  * updated, unless it was force-disabled by a previous prctl
1349                  * call. Currently, this is possible on an AMD CPU which has the
1350                  * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the
1351                  * kernel is booted with 'spectre_v2_user=seccomp', then
1352                  * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and
1353                  * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED.
1354                  */
1355                 if (!is_spec_ib_user_controlled() ||
1356                     task_spec_ib_force_disable(task))
1357                         return -EPERM;
1358
1359                 task_clear_spec_ib_disable(task);
1360                 task_update_spec_tif(task);
1361                 break;
1362         case PR_SPEC_DISABLE:
1363         case PR_SPEC_FORCE_DISABLE:
1364                 /*
1365                  * Indirect branch speculation is always allowed when
1366                  * mitigation is force disabled.
1367                  */
1368                 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
1369                     spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
1370                         return -EPERM;
1371
1372                 if (!is_spec_ib_user_controlled())
1373                         return 0;
1374
1375                 task_set_spec_ib_disable(task);
1376                 if (ctrl == PR_SPEC_FORCE_DISABLE)
1377                         task_set_spec_ib_force_disable(task);
1378                 task_update_spec_tif(task);
1379                 break;
1380         default:
1381                 return -ERANGE;
1382         }
1383         return 0;
1384 }
1385
1386 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
1387                              unsigned long ctrl)
1388 {
1389         switch (which) {
1390         case PR_SPEC_STORE_BYPASS:
1391                 return ssb_prctl_set(task, ctrl);
1392         case PR_SPEC_INDIRECT_BRANCH:
1393                 return ib_prctl_set(task, ctrl);
1394         default:
1395                 return -ENODEV;
1396         }
1397 }
1398
1399 #ifdef CONFIG_SECCOMP
1400 void arch_seccomp_spec_mitigate(struct task_struct *task)
1401 {
1402         if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
1403                 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
1404         if (spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP ||
1405             spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP)
1406                 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE);
1407 }
1408 #endif
1409
1410 static int ssb_prctl_get(struct task_struct *task)
1411 {
1412         switch (ssb_mode) {
1413         case SPEC_STORE_BYPASS_DISABLE:
1414                 return PR_SPEC_DISABLE;
1415         case SPEC_STORE_BYPASS_SECCOMP:
1416         case SPEC_STORE_BYPASS_PRCTL:
1417                 if (task_spec_ssb_force_disable(task))
1418                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
1419                 if (task_spec_ssb_disable(task))
1420                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
1421                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
1422         default:
1423                 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
1424                         return PR_SPEC_ENABLE;
1425                 return PR_SPEC_NOT_AFFECTED;
1426         }
1427 }
1428
1429 static int ib_prctl_get(struct task_struct *task)
1430 {
1431         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
1432                 return PR_SPEC_NOT_AFFECTED;
1433
1434         if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
1435             spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
1436                 return PR_SPEC_ENABLE;
1437         else if (is_spec_ib_user_controlled()) {
1438                 if (task_spec_ib_force_disable(task))
1439                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
1440                 if (task_spec_ib_disable(task))
1441                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
1442                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
1443         } else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT ||
1444             spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT ||
1445             spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED)
1446                 return PR_SPEC_DISABLE;
1447         else
1448                 return PR_SPEC_NOT_AFFECTED;
1449 }
1450
1451 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
1452 {
1453         switch (which) {
1454         case PR_SPEC_STORE_BYPASS:
1455                 return ssb_prctl_get(task);
1456         case PR_SPEC_INDIRECT_BRANCH:
1457                 return ib_prctl_get(task);
1458         default:
1459                 return -ENODEV;
1460         }
1461 }
1462
1463 void x86_spec_ctrl_setup_ap(void)
1464 {
1465         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
1466                 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
1467
1468         if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
1469                 x86_amd_ssb_disable();
1470 }
1471
1472 bool itlb_multihit_kvm_mitigation;
1473 EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation);
1474
1475 #undef pr_fmt
1476 #define pr_fmt(fmt)     "L1TF: " fmt
1477
1478 /* Default mitigation for L1TF-affected CPUs */
1479 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH;
1480 #if IS_ENABLED(CONFIG_KVM_INTEL)
1481 EXPORT_SYMBOL_GPL(l1tf_mitigation);
1482 #endif
1483 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
1484 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation);
1485
1486 /*
1487  * These CPUs all support 44bits physical address space internally in the
1488  * cache but CPUID can report a smaller number of physical address bits.
1489  *
1490  * The L1TF mitigation uses the top most address bit for the inversion of
1491  * non present PTEs. When the installed memory reaches into the top most
1492  * address bit due to memory holes, which has been observed on machines
1493  * which report 36bits physical address bits and have 32G RAM installed,
1494  * then the mitigation range check in l1tf_select_mitigation() triggers.
1495  * This is a false positive because the mitigation is still possible due to
1496  * the fact that the cache uses 44bit internally. Use the cache bits
1497  * instead of the reported physical bits and adjust them on the affected
1498  * machines to 44bit if the reported bits are less than 44.
1499  */
1500 static void override_cache_bits(struct cpuinfo_x86 *c)
1501 {
1502         if (c->x86 != 6)
1503                 return;
1504
1505         switch (c->x86_model) {
1506         case INTEL_FAM6_NEHALEM:
1507         case INTEL_FAM6_WESTMERE:
1508         case INTEL_FAM6_SANDYBRIDGE:
1509         case INTEL_FAM6_IVYBRIDGE:
1510         case INTEL_FAM6_HASWELL_CORE:
1511         case INTEL_FAM6_HASWELL_ULT:
1512         case INTEL_FAM6_HASWELL_GT3E:
1513         case INTEL_FAM6_BROADWELL_CORE:
1514         case INTEL_FAM6_BROADWELL_GT3E:
1515         case INTEL_FAM6_SKYLAKE_MOBILE:
1516         case INTEL_FAM6_SKYLAKE_DESKTOP:
1517         case INTEL_FAM6_KABYLAKE_MOBILE:
1518         case INTEL_FAM6_KABYLAKE_DESKTOP:
1519                 if (c->x86_cache_bits < 44)
1520                         c->x86_cache_bits = 44;
1521                 break;
1522         }
1523 }
1524
1525 static void __init l1tf_select_mitigation(void)
1526 {
1527         u64 half_pa;
1528
1529         if (!boot_cpu_has_bug(X86_BUG_L1TF))
1530                 return;
1531
1532         if (cpu_mitigations_off())
1533                 l1tf_mitigation = L1TF_MITIGATION_OFF;
1534         else if (cpu_mitigations_auto_nosmt())
1535                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
1536
1537         override_cache_bits(&boot_cpu_data);
1538
1539         switch (l1tf_mitigation) {
1540         case L1TF_MITIGATION_OFF:
1541         case L1TF_MITIGATION_FLUSH_NOWARN:
1542         case L1TF_MITIGATION_FLUSH:
1543                 break;
1544         case L1TF_MITIGATION_FLUSH_NOSMT:
1545         case L1TF_MITIGATION_FULL:
1546                 cpu_smt_disable(false);
1547                 break;
1548         case L1TF_MITIGATION_FULL_FORCE:
1549                 cpu_smt_disable(true);
1550                 break;
1551         }
1552
1553 #if CONFIG_PGTABLE_LEVELS == 2
1554         pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
1555         return;
1556 #endif
1557
1558         half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT;
1559         if (l1tf_mitigation != L1TF_MITIGATION_OFF &&
1560                         e820_any_mapped(half_pa, ULLONG_MAX - half_pa, E820_RAM)) {
1561                 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
1562                 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
1563                                 half_pa);
1564                 pr_info("However, doing so will make a part of your RAM unusable.\n");
1565                 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n");
1566                 return;
1567         }
1568
1569         setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV);
1570 }
1571
1572 static int __init l1tf_cmdline(char *str)
1573 {
1574         if (!boot_cpu_has_bug(X86_BUG_L1TF))
1575                 return 0;
1576
1577         if (!str)
1578                 return -EINVAL;
1579
1580         if (!strcmp(str, "off"))
1581                 l1tf_mitigation = L1TF_MITIGATION_OFF;
1582         else if (!strcmp(str, "flush,nowarn"))
1583                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN;
1584         else if (!strcmp(str, "flush"))
1585                 l1tf_mitigation = L1TF_MITIGATION_FLUSH;
1586         else if (!strcmp(str, "flush,nosmt"))
1587                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
1588         else if (!strcmp(str, "full"))
1589                 l1tf_mitigation = L1TF_MITIGATION_FULL;
1590         else if (!strcmp(str, "full,force"))
1591                 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE;
1592
1593         return 0;
1594 }
1595 early_param("l1tf", l1tf_cmdline);
1596
1597 #undef pr_fmt
1598 #define pr_fmt(fmt) fmt
1599
1600 #ifdef CONFIG_SYSFS
1601
1602 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
1603
1604 #if IS_ENABLED(CONFIG_KVM_INTEL)
1605 static const char * const l1tf_vmx_states[] = {
1606         [VMENTER_L1D_FLUSH_AUTO]                = "auto",
1607         [VMENTER_L1D_FLUSH_NEVER]               = "vulnerable",
1608         [VMENTER_L1D_FLUSH_COND]                = "conditional cache flushes",
1609         [VMENTER_L1D_FLUSH_ALWAYS]              = "cache flushes",
1610         [VMENTER_L1D_FLUSH_EPT_DISABLED]        = "EPT disabled",
1611         [VMENTER_L1D_FLUSH_NOT_REQUIRED]        = "flush not necessary"
1612 };
1613
1614 static ssize_t l1tf_show_state(char *buf)
1615 {
1616         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO)
1617                 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
1618
1619         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED ||
1620             (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER &&
1621              sched_smt_active())) {
1622                 return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG,
1623                                l1tf_vmx_states[l1tf_vmx_mitigation]);
1624         }
1625
1626         return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG,
1627                        l1tf_vmx_states[l1tf_vmx_mitigation],
1628                        sched_smt_active() ? "vulnerable" : "disabled");
1629 }
1630
1631 static ssize_t itlb_multihit_show_state(char *buf)
1632 {
1633         if (itlb_multihit_kvm_mitigation)
1634                 return sprintf(buf, "KVM: Mitigation: Split huge pages\n");
1635         else
1636                 return sprintf(buf, "KVM: Vulnerable\n");
1637 }
1638 #else
1639 static ssize_t l1tf_show_state(char *buf)
1640 {
1641         return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
1642 }
1643
1644 static ssize_t itlb_multihit_show_state(char *buf)
1645 {
1646         return sprintf(buf, "Processor vulnerable\n");
1647 }
1648 #endif
1649
1650 static ssize_t mds_show_state(char *buf)
1651 {
1652 #ifdef CONFIG_HYPERVISOR_GUEST
1653         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
1654                 return sprintf(buf, "%s; SMT Host state unknown\n",
1655                                mds_strings[mds_mitigation]);
1656         }
1657 #endif
1658
1659         if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) {
1660                 return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
1661                                (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" :
1662                                 sched_smt_active() ? "mitigated" : "disabled"));
1663         }
1664
1665         return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
1666                        sched_smt_active() ? "vulnerable" : "disabled");
1667 }
1668
1669 static ssize_t tsx_async_abort_show_state(char *buf)
1670 {
1671         if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) ||
1672             (taa_mitigation == TAA_MITIGATION_OFF))
1673                 return sprintf(buf, "%s\n", taa_strings[taa_mitigation]);
1674
1675         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
1676                 return sprintf(buf, "%s; SMT Host state unknown\n",
1677                                taa_strings[taa_mitigation]);
1678         }
1679
1680         return sprintf(buf, "%s; SMT %s\n", taa_strings[taa_mitigation],
1681                        sched_smt_active() ? "vulnerable" : "disabled");
1682 }
1683
1684 static char *stibp_state(void)
1685 {
1686         if (spectre_v2_in_eibrs_mode(spectre_v2_enabled))
1687                 return "";
1688
1689         switch (spectre_v2_user_stibp) {
1690         case SPECTRE_V2_USER_NONE:
1691                 return ", STIBP: disabled";
1692         case SPECTRE_V2_USER_STRICT:
1693                 return ", STIBP: forced";
1694         case SPECTRE_V2_USER_STRICT_PREFERRED:
1695                 return ", STIBP: always-on";
1696         case SPECTRE_V2_USER_PRCTL:
1697         case SPECTRE_V2_USER_SECCOMP:
1698                 if (static_key_enabled(&switch_to_cond_stibp))
1699                         return ", STIBP: conditional";
1700         }
1701         return "";
1702 }
1703
1704 static char *ibpb_state(void)
1705 {
1706         if (boot_cpu_has(X86_FEATURE_IBPB)) {
1707                 if (static_key_enabled(&switch_mm_always_ibpb))
1708                         return ", IBPB: always-on";
1709                 if (static_key_enabled(&switch_mm_cond_ibpb))
1710                         return ", IBPB: conditional";
1711                 return ", IBPB: disabled";
1712         }
1713         return "";
1714 }
1715
1716 static ssize_t spectre_v2_show_state(char *buf)
1717 {
1718         if (spectre_v2_enabled == SPECTRE_V2_LFENCE)
1719                 return sprintf(buf, "Vulnerable: LFENCE\n");
1720
1721         if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
1722                 return sprintf(buf, "Vulnerable: eIBRS with unprivileged eBPF\n");
1723
1724         if (sched_smt_active() && unprivileged_ebpf_enabled() &&
1725             spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
1726                 return sprintf(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n");
1727
1728         return sprintf(buf, "%s%s%s%s%s%s\n",
1729                        spectre_v2_strings[spectre_v2_enabled],
1730                        ibpb_state(),
1731                        boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
1732                        stibp_state(),
1733                        boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "",
1734                        spectre_v2_module_string());
1735 }
1736
1737 static ssize_t srbds_show_state(char *buf)
1738 {
1739         return sprintf(buf, "%s\n", srbds_strings[srbds_mitigation]);
1740 }
1741
1742 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
1743                                char *buf, unsigned int bug)
1744 {
1745         if (!boot_cpu_has_bug(bug))
1746                 return sprintf(buf, "Not affected\n");
1747
1748         switch (bug) {
1749         case X86_BUG_CPU_MELTDOWN:
1750                 if (boot_cpu_has(X86_FEATURE_KAISER))
1751                         return sprintf(buf, "Mitigation: PTI\n");
1752
1753                 break;
1754
1755         case X86_BUG_SPECTRE_V1:
1756                 return sprintf(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]);
1757
1758         case X86_BUG_SPECTRE_V2:
1759                 return spectre_v2_show_state(buf);
1760
1761         case X86_BUG_SPEC_STORE_BYPASS:
1762                 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
1763
1764         case X86_BUG_L1TF:
1765                 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV))
1766                         return l1tf_show_state(buf);
1767                 break;
1768
1769         case X86_BUG_MDS:
1770                 return mds_show_state(buf);
1771
1772         case X86_BUG_TAA:
1773                 return tsx_async_abort_show_state(buf);
1774
1775         case X86_BUG_ITLB_MULTIHIT:
1776                 return itlb_multihit_show_state(buf);
1777
1778         case X86_BUG_SRBDS:
1779                 return srbds_show_state(buf);
1780
1781         default:
1782                 break;
1783         }
1784
1785         return sprintf(buf, "Vulnerable\n");
1786 }
1787
1788 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
1789 {
1790         return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
1791 }
1792
1793 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
1794 {
1795         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
1796 }
1797
1798 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
1799 {
1800         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
1801 }
1802
1803 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
1804 {
1805         return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
1806 }
1807
1808 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf)
1809 {
1810         return cpu_show_common(dev, attr, buf, X86_BUG_L1TF);
1811 }
1812
1813 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf)
1814 {
1815         return cpu_show_common(dev, attr, buf, X86_BUG_MDS);
1816 }
1817
1818 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf)
1819 {
1820         return cpu_show_common(dev, attr, buf, X86_BUG_TAA);
1821 }
1822
1823 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf)
1824 {
1825         return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT);
1826 }
1827
1828 ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *buf)
1829 {
1830         return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS);
1831 }
1832 #endif