arch/x86/kernel/cpu/bugs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Copyright (C) 1994  Linus Torvalds
   4  *
   5  *  Cyrix stuff, June 1998 by:
   6  *      - Rafael R. Reilova (moved everything from head.S),
   7  *        <rreilova@ececs.uc.edu>
   8  *      - Channing Corn (tests & fixes),
   9  *      - Andrew D. Balsa (code cleanup).
  10  */
  11 #include <linux/init.h>
  12 #include <linux/cpu.h>
  13 #include <linux/module.h>
  14 #include <linux/nospec.h>
  15 #include <linux/prctl.h>
  16 #include <linux/sched/smt.h>
  17 #include <linux/pgtable.h>
  18 #include <linux/bpf.h>
  19
  20 #include <asm/spec-ctrl.h>
  21 #include <asm/cmdline.h>
  22 #include <asm/bugs.h>
  23 #include <asm/processor.h>
  24 #include <asm/processor-flags.h>
  25 #include <asm/fpu/internal.h>
  26 #include <asm/msr.h>
  27 #include <asm/vmx.h>
  28 #include <asm/paravirt.h>
  29 #include <asm/intel-family.h>
  30 #include <asm/e820/api.h>
  31 #include <asm/hypervisor.h>
  32 #include <asm/tlbflush.h>
  33
  34 #include "cpu.h"
  35
  36 static void __init spectre_v1_select_mitigation(void);
  37 static void __init spectre_v2_select_mitigation(void);
  38 static void __init retbleed_select_mitigation(void);
  39 static void __init spectre_v2_user_select_mitigation(void);
  40 static void __init ssb_select_mitigation(void);
  41 static void __init l1tf_select_mitigation(void);
  42 static void __init mds_select_mitigation(void);
  43 static void __init md_clear_update_mitigation(void);
  44 static void __init md_clear_select_mitigation(void);
  45 static void __init taa_select_mitigation(void);
  46 static void __init mmio_select_mitigation(void);
  47 static void __init srbds_select_mitigation(void);
  48 static void __init l1d_flush_select_mitigation(void);
  49 static void __init gds_select_mitigation(void);
  50 static void __init srso_select_mitigation(void);
  51
  52 /* The base value of the SPEC_CTRL MSR without task-specific bits set */
  53 u64 x86_spec_ctrl_base;
  54 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
  55
  56 /* The current value of the SPEC_CTRL MSR with task-specific bits set */
  57 DEFINE_PER_CPU(u64, x86_spec_ctrl_current);
  58 EXPORT_SYMBOL_GPL(x86_spec_ctrl_current);
  59
  60 u64 x86_pred_cmd __ro_after_init = PRED_CMD_IBPB;
  61 EXPORT_SYMBOL_GPL(x86_pred_cmd);
  62
  63 static DEFINE_MUTEX(spec_ctrl_mutex);
  64
  65 void (*x86_return_thunk)(void) __ro_after_init = &__x86_return_thunk;
  66
  67 /* Update SPEC_CTRL MSR and its cached copy unconditionally */
  68 static void update_spec_ctrl(u64 val)
  69 {
  70         this_cpu_write(x86_spec_ctrl_current, val);
  71         wrmsrl(MSR_IA32_SPEC_CTRL, val);
  72 }
  73
  74 /*
  75  * Keep track of the SPEC_CTRL MSR value for the current task, which may differ
  76  * from x86_spec_ctrl_base due to STIBP/SSB in __speculation_ctrl_update().
  77  */
  78 void update_spec_ctrl_cond(u64 val)
  79 {
  80         if (this_cpu_read(x86_spec_ctrl_current) == val)
  81                 return;
  82
  83         this_cpu_write(x86_spec_ctrl_current, val);
  84
  85         /*
  86          * When KERNEL_IBRS this MSR is written on return-to-user, unless
  87          * forced the update can be delayed until that time.
  88          */
  89         if (!cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS))
  90                 wrmsrl(MSR_IA32_SPEC_CTRL, val);
  91 }
  92
  93 u64 spec_ctrl_current(void)
  94 {
  95         return this_cpu_read(x86_spec_ctrl_current);
  96 }
  97 EXPORT_SYMBOL_GPL(spec_ctrl_current);
  98
  99 /*
 100  * AMD specific MSR info for Speculative Store Bypass control.
 101  * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
 102  */
 103 u64 __ro_after_init x86_amd_ls_cfg_base;
 104 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
 105
 106 /* Control conditional STIBP in switch_to() */
 107 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp);
 108 /* Control conditional IBPB in switch_mm() */
 109 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
 110 /* Control unconditional IBPB in switch_mm() */
 111 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
 112
 113 /* Control MDS CPU buffer clear before returning to user space */
 114 DEFINE_STATIC_KEY_FALSE(mds_user_clear);
 115 EXPORT_SYMBOL_GPL(mds_user_clear);
 116 /* Control MDS CPU buffer clear before idling (halt, mwait) */
 117 DEFINE_STATIC_KEY_FALSE(mds_idle_clear);
 118 EXPORT_SYMBOL_GPL(mds_idle_clear);
 119
 120 /*
 121  * Controls whether l1d flush based mitigations are enabled,
 122  * based on hw features and admin setting via boot parameter
 123  * defaults to false
 124  */
 125 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush);
 126
 127 /* Controls CPU Fill buffer clear before KVM guest MMIO accesses */
 128 DEFINE_STATIC_KEY_FALSE(mmio_stale_data_clear);
 129 EXPORT_SYMBOL_GPL(mmio_stale_data_clear);
 130
 131 void __init cpu_select_mitigations(void)
 132 {
 133         /*
 134          * Read the SPEC_CTRL MSR to account for reserved bits which may
 135          * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
 136          * init code as it is not enumerated and depends on the family.
 137          */
 138         if (cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL)) {
 139                 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 140
 141                 /*
 142                  * Previously running kernel (kexec), may have some controls
 143                  * turned ON. Clear them and let the mitigations setup below
 144                  * rediscover them based on configuration.
 145                  */
 146                 x86_spec_ctrl_base &= ~SPEC_CTRL_MITIGATIONS_MASK;
 147         }
 148
 149         /* Select the proper CPU mitigations before patching alternatives: */
 150         spectre_v1_select_mitigation();
 151         spectre_v2_select_mitigation();
 152         /*
 153          * retbleed_select_mitigation() relies on the state set by
 154          * spectre_v2_select_mitigation(); specifically it wants to know about
 155          * spectre_v2=ibrs.
 156          */
 157         retbleed_select_mitigation();
 158         /*
 159          * spectre_v2_user_select_mitigation() relies on the state set by
 160          * retbleed_select_mitigation(); specifically the STIBP selection is
 161          * forced for UNRET or IBPB.
 162          */
 163         spectre_v2_user_select_mitigation();
 164         ssb_select_mitigation();
 165         l1tf_select_mitigation();
 166         md_clear_select_mitigation();
 167         srbds_select_mitigation();
 168         l1d_flush_select_mitigation();
 169
 170         /*
 171          * srso_select_mitigation() depends and must run after
 172          * retbleed_select_mitigation().
 173          */
 174         srso_select_mitigation();
 175         gds_select_mitigation();
 176 }
 177
 178 /*
 179  * NOTE: For VMX, this function is not called in the vmexit path.
 180  * It uses vmx_spec_ctrl_restore_host() instead.
 181  */
 182 void
 183 x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
 184 {
 185         u64 msrval, guestval = guest_spec_ctrl, hostval = spec_ctrl_current();
 186         struct thread_info *ti = current_thread_info();
 187
 188         if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) {
 189                 if (hostval != guestval) {
 190                         msrval = setguest ? guestval : hostval;
 191                         wrmsrl(MSR_IA32_SPEC_CTRL, msrval);
 192                 }
 193         }
 194
 195         /*
 196          * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
 197          * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
 198          */
 199         if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
 200             !static_cpu_has(X86_FEATURE_VIRT_SSBD))
 201                 return;
 202
 203         /*
 204          * If the host has SSBD mitigation enabled, force it in the host's
 205          * virtual MSR value. If its not permanently enabled, evaluate
 206          * current's TIF_SSBD thread flag.
 207          */
 208         if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
 209                 hostval = SPEC_CTRL_SSBD;
 210         else
 211                 hostval = ssbd_tif_to_spec_ctrl(ti->flags);
 212
 213         /* Sanitize the guest value */
 214         guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
 215
 216         if (hostval != guestval) {
 217                 unsigned long tif;
 218
 219                 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
 220                                  ssbd_spec_ctrl_to_tif(hostval);
 221
 222                 speculation_ctrl_update(tif);
 223         }
 224 }
 225 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
 226
 227 static void x86_amd_ssb_disable(void)
 228 {
 229         u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
 230
 231         if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
 232                 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
 233         else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
 234                 wrmsrl(MSR_AMD64_LS_CFG, msrval);
 235 }
 236
 237 #undef pr_fmt
 238 #define pr_fmt(fmt)     "MDS: " fmt
 239
 240 /* Default mitigation for MDS-affected CPUs */
 241 static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL;
 242 static bool mds_nosmt __ro_after_init = false;
 243
 244 static const char * const mds_strings[] = {
 245         [MDS_MITIGATION_OFF]    = "Vulnerable",
 246         [MDS_MITIGATION_FULL]   = "Mitigation: Clear CPU buffers",
 247         [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode",
 248 };
 249
 250 static void __init mds_select_mitigation(void)
 251 {
 252         if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) {
 253                 mds_mitigation = MDS_MITIGATION_OFF;
 254                 return;
 255         }
 256
 257         if (mds_mitigation == MDS_MITIGATION_FULL) {
 258                 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR))
 259                         mds_mitigation = MDS_MITIGATION_VMWERV;
 260
 261                 static_branch_enable(&mds_user_clear);
 262
 263                 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) &&
 264                     (mds_nosmt || cpu_mitigations_auto_nosmt()))
 265                         cpu_smt_disable(false);
 266         }
 267 }
 268
 269 static int __init mds_cmdline(char *str)
 270 {
 271         if (!boot_cpu_has_bug(X86_BUG_MDS))
 272                 return 0;
 273
 274         if (!str)
 275                 return -EINVAL;
 276
 277         if (!strcmp(str, "off"))
 278                 mds_mitigation = MDS_MITIGATION_OFF;
 279         else if (!strcmp(str, "full"))
 280                 mds_mitigation = MDS_MITIGATION_FULL;
 281         else if (!strcmp(str, "full,nosmt")) {
 282                 mds_mitigation = MDS_MITIGATION_FULL;
 283                 mds_nosmt = true;
 284         }
 285
 286         return 0;
 287 }
 288 early_param("mds", mds_cmdline);
 289
 290 #undef pr_fmt
 291 #define pr_fmt(fmt)     "TAA: " fmt
 292
 293 enum taa_mitigations {
 294         TAA_MITIGATION_OFF,
 295         TAA_MITIGATION_UCODE_NEEDED,
 296         TAA_MITIGATION_VERW,
 297         TAA_MITIGATION_TSX_DISABLED,
 298 };
 299
 300 /* Default mitigation for TAA-affected CPUs */
 301 static enum taa_mitigations taa_mitigation __ro_after_init = TAA_MITIGATION_VERW;
 302 static bool taa_nosmt __ro_after_init;
 303
 304 static const char * const taa_strings[] = {
 305         [TAA_MITIGATION_OFF]            = "Vulnerable",
 306         [TAA_MITIGATION_UCODE_NEEDED]   = "Vulnerable: Clear CPU buffers attempted, no microcode",
 307         [TAA_MITIGATION_VERW]           = "Mitigation: Clear CPU buffers",
 308         [TAA_MITIGATION_TSX_DISABLED]   = "Mitigation: TSX disabled",
 309 };
 310
 311 static void __init taa_select_mitigation(void)
 312 {
 313         u64 ia32_cap;
 314
 315         if (!boot_cpu_has_bug(X86_BUG_TAA)) {
 316                 taa_mitigation = TAA_MITIGATION_OFF;
 317                 return;
 318         }
 319
 320         /* TSX previously disabled by tsx=off */
 321         if (!boot_cpu_has(X86_FEATURE_RTM)) {
 322                 taa_mitigation = TAA_MITIGATION_TSX_DISABLED;
 323                 return;
 324         }
 325
 326         if (cpu_mitigations_off()) {
 327                 taa_mitigation = TAA_MITIGATION_OFF;
 328                 return;
 329         }
 330
 331         /*
 332          * TAA mitigation via VERW is turned off if both
 333          * tsx_async_abort=off and mds=off are specified.
 334          */
 335         if (taa_mitigation == TAA_MITIGATION_OFF &&
 336             mds_mitigation == MDS_MITIGATION_OFF)
 337                 return;
 338
 339         if (boot_cpu_has(X86_FEATURE_MD_CLEAR))
 340                 taa_mitigation = TAA_MITIGATION_VERW;
 341         else
 342                 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
 343
 344         /*
 345          * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1.
 346          * A microcode update fixes this behavior to clear CPU buffers. It also
 347          * adds support for MSR_IA32_TSX_CTRL which is enumerated by the
 348          * ARCH_CAP_TSX_CTRL_MSR bit.
 349          *
 350          * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode
 351          * update is required.
 352          */
 353         ia32_cap = x86_read_arch_cap_msr();
 354         if ( (ia32_cap & ARCH_CAP_MDS_NO) &&
 355             !(ia32_cap & ARCH_CAP_TSX_CTRL_MSR))
 356                 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
 357
 358         /*
 359          * TSX is enabled, select alternate mitigation for TAA which is
 360          * the same as MDS. Enable MDS static branch to clear CPU buffers.
 361          *
 362          * For guests that can't determine whether the correct microcode is
 363          * present on host, enable the mitigation for UCODE_NEEDED as well.
 364          */
 365         static_branch_enable(&mds_user_clear);
 366
 367         if (taa_nosmt || cpu_mitigations_auto_nosmt())
 368                 cpu_smt_disable(false);
 369 }
 370
 371 static int __init tsx_async_abort_parse_cmdline(char *str)
 372 {
 373         if (!boot_cpu_has_bug(X86_BUG_TAA))
 374                 return 0;
 375
 376         if (!str)
 377                 return -EINVAL;
 378
 379         if (!strcmp(str, "off")) {
 380                 taa_mitigation = TAA_MITIGATION_OFF;
 381         } else if (!strcmp(str, "full")) {
 382                 taa_mitigation = TAA_MITIGATION_VERW;
 383         } else if (!strcmp(str, "full,nosmt")) {
 384                 taa_mitigation = TAA_MITIGATION_VERW;
 385                 taa_nosmt = true;
 386         }
 387
 388         return 0;
 389 }
 390 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline);
 391
 392 #undef pr_fmt
 393 #define pr_fmt(fmt)     "MMIO Stale Data: " fmt
 394
 395 enum mmio_mitigations {
 396         MMIO_MITIGATION_OFF,
 397         MMIO_MITIGATION_UCODE_NEEDED,
 398         MMIO_MITIGATION_VERW,
 399 };
 400
 401 /* Default mitigation for Processor MMIO Stale Data vulnerabilities */
 402 static enum mmio_mitigations mmio_mitigation __ro_after_init = MMIO_MITIGATION_VERW;
 403 static bool mmio_nosmt __ro_after_init = false;
 404
 405 static const char * const mmio_strings[] = {
 406         [MMIO_MITIGATION_OFF]           = "Vulnerable",
 407         [MMIO_MITIGATION_UCODE_NEEDED]  = "Vulnerable: Clear CPU buffers attempted, no microcode",
 408         [MMIO_MITIGATION_VERW]          = "Mitigation: Clear CPU buffers",
 409 };
 410
 411 static void __init mmio_select_mitigation(void)
 412 {
 413         u64 ia32_cap;
 414
 415         if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA) ||
 416              boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN) ||
 417              cpu_mitigations_off()) {
 418                 mmio_mitigation = MMIO_MITIGATION_OFF;
 419                 return;
 420         }
 421
 422         if (mmio_mitigation == MMIO_MITIGATION_OFF)
 423                 return;
 424
 425         ia32_cap = x86_read_arch_cap_msr();
 426
 427         /*
 428          * Enable CPU buffer clear mitigation for host and VMM, if also affected
 429          * by MDS or TAA. Otherwise, enable mitigation for VMM only.
 430          */
 431         if (boot_cpu_has_bug(X86_BUG_MDS) || (boot_cpu_has_bug(X86_BUG_TAA) &&
 432                                               boot_cpu_has(X86_FEATURE_RTM)))
 433                 static_branch_enable(&mds_user_clear);
 434         else
 435                 static_branch_enable(&mmio_stale_data_clear);
 436
 437         /*
 438          * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can
 439          * be propagated to uncore buffers, clearing the Fill buffers on idle
 440          * is required irrespective of SMT state.
 441          */
 442         if (!(ia32_cap & ARCH_CAP_FBSDP_NO))
 443                 static_branch_enable(&mds_idle_clear);
 444
 445         /*
 446          * Check if the system has the right microcode.
 447          *
 448          * CPU Fill buffer clear mitigation is enumerated by either an explicit
 449          * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS
 450          * affected systems.
 451          */
 452         if ((ia32_cap & ARCH_CAP_FB_CLEAR) ||
 453             (boot_cpu_has(X86_FEATURE_MD_CLEAR) &&
 454              boot_cpu_has(X86_FEATURE_FLUSH_L1D) &&
 455              !(ia32_cap & ARCH_CAP_MDS_NO)))
 456                 mmio_mitigation = MMIO_MITIGATION_VERW;
 457         else
 458                 mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED;
 459
 460         if (mmio_nosmt || cpu_mitigations_auto_nosmt())
 461                 cpu_smt_disable(false);
 462 }
 463
 464 static int __init mmio_stale_data_parse_cmdline(char *str)
 465 {
 466         if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA))
 467                 return 0;
 468
 469         if (!str)
 470                 return -EINVAL;
 471
 472         if (!strcmp(str, "off")) {
 473                 mmio_mitigation = MMIO_MITIGATION_OFF;
 474         } else if (!strcmp(str, "full")) {
 475                 mmio_mitigation = MMIO_MITIGATION_VERW;
 476         } else if (!strcmp(str, "full,nosmt")) {
 477                 mmio_mitigation = MMIO_MITIGATION_VERW;
 478                 mmio_nosmt = true;
 479         }
 480
 481         return 0;
 482 }
 483 early_param("mmio_stale_data", mmio_stale_data_parse_cmdline);
 484
 485 #undef pr_fmt
 486 #define pr_fmt(fmt)     "" fmt
 487
 488 static void __init md_clear_update_mitigation(void)
 489 {
 490         if (cpu_mitigations_off())
 491                 return;
 492
 493         if (!static_key_enabled(&mds_user_clear))
 494                 goto out;
 495
 496         /*
 497          * mds_user_clear is now enabled. Update MDS, TAA and MMIO Stale Data
 498          * mitigation, if necessary.
 499          */
 500         if (mds_mitigation == MDS_MITIGATION_OFF &&
 501             boot_cpu_has_bug(X86_BUG_MDS)) {
 502                 mds_mitigation = MDS_MITIGATION_FULL;
 503                 mds_select_mitigation();
 504         }
 505         if (taa_mitigation == TAA_MITIGATION_OFF &&
 506             boot_cpu_has_bug(X86_BUG_TAA)) {
 507                 taa_mitigation = TAA_MITIGATION_VERW;
 508                 taa_select_mitigation();
 509         }
 510         if (mmio_mitigation == MMIO_MITIGATION_OFF &&
 511             boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) {
 512                 mmio_mitigation = MMIO_MITIGATION_VERW;
 513                 mmio_select_mitigation();
 514         }
 515 out:
 516         if (boot_cpu_has_bug(X86_BUG_MDS))
 517                 pr_info("MDS: %s\n", mds_strings[mds_mitigation]);
 518         if (boot_cpu_has_bug(X86_BUG_TAA))
 519                 pr_info("TAA: %s\n", taa_strings[taa_mitigation]);
 520         if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA))
 521                 pr_info("MMIO Stale Data: %s\n", mmio_strings[mmio_mitigation]);
 522         else if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN))
 523                 pr_info("MMIO Stale Data: Unknown: No mitigations\n");
 524 }
 525
 526 static void __init md_clear_select_mitigation(void)
 527 {
 528         mds_select_mitigation();
 529         taa_select_mitigation();
 530         mmio_select_mitigation();
 531
 532         /*
 533          * As MDS, TAA and MMIO Stale Data mitigations are inter-related, update
 534          * and print their mitigation after MDS, TAA and MMIO Stale Data
 535          * mitigation selection is done.
 536          */
 537         md_clear_update_mitigation();
 538 }
 539
 540 #undef pr_fmt
 541 #define pr_fmt(fmt)     "SRBDS: " fmt
 542
 543 enum srbds_mitigations {
 544         SRBDS_MITIGATION_OFF,
 545         SRBDS_MITIGATION_UCODE_NEEDED,
 546         SRBDS_MITIGATION_FULL,
 547         SRBDS_MITIGATION_TSX_OFF,
 548         SRBDS_MITIGATION_HYPERVISOR,
 549 };
 550
 551 static enum srbds_mitigations srbds_mitigation __ro_after_init = SRBDS_MITIGATION_FULL;
 552
 553 static const char * const srbds_strings[] = {
 554         [SRBDS_MITIGATION_OFF]          = "Vulnerable",
 555         [SRBDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode",
 556         [SRBDS_MITIGATION_FULL]         = "Mitigation: Microcode",
 557         [SRBDS_MITIGATION_TSX_OFF]      = "Mitigation: TSX disabled",
 558         [SRBDS_MITIGATION_HYPERVISOR]   = "Unknown: Dependent on hypervisor status",
 559 };
 560
 561 static bool srbds_off;
 562
 563 void update_srbds_msr(void)
 564 {
 565         u64 mcu_ctrl;
 566
 567         if (!boot_cpu_has_bug(X86_BUG_SRBDS))
 568                 return;
 569
 570         if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
 571                 return;
 572
 573         if (srbds_mitigation == SRBDS_MITIGATION_UCODE_NEEDED)
 574                 return;
 575
 576         rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 577
 578         switch (srbds_mitigation) {
 579         case SRBDS_MITIGATION_OFF:
 580         case SRBDS_MITIGATION_TSX_OFF:
 581                 mcu_ctrl |= RNGDS_MITG_DIS;
 582                 break;
 583         case SRBDS_MITIGATION_FULL:
 584                 mcu_ctrl &= ~RNGDS_MITG_DIS;
 585                 break;
 586         default:
 587                 break;
 588         }
 589
 590         wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 591 }
 592
 593 static void __init srbds_select_mitigation(void)
 594 {
 595         u64 ia32_cap;
 596
 597         if (!boot_cpu_has_bug(X86_BUG_SRBDS))
 598                 return;
 599
 600         /*
 601          * Check to see if this is one of the MDS_NO systems supporting TSX that
 602          * are only exposed to SRBDS when TSX is enabled or when CPU is affected
 603          * by Processor MMIO Stale Data vulnerability.
 604          */
 605         ia32_cap = x86_read_arch_cap_msr();
 606         if ((ia32_cap & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM) &&
 607             !boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA))
 608                 srbds_mitigation = SRBDS_MITIGATION_TSX_OFF;
 609         else if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
 610                 srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR;
 611         else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL))
 612                 srbds_mitigation = SRBDS_MITIGATION_UCODE_NEEDED;
 613         else if (cpu_mitigations_off() || srbds_off)
 614                 srbds_mitigation = SRBDS_MITIGATION_OFF;
 615
 616         update_srbds_msr();
 617         pr_info("%s\n", srbds_strings[srbds_mitigation]);
 618 }
 619
 620 static int __init srbds_parse_cmdline(char *str)
 621 {
 622         if (!str)
 623                 return -EINVAL;
 624
 625         if (!boot_cpu_has_bug(X86_BUG_SRBDS))
 626                 return 0;
 627
 628         srbds_off = !strcmp(str, "off");
 629         return 0;
 630 }
 631 early_param("srbds", srbds_parse_cmdline);
 632
 633 #undef pr_fmt
 634 #define pr_fmt(fmt)     "L1D Flush : " fmt
 635
 636 enum l1d_flush_mitigations {
 637         L1D_FLUSH_OFF = 0,
 638         L1D_FLUSH_ON,
 639 };
 640
 641 static enum l1d_flush_mitigations l1d_flush_mitigation __initdata = L1D_FLUSH_OFF;
 642
 643 static void __init l1d_flush_select_mitigation(void)
 644 {
 645         if (!l1d_flush_mitigation || !boot_cpu_has(X86_FEATURE_FLUSH_L1D))
 646                 return;
 647
 648         static_branch_enable(&switch_mm_cond_l1d_flush);
 649         pr_info("Conditional flush on switch_mm() enabled\n");
 650 }
 651
 652 static int __init l1d_flush_parse_cmdline(char *str)
 653 {
 654         if (!strcmp(str, "on"))
 655                 l1d_flush_mitigation = L1D_FLUSH_ON;
 656
 657         return 0;
 658 }
 659 early_param("l1d_flush", l1d_flush_parse_cmdline);
 660
 661 #undef pr_fmt
 662 #define pr_fmt(fmt)     "GDS: " fmt
 663
 664 enum gds_mitigations {
 665         GDS_MITIGATION_OFF,
 666         GDS_MITIGATION_UCODE_NEEDED,
 667         GDS_MITIGATION_FORCE,
 668         GDS_MITIGATION_FULL,
 669         GDS_MITIGATION_FULL_LOCKED,
 670         GDS_MITIGATION_HYPERVISOR,
 671 };
 672
 673 #if IS_ENABLED(CONFIG_GDS_FORCE_MITIGATION)
 674 static enum gds_mitigations gds_mitigation __ro_after_init = GDS_MITIGATION_FORCE;
 675 #else
 676 static enum gds_mitigations gds_mitigation __ro_after_init = GDS_MITIGATION_FULL;
 677 #endif
 678
 679 static const char * const gds_strings[] = {
 680         [GDS_MITIGATION_OFF]            = "Vulnerable",
 681         [GDS_MITIGATION_UCODE_NEEDED]   = "Vulnerable: No microcode",
 682         [GDS_MITIGATION_FORCE]          = "Mitigation: AVX disabled, no microcode",
 683         [GDS_MITIGATION_FULL]           = "Mitigation: Microcode",
 684         [GDS_MITIGATION_FULL_LOCKED]    = "Mitigation: Microcode (locked)",
 685         [GDS_MITIGATION_HYPERVISOR]     = "Unknown: Dependent on hypervisor status",
 686 };
 687
 688 bool gds_ucode_mitigated(void)
 689 {
 690         return (gds_mitigation == GDS_MITIGATION_FULL ||
 691                 gds_mitigation == GDS_MITIGATION_FULL_LOCKED);
 692 }
 693 EXPORT_SYMBOL_GPL(gds_ucode_mitigated);
 694
 695 void update_gds_msr(void)
 696 {
 697         u64 mcu_ctrl_after;
 698         u64 mcu_ctrl;
 699
 700         switch (gds_mitigation) {
 701         case GDS_MITIGATION_OFF:
 702                 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 703                 mcu_ctrl |= GDS_MITG_DIS;
 704                 break;
 705         case GDS_MITIGATION_FULL_LOCKED:
 706                 /*
 707                  * The LOCKED state comes from the boot CPU. APs might not have
 708                  * the same state. Make sure the mitigation is enabled on all
 709                  * CPUs.
 710                  */
 711         case GDS_MITIGATION_FULL:
 712                 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 713                 mcu_ctrl &= ~GDS_MITG_DIS;
 714                 break;
 715         case GDS_MITIGATION_FORCE:
 716         case GDS_MITIGATION_UCODE_NEEDED:
 717         case GDS_MITIGATION_HYPERVISOR:
 718                 return;
 719         };
 720
 721         wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 722
 723         /*
 724          * Check to make sure that the WRMSR value was not ignored. Writes to
 725          * GDS_MITG_DIS will be ignored if this processor is locked but the boot
 726          * processor was not.
 727          */
 728         rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl_after);
 729         WARN_ON_ONCE(mcu_ctrl != mcu_ctrl_after);
 730 }
 731
 732 static void __init gds_select_mitigation(void)
 733 {
 734         u64 mcu_ctrl;
 735
 736         if (!boot_cpu_has_bug(X86_BUG_GDS))
 737                 return;
 738
 739         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
 740                 gds_mitigation = GDS_MITIGATION_HYPERVISOR;
 741                 goto out;
 742         }
 743
 744         if (cpu_mitigations_off())
 745                 gds_mitigation = GDS_MITIGATION_OFF;
 746         /* Will verify below that mitigation _can_ be disabled */
 747
 748         /* No microcode */
 749         if (!(x86_read_arch_cap_msr() & ARCH_CAP_GDS_CTRL)) {
 750                 if (gds_mitigation == GDS_MITIGATION_FORCE) {
 751                         /*
 752                          * This only needs to be done on the boot CPU so do it
 753                          * here rather than in update_gds_msr()
 754                          */
 755                         setup_clear_cpu_cap(X86_FEATURE_AVX);
 756                         pr_warn("Microcode update needed! Disabling AVX as mitigation.\n");
 757                 } else {
 758                         gds_mitigation = GDS_MITIGATION_UCODE_NEEDED;
 759                 }
 760                 goto out;
 761         }
 762
 763         /* Microcode has mitigation, use it */
 764         if (gds_mitigation == GDS_MITIGATION_FORCE)
 765                 gds_mitigation = GDS_MITIGATION_FULL;
 766
 767         rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 768         if (mcu_ctrl & GDS_MITG_LOCKED) {
 769                 if (gds_mitigation == GDS_MITIGATION_OFF)
 770                         pr_warn("Mitigation locked. Disable failed.\n");
 771
 772                 /*
 773                  * The mitigation is selected from the boot CPU. All other CPUs
 774                  * _should_ have the same state. If the boot CPU isn't locked
 775                  * but others are then update_gds_msr() will WARN() of the state
 776                  * mismatch. If the boot CPU is locked update_gds_msr() will
 777                  * ensure the other CPUs have the mitigation enabled.
 778                  */
 779                 gds_mitigation = GDS_MITIGATION_FULL_LOCKED;
 780         }
 781
 782         update_gds_msr();
 783 out:
 784         pr_info("%s\n", gds_strings[gds_mitigation]);
 785 }
 786
 787 static int __init gds_parse_cmdline(char *str)
 788 {
 789         if (!str)
 790                 return -EINVAL;
 791
 792         if (!boot_cpu_has_bug(X86_BUG_GDS))
 793                 return 0;
 794
 795         if (!strcmp(str, "off"))
 796                 gds_mitigation = GDS_MITIGATION_OFF;
 797         else if (!strcmp(str, "force"))
 798                 gds_mitigation = GDS_MITIGATION_FORCE;
 799
 800         return 0;
 801 }
 802 early_param("gather_data_sampling", gds_parse_cmdline);
 803
 804 #undef pr_fmt
 805 #define pr_fmt(fmt)     "Spectre V1 : " fmt
 806
 807 enum spectre_v1_mitigation {
 808         SPECTRE_V1_MITIGATION_NONE,
 809         SPECTRE_V1_MITIGATION_AUTO,
 810 };
 811
 812 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init =
 813         SPECTRE_V1_MITIGATION_AUTO;
 814
 815 static const char * const spectre_v1_strings[] = {
 816         [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers",
 817         [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization",
 818 };
 819
 820 /*
 821  * Does SMAP provide full mitigation against speculative kernel access to
 822  * userspace?
 823  */
 824 static bool smap_works_speculatively(void)
 825 {
 826         if (!boot_cpu_has(X86_FEATURE_SMAP))
 827                 return false;
 828
 829         /*
 830          * On CPUs which are vulnerable to Meltdown, SMAP does not
 831          * prevent speculative access to user data in the L1 cache.
 832          * Consider SMAP to be non-functional as a mitigation on these
 833          * CPUs.
 834          */
 835         if (boot_cpu_has(X86_BUG_CPU_MELTDOWN))
 836                 return false;
 837
 838         return true;
 839 }
 840
 841 static void __init spectre_v1_select_mitigation(void)
 842 {
 843         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) {
 844                 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
 845                 return;
 846         }
 847
 848         if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) {
 849                 /*
 850                  * With Spectre v1, a user can speculatively control either
 851                  * path of a conditional swapgs with a user-controlled GS
 852                  * value.  The mitigation is to add lfences to both code paths.
 853                  *
 854                  * If FSGSBASE is enabled, the user can put a kernel address in
 855                  * GS, in which case SMAP provides no protection.
 856                  *
 857                  * If FSGSBASE is disabled, the user can only put a user space
 858                  * address in GS.  That makes an attack harder, but still
 859                  * possible if there's no SMAP protection.
 860                  */
 861                 if (boot_cpu_has(X86_FEATURE_FSGSBASE) ||
 862                     !smap_works_speculatively()) {
 863                         /*
 864                          * Mitigation can be provided from SWAPGS itself or
 865                          * PTI as the CR3 write in the Meltdown mitigation
 866                          * is serializing.
 867                          *
 868                          * If neither is there, mitigate with an LFENCE to
 869                          * stop speculation through swapgs.
 870                          */
 871                         if (boot_cpu_has_bug(X86_BUG_SWAPGS) &&
 872                             !boot_cpu_has(X86_FEATURE_PTI))
 873                                 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER);
 874
 875                         /*
 876                          * Enable lfences in the kernel entry (non-swapgs)
 877                          * paths, to prevent user entry from speculatively
 878                          * skipping swapgs.
 879                          */
 880                         setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL);
 881                 }
 882         }
 883
 884         pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]);
 885 }
 886
 887 static int __init nospectre_v1_cmdline(char *str)
 888 {
 889         spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
 890         return 0;
 891 }
 892 early_param("nospectre_v1", nospectre_v1_cmdline);
 893
 894 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
 895         SPECTRE_V2_NONE;
 896
 897 #undef pr_fmt
 898 #define pr_fmt(fmt)     "RETBleed: " fmt
 899
 900 enum retbleed_mitigation {
 901         RETBLEED_MITIGATION_NONE,
 902         RETBLEED_MITIGATION_UNRET,
 903         RETBLEED_MITIGATION_IBPB,
 904         RETBLEED_MITIGATION_IBRS,
 905         RETBLEED_MITIGATION_EIBRS,
 906 };
 907
 908 enum retbleed_mitigation_cmd {
 909         RETBLEED_CMD_OFF,
 910         RETBLEED_CMD_AUTO,
 911         RETBLEED_CMD_UNRET,
 912         RETBLEED_CMD_IBPB,
 913 };
 914
 915 const char * const retbleed_strings[] = {
 916         [RETBLEED_MITIGATION_NONE]      = "Vulnerable",
 917         [RETBLEED_MITIGATION_UNRET]     = "Mitigation: untrained return thunk",
 918         [RETBLEED_MITIGATION_IBPB]      = "Mitigation: IBPB",
 919         [RETBLEED_MITIGATION_IBRS]      = "Mitigation: IBRS",
 920         [RETBLEED_MITIGATION_EIBRS]     = "Mitigation: Enhanced IBRS",
 921 };
 922
 923 static enum retbleed_mitigation retbleed_mitigation __ro_after_init =
 924         RETBLEED_MITIGATION_NONE;
 925 static enum retbleed_mitigation_cmd retbleed_cmd __ro_after_init =
 926         RETBLEED_CMD_AUTO;
 927
 928 static int __ro_after_init retbleed_nosmt = false;
 929
 930 static int __init retbleed_parse_cmdline(char *str)
 931 {
 932         if (!str)
 933                 return -EINVAL;
 934
 935         while (str) {
 936                 char *next = strchr(str, ',');
 937                 if (next) {
 938                         *next = 0;
 939                         next++;
 940                 }
 941
 942                 if (!strcmp(str, "off")) {
 943                         retbleed_cmd = RETBLEED_CMD_OFF;
 944                 } else if (!strcmp(str, "auto")) {
 945                         retbleed_cmd = RETBLEED_CMD_AUTO;
 946                 } else if (!strcmp(str, "unret")) {
 947                         retbleed_cmd = RETBLEED_CMD_UNRET;
 948                 } else if (!strcmp(str, "ibpb")) {
 949                         retbleed_cmd = RETBLEED_CMD_IBPB;
 950                 } else if (!strcmp(str, "nosmt")) {
 951                         retbleed_nosmt = true;
 952                 } else {
 953                         pr_err("Ignoring unknown retbleed option (%s).", str);
 954                 }
 955
 956                 str = next;
 957         }
 958
 959         return 0;
 960 }
 961 early_param("retbleed", retbleed_parse_cmdline);
 962
 963 #define RETBLEED_UNTRAIN_MSG "WARNING: BTB untrained return thunk mitigation is only effective on AMD/Hygon!\n"
 964 #define RETBLEED_INTEL_MSG "WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!\n"
 965
 966 static void __init retbleed_select_mitigation(void)
 967 {
 968         bool mitigate_smt = false;
 969
 970         if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off())
 971                 return;
 972
 973         switch (retbleed_cmd) {
 974         case RETBLEED_CMD_OFF:
 975                 return;
 976
 977         case RETBLEED_CMD_UNRET:
 978                 if (IS_ENABLED(CONFIG_CPU_UNRET_ENTRY)) {
 979                         retbleed_mitigation = RETBLEED_MITIGATION_UNRET;
 980                 } else {
 981                         pr_err("WARNING: kernel not compiled with CPU_UNRET_ENTRY.\n");
 982                         goto do_cmd_auto;
 983                 }
 984                 break;
 985
 986         case RETBLEED_CMD_IBPB:
 987                 if (!boot_cpu_has(X86_FEATURE_IBPB)) {
 988                         pr_err("WARNING: CPU does not support IBPB.\n");
 989                         goto do_cmd_auto;
 990                 } else if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY)) {
 991                         retbleed_mitigation = RETBLEED_MITIGATION_IBPB;
 992                 } else {
 993                         pr_err("WARNING: kernel not compiled with CPU_IBPB_ENTRY.\n");
 994                         goto do_cmd_auto;
 995                 }
 996                 break;
 997
 998 do_cmd_auto:
 999         case RETBLEED_CMD_AUTO:
1000         default:
1001                 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
1002                     boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
1003                         if (IS_ENABLED(CONFIG_CPU_UNRET_ENTRY))
1004                                 retbleed_mitigation = RETBLEED_MITIGATION_UNRET;
1005                         else if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY) && boot_cpu_has(X86_FEATURE_IBPB))
1006                                 retbleed_mitigation = RETBLEED_MITIGATION_IBPB;
1007                 }
1008
1009                 /*
1010                  * The Intel mitigation (IBRS or eIBRS) was already selected in
1011                  * spectre_v2_select_mitigation().  'retbleed_mitigation' will
1012                  * be set accordingly below.
1013                  */
1014
1015                 break;
1016         }
1017
1018         switch (retbleed_mitigation) {
1019         case RETBLEED_MITIGATION_UNRET:
1020                 setup_force_cpu_cap(X86_FEATURE_RETHUNK);
1021                 setup_force_cpu_cap(X86_FEATURE_UNRET);
1022
1023                 if (IS_ENABLED(CONFIG_RETHUNK))
1024                         x86_return_thunk = retbleed_return_thunk;
1025
1026                 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
1027                     boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
1028                         pr_err(RETBLEED_UNTRAIN_MSG);
1029
1030                 mitigate_smt = true;
1031                 break;
1032
1033         case RETBLEED_MITIGATION_IBPB:
1034                 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB);
1035                 mitigate_smt = true;
1036                 break;
1037
1038         default:
1039                 break;
1040         }
1041
1042         if (mitigate_smt && !boot_cpu_has(X86_FEATURE_STIBP) &&
1043             (retbleed_nosmt || cpu_mitigations_auto_nosmt()))
1044                 cpu_smt_disable(false);
1045
1046         /*
1047          * Let IBRS trump all on Intel without affecting the effects of the
1048          * retbleed= cmdline option.
1049          */
1050         if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
1051                 switch (spectre_v2_enabled) {
1052                 case SPECTRE_V2_IBRS:
1053                         retbleed_mitigation = RETBLEED_MITIGATION_IBRS;
1054                         break;
1055                 case SPECTRE_V2_EIBRS:
1056                 case SPECTRE_V2_EIBRS_RETPOLINE:
1057                 case SPECTRE_V2_EIBRS_LFENCE:
1058                         retbleed_mitigation = RETBLEED_MITIGATION_EIBRS;
1059                         break;
1060                 default:
1061                         pr_err(RETBLEED_INTEL_MSG);
1062                 }
1063         }
1064
1065         pr_info("%s\n", retbleed_strings[retbleed_mitigation]);
1066 }
1067
1068 #undef pr_fmt
1069 #define pr_fmt(fmt)     "Spectre V2 : " fmt
1070
1071 static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init =
1072         SPECTRE_V2_USER_NONE;
1073 static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init =
1074         SPECTRE_V2_USER_NONE;
1075
1076 #ifdef CONFIG_RETPOLINE
1077 static bool spectre_v2_bad_module;
1078
1079 bool retpoline_module_ok(bool has_retpoline)
1080 {
1081         if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
1082                 return true;
1083
1084         pr_err("System may be vulnerable to spectre v2\n");
1085         spectre_v2_bad_module = true;
1086         return false;
1087 }
1088
1089 static inline const char *spectre_v2_module_string(void)
1090 {
1091         return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
1092 }
1093 #else
1094 static inline const char *spectre_v2_module_string(void) { return ""; }
1095 #endif
1096
1097 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n"
1098 #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n"
1099 #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n"
1100 #define SPECTRE_V2_IBRS_PERF_MSG "WARNING: IBRS mitigation selected on Enhanced IBRS CPU, this may cause unnecessary performance loss\n"
1101
1102 #ifdef CONFIG_BPF_SYSCALL
1103 void unpriv_ebpf_notify(int new_state)
1104 {
1105         if (new_state)
1106                 return;
1107
1108         /* Unprivileged eBPF is enabled */
1109
1110         switch (spectre_v2_enabled) {
1111         case SPECTRE_V2_EIBRS:
1112                 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
1113                 break;
1114         case SPECTRE_V2_EIBRS_LFENCE:
1115                 if (sched_smt_active())
1116                         pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
1117                 break;
1118         default:
1119                 break;
1120         }
1121 }
1122 #endif
1123
1124 static inline bool match_option(const char *arg, int arglen, const char *opt)
1125 {
1126         int len = strlen(opt);
1127
1128         return len == arglen && !strncmp(arg, opt, len);
1129 }
1130
1131 /* The kernel command line selection for spectre v2 */
1132 enum spectre_v2_mitigation_cmd {
1133         SPECTRE_V2_CMD_NONE,
1134         SPECTRE_V2_CMD_AUTO,
1135         SPECTRE_V2_CMD_FORCE,
1136         SPECTRE_V2_CMD_RETPOLINE,
1137         SPECTRE_V2_CMD_RETPOLINE_GENERIC,
1138         SPECTRE_V2_CMD_RETPOLINE_LFENCE,
1139         SPECTRE_V2_CMD_EIBRS,
1140         SPECTRE_V2_CMD_EIBRS_RETPOLINE,
1141         SPECTRE_V2_CMD_EIBRS_LFENCE,
1142         SPECTRE_V2_CMD_IBRS,
1143 };
1144
1145 enum spectre_v2_user_cmd {
1146         SPECTRE_V2_USER_CMD_NONE,
1147         SPECTRE_V2_USER_CMD_AUTO,
1148         SPECTRE_V2_USER_CMD_FORCE,
1149         SPECTRE_V2_USER_CMD_PRCTL,
1150         SPECTRE_V2_USER_CMD_PRCTL_IBPB,
1151         SPECTRE_V2_USER_CMD_SECCOMP,
1152         SPECTRE_V2_USER_CMD_SECCOMP_IBPB,
1153 };
1154
1155 static const char * const spectre_v2_user_strings[] = {
1156         [SPECTRE_V2_USER_NONE]                  = "User space: Vulnerable",
1157         [SPECTRE_V2_USER_STRICT]                = "User space: Mitigation: STIBP protection",
1158         [SPECTRE_V2_USER_STRICT_PREFERRED]      = "User space: Mitigation: STIBP always-on protection",
1159         [SPECTRE_V2_USER_PRCTL]                 = "User space: Mitigation: STIBP via prctl",
1160         [SPECTRE_V2_USER_SECCOMP]               = "User space: Mitigation: STIBP via seccomp and prctl",
1161 };
1162
1163 static const struct {
1164         const char                      *option;
1165         enum spectre_v2_user_cmd        cmd;
1166         bool                            secure;
1167 } v2_user_options[] __initconst = {
1168         { "auto",               SPECTRE_V2_USER_CMD_AUTO,               false },
1169         { "off",                SPECTRE_V2_USER_CMD_NONE,               false },
1170         { "on",                 SPECTRE_V2_USER_CMD_FORCE,              true  },
1171         { "prctl",              SPECTRE_V2_USER_CMD_PRCTL,              false },
1172         { "prctl,ibpb",         SPECTRE_V2_USER_CMD_PRCTL_IBPB,         false },
1173         { "seccomp",            SPECTRE_V2_USER_CMD_SECCOMP,            false },
1174         { "seccomp,ibpb",       SPECTRE_V2_USER_CMD_SECCOMP_IBPB,       false },
1175 };
1176
1177 static void __init spec_v2_user_print_cond(const char *reason, bool secure)
1178 {
1179         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
1180                 pr_info("spectre_v2_user=%s forced on command line.\n", reason);
1181 }
1182
1183 static __ro_after_init enum spectre_v2_mitigation_cmd spectre_v2_cmd;
1184
1185 static enum spectre_v2_user_cmd __init
1186 spectre_v2_parse_user_cmdline(void)
1187 {
1188         char arg[20];
1189         int ret, i;
1190
1191         switch (spectre_v2_cmd) {
1192         case SPECTRE_V2_CMD_NONE:
1193                 return SPECTRE_V2_USER_CMD_NONE;
1194         case SPECTRE_V2_CMD_FORCE:
1195                 return SPECTRE_V2_USER_CMD_FORCE;
1196         default:
1197                 break;
1198         }
1199
1200         ret = cmdline_find_option(boot_command_line, "spectre_v2_user",
1201                                   arg, sizeof(arg));
1202         if (ret < 0)
1203                 return SPECTRE_V2_USER_CMD_AUTO;
1204
1205         for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) {
1206                 if (match_option(arg, ret, v2_user_options[i].option)) {
1207                         spec_v2_user_print_cond(v2_user_options[i].option,
1208                                                 v2_user_options[i].secure);
1209                         return v2_user_options[i].cmd;
1210                 }
1211         }
1212
1213         pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg);
1214         return SPECTRE_V2_USER_CMD_AUTO;
1215 }
1216
1217 static inline bool spectre_v2_in_eibrs_mode(enum spectre_v2_mitigation mode)
1218 {
1219         return mode == SPECTRE_V2_EIBRS ||
1220                mode == SPECTRE_V2_EIBRS_RETPOLINE ||
1221                mode == SPECTRE_V2_EIBRS_LFENCE;
1222 }
1223
1224 static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode)
1225 {
1226         return spectre_v2_in_eibrs_mode(mode) || mode == SPECTRE_V2_IBRS;
1227 }
1228
1229 static void __init
1230 spectre_v2_user_select_mitigation(void)
1231 {
1232         enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE;
1233         bool smt_possible = IS_ENABLED(CONFIG_SMP);
1234         enum spectre_v2_user_cmd cmd;
1235
1236         if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP))
1237                 return;
1238
1239         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
1240             cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
1241                 smt_possible = false;
1242
1243         cmd = spectre_v2_parse_user_cmdline();
1244         switch (cmd) {
1245         case SPECTRE_V2_USER_CMD_NONE:
1246                 goto set_mode;
1247         case SPECTRE_V2_USER_CMD_FORCE:
1248                 mode = SPECTRE_V2_USER_STRICT;
1249                 break;
1250         case SPECTRE_V2_USER_CMD_PRCTL:
1251         case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
1252                 mode = SPECTRE_V2_USER_PRCTL;
1253                 break;
1254         case SPECTRE_V2_USER_CMD_AUTO:
1255         case SPECTRE_V2_USER_CMD_SECCOMP:
1256         case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
1257                 if (IS_ENABLED(CONFIG_SECCOMP))
1258                         mode = SPECTRE_V2_USER_SECCOMP;
1259                 else
1260                         mode = SPECTRE_V2_USER_PRCTL;
1261                 break;
1262         }
1263
1264         /* Initialize Indirect Branch Prediction Barrier */
1265         if (boot_cpu_has(X86_FEATURE_IBPB)) {
1266                 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
1267
1268                 spectre_v2_user_ibpb = mode;
1269                 switch (cmd) {
1270                 case SPECTRE_V2_USER_CMD_FORCE:
1271                 case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
1272                 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
1273                         static_branch_enable(&switch_mm_always_ibpb);
1274                         spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT;
1275                         break;
1276                 case SPECTRE_V2_USER_CMD_PRCTL:
1277                 case SPECTRE_V2_USER_CMD_AUTO:
1278                 case SPECTRE_V2_USER_CMD_SECCOMP:
1279                         static_branch_enable(&switch_mm_cond_ibpb);
1280                         break;
1281                 default:
1282                         break;
1283                 }
1284
1285                 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
1286                         static_key_enabled(&switch_mm_always_ibpb) ?
1287                         "always-on" : "conditional");
1288         }
1289
1290         /*
1291          * If no STIBP, enhanced IBRS is enabled, or SMT impossible, STIBP
1292          * is not required.
1293          *
1294          * Enhanced IBRS also protects against cross-thread branch target
1295          * injection in user-mode as the IBRS bit remains always set which
1296          * implicitly enables cross-thread protections.  However, in legacy IBRS
1297          * mode, the IBRS bit is set only on kernel entry and cleared on return
1298          * to userspace. This disables the implicit cross-thread protection,
1299          * so allow for STIBP to be selected in that case.
1300          */
1301         if (!boot_cpu_has(X86_FEATURE_STIBP) ||
1302             !smt_possible ||
1303             spectre_v2_in_eibrs_mode(spectre_v2_enabled))
1304                 return;
1305
1306         /*
1307          * At this point, an STIBP mode other than "off" has been set.
1308          * If STIBP support is not being forced, check if STIBP always-on
1309          * is preferred.
1310          */
1311         if (mode != SPECTRE_V2_USER_STRICT &&
1312             boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON))
1313                 mode = SPECTRE_V2_USER_STRICT_PREFERRED;
1314
1315         if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET ||
1316             retbleed_mitigation == RETBLEED_MITIGATION_IBPB) {
1317                 if (mode != SPECTRE_V2_USER_STRICT &&
1318                     mode != SPECTRE_V2_USER_STRICT_PREFERRED)
1319                         pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n");
1320                 mode = SPECTRE_V2_USER_STRICT_PREFERRED;
1321         }
1322
1323         spectre_v2_user_stibp = mode;
1324
1325 set_mode:
1326         pr_info("%s\n", spectre_v2_user_strings[mode]);
1327 }
1328
1329 static const char * const spectre_v2_strings[] = {
1330         [SPECTRE_V2_NONE]                       = "Vulnerable",
1331         [SPECTRE_V2_RETPOLINE]                  = "Mitigation: Retpolines",
1332         [SPECTRE_V2_LFENCE]                     = "Mitigation: LFENCE",
1333         [SPECTRE_V2_EIBRS]                      = "Mitigation: Enhanced IBRS",
1334         [SPECTRE_V2_EIBRS_LFENCE]               = "Mitigation: Enhanced IBRS + LFENCE",
1335         [SPECTRE_V2_EIBRS_RETPOLINE]            = "Mitigation: Enhanced IBRS + Retpolines",
1336         [SPECTRE_V2_IBRS]                       = "Mitigation: IBRS",
1337 };
1338
1339 static const struct {
1340         const char *option;
1341         enum spectre_v2_mitigation_cmd cmd;
1342         bool secure;
1343 } mitigation_options[] __initconst = {
1344         { "off",                SPECTRE_V2_CMD_NONE,              false },
1345         { "on",                 SPECTRE_V2_CMD_FORCE,             true  },
1346         { "retpoline",          SPECTRE_V2_CMD_RETPOLINE,         false },
1347         { "retpoline,amd",      SPECTRE_V2_CMD_RETPOLINE_LFENCE,  false },
1348         { "retpoline,lfence",   SPECTRE_V2_CMD_RETPOLINE_LFENCE,  false },
1349         { "retpoline,generic",  SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
1350         { "eibrs",              SPECTRE_V2_CMD_EIBRS,             false },
1351         { "eibrs,lfence",       SPECTRE_V2_CMD_EIBRS_LFENCE,      false },
1352         { "eibrs,retpoline",    SPECTRE_V2_CMD_EIBRS_RETPOLINE,   false },
1353         { "auto",               SPECTRE_V2_CMD_AUTO,              false },
1354         { "ibrs",               SPECTRE_V2_CMD_IBRS,              false },
1355 };
1356
1357 static void __init spec_v2_print_cond(const char *reason, bool secure)
1358 {
1359         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
1360                 pr_info("%s selected on command line.\n", reason);
1361 }
1362
1363 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
1364 {
1365         enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
1366         char arg[20];
1367         int ret, i;
1368
1369         if (cmdline_find_option_bool(boot_command_line, "nospectre_v2") ||
1370             cpu_mitigations_off())
1371                 return SPECTRE_V2_CMD_NONE;
1372
1373         ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
1374         if (ret < 0)
1375                 return SPECTRE_V2_CMD_AUTO;
1376
1377         for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
1378                 if (!match_option(arg, ret, mitigation_options[i].option))
1379                         continue;
1380                 cmd = mitigation_options[i].cmd;
1381                 break;
1382         }
1383
1384         if (i >= ARRAY_SIZE(mitigation_options)) {
1385                 pr_err("unknown option (%s). Switching to AUTO select\n", arg);
1386                 return SPECTRE_V2_CMD_AUTO;
1387         }
1388
1389         if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
1390              cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
1391              cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC ||
1392              cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
1393              cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
1394             !IS_ENABLED(CONFIG_RETPOLINE)) {
1395                 pr_err("%s selected but not compiled in. Switching to AUTO select\n",
1396                        mitigation_options[i].option);
1397                 return SPECTRE_V2_CMD_AUTO;
1398         }
1399
1400         if ((cmd == SPECTRE_V2_CMD_EIBRS ||
1401              cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
1402              cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
1403             !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
1404                 pr_err("%s selected but CPU doesn't have eIBRS. Switching to AUTO select\n",
1405                        mitigation_options[i].option);
1406                 return SPECTRE_V2_CMD_AUTO;
1407         }
1408
1409         if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
1410              cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) &&
1411             !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
1412                 pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n",
1413                        mitigation_options[i].option);
1414                 return SPECTRE_V2_CMD_AUTO;
1415         }
1416
1417         if (cmd == SPECTRE_V2_CMD_IBRS && !IS_ENABLED(CONFIG_CPU_IBRS_ENTRY)) {
1418                 pr_err("%s selected but not compiled in. Switching to AUTO select\n",
1419                        mitigation_options[i].option);
1420                 return SPECTRE_V2_CMD_AUTO;
1421         }
1422
1423         if (cmd == SPECTRE_V2_CMD_IBRS && boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
1424                 pr_err("%s selected but not Intel CPU. Switching to AUTO select\n",
1425                        mitigation_options[i].option);
1426                 return SPECTRE_V2_CMD_AUTO;
1427         }
1428
1429         if (cmd == SPECTRE_V2_CMD_IBRS && !boot_cpu_has(X86_FEATURE_IBRS)) {
1430                 pr_err("%s selected but CPU doesn't have IBRS. Switching to AUTO select\n",
1431                        mitigation_options[i].option);
1432                 return SPECTRE_V2_CMD_AUTO;
1433         }
1434
1435         if (cmd == SPECTRE_V2_CMD_IBRS && boot_cpu_has(X86_FEATURE_XENPV)) {
1436                 pr_err("%s selected but running as XenPV guest. Switching to AUTO select\n",
1437                        mitigation_options[i].option);
1438                 return SPECTRE_V2_CMD_AUTO;
1439         }
1440
1441         spec_v2_print_cond(mitigation_options[i].option,
1442                            mitigation_options[i].secure);
1443         return cmd;
1444 }
1445
1446 static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void)
1447 {
1448         if (!IS_ENABLED(CONFIG_RETPOLINE)) {
1449                 pr_err("Kernel not compiled with retpoline; no mitigation available!");
1450                 return SPECTRE_V2_NONE;
1451         }
1452
1453         return SPECTRE_V2_RETPOLINE;
1454 }
1455
1456 /* Disable in-kernel use of non-RSB RET predictors */
1457 static void __init spec_ctrl_disable_kernel_rrsba(void)
1458 {
1459         u64 ia32_cap;
1460
1461         if (!boot_cpu_has(X86_FEATURE_RRSBA_CTRL))
1462                 return;
1463
1464         ia32_cap = x86_read_arch_cap_msr();
1465
1466         if (ia32_cap & ARCH_CAP_RRSBA) {
1467                 x86_spec_ctrl_base |= SPEC_CTRL_RRSBA_DIS_S;
1468                 update_spec_ctrl(x86_spec_ctrl_base);
1469         }
1470 }
1471
1472 static void __init spectre_v2_determine_rsb_fill_type_at_vmexit(enum spectre_v2_mitigation mode)
1473 {
1474         /*
1475          * Similar to context switches, there are two types of RSB attacks
1476          * after VM exit:
1477          *
1478          * 1) RSB underflow
1479          *
1480          * 2) Poisoned RSB entry
1481          *
1482          * When retpoline is enabled, both are mitigated by filling/clearing
1483          * the RSB.
1484          *
1485          * When IBRS is enabled, while #1 would be mitigated by the IBRS branch
1486          * prediction isolation protections, RSB still needs to be cleared
1487          * because of #2.  Note that SMEP provides no protection here, unlike
1488          * user-space-poisoned RSB entries.
1489          *
1490          * eIBRS should protect against RSB poisoning, but if the EIBRS_PBRSB
1491          * bug is present then a LITE version of RSB protection is required,
1492          * just a single call needs to retire before a RET is executed.
1493          */
1494         switch (mode) {
1495         case SPECTRE_V2_NONE:
1496                 return;
1497
1498         case SPECTRE_V2_EIBRS_LFENCE:
1499         case SPECTRE_V2_EIBRS:
1500                 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) {
1501                         setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT_LITE);
1502                         pr_info("Spectre v2 / PBRSB-eIBRS: Retire a single CALL on VMEXIT\n");
1503                 }
1504                 return;
1505
1506         case SPECTRE_V2_EIBRS_RETPOLINE:
1507         case SPECTRE_V2_RETPOLINE:
1508         case SPECTRE_V2_LFENCE:
1509         case SPECTRE_V2_IBRS:
1510                 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT);
1511                 pr_info("Spectre v2 / SpectreRSB : Filling RSB on VMEXIT\n");
1512                 return;
1513         }
1514
1515         pr_warn_once("Unknown Spectre v2 mode, disabling RSB mitigation at VM exit");
1516         dump_stack();
1517 }
1518
1519 static void __init spectre_v2_select_mitigation(void)
1520 {
1521         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
1522         enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
1523
1524         /*
1525          * If the CPU is not affected and the command line mode is NONE or AUTO
1526          * then nothing to do.
1527          */
1528         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
1529             (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
1530                 return;
1531
1532         switch (cmd) {
1533         case SPECTRE_V2_CMD_NONE:
1534                 return;
1535
1536         case SPECTRE_V2_CMD_FORCE:
1537         case SPECTRE_V2_CMD_AUTO:
1538                 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
1539                         mode = SPECTRE_V2_EIBRS;
1540                         break;
1541                 }
1542
1543                 if (IS_ENABLED(CONFIG_CPU_IBRS_ENTRY) &&
1544                     boot_cpu_has_bug(X86_BUG_RETBLEED) &&
1545                     retbleed_cmd != RETBLEED_CMD_OFF &&
1546                     boot_cpu_has(X86_FEATURE_IBRS) &&
1547                     boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
1548                         mode = SPECTRE_V2_IBRS;
1549                         break;
1550                 }
1551
1552                 mode = spectre_v2_select_retpoline();
1553                 break;
1554
1555         case SPECTRE_V2_CMD_RETPOLINE_LFENCE:
1556                 pr_err(SPECTRE_V2_LFENCE_MSG);
1557                 mode = SPECTRE_V2_LFENCE;
1558                 break;
1559
1560         case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
1561                 mode = SPECTRE_V2_RETPOLINE;
1562                 break;
1563
1564         case SPECTRE_V2_CMD_RETPOLINE:
1565                 mode = spectre_v2_select_retpoline();
1566                 break;
1567
1568         case SPECTRE_V2_CMD_IBRS:
1569                 mode = SPECTRE_V2_IBRS;
1570                 break;
1571
1572         case SPECTRE_V2_CMD_EIBRS:
1573                 mode = SPECTRE_V2_EIBRS;
1574                 break;
1575
1576         case SPECTRE_V2_CMD_EIBRS_LFENCE:
1577                 mode = SPECTRE_V2_EIBRS_LFENCE;
1578                 break;
1579
1580         case SPECTRE_V2_CMD_EIBRS_RETPOLINE:
1581                 mode = SPECTRE_V2_EIBRS_RETPOLINE;
1582                 break;
1583         }
1584
1585         if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
1586                 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
1587
1588         if (spectre_v2_in_ibrs_mode(mode)) {
1589                 x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
1590                 update_spec_ctrl(x86_spec_ctrl_base);
1591         }
1592
1593         switch (mode) {
1594         case SPECTRE_V2_NONE:
1595         case SPECTRE_V2_EIBRS:
1596                 break;
1597
1598         case SPECTRE_V2_IBRS:
1599                 setup_force_cpu_cap(X86_FEATURE_KERNEL_IBRS);
1600                 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED))
1601                         pr_warn(SPECTRE_V2_IBRS_PERF_MSG);
1602                 break;
1603
1604         case SPECTRE_V2_LFENCE:
1605         case SPECTRE_V2_EIBRS_LFENCE:
1606                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE);
1607                 fallthrough;
1608
1609         case SPECTRE_V2_RETPOLINE:
1610         case SPECTRE_V2_EIBRS_RETPOLINE:
1611                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
1612                 break;
1613         }
1614
1615         /*
1616          * Disable alternate RSB predictions in kernel when indirect CALLs and
1617          * JMPs gets protection against BHI and Intramode-BTI, but RET
1618          * prediction from a non-RSB predictor is still a risk.
1619          */
1620         if (mode == SPECTRE_V2_EIBRS_LFENCE ||
1621             mode == SPECTRE_V2_EIBRS_RETPOLINE ||
1622             mode == SPECTRE_V2_RETPOLINE)
1623                 spec_ctrl_disable_kernel_rrsba();
1624
1625         spectre_v2_enabled = mode;
1626         pr_info("%s\n", spectre_v2_strings[mode]);
1627
1628         /*
1629          * If Spectre v2 protection has been enabled, fill the RSB during a
1630          * context switch.  In general there are two types of RSB attacks
1631          * across context switches, for which the CALLs/RETs may be unbalanced.
1632          *
1633          * 1) RSB underflow
1634          *
1635          *    Some Intel parts have "bottomless RSB".  When the RSB is empty,
1636          *    speculated return targets may come from the branch predictor,
1637          *    which could have a user-poisoned BTB or BHB entry.
1638          *
1639          *    AMD has it even worse: *all* returns are speculated from the BTB,
1640          *    regardless of the state of the RSB.
1641          *
1642          *    When IBRS or eIBRS is enabled, the "user -> kernel" attack
1643          *    scenario is mitigated by the IBRS branch prediction isolation
1644          *    properties, so the RSB buffer filling wouldn't be necessary to
1645          *    protect against this type of attack.
1646          *
1647          *    The "user -> user" attack scenario is mitigated by RSB filling.
1648          *
1649          * 2) Poisoned RSB entry
1650          *
1651          *    If the 'next' in-kernel return stack is shorter than 'prev',
1652          *    'next' could be tricked into speculating with a user-poisoned RSB
1653          *    entry.
1654          *
1655          *    The "user -> kernel" attack scenario is mitigated by SMEP and
1656          *    eIBRS.
1657          *
1658          *    The "user -> user" scenario, also known as SpectreBHB, requires
1659          *    RSB clearing.
1660          *
1661          * So to mitigate all cases, unconditionally fill RSB on context
1662          * switches.
1663          *
1664          * FIXME: Is this pointless for retbleed-affected AMD?
1665          */
1666         setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
1667         pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
1668
1669         spectre_v2_determine_rsb_fill_type_at_vmexit(mode);
1670
1671         /*
1672          * Retpoline protects the kernel, but doesn't protect firmware.  IBRS
1673          * and Enhanced IBRS protect firmware too, so enable IBRS around
1674          * firmware calls only when IBRS / Enhanced IBRS aren't otherwise
1675          * enabled.
1676          *
1677          * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
1678          * the user might select retpoline on the kernel command line and if
1679          * the CPU supports Enhanced IBRS, kernel might un-intentionally not
1680          * enable IBRS around firmware calls.
1681          */
1682         if (boot_cpu_has_bug(X86_BUG_RETBLEED) &&
1683             boot_cpu_has(X86_FEATURE_IBPB) &&
1684             (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
1685              boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)) {
1686
1687                 if (retbleed_cmd != RETBLEED_CMD_IBPB) {
1688                         setup_force_cpu_cap(X86_FEATURE_USE_IBPB_FW);
1689                         pr_info("Enabling Speculation Barrier for firmware calls\n");
1690                 }
1691
1692         } else if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_ibrs_mode(mode)) {
1693                 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
1694                 pr_info("Enabling Restricted Speculation for firmware calls\n");
1695         }
1696
1697         /* Set up IBPB and STIBP depending on the general spectre V2 command */
1698         spectre_v2_cmd = cmd;
1699 }
1700
1701 static void update_stibp_msr(void * __unused)
1702 {
1703         u64 val = spec_ctrl_current() | (x86_spec_ctrl_base & SPEC_CTRL_STIBP);
1704         update_spec_ctrl(val);
1705 }
1706
1707 /* Update x86_spec_ctrl_base in case SMT state changed. */
1708 static void update_stibp_strict(void)
1709 {
1710         u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP;
1711
1712         if (sched_smt_active())
1713                 mask |= SPEC_CTRL_STIBP;
1714
1715         if (mask == x86_spec_ctrl_base)
1716                 return;
1717
1718         pr_info("Update user space SMT mitigation: STIBP %s\n",
1719                 mask & SPEC_CTRL_STIBP ? "always-on" : "off");
1720         x86_spec_ctrl_base = mask;
1721         on_each_cpu(update_stibp_msr, NULL, 1);
1722 }
1723
1724 /* Update the static key controlling the evaluation of TIF_SPEC_IB */
1725 static void update_indir_branch_cond(void)
1726 {
1727         if (sched_smt_active())
1728                 static_branch_enable(&switch_to_cond_stibp);
1729         else
1730                 static_branch_disable(&switch_to_cond_stibp);
1731 }
1732
1733 #undef pr_fmt
1734 #define pr_fmt(fmt) fmt
1735
1736 /* Update the static key controlling the MDS CPU buffer clear in idle */
1737 static void update_mds_branch_idle(void)
1738 {
1739         u64 ia32_cap = x86_read_arch_cap_msr();
1740
1741         /*
1742          * Enable the idle clearing if SMT is active on CPUs which are
1743          * affected only by MSBDS and not any other MDS variant.
1744          *
1745          * The other variants cannot be mitigated when SMT is enabled, so
1746          * clearing the buffers on idle just to prevent the Store Buffer
1747          * repartitioning leak would be a window dressing exercise.
1748          */
1749         if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY))
1750                 return;
1751
1752         if (sched_smt_active()) {
1753                 static_branch_enable(&mds_idle_clear);
1754         } else if (mmio_mitigation == MMIO_MITIGATION_OFF ||
1755                    (ia32_cap & ARCH_CAP_FBSDP_NO)) {
1756                 static_branch_disable(&mds_idle_clear);
1757         }
1758 }
1759
1760 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n"
1761 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n"
1762 #define MMIO_MSG_SMT "MMIO Stale Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mmio_stale_data.html for more details.\n"
1763
1764 void cpu_bugs_smt_update(void)
1765 {
1766         mutex_lock(&spec_ctrl_mutex);
1767
1768         if (sched_smt_active() && unprivileged_ebpf_enabled() &&
1769             spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
1770                 pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
1771
1772         switch (spectre_v2_user_stibp) {
1773         case SPECTRE_V2_USER_NONE:
1774                 break;
1775         case SPECTRE_V2_USER_STRICT:
1776         case SPECTRE_V2_USER_STRICT_PREFERRED:
1777                 update_stibp_strict();
1778                 break;
1779         case SPECTRE_V2_USER_PRCTL:
1780         case SPECTRE_V2_USER_SECCOMP:
1781                 update_indir_branch_cond();
1782                 break;
1783         }
1784
1785         switch (mds_mitigation) {
1786         case MDS_MITIGATION_FULL:
1787         case MDS_MITIGATION_VMWERV:
1788                 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY))
1789                         pr_warn_once(MDS_MSG_SMT);
1790                 update_mds_branch_idle();
1791                 break;
1792         case MDS_MITIGATION_OFF:
1793                 break;
1794         }
1795
1796         switch (taa_mitigation) {
1797         case TAA_MITIGATION_VERW:
1798         case TAA_MITIGATION_UCODE_NEEDED:
1799                 if (sched_smt_active())
1800                         pr_warn_once(TAA_MSG_SMT);
1801                 break;
1802         case TAA_MITIGATION_TSX_DISABLED:
1803         case TAA_MITIGATION_OFF:
1804                 break;
1805         }
1806
1807         switch (mmio_mitigation) {
1808         case MMIO_MITIGATION_VERW:
1809         case MMIO_MITIGATION_UCODE_NEEDED:
1810                 if (sched_smt_active())
1811                         pr_warn_once(MMIO_MSG_SMT);
1812                 break;
1813         case MMIO_MITIGATION_OFF:
1814                 break;
1815         }
1816
1817         mutex_unlock(&spec_ctrl_mutex);
1818 }
1819
1820 #undef pr_fmt
1821 #define pr_fmt(fmt)     "Speculative Store Bypass: " fmt
1822
1823 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE;
1824
1825 /* The kernel command line selection */
1826 enum ssb_mitigation_cmd {
1827         SPEC_STORE_BYPASS_CMD_NONE,
1828         SPEC_STORE_BYPASS_CMD_AUTO,
1829         SPEC_STORE_BYPASS_CMD_ON,
1830         SPEC_STORE_BYPASS_CMD_PRCTL,
1831         SPEC_STORE_BYPASS_CMD_SECCOMP,
1832 };
1833
1834 static const char * const ssb_strings[] = {
1835         [SPEC_STORE_BYPASS_NONE]        = "Vulnerable",
1836         [SPEC_STORE_BYPASS_DISABLE]     = "Mitigation: Speculative Store Bypass disabled",
1837         [SPEC_STORE_BYPASS_PRCTL]       = "Mitigation: Speculative Store Bypass disabled via prctl",
1838         [SPEC_STORE_BYPASS_SECCOMP]     = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
1839 };
1840
1841 static const struct {
1842         const char *option;
1843         enum ssb_mitigation_cmd cmd;
1844 } ssb_mitigation_options[]  __initconst = {
1845         { "auto",       SPEC_STORE_BYPASS_CMD_AUTO },    /* Platform decides */
1846         { "on",         SPEC_STORE_BYPASS_CMD_ON },      /* Disable Speculative Store Bypass */
1847         { "off",        SPEC_STORE_BYPASS_CMD_NONE },    /* Don't touch Speculative Store Bypass */
1848         { "prctl",      SPEC_STORE_BYPASS_CMD_PRCTL },   /* Disable Speculative Store Bypass via prctl */
1849         { "seccomp",    SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */
1850 };
1851
1852 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
1853 {
1854         enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
1855         char arg[20];
1856         int ret, i;
1857
1858         if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable") ||
1859             cpu_mitigations_off()) {
1860                 return SPEC_STORE_BYPASS_CMD_NONE;
1861         } else {
1862                 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
1863                                           arg, sizeof(arg));
1864                 if (ret < 0)
1865                         return SPEC_STORE_BYPASS_CMD_AUTO;
1866
1867                 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
1868                         if (!match_option(arg, ret, ssb_mitigation_options[i].option))
1869                                 continue;
1870
1871                         cmd = ssb_mitigation_options[i].cmd;
1872                         break;
1873                 }
1874
1875                 if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
1876                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
1877                         return SPEC_STORE_BYPASS_CMD_AUTO;
1878                 }
1879         }
1880
1881         return cmd;
1882 }
1883
1884 static enum ssb_mitigation __init __ssb_select_mitigation(void)
1885 {
1886         enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
1887         enum ssb_mitigation_cmd cmd;
1888
1889         if (!boot_cpu_has(X86_FEATURE_SSBD))
1890                 return mode;
1891
1892         cmd = ssb_parse_cmdline();
1893         if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
1894             (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
1895              cmd == SPEC_STORE_BYPASS_CMD_AUTO))
1896                 return mode;
1897
1898         switch (cmd) {
1899         case SPEC_STORE_BYPASS_CMD_AUTO:
1900         case SPEC_STORE_BYPASS_CMD_SECCOMP:
1901                 /*
1902                  * Choose prctl+seccomp as the default mode if seccomp is
1903                  * enabled.
1904                  */
1905                 if (IS_ENABLED(CONFIG_SECCOMP))
1906                         mode = SPEC_STORE_BYPASS_SECCOMP;
1907                 else
1908                         mode = SPEC_STORE_BYPASS_PRCTL;
1909                 break;
1910         case SPEC_STORE_BYPASS_CMD_ON:
1911                 mode = SPEC_STORE_BYPASS_DISABLE;
1912                 break;
1913         case SPEC_STORE_BYPASS_CMD_PRCTL:
1914                 mode = SPEC_STORE_BYPASS_PRCTL;
1915                 break;
1916         case SPEC_STORE_BYPASS_CMD_NONE:
1917                 break;
1918         }
1919
1920         /*
1921          * We have three CPU feature flags that are in play here:
1922          *  - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
1923          *  - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
1924          *  - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
1925          */
1926         if (mode == SPEC_STORE_BYPASS_DISABLE) {
1927                 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
1928                 /*
1929                  * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
1930                  * use a completely different MSR and bit dependent on family.
1931                  */
1932                 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
1933                     !static_cpu_has(X86_FEATURE_AMD_SSBD)) {
1934                         x86_amd_ssb_disable();
1935                 } else {
1936                         x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
1937                         update_spec_ctrl(x86_spec_ctrl_base);
1938                 }
1939         }
1940
1941         return mode;
1942 }
1943
1944 static void ssb_select_mitigation(void)
1945 {
1946         ssb_mode = __ssb_select_mitigation();
1947
1948         if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
1949                 pr_info("%s\n", ssb_strings[ssb_mode]);
1950 }
1951
1952 #undef pr_fmt
1953 #define pr_fmt(fmt)     "Speculation prctl: " fmt
1954
1955 static void task_update_spec_tif(struct task_struct *tsk)
1956 {
1957         /* Force the update of the real TIF bits */
1958         set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE);
1959
1960         /*
1961          * Immediately update the speculation control MSRs for the current
1962          * task, but for a non-current task delay setting the CPU
1963          * mitigation until it is scheduled next.
1964          *
1965          * This can only happen for SECCOMP mitigation. For PRCTL it's
1966          * always the current task.
1967          */
1968         if (tsk == current)
1969                 speculation_ctrl_update_current();
1970 }
1971
1972 static int l1d_flush_prctl_set(struct task_struct *task, unsigned long ctrl)
1973 {
1974
1975         if (!static_branch_unlikely(&switch_mm_cond_l1d_flush))
1976                 return -EPERM;
1977
1978         switch (ctrl) {
1979         case PR_SPEC_ENABLE:
1980                 set_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH);
1981                 return 0;
1982         case PR_SPEC_DISABLE:
1983                 clear_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH);
1984                 return 0;
1985         default:
1986                 return -ERANGE;
1987         }
1988 }
1989
1990 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
1991 {
1992         if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
1993             ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
1994                 return -ENXIO;
1995
1996         switch (ctrl) {
1997         case PR_SPEC_ENABLE:
1998                 /* If speculation is force disabled, enable is not allowed */
1999                 if (task_spec_ssb_force_disable(task))
2000                         return -EPERM;
2001                 task_clear_spec_ssb_disable(task);
2002                 task_clear_spec_ssb_noexec(task);
2003                 task_update_spec_tif(task);
2004                 break;
2005         case PR_SPEC_DISABLE:
2006                 task_set_spec_ssb_disable(task);
2007                 task_clear_spec_ssb_noexec(task);
2008                 task_update_spec_tif(task);
2009                 break;
2010         case PR_SPEC_FORCE_DISABLE:
2011                 task_set_spec_ssb_disable(task);
2012                 task_set_spec_ssb_force_disable(task);
2013                 task_clear_spec_ssb_noexec(task);
2014                 task_update_spec_tif(task);
2015                 break;
2016         case PR_SPEC_DISABLE_NOEXEC:
2017                 if (task_spec_ssb_force_disable(task))
2018                         return -EPERM;
2019                 task_set_spec_ssb_disable(task);
2020                 task_set_spec_ssb_noexec(task);
2021                 task_update_spec_tif(task);
2022                 break;
2023         default:
2024                 return -ERANGE;
2025         }
2026         return 0;
2027 }
2028
2029 static bool is_spec_ib_user_controlled(void)
2030 {
2031         return spectre_v2_user_ibpb == SPECTRE_V2_USER_PRCTL ||
2032                 spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP ||
2033                 spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL ||
2034                 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP;
2035 }
2036
2037 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl)
2038 {
2039         switch (ctrl) {
2040         case PR_SPEC_ENABLE:
2041                 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
2042                     spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
2043                         return 0;
2044
2045                 /*
2046                  * With strict mode for both IBPB and STIBP, the instruction
2047                  * code paths avoid checking this task flag and instead,
2048                  * unconditionally run the instruction. However, STIBP and IBPB
2049                  * are independent and either can be set to conditionally
2050                  * enabled regardless of the mode of the other.
2051                  *
2052                  * If either is set to conditional, allow the task flag to be
2053                  * updated, unless it was force-disabled by a previous prctl
2054                  * call. Currently, this is possible on an AMD CPU which has the
2055                  * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the
2056                  * kernel is booted with 'spectre_v2_user=seccomp', then
2057                  * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and
2058                  * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED.
2059                  */
2060                 if (!is_spec_ib_user_controlled() ||
2061                     task_spec_ib_force_disable(task))
2062                         return -EPERM;
2063
2064                 task_clear_spec_ib_disable(task);
2065                 task_update_spec_tif(task);
2066                 break;
2067         case PR_SPEC_DISABLE:
2068         case PR_SPEC_FORCE_DISABLE:
2069                 /*
2070                  * Indirect branch speculation is always allowed when
2071                  * mitigation is force disabled.
2072                  */
2073                 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
2074                     spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
2075                         return -EPERM;
2076
2077                 if (!is_spec_ib_user_controlled())
2078                         return 0;
2079
2080                 task_set_spec_ib_disable(task);
2081                 if (ctrl == PR_SPEC_FORCE_DISABLE)
2082                         task_set_spec_ib_force_disable(task);
2083                 task_update_spec_tif(task);
2084                 if (task == current)
2085                         indirect_branch_prediction_barrier();
2086                 break;
2087         default:
2088                 return -ERANGE;
2089         }
2090         return 0;
2091 }
2092
2093 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
2094                              unsigned long ctrl)
2095 {
2096         switch (which) {
2097         case PR_SPEC_STORE_BYPASS:
2098                 return ssb_prctl_set(task, ctrl);
2099         case PR_SPEC_INDIRECT_BRANCH:
2100                 return ib_prctl_set(task, ctrl);
2101         case PR_SPEC_L1D_FLUSH:
2102                 return l1d_flush_prctl_set(task, ctrl);
2103         default:
2104                 return -ENODEV;
2105         }
2106 }
2107
2108 #ifdef CONFIG_SECCOMP
2109 void arch_seccomp_spec_mitigate(struct task_struct *task)
2110 {
2111         if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
2112                 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
2113         if (spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP ||
2114             spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP)
2115                 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE);
2116 }
2117 #endif
2118
2119 static int l1d_flush_prctl_get(struct task_struct *task)
2120 {
2121         if (!static_branch_unlikely(&switch_mm_cond_l1d_flush))
2122                 return PR_SPEC_FORCE_DISABLE;
2123
2124         if (test_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH))
2125                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
2126         else
2127                 return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
2128 }
2129
2130 static int ssb_prctl_get(struct task_struct *task)
2131 {
2132         switch (ssb_mode) {
2133         case SPEC_STORE_BYPASS_DISABLE:
2134                 return PR_SPEC_DISABLE;
2135         case SPEC_STORE_BYPASS_SECCOMP:
2136         case SPEC_STORE_BYPASS_PRCTL:
2137                 if (task_spec_ssb_force_disable(task))
2138                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
2139                 if (task_spec_ssb_noexec(task))
2140                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC;
2141                 if (task_spec_ssb_disable(task))
2142                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
2143                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
2144         default:
2145                 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
2146                         return PR_SPEC_ENABLE;
2147                 return PR_SPEC_NOT_AFFECTED;
2148         }
2149 }
2150
2151 static int ib_prctl_get(struct task_struct *task)
2152 {
2153         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
2154                 return PR_SPEC_NOT_AFFECTED;
2155
2156         if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
2157             spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
2158                 return PR_SPEC_ENABLE;
2159         else if (is_spec_ib_user_controlled()) {
2160                 if (task_spec_ib_force_disable(task))
2161                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
2162                 if (task_spec_ib_disable(task))
2163                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
2164                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
2165         } else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT ||
2166             spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT ||
2167             spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED)
2168                 return PR_SPEC_DISABLE;
2169         else
2170                 return PR_SPEC_NOT_AFFECTED;
2171 }
2172
2173 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
2174 {
2175         switch (which) {
2176         case PR_SPEC_STORE_BYPASS:
2177                 return ssb_prctl_get(task);
2178         case PR_SPEC_INDIRECT_BRANCH:
2179                 return ib_prctl_get(task);
2180         case PR_SPEC_L1D_FLUSH:
2181                 return l1d_flush_prctl_get(task);
2182         default:
2183                 return -ENODEV;
2184         }
2185 }
2186
2187 void x86_spec_ctrl_setup_ap(void)
2188 {
2189         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
2190                 update_spec_ctrl(x86_spec_ctrl_base);
2191
2192         if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
2193                 x86_amd_ssb_disable();
2194 }
2195
2196 bool itlb_multihit_kvm_mitigation;
2197 EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation);
2198
2199 #undef pr_fmt
2200 #define pr_fmt(fmt)     "L1TF: " fmt
2201
2202 /* Default mitigation for L1TF-affected CPUs */
2203 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH;
2204 #if IS_ENABLED(CONFIG_KVM_INTEL)
2205 EXPORT_SYMBOL_GPL(l1tf_mitigation);
2206 #endif
2207 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
2208 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation);
2209
2210 /*
2211  * These CPUs all support 44bits physical address space internally in the
2212  * cache but CPUID can report a smaller number of physical address bits.
2213  *
2214  * The L1TF mitigation uses the top most address bit for the inversion of
2215  * non present PTEs. When the installed memory reaches into the top most
2216  * address bit due to memory holes, which has been observed on machines
2217  * which report 36bits physical address bits and have 32G RAM installed,
2218  * then the mitigation range check in l1tf_select_mitigation() triggers.
2219  * This is a false positive because the mitigation is still possible due to
2220  * the fact that the cache uses 44bit internally. Use the cache bits
2221  * instead of the reported physical bits and adjust them on the affected
2222  * machines to 44bit if the reported bits are less than 44.
2223  */
2224 static void override_cache_bits(struct cpuinfo_x86 *c)
2225 {
2226         if (c->x86 != 6)
2227                 return;
2228
2229         switch (c->x86_model) {
2230         case INTEL_FAM6_NEHALEM:
2231         case INTEL_FAM6_WESTMERE:
2232         case INTEL_FAM6_SANDYBRIDGE:
2233         case INTEL_FAM6_IVYBRIDGE:
2234         case INTEL_FAM6_HASWELL:
2235         case INTEL_FAM6_HASWELL_L:
2236         case INTEL_FAM6_HASWELL_G:
2237         case INTEL_FAM6_BROADWELL:
2238         case INTEL_FAM6_BROADWELL_G:
2239         case INTEL_FAM6_SKYLAKE_L:
2240         case INTEL_FAM6_SKYLAKE:
2241         case INTEL_FAM6_KABYLAKE_L:
2242         case INTEL_FAM6_KABYLAKE:
2243                 if (c->x86_cache_bits < 44)
2244                         c->x86_cache_bits = 44;
2245                 break;
2246         }
2247 }
2248
2249 static void __init l1tf_select_mitigation(void)
2250 {
2251         u64 half_pa;
2252
2253         if (!boot_cpu_has_bug(X86_BUG_L1TF))
2254                 return;
2255
2256         if (cpu_mitigations_off())
2257                 l1tf_mitigation = L1TF_MITIGATION_OFF;
2258         else if (cpu_mitigations_auto_nosmt())
2259                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
2260
2261         override_cache_bits(&boot_cpu_data);
2262
2263         switch (l1tf_mitigation) {
2264         case L1TF_MITIGATION_OFF:
2265         case L1TF_MITIGATION_FLUSH_NOWARN:
2266         case L1TF_MITIGATION_FLUSH:
2267                 break;
2268         case L1TF_MITIGATION_FLUSH_NOSMT:
2269         case L1TF_MITIGATION_FULL:
2270                 cpu_smt_disable(false);
2271                 break;
2272         case L1TF_MITIGATION_FULL_FORCE:
2273                 cpu_smt_disable(true);
2274                 break;
2275         }
2276
2277 #if CONFIG_PGTABLE_LEVELS == 2
2278         pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
2279         return;
2280 #endif
2281
2282         half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT;
2283         if (l1tf_mitigation != L1TF_MITIGATION_OFF &&
2284                         e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) {
2285                 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
2286                 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
2287                                 half_pa);
2288                 pr_info("However, doing so will make a part of your RAM unusable.\n");
2289                 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n");
2290                 return;
2291         }
2292
2293         setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV);
2294 }
2295
2296 static int __init l1tf_cmdline(char *str)
2297 {
2298         if (!boot_cpu_has_bug(X86_BUG_L1TF))
2299                 return 0;
2300
2301         if (!str)
2302                 return -EINVAL;
2303
2304         if (!strcmp(str, "off"))
2305                 l1tf_mitigation = L1TF_MITIGATION_OFF;
2306         else if (!strcmp(str, "flush,nowarn"))
2307                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN;
2308         else if (!strcmp(str, "flush"))
2309                 l1tf_mitigation = L1TF_MITIGATION_FLUSH;
2310         else if (!strcmp(str, "flush,nosmt"))
2311                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
2312         else if (!strcmp(str, "full"))
2313                 l1tf_mitigation = L1TF_MITIGATION_FULL;
2314         else if (!strcmp(str, "full,force"))
2315                 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE;
2316
2317         return 0;
2318 }
2319 early_param("l1tf", l1tf_cmdline);
2320
2321 #undef pr_fmt
2322 #define pr_fmt(fmt)     "Speculative Return Stack Overflow: " fmt
2323
2324 enum srso_mitigation {
2325         SRSO_MITIGATION_NONE,
2326         SRSO_MITIGATION_MICROCODE,
2327         SRSO_MITIGATION_SAFE_RET,
2328         SRSO_MITIGATION_IBPB,
2329         SRSO_MITIGATION_IBPB_ON_VMEXIT,
2330 };
2331
2332 enum srso_mitigation_cmd {
2333         SRSO_CMD_OFF,
2334         SRSO_CMD_MICROCODE,
2335         SRSO_CMD_SAFE_RET,
2336         SRSO_CMD_IBPB,
2337         SRSO_CMD_IBPB_ON_VMEXIT,
2338 };
2339
2340 static const char * const srso_strings[] = {
2341         [SRSO_MITIGATION_NONE]           = "Vulnerable",
2342         [SRSO_MITIGATION_MICROCODE]      = "Mitigation: microcode",
2343         [SRSO_MITIGATION_SAFE_RET]       = "Mitigation: safe RET",
2344         [SRSO_MITIGATION_IBPB]           = "Mitigation: IBPB",
2345         [SRSO_MITIGATION_IBPB_ON_VMEXIT] = "Mitigation: IBPB on VMEXIT only"
2346 };
2347
2348 static enum srso_mitigation srso_mitigation __ro_after_init = SRSO_MITIGATION_NONE;
2349 static enum srso_mitigation_cmd srso_cmd __ro_after_init = SRSO_CMD_SAFE_RET;
2350
2351 static int __init srso_parse_cmdline(char *str)
2352 {
2353         if (!str)
2354                 return -EINVAL;
2355
2356         if (!strcmp(str, "off"))
2357                 srso_cmd = SRSO_CMD_OFF;
2358         else if (!strcmp(str, "microcode"))
2359                 srso_cmd = SRSO_CMD_MICROCODE;
2360         else if (!strcmp(str, "safe-ret"))
2361                 srso_cmd = SRSO_CMD_SAFE_RET;
2362         else if (!strcmp(str, "ibpb"))
2363                 srso_cmd = SRSO_CMD_IBPB;
2364         else if (!strcmp(str, "ibpb-vmexit"))
2365                 srso_cmd = SRSO_CMD_IBPB_ON_VMEXIT;
2366         else
2367                 pr_err("Ignoring unknown SRSO option (%s).", str);
2368
2369         return 0;
2370 }
2371 early_param("spec_rstack_overflow", srso_parse_cmdline);
2372
2373 #define SRSO_NOTICE "WARNING: See https://kernel.org/doc/html/latest/admin-guide/hw-vuln/srso.html for mitigation options."
2374
2375 static void __init srso_select_mitigation(void)
2376 {
2377         bool has_microcode;
2378
2379         if (!boot_cpu_has_bug(X86_BUG_SRSO) || cpu_mitigations_off())
2380                 goto pred_cmd;
2381
2382         /*
2383          * The first check is for the kernel running as a guest in order
2384          * for guests to verify whether IBPB is a viable mitigation.
2385          */
2386         has_microcode = boot_cpu_has(X86_FEATURE_IBPB_BRTYPE) || cpu_has_ibpb_brtype_microcode();
2387         if (!has_microcode) {
2388                 pr_warn("IBPB-extending microcode not applied!\n");
2389                 pr_warn(SRSO_NOTICE);
2390         } else {
2391                 /*
2392                  * Enable the synthetic (even if in a real CPUID leaf)
2393                  * flags for guests.
2394                  */
2395                 setup_force_cpu_cap(X86_FEATURE_IBPB_BRTYPE);
2396
2397                 /*
2398                  * Zen1/2 with SMT off aren't vulnerable after the right
2399                  * IBPB microcode has been applied.
2400                  */
2401                 if (boot_cpu_data.x86 < 0x19 && !cpu_smt_possible()) {
2402                         setup_force_cpu_cap(X86_FEATURE_SRSO_NO);
2403                         return;
2404                 }
2405         }
2406
2407         if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB) {
2408                 if (has_microcode) {
2409                         pr_err("Retbleed IBPB mitigation enabled, using same for SRSO\n");
2410                         srso_mitigation = SRSO_MITIGATION_IBPB;
2411                         goto pred_cmd;
2412                 }
2413         }
2414
2415         switch (srso_cmd) {
2416         case SRSO_CMD_OFF:
2417                 goto pred_cmd;
2418
2419         case SRSO_CMD_MICROCODE:
2420                 if (has_microcode) {
2421                         srso_mitigation = SRSO_MITIGATION_MICROCODE;
2422                         pr_warn(SRSO_NOTICE);
2423                 }
2424                 break;
2425
2426         case SRSO_CMD_SAFE_RET:
2427                 if (IS_ENABLED(CONFIG_CPU_SRSO)) {
2428                         /*
2429                          * Enable the return thunk for generated code
2430                          * like ftrace, static_call, etc.
2431                          */
2432                         setup_force_cpu_cap(X86_FEATURE_RETHUNK);
2433                         setup_force_cpu_cap(X86_FEATURE_UNRET);
2434
2435                         if (boot_cpu_data.x86 == 0x19) {
2436                                 setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS);
2437                                 x86_return_thunk = srso_alias_return_thunk;
2438                         } else {
2439                                 setup_force_cpu_cap(X86_FEATURE_SRSO);
2440                                 x86_return_thunk = srso_return_thunk;
2441                         }
2442                         srso_mitigation = SRSO_MITIGATION_SAFE_RET;
2443                 } else {
2444                         pr_err("WARNING: kernel not compiled with CPU_SRSO.\n");
2445                         goto pred_cmd;
2446                 }
2447                 break;
2448
2449         case SRSO_CMD_IBPB:
2450                 if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY)) {
2451                         if (has_microcode) {
2452                                 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB);
2453                                 srso_mitigation = SRSO_MITIGATION_IBPB;
2454                         }
2455                 } else {
2456                         pr_err("WARNING: kernel not compiled with CPU_IBPB_ENTRY.\n");
2457                         goto pred_cmd;
2458                 }
2459                 break;
2460
2461         case SRSO_CMD_IBPB_ON_VMEXIT:
2462                 if (IS_ENABLED(CONFIG_CPU_SRSO)) {
2463                         if (!boot_cpu_has(X86_FEATURE_ENTRY_IBPB) && has_microcode) {
2464                                 setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT);
2465                                 srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT;
2466                         }
2467                 } else {
2468                         pr_err("WARNING: kernel not compiled with CPU_SRSO.\n");
2469                         goto pred_cmd;
2470                 }
2471                 break;
2472
2473         default:
2474                 break;
2475         }
2476
2477         pr_info("%s%s\n", srso_strings[srso_mitigation], (has_microcode ? "" : ", no microcode"));
2478
2479 pred_cmd:
2480         if ((boot_cpu_has(X86_FEATURE_SRSO_NO) || srso_cmd == SRSO_CMD_OFF) &&
2481              boot_cpu_has(X86_FEATURE_SBPB))
2482                 x86_pred_cmd = PRED_CMD_SBPB;
2483 }
2484
2485 #undef pr_fmt
2486 #define pr_fmt(fmt) fmt
2487
2488 #ifdef CONFIG_SYSFS
2489
2490 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
2491
2492 #if IS_ENABLED(CONFIG_KVM_INTEL)
2493 static const char * const l1tf_vmx_states[] = {
2494         [VMENTER_L1D_FLUSH_AUTO]                = "auto",
2495         [VMENTER_L1D_FLUSH_NEVER]               = "vulnerable",
2496         [VMENTER_L1D_FLUSH_COND]                = "conditional cache flushes",
2497         [VMENTER_L1D_FLUSH_ALWAYS]              = "cache flushes",
2498         [VMENTER_L1D_FLUSH_EPT_DISABLED]        = "EPT disabled",
2499         [VMENTER_L1D_FLUSH_NOT_REQUIRED]        = "flush not necessary"
2500 };
2501
2502 static ssize_t l1tf_show_state(char *buf)
2503 {
2504         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO)
2505                 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
2506
2507         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED ||
2508             (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER &&
2509              sched_smt_active())) {
2510                 return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG,
2511                                l1tf_vmx_states[l1tf_vmx_mitigation]);
2512         }
2513
2514         return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG,
2515                        l1tf_vmx_states[l1tf_vmx_mitigation],
2516                        sched_smt_active() ? "vulnerable" : "disabled");
2517 }
2518
2519 static ssize_t itlb_multihit_show_state(char *buf)
2520 {
2521         if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) ||
2522             !boot_cpu_has(X86_FEATURE_VMX))
2523                 return sprintf(buf, "KVM: Mitigation: VMX unsupported\n");
2524         else if (!(cr4_read_shadow() & X86_CR4_VMXE))
2525                 return sprintf(buf, "KVM: Mitigation: VMX disabled\n");
2526         else if (itlb_multihit_kvm_mitigation)
2527                 return sprintf(buf, "KVM: Mitigation: Split huge pages\n");
2528         else
2529                 return sprintf(buf, "KVM: Vulnerable\n");
2530 }
2531 #else
2532 static ssize_t l1tf_show_state(char *buf)
2533 {
2534         return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
2535 }
2536
2537 static ssize_t itlb_multihit_show_state(char *buf)
2538 {
2539         return sprintf(buf, "Processor vulnerable\n");
2540 }
2541 #endif
2542
2543 static ssize_t mds_show_state(char *buf)
2544 {
2545         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
2546                 return sprintf(buf, "%s; SMT Host state unknown\n",
2547                                mds_strings[mds_mitigation]);
2548         }
2549
2550         if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) {
2551                 return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
2552                                (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" :
2553                                 sched_smt_active() ? "mitigated" : "disabled"));
2554         }
2555
2556         return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
2557                        sched_smt_active() ? "vulnerable" : "disabled");
2558 }
2559
2560 static ssize_t tsx_async_abort_show_state(char *buf)
2561 {
2562         if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) ||
2563             (taa_mitigation == TAA_MITIGATION_OFF))
2564                 return sprintf(buf, "%s\n", taa_strings[taa_mitigation]);
2565
2566         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
2567                 return sprintf(buf, "%s; SMT Host state unknown\n",
2568                                taa_strings[taa_mitigation]);
2569         }
2570
2571         return sprintf(buf, "%s; SMT %s\n", taa_strings[taa_mitigation],
2572                        sched_smt_active() ? "vulnerable" : "disabled");
2573 }
2574
2575 static ssize_t mmio_stale_data_show_state(char *buf)
2576 {
2577         if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN))
2578                 return sysfs_emit(buf, "Unknown: No mitigations\n");
2579
2580         if (mmio_mitigation == MMIO_MITIGATION_OFF)
2581                 return sysfs_emit(buf, "%s\n", mmio_strings[mmio_mitigation]);
2582
2583         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
2584                 return sysfs_emit(buf, "%s; SMT Host state unknown\n",
2585                                   mmio_strings[mmio_mitigation]);
2586         }
2587
2588         return sysfs_emit(buf, "%s; SMT %s\n", mmio_strings[mmio_mitigation],
2589                           sched_smt_active() ? "vulnerable" : "disabled");
2590 }
2591
2592 static char *stibp_state(void)
2593 {
2594         if (spectre_v2_in_eibrs_mode(spectre_v2_enabled))
2595                 return "";
2596
2597         switch (spectre_v2_user_stibp) {
2598         case SPECTRE_V2_USER_NONE:
2599                 return ", STIBP: disabled";
2600         case SPECTRE_V2_USER_STRICT:
2601                 return ", STIBP: forced";
2602         case SPECTRE_V2_USER_STRICT_PREFERRED:
2603                 return ", STIBP: always-on";
2604         case SPECTRE_V2_USER_PRCTL:
2605         case SPECTRE_V2_USER_SECCOMP:
2606                 if (static_key_enabled(&switch_to_cond_stibp))
2607                         return ", STIBP: conditional";
2608         }
2609         return "";
2610 }
2611
2612 static char *ibpb_state(void)
2613 {
2614         if (boot_cpu_has(X86_FEATURE_IBPB)) {
2615                 if (static_key_enabled(&switch_mm_always_ibpb))
2616                         return ", IBPB: always-on";
2617                 if (static_key_enabled(&switch_mm_cond_ibpb))
2618                         return ", IBPB: conditional";
2619                 return ", IBPB: disabled";
2620         }
2621         return "";
2622 }
2623
2624 static char *pbrsb_eibrs_state(void)
2625 {
2626         if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) {
2627                 if (boot_cpu_has(X86_FEATURE_RSB_VMEXIT_LITE) ||
2628                     boot_cpu_has(X86_FEATURE_RSB_VMEXIT))
2629                         return ", PBRSB-eIBRS: SW sequence";
2630                 else
2631                         return ", PBRSB-eIBRS: Vulnerable";
2632         } else {
2633                 return ", PBRSB-eIBRS: Not affected";
2634         }
2635 }
2636
2637 static ssize_t spectre_v2_show_state(char *buf)
2638 {
2639         if (spectre_v2_enabled == SPECTRE_V2_LFENCE)
2640                 return sprintf(buf, "Vulnerable: LFENCE\n");
2641
2642         if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
2643                 return sprintf(buf, "Vulnerable: eIBRS with unprivileged eBPF\n");
2644
2645         if (sched_smt_active() && unprivileged_ebpf_enabled() &&
2646             spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
2647                 return sprintf(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n");
2648
2649         return sprintf(buf, "%s%s%s%s%s%s%s\n",
2650                        spectre_v2_strings[spectre_v2_enabled],
2651                        ibpb_state(),
2652                        boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
2653                        stibp_state(),
2654                        boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "",
2655                        pbrsb_eibrs_state(),
2656                        spectre_v2_module_string());
2657 }
2658
2659 static ssize_t srbds_show_state(char *buf)
2660 {
2661         return sprintf(buf, "%s\n", srbds_strings[srbds_mitigation]);
2662 }
2663
2664 static ssize_t retbleed_show_state(char *buf)
2665 {
2666         if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET ||
2667             retbleed_mitigation == RETBLEED_MITIGATION_IBPB) {
2668             if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
2669                 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
2670                     return sprintf(buf, "Vulnerable: untrained return thunk / IBPB on non-AMD based uarch\n");
2671
2672             return sprintf(buf, "%s; SMT %s\n",
2673                            retbleed_strings[retbleed_mitigation],
2674                            !sched_smt_active() ? "disabled" :
2675                            spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT ||
2676                            spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED ?
2677                            "enabled with STIBP protection" : "vulnerable");
2678         }
2679
2680         return sprintf(buf, "%s\n", retbleed_strings[retbleed_mitigation]);
2681 }
2682
2683 static ssize_t gds_show_state(char *buf)
2684 {
2685         return sysfs_emit(buf, "%s\n", gds_strings[gds_mitigation]);
2686 }
2687
2688 static ssize_t srso_show_state(char *buf)
2689 {
2690         if (boot_cpu_has(X86_FEATURE_SRSO_NO))
2691                 return sysfs_emit(buf, "Mitigation: SMT disabled\n");
2692
2693         return sysfs_emit(buf, "%s%s\n",
2694                           srso_strings[srso_mitigation],
2695                           boot_cpu_has(X86_FEATURE_IBPB_BRTYPE) ? "" : ", no microcode");
2696 }
2697
2698 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
2699                                char *buf, unsigned int bug)
2700 {
2701         if (!boot_cpu_has_bug(bug))
2702                 return sprintf(buf, "Not affected\n");
2703
2704         switch (bug) {
2705         case X86_BUG_CPU_MELTDOWN:
2706                 if (boot_cpu_has(X86_FEATURE_PTI))
2707                         return sprintf(buf, "Mitigation: PTI\n");
2708
2709                 if (hypervisor_is_type(X86_HYPER_XEN_PV))
2710                         return sprintf(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n");
2711
2712                 break;
2713
2714         case X86_BUG_SPECTRE_V1:
2715                 return sprintf(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]);
2716
2717         case X86_BUG_SPECTRE_V2:
2718                 return spectre_v2_show_state(buf);
2719
2720         case X86_BUG_SPEC_STORE_BYPASS:
2721                 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
2722
2723         case X86_BUG_L1TF:
2724                 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV))
2725                         return l1tf_show_state(buf);
2726                 break;
2727
2728         case X86_BUG_MDS:
2729                 return mds_show_state(buf);
2730
2731         case X86_BUG_TAA:
2732                 return tsx_async_abort_show_state(buf);
2733
2734         case X86_BUG_ITLB_MULTIHIT:
2735                 return itlb_multihit_show_state(buf);
2736
2737         case X86_BUG_SRBDS:
2738                 return srbds_show_state(buf);
2739
2740         case X86_BUG_MMIO_STALE_DATA:
2741         case X86_BUG_MMIO_UNKNOWN:
2742                 return mmio_stale_data_show_state(buf);
2743
2744         case X86_BUG_RETBLEED:
2745                 return retbleed_show_state(buf);
2746
2747         case X86_BUG_GDS:
2748                 return gds_show_state(buf);
2749
2750         case X86_BUG_SRSO:
2751                 return srso_show_state(buf);
2752
2753         default:
2754                 break;
2755         }
2756
2757         return sprintf(buf, "Vulnerable\n");
2758 }
2759
2760 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
2761 {
2762         return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
2763 }
2764
2765 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
2766 {
2767         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
2768 }
2769
2770 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
2771 {
2772         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
2773 }
2774
2775 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
2776 {
2777         return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
2778 }
2779
2780 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf)
2781 {
2782         return cpu_show_common(dev, attr, buf, X86_BUG_L1TF);
2783 }
2784
2785 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf)
2786 {
2787         return cpu_show_common(dev, attr, buf, X86_BUG_MDS);
2788 }
2789
2790 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf)
2791 {
2792         return cpu_show_common(dev, attr, buf, X86_BUG_TAA);
2793 }
2794
2795 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf)
2796 {
2797         return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT);
2798 }
2799
2800 ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *buf)
2801 {
2802         return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS);
2803 }
2804
2805 ssize_t cpu_show_mmio_stale_data(struct device *dev, struct device_attribute *attr, char *buf)
2806 {
2807         if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN))
2808                 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_UNKNOWN);
2809         else
2810                 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA);
2811 }
2812
2813 ssize_t cpu_show_retbleed(struct device *dev, struct device_attribute *attr, char *buf)
2814 {
2815         return cpu_show_common(dev, attr, buf, X86_BUG_RETBLEED);
2816 }
2817
2818 ssize_t cpu_show_gds(struct device *dev, struct device_attribute *attr, char *buf)
2819 {
2820         return cpu_show_common(dev, attr, buf, X86_BUG_GDS);
2821 }
2822
2823 ssize_t cpu_show_spec_rstack_overflow(struct device *dev, struct device_attribute *attr, char *buf)
2824 {
2825         return cpu_show_common(dev, attr, buf, X86_BUG_SRSO);
2826 }
2827 #endif