x86/kernel/cpu/common.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* cpu_feature_enabled() cannot be used this early */
   3 #define USE_EARLY_PGTABLE_L5
   4
   5 #include <linux/memblock.h>
   6 #include <linux/linkage.h>
   7 #include <linux/bitops.h>
   8 #include <linux/kernel.h>
   9 #include <linux/export.h>
  10 #include <linux/percpu.h>
  11 #include <linux/string.h>
  12 #include <linux/ctype.h>
  13 #include <linux/delay.h>
  14 #include <linux/sched/mm.h>
  15 #include <linux/sched/clock.h>
  16 #include <linux/sched/task.h>
  17 #include <linux/sched/smt.h>
  18 #include <linux/init.h>
  19 #include <linux/kprobes.h>
  20 #include <linux/kgdb.h>
  21 #include <linux/mem_encrypt.h>
  22 #include <linux/smp.h>
  23 #include <linux/cpu.h>
  24 #include <linux/io.h>
  25 #include <linux/syscore_ops.h>
  26 #include <linux/pgtable.h>
  27 #include <linux/utsname.h>
  28
  29 #include <asm/alternative.h>
  30 #include <asm/cmdline.h>
  31 #include <asm/stackprotector.h>
  32 #include <asm/perf_event.h>
  33 #include <asm/mmu_context.h>
  34 #include <asm/doublefault.h>
  35 #include <asm/archrandom.h>
  36 #include <asm/hypervisor.h>
  37 #include <asm/processor.h>
  38 #include <asm/tlbflush.h>
  39 #include <asm/debugreg.h>
  40 #include <asm/sections.h>
  41 #include <asm/vsyscall.h>
  42 #include <linux/topology.h>
  43 #include <linux/cpumask.h>
  44 #include <linux/atomic.h>
  45 #include <asm/proto.h>
  46 #include <asm/setup.h>
  47 #include <asm/apic.h>
  48 #include <asm/desc.h>
  49 #include <asm/fpu/api.h>
  50 #include <asm/mtrr.h>
  51 #include <asm/hwcap2.h>
  52 #include <linux/numa.h>
  53 #include <asm/numa.h>
  54 #include <asm/asm.h>
  55 #include <asm/bugs.h>
  56 #include <asm/cpu.h>
  57 #include <asm/mce.h>
  58 #include <asm/msr.h>
  59 #include <asm/memtype.h>
  60 #include <asm/microcode.h>
  61 #include <asm/microcode_intel.h>
  62 #include <asm/intel-family.h>
  63 #include <asm/cpu_device_id.h>
  64 #include <asm/uv/uv.h>
  65 #include <asm/set_memory.h>
  66 #include <asm/traps.h>
  67 #include <asm/sev.h>
  68
  69 #include "cpu.h"
  70
  71 u32 elf_hwcap2 __read_mostly;
  72
  73 /* all of these masks are initialized in setup_cpu_local_masks() */
  74 cpumask_var_t cpu_initialized_mask;
  75 cpumask_var_t cpu_callout_mask;
  76 cpumask_var_t cpu_callin_mask;
  77
  78 /* representing cpus for which sibling maps can be computed */
  79 cpumask_var_t cpu_sibling_setup_mask;
  80
  81 /* Number of siblings per CPU package */
  82 int smp_num_siblings = 1;
  83 EXPORT_SYMBOL(smp_num_siblings);
  84
  85 /* Last level cache ID of each logical CPU */
  86 DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID;
  87
  88 u16 get_llc_id(unsigned int cpu)
  89 {
  90         return per_cpu(cpu_llc_id, cpu);
  91 }
  92 EXPORT_SYMBOL_GPL(get_llc_id);
  93
  94 /* L2 cache ID of each logical CPU */
  95 DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_l2c_id) = BAD_APICID;
  96
  97 static struct ppin_info {
  98         int     feature;
  99         int     msr_ppin_ctl;
 100         int     msr_ppin;
 101 } ppin_info[] = {
 102         [X86_VENDOR_INTEL] = {
 103                 .feature = X86_FEATURE_INTEL_PPIN,
 104                 .msr_ppin_ctl = MSR_PPIN_CTL,
 105                 .msr_ppin = MSR_PPIN
 106         },
 107         [X86_VENDOR_AMD] = {
 108                 .feature = X86_FEATURE_AMD_PPIN,
 109                 .msr_ppin_ctl = MSR_AMD_PPIN_CTL,
 110                 .msr_ppin = MSR_AMD_PPIN
 111         },
 112 };
 113
 114 static const struct x86_cpu_id ppin_cpuids[] = {
 115         X86_MATCH_FEATURE(X86_FEATURE_AMD_PPIN, &ppin_info[X86_VENDOR_AMD]),
 116         X86_MATCH_FEATURE(X86_FEATURE_INTEL_PPIN, &ppin_info[X86_VENDOR_INTEL]),
 117
 118         /* Legacy models without CPUID enumeration */
 119         X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &ppin_info[X86_VENDOR_INTEL]),
 120         X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &ppin_info[X86_VENDOR_INTEL]),
 121         X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &ppin_info[X86_VENDOR_INTEL]),
 122         X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &ppin_info[X86_VENDOR_INTEL]),
 123         X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &ppin_info[X86_VENDOR_INTEL]),
 124         X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &ppin_info[X86_VENDOR_INTEL]),
 125         X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &ppin_info[X86_VENDOR_INTEL]),
 126         X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &ppin_info[X86_VENDOR_INTEL]),
 127         X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &ppin_info[X86_VENDOR_INTEL]),
 128         X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &ppin_info[X86_VENDOR_INTEL]),
 129
 130         {}
 131 };
 132
 133 static void ppin_init(struct cpuinfo_x86 *c)
 134 {
 135         const struct x86_cpu_id *id;
 136         unsigned long long val;
 137         struct ppin_info *info;
 138
 139         id = x86_match_cpu(ppin_cpuids);
 140         if (!id)
 141                 return;
 142
 143         /*
 144          * Testing the presence of the MSR is not enough. Need to check
 145          * that the PPIN_CTL allows reading of the PPIN.
 146          */
 147         info = (struct ppin_info *)id->driver_data;
 148
 149         if (rdmsrl_safe(info->msr_ppin_ctl, &val))
 150                 goto clear_ppin;
 151
 152         if ((val & 3UL) == 1UL) {
 153                 /* PPIN locked in disabled mode */
 154                 goto clear_ppin;
 155         }
 156
 157         /* If PPIN is disabled, try to enable */
 158         if (!(val & 2UL)) {
 159                 wrmsrl_safe(info->msr_ppin_ctl,  val | 2UL);
 160                 rdmsrl_safe(info->msr_ppin_ctl, &val);
 161         }
 162
 163         /* Is the enable bit set? */
 164         if (val & 2UL) {
 165                 c->ppin = __rdmsr(info->msr_ppin);
 166                 set_cpu_cap(c, info->feature);
 167                 return;
 168         }
 169
 170 clear_ppin:
 171         clear_cpu_cap(c, info->feature);
 172 }
 173
 174 /* correctly size the local cpu masks */
 175 void __init setup_cpu_local_masks(void)
 176 {
 177         alloc_bootmem_cpumask_var(&cpu_initialized_mask);
 178         alloc_bootmem_cpumask_var(&cpu_callin_mask);
 179         alloc_bootmem_cpumask_var(&cpu_callout_mask);
 180         alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
 181 }
 182
 183 static void default_init(struct cpuinfo_x86 *c)
 184 {
 185 #ifdef CONFIG_X86_64
 186         cpu_detect_cache_sizes(c);
 187 #else
 188         /* Not much we can do here... */
 189         /* Check if at least it has cpuid */
 190         if (c->cpuid_level == -1) {
 191                 /* No cpuid. It must be an ancient CPU */
 192                 if (c->x86 == 4)
 193                         strcpy(c->x86_model_id, "486");
 194                 else if (c->x86 == 3)
 195                         strcpy(c->x86_model_id, "386");
 196         }
 197 #endif
 198 }
 199
 200 static const struct cpu_dev default_cpu = {
 201         .c_init         = default_init,
 202         .c_vendor       = "Unknown",
 203         .c_x86_vendor   = X86_VENDOR_UNKNOWN,
 204 };
 205
 206 static const struct cpu_dev *this_cpu = &default_cpu;
 207
 208 DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
 209 #ifdef CONFIG_X86_64
 210         /*
 211          * We need valid kernel segments for data and code in long mode too
 212          * IRET will check the segment types  kkeil 2000/10/28
 213          * Also sysret mandates a special GDT layout
 214          *
 215          * TLS descriptors are currently at a different place compared to i386.
 216          * Hopefully nobody expects them at a fixed place (Wine?)
 217          */
 218         [GDT_ENTRY_KERNEL32_CS]         = GDT_ENTRY_INIT(0xc09b, 0, 0xfffff),
 219         [GDT_ENTRY_KERNEL_CS]           = GDT_ENTRY_INIT(0xa09b, 0, 0xfffff),
 220         [GDT_ENTRY_KERNEL_DS]           = GDT_ENTRY_INIT(0xc093, 0, 0xfffff),
 221         [GDT_ENTRY_DEFAULT_USER32_CS]   = GDT_ENTRY_INIT(0xc0fb, 0, 0xfffff),
 222         [GDT_ENTRY_DEFAULT_USER_DS]     = GDT_ENTRY_INIT(0xc0f3, 0, 0xfffff),
 223         [GDT_ENTRY_DEFAULT_USER_CS]     = GDT_ENTRY_INIT(0xa0fb, 0, 0xfffff),
 224 #else
 225         [GDT_ENTRY_KERNEL_CS]           = GDT_ENTRY_INIT(0xc09a, 0, 0xfffff),
 226         [GDT_ENTRY_KERNEL_DS]           = GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
 227         [GDT_ENTRY_DEFAULT_USER_CS]     = GDT_ENTRY_INIT(0xc0fa, 0, 0xfffff),
 228         [GDT_ENTRY_DEFAULT_USER_DS]     = GDT_ENTRY_INIT(0xc0f2, 0, 0xfffff),
 229         /*
 230          * Segments used for calling PnP BIOS have byte granularity.
 231          * They code segments and data segments have fixed 64k limits,
 232          * the transfer segment sizes are set at run time.
 233          */
 234         /* 32-bit code */
 235         [GDT_ENTRY_PNPBIOS_CS32]        = GDT_ENTRY_INIT(0x409a, 0, 0xffff),
 236         /* 16-bit code */
 237         [GDT_ENTRY_PNPBIOS_CS16]        = GDT_ENTRY_INIT(0x009a, 0, 0xffff),
 238         /* 16-bit data */
 239         [GDT_ENTRY_PNPBIOS_DS]          = GDT_ENTRY_INIT(0x0092, 0, 0xffff),
 240         /* 16-bit data */
 241         [GDT_ENTRY_PNPBIOS_TS1]         = GDT_ENTRY_INIT(0x0092, 0, 0),
 242         /* 16-bit data */
 243         [GDT_ENTRY_PNPBIOS_TS2]         = GDT_ENTRY_INIT(0x0092, 0, 0),
 244         /*
 245          * The APM segments have byte granularity and their bases
 246          * are set at run time.  All have 64k limits.
 247          */
 248         /* 32-bit code */
 249         [GDT_ENTRY_APMBIOS_BASE]        = GDT_ENTRY_INIT(0x409a, 0, 0xffff),
 250         /* 16-bit code */
 251         [GDT_ENTRY_APMBIOS_BASE+1]      = GDT_ENTRY_INIT(0x009a, 0, 0xffff),
 252         /* data */
 253         [GDT_ENTRY_APMBIOS_BASE+2]      = GDT_ENTRY_INIT(0x4092, 0, 0xffff),
 254
 255         [GDT_ENTRY_ESPFIX_SS]           = GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
 256         [GDT_ENTRY_PERCPU]              = GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
 257 #endif
 258 } };
 259 EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
 260
 261 #ifdef CONFIG_X86_64
 262 static int __init x86_nopcid_setup(char *s)
 263 {
 264         /* nopcid doesn't accept parameters */
 265         if (s)
 266                 return -EINVAL;
 267
 268         /* do not emit a message if the feature is not present */
 269         if (!boot_cpu_has(X86_FEATURE_PCID))
 270                 return 0;
 271
 272         setup_clear_cpu_cap(X86_FEATURE_PCID);
 273         pr_info("nopcid: PCID feature disabled\n");
 274         return 0;
 275 }
 276 early_param("nopcid", x86_nopcid_setup);
 277 #endif
 278
 279 static int __init x86_noinvpcid_setup(char *s)
 280 {
 281         /* noinvpcid doesn't accept parameters */
 282         if (s)
 283                 return -EINVAL;
 284
 285         /* do not emit a message if the feature is not present */
 286         if (!boot_cpu_has(X86_FEATURE_INVPCID))
 287                 return 0;
 288
 289         setup_clear_cpu_cap(X86_FEATURE_INVPCID);
 290         pr_info("noinvpcid: INVPCID feature disabled\n");
 291         return 0;
 292 }
 293 early_param("noinvpcid", x86_noinvpcid_setup);
 294
 295 #ifdef CONFIG_X86_32
 296 static int cachesize_override = -1;
 297 static int disable_x86_serial_nr = 1;
 298
 299 static int __init cachesize_setup(char *str)
 300 {
 301         get_option(&str, &cachesize_override);
 302         return 1;
 303 }
 304 __setup("cachesize=", cachesize_setup);
 305
 306 /* Standard macro to see if a specific flag is changeable */
 307 static inline int flag_is_changeable_p(u32 flag)
 308 {
 309         u32 f1, f2;
 310
 311         /*
 312          * Cyrix and IDT cpus allow disabling of CPUID
 313          * so the code below may return different results
 314          * when it is executed before and after enabling
 315          * the CPUID. Add "volatile" to not allow gcc to
 316          * optimize the subsequent calls to this function.
 317          */
 318         asm volatile ("pushfl           \n\t"
 319                       "pushfl           \n\t"
 320                       "popl %0          \n\t"
 321                       "movl %0, %1      \n\t"
 322                       "xorl %2, %0      \n\t"
 323                       "pushl %0         \n\t"
 324                       "popfl            \n\t"
 325                       "pushfl           \n\t"
 326                       "popl %0          \n\t"
 327                       "popfl            \n\t"
 328
 329                       : "=&r" (f1), "=&r" (f2)
 330                       : "ir" (flag));
 331
 332         return ((f1^f2) & flag) != 0;
 333 }
 334
 335 /* Probe for the CPUID instruction */
 336 int have_cpuid_p(void)
 337 {
 338         return flag_is_changeable_p(X86_EFLAGS_ID);
 339 }
 340
 341 static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
 342 {
 343         unsigned long lo, hi;
 344
 345         if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
 346                 return;
 347
 348         /* Disable processor serial number: */
 349
 350         rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
 351         lo |= 0x200000;
 352         wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
 353
 354         pr_notice("CPU serial number disabled.\n");
 355         clear_cpu_cap(c, X86_FEATURE_PN);
 356
 357         /* Disabling the serial number may affect the cpuid level */
 358         c->cpuid_level = cpuid_eax(0);
 359 }
 360
 361 static int __init x86_serial_nr_setup(char *s)
 362 {
 363         disable_x86_serial_nr = 0;
 364         return 1;
 365 }
 366 __setup("serialnumber", x86_serial_nr_setup);
 367 #else
 368 static inline int flag_is_changeable_p(u32 flag)
 369 {
 370         return 1;
 371 }
 372 static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
 373 {
 374 }
 375 #endif
 376
 377 static __always_inline void setup_smep(struct cpuinfo_x86 *c)
 378 {
 379         if (cpu_has(c, X86_FEATURE_SMEP))
 380                 cr4_set_bits(X86_CR4_SMEP);
 381 }
 382
 383 static __always_inline void setup_smap(struct cpuinfo_x86 *c)
 384 {
 385         unsigned long eflags = native_save_fl();
 386
 387         /* This should have been cleared long ago */
 388         BUG_ON(eflags & X86_EFLAGS_AC);
 389
 390         if (cpu_has(c, X86_FEATURE_SMAP))
 391                 cr4_set_bits(X86_CR4_SMAP);
 392 }
 393
 394 static __always_inline void setup_umip(struct cpuinfo_x86 *c)
 395 {
 396         /* Check the boot processor, plus build option for UMIP. */
 397         if (!cpu_feature_enabled(X86_FEATURE_UMIP))
 398                 goto out;
 399
 400         /* Check the current processor's cpuid bits. */
 401         if (!cpu_has(c, X86_FEATURE_UMIP))
 402                 goto out;
 403
 404         cr4_set_bits(X86_CR4_UMIP);
 405
 406         pr_info_once("x86/cpu: User Mode Instruction Prevention (UMIP) activated\n");
 407
 408         return;
 409
 410 out:
 411         /*
 412          * Make sure UMIP is disabled in case it was enabled in a
 413          * previous boot (e.g., via kexec).
 414          */
 415         cr4_clear_bits(X86_CR4_UMIP);
 416 }
 417
 418 /* These bits should not change their value after CPU init is finished. */
 419 static const unsigned long cr4_pinned_mask =
 420         X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP |
 421         X86_CR4_FSGSBASE | X86_CR4_CET;
 422 static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning);
 423 static unsigned long cr4_pinned_bits __ro_after_init;
 424
 425 void native_write_cr0(unsigned long val)
 426 {
 427         unsigned long bits_missing = 0;
 428
 429 set_register:
 430         asm volatile("mov %0,%%cr0": "+r" (val) : : "memory");
 431
 432         if (static_branch_likely(&cr_pinning)) {
 433                 if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) {
 434                         bits_missing = X86_CR0_WP;
 435                         val |= bits_missing;
 436                         goto set_register;
 437                 }
 438                 /* Warn after we've set the missing bits. */
 439                 WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n");
 440         }
 441 }
 442 EXPORT_SYMBOL(native_write_cr0);
 443
 444 void __no_profile native_write_cr4(unsigned long val)
 445 {
 446         unsigned long bits_changed = 0;
 447
 448 set_register:
 449         asm volatile("mov %0,%%cr4": "+r" (val) : : "memory");
 450
 451         if (static_branch_likely(&cr_pinning)) {
 452                 if (unlikely((val & cr4_pinned_mask) != cr4_pinned_bits)) {
 453                         bits_changed = (val & cr4_pinned_mask) ^ cr4_pinned_bits;
 454                         val = (val & ~cr4_pinned_mask) | cr4_pinned_bits;
 455                         goto set_register;
 456                 }
 457                 /* Warn after we've corrected the changed bits. */
 458                 WARN_ONCE(bits_changed, "pinned CR4 bits changed: 0x%lx!?\n",
 459                           bits_changed);
 460         }
 461 }
 462 #if IS_MODULE(CONFIG_LKDTM)
 463 EXPORT_SYMBOL_GPL(native_write_cr4);
 464 #endif
 465
 466 void cr4_update_irqsoff(unsigned long set, unsigned long clear)
 467 {
 468         unsigned long newval, cr4 = this_cpu_read(cpu_tlbstate.cr4);
 469
 470         lockdep_assert_irqs_disabled();
 471
 472         newval = (cr4 & ~clear) | set;
 473         if (newval != cr4) {
 474                 this_cpu_write(cpu_tlbstate.cr4, newval);
 475                 __write_cr4(newval);
 476         }
 477 }
 478 EXPORT_SYMBOL(cr4_update_irqsoff);
 479
 480 /* Read the CR4 shadow. */
 481 unsigned long cr4_read_shadow(void)
 482 {
 483         return this_cpu_read(cpu_tlbstate.cr4);
 484 }
 485 EXPORT_SYMBOL_GPL(cr4_read_shadow);
 486
 487 void cr4_init(void)
 488 {
 489         unsigned long cr4 = __read_cr4();
 490
 491         if (boot_cpu_has(X86_FEATURE_PCID))
 492                 cr4 |= X86_CR4_PCIDE;
 493         if (static_branch_likely(&cr_pinning))
 494                 cr4 = (cr4 & ~cr4_pinned_mask) | cr4_pinned_bits;
 495
 496         __write_cr4(cr4);
 497
 498         /* Initialize cr4 shadow for this CPU. */
 499         this_cpu_write(cpu_tlbstate.cr4, cr4);
 500 }
 501
 502 /*
 503  * Once CPU feature detection is finished (and boot params have been
 504  * parsed), record any of the sensitive CR bits that are set, and
 505  * enable CR pinning.
 506  */
 507 static void __init setup_cr_pinning(void)
 508 {
 509         cr4_pinned_bits = this_cpu_read(cpu_tlbstate.cr4) & cr4_pinned_mask;
 510         static_key_enable(&cr_pinning.key);
 511 }
 512
 513 static __init int x86_nofsgsbase_setup(char *arg)
 514 {
 515         /* Require an exact match without trailing characters. */
 516         if (strlen(arg))
 517                 return 0;
 518
 519         /* Do not emit a message if the feature is not present. */
 520         if (!boot_cpu_has(X86_FEATURE_FSGSBASE))
 521                 return 1;
 522
 523         setup_clear_cpu_cap(X86_FEATURE_FSGSBASE);
 524         pr_info("FSGSBASE disabled via kernel command line\n");
 525         return 1;
 526 }
 527 __setup("nofsgsbase", x86_nofsgsbase_setup);
 528
 529 /*
 530  * Protection Keys are not available in 32-bit mode.
 531  */
 532 static bool pku_disabled;
 533
 534 static __always_inline void setup_pku(struct cpuinfo_x86 *c)
 535 {
 536         if (c == &boot_cpu_data) {
 537                 if (pku_disabled || !cpu_feature_enabled(X86_FEATURE_PKU))
 538                         return;
 539                 /*
 540                  * Setting CR4.PKE will cause the X86_FEATURE_OSPKE cpuid
 541                  * bit to be set.  Enforce it.
 542                  */
 543                 setup_force_cpu_cap(X86_FEATURE_OSPKE);
 544
 545         } else if (!cpu_feature_enabled(X86_FEATURE_OSPKE)) {
 546                 return;
 547         }
 548
 549         cr4_set_bits(X86_CR4_PKE);
 550         /* Load the default PKRU value */
 551         pkru_write_default();
 552 }
 553
 554 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
 555 static __init int setup_disable_pku(char *arg)
 556 {
 557         /*
 558          * Do not clear the X86_FEATURE_PKU bit.  All of the
 559          * runtime checks are against OSPKE so clearing the
 560          * bit does nothing.
 561          *
 562          * This way, we will see "pku" in cpuinfo, but not
 563          * "ospke", which is exactly what we want.  It shows
 564          * that the CPU has PKU, but the OS has not enabled it.
 565          * This happens to be exactly how a system would look
 566          * if we disabled the config option.
 567          */
 568         pr_info("x86: 'nopku' specified, disabling Memory Protection Keys\n");
 569         pku_disabled = true;
 570         return 1;
 571 }
 572 __setup("nopku", setup_disable_pku);
 573 #endif /* CONFIG_X86_64 */
 574
 575 #ifdef CONFIG_X86_KERNEL_IBT
 576
 577 __noendbr u64 ibt_save(void)
 578 {
 579         u64 msr = 0;
 580
 581         if (cpu_feature_enabled(X86_FEATURE_IBT)) {
 582                 rdmsrl(MSR_IA32_S_CET, msr);
 583                 wrmsrl(MSR_IA32_S_CET, msr & ~CET_ENDBR_EN);
 584         }
 585
 586         return msr;
 587 }
 588
 589 __noendbr void ibt_restore(u64 save)
 590 {
 591         u64 msr;
 592
 593         if (cpu_feature_enabled(X86_FEATURE_IBT)) {
 594                 rdmsrl(MSR_IA32_S_CET, msr);
 595                 msr &= ~CET_ENDBR_EN;
 596                 msr |= (save & CET_ENDBR_EN);
 597                 wrmsrl(MSR_IA32_S_CET, msr);
 598         }
 599 }
 600
 601 #endif
 602
 603 static __always_inline void setup_cet(struct cpuinfo_x86 *c)
 604 {
 605         u64 msr = CET_ENDBR_EN;
 606
 607         if (!HAS_KERNEL_IBT ||
 608             !cpu_feature_enabled(X86_FEATURE_IBT))
 609                 return;
 610
 611         wrmsrl(MSR_IA32_S_CET, msr);
 612         cr4_set_bits(X86_CR4_CET);
 613
 614         if (!ibt_selftest()) {
 615                 pr_err("IBT selftest: Failed!\n");
 616                 setup_clear_cpu_cap(X86_FEATURE_IBT);
 617                 return;
 618         }
 619 }
 620
 621 __noendbr void cet_disable(void)
 622 {
 623         if (cpu_feature_enabled(X86_FEATURE_IBT))
 624                 wrmsrl(MSR_IA32_S_CET, 0);
 625 }
 626
 627 /*
 628  * Some CPU features depend on higher CPUID levels, which may not always
 629  * be available due to CPUID level capping or broken virtualization
 630  * software.  Add those features to this table to auto-disable them.
 631  */
 632 struct cpuid_dependent_feature {
 633         u32 feature;
 634         u32 level;
 635 };
 636
 637 static const struct cpuid_dependent_feature
 638 cpuid_dependent_features[] = {
 639         { X86_FEATURE_MWAIT,            0x00000005 },
 640         { X86_FEATURE_DCA,              0x00000009 },
 641         { X86_FEATURE_XSAVE,            0x0000000d },
 642         { 0, 0 }
 643 };
 644
 645 static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
 646 {
 647         const struct cpuid_dependent_feature *df;
 648
 649         for (df = cpuid_dependent_features; df->feature; df++) {
 650
 651                 if (!cpu_has(c, df->feature))
 652                         continue;
 653                 /*
 654                  * Note: cpuid_level is set to -1 if unavailable, but
 655                  * extended_extended_level is set to 0 if unavailable
 656                  * and the legitimate extended levels are all negative
 657                  * when signed; hence the weird messing around with
 658                  * signs here...
 659                  */
 660                 if (!((s32)df->level < 0 ?
 661                      (u32)df->level > (u32)c->extended_cpuid_level :
 662                      (s32)df->level > (s32)c->cpuid_level))
 663                         continue;
 664
 665                 clear_cpu_cap(c, df->feature);
 666                 if (!warn)
 667                         continue;
 668
 669                 pr_warn("CPU: CPU feature " X86_CAP_FMT " disabled, no CPUID level 0x%x\n",
 670                         x86_cap_flag(df->feature), df->level);
 671         }
 672 }
 673
 674 /*
 675  * Naming convention should be: <Name> [(<Codename>)]
 676  * This table only is used unless init_<vendor>() below doesn't set it;
 677  * in particular, if CPUID levels 0x80000002..4 are supported, this
 678  * isn't used
 679  */
 680
 681 /* Look up CPU names by table lookup. */
 682 static const char *table_lookup_model(struct cpuinfo_x86 *c)
 683 {
 684 #ifdef CONFIG_X86_32
 685         const struct legacy_cpu_model_info *info;
 686
 687         if (c->x86_model >= 16)
 688                 return NULL;    /* Range check */
 689
 690         if (!this_cpu)
 691                 return NULL;
 692
 693         info = this_cpu->legacy_models;
 694
 695         while (info->family) {
 696                 if (info->family == c->x86)
 697                         return info->model_names[c->x86_model];
 698                 info++;
 699         }
 700 #endif
 701         return NULL;            /* Not found */
 702 }
 703
 704 /* Aligned to unsigned long to avoid split lock in atomic bitmap ops */
 705 __u32 cpu_caps_cleared[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
 706 __u32 cpu_caps_set[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
 707
 708 void load_percpu_segment(int cpu)
 709 {
 710 #ifdef CONFIG_X86_32
 711         loadsegment(fs, __KERNEL_PERCPU);
 712 #else
 713         __loadsegment_simple(gs, 0);
 714         wrmsrl(MSR_GS_BASE, cpu_kernelmode_gs_base(cpu));
 715 #endif
 716 }
 717
 718 #ifdef CONFIG_X86_32
 719 /* The 32-bit entry code needs to find cpu_entry_area. */
 720 DEFINE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
 721 #endif
 722
 723 /* Load the original GDT from the per-cpu structure */
 724 void load_direct_gdt(int cpu)
 725 {
 726         struct desc_ptr gdt_descr;
 727
 728         gdt_descr.address = (long)get_cpu_gdt_rw(cpu);
 729         gdt_descr.size = GDT_SIZE - 1;
 730         load_gdt(&gdt_descr);
 731 }
 732 EXPORT_SYMBOL_GPL(load_direct_gdt);
 733
 734 /* Load a fixmap remapping of the per-cpu GDT */
 735 void load_fixmap_gdt(int cpu)
 736 {
 737         struct desc_ptr gdt_descr;
 738
 739         gdt_descr.address = (long)get_cpu_gdt_ro(cpu);
 740         gdt_descr.size = GDT_SIZE - 1;
 741         load_gdt(&gdt_descr);
 742 }
 743 EXPORT_SYMBOL_GPL(load_fixmap_gdt);
 744
 745 /*
 746  * Current gdt points %fs at the "master" per-cpu area: after this,
 747  * it's on the real one.
 748  */
 749 void switch_to_new_gdt(int cpu)
 750 {
 751         /* Load the original GDT */
 752         load_direct_gdt(cpu);
 753         /* Reload the per-cpu base */
 754         load_percpu_segment(cpu);
 755 }
 756
 757 static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
 758
 759 static void get_model_name(struct cpuinfo_x86 *c)
 760 {
 761         unsigned int *v;
 762         char *p, *q, *s;
 763
 764         if (c->extended_cpuid_level < 0x80000004)
 765                 return;
 766
 767         v = (unsigned int *)c->x86_model_id;
 768         cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
 769         cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
 770         cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
 771         c->x86_model_id[48] = 0;
 772
 773         /* Trim whitespace */
 774         p = q = s = &c->x86_model_id[0];
 775
 776         while (*p == ' ')
 777                 p++;
 778
 779         while (*p) {
 780                 /* Note the last non-whitespace index */
 781                 if (!isspace(*p))
 782                         s = q;
 783
 784                 *q++ = *p++;
 785         }
 786
 787         *(s + 1) = '\0';
 788 }
 789
 790 void detect_num_cpu_cores(struct cpuinfo_x86 *c)
 791 {
 792         unsigned int eax, ebx, ecx, edx;
 793
 794         c->x86_max_cores = 1;
 795         if (!IS_ENABLED(CONFIG_SMP) || c->cpuid_level < 4)
 796                 return;
 797
 798         cpuid_count(4, 0, &eax, &ebx, &ecx, &edx);
 799         if (eax & 0x1f)
 800                 c->x86_max_cores = (eax >> 26) + 1;
 801 }
 802
 803 void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
 804 {
 805         unsigned int n, dummy, ebx, ecx, edx, l2size;
 806
 807         n = c->extended_cpuid_level;
 808
 809         if (n >= 0x80000005) {
 810                 cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
 811                 c->x86_cache_size = (ecx>>24) + (edx>>24);
 812 #ifdef CONFIG_X86_64
 813                 /* On K8 L1 TLB is inclusive, so don't count it */
 814                 c->x86_tlbsize = 0;
 815 #endif
 816         }
 817
 818         if (n < 0x80000006)     /* Some chips just has a large L1. */
 819                 return;
 820
 821         cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
 822         l2size = ecx >> 16;
 823
 824 #ifdef CONFIG_X86_64
 825         c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
 826 #else
 827         /* do processor-specific cache resizing */
 828         if (this_cpu->legacy_cache_size)
 829                 l2size = this_cpu->legacy_cache_size(c, l2size);
 830
 831         /* Allow user to override all this if necessary. */
 832         if (cachesize_override != -1)
 833                 l2size = cachesize_override;
 834
 835         if (l2size == 0)
 836                 return;         /* Again, no L2 cache is possible */
 837 #endif
 838
 839         c->x86_cache_size = l2size;
 840 }
 841
 842 u16 __read_mostly tlb_lli_4k[NR_INFO];
 843 u16 __read_mostly tlb_lli_2m[NR_INFO];
 844 u16 __read_mostly tlb_lli_4m[NR_INFO];
 845 u16 __read_mostly tlb_lld_4k[NR_INFO];
 846 u16 __read_mostly tlb_lld_2m[NR_INFO];
 847 u16 __read_mostly tlb_lld_4m[NR_INFO];
 848 u16 __read_mostly tlb_lld_1g[NR_INFO];
 849
 850 static void cpu_detect_tlb(struct cpuinfo_x86 *c)
 851 {
 852         if (this_cpu->c_detect_tlb)
 853                 this_cpu->c_detect_tlb(c);
 854
 855         pr_info("Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
 856                 tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
 857                 tlb_lli_4m[ENTRIES]);
 858
 859         pr_info("Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d, 1GB %d\n",
 860                 tlb_lld_4k[ENTRIES], tlb_lld_2m[ENTRIES],
 861                 tlb_lld_4m[ENTRIES], tlb_lld_1g[ENTRIES]);
 862 }
 863
 864 int detect_ht_early(struct cpuinfo_x86 *c)
 865 {
 866 #ifdef CONFIG_SMP
 867         u32 eax, ebx, ecx, edx;
 868
 869         if (!cpu_has(c, X86_FEATURE_HT))
 870                 return -1;
 871
 872         if (cpu_has(c, X86_FEATURE_CMP_LEGACY))
 873                 return -1;
 874
 875         if (cpu_has(c, X86_FEATURE_XTOPOLOGY))
 876                 return -1;
 877
 878         cpuid(1, &eax, &ebx, &ecx, &edx);
 879
 880         smp_num_siblings = (ebx & 0xff0000) >> 16;
 881         if (smp_num_siblings == 1)
 882                 pr_info_once("CPU0: Hyper-Threading is disabled\n");
 883 #endif
 884         return 0;
 885 }
 886
 887 void detect_ht(struct cpuinfo_x86 *c)
 888 {
 889 #ifdef CONFIG_SMP
 890         int index_msb, core_bits;
 891
 892         if (detect_ht_early(c) < 0)
 893                 return;
 894
 895         index_msb = get_count_order(smp_num_siblings);
 896         c->phys_proc_id = apic->phys_pkg_id(c->initial_apicid, index_msb);
 897
 898         smp_num_siblings = smp_num_siblings / c->x86_max_cores;
 899
 900         index_msb = get_count_order(smp_num_siblings);
 901
 902         core_bits = get_count_order(c->x86_max_cores);
 903
 904         c->cpu_core_id = apic->phys_pkg_id(c->initial_apicid, index_msb) &
 905                                        ((1 << core_bits) - 1);
 906 #endif
 907 }
 908
 909 static void get_cpu_vendor(struct cpuinfo_x86 *c)
 910 {
 911         char *v = c->x86_vendor_id;
 912         int i;
 913
 914         for (i = 0; i < X86_VENDOR_NUM; i++) {
 915                 if (!cpu_devs[i])
 916                         break;
 917
 918                 if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
 919                     (cpu_devs[i]->c_ident[1] &&
 920                      !strcmp(v, cpu_devs[i]->c_ident[1]))) {
 921
 922                         this_cpu = cpu_devs[i];
 923                         c->x86_vendor = this_cpu->c_x86_vendor;
 924                         return;
 925                 }
 926         }
 927
 928         pr_err_once("CPU: vendor_id '%s' unknown, using generic init.\n" \
 929                     "CPU: Your system may be unstable.\n", v);
 930
 931         c->x86_vendor = X86_VENDOR_UNKNOWN;
 932         this_cpu = &default_cpu;
 933 }
 934
 935 void cpu_detect(struct cpuinfo_x86 *c)
 936 {
 937         /* Get vendor name */
 938         cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
 939               (unsigned int *)&c->x86_vendor_id[0],
 940               (unsigned int *)&c->x86_vendor_id[8],
 941               (unsigned int *)&c->x86_vendor_id[4]);
 942
 943         c->x86 = 4;
 944         /* Intel-defined flags: level 0x00000001 */
 945         if (c->cpuid_level >= 0x00000001) {
 946                 u32 junk, tfms, cap0, misc;
 947
 948                 cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
 949                 c->x86          = x86_family(tfms);
 950                 c->x86_model    = x86_model(tfms);
 951                 c->x86_stepping = x86_stepping(tfms);
 952
 953                 if (cap0 & (1<<19)) {
 954                         c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
 955                         c->x86_cache_alignment = c->x86_clflush_size;
 956                 }
 957         }
 958 }
 959
 960 static void apply_forced_caps(struct cpuinfo_x86 *c)
 961 {
 962         int i;
 963
 964         for (i = 0; i < NCAPINTS + NBUGINTS; i++) {
 965                 c->x86_capability[i] &= ~cpu_caps_cleared[i];
 966                 c->x86_capability[i] |= cpu_caps_set[i];
 967         }
 968 }
 969
 970 static void init_speculation_control(struct cpuinfo_x86 *c)
 971 {
 972         /*
 973          * The Intel SPEC_CTRL CPUID bit implies IBRS and IBPB support,
 974          * and they also have a different bit for STIBP support. Also,
 975          * a hypervisor might have set the individual AMD bits even on
 976          * Intel CPUs, for finer-grained selection of what's available.
 977          */
 978         if (cpu_has(c, X86_FEATURE_SPEC_CTRL)) {
 979                 set_cpu_cap(c, X86_FEATURE_IBRS);
 980                 set_cpu_cap(c, X86_FEATURE_IBPB);
 981                 set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
 982         }
 983
 984         if (cpu_has(c, X86_FEATURE_INTEL_STIBP))
 985                 set_cpu_cap(c, X86_FEATURE_STIBP);
 986
 987         if (cpu_has(c, X86_FEATURE_SPEC_CTRL_SSBD) ||
 988             cpu_has(c, X86_FEATURE_VIRT_SSBD))
 989                 set_cpu_cap(c, X86_FEATURE_SSBD);
 990
 991         if (cpu_has(c, X86_FEATURE_AMD_IBRS)) {
 992                 set_cpu_cap(c, X86_FEATURE_IBRS);
 993                 set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
 994         }
 995
 996         if (cpu_has(c, X86_FEATURE_AMD_IBPB))
 997                 set_cpu_cap(c, X86_FEATURE_IBPB);
 998
 999         if (cpu_has(c, X86_FEATURE_AMD_STIBP)) {
1000                 set_cpu_cap(c, X86_FEATURE_STIBP);
1001                 set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
1002         }
1003
1004         if (cpu_has(c, X86_FEATURE_AMD_SSBD)) {
1005                 set_cpu_cap(c, X86_FEATURE_SSBD);
1006                 set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
1007                 clear_cpu_cap(c, X86_FEATURE_VIRT_SSBD);
1008         }
1009 }
1010
1011 void get_cpu_cap(struct cpuinfo_x86 *c)
1012 {
1013         u32 eax, ebx, ecx, edx;
1014
1015         /* Intel-defined flags: level 0x00000001 */
1016         if (c->cpuid_level >= 0x00000001) {
1017                 cpuid(0x00000001, &eax, &ebx, &ecx, &edx);
1018
1019                 c->x86_capability[CPUID_1_ECX] = ecx;
1020                 c->x86_capability[CPUID_1_EDX] = edx;
1021         }
1022
1023         /* Thermal and Power Management Leaf: level 0x00000006 (eax) */
1024         if (c->cpuid_level >= 0x00000006)
1025                 c->x86_capability[CPUID_6_EAX] = cpuid_eax(0x00000006);
1026
1027         /* Additional Intel-defined flags: level 0x00000007 */
1028         if (c->cpuid_level >= 0x00000007) {
1029                 cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);
1030                 c->x86_capability[CPUID_7_0_EBX] = ebx;
1031                 c->x86_capability[CPUID_7_ECX] = ecx;
1032                 c->x86_capability[CPUID_7_EDX] = edx;
1033
1034                 /* Check valid sub-leaf index before accessing it */
1035                 if (eax >= 1) {
1036                         cpuid_count(0x00000007, 1, &eax, &ebx, &ecx, &edx);
1037                         c->x86_capability[CPUID_7_1_EAX] = eax;
1038                 }
1039         }
1040
1041         /* Extended state features: level 0x0000000d */
1042         if (c->cpuid_level >= 0x0000000d) {
1043                 cpuid_count(0x0000000d, 1, &eax, &ebx, &ecx, &edx);
1044
1045                 c->x86_capability[CPUID_D_1_EAX] = eax;
1046         }
1047
1048         /* AMD-defined flags: level 0x80000001 */
1049         eax = cpuid_eax(0x80000000);
1050         c->extended_cpuid_level = eax;
1051
1052         if ((eax & 0xffff0000) == 0x80000000) {
1053                 if (eax >= 0x80000001) {
1054                         cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
1055
1056                         c->x86_capability[CPUID_8000_0001_ECX] = ecx;
1057                         c->x86_capability[CPUID_8000_0001_EDX] = edx;
1058                 }
1059         }
1060
1061         if (c->extended_cpuid_level >= 0x80000007) {
1062                 cpuid(0x80000007, &eax, &ebx, &ecx, &edx);
1063
1064                 c->x86_capability[CPUID_8000_0007_EBX] = ebx;
1065                 c->x86_power = edx;
1066         }
1067
1068         if (c->extended_cpuid_level >= 0x80000008) {
1069                 cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
1070                 c->x86_capability[CPUID_8000_0008_EBX] = ebx;
1071         }
1072
1073         if (c->extended_cpuid_level >= 0x8000000a)
1074                 c->x86_capability[CPUID_8000_000A_EDX] = cpuid_edx(0x8000000a);
1075
1076         if (c->extended_cpuid_level >= 0x8000001f)
1077                 c->x86_capability[CPUID_8000_001F_EAX] = cpuid_eax(0x8000001f);
1078
1079         if (c->extended_cpuid_level >= 0x80000021)
1080                 c->x86_capability[CPUID_8000_0021_EAX] = cpuid_eax(0x80000021);
1081
1082         init_scattered_cpuid_features(c);
1083         init_speculation_control(c);
1084
1085         /*
1086          * Clear/Set all flags overridden by options, after probe.
1087          * This needs to happen each time we re-probe, which may happen
1088          * several times during CPU initialization.
1089          */
1090         apply_forced_caps(c);
1091 }
1092
1093 void get_cpu_address_sizes(struct cpuinfo_x86 *c)
1094 {
1095         u32 eax, ebx, ecx, edx;
1096
1097         if (c->extended_cpuid_level >= 0x80000008) {
1098                 cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
1099
1100                 c->x86_virt_bits = (eax >> 8) & 0xff;
1101                 c->x86_phys_bits = eax & 0xff;
1102         }
1103 #ifdef CONFIG_X86_32
1104         else if (cpu_has(c, X86_FEATURE_PAE) || cpu_has(c, X86_FEATURE_PSE36))
1105                 c->x86_phys_bits = 36;
1106 #endif
1107         c->x86_cache_bits = c->x86_phys_bits;
1108 }
1109
1110 static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
1111 {
1112 #ifdef CONFIG_X86_32
1113         int i;
1114
1115         /*
1116          * First of all, decide if this is a 486 or higher
1117          * It's a 486 if we can modify the AC flag
1118          */
1119         if (flag_is_changeable_p(X86_EFLAGS_AC))
1120                 c->x86 = 4;
1121         else
1122                 c->x86 = 3;
1123
1124         for (i = 0; i < X86_VENDOR_NUM; i++)
1125                 if (cpu_devs[i] && cpu_devs[i]->c_identify) {
1126                         c->x86_vendor_id[0] = 0;
1127                         cpu_devs[i]->c_identify(c);
1128                         if (c->x86_vendor_id[0]) {
1129                                 get_cpu_vendor(c);
1130                                 break;
1131                         }
1132                 }
1133 #endif
1134 }
1135
1136 #define NO_SPECULATION          BIT(0)
1137 #define NO_MELTDOWN             BIT(1)
1138 #define NO_SSB                  BIT(2)
1139 #define NO_L1TF                 BIT(3)
1140 #define NO_MDS                  BIT(4)
1141 #define MSBDS_ONLY              BIT(5)
1142 #define NO_SWAPGS               BIT(6)
1143 #define NO_ITLB_MULTIHIT        BIT(7)
1144 #define NO_SPECTRE_V2           BIT(8)
1145 #define NO_MMIO                 BIT(9)
1146 #define NO_EIBRS_PBRSB          BIT(10)
1147 #define NO_BHI                  BIT(11)
1148
1149 #define VULNWL(vendor, family, model, whitelist)        \
1150         X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, whitelist)
1151
1152 #define VULNWL_INTEL(model, whitelist)          \
1153         VULNWL(INTEL, 6, INTEL_FAM6_##model, whitelist)
1154
1155 #define VULNWL_AMD(family, whitelist)           \
1156         VULNWL(AMD, family, X86_MODEL_ANY, whitelist)
1157
1158 #define VULNWL_HYGON(family, whitelist)         \
1159         VULNWL(HYGON, family, X86_MODEL_ANY, whitelist)
1160
1161 static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = {
1162         VULNWL(ANY,     4, X86_MODEL_ANY,       NO_SPECULATION),
1163         VULNWL(CENTAUR, 5, X86_MODEL_ANY,       NO_SPECULATION),
1164         VULNWL(INTEL,   5, X86_MODEL_ANY,       NO_SPECULATION),
1165         VULNWL(NSC,     5, X86_MODEL_ANY,       NO_SPECULATION),
1166         VULNWL(VORTEX,  5, X86_MODEL_ANY,       NO_SPECULATION),
1167         VULNWL(VORTEX,  6, X86_MODEL_ANY,       NO_SPECULATION),
1168
1169         /* Intel Family 6 */
1170         VULNWL_INTEL(TIGERLAKE,                 NO_MMIO),
1171         VULNWL_INTEL(TIGERLAKE_L,               NO_MMIO),
1172         VULNWL_INTEL(ALDERLAKE,                 NO_MMIO),
1173         VULNWL_INTEL(ALDERLAKE_L,               NO_MMIO),
1174
1175         VULNWL_INTEL(ATOM_SALTWELL,             NO_SPECULATION | NO_ITLB_MULTIHIT),
1176         VULNWL_INTEL(ATOM_SALTWELL_TABLET,      NO_SPECULATION | NO_ITLB_MULTIHIT),
1177         VULNWL_INTEL(ATOM_SALTWELL_MID,         NO_SPECULATION | NO_ITLB_MULTIHIT),
1178         VULNWL_INTEL(ATOM_BONNELL,              NO_SPECULATION | NO_ITLB_MULTIHIT),
1179         VULNWL_INTEL(ATOM_BONNELL_MID,          NO_SPECULATION | NO_ITLB_MULTIHIT),
1180
1181         VULNWL_INTEL(ATOM_SILVERMONT,           NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1182         VULNWL_INTEL(ATOM_SILVERMONT_D,         NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1183         VULNWL_INTEL(ATOM_SILVERMONT_MID,       NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1184         VULNWL_INTEL(ATOM_AIRMONT,              NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1185         VULNWL_INTEL(XEON_PHI_KNL,              NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1186         VULNWL_INTEL(XEON_PHI_KNM,              NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1187
1188         VULNWL_INTEL(CORE_YONAH,                NO_SSB),
1189
1190         VULNWL_INTEL(ATOM_AIRMONT_MID,          NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1191         VULNWL_INTEL(ATOM_AIRMONT_NP,           NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT),
1192
1193         VULNWL_INTEL(ATOM_GOLDMONT,             NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1194         VULNWL_INTEL(ATOM_GOLDMONT_D,           NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1195         VULNWL_INTEL(ATOM_GOLDMONT_PLUS,        NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB),
1196
1197         /*
1198          * Technically, swapgs isn't serializing on AMD (despite it previously
1199          * being documented as such in the APM).  But according to AMD, %gs is
1200          * updated non-speculatively, and the issuing of %gs-relative memory
1201          * operands will be blocked until the %gs update completes, which is
1202          * good enough for our purposes.
1203          */
1204
1205         VULNWL_INTEL(ATOM_TREMONT,              NO_EIBRS_PBRSB),
1206         VULNWL_INTEL(ATOM_TREMONT_L,            NO_EIBRS_PBRSB),
1207         VULNWL_INTEL(ATOM_TREMONT_D,            NO_ITLB_MULTIHIT | NO_EIBRS_PBRSB),
1208
1209         /* AMD Family 0xf - 0x12 */
1210         VULNWL_AMD(0x0f,        NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
1211         VULNWL_AMD(0x10,        NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
1212         VULNWL_AMD(0x11,        NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
1213         VULNWL_AMD(0x12,        NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
1214
1215         /* FAMILY_ANY must be last, otherwise 0x0f - 0x12 matches won't work */
1216         VULNWL_AMD(X86_FAMILY_ANY,      NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB | NO_BHI),
1217         VULNWL_HYGON(X86_FAMILY_ANY,    NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB | NO_BHI),
1218
1219         /* Zhaoxin Family 7 */
1220         VULNWL(CENTAUR, 7, X86_MODEL_ANY,       NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO | NO_BHI),
1221         VULNWL(ZHAOXIN, 7, X86_MODEL_ANY,       NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO | NO_BHI),
1222         {}
1223 };
1224
1225 #define VULNBL(vendor, family, model, blacklist)        \
1226         X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, blacklist)
1227
1228 #define VULNBL_INTEL_STEPPINGS(model, steppings, issues)                   \
1229         X86_MATCH_VENDOR_FAM_MODEL_STEPPINGS_FEATURE(INTEL, 6,             \
1230                                             INTEL_FAM6_##model, steppings, \
1231                                             X86_FEATURE_ANY, issues)
1232
1233 #define VULNBL_AMD(family, blacklist)           \
1234         VULNBL(AMD, family, X86_MODEL_ANY, blacklist)
1235
1236 #define VULNBL_HYGON(family, blacklist)         \
1237         VULNBL(HYGON, family, X86_MODEL_ANY, blacklist)
1238
1239 #define SRBDS           BIT(0)
1240 /* CPU is affected by X86_BUG_MMIO_STALE_DATA */
1241 #define MMIO            BIT(1)
1242 /* CPU is affected by Shared Buffers Data Sampling (SBDS), a variant of X86_BUG_MMIO_STALE_DATA */
1243 #define MMIO_SBDS       BIT(2)
1244 /* CPU is affected by RETbleed, speculating where you would not expect it */
1245 #define RETBLEED        BIT(3)
1246 /* CPU is affected by SMT (cross-thread) return predictions */
1247 #define SMT_RSB         BIT(4)
1248 /* CPU is affected by SRSO */
1249 #define SRSO            BIT(5)
1250 /* CPU is affected by GDS */
1251 #define GDS             BIT(6)
1252 /* CPU is affected by Register File Data Sampling */
1253 #define RFDS            BIT(7)
1254
1255 static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = {
1256         VULNBL_INTEL_STEPPINGS(IVYBRIDGE,       X86_STEPPING_ANY,               SRBDS),
1257         VULNBL_INTEL_STEPPINGS(HASWELL,         X86_STEPPING_ANY,               SRBDS),
1258         VULNBL_INTEL_STEPPINGS(HASWELL_L,       X86_STEPPING_ANY,               SRBDS),
1259         VULNBL_INTEL_STEPPINGS(HASWELL_G,       X86_STEPPING_ANY,               SRBDS),
1260         VULNBL_INTEL_STEPPINGS(HASWELL_X,       X86_STEPPING_ANY,               MMIO),
1261         VULNBL_INTEL_STEPPINGS(BROADWELL_D,     X86_STEPPING_ANY,               MMIO),
1262         VULNBL_INTEL_STEPPINGS(BROADWELL_G,     X86_STEPPING_ANY,               SRBDS),
1263         VULNBL_INTEL_STEPPINGS(BROADWELL_X,     X86_STEPPING_ANY,               MMIO),
1264         VULNBL_INTEL_STEPPINGS(BROADWELL,       X86_STEPPING_ANY,               SRBDS),
1265         VULNBL_INTEL_STEPPINGS(SKYLAKE_X,       X86_STEPPING_ANY,               MMIO | RETBLEED | GDS),
1266         VULNBL_INTEL_STEPPINGS(SKYLAKE_L,       X86_STEPPING_ANY,               MMIO | RETBLEED | GDS | SRBDS),
1267         VULNBL_INTEL_STEPPINGS(SKYLAKE,         X86_STEPPING_ANY,               MMIO | RETBLEED | GDS | SRBDS),
1268         VULNBL_INTEL_STEPPINGS(KABYLAKE_L,      X86_STEPPING_ANY,               MMIO | RETBLEED | GDS | SRBDS),
1269         VULNBL_INTEL_STEPPINGS(KABYLAKE,        X86_STEPPING_ANY,               MMIO | RETBLEED | GDS | SRBDS),
1270         VULNBL_INTEL_STEPPINGS(CANNONLAKE_L,    X86_STEPPING_ANY,               RETBLEED),
1271         VULNBL_INTEL_STEPPINGS(ICELAKE_L,       X86_STEPPING_ANY,               MMIO | MMIO_SBDS | RETBLEED | GDS),
1272         VULNBL_INTEL_STEPPINGS(ICELAKE_D,       X86_STEPPING_ANY,               MMIO | GDS),
1273         VULNBL_INTEL_STEPPINGS(ICELAKE_X,       X86_STEPPING_ANY,               MMIO | GDS),
1274         VULNBL_INTEL_STEPPINGS(COMETLAKE,       X86_STEPPING_ANY,               MMIO | MMIO_SBDS | RETBLEED | GDS),
1275         VULNBL_INTEL_STEPPINGS(COMETLAKE_L,     X86_STEPPINGS(0x0, 0x0),        MMIO | RETBLEED),
1276         VULNBL_INTEL_STEPPINGS(COMETLAKE_L,     X86_STEPPING_ANY,               MMIO | MMIO_SBDS | RETBLEED | GDS),
1277         VULNBL_INTEL_STEPPINGS(TIGERLAKE_L,     X86_STEPPING_ANY,               GDS),
1278         VULNBL_INTEL_STEPPINGS(TIGERLAKE,       X86_STEPPING_ANY,               GDS),
1279         VULNBL_INTEL_STEPPINGS(LAKEFIELD,       X86_STEPPING_ANY,               MMIO | MMIO_SBDS | RETBLEED),
1280         VULNBL_INTEL_STEPPINGS(ROCKETLAKE,      X86_STEPPING_ANY,               MMIO | RETBLEED | GDS),
1281         VULNBL_INTEL_STEPPINGS(ALDERLAKE,       X86_STEPPING_ANY,               RFDS),
1282         VULNBL_INTEL_STEPPINGS(ALDERLAKE_L,     X86_STEPPING_ANY,               RFDS),
1283         VULNBL_INTEL_STEPPINGS(RAPTORLAKE,      X86_STEPPING_ANY,               RFDS),
1284         VULNBL_INTEL_STEPPINGS(RAPTORLAKE_P,    X86_STEPPING_ANY,               RFDS),
1285         VULNBL_INTEL_STEPPINGS(RAPTORLAKE_S,    X86_STEPPING_ANY,               RFDS),
1286         VULNBL_INTEL_STEPPINGS(ALDERLAKE_N,     X86_STEPPING_ANY,               RFDS),
1287         VULNBL_INTEL_STEPPINGS(ATOM_TREMONT,    X86_STEPPING_ANY,               MMIO | MMIO_SBDS | RFDS),
1288         VULNBL_INTEL_STEPPINGS(ATOM_TREMONT_D,  X86_STEPPING_ANY,               MMIO | RFDS),
1289         VULNBL_INTEL_STEPPINGS(ATOM_TREMONT_L,  X86_STEPPING_ANY,               MMIO | MMIO_SBDS | RFDS),
1290         VULNBL_INTEL_STEPPINGS(ATOM_GOLDMONT,   X86_STEPPING_ANY,               RFDS),
1291         VULNBL_INTEL_STEPPINGS(ATOM_GOLDMONT_D, X86_STEPPING_ANY,               RFDS),
1292         VULNBL_INTEL_STEPPINGS(ATOM_GOLDMONT_PLUS, X86_STEPPING_ANY,            RFDS),
1293
1294         VULNBL_AMD(0x15, RETBLEED),
1295         VULNBL_AMD(0x16, RETBLEED),
1296         VULNBL_AMD(0x17, RETBLEED | SMT_RSB | SRSO),
1297         VULNBL_HYGON(0x18, RETBLEED | SMT_RSB | SRSO),
1298         VULNBL_AMD(0x19, SRSO),
1299         {}
1300 };
1301
1302 static bool __init cpu_matches(const struct x86_cpu_id *table, unsigned long which)
1303 {
1304         const struct x86_cpu_id *m = x86_match_cpu(table);
1305
1306         return m && !!(m->driver_data & which);
1307 }
1308
1309 u64 x86_read_arch_cap_msr(void)
1310 {
1311         u64 x86_arch_cap_msr = 0;
1312
1313         if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
1314                 rdmsrl(MSR_IA32_ARCH_CAPABILITIES, x86_arch_cap_msr);
1315
1316         return x86_arch_cap_msr;
1317 }
1318
1319 static bool arch_cap_mmio_immune(u64 x86_arch_cap_msr)
1320 {
1321         return (x86_arch_cap_msr & ARCH_CAP_FBSDP_NO &&
1322                 x86_arch_cap_msr & ARCH_CAP_PSDP_NO &&
1323                 x86_arch_cap_msr & ARCH_CAP_SBDR_SSDP_NO);
1324 }
1325
1326 static bool __init vulnerable_to_rfds(u64 x86_arch_cap_msr)
1327 {
1328         /* The "immunity" bit trumps everything else: */
1329         if (x86_arch_cap_msr & ARCH_CAP_RFDS_NO)
1330                 return false;
1331
1332         /*
1333          * VMMs set ARCH_CAP_RFDS_CLEAR for processors not in the blacklist to
1334          * indicate that mitigation is needed because guest is running on a
1335          * vulnerable hardware or may migrate to such hardware:
1336          */
1337         if (x86_arch_cap_msr & ARCH_CAP_RFDS_CLEAR)
1338                 return true;
1339
1340         /* Only consult the blacklist when there is no enumeration: */
1341         return cpu_matches(cpu_vuln_blacklist, RFDS);
1342 }
1343
1344 static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c)
1345 {
1346         u64 x86_arch_cap_msr = x86_read_arch_cap_msr();
1347
1348         /* Set ITLB_MULTIHIT bug if cpu is not in the whitelist and not mitigated */
1349         if (!cpu_matches(cpu_vuln_whitelist, NO_ITLB_MULTIHIT) &&
1350             !(x86_arch_cap_msr & ARCH_CAP_PSCHANGE_MC_NO))
1351                 setup_force_cpu_bug(X86_BUG_ITLB_MULTIHIT);
1352
1353         if (cpu_matches(cpu_vuln_whitelist, NO_SPECULATION))
1354                 return;
1355
1356         setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
1357
1358         if (!cpu_matches(cpu_vuln_whitelist, NO_SPECTRE_V2))
1359                 setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
1360
1361         if (!cpu_matches(cpu_vuln_whitelist, NO_SSB) &&
1362             !(x86_arch_cap_msr & ARCH_CAP_SSB_NO) &&
1363            !cpu_has(c, X86_FEATURE_AMD_SSB_NO))
1364                 setup_force_cpu_bug(X86_BUG_SPEC_STORE_BYPASS);
1365
1366         /*
1367          * AMD's AutoIBRS is equivalent to Intel's eIBRS - use the Intel feature
1368          * flag and protect from vendor-specific bugs via the whitelist.
1369          */
1370         if ((x86_arch_cap_msr & ARCH_CAP_IBRS_ALL) || cpu_has(c, X86_FEATURE_AUTOIBRS)) {
1371                 setup_force_cpu_cap(X86_FEATURE_IBRS_ENHANCED);
1372                 if (!cpu_matches(cpu_vuln_whitelist, NO_EIBRS_PBRSB) &&
1373                     !(x86_arch_cap_msr & ARCH_CAP_PBRSB_NO))
1374                         setup_force_cpu_bug(X86_BUG_EIBRS_PBRSB);
1375         }
1376
1377         if (!cpu_matches(cpu_vuln_whitelist, NO_MDS) &&
1378             !(x86_arch_cap_msr & ARCH_CAP_MDS_NO)) {
1379                 setup_force_cpu_bug(X86_BUG_MDS);
1380                 if (cpu_matches(cpu_vuln_whitelist, MSBDS_ONLY))
1381                         setup_force_cpu_bug(X86_BUG_MSBDS_ONLY);
1382         }
1383
1384         if (!cpu_matches(cpu_vuln_whitelist, NO_SWAPGS))
1385                 setup_force_cpu_bug(X86_BUG_SWAPGS);
1386
1387         /*
1388          * When the CPU is not mitigated for TAA (TAA_NO=0) set TAA bug when:
1389          *      - TSX is supported or
1390          *      - TSX_CTRL is present
1391          *
1392          * TSX_CTRL check is needed for cases when TSX could be disabled before
1393          * the kernel boot e.g. kexec.
1394          * TSX_CTRL check alone is not sufficient for cases when the microcode
1395          * update is not present or running as guest that don't get TSX_CTRL.
1396          */
1397         if (!(x86_arch_cap_msr & ARCH_CAP_TAA_NO) &&
1398             (cpu_has(c, X86_FEATURE_RTM) ||
1399              (x86_arch_cap_msr & ARCH_CAP_TSX_CTRL_MSR)))
1400                 setup_force_cpu_bug(X86_BUG_TAA);
1401
1402         /*
1403          * SRBDS affects CPUs which support RDRAND or RDSEED and are listed
1404          * in the vulnerability blacklist.
1405          *
1406          * Some of the implications and mitigation of Shared Buffers Data
1407          * Sampling (SBDS) are similar to SRBDS. Give SBDS same treatment as
1408          * SRBDS.
1409          */
1410         if ((cpu_has(c, X86_FEATURE_RDRAND) ||
1411              cpu_has(c, X86_FEATURE_RDSEED)) &&
1412             cpu_matches(cpu_vuln_blacklist, SRBDS | MMIO_SBDS))
1413                     setup_force_cpu_bug(X86_BUG_SRBDS);
1414
1415         /*
1416          * Processor MMIO Stale Data bug enumeration
1417          *
1418          * Affected CPU list is generally enough to enumerate the vulnerability,
1419          * but for virtualization case check for ARCH_CAP MSR bits also, VMM may
1420          * not want the guest to enumerate the bug.
1421          *
1422          * Set X86_BUG_MMIO_UNKNOWN for CPUs that are neither in the blacklist,
1423          * nor in the whitelist and also don't enumerate MSR ARCH_CAP MMIO bits.
1424          */
1425         if (!arch_cap_mmio_immune(x86_arch_cap_msr)) {
1426                 if (cpu_matches(cpu_vuln_blacklist, MMIO))
1427                         setup_force_cpu_bug(X86_BUG_MMIO_STALE_DATA);
1428                 else if (!cpu_matches(cpu_vuln_whitelist, NO_MMIO))
1429                         setup_force_cpu_bug(X86_BUG_MMIO_UNKNOWN);
1430         }
1431
1432         if (!cpu_has(c, X86_FEATURE_BTC_NO)) {
1433                 if (cpu_matches(cpu_vuln_blacklist, RETBLEED) || (x86_arch_cap_msr & ARCH_CAP_RSBA))
1434                         setup_force_cpu_bug(X86_BUG_RETBLEED);
1435         }
1436
1437         if (cpu_matches(cpu_vuln_blacklist, SMT_RSB))
1438                 setup_force_cpu_bug(X86_BUG_SMT_RSB);
1439
1440         /*
1441          * Check if CPU is vulnerable to GDS. If running in a virtual machine on
1442          * an affected processor, the VMM may have disabled the use of GATHER by
1443          * disabling AVX2. The only way to do this in HW is to clear XCR0[2],
1444          * which means that AVX will be disabled.
1445          */
1446         if (cpu_matches(cpu_vuln_blacklist, GDS) && !(x86_arch_cap_msr & ARCH_CAP_GDS_NO) &&
1447             boot_cpu_has(X86_FEATURE_AVX))
1448                 setup_force_cpu_bug(X86_BUG_GDS);
1449
1450         if (!cpu_has(c, X86_FEATURE_SRSO_NO)) {
1451                 if (cpu_matches(cpu_vuln_blacklist, SRSO))
1452                         setup_force_cpu_bug(X86_BUG_SRSO);
1453         }
1454
1455         if (vulnerable_to_rfds(x86_arch_cap_msr))
1456                 setup_force_cpu_bug(X86_BUG_RFDS);
1457
1458         /* When virtualized, eIBRS could be hidden, assume vulnerable */
1459         if (!(x86_arch_cap_msr & ARCH_CAP_BHI_NO) &&
1460             !cpu_matches(cpu_vuln_whitelist, NO_BHI) &&
1461             (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED) ||
1462              boot_cpu_has(X86_FEATURE_HYPERVISOR)))
1463                 setup_force_cpu_bug(X86_BUG_BHI);
1464
1465         if (cpu_matches(cpu_vuln_whitelist, NO_MELTDOWN))
1466                 return;
1467
1468         /* Rogue Data Cache Load? No! */
1469         if (x86_arch_cap_msr & ARCH_CAP_RDCL_NO)
1470                 return;
1471
1472         setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN);
1473
1474         if (cpu_matches(cpu_vuln_whitelist, NO_L1TF))
1475                 return;
1476
1477         setup_force_cpu_bug(X86_BUG_L1TF);
1478 }
1479
1480 /*
1481  * The NOPL instruction is supposed to exist on all CPUs of family >= 6;
1482  * unfortunately, that's not true in practice because of early VIA
1483  * chips and (more importantly) broken virtualizers that are not easy
1484  * to detect. In the latter case it doesn't even *fail* reliably, so
1485  * probing for it doesn't even work. Disable it completely on 32-bit
1486  * unless we can find a reliable way to detect all the broken cases.
1487  * Enable it explicitly on 64-bit for non-constant inputs of cpu_has().
1488  */
1489 static void detect_nopl(void)
1490 {
1491 #ifdef CONFIG_X86_32
1492         setup_clear_cpu_cap(X86_FEATURE_NOPL);
1493 #else
1494         setup_force_cpu_cap(X86_FEATURE_NOPL);
1495 #endif
1496 }
1497
1498 /*
1499  * We parse cpu parameters early because fpu__init_system() is executed
1500  * before parse_early_param().
1501  */
1502 static void __init cpu_parse_early_param(void)
1503 {
1504         char arg[128];
1505         char *argptr = arg, *opt;
1506         int arglen, taint = 0;
1507
1508 #ifdef CONFIG_X86_32
1509         if (cmdline_find_option_bool(boot_command_line, "no387"))
1510 #ifdef CONFIG_MATH_EMULATION
1511                 setup_clear_cpu_cap(X86_FEATURE_FPU);
1512 #else
1513                 pr_err("Option 'no387' required CONFIG_MATH_EMULATION enabled.\n");
1514 #endif
1515
1516         if (cmdline_find_option_bool(boot_command_line, "nofxsr"))
1517                 setup_clear_cpu_cap(X86_FEATURE_FXSR);
1518 #endif
1519
1520         if (cmdline_find_option_bool(boot_command_line, "noxsave"))
1521                 setup_clear_cpu_cap(X86_FEATURE_XSAVE);
1522
1523         if (cmdline_find_option_bool(boot_command_line, "noxsaveopt"))
1524                 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
1525
1526         if (cmdline_find_option_bool(boot_command_line, "noxsaves"))
1527                 setup_clear_cpu_cap(X86_FEATURE_XSAVES);
1528
1529         arglen = cmdline_find_option(boot_command_line, "clearcpuid", arg, sizeof(arg));
1530         if (arglen <= 0)
1531                 return;
1532
1533         pr_info("Clearing CPUID bits:");
1534
1535         while (argptr) {
1536                 bool found __maybe_unused = false;
1537                 unsigned int bit;
1538
1539                 opt = strsep(&argptr, ",");
1540
1541                 /*
1542                  * Handle naked numbers first for feature flags which don't
1543                  * have names.
1544                  */
1545                 if (!kstrtouint(opt, 10, &bit)) {
1546                         if (bit < NCAPINTS * 32) {
1547
1548 #ifdef CONFIG_X86_FEATURE_NAMES
1549                                 /* empty-string, i.e., ""-defined feature flags */
1550                                 if (!x86_cap_flags[bit])
1551                                         pr_cont(" " X86_CAP_FMT_NUM, x86_cap_flag_num(bit));
1552                                 else
1553 #endif
1554                                         pr_cont(" " X86_CAP_FMT, x86_cap_flag(bit));
1555
1556                                 setup_clear_cpu_cap(bit);
1557                                 taint++;
1558                         }
1559                         /*
1560                          * The assumption is that there are no feature names with only
1561                          * numbers in the name thus go to the next argument.
1562                          */
1563                         continue;
1564                 }
1565
1566 #ifdef CONFIG_X86_FEATURE_NAMES
1567                 for (bit = 0; bit < 32 * NCAPINTS; bit++) {
1568                         if (!x86_cap_flag(bit))
1569                                 continue;
1570
1571                         if (strcmp(x86_cap_flag(bit), opt))
1572                                 continue;
1573
1574                         pr_cont(" %s", opt);
1575                         setup_clear_cpu_cap(bit);
1576                         taint++;
1577                         found = true;
1578                         break;
1579                 }
1580
1581                 if (!found)
1582                         pr_cont(" (unknown: %s)", opt);
1583 #endif
1584         }
1585         pr_cont("\n");
1586
1587         if (taint)
1588                 add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
1589 }
1590
1591 /*
1592  * Do minimum CPU detection early.
1593  * Fields really needed: vendor, cpuid_level, family, model, mask,
1594  * cache alignment.
1595  * The others are not touched to avoid unwanted side effects.
1596  *
1597  * WARNING: this function is only called on the boot CPU.  Don't add code
1598  * here that is supposed to run on all CPUs.
1599  */
1600 static void __init early_identify_cpu(struct cpuinfo_x86 *c)
1601 {
1602 #ifdef CONFIG_X86_64
1603         c->x86_clflush_size = 64;
1604         c->x86_phys_bits = 36;
1605         c->x86_virt_bits = 48;
1606 #else
1607         c->x86_clflush_size = 32;
1608         c->x86_phys_bits = 32;
1609         c->x86_virt_bits = 32;
1610 #endif
1611         c->x86_cache_alignment = c->x86_clflush_size;
1612
1613         memset(&c->x86_capability, 0, sizeof(c->x86_capability));
1614         c->extended_cpuid_level = 0;
1615
1616         if (!have_cpuid_p())
1617                 identify_cpu_without_cpuid(c);
1618
1619         /* cyrix could have cpuid enabled via c_identify()*/
1620         if (have_cpuid_p()) {
1621                 cpu_detect(c);
1622                 get_cpu_vendor(c);
1623                 get_cpu_cap(c);
1624                 get_cpu_address_sizes(c);
1625                 setup_force_cpu_cap(X86_FEATURE_CPUID);
1626                 cpu_parse_early_param();
1627
1628                 if (this_cpu->c_early_init)
1629                         this_cpu->c_early_init(c);
1630
1631                 c->cpu_index = 0;
1632                 filter_cpuid_features(c, false);
1633
1634                 if (this_cpu->c_bsp_init)
1635                         this_cpu->c_bsp_init(c);
1636         } else {
1637                 setup_clear_cpu_cap(X86_FEATURE_CPUID);
1638         }
1639
1640         setup_force_cpu_cap(X86_FEATURE_ALWAYS);
1641
1642         cpu_set_bug_bits(c);
1643
1644         sld_setup(c);
1645
1646 #ifdef CONFIG_X86_32
1647         /*
1648          * Regardless of whether PCID is enumerated, the SDM says
1649          * that it can't be enabled in 32-bit mode.
1650          */
1651         setup_clear_cpu_cap(X86_FEATURE_PCID);
1652 #endif
1653
1654         /*
1655          * Later in the boot process pgtable_l5_enabled() relies on
1656          * cpu_feature_enabled(X86_FEATURE_LA57). If 5-level paging is not
1657          * enabled by this point we need to clear the feature bit to avoid
1658          * false-positives at the later stage.
1659          *
1660          * pgtable_l5_enabled() can be false here for several reasons:
1661          *  - 5-level paging is disabled compile-time;
1662          *  - it's 32-bit kernel;
1663          *  - machine doesn't support 5-level paging;
1664          *  - user specified 'no5lvl' in kernel command line.
1665          */
1666         if (!pgtable_l5_enabled())
1667                 setup_clear_cpu_cap(X86_FEATURE_LA57);
1668
1669         detect_nopl();
1670 }
1671
1672 void __init early_cpu_init(void)
1673 {
1674         const struct cpu_dev *const *cdev;
1675         int count = 0;
1676
1677 #ifdef CONFIG_PROCESSOR_SELECT
1678         pr_info("KERNEL supported cpus:\n");
1679 #endif
1680
1681         for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
1682                 const struct cpu_dev *cpudev = *cdev;
1683
1684                 if (count >= X86_VENDOR_NUM)
1685                         break;
1686                 cpu_devs[count] = cpudev;
1687                 count++;
1688
1689 #ifdef CONFIG_PROCESSOR_SELECT
1690                 {
1691                         unsigned int j;
1692
1693                         for (j = 0; j < 2; j++) {
1694                                 if (!cpudev->c_ident[j])
1695                                         continue;
1696                                 pr_info("  %s %s\n", cpudev->c_vendor,
1697                                         cpudev->c_ident[j]);
1698                         }
1699                 }
1700 #endif
1701         }
1702         early_identify_cpu(&boot_cpu_data);
1703 }
1704
1705 static bool detect_null_seg_behavior(void)
1706 {
1707         /*
1708          * Empirically, writing zero to a segment selector on AMD does
1709          * not clear the base, whereas writing zero to a segment
1710          * selector on Intel does clear the base.  Intel's behavior
1711          * allows slightly faster context switches in the common case
1712          * where GS is unused by the prev and next threads.
1713          *
1714          * Since neither vendor documents this anywhere that I can see,
1715          * detect it directly instead of hard-coding the choice by
1716          * vendor.
1717          *
1718          * I've designated AMD's behavior as the "bug" because it's
1719          * counterintuitive and less friendly.
1720          */
1721
1722         unsigned long old_base, tmp;
1723         rdmsrl(MSR_FS_BASE, old_base);
1724         wrmsrl(MSR_FS_BASE, 1);
1725         loadsegment(fs, 0);
1726         rdmsrl(MSR_FS_BASE, tmp);
1727         wrmsrl(MSR_FS_BASE, old_base);
1728         return tmp == 0;
1729 }
1730
1731 void check_null_seg_clears_base(struct cpuinfo_x86 *c)
1732 {
1733         /* BUG_NULL_SEG is only relevant with 64bit userspace */
1734         if (!IS_ENABLED(CONFIG_X86_64))
1735                 return;
1736
1737         /* Zen3 CPUs advertise Null Selector Clears Base in CPUID. */
1738         if (c->extended_cpuid_level >= 0x80000021 &&
1739             cpuid_eax(0x80000021) & BIT(6))
1740                 return;
1741
1742         /*
1743          * CPUID bit above wasn't set. If this kernel is still running
1744          * as a HV guest, then the HV has decided not to advertize
1745          * that CPUID bit for whatever reason.  For example, one
1746          * member of the migration pool might be vulnerable.  Which
1747          * means, the bug is present: set the BUG flag and return.
1748          */
1749         if (cpu_has(c, X86_FEATURE_HYPERVISOR)) {
1750                 set_cpu_bug(c, X86_BUG_NULL_SEG);
1751                 return;
1752         }
1753
1754         /*
1755          * Zen2 CPUs also have this behaviour, but no CPUID bit.
1756          * 0x18 is the respective family for Hygon.
1757          */
1758         if ((c->x86 == 0x17 || c->x86 == 0x18) &&
1759             detect_null_seg_behavior())
1760                 return;
1761
1762         /* All the remaining ones are affected */
1763         set_cpu_bug(c, X86_BUG_NULL_SEG);
1764 }
1765
1766 static void generic_identify(struct cpuinfo_x86 *c)
1767 {
1768         c->extended_cpuid_level = 0;
1769
1770         if (!have_cpuid_p())
1771                 identify_cpu_without_cpuid(c);
1772
1773         /* cyrix could have cpuid enabled via c_identify()*/
1774         if (!have_cpuid_p())
1775                 return;
1776
1777         cpu_detect(c);
1778
1779         get_cpu_vendor(c);
1780
1781         get_cpu_cap(c);
1782
1783         get_cpu_address_sizes(c);
1784
1785         if (c->cpuid_level >= 0x00000001) {
1786                 c->initial_apicid = (cpuid_ebx(1) >> 24) & 0xFF;
1787 #ifdef CONFIG_X86_32
1788 # ifdef CONFIG_SMP
1789                 c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
1790 # else
1791                 c->apicid = c->initial_apicid;
1792 # endif
1793 #endif
1794                 c->phys_proc_id = c->initial_apicid;
1795         }
1796
1797         get_model_name(c); /* Default name */
1798
1799         /*
1800          * ESPFIX is a strange bug.  All real CPUs have it.  Paravirt
1801          * systems that run Linux at CPL > 0 may or may not have the
1802          * issue, but, even if they have the issue, there's absolutely
1803          * nothing we can do about it because we can't use the real IRET
1804          * instruction.
1805          *
1806          * NB: For the time being, only 32-bit kernels support
1807          * X86_BUG_ESPFIX as such.  64-bit kernels directly choose
1808          * whether to apply espfix using paravirt hooks.  If any
1809          * non-paravirt system ever shows up that does *not* have the
1810          * ESPFIX issue, we can change this.
1811          */
1812 #ifdef CONFIG_X86_32
1813         set_cpu_bug(c, X86_BUG_ESPFIX);
1814 #endif
1815 }
1816
1817 /*
1818  * Validate that ACPI/mptables have the same information about the
1819  * effective APIC id and update the package map.
1820  */
1821 static void validate_apic_and_package_id(struct cpuinfo_x86 *c)
1822 {
1823 #ifdef CONFIG_SMP
1824         unsigned int apicid, cpu = smp_processor_id();
1825
1826         apicid = apic->cpu_present_to_apicid(cpu);
1827
1828         if (apicid != c->apicid) {
1829                 pr_err(FW_BUG "CPU%u: APIC id mismatch. Firmware: %x APIC: %x\n",
1830                        cpu, apicid, c->initial_apicid);
1831         }
1832         BUG_ON(topology_update_package_map(c->phys_proc_id, cpu));
1833         BUG_ON(topology_update_die_map(c->cpu_die_id, cpu));
1834 #else
1835         c->logical_proc_id = 0;
1836 #endif
1837 }
1838
1839 /*
1840  * This does the hard work of actually picking apart the CPU stuff...
1841  */
1842 static void identify_cpu(struct cpuinfo_x86 *c)
1843 {
1844         int i;
1845
1846         c->loops_per_jiffy = loops_per_jiffy;
1847         c->x86_cache_size = 0;
1848         c->x86_vendor = X86_VENDOR_UNKNOWN;
1849         c->x86_model = c->x86_stepping = 0;     /* So far unknown... */
1850         c->x86_vendor_id[0] = '\0'; /* Unset */
1851         c->x86_model_id[0] = '\0';  /* Unset */
1852         c->x86_max_cores = 1;
1853         c->x86_coreid_bits = 0;
1854         c->cu_id = 0xff;
1855 #ifdef CONFIG_X86_64
1856         c->x86_clflush_size = 64;
1857         c->x86_phys_bits = 36;
1858         c->x86_virt_bits = 48;
1859 #else
1860         c->cpuid_level = -1;    /* CPUID not detected */
1861         c->x86_clflush_size = 32;
1862         c->x86_phys_bits = 32;
1863         c->x86_virt_bits = 32;
1864 #endif
1865         c->x86_cache_alignment = c->x86_clflush_size;
1866         memset(&c->x86_capability, 0, sizeof(c->x86_capability));
1867 #ifdef CONFIG_X86_VMX_FEATURE_NAMES
1868         memset(&c->vmx_capability, 0, sizeof(c->vmx_capability));
1869 #endif
1870
1871         generic_identify(c);
1872
1873         if (this_cpu->c_identify)
1874                 this_cpu->c_identify(c);
1875
1876         /* Clear/Set all flags overridden by options, after probe */
1877         apply_forced_caps(c);
1878
1879 #ifdef CONFIG_X86_64
1880         c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
1881 #endif
1882
1883         /*
1884          * Vendor-specific initialization.  In this section we
1885          * canonicalize the feature flags, meaning if there are
1886          * features a certain CPU supports which CPUID doesn't
1887          * tell us, CPUID claiming incorrect flags, or other bugs,
1888          * we handle them here.
1889          *
1890          * At the end of this section, c->x86_capability better
1891          * indicate the features this CPU genuinely supports!
1892          */
1893         if (this_cpu->c_init)
1894                 this_cpu->c_init(c);
1895
1896         /* Disable the PN if appropriate */
1897         squash_the_stupid_serial_number(c);
1898
1899         /* Set up SMEP/SMAP/UMIP */
1900         setup_smep(c);
1901         setup_smap(c);
1902         setup_umip(c);
1903
1904         /* Enable FSGSBASE instructions if available. */
1905         if (cpu_has(c, X86_FEATURE_FSGSBASE)) {
1906                 cr4_set_bits(X86_CR4_FSGSBASE);
1907                 elf_hwcap2 |= HWCAP2_FSGSBASE;
1908         }
1909
1910         /*
1911          * The vendor-specific functions might have changed features.
1912          * Now we do "generic changes."
1913          */
1914
1915         /* Filter out anything that depends on CPUID levels we don't have */
1916         filter_cpuid_features(c, true);
1917
1918         /* If the model name is still unset, do table lookup. */
1919         if (!c->x86_model_id[0]) {
1920                 const char *p;
1921                 p = table_lookup_model(c);
1922                 if (p)
1923                         strcpy(c->x86_model_id, p);
1924                 else
1925                         /* Last resort... */
1926                         sprintf(c->x86_model_id, "%02x/%02x",
1927                                 c->x86, c->x86_model);
1928         }
1929
1930 #ifdef CONFIG_X86_64
1931         detect_ht(c);
1932 #endif
1933
1934         x86_init_rdrand(c);
1935         setup_pku(c);
1936         setup_cet(c);
1937
1938         /*
1939          * Clear/Set all flags overridden by options, need do it
1940          * before following smp all cpus cap AND.
1941          */
1942         apply_forced_caps(c);
1943
1944         /*
1945          * On SMP, boot_cpu_data holds the common feature set between
1946          * all CPUs; so make sure that we indicate which features are
1947          * common between the CPUs.  The first time this routine gets
1948          * executed, c == &boot_cpu_data.
1949          */
1950         if (c != &boot_cpu_data) {
1951                 /* AND the already accumulated flags with these */
1952                 for (i = 0; i < NCAPINTS; i++)
1953                         boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
1954
1955                 /* OR, i.e. replicate the bug flags */
1956                 for (i = NCAPINTS; i < NCAPINTS + NBUGINTS; i++)
1957                         c->x86_capability[i] |= boot_cpu_data.x86_capability[i];
1958         }
1959
1960         ppin_init(c);
1961
1962         /* Init Machine Check Exception if available. */
1963         mcheck_cpu_init(c);
1964
1965         select_idle_routine(c);
1966
1967 #ifdef CONFIG_NUMA
1968         numa_add_cpu(smp_processor_id());
1969 #endif
1970 }
1971
1972 /*
1973  * Set up the CPU state needed to execute SYSENTER/SYSEXIT instructions
1974  * on 32-bit kernels:
1975  */
1976 #ifdef CONFIG_X86_32
1977 void enable_sep_cpu(void)
1978 {
1979         struct tss_struct *tss;
1980         int cpu;
1981
1982         if (!boot_cpu_has(X86_FEATURE_SEP))
1983                 return;
1984
1985         cpu = get_cpu();
1986         tss = &per_cpu(cpu_tss_rw, cpu);
1987
1988         /*
1989          * We cache MSR_IA32_SYSENTER_CS's value in the TSS's ss1 field --
1990          * see the big comment in struct x86_hw_tss's definition.
1991          */
1992
1993         tss->x86_tss.ss1 = __KERNEL_CS;
1994         wrmsr(MSR_IA32_SYSENTER_CS, tss->x86_tss.ss1, 0);
1995         wrmsr(MSR_IA32_SYSENTER_ESP, (unsigned long)(cpu_entry_stack(cpu) + 1), 0);
1996         wrmsr(MSR_IA32_SYSENTER_EIP, (unsigned long)entry_SYSENTER_32, 0);
1997
1998         put_cpu();
1999 }
2000 #endif
2001
2002 void __init identify_boot_cpu(void)
2003 {
2004         identify_cpu(&boot_cpu_data);
2005         if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT))
2006                 pr_info("CET detected: Indirect Branch Tracking enabled\n");
2007 #ifdef CONFIG_X86_32
2008         sysenter_setup();
2009         enable_sep_cpu();
2010 #endif
2011         cpu_detect_tlb(&boot_cpu_data);
2012         setup_cr_pinning();
2013
2014         tsx_init();
2015 }
2016
2017 void identify_secondary_cpu(struct cpuinfo_x86 *c)
2018 {
2019         BUG_ON(c == &boot_cpu_data);
2020         identify_cpu(c);
2021 #ifdef CONFIG_X86_32
2022         enable_sep_cpu();
2023 #endif
2024         mtrr_ap_init();
2025         validate_apic_and_package_id(c);
2026         x86_spec_ctrl_setup_ap();
2027         update_srbds_msr();
2028         if (boot_cpu_has_bug(X86_BUG_GDS))
2029                 update_gds_msr();
2030
2031         tsx_ap_init();
2032 }
2033
2034 void print_cpu_info(struct cpuinfo_x86 *c)
2035 {
2036         const char *vendor = NULL;
2037
2038         if (c->x86_vendor < X86_VENDOR_NUM) {
2039                 vendor = this_cpu->c_vendor;
2040         } else {
2041                 if (c->cpuid_level >= 0)
2042                         vendor = c->x86_vendor_id;
2043         }
2044
2045         if (vendor && !strstr(c->x86_model_id, vendor))
2046                 pr_cont("%s ", vendor);
2047
2048         if (c->x86_model_id[0])
2049                 pr_cont("%s", c->x86_model_id);
2050         else
2051                 pr_cont("%d86", c->x86);
2052
2053         pr_cont(" (family: 0x%x, model: 0x%x", c->x86, c->x86_model);
2054
2055         if (c->x86_stepping || c->cpuid_level >= 0)
2056                 pr_cont(", stepping: 0x%x)\n", c->x86_stepping);
2057         else
2058                 pr_cont(")\n");
2059 }
2060
2061 /*
2062  * clearcpuid= was already parsed in cpu_parse_early_param().  This dummy
2063  * function prevents it from becoming an environment variable for init.
2064  */
2065 static __init int setup_clearcpuid(char *arg)
2066 {
2067         return 1;
2068 }
2069 __setup("clearcpuid=", setup_clearcpuid);
2070
2071 #ifdef CONFIG_X86_64
2072 DEFINE_PER_CPU_FIRST(struct fixed_percpu_data,
2073                      fixed_percpu_data) __aligned(PAGE_SIZE) __visible;
2074 EXPORT_PER_CPU_SYMBOL_GPL(fixed_percpu_data);
2075
2076 /*
2077  * The following percpu variables are hot.  Align current_task to
2078  * cacheline size such that they fall in the same cacheline.
2079  */
2080 DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
2081         &init_task;
2082 EXPORT_PER_CPU_SYMBOL(current_task);
2083
2084 DEFINE_PER_CPU(void *, hardirq_stack_ptr);
2085 DEFINE_PER_CPU(bool, hardirq_stack_inuse);
2086
2087 DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
2088 EXPORT_PER_CPU_SYMBOL(__preempt_count);
2089
2090 DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) = TOP_OF_INIT_STACK;
2091
2092 static void wrmsrl_cstar(unsigned long val)
2093 {
2094         /*
2095          * Intel CPUs do not support 32-bit SYSCALL. Writing to MSR_CSTAR
2096          * is so far ignored by the CPU, but raises a #VE trap in a TDX
2097          * guest. Avoid the pointless write on all Intel CPUs.
2098          */
2099         if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
2100                 wrmsrl(MSR_CSTAR, val);
2101 }
2102
2103 /* May not be marked __init: used by software suspend */
2104 void syscall_init(void)
2105 {
2106         wrmsr(MSR_STAR, 0, (__USER32_CS << 16) | __KERNEL_CS);
2107         wrmsrl(MSR_LSTAR, (unsigned long)entry_SYSCALL_64);
2108
2109 #ifdef CONFIG_IA32_EMULATION
2110         wrmsrl_cstar((unsigned long)entry_SYSCALL_compat);
2111         /*
2112          * This only works on Intel CPUs.
2113          * On AMD CPUs these MSRs are 32-bit, CPU truncates MSR_IA32_SYSENTER_EIP.
2114          * This does not cause SYSENTER to jump to the wrong location, because
2115          * AMD doesn't allow SYSENTER in long mode (either 32- or 64-bit).
2116          */
2117         wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)__KERNEL_CS);
2118         wrmsrl_safe(MSR_IA32_SYSENTER_ESP,
2119                     (unsigned long)(cpu_entry_stack(smp_processor_id()) + 1));
2120         wrmsrl_safe(MSR_IA32_SYSENTER_EIP, (u64)entry_SYSENTER_compat);
2121 #else
2122         wrmsrl_cstar((unsigned long)ignore_sysret);
2123         wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)GDT_ENTRY_INVALID_SEG);
2124         wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
2125         wrmsrl_safe(MSR_IA32_SYSENTER_EIP, 0ULL);
2126 #endif
2127
2128         /*
2129          * Flags to clear on syscall; clear as much as possible
2130          * to minimize user space-kernel interference.
2131          */
2132         wrmsrl(MSR_SYSCALL_MASK,
2133                X86_EFLAGS_CF|X86_EFLAGS_PF|X86_EFLAGS_AF|
2134                X86_EFLAGS_ZF|X86_EFLAGS_SF|X86_EFLAGS_TF|
2135                X86_EFLAGS_IF|X86_EFLAGS_DF|X86_EFLAGS_OF|
2136                X86_EFLAGS_IOPL|X86_EFLAGS_NT|X86_EFLAGS_RF|
2137                X86_EFLAGS_AC|X86_EFLAGS_ID);
2138 }
2139
2140 #else   /* CONFIG_X86_64 */
2141
2142 DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
2143 EXPORT_PER_CPU_SYMBOL(current_task);
2144 DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
2145 EXPORT_PER_CPU_SYMBOL(__preempt_count);
2146
2147 /*
2148  * On x86_32, vm86 modifies tss.sp0, so sp0 isn't a reliable way to find
2149  * the top of the kernel stack.  Use an extra percpu variable to track the
2150  * top of the kernel stack directly.
2151  */
2152 DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) =
2153         (unsigned long)&init_thread_union + THREAD_SIZE;
2154 EXPORT_PER_CPU_SYMBOL(cpu_current_top_of_stack);
2155
2156 #ifdef CONFIG_STACKPROTECTOR
2157 DEFINE_PER_CPU(unsigned long, __stack_chk_guard);
2158 EXPORT_PER_CPU_SYMBOL(__stack_chk_guard);
2159 #endif
2160
2161 #endif  /* CONFIG_X86_64 */
2162
2163 /*
2164  * Clear all 6 debug registers:
2165  */
2166 static void clear_all_debug_regs(void)
2167 {
2168         int i;
2169
2170         for (i = 0; i < 8; i++) {
2171                 /* Ignore db4, db5 */
2172                 if ((i == 4) || (i == 5))
2173                         continue;
2174
2175                 set_debugreg(0, i);
2176         }
2177 }
2178
2179 #ifdef CONFIG_KGDB
2180 /*
2181  * Restore debug regs if using kgdbwait and you have a kernel debugger
2182  * connection established.
2183  */
2184 static void dbg_restore_debug_regs(void)
2185 {
2186         if (unlikely(kgdb_connected && arch_kgdb_ops.correct_hw_break))
2187                 arch_kgdb_ops.correct_hw_break();
2188 }
2189 #else /* ! CONFIG_KGDB */
2190 #define dbg_restore_debug_regs()
2191 #endif /* ! CONFIG_KGDB */
2192
2193 static void wait_for_master_cpu(int cpu)
2194 {
2195 #ifdef CONFIG_SMP
2196         /*
2197          * wait for ACK from master CPU before continuing
2198          * with AP initialization
2199          */
2200         WARN_ON(cpumask_test_and_set_cpu(cpu, cpu_initialized_mask));
2201         while (!cpumask_test_cpu(cpu, cpu_callout_mask))
2202                 cpu_relax();
2203 #endif
2204 }
2205
2206 #ifdef CONFIG_X86_64
2207 static inline void setup_getcpu(int cpu)
2208 {
2209         unsigned long cpudata = vdso_encode_cpunode(cpu, early_cpu_to_node(cpu));
2210         struct desc_struct d = { };
2211
2212         if (boot_cpu_has(X86_FEATURE_RDTSCP) || boot_cpu_has(X86_FEATURE_RDPID))
2213                 wrmsr(MSR_TSC_AUX, cpudata, 0);
2214
2215         /* Store CPU and node number in limit. */
2216         d.limit0 = cpudata;
2217         d.limit1 = cpudata >> 16;
2218
2219         d.type = 5;             /* RO data, expand down, accessed */
2220         d.dpl = 3;              /* Visible to user code */
2221         d.s = 1;                /* Not a system segment */
2222         d.p = 1;                /* Present */
2223         d.d = 1;                /* 32-bit */
2224
2225         write_gdt_entry(get_cpu_gdt_rw(cpu), GDT_ENTRY_CPUNODE, &d, DESCTYPE_S);
2226 }
2227
2228 static inline void ucode_cpu_init(int cpu)
2229 {
2230         if (cpu)
2231                 load_ucode_ap();
2232 }
2233
2234 static inline void tss_setup_ist(struct tss_struct *tss)
2235 {
2236         /* Set up the per-CPU TSS IST stacks */
2237         tss->x86_tss.ist[IST_INDEX_DF] = __this_cpu_ist_top_va(DF);
2238         tss->x86_tss.ist[IST_INDEX_NMI] = __this_cpu_ist_top_va(NMI);
2239         tss->x86_tss.ist[IST_INDEX_DB] = __this_cpu_ist_top_va(DB);
2240         tss->x86_tss.ist[IST_INDEX_MCE] = __this_cpu_ist_top_va(MCE);
2241         /* Only mapped when SEV-ES is active */
2242         tss->x86_tss.ist[IST_INDEX_VC] = __this_cpu_ist_top_va(VC);
2243 }
2244
2245 #else /* CONFIG_X86_64 */
2246
2247 static inline void setup_getcpu(int cpu) { }
2248
2249 static inline void ucode_cpu_init(int cpu)
2250 {
2251         show_ucode_info_early();
2252 }
2253
2254 static inline void tss_setup_ist(struct tss_struct *tss) { }
2255
2256 #endif /* !CONFIG_X86_64 */
2257
2258 static inline void tss_setup_io_bitmap(struct tss_struct *tss)
2259 {
2260         tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET_INVALID;
2261
2262 #ifdef CONFIG_X86_IOPL_IOPERM
2263         tss->io_bitmap.prev_max = 0;
2264         tss->io_bitmap.prev_sequence = 0;
2265         memset(tss->io_bitmap.bitmap, 0xff, sizeof(tss->io_bitmap.bitmap));
2266         /*
2267          * Invalidate the extra array entry past the end of the all
2268          * permission bitmap as required by the hardware.
2269          */
2270         tss->io_bitmap.mapall[IO_BITMAP_LONGS] = ~0UL;
2271 #endif
2272 }
2273
2274 /*
2275  * Setup everything needed to handle exceptions from the IDT, including the IST
2276  * exceptions which use paranoid_entry().
2277  */
2278 void cpu_init_exception_handling(void)
2279 {
2280         struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw);
2281         int cpu = raw_smp_processor_id();
2282
2283         /* paranoid_entry() gets the CPU number from the GDT */
2284         setup_getcpu(cpu);
2285
2286         /* IST vectors need TSS to be set up. */
2287         tss_setup_ist(tss);
2288         tss_setup_io_bitmap(tss);
2289         set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
2290
2291         load_TR_desc();
2292
2293         /* GHCB needs to be setup to handle #VC. */
2294         setup_ghcb();
2295
2296         /* Finally load the IDT */
2297         load_current_idt();
2298 }
2299
2300 /*
2301  * cpu_init() initializes state that is per-CPU. Some data is already
2302  * initialized (naturally) in the bootstrap process, such as the GDT.  We
2303  * reload it nevertheless, this function acts as a 'CPU state barrier',
2304  * nothing should get across.
2305  */
2306 void cpu_init(void)
2307 {
2308         struct task_struct *cur = current;
2309         int cpu = raw_smp_processor_id();
2310
2311         wait_for_master_cpu(cpu);
2312
2313         ucode_cpu_init(cpu);
2314
2315 #ifdef CONFIG_NUMA
2316         if (this_cpu_read(numa_node) == 0 &&
2317             early_cpu_to_node(cpu) != NUMA_NO_NODE)
2318                 set_numa_node(early_cpu_to_node(cpu));
2319 #endif
2320         pr_debug("Initializing CPU#%d\n", cpu);
2321
2322         if (IS_ENABLED(CONFIG_X86_64) || cpu_feature_enabled(X86_FEATURE_VME) ||
2323             boot_cpu_has(X86_FEATURE_TSC) || boot_cpu_has(X86_FEATURE_DE))
2324                 cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
2325
2326         /*
2327          * Initialize the per-CPU GDT with the boot GDT,
2328          * and set up the GDT descriptor:
2329          */
2330         switch_to_new_gdt(cpu);
2331
2332         if (IS_ENABLED(CONFIG_X86_64)) {
2333                 loadsegment(fs, 0);
2334                 memset(cur->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
2335                 syscall_init();
2336
2337                 wrmsrl(MSR_FS_BASE, 0);
2338                 wrmsrl(MSR_KERNEL_GS_BASE, 0);
2339                 barrier();
2340
2341                 x2apic_setup();
2342         }
2343
2344         mmgrab(&init_mm);
2345         cur->active_mm = &init_mm;
2346         BUG_ON(cur->mm);
2347         initialize_tlbstate_and_flush();
2348         enter_lazy_tlb(&init_mm, cur);
2349
2350         /*
2351          * sp0 points to the entry trampoline stack regardless of what task
2352          * is running.
2353          */
2354         load_sp0((unsigned long)(cpu_entry_stack(cpu) + 1));
2355
2356         load_mm_ldt(&init_mm);
2357
2358         clear_all_debug_regs();
2359         dbg_restore_debug_regs();
2360
2361         doublefault_init_cpu_tss();
2362
2363         if (is_uv_system())
2364                 uv_cpu_init();
2365
2366         load_fixmap_gdt(cpu);
2367 }
2368
2369 #ifdef CONFIG_SMP
2370 void cpu_init_secondary(void)
2371 {
2372         /*
2373          * Relies on the BP having set-up the IDT tables, which are loaded
2374          * on this CPU in cpu_init_exception_handling().
2375          */
2376         cpu_init_exception_handling();
2377         cpu_init();
2378         fpu__init_cpu();
2379 }
2380 #endif
2381
2382 #ifdef CONFIG_MICROCODE_LATE_LOADING
2383 /**
2384  * store_cpu_caps() - Store a snapshot of CPU capabilities
2385  * @curr_info: Pointer where to store it
2386  *
2387  * Returns: None
2388  */
2389 void store_cpu_caps(struct cpuinfo_x86 *curr_info)
2390 {
2391         /* Reload CPUID max function as it might've changed. */
2392         curr_info->cpuid_level = cpuid_eax(0);
2393
2394         /* Copy all capability leafs and pick up the synthetic ones. */
2395         memcpy(&curr_info->x86_capability, &boot_cpu_data.x86_capability,
2396                sizeof(curr_info->x86_capability));
2397
2398         /* Get the hardware CPUID leafs */
2399         get_cpu_cap(curr_info);
2400 }
2401
2402 /**
2403  * microcode_check() - Check if any CPU capabilities changed after an update.
2404  * @prev_info:  CPU capabilities stored before an update.
2405  *
2406  * The microcode loader calls this upon late microcode load to recheck features,
2407  * only when microcode has been updated. Caller holds microcode_mutex and CPU
2408  * hotplug lock.
2409  *
2410  * Return: None
2411  */
2412 void microcode_check(struct cpuinfo_x86 *prev_info)
2413 {
2414         struct cpuinfo_x86 curr_info;
2415
2416         perf_check_microcode();
2417
2418         amd_check_microcode();
2419
2420         store_cpu_caps(&curr_info);
2421
2422         if (!memcmp(&prev_info->x86_capability, &curr_info.x86_capability,
2423                     sizeof(prev_info->x86_capability)))
2424                 return;
2425
2426         pr_warn("x86/CPU: CPU features have changed after loading microcode, but might not take effect.\n");
2427         pr_warn("x86/CPU: Please consider either early loading through initrd/built-in or a potential BIOS update.\n");
2428 }
2429 #endif
2430
2431 /*
2432  * Invoked from core CPU hotplug code after hotplug operations
2433  */
2434 void arch_smt_update(void)
2435 {
2436         /* Handle the speculative execution misfeatures */
2437         cpu_bugs_smt_update();
2438         /* Check whether IPI broadcasting can be enabled */
2439         apic_smt_update();
2440 }
2441
2442 void __init arch_cpu_finalize_init(void)
2443 {
2444         identify_boot_cpu();
2445
2446         /*
2447          * identify_boot_cpu() initialized SMT support information, let the
2448          * core code know.
2449          */
2450         cpu_smt_check_topology();
2451
2452         if (!IS_ENABLED(CONFIG_SMP)) {
2453                 pr_info("CPU: ");
2454                 print_cpu_info(&boot_cpu_data);
2455         }
2456
2457         cpu_select_mitigations();
2458
2459         arch_smt_update();
2460
2461         if (IS_ENABLED(CONFIG_X86_32)) {
2462                 /*
2463                  * Check whether this is a real i386 which is not longer
2464                  * supported and fixup the utsname.
2465                  */
2466                 if (boot_cpu_data.x86 < 4)
2467                         panic("Kernel requires i486+ for 'invlpg' and other features");
2468
2469                 init_utsname()->machine[1] =
2470                         '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
2471         }
2472
2473         /*
2474          * Must be before alternatives because it might set or clear
2475          * feature bits.
2476          */
2477         fpu__init_system();
2478         fpu__init_cpu();
2479
2480         alternative_instructions();
2481
2482         if (IS_ENABLED(CONFIG_X86_64)) {
2483                 /*
2484                  * Make sure the first 2MB area is not mapped by huge pages
2485                  * There are typically fixed size MTRRs in there and overlapping
2486                  * MTRRs into large pages causes slow downs.
2487                  *
2488                  * Right now we don't do that with gbpages because there seems
2489                  * very little benefit for that case.
2490                  */
2491                 if (!direct_gbpages)
2492                         set_memory_4k((unsigned long)__va(0), 1);
2493         } else {
2494                 fpu__init_check_bugs();
2495         }
2496
2497         /*
2498          * This needs to be called before any devices perform DMA
2499          * operations that might use the SWIOTLB bounce buffers. It will
2500          * mark the bounce buffers as decrypted so that their usage will
2501          * not cause "plain-text" data to be decrypted when accessed. It
2502          * must be called after late_time_init() so that Hyper-V x86/x64
2503          * hypercalls work when the SWIOTLB bounce buffers are decrypted.
2504          */
2505         mem_encrypt_init();
2506 }