drivers/thermal/intel/therm_throt.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Thermal throttle event support code (such as syslog messaging and rate
   4  * limiting) that was factored out from x86_64 (mce_intel.c) and i386 (p4.c).
   5  *
   6  * This allows consistent reporting of CPU thermal throttle events.
   7  *
   8  * Maintains a counter in /sys that keeps track of the number of thermal
   9  * events, such that the user knows how bad the thermal problem might be
  10  * (since the logging to syslog is rate limited).
  11  *
  12  * Author: Dmitriy Zavin (dmitriyz@google.com)
  13  *
  14  * Credits: Adapted from Zwane Mwaikambo's original code in mce_intel.c.
  15  *          Inspired by Ross Biro's and Al Borchers' counter code.
  16  */
  17 #include <linux/interrupt.h>
  18 #include <linux/notifier.h>
  19 #include <linux/jiffies.h>
  20 #include <linux/kernel.h>
  21 #include <linux/percpu.h>
  22 #include <linux/export.h>
  23 #include <linux/types.h>
  24 #include <linux/init.h>
  25 #include <linux/smp.h>
  26 #include <linux/cpu.h>
  27
  28 #include <asm/processor.h>
  29 #include <asm/thermal.h>
  30 #include <asm/traps.h>
  31 #include <asm/apic.h>
  32 #include <asm/irq.h>
  33 #include <asm/msr.h>
  34
  35 #include "intel_hfi.h"
  36 #include "thermal_interrupt.h"
  37
  38 /* How long to wait between reporting thermal events */
  39 #define CHECK_INTERVAL          (300 * HZ)
  40
  41 #define THERMAL_THROTTLING_EVENT        0
  42 #define POWER_LIMIT_EVENT               1
  43
  44 /**
  45  * struct _thermal_state - Represent the current thermal event state
  46  * @next_check:                 Stores the next timestamp, when it is allowed
  47  *                              to log the next warning message.
  48  * @last_interrupt_time:        Stores the timestamp for the last threshold
  49  *                              high event.
  50  * @therm_work:                 Delayed workqueue structure
  51  * @count:                      Stores the current running count for thermal
  52  *                              or power threshold interrupts.
  53  * @last_count:                 Stores the previous running count for thermal
  54  *                              or power threshold interrupts.
  55  * @max_time_ms:                This shows the maximum amount of time CPU was
  56  *                              in throttled state for a single thermal
  57  *                              threshold high to low state.
  58  * @total_time_ms:              This is a cumulative time during which CPU was
  59  *                              in the throttled state.
  60  * @rate_control_active:        Set when a throttling message is logged.
  61  *                              This is used for the purpose of rate-control.
  62  * @new_event:                  Stores the last high/low status of the
  63  *                              THERM_STATUS_PROCHOT or
  64  *                              THERM_STATUS_POWER_LIMIT.
  65  * @level:                      Stores whether this _thermal_state instance is
  66  *                              for a CORE level or for PACKAGE level.
  67  * @sample_index:               Index for storing the next sample in the buffer
  68  *                              temp_samples[].
  69  * @sample_count:               Total number of samples collected in the buffer
  70  *                              temp_samples[].
  71  * @average:                    The last moving average of temperature samples
  72  * @baseline_temp:              Temperature at which thermal threshold high
  73  *                              interrupt was generated.
  74  * @temp_samples:               Storage for temperature samples to calculate
  75  *                              moving average.
  76  *
  77  * This structure is used to represent data related to thermal state for a CPU.
  78  * There is a separate storage for core and package level for each CPU.
  79  */
  80 struct _thermal_state {
  81         u64                     next_check;
  82         u64                     last_interrupt_time;
  83         struct delayed_work     therm_work;
  84         unsigned long           count;
  85         unsigned long           last_count;
  86         unsigned long           max_time_ms;
  87         unsigned long           total_time_ms;
  88         bool                    rate_control_active;
  89         bool                    new_event;
  90         u8                      level;
  91         u8                      sample_index;
  92         u8                      sample_count;
  93         u8                      average;
  94         u8                      baseline_temp;
  95         u8                      temp_samples[3];
  96 };
  97
  98 struct thermal_state {
  99         struct _thermal_state core_throttle;
 100         struct _thermal_state core_power_limit;
 101         struct _thermal_state package_throttle;
 102         struct _thermal_state package_power_limit;
 103         struct _thermal_state core_thresh0;
 104         struct _thermal_state core_thresh1;
 105         struct _thermal_state pkg_thresh0;
 106         struct _thermal_state pkg_thresh1;
 107 };
 108
 109 /* Callback to handle core threshold interrupts */
 110 int (*platform_thermal_notify)(__u64 msr_val);
 111 EXPORT_SYMBOL(platform_thermal_notify);
 112
 113 /* Callback to handle core package threshold_interrupts */
 114 int (*platform_thermal_package_notify)(__u64 msr_val);
 115 EXPORT_SYMBOL_GPL(platform_thermal_package_notify);
 116
 117 /* Callback support of rate control, return true, if
 118  * callback has rate control */
 119 bool (*platform_thermal_package_rate_control)(void);
 120 EXPORT_SYMBOL_GPL(platform_thermal_package_rate_control);
 121
 122
 123 static DEFINE_PER_CPU(struct thermal_state, thermal_state);
 124
 125 static atomic_t therm_throt_en  = ATOMIC_INIT(0);
 126
 127 static u32 lvtthmr_init __read_mostly;
 128
 129 #ifdef CONFIG_SYSFS
 130 #define define_therm_throt_device_one_ro(_name)                         \
 131         static DEVICE_ATTR(_name, 0444,                                 \
 132                            therm_throt_device_show_##_name,             \
 133                                    NULL)                                \
 134
 135 #define define_therm_throt_device_show_func(event, name)                \
 136                                                                         \
 137 static ssize_t therm_throt_device_show_##event##_##name(                \
 138                         struct device *dev,                             \
 139                         struct device_attribute *attr,                  \
 140                         char *buf)                                      \
 141 {                                                                       \
 142         unsigned int cpu = dev->id;                                     \
 143         ssize_t ret;                                                    \
 144                                                                         \
 145         preempt_disable();      /* CPU hotplug */                       \
 146         if (cpu_online(cpu)) {                                          \
 147                 ret = sprintf(buf, "%lu\n",                             \
 148                               per_cpu(thermal_state, cpu).event.name);  \
 149         } else                                                          \
 150                 ret = 0;                                                \
 151         preempt_enable();                                               \
 152                                                                         \
 153         return ret;                                                     \
 154 }
 155
 156 define_therm_throt_device_show_func(core_throttle, count);
 157 define_therm_throt_device_one_ro(core_throttle_count);
 158
 159 define_therm_throt_device_show_func(core_power_limit, count);
 160 define_therm_throt_device_one_ro(core_power_limit_count);
 161
 162 define_therm_throt_device_show_func(package_throttle, count);
 163 define_therm_throt_device_one_ro(package_throttle_count);
 164
 165 define_therm_throt_device_show_func(package_power_limit, count);
 166 define_therm_throt_device_one_ro(package_power_limit_count);
 167
 168 define_therm_throt_device_show_func(core_throttle, max_time_ms);
 169 define_therm_throt_device_one_ro(core_throttle_max_time_ms);
 170
 171 define_therm_throt_device_show_func(package_throttle, max_time_ms);
 172 define_therm_throt_device_one_ro(package_throttle_max_time_ms);
 173
 174 define_therm_throt_device_show_func(core_throttle, total_time_ms);
 175 define_therm_throt_device_one_ro(core_throttle_total_time_ms);
 176
 177 define_therm_throt_device_show_func(package_throttle, total_time_ms);
 178 define_therm_throt_device_one_ro(package_throttle_total_time_ms);
 179
 180 static struct attribute *thermal_throttle_attrs[] = {
 181         &dev_attr_core_throttle_count.attr,
 182         &dev_attr_core_throttle_max_time_ms.attr,
 183         &dev_attr_core_throttle_total_time_ms.attr,
 184         NULL
 185 };
 186
 187 static const struct attribute_group thermal_attr_group = {
 188         .attrs  = thermal_throttle_attrs,
 189         .name   = "thermal_throttle"
 190 };
 191 #endif /* CONFIG_SYSFS */
 192
 193 #define CORE_LEVEL      0
 194 #define PACKAGE_LEVEL   1
 195
 196 #define THERM_THROT_POLL_INTERVAL       HZ
 197 #define THERM_STATUS_PROCHOT_LOG        BIT(1)
 198
 199 #define THERM_STATUS_CLEAR_CORE_MASK (BIT(1) | BIT(3) | BIT(5) | BIT(7) | BIT(9) | BIT(11) | BIT(13) | BIT(15))
 200 #define THERM_STATUS_CLEAR_PKG_MASK  (BIT(1) | BIT(3) | BIT(5) | BIT(7) | BIT(9) | BIT(11))
 201
 202 static void clear_therm_status_log(int level)
 203 {
 204         int msr;
 205         u64 mask, msr_val;
 206
 207         if (level == CORE_LEVEL) {
 208                 msr  = MSR_IA32_THERM_STATUS;
 209                 mask = THERM_STATUS_CLEAR_CORE_MASK;
 210         } else {
 211                 msr  = MSR_IA32_PACKAGE_THERM_STATUS;
 212                 mask = THERM_STATUS_CLEAR_PKG_MASK;
 213         }
 214
 215         rdmsrl(msr, msr_val);
 216         msr_val &= mask;
 217         wrmsrl(msr, msr_val & ~THERM_STATUS_PROCHOT_LOG);
 218 }
 219
 220 static void get_therm_status(int level, bool *proc_hot, u8 *temp)
 221 {
 222         int msr;
 223         u64 msr_val;
 224
 225         if (level == CORE_LEVEL)
 226                 msr = MSR_IA32_THERM_STATUS;
 227         else
 228                 msr = MSR_IA32_PACKAGE_THERM_STATUS;
 229
 230         rdmsrl(msr, msr_val);
 231         if (msr_val & THERM_STATUS_PROCHOT_LOG)
 232                 *proc_hot = true;
 233         else
 234                 *proc_hot = false;
 235
 236         *temp = (msr_val >> 16) & 0x7F;
 237 }
 238
 239 static void __maybe_unused throttle_active_work(struct work_struct *work)
 240 {
 241         struct _thermal_state *state = container_of(to_delayed_work(work),
 242                                                 struct _thermal_state, therm_work);
 243         unsigned int i, avg, this_cpu = smp_processor_id();
 244         u64 now = get_jiffies_64();
 245         bool hot;
 246         u8 temp;
 247
 248         get_therm_status(state->level, &hot, &temp);
 249         /* temperature value is offset from the max so lesser means hotter */
 250         if (!hot && temp > state->baseline_temp) {
 251                 if (state->rate_control_active)
 252                         pr_info("CPU%d: %s temperature/speed normal (total events = %lu)\n",
 253                                 this_cpu,
 254                                 state->level == CORE_LEVEL ? "Core" : "Package",
 255                                 state->count);
 256
 257                 state->rate_control_active = false;
 258                 return;
 259         }
 260
 261         if (time_before64(now, state->next_check) &&
 262                           state->rate_control_active)
 263                 goto re_arm;
 264
 265         state->next_check = now + CHECK_INTERVAL;
 266
 267         if (state->count != state->last_count) {
 268                 /* There was one new thermal interrupt */
 269                 state->last_count = state->count;
 270                 state->average = 0;
 271                 state->sample_count = 0;
 272                 state->sample_index = 0;
 273         }
 274
 275         state->temp_samples[state->sample_index] = temp;
 276         state->sample_count++;
 277         state->sample_index = (state->sample_index + 1) % ARRAY_SIZE(state->temp_samples);
 278         if (state->sample_count < ARRAY_SIZE(state->temp_samples))
 279                 goto re_arm;
 280
 281         avg = 0;
 282         for (i = 0; i < ARRAY_SIZE(state->temp_samples); ++i)
 283                 avg += state->temp_samples[i];
 284
 285         avg /= ARRAY_SIZE(state->temp_samples);
 286
 287         if (state->average > avg) {
 288                 pr_warn("CPU%d: %s temperature is above threshold, cpu clock is throttled (total events = %lu)\n",
 289                         this_cpu,
 290                         state->level == CORE_LEVEL ? "Core" : "Package",
 291                         state->count);
 292                 state->rate_control_active = true;
 293         }
 294
 295         state->average = avg;
 296
 297 re_arm:
 298         clear_therm_status_log(state->level);
 299         schedule_delayed_work_on(this_cpu, &state->therm_work, THERM_THROT_POLL_INTERVAL);
 300 }
 301
 302 /***
 303  * therm_throt_process - Process thermal throttling event from interrupt
 304  * @curr: Whether the condition is current or not (boolean), since the
 305  *        thermal interrupt normally gets called both when the thermal
 306  *        event begins and once the event has ended.
 307  *
 308  * This function is called by the thermal interrupt after the
 309  * IRQ has been acknowledged.
 310  *
 311  * It will take care of rate limiting and printing messages to the syslog.
 312  */
 313 static void therm_throt_process(bool new_event, int event, int level)
 314 {
 315         struct _thermal_state *state;
 316         unsigned int this_cpu = smp_processor_id();
 317         bool old_event;
 318         u64 now;
 319         struct thermal_state *pstate = &per_cpu(thermal_state, this_cpu);
 320
 321         now = get_jiffies_64();
 322         if (level == CORE_LEVEL) {
 323                 if (event == THERMAL_THROTTLING_EVENT)
 324                         state = &pstate->core_throttle;
 325                 else if (event == POWER_LIMIT_EVENT)
 326                         state = &pstate->core_power_limit;
 327                 else
 328                         return;
 329         } else if (level == PACKAGE_LEVEL) {
 330                 if (event == THERMAL_THROTTLING_EVENT)
 331                         state = &pstate->package_throttle;
 332                 else if (event == POWER_LIMIT_EVENT)
 333                         state = &pstate->package_power_limit;
 334                 else
 335                         return;
 336         } else
 337                 return;
 338
 339         old_event = state->new_event;
 340         state->new_event = new_event;
 341
 342         if (new_event)
 343                 state->count++;
 344
 345         if (event != THERMAL_THROTTLING_EVENT)
 346                 return;
 347
 348         if (new_event && !state->last_interrupt_time) {
 349                 bool hot;
 350                 u8 temp;
 351
 352                 get_therm_status(state->level, &hot, &temp);
 353                 /*
 354                  * Ignore short temperature spike as the system is not close
 355                  * to PROCHOT. 10C offset is large enough to ignore. It is
 356                  * already dropped from the high threshold temperature.
 357                  */
 358                 if (temp > 10)
 359                         return;
 360
 361                 state->baseline_temp = temp;
 362                 state->last_interrupt_time = now;
 363                 schedule_delayed_work_on(this_cpu, &state->therm_work, THERM_THROT_POLL_INTERVAL);
 364         } else if (old_event && state->last_interrupt_time) {
 365                 unsigned long throttle_time;
 366
 367                 throttle_time = jiffies_delta_to_msecs(now - state->last_interrupt_time);
 368                 if (throttle_time > state->max_time_ms)
 369                         state->max_time_ms = throttle_time;
 370                 state->total_time_ms += throttle_time;
 371                 state->last_interrupt_time = 0;
 372         }
 373 }
 374
 375 static int thresh_event_valid(int level, int event)
 376 {
 377         struct _thermal_state *state;
 378         unsigned int this_cpu = smp_processor_id();
 379         struct thermal_state *pstate = &per_cpu(thermal_state, this_cpu);
 380         u64 now = get_jiffies_64();
 381
 382         if (level == PACKAGE_LEVEL)
 383                 state = (event == 0) ? &pstate->pkg_thresh0 :
 384                                                 &pstate->pkg_thresh1;
 385         else
 386                 state = (event == 0) ? &pstate->core_thresh0 :
 387                                                 &pstate->core_thresh1;
 388
 389         if (time_before64(now, state->next_check))
 390                 return 0;
 391
 392         state->next_check = now + CHECK_INTERVAL;
 393
 394         return 1;
 395 }
 396
 397 static bool int_pln_enable;
 398 static int __init int_pln_enable_setup(char *s)
 399 {
 400         int_pln_enable = true;
 401
 402         return 1;
 403 }
 404 __setup("int_pln_enable", int_pln_enable_setup);
 405
 406 #ifdef CONFIG_SYSFS
 407 /* Add/Remove thermal_throttle interface for CPU device: */
 408 static int thermal_throttle_add_dev(struct device *dev, unsigned int cpu)
 409 {
 410         int err;
 411         struct cpuinfo_x86 *c = &cpu_data(cpu);
 412
 413         err = sysfs_create_group(&dev->kobj, &thermal_attr_group);
 414         if (err)
 415                 return err;
 416
 417         if (cpu_has(c, X86_FEATURE_PLN) && int_pln_enable) {
 418                 err = sysfs_add_file_to_group(&dev->kobj,
 419                                               &dev_attr_core_power_limit_count.attr,
 420                                               thermal_attr_group.name);
 421                 if (err)
 422                         goto del_group;
 423         }
 424
 425         if (cpu_has(c, X86_FEATURE_PTS)) {
 426                 err = sysfs_add_file_to_group(&dev->kobj,
 427                                               &dev_attr_package_throttle_count.attr,
 428                                               thermal_attr_group.name);
 429                 if (err)
 430                         goto del_group;
 431
 432                 err = sysfs_add_file_to_group(&dev->kobj,
 433                                               &dev_attr_package_throttle_max_time_ms.attr,
 434                                               thermal_attr_group.name);
 435                 if (err)
 436                         goto del_group;
 437
 438                 err = sysfs_add_file_to_group(&dev->kobj,
 439                                               &dev_attr_package_throttle_total_time_ms.attr,
 440                                               thermal_attr_group.name);
 441                 if (err)
 442                         goto del_group;
 443
 444                 if (cpu_has(c, X86_FEATURE_PLN) && int_pln_enable) {
 445                         err = sysfs_add_file_to_group(&dev->kobj,
 446                                         &dev_attr_package_power_limit_count.attr,
 447                                         thermal_attr_group.name);
 448                         if (err)
 449                                 goto del_group;
 450                 }
 451         }
 452
 453         return 0;
 454
 455 del_group:
 456         sysfs_remove_group(&dev->kobj, &thermal_attr_group);
 457
 458         return err;
 459 }
 460
 461 static void thermal_throttle_remove_dev(struct device *dev)
 462 {
 463         sysfs_remove_group(&dev->kobj, &thermal_attr_group);
 464 }
 465
 466 /* Get notified when a cpu comes on/off. Be hotplug friendly. */
 467 static int thermal_throttle_online(unsigned int cpu)
 468 {
 469         struct thermal_state *state = &per_cpu(thermal_state, cpu);
 470         struct device *dev = get_cpu_device(cpu);
 471         u32 l;
 472
 473         state->package_throttle.level = PACKAGE_LEVEL;
 474         state->core_throttle.level = CORE_LEVEL;
 475
 476         INIT_DELAYED_WORK(&state->package_throttle.therm_work, throttle_active_work);
 477         INIT_DELAYED_WORK(&state->core_throttle.therm_work, throttle_active_work);
 478
 479         /*
 480          * The first CPU coming online will enable the HFI. Usually this causes
 481          * hardware to issue an HFI thermal interrupt. Such interrupt will reach
 482          * the CPU once we enable the thermal vector in the local APIC.
 483          */
 484         intel_hfi_online(cpu);
 485
 486         /* Unmask the thermal vector after the above workqueues are initialized. */
 487         l = apic_read(APIC_LVTTHMR);
 488         apic_write(APIC_LVTTHMR, l & ~APIC_LVT_MASKED);
 489
 490         return thermal_throttle_add_dev(dev, cpu);
 491 }
 492
 493 static int thermal_throttle_offline(unsigned int cpu)
 494 {
 495         struct thermal_state *state = &per_cpu(thermal_state, cpu);
 496         struct device *dev = get_cpu_device(cpu);
 497         u32 l;
 498
 499         /* Mask the thermal vector before draining evtl. pending work */
 500         l = apic_read(APIC_LVTTHMR);
 501         apic_write(APIC_LVTTHMR, l | APIC_LVT_MASKED);
 502
 503         intel_hfi_offline(cpu);
 504
 505         cancel_delayed_work_sync(&state->package_throttle.therm_work);
 506         cancel_delayed_work_sync(&state->core_throttle.therm_work);
 507
 508         state->package_throttle.rate_control_active = false;
 509         state->core_throttle.rate_control_active = false;
 510
 511         thermal_throttle_remove_dev(dev);
 512         return 0;
 513 }
 514
 515 static __init int thermal_throttle_init_device(void)
 516 {
 517         int ret;
 518
 519         if (!atomic_read(&therm_throt_en))
 520                 return 0;
 521
 522         intel_hfi_init();
 523
 524         ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/therm:online",
 525                                 thermal_throttle_online,
 526                                 thermal_throttle_offline);
 527         return ret < 0 ? ret : 0;
 528 }
 529 device_initcall(thermal_throttle_init_device);
 530
 531 #endif /* CONFIG_SYSFS */
 532
 533 static void notify_package_thresholds(__u64 msr_val)
 534 {
 535         bool notify_thres_0 = false;
 536         bool notify_thres_1 = false;
 537
 538         if (!platform_thermal_package_notify)
 539                 return;
 540
 541         /* lower threshold check */
 542         if (msr_val & THERM_LOG_THRESHOLD0)
 543                 notify_thres_0 = true;
 544         /* higher threshold check */
 545         if (msr_val & THERM_LOG_THRESHOLD1)
 546                 notify_thres_1 = true;
 547
 548         if (!notify_thres_0 && !notify_thres_1)
 549                 return;
 550
 551         if (platform_thermal_package_rate_control &&
 552                 platform_thermal_package_rate_control()) {
 553                 /* Rate control is implemented in callback */
 554                 platform_thermal_package_notify(msr_val);
 555                 return;
 556         }
 557
 558         /* lower threshold reached */
 559         if (notify_thres_0 && thresh_event_valid(PACKAGE_LEVEL, 0))
 560                 platform_thermal_package_notify(msr_val);
 561         /* higher threshold reached */
 562         if (notify_thres_1 && thresh_event_valid(PACKAGE_LEVEL, 1))
 563                 platform_thermal_package_notify(msr_val);
 564 }
 565
 566 static void notify_thresholds(__u64 msr_val)
 567 {
 568         /* check whether the interrupt handler is defined;
 569          * otherwise simply return
 570          */
 571         if (!platform_thermal_notify)
 572                 return;
 573
 574         /* lower threshold reached */
 575         if ((msr_val & THERM_LOG_THRESHOLD0) &&
 576                         thresh_event_valid(CORE_LEVEL, 0))
 577                 platform_thermal_notify(msr_val);
 578         /* higher threshold reached */
 579         if ((msr_val & THERM_LOG_THRESHOLD1) &&
 580                         thresh_event_valid(CORE_LEVEL, 1))
 581                 platform_thermal_notify(msr_val);
 582 }
 583
 584 void __weak notify_hwp_interrupt(void)
 585 {
 586         wrmsrl_safe(MSR_HWP_STATUS, 0);
 587 }
 588
 589 /* Thermal transition interrupt handler */
 590 void intel_thermal_interrupt(void)
 591 {
 592         __u64 msr_val;
 593
 594         if (static_cpu_has(X86_FEATURE_HWP))
 595                 notify_hwp_interrupt();
 596
 597         rdmsrl(MSR_IA32_THERM_STATUS, msr_val);
 598
 599         /* Check for violation of core thermal thresholds*/
 600         notify_thresholds(msr_val);
 601
 602         therm_throt_process(msr_val & THERM_STATUS_PROCHOT,
 603                             THERMAL_THROTTLING_EVENT,
 604                             CORE_LEVEL);
 605
 606         if (this_cpu_has(X86_FEATURE_PLN) && int_pln_enable)
 607                 therm_throt_process(msr_val & THERM_STATUS_POWER_LIMIT,
 608                                         POWER_LIMIT_EVENT,
 609                                         CORE_LEVEL);
 610
 611         if (this_cpu_has(X86_FEATURE_PTS)) {
 612                 rdmsrl(MSR_IA32_PACKAGE_THERM_STATUS, msr_val);
 613                 /* check violations of package thermal thresholds */
 614                 notify_package_thresholds(msr_val);
 615                 therm_throt_process(msr_val & PACKAGE_THERM_STATUS_PROCHOT,
 616                                         THERMAL_THROTTLING_EVENT,
 617                                         PACKAGE_LEVEL);
 618                 if (this_cpu_has(X86_FEATURE_PLN) && int_pln_enable)
 619                         therm_throt_process(msr_val &
 620                                         PACKAGE_THERM_STATUS_POWER_LIMIT,
 621                                         POWER_LIMIT_EVENT,
 622                                         PACKAGE_LEVEL);
 623
 624                 if (this_cpu_has(X86_FEATURE_HFI))
 625                         intel_hfi_process_event(msr_val &
 626                                                 PACKAGE_THERM_STATUS_HFI_UPDATED);
 627         }
 628 }
 629
 630 /* Thermal monitoring depends on APIC, ACPI and clock modulation */
 631 static int intel_thermal_supported(struct cpuinfo_x86 *c)
 632 {
 633         if (!boot_cpu_has(X86_FEATURE_APIC))
 634                 return 0;
 635         if (!cpu_has(c, X86_FEATURE_ACPI) || !cpu_has(c, X86_FEATURE_ACC))
 636                 return 0;
 637         return 1;
 638 }
 639
 640 bool x86_thermal_enabled(void)
 641 {
 642         return atomic_read(&therm_throt_en);
 643 }
 644
 645 void __init therm_lvt_init(void)
 646 {
 647         /*
 648          * This function is only called on boot CPU. Save the init thermal
 649          * LVT value on BSP and use that value to restore APs' thermal LVT
 650          * entry BIOS programmed later
 651          */
 652         if (intel_thermal_supported(&boot_cpu_data))
 653                 lvtthmr_init = apic_read(APIC_LVTTHMR);
 654 }
 655
 656 void intel_init_thermal(struct cpuinfo_x86 *c)
 657 {
 658         unsigned int cpu = smp_processor_id();
 659         int tm2 = 0;
 660         u32 l, h;
 661
 662         if (!intel_thermal_supported(c))
 663                 return;
 664
 665         /*
 666          * First check if its enabled already, in which case there might
 667          * be some SMM goo which handles it, so we can't even put a handler
 668          * since it might be delivered via SMI already:
 669          */
 670         rdmsr(MSR_IA32_MISC_ENABLE, l, h);
 671
 672         h = lvtthmr_init;
 673         /*
 674          * The initial value of thermal LVT entries on all APs always reads
 675          * 0x10000 because APs are woken up by BSP issuing INIT-SIPI-SIPI
 676          * sequence to them and LVT registers are reset to 0s except for
 677          * the mask bits which are set to 1s when APs receive INIT IPI.
 678          * If BIOS takes over the thermal interrupt and sets its interrupt
 679          * delivery mode to SMI (not fixed), it restores the value that the
 680          * BIOS has programmed on AP based on BSP's info we saved since BIOS
 681          * is always setting the same value for all threads/cores.
 682          */
 683         if ((h & APIC_DM_FIXED_MASK) != APIC_DM_FIXED)
 684                 apic_write(APIC_LVTTHMR, lvtthmr_init);
 685
 686
 687         if ((l & MSR_IA32_MISC_ENABLE_TM1) && (h & APIC_DM_SMI)) {
 688                 if (system_state == SYSTEM_BOOTING)
 689                         pr_debug("CPU%d: Thermal monitoring handled by SMI\n", cpu);
 690                 return;
 691         }
 692
 693         /* early Pentium M models use different method for enabling TM2 */
 694         if (cpu_has(c, X86_FEATURE_TM2)) {
 695                 if (c->x86 == 6 && (c->x86_model == 9 || c->x86_model == 13)) {
 696                         rdmsr(MSR_THERM2_CTL, l, h);
 697                         if (l & MSR_THERM2_CTL_TM_SELECT)
 698                                 tm2 = 1;
 699                 } else if (l & MSR_IA32_MISC_ENABLE_TM2)
 700                         tm2 = 1;
 701         }
 702
 703         /* We'll mask the thermal vector in the lapic till we're ready: */
 704         h = THERMAL_APIC_VECTOR | APIC_DM_FIXED | APIC_LVT_MASKED;
 705         apic_write(APIC_LVTTHMR, h);
 706
 707         rdmsr(MSR_IA32_THERM_INTERRUPT, l, h);
 708         if (cpu_has(c, X86_FEATURE_PLN) && !int_pln_enable)
 709                 wrmsr(MSR_IA32_THERM_INTERRUPT,
 710                         (l | (THERM_INT_LOW_ENABLE
 711                         | THERM_INT_HIGH_ENABLE)) & ~THERM_INT_PLN_ENABLE, h);
 712         else if (cpu_has(c, X86_FEATURE_PLN) && int_pln_enable)
 713                 wrmsr(MSR_IA32_THERM_INTERRUPT,
 714                         l | (THERM_INT_LOW_ENABLE
 715                         | THERM_INT_HIGH_ENABLE | THERM_INT_PLN_ENABLE), h);
 716         else
 717                 wrmsr(MSR_IA32_THERM_INTERRUPT,
 718                       l | (THERM_INT_LOW_ENABLE | THERM_INT_HIGH_ENABLE), h);
 719
 720         if (cpu_has(c, X86_FEATURE_PTS)) {
 721                 rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
 722                 if (cpu_has(c, X86_FEATURE_PLN) && !int_pln_enable)
 723                         wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
 724                                 (l | (PACKAGE_THERM_INT_LOW_ENABLE
 725                                 | PACKAGE_THERM_INT_HIGH_ENABLE))
 726                                 & ~PACKAGE_THERM_INT_PLN_ENABLE, h);
 727                 else if (cpu_has(c, X86_FEATURE_PLN) && int_pln_enable)
 728                         wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
 729                                 l | (PACKAGE_THERM_INT_LOW_ENABLE
 730                                 | PACKAGE_THERM_INT_HIGH_ENABLE
 731                                 | PACKAGE_THERM_INT_PLN_ENABLE), h);
 732                 else
 733                         wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
 734                               l | (PACKAGE_THERM_INT_LOW_ENABLE
 735                                 | PACKAGE_THERM_INT_HIGH_ENABLE), h);
 736
 737                 if (cpu_has(c, X86_FEATURE_HFI)) {
 738                         rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
 739                         wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
 740                               l | PACKAGE_THERM_INT_HFI_ENABLE, h);
 741                 }
 742         }
 743
 744         rdmsr(MSR_IA32_MISC_ENABLE, l, h);
 745         wrmsr(MSR_IA32_MISC_ENABLE, l | MSR_IA32_MISC_ENABLE_TM1, h);
 746
 747         pr_info_once("CPU0: Thermal monitoring enabled (%s)\n",
 748                       tm2 ? "TM2" : "TM1");
 749
 750         /* enable thermal throttle processing */
 751         atomic_set(&therm_throt_en, 1);
 752 }