kernel/cpu.c

   1 /* CPU control.
   2  * (C) 2001, 2002, 2003, 2004 Rusty Russell
   3  *
   4  * This code is licenced under the GPL.
   5  */
   6 #include <linux/proc_fs.h>
   7 #include <linux/smp.h>
   8 #include <linux/init.h>
   9 #include <linux/notifier.h>
  10 #include <linux/sched/signal.h>
  11 #include <linux/sched/hotplug.h>
  12 #include <linux/sched/task.h>
  13 #include <linux/sched/smt.h>
  14 #include <linux/unistd.h>
  15 #include <linux/cpu.h>
  16 #include <linux/oom.h>
  17 #include <linux/rcupdate.h>
  18 #include <linux/export.h>
  19 #include <linux/bug.h>
  20 #include <linux/kthread.h>
  21 #include <linux/stop_machine.h>
  22 #include <linux/mutex.h>
  23 #include <linux/gfp.h>
  24 #include <linux/suspend.h>
  25 #include <linux/lockdep.h>
  26 #include <linux/tick.h>
  27 #include <linux/irq.h>
  28 #include <linux/nmi.h>
  29 #include <linux/smpboot.h>
  30 #include <linux/relay.h>
  31 #include <linux/slab.h>
  32 #include <linux/percpu-rwsem.h>
  33 #include <linux/cpuset.h>
  34
  35 #include <trace/events/power.h>
  36 #define CREATE_TRACE_POINTS
  37 #include <trace/events/cpuhp.h>
  38
  39 #include "smpboot.h"
  40
  41 /**
  42  * cpuhp_cpu_state - Per cpu hotplug state storage
  43  * @state:      The current cpu state
  44  * @target:     The target state
  45  * @thread:     Pointer to the hotplug thread
  46  * @should_run: Thread should execute
  47  * @rollback:   Perform a rollback
  48  * @single:     Single callback invocation
  49  * @bringup:    Single callback bringup or teardown selector
  50  * @cb_state:   The state for a single callback (install/uninstall)
  51  * @result:     Result of the operation
  52  * @done_up:    Signal completion to the issuer of the task for cpu-up
  53  * @done_down:  Signal completion to the issuer of the task for cpu-down
  54  */
  55 struct cpuhp_cpu_state {
  56         enum cpuhp_state        state;
  57         enum cpuhp_state        target;
  58         enum cpuhp_state        fail;
  59 #ifdef CONFIG_SMP
  60         struct task_struct      *thread;
  61         bool                    should_run;
  62         bool                    rollback;
  63         bool                    single;
  64         bool                    bringup;
  65         bool                    booted_once;
  66         struct hlist_node       *node;
  67         struct hlist_node       *last;
  68         enum cpuhp_state        cb_state;
  69         int                     result;
  70         struct completion       done_up;
  71         struct completion       done_down;
  72 #endif
  73 };
  74
  75 static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
  76         .fail = CPUHP_INVALID,
  77 };
  78
  79 #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
  80 static struct lockdep_map cpuhp_state_up_map =
  81         STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
  82 static struct lockdep_map cpuhp_state_down_map =
  83         STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
  84
  85
  86 static void inline cpuhp_lock_acquire(bool bringup)
  87 {
  88         lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
  89 }
  90
  91 static void inline cpuhp_lock_release(bool bringup)
  92 {
  93         lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
  94 }
  95 #else
  96
  97 static void inline cpuhp_lock_acquire(bool bringup) { }
  98 static void inline cpuhp_lock_release(bool bringup) { }
  99
 100 #endif
 101
 102 /**
 103  * cpuhp_step - Hotplug state machine step
 104  * @name:       Name of the step
 105  * @startup:    Startup function of the step
 106  * @teardown:   Teardown function of the step
 107  * @skip_onerr: Do not invoke the functions on error rollback
 108  *              Will go away once the notifiers are gone
 109  * @cant_stop:  Bringup/teardown can't be stopped at this step
 110  */
 111 struct cpuhp_step {
 112         const char              *name;
 113         union {
 114                 int             (*single)(unsigned int cpu);
 115                 int             (*multi)(unsigned int cpu,
 116                                          struct hlist_node *node);
 117         } startup;
 118         union {
 119                 int             (*single)(unsigned int cpu);
 120                 int             (*multi)(unsigned int cpu,
 121                                          struct hlist_node *node);
 122         } teardown;
 123         struct hlist_head       list;
 124         bool                    skip_onerr;
 125         bool                    cant_stop;
 126         bool                    multi_instance;
 127 };
 128
 129 static DEFINE_MUTEX(cpuhp_state_mutex);
 130 static struct cpuhp_step cpuhp_bp_states[];
 131 static struct cpuhp_step cpuhp_ap_states[];
 132
 133 static bool cpuhp_is_ap_state(enum cpuhp_state state)
 134 {
 135         /*
 136          * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
 137          * purposes as that state is handled explicitly in cpu_down.
 138          */
 139         return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
 140 }
 141
 142 static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
 143 {
 144         struct cpuhp_step *sp;
 145
 146         sp = cpuhp_is_ap_state(state) ? cpuhp_ap_states : cpuhp_bp_states;
 147         return sp + state;
 148 }
 149
 150 /**
 151  * cpuhp_invoke_callback _ Invoke the callbacks for a given state
 152  * @cpu:        The cpu for which the callback should be invoked
 153  * @state:      The state to do callbacks for
 154  * @bringup:    True if the bringup callback should be invoked
 155  * @node:       For multi-instance, do a single entry callback for install/remove
 156  * @lastp:      For multi-instance rollback, remember how far we got
 157  *
 158  * Called from cpu hotplug and from the state register machinery.
 159  */
 160 static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
 161                                  bool bringup, struct hlist_node *node,
 162                                  struct hlist_node **lastp)
 163 {
 164         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 165         struct cpuhp_step *step = cpuhp_get_step(state);
 166         int (*cbm)(unsigned int cpu, struct hlist_node *node);
 167         int (*cb)(unsigned int cpu);
 168         int ret, cnt;
 169
 170         if (st->fail == state) {
 171                 st->fail = CPUHP_INVALID;
 172
 173                 if (!(bringup ? step->startup.single : step->teardown.single))
 174                         return 0;
 175
 176                 return -EAGAIN;
 177         }
 178
 179         if (!step->multi_instance) {
 180                 WARN_ON_ONCE(lastp && *lastp);
 181                 cb = bringup ? step->startup.single : step->teardown.single;
 182                 if (!cb)
 183                         return 0;
 184                 trace_cpuhp_enter(cpu, st->target, state, cb);
 185                 ret = cb(cpu);
 186                 trace_cpuhp_exit(cpu, st->state, state, ret);
 187                 return ret;
 188         }
 189         cbm = bringup ? step->startup.multi : step->teardown.multi;
 190         if (!cbm)
 191                 return 0;
 192
 193         /* Single invocation for instance add/remove */
 194         if (node) {
 195                 WARN_ON_ONCE(lastp && *lastp);
 196                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 197                 ret = cbm(cpu, node);
 198                 trace_cpuhp_exit(cpu, st->state, state, ret);
 199                 return ret;
 200         }
 201
 202         /* State transition. Invoke on all instances */
 203         cnt = 0;
 204         hlist_for_each(node, &step->list) {
 205                 if (lastp && node == *lastp)
 206                         break;
 207
 208                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 209                 ret = cbm(cpu, node);
 210                 trace_cpuhp_exit(cpu, st->state, state, ret);
 211                 if (ret) {
 212                         if (!lastp)
 213                                 goto err;
 214
 215                         *lastp = node;
 216                         return ret;
 217                 }
 218                 cnt++;
 219         }
 220         if (lastp)
 221                 *lastp = NULL;
 222         return 0;
 223 err:
 224         /* Rollback the instances if one failed */
 225         cbm = !bringup ? step->startup.multi : step->teardown.multi;
 226         if (!cbm)
 227                 return ret;
 228
 229         hlist_for_each(node, &step->list) {
 230                 if (!cnt--)
 231                         break;
 232
 233                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 234                 ret = cbm(cpu, node);
 235                 trace_cpuhp_exit(cpu, st->state, state, ret);
 236                 /*
 237                  * Rollback must not fail,
 238                  */
 239                 WARN_ON_ONCE(ret);
 240         }
 241         return ret;
 242 }
 243
 244 #ifdef CONFIG_SMP
 245 static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
 246 {
 247         struct completion *done = bringup ? &st->done_up : &st->done_down;
 248         wait_for_completion(done);
 249 }
 250
 251 static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
 252 {
 253         struct completion *done = bringup ? &st->done_up : &st->done_down;
 254         complete(done);
 255 }
 256
 257 /*
 258  * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
 259  */
 260 static bool cpuhp_is_atomic_state(enum cpuhp_state state)
 261 {
 262         return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
 263 }
 264
 265 /* Serializes the updates to cpu_online_mask, cpu_present_mask */
 266 static DEFINE_MUTEX(cpu_add_remove_lock);
 267 bool cpuhp_tasks_frozen;
 268 EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
 269
 270 /*
 271  * The following two APIs (cpu_maps_update_begin/done) must be used when
 272  * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
 273  */
 274 void cpu_maps_update_begin(void)
 275 {
 276         mutex_lock(&cpu_add_remove_lock);
 277 }
 278
 279 void cpu_maps_update_done(void)
 280 {
 281         mutex_unlock(&cpu_add_remove_lock);
 282 }
 283
 284 /*
 285  * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
 286  * Should always be manipulated under cpu_add_remove_lock
 287  */
 288 static int cpu_hotplug_disabled;
 289
 290 #ifdef CONFIG_HOTPLUG_CPU
 291
 292 DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
 293
 294 void cpus_read_lock(void)
 295 {
 296         percpu_down_read(&cpu_hotplug_lock);
 297 }
 298 EXPORT_SYMBOL_GPL(cpus_read_lock);
 299
 300 void cpus_read_unlock(void)
 301 {
 302         percpu_up_read(&cpu_hotplug_lock);
 303 }
 304 EXPORT_SYMBOL_GPL(cpus_read_unlock);
 305
 306 void cpus_write_lock(void)
 307 {
 308         percpu_down_write(&cpu_hotplug_lock);
 309 }
 310
 311 void cpus_write_unlock(void)
 312 {
 313         percpu_up_write(&cpu_hotplug_lock);
 314 }
 315
 316 void lockdep_assert_cpus_held(void)
 317 {
 318         /*
 319          * We can't have hotplug operations before userspace starts running,
 320          * and some init codepaths will knowingly not take the hotplug lock.
 321          * This is all valid, so mute lockdep until it makes sense to report
 322          * unheld locks.
 323          */
 324         if (system_state < SYSTEM_RUNNING)
 325                 return;
 326
 327         percpu_rwsem_assert_held(&cpu_hotplug_lock);
 328 }
 329
 330 /*
 331  * Wait for currently running CPU hotplug operations to complete (if any) and
 332  * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
 333  * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
 334  * hotplug path before performing hotplug operations. So acquiring that lock
 335  * guarantees mutual exclusion from any currently running hotplug operations.
 336  */
 337 void cpu_hotplug_disable(void)
 338 {
 339         cpu_maps_update_begin();
 340         cpu_hotplug_disabled++;
 341         cpu_maps_update_done();
 342 }
 343 EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
 344
 345 static void __cpu_hotplug_enable(void)
 346 {
 347         if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
 348                 return;
 349         cpu_hotplug_disabled--;
 350 }
 351
 352 void cpu_hotplug_enable(void)
 353 {
 354         cpu_maps_update_begin();
 355         __cpu_hotplug_enable();
 356         cpu_maps_update_done();
 357 }
 358 EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
 359 #endif  /* CONFIG_HOTPLUG_CPU */
 360
 361 /*
 362  * Architectures that need SMT-specific errata handling during SMT hotplug
 363  * should override this.
 364  */
 365 void __weak arch_smt_update(void) { }
 366
 367 #ifdef CONFIG_HOTPLUG_SMT
 368 enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;
 369
 370 void __init cpu_smt_disable(bool force)
 371 {
 372         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
 373                 cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
 374                 return;
 375
 376         if (force) {
 377                 pr_info("SMT: Force disabled\n");
 378                 cpu_smt_control = CPU_SMT_FORCE_DISABLED;
 379         } else {
 380                 pr_info("SMT: disabled\n");
 381                 cpu_smt_control = CPU_SMT_DISABLED;
 382         }
 383 }
 384
 385 /*
 386  * The decision whether SMT is supported can only be done after the full
 387  * CPU identification. Called from architecture code.
 388  */
 389 void __init cpu_smt_check_topology(void)
 390 {
 391         if (!topology_smt_supported())
 392                 cpu_smt_control = CPU_SMT_NOT_SUPPORTED;
 393 }
 394
 395 static int __init smt_cmdline_disable(char *str)
 396 {
 397         cpu_smt_disable(str && !strcmp(str, "force"));
 398         return 0;
 399 }
 400 early_param("nosmt", smt_cmdline_disable);
 401
 402 static inline bool cpu_smt_allowed(unsigned int cpu)
 403 {
 404         if (cpu_smt_control == CPU_SMT_ENABLED)
 405                 return true;
 406
 407         if (topology_is_primary_thread(cpu))
 408                 return true;
 409
 410         /*
 411          * On x86 it's required to boot all logical CPUs at least once so
 412          * that the init code can get a chance to set CR4.MCE on each
 413          * CPU. Otherwise, a broadacasted MCE observing CR4.MCE=0b on any
 414          * core will shutdown the machine.
 415          */
 416         return !per_cpu(cpuhp_state, cpu).booted_once;
 417 }
 418 #else
 419 static inline bool cpu_smt_allowed(unsigned int cpu) { return true; }
 420 #endif
 421
 422 static inline enum cpuhp_state
 423 cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target)
 424 {
 425         enum cpuhp_state prev_state = st->state;
 426
 427         st->rollback = false;
 428         st->last = NULL;
 429
 430         st->target = target;
 431         st->single = false;
 432         st->bringup = st->state < target;
 433
 434         return prev_state;
 435 }
 436
 437 static inline void
 438 cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state)
 439 {
 440         st->rollback = true;
 441
 442         /*
 443          * If we have st->last we need to undo partial multi_instance of this
 444          * state first. Otherwise start undo at the previous state.
 445          */
 446         if (!st->last) {
 447                 if (st->bringup)
 448                         st->state--;
 449                 else
 450                         st->state++;
 451         }
 452
 453         st->target = prev_state;
 454         st->bringup = !st->bringup;
 455 }
 456
 457 /* Regular hotplug invocation of the AP hotplug thread */
 458 static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
 459 {
 460         if (!st->single && st->state == st->target)
 461                 return;
 462
 463         st->result = 0;
 464         /*
 465          * Make sure the above stores are visible before should_run becomes
 466          * true. Paired with the mb() above in cpuhp_thread_fun()
 467          */
 468         smp_mb();
 469         st->should_run = true;
 470         wake_up_process(st->thread);
 471         wait_for_ap_thread(st, st->bringup);
 472 }
 473
 474 static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target)
 475 {
 476         enum cpuhp_state prev_state;
 477         int ret;
 478
 479         prev_state = cpuhp_set_state(st, target);
 480         __cpuhp_kick_ap(st);
 481         if ((ret = st->result)) {
 482                 cpuhp_reset_state(st, prev_state);
 483                 __cpuhp_kick_ap(st);
 484         }
 485
 486         return ret;
 487 }
 488
 489 static int bringup_wait_for_ap(unsigned int cpu)
 490 {
 491         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 492
 493         /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
 494         wait_for_ap_thread(st, true);
 495         if (WARN_ON_ONCE((!cpu_online(cpu))))
 496                 return -ECANCELED;
 497
 498         /* Unpark the hotplug thread of the target cpu */
 499         kthread_unpark(st->thread);
 500
 501         /*
 502          * SMT soft disabling on X86 requires to bring the CPU out of the
 503          * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit.  The
 504          * CPU marked itself as booted_once in cpu_notify_starting() so the
 505          * cpu_smt_allowed() check will now return false if this is not the
 506          * primary sibling.
 507          */
 508         if (!cpu_smt_allowed(cpu))
 509                 return -ECANCELED;
 510
 511         if (st->target <= CPUHP_AP_ONLINE_IDLE)
 512                 return 0;
 513
 514         return cpuhp_kick_ap(st, st->target);
 515 }
 516
 517 static int bringup_cpu(unsigned int cpu)
 518 {
 519         struct task_struct *idle = idle_thread_get(cpu);
 520         int ret;
 521
 522         /*
 523          * Some architectures have to walk the irq descriptors to
 524          * setup the vector space for the cpu which comes online.
 525          * Prevent irq alloc/free across the bringup.
 526          */
 527         irq_lock_sparse();
 528
 529         /* Arch-specific enabling code. */
 530         ret = __cpu_up(cpu, idle);
 531         irq_unlock_sparse();
 532         if (ret)
 533                 return ret;
 534         return bringup_wait_for_ap(cpu);
 535 }
 536
 537 /*
 538  * Hotplug state machine related functions
 539  */
 540
 541 static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st)
 542 {
 543         for (st->state--; st->state > st->target; st->state--) {
 544                 struct cpuhp_step *step = cpuhp_get_step(st->state);
 545
 546                 if (!step->skip_onerr)
 547                         cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
 548         }
 549 }
 550
 551 static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st)
 552 {
 553         if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
 554                 return true;
 555         /*
 556          * When CPU hotplug is disabled, then taking the CPU down is not
 557          * possible because takedown_cpu() and the architecture and
 558          * subsystem specific mechanisms are not available. So the CPU
 559          * which would be completely unplugged again needs to stay around
 560          * in the current state.
 561          */
 562         return st->state <= CPUHP_BRINGUP_CPU;
 563 }
 564
 565 static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
 566                               enum cpuhp_state target)
 567 {
 568         enum cpuhp_state prev_state = st->state;
 569         int ret = 0;
 570
 571         while (st->state < target) {
 572                 st->state++;
 573                 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
 574                 if (ret) {
 575                         if (can_rollback_cpu(st)) {
 576                                 st->target = prev_state;
 577                                 undo_cpu_up(cpu, st);
 578                         }
 579                         break;
 580                 }
 581         }
 582         return ret;
 583 }
 584
 585 /*
 586  * The cpu hotplug threads manage the bringup and teardown of the cpus
 587  */
 588 static void cpuhp_create(unsigned int cpu)
 589 {
 590         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 591
 592         init_completion(&st->done_up);
 593         init_completion(&st->done_down);
 594 }
 595
 596 static int cpuhp_should_run(unsigned int cpu)
 597 {
 598         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 599
 600         return st->should_run;
 601 }
 602
 603 /*
 604  * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
 605  * callbacks when a state gets [un]installed at runtime.
 606  *
 607  * Each invocation of this function by the smpboot thread does a single AP
 608  * state callback.
 609  *
 610  * It has 3 modes of operation:
 611  *  - single: runs st->cb_state
 612  *  - up:     runs ++st->state, while st->state < st->target
 613  *  - down:   runs st->state--, while st->state > st->target
 614  *
 615  * When complete or on error, should_run is cleared and the completion is fired.
 616  */
 617 static void cpuhp_thread_fun(unsigned int cpu)
 618 {
 619         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 620         bool bringup = st->bringup;
 621         enum cpuhp_state state;
 622
 623         if (WARN_ON_ONCE(!st->should_run))
 624                 return;
 625
 626         /*
 627          * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
 628          * that if we see ->should_run we also see the rest of the state.
 629          */
 630         smp_mb();
 631
 632         cpuhp_lock_acquire(bringup);
 633
 634         if (st->single) {
 635                 state = st->cb_state;
 636                 st->should_run = false;
 637         } else {
 638                 if (bringup) {
 639                         st->state++;
 640                         state = st->state;
 641                         st->should_run = (st->state < st->target);
 642                         WARN_ON_ONCE(st->state > st->target);
 643                 } else {
 644                         state = st->state;
 645                         st->state--;
 646                         st->should_run = (st->state > st->target);
 647                         WARN_ON_ONCE(st->state < st->target);
 648                 }
 649         }
 650
 651         WARN_ON_ONCE(!cpuhp_is_ap_state(state));
 652
 653         if (st->rollback) {
 654                 struct cpuhp_step *step = cpuhp_get_step(state);
 655                 if (step->skip_onerr)
 656                         goto next;
 657         }
 658
 659         if (cpuhp_is_atomic_state(state)) {
 660                 local_irq_disable();
 661                 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
 662                 local_irq_enable();
 663
 664                 /*
 665                  * STARTING/DYING must not fail!
 666                  */
 667                 WARN_ON_ONCE(st->result);
 668         } else {
 669                 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
 670         }
 671
 672         if (st->result) {
 673                 /*
 674                  * If we fail on a rollback, we're up a creek without no
 675                  * paddle, no way forward, no way back. We loose, thanks for
 676                  * playing.
 677                  */
 678                 WARN_ON_ONCE(st->rollback);
 679                 st->should_run = false;
 680         }
 681
 682 next:
 683         cpuhp_lock_release(bringup);
 684
 685         if (!st->should_run)
 686                 complete_ap_thread(st, bringup);
 687 }
 688
 689 /* Invoke a single callback on a remote cpu */
 690 static int
 691 cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
 692                          struct hlist_node *node)
 693 {
 694         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 695         int ret;
 696
 697         if (!cpu_online(cpu))
 698                 return 0;
 699
 700         cpuhp_lock_acquire(false);
 701         cpuhp_lock_release(false);
 702
 703         cpuhp_lock_acquire(true);
 704         cpuhp_lock_release(true);
 705
 706         /*
 707          * If we are up and running, use the hotplug thread. For early calls
 708          * we invoke the thread function directly.
 709          */
 710         if (!st->thread)
 711                 return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
 712
 713         st->rollback = false;
 714         st->last = NULL;
 715
 716         st->node = node;
 717         st->bringup = bringup;
 718         st->cb_state = state;
 719         st->single = true;
 720
 721         __cpuhp_kick_ap(st);
 722
 723         /*
 724          * If we failed and did a partial, do a rollback.
 725          */
 726         if ((ret = st->result) && st->last) {
 727                 st->rollback = true;
 728                 st->bringup = !bringup;
 729
 730                 __cpuhp_kick_ap(st);
 731         }
 732
 733         /*
 734          * Clean up the leftovers so the next hotplug operation wont use stale
 735          * data.
 736          */
 737         st->node = st->last = NULL;
 738         return ret;
 739 }
 740
 741 static int cpuhp_kick_ap_work(unsigned int cpu)
 742 {
 743         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 744         enum cpuhp_state prev_state = st->state;
 745         int ret;
 746
 747         cpuhp_lock_acquire(false);
 748         cpuhp_lock_release(false);
 749
 750         cpuhp_lock_acquire(true);
 751         cpuhp_lock_release(true);
 752
 753         trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
 754         ret = cpuhp_kick_ap(st, st->target);
 755         trace_cpuhp_exit(cpu, st->state, prev_state, ret);
 756
 757         return ret;
 758 }
 759
 760 static struct smp_hotplug_thread cpuhp_threads = {
 761         .store                  = &cpuhp_state.thread,
 762         .create                 = &cpuhp_create,
 763         .thread_should_run      = cpuhp_should_run,
 764         .thread_fn              = cpuhp_thread_fun,
 765         .thread_comm            = "cpuhp/%u",
 766         .selfparking            = true,
 767 };
 768
 769 void __init cpuhp_threads_init(void)
 770 {
 771         BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
 772         kthread_unpark(this_cpu_read(cpuhp_state.thread));
 773 }
 774
 775 /*
 776  *
 777  * Serialize hotplug trainwrecks outside of the cpu_hotplug_lock
 778  * protected region.
 779  *
 780  * The operation is still serialized against concurrent CPU hotplug via
 781  * cpu_add_remove_lock, i.e. CPU map protection.  But it is _not_
 782  * serialized against other hotplug related activity like adding or
 783  * removing of state callbacks and state instances, which invoke either the
 784  * startup or the teardown callback of the affected state.
 785  *
 786  * This is required for subsystems which are unfixable vs. CPU hotplug and
 787  * evade lock inversion problems by scheduling work which has to be
 788  * completed _before_ cpu_up()/_cpu_down() returns.
 789  *
 790  * Don't even think about adding anything to this for any new code or even
 791  * drivers. It's only purpose is to keep existing lock order trainwrecks
 792  * working.
 793  *
 794  * For cpu_down() there might be valid reasons to finish cleanups which are
 795  * not required to be done under cpu_hotplug_lock, but that's a different
 796  * story and would be not invoked via this.
 797  */
 798 static void cpu_up_down_serialize_trainwrecks(bool tasks_frozen)
 799 {
 800         /*
 801          * cpusets delegate hotplug operations to a worker to "solve" the
 802          * lock order problems. Wait for the worker, but only if tasks are
 803          * _not_ frozen (suspend, hibernate) as that would wait forever.
 804          *
 805          * The wait is required because otherwise the hotplug operation
 806          * returns with inconsistent state, which could even be observed in
 807          * user space when a new CPU is brought up. The CPU plug uevent
 808          * would be delivered and user space reacting on it would fail to
 809          * move tasks to the newly plugged CPU up to the point where the
 810          * work has finished because up to that point the newly plugged CPU
 811          * is not assignable in cpusets/cgroups. On unplug that's not
 812          * necessarily a visible issue, but it is still inconsistent state,
 813          * which is the real problem which needs to be "fixed". This can't
 814          * prevent the transient state between scheduling the work and
 815          * returning from waiting for it.
 816          */
 817         if (!tasks_frozen)
 818                 cpuset_wait_for_hotplug();
 819 }
 820
 821 #ifdef CONFIG_HOTPLUG_CPU
 822 #ifndef arch_clear_mm_cpumask_cpu
 823 #define arch_clear_mm_cpumask_cpu(cpu, mm) cpumask_clear_cpu(cpu, mm_cpumask(mm))
 824 #endif
 825
 826 /**
 827  * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
 828  * @cpu: a CPU id
 829  *
 830  * This function walks all processes, finds a valid mm struct for each one and
 831  * then clears a corresponding bit in mm's cpumask.  While this all sounds
 832  * trivial, there are various non-obvious corner cases, which this function
 833  * tries to solve in a safe manner.
 834  *
 835  * Also note that the function uses a somewhat relaxed locking scheme, so it may
 836  * be called only for an already offlined CPU.
 837  */
 838 void clear_tasks_mm_cpumask(int cpu)
 839 {
 840         struct task_struct *p;
 841
 842         /*
 843          * This function is called after the cpu is taken down and marked
 844          * offline, so its not like new tasks will ever get this cpu set in
 845          * their mm mask. -- Peter Zijlstra
 846          * Thus, we may use rcu_read_lock() here, instead of grabbing
 847          * full-fledged tasklist_lock.
 848          */
 849         WARN_ON(cpu_online(cpu));
 850         rcu_read_lock();
 851         for_each_process(p) {
 852                 struct task_struct *t;
 853
 854                 /*
 855                  * Main thread might exit, but other threads may still have
 856                  * a valid mm. Find one.
 857                  */
 858                 t = find_lock_task_mm(p);
 859                 if (!t)
 860                         continue;
 861                 arch_clear_mm_cpumask_cpu(cpu, t->mm);
 862                 task_unlock(t);
 863         }
 864         rcu_read_unlock();
 865 }
 866
 867 /* Take this CPU down. */
 868 static int take_cpu_down(void *_param)
 869 {
 870         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 871         enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
 872         int err, cpu = smp_processor_id();
 873         int ret;
 874
 875         /* Ensure this CPU doesn't handle any more interrupts. */
 876         err = __cpu_disable();
 877         if (err < 0)
 878                 return err;
 879
 880         /*
 881          * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not
 882          * do this step again.
 883          */
 884         WARN_ON(st->state != CPUHP_TEARDOWN_CPU);
 885         st->state--;
 886         /* Invoke the former CPU_DYING callbacks */
 887         for (; st->state > target; st->state--) {
 888                 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
 889                 /*
 890                  * DYING must not fail!
 891                  */
 892                 WARN_ON_ONCE(ret);
 893         }
 894
 895         /* Give up timekeeping duties */
 896         tick_handover_do_timer();
 897         /* Park the stopper thread */
 898         stop_machine_park(cpu);
 899         return 0;
 900 }
 901
 902 static int takedown_cpu(unsigned int cpu)
 903 {
 904         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 905         int err;
 906
 907         /* Park the smpboot threads */
 908         kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread);
 909
 910         /*
 911          * Prevent irq alloc/free while the dying cpu reorganizes the
 912          * interrupt affinities.
 913          */
 914         irq_lock_sparse();
 915
 916         /*
 917          * So now all preempt/rcu users must observe !cpu_active().
 918          */
 919         err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
 920         if (err) {
 921                 /* CPU refused to die */
 922                 irq_unlock_sparse();
 923                 /* Unpark the hotplug thread so we can rollback there */
 924                 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread);
 925                 return err;
 926         }
 927         BUG_ON(cpu_online(cpu));
 928
 929         /*
 930          * The CPUHP_AP_SCHED_MIGRATE_DYING callback will have removed all
 931          * runnable tasks from the cpu, there's only the idle task left now
 932          * that the migration thread is done doing the stop_machine thing.
 933          *
 934          * Wait for the stop thread to go away.
 935          */
 936         wait_for_ap_thread(st, false);
 937         BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
 938
 939         /* Interrupts are moved away from the dying cpu, reenable alloc/free */
 940         irq_unlock_sparse();
 941
 942         hotplug_cpu__broadcast_tick_pull(cpu);
 943         /* This actually kills the CPU. */
 944         __cpu_die(cpu);
 945
 946         tick_cleanup_dead_cpu(cpu);
 947         rcutree_migrate_callbacks(cpu);
 948         return 0;
 949 }
 950
 951 static void cpuhp_complete_idle_dead(void *arg)
 952 {
 953         struct cpuhp_cpu_state *st = arg;
 954
 955         complete_ap_thread(st, false);
 956 }
 957
 958 void cpuhp_report_idle_dead(void)
 959 {
 960         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 961
 962         BUG_ON(st->state != CPUHP_AP_OFFLINE);
 963         rcu_report_dead(smp_processor_id());
 964         st->state = CPUHP_AP_IDLE_DEAD;
 965         /*
 966          * We cannot call complete after rcu_report_dead() so we delegate it
 967          * to an online cpu.
 968          */
 969         smp_call_function_single(cpumask_first(cpu_online_mask),
 970                                  cpuhp_complete_idle_dead, st, 0);
 971 }
 972
 973 static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st)
 974 {
 975         for (st->state++; st->state < st->target; st->state++) {
 976                 struct cpuhp_step *step = cpuhp_get_step(st->state);
 977
 978                 if (!step->skip_onerr)
 979                         cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
 980         }
 981 }
 982
 983 static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
 984                                 enum cpuhp_state target)
 985 {
 986         enum cpuhp_state prev_state = st->state;
 987         int ret = 0;
 988
 989         for (; st->state > target; st->state--) {
 990                 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
 991                 if (ret) {
 992                         st->target = prev_state;
 993                         if (st->state < prev_state)
 994                                 undo_cpu_down(cpu, st);
 995                         break;
 996                 }
 997         }
 998         return ret;
 999 }
1000
1001 /* Requires cpu_add_remove_lock to be held */
1002 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
1003                            enum cpuhp_state target)
1004 {
1005         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1006         int prev_state, ret = 0;
1007
1008         if (num_online_cpus() == 1)
1009                 return -EBUSY;
1010
1011         if (!cpu_present(cpu))
1012                 return -EINVAL;
1013
1014         cpus_write_lock();
1015
1016         cpuhp_tasks_frozen = tasks_frozen;
1017
1018         prev_state = cpuhp_set_state(st, target);
1019         /*
1020          * If the current CPU state is in the range of the AP hotplug thread,
1021          * then we need to kick the thread.
1022          */
1023         if (st->state > CPUHP_TEARDOWN_CPU) {
1024                 st->target = max((int)target, CPUHP_TEARDOWN_CPU);
1025                 ret = cpuhp_kick_ap_work(cpu);
1026                 /*
1027                  * The AP side has done the error rollback already. Just
1028                  * return the error code..
1029                  */
1030                 if (ret)
1031                         goto out;
1032
1033                 /*
1034                  * We might have stopped still in the range of the AP hotplug
1035                  * thread. Nothing to do anymore.
1036                  */
1037                 if (st->state > CPUHP_TEARDOWN_CPU)
1038                         goto out;
1039
1040                 st->target = target;
1041         }
1042         /*
1043          * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
1044          * to do the further cleanups.
1045          */
1046         ret = cpuhp_down_callbacks(cpu, st, target);
1047         if (ret && st->state == CPUHP_TEARDOWN_CPU && st->state < prev_state) {
1048                 cpuhp_reset_state(st, prev_state);
1049                 __cpuhp_kick_ap(st);
1050         }
1051
1052 out:
1053         cpus_write_unlock();
1054         /*
1055          * Do post unplug cleanup. This is still protected against
1056          * concurrent CPU hotplug via cpu_add_remove_lock.
1057          */
1058         lockup_detector_cleanup();
1059         arch_smt_update();
1060         cpu_up_down_serialize_trainwrecks(tasks_frozen);
1061         return ret;
1062 }
1063
1064 static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
1065 {
1066         if (cpu_hotplug_disabled)
1067                 return -EBUSY;
1068         return _cpu_down(cpu, 0, target);
1069 }
1070
1071 static int do_cpu_down(unsigned int cpu, enum cpuhp_state target)
1072 {
1073         int err;
1074
1075         cpu_maps_update_begin();
1076         err = cpu_down_maps_locked(cpu, target);
1077         cpu_maps_update_done();
1078         return err;
1079 }
1080
1081 int cpu_down(unsigned int cpu)
1082 {
1083         return do_cpu_down(cpu, CPUHP_OFFLINE);
1084 }
1085 EXPORT_SYMBOL(cpu_down);
1086
1087 #else
1088 #define takedown_cpu            NULL
1089 #endif /*CONFIG_HOTPLUG_CPU*/
1090
1091 /**
1092  * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1093  * @cpu: cpu that just started
1094  *
1095  * It must be called by the arch code on the new cpu, before the new cpu
1096  * enables interrupts and before the "boot" cpu returns from __cpu_up().
1097  */
1098 void notify_cpu_starting(unsigned int cpu)
1099 {
1100         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1101         enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
1102         int ret;
1103
1104         rcu_cpu_starting(cpu);  /* Enables RCU usage on this CPU. */
1105         st->booted_once = true;
1106         while (st->state < target) {
1107                 st->state++;
1108                 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
1109                 /*
1110                  * STARTING must not fail!
1111                  */
1112                 WARN_ON_ONCE(ret);
1113         }
1114 }
1115
1116 /*
1117  * Called from the idle task. Wake up the controlling task which brings the
1118  * hotplug thread of the upcoming CPU up and then delegates the rest of the
1119  * online bringup to the hotplug thread.
1120  */
1121 void cpuhp_online_idle(enum cpuhp_state state)
1122 {
1123         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1124
1125         /* Happens for the boot cpu */
1126         if (state != CPUHP_AP_ONLINE_IDLE)
1127                 return;
1128
1129         /*
1130          * Unpart the stopper thread before we start the idle loop (and start
1131          * scheduling); this ensures the stopper task is always available.
1132          */
1133         stop_machine_unpark(smp_processor_id());
1134
1135         st->state = CPUHP_AP_ONLINE_IDLE;
1136         complete_ap_thread(st, true);
1137 }
1138
1139 /* Requires cpu_add_remove_lock to be held */
1140 static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
1141 {
1142         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1143         struct task_struct *idle;
1144         int ret = 0;
1145
1146         cpus_write_lock();
1147
1148         if (!cpu_present(cpu)) {
1149                 ret = -EINVAL;
1150                 goto out;
1151         }
1152
1153         /*
1154          * The caller of do_cpu_up might have raced with another
1155          * caller. Ignore it for now.
1156          */
1157         if (st->state >= target)
1158                 goto out;
1159
1160         if (st->state == CPUHP_OFFLINE) {
1161                 /* Let it fail before we try to bring the cpu up */
1162                 idle = idle_thread_get(cpu);
1163                 if (IS_ERR(idle)) {
1164                         ret = PTR_ERR(idle);
1165                         goto out;
1166                 }
1167         }
1168
1169         cpuhp_tasks_frozen = tasks_frozen;
1170
1171         cpuhp_set_state(st, target);
1172         /*
1173          * If the current CPU state is in the range of the AP hotplug thread,
1174          * then we need to kick the thread once more.
1175          */
1176         if (st->state > CPUHP_BRINGUP_CPU) {
1177                 ret = cpuhp_kick_ap_work(cpu);
1178                 /*
1179                  * The AP side has done the error rollback already. Just
1180                  * return the error code..
1181                  */
1182                 if (ret)
1183                         goto out;
1184         }
1185
1186         /*
1187          * Try to reach the target state. We max out on the BP at
1188          * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
1189          * responsible for bringing it up to the target state.
1190          */
1191         target = min((int)target, CPUHP_BRINGUP_CPU);
1192         ret = cpuhp_up_callbacks(cpu, st, target);
1193 out:
1194         cpus_write_unlock();
1195         arch_smt_update();
1196         cpu_up_down_serialize_trainwrecks(tasks_frozen);
1197         return ret;
1198 }
1199
1200 static int do_cpu_up(unsigned int cpu, enum cpuhp_state target)
1201 {
1202         int err = 0;
1203
1204         if (!cpu_possible(cpu)) {
1205                 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1206                        cpu);
1207 #if defined(CONFIG_IA64)
1208                 pr_err("please check additional_cpus= boot parameter\n");
1209 #endif
1210                 return -EINVAL;
1211         }
1212
1213         err = try_online_node(cpu_to_node(cpu));
1214         if (err)
1215                 return err;
1216
1217         cpu_maps_update_begin();
1218
1219         if (cpu_hotplug_disabled) {
1220                 err = -EBUSY;
1221                 goto out;
1222         }
1223         if (!cpu_smt_allowed(cpu)) {
1224                 err = -EPERM;
1225                 goto out;
1226         }
1227
1228         err = _cpu_up(cpu, 0, target);
1229 out:
1230         cpu_maps_update_done();
1231         return err;
1232 }
1233
1234 int cpu_up(unsigned int cpu)
1235 {
1236         return do_cpu_up(cpu, CPUHP_ONLINE);
1237 }
1238 EXPORT_SYMBOL_GPL(cpu_up);
1239
1240 #ifdef CONFIG_PM_SLEEP_SMP
1241 static cpumask_var_t frozen_cpus;
1242
1243 int freeze_secondary_cpus(int primary)
1244 {
1245         int cpu, error = 0;
1246
1247         cpu_maps_update_begin();
1248         if (!cpu_online(primary))
1249                 primary = cpumask_first(cpu_online_mask);
1250         /*
1251          * We take down all of the non-boot CPUs in one shot to avoid races
1252          * with the userspace trying to use the CPU hotplug at the same time
1253          */
1254         cpumask_clear(frozen_cpus);
1255
1256         pr_info("Disabling non-boot CPUs ...\n");
1257         for_each_online_cpu(cpu) {
1258                 if (cpu == primary)
1259                         continue;
1260                 trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
1261                 error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
1262                 trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
1263                 if (!error)
1264                         cpumask_set_cpu(cpu, frozen_cpus);
1265                 else {
1266                         pr_err("Error taking CPU%d down: %d\n", cpu, error);
1267                         break;
1268                 }
1269         }
1270
1271         if (!error)
1272                 BUG_ON(num_online_cpus() > 1);
1273         else
1274                 pr_err("Non-boot CPUs are not disabled\n");
1275
1276         /*
1277          * Make sure the CPUs won't be enabled by someone else. We need to do
1278          * this even in case of failure as all disable_nonboot_cpus() users are
1279          * supposed to do enable_nonboot_cpus() on the failure path.
1280          */
1281         cpu_hotplug_disabled++;
1282
1283         cpu_maps_update_done();
1284         return error;
1285 }
1286
1287 void __weak arch_enable_nonboot_cpus_begin(void)
1288 {
1289 }
1290
1291 void __weak arch_enable_nonboot_cpus_end(void)
1292 {
1293 }
1294
1295 void enable_nonboot_cpus(void)
1296 {
1297         int cpu, error;
1298
1299         /* Allow everyone to use the CPU hotplug again */
1300         cpu_maps_update_begin();
1301         __cpu_hotplug_enable();
1302         if (cpumask_empty(frozen_cpus))
1303                 goto out;
1304
1305         pr_info("Enabling non-boot CPUs ...\n");
1306
1307         arch_enable_nonboot_cpus_begin();
1308
1309         for_each_cpu(cpu, frozen_cpus) {
1310                 trace_suspend_resume(TPS("CPU_ON"), cpu, true);
1311                 error = _cpu_up(cpu, 1, CPUHP_ONLINE);
1312                 trace_suspend_resume(TPS("CPU_ON"), cpu, false);
1313                 if (!error) {
1314                         pr_info("CPU%d is up\n", cpu);
1315                         continue;
1316                 }
1317                 pr_warn("Error taking CPU%d up: %d\n", cpu, error);
1318         }
1319
1320         arch_enable_nonboot_cpus_end();
1321
1322         cpumask_clear(frozen_cpus);
1323 out:
1324         cpu_maps_update_done();
1325 }
1326
1327 static int __init alloc_frozen_cpus(void)
1328 {
1329         if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
1330                 return -ENOMEM;
1331         return 0;
1332 }
1333 core_initcall(alloc_frozen_cpus);
1334
1335 /*
1336  * When callbacks for CPU hotplug notifications are being executed, we must
1337  * ensure that the state of the system with respect to the tasks being frozen
1338  * or not, as reported by the notification, remains unchanged *throughout the
1339  * duration* of the execution of the callbacks.
1340  * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1341  *
1342  * This synchronization is implemented by mutually excluding regular CPU
1343  * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1344  * Hibernate notifications.
1345  */
1346 static int
1347 cpu_hotplug_pm_callback(struct notifier_block *nb,
1348                         unsigned long action, void *ptr)
1349 {
1350         switch (action) {
1351
1352         case PM_SUSPEND_PREPARE:
1353         case PM_HIBERNATION_PREPARE:
1354                 cpu_hotplug_disable();
1355                 break;
1356
1357         case PM_POST_SUSPEND:
1358         case PM_POST_HIBERNATION:
1359                 cpu_hotplug_enable();
1360                 break;
1361
1362         default:
1363                 return NOTIFY_DONE;
1364         }
1365
1366         return NOTIFY_OK;
1367 }
1368
1369
1370 static int __init cpu_hotplug_pm_sync_init(void)
1371 {
1372         /*
1373          * cpu_hotplug_pm_callback has higher priority than x86
1374          * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1375          * to disable cpu hotplug to avoid cpu hotplug race.
1376          */
1377         pm_notifier(cpu_hotplug_pm_callback, 0);
1378         return 0;
1379 }
1380 core_initcall(cpu_hotplug_pm_sync_init);
1381
1382 #endif /* CONFIG_PM_SLEEP_SMP */
1383
1384 int __boot_cpu_id;
1385
1386 #endif /* CONFIG_SMP */
1387
1388 /* Boot processor state steps */
1389 static struct cpuhp_step cpuhp_bp_states[] = {
1390         [CPUHP_OFFLINE] = {
1391                 .name                   = "offline",
1392                 .startup.single         = NULL,
1393                 .teardown.single        = NULL,
1394         },
1395 #ifdef CONFIG_SMP
1396         [CPUHP_CREATE_THREADS]= {
1397                 .name                   = "threads:prepare",
1398                 .startup.single         = smpboot_create_threads,
1399                 .teardown.single        = NULL,
1400                 .cant_stop              = true,
1401         },
1402         [CPUHP_PERF_PREPARE] = {
1403                 .name                   = "perf:prepare",
1404                 .startup.single         = perf_event_init_cpu,
1405                 .teardown.single        = perf_event_exit_cpu,
1406         },
1407         [CPUHP_WORKQUEUE_PREP] = {
1408                 .name                   = "workqueue:prepare",
1409                 .startup.single         = workqueue_prepare_cpu,
1410                 .teardown.single        = NULL,
1411         },
1412         [CPUHP_HRTIMERS_PREPARE] = {
1413                 .name                   = "hrtimers:prepare",
1414                 .startup.single         = hrtimers_prepare_cpu,
1415                 .teardown.single        = hrtimers_dead_cpu,
1416         },
1417         [CPUHP_SMPCFD_PREPARE] = {
1418                 .name                   = "smpcfd:prepare",
1419                 .startup.single         = smpcfd_prepare_cpu,
1420                 .teardown.single        = smpcfd_dead_cpu,
1421         },
1422         [CPUHP_RELAY_PREPARE] = {
1423                 .name                   = "relay:prepare",
1424                 .startup.single         = relay_prepare_cpu,
1425                 .teardown.single        = NULL,
1426         },
1427         [CPUHP_SLAB_PREPARE] = {
1428                 .name                   = "slab:prepare",
1429                 .startup.single         = slab_prepare_cpu,
1430                 .teardown.single        = slab_dead_cpu,
1431         },
1432         [CPUHP_RCUTREE_PREP] = {
1433                 .name                   = "RCU/tree:prepare",
1434                 .startup.single         = rcutree_prepare_cpu,
1435                 .teardown.single        = rcutree_dead_cpu,
1436         },
1437         /*
1438          * On the tear-down path, timers_dead_cpu() must be invoked
1439          * before blk_mq_queue_reinit_notify() from notify_dead(),
1440          * otherwise a RCU stall occurs.
1441          */
1442         [CPUHP_TIMERS_PREPARE] = {
1443                 .name                   = "timers:dead",
1444                 .startup.single         = timers_prepare_cpu,
1445                 .teardown.single        = timers_dead_cpu,
1446         },
1447         /* Kicks the plugged cpu into life */
1448         [CPUHP_BRINGUP_CPU] = {
1449                 .name                   = "cpu:bringup",
1450                 .startup.single         = bringup_cpu,
1451                 .teardown.single        = NULL,
1452                 .cant_stop              = true,
1453         },
1454         /*
1455          * Handled on controll processor until the plugged processor manages
1456          * this itself.
1457          */
1458         [CPUHP_TEARDOWN_CPU] = {
1459                 .name                   = "cpu:teardown",
1460                 .startup.single         = NULL,
1461                 .teardown.single        = takedown_cpu,
1462                 .cant_stop              = true,
1463         },
1464 #else
1465         [CPUHP_BRINGUP_CPU] = { },
1466 #endif
1467 };
1468
1469 /* Application processor state steps */
1470 static struct cpuhp_step cpuhp_ap_states[] = {
1471 #ifdef CONFIG_SMP
1472         /* Final state before CPU kills itself */
1473         [CPUHP_AP_IDLE_DEAD] = {
1474                 .name                   = "idle:dead",
1475         },
1476         /*
1477          * Last state before CPU enters the idle loop to die. Transient state
1478          * for synchronization.
1479          */
1480         [CPUHP_AP_OFFLINE] = {
1481                 .name                   = "ap:offline",
1482                 .cant_stop              = true,
1483         },
1484         /* First state is scheduler control. Interrupts are disabled */
1485         [CPUHP_AP_SCHED_STARTING] = {
1486                 .name                   = "sched:starting",
1487                 .startup.single         = sched_cpu_starting,
1488                 .teardown.single        = sched_cpu_dying,
1489         },
1490         [CPUHP_AP_RCUTREE_DYING] = {
1491                 .name                   = "RCU/tree:dying",
1492                 .startup.single         = NULL,
1493                 .teardown.single        = rcutree_dying_cpu,
1494         },
1495         [CPUHP_AP_SMPCFD_DYING] = {
1496                 .name                   = "smpcfd:dying",
1497                 .startup.single         = NULL,
1498                 .teardown.single        = smpcfd_dying_cpu,
1499         },
1500         /* Entry state on starting. Interrupts enabled from here on. Transient
1501          * state for synchronsization */
1502         [CPUHP_AP_ONLINE] = {
1503                 .name                   = "ap:online",
1504         },
1505         /* Handle smpboot threads park/unpark */
1506         [CPUHP_AP_SMPBOOT_THREADS] = {
1507                 .name                   = "smpboot/threads:online",
1508                 .startup.single         = smpboot_unpark_threads,
1509                 .teardown.single        = smpboot_park_threads,
1510         },
1511         [CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
1512                 .name                   = "irq/affinity:online",
1513                 .startup.single         = irq_affinity_online_cpu,
1514                 .teardown.single        = NULL,
1515         },
1516         [CPUHP_AP_PERF_ONLINE] = {
1517                 .name                   = "perf:online",
1518                 .startup.single         = perf_event_init_cpu,
1519                 .teardown.single        = perf_event_exit_cpu,
1520         },
1521         [CPUHP_AP_WORKQUEUE_ONLINE] = {
1522                 .name                   = "workqueue:online",
1523                 .startup.single         = workqueue_online_cpu,
1524                 .teardown.single        = workqueue_offline_cpu,
1525         },
1526         [CPUHP_AP_RCUTREE_ONLINE] = {
1527                 .name                   = "RCU/tree:online",
1528                 .startup.single         = rcutree_online_cpu,
1529                 .teardown.single        = rcutree_offline_cpu,
1530         },
1531 #endif
1532         /*
1533          * The dynamically registered state space is here
1534          */
1535
1536 #ifdef CONFIG_SMP
1537         /* Last state is scheduler control setting the cpu active */
1538         [CPUHP_AP_ACTIVE] = {
1539                 .name                   = "sched:active",
1540                 .startup.single         = sched_cpu_activate,
1541                 .teardown.single        = sched_cpu_deactivate,
1542         },
1543 #endif
1544
1545         /* CPU is fully up and running. */
1546         [CPUHP_ONLINE] = {
1547                 .name                   = "online",
1548                 .startup.single         = NULL,
1549                 .teardown.single        = NULL,
1550         },
1551 };
1552
1553 /* Sanity check for callbacks */
1554 static int cpuhp_cb_check(enum cpuhp_state state)
1555 {
1556         if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
1557                 return -EINVAL;
1558         return 0;
1559 }
1560
1561 /*
1562  * Returns a free for dynamic slot assignment of the Online state. The states
1563  * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1564  * by having no name assigned.
1565  */
1566 static int cpuhp_reserve_state(enum cpuhp_state state)
1567 {
1568         enum cpuhp_state i, end;
1569         struct cpuhp_step *step;
1570
1571         switch (state) {
1572         case CPUHP_AP_ONLINE_DYN:
1573                 step = cpuhp_ap_states + CPUHP_AP_ONLINE_DYN;
1574                 end = CPUHP_AP_ONLINE_DYN_END;
1575                 break;
1576         case CPUHP_BP_PREPARE_DYN:
1577                 step = cpuhp_bp_states + CPUHP_BP_PREPARE_DYN;
1578                 end = CPUHP_BP_PREPARE_DYN_END;
1579                 break;
1580         default:
1581                 return -EINVAL;
1582         }
1583
1584         for (i = state; i <= end; i++, step++) {
1585                 if (!step->name)
1586                         return i;
1587         }
1588         WARN(1, "No more dynamic states available for CPU hotplug\n");
1589         return -ENOSPC;
1590 }
1591
1592 static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
1593                                  int (*startup)(unsigned int cpu),
1594                                  int (*teardown)(unsigned int cpu),
1595                                  bool multi_instance)
1596 {
1597         /* (Un)Install the callbacks for further cpu hotplug operations */
1598         struct cpuhp_step *sp;
1599         int ret = 0;
1600
1601         /*
1602          * If name is NULL, then the state gets removed.
1603          *
1604          * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
1605          * the first allocation from these dynamic ranges, so the removal
1606          * would trigger a new allocation and clear the wrong (already
1607          * empty) state, leaving the callbacks of the to be cleared state
1608          * dangling, which causes wreckage on the next hotplug operation.
1609          */
1610         if (name && (state == CPUHP_AP_ONLINE_DYN ||
1611                      state == CPUHP_BP_PREPARE_DYN)) {
1612                 ret = cpuhp_reserve_state(state);
1613                 if (ret < 0)
1614                         return ret;
1615                 state = ret;
1616         }
1617         sp = cpuhp_get_step(state);
1618         if (name && sp->name)
1619                 return -EBUSY;
1620
1621         sp->startup.single = startup;
1622         sp->teardown.single = teardown;
1623         sp->name = name;
1624         sp->multi_instance = multi_instance;
1625         INIT_HLIST_HEAD(&sp->list);
1626         return ret;
1627 }
1628
1629 static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
1630 {
1631         return cpuhp_get_step(state)->teardown.single;
1632 }
1633
1634 /*
1635  * Call the startup/teardown function for a step either on the AP or
1636  * on the current CPU.
1637  */
1638 static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
1639                             struct hlist_node *node)
1640 {
1641         struct cpuhp_step *sp = cpuhp_get_step(state);
1642         int ret;
1643
1644         /*
1645          * If there's nothing to do, we done.
1646          * Relies on the union for multi_instance.
1647          */
1648         if ((bringup && !sp->startup.single) ||
1649             (!bringup && !sp->teardown.single))
1650                 return 0;
1651         /*
1652          * The non AP bound callbacks can fail on bringup. On teardown
1653          * e.g. module removal we crash for now.
1654          */
1655 #ifdef CONFIG_SMP
1656         if (cpuhp_is_ap_state(state))
1657                 ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
1658         else
1659                 ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1660 #else
1661         ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1662 #endif
1663         BUG_ON(ret && !bringup);
1664         return ret;
1665 }
1666
1667 /*
1668  * Called from __cpuhp_setup_state on a recoverable failure.
1669  *
1670  * Note: The teardown callbacks for rollback are not allowed to fail!
1671  */
1672 static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
1673                                    struct hlist_node *node)
1674 {
1675         int cpu;
1676
1677         /* Roll back the already executed steps on the other cpus */
1678         for_each_present_cpu(cpu) {
1679                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1680                 int cpustate = st->state;
1681
1682                 if (cpu >= failedcpu)
1683                         break;
1684
1685                 /* Did we invoke the startup call on that cpu ? */
1686                 if (cpustate >= state)
1687                         cpuhp_issue_call(cpu, state, false, node);
1688         }
1689 }
1690
1691 int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
1692                                           struct hlist_node *node,
1693                                           bool invoke)
1694 {
1695         struct cpuhp_step *sp;
1696         int cpu;
1697         int ret;
1698
1699         lockdep_assert_cpus_held();
1700
1701         sp = cpuhp_get_step(state);
1702         if (sp->multi_instance == false)
1703                 return -EINVAL;
1704
1705         mutex_lock(&cpuhp_state_mutex);
1706
1707         if (!invoke || !sp->startup.multi)
1708                 goto add_node;
1709
1710         /*
1711          * Try to call the startup callback for each present cpu
1712          * depending on the hotplug state of the cpu.
1713          */
1714         for_each_present_cpu(cpu) {
1715                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1716                 int cpustate = st->state;
1717
1718                 if (cpustate < state)
1719                         continue;
1720
1721                 ret = cpuhp_issue_call(cpu, state, true, node);
1722                 if (ret) {
1723                         if (sp->teardown.multi)
1724                                 cpuhp_rollback_install(cpu, state, node);
1725                         goto unlock;
1726                 }
1727         }
1728 add_node:
1729         ret = 0;
1730         hlist_add_head(node, &sp->list);
1731 unlock:
1732         mutex_unlock(&cpuhp_state_mutex);
1733         return ret;
1734 }
1735
1736 int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
1737                                bool invoke)
1738 {
1739         int ret;
1740
1741         cpus_read_lock();
1742         ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
1743         cpus_read_unlock();
1744         return ret;
1745 }
1746 EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
1747
1748 /**
1749  * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
1750  * @state:              The state to setup
1751  * @invoke:             If true, the startup function is invoked for cpus where
1752  *                      cpu state >= @state
1753  * @startup:            startup callback function
1754  * @teardown:           teardown callback function
1755  * @multi_instance:     State is set up for multiple instances which get
1756  *                      added afterwards.
1757  *
1758  * The caller needs to hold cpus read locked while calling this function.
1759  * Returns:
1760  *   On success:
1761  *      Positive state number if @state is CPUHP_AP_ONLINE_DYN
1762  *      0 for all other states
1763  *   On failure: proper (negative) error code
1764  */
1765 int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
1766                                    const char *name, bool invoke,
1767                                    int (*startup)(unsigned int cpu),
1768                                    int (*teardown)(unsigned int cpu),
1769                                    bool multi_instance)
1770 {
1771         int cpu, ret = 0;
1772         bool dynstate;
1773
1774         lockdep_assert_cpus_held();
1775
1776         if (cpuhp_cb_check(state) || !name)
1777                 return -EINVAL;
1778
1779         mutex_lock(&cpuhp_state_mutex);
1780
1781         ret = cpuhp_store_callbacks(state, name, startup, teardown,
1782                                     multi_instance);
1783
1784         dynstate = state == CPUHP_AP_ONLINE_DYN;
1785         if (ret > 0 && dynstate) {
1786                 state = ret;
1787                 ret = 0;
1788         }
1789
1790         if (ret || !invoke || !startup)
1791                 goto out;
1792
1793         /*
1794          * Try to call the startup callback for each present cpu
1795          * depending on the hotplug state of the cpu.
1796          */
1797         for_each_present_cpu(cpu) {
1798                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1799                 int cpustate = st->state;
1800
1801                 if (cpustate < state)
1802                         continue;
1803
1804                 ret = cpuhp_issue_call(cpu, state, true, NULL);
1805                 if (ret) {
1806                         if (teardown)
1807                                 cpuhp_rollback_install(cpu, state, NULL);
1808                         cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1809                         goto out;
1810                 }
1811         }
1812 out:
1813         mutex_unlock(&cpuhp_state_mutex);
1814         /*
1815          * If the requested state is CPUHP_AP_ONLINE_DYN, return the
1816          * dynamically allocated state in case of success.
1817          */
1818         if (!ret && dynstate)
1819                 return state;
1820         return ret;
1821 }
1822 EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);
1823
1824 int __cpuhp_setup_state(enum cpuhp_state state,
1825                         const char *name, bool invoke,
1826                         int (*startup)(unsigned int cpu),
1827                         int (*teardown)(unsigned int cpu),
1828                         bool multi_instance)
1829 {
1830         int ret;
1831
1832         cpus_read_lock();
1833         ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
1834                                              teardown, multi_instance);
1835         cpus_read_unlock();
1836         return ret;
1837 }
1838 EXPORT_SYMBOL(__cpuhp_setup_state);
1839
1840 int __cpuhp_state_remove_instance(enum cpuhp_state state,
1841                                   struct hlist_node *node, bool invoke)
1842 {
1843         struct cpuhp_step *sp = cpuhp_get_step(state);
1844         int cpu;
1845
1846         BUG_ON(cpuhp_cb_check(state));
1847
1848         if (!sp->multi_instance)
1849                 return -EINVAL;
1850
1851         cpus_read_lock();
1852         mutex_lock(&cpuhp_state_mutex);
1853
1854         if (!invoke || !cpuhp_get_teardown_cb(state))
1855                 goto remove;
1856         /*
1857          * Call the teardown callback for each present cpu depending
1858          * on the hotplug state of the cpu. This function is not
1859          * allowed to fail currently!
1860          */
1861         for_each_present_cpu(cpu) {
1862                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1863                 int cpustate = st->state;
1864
1865                 if (cpustate >= state)
1866                         cpuhp_issue_call(cpu, state, false, node);
1867         }
1868
1869 remove:
1870         hlist_del(node);
1871         mutex_unlock(&cpuhp_state_mutex);
1872         cpus_read_unlock();
1873
1874         return 0;
1875 }
1876 EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
1877
1878 /**
1879  * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
1880  * @state:      The state to remove
1881  * @invoke:     If true, the teardown function is invoked for cpus where
1882  *              cpu state >= @state
1883  *
1884  * The caller needs to hold cpus read locked while calling this function.
1885  * The teardown callback is currently not allowed to fail. Think
1886  * about module removal!
1887  */
1888 void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
1889 {
1890         struct cpuhp_step *sp = cpuhp_get_step(state);
1891         int cpu;
1892
1893         BUG_ON(cpuhp_cb_check(state));
1894
1895         lockdep_assert_cpus_held();
1896
1897         mutex_lock(&cpuhp_state_mutex);
1898         if (sp->multi_instance) {
1899                 WARN(!hlist_empty(&sp->list),
1900                      "Error: Removing state %d which has instances left.\n",
1901                      state);
1902                 goto remove;
1903         }
1904
1905         if (!invoke || !cpuhp_get_teardown_cb(state))
1906                 goto remove;
1907
1908         /*
1909          * Call the teardown callback for each present cpu depending
1910          * on the hotplug state of the cpu. This function is not
1911          * allowed to fail currently!
1912          */
1913         for_each_present_cpu(cpu) {
1914                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1915                 int cpustate = st->state;
1916
1917                 if (cpustate >= state)
1918                         cpuhp_issue_call(cpu, state, false, NULL);
1919         }
1920 remove:
1921         cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1922         mutex_unlock(&cpuhp_state_mutex);
1923 }
1924 EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);
1925
1926 void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
1927 {
1928         cpus_read_lock();
1929         __cpuhp_remove_state_cpuslocked(state, invoke);
1930         cpus_read_unlock();
1931 }
1932 EXPORT_SYMBOL(__cpuhp_remove_state);
1933
1934 #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
1935 static ssize_t show_cpuhp_state(struct device *dev,
1936                                 struct device_attribute *attr, char *buf)
1937 {
1938         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1939
1940         return sprintf(buf, "%d\n", st->state);
1941 }
1942 static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL);
1943
1944 static ssize_t write_cpuhp_target(struct device *dev,
1945                                   struct device_attribute *attr,
1946                                   const char *buf, size_t count)
1947 {
1948         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1949         struct cpuhp_step *sp;
1950         int target, ret;
1951
1952         ret = kstrtoint(buf, 10, &target);
1953         if (ret)
1954                 return ret;
1955
1956 #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
1957         if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
1958                 return -EINVAL;
1959 #else
1960         if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
1961                 return -EINVAL;
1962 #endif
1963
1964         ret = lock_device_hotplug_sysfs();
1965         if (ret)
1966                 return ret;
1967
1968         mutex_lock(&cpuhp_state_mutex);
1969         sp = cpuhp_get_step(target);
1970         ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
1971         mutex_unlock(&cpuhp_state_mutex);
1972         if (ret)
1973                 goto out;
1974
1975         if (st->state < target)
1976                 ret = do_cpu_up(dev->id, target);
1977         else
1978                 ret = do_cpu_down(dev->id, target);
1979 out:
1980         unlock_device_hotplug();
1981         return ret ? ret : count;
1982 }
1983
1984 static ssize_t show_cpuhp_target(struct device *dev,
1985                                  struct device_attribute *attr, char *buf)
1986 {
1987         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1988
1989         return sprintf(buf, "%d\n", st->target);
1990 }
1991 static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target);
1992
1993
1994 static ssize_t write_cpuhp_fail(struct device *dev,
1995                                 struct device_attribute *attr,
1996                                 const char *buf, size_t count)
1997 {
1998         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1999         struct cpuhp_step *sp;
2000         int fail, ret;
2001
2002         ret = kstrtoint(buf, 10, &fail);
2003         if (ret)
2004                 return ret;
2005
2006         if (fail < CPUHP_OFFLINE || fail > CPUHP_ONLINE)
2007                 return -EINVAL;
2008
2009         /*
2010          * Cannot fail STARTING/DYING callbacks.
2011          */
2012         if (cpuhp_is_atomic_state(fail))
2013                 return -EINVAL;
2014
2015         /*
2016          * Cannot fail anything that doesn't have callbacks.
2017          */
2018         mutex_lock(&cpuhp_state_mutex);
2019         sp = cpuhp_get_step(fail);
2020         if (!sp->startup.single && !sp->teardown.single)
2021                 ret = -EINVAL;
2022         mutex_unlock(&cpuhp_state_mutex);
2023         if (ret)
2024                 return ret;
2025
2026         st->fail = fail;
2027
2028         return count;
2029 }
2030
2031 static ssize_t show_cpuhp_fail(struct device *dev,
2032                                struct device_attribute *attr, char *buf)
2033 {
2034         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2035
2036         return sprintf(buf, "%d\n", st->fail);
2037 }
2038
2039 static DEVICE_ATTR(fail, 0644, show_cpuhp_fail, write_cpuhp_fail);
2040
2041 static struct attribute *cpuhp_cpu_attrs[] = {
2042         &dev_attr_state.attr,
2043         &dev_attr_target.attr,
2044         &dev_attr_fail.attr,
2045         NULL
2046 };
2047
2048 static const struct attribute_group cpuhp_cpu_attr_group = {
2049         .attrs = cpuhp_cpu_attrs,
2050         .name = "hotplug",
2051         NULL
2052 };
2053
2054 static ssize_t show_cpuhp_states(struct device *dev,
2055                                  struct device_attribute *attr, char *buf)
2056 {
2057         ssize_t cur, res = 0;
2058         int i;
2059
2060         mutex_lock(&cpuhp_state_mutex);
2061         for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
2062                 struct cpuhp_step *sp = cpuhp_get_step(i);
2063
2064                 if (sp->name) {
2065                         cur = sprintf(buf, "%3d: %s\n", i, sp->name);
2066                         buf += cur;
2067                         res += cur;
2068                 }
2069         }
2070         mutex_unlock(&cpuhp_state_mutex);
2071         return res;
2072 }
2073 static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL);
2074
2075 static struct attribute *cpuhp_cpu_root_attrs[] = {
2076         &dev_attr_states.attr,
2077         NULL
2078 };
2079
2080 static const struct attribute_group cpuhp_cpu_root_attr_group = {
2081         .attrs = cpuhp_cpu_root_attrs,
2082         .name = "hotplug",
2083         NULL
2084 };
2085
2086 #ifdef CONFIG_HOTPLUG_SMT
2087
2088 static const char *smt_states[] = {
2089         [CPU_SMT_ENABLED]               = "on",
2090         [CPU_SMT_DISABLED]              = "off",
2091         [CPU_SMT_FORCE_DISABLED]        = "forceoff",
2092         [CPU_SMT_NOT_SUPPORTED]         = "notsupported",
2093 };
2094
2095 static ssize_t
2096 show_smt_control(struct device *dev, struct device_attribute *attr, char *buf)
2097 {
2098         return snprintf(buf, PAGE_SIZE - 2, "%s\n", smt_states[cpu_smt_control]);
2099 }
2100
2101 static void cpuhp_offline_cpu_device(unsigned int cpu)
2102 {
2103         struct device *dev = get_cpu_device(cpu);
2104
2105         dev->offline = true;
2106         /* Tell user space about the state change */
2107         kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
2108 }
2109
2110 static void cpuhp_online_cpu_device(unsigned int cpu)
2111 {
2112         struct device *dev = get_cpu_device(cpu);
2113
2114         dev->offline = false;
2115         /* Tell user space about the state change */
2116         kobject_uevent(&dev->kobj, KOBJ_ONLINE);
2117 }
2118
2119 int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
2120 {
2121         int cpu, ret = 0;
2122
2123         cpu_maps_update_begin();
2124         for_each_online_cpu(cpu) {
2125                 if (topology_is_primary_thread(cpu))
2126                         continue;
2127                 ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
2128                 if (ret)
2129                         break;
2130                 /*
2131                  * As this needs to hold the cpu maps lock it's impossible
2132                  * to call device_offline() because that ends up calling
2133                  * cpu_down() which takes cpu maps lock. cpu maps lock
2134                  * needs to be held as this might race against in kernel
2135                  * abusers of the hotplug machinery (thermal management).
2136                  *
2137                  * So nothing would update device:offline state. That would
2138                  * leave the sysfs entry stale and prevent onlining after
2139                  * smt control has been changed to 'off' again. This is
2140                  * called under the sysfs hotplug lock, so it is properly
2141                  * serialized against the regular offline usage.
2142                  */
2143                 cpuhp_offline_cpu_device(cpu);
2144         }
2145         if (!ret)
2146                 cpu_smt_control = ctrlval;
2147         cpu_maps_update_done();
2148         return ret;
2149 }
2150
2151 int cpuhp_smt_enable(void)
2152 {
2153         int cpu, ret = 0;
2154
2155         cpu_maps_update_begin();
2156         cpu_smt_control = CPU_SMT_ENABLED;
2157         for_each_present_cpu(cpu) {
2158                 /* Skip online CPUs and CPUs on offline nodes */
2159                 if (cpu_online(cpu) || !node_online(cpu_to_node(cpu)))
2160                         continue;
2161                 ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
2162                 if (ret)
2163                         break;
2164                 /* See comment in cpuhp_smt_disable() */
2165                 cpuhp_online_cpu_device(cpu);
2166         }
2167         cpu_maps_update_done();
2168         return ret;
2169 }
2170
2171 static ssize_t
2172 store_smt_control(struct device *dev, struct device_attribute *attr,
2173                   const char *buf, size_t count)
2174 {
2175         int ctrlval, ret;
2176
2177         if (sysfs_streq(buf, "on"))
2178                 ctrlval = CPU_SMT_ENABLED;
2179         else if (sysfs_streq(buf, "off"))
2180                 ctrlval = CPU_SMT_DISABLED;
2181         else if (sysfs_streq(buf, "forceoff"))
2182                 ctrlval = CPU_SMT_FORCE_DISABLED;
2183         else
2184                 return -EINVAL;
2185
2186         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED)
2187                 return -EPERM;
2188
2189         if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
2190                 return -ENODEV;
2191
2192         ret = lock_device_hotplug_sysfs();
2193         if (ret)
2194                 return ret;
2195
2196         if (ctrlval != cpu_smt_control) {
2197                 switch (ctrlval) {
2198                 case CPU_SMT_ENABLED:
2199                         ret = cpuhp_smt_enable();
2200                         break;
2201                 case CPU_SMT_DISABLED:
2202                 case CPU_SMT_FORCE_DISABLED:
2203                         ret = cpuhp_smt_disable(ctrlval);
2204                         break;
2205                 }
2206         }
2207
2208         unlock_device_hotplug();
2209         return ret ? ret : count;
2210 }
2211 static DEVICE_ATTR(control, 0644, show_smt_control, store_smt_control);
2212
2213 static ssize_t
2214 show_smt_active(struct device *dev, struct device_attribute *attr, char *buf)
2215 {
2216         bool active = topology_max_smt_threads() > 1;
2217
2218         return snprintf(buf, PAGE_SIZE - 2, "%d\n", active);
2219 }
2220 static DEVICE_ATTR(active, 0444, show_smt_active, NULL);
2221
2222 static struct attribute *cpuhp_smt_attrs[] = {
2223         &dev_attr_control.attr,
2224         &dev_attr_active.attr,
2225         NULL
2226 };
2227
2228 static const struct attribute_group cpuhp_smt_attr_group = {
2229         .attrs = cpuhp_smt_attrs,
2230         .name = "smt",
2231         NULL
2232 };
2233
2234 static int __init cpu_smt_state_init(void)
2235 {
2236         return sysfs_create_group(&cpu_subsys.dev_root->kobj,
2237                                   &cpuhp_smt_attr_group);
2238 }
2239
2240 #else
2241 static inline int cpu_smt_state_init(void) { return 0; }
2242 #endif
2243
2244 static int __init cpuhp_sysfs_init(void)
2245 {
2246         int cpu, ret;
2247
2248         ret = cpu_smt_state_init();
2249         if (ret)
2250                 return ret;
2251
2252         ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
2253                                  &cpuhp_cpu_root_attr_group);
2254         if (ret)
2255                 return ret;
2256
2257         for_each_possible_cpu(cpu) {
2258                 struct device *dev = get_cpu_device(cpu);
2259
2260                 if (!dev)
2261                         continue;
2262                 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
2263                 if (ret)
2264                         return ret;
2265         }
2266         return 0;
2267 }
2268 device_initcall(cpuhp_sysfs_init);
2269 #endif
2270
2271 /*
2272  * cpu_bit_bitmap[] is a special, "compressed" data structure that
2273  * represents all NR_CPUS bits binary values of 1<<nr.
2274  *
2275  * It is used by cpumask_of() to get a constant address to a CPU
2276  * mask value that has a single bit set only.
2277  */
2278
2279 /* cpu_bit_bitmap[0] is empty - so we can back into it */
2280 #define MASK_DECLARE_1(x)       [x+1][0] = (1UL << (x))
2281 #define MASK_DECLARE_2(x)       MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
2282 #define MASK_DECLARE_4(x)       MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
2283 #define MASK_DECLARE_8(x)       MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
2284
2285 const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
2286
2287         MASK_DECLARE_8(0),      MASK_DECLARE_8(8),
2288         MASK_DECLARE_8(16),     MASK_DECLARE_8(24),
2289 #if BITS_PER_LONG > 32
2290         MASK_DECLARE_8(32),     MASK_DECLARE_8(40),
2291         MASK_DECLARE_8(48),     MASK_DECLARE_8(56),
2292 #endif
2293 };
2294 EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
2295
2296 const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
2297 EXPORT_SYMBOL(cpu_all_bits);
2298
2299 #ifdef CONFIG_INIT_ALL_POSSIBLE
2300 struct cpumask __cpu_possible_mask __read_mostly
2301         = {CPU_BITS_ALL};
2302 #else
2303 struct cpumask __cpu_possible_mask __read_mostly;
2304 #endif
2305 EXPORT_SYMBOL(__cpu_possible_mask);
2306
2307 struct cpumask __cpu_online_mask __read_mostly;
2308 EXPORT_SYMBOL(__cpu_online_mask);
2309
2310 struct cpumask __cpu_present_mask __read_mostly;
2311 EXPORT_SYMBOL(__cpu_present_mask);
2312
2313 struct cpumask __cpu_active_mask __read_mostly;
2314 EXPORT_SYMBOL(__cpu_active_mask);
2315
2316 void init_cpu_present(const struct cpumask *src)
2317 {
2318         cpumask_copy(&__cpu_present_mask, src);
2319 }
2320
2321 void init_cpu_possible(const struct cpumask *src)
2322 {
2323         cpumask_copy(&__cpu_possible_mask, src);
2324 }
2325
2326 void init_cpu_online(const struct cpumask *src)
2327 {
2328         cpumask_copy(&__cpu_online_mask, src);
2329 }
2330
2331 /*
2332  * Activate the first processor.
2333  */
2334 void __init boot_cpu_init(void)
2335 {
2336         int cpu = smp_processor_id();
2337
2338         /* Mark the boot cpu "present", "online" etc for SMP and UP case */
2339         set_cpu_online(cpu, true);
2340         set_cpu_active(cpu, true);
2341         set_cpu_present(cpu, true);
2342         set_cpu_possible(cpu, true);
2343
2344 #ifdef CONFIG_SMP
2345         __boot_cpu_id = cpu;
2346 #endif
2347 }
2348
2349 /*
2350  * Must be called _AFTER_ setting up the per_cpu areas
2351  */
2352 void __init boot_cpu_hotplug_init(void)
2353 {
2354 #ifdef CONFIG_SMP
2355         this_cpu_write(cpuhp_state.booted_once, true);
2356 #endif
2357         this_cpu_write(cpuhp_state.state, CPUHP_ONLINE);
2358 }
2359
2360 /*
2361  * These are used for a global "mitigations=" cmdline option for toggling
2362  * optional CPU mitigations.
2363  */
2364 enum cpu_mitigations {
2365         CPU_MITIGATIONS_OFF,
2366         CPU_MITIGATIONS_AUTO,
2367         CPU_MITIGATIONS_AUTO_NOSMT,
2368 };
2369
2370 static enum cpu_mitigations cpu_mitigations __ro_after_init =
2371         CPU_MITIGATIONS_AUTO;
2372
2373 static int __init mitigations_parse_cmdline(char *arg)
2374 {
2375         if (!strcmp(arg, "off"))
2376                 cpu_mitigations = CPU_MITIGATIONS_OFF;
2377         else if (!strcmp(arg, "auto"))
2378                 cpu_mitigations = CPU_MITIGATIONS_AUTO;
2379         else if (!strcmp(arg, "auto,nosmt"))
2380                 cpu_mitigations = CPU_MITIGATIONS_AUTO_NOSMT;
2381         else
2382                 pr_crit("Unsupported mitigations=%s, system may still be vulnerable\n",
2383                         arg);
2384
2385         return 0;
2386 }
2387 early_param("mitigations", mitigations_parse_cmdline);
2388
2389 /* mitigations=off */
2390 bool cpu_mitigations_off(void)
2391 {
2392         return cpu_mitigations == CPU_MITIGATIONS_OFF;
2393 }
2394 EXPORT_SYMBOL_GPL(cpu_mitigations_off);
2395
2396 /* mitigations=auto,nosmt */
2397 bool cpu_mitigations_auto_nosmt(void)
2398 {
2399         return cpu_mitigations == CPU_MITIGATIONS_AUTO_NOSMT;
2400 }
2401 EXPORT_SYMBOL_GPL(cpu_mitigations_auto_nosmt);