kernel/cpu.c

   1 /* CPU control.
   2  * (C) 2001, 2002, 2003, 2004 Rusty Russell
   3  *
   4  * This code is licenced under the GPL.
   5  */
   6 #include <linux/proc_fs.h>
   7 #include <linux/smp.h>
   8 #include <linux/init.h>
   9 #include <linux/notifier.h>
  10 #include <linux/sched/signal.h>
  11 #include <linux/sched/hotplug.h>
  12 #include <linux/sched/task.h>
  13 #include <linux/sched/smt.h>
  14 #include <linux/unistd.h>
  15 #include <linux/cpu.h>
  16 #include <linux/oom.h>
  17 #include <linux/rcupdate.h>
  18 #include <linux/export.h>
  19 #include <linux/bug.h>
  20 #include <linux/kthread.h>
  21 #include <linux/stop_machine.h>
  22 #include <linux/mutex.h>
  23 #include <linux/gfp.h>
  24 #include <linux/suspend.h>
  25 #include <linux/lockdep.h>
  26 #include <linux/tick.h>
  27 #include <linux/irq.h>
  28 #include <linux/nmi.h>
  29 #include <linux/smpboot.h>
  30 #include <linux/relay.h>
  31 #include <linux/slab.h>
  32 #include <linux/percpu-rwsem.h>
  33 #include <linux/cpuset.h>
  34 #include <linux/random.h>
  35
  36 #include <trace/events/power.h>
  37 #define CREATE_TRACE_POINTS
  38 #include <trace/events/cpuhp.h>
  39
  40 #include "smpboot.h"
  41
  42 /**
  43  * cpuhp_cpu_state - Per cpu hotplug state storage
  44  * @state:      The current cpu state
  45  * @target:     The target state
  46  * @thread:     Pointer to the hotplug thread
  47  * @should_run: Thread should execute
  48  * @rollback:   Perform a rollback
  49  * @single:     Single callback invocation
  50  * @bringup:    Single callback bringup or teardown selector
  51  * @cb_state:   The state for a single callback (install/uninstall)
  52  * @result:     Result of the operation
  53  * @done_up:    Signal completion to the issuer of the task for cpu-up
  54  * @done_down:  Signal completion to the issuer of the task for cpu-down
  55  */
  56 struct cpuhp_cpu_state {
  57         enum cpuhp_state        state;
  58         enum cpuhp_state        target;
  59         enum cpuhp_state        fail;
  60 #ifdef CONFIG_SMP
  61         struct task_struct      *thread;
  62         bool                    should_run;
  63         bool                    rollback;
  64         bool                    single;
  65         bool                    bringup;
  66         bool                    booted_once;
  67         struct hlist_node       *node;
  68         struct hlist_node       *last;
  69         enum cpuhp_state        cb_state;
  70         int                     result;
  71         struct completion       done_up;
  72         struct completion       done_down;
  73 #endif
  74 };
  75
  76 static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
  77         .fail = CPUHP_INVALID,
  78 };
  79
  80 #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
  81 static struct lockdep_map cpuhp_state_up_map =
  82         STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
  83 static struct lockdep_map cpuhp_state_down_map =
  84         STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
  85
  86
  87 static void inline cpuhp_lock_acquire(bool bringup)
  88 {
  89         lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
  90 }
  91
  92 static void inline cpuhp_lock_release(bool bringup)
  93 {
  94         lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
  95 }
  96 #else
  97
  98 static void inline cpuhp_lock_acquire(bool bringup) { }
  99 static void inline cpuhp_lock_release(bool bringup) { }
 100
 101 #endif
 102
 103 /**
 104  * cpuhp_step - Hotplug state machine step
 105  * @name:       Name of the step
 106  * @startup:    Startup function of the step
 107  * @teardown:   Teardown function of the step
 108  * @skip_onerr: Do not invoke the functions on error rollback
 109  *              Will go away once the notifiers are gone
 110  * @cant_stop:  Bringup/teardown can't be stopped at this step
 111  */
 112 struct cpuhp_step {
 113         const char              *name;
 114         union {
 115                 int             (*single)(unsigned int cpu);
 116                 int             (*multi)(unsigned int cpu,
 117                                          struct hlist_node *node);
 118         } startup;
 119         union {
 120                 int             (*single)(unsigned int cpu);
 121                 int             (*multi)(unsigned int cpu,
 122                                          struct hlist_node *node);
 123         } teardown;
 124         struct hlist_head       list;
 125         bool                    skip_onerr;
 126         bool                    cant_stop;
 127         bool                    multi_instance;
 128 };
 129
 130 static DEFINE_MUTEX(cpuhp_state_mutex);
 131 static struct cpuhp_step cpuhp_bp_states[];
 132 static struct cpuhp_step cpuhp_ap_states[];
 133
 134 static bool cpuhp_is_ap_state(enum cpuhp_state state)
 135 {
 136         /*
 137          * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
 138          * purposes as that state is handled explicitly in cpu_down.
 139          */
 140         return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
 141 }
 142
 143 static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
 144 {
 145         struct cpuhp_step *sp;
 146
 147         sp = cpuhp_is_ap_state(state) ? cpuhp_ap_states : cpuhp_bp_states;
 148         return sp + state;
 149 }
 150
 151 /**
 152  * cpuhp_invoke_callback _ Invoke the callbacks for a given state
 153  * @cpu:        The cpu for which the callback should be invoked
 154  * @state:      The state to do callbacks for
 155  * @bringup:    True if the bringup callback should be invoked
 156  * @node:       For multi-instance, do a single entry callback for install/remove
 157  * @lastp:      For multi-instance rollback, remember how far we got
 158  *
 159  * Called from cpu hotplug and from the state register machinery.
 160  */
 161 static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
 162                                  bool bringup, struct hlist_node *node,
 163                                  struct hlist_node **lastp)
 164 {
 165         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 166         struct cpuhp_step *step = cpuhp_get_step(state);
 167         int (*cbm)(unsigned int cpu, struct hlist_node *node);
 168         int (*cb)(unsigned int cpu);
 169         int ret, cnt;
 170
 171         if (st->fail == state) {
 172                 st->fail = CPUHP_INVALID;
 173
 174                 if (!(bringup ? step->startup.single : step->teardown.single))
 175                         return 0;
 176
 177                 return -EAGAIN;
 178         }
 179
 180         if (!step->multi_instance) {
 181                 WARN_ON_ONCE(lastp && *lastp);
 182                 cb = bringup ? step->startup.single : step->teardown.single;
 183                 if (!cb)
 184                         return 0;
 185                 trace_cpuhp_enter(cpu, st->target, state, cb);
 186                 ret = cb(cpu);
 187                 trace_cpuhp_exit(cpu, st->state, state, ret);
 188                 return ret;
 189         }
 190         cbm = bringup ? step->startup.multi : step->teardown.multi;
 191         if (!cbm)
 192                 return 0;
 193
 194         /* Single invocation for instance add/remove */
 195         if (node) {
 196                 WARN_ON_ONCE(lastp && *lastp);
 197                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 198                 ret = cbm(cpu, node);
 199                 trace_cpuhp_exit(cpu, st->state, state, ret);
 200                 return ret;
 201         }
 202
 203         /* State transition. Invoke on all instances */
 204         cnt = 0;
 205         hlist_for_each(node, &step->list) {
 206                 if (lastp && node == *lastp)
 207                         break;
 208
 209                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 210                 ret = cbm(cpu, node);
 211                 trace_cpuhp_exit(cpu, st->state, state, ret);
 212                 if (ret) {
 213                         if (!lastp)
 214                                 goto err;
 215
 216                         *lastp = node;
 217                         return ret;
 218                 }
 219                 cnt++;
 220         }
 221         if (lastp)
 222                 *lastp = NULL;
 223         return 0;
 224 err:
 225         /* Rollback the instances if one failed */
 226         cbm = !bringup ? step->startup.multi : step->teardown.multi;
 227         if (!cbm)
 228                 return ret;
 229
 230         hlist_for_each(node, &step->list) {
 231                 if (!cnt--)
 232                         break;
 233
 234                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 235                 ret = cbm(cpu, node);
 236                 trace_cpuhp_exit(cpu, st->state, state, ret);
 237                 /*
 238                  * Rollback must not fail,
 239                  */
 240                 WARN_ON_ONCE(ret);
 241         }
 242         return ret;
 243 }
 244
 245 #ifdef CONFIG_SMP
 246 static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
 247 {
 248         struct completion *done = bringup ? &st->done_up : &st->done_down;
 249         wait_for_completion(done);
 250 }
 251
 252 static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
 253 {
 254         struct completion *done = bringup ? &st->done_up : &st->done_down;
 255         complete(done);
 256 }
 257
 258 /*
 259  * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
 260  */
 261 static bool cpuhp_is_atomic_state(enum cpuhp_state state)
 262 {
 263         return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
 264 }
 265
 266 /* Serializes the updates to cpu_online_mask, cpu_present_mask */
 267 static DEFINE_MUTEX(cpu_add_remove_lock);
 268 bool cpuhp_tasks_frozen;
 269 EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
 270
 271 /*
 272  * The following two APIs (cpu_maps_update_begin/done) must be used when
 273  * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
 274  */
 275 void cpu_maps_update_begin(void)
 276 {
 277         mutex_lock(&cpu_add_remove_lock);
 278 }
 279
 280 void cpu_maps_update_done(void)
 281 {
 282         mutex_unlock(&cpu_add_remove_lock);
 283 }
 284
 285 /*
 286  * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
 287  * Should always be manipulated under cpu_add_remove_lock
 288  */
 289 static int cpu_hotplug_disabled;
 290
 291 #ifdef CONFIG_HOTPLUG_CPU
 292
 293 DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
 294
 295 void cpus_read_lock(void)
 296 {
 297         percpu_down_read(&cpu_hotplug_lock);
 298 }
 299 EXPORT_SYMBOL_GPL(cpus_read_lock);
 300
 301 void cpus_read_unlock(void)
 302 {
 303         percpu_up_read(&cpu_hotplug_lock);
 304 }
 305 EXPORT_SYMBOL_GPL(cpus_read_unlock);
 306
 307 void cpus_write_lock(void)
 308 {
 309         percpu_down_write(&cpu_hotplug_lock);
 310 }
 311
 312 void cpus_write_unlock(void)
 313 {
 314         percpu_up_write(&cpu_hotplug_lock);
 315 }
 316
 317 void lockdep_assert_cpus_held(void)
 318 {
 319         /*
 320          * We can't have hotplug operations before userspace starts running,
 321          * and some init codepaths will knowingly not take the hotplug lock.
 322          * This is all valid, so mute lockdep until it makes sense to report
 323          * unheld locks.
 324          */
 325         if (system_state < SYSTEM_RUNNING)
 326                 return;
 327
 328         percpu_rwsem_assert_held(&cpu_hotplug_lock);
 329 }
 330
 331 /*
 332  * Wait for currently running CPU hotplug operations to complete (if any) and
 333  * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
 334  * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
 335  * hotplug path before performing hotplug operations. So acquiring that lock
 336  * guarantees mutual exclusion from any currently running hotplug operations.
 337  */
 338 void cpu_hotplug_disable(void)
 339 {
 340         cpu_maps_update_begin();
 341         cpu_hotplug_disabled++;
 342         cpu_maps_update_done();
 343 }
 344 EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
 345
 346 static void __cpu_hotplug_enable(void)
 347 {
 348         if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
 349                 return;
 350         cpu_hotplug_disabled--;
 351 }
 352
 353 void cpu_hotplug_enable(void)
 354 {
 355         cpu_maps_update_begin();
 356         __cpu_hotplug_enable();
 357         cpu_maps_update_done();
 358 }
 359 EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
 360 #endif  /* CONFIG_HOTPLUG_CPU */
 361
 362 /*
 363  * Architectures that need SMT-specific errata handling during SMT hotplug
 364  * should override this.
 365  */
 366 void __weak arch_smt_update(void) { }
 367
 368 #ifdef CONFIG_HOTPLUG_SMT
 369 enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;
 370
 371 void __init cpu_smt_disable(bool force)
 372 {
 373         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
 374                 cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
 375                 return;
 376
 377         if (force) {
 378                 pr_info("SMT: Force disabled\n");
 379                 cpu_smt_control = CPU_SMT_FORCE_DISABLED;
 380         } else {
 381                 pr_info("SMT: disabled\n");
 382                 cpu_smt_control = CPU_SMT_DISABLED;
 383         }
 384 }
 385
 386 /*
 387  * The decision whether SMT is supported can only be done after the full
 388  * CPU identification. Called from architecture code.
 389  */
 390 void __init cpu_smt_check_topology(void)
 391 {
 392         if (!topology_smt_supported())
 393                 cpu_smt_control = CPU_SMT_NOT_SUPPORTED;
 394 }
 395
 396 static int __init smt_cmdline_disable(char *str)
 397 {
 398         cpu_smt_disable(str && !strcmp(str, "force"));
 399         return 0;
 400 }
 401 early_param("nosmt", smt_cmdline_disable);
 402
 403 static inline bool cpu_smt_allowed(unsigned int cpu)
 404 {
 405         if (cpu_smt_control == CPU_SMT_ENABLED)
 406                 return true;
 407
 408         if (topology_is_primary_thread(cpu))
 409                 return true;
 410
 411         /*
 412          * On x86 it's required to boot all logical CPUs at least once so
 413          * that the init code can get a chance to set CR4.MCE on each
 414          * CPU. Otherwise, a broadacasted MCE observing CR4.MCE=0b on any
 415          * core will shutdown the machine.
 416          */
 417         return !per_cpu(cpuhp_state, cpu).booted_once;
 418 }
 419 #else
 420 static inline bool cpu_smt_allowed(unsigned int cpu) { return true; }
 421 #endif
 422
 423 static inline enum cpuhp_state
 424 cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target)
 425 {
 426         enum cpuhp_state prev_state = st->state;
 427
 428         st->rollback = false;
 429         st->last = NULL;
 430
 431         st->target = target;
 432         st->single = false;
 433         st->bringup = st->state < target;
 434
 435         return prev_state;
 436 }
 437
 438 static inline void
 439 cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state)
 440 {
 441         st->rollback = true;
 442
 443         /*
 444          * If we have st->last we need to undo partial multi_instance of this
 445          * state first. Otherwise start undo at the previous state.
 446          */
 447         if (!st->last) {
 448                 if (st->bringup)
 449                         st->state--;
 450                 else
 451                         st->state++;
 452         }
 453
 454         st->target = prev_state;
 455         st->bringup = !st->bringup;
 456 }
 457
 458 /* Regular hotplug invocation of the AP hotplug thread */
 459 static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
 460 {
 461         if (!st->single && st->state == st->target)
 462                 return;
 463
 464         st->result = 0;
 465         /*
 466          * Make sure the above stores are visible before should_run becomes
 467          * true. Paired with the mb() above in cpuhp_thread_fun()
 468          */
 469         smp_mb();
 470         st->should_run = true;
 471         wake_up_process(st->thread);
 472         wait_for_ap_thread(st, st->bringup);
 473 }
 474
 475 static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target)
 476 {
 477         enum cpuhp_state prev_state;
 478         int ret;
 479
 480         prev_state = cpuhp_set_state(st, target);
 481         __cpuhp_kick_ap(st);
 482         if ((ret = st->result)) {
 483                 cpuhp_reset_state(st, prev_state);
 484                 __cpuhp_kick_ap(st);
 485         }
 486
 487         return ret;
 488 }
 489
 490 static int bringup_wait_for_ap(unsigned int cpu)
 491 {
 492         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 493
 494         /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
 495         wait_for_ap_thread(st, true);
 496         if (WARN_ON_ONCE((!cpu_online(cpu))))
 497                 return -ECANCELED;
 498
 499         /* Unpark the hotplug thread of the target cpu */
 500         kthread_unpark(st->thread);
 501
 502         /*
 503          * SMT soft disabling on X86 requires to bring the CPU out of the
 504          * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit.  The
 505          * CPU marked itself as booted_once in cpu_notify_starting() so the
 506          * cpu_smt_allowed() check will now return false if this is not the
 507          * primary sibling.
 508          */
 509         if (!cpu_smt_allowed(cpu))
 510                 return -ECANCELED;
 511
 512         if (st->target <= CPUHP_AP_ONLINE_IDLE)
 513                 return 0;
 514
 515         return cpuhp_kick_ap(st, st->target);
 516 }
 517
 518 static int bringup_cpu(unsigned int cpu)
 519 {
 520         struct task_struct *idle = idle_thread_get(cpu);
 521         int ret;
 522
 523         /*
 524          * Some architectures have to walk the irq descriptors to
 525          * setup the vector space for the cpu which comes online.
 526          * Prevent irq alloc/free across the bringup.
 527          */
 528         irq_lock_sparse();
 529
 530         /* Arch-specific enabling code. */
 531         ret = __cpu_up(cpu, idle);
 532         irq_unlock_sparse();
 533         if (ret)
 534                 return ret;
 535         return bringup_wait_for_ap(cpu);
 536 }
 537
 538 /*
 539  * Hotplug state machine related functions
 540  */
 541
 542 static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st)
 543 {
 544         for (st->state--; st->state > st->target; st->state--) {
 545                 struct cpuhp_step *step = cpuhp_get_step(st->state);
 546
 547                 if (!step->skip_onerr)
 548                         cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
 549         }
 550 }
 551
 552 static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st)
 553 {
 554         if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
 555                 return true;
 556         /*
 557          * When CPU hotplug is disabled, then taking the CPU down is not
 558          * possible because takedown_cpu() and the architecture and
 559          * subsystem specific mechanisms are not available. So the CPU
 560          * which would be completely unplugged again needs to stay around
 561          * in the current state.
 562          */
 563         return st->state <= CPUHP_BRINGUP_CPU;
 564 }
 565
 566 static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
 567                               enum cpuhp_state target)
 568 {
 569         enum cpuhp_state prev_state = st->state;
 570         int ret = 0;
 571
 572         while (st->state < target) {
 573                 st->state++;
 574                 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
 575                 if (ret) {
 576                         if (can_rollback_cpu(st)) {
 577                                 st->target = prev_state;
 578                                 undo_cpu_up(cpu, st);
 579                         }
 580                         break;
 581                 }
 582         }
 583         return ret;
 584 }
 585
 586 /*
 587  * The cpu hotplug threads manage the bringup and teardown of the cpus
 588  */
 589 static void cpuhp_create(unsigned int cpu)
 590 {
 591         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 592
 593         init_completion(&st->done_up);
 594         init_completion(&st->done_down);
 595 }
 596
 597 static int cpuhp_should_run(unsigned int cpu)
 598 {
 599         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 600
 601         return st->should_run;
 602 }
 603
 604 /*
 605  * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
 606  * callbacks when a state gets [un]installed at runtime.
 607  *
 608  * Each invocation of this function by the smpboot thread does a single AP
 609  * state callback.
 610  *
 611  * It has 3 modes of operation:
 612  *  - single: runs st->cb_state
 613  *  - up:     runs ++st->state, while st->state < st->target
 614  *  - down:   runs st->state--, while st->state > st->target
 615  *
 616  * When complete or on error, should_run is cleared and the completion is fired.
 617  */
 618 static void cpuhp_thread_fun(unsigned int cpu)
 619 {
 620         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 621         bool bringup = st->bringup;
 622         enum cpuhp_state state;
 623
 624         if (WARN_ON_ONCE(!st->should_run))
 625                 return;
 626
 627         /*
 628          * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
 629          * that if we see ->should_run we also see the rest of the state.
 630          */
 631         smp_mb();
 632
 633         cpuhp_lock_acquire(bringup);
 634
 635         if (st->single) {
 636                 state = st->cb_state;
 637                 st->should_run = false;
 638         } else {
 639                 if (bringup) {
 640                         st->state++;
 641                         state = st->state;
 642                         st->should_run = (st->state < st->target);
 643                         WARN_ON_ONCE(st->state > st->target);
 644                 } else {
 645                         state = st->state;
 646                         st->state--;
 647                         st->should_run = (st->state > st->target);
 648                         WARN_ON_ONCE(st->state < st->target);
 649                 }
 650         }
 651
 652         WARN_ON_ONCE(!cpuhp_is_ap_state(state));
 653
 654         if (st->rollback) {
 655                 struct cpuhp_step *step = cpuhp_get_step(state);
 656                 if (step->skip_onerr)
 657                         goto next;
 658         }
 659
 660         if (cpuhp_is_atomic_state(state)) {
 661                 local_irq_disable();
 662                 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
 663                 local_irq_enable();
 664
 665                 /*
 666                  * STARTING/DYING must not fail!
 667                  */
 668                 WARN_ON_ONCE(st->result);
 669         } else {
 670                 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
 671         }
 672
 673         if (st->result) {
 674                 /*
 675                  * If we fail on a rollback, we're up a creek without no
 676                  * paddle, no way forward, no way back. We loose, thanks for
 677                  * playing.
 678                  */
 679                 WARN_ON_ONCE(st->rollback);
 680                 st->should_run = false;
 681         }
 682
 683 next:
 684         cpuhp_lock_release(bringup);
 685
 686         if (!st->should_run)
 687                 complete_ap_thread(st, bringup);
 688 }
 689
 690 /* Invoke a single callback on a remote cpu */
 691 static int
 692 cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
 693                          struct hlist_node *node)
 694 {
 695         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 696         int ret;
 697
 698         if (!cpu_online(cpu))
 699                 return 0;
 700
 701         cpuhp_lock_acquire(false);
 702         cpuhp_lock_release(false);
 703
 704         cpuhp_lock_acquire(true);
 705         cpuhp_lock_release(true);
 706
 707         /*
 708          * If we are up and running, use the hotplug thread. For early calls
 709          * we invoke the thread function directly.
 710          */
 711         if (!st->thread)
 712                 return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
 713
 714         st->rollback = false;
 715         st->last = NULL;
 716
 717         st->node = node;
 718         st->bringup = bringup;
 719         st->cb_state = state;
 720         st->single = true;
 721
 722         __cpuhp_kick_ap(st);
 723
 724         /*
 725          * If we failed and did a partial, do a rollback.
 726          */
 727         if ((ret = st->result) && st->last) {
 728                 st->rollback = true;
 729                 st->bringup = !bringup;
 730
 731                 __cpuhp_kick_ap(st);
 732         }
 733
 734         /*
 735          * Clean up the leftovers so the next hotplug operation wont use stale
 736          * data.
 737          */
 738         st->node = st->last = NULL;
 739         return ret;
 740 }
 741
 742 static int cpuhp_kick_ap_work(unsigned int cpu)
 743 {
 744         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 745         enum cpuhp_state prev_state = st->state;
 746         int ret;
 747
 748         cpuhp_lock_acquire(false);
 749         cpuhp_lock_release(false);
 750
 751         cpuhp_lock_acquire(true);
 752         cpuhp_lock_release(true);
 753
 754         trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
 755         ret = cpuhp_kick_ap(st, st->target);
 756         trace_cpuhp_exit(cpu, st->state, prev_state, ret);
 757
 758         return ret;
 759 }
 760
 761 static struct smp_hotplug_thread cpuhp_threads = {
 762         .store                  = &cpuhp_state.thread,
 763         .create                 = &cpuhp_create,
 764         .thread_should_run      = cpuhp_should_run,
 765         .thread_fn              = cpuhp_thread_fun,
 766         .thread_comm            = "cpuhp/%u",
 767         .selfparking            = true,
 768 };
 769
 770 void __init cpuhp_threads_init(void)
 771 {
 772         BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
 773         kthread_unpark(this_cpu_read(cpuhp_state.thread));
 774 }
 775
 776 /*
 777  *
 778  * Serialize hotplug trainwrecks outside of the cpu_hotplug_lock
 779  * protected region.
 780  *
 781  * The operation is still serialized against concurrent CPU hotplug via
 782  * cpu_add_remove_lock, i.e. CPU map protection.  But it is _not_
 783  * serialized against other hotplug related activity like adding or
 784  * removing of state callbacks and state instances, which invoke either the
 785  * startup or the teardown callback of the affected state.
 786  *
 787  * This is required for subsystems which are unfixable vs. CPU hotplug and
 788  * evade lock inversion problems by scheduling work which has to be
 789  * completed _before_ cpu_up()/_cpu_down() returns.
 790  *
 791  * Don't even think about adding anything to this for any new code or even
 792  * drivers. It's only purpose is to keep existing lock order trainwrecks
 793  * working.
 794  *
 795  * For cpu_down() there might be valid reasons to finish cleanups which are
 796  * not required to be done under cpu_hotplug_lock, but that's a different
 797  * story and would be not invoked via this.
 798  */
 799 static void cpu_up_down_serialize_trainwrecks(bool tasks_frozen)
 800 {
 801         /*
 802          * cpusets delegate hotplug operations to a worker to "solve" the
 803          * lock order problems. Wait for the worker, but only if tasks are
 804          * _not_ frozen (suspend, hibernate) as that would wait forever.
 805          *
 806          * The wait is required because otherwise the hotplug operation
 807          * returns with inconsistent state, which could even be observed in
 808          * user space when a new CPU is brought up. The CPU plug uevent
 809          * would be delivered and user space reacting on it would fail to
 810          * move tasks to the newly plugged CPU up to the point where the
 811          * work has finished because up to that point the newly plugged CPU
 812          * is not assignable in cpusets/cgroups. On unplug that's not
 813          * necessarily a visible issue, but it is still inconsistent state,
 814          * which is the real problem which needs to be "fixed". This can't
 815          * prevent the transient state between scheduling the work and
 816          * returning from waiting for it.
 817          */
 818         if (!tasks_frozen)
 819                 cpuset_wait_for_hotplug();
 820 }
 821
 822 #ifdef CONFIG_HOTPLUG_CPU
 823 #ifndef arch_clear_mm_cpumask_cpu
 824 #define arch_clear_mm_cpumask_cpu(cpu, mm) cpumask_clear_cpu(cpu, mm_cpumask(mm))
 825 #endif
 826
 827 /**
 828  * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
 829  * @cpu: a CPU id
 830  *
 831  * This function walks all processes, finds a valid mm struct for each one and
 832  * then clears a corresponding bit in mm's cpumask.  While this all sounds
 833  * trivial, there are various non-obvious corner cases, which this function
 834  * tries to solve in a safe manner.
 835  *
 836  * Also note that the function uses a somewhat relaxed locking scheme, so it may
 837  * be called only for an already offlined CPU.
 838  */
 839 void clear_tasks_mm_cpumask(int cpu)
 840 {
 841         struct task_struct *p;
 842
 843         /*
 844          * This function is called after the cpu is taken down and marked
 845          * offline, so its not like new tasks will ever get this cpu set in
 846          * their mm mask. -- Peter Zijlstra
 847          * Thus, we may use rcu_read_lock() here, instead of grabbing
 848          * full-fledged tasklist_lock.
 849          */
 850         WARN_ON(cpu_online(cpu));
 851         rcu_read_lock();
 852         for_each_process(p) {
 853                 struct task_struct *t;
 854
 855                 /*
 856                  * Main thread might exit, but other threads may still have
 857                  * a valid mm. Find one.
 858                  */
 859                 t = find_lock_task_mm(p);
 860                 if (!t)
 861                         continue;
 862                 arch_clear_mm_cpumask_cpu(cpu, t->mm);
 863                 task_unlock(t);
 864         }
 865         rcu_read_unlock();
 866 }
 867
 868 /* Take this CPU down. */
 869 static int take_cpu_down(void *_param)
 870 {
 871         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 872         enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
 873         int err, cpu = smp_processor_id();
 874         int ret;
 875
 876         /* Ensure this CPU doesn't handle any more interrupts. */
 877         err = __cpu_disable();
 878         if (err < 0)
 879                 return err;
 880
 881         /*
 882          * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not
 883          * do this step again.
 884          */
 885         WARN_ON(st->state != CPUHP_TEARDOWN_CPU);
 886         st->state--;
 887         /* Invoke the former CPU_DYING callbacks */
 888         for (; st->state > target; st->state--) {
 889                 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
 890                 /*
 891                  * DYING must not fail!
 892                  */
 893                 WARN_ON_ONCE(ret);
 894         }
 895
 896         /* Give up timekeeping duties */
 897         tick_handover_do_timer();
 898         /* Park the stopper thread */
 899         stop_machine_park(cpu);
 900         return 0;
 901 }
 902
 903 static int takedown_cpu(unsigned int cpu)
 904 {
 905         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 906         int err;
 907
 908         /* Park the smpboot threads */
 909         kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread);
 910
 911         /*
 912          * Prevent irq alloc/free while the dying cpu reorganizes the
 913          * interrupt affinities.
 914          */
 915         irq_lock_sparse();
 916
 917         /*
 918          * So now all preempt/rcu users must observe !cpu_active().
 919          */
 920         err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
 921         if (err) {
 922                 /* CPU refused to die */
 923                 irq_unlock_sparse();
 924                 /* Unpark the hotplug thread so we can rollback there */
 925                 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread);
 926                 return err;
 927         }
 928         BUG_ON(cpu_online(cpu));
 929
 930         /*
 931          * The CPUHP_AP_SCHED_MIGRATE_DYING callback will have removed all
 932          * runnable tasks from the cpu, there's only the idle task left now
 933          * that the migration thread is done doing the stop_machine thing.
 934          *
 935          * Wait for the stop thread to go away.
 936          */
 937         wait_for_ap_thread(st, false);
 938         BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
 939
 940         /* Interrupts are moved away from the dying cpu, reenable alloc/free */
 941         irq_unlock_sparse();
 942
 943         hotplug_cpu__broadcast_tick_pull(cpu);
 944         /* This actually kills the CPU. */
 945         __cpu_die(cpu);
 946
 947         tick_cleanup_dead_cpu(cpu);
 948         rcutree_migrate_callbacks(cpu);
 949         return 0;
 950 }
 951
 952 static void cpuhp_complete_idle_dead(void *arg)
 953 {
 954         struct cpuhp_cpu_state *st = arg;
 955
 956         complete_ap_thread(st, false);
 957 }
 958
 959 void cpuhp_report_idle_dead(void)
 960 {
 961         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 962
 963         BUG_ON(st->state != CPUHP_AP_OFFLINE);
 964         rcu_report_dead(smp_processor_id());
 965         st->state = CPUHP_AP_IDLE_DEAD;
 966         /*
 967          * We cannot call complete after rcu_report_dead() so we delegate it
 968          * to an online cpu.
 969          */
 970         smp_call_function_single(cpumask_first(cpu_online_mask),
 971                                  cpuhp_complete_idle_dead, st, 0);
 972 }
 973
 974 static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st)
 975 {
 976         for (st->state++; st->state < st->target; st->state++) {
 977                 struct cpuhp_step *step = cpuhp_get_step(st->state);
 978
 979                 if (!step->skip_onerr)
 980                         cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
 981         }
 982 }
 983
 984 static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
 985                                 enum cpuhp_state target)
 986 {
 987         enum cpuhp_state prev_state = st->state;
 988         int ret = 0;
 989
 990         for (; st->state > target; st->state--) {
 991                 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
 992                 if (ret) {
 993                         st->target = prev_state;
 994                         if (st->state < prev_state)
 995                                 undo_cpu_down(cpu, st);
 996                         break;
 997                 }
 998         }
 999         return ret;
1000 }
1001
1002 /* Requires cpu_add_remove_lock to be held */
1003 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
1004                            enum cpuhp_state target)
1005 {
1006         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1007         int prev_state, ret = 0;
1008
1009         if (num_online_cpus() == 1)
1010                 return -EBUSY;
1011
1012         if (!cpu_present(cpu))
1013                 return -EINVAL;
1014
1015         cpus_write_lock();
1016
1017         cpuhp_tasks_frozen = tasks_frozen;
1018
1019         prev_state = cpuhp_set_state(st, target);
1020         /*
1021          * If the current CPU state is in the range of the AP hotplug thread,
1022          * then we need to kick the thread.
1023          */
1024         if (st->state > CPUHP_TEARDOWN_CPU) {
1025                 st->target = max((int)target, CPUHP_TEARDOWN_CPU);
1026                 ret = cpuhp_kick_ap_work(cpu);
1027                 /*
1028                  * The AP side has done the error rollback already. Just
1029                  * return the error code..
1030                  */
1031                 if (ret)
1032                         goto out;
1033
1034                 /*
1035                  * We might have stopped still in the range of the AP hotplug
1036                  * thread. Nothing to do anymore.
1037                  */
1038                 if (st->state > CPUHP_TEARDOWN_CPU)
1039                         goto out;
1040
1041                 st->target = target;
1042         }
1043         /*
1044          * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
1045          * to do the further cleanups.
1046          */
1047         ret = cpuhp_down_callbacks(cpu, st, target);
1048         if (ret && st->state == CPUHP_TEARDOWN_CPU && st->state < prev_state) {
1049                 cpuhp_reset_state(st, prev_state);
1050                 __cpuhp_kick_ap(st);
1051         }
1052
1053 out:
1054         cpus_write_unlock();
1055         /*
1056          * Do post unplug cleanup. This is still protected against
1057          * concurrent CPU hotplug via cpu_add_remove_lock.
1058          */
1059         lockup_detector_cleanup();
1060         arch_smt_update();
1061         cpu_up_down_serialize_trainwrecks(tasks_frozen);
1062         return ret;
1063 }
1064
1065 static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
1066 {
1067         if (cpu_hotplug_disabled)
1068                 return -EBUSY;
1069         return _cpu_down(cpu, 0, target);
1070 }
1071
1072 static int do_cpu_down(unsigned int cpu, enum cpuhp_state target)
1073 {
1074         int err;
1075
1076         cpu_maps_update_begin();
1077         err = cpu_down_maps_locked(cpu, target);
1078         cpu_maps_update_done();
1079         return err;
1080 }
1081
1082 int cpu_down(unsigned int cpu)
1083 {
1084         return do_cpu_down(cpu, CPUHP_OFFLINE);
1085 }
1086 EXPORT_SYMBOL(cpu_down);
1087
1088 #else
1089 #define takedown_cpu            NULL
1090 #endif /*CONFIG_HOTPLUG_CPU*/
1091
1092 /**
1093  * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1094  * @cpu: cpu that just started
1095  *
1096  * It must be called by the arch code on the new cpu, before the new cpu
1097  * enables interrupts and before the "boot" cpu returns from __cpu_up().
1098  */
1099 void notify_cpu_starting(unsigned int cpu)
1100 {
1101         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1102         enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
1103         int ret;
1104
1105         rcu_cpu_starting(cpu);  /* Enables RCU usage on this CPU. */
1106         st->booted_once = true;
1107         while (st->state < target) {
1108                 st->state++;
1109                 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
1110                 /*
1111                  * STARTING must not fail!
1112                  */
1113                 WARN_ON_ONCE(ret);
1114         }
1115 }
1116
1117 /*
1118  * Called from the idle task. Wake up the controlling task which brings the
1119  * hotplug thread of the upcoming CPU up and then delegates the rest of the
1120  * online bringup to the hotplug thread.
1121  */
1122 void cpuhp_online_idle(enum cpuhp_state state)
1123 {
1124         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1125
1126         /* Happens for the boot cpu */
1127         if (state != CPUHP_AP_ONLINE_IDLE)
1128                 return;
1129
1130         /*
1131          * Unpart the stopper thread before we start the idle loop (and start
1132          * scheduling); this ensures the stopper task is always available.
1133          */
1134         stop_machine_unpark(smp_processor_id());
1135
1136         st->state = CPUHP_AP_ONLINE_IDLE;
1137         complete_ap_thread(st, true);
1138 }
1139
1140 /* Requires cpu_add_remove_lock to be held */
1141 static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
1142 {
1143         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1144         struct task_struct *idle;
1145         int ret = 0;
1146
1147         cpus_write_lock();
1148
1149         if (!cpu_present(cpu)) {
1150                 ret = -EINVAL;
1151                 goto out;
1152         }
1153
1154         /*
1155          * The caller of do_cpu_up might have raced with another
1156          * caller. Ignore it for now.
1157          */
1158         if (st->state >= target)
1159                 goto out;
1160
1161         if (st->state == CPUHP_OFFLINE) {
1162                 /* Let it fail before we try to bring the cpu up */
1163                 idle = idle_thread_get(cpu);
1164                 if (IS_ERR(idle)) {
1165                         ret = PTR_ERR(idle);
1166                         goto out;
1167                 }
1168         }
1169
1170         cpuhp_tasks_frozen = tasks_frozen;
1171
1172         cpuhp_set_state(st, target);
1173         /*
1174          * If the current CPU state is in the range of the AP hotplug thread,
1175          * then we need to kick the thread once more.
1176          */
1177         if (st->state > CPUHP_BRINGUP_CPU) {
1178                 ret = cpuhp_kick_ap_work(cpu);
1179                 /*
1180                  * The AP side has done the error rollback already. Just
1181                  * return the error code..
1182                  */
1183                 if (ret)
1184                         goto out;
1185         }
1186
1187         /*
1188          * Try to reach the target state. We max out on the BP at
1189          * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
1190          * responsible for bringing it up to the target state.
1191          */
1192         target = min((int)target, CPUHP_BRINGUP_CPU);
1193         ret = cpuhp_up_callbacks(cpu, st, target);
1194 out:
1195         cpus_write_unlock();
1196         arch_smt_update();
1197         cpu_up_down_serialize_trainwrecks(tasks_frozen);
1198         return ret;
1199 }
1200
1201 static int do_cpu_up(unsigned int cpu, enum cpuhp_state target)
1202 {
1203         int err = 0;
1204
1205         if (!cpu_possible(cpu)) {
1206                 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1207                        cpu);
1208 #if defined(CONFIG_IA64)
1209                 pr_err("please check additional_cpus= boot parameter\n");
1210 #endif
1211                 return -EINVAL;
1212         }
1213
1214         err = try_online_node(cpu_to_node(cpu));
1215         if (err)
1216                 return err;
1217
1218         cpu_maps_update_begin();
1219
1220         if (cpu_hotplug_disabled) {
1221                 err = -EBUSY;
1222                 goto out;
1223         }
1224         if (!cpu_smt_allowed(cpu)) {
1225                 err = -EPERM;
1226                 goto out;
1227         }
1228
1229         err = _cpu_up(cpu, 0, target);
1230 out:
1231         cpu_maps_update_done();
1232         return err;
1233 }
1234
1235 int cpu_up(unsigned int cpu)
1236 {
1237         return do_cpu_up(cpu, CPUHP_ONLINE);
1238 }
1239 EXPORT_SYMBOL_GPL(cpu_up);
1240
1241 #ifdef CONFIG_PM_SLEEP_SMP
1242 static cpumask_var_t frozen_cpus;
1243
1244 int freeze_secondary_cpus(int primary)
1245 {
1246         int cpu, error = 0;
1247
1248         cpu_maps_update_begin();
1249         if (!cpu_online(primary))
1250                 primary = cpumask_first(cpu_online_mask);
1251         /*
1252          * We take down all of the non-boot CPUs in one shot to avoid races
1253          * with the userspace trying to use the CPU hotplug at the same time
1254          */
1255         cpumask_clear(frozen_cpus);
1256
1257         pr_info("Disabling non-boot CPUs ...\n");
1258         for_each_online_cpu(cpu) {
1259                 if (cpu == primary)
1260                         continue;
1261                 trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
1262                 error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
1263                 trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
1264                 if (!error)
1265                         cpumask_set_cpu(cpu, frozen_cpus);
1266                 else {
1267                         pr_err("Error taking CPU%d down: %d\n", cpu, error);
1268                         break;
1269                 }
1270         }
1271
1272         if (!error)
1273                 BUG_ON(num_online_cpus() > 1);
1274         else
1275                 pr_err("Non-boot CPUs are not disabled\n");
1276
1277         /*
1278          * Make sure the CPUs won't be enabled by someone else. We need to do
1279          * this even in case of failure as all disable_nonboot_cpus() users are
1280          * supposed to do enable_nonboot_cpus() on the failure path.
1281          */
1282         cpu_hotplug_disabled++;
1283
1284         cpu_maps_update_done();
1285         return error;
1286 }
1287
1288 void __weak arch_enable_nonboot_cpus_begin(void)
1289 {
1290 }
1291
1292 void __weak arch_enable_nonboot_cpus_end(void)
1293 {
1294 }
1295
1296 void enable_nonboot_cpus(void)
1297 {
1298         int cpu, error;
1299
1300         /* Allow everyone to use the CPU hotplug again */
1301         cpu_maps_update_begin();
1302         __cpu_hotplug_enable();
1303         if (cpumask_empty(frozen_cpus))
1304                 goto out;
1305
1306         pr_info("Enabling non-boot CPUs ...\n");
1307
1308         arch_enable_nonboot_cpus_begin();
1309
1310         for_each_cpu(cpu, frozen_cpus) {
1311                 trace_suspend_resume(TPS("CPU_ON"), cpu, true);
1312                 error = _cpu_up(cpu, 1, CPUHP_ONLINE);
1313                 trace_suspend_resume(TPS("CPU_ON"), cpu, false);
1314                 if (!error) {
1315                         pr_info("CPU%d is up\n", cpu);
1316                         continue;
1317                 }
1318                 pr_warn("Error taking CPU%d up: %d\n", cpu, error);
1319         }
1320
1321         arch_enable_nonboot_cpus_end();
1322
1323         cpumask_clear(frozen_cpus);
1324 out:
1325         cpu_maps_update_done();
1326 }
1327
1328 static int __init alloc_frozen_cpus(void)
1329 {
1330         if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
1331                 return -ENOMEM;
1332         return 0;
1333 }
1334 core_initcall(alloc_frozen_cpus);
1335
1336 /*
1337  * When callbacks for CPU hotplug notifications are being executed, we must
1338  * ensure that the state of the system with respect to the tasks being frozen
1339  * or not, as reported by the notification, remains unchanged *throughout the
1340  * duration* of the execution of the callbacks.
1341  * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1342  *
1343  * This synchronization is implemented by mutually excluding regular CPU
1344  * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1345  * Hibernate notifications.
1346  */
1347 static int
1348 cpu_hotplug_pm_callback(struct notifier_block *nb,
1349                         unsigned long action, void *ptr)
1350 {
1351         switch (action) {
1352
1353         case PM_SUSPEND_PREPARE:
1354         case PM_HIBERNATION_PREPARE:
1355                 cpu_hotplug_disable();
1356                 break;
1357
1358         case PM_POST_SUSPEND:
1359         case PM_POST_HIBERNATION:
1360                 cpu_hotplug_enable();
1361                 break;
1362
1363         default:
1364                 return NOTIFY_DONE;
1365         }
1366
1367         return NOTIFY_OK;
1368 }
1369
1370
1371 static int __init cpu_hotplug_pm_sync_init(void)
1372 {
1373         /*
1374          * cpu_hotplug_pm_callback has higher priority than x86
1375          * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1376          * to disable cpu hotplug to avoid cpu hotplug race.
1377          */
1378         pm_notifier(cpu_hotplug_pm_callback, 0);
1379         return 0;
1380 }
1381 core_initcall(cpu_hotplug_pm_sync_init);
1382
1383 #endif /* CONFIG_PM_SLEEP_SMP */
1384
1385 int __boot_cpu_id;
1386
1387 #endif /* CONFIG_SMP */
1388
1389 /* Boot processor state steps */
1390 static struct cpuhp_step cpuhp_bp_states[] = {
1391         [CPUHP_OFFLINE] = {
1392                 .name                   = "offline",
1393                 .startup.single         = NULL,
1394                 .teardown.single        = NULL,
1395         },
1396 #ifdef CONFIG_SMP
1397         [CPUHP_CREATE_THREADS]= {
1398                 .name                   = "threads:prepare",
1399                 .startup.single         = smpboot_create_threads,
1400                 .teardown.single        = NULL,
1401                 .cant_stop              = true,
1402         },
1403         [CPUHP_PERF_PREPARE] = {
1404                 .name                   = "perf:prepare",
1405                 .startup.single         = perf_event_init_cpu,
1406                 .teardown.single        = perf_event_exit_cpu,
1407         },
1408         [CPUHP_RANDOM_PREPARE] = {
1409                 .name                   = "random:prepare",
1410                 .startup.single         = random_prepare_cpu,
1411                 .teardown.single        = NULL,
1412         },
1413         [CPUHP_WORKQUEUE_PREP] = {
1414                 .name                   = "workqueue:prepare",
1415                 .startup.single         = workqueue_prepare_cpu,
1416                 .teardown.single        = NULL,
1417         },
1418         [CPUHP_HRTIMERS_PREPARE] = {
1419                 .name                   = "hrtimers:prepare",
1420                 .startup.single         = hrtimers_prepare_cpu,
1421                 .teardown.single        = hrtimers_dead_cpu,
1422         },
1423         [CPUHP_SMPCFD_PREPARE] = {
1424                 .name                   = "smpcfd:prepare",
1425                 .startup.single         = smpcfd_prepare_cpu,
1426                 .teardown.single        = smpcfd_dead_cpu,
1427         },
1428         [CPUHP_RELAY_PREPARE] = {
1429                 .name                   = "relay:prepare",
1430                 .startup.single         = relay_prepare_cpu,
1431                 .teardown.single        = NULL,
1432         },
1433         [CPUHP_SLAB_PREPARE] = {
1434                 .name                   = "slab:prepare",
1435                 .startup.single         = slab_prepare_cpu,
1436                 .teardown.single        = slab_dead_cpu,
1437         },
1438         [CPUHP_RCUTREE_PREP] = {
1439                 .name                   = "RCU/tree:prepare",
1440                 .startup.single         = rcutree_prepare_cpu,
1441                 .teardown.single        = rcutree_dead_cpu,
1442         },
1443         /*
1444          * On the tear-down path, timers_dead_cpu() must be invoked
1445          * before blk_mq_queue_reinit_notify() from notify_dead(),
1446          * otherwise a RCU stall occurs.
1447          */
1448         [CPUHP_TIMERS_PREPARE] = {
1449                 .name                   = "timers:dead",
1450                 .startup.single         = timers_prepare_cpu,
1451                 .teardown.single        = timers_dead_cpu,
1452         },
1453         /* Kicks the plugged cpu into life */
1454         [CPUHP_BRINGUP_CPU] = {
1455                 .name                   = "cpu:bringup",
1456                 .startup.single         = bringup_cpu,
1457                 .teardown.single        = NULL,
1458                 .cant_stop              = true,
1459         },
1460         /*
1461          * Handled on controll processor until the plugged processor manages
1462          * this itself.
1463          */
1464         [CPUHP_TEARDOWN_CPU] = {
1465                 .name                   = "cpu:teardown",
1466                 .startup.single         = NULL,
1467                 .teardown.single        = takedown_cpu,
1468                 .cant_stop              = true,
1469         },
1470 #else
1471         [CPUHP_BRINGUP_CPU] = { },
1472 #endif
1473 };
1474
1475 /* Application processor state steps */
1476 static struct cpuhp_step cpuhp_ap_states[] = {
1477 #ifdef CONFIG_SMP
1478         /* Final state before CPU kills itself */
1479         [CPUHP_AP_IDLE_DEAD] = {
1480                 .name                   = "idle:dead",
1481         },
1482         /*
1483          * Last state before CPU enters the idle loop to die. Transient state
1484          * for synchronization.
1485          */
1486         [CPUHP_AP_OFFLINE] = {
1487                 .name                   = "ap:offline",
1488                 .cant_stop              = true,
1489         },
1490         /* First state is scheduler control. Interrupts are disabled */
1491         [CPUHP_AP_SCHED_STARTING] = {
1492                 .name                   = "sched:starting",
1493                 .startup.single         = sched_cpu_starting,
1494                 .teardown.single        = sched_cpu_dying,
1495         },
1496         [CPUHP_AP_RCUTREE_DYING] = {
1497                 .name                   = "RCU/tree:dying",
1498                 .startup.single         = NULL,
1499                 .teardown.single        = rcutree_dying_cpu,
1500         },
1501         [CPUHP_AP_SMPCFD_DYING] = {
1502                 .name                   = "smpcfd:dying",
1503                 .startup.single         = NULL,
1504                 .teardown.single        = smpcfd_dying_cpu,
1505         },
1506         /* Entry state on starting. Interrupts enabled from here on. Transient
1507          * state for synchronsization */
1508         [CPUHP_AP_ONLINE] = {
1509                 .name                   = "ap:online",
1510         },
1511         /* Handle smpboot threads park/unpark */
1512         [CPUHP_AP_SMPBOOT_THREADS] = {
1513                 .name                   = "smpboot/threads:online",
1514                 .startup.single         = smpboot_unpark_threads,
1515                 .teardown.single        = smpboot_park_threads,
1516         },
1517         [CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
1518                 .name                   = "irq/affinity:online",
1519                 .startup.single         = irq_affinity_online_cpu,
1520                 .teardown.single        = NULL,
1521         },
1522         [CPUHP_AP_PERF_ONLINE] = {
1523                 .name                   = "perf:online",
1524                 .startup.single         = perf_event_init_cpu,
1525                 .teardown.single        = perf_event_exit_cpu,
1526         },
1527         [CPUHP_AP_WORKQUEUE_ONLINE] = {
1528                 .name                   = "workqueue:online",
1529                 .startup.single         = workqueue_online_cpu,
1530                 .teardown.single        = workqueue_offline_cpu,
1531         },
1532         [CPUHP_AP_RANDOM_ONLINE] = {
1533                 .name                   = "random:online",
1534                 .startup.single         = random_online_cpu,
1535                 .teardown.single        = NULL,
1536         },
1537         [CPUHP_AP_RCUTREE_ONLINE] = {
1538                 .name                   = "RCU/tree:online",
1539                 .startup.single         = rcutree_online_cpu,
1540                 .teardown.single        = rcutree_offline_cpu,
1541         },
1542 #endif
1543         /*
1544          * The dynamically registered state space is here
1545          */
1546
1547 #ifdef CONFIG_SMP
1548         /* Last state is scheduler control setting the cpu active */
1549         [CPUHP_AP_ACTIVE] = {
1550                 .name                   = "sched:active",
1551                 .startup.single         = sched_cpu_activate,
1552                 .teardown.single        = sched_cpu_deactivate,
1553         },
1554 #endif
1555
1556         /* CPU is fully up and running. */
1557         [CPUHP_ONLINE] = {
1558                 .name                   = "online",
1559                 .startup.single         = NULL,
1560                 .teardown.single        = NULL,
1561         },
1562 };
1563
1564 /* Sanity check for callbacks */
1565 static int cpuhp_cb_check(enum cpuhp_state state)
1566 {
1567         if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
1568                 return -EINVAL;
1569         return 0;
1570 }
1571
1572 /*
1573  * Returns a free for dynamic slot assignment of the Online state. The states
1574  * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1575  * by having no name assigned.
1576  */
1577 static int cpuhp_reserve_state(enum cpuhp_state state)
1578 {
1579         enum cpuhp_state i, end;
1580         struct cpuhp_step *step;
1581
1582         switch (state) {
1583         case CPUHP_AP_ONLINE_DYN:
1584                 step = cpuhp_ap_states + CPUHP_AP_ONLINE_DYN;
1585                 end = CPUHP_AP_ONLINE_DYN_END;
1586                 break;
1587         case CPUHP_BP_PREPARE_DYN:
1588                 step = cpuhp_bp_states + CPUHP_BP_PREPARE_DYN;
1589                 end = CPUHP_BP_PREPARE_DYN_END;
1590                 break;
1591         default:
1592                 return -EINVAL;
1593         }
1594
1595         for (i = state; i <= end; i++, step++) {
1596                 if (!step->name)
1597                         return i;
1598         }
1599         WARN(1, "No more dynamic states available for CPU hotplug\n");
1600         return -ENOSPC;
1601 }
1602
1603 static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
1604                                  int (*startup)(unsigned int cpu),
1605                                  int (*teardown)(unsigned int cpu),
1606                                  bool multi_instance)
1607 {
1608         /* (Un)Install the callbacks for further cpu hotplug operations */
1609         struct cpuhp_step *sp;
1610         int ret = 0;
1611
1612         /*
1613          * If name is NULL, then the state gets removed.
1614          *
1615          * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
1616          * the first allocation from these dynamic ranges, so the removal
1617          * would trigger a new allocation and clear the wrong (already
1618          * empty) state, leaving the callbacks of the to be cleared state
1619          * dangling, which causes wreckage on the next hotplug operation.
1620          */
1621         if (name && (state == CPUHP_AP_ONLINE_DYN ||
1622                      state == CPUHP_BP_PREPARE_DYN)) {
1623                 ret = cpuhp_reserve_state(state);
1624                 if (ret < 0)
1625                         return ret;
1626                 state = ret;
1627         }
1628         sp = cpuhp_get_step(state);
1629         if (name && sp->name)
1630                 return -EBUSY;
1631
1632         sp->startup.single = startup;
1633         sp->teardown.single = teardown;
1634         sp->name = name;
1635         sp->multi_instance = multi_instance;
1636         INIT_HLIST_HEAD(&sp->list);
1637         return ret;
1638 }
1639
1640 static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
1641 {
1642         return cpuhp_get_step(state)->teardown.single;
1643 }
1644
1645 /*
1646  * Call the startup/teardown function for a step either on the AP or
1647  * on the current CPU.
1648  */
1649 static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
1650                             struct hlist_node *node)
1651 {
1652         struct cpuhp_step *sp = cpuhp_get_step(state);
1653         int ret;
1654
1655         /*
1656          * If there's nothing to do, we done.
1657          * Relies on the union for multi_instance.
1658          */
1659         if ((bringup && !sp->startup.single) ||
1660             (!bringup && !sp->teardown.single))
1661                 return 0;
1662         /*
1663          * The non AP bound callbacks can fail on bringup. On teardown
1664          * e.g. module removal we crash for now.
1665          */
1666 #ifdef CONFIG_SMP
1667         if (cpuhp_is_ap_state(state))
1668                 ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
1669         else
1670                 ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1671 #else
1672         ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1673 #endif
1674         BUG_ON(ret && !bringup);
1675         return ret;
1676 }
1677
1678 /*
1679  * Called from __cpuhp_setup_state on a recoverable failure.
1680  *
1681  * Note: The teardown callbacks for rollback are not allowed to fail!
1682  */
1683 static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
1684                                    struct hlist_node *node)
1685 {
1686         int cpu;
1687
1688         /* Roll back the already executed steps on the other cpus */
1689         for_each_present_cpu(cpu) {
1690                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1691                 int cpustate = st->state;
1692
1693                 if (cpu >= failedcpu)
1694                         break;
1695
1696                 /* Did we invoke the startup call on that cpu ? */
1697                 if (cpustate >= state)
1698                         cpuhp_issue_call(cpu, state, false, node);
1699         }
1700 }
1701
1702 int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
1703                                           struct hlist_node *node,
1704                                           bool invoke)
1705 {
1706         struct cpuhp_step *sp;
1707         int cpu;
1708         int ret;
1709
1710         lockdep_assert_cpus_held();
1711
1712         sp = cpuhp_get_step(state);
1713         if (sp->multi_instance == false)
1714                 return -EINVAL;
1715
1716         mutex_lock(&cpuhp_state_mutex);
1717
1718         if (!invoke || !sp->startup.multi)
1719                 goto add_node;
1720
1721         /*
1722          * Try to call the startup callback for each present cpu
1723          * depending on the hotplug state of the cpu.
1724          */
1725         for_each_present_cpu(cpu) {
1726                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1727                 int cpustate = st->state;
1728
1729                 if (cpustate < state)
1730                         continue;
1731
1732                 ret = cpuhp_issue_call(cpu, state, true, node);
1733                 if (ret) {
1734                         if (sp->teardown.multi)
1735                                 cpuhp_rollback_install(cpu, state, node);
1736                         goto unlock;
1737                 }
1738         }
1739 add_node:
1740         ret = 0;
1741         hlist_add_head(node, &sp->list);
1742 unlock:
1743         mutex_unlock(&cpuhp_state_mutex);
1744         return ret;
1745 }
1746
1747 int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
1748                                bool invoke)
1749 {
1750         int ret;
1751
1752         cpus_read_lock();
1753         ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
1754         cpus_read_unlock();
1755         return ret;
1756 }
1757 EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
1758
1759 /**
1760  * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
1761  * @state:              The state to setup
1762  * @invoke:             If true, the startup function is invoked for cpus where
1763  *                      cpu state >= @state
1764  * @startup:            startup callback function
1765  * @teardown:           teardown callback function
1766  * @multi_instance:     State is set up for multiple instances which get
1767  *                      added afterwards.
1768  *
1769  * The caller needs to hold cpus read locked while calling this function.
1770  * Returns:
1771  *   On success:
1772  *      Positive state number if @state is CPUHP_AP_ONLINE_DYN
1773  *      0 for all other states
1774  *   On failure: proper (negative) error code
1775  */
1776 int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
1777                                    const char *name, bool invoke,
1778                                    int (*startup)(unsigned int cpu),
1779                                    int (*teardown)(unsigned int cpu),
1780                                    bool multi_instance)
1781 {
1782         int cpu, ret = 0;
1783         bool dynstate;
1784
1785         lockdep_assert_cpus_held();
1786
1787         if (cpuhp_cb_check(state) || !name)
1788                 return -EINVAL;
1789
1790         mutex_lock(&cpuhp_state_mutex);
1791
1792         ret = cpuhp_store_callbacks(state, name, startup, teardown,
1793                                     multi_instance);
1794
1795         dynstate = state == CPUHP_AP_ONLINE_DYN;
1796         if (ret > 0 && dynstate) {
1797                 state = ret;
1798                 ret = 0;
1799         }
1800
1801         if (ret || !invoke || !startup)
1802                 goto out;
1803
1804         /*
1805          * Try to call the startup callback for each present cpu
1806          * depending on the hotplug state of the cpu.
1807          */
1808         for_each_present_cpu(cpu) {
1809                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1810                 int cpustate = st->state;
1811
1812                 if (cpustate < state)
1813                         continue;
1814
1815                 ret = cpuhp_issue_call(cpu, state, true, NULL);
1816                 if (ret) {
1817                         if (teardown)
1818                                 cpuhp_rollback_install(cpu, state, NULL);
1819                         cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1820                         goto out;
1821                 }
1822         }
1823 out:
1824         mutex_unlock(&cpuhp_state_mutex);
1825         /*
1826          * If the requested state is CPUHP_AP_ONLINE_DYN, return the
1827          * dynamically allocated state in case of success.
1828          */
1829         if (!ret && dynstate)
1830                 return state;
1831         return ret;
1832 }
1833 EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);
1834
1835 int __cpuhp_setup_state(enum cpuhp_state state,
1836                         const char *name, bool invoke,
1837                         int (*startup)(unsigned int cpu),
1838                         int (*teardown)(unsigned int cpu),
1839                         bool multi_instance)
1840 {
1841         int ret;
1842
1843         cpus_read_lock();
1844         ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
1845                                              teardown, multi_instance);
1846         cpus_read_unlock();
1847         return ret;
1848 }
1849 EXPORT_SYMBOL(__cpuhp_setup_state);
1850
1851 int __cpuhp_state_remove_instance(enum cpuhp_state state,
1852                                   struct hlist_node *node, bool invoke)
1853 {
1854         struct cpuhp_step *sp = cpuhp_get_step(state);
1855         int cpu;
1856
1857         BUG_ON(cpuhp_cb_check(state));
1858
1859         if (!sp->multi_instance)
1860                 return -EINVAL;
1861
1862         cpus_read_lock();
1863         mutex_lock(&cpuhp_state_mutex);
1864
1865         if (!invoke || !cpuhp_get_teardown_cb(state))
1866                 goto remove;
1867         /*
1868          * Call the teardown callback for each present cpu depending
1869          * on the hotplug state of the cpu. This function is not
1870          * allowed to fail currently!
1871          */
1872         for_each_present_cpu(cpu) {
1873                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1874                 int cpustate = st->state;
1875
1876                 if (cpustate >= state)
1877                         cpuhp_issue_call(cpu, state, false, node);
1878         }
1879
1880 remove:
1881         hlist_del(node);
1882         mutex_unlock(&cpuhp_state_mutex);
1883         cpus_read_unlock();
1884
1885         return 0;
1886 }
1887 EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
1888
1889 /**
1890  * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
1891  * @state:      The state to remove
1892  * @invoke:     If true, the teardown function is invoked for cpus where
1893  *              cpu state >= @state
1894  *
1895  * The caller needs to hold cpus read locked while calling this function.
1896  * The teardown callback is currently not allowed to fail. Think
1897  * about module removal!
1898  */
1899 void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
1900 {
1901         struct cpuhp_step *sp = cpuhp_get_step(state);
1902         int cpu;
1903
1904         BUG_ON(cpuhp_cb_check(state));
1905
1906         lockdep_assert_cpus_held();
1907
1908         mutex_lock(&cpuhp_state_mutex);
1909         if (sp->multi_instance) {
1910                 WARN(!hlist_empty(&sp->list),
1911                      "Error: Removing state %d which has instances left.\n",
1912                      state);
1913                 goto remove;
1914         }
1915
1916         if (!invoke || !cpuhp_get_teardown_cb(state))
1917                 goto remove;
1918
1919         /*
1920          * Call the teardown callback for each present cpu depending
1921          * on the hotplug state of the cpu. This function is not
1922          * allowed to fail currently!
1923          */
1924         for_each_present_cpu(cpu) {
1925                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1926                 int cpustate = st->state;
1927
1928                 if (cpustate >= state)
1929                         cpuhp_issue_call(cpu, state, false, NULL);
1930         }
1931 remove:
1932         cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1933         mutex_unlock(&cpuhp_state_mutex);
1934 }
1935 EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);
1936
1937 void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
1938 {
1939         cpus_read_lock();
1940         __cpuhp_remove_state_cpuslocked(state, invoke);
1941         cpus_read_unlock();
1942 }
1943 EXPORT_SYMBOL(__cpuhp_remove_state);
1944
1945 #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
1946 static ssize_t show_cpuhp_state(struct device *dev,
1947                                 struct device_attribute *attr, char *buf)
1948 {
1949         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1950
1951         return sprintf(buf, "%d\n", st->state);
1952 }
1953 static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL);
1954
1955 static ssize_t write_cpuhp_target(struct device *dev,
1956                                   struct device_attribute *attr,
1957                                   const char *buf, size_t count)
1958 {
1959         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1960         struct cpuhp_step *sp;
1961         int target, ret;
1962
1963         ret = kstrtoint(buf, 10, &target);
1964         if (ret)
1965                 return ret;
1966
1967 #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
1968         if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
1969                 return -EINVAL;
1970 #else
1971         if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
1972                 return -EINVAL;
1973 #endif
1974
1975         ret = lock_device_hotplug_sysfs();
1976         if (ret)
1977                 return ret;
1978
1979         mutex_lock(&cpuhp_state_mutex);
1980         sp = cpuhp_get_step(target);
1981         ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
1982         mutex_unlock(&cpuhp_state_mutex);
1983         if (ret)
1984                 goto out;
1985
1986         if (st->state < target)
1987                 ret = do_cpu_up(dev->id, target);
1988         else
1989                 ret = do_cpu_down(dev->id, target);
1990 out:
1991         unlock_device_hotplug();
1992         return ret ? ret : count;
1993 }
1994
1995 static ssize_t show_cpuhp_target(struct device *dev,
1996                                  struct device_attribute *attr, char *buf)
1997 {
1998         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1999
2000         return sprintf(buf, "%d\n", st->target);
2001 }
2002 static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target);
2003
2004
2005 static ssize_t write_cpuhp_fail(struct device *dev,
2006                                 struct device_attribute *attr,
2007                                 const char *buf, size_t count)
2008 {
2009         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2010         struct cpuhp_step *sp;
2011         int fail, ret;
2012
2013         ret = kstrtoint(buf, 10, &fail);
2014         if (ret)
2015                 return ret;
2016
2017         if (fail < CPUHP_OFFLINE || fail > CPUHP_ONLINE)
2018                 return -EINVAL;
2019
2020         /*
2021          * Cannot fail STARTING/DYING callbacks.
2022          */
2023         if (cpuhp_is_atomic_state(fail))
2024                 return -EINVAL;
2025
2026         /*
2027          * Cannot fail anything that doesn't have callbacks.
2028          */
2029         mutex_lock(&cpuhp_state_mutex);
2030         sp = cpuhp_get_step(fail);
2031         if (!sp->startup.single && !sp->teardown.single)
2032                 ret = -EINVAL;
2033         mutex_unlock(&cpuhp_state_mutex);
2034         if (ret)
2035                 return ret;
2036
2037         st->fail = fail;
2038
2039         return count;
2040 }
2041
2042 static ssize_t show_cpuhp_fail(struct device *dev,
2043                                struct device_attribute *attr, char *buf)
2044 {
2045         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2046
2047         return sprintf(buf, "%d\n", st->fail);
2048 }
2049
2050 static DEVICE_ATTR(fail, 0644, show_cpuhp_fail, write_cpuhp_fail);
2051
2052 static struct attribute *cpuhp_cpu_attrs[] = {
2053         &dev_attr_state.attr,
2054         &dev_attr_target.attr,
2055         &dev_attr_fail.attr,
2056         NULL
2057 };
2058
2059 static const struct attribute_group cpuhp_cpu_attr_group = {
2060         .attrs = cpuhp_cpu_attrs,
2061         .name = "hotplug",
2062         NULL
2063 };
2064
2065 static ssize_t show_cpuhp_states(struct device *dev,
2066                                  struct device_attribute *attr, char *buf)
2067 {
2068         ssize_t cur, res = 0;
2069         int i;
2070
2071         mutex_lock(&cpuhp_state_mutex);
2072         for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
2073                 struct cpuhp_step *sp = cpuhp_get_step(i);
2074
2075                 if (sp->name) {
2076                         cur = sprintf(buf, "%3d: %s\n", i, sp->name);
2077                         buf += cur;
2078                         res += cur;
2079                 }
2080         }
2081         mutex_unlock(&cpuhp_state_mutex);
2082         return res;
2083 }
2084 static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL);
2085
2086 static struct attribute *cpuhp_cpu_root_attrs[] = {
2087         &dev_attr_states.attr,
2088         NULL
2089 };
2090
2091 static const struct attribute_group cpuhp_cpu_root_attr_group = {
2092         .attrs = cpuhp_cpu_root_attrs,
2093         .name = "hotplug",
2094         NULL
2095 };
2096
2097 #ifdef CONFIG_HOTPLUG_SMT
2098
2099 static const char *smt_states[] = {
2100         [CPU_SMT_ENABLED]               = "on",
2101         [CPU_SMT_DISABLED]              = "off",
2102         [CPU_SMT_FORCE_DISABLED]        = "forceoff",
2103         [CPU_SMT_NOT_SUPPORTED]         = "notsupported",
2104 };
2105
2106 static ssize_t
2107 show_smt_control(struct device *dev, struct device_attribute *attr, char *buf)
2108 {
2109         return snprintf(buf, PAGE_SIZE - 2, "%s\n", smt_states[cpu_smt_control]);
2110 }
2111
2112 static void cpuhp_offline_cpu_device(unsigned int cpu)
2113 {
2114         struct device *dev = get_cpu_device(cpu);
2115
2116         dev->offline = true;
2117         /* Tell user space about the state change */
2118         kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
2119 }
2120
2121 static void cpuhp_online_cpu_device(unsigned int cpu)
2122 {
2123         struct device *dev = get_cpu_device(cpu);
2124
2125         dev->offline = false;
2126         /* Tell user space about the state change */
2127         kobject_uevent(&dev->kobj, KOBJ_ONLINE);
2128 }
2129
2130 int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
2131 {
2132         int cpu, ret = 0;
2133
2134         cpu_maps_update_begin();
2135         for_each_online_cpu(cpu) {
2136                 if (topology_is_primary_thread(cpu))
2137                         continue;
2138                 ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
2139                 if (ret)
2140                         break;
2141                 /*
2142                  * As this needs to hold the cpu maps lock it's impossible
2143                  * to call device_offline() because that ends up calling
2144                  * cpu_down() which takes cpu maps lock. cpu maps lock
2145                  * needs to be held as this might race against in kernel
2146                  * abusers of the hotplug machinery (thermal management).
2147                  *
2148                  * So nothing would update device:offline state. That would
2149                  * leave the sysfs entry stale and prevent onlining after
2150                  * smt control has been changed to 'off' again. This is
2151                  * called under the sysfs hotplug lock, so it is properly
2152                  * serialized against the regular offline usage.
2153                  */
2154                 cpuhp_offline_cpu_device(cpu);
2155         }
2156         if (!ret)
2157                 cpu_smt_control = ctrlval;
2158         cpu_maps_update_done();
2159         return ret;
2160 }
2161
2162 int cpuhp_smt_enable(void)
2163 {
2164         int cpu, ret = 0;
2165
2166         cpu_maps_update_begin();
2167         cpu_smt_control = CPU_SMT_ENABLED;
2168         for_each_present_cpu(cpu) {
2169                 /* Skip online CPUs and CPUs on offline nodes */
2170                 if (cpu_online(cpu) || !node_online(cpu_to_node(cpu)))
2171                         continue;
2172                 ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
2173                 if (ret)
2174                         break;
2175                 /* See comment in cpuhp_smt_disable() */
2176                 cpuhp_online_cpu_device(cpu);
2177         }
2178         cpu_maps_update_done();
2179         return ret;
2180 }
2181
2182 static ssize_t
2183 store_smt_control(struct device *dev, struct device_attribute *attr,
2184                   const char *buf, size_t count)
2185 {
2186         int ctrlval, ret;
2187
2188         if (sysfs_streq(buf, "on"))
2189                 ctrlval = CPU_SMT_ENABLED;
2190         else if (sysfs_streq(buf, "off"))
2191                 ctrlval = CPU_SMT_DISABLED;
2192         else if (sysfs_streq(buf, "forceoff"))
2193                 ctrlval = CPU_SMT_FORCE_DISABLED;
2194         else
2195                 return -EINVAL;
2196
2197         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED)
2198                 return -EPERM;
2199
2200         if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
2201                 return -ENODEV;
2202
2203         ret = lock_device_hotplug_sysfs();
2204         if (ret)
2205                 return ret;
2206
2207         if (ctrlval != cpu_smt_control) {
2208                 switch (ctrlval) {
2209                 case CPU_SMT_ENABLED:
2210                         ret = cpuhp_smt_enable();
2211                         break;
2212                 case CPU_SMT_DISABLED:
2213                 case CPU_SMT_FORCE_DISABLED:
2214                         ret = cpuhp_smt_disable(ctrlval);
2215                         break;
2216                 }
2217         }
2218
2219         unlock_device_hotplug();
2220         return ret ? ret : count;
2221 }
2222 static DEVICE_ATTR(control, 0644, show_smt_control, store_smt_control);
2223
2224 static ssize_t
2225 show_smt_active(struct device *dev, struct device_attribute *attr, char *buf)
2226 {
2227         bool active = topology_max_smt_threads() > 1;
2228
2229         return snprintf(buf, PAGE_SIZE - 2, "%d\n", active);
2230 }
2231 static DEVICE_ATTR(active, 0444, show_smt_active, NULL);
2232
2233 static struct attribute *cpuhp_smt_attrs[] = {
2234         &dev_attr_control.attr,
2235         &dev_attr_active.attr,
2236         NULL
2237 };
2238
2239 static const struct attribute_group cpuhp_smt_attr_group = {
2240         .attrs = cpuhp_smt_attrs,
2241         .name = "smt",
2242         NULL
2243 };
2244
2245 static int __init cpu_smt_state_init(void)
2246 {
2247         return sysfs_create_group(&cpu_subsys.dev_root->kobj,
2248                                   &cpuhp_smt_attr_group);
2249 }
2250
2251 #else
2252 static inline int cpu_smt_state_init(void) { return 0; }
2253 #endif
2254
2255 static int __init cpuhp_sysfs_init(void)
2256 {
2257         int cpu, ret;
2258
2259         ret = cpu_smt_state_init();
2260         if (ret)
2261                 return ret;
2262
2263         ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
2264                                  &cpuhp_cpu_root_attr_group);
2265         if (ret)
2266                 return ret;
2267
2268         for_each_possible_cpu(cpu) {
2269                 struct device *dev = get_cpu_device(cpu);
2270
2271                 if (!dev)
2272                         continue;
2273                 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
2274                 if (ret)
2275                         return ret;
2276         }
2277         return 0;
2278 }
2279 device_initcall(cpuhp_sysfs_init);
2280 #endif
2281
2282 /*
2283  * cpu_bit_bitmap[] is a special, "compressed" data structure that
2284  * represents all NR_CPUS bits binary values of 1<<nr.
2285  *
2286  * It is used by cpumask_of() to get a constant address to a CPU
2287  * mask value that has a single bit set only.
2288  */
2289
2290 /* cpu_bit_bitmap[0] is empty - so we can back into it */
2291 #define MASK_DECLARE_1(x)       [x+1][0] = (1UL << (x))
2292 #define MASK_DECLARE_2(x)       MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
2293 #define MASK_DECLARE_4(x)       MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
2294 #define MASK_DECLARE_8(x)       MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
2295
2296 const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
2297
2298         MASK_DECLARE_8(0),      MASK_DECLARE_8(8),
2299         MASK_DECLARE_8(16),     MASK_DECLARE_8(24),
2300 #if BITS_PER_LONG > 32
2301         MASK_DECLARE_8(32),     MASK_DECLARE_8(40),
2302         MASK_DECLARE_8(48),     MASK_DECLARE_8(56),
2303 #endif
2304 };
2305 EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
2306
2307 const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
2308 EXPORT_SYMBOL(cpu_all_bits);
2309
2310 #ifdef CONFIG_INIT_ALL_POSSIBLE
2311 struct cpumask __cpu_possible_mask __read_mostly
2312         = {CPU_BITS_ALL};
2313 #else
2314 struct cpumask __cpu_possible_mask __read_mostly;
2315 #endif
2316 EXPORT_SYMBOL(__cpu_possible_mask);
2317
2318 struct cpumask __cpu_online_mask __read_mostly;
2319 EXPORT_SYMBOL(__cpu_online_mask);
2320
2321 struct cpumask __cpu_present_mask __read_mostly;
2322 EXPORT_SYMBOL(__cpu_present_mask);
2323
2324 struct cpumask __cpu_active_mask __read_mostly;
2325 EXPORT_SYMBOL(__cpu_active_mask);
2326
2327 void init_cpu_present(const struct cpumask *src)
2328 {
2329         cpumask_copy(&__cpu_present_mask, src);
2330 }
2331
2332 void init_cpu_possible(const struct cpumask *src)
2333 {
2334         cpumask_copy(&__cpu_possible_mask, src);
2335 }
2336
2337 void init_cpu_online(const struct cpumask *src)
2338 {
2339         cpumask_copy(&__cpu_online_mask, src);
2340 }
2341
2342 /*
2343  * Activate the first processor.
2344  */
2345 void __init boot_cpu_init(void)
2346 {
2347         int cpu = smp_processor_id();
2348
2349         /* Mark the boot cpu "present", "online" etc for SMP and UP case */
2350         set_cpu_online(cpu, true);
2351         set_cpu_active(cpu, true);
2352         set_cpu_present(cpu, true);
2353         set_cpu_possible(cpu, true);
2354
2355 #ifdef CONFIG_SMP
2356         __boot_cpu_id = cpu;
2357 #endif
2358 }
2359
2360 /*
2361  * Must be called _AFTER_ setting up the per_cpu areas
2362  */
2363 void __init boot_cpu_hotplug_init(void)
2364 {
2365 #ifdef CONFIG_SMP
2366         this_cpu_write(cpuhp_state.booted_once, true);
2367 #endif
2368         this_cpu_write(cpuhp_state.state, CPUHP_ONLINE);
2369 }
2370
2371 /*
2372  * These are used for a global "mitigations=" cmdline option for toggling
2373  * optional CPU mitigations.
2374  */
2375 enum cpu_mitigations {
2376         CPU_MITIGATIONS_OFF,
2377         CPU_MITIGATIONS_AUTO,
2378         CPU_MITIGATIONS_AUTO_NOSMT,
2379 };
2380
2381 static enum cpu_mitigations cpu_mitigations __ro_after_init =
2382         CPU_MITIGATIONS_AUTO;
2383
2384 static int __init mitigations_parse_cmdline(char *arg)
2385 {
2386         if (!strcmp(arg, "off"))
2387                 cpu_mitigations = CPU_MITIGATIONS_OFF;
2388         else if (!strcmp(arg, "auto"))
2389                 cpu_mitigations = CPU_MITIGATIONS_AUTO;
2390         else if (!strcmp(arg, "auto,nosmt"))
2391                 cpu_mitigations = CPU_MITIGATIONS_AUTO_NOSMT;
2392         else
2393                 pr_crit("Unsupported mitigations=%s, system may still be vulnerable\n",
2394                         arg);
2395
2396         return 0;
2397 }
2398 early_param("mitigations", mitigations_parse_cmdline);
2399
2400 /* mitigations=off */
2401 bool cpu_mitigations_off(void)
2402 {
2403         return cpu_mitigations == CPU_MITIGATIONS_OFF;
2404 }
2405 EXPORT_SYMBOL_GPL(cpu_mitigations_off);
2406
2407 /* mitigations=auto,nosmt */
2408 bool cpu_mitigations_auto_nosmt(void)
2409 {
2410         return cpu_mitigations == CPU_MITIGATIONS_AUTO_NOSMT;
2411 }
2412 EXPORT_SYMBOL_GPL(cpu_mitigations_auto_nosmt);