arch/powerpc/platforms/pseries/hotplug-cpu.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * pseries CPU Hotplug infrastructure.
   4  *
   5  * Split out from arch/powerpc/platforms/pseries/setup.c
   6  *  arch/powerpc/kernel/rtas.c, and arch/powerpc/platforms/pseries/smp.c
   7  *
   8  * Peter Bergner, IBM   March 2001.
   9  * Copyright (C) 2001 IBM.
  10  * Dave Engebretsen, Peter Bergner, and
  11  * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com
  12  * Plus various changes from other IBM teams...
  13  *
  14  * Copyright (C) 2006 Michael Ellerman, IBM Corporation
  15  */
  16
  17 #define pr_fmt(fmt)     "pseries-hotplug-cpu: " fmt
  18
  19 #include <linux/kernel.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/delay.h>
  22 #include <linux/sched.h>        /* for idle_task_exit */
  23 #include <linux/sched/hotplug.h>
  24 #include <linux/cpu.h>
  25 #include <linux/of.h>
  26 #include <linux/slab.h>
  27 #include <asm/prom.h>
  28 #include <asm/rtas.h>
  29 #include <asm/firmware.h>
  30 #include <asm/machdep.h>
  31 #include <asm/vdso_datapage.h>
  32 #include <asm/xics.h>
  33 #include <asm/xive.h>
  34 #include <asm/plpar_wrappers.h>
  35 #include <asm/topology.h>
  36
  37 #include "pseries.h"
  38 #include "offline_states.h"
  39
  40 /* This version can't take the spinlock, because it never returns */
  41 static int rtas_stop_self_token = RTAS_UNKNOWN_SERVICE;
  42
  43 static DEFINE_PER_CPU(enum cpu_state_vals, preferred_offline_state) =
  44                                                         CPU_STATE_OFFLINE;
  45 static DEFINE_PER_CPU(enum cpu_state_vals, current_state) = CPU_STATE_OFFLINE;
  46
  47 static enum cpu_state_vals default_offline_state = CPU_STATE_OFFLINE;
  48
  49 static bool cede_offline_enabled __read_mostly = true;
  50
  51 /*
  52  * Enable/disable cede_offline when available.
  53  */
  54 static int __init setup_cede_offline(char *str)
  55 {
  56         return (kstrtobool(str, &cede_offline_enabled) == 0);
  57 }
  58
  59 __setup("cede_offline=", setup_cede_offline);
  60
  61 enum cpu_state_vals get_cpu_current_state(int cpu)
  62 {
  63         return per_cpu(current_state, cpu);
  64 }
  65
  66 void set_cpu_current_state(int cpu, enum cpu_state_vals state)
  67 {
  68         per_cpu(current_state, cpu) = state;
  69 }
  70
  71 enum cpu_state_vals get_preferred_offline_state(int cpu)
  72 {
  73         return per_cpu(preferred_offline_state, cpu);
  74 }
  75
  76 void set_preferred_offline_state(int cpu, enum cpu_state_vals state)
  77 {
  78         per_cpu(preferred_offline_state, cpu) = state;
  79 }
  80
  81 void set_default_offline_state(int cpu)
  82 {
  83         per_cpu(preferred_offline_state, cpu) = default_offline_state;
  84 }
  85
  86 static void rtas_stop_self(void)
  87 {
  88         static struct rtas_args args;
  89
  90         local_irq_disable();
  91
  92         BUG_ON(rtas_stop_self_token == RTAS_UNKNOWN_SERVICE);
  93
  94         rtas_call_unlocked(&args, rtas_stop_self_token, 0, 1, NULL);
  95
  96         panic("Alas, I survived.\n");
  97 }
  98
  99 static void pseries_mach_cpu_die(void)
 100 {
 101         unsigned int cpu = smp_processor_id();
 102         unsigned int hwcpu = hard_smp_processor_id();
 103         u8 cede_latency_hint = 0;
 104
 105         local_irq_disable();
 106         idle_task_exit();
 107         if (xive_enabled())
 108                 xive_teardown_cpu();
 109         else
 110                 xics_teardown_cpu();
 111
 112         if (get_preferred_offline_state(cpu) == CPU_STATE_INACTIVE) {
 113                 set_cpu_current_state(cpu, CPU_STATE_INACTIVE);
 114                 if (ppc_md.suspend_disable_cpu)
 115                         ppc_md.suspend_disable_cpu();
 116
 117                 cede_latency_hint = 2;
 118
 119                 get_lppaca()->idle = 1;
 120                 if (!lppaca_shared_proc(get_lppaca()))
 121                         get_lppaca()->donate_dedicated_cpu = 1;
 122
 123                 while (get_preferred_offline_state(cpu) == CPU_STATE_INACTIVE) {
 124                         while (!prep_irq_for_idle()) {
 125                                 local_irq_enable();
 126                                 local_irq_disable();
 127                         }
 128
 129                         extended_cede_processor(cede_latency_hint);
 130                 }
 131
 132                 local_irq_disable();
 133
 134                 if (!lppaca_shared_proc(get_lppaca()))
 135                         get_lppaca()->donate_dedicated_cpu = 0;
 136                 get_lppaca()->idle = 0;
 137
 138                 if (get_preferred_offline_state(cpu) == CPU_STATE_ONLINE) {
 139                         unregister_slb_shadow(hwcpu);
 140
 141                         hard_irq_disable();
 142                         /*
 143                          * Call to start_secondary_resume() will not return.
 144                          * Kernel stack will be reset and start_secondary()
 145                          * will be called to continue the online operation.
 146                          */
 147                         start_secondary_resume();
 148                 }
 149         }
 150
 151         /* Requested state is CPU_STATE_OFFLINE at this point */
 152         WARN_ON(get_preferred_offline_state(cpu) != CPU_STATE_OFFLINE);
 153
 154         set_cpu_current_state(cpu, CPU_STATE_OFFLINE);
 155         unregister_slb_shadow(hwcpu);
 156         rtas_stop_self();
 157
 158         /* Should never get here... */
 159         BUG();
 160         for(;;);
 161 }
 162
 163 static int pseries_cpu_disable(void)
 164 {
 165         int cpu = smp_processor_id();
 166
 167         set_cpu_online(cpu, false);
 168         vdso_data->processorCount--;
 169
 170         /*fix boot_cpuid here*/
 171         if (cpu == boot_cpuid)
 172                 boot_cpuid = cpumask_any(cpu_online_mask);
 173
 174         /* FIXME: abstract this to not be platform specific later on */
 175         if (xive_enabled())
 176                 xive_smp_disable_cpu();
 177         else
 178                 xics_migrate_irqs_away();
 179         return 0;
 180 }
 181
 182 /*
 183  * pseries_cpu_die: Wait for the cpu to die.
 184  * @cpu: logical processor id of the CPU whose death we're awaiting.
 185  *
 186  * This function is called from the context of the thread which is performing
 187  * the cpu-offline. Here we wait for long enough to allow the cpu in question
 188  * to self-destroy so that the cpu-offline thread can send the CPU_DEAD
 189  * notifications.
 190  *
 191  * OTOH, pseries_mach_cpu_die() is called by the @cpu when it wants to
 192  * self-destruct.
 193  */
 194 static void pseries_cpu_die(unsigned int cpu)
 195 {
 196         int tries;
 197         int cpu_status = 1;
 198         unsigned int pcpu = get_hard_smp_processor_id(cpu);
 199
 200         if (get_preferred_offline_state(cpu) == CPU_STATE_INACTIVE) {
 201                 cpu_status = 1;
 202                 for (tries = 0; tries < 5000; tries++) {
 203                         if (get_cpu_current_state(cpu) == CPU_STATE_INACTIVE) {
 204                                 cpu_status = 0;
 205                                 break;
 206                         }
 207                         msleep(1);
 208                 }
 209         } else if (get_preferred_offline_state(cpu) == CPU_STATE_OFFLINE) {
 210
 211                 for (tries = 0; tries < 25; tries++) {
 212                         cpu_status = smp_query_cpu_stopped(pcpu);
 213                         if (cpu_status == QCSS_STOPPED ||
 214                             cpu_status == QCSS_HARDWARE_ERROR)
 215                                 break;
 216                         cpu_relax();
 217                 }
 218         }
 219
 220         if (cpu_status != 0) {
 221                 printk("Querying DEAD? cpu %i (%i) shows %i\n",
 222                        cpu, pcpu, cpu_status);
 223         }
 224
 225         /* Isolation and deallocation are definitely done by
 226          * drslot_chrp_cpu.  If they were not they would be
 227          * done here.  Change isolate state to Isolate and
 228          * change allocation-state to Unusable.
 229          */
 230         paca_ptrs[cpu]->cpu_start = 0;
 231 }
 232
 233 /*
 234  * Update cpu_present_mask and paca(s) for a new cpu node.  The wrinkle
 235  * here is that a cpu device node may represent up to two logical cpus
 236  * in the SMT case.  We must honor the assumption in other code that
 237  * the logical ids for sibling SMT threads x and y are adjacent, such
 238  * that x^1 == y and y^1 == x.
 239  */
 240 static int pseries_add_processor(struct device_node *np)
 241 {
 242         unsigned int cpu;
 243         cpumask_var_t candidate_mask, tmp;
 244         int err = -ENOSPC, len, nthreads, i;
 245         const __be32 *intserv;
 246
 247         intserv = of_get_property(np, "ibm,ppc-interrupt-server#s", &len);
 248         if (!intserv)
 249                 return 0;
 250
 251         zalloc_cpumask_var(&candidate_mask, GFP_KERNEL);
 252         zalloc_cpumask_var(&tmp, GFP_KERNEL);
 253
 254         nthreads = len / sizeof(u32);
 255         for (i = 0; i < nthreads; i++)
 256                 cpumask_set_cpu(i, tmp);
 257
 258         cpu_maps_update_begin();
 259
 260         BUG_ON(!cpumask_subset(cpu_present_mask, cpu_possible_mask));
 261
 262         /* Get a bitmap of unoccupied slots. */
 263         cpumask_xor(candidate_mask, cpu_possible_mask, cpu_present_mask);
 264         if (cpumask_empty(candidate_mask)) {
 265                 /* If we get here, it most likely means that NR_CPUS is
 266                  * less than the partition's max processors setting.
 267                  */
 268                 printk(KERN_ERR "Cannot add cpu %pOF; this system configuration"
 269                        " supports %d logical cpus.\n", np,
 270                        num_possible_cpus());
 271                 goto out_unlock;
 272         }
 273
 274         while (!cpumask_empty(tmp))
 275                 if (cpumask_subset(tmp, candidate_mask))
 276                         /* Found a range where we can insert the new cpu(s) */
 277                         break;
 278                 else
 279                         cpumask_shift_left(tmp, tmp, nthreads);
 280
 281         if (cpumask_empty(tmp)) {
 282                 printk(KERN_ERR "Unable to find space in cpu_present_mask for"
 283                        " processor %pOFn with %d thread(s)\n", np,
 284                        nthreads);
 285                 goto out_unlock;
 286         }
 287
 288         for_each_cpu(cpu, tmp) {
 289                 BUG_ON(cpu_present(cpu));
 290                 set_cpu_present(cpu, true);
 291                 set_hard_smp_processor_id(cpu, be32_to_cpu(*intserv++));
 292         }
 293         err = 0;
 294 out_unlock:
 295         cpu_maps_update_done();
 296         free_cpumask_var(candidate_mask);
 297         free_cpumask_var(tmp);
 298         return err;
 299 }
 300
 301 /*
 302  * Update the present map for a cpu node which is going away, and set
 303  * the hard id in the paca(s) to -1 to be consistent with boot time
 304  * convention for non-present cpus.
 305  */
 306 static void pseries_remove_processor(struct device_node *np)
 307 {
 308         unsigned int cpu;
 309         int len, nthreads, i;
 310         const __be32 *intserv;
 311         u32 thread;
 312
 313         intserv = of_get_property(np, "ibm,ppc-interrupt-server#s", &len);
 314         if (!intserv)
 315                 return;
 316
 317         nthreads = len / sizeof(u32);
 318
 319         cpu_maps_update_begin();
 320         for (i = 0; i < nthreads; i++) {
 321                 thread = be32_to_cpu(intserv[i]);
 322                 for_each_present_cpu(cpu) {
 323                         if (get_hard_smp_processor_id(cpu) != thread)
 324                                 continue;
 325                         BUG_ON(cpu_online(cpu));
 326                         set_cpu_present(cpu, false);
 327                         set_hard_smp_processor_id(cpu, -1);
 328                         update_numa_cpu_lookup_table(cpu, -1);
 329                         break;
 330                 }
 331                 if (cpu >= nr_cpu_ids)
 332                         printk(KERN_WARNING "Could not find cpu to remove "
 333                                "with physical id 0x%x\n", thread);
 334         }
 335         cpu_maps_update_done();
 336 }
 337
 338 static int dlpar_online_cpu(struct device_node *dn)
 339 {
 340         int rc = 0;
 341         unsigned int cpu;
 342         int len, nthreads, i;
 343         const __be32 *intserv;
 344         u32 thread;
 345
 346         intserv = of_get_property(dn, "ibm,ppc-interrupt-server#s", &len);
 347         if (!intserv)
 348                 return -EINVAL;
 349
 350         nthreads = len / sizeof(u32);
 351
 352         cpu_maps_update_begin();
 353         for (i = 0; i < nthreads; i++) {
 354                 thread = be32_to_cpu(intserv[i]);
 355                 for_each_present_cpu(cpu) {
 356                         if (get_hard_smp_processor_id(cpu) != thread)
 357                                 continue;
 358                         BUG_ON(get_cpu_current_state(cpu)
 359                                         != CPU_STATE_OFFLINE);
 360                         cpu_maps_update_done();
 361                         timed_topology_update(1);
 362                         find_and_online_cpu_nid(cpu);
 363                         rc = device_online(get_cpu_device(cpu));
 364                         if (rc)
 365                                 goto out;
 366                         cpu_maps_update_begin();
 367
 368                         break;
 369                 }
 370                 if (cpu == num_possible_cpus())
 371                         printk(KERN_WARNING "Could not find cpu to online "
 372                                "with physical id 0x%x\n", thread);
 373         }
 374         cpu_maps_update_done();
 375
 376 out:
 377         return rc;
 378
 379 }
 380
 381 static bool dlpar_cpu_exists(struct device_node *parent, u32 drc_index)
 382 {
 383         struct device_node *child = NULL;
 384         u32 my_drc_index;
 385         bool found;
 386         int rc;
 387
 388         /* Assume cpu doesn't exist */
 389         found = false;
 390
 391         for_each_child_of_node(parent, child) {
 392                 rc = of_property_read_u32(child, "ibm,my-drc-index",
 393                                           &my_drc_index);
 394                 if (rc)
 395                         continue;
 396
 397                 if (my_drc_index == drc_index) {
 398                         of_node_put(child);
 399                         found = true;
 400                         break;
 401                 }
 402         }
 403
 404         return found;
 405 }
 406
 407 static bool valid_cpu_drc_index(struct device_node *parent, u32 drc_index)
 408 {
 409         bool found = false;
 410         int rc, index;
 411
 412         index = 0;
 413         while (!found) {
 414                 u32 drc;
 415
 416                 rc = of_property_read_u32_index(parent, "ibm,drc-indexes",
 417                                                 index++, &drc);
 418                 if (rc)
 419                         break;
 420
 421                 if (drc == drc_index)
 422                         found = true;
 423         }
 424
 425         return found;
 426 }
 427
 428 static ssize_t dlpar_cpu_add(u32 drc_index)
 429 {
 430         struct device_node *dn, *parent;
 431         int rc, saved_rc;
 432
 433         pr_debug("Attempting to add CPU, drc index: %x\n", drc_index);
 434
 435         parent = of_find_node_by_path("/cpus");
 436         if (!parent) {
 437                 pr_warn("Failed to find CPU root node \"/cpus\"\n");
 438                 return -ENODEV;
 439         }
 440
 441         if (dlpar_cpu_exists(parent, drc_index)) {
 442                 of_node_put(parent);
 443                 pr_warn("CPU with drc index %x already exists\n", drc_index);
 444                 return -EINVAL;
 445         }
 446
 447         if (!valid_cpu_drc_index(parent, drc_index)) {
 448                 of_node_put(parent);
 449                 pr_warn("Cannot find CPU (drc index %x) to add.\n", drc_index);
 450                 return -EINVAL;
 451         }
 452
 453         rc = dlpar_acquire_drc(drc_index);
 454         if (rc) {
 455                 pr_warn("Failed to acquire DRC, rc: %d, drc index: %x\n",
 456                         rc, drc_index);
 457                 of_node_put(parent);
 458                 return -EINVAL;
 459         }
 460
 461         dn = dlpar_configure_connector(cpu_to_be32(drc_index), parent);
 462         if (!dn) {
 463                 pr_warn("Failed call to configure-connector, drc index: %x\n",
 464                         drc_index);
 465                 dlpar_release_drc(drc_index);
 466                 of_node_put(parent);
 467                 return -EINVAL;
 468         }
 469
 470         rc = dlpar_attach_node(dn, parent);
 471
 472         /* Regardless we are done with parent now */
 473         of_node_put(parent);
 474
 475         if (rc) {
 476                 saved_rc = rc;
 477                 pr_warn("Failed to attach node %pOFn, rc: %d, drc index: %x\n",
 478                         dn, rc, drc_index);
 479
 480                 rc = dlpar_release_drc(drc_index);
 481                 if (!rc)
 482                         dlpar_free_cc_nodes(dn);
 483
 484                 return saved_rc;
 485         }
 486
 487         rc = dlpar_online_cpu(dn);
 488         if (rc) {
 489                 saved_rc = rc;
 490                 pr_warn("Failed to online cpu %pOFn, rc: %d, drc index: %x\n",
 491                         dn, rc, drc_index);
 492
 493                 rc = dlpar_detach_node(dn);
 494                 if (!rc)
 495                         dlpar_release_drc(drc_index);
 496
 497                 return saved_rc;
 498         }
 499
 500         pr_debug("Successfully added CPU %pOFn, drc index: %x\n", dn,
 501                  drc_index);
 502         return rc;
 503 }
 504
 505 static int dlpar_offline_cpu(struct device_node *dn)
 506 {
 507         int rc = 0;
 508         unsigned int cpu;
 509         int len, nthreads, i;
 510         const __be32 *intserv;
 511         u32 thread;
 512
 513         intserv = of_get_property(dn, "ibm,ppc-interrupt-server#s", &len);
 514         if (!intserv)
 515                 return -EINVAL;
 516
 517         nthreads = len / sizeof(u32);
 518
 519         cpu_maps_update_begin();
 520         for (i = 0; i < nthreads; i++) {
 521                 thread = be32_to_cpu(intserv[i]);
 522                 for_each_present_cpu(cpu) {
 523                         if (get_hard_smp_processor_id(cpu) != thread)
 524                                 continue;
 525
 526                         if (get_cpu_current_state(cpu) == CPU_STATE_OFFLINE)
 527                                 break;
 528
 529                         if (get_cpu_current_state(cpu) == CPU_STATE_ONLINE) {
 530                                 set_preferred_offline_state(cpu,
 531                                                             CPU_STATE_OFFLINE);
 532                                 cpu_maps_update_done();
 533                                 timed_topology_update(1);
 534                                 rc = device_offline(get_cpu_device(cpu));
 535                                 if (rc)
 536                                         goto out;
 537                                 cpu_maps_update_begin();
 538                                 break;
 539
 540                         }
 541
 542                         /*
 543                          * The cpu is in CPU_STATE_INACTIVE.
 544                          * Upgrade it's state to CPU_STATE_OFFLINE.
 545                          */
 546                         set_preferred_offline_state(cpu, CPU_STATE_OFFLINE);
 547                         BUG_ON(plpar_hcall_norets(H_PROD, thread)
 548                                                                 != H_SUCCESS);
 549                         __cpu_die(cpu);
 550                         break;
 551                 }
 552                 if (cpu == num_possible_cpus())
 553                         printk(KERN_WARNING "Could not find cpu to offline with physical id 0x%x\n", thread);
 554         }
 555         cpu_maps_update_done();
 556
 557 out:
 558         return rc;
 559
 560 }
 561
 562 static ssize_t dlpar_cpu_remove(struct device_node *dn, u32 drc_index)
 563 {
 564         int rc;
 565
 566         pr_debug("Attempting to remove CPU %pOFn, drc index: %x\n",
 567                  dn, drc_index);
 568
 569         rc = dlpar_offline_cpu(dn);
 570         if (rc) {
 571                 pr_warn("Failed to offline CPU %pOFn, rc: %d\n", dn, rc);
 572                 return -EINVAL;
 573         }
 574
 575         rc = dlpar_release_drc(drc_index);
 576         if (rc) {
 577                 pr_warn("Failed to release drc (%x) for CPU %pOFn, rc: %d\n",
 578                         drc_index, dn, rc);
 579                 dlpar_online_cpu(dn);
 580                 return rc;
 581         }
 582
 583         rc = dlpar_detach_node(dn);
 584         if (rc) {
 585                 int saved_rc = rc;
 586
 587                 pr_warn("Failed to detach CPU %pOFn, rc: %d", dn, rc);
 588
 589                 rc = dlpar_acquire_drc(drc_index);
 590                 if (!rc)
 591                         dlpar_online_cpu(dn);
 592
 593                 return saved_rc;
 594         }
 595
 596         pr_debug("Successfully removed CPU, drc index: %x\n", drc_index);
 597         return 0;
 598 }
 599
 600 static struct device_node *cpu_drc_index_to_dn(u32 drc_index)
 601 {
 602         struct device_node *dn;
 603         u32 my_index;
 604         int rc;
 605
 606         for_each_node_by_type(dn, "cpu") {
 607                 rc = of_property_read_u32(dn, "ibm,my-drc-index", &my_index);
 608                 if (rc)
 609                         continue;
 610
 611                 if (my_index == drc_index)
 612                         break;
 613         }
 614
 615         return dn;
 616 }
 617
 618 static int dlpar_cpu_remove_by_index(u32 drc_index)
 619 {
 620         struct device_node *dn;
 621         int rc;
 622
 623         dn = cpu_drc_index_to_dn(drc_index);
 624         if (!dn) {
 625                 pr_warn("Cannot find CPU (drc index %x) to remove\n",
 626                         drc_index);
 627                 return -ENODEV;
 628         }
 629
 630         rc = dlpar_cpu_remove(dn, drc_index);
 631         of_node_put(dn);
 632         return rc;
 633 }
 634
 635 static int find_dlpar_cpus_to_remove(u32 *cpu_drcs, int cpus_to_remove)
 636 {
 637         struct device_node *dn;
 638         int cpus_found = 0;
 639         int rc;
 640
 641         /* We want to find cpus_to_remove + 1 CPUs to ensure we do not
 642          * remove the last CPU.
 643          */
 644         for_each_node_by_type(dn, "cpu") {
 645                 cpus_found++;
 646
 647                 if (cpus_found > cpus_to_remove) {
 648                         of_node_put(dn);
 649                         break;
 650                 }
 651
 652                 /* Note that cpus_found is always 1 ahead of the index
 653                  * into the cpu_drcs array, so we use cpus_found - 1
 654                  */
 655                 rc = of_property_read_u32(dn, "ibm,my-drc-index",
 656                                           &cpu_drcs[cpus_found - 1]);
 657                 if (rc) {
 658                         pr_warn("Error occurred getting drc-index for %pOFn\n",
 659                                 dn);
 660                         of_node_put(dn);
 661                         return -1;
 662                 }
 663         }
 664
 665         if (cpus_found < cpus_to_remove) {
 666                 pr_warn("Failed to find enough CPUs (%d of %d) to remove\n",
 667                         cpus_found, cpus_to_remove);
 668         } else if (cpus_found == cpus_to_remove) {
 669                 pr_warn("Cannot remove all CPUs\n");
 670         }
 671
 672         return cpus_found;
 673 }
 674
 675 static int dlpar_cpu_remove_by_count(u32 cpus_to_remove)
 676 {
 677         u32 *cpu_drcs;
 678         int cpus_found;
 679         int cpus_removed = 0;
 680         int i, rc;
 681
 682         pr_debug("Attempting to hot-remove %d CPUs\n", cpus_to_remove);
 683
 684         cpu_drcs = kcalloc(cpus_to_remove, sizeof(*cpu_drcs), GFP_KERNEL);
 685         if (!cpu_drcs)
 686                 return -EINVAL;
 687
 688         cpus_found = find_dlpar_cpus_to_remove(cpu_drcs, cpus_to_remove);
 689         if (cpus_found <= cpus_to_remove) {
 690                 kfree(cpu_drcs);
 691                 return -EINVAL;
 692         }
 693
 694         for (i = 0; i < cpus_to_remove; i++) {
 695                 rc = dlpar_cpu_remove_by_index(cpu_drcs[i]);
 696                 if (rc)
 697                         break;
 698
 699                 cpus_removed++;
 700         }
 701
 702         if (cpus_removed != cpus_to_remove) {
 703                 pr_warn("CPU hot-remove failed, adding back removed CPUs\n");
 704
 705                 for (i = 0; i < cpus_removed; i++)
 706                         dlpar_cpu_add(cpu_drcs[i]);
 707
 708                 rc = -EINVAL;
 709         } else {
 710                 rc = 0;
 711         }
 712
 713         kfree(cpu_drcs);
 714         return rc;
 715 }
 716
 717 static int find_dlpar_cpus_to_add(u32 *cpu_drcs, u32 cpus_to_add)
 718 {
 719         struct device_node *parent;
 720         int cpus_found = 0;
 721         int index, rc;
 722
 723         parent = of_find_node_by_path("/cpus");
 724         if (!parent) {
 725                 pr_warn("Could not find CPU root node in device tree\n");
 726                 kfree(cpu_drcs);
 727                 return -1;
 728         }
 729
 730         /* Search the ibm,drc-indexes array for possible CPU drcs to
 731          * add. Note that the format of the ibm,drc-indexes array is
 732          * the number of entries in the array followed by the array
 733          * of drc values so we start looking at index = 1.
 734          */
 735         index = 1;
 736         while (cpus_found < cpus_to_add) {
 737                 u32 drc;
 738
 739                 rc = of_property_read_u32_index(parent, "ibm,drc-indexes",
 740                                                 index++, &drc);
 741                 if (rc)
 742                         break;
 743
 744                 if (dlpar_cpu_exists(parent, drc))
 745                         continue;
 746
 747                 cpu_drcs[cpus_found++] = drc;
 748         }
 749
 750         of_node_put(parent);
 751         return cpus_found;
 752 }
 753
 754 static int dlpar_cpu_add_by_count(u32 cpus_to_add)
 755 {
 756         u32 *cpu_drcs;
 757         int cpus_added = 0;
 758         int cpus_found;
 759         int i, rc;
 760
 761         pr_debug("Attempting to hot-add %d CPUs\n", cpus_to_add);
 762
 763         cpu_drcs = kcalloc(cpus_to_add, sizeof(*cpu_drcs), GFP_KERNEL);
 764         if (!cpu_drcs)
 765                 return -EINVAL;
 766
 767         cpus_found = find_dlpar_cpus_to_add(cpu_drcs, cpus_to_add);
 768         if (cpus_found < cpus_to_add) {
 769                 pr_warn("Failed to find enough CPUs (%d of %d) to add\n",
 770                         cpus_found, cpus_to_add);
 771                 kfree(cpu_drcs);
 772                 return -EINVAL;
 773         }
 774
 775         for (i = 0; i < cpus_to_add; i++) {
 776                 rc = dlpar_cpu_add(cpu_drcs[i]);
 777                 if (rc)
 778                         break;
 779
 780                 cpus_added++;
 781         }
 782
 783         if (cpus_added < cpus_to_add) {
 784                 pr_warn("CPU hot-add failed, removing any added CPUs\n");
 785
 786                 for (i = 0; i < cpus_added; i++)
 787                         dlpar_cpu_remove_by_index(cpu_drcs[i]);
 788
 789                 rc = -EINVAL;
 790         } else {
 791                 rc = 0;
 792         }
 793
 794         kfree(cpu_drcs);
 795         return rc;
 796 }
 797
 798 int dlpar_cpu_readd(int cpu)
 799 {
 800         struct device_node *dn;
 801         struct device *dev;
 802         u32 drc_index;
 803         int rc;
 804
 805         dev = get_cpu_device(cpu);
 806         dn = dev->of_node;
 807
 808         rc = of_property_read_u32(dn, "ibm,my-drc-index", &drc_index);
 809
 810         rc = dlpar_cpu_remove_by_index(drc_index);
 811         if (!rc)
 812                 rc = dlpar_cpu_add(drc_index);
 813
 814         return rc;
 815 }
 816
 817 int dlpar_cpu(struct pseries_hp_errorlog *hp_elog)
 818 {
 819         u32 count, drc_index;
 820         int rc;
 821
 822         count = hp_elog->_drc_u.drc_count;
 823         drc_index = hp_elog->_drc_u.drc_index;
 824
 825         lock_device_hotplug();
 826
 827         switch (hp_elog->action) {
 828         case PSERIES_HP_ELOG_ACTION_REMOVE:
 829                 if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_COUNT)
 830                         rc = dlpar_cpu_remove_by_count(count);
 831                 else if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_INDEX)
 832                         rc = dlpar_cpu_remove_by_index(drc_index);
 833                 else
 834                         rc = -EINVAL;
 835                 break;
 836         case PSERIES_HP_ELOG_ACTION_ADD:
 837                 if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_COUNT)
 838                         rc = dlpar_cpu_add_by_count(count);
 839                 else if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_INDEX)
 840                         rc = dlpar_cpu_add(drc_index);
 841                 else
 842                         rc = -EINVAL;
 843                 break;
 844         default:
 845                 pr_err("Invalid action (%d) specified\n", hp_elog->action);
 846                 rc = -EINVAL;
 847                 break;
 848         }
 849
 850         unlock_device_hotplug();
 851         return rc;
 852 }
 853
 854 #ifdef CONFIG_ARCH_CPU_PROBE_RELEASE
 855
 856 static ssize_t dlpar_cpu_probe(const char *buf, size_t count)
 857 {
 858         u32 drc_index;
 859         int rc;
 860
 861         rc = kstrtou32(buf, 0, &drc_index);
 862         if (rc)
 863                 return -EINVAL;
 864
 865         rc = dlpar_cpu_add(drc_index);
 866
 867         return rc ? rc : count;
 868 }
 869
 870 static ssize_t dlpar_cpu_release(const char *buf, size_t count)
 871 {
 872         struct device_node *dn;
 873         u32 drc_index;
 874         int rc;
 875
 876         dn = of_find_node_by_path(buf);
 877         if (!dn)
 878                 return -EINVAL;
 879
 880         rc = of_property_read_u32(dn, "ibm,my-drc-index", &drc_index);
 881         if (rc) {
 882                 of_node_put(dn);
 883                 return -EINVAL;
 884         }
 885
 886         rc = dlpar_cpu_remove(dn, drc_index);
 887         of_node_put(dn);
 888
 889         return rc ? rc : count;
 890 }
 891
 892 #endif /* CONFIG_ARCH_CPU_PROBE_RELEASE */
 893
 894 static int pseries_smp_notifier(struct notifier_block *nb,
 895                                 unsigned long action, void *data)
 896 {
 897         struct of_reconfig_data *rd = data;
 898         int err = 0;
 899
 900         switch (action) {
 901         case OF_RECONFIG_ATTACH_NODE:
 902                 err = pseries_add_processor(rd->dn);
 903                 break;
 904         case OF_RECONFIG_DETACH_NODE:
 905                 pseries_remove_processor(rd->dn);
 906                 break;
 907         }
 908         return notifier_from_errno(err);
 909 }
 910
 911 static struct notifier_block pseries_smp_nb = {
 912         .notifier_call = pseries_smp_notifier,
 913 };
 914
 915 #define MAX_CEDE_LATENCY_LEVELS         4
 916 #define CEDE_LATENCY_PARAM_LENGTH       10
 917 #define CEDE_LATENCY_PARAM_MAX_LENGTH   \
 918         (MAX_CEDE_LATENCY_LEVELS * CEDE_LATENCY_PARAM_LENGTH * sizeof(char))
 919 #define CEDE_LATENCY_TOKEN              45
 920
 921 static char cede_parameters[CEDE_LATENCY_PARAM_MAX_LENGTH];
 922
 923 static int parse_cede_parameters(void)
 924 {
 925         memset(cede_parameters, 0, CEDE_LATENCY_PARAM_MAX_LENGTH);
 926         return rtas_call(rtas_token("ibm,get-system-parameter"), 3, 1,
 927                          NULL,
 928                          CEDE_LATENCY_TOKEN,
 929                          __pa(cede_parameters),
 930                          CEDE_LATENCY_PARAM_MAX_LENGTH);
 931 }
 932
 933 static int __init pseries_cpu_hotplug_init(void)
 934 {
 935         int cpu;
 936         int qcss_tok;
 937
 938 #ifdef CONFIG_ARCH_CPU_PROBE_RELEASE
 939         ppc_md.cpu_probe = dlpar_cpu_probe;
 940         ppc_md.cpu_release = dlpar_cpu_release;
 941 #endif /* CONFIG_ARCH_CPU_PROBE_RELEASE */
 942
 943         rtas_stop_self_token = rtas_token("stop-self");
 944         qcss_tok = rtas_token("query-cpu-stopped-state");
 945
 946         if (rtas_stop_self_token == RTAS_UNKNOWN_SERVICE ||
 947                         qcss_tok == RTAS_UNKNOWN_SERVICE) {
 948                 printk(KERN_INFO "CPU Hotplug not supported by firmware "
 949                                 "- disabling.\n");
 950                 return 0;
 951         }
 952
 953         ppc_md.cpu_die = pseries_mach_cpu_die;
 954         smp_ops->cpu_disable = pseries_cpu_disable;
 955         smp_ops->cpu_die = pseries_cpu_die;
 956
 957         /* Processors can be added/removed only on LPAR */
 958         if (firmware_has_feature(FW_FEATURE_LPAR)) {
 959                 of_reconfig_notifier_register(&pseries_smp_nb);
 960                 cpu_maps_update_begin();
 961                 if (cede_offline_enabled && parse_cede_parameters() == 0) {
 962                         default_offline_state = CPU_STATE_INACTIVE;
 963                         for_each_online_cpu(cpu)
 964                                 set_default_offline_state(cpu);
 965                 }
 966                 cpu_maps_update_done();
 967         }
 968
 969         return 0;
 970 }
 971 machine_arch_initcall(pseries, pseries_cpu_hotplug_init);