arch/x86/xen/smp.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/smp.h>
   3 #include <linux/cpu.h>
   4 #include <linux/slab.h>
   5 #include <linux/cpumask.h>
   6 #include <linux/percpu.h>
   7
   8 #include <xen/events.h>
   9
  10 #include <xen/hvc-console.h>
  11 #include "xen-ops.h"
  12 #include "smp.h"
  13
  14 static DEFINE_PER_CPU(struct xen_common_irq, xen_resched_irq) = { .irq = -1 };
  15 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfunc_irq) = { .irq = -1 };
  16 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfuncsingle_irq) = { .irq = -1 };
  17 static DEFINE_PER_CPU(struct xen_common_irq, xen_debug_irq) = { .irq = -1 };
  18
  19 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id);
  20 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
  21
  22 /*
  23  * Reschedule call back.
  24  */
  25 static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
  26 {
  27         inc_irq_stat(irq_resched_count);
  28         scheduler_ipi();
  29
  30         return IRQ_HANDLED;
  31 }
  32
  33 void xen_smp_intr_free(unsigned int cpu)
  34 {
  35         kfree(per_cpu(xen_resched_irq, cpu).name);
  36         per_cpu(xen_resched_irq, cpu).name = NULL;
  37         if (per_cpu(xen_resched_irq, cpu).irq >= 0) {
  38                 unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu).irq, NULL);
  39                 per_cpu(xen_resched_irq, cpu).irq = -1;
  40         }
  41         kfree(per_cpu(xen_callfunc_irq, cpu).name);
  42         per_cpu(xen_callfunc_irq, cpu).name = NULL;
  43         if (per_cpu(xen_callfunc_irq, cpu).irq >= 0) {
  44                 unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu).irq, NULL);
  45                 per_cpu(xen_callfunc_irq, cpu).irq = -1;
  46         }
  47         kfree(per_cpu(xen_debug_irq, cpu).name);
  48         per_cpu(xen_debug_irq, cpu).name = NULL;
  49         if (per_cpu(xen_debug_irq, cpu).irq >= 0) {
  50                 unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu).irq, NULL);
  51                 per_cpu(xen_debug_irq, cpu).irq = -1;
  52         }
  53         kfree(per_cpu(xen_callfuncsingle_irq, cpu).name);
  54         per_cpu(xen_callfuncsingle_irq, cpu).name = NULL;
  55         if (per_cpu(xen_callfuncsingle_irq, cpu).irq >= 0) {
  56                 unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu).irq,
  57                                        NULL);
  58                 per_cpu(xen_callfuncsingle_irq, cpu).irq = -1;
  59         }
  60 }
  61
  62 int xen_smp_intr_init(unsigned int cpu)
  63 {
  64         int rc;
  65         char *resched_name, *callfunc_name, *debug_name;
  66
  67         resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu);
  68         if (!resched_name)
  69                 goto fail_mem;
  70         per_cpu(xen_resched_irq, cpu).name = resched_name;
  71         rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR,
  72                                     cpu,
  73                                     xen_reschedule_interrupt,
  74                                     IRQF_PERCPU|IRQF_NOBALANCING,
  75                                     resched_name,
  76                                     NULL);
  77         if (rc < 0)
  78                 goto fail;
  79         per_cpu(xen_resched_irq, cpu).irq = rc;
  80
  81         callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu);
  82         if (!callfunc_name)
  83                 goto fail_mem;
  84         per_cpu(xen_callfunc_irq, cpu).name = callfunc_name;
  85         rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR,
  86                                     cpu,
  87                                     xen_call_function_interrupt,
  88                                     IRQF_PERCPU|IRQF_NOBALANCING,
  89                                     callfunc_name,
  90                                     NULL);
  91         if (rc < 0)
  92                 goto fail;
  93         per_cpu(xen_callfunc_irq, cpu).irq = rc;
  94
  95         if (!xen_fifo_events) {
  96                 debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu);
  97                 if (!debug_name)
  98                         goto fail_mem;
  99
 100                 per_cpu(xen_debug_irq, cpu).name = debug_name;
 101                 rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu,
 102                                              xen_debug_interrupt,
 103                                              IRQF_PERCPU | IRQF_NOBALANCING,
 104                                              debug_name, NULL);
 105                 if (rc < 0)
 106                         goto fail;
 107                 per_cpu(xen_debug_irq, cpu).irq = rc;
 108         }
 109
 110         callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu);
 111         if (!callfunc_name)
 112                 goto fail_mem;
 113
 114         per_cpu(xen_callfuncsingle_irq, cpu).name = callfunc_name;
 115         rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR,
 116                                     cpu,
 117                                     xen_call_function_single_interrupt,
 118                                     IRQF_PERCPU|IRQF_NOBALANCING,
 119                                     callfunc_name,
 120                                     NULL);
 121         if (rc < 0)
 122                 goto fail;
 123         per_cpu(xen_callfuncsingle_irq, cpu).irq = rc;
 124
 125         return 0;
 126
 127  fail_mem:
 128         rc = -ENOMEM;
 129  fail:
 130         xen_smp_intr_free(cpu);
 131         return rc;
 132 }
 133
 134 void __init xen_smp_cpus_done(unsigned int max_cpus)
 135 {
 136         int cpu, rc, count = 0;
 137
 138         if (xen_hvm_domain())
 139                 native_smp_cpus_done(max_cpus);
 140         else
 141                 calculate_max_logical_packages();
 142
 143         if (xen_have_vcpu_info_placement)
 144                 return;
 145
 146         for_each_online_cpu(cpu) {
 147                 if (xen_vcpu_nr(cpu) < MAX_VIRT_CPUS)
 148                         continue;
 149
 150                 rc = cpu_down(cpu);
 151
 152                 if (rc == 0) {
 153                         /*
 154                          * Reset vcpu_info so this cpu cannot be onlined again.
 155                          */
 156                         xen_vcpu_info_reset(cpu);
 157                         count++;
 158                 } else {
 159                         pr_warn("%s: failed to bring CPU %d down, error %d\n",
 160                                 __func__, cpu, rc);
 161                 }
 162         }
 163         WARN(count, "%s: brought %d CPUs offline\n", __func__, count);
 164 }
 165
 166 void xen_smp_send_reschedule(int cpu)
 167 {
 168         xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR);
 169 }
 170
 171 static void __xen_send_IPI_mask(const struct cpumask *mask,
 172                               int vector)
 173 {
 174         unsigned cpu;
 175
 176         for_each_cpu_and(cpu, mask, cpu_online_mask)
 177                 xen_send_IPI_one(cpu, vector);
 178 }
 179
 180 void xen_smp_send_call_function_ipi(const struct cpumask *mask)
 181 {
 182         int cpu;
 183
 184         __xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);
 185
 186         /* Make sure other vcpus get a chance to run if they need to. */
 187         for_each_cpu(cpu, mask) {
 188                 if (xen_vcpu_stolen(cpu)) {
 189                         HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
 190                         break;
 191                 }
 192         }
 193 }
 194
 195 void xen_smp_send_call_function_single_ipi(int cpu)
 196 {
 197         __xen_send_IPI_mask(cpumask_of(cpu),
 198                           XEN_CALL_FUNCTION_SINGLE_VECTOR);
 199 }
 200
 201 static inline int xen_map_vector(int vector)
 202 {
 203         int xen_vector;
 204
 205         switch (vector) {
 206         case RESCHEDULE_VECTOR:
 207                 xen_vector = XEN_RESCHEDULE_VECTOR;
 208                 break;
 209         case CALL_FUNCTION_VECTOR:
 210                 xen_vector = XEN_CALL_FUNCTION_VECTOR;
 211                 break;
 212         case CALL_FUNCTION_SINGLE_VECTOR:
 213                 xen_vector = XEN_CALL_FUNCTION_SINGLE_VECTOR;
 214                 break;
 215         case IRQ_WORK_VECTOR:
 216                 xen_vector = XEN_IRQ_WORK_VECTOR;
 217                 break;
 218 #ifdef CONFIG_X86_64
 219         case NMI_VECTOR:
 220         case APIC_DM_NMI: /* Some use that instead of NMI_VECTOR */
 221                 xen_vector = XEN_NMI_VECTOR;
 222                 break;
 223 #endif
 224         default:
 225                 xen_vector = -1;
 226                 printk(KERN_ERR "xen: vector 0x%x is not implemented\n",
 227                         vector);
 228         }
 229
 230         return xen_vector;
 231 }
 232
 233 void xen_send_IPI_mask(const struct cpumask *mask,
 234                               int vector)
 235 {
 236         int xen_vector = xen_map_vector(vector);
 237
 238         if (xen_vector >= 0)
 239                 __xen_send_IPI_mask(mask, xen_vector);
 240 }
 241
 242 void xen_send_IPI_all(int vector)
 243 {
 244         int xen_vector = xen_map_vector(vector);
 245
 246         if (xen_vector >= 0)
 247                 __xen_send_IPI_mask(cpu_online_mask, xen_vector);
 248 }
 249
 250 void xen_send_IPI_self(int vector)
 251 {
 252         int xen_vector = xen_map_vector(vector);
 253
 254         if (xen_vector >= 0)
 255                 xen_send_IPI_one(smp_processor_id(), xen_vector);
 256 }
 257
 258 void xen_send_IPI_mask_allbutself(const struct cpumask *mask,
 259                                 int vector)
 260 {
 261         unsigned cpu;
 262         unsigned int this_cpu = smp_processor_id();
 263         int xen_vector = xen_map_vector(vector);
 264
 265         if (!(num_online_cpus() > 1) || (xen_vector < 0))
 266                 return;
 267
 268         for_each_cpu_and(cpu, mask, cpu_online_mask) {
 269                 if (this_cpu == cpu)
 270                         continue;
 271
 272                 xen_send_IPI_one(cpu, xen_vector);
 273         }
 274 }
 275
 276 void xen_send_IPI_allbutself(int vector)
 277 {
 278         xen_send_IPI_mask_allbutself(cpu_online_mask, vector);
 279 }
 280
 281 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
 282 {
 283         irq_enter();
 284         generic_smp_call_function_interrupt();
 285         inc_irq_stat(irq_call_count);
 286         irq_exit();
 287
 288         return IRQ_HANDLED;
 289 }
 290
 291 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
 292 {
 293         irq_enter();
 294         generic_smp_call_function_single_interrupt();
 295         inc_irq_stat(irq_call_count);
 296         irq_exit();
 297
 298         return IRQ_HANDLED;
 299 }