arch/x86/kvm/vmx/posted_intr.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 #include <linux/kvm_host.h>
   3
   4 #include <asm/irq_remapping.h>
   5 #include <asm/cpu.h>
   6
   7 #include "lapic.h"
   8 #include "irq.h"
   9 #include "posted_intr.h"
  10 #include "trace.h"
  11 #include "vmx.h"
  12
  13 /*
  14  * Maintain a per-CPU list of vCPUs that need to be awakened by wakeup_handler()
  15  * when a WAKEUP_VECTOR interrupted is posted.  vCPUs are added to the list when
  16  * the vCPU is scheduled out and is blocking (e.g. in HLT) with IRQs enabled.
  17  * The vCPUs posted interrupt descriptor is updated at the same time to set its
  18  * notification vector to WAKEUP_VECTOR, so that posted interrupt from devices
  19  * wake the target vCPUs.  vCPUs are removed from the list and the notification
  20  * vector is reset when the vCPU is scheduled in.
  21  */
  22 static DEFINE_PER_CPU(struct list_head, wakeup_vcpus_on_cpu);
  23 /*
  24  * Protect the per-CPU list with a per-CPU spinlock to handle task migration.
  25  * When a blocking vCPU is awakened _and_ migrated to a different pCPU, the
  26  * ->sched_in() path will need to take the vCPU off the list of the _previous_
  27  * CPU.  IRQs must be disabled when taking this lock, otherwise deadlock will
  28  * occur if a wakeup IRQ arrives and attempts to acquire the lock.
  29  */
  30 static DEFINE_PER_CPU(raw_spinlock_t, wakeup_vcpus_on_cpu_lock);
  31
  32 static inline struct pi_desc *vcpu_to_pi_desc(struct kvm_vcpu *vcpu)
  33 {
  34         return &(to_vmx(vcpu)->pi_desc);
  35 }
  36
  37 static int pi_try_set_control(struct pi_desc *pi_desc, u64 old, u64 new)
  38 {
  39         /*
  40          * PID.ON can be set at any time by a different vCPU or by hardware,
  41          * e.g. a device.  PID.control must be written atomically, and the
  42          * update must be retried with a fresh snapshot an ON change causes
  43          * the cmpxchg to fail.
  44          */
  45         if (cmpxchg64(&pi_desc->control, old, new) != old)
  46                 return -EBUSY;
  47
  48         return 0;
  49 }
  50
  51 void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu)
  52 {
  53         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
  54         struct vcpu_vmx *vmx = to_vmx(vcpu);
  55         struct pi_desc old, new;
  56         unsigned long flags;
  57         unsigned int dest;
  58
  59         /*
  60          * To simplify hot-plug and dynamic toggling of APICv, keep PI.NDST and
  61          * PI.SN up-to-date even if there is no assigned device or if APICv is
  62          * deactivated due to a dynamic inhibit bit, e.g. for Hyper-V's SyncIC.
  63          */
  64         if (!enable_apicv || !lapic_in_kernel(vcpu))
  65                 return;
  66
  67         /*
  68          * If the vCPU wasn't on the wakeup list and wasn't migrated, then the
  69          * full update can be skipped as neither the vector nor the destination
  70          * needs to be changed.
  71          */
  72         if (pi_desc->nv != POSTED_INTR_WAKEUP_VECTOR && vcpu->cpu == cpu) {
  73                 /*
  74                  * Clear SN if it was set due to being preempted.  Again, do
  75                  * this even if there is no assigned device for simplicity.
  76                  */
  77                 if (pi_test_and_clear_sn(pi_desc))
  78                         goto after_clear_sn;
  79                 return;
  80         }
  81
  82         local_irq_save(flags);
  83
  84         /*
  85          * If the vCPU was waiting for wakeup, remove the vCPU from the wakeup
  86          * list of the _previous_ pCPU, which will not be the same as the
  87          * current pCPU if the task was migrated.
  88          */
  89         if (pi_desc->nv == POSTED_INTR_WAKEUP_VECTOR) {
  90                 raw_spin_lock(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu));
  91                 list_del(&vmx->pi_wakeup_list);
  92                 raw_spin_unlock(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu));
  93         }
  94
  95         dest = cpu_physical_id(cpu);
  96         if (!x2apic_mode)
  97                 dest = (dest << 8) & 0xFF00;
  98
  99         do {
 100                 old.control = new.control = READ_ONCE(pi_desc->control);
 101
 102                 /*
 103                  * Clear SN (as above) and refresh the destination APIC ID to
 104                  * handle task migration (@cpu != vcpu->cpu).
 105                  */
 106                 new.ndst = dest;
 107                 new.sn = 0;
 108
 109                 /*
 110                  * Restore the notification vector; in the blocking case, the
 111                  * descriptor was modified on "put" to use the wakeup vector.
 112                  */
 113                 new.nv = POSTED_INTR_VECTOR;
 114         } while (pi_try_set_control(pi_desc, old.control, new.control));
 115
 116         local_irq_restore(flags);
 117
 118 after_clear_sn:
 119
 120         /*
 121          * Clear SN before reading the bitmap.  The VT-d firmware
 122          * writes the bitmap and reads SN atomically (5.2.3 in the
 123          * spec), so it doesn't really have a memory barrier that
 124          * pairs with this, but we cannot do that and we need one.
 125          */
 126         smp_mb__after_atomic();
 127
 128         if (!pi_is_pir_empty(pi_desc))
 129                 pi_set_on(pi_desc);
 130 }
 131
 132 static bool vmx_can_use_vtd_pi(struct kvm *kvm)
 133 {
 134         return irqchip_in_kernel(kvm) && enable_apicv &&
 135                 kvm_arch_has_assigned_device(kvm) &&
 136                 irq_remapping_cap(IRQ_POSTING_CAP);
 137 }
 138
 139 /*
 140  * Put the vCPU on this pCPU's list of vCPUs that needs to be awakened and set
 141  * WAKEUP as the notification vector in the PI descriptor.
 142  */
 143 static void pi_enable_wakeup_handler(struct kvm_vcpu *vcpu)
 144 {
 145         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
 146         struct vcpu_vmx *vmx = to_vmx(vcpu);
 147         struct pi_desc old, new;
 148         unsigned long flags;
 149
 150         local_irq_save(flags);
 151
 152         raw_spin_lock(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu));
 153         list_add_tail(&vmx->pi_wakeup_list,
 154                       &per_cpu(wakeup_vcpus_on_cpu, vcpu->cpu));
 155         raw_spin_unlock(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu));
 156
 157         WARN(pi_desc->sn, "PI descriptor SN field set before blocking");
 158
 159         do {
 160                 old.control = new.control = READ_ONCE(pi_desc->control);
 161
 162                 /* set 'NV' to 'wakeup vector' */
 163                 new.nv = POSTED_INTR_WAKEUP_VECTOR;
 164         } while (pi_try_set_control(pi_desc, old.control, new.control));
 165
 166         /*
 167          * Send a wakeup IPI to this CPU if an interrupt may have been posted
 168          * before the notification vector was updated, in which case the IRQ
 169          * will arrive on the non-wakeup vector.  An IPI is needed as calling
 170          * try_to_wake_up() from ->sched_out() isn't allowed (IRQs are not
 171          * enabled until it is safe to call try_to_wake_up() on the task being
 172          * scheduled out).
 173          */
 174         if (pi_test_on(&new))
 175                 apic->send_IPI_self(POSTED_INTR_WAKEUP_VECTOR);
 176
 177         local_irq_restore(flags);
 178 }
 179
 180 void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu)
 181 {
 182         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
 183
 184         if (!vmx_can_use_vtd_pi(vcpu->kvm))
 185                 return;
 186
 187         if (kvm_vcpu_is_blocking(vcpu) && !vmx_interrupt_blocked(vcpu))
 188                 pi_enable_wakeup_handler(vcpu);
 189
 190         /*
 191          * Set SN when the vCPU is preempted.  Note, the vCPU can both be seen
 192          * as blocking and preempted, e.g. if it's preempted between setting
 193          * its wait state and manually scheduling out.
 194          */
 195         if (vcpu->preempted)
 196                 pi_set_sn(pi_desc);
 197 }
 198
 199 /*
 200  * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR.
 201  */
 202 void pi_wakeup_handler(void)
 203 {
 204         int cpu = smp_processor_id();
 205         struct list_head *wakeup_list = &per_cpu(wakeup_vcpus_on_cpu, cpu);
 206         raw_spinlock_t *spinlock = &per_cpu(wakeup_vcpus_on_cpu_lock, cpu);
 207         struct vcpu_vmx *vmx;
 208
 209         raw_spin_lock(spinlock);
 210         list_for_each_entry(vmx, wakeup_list, pi_wakeup_list) {
 211
 212                 if (pi_test_on(&vmx->pi_desc))
 213                         kvm_vcpu_wake_up(&vmx->vcpu);
 214         }
 215         raw_spin_unlock(spinlock);
 216 }
 217
 218 void __init pi_init_cpu(int cpu)
 219 {
 220         INIT_LIST_HEAD(&per_cpu(wakeup_vcpus_on_cpu, cpu));
 221         raw_spin_lock_init(&per_cpu(wakeup_vcpus_on_cpu_lock, cpu));
 222 }
 223
 224 bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu)
 225 {
 226         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
 227
 228         return pi_test_on(pi_desc) ||
 229                 (pi_test_sn(pi_desc) && !pi_is_pir_empty(pi_desc));
 230 }
 231
 232
 233 /*
 234  * Bail out of the block loop if the VM has an assigned
 235  * device, but the blocking vCPU didn't reconfigure the
 236  * PI.NV to the wakeup vector, i.e. the assigned device
 237  * came along after the initial check in vmx_vcpu_pi_put().
 238  */
 239 void vmx_pi_start_assignment(struct kvm *kvm)
 240 {
 241         if (!irq_remapping_cap(IRQ_POSTING_CAP))
 242                 return;
 243
 244         kvm_make_all_cpus_request(kvm, KVM_REQ_UNBLOCK);
 245 }
 246
 247 /*
 248  * vmx_pi_update_irte - set IRTE for Posted-Interrupts
 249  *
 250  * @kvm: kvm
 251  * @host_irq: host irq of the interrupt
 252  * @guest_irq: gsi of the interrupt
 253  * @set: set or unset PI
 254  * returns 0 on success, < 0 on failure
 255  */
 256 int vmx_pi_update_irte(struct kvm *kvm, unsigned int host_irq,
 257                        uint32_t guest_irq, bool set)
 258 {
 259         struct kvm_kernel_irq_routing_entry *e;
 260         struct kvm_irq_routing_table *irq_rt;
 261         struct kvm_lapic_irq irq;
 262         struct kvm_vcpu *vcpu;
 263         struct vcpu_data vcpu_info;
 264         int idx, ret = 0;
 265
 266         if (!vmx_can_use_vtd_pi(kvm))
 267                 return 0;
 268
 269         idx = srcu_read_lock(&kvm->irq_srcu);
 270         irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
 271         if (guest_irq >= irq_rt->nr_rt_entries ||
 272             hlist_empty(&irq_rt->map[guest_irq])) {
 273                 pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n",
 274                              guest_irq, irq_rt->nr_rt_entries);
 275                 goto out;
 276         }
 277
 278         hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
 279                 if (e->type != KVM_IRQ_ROUTING_MSI)
 280                         continue;
 281                 /*
 282                  * VT-d PI cannot support posting multicast/broadcast
 283                  * interrupts to a vCPU, we still use interrupt remapping
 284                  * for these kind of interrupts.
 285                  *
 286                  * For lowest-priority interrupts, we only support
 287                  * those with single CPU as the destination, e.g. user
 288                  * configures the interrupts via /proc/irq or uses
 289                  * irqbalance to make the interrupts single-CPU.
 290                  *
 291                  * We will support full lowest-priority interrupt later.
 292                  *
 293                  * In addition, we can only inject generic interrupts using
 294                  * the PI mechanism, refuse to route others through it.
 295                  */
 296
 297                 kvm_set_msi_irq(kvm, e, &irq);
 298                 if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) ||
 299                     !kvm_irq_is_postable(&irq)) {
 300                         /*
 301                          * Make sure the IRTE is in remapped mode if
 302                          * we don't handle it in posted mode.
 303                          */
 304                         ret = irq_set_vcpu_affinity(host_irq, NULL);
 305                         if (ret < 0) {
 306                                 printk(KERN_INFO
 307                                    "failed to back to remapped mode, irq: %u\n",
 308                                    host_irq);
 309                                 goto out;
 310                         }
 311
 312                         continue;
 313                 }
 314
 315                 vcpu_info.pi_desc_addr = __pa(vcpu_to_pi_desc(vcpu));
 316                 vcpu_info.vector = irq.vector;
 317
 318                 trace_kvm_pi_irte_update(host_irq, vcpu->vcpu_id, e->gsi,
 319                                 vcpu_info.vector, vcpu_info.pi_desc_addr, set);
 320
 321                 if (set)
 322                         ret = irq_set_vcpu_affinity(host_irq, &vcpu_info);
 323                 else
 324                         ret = irq_set_vcpu_affinity(host_irq, NULL);
 325
 326                 if (ret < 0) {
 327                         printk(KERN_INFO "%s: failed to update PI IRTE\n",
 328                                         __func__);
 329                         goto out;
 330                 }
 331         }
 332
 333         ret = 0;
 334 out:
 335         srcu_read_unlock(&kvm->irq_srcu, idx);
 336         return ret;
 337 }