arch/arm64/kvm/vgic/vgic-mmio.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * VGIC MMIO handling functions
   4  */
   5
   6 #include <linux/bitops.h>
   7 #include <linux/bsearch.h>
   8 #include <linux/interrupt.h>
   9 #include <linux/irq.h>
  10 #include <linux/kvm.h>
  11 #include <linux/kvm_host.h>
  12 #include <kvm/iodev.h>
  13 #include <kvm/arm_arch_timer.h>
  14 #include <kvm/arm_vgic.h>
  15
  16 #include "vgic.h"
  17 #include "vgic-mmio.h"
  18
  19 unsigned long vgic_mmio_read_raz(struct kvm_vcpu *vcpu,
  20                                  gpa_t addr, unsigned int len)
  21 {
  22         return 0;
  23 }
  24
  25 unsigned long vgic_mmio_read_rao(struct kvm_vcpu *vcpu,
  26                                  gpa_t addr, unsigned int len)
  27 {
  28         return -1UL;
  29 }
  30
  31 void vgic_mmio_write_wi(struct kvm_vcpu *vcpu, gpa_t addr,
  32                         unsigned int len, unsigned long val)
  33 {
  34         /* Ignore */
  35 }
  36
  37 int vgic_mmio_uaccess_write_wi(struct kvm_vcpu *vcpu, gpa_t addr,
  38                                unsigned int len, unsigned long val)
  39 {
  40         /* Ignore */
  41         return 0;
  42 }
  43
  44 unsigned long vgic_mmio_read_group(struct kvm_vcpu *vcpu,
  45                                    gpa_t addr, unsigned int len)
  46 {
  47         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
  48         u32 value = 0;
  49         int i;
  50
  51         /* Loop over all IRQs affected by this read */
  52         for (i = 0; i < len * 8; i++) {
  53                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
  54
  55                 if (irq->group)
  56                         value |= BIT(i);
  57
  58                 vgic_put_irq(vcpu->kvm, irq);
  59         }
  60
  61         return value;
  62 }
  63
  64 static void vgic_update_vsgi(struct vgic_irq *irq)
  65 {
  66         WARN_ON(its_prop_update_vsgi(irq->host_irq, irq->priority, irq->group));
  67 }
  68
  69 void vgic_mmio_write_group(struct kvm_vcpu *vcpu, gpa_t addr,
  70                            unsigned int len, unsigned long val)
  71 {
  72         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
  73         int i;
  74         unsigned long flags;
  75
  76         for (i = 0; i < len * 8; i++) {
  77                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
  78
  79                 raw_spin_lock_irqsave(&irq->irq_lock, flags);
  80                 irq->group = !!(val & BIT(i));
  81                 if (irq->hw && vgic_irq_is_sgi(irq->intid)) {
  82                         vgic_update_vsgi(irq);
  83                         raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
  84                 } else {
  85                         vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
  86                 }
  87
  88                 vgic_put_irq(vcpu->kvm, irq);
  89         }
  90 }
  91
  92 /*
  93  * Read accesses to both GICD_ICENABLER and GICD_ISENABLER return the value
  94  * of the enabled bit, so there is only one function for both here.
  95  */
  96 unsigned long vgic_mmio_read_enable(struct kvm_vcpu *vcpu,
  97                                     gpa_t addr, unsigned int len)
  98 {
  99         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 100         u32 value = 0;
 101         int i;
 102
 103         /* Loop over all IRQs affected by this read */
 104         for (i = 0; i < len * 8; i++) {
 105                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 106
 107                 if (irq->enabled)
 108                         value |= (1U << i);
 109
 110                 vgic_put_irq(vcpu->kvm, irq);
 111         }
 112
 113         return value;
 114 }
 115
 116 void vgic_mmio_write_senable(struct kvm_vcpu *vcpu,
 117                              gpa_t addr, unsigned int len,
 118                              unsigned long val)
 119 {
 120         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 121         int i;
 122         unsigned long flags;
 123
 124         for_each_set_bit(i, &val, len * 8) {
 125                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 126
 127                 raw_spin_lock_irqsave(&irq->irq_lock, flags);
 128                 if (irq->hw && vgic_irq_is_sgi(irq->intid)) {
 129                         if (!irq->enabled) {
 130                                 struct irq_data *data;
 131
 132                                 irq->enabled = true;
 133                                 data = &irq_to_desc(irq->host_irq)->irq_data;
 134                                 while (irqd_irq_disabled(data))
 135                                         enable_irq(irq->host_irq);
 136                         }
 137
 138                         raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 139                         vgic_put_irq(vcpu->kvm, irq);
 140
 141                         continue;
 142                 } else if (vgic_irq_is_mapped_level(irq)) {
 143                         bool was_high = irq->line_level;
 144
 145                         /*
 146                          * We need to update the state of the interrupt because
 147                          * the guest might have changed the state of the device
 148                          * while the interrupt was disabled at the VGIC level.
 149                          */
 150                         irq->line_level = vgic_get_phys_line_level(irq);
 151                         /*
 152                          * Deactivate the physical interrupt so the GIC will let
 153                          * us know when it is asserted again.
 154                          */
 155                         if (!irq->active && was_high && !irq->line_level)
 156                                 vgic_irq_set_phys_active(irq, false);
 157                 }
 158                 irq->enabled = true;
 159                 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
 160
 161                 vgic_put_irq(vcpu->kvm, irq);
 162         }
 163 }
 164
 165 void vgic_mmio_write_cenable(struct kvm_vcpu *vcpu,
 166                              gpa_t addr, unsigned int len,
 167                              unsigned long val)
 168 {
 169         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 170         int i;
 171         unsigned long flags;
 172
 173         for_each_set_bit(i, &val, len * 8) {
 174                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 175
 176                 raw_spin_lock_irqsave(&irq->irq_lock, flags);
 177                 if (irq->hw && vgic_irq_is_sgi(irq->intid) && irq->enabled)
 178                         disable_irq_nosync(irq->host_irq);
 179
 180                 irq->enabled = false;
 181
 182                 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 183                 vgic_put_irq(vcpu->kvm, irq);
 184         }
 185 }
 186
 187 int vgic_uaccess_write_senable(struct kvm_vcpu *vcpu,
 188                                gpa_t addr, unsigned int len,
 189                                unsigned long val)
 190 {
 191         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 192         int i;
 193         unsigned long flags;
 194
 195         for_each_set_bit(i, &val, len * 8) {
 196                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 197
 198                 raw_spin_lock_irqsave(&irq->irq_lock, flags);
 199                 irq->enabled = true;
 200                 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
 201
 202                 vgic_put_irq(vcpu->kvm, irq);
 203         }
 204
 205         return 0;
 206 }
 207
 208 int vgic_uaccess_write_cenable(struct kvm_vcpu *vcpu,
 209                                gpa_t addr, unsigned int len,
 210                                unsigned long val)
 211 {
 212         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 213         int i;
 214         unsigned long flags;
 215
 216         for_each_set_bit(i, &val, len * 8) {
 217                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 218
 219                 raw_spin_lock_irqsave(&irq->irq_lock, flags);
 220                 irq->enabled = false;
 221                 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 222
 223                 vgic_put_irq(vcpu->kvm, irq);
 224         }
 225
 226         return 0;
 227 }
 228
 229 static unsigned long __read_pending(struct kvm_vcpu *vcpu,
 230                                     gpa_t addr, unsigned int len,
 231                                     bool is_user)
 232 {
 233         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 234         u32 value = 0;
 235         int i;
 236
 237         /* Loop over all IRQs affected by this read */
 238         for (i = 0; i < len * 8; i++) {
 239                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 240                 unsigned long flags;
 241                 bool val;
 242
 243                 raw_spin_lock_irqsave(&irq->irq_lock, flags);
 244                 if (irq->hw && vgic_irq_is_sgi(irq->intid)) {
 245                         int err;
 246
 247                         val = false;
 248                         err = irq_get_irqchip_state(irq->host_irq,
 249                                                     IRQCHIP_STATE_PENDING,
 250                                                     &val);
 251                         WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);
 252                 } else if (!is_user && vgic_irq_is_mapped_level(irq)) {
 253                         val = vgic_get_phys_line_level(irq);
 254                 } else {
 255                         val = irq_is_pending(irq);
 256                 }
 257
 258                 value |= ((u32)val << i);
 259                 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 260
 261                 vgic_put_irq(vcpu->kvm, irq);
 262         }
 263
 264         return value;
 265 }
 266
 267 unsigned long vgic_mmio_read_pending(struct kvm_vcpu *vcpu,
 268                                      gpa_t addr, unsigned int len)
 269 {
 270         return __read_pending(vcpu, addr, len, false);
 271 }
 272
 273 unsigned long vgic_uaccess_read_pending(struct kvm_vcpu *vcpu,
 274                                         gpa_t addr, unsigned int len)
 275 {
 276         return __read_pending(vcpu, addr, len, true);
 277 }
 278
 279 static bool is_vgic_v2_sgi(struct kvm_vcpu *vcpu, struct vgic_irq *irq)
 280 {
 281         return (vgic_irq_is_sgi(irq->intid) &&
 282                 vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2);
 283 }
 284
 285 void vgic_mmio_write_spending(struct kvm_vcpu *vcpu,
 286                               gpa_t addr, unsigned int len,
 287                               unsigned long val)
 288 {
 289         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 290         int i;
 291         unsigned long flags;
 292
 293         for_each_set_bit(i, &val, len * 8) {
 294                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 295
 296                 /* GICD_ISPENDR0 SGI bits are WI */
 297                 if (is_vgic_v2_sgi(vcpu, irq)) {
 298                         vgic_put_irq(vcpu->kvm, irq);
 299                         continue;
 300                 }
 301
 302                 raw_spin_lock_irqsave(&irq->irq_lock, flags);
 303
 304                 if (irq->hw && vgic_irq_is_sgi(irq->intid)) {
 305                         /* HW SGI? Ask the GIC to inject it */
 306                         int err;
 307                         err = irq_set_irqchip_state(irq->host_irq,
 308                                                     IRQCHIP_STATE_PENDING,
 309                                                     true);
 310                         WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);
 311
 312                         raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 313                         vgic_put_irq(vcpu->kvm, irq);
 314
 315                         continue;
 316                 }
 317
 318                 irq->pending_latch = true;
 319                 if (irq->hw)
 320                         vgic_irq_set_phys_active(irq, true);
 321
 322                 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
 323                 vgic_put_irq(vcpu->kvm, irq);
 324         }
 325 }
 326
 327 int vgic_uaccess_write_spending(struct kvm_vcpu *vcpu,
 328                                 gpa_t addr, unsigned int len,
 329                                 unsigned long val)
 330 {
 331         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 332         int i;
 333         unsigned long flags;
 334
 335         for_each_set_bit(i, &val, len * 8) {
 336                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 337
 338                 raw_spin_lock_irqsave(&irq->irq_lock, flags);
 339                 irq->pending_latch = true;
 340
 341                 /*
 342                  * GICv2 SGIs are terribly broken. We can't restore
 343                  * the source of the interrupt, so just pick the vcpu
 344                  * itself as the source...
 345                  */
 346                 if (is_vgic_v2_sgi(vcpu, irq))
 347                         irq->source |= BIT(vcpu->vcpu_id);
 348
 349                 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
 350
 351                 vgic_put_irq(vcpu->kvm, irq);
 352         }
 353
 354         return 0;
 355 }
 356
 357 /* Must be called with irq->irq_lock held */
 358 static void vgic_hw_irq_cpending(struct kvm_vcpu *vcpu, struct vgic_irq *irq)
 359 {
 360         irq->pending_latch = false;
 361
 362         /*
 363          * We don't want the guest to effectively mask the physical
 364          * interrupt by doing a write to SPENDR followed by a write to
 365          * CPENDR for HW interrupts, so we clear the active state on
 366          * the physical side if the virtual interrupt is not active.
 367          * This may lead to taking an additional interrupt on the
 368          * host, but that should not be a problem as the worst that
 369          * can happen is an additional vgic injection.  We also clear
 370          * the pending state to maintain proper semantics for edge HW
 371          * interrupts.
 372          */
 373         vgic_irq_set_phys_pending(irq, false);
 374         if (!irq->active)
 375                 vgic_irq_set_phys_active(irq, false);
 376 }
 377
 378 void vgic_mmio_write_cpending(struct kvm_vcpu *vcpu,
 379                               gpa_t addr, unsigned int len,
 380                               unsigned long val)
 381 {
 382         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 383         int i;
 384         unsigned long flags;
 385
 386         for_each_set_bit(i, &val, len * 8) {
 387                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 388
 389                 /* GICD_ICPENDR0 SGI bits are WI */
 390                 if (is_vgic_v2_sgi(vcpu, irq)) {
 391                         vgic_put_irq(vcpu->kvm, irq);
 392                         continue;
 393                 }
 394
 395                 raw_spin_lock_irqsave(&irq->irq_lock, flags);
 396
 397                 if (irq->hw && vgic_irq_is_sgi(irq->intid)) {
 398                         /* HW SGI? Ask the GIC to clear its pending bit */
 399                         int err;
 400                         err = irq_set_irqchip_state(irq->host_irq,
 401                                                     IRQCHIP_STATE_PENDING,
 402                                                     false);
 403                         WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);
 404
 405                         raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 406                         vgic_put_irq(vcpu->kvm, irq);
 407
 408                         continue;
 409                 }
 410
 411                 if (irq->hw)
 412                         vgic_hw_irq_cpending(vcpu, irq);
 413                 else
 414                         irq->pending_latch = false;
 415
 416                 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 417                 vgic_put_irq(vcpu->kvm, irq);
 418         }
 419 }
 420
 421 int vgic_uaccess_write_cpending(struct kvm_vcpu *vcpu,
 422                                 gpa_t addr, unsigned int len,
 423                                 unsigned long val)
 424 {
 425         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 426         int i;
 427         unsigned long flags;
 428
 429         for_each_set_bit(i, &val, len * 8) {
 430                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 431
 432                 raw_spin_lock_irqsave(&irq->irq_lock, flags);
 433                 /*
 434                  * More fun with GICv2 SGIs! If we're clearing one of them
 435                  * from userspace, which source vcpu to clear? Let's not
 436                  * even think of it, and blow the whole set.
 437                  */
 438                 if (is_vgic_v2_sgi(vcpu, irq))
 439                         irq->source = 0;
 440
 441                 irq->pending_latch = false;
 442
 443                 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 444
 445                 vgic_put_irq(vcpu->kvm, irq);
 446         }
 447
 448         return 0;
 449 }
 450
 451 /*
 452  * If we are fiddling with an IRQ's active state, we have to make sure the IRQ
 453  * is not queued on some running VCPU's LRs, because then the change to the
 454  * active state can be overwritten when the VCPU's state is synced coming back
 455  * from the guest.
 456  *
 457  * For shared interrupts as well as GICv3 private interrupts, we have to
 458  * stop all the VCPUs because interrupts can be migrated while we don't hold
 459  * the IRQ locks and we don't want to be chasing moving targets.
 460  *
 461  * For GICv2 private interrupts we don't have to do anything because
 462  * userspace accesses to the VGIC state already require all VCPUs to be
 463  * stopped, and only the VCPU itself can modify its private interrupts
 464  * active state, which guarantees that the VCPU is not running.
 465  */
 466 static void vgic_access_active_prepare(struct kvm_vcpu *vcpu, u32 intid)
 467 {
 468         if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3 ||
 469             intid >= VGIC_NR_PRIVATE_IRQS)
 470                 kvm_arm_halt_guest(vcpu->kvm);
 471 }
 472
 473 /* See vgic_access_active_prepare */
 474 static void vgic_access_active_finish(struct kvm_vcpu *vcpu, u32 intid)
 475 {
 476         if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3 ||
 477             intid >= VGIC_NR_PRIVATE_IRQS)
 478                 kvm_arm_resume_guest(vcpu->kvm);
 479 }
 480
 481 static unsigned long __vgic_mmio_read_active(struct kvm_vcpu *vcpu,
 482                                              gpa_t addr, unsigned int len)
 483 {
 484         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 485         u32 value = 0;
 486         int i;
 487
 488         /* Loop over all IRQs affected by this read */
 489         for (i = 0; i < len * 8; i++) {
 490                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 491
 492                 /*
 493                  * Even for HW interrupts, don't evaluate the HW state as
 494                  * all the guest is interested in is the virtual state.
 495                  */
 496                 if (irq->active)
 497                         value |= (1U << i);
 498
 499                 vgic_put_irq(vcpu->kvm, irq);
 500         }
 501
 502         return value;
 503 }
 504
 505 unsigned long vgic_mmio_read_active(struct kvm_vcpu *vcpu,
 506                                     gpa_t addr, unsigned int len)
 507 {
 508         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 509         u32 val;
 510
 511         mutex_lock(&vcpu->kvm->lock);
 512         vgic_access_active_prepare(vcpu, intid);
 513
 514         val = __vgic_mmio_read_active(vcpu, addr, len);
 515
 516         vgic_access_active_finish(vcpu, intid);
 517         mutex_unlock(&vcpu->kvm->lock);
 518
 519         return val;
 520 }
 521
 522 unsigned long vgic_uaccess_read_active(struct kvm_vcpu *vcpu,
 523                                     gpa_t addr, unsigned int len)
 524 {
 525         return __vgic_mmio_read_active(vcpu, addr, len);
 526 }
 527
 528 /* Must be called with irq->irq_lock held */
 529 static void vgic_hw_irq_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq,
 530                                       bool active, bool is_uaccess)
 531 {
 532         if (is_uaccess)
 533                 return;
 534
 535         irq->active = active;
 536         vgic_irq_set_phys_active(irq, active);
 537 }
 538
 539 static void vgic_mmio_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq,
 540                                     bool active)
 541 {
 542         unsigned long flags;
 543         struct kvm_vcpu *requester_vcpu = kvm_get_running_vcpu();
 544
 545         raw_spin_lock_irqsave(&irq->irq_lock, flags);
 546
 547         if (irq->hw && !vgic_irq_is_sgi(irq->intid)) {
 548                 vgic_hw_irq_change_active(vcpu, irq, active, !requester_vcpu);
 549         } else if (irq->hw && vgic_irq_is_sgi(irq->intid)) {
 550                 /*
 551                  * GICv4.1 VSGI feature doesn't track an active state,
 552                  * so let's not kid ourselves, there is nothing we can
 553                  * do here.
 554                  */
 555                 irq->active = false;
 556         } else {
 557                 u32 model = vcpu->kvm->arch.vgic.vgic_model;
 558                 u8 active_source;
 559
 560                 irq->active = active;
 561
 562                 /*
 563                  * The GICv2 architecture indicates that the source CPUID for
 564                  * an SGI should be provided during an EOI which implies that
 565                  * the active state is stored somewhere, but at the same time
 566                  * this state is not architecturally exposed anywhere and we
 567                  * have no way of knowing the right source.
 568                  *
 569                  * This may lead to a VCPU not being able to receive
 570                  * additional instances of a particular SGI after migration
 571                  * for a GICv2 VM on some GIC implementations.  Oh well.
 572                  */
 573                 active_source = (requester_vcpu) ? requester_vcpu->vcpu_id : 0;
 574
 575                 if (model == KVM_DEV_TYPE_ARM_VGIC_V2 &&
 576                     active && vgic_irq_is_sgi(irq->intid))
 577                         irq->active_source = active_source;
 578         }
 579
 580         if (irq->active)
 581                 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
 582         else
 583                 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 584 }
 585
 586 static void __vgic_mmio_write_cactive(struct kvm_vcpu *vcpu,
 587                                       gpa_t addr, unsigned int len,
 588                                       unsigned long val)
 589 {
 590         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 591         int i;
 592
 593         for_each_set_bit(i, &val, len * 8) {
 594                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 595                 vgic_mmio_change_active(vcpu, irq, false);
 596                 vgic_put_irq(vcpu->kvm, irq);
 597         }
 598 }
 599
 600 void vgic_mmio_write_cactive(struct kvm_vcpu *vcpu,
 601                              gpa_t addr, unsigned int len,
 602                              unsigned long val)
 603 {
 604         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 605
 606         mutex_lock(&vcpu->kvm->lock);
 607         vgic_access_active_prepare(vcpu, intid);
 608
 609         __vgic_mmio_write_cactive(vcpu, addr, len, val);
 610
 611         vgic_access_active_finish(vcpu, intid);
 612         mutex_unlock(&vcpu->kvm->lock);
 613 }
 614
 615 int vgic_mmio_uaccess_write_cactive(struct kvm_vcpu *vcpu,
 616                                      gpa_t addr, unsigned int len,
 617                                      unsigned long val)
 618 {
 619         __vgic_mmio_write_cactive(vcpu, addr, len, val);
 620         return 0;
 621 }
 622
 623 static void __vgic_mmio_write_sactive(struct kvm_vcpu *vcpu,
 624                                       gpa_t addr, unsigned int len,
 625                                       unsigned long val)
 626 {
 627         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 628         int i;
 629
 630         for_each_set_bit(i, &val, len * 8) {
 631                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 632                 vgic_mmio_change_active(vcpu, irq, true);
 633                 vgic_put_irq(vcpu->kvm, irq);
 634         }
 635 }
 636
 637 void vgic_mmio_write_sactive(struct kvm_vcpu *vcpu,
 638                              gpa_t addr, unsigned int len,
 639                              unsigned long val)
 640 {
 641         u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 642
 643         mutex_lock(&vcpu->kvm->lock);
 644         vgic_access_active_prepare(vcpu, intid);
 645
 646         __vgic_mmio_write_sactive(vcpu, addr, len, val);
 647
 648         vgic_access_active_finish(vcpu, intid);
 649         mutex_unlock(&vcpu->kvm->lock);
 650 }
 651
 652 int vgic_mmio_uaccess_write_sactive(struct kvm_vcpu *vcpu,
 653                                      gpa_t addr, unsigned int len,
 654                                      unsigned long val)
 655 {
 656         __vgic_mmio_write_sactive(vcpu, addr, len, val);
 657         return 0;
 658 }
 659
 660 unsigned long vgic_mmio_read_priority(struct kvm_vcpu *vcpu,
 661                                       gpa_t addr, unsigned int len)
 662 {
 663         u32 intid = VGIC_ADDR_TO_INTID(addr, 8);
 664         int i;
 665         u64 val = 0;
 666
 667         for (i = 0; i < len; i++) {
 668                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 669
 670                 val |= (u64)irq->priority << (i * 8);
 671
 672                 vgic_put_irq(vcpu->kvm, irq);
 673         }
 674
 675         return val;
 676 }
 677
 678 /*
 679  * We currently don't handle changing the priority of an interrupt that
 680  * is already pending on a VCPU. If there is a need for this, we would
 681  * need to make this VCPU exit and re-evaluate the priorities, potentially
 682  * leading to this interrupt getting presented now to the guest (if it has
 683  * been masked by the priority mask before).
 684  */
 685 void vgic_mmio_write_priority(struct kvm_vcpu *vcpu,
 686                               gpa_t addr, unsigned int len,
 687                               unsigned long val)
 688 {
 689         u32 intid = VGIC_ADDR_TO_INTID(addr, 8);
 690         int i;
 691         unsigned long flags;
 692
 693         for (i = 0; i < len; i++) {
 694                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 695
 696                 raw_spin_lock_irqsave(&irq->irq_lock, flags);
 697                 /* Narrow the priority range to what we actually support */
 698                 irq->priority = (val >> (i * 8)) & GENMASK(7, 8 - VGIC_PRI_BITS);
 699                 if (irq->hw && vgic_irq_is_sgi(irq->intid))
 700                         vgic_update_vsgi(irq);
 701                 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 702
 703                 vgic_put_irq(vcpu->kvm, irq);
 704         }
 705 }
 706
 707 unsigned long vgic_mmio_read_config(struct kvm_vcpu *vcpu,
 708                                     gpa_t addr, unsigned int len)
 709 {
 710         u32 intid = VGIC_ADDR_TO_INTID(addr, 2);
 711         u32 value = 0;
 712         int i;
 713
 714         for (i = 0; i < len * 4; i++) {
 715                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 716
 717                 if (irq->config == VGIC_CONFIG_EDGE)
 718                         value |= (2U << (i * 2));
 719
 720                 vgic_put_irq(vcpu->kvm, irq);
 721         }
 722
 723         return value;
 724 }
 725
 726 void vgic_mmio_write_config(struct kvm_vcpu *vcpu,
 727                             gpa_t addr, unsigned int len,
 728                             unsigned long val)
 729 {
 730         u32 intid = VGIC_ADDR_TO_INTID(addr, 2);
 731         int i;
 732         unsigned long flags;
 733
 734         for (i = 0; i < len * 4; i++) {
 735                 struct vgic_irq *irq;
 736
 737                 /*
 738                  * The configuration cannot be changed for SGIs in general,
 739                  * for PPIs this is IMPLEMENTATION DEFINED. The arch timer
 740                  * code relies on PPIs being level triggered, so we also
 741                  * make them read-only here.
 742                  */
 743                 if (intid + i < VGIC_NR_PRIVATE_IRQS)
 744                         continue;
 745
 746                 irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 747                 raw_spin_lock_irqsave(&irq->irq_lock, flags);
 748
 749                 if (test_bit(i * 2 + 1, &val))
 750                         irq->config = VGIC_CONFIG_EDGE;
 751                 else
 752                         irq->config = VGIC_CONFIG_LEVEL;
 753
 754                 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 755                 vgic_put_irq(vcpu->kvm, irq);
 756         }
 757 }
 758
 759 u64 vgic_read_irq_line_level_info(struct kvm_vcpu *vcpu, u32 intid)
 760 {
 761         int i;
 762         u64 val = 0;
 763         int nr_irqs = vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;
 764
 765         for (i = 0; i < 32; i++) {
 766                 struct vgic_irq *irq;
 767
 768                 if ((intid + i) < VGIC_NR_SGIS || (intid + i) >= nr_irqs)
 769                         continue;
 770
 771                 irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 772                 if (irq->config == VGIC_CONFIG_LEVEL && irq->line_level)
 773                         val |= (1U << i);
 774
 775                 vgic_put_irq(vcpu->kvm, irq);
 776         }
 777
 778         return val;
 779 }
 780
 781 void vgic_write_irq_line_level_info(struct kvm_vcpu *vcpu, u32 intid,
 782                                     const u64 val)
 783 {
 784         int i;
 785         int nr_irqs = vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;
 786         unsigned long flags;
 787
 788         for (i = 0; i < 32; i++) {
 789                 struct vgic_irq *irq;
 790                 bool new_level;
 791
 792                 if ((intid + i) < VGIC_NR_SGIS || (intid + i) >= nr_irqs)
 793                         continue;
 794
 795                 irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 796
 797                 /*
 798                  * Line level is set irrespective of irq type
 799                  * (level or edge) to avoid dependency that VM should
 800                  * restore irq config before line level.
 801                  */
 802                 new_level = !!(val & (1U << i));
 803                 raw_spin_lock_irqsave(&irq->irq_lock, flags);
 804                 irq->line_level = new_level;
 805                 if (new_level)
 806                         vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
 807                 else
 808                         raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 809
 810                 vgic_put_irq(vcpu->kvm, irq);
 811         }
 812 }
 813
 814 static int match_region(const void *key, const void *elt)
 815 {
 816         const unsigned int offset = (unsigned long)key;
 817         const struct vgic_register_region *region = elt;
 818
 819         if (offset < region->reg_offset)
 820                 return -1;
 821
 822         if (offset >= region->reg_offset + region->len)
 823                 return 1;
 824
 825         return 0;
 826 }
 827
 828 const struct vgic_register_region *
 829 vgic_find_mmio_region(const struct vgic_register_region *regions,
 830                       int nr_regions, unsigned int offset)
 831 {
 832         return bsearch((void *)(uintptr_t)offset, regions, nr_regions,
 833                        sizeof(regions[0]), match_region);
 834 }
 835
 836 void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
 837 {
 838         if (kvm_vgic_global_state.type == VGIC_V2)
 839                 vgic_v2_set_vmcr(vcpu, vmcr);
 840         else
 841                 vgic_v3_set_vmcr(vcpu, vmcr);
 842 }
 843
 844 void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
 845 {
 846         if (kvm_vgic_global_state.type == VGIC_V2)
 847                 vgic_v2_get_vmcr(vcpu, vmcr);
 848         else
 849                 vgic_v3_get_vmcr(vcpu, vmcr);
 850 }
 851
 852 /*
 853  * kvm_mmio_read_buf() returns a value in a format where it can be converted
 854  * to a byte array and be directly observed as the guest wanted it to appear
 855  * in memory if it had done the store itself, which is LE for the GIC, as the
 856  * guest knows the GIC is always LE.
 857  *
 858  * We convert this value to the CPUs native format to deal with it as a data
 859  * value.
 860  */
 861 unsigned long vgic_data_mmio_bus_to_host(const void *val, unsigned int len)
 862 {
 863         unsigned long data = kvm_mmio_read_buf(val, len);
 864
 865         switch (len) {
 866         case 1:
 867                 return data;
 868         case 2:
 869                 return le16_to_cpu(data);
 870         case 4:
 871                 return le32_to_cpu(data);
 872         default:
 873                 return le64_to_cpu(data);
 874         }
 875 }
 876
 877 /*
 878  * kvm_mmio_write_buf() expects a value in a format such that if converted to
 879  * a byte array it is observed as the guest would see it if it could perform
 880  * the load directly.  Since the GIC is LE, and the guest knows this, the
 881  * guest expects a value in little endian format.
 882  *
 883  * We convert the data value from the CPUs native format to LE so that the
 884  * value is returned in the proper format.
 885  */
 886 void vgic_data_host_to_mmio_bus(void *buf, unsigned int len,
 887                                 unsigned long data)
 888 {
 889         switch (len) {
 890         case 1:
 891                 break;
 892         case 2:
 893                 data = cpu_to_le16(data);
 894                 break;
 895         case 4:
 896                 data = cpu_to_le32(data);
 897                 break;
 898         default:
 899                 data = cpu_to_le64(data);
 900         }
 901
 902         kvm_mmio_write_buf(buf, len, data);
 903 }
 904
 905 static
 906 struct vgic_io_device *kvm_to_vgic_iodev(const struct kvm_io_device *dev)
 907 {
 908         return container_of(dev, struct vgic_io_device, dev);
 909 }
 910
 911 static bool check_region(const struct kvm *kvm,
 912                          const struct vgic_register_region *region,
 913                          gpa_t addr, int len)
 914 {
 915         int flags, nr_irqs = kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;
 916
 917         switch (len) {
 918         case sizeof(u8):
 919                 flags = VGIC_ACCESS_8bit;
 920                 break;
 921         case sizeof(u32):
 922                 flags = VGIC_ACCESS_32bit;
 923                 break;
 924         case sizeof(u64):
 925                 flags = VGIC_ACCESS_64bit;
 926                 break;
 927         default:
 928                 return false;
 929         }
 930
 931         if ((region->access_flags & flags) && IS_ALIGNED(addr, len)) {
 932                 if (!region->bits_per_irq)
 933                         return true;
 934
 935                 /* Do we access a non-allocated IRQ? */
 936                 return VGIC_ADDR_TO_INTID(addr, region->bits_per_irq) < nr_irqs;
 937         }
 938
 939         return false;
 940 }
 941
 942 const struct vgic_register_region *
 943 vgic_get_mmio_region(struct kvm_vcpu *vcpu, struct vgic_io_device *iodev,
 944                      gpa_t addr, int len)
 945 {
 946         const struct vgic_register_region *region;
 947
 948         region = vgic_find_mmio_region(iodev->regions, iodev->nr_regions,
 949                                        addr - iodev->base_addr);
 950         if (!region || !check_region(vcpu->kvm, region, addr, len))
 951                 return NULL;
 952
 953         return region;
 954 }
 955
 956 static int vgic_uaccess_read(struct kvm_vcpu *vcpu, struct vgic_io_device *iodev,
 957                              gpa_t addr, u32 *val)
 958 {
 959         const struct vgic_register_region *region;
 960         struct kvm_vcpu *r_vcpu;
 961
 962         region = vgic_get_mmio_region(vcpu, iodev, addr, sizeof(u32));
 963         if (!region) {
 964                 *val = 0;
 965                 return 0;
 966         }
 967
 968         r_vcpu = iodev->redist_vcpu ? iodev->redist_vcpu : vcpu;
 969         if (region->uaccess_read)
 970                 *val = region->uaccess_read(r_vcpu, addr, sizeof(u32));
 971         else
 972                 *val = region->read(r_vcpu, addr, sizeof(u32));
 973
 974         return 0;
 975 }
 976
 977 static int vgic_uaccess_write(struct kvm_vcpu *vcpu, struct vgic_io_device *iodev,
 978                               gpa_t addr, const u32 *val)
 979 {
 980         const struct vgic_register_region *region;
 981         struct kvm_vcpu *r_vcpu;
 982
 983         region = vgic_get_mmio_region(vcpu, iodev, addr, sizeof(u32));
 984         if (!region)
 985                 return 0;
 986
 987         r_vcpu = iodev->redist_vcpu ? iodev->redist_vcpu : vcpu;
 988         if (region->uaccess_write)
 989                 return region->uaccess_write(r_vcpu, addr, sizeof(u32), *val);
 990
 991         region->write(r_vcpu, addr, sizeof(u32), *val);
 992         return 0;
 993 }
 994
 995 /*
 996  * Userland access to VGIC registers.
 997  */
 998 int vgic_uaccess(struct kvm_vcpu *vcpu, struct vgic_io_device *dev,
 999                  bool is_write, int offset, u32 *val)
1000 {
1001         if (is_write)
1002                 return vgic_uaccess_write(vcpu, dev, offset, val);
1003         else
1004                 return vgic_uaccess_read(vcpu, dev, offset, val);
1005 }
1006
1007 static int dispatch_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
1008                               gpa_t addr, int len, void *val)
1009 {
1010         struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev);
1011         const struct vgic_register_region *region;
1012         unsigned long data = 0;
1013
1014         region = vgic_get_mmio_region(vcpu, iodev, addr, len);
1015         if (!region) {
1016                 memset(val, 0, len);
1017                 return 0;
1018         }
1019
1020         switch (iodev->iodev_type) {
1021         case IODEV_CPUIF:
1022                 data = region->read(vcpu, addr, len);
1023                 break;
1024         case IODEV_DIST:
1025                 data = region->read(vcpu, addr, len);
1026                 break;
1027         case IODEV_REDIST:
1028                 data = region->read(iodev->redist_vcpu, addr, len);
1029                 break;
1030         case IODEV_ITS:
1031                 data = region->its_read(vcpu->kvm, iodev->its, addr, len);
1032                 break;
1033         }
1034
1035         vgic_data_host_to_mmio_bus(val, len, data);
1036         return 0;
1037 }
1038
1039 static int dispatch_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
1040                                gpa_t addr, int len, const void *val)
1041 {
1042         struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev);
1043         const struct vgic_register_region *region;
1044         unsigned long data = vgic_data_mmio_bus_to_host(val, len);
1045
1046         region = vgic_get_mmio_region(vcpu, iodev, addr, len);
1047         if (!region)
1048                 return 0;
1049
1050         switch (iodev->iodev_type) {
1051         case IODEV_CPUIF:
1052                 region->write(vcpu, addr, len, data);
1053                 break;
1054         case IODEV_DIST:
1055                 region->write(vcpu, addr, len, data);
1056                 break;
1057         case IODEV_REDIST:
1058                 region->write(iodev->redist_vcpu, addr, len, data);
1059                 break;
1060         case IODEV_ITS:
1061                 region->its_write(vcpu->kvm, iodev->its, addr, len, data);
1062                 break;
1063         }
1064
1065         return 0;
1066 }
1067
1068 struct kvm_io_device_ops kvm_io_gic_ops = {
1069         .read = dispatch_mmio_read,
1070         .write = dispatch_mmio_write,
1071 };
1072
1073 int vgic_register_dist_iodev(struct kvm *kvm, gpa_t dist_base_address,
1074                              enum vgic_type type)
1075 {
1076         struct vgic_io_device *io_device = &kvm->arch.vgic.dist_iodev;
1077         int ret = 0;
1078         unsigned int len;
1079
1080         switch (type) {
1081         case VGIC_V2:
1082                 len = vgic_v2_init_dist_iodev(io_device);
1083                 break;
1084         case VGIC_V3:
1085                 len = vgic_v3_init_dist_iodev(io_device);
1086                 break;
1087         default:
1088                 BUG_ON(1);
1089         }
1090
1091         io_device->base_addr = dist_base_address;
1092         io_device->iodev_type = IODEV_DIST;
1093         io_device->redist_vcpu = NULL;
1094
1095         mutex_lock(&kvm->slots_lock);
1096         ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, dist_base_address,
1097                                       len, &io_device->dev);
1098         mutex_unlock(&kvm->slots_lock);
1099
1100         return ret;
1101 }