arch/s390/kvm/interrupt.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * handling kvm guest interrupts
   4  *
   5  * Copyright IBM Corp. 2008, 2020
   6  *
   7  *    Author(s): Carsten Otte <cotte@de.ibm.com>
   8  */
   9
  10 #define KMSG_COMPONENT "kvm-s390"
  11 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  12
  13 #include <linux/interrupt.h>
  14 #include <linux/kvm_host.h>
  15 #include <linux/hrtimer.h>
  16 #include <linux/mmu_context.h>
  17 #include <linux/nospec.h>
  18 #include <linux/signal.h>
  19 #include <linux/slab.h>
  20 #include <linux/bitmap.h>
  21 #include <linux/vmalloc.h>
  22 #include <asm/asm-offsets.h>
  23 #include <asm/dis.h>
  24 #include <linux/uaccess.h>
  25 #include <asm/sclp.h>
  26 #include <asm/isc.h>
  27 #include <asm/gmap.h>
  28 #include <asm/switch_to.h>
  29 #include <asm/nmi.h>
  30 #include <asm/airq.h>
  31 #include <asm/tpi.h>
  32 #include "kvm-s390.h"
  33 #include "gaccess.h"
  34 #include "trace-s390.h"
  35 #include "pci.h"
  36
  37 #define PFAULT_INIT 0x0600
  38 #define PFAULT_DONE 0x0680
  39 #define VIRTIO_PARAM 0x0d00
  40
  41 static struct kvm_s390_gib *gib;
  42
  43 /* handle external calls via sigp interpretation facility */
  44 static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id)
  45 {
  46         int c, scn;
  47
  48         if (!kvm_s390_test_cpuflags(vcpu, CPUSTAT_ECALL_PEND))
  49                 return 0;
  50
  51         BUG_ON(!kvm_s390_use_sca_entries());
  52         read_lock(&vcpu->kvm->arch.sca_lock);
  53         if (vcpu->kvm->arch.use_esca) {
  54                 struct esca_block *sca = vcpu->kvm->arch.sca;
  55                 union esca_sigp_ctrl sigp_ctrl =
  56                         sca->cpu[vcpu->vcpu_id].sigp_ctrl;
  57
  58                 c = sigp_ctrl.c;
  59                 scn = sigp_ctrl.scn;
  60         } else {
  61                 struct bsca_block *sca = vcpu->kvm->arch.sca;
  62                 union bsca_sigp_ctrl sigp_ctrl =
  63                         sca->cpu[vcpu->vcpu_id].sigp_ctrl;
  64
  65                 c = sigp_ctrl.c;
  66                 scn = sigp_ctrl.scn;
  67         }
  68         read_unlock(&vcpu->kvm->arch.sca_lock);
  69
  70         if (src_id)
  71                 *src_id = scn;
  72
  73         return c;
  74 }
  75
  76 static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id)
  77 {
  78         int expect, rc;
  79
  80         BUG_ON(!kvm_s390_use_sca_entries());
  81         read_lock(&vcpu->kvm->arch.sca_lock);
  82         if (vcpu->kvm->arch.use_esca) {
  83                 struct esca_block *sca = vcpu->kvm->arch.sca;
  84                 union esca_sigp_ctrl *sigp_ctrl =
  85                         &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
  86                 union esca_sigp_ctrl new_val = {0}, old_val;
  87
  88                 old_val = READ_ONCE(*sigp_ctrl);
  89                 new_val.scn = src_id;
  90                 new_val.c = 1;
  91                 old_val.c = 0;
  92
  93                 expect = old_val.value;
  94                 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
  95         } else {
  96                 struct bsca_block *sca = vcpu->kvm->arch.sca;
  97                 union bsca_sigp_ctrl *sigp_ctrl =
  98                         &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
  99                 union bsca_sigp_ctrl new_val = {0}, old_val;
 100
 101                 old_val = READ_ONCE(*sigp_ctrl);
 102                 new_val.scn = src_id;
 103                 new_val.c = 1;
 104                 old_val.c = 0;
 105
 106                 expect = old_val.value;
 107                 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
 108         }
 109         read_unlock(&vcpu->kvm->arch.sca_lock);
 110
 111         if (rc != expect) {
 112                 /* another external call is pending */
 113                 return -EBUSY;
 114         }
 115         kvm_s390_set_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
 116         return 0;
 117 }
 118
 119 static void sca_clear_ext_call(struct kvm_vcpu *vcpu)
 120 {
 121         int rc, expect;
 122
 123         if (!kvm_s390_use_sca_entries())
 124                 return;
 125         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
 126         read_lock(&vcpu->kvm->arch.sca_lock);
 127         if (vcpu->kvm->arch.use_esca) {
 128                 struct esca_block *sca = vcpu->kvm->arch.sca;
 129                 union esca_sigp_ctrl *sigp_ctrl =
 130                         &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
 131                 union esca_sigp_ctrl old;
 132
 133                 old = READ_ONCE(*sigp_ctrl);
 134                 expect = old.value;
 135                 rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
 136         } else {
 137                 struct bsca_block *sca = vcpu->kvm->arch.sca;
 138                 union bsca_sigp_ctrl *sigp_ctrl =
 139                         &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
 140                 union bsca_sigp_ctrl old;
 141
 142                 old = READ_ONCE(*sigp_ctrl);
 143                 expect = old.value;
 144                 rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
 145         }
 146         read_unlock(&vcpu->kvm->arch.sca_lock);
 147         WARN_ON(rc != expect); /* cannot clear? */
 148 }
 149
 150 int psw_extint_disabled(struct kvm_vcpu *vcpu)
 151 {
 152         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT);
 153 }
 154
 155 static int psw_ioint_disabled(struct kvm_vcpu *vcpu)
 156 {
 157         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO);
 158 }
 159
 160 static int psw_mchk_disabled(struct kvm_vcpu *vcpu)
 161 {
 162         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_MCHECK);
 163 }
 164
 165 static int psw_interrupts_disabled(struct kvm_vcpu *vcpu)
 166 {
 167         return psw_extint_disabled(vcpu) &&
 168                psw_ioint_disabled(vcpu) &&
 169                psw_mchk_disabled(vcpu);
 170 }
 171
 172 static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu)
 173 {
 174         if (psw_extint_disabled(vcpu) ||
 175             !(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
 176                 return 0;
 177         if (guestdbg_enabled(vcpu) && guestdbg_sstep_enabled(vcpu))
 178                 /* No timer interrupts when single stepping */
 179                 return 0;
 180         return 1;
 181 }
 182
 183 static int ckc_irq_pending(struct kvm_vcpu *vcpu)
 184 {
 185         const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
 186         const u64 ckc = vcpu->arch.sie_block->ckc;
 187
 188         if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
 189                 if ((s64)ckc >= (s64)now)
 190                         return 0;
 191         } else if (ckc >= now) {
 192                 return 0;
 193         }
 194         return ckc_interrupts_enabled(vcpu);
 195 }
 196
 197 static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu)
 198 {
 199         return !psw_extint_disabled(vcpu) &&
 200                (vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK);
 201 }
 202
 203 static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu)
 204 {
 205         if (!cpu_timer_interrupts_enabled(vcpu))
 206                 return 0;
 207         return kvm_s390_get_cpu_timer(vcpu) >> 63;
 208 }
 209
 210 static uint64_t isc_to_isc_bits(int isc)
 211 {
 212         return (0x80 >> isc) << 24;
 213 }
 214
 215 static inline u32 isc_to_int_word(u8 isc)
 216 {
 217         return ((u32)isc << 27) | 0x80000000;
 218 }
 219
 220 static inline u8 int_word_to_isc(u32 int_word)
 221 {
 222         return (int_word & 0x38000000) >> 27;
 223 }
 224
 225 /*
 226  * To use atomic bitmap functions, we have to provide a bitmap address
 227  * that is u64 aligned. However, the ipm might be u32 aligned.
 228  * Therefore, we logically start the bitmap at the very beginning of the
 229  * struct and fixup the bit number.
 230  */
 231 #define IPM_BIT_OFFSET (offsetof(struct kvm_s390_gisa, ipm) * BITS_PER_BYTE)
 232
 233 /**
 234  * gisa_set_iam - change the GISA interruption alert mask
 235  *
 236  * @gisa: gisa to operate on
 237  * @iam: new IAM value to use
 238  *
 239  * Change the IAM atomically with the next alert address and the IPM
 240  * of the GISA if the GISA is not part of the GIB alert list. All three
 241  * fields are located in the first long word of the GISA.
 242  *
 243  * Returns: 0 on success
 244  *          -EBUSY in case the gisa is part of the alert list
 245  */
 246 static inline int gisa_set_iam(struct kvm_s390_gisa *gisa, u8 iam)
 247 {
 248         u64 word, _word;
 249
 250         do {
 251                 word = READ_ONCE(gisa->u64.word[0]);
 252                 if ((u64)gisa != word >> 32)
 253                         return -EBUSY;
 254                 _word = (word & ~0xffUL) | iam;
 255         } while (cmpxchg(&gisa->u64.word[0], word, _word) != word);
 256
 257         return 0;
 258 }
 259
 260 /**
 261  * gisa_clear_ipm - clear the GISA interruption pending mask
 262  *
 263  * @gisa: gisa to operate on
 264  *
 265  * Clear the IPM atomically with the next alert address and the IAM
 266  * of the GISA unconditionally. All three fields are located in the
 267  * first long word of the GISA.
 268  */
 269 static inline void gisa_clear_ipm(struct kvm_s390_gisa *gisa)
 270 {
 271         u64 word, _word;
 272
 273         do {
 274                 word = READ_ONCE(gisa->u64.word[0]);
 275                 _word = word & ~(0xffUL << 24);
 276         } while (cmpxchg(&gisa->u64.word[0], word, _word) != word);
 277 }
 278
 279 /**
 280  * gisa_get_ipm_or_restore_iam - return IPM or restore GISA IAM
 281  *
 282  * @gi: gisa interrupt struct to work on
 283  *
 284  * Atomically restores the interruption alert mask if none of the
 285  * relevant ISCs are pending and return the IPM.
 286  *
 287  * Returns: the relevant pending ISCs
 288  */
 289 static inline u8 gisa_get_ipm_or_restore_iam(struct kvm_s390_gisa_interrupt *gi)
 290 {
 291         u8 pending_mask, alert_mask;
 292         u64 word, _word;
 293
 294         do {
 295                 word = READ_ONCE(gi->origin->u64.word[0]);
 296                 alert_mask = READ_ONCE(gi->alert.mask);
 297                 pending_mask = (u8)(word >> 24) & alert_mask;
 298                 if (pending_mask)
 299                         return pending_mask;
 300                 _word = (word & ~0xffUL) | alert_mask;
 301         } while (cmpxchg(&gi->origin->u64.word[0], word, _word) != word);
 302
 303         return 0;
 304 }
 305
 306 static inline void gisa_set_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
 307 {
 308         set_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
 309 }
 310
 311 static inline u8 gisa_get_ipm(struct kvm_s390_gisa *gisa)
 312 {
 313         return READ_ONCE(gisa->ipm);
 314 }
 315
 316 static inline int gisa_tac_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
 317 {
 318         return test_and_clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
 319 }
 320
 321 static inline unsigned long pending_irqs_no_gisa(struct kvm_vcpu *vcpu)
 322 {
 323         unsigned long pending = vcpu->kvm->arch.float_int.pending_irqs |
 324                                 vcpu->arch.local_int.pending_irqs;
 325
 326         pending &= ~vcpu->kvm->arch.float_int.masked_irqs;
 327         return pending;
 328 }
 329
 330 static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu)
 331 {
 332         struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
 333         unsigned long pending_mask;
 334
 335         pending_mask = pending_irqs_no_gisa(vcpu);
 336         if (gi->origin)
 337                 pending_mask |= gisa_get_ipm(gi->origin) << IRQ_PEND_IO_ISC_7;
 338         return pending_mask;
 339 }
 340
 341 static inline int isc_to_irq_type(unsigned long isc)
 342 {
 343         return IRQ_PEND_IO_ISC_0 - isc;
 344 }
 345
 346 static inline int irq_type_to_isc(unsigned long irq_type)
 347 {
 348         return IRQ_PEND_IO_ISC_0 - irq_type;
 349 }
 350
 351 static unsigned long disable_iscs(struct kvm_vcpu *vcpu,
 352                                    unsigned long active_mask)
 353 {
 354         int i;
 355
 356         for (i = 0; i <= MAX_ISC; i++)
 357                 if (!(vcpu->arch.sie_block->gcr[6] & isc_to_isc_bits(i)))
 358                         active_mask &= ~(1UL << (isc_to_irq_type(i)));
 359
 360         return active_mask;
 361 }
 362
 363 static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu)
 364 {
 365         unsigned long active_mask;
 366
 367         active_mask = pending_irqs(vcpu);
 368         if (!active_mask)
 369                 return 0;
 370
 371         if (psw_extint_disabled(vcpu))
 372                 active_mask &= ~IRQ_PEND_EXT_MASK;
 373         if (psw_ioint_disabled(vcpu))
 374                 active_mask &= ~IRQ_PEND_IO_MASK;
 375         else
 376                 active_mask = disable_iscs(vcpu, active_mask);
 377         if (!(vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
 378                 __clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask);
 379         if (!(vcpu->arch.sie_block->gcr[0] & CR0_EMERGENCY_SIGNAL_SUBMASK))
 380                 __clear_bit(IRQ_PEND_EXT_EMERGENCY, &active_mask);
 381         if (!(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
 382                 __clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask);
 383         if (!(vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK))
 384                 __clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask);
 385         if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK)) {
 386                 __clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask);
 387                 __clear_bit(IRQ_PEND_EXT_SERVICE_EV, &active_mask);
 388         }
 389         if (psw_mchk_disabled(vcpu))
 390                 active_mask &= ~IRQ_PEND_MCHK_MASK;
 391         /* PV guest cpus can have a single interruption injected at a time. */
 392         if (kvm_s390_pv_cpu_get_handle(vcpu) &&
 393             vcpu->arch.sie_block->iictl != IICTL_CODE_NONE)
 394                 active_mask &= ~(IRQ_PEND_EXT_II_MASK |
 395                                  IRQ_PEND_IO_MASK |
 396                                  IRQ_PEND_MCHK_MASK);
 397         /*
 398          * Check both floating and local interrupt's cr14 because
 399          * bit IRQ_PEND_MCHK_REP could be set in both cases.
 400          */
 401         if (!(vcpu->arch.sie_block->gcr[14] &
 402            (vcpu->kvm->arch.float_int.mchk.cr14 |
 403            vcpu->arch.local_int.irq.mchk.cr14)))
 404                 __clear_bit(IRQ_PEND_MCHK_REP, &active_mask);
 405
 406         /*
 407          * STOP irqs will never be actively delivered. They are triggered via
 408          * intercept requests and cleared when the stop intercept is performed.
 409          */
 410         __clear_bit(IRQ_PEND_SIGP_STOP, &active_mask);
 411
 412         return active_mask;
 413 }
 414
 415 static void __set_cpu_idle(struct kvm_vcpu *vcpu)
 416 {
 417         kvm_s390_set_cpuflags(vcpu, CPUSTAT_WAIT);
 418         set_bit(vcpu->vcpu_idx, vcpu->kvm->arch.idle_mask);
 419 }
 420
 421 static void __unset_cpu_idle(struct kvm_vcpu *vcpu)
 422 {
 423         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_WAIT);
 424         clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.idle_mask);
 425 }
 426
 427 static void __reset_intercept_indicators(struct kvm_vcpu *vcpu)
 428 {
 429         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IO_INT | CPUSTAT_EXT_INT |
 430                                       CPUSTAT_STOP_INT);
 431         vcpu->arch.sie_block->lctl = 0x0000;
 432         vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT);
 433
 434         if (guestdbg_enabled(vcpu)) {
 435                 vcpu->arch.sie_block->lctl |= (LCTL_CR0 | LCTL_CR9 |
 436                                                LCTL_CR10 | LCTL_CR11);
 437                 vcpu->arch.sie_block->ictl |= (ICTL_STCTL | ICTL_PINT);
 438         }
 439 }
 440
 441 static void set_intercept_indicators_io(struct kvm_vcpu *vcpu)
 442 {
 443         if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_IO_MASK))
 444                 return;
 445         if (psw_ioint_disabled(vcpu))
 446                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IO_INT);
 447         else
 448                 vcpu->arch.sie_block->lctl |= LCTL_CR6;
 449 }
 450
 451 static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu)
 452 {
 453         if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_EXT_MASK))
 454                 return;
 455         if (psw_extint_disabled(vcpu))
 456                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
 457         else
 458                 vcpu->arch.sie_block->lctl |= LCTL_CR0;
 459 }
 460
 461 static void set_intercept_indicators_mchk(struct kvm_vcpu *vcpu)
 462 {
 463         if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_MCHK_MASK))
 464                 return;
 465         if (psw_mchk_disabled(vcpu))
 466                 vcpu->arch.sie_block->ictl |= ICTL_LPSW;
 467         else
 468                 vcpu->arch.sie_block->lctl |= LCTL_CR14;
 469 }
 470
 471 static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu)
 472 {
 473         if (kvm_s390_is_stop_irq_pending(vcpu))
 474                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
 475 }
 476
 477 /* Set interception request for non-deliverable interrupts */
 478 static void set_intercept_indicators(struct kvm_vcpu *vcpu)
 479 {
 480         set_intercept_indicators_io(vcpu);
 481         set_intercept_indicators_ext(vcpu);
 482         set_intercept_indicators_mchk(vcpu);
 483         set_intercept_indicators_stop(vcpu);
 484 }
 485
 486 static int __must_check __deliver_cpu_timer(struct kvm_vcpu *vcpu)
 487 {
 488         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 489         int rc = 0;
 490
 491         vcpu->stat.deliver_cputm++;
 492         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
 493                                          0, 0);
 494         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 495                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 496                 vcpu->arch.sie_block->eic = EXT_IRQ_CPU_TIMER;
 497         } else {
 498                 rc  = put_guest_lc(vcpu, EXT_IRQ_CPU_TIMER,
 499                                    (u16 *)__LC_EXT_INT_CODE);
 500                 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
 501                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 502                                      &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 503                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 504                                     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 505         }
 506         clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
 507         return rc ? -EFAULT : 0;
 508 }
 509
 510 static int __must_check __deliver_ckc(struct kvm_vcpu *vcpu)
 511 {
 512         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 513         int rc = 0;
 514
 515         vcpu->stat.deliver_ckc++;
 516         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
 517                                          0, 0);
 518         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 519                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 520                 vcpu->arch.sie_block->eic = EXT_IRQ_CLK_COMP;
 521         } else {
 522                 rc  = put_guest_lc(vcpu, EXT_IRQ_CLK_COMP,
 523                                    (u16 __user *)__LC_EXT_INT_CODE);
 524                 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
 525                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 526                                      &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 527                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 528                                     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 529         }
 530         clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
 531         return rc ? -EFAULT : 0;
 532 }
 533
 534 static int __must_check __deliver_pfault_init(struct kvm_vcpu *vcpu)
 535 {
 536         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 537         struct kvm_s390_ext_info ext;
 538         int rc;
 539
 540         spin_lock(&li->lock);
 541         ext = li->irq.ext;
 542         clear_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
 543         li->irq.ext.ext_params2 = 0;
 544         spin_unlock(&li->lock);
 545
 546         VCPU_EVENT(vcpu, 4, "deliver: pfault init token 0x%llx",
 547                    ext.ext_params2);
 548         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 549                                          KVM_S390_INT_PFAULT_INIT,
 550                                          0, ext.ext_params2);
 551
 552         rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *) __LC_EXT_INT_CODE);
 553         rc |= put_guest_lc(vcpu, PFAULT_INIT, (u16 *) __LC_EXT_CPU_ADDR);
 554         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 555                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 556         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 557                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 558         rc |= put_guest_lc(vcpu, ext.ext_params2, (u64 *) __LC_EXT_PARAMS2);
 559         return rc ? -EFAULT : 0;
 560 }
 561
 562 static int __write_machine_check(struct kvm_vcpu *vcpu,
 563                                  struct kvm_s390_mchk_info *mchk)
 564 {
 565         unsigned long ext_sa_addr;
 566         unsigned long lc;
 567         freg_t fprs[NUM_FPRS];
 568         union mci mci;
 569         int rc;
 570
 571         /*
 572          * All other possible payload for a machine check (e.g. the register
 573          * contents in the save area) will be handled by the ultravisor, as
 574          * the hypervisor does not not have the needed information for
 575          * protected guests.
 576          */
 577         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 578                 vcpu->arch.sie_block->iictl = IICTL_CODE_MCHK;
 579                 vcpu->arch.sie_block->mcic = mchk->mcic;
 580                 vcpu->arch.sie_block->faddr = mchk->failing_storage_address;
 581                 vcpu->arch.sie_block->edc = mchk->ext_damage_code;
 582                 return 0;
 583         }
 584
 585         mci.val = mchk->mcic;
 586         /* take care of lazy register loading */
 587         save_fpu_regs();
 588         save_access_regs(vcpu->run->s.regs.acrs);
 589         if (MACHINE_HAS_GS && vcpu->arch.gs_enabled)
 590                 save_gs_cb(current->thread.gs_cb);
 591
 592         /* Extended save area */
 593         rc = read_guest_lc(vcpu, __LC_MCESAD, &ext_sa_addr,
 594                            sizeof(unsigned long));
 595         /* Only bits 0 through 63-LC are used for address formation */
 596         lc = ext_sa_addr & MCESA_LC_MASK;
 597         if (test_kvm_facility(vcpu->kvm, 133)) {
 598                 switch (lc) {
 599                 case 0:
 600                 case 10:
 601                         ext_sa_addr &= ~0x3ffUL;
 602                         break;
 603                 case 11:
 604                         ext_sa_addr &= ~0x7ffUL;
 605                         break;
 606                 case 12:
 607                         ext_sa_addr &= ~0xfffUL;
 608                         break;
 609                 default:
 610                         ext_sa_addr = 0;
 611                         break;
 612                 }
 613         } else {
 614                 ext_sa_addr &= ~0x3ffUL;
 615         }
 616
 617         if (!rc && mci.vr && ext_sa_addr && test_kvm_facility(vcpu->kvm, 129)) {
 618                 if (write_guest_abs(vcpu, ext_sa_addr, vcpu->run->s.regs.vrs,
 619                                     512))
 620                         mci.vr = 0;
 621         } else {
 622                 mci.vr = 0;
 623         }
 624         if (!rc && mci.gs && ext_sa_addr && test_kvm_facility(vcpu->kvm, 133)
 625             && (lc == 11 || lc == 12)) {
 626                 if (write_guest_abs(vcpu, ext_sa_addr + 1024,
 627                                     &vcpu->run->s.regs.gscb, 32))
 628                         mci.gs = 0;
 629         } else {
 630                 mci.gs = 0;
 631         }
 632
 633         /* General interruption information */
 634         rc |= put_guest_lc(vcpu, 1, (u8 __user *) __LC_AR_MODE_ID);
 635         rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW,
 636                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 637         rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW,
 638                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 639         rc |= put_guest_lc(vcpu, mci.val, (u64 __user *) __LC_MCCK_CODE);
 640
 641         /* Register-save areas */
 642         if (MACHINE_HAS_VX) {
 643                 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
 644                 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, fprs, 128);
 645         } else {
 646                 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA,
 647                                      vcpu->run->s.regs.fprs, 128);
 648         }
 649         rc |= write_guest_lc(vcpu, __LC_GPREGS_SAVE_AREA,
 650                              vcpu->run->s.regs.gprs, 128);
 651         rc |= put_guest_lc(vcpu, current->thread.fpu.fpc,
 652                            (u32 __user *) __LC_FP_CREG_SAVE_AREA);
 653         rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->todpr,
 654                            (u32 __user *) __LC_TOD_PROGREG_SAVE_AREA);
 655         rc |= put_guest_lc(vcpu, kvm_s390_get_cpu_timer(vcpu),
 656                            (u64 __user *) __LC_CPU_TIMER_SAVE_AREA);
 657         rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->ckc >> 8,
 658                            (u64 __user *) __LC_CLOCK_COMP_SAVE_AREA);
 659         rc |= write_guest_lc(vcpu, __LC_AREGS_SAVE_AREA,
 660                              &vcpu->run->s.regs.acrs, 64);
 661         rc |= write_guest_lc(vcpu, __LC_CREGS_SAVE_AREA,
 662                              &vcpu->arch.sie_block->gcr, 128);
 663
 664         /* Extended interruption information */
 665         rc |= put_guest_lc(vcpu, mchk->ext_damage_code,
 666                            (u32 __user *) __LC_EXT_DAMAGE_CODE);
 667         rc |= put_guest_lc(vcpu, mchk->failing_storage_address,
 668                            (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR);
 669         rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA, &mchk->fixed_logout,
 670                              sizeof(mchk->fixed_logout));
 671         return rc ? -EFAULT : 0;
 672 }
 673
 674 static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu)
 675 {
 676         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
 677         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 678         struct kvm_s390_mchk_info mchk = {};
 679         int deliver = 0;
 680         int rc = 0;
 681
 682         spin_lock(&fi->lock);
 683         spin_lock(&li->lock);
 684         if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) ||
 685             test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) {
 686                 /*
 687                  * If there was an exigent machine check pending, then any
 688                  * repressible machine checks that might have been pending
 689                  * are indicated along with it, so always clear bits for
 690                  * repressible and exigent interrupts
 691                  */
 692                 mchk = li->irq.mchk;
 693                 clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
 694                 clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs);
 695                 memset(&li->irq.mchk, 0, sizeof(mchk));
 696                 deliver = 1;
 697         }
 698         /*
 699          * We indicate floating repressible conditions along with
 700          * other pending conditions. Channel Report Pending and Channel
 701          * Subsystem damage are the only two and are indicated by
 702          * bits in mcic and masked in cr14.
 703          */
 704         if (test_and_clear_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
 705                 mchk.mcic |= fi->mchk.mcic;
 706                 mchk.cr14 |= fi->mchk.cr14;
 707                 memset(&fi->mchk, 0, sizeof(mchk));
 708                 deliver = 1;
 709         }
 710         spin_unlock(&li->lock);
 711         spin_unlock(&fi->lock);
 712
 713         if (deliver) {
 714                 VCPU_EVENT(vcpu, 3, "deliver: machine check mcic 0x%llx",
 715                            mchk.mcic);
 716                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 717                                                  KVM_S390_MCHK,
 718                                                  mchk.cr14, mchk.mcic);
 719                 vcpu->stat.deliver_machine_check++;
 720                 rc = __write_machine_check(vcpu, &mchk);
 721         }
 722         return rc;
 723 }
 724
 725 static int __must_check __deliver_restart(struct kvm_vcpu *vcpu)
 726 {
 727         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 728         int rc = 0;
 729
 730         VCPU_EVENT(vcpu, 3, "%s", "deliver: cpu restart");
 731         vcpu->stat.deliver_restart_signal++;
 732         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);
 733
 734         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 735                 vcpu->arch.sie_block->iictl = IICTL_CODE_RESTART;
 736         } else {
 737                 rc  = write_guest_lc(vcpu,
 738                                      offsetof(struct lowcore, restart_old_psw),
 739                                      &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 740                 rc |= read_guest_lc(vcpu, offsetof(struct lowcore, restart_psw),
 741                                     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 742         }
 743         clear_bit(IRQ_PEND_RESTART, &li->pending_irqs);
 744         return rc ? -EFAULT : 0;
 745 }
 746
 747 static int __must_check __deliver_set_prefix(struct kvm_vcpu *vcpu)
 748 {
 749         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 750         struct kvm_s390_prefix_info prefix;
 751
 752         spin_lock(&li->lock);
 753         prefix = li->irq.prefix;
 754         li->irq.prefix.address = 0;
 755         clear_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
 756         spin_unlock(&li->lock);
 757
 758         vcpu->stat.deliver_prefix_signal++;
 759         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 760                                          KVM_S390_SIGP_SET_PREFIX,
 761                                          prefix.address, 0);
 762
 763         kvm_s390_set_prefix(vcpu, prefix.address);
 764         return 0;
 765 }
 766
 767 static int __must_check __deliver_emergency_signal(struct kvm_vcpu *vcpu)
 768 {
 769         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 770         int rc;
 771         int cpu_addr;
 772
 773         spin_lock(&li->lock);
 774         cpu_addr = find_first_bit(li->sigp_emerg_pending, KVM_MAX_VCPUS);
 775         clear_bit(cpu_addr, li->sigp_emerg_pending);
 776         if (bitmap_empty(li->sigp_emerg_pending, KVM_MAX_VCPUS))
 777                 clear_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
 778         spin_unlock(&li->lock);
 779
 780         VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp emerg");
 781         vcpu->stat.deliver_emergency_signal++;
 782         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
 783                                          cpu_addr, 0);
 784         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 785                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 786                 vcpu->arch.sie_block->eic = EXT_IRQ_EMERGENCY_SIG;
 787                 vcpu->arch.sie_block->extcpuaddr = cpu_addr;
 788                 return 0;
 789         }
 790
 791         rc  = put_guest_lc(vcpu, EXT_IRQ_EMERGENCY_SIG,
 792                            (u16 *)__LC_EXT_INT_CODE);
 793         rc |= put_guest_lc(vcpu, cpu_addr, (u16 *)__LC_EXT_CPU_ADDR);
 794         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 795                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 796         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 797                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 798         return rc ? -EFAULT : 0;
 799 }
 800
 801 static int __must_check __deliver_external_call(struct kvm_vcpu *vcpu)
 802 {
 803         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 804         struct kvm_s390_extcall_info extcall;
 805         int rc;
 806
 807         spin_lock(&li->lock);
 808         extcall = li->irq.extcall;
 809         li->irq.extcall.code = 0;
 810         clear_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
 811         spin_unlock(&li->lock);
 812
 813         VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp ext call");
 814         vcpu->stat.deliver_external_call++;
 815         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 816                                          KVM_S390_INT_EXTERNAL_CALL,
 817                                          extcall.code, 0);
 818         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 819                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 820                 vcpu->arch.sie_block->eic = EXT_IRQ_EXTERNAL_CALL;
 821                 vcpu->arch.sie_block->extcpuaddr = extcall.code;
 822                 return 0;
 823         }
 824
 825         rc  = put_guest_lc(vcpu, EXT_IRQ_EXTERNAL_CALL,
 826                            (u16 *)__LC_EXT_INT_CODE);
 827         rc |= put_guest_lc(vcpu, extcall.code, (u16 *)__LC_EXT_CPU_ADDR);
 828         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 829                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 830         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, &vcpu->arch.sie_block->gpsw,
 831                             sizeof(psw_t));
 832         return rc ? -EFAULT : 0;
 833 }
 834
 835 static int __deliver_prog_pv(struct kvm_vcpu *vcpu, u16 code)
 836 {
 837         switch (code) {
 838         case PGM_SPECIFICATION:
 839                 vcpu->arch.sie_block->iictl = IICTL_CODE_SPECIFICATION;
 840                 break;
 841         case PGM_OPERAND:
 842                 vcpu->arch.sie_block->iictl = IICTL_CODE_OPERAND;
 843                 break;
 844         default:
 845                 return -EINVAL;
 846         }
 847         return 0;
 848 }
 849
 850 static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
 851 {
 852         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 853         struct kvm_s390_pgm_info pgm_info;
 854         int rc = 0, nullifying = false;
 855         u16 ilen;
 856
 857         spin_lock(&li->lock);
 858         pgm_info = li->irq.pgm;
 859         clear_bit(IRQ_PEND_PROG, &li->pending_irqs);
 860         memset(&li->irq.pgm, 0, sizeof(pgm_info));
 861         spin_unlock(&li->lock);
 862
 863         ilen = pgm_info.flags & KVM_S390_PGM_FLAGS_ILC_MASK;
 864         VCPU_EVENT(vcpu, 3, "deliver: program irq code 0x%x, ilen:%d",
 865                    pgm_info.code, ilen);
 866         vcpu->stat.deliver_program++;
 867         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
 868                                          pgm_info.code, 0);
 869
 870         /* PER is handled by the ultravisor */
 871         if (kvm_s390_pv_cpu_is_protected(vcpu))
 872                 return __deliver_prog_pv(vcpu, pgm_info.code & ~PGM_PER);
 873
 874         switch (pgm_info.code & ~PGM_PER) {
 875         case PGM_AFX_TRANSLATION:
 876         case PGM_ASX_TRANSLATION:
 877         case PGM_EX_TRANSLATION:
 878         case PGM_LFX_TRANSLATION:
 879         case PGM_LSTE_SEQUENCE:
 880         case PGM_LSX_TRANSLATION:
 881         case PGM_LX_TRANSLATION:
 882         case PGM_PRIMARY_AUTHORITY:
 883         case PGM_SECONDARY_AUTHORITY:
 884                 nullifying = true;
 885                 fallthrough;
 886         case PGM_SPACE_SWITCH:
 887                 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
 888                                   (u64 *)__LC_TRANS_EXC_CODE);
 889                 break;
 890         case PGM_ALEN_TRANSLATION:
 891         case PGM_ALE_SEQUENCE:
 892         case PGM_ASTE_INSTANCE:
 893         case PGM_ASTE_SEQUENCE:
 894         case PGM_ASTE_VALIDITY:
 895         case PGM_EXTENDED_AUTHORITY:
 896                 rc = put_guest_lc(vcpu, pgm_info.exc_access_id,
 897                                   (u8 *)__LC_EXC_ACCESS_ID);
 898                 nullifying = true;
 899                 break;
 900         case PGM_ASCE_TYPE:
 901         case PGM_PAGE_TRANSLATION:
 902         case PGM_REGION_FIRST_TRANS:
 903         case PGM_REGION_SECOND_TRANS:
 904         case PGM_REGION_THIRD_TRANS:
 905         case PGM_SEGMENT_TRANSLATION:
 906                 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
 907                                   (u64 *)__LC_TRANS_EXC_CODE);
 908                 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
 909                                    (u8 *)__LC_EXC_ACCESS_ID);
 910                 rc |= put_guest_lc(vcpu, pgm_info.op_access_id,
 911                                    (u8 *)__LC_OP_ACCESS_ID);
 912                 nullifying = true;
 913                 break;
 914         case PGM_MONITOR:
 915                 rc = put_guest_lc(vcpu, pgm_info.mon_class_nr,
 916                                   (u16 *)__LC_MON_CLASS_NR);
 917                 rc |= put_guest_lc(vcpu, pgm_info.mon_code,
 918                                    (u64 *)__LC_MON_CODE);
 919                 break;
 920         case PGM_VECTOR_PROCESSING:
 921         case PGM_DATA:
 922                 rc = put_guest_lc(vcpu, pgm_info.data_exc_code,
 923                                   (u32 *)__LC_DATA_EXC_CODE);
 924                 break;
 925         case PGM_PROTECTION:
 926                 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
 927                                   (u64 *)__LC_TRANS_EXC_CODE);
 928                 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
 929                                    (u8 *)__LC_EXC_ACCESS_ID);
 930                 break;
 931         case PGM_STACK_FULL:
 932         case PGM_STACK_EMPTY:
 933         case PGM_STACK_SPECIFICATION:
 934         case PGM_STACK_TYPE:
 935         case PGM_STACK_OPERATION:
 936         case PGM_TRACE_TABEL:
 937         case PGM_CRYPTO_OPERATION:
 938                 nullifying = true;
 939                 break;
 940         }
 941
 942         if (pgm_info.code & PGM_PER) {
 943                 rc |= put_guest_lc(vcpu, pgm_info.per_code,
 944                                    (u8 *) __LC_PER_CODE);
 945                 rc |= put_guest_lc(vcpu, pgm_info.per_atmid,
 946                                    (u8 *)__LC_PER_ATMID);
 947                 rc |= put_guest_lc(vcpu, pgm_info.per_address,
 948                                    (u64 *) __LC_PER_ADDRESS);
 949                 rc |= put_guest_lc(vcpu, pgm_info.per_access_id,
 950                                    (u8 *) __LC_PER_ACCESS_ID);
 951         }
 952
 953         if (nullifying && !(pgm_info.flags & KVM_S390_PGM_FLAGS_NO_REWIND))
 954                 kvm_s390_rewind_psw(vcpu, ilen);
 955
 956         /* bit 1+2 of the target are the ilc, so we can directly use ilen */
 957         rc |= put_guest_lc(vcpu, ilen, (u16 *) __LC_PGM_ILC);
 958         rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->gbea,
 959                                  (u64 *) __LC_PGM_LAST_BREAK);
 960         rc |= put_guest_lc(vcpu, pgm_info.code,
 961                            (u16 *)__LC_PGM_INT_CODE);
 962         rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW,
 963                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 964         rc |= read_guest_lc(vcpu, __LC_PGM_NEW_PSW,
 965                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 966         return rc ? -EFAULT : 0;
 967 }
 968
 969 #define SCCB_MASK 0xFFFFFFF8
 970 #define SCCB_EVENT_PENDING 0x3
 971
 972 static int write_sclp(struct kvm_vcpu *vcpu, u32 parm)
 973 {
 974         int rc;
 975
 976         if (kvm_s390_pv_cpu_get_handle(vcpu)) {
 977                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 978                 vcpu->arch.sie_block->eic = EXT_IRQ_SERVICE_SIG;
 979                 vcpu->arch.sie_block->eiparams = parm;
 980                 return 0;
 981         }
 982
 983         rc  = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE);
 984         rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
 985         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 986                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 987         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 988                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 989         rc |= put_guest_lc(vcpu, parm,
 990                            (u32 *)__LC_EXT_PARAMS);
 991
 992         return rc ? -EFAULT : 0;
 993 }
 994
 995 static int __must_check __deliver_service(struct kvm_vcpu *vcpu)
 996 {
 997         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
 998         struct kvm_s390_ext_info ext;
 999
1000         spin_lock(&fi->lock);
1001         if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs) ||
1002             !(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) {
1003                 spin_unlock(&fi->lock);
1004                 return 0;
1005         }
1006         ext = fi->srv_signal;
1007         memset(&fi->srv_signal, 0, sizeof(ext));
1008         clear_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
1009         clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
1010         if (kvm_s390_pv_cpu_is_protected(vcpu))
1011                 set_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs);
1012         spin_unlock(&fi->lock);
1013
1014         VCPU_EVENT(vcpu, 4, "deliver: sclp parameter 0x%x",
1015                    ext.ext_params);
1016         vcpu->stat.deliver_service_signal++;
1017         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
1018                                          ext.ext_params, 0);
1019
1020         return write_sclp(vcpu, ext.ext_params);
1021 }
1022
1023 static int __must_check __deliver_service_ev(struct kvm_vcpu *vcpu)
1024 {
1025         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1026         struct kvm_s390_ext_info ext;
1027
1028         spin_lock(&fi->lock);
1029         if (!(test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs))) {
1030                 spin_unlock(&fi->lock);
1031                 return 0;
1032         }
1033         ext = fi->srv_signal;
1034         /* only clear the event bit */
1035         fi->srv_signal.ext_params &= ~SCCB_EVENT_PENDING;
1036         clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
1037         spin_unlock(&fi->lock);
1038
1039         VCPU_EVENT(vcpu, 4, "%s", "deliver: sclp parameter event");
1040         vcpu->stat.deliver_service_signal++;
1041         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
1042                                          ext.ext_params, 0);
1043
1044         return write_sclp(vcpu, SCCB_EVENT_PENDING);
1045 }
1046
1047 static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu)
1048 {
1049         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1050         struct kvm_s390_interrupt_info *inti;
1051         int rc = 0;
1052
1053         spin_lock(&fi->lock);
1054         inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT],
1055                                         struct kvm_s390_interrupt_info,
1056                                         list);
1057         if (inti) {
1058                 list_del(&inti->list);
1059                 fi->counters[FIRQ_CNTR_PFAULT] -= 1;
1060         }
1061         if (list_empty(&fi->lists[FIRQ_LIST_PFAULT]))
1062                 clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
1063         spin_unlock(&fi->lock);
1064
1065         if (inti) {
1066                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1067                                                  KVM_S390_INT_PFAULT_DONE, 0,
1068                                                  inti->ext.ext_params2);
1069                 VCPU_EVENT(vcpu, 4, "deliver: pfault done token 0x%llx",
1070                            inti->ext.ext_params2);
1071
1072                 rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
1073                                 (u16 *)__LC_EXT_INT_CODE);
1074                 rc |= put_guest_lc(vcpu, PFAULT_DONE,
1075                                 (u16 *)__LC_EXT_CPU_ADDR);
1076                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
1077                                 &vcpu->arch.sie_block->gpsw,
1078                                 sizeof(psw_t));
1079                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
1080                                 &vcpu->arch.sie_block->gpsw,
1081                                 sizeof(psw_t));
1082                 rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
1083                                 (u64 *)__LC_EXT_PARAMS2);
1084                 kfree(inti);
1085         }
1086         return rc ? -EFAULT : 0;
1087 }
1088
1089 static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu)
1090 {
1091         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1092         struct kvm_s390_interrupt_info *inti;
1093         int rc = 0;
1094
1095         spin_lock(&fi->lock);
1096         inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO],
1097                                         struct kvm_s390_interrupt_info,
1098                                         list);
1099         if (inti) {
1100                 VCPU_EVENT(vcpu, 4,
1101                            "deliver: virtio parm: 0x%x,parm64: 0x%llx",
1102                            inti->ext.ext_params, inti->ext.ext_params2);
1103                 vcpu->stat.deliver_virtio++;
1104                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1105                                 inti->type,
1106                                 inti->ext.ext_params,
1107                                 inti->ext.ext_params2);
1108                 list_del(&inti->list);
1109                 fi->counters[FIRQ_CNTR_VIRTIO] -= 1;
1110         }
1111         if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO]))
1112                 clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
1113         spin_unlock(&fi->lock);
1114
1115         if (inti) {
1116                 rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
1117                                 (u16 *)__LC_EXT_INT_CODE);
1118                 rc |= put_guest_lc(vcpu, VIRTIO_PARAM,
1119                                 (u16 *)__LC_EXT_CPU_ADDR);
1120                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
1121                                 &vcpu->arch.sie_block->gpsw,
1122                                 sizeof(psw_t));
1123                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
1124                                 &vcpu->arch.sie_block->gpsw,
1125                                 sizeof(psw_t));
1126                 rc |= put_guest_lc(vcpu, inti->ext.ext_params,
1127                                 (u32 *)__LC_EXT_PARAMS);
1128                 rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
1129                                 (u64 *)__LC_EXT_PARAMS2);
1130                 kfree(inti);
1131         }
1132         return rc ? -EFAULT : 0;
1133 }
1134
1135 static int __do_deliver_io(struct kvm_vcpu *vcpu, struct kvm_s390_io_info *io)
1136 {
1137         int rc;
1138
1139         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
1140                 vcpu->arch.sie_block->iictl = IICTL_CODE_IO;
1141                 vcpu->arch.sie_block->subchannel_id = io->subchannel_id;
1142                 vcpu->arch.sie_block->subchannel_nr = io->subchannel_nr;
1143                 vcpu->arch.sie_block->io_int_parm = io->io_int_parm;
1144                 vcpu->arch.sie_block->io_int_word = io->io_int_word;
1145                 return 0;
1146         }
1147
1148         rc  = put_guest_lc(vcpu, io->subchannel_id, (u16 *)__LC_SUBCHANNEL_ID);
1149         rc |= put_guest_lc(vcpu, io->subchannel_nr, (u16 *)__LC_SUBCHANNEL_NR);
1150         rc |= put_guest_lc(vcpu, io->io_int_parm, (u32 *)__LC_IO_INT_PARM);
1151         rc |= put_guest_lc(vcpu, io->io_int_word, (u32 *)__LC_IO_INT_WORD);
1152         rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW,
1153                              &vcpu->arch.sie_block->gpsw,
1154                              sizeof(psw_t));
1155         rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW,
1156                             &vcpu->arch.sie_block->gpsw,
1157                             sizeof(psw_t));
1158         return rc ? -EFAULT : 0;
1159 }
1160
1161 static int __must_check __deliver_io(struct kvm_vcpu *vcpu,
1162                                      unsigned long irq_type)
1163 {
1164         struct list_head *isc_list;
1165         struct kvm_s390_float_interrupt *fi;
1166         struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
1167         struct kvm_s390_interrupt_info *inti = NULL;
1168         struct kvm_s390_io_info io;
1169         u32 isc;
1170         int rc = 0;
1171
1172         fi = &vcpu->kvm->arch.float_int;
1173
1174         spin_lock(&fi->lock);
1175         isc = irq_type_to_isc(irq_type);
1176         isc_list = &fi->lists[isc];
1177         inti = list_first_entry_or_null(isc_list,
1178                                         struct kvm_s390_interrupt_info,
1179                                         list);
1180         if (inti) {
1181                 if (inti->type & KVM_S390_INT_IO_AI_MASK)
1182                         VCPU_EVENT(vcpu, 4, "%s", "deliver: I/O (AI)");
1183                 else
1184                         VCPU_EVENT(vcpu, 4, "deliver: I/O %x ss %x schid %04x",
1185                         inti->io.subchannel_id >> 8,
1186                         inti->io.subchannel_id >> 1 & 0x3,
1187                         inti->io.subchannel_nr);
1188
1189                 vcpu->stat.deliver_io++;
1190                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1191                                 inti->type,
1192                                 ((__u32)inti->io.subchannel_id << 16) |
1193                                 inti->io.subchannel_nr,
1194                                 ((__u64)inti->io.io_int_parm << 32) |
1195                                 inti->io.io_int_word);
1196                 list_del(&inti->list);
1197                 fi->counters[FIRQ_CNTR_IO] -= 1;
1198         }
1199         if (list_empty(isc_list))
1200                 clear_bit(irq_type, &fi->pending_irqs);
1201         spin_unlock(&fi->lock);
1202
1203         if (inti) {
1204                 rc = __do_deliver_io(vcpu, &(inti->io));
1205                 kfree(inti);
1206                 goto out;
1207         }
1208
1209         if (gi->origin && gisa_tac_ipm_gisc(gi->origin, isc)) {
1210                 /*
1211                  * in case an adapter interrupt was not delivered
1212                  * in SIE context KVM will handle the delivery
1213                  */
1214                 VCPU_EVENT(vcpu, 4, "%s isc %u", "deliver: I/O (AI/gisa)", isc);
1215                 memset(&io, 0, sizeof(io));
1216                 io.io_int_word = isc_to_int_word(isc);
1217                 vcpu->stat.deliver_io++;
1218                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1219                         KVM_S390_INT_IO(1, 0, 0, 0),
1220                         ((__u32)io.subchannel_id << 16) |
1221                         io.subchannel_nr,
1222                         ((__u64)io.io_int_parm << 32) |
1223                         io.io_int_word);
1224                 rc = __do_deliver_io(vcpu, &io);
1225         }
1226 out:
1227         return rc;
1228 }
1229
1230 /* Check whether an external call is pending (deliverable or not) */
1231 int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu)
1232 {
1233         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1234
1235         if (!sclp.has_sigpif)
1236                 return test_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
1237
1238         return sca_ext_call_pending(vcpu, NULL);
1239 }
1240
1241 int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop)
1242 {
1243         if (deliverable_irqs(vcpu))
1244                 return 1;
1245
1246         if (kvm_cpu_has_pending_timer(vcpu))
1247                 return 1;
1248
1249         /* external call pending and deliverable */
1250         if (kvm_s390_ext_call_pending(vcpu) &&
1251             !psw_extint_disabled(vcpu) &&
1252             (vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
1253                 return 1;
1254
1255         if (!exclude_stop && kvm_s390_is_stop_irq_pending(vcpu))
1256                 return 1;
1257         return 0;
1258 }
1259
1260 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
1261 {
1262         return ckc_irq_pending(vcpu) || cpu_timer_irq_pending(vcpu);
1263 }
1264
1265 static u64 __calculate_sltime(struct kvm_vcpu *vcpu)
1266 {
1267         const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
1268         const u64 ckc = vcpu->arch.sie_block->ckc;
1269         u64 cputm, sltime = 0;
1270
1271         if (ckc_interrupts_enabled(vcpu)) {
1272                 if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
1273                         if ((s64)now < (s64)ckc)
1274                                 sltime = tod_to_ns((s64)ckc - (s64)now);
1275                 } else if (now < ckc) {
1276                         sltime = tod_to_ns(ckc - now);
1277                 }
1278                 /* already expired */
1279                 if (!sltime)
1280                         return 0;
1281                 if (cpu_timer_interrupts_enabled(vcpu)) {
1282                         cputm = kvm_s390_get_cpu_timer(vcpu);
1283                         /* already expired? */
1284                         if (cputm >> 63)
1285                                 return 0;
1286                         return min_t(u64, sltime, tod_to_ns(cputm));
1287                 }
1288         } else if (cpu_timer_interrupts_enabled(vcpu)) {
1289                 sltime = kvm_s390_get_cpu_timer(vcpu);
1290                 /* already expired? */
1291                 if (sltime >> 63)
1292                         return 0;
1293         }
1294         return sltime;
1295 }
1296
1297 int kvm_s390_handle_wait(struct kvm_vcpu *vcpu)
1298 {
1299         struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
1300         u64 sltime;
1301
1302         vcpu->stat.exit_wait_state++;
1303
1304         /* fast path */
1305         if (kvm_arch_vcpu_runnable(vcpu))
1306                 return 0;
1307
1308         if (psw_interrupts_disabled(vcpu)) {
1309                 VCPU_EVENT(vcpu, 3, "%s", "disabled wait");
1310                 return -EOPNOTSUPP; /* disabled wait */
1311         }
1312
1313         if (gi->origin &&
1314             (gisa_get_ipm_or_restore_iam(gi) &
1315              vcpu->arch.sie_block->gcr[6] >> 24))
1316                 return 0;
1317
1318         if (!ckc_interrupts_enabled(vcpu) &&
1319             !cpu_timer_interrupts_enabled(vcpu)) {
1320                 VCPU_EVENT(vcpu, 3, "%s", "enabled wait w/o timer");
1321                 __set_cpu_idle(vcpu);
1322                 goto no_timer;
1323         }
1324
1325         sltime = __calculate_sltime(vcpu);
1326         if (!sltime)
1327                 return 0;
1328
1329         __set_cpu_idle(vcpu);
1330         hrtimer_start(&vcpu->arch.ckc_timer, sltime, HRTIMER_MODE_REL);
1331         VCPU_EVENT(vcpu, 4, "enabled wait: %llu ns", sltime);
1332 no_timer:
1333         kvm_vcpu_srcu_read_unlock(vcpu);
1334         kvm_vcpu_halt(vcpu);
1335         vcpu->valid_wakeup = false;
1336         __unset_cpu_idle(vcpu);
1337         kvm_vcpu_srcu_read_lock(vcpu);
1338
1339         hrtimer_cancel(&vcpu->arch.ckc_timer);
1340         return 0;
1341 }
1342
1343 void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu)
1344 {
1345         vcpu->valid_wakeup = true;
1346         kvm_vcpu_wake_up(vcpu);
1347
1348         /*
1349          * The VCPU might not be sleeping but rather executing VSIE. Let's
1350          * kick it, so it leaves the SIE to process the request.
1351          */
1352         kvm_s390_vsie_kick(vcpu);
1353 }
1354
1355 enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer)
1356 {
1357         struct kvm_vcpu *vcpu;
1358         u64 sltime;
1359
1360         vcpu = container_of(timer, struct kvm_vcpu, arch.ckc_timer);
1361         sltime = __calculate_sltime(vcpu);
1362
1363         /*
1364          * If the monotonic clock runs faster than the tod clock we might be
1365          * woken up too early and have to go back to sleep to avoid deadlocks.
1366          */
1367         if (sltime && hrtimer_forward_now(timer, ns_to_ktime(sltime)))
1368                 return HRTIMER_RESTART;
1369         kvm_s390_vcpu_wakeup(vcpu);
1370         return HRTIMER_NORESTART;
1371 }
1372
1373 void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu)
1374 {
1375         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1376
1377         spin_lock(&li->lock);
1378         li->pending_irqs = 0;
1379         bitmap_zero(li->sigp_emerg_pending, KVM_MAX_VCPUS);
1380         memset(&li->irq, 0, sizeof(li->irq));
1381         spin_unlock(&li->lock);
1382
1383         sca_clear_ext_call(vcpu);
1384 }
1385
1386 int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
1387 {
1388         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1389         int rc = 0;
1390         bool delivered = false;
1391         unsigned long irq_type;
1392         unsigned long irqs;
1393
1394         __reset_intercept_indicators(vcpu);
1395
1396         /* pending ckc conditions might have been invalidated */
1397         clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1398         if (ckc_irq_pending(vcpu))
1399                 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1400
1401         /* pending cpu timer conditions might have been invalidated */
1402         clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1403         if (cpu_timer_irq_pending(vcpu))
1404                 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1405
1406         while ((irqs = deliverable_irqs(vcpu)) && !rc) {
1407                 /* bits are in the reverse order of interrupt priority */
1408                 irq_type = find_last_bit(&irqs, IRQ_PEND_COUNT);
1409                 switch (irq_type) {
1410                 case IRQ_PEND_IO_ISC_0:
1411                 case IRQ_PEND_IO_ISC_1:
1412                 case IRQ_PEND_IO_ISC_2:
1413                 case IRQ_PEND_IO_ISC_3:
1414                 case IRQ_PEND_IO_ISC_4:
1415                 case IRQ_PEND_IO_ISC_5:
1416                 case IRQ_PEND_IO_ISC_6:
1417                 case IRQ_PEND_IO_ISC_7:
1418                         rc = __deliver_io(vcpu, irq_type);
1419                         break;
1420                 case IRQ_PEND_MCHK_EX:
1421                 case IRQ_PEND_MCHK_REP:
1422                         rc = __deliver_machine_check(vcpu);
1423                         break;
1424                 case IRQ_PEND_PROG:
1425                         rc = __deliver_prog(vcpu);
1426                         break;
1427                 case IRQ_PEND_EXT_EMERGENCY:
1428                         rc = __deliver_emergency_signal(vcpu);
1429                         break;
1430                 case IRQ_PEND_EXT_EXTERNAL:
1431                         rc = __deliver_external_call(vcpu);
1432                         break;
1433                 case IRQ_PEND_EXT_CLOCK_COMP:
1434                         rc = __deliver_ckc(vcpu);
1435                         break;
1436                 case IRQ_PEND_EXT_CPU_TIMER:
1437                         rc = __deliver_cpu_timer(vcpu);
1438                         break;
1439                 case IRQ_PEND_RESTART:
1440                         rc = __deliver_restart(vcpu);
1441                         break;
1442                 case IRQ_PEND_SET_PREFIX:
1443                         rc = __deliver_set_prefix(vcpu);
1444                         break;
1445                 case IRQ_PEND_PFAULT_INIT:
1446                         rc = __deliver_pfault_init(vcpu);
1447                         break;
1448                 case IRQ_PEND_EXT_SERVICE:
1449                         rc = __deliver_service(vcpu);
1450                         break;
1451                 case IRQ_PEND_EXT_SERVICE_EV:
1452                         rc = __deliver_service_ev(vcpu);
1453                         break;
1454                 case IRQ_PEND_PFAULT_DONE:
1455                         rc = __deliver_pfault_done(vcpu);
1456                         break;
1457                 case IRQ_PEND_VIRTIO:
1458                         rc = __deliver_virtio(vcpu);
1459                         break;
1460                 default:
1461                         WARN_ONCE(1, "Unknown pending irq type %ld", irq_type);
1462                         clear_bit(irq_type, &li->pending_irqs);
1463                 }
1464                 delivered |= !rc;
1465         }
1466
1467         /*
1468          * We delivered at least one interrupt and modified the PC. Force a
1469          * singlestep event now.
1470          */
1471         if (delivered && guestdbg_sstep_enabled(vcpu)) {
1472                 struct kvm_debug_exit_arch *debug_exit = &vcpu->run->debug.arch;
1473
1474                 debug_exit->addr = vcpu->arch.sie_block->gpsw.addr;
1475                 debug_exit->type = KVM_SINGLESTEP;
1476                 vcpu->guest_debug |= KVM_GUESTDBG_EXIT_PENDING;
1477         }
1478
1479         set_intercept_indicators(vcpu);
1480
1481         return rc;
1482 }
1483
1484 static int __inject_prog(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1485 {
1486         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1487
1488         vcpu->stat.inject_program++;
1489         VCPU_EVENT(vcpu, 3, "inject: program irq code 0x%x", irq->u.pgm.code);
1490         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
1491                                    irq->u.pgm.code, 0);
1492
1493         if (!(irq->u.pgm.flags & KVM_S390_PGM_FLAGS_ILC_VALID)) {
1494                 /* auto detection if no valid ILC was given */
1495                 irq->u.pgm.flags &= ~KVM_S390_PGM_FLAGS_ILC_MASK;
1496                 irq->u.pgm.flags |= kvm_s390_get_ilen(vcpu);
1497                 irq->u.pgm.flags |= KVM_S390_PGM_FLAGS_ILC_VALID;
1498         }
1499
1500         if (irq->u.pgm.code == PGM_PER) {
1501                 li->irq.pgm.code |= PGM_PER;
1502                 li->irq.pgm.flags = irq->u.pgm.flags;
1503                 /* only modify PER related information */
1504                 li->irq.pgm.per_address = irq->u.pgm.per_address;
1505                 li->irq.pgm.per_code = irq->u.pgm.per_code;
1506                 li->irq.pgm.per_atmid = irq->u.pgm.per_atmid;
1507                 li->irq.pgm.per_access_id = irq->u.pgm.per_access_id;
1508         } else if (!(irq->u.pgm.code & PGM_PER)) {
1509                 li->irq.pgm.code = (li->irq.pgm.code & PGM_PER) |
1510                                    irq->u.pgm.code;
1511                 li->irq.pgm.flags = irq->u.pgm.flags;
1512                 /* only modify non-PER information */
1513                 li->irq.pgm.trans_exc_code = irq->u.pgm.trans_exc_code;
1514                 li->irq.pgm.mon_code = irq->u.pgm.mon_code;
1515                 li->irq.pgm.data_exc_code = irq->u.pgm.data_exc_code;
1516                 li->irq.pgm.mon_class_nr = irq->u.pgm.mon_class_nr;
1517                 li->irq.pgm.exc_access_id = irq->u.pgm.exc_access_id;
1518                 li->irq.pgm.op_access_id = irq->u.pgm.op_access_id;
1519         } else {
1520                 li->irq.pgm = irq->u.pgm;
1521         }
1522         set_bit(IRQ_PEND_PROG, &li->pending_irqs);
1523         return 0;
1524 }
1525
1526 static int __inject_pfault_init(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1527 {
1528         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1529
1530         vcpu->stat.inject_pfault_init++;
1531         VCPU_EVENT(vcpu, 4, "inject: pfault init parameter block at 0x%llx",
1532                    irq->u.ext.ext_params2);
1533         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_PFAULT_INIT,
1534                                    irq->u.ext.ext_params,
1535                                    irq->u.ext.ext_params2);
1536
1537         li->irq.ext = irq->u.ext;
1538         set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
1539         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1540         return 0;
1541 }
1542
1543 static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1544 {
1545         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1546         struct kvm_s390_extcall_info *extcall = &li->irq.extcall;
1547         uint16_t src_id = irq->u.extcall.code;
1548
1549         vcpu->stat.inject_external_call++;
1550         VCPU_EVENT(vcpu, 4, "inject: external call source-cpu:%u",
1551                    src_id);
1552         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EXTERNAL_CALL,
1553                                    src_id, 0);
1554
1555         /* sending vcpu invalid */
1556         if (kvm_get_vcpu_by_id(vcpu->kvm, src_id) == NULL)
1557                 return -EINVAL;
1558
1559         if (sclp.has_sigpif && !kvm_s390_pv_cpu_get_handle(vcpu))
1560                 return sca_inject_ext_call(vcpu, src_id);
1561
1562         if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs))
1563                 return -EBUSY;
1564         *extcall = irq->u.extcall;
1565         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1566         return 0;
1567 }
1568
1569 static int __inject_set_prefix(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1570 {
1571         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1572         struct kvm_s390_prefix_info *prefix = &li->irq.prefix;
1573
1574         vcpu->stat.inject_set_prefix++;
1575         VCPU_EVENT(vcpu, 3, "inject: set prefix to %x",
1576                    irq->u.prefix.address);
1577         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_SET_PREFIX,
1578                                    irq->u.prefix.address, 0);
1579
1580         if (!is_vcpu_stopped(vcpu))
1581                 return -EBUSY;
1582
1583         *prefix = irq->u.prefix;
1584         set_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
1585         return 0;
1586 }
1587
1588 #define KVM_S390_STOP_SUPP_FLAGS (KVM_S390_STOP_FLAG_STORE_STATUS)
1589 static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1590 {
1591         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1592         struct kvm_s390_stop_info *stop = &li->irq.stop;
1593         int rc = 0;
1594
1595         vcpu->stat.inject_stop_signal++;
1596         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_STOP, 0, 0);
1597
1598         if (irq->u.stop.flags & ~KVM_S390_STOP_SUPP_FLAGS)
1599                 return -EINVAL;
1600
1601         if (is_vcpu_stopped(vcpu)) {
1602                 if (irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS)
1603                         rc = kvm_s390_store_status_unloaded(vcpu,
1604                                                 KVM_S390_STORE_STATUS_NOADDR);
1605                 return rc;
1606         }
1607
1608         if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs))
1609                 return -EBUSY;
1610         stop->flags = irq->u.stop.flags;
1611         kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
1612         return 0;
1613 }
1614
1615 static int __inject_sigp_restart(struct kvm_vcpu *vcpu)
1616 {
1617         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1618
1619         vcpu->stat.inject_restart++;
1620         VCPU_EVENT(vcpu, 3, "%s", "inject: restart int");
1621         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);
1622
1623         set_bit(IRQ_PEND_RESTART, &li->pending_irqs);
1624         return 0;
1625 }
1626
1627 static int __inject_sigp_emergency(struct kvm_vcpu *vcpu,
1628                                    struct kvm_s390_irq *irq)
1629 {
1630         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1631
1632         vcpu->stat.inject_emergency_signal++;
1633         VCPU_EVENT(vcpu, 4, "inject: emergency from cpu %u",
1634                    irq->u.emerg.code);
1635         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
1636                                    irq->u.emerg.code, 0);
1637
1638         /* sending vcpu invalid */
1639         if (kvm_get_vcpu_by_id(vcpu->kvm, irq->u.emerg.code) == NULL)
1640                 return -EINVAL;
1641
1642         set_bit(irq->u.emerg.code, li->sigp_emerg_pending);
1643         set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
1644         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1645         return 0;
1646 }
1647
1648 static int __inject_mchk(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1649 {
1650         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1651         struct kvm_s390_mchk_info *mchk = &li->irq.mchk;
1652
1653         vcpu->stat.inject_mchk++;
1654         VCPU_EVENT(vcpu, 3, "inject: machine check mcic 0x%llx",
1655                    irq->u.mchk.mcic);
1656         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_MCHK, 0,
1657                                    irq->u.mchk.mcic);
1658
1659         /*
1660          * Because repressible machine checks can be indicated along with
1661          * exigent machine checks (PoP, Chapter 11, Interruption action)
1662          * we need to combine cr14, mcic and external damage code.
1663          * Failing storage address and the logout area should not be or'ed
1664          * together, we just indicate the last occurrence of the corresponding
1665          * machine check
1666          */
1667         mchk->cr14 |= irq->u.mchk.cr14;
1668         mchk->mcic |= irq->u.mchk.mcic;
1669         mchk->ext_damage_code |= irq->u.mchk.ext_damage_code;
1670         mchk->failing_storage_address = irq->u.mchk.failing_storage_address;
1671         memcpy(&mchk->fixed_logout, &irq->u.mchk.fixed_logout,
1672                sizeof(mchk->fixed_logout));
1673         if (mchk->mcic & MCHK_EX_MASK)
1674                 set_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
1675         else if (mchk->mcic & MCHK_REP_MASK)
1676                 set_bit(IRQ_PEND_MCHK_REP,  &li->pending_irqs);
1677         return 0;
1678 }
1679
1680 static int __inject_ckc(struct kvm_vcpu *vcpu)
1681 {
1682         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1683
1684         vcpu->stat.inject_ckc++;
1685         VCPU_EVENT(vcpu, 3, "%s", "inject: clock comparator external");
1686         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
1687                                    0, 0);
1688
1689         set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1690         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1691         return 0;
1692 }
1693
1694 static int __inject_cpu_timer(struct kvm_vcpu *vcpu)
1695 {
1696         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1697
1698         vcpu->stat.inject_cputm++;
1699         VCPU_EVENT(vcpu, 3, "%s", "inject: cpu timer external");
1700         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
1701                                    0, 0);
1702
1703         set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1704         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1705         return 0;
1706 }
1707
1708 static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm,
1709                                                   int isc, u32 schid)
1710 {
1711         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1712         struct list_head *isc_list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1713         struct kvm_s390_interrupt_info *iter;
1714         u16 id = (schid & 0xffff0000U) >> 16;
1715         u16 nr = schid & 0x0000ffffU;
1716
1717         spin_lock(&fi->lock);
1718         list_for_each_entry(iter, isc_list, list) {
1719                 if (schid && (id != iter->io.subchannel_id ||
1720                               nr != iter->io.subchannel_nr))
1721                         continue;
1722                 /* found an appropriate entry */
1723                 list_del_init(&iter->list);
1724                 fi->counters[FIRQ_CNTR_IO] -= 1;
1725                 if (list_empty(isc_list))
1726                         clear_bit(isc_to_irq_type(isc), &fi->pending_irqs);
1727                 spin_unlock(&fi->lock);
1728                 return iter;
1729         }
1730         spin_unlock(&fi->lock);
1731         return NULL;
1732 }
1733
1734 static struct kvm_s390_interrupt_info *get_top_io_int(struct kvm *kvm,
1735                                                       u64 isc_mask, u32 schid)
1736 {
1737         struct kvm_s390_interrupt_info *inti = NULL;
1738         int isc;
1739
1740         for (isc = 0; isc <= MAX_ISC && !inti; isc++) {
1741                 if (isc_mask & isc_to_isc_bits(isc))
1742                         inti = get_io_int(kvm, isc, schid);
1743         }
1744         return inti;
1745 }
1746
1747 static int get_top_gisa_isc(struct kvm *kvm, u64 isc_mask, u32 schid)
1748 {
1749         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1750         unsigned long active_mask;
1751         int isc;
1752
1753         if (schid)
1754                 goto out;
1755         if (!gi->origin)
1756                 goto out;
1757
1758         active_mask = (isc_mask & gisa_get_ipm(gi->origin) << 24) << 32;
1759         while (active_mask) {
1760                 isc = __fls(active_mask) ^ (BITS_PER_LONG - 1);
1761                 if (gisa_tac_ipm_gisc(gi->origin, isc))
1762                         return isc;
1763                 clear_bit_inv(isc, &active_mask);
1764         }
1765 out:
1766         return -EINVAL;
1767 }
1768
1769 /*
1770  * Dequeue and return an I/O interrupt matching any of the interruption
1771  * subclasses as designated by the isc mask in cr6 and the schid (if != 0).
1772  * Take into account the interrupts pending in the interrupt list and in GISA.
1773  *
1774  * Note that for a guest that does not enable I/O interrupts
1775  * but relies on TPI, a flood of classic interrupts may starve
1776  * out adapter interrupts on the same isc. Linux does not do
1777  * that, and it is possible to work around the issue by configuring
1778  * different iscs for classic and adapter interrupts in the guest,
1779  * but we may want to revisit this in the future.
1780  */
1781 struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
1782                                                     u64 isc_mask, u32 schid)
1783 {
1784         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1785         struct kvm_s390_interrupt_info *inti, *tmp_inti;
1786         int isc;
1787
1788         inti = get_top_io_int(kvm, isc_mask, schid);
1789
1790         isc = get_top_gisa_isc(kvm, isc_mask, schid);
1791         if (isc < 0)
1792                 /* no AI in GISA */
1793                 goto out;
1794
1795         if (!inti)
1796                 /* AI in GISA but no classical IO int */
1797                 goto gisa_out;
1798
1799         /* both types of interrupts present */
1800         if (int_word_to_isc(inti->io.io_int_word) <= isc) {
1801                 /* classical IO int with higher priority */
1802                 gisa_set_ipm_gisc(gi->origin, isc);
1803                 goto out;
1804         }
1805 gisa_out:
1806         tmp_inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
1807         if (tmp_inti) {
1808                 tmp_inti->type = KVM_S390_INT_IO(1, 0, 0, 0);
1809                 tmp_inti->io.io_int_word = isc_to_int_word(isc);
1810                 if (inti)
1811                         kvm_s390_reinject_io_int(kvm, inti);
1812                 inti = tmp_inti;
1813         } else
1814                 gisa_set_ipm_gisc(gi->origin, isc);
1815 out:
1816         return inti;
1817 }
1818
1819 static int __inject_service(struct kvm *kvm,
1820                              struct kvm_s390_interrupt_info *inti)
1821 {
1822         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1823
1824         kvm->stat.inject_service_signal++;
1825         spin_lock(&fi->lock);
1826         fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING;
1827
1828         /* We always allow events, track them separately from the sccb ints */
1829         if (fi->srv_signal.ext_params & SCCB_EVENT_PENDING)
1830                 set_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
1831
1832         /*
1833          * Early versions of the QEMU s390 bios will inject several
1834          * service interrupts after another without handling a
1835          * condition code indicating busy.
1836          * We will silently ignore those superfluous sccb values.
1837          * A future version of QEMU will take care of serialization
1838          * of servc requests
1839          */
1840         if (fi->srv_signal.ext_params & SCCB_MASK)
1841                 goto out;
1842         fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK;
1843         set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
1844 out:
1845         spin_unlock(&fi->lock);
1846         kfree(inti);
1847         return 0;
1848 }
1849
1850 static int __inject_virtio(struct kvm *kvm,
1851                             struct kvm_s390_interrupt_info *inti)
1852 {
1853         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1854
1855         kvm->stat.inject_virtio++;
1856         spin_lock(&fi->lock);
1857         if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) {
1858                 spin_unlock(&fi->lock);
1859                 return -EBUSY;
1860         }
1861         fi->counters[FIRQ_CNTR_VIRTIO] += 1;
1862         list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]);
1863         set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
1864         spin_unlock(&fi->lock);
1865         return 0;
1866 }
1867
1868 static int __inject_pfault_done(struct kvm *kvm,
1869                                  struct kvm_s390_interrupt_info *inti)
1870 {
1871         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1872
1873         kvm->stat.inject_pfault_done++;
1874         spin_lock(&fi->lock);
1875         if (fi->counters[FIRQ_CNTR_PFAULT] >=
1876                 (ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) {
1877                 spin_unlock(&fi->lock);
1878                 return -EBUSY;
1879         }
1880         fi->counters[FIRQ_CNTR_PFAULT] += 1;
1881         list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]);
1882         set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
1883         spin_unlock(&fi->lock);
1884         return 0;
1885 }
1886
1887 #define CR_PENDING_SUBCLASS 28
1888 static int __inject_float_mchk(struct kvm *kvm,
1889                                 struct kvm_s390_interrupt_info *inti)
1890 {
1891         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1892
1893         kvm->stat.inject_float_mchk++;
1894         spin_lock(&fi->lock);
1895         fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS);
1896         fi->mchk.mcic |= inti->mchk.mcic;
1897         set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs);
1898         spin_unlock(&fi->lock);
1899         kfree(inti);
1900         return 0;
1901 }
1902
1903 static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1904 {
1905         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1906         struct kvm_s390_float_interrupt *fi;
1907         struct list_head *list;
1908         int isc;
1909
1910         kvm->stat.inject_io++;
1911         isc = int_word_to_isc(inti->io.io_int_word);
1912
1913         /*
1914          * We do not use the lock checking variant as this is just a
1915          * performance optimization and we do not hold the lock here.
1916          * This is ok as the code will pick interrupts from both "lists"
1917          * for delivery.
1918          */
1919         if (gi->origin && inti->type & KVM_S390_INT_IO_AI_MASK) {
1920                 VM_EVENT(kvm, 4, "%s isc %1u", "inject: I/O (AI/gisa)", isc);
1921                 gisa_set_ipm_gisc(gi->origin, isc);
1922                 kfree(inti);
1923                 return 0;
1924         }
1925
1926         fi = &kvm->arch.float_int;
1927         spin_lock(&fi->lock);
1928         if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) {
1929                 spin_unlock(&fi->lock);
1930                 return -EBUSY;
1931         }
1932         fi->counters[FIRQ_CNTR_IO] += 1;
1933
1934         if (inti->type & KVM_S390_INT_IO_AI_MASK)
1935                 VM_EVENT(kvm, 4, "%s", "inject: I/O (AI)");
1936         else
1937                 VM_EVENT(kvm, 4, "inject: I/O %x ss %x schid %04x",
1938                         inti->io.subchannel_id >> 8,
1939                         inti->io.subchannel_id >> 1 & 0x3,
1940                         inti->io.subchannel_nr);
1941         list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1942         list_add_tail(&inti->list, list);
1943         set_bit(isc_to_irq_type(isc), &fi->pending_irqs);
1944         spin_unlock(&fi->lock);
1945         return 0;
1946 }
1947
1948 /*
1949  * Find a destination VCPU for a floating irq and kick it.
1950  */
1951 static void __floating_irq_kick(struct kvm *kvm, u64 type)
1952 {
1953         struct kvm_vcpu *dst_vcpu;
1954         int sigcpu, online_vcpus, nr_tries = 0;
1955
1956         online_vcpus = atomic_read(&kvm->online_vcpus);
1957         if (!online_vcpus)
1958                 return;
1959
1960         /* find idle VCPUs first, then round robin */
1961         sigcpu = find_first_bit(kvm->arch.idle_mask, online_vcpus);
1962         if (sigcpu == online_vcpus) {
1963                 do {
1964                         sigcpu = kvm->arch.float_int.next_rr_cpu++;
1965                         kvm->arch.float_int.next_rr_cpu %= online_vcpus;
1966                         /* avoid endless loops if all vcpus are stopped */
1967                         if (nr_tries++ >= online_vcpus)
1968                                 return;
1969                 } while (is_vcpu_stopped(kvm_get_vcpu(kvm, sigcpu)));
1970         }
1971         dst_vcpu = kvm_get_vcpu(kvm, sigcpu);
1972
1973         /* make the VCPU drop out of the SIE, or wake it up if sleeping */
1974         switch (type) {
1975         case KVM_S390_MCHK:
1976                 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_STOP_INT);
1977                 break;
1978         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1979                 if (!(type & KVM_S390_INT_IO_AI_MASK &&
1980                       kvm->arch.gisa_int.origin) ||
1981                       kvm_s390_pv_cpu_get_handle(dst_vcpu))
1982                         kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_IO_INT);
1983                 break;
1984         default:
1985                 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_EXT_INT);
1986                 break;
1987         }
1988         kvm_s390_vcpu_wakeup(dst_vcpu);
1989 }
1990
1991 static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1992 {
1993         u64 type = READ_ONCE(inti->type);
1994         int rc;
1995
1996         switch (type) {
1997         case KVM_S390_MCHK:
1998                 rc = __inject_float_mchk(kvm, inti);
1999                 break;
2000         case KVM_S390_INT_VIRTIO:
2001                 rc = __inject_virtio(kvm, inti);
2002                 break;
2003         case KVM_S390_INT_SERVICE:
2004                 rc = __inject_service(kvm, inti);
2005                 break;
2006         case KVM_S390_INT_PFAULT_DONE:
2007                 rc = __inject_pfault_done(kvm, inti);
2008                 break;
2009         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2010                 rc = __inject_io(kvm, inti);
2011                 break;
2012         default:
2013                 rc = -EINVAL;
2014         }
2015         if (rc)
2016                 return rc;
2017
2018         __floating_irq_kick(kvm, type);
2019         return 0;
2020 }
2021
2022 int kvm_s390_inject_vm(struct kvm *kvm,
2023                        struct kvm_s390_interrupt *s390int)
2024 {
2025         struct kvm_s390_interrupt_info *inti;
2026         int rc;
2027
2028         inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
2029         if (!inti)
2030                 return -ENOMEM;
2031
2032         inti->type = s390int->type;
2033         switch (inti->type) {
2034         case KVM_S390_INT_VIRTIO:
2035                 VM_EVENT(kvm, 5, "inject: virtio parm:%x,parm64:%llx",
2036                          s390int->parm, s390int->parm64);
2037                 inti->ext.ext_params = s390int->parm;
2038                 inti->ext.ext_params2 = s390int->parm64;
2039                 break;
2040         case KVM_S390_INT_SERVICE:
2041                 VM_EVENT(kvm, 4, "inject: sclp parm:%x", s390int->parm);
2042                 inti->ext.ext_params = s390int->parm;
2043                 break;
2044         case KVM_S390_INT_PFAULT_DONE:
2045                 inti->ext.ext_params2 = s390int->parm64;
2046                 break;
2047         case KVM_S390_MCHK:
2048                 VM_EVENT(kvm, 3, "inject: machine check mcic 0x%llx",
2049                          s390int->parm64);
2050                 inti->mchk.cr14 = s390int->parm; /* upper bits are not used */
2051                 inti->mchk.mcic = s390int->parm64;
2052                 break;
2053         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2054                 inti->io.subchannel_id = s390int->parm >> 16;
2055                 inti->io.subchannel_nr = s390int->parm & 0x0000ffffu;
2056                 inti->io.io_int_parm = s390int->parm64 >> 32;
2057                 inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull;
2058                 break;
2059         default:
2060                 kfree(inti);
2061                 return -EINVAL;
2062         }
2063         trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64,
2064                                  2);
2065
2066         rc = __inject_vm(kvm, inti);
2067         if (rc)
2068                 kfree(inti);
2069         return rc;
2070 }
2071
2072 int kvm_s390_reinject_io_int(struct kvm *kvm,
2073                               struct kvm_s390_interrupt_info *inti)
2074 {
2075         return __inject_vm(kvm, inti);
2076 }
2077
2078 int s390int_to_s390irq(struct kvm_s390_interrupt *s390int,
2079                        struct kvm_s390_irq *irq)
2080 {
2081         irq->type = s390int->type;
2082         switch (irq->type) {
2083         case KVM_S390_PROGRAM_INT:
2084                 if (s390int->parm & 0xffff0000)
2085                         return -EINVAL;
2086                 irq->u.pgm.code = s390int->parm;
2087                 break;
2088         case KVM_S390_SIGP_SET_PREFIX:
2089                 irq->u.prefix.address = s390int->parm;
2090                 break;
2091         case KVM_S390_SIGP_STOP:
2092                 irq->u.stop.flags = s390int->parm;
2093                 break;
2094         case KVM_S390_INT_EXTERNAL_CALL:
2095                 if (s390int->parm & 0xffff0000)
2096                         return -EINVAL;
2097                 irq->u.extcall.code = s390int->parm;
2098                 break;
2099         case KVM_S390_INT_EMERGENCY:
2100                 if (s390int->parm & 0xffff0000)
2101                         return -EINVAL;
2102                 irq->u.emerg.code = s390int->parm;
2103                 break;
2104         case KVM_S390_MCHK:
2105                 irq->u.mchk.mcic = s390int->parm64;
2106                 break;
2107         case KVM_S390_INT_PFAULT_INIT:
2108                 irq->u.ext.ext_params = s390int->parm;
2109                 irq->u.ext.ext_params2 = s390int->parm64;
2110                 break;
2111         case KVM_S390_RESTART:
2112         case KVM_S390_INT_CLOCK_COMP:
2113         case KVM_S390_INT_CPU_TIMER:
2114                 break;
2115         default:
2116                 return -EINVAL;
2117         }
2118         return 0;
2119 }
2120
2121 int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu)
2122 {
2123         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2124
2125         return test_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
2126 }
2127
2128 int kvm_s390_is_restart_irq_pending(struct kvm_vcpu *vcpu)
2129 {
2130         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2131
2132         return test_bit(IRQ_PEND_RESTART, &li->pending_irqs);
2133 }
2134
2135 void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu)
2136 {
2137         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2138
2139         spin_lock(&li->lock);
2140         li->irq.stop.flags = 0;
2141         clear_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
2142         spin_unlock(&li->lock);
2143 }
2144
2145 static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
2146 {
2147         int rc;
2148
2149         switch (irq->type) {
2150         case KVM_S390_PROGRAM_INT:
2151                 rc = __inject_prog(vcpu, irq);
2152                 break;
2153         case KVM_S390_SIGP_SET_PREFIX:
2154                 rc = __inject_set_prefix(vcpu, irq);
2155                 break;
2156         case KVM_S390_SIGP_STOP:
2157                 rc = __inject_sigp_stop(vcpu, irq);
2158                 break;
2159         case KVM_S390_RESTART:
2160                 rc = __inject_sigp_restart(vcpu);
2161                 break;
2162         case KVM_S390_INT_CLOCK_COMP:
2163                 rc = __inject_ckc(vcpu);
2164                 break;
2165         case KVM_S390_INT_CPU_TIMER:
2166                 rc = __inject_cpu_timer(vcpu);
2167                 break;
2168         case KVM_S390_INT_EXTERNAL_CALL:
2169                 rc = __inject_extcall(vcpu, irq);
2170                 break;
2171         case KVM_S390_INT_EMERGENCY:
2172                 rc = __inject_sigp_emergency(vcpu, irq);
2173                 break;
2174         case KVM_S390_MCHK:
2175                 rc = __inject_mchk(vcpu, irq);
2176                 break;
2177         case KVM_S390_INT_PFAULT_INIT:
2178                 rc = __inject_pfault_init(vcpu, irq);
2179                 break;
2180         case KVM_S390_INT_VIRTIO:
2181         case KVM_S390_INT_SERVICE:
2182         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2183         default:
2184                 rc = -EINVAL;
2185         }
2186
2187         return rc;
2188 }
2189
2190 int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
2191 {
2192         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2193         int rc;
2194
2195         spin_lock(&li->lock);
2196         rc = do_inject_vcpu(vcpu, irq);
2197         spin_unlock(&li->lock);
2198         if (!rc)
2199                 kvm_s390_vcpu_wakeup(vcpu);
2200         return rc;
2201 }
2202
2203 static inline void clear_irq_list(struct list_head *_list)
2204 {
2205         struct kvm_s390_interrupt_info *inti, *n;
2206
2207         list_for_each_entry_safe(inti, n, _list, list) {
2208                 list_del(&inti->list);
2209                 kfree(inti);
2210         }
2211 }
2212
2213 static void inti_to_irq(struct kvm_s390_interrupt_info *inti,
2214                        struct kvm_s390_irq *irq)
2215 {
2216         irq->type = inti->type;
2217         switch (inti->type) {
2218         case KVM_S390_INT_PFAULT_INIT:
2219         case KVM_S390_INT_PFAULT_DONE:
2220         case KVM_S390_INT_VIRTIO:
2221                 irq->u.ext = inti->ext;
2222                 break;
2223         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2224                 irq->u.io = inti->io;
2225                 break;
2226         }
2227 }
2228
2229 void kvm_s390_clear_float_irqs(struct kvm *kvm)
2230 {
2231         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2232         int i;
2233
2234         mutex_lock(&kvm->lock);
2235         if (!kvm_s390_pv_is_protected(kvm))
2236                 fi->masked_irqs = 0;
2237         mutex_unlock(&kvm->lock);
2238         spin_lock(&fi->lock);
2239         fi->pending_irqs = 0;
2240         memset(&fi->srv_signal, 0, sizeof(fi->srv_signal));
2241         memset(&fi->mchk, 0, sizeof(fi->mchk));
2242         for (i = 0; i < FIRQ_LIST_COUNT; i++)
2243                 clear_irq_list(&fi->lists[i]);
2244         for (i = 0; i < FIRQ_MAX_COUNT; i++)
2245                 fi->counters[i] = 0;
2246         spin_unlock(&fi->lock);
2247         kvm_s390_gisa_clear(kvm);
2248 };
2249
2250 static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len)
2251 {
2252         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
2253         struct kvm_s390_interrupt_info *inti;
2254         struct kvm_s390_float_interrupt *fi;
2255         struct kvm_s390_irq *buf;
2256         struct kvm_s390_irq *irq;
2257         int max_irqs;
2258         int ret = 0;
2259         int n = 0;
2260         int i;
2261
2262         if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0)
2263                 return -EINVAL;
2264
2265         /*
2266          * We are already using -ENOMEM to signal
2267          * userspace it may retry with a bigger buffer,
2268          * so we need to use something else for this case
2269          */
2270         buf = vzalloc(len);
2271         if (!buf)
2272                 return -ENOBUFS;
2273
2274         max_irqs = len / sizeof(struct kvm_s390_irq);
2275
2276         if (gi->origin && gisa_get_ipm(gi->origin)) {
2277                 for (i = 0; i <= MAX_ISC; i++) {
2278                         if (n == max_irqs) {
2279                                 /* signal userspace to try again */
2280                                 ret = -ENOMEM;
2281                                 goto out_nolock;
2282                         }
2283                         if (gisa_tac_ipm_gisc(gi->origin, i)) {
2284                                 irq = (struct kvm_s390_irq *) &buf[n];
2285                                 irq->type = KVM_S390_INT_IO(1, 0, 0, 0);
2286                                 irq->u.io.io_int_word = isc_to_int_word(i);
2287                                 n++;
2288                         }
2289                 }
2290         }
2291         fi = &kvm->arch.float_int;
2292         spin_lock(&fi->lock);
2293         for (i = 0; i < FIRQ_LIST_COUNT; i++) {
2294                 list_for_each_entry(inti, &fi->lists[i], list) {
2295                         if (n == max_irqs) {
2296                                 /* signal userspace to try again */
2297                                 ret = -ENOMEM;
2298                                 goto out;
2299                         }
2300                         inti_to_irq(inti, &buf[n]);
2301                         n++;
2302                 }
2303         }
2304         if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs) ||
2305             test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs)) {
2306                 if (n == max_irqs) {
2307                         /* signal userspace to try again */
2308                         ret = -ENOMEM;
2309                         goto out;
2310                 }
2311                 irq = (struct kvm_s390_irq *) &buf[n];
2312                 irq->type = KVM_S390_INT_SERVICE;
2313                 irq->u.ext = fi->srv_signal;
2314                 n++;
2315         }
2316         if (test_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
2317                 if (n == max_irqs) {
2318                                 /* signal userspace to try again */
2319                                 ret = -ENOMEM;
2320                                 goto out;
2321                 }
2322                 irq = (struct kvm_s390_irq *) &buf[n];
2323                 irq->type = KVM_S390_MCHK;
2324                 irq->u.mchk = fi->mchk;
2325                 n++;
2326 }
2327
2328 out:
2329         spin_unlock(&fi->lock);
2330 out_nolock:
2331         if (!ret && n > 0) {
2332                 if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n))
2333                         ret = -EFAULT;
2334         }
2335         vfree(buf);
2336
2337         return ret < 0 ? ret : n;
2338 }
2339
2340 static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr)
2341 {
2342         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2343         struct kvm_s390_ais_all ais;
2344
2345         if (attr->attr < sizeof(ais))
2346                 return -EINVAL;
2347
2348         if (!test_kvm_facility(kvm, 72))
2349                 return -EOPNOTSUPP;
2350
2351         mutex_lock(&fi->ais_lock);
2352         ais.simm = fi->simm;
2353         ais.nimm = fi->nimm;
2354         mutex_unlock(&fi->ais_lock);
2355
2356         if (copy_to_user((void __user *)attr->addr, &ais, sizeof(ais)))
2357                 return -EFAULT;
2358
2359         return 0;
2360 }
2361
2362 static int flic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2363 {
2364         int r;
2365
2366         switch (attr->group) {
2367         case KVM_DEV_FLIC_GET_ALL_IRQS:
2368                 r = get_all_floating_irqs(dev->kvm, (u8 __user *) attr->addr,
2369                                           attr->attr);
2370                 break;
2371         case KVM_DEV_FLIC_AISM_ALL:
2372                 r = flic_ais_mode_get_all(dev->kvm, attr);
2373                 break;
2374         default:
2375                 r = -EINVAL;
2376         }
2377
2378         return r;
2379 }
2380
2381 static inline int copy_irq_from_user(struct kvm_s390_interrupt_info *inti,
2382                                      u64 addr)
2383 {
2384         struct kvm_s390_irq __user *uptr = (struct kvm_s390_irq __user *) addr;
2385         void *target = NULL;
2386         void __user *source;
2387         u64 size;
2388
2389         if (get_user(inti->type, (u64 __user *)addr))
2390                 return -EFAULT;
2391
2392         switch (inti->type) {
2393         case KVM_S390_INT_PFAULT_INIT:
2394         case KVM_S390_INT_PFAULT_DONE:
2395         case KVM_S390_INT_VIRTIO:
2396         case KVM_S390_INT_SERVICE:
2397                 target = (void *) &inti->ext;
2398                 source = &uptr->u.ext;
2399                 size = sizeof(inti->ext);
2400                 break;
2401         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2402                 target = (void *) &inti->io;
2403                 source = &uptr->u.io;
2404                 size = sizeof(inti->io);
2405                 break;
2406         case KVM_S390_MCHK:
2407                 target = (void *) &inti->mchk;
2408                 source = &uptr->u.mchk;
2409                 size = sizeof(inti->mchk);
2410                 break;
2411         default:
2412                 return -EINVAL;
2413         }
2414
2415         if (copy_from_user(target, source, size))
2416                 return -EFAULT;
2417
2418         return 0;
2419 }
2420
2421 static int enqueue_floating_irq(struct kvm_device *dev,
2422                                 struct kvm_device_attr *attr)
2423 {
2424         struct kvm_s390_interrupt_info *inti = NULL;
2425         int r = 0;
2426         int len = attr->attr;
2427
2428         if (len % sizeof(struct kvm_s390_irq) != 0)
2429                 return -EINVAL;
2430         else if (len > KVM_S390_FLIC_MAX_BUFFER)
2431                 return -EINVAL;
2432
2433         while (len >= sizeof(struct kvm_s390_irq)) {
2434                 inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
2435                 if (!inti)
2436                         return -ENOMEM;
2437
2438                 r = copy_irq_from_user(inti, attr->addr);
2439                 if (r) {
2440                         kfree(inti);
2441                         return r;
2442                 }
2443                 r = __inject_vm(dev->kvm, inti);
2444                 if (r) {
2445                         kfree(inti);
2446                         return r;
2447                 }
2448                 len -= sizeof(struct kvm_s390_irq);
2449                 attr->addr += sizeof(struct kvm_s390_irq);
2450         }
2451
2452         return r;
2453 }
2454
2455 static struct s390_io_adapter *get_io_adapter(struct kvm *kvm, unsigned int id)
2456 {
2457         if (id >= MAX_S390_IO_ADAPTERS)
2458                 return NULL;
2459         id = array_index_nospec(id, MAX_S390_IO_ADAPTERS);
2460         return kvm->arch.adapters[id];
2461 }
2462
2463 static int register_io_adapter(struct kvm_device *dev,
2464                                struct kvm_device_attr *attr)
2465 {
2466         struct s390_io_adapter *adapter;
2467         struct kvm_s390_io_adapter adapter_info;
2468
2469         if (copy_from_user(&adapter_info,
2470                            (void __user *)attr->addr, sizeof(adapter_info)))
2471                 return -EFAULT;
2472
2473         if (adapter_info.id >= MAX_S390_IO_ADAPTERS)
2474                 return -EINVAL;
2475
2476         adapter_info.id = array_index_nospec(adapter_info.id,
2477                                              MAX_S390_IO_ADAPTERS);
2478
2479         if (dev->kvm->arch.adapters[adapter_info.id] != NULL)
2480                 return -EINVAL;
2481
2482         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL_ACCOUNT);
2483         if (!adapter)
2484                 return -ENOMEM;
2485
2486         adapter->id = adapter_info.id;
2487         adapter->isc = adapter_info.isc;
2488         adapter->maskable = adapter_info.maskable;
2489         adapter->masked = false;
2490         adapter->swap = adapter_info.swap;
2491         adapter->suppressible = (adapter_info.flags) &
2492                                 KVM_S390_ADAPTER_SUPPRESSIBLE;
2493         dev->kvm->arch.adapters[adapter->id] = adapter;
2494
2495         return 0;
2496 }
2497
2498 int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked)
2499 {
2500         int ret;
2501         struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
2502
2503         if (!adapter || !adapter->maskable)
2504                 return -EINVAL;
2505         ret = adapter->masked;
2506         adapter->masked = masked;
2507         return ret;
2508 }
2509
2510 void kvm_s390_destroy_adapters(struct kvm *kvm)
2511 {
2512         int i;
2513
2514         for (i = 0; i < MAX_S390_IO_ADAPTERS; i++)
2515                 kfree(kvm->arch.adapters[i]);
2516 }
2517
2518 static int modify_io_adapter(struct kvm_device *dev,
2519                              struct kvm_device_attr *attr)
2520 {
2521         struct kvm_s390_io_adapter_req req;
2522         struct s390_io_adapter *adapter;
2523         int ret;
2524
2525         if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req)))
2526                 return -EFAULT;
2527
2528         adapter = get_io_adapter(dev->kvm, req.id);
2529         if (!adapter)
2530                 return -EINVAL;
2531         switch (req.type) {
2532         case KVM_S390_IO_ADAPTER_MASK:
2533                 ret = kvm_s390_mask_adapter(dev->kvm, req.id, req.mask);
2534                 if (ret > 0)
2535                         ret = 0;
2536                 break;
2537         /*
2538          * The following operations are no longer needed and therefore no-ops.
2539          * The gpa to hva translation is done when an IRQ route is set up. The
2540          * set_irq code uses get_user_pages_remote() to do the actual write.
2541          */
2542         case KVM_S390_IO_ADAPTER_MAP:
2543         case KVM_S390_IO_ADAPTER_UNMAP:
2544                 ret = 0;
2545                 break;
2546         default:
2547                 ret = -EINVAL;
2548         }
2549
2550         return ret;
2551 }
2552
2553 static int clear_io_irq(struct kvm *kvm, struct kvm_device_attr *attr)
2554
2555 {
2556         const u64 isc_mask = 0xffUL << 24; /* all iscs set */
2557         u32 schid;
2558
2559         if (attr->flags)
2560                 return -EINVAL;
2561         if (attr->attr != sizeof(schid))
2562                 return -EINVAL;
2563         if (copy_from_user(&schid, (void __user *) attr->addr, sizeof(schid)))
2564                 return -EFAULT;
2565         if (!schid)
2566                 return -EINVAL;
2567         kfree(kvm_s390_get_io_int(kvm, isc_mask, schid));
2568         /*
2569          * If userspace is conforming to the architecture, we can have at most
2570          * one pending I/O interrupt per subchannel, so this is effectively a
2571          * clear all.
2572          */
2573         return 0;
2574 }
2575
2576 static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr)
2577 {
2578         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2579         struct kvm_s390_ais_req req;
2580         int ret = 0;
2581
2582         if (!test_kvm_facility(kvm, 72))
2583                 return -EOPNOTSUPP;
2584
2585         if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req)))
2586                 return -EFAULT;
2587
2588         if (req.isc > MAX_ISC)
2589                 return -EINVAL;
2590
2591         trace_kvm_s390_modify_ais_mode(req.isc,
2592                                        (fi->simm & AIS_MODE_MASK(req.isc)) ?
2593                                        (fi->nimm & AIS_MODE_MASK(req.isc)) ?
2594                                        2 : KVM_S390_AIS_MODE_SINGLE :
2595                                        KVM_S390_AIS_MODE_ALL, req.mode);
2596
2597         mutex_lock(&fi->ais_lock);
2598         switch (req.mode) {
2599         case KVM_S390_AIS_MODE_ALL:
2600                 fi->simm &= ~AIS_MODE_MASK(req.isc);
2601                 fi->nimm &= ~AIS_MODE_MASK(req.isc);
2602                 break;
2603         case KVM_S390_AIS_MODE_SINGLE:
2604                 fi->simm |= AIS_MODE_MASK(req.isc);
2605                 fi->nimm &= ~AIS_MODE_MASK(req.isc);
2606                 break;
2607         default:
2608                 ret = -EINVAL;
2609         }
2610         mutex_unlock(&fi->ais_lock);
2611
2612         return ret;
2613 }
2614
2615 static int kvm_s390_inject_airq(struct kvm *kvm,
2616                                 struct s390_io_adapter *adapter)
2617 {
2618         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2619         struct kvm_s390_interrupt s390int = {
2620                 .type = KVM_S390_INT_IO(1, 0, 0, 0),
2621                 .parm = 0,
2622                 .parm64 = isc_to_int_word(adapter->isc),
2623         };
2624         int ret = 0;
2625
2626         if (!test_kvm_facility(kvm, 72) || !adapter->suppressible)
2627                 return kvm_s390_inject_vm(kvm, &s390int);
2628
2629         mutex_lock(&fi->ais_lock);
2630         if (fi->nimm & AIS_MODE_MASK(adapter->isc)) {
2631                 trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc);
2632                 goto out;
2633         }
2634
2635         ret = kvm_s390_inject_vm(kvm, &s390int);
2636         if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) {
2637                 fi->nimm |= AIS_MODE_MASK(adapter->isc);
2638                 trace_kvm_s390_modify_ais_mode(adapter->isc,
2639                                                KVM_S390_AIS_MODE_SINGLE, 2);
2640         }
2641 out:
2642         mutex_unlock(&fi->ais_lock);
2643         return ret;
2644 }
2645
2646 static int flic_inject_airq(struct kvm *kvm, struct kvm_device_attr *attr)
2647 {
2648         unsigned int id = attr->attr;
2649         struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
2650
2651         if (!adapter)
2652                 return -EINVAL;
2653
2654         return kvm_s390_inject_airq(kvm, adapter);
2655 }
2656
2657 static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr)
2658 {
2659         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2660         struct kvm_s390_ais_all ais;
2661
2662         if (!test_kvm_facility(kvm, 72))
2663                 return -EOPNOTSUPP;
2664
2665         if (copy_from_user(&ais, (void __user *)attr->addr, sizeof(ais)))
2666                 return -EFAULT;
2667
2668         mutex_lock(&fi->ais_lock);
2669         fi->simm = ais.simm;
2670         fi->nimm = ais.nimm;
2671         mutex_unlock(&fi->ais_lock);
2672
2673         return 0;
2674 }
2675
2676 static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2677 {
2678         int r = 0;
2679         unsigned long i;
2680         struct kvm_vcpu *vcpu;
2681
2682         switch (attr->group) {
2683         case KVM_DEV_FLIC_ENQUEUE:
2684                 r = enqueue_floating_irq(dev, attr);
2685                 break;
2686         case KVM_DEV_FLIC_CLEAR_IRQS:
2687                 kvm_s390_clear_float_irqs(dev->kvm);
2688                 break;
2689         case KVM_DEV_FLIC_APF_ENABLE:
2690                 dev->kvm->arch.gmap->pfault_enabled = 1;
2691                 break;
2692         case KVM_DEV_FLIC_APF_DISABLE_WAIT:
2693                 dev->kvm->arch.gmap->pfault_enabled = 0;
2694                 /*
2695                  * Make sure no async faults are in transition when
2696                  * clearing the queues. So we don't need to worry
2697                  * about late coming workers.
2698                  */
2699                 synchronize_srcu(&dev->kvm->srcu);
2700                 kvm_for_each_vcpu(i, vcpu, dev->kvm)
2701                         kvm_clear_async_pf_completion_queue(vcpu);
2702                 break;
2703         case KVM_DEV_FLIC_ADAPTER_REGISTER:
2704                 r = register_io_adapter(dev, attr);
2705                 break;
2706         case KVM_DEV_FLIC_ADAPTER_MODIFY:
2707                 r = modify_io_adapter(dev, attr);
2708                 break;
2709         case KVM_DEV_FLIC_CLEAR_IO_IRQ:
2710                 r = clear_io_irq(dev->kvm, attr);
2711                 break;
2712         case KVM_DEV_FLIC_AISM:
2713                 r = modify_ais_mode(dev->kvm, attr);
2714                 break;
2715         case KVM_DEV_FLIC_AIRQ_INJECT:
2716                 r = flic_inject_airq(dev->kvm, attr);
2717                 break;
2718         case KVM_DEV_FLIC_AISM_ALL:
2719                 r = flic_ais_mode_set_all(dev->kvm, attr);
2720                 break;
2721         default:
2722                 r = -EINVAL;
2723         }
2724
2725         return r;
2726 }
2727
2728 static int flic_has_attr(struct kvm_device *dev,
2729                              struct kvm_device_attr *attr)
2730 {
2731         switch (attr->group) {
2732         case KVM_DEV_FLIC_GET_ALL_IRQS:
2733         case KVM_DEV_FLIC_ENQUEUE:
2734         case KVM_DEV_FLIC_CLEAR_IRQS:
2735         case KVM_DEV_FLIC_APF_ENABLE:
2736         case KVM_DEV_FLIC_APF_DISABLE_WAIT:
2737         case KVM_DEV_FLIC_ADAPTER_REGISTER:
2738         case KVM_DEV_FLIC_ADAPTER_MODIFY:
2739         case KVM_DEV_FLIC_CLEAR_IO_IRQ:
2740         case KVM_DEV_FLIC_AISM:
2741         case KVM_DEV_FLIC_AIRQ_INJECT:
2742         case KVM_DEV_FLIC_AISM_ALL:
2743                 return 0;
2744         }
2745         return -ENXIO;
2746 }
2747
2748 static int flic_create(struct kvm_device *dev, u32 type)
2749 {
2750         if (!dev)
2751                 return -EINVAL;
2752         if (dev->kvm->arch.flic)
2753                 return -EINVAL;
2754         dev->kvm->arch.flic = dev;
2755         return 0;
2756 }
2757
2758 static void flic_destroy(struct kvm_device *dev)
2759 {
2760         dev->kvm->arch.flic = NULL;
2761         kfree(dev);
2762 }
2763
2764 /* s390 floating irq controller (flic) */
2765 struct kvm_device_ops kvm_flic_ops = {
2766         .name = "kvm-flic",
2767         .get_attr = flic_get_attr,
2768         .set_attr = flic_set_attr,
2769         .has_attr = flic_has_attr,
2770         .create = flic_create,
2771         .destroy = flic_destroy,
2772 };
2773
2774 static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap)
2775 {
2776         unsigned long bit;
2777
2778         bit = bit_nr + (addr % PAGE_SIZE) * 8;
2779
2780         return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit;
2781 }
2782
2783 static struct page *get_map_page(struct kvm *kvm, u64 uaddr)
2784 {
2785         struct page *page = NULL;
2786
2787         mmap_read_lock(kvm->mm);
2788         get_user_pages_remote(kvm->mm, uaddr, 1, FOLL_WRITE,
2789                               &page, NULL);
2790         mmap_read_unlock(kvm->mm);
2791         return page;
2792 }
2793
2794 static int adapter_indicators_set(struct kvm *kvm,
2795                                   struct s390_io_adapter *adapter,
2796                                   struct kvm_s390_adapter_int *adapter_int)
2797 {
2798         unsigned long bit;
2799         int summary_set, idx;
2800         struct page *ind_page, *summary_page;
2801         void *map;
2802
2803         ind_page = get_map_page(kvm, adapter_int->ind_addr);
2804         if (!ind_page)
2805                 return -1;
2806         summary_page = get_map_page(kvm, adapter_int->summary_addr);
2807         if (!summary_page) {
2808                 put_page(ind_page);
2809                 return -1;
2810         }
2811
2812         idx = srcu_read_lock(&kvm->srcu);
2813         map = page_address(ind_page);
2814         bit = get_ind_bit(adapter_int->ind_addr,
2815                           adapter_int->ind_offset, adapter->swap);
2816         set_bit(bit, map);
2817         mark_page_dirty(kvm, adapter_int->ind_addr >> PAGE_SHIFT);
2818         set_page_dirty_lock(ind_page);
2819         map = page_address(summary_page);
2820         bit = get_ind_bit(adapter_int->summary_addr,
2821                           adapter_int->summary_offset, adapter->swap);
2822         summary_set = test_and_set_bit(bit, map);
2823         mark_page_dirty(kvm, adapter_int->summary_addr >> PAGE_SHIFT);
2824         set_page_dirty_lock(summary_page);
2825         srcu_read_unlock(&kvm->srcu, idx);
2826
2827         put_page(ind_page);
2828         put_page(summary_page);
2829         return summary_set ? 0 : 1;
2830 }
2831
2832 /*
2833  * < 0 - not injected due to error
2834  * = 0 - coalesced, summary indicator already active
2835  * > 0 - injected interrupt
2836  */
2837 static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e,
2838                            struct kvm *kvm, int irq_source_id, int level,
2839                            bool line_status)
2840 {
2841         int ret;
2842         struct s390_io_adapter *adapter;
2843
2844         /* We're only interested in the 0->1 transition. */
2845         if (!level)
2846                 return 0;
2847         adapter = get_io_adapter(kvm, e->adapter.adapter_id);
2848         if (!adapter)
2849                 return -1;
2850         ret = adapter_indicators_set(kvm, adapter, &e->adapter);
2851         if ((ret > 0) && !adapter->masked) {
2852                 ret = kvm_s390_inject_airq(kvm, adapter);
2853                 if (ret == 0)
2854                         ret = 1;
2855         }
2856         return ret;
2857 }
2858
2859 /*
2860  * Inject the machine check to the guest.
2861  */
2862 void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu,
2863                                      struct mcck_volatile_info *mcck_info)
2864 {
2865         struct kvm_s390_interrupt_info inti;
2866         struct kvm_s390_irq irq;
2867         struct kvm_s390_mchk_info *mchk;
2868         union mci mci;
2869         __u64 cr14 = 0;         /* upper bits are not used */
2870         int rc;
2871
2872         mci.val = mcck_info->mcic;
2873         if (mci.sr)
2874                 cr14 |= CR14_RECOVERY_SUBMASK;
2875         if (mci.dg)
2876                 cr14 |= CR14_DEGRADATION_SUBMASK;
2877         if (mci.w)
2878                 cr14 |= CR14_WARNING_SUBMASK;
2879
2880         mchk = mci.ck ? &inti.mchk : &irq.u.mchk;
2881         mchk->cr14 = cr14;
2882         mchk->mcic = mcck_info->mcic;
2883         mchk->ext_damage_code = mcck_info->ext_damage_code;
2884         mchk->failing_storage_address = mcck_info->failing_storage_address;
2885         if (mci.ck) {
2886                 /* Inject the floating machine check */
2887                 inti.type = KVM_S390_MCHK;
2888                 rc = __inject_vm(vcpu->kvm, &inti);
2889         } else {
2890                 /* Inject the machine check to specified vcpu */
2891                 irq.type = KVM_S390_MCHK;
2892                 rc = kvm_s390_inject_vcpu(vcpu, &irq);
2893         }
2894         WARN_ON_ONCE(rc);
2895 }
2896
2897 int kvm_set_routing_entry(struct kvm *kvm,
2898                           struct kvm_kernel_irq_routing_entry *e,
2899                           const struct kvm_irq_routing_entry *ue)
2900 {
2901         u64 uaddr;
2902
2903         switch (ue->type) {
2904         /* we store the userspace addresses instead of the guest addresses */
2905         case KVM_IRQ_ROUTING_S390_ADAPTER:
2906                 e->set = set_adapter_int;
2907                 uaddr =  gmap_translate(kvm->arch.gmap, ue->u.adapter.summary_addr);
2908                 if (uaddr == -EFAULT)
2909                         return -EFAULT;
2910                 e->adapter.summary_addr = uaddr;
2911                 uaddr =  gmap_translate(kvm->arch.gmap, ue->u.adapter.ind_addr);
2912                 if (uaddr == -EFAULT)
2913                         return -EFAULT;
2914                 e->adapter.ind_addr = uaddr;
2915                 e->adapter.summary_offset = ue->u.adapter.summary_offset;
2916                 e->adapter.ind_offset = ue->u.adapter.ind_offset;
2917                 e->adapter.adapter_id = ue->u.adapter.adapter_id;
2918                 return 0;
2919         default:
2920                 return -EINVAL;
2921         }
2922 }
2923
2924 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
2925                 int irq_source_id, int level, bool line_status)
2926 {
2927         return -EINVAL;
2928 }
2929
2930 int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len)
2931 {
2932         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2933         struct kvm_s390_irq *buf;
2934         int r = 0;
2935         int n;
2936
2937         buf = vmalloc(len);
2938         if (!buf)
2939                 return -ENOMEM;
2940
2941         if (copy_from_user((void *) buf, irqstate, len)) {
2942                 r = -EFAULT;
2943                 goto out_free;
2944         }
2945
2946         /*
2947          * Don't allow setting the interrupt state
2948          * when there are already interrupts pending
2949          */
2950         spin_lock(&li->lock);
2951         if (li->pending_irqs) {
2952                 r = -EBUSY;
2953                 goto out_unlock;
2954         }
2955
2956         for (n = 0; n < len / sizeof(*buf); n++) {
2957                 r = do_inject_vcpu(vcpu, &buf[n]);
2958                 if (r)
2959                         break;
2960         }
2961
2962 out_unlock:
2963         spin_unlock(&li->lock);
2964 out_free:
2965         vfree(buf);
2966
2967         return r;
2968 }
2969
2970 static void store_local_irq(struct kvm_s390_local_interrupt *li,
2971                             struct kvm_s390_irq *irq,
2972                             unsigned long irq_type)
2973 {
2974         switch (irq_type) {
2975         case IRQ_PEND_MCHK_EX:
2976         case IRQ_PEND_MCHK_REP:
2977                 irq->type = KVM_S390_MCHK;
2978                 irq->u.mchk = li->irq.mchk;
2979                 break;
2980         case IRQ_PEND_PROG:
2981                 irq->type = KVM_S390_PROGRAM_INT;
2982                 irq->u.pgm = li->irq.pgm;
2983                 break;
2984         case IRQ_PEND_PFAULT_INIT:
2985                 irq->type = KVM_S390_INT_PFAULT_INIT;
2986                 irq->u.ext = li->irq.ext;
2987                 break;
2988         case IRQ_PEND_EXT_EXTERNAL:
2989                 irq->type = KVM_S390_INT_EXTERNAL_CALL;
2990                 irq->u.extcall = li->irq.extcall;
2991                 break;
2992         case IRQ_PEND_EXT_CLOCK_COMP:
2993                 irq->type = KVM_S390_INT_CLOCK_COMP;
2994                 break;
2995         case IRQ_PEND_EXT_CPU_TIMER:
2996                 irq->type = KVM_S390_INT_CPU_TIMER;
2997                 break;
2998         case IRQ_PEND_SIGP_STOP:
2999                 irq->type = KVM_S390_SIGP_STOP;
3000                 irq->u.stop = li->irq.stop;
3001                 break;
3002         case IRQ_PEND_RESTART:
3003                 irq->type = KVM_S390_RESTART;
3004                 break;
3005         case IRQ_PEND_SET_PREFIX:
3006                 irq->type = KVM_S390_SIGP_SET_PREFIX;
3007                 irq->u.prefix = li->irq.prefix;
3008                 break;
3009         }
3010 }
3011
3012 int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len)
3013 {
3014         int scn;
3015         DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS);
3016         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
3017         unsigned long pending_irqs;
3018         struct kvm_s390_irq irq;
3019         unsigned long irq_type;
3020         int cpuaddr;
3021         int n = 0;
3022
3023         spin_lock(&li->lock);
3024         pending_irqs = li->pending_irqs;
3025         memcpy(&sigp_emerg_pending, &li->sigp_emerg_pending,
3026                sizeof(sigp_emerg_pending));
3027         spin_unlock(&li->lock);
3028
3029         for_each_set_bit(irq_type, &pending_irqs, IRQ_PEND_COUNT) {
3030                 memset(&irq, 0, sizeof(irq));
3031                 if (irq_type == IRQ_PEND_EXT_EMERGENCY)
3032                         continue;
3033                 if (n + sizeof(irq) > len)
3034                         return -ENOBUFS;
3035                 store_local_irq(&vcpu->arch.local_int, &irq, irq_type);
3036                 if (copy_to_user(&buf[n], &irq, sizeof(irq)))
3037                         return -EFAULT;
3038                 n += sizeof(irq);
3039         }
3040
3041         if (test_bit(IRQ_PEND_EXT_EMERGENCY, &pending_irqs)) {
3042                 for_each_set_bit(cpuaddr, sigp_emerg_pending, KVM_MAX_VCPUS) {
3043                         memset(&irq, 0, sizeof(irq));
3044                         if (n + sizeof(irq) > len)
3045                                 return -ENOBUFS;
3046                         irq.type = KVM_S390_INT_EMERGENCY;
3047                         irq.u.emerg.code = cpuaddr;
3048                         if (copy_to_user(&buf[n], &irq, sizeof(irq)))
3049                                 return -EFAULT;
3050                         n += sizeof(irq);
3051                 }
3052         }
3053
3054         if (sca_ext_call_pending(vcpu, &scn)) {
3055                 if (n + sizeof(irq) > len)
3056                         return -ENOBUFS;
3057                 memset(&irq, 0, sizeof(irq));
3058                 irq.type = KVM_S390_INT_EXTERNAL_CALL;
3059                 irq.u.extcall.code = scn;
3060                 if (copy_to_user(&buf[n], &irq, sizeof(irq)))
3061                         return -EFAULT;
3062                 n += sizeof(irq);
3063         }
3064
3065         return n;
3066 }
3067
3068 static void __airqs_kick_single_vcpu(struct kvm *kvm, u8 deliverable_mask)
3069 {
3070         int vcpu_idx, online_vcpus = atomic_read(&kvm->online_vcpus);
3071         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3072         struct kvm_vcpu *vcpu;
3073         u8 vcpu_isc_mask;
3074
3075         for_each_set_bit(vcpu_idx, kvm->arch.idle_mask, online_vcpus) {
3076                 vcpu = kvm_get_vcpu(kvm, vcpu_idx);
3077                 if (psw_ioint_disabled(vcpu))
3078                         continue;
3079                 vcpu_isc_mask = (u8)(vcpu->arch.sie_block->gcr[6] >> 24);
3080                 if (deliverable_mask & vcpu_isc_mask) {
3081                         /* lately kicked but not yet running */
3082                         if (test_and_set_bit(vcpu_idx, gi->kicked_mask))
3083                                 return;
3084                         kvm_s390_vcpu_wakeup(vcpu);
3085                         return;
3086                 }
3087         }
3088 }
3089
3090 static enum hrtimer_restart gisa_vcpu_kicker(struct hrtimer *timer)
3091 {
3092         struct kvm_s390_gisa_interrupt *gi =
3093                 container_of(timer, struct kvm_s390_gisa_interrupt, timer);
3094         struct kvm *kvm =
3095                 container_of(gi->origin, struct sie_page2, gisa)->kvm;
3096         u8 pending_mask;
3097
3098         pending_mask = gisa_get_ipm_or_restore_iam(gi);
3099         if (pending_mask) {
3100                 __airqs_kick_single_vcpu(kvm, pending_mask);
3101                 hrtimer_forward_now(timer, ns_to_ktime(gi->expires));
3102                 return HRTIMER_RESTART;
3103         }
3104
3105         return HRTIMER_NORESTART;
3106 }
3107
3108 #define NULL_GISA_ADDR 0x00000000UL
3109 #define NONE_GISA_ADDR 0x00000001UL
3110 #define GISA_ADDR_MASK 0xfffff000UL
3111
3112 static void process_gib_alert_list(void)
3113 {
3114         struct kvm_s390_gisa_interrupt *gi;
3115         u32 final, gisa_phys, origin = 0UL;
3116         struct kvm_s390_gisa *gisa;
3117         struct kvm *kvm;
3118
3119         do {
3120                 /*
3121                  * If the NONE_GISA_ADDR is still stored in the alert list
3122                  * origin, we will leave the outer loop. No further GISA has
3123                  * been added to the alert list by millicode while processing
3124                  * the current alert list.
3125                  */
3126                 final = (origin & NONE_GISA_ADDR);
3127                 /*
3128                  * Cut off the alert list and store the NONE_GISA_ADDR in the
3129                  * alert list origin to avoid further GAL interruptions.
3130                  * A new alert list can be build up by millicode in parallel
3131                  * for guests not in the yet cut-off alert list. When in the
3132                  * final loop, store the NULL_GISA_ADDR instead. This will re-
3133                  * enable GAL interruptions on the host again.
3134                  */
3135                 origin = xchg(&gib->alert_list_origin,
3136                               (!final) ? NONE_GISA_ADDR : NULL_GISA_ADDR);
3137                 /*
3138                  * Loop through the just cut-off alert list and start the
3139                  * gisa timers to kick idle vcpus to consume the pending
3140                  * interruptions asap.
3141                  */
3142                 while (origin & GISA_ADDR_MASK) {
3143                         gisa_phys = origin;
3144                         gisa = phys_to_virt(gisa_phys);
3145                         origin = gisa->next_alert;
3146                         gisa->next_alert = gisa_phys;
3147                         kvm = container_of(gisa, struct sie_page2, gisa)->kvm;
3148                         gi = &kvm->arch.gisa_int;
3149                         if (hrtimer_active(&gi->timer))
3150                                 hrtimer_cancel(&gi->timer);
3151                         hrtimer_start(&gi->timer, 0, HRTIMER_MODE_REL);
3152                 }
3153         } while (!final);
3154
3155 }
3156
3157 void kvm_s390_gisa_clear(struct kvm *kvm)
3158 {
3159         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3160
3161         if (!gi->origin)
3162                 return;
3163         gisa_clear_ipm(gi->origin);
3164         VM_EVENT(kvm, 3, "gisa 0x%pK cleared", gi->origin);
3165 }
3166
3167 void kvm_s390_gisa_init(struct kvm *kvm)
3168 {
3169         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3170
3171         if (!css_general_characteristics.aiv)
3172                 return;
3173         gi->origin = &kvm->arch.sie_page2->gisa;
3174         gi->alert.mask = 0;
3175         spin_lock_init(&gi->alert.ref_lock);
3176         gi->expires = 50 * 1000; /* 50 usec */
3177         hrtimer_init(&gi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3178         gi->timer.function = gisa_vcpu_kicker;
3179         memset(gi->origin, 0, sizeof(struct kvm_s390_gisa));
3180         gi->origin->next_alert = (u32)virt_to_phys(gi->origin);
3181         VM_EVENT(kvm, 3, "gisa 0x%pK initialized", gi->origin);
3182 }
3183
3184 void kvm_s390_gisa_enable(struct kvm *kvm)
3185 {
3186         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3187         struct kvm_vcpu *vcpu;
3188         unsigned long i;
3189         u32 gisa_desc;
3190
3191         if (gi->origin)
3192                 return;
3193         kvm_s390_gisa_init(kvm);
3194         gisa_desc = kvm_s390_get_gisa_desc(kvm);
3195         if (!gisa_desc)
3196                 return;
3197         kvm_for_each_vcpu(i, vcpu, kvm) {
3198                 mutex_lock(&vcpu->mutex);
3199                 vcpu->arch.sie_block->gd = gisa_desc;
3200                 vcpu->arch.sie_block->eca |= ECA_AIV;
3201                 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
3202                            vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
3203                 mutex_unlock(&vcpu->mutex);
3204         }
3205 }
3206
3207 void kvm_s390_gisa_destroy(struct kvm *kvm)
3208 {
3209         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3210         struct kvm_s390_gisa *gisa = gi->origin;
3211
3212         if (!gi->origin)
3213                 return;
3214         WARN(gi->alert.mask != 0x00,
3215              "unexpected non zero alert.mask 0x%02x",
3216              gi->alert.mask);
3217         gi->alert.mask = 0x00;
3218         if (gisa_set_iam(gi->origin, gi->alert.mask))
3219                 process_gib_alert_list();
3220         hrtimer_cancel(&gi->timer);
3221         gi->origin = NULL;
3222         VM_EVENT(kvm, 3, "gisa 0x%pK destroyed", gisa);
3223 }
3224
3225 void kvm_s390_gisa_disable(struct kvm *kvm)
3226 {
3227         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3228         struct kvm_vcpu *vcpu;
3229         unsigned long i;
3230
3231         if (!gi->origin)
3232                 return;
3233         kvm_for_each_vcpu(i, vcpu, kvm) {
3234                 mutex_lock(&vcpu->mutex);
3235                 vcpu->arch.sie_block->eca &= ~ECA_AIV;
3236                 vcpu->arch.sie_block->gd = 0U;
3237                 mutex_unlock(&vcpu->mutex);
3238                 VCPU_EVENT(vcpu, 3, "AIV disabled for cpu %03u", vcpu->vcpu_id);
3239         }
3240         kvm_s390_gisa_destroy(kvm);
3241 }
3242
3243 /**
3244  * kvm_s390_gisc_register - register a guest ISC
3245  *
3246  * @kvm:  the kernel vm to work with
3247  * @gisc: the guest interruption sub class to register
3248  *
3249  * The function extends the vm specific alert mask to use.
3250  * The effective IAM mask in the GISA is updated as well
3251  * in case the GISA is not part of the GIB alert list.
3252  * It will be updated latest when the IAM gets restored
3253  * by gisa_get_ipm_or_restore_iam().
3254  *
3255  * Returns: the nonspecific ISC (NISC) the gib alert mechanism
3256  *          has registered with the channel subsystem.
3257  *          -ENODEV in case the vm uses no GISA
3258  *          -ERANGE in case the guest ISC is invalid
3259  */
3260 int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc)
3261 {
3262         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3263
3264         if (!gi->origin)
3265                 return -ENODEV;
3266         if (gisc > MAX_ISC)
3267                 return -ERANGE;
3268
3269         spin_lock(&gi->alert.ref_lock);
3270         gi->alert.ref_count[gisc]++;
3271         if (gi->alert.ref_count[gisc] == 1) {
3272                 gi->alert.mask |= 0x80 >> gisc;
3273                 gisa_set_iam(gi->origin, gi->alert.mask);
3274         }
3275         spin_unlock(&gi->alert.ref_lock);
3276
3277         return gib->nisc;
3278 }
3279 EXPORT_SYMBOL_GPL(kvm_s390_gisc_register);
3280
3281 /**
3282  * kvm_s390_gisc_unregister - unregister a guest ISC
3283  *
3284  * @kvm:  the kernel vm to work with
3285  * @gisc: the guest interruption sub class to register
3286  *
3287  * The function reduces the vm specific alert mask to use.
3288  * The effective IAM mask in the GISA is updated as well
3289  * in case the GISA is not part of the GIB alert list.
3290  * It will be updated latest when the IAM gets restored
3291  * by gisa_get_ipm_or_restore_iam().
3292  *
3293  * Returns: the nonspecific ISC (NISC) the gib alert mechanism
3294  *          has registered with the channel subsystem.
3295  *          -ENODEV in case the vm uses no GISA
3296  *          -ERANGE in case the guest ISC is invalid
3297  *          -EINVAL in case the guest ISC is not registered
3298  */
3299 int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc)
3300 {
3301         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3302         int rc = 0;
3303
3304         if (!gi->origin)
3305                 return -ENODEV;
3306         if (gisc > MAX_ISC)
3307                 return -ERANGE;
3308
3309         spin_lock(&gi->alert.ref_lock);
3310         if (gi->alert.ref_count[gisc] == 0) {
3311                 rc = -EINVAL;
3312                 goto out;
3313         }
3314         gi->alert.ref_count[gisc]--;
3315         if (gi->alert.ref_count[gisc] == 0) {
3316                 gi->alert.mask &= ~(0x80 >> gisc);
3317                 gisa_set_iam(gi->origin, gi->alert.mask);
3318         }
3319 out:
3320         spin_unlock(&gi->alert.ref_lock);
3321
3322         return rc;
3323 }
3324 EXPORT_SYMBOL_GPL(kvm_s390_gisc_unregister);
3325
3326 static void aen_host_forward(unsigned long si)
3327 {
3328         struct kvm_s390_gisa_interrupt *gi;
3329         struct zpci_gaite *gaite;
3330         struct kvm *kvm;
3331
3332         gaite = (struct zpci_gaite *)aift->gait +
3333                 (si * sizeof(struct zpci_gaite));
3334         if (gaite->count == 0)
3335                 return;
3336         if (gaite->aisb != 0)
3337                 set_bit_inv(gaite->aisbo, phys_to_virt(gaite->aisb));
3338
3339         kvm = kvm_s390_pci_si_to_kvm(aift, si);
3340         if (!kvm)
3341                 return;
3342         gi = &kvm->arch.gisa_int;
3343
3344         if (!(gi->origin->g1.simm & AIS_MODE_MASK(gaite->gisc)) ||
3345             !(gi->origin->g1.nimm & AIS_MODE_MASK(gaite->gisc))) {
3346                 gisa_set_ipm_gisc(gi->origin, gaite->gisc);
3347                 if (hrtimer_active(&gi->timer))
3348                         hrtimer_cancel(&gi->timer);
3349                 hrtimer_start(&gi->timer, 0, HRTIMER_MODE_REL);
3350                 kvm->stat.aen_forward++;
3351         }
3352 }
3353
3354 static void aen_process_gait(u8 isc)
3355 {
3356         bool found = false, first = true;
3357         union zpci_sic_iib iib = {{0}};
3358         unsigned long si, flags;
3359
3360         spin_lock_irqsave(&aift->gait_lock, flags);
3361
3362         if (!aift->gait) {
3363                 spin_unlock_irqrestore(&aift->gait_lock, flags);
3364                 return;
3365         }
3366
3367         for (si = 0;;) {
3368                 /* Scan adapter summary indicator bit vector */
3369                 si = airq_iv_scan(aift->sbv, si, airq_iv_end(aift->sbv));
3370                 if (si == -1UL) {
3371                         if (first || found) {
3372                                 /* Re-enable interrupts. */
3373                                 zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, isc,
3374                                                   &iib);
3375                                 first = found = false;
3376                         } else {
3377                                 /* Interrupts on and all bits processed */
3378                                 break;
3379                         }
3380                         found = false;
3381                         si = 0;
3382                         /* Scan again after re-enabling interrupts */
3383                         continue;
3384                 }
3385                 found = true;
3386                 aen_host_forward(si);
3387         }
3388
3389         spin_unlock_irqrestore(&aift->gait_lock, flags);
3390 }
3391
3392 static void gib_alert_irq_handler(struct airq_struct *airq,
3393                                   struct tpi_info *tpi_info)
3394 {
3395         struct tpi_adapter_info *info = (struct tpi_adapter_info *)tpi_info;
3396
3397         inc_irq_stat(IRQIO_GAL);
3398
3399         if ((info->forward || info->error) &&
3400             IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) {
3401                 aen_process_gait(info->isc);
3402                 if (info->aism != 0)
3403                         process_gib_alert_list();
3404         } else {
3405                 process_gib_alert_list();
3406         }
3407 }
3408
3409 static struct airq_struct gib_alert_irq = {
3410         .handler = gib_alert_irq_handler,
3411 };
3412
3413 void kvm_s390_gib_destroy(void)
3414 {
3415         if (!gib)
3416                 return;
3417         if (kvm_s390_pci_interp_allowed() && aift) {
3418                 mutex_lock(&aift->aift_lock);
3419                 kvm_s390_pci_aen_exit();
3420                 mutex_unlock(&aift->aift_lock);
3421         }
3422         chsc_sgib(0);
3423         unregister_adapter_interrupt(&gib_alert_irq);
3424         free_page((unsigned long)gib);
3425         gib = NULL;
3426 }
3427
3428 int __init kvm_s390_gib_init(u8 nisc)
3429 {
3430         u32 gib_origin;
3431         int rc = 0;
3432
3433         if (!css_general_characteristics.aiv) {
3434                 KVM_EVENT(3, "%s", "gib not initialized, no AIV facility");
3435                 goto out;
3436         }
3437
3438         gib = (struct kvm_s390_gib *)get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA);
3439         if (!gib) {
3440                 rc = -ENOMEM;
3441                 goto out;
3442         }
3443
3444         gib_alert_irq.isc = nisc;
3445         if (register_adapter_interrupt(&gib_alert_irq)) {
3446                 pr_err("Registering the GIB alert interruption handler failed\n");
3447                 rc = -EIO;
3448                 goto out_free_gib;
3449         }
3450         /* adapter interrupts used for AP (applicable here) don't use the LSI */
3451         *gib_alert_irq.lsi_ptr = 0xff;
3452
3453         gib->nisc = nisc;
3454         gib_origin = virt_to_phys(gib);
3455         if (chsc_sgib(gib_origin)) {
3456                 pr_err("Associating the GIB with the AIV facility failed\n");
3457                 free_page((unsigned long)gib);
3458                 gib = NULL;
3459                 rc = -EIO;
3460                 goto out_unreg_gal;
3461         }
3462
3463         if (kvm_s390_pci_interp_allowed()) {
3464                 if (kvm_s390_pci_aen_init(nisc)) {
3465                         pr_err("Initializing AEN for PCI failed\n");
3466                         rc = -EIO;
3467                         goto out_unreg_gal;
3468                 }
3469         }
3470
3471         KVM_EVENT(3, "gib 0x%pK (nisc=%d) initialized", gib, gib->nisc);
3472         goto out;
3473
3474 out_unreg_gal:
3475         unregister_adapter_interrupt(&gib_alert_irq);
3476 out_free_gib:
3477         free_page((unsigned long)gib);
3478         gib = NULL;
3479 out:
3480         return rc;
3481 }