s390/kvm/interrupt.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * handling kvm guest interrupts
   4  *
   5  * Copyright IBM Corp. 2008, 2020
   6  *
   7  *    Author(s): Carsten Otte <cotte@de.ibm.com>
   8  */
   9
  10 #define KMSG_COMPONENT "kvm-s390"
  11 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  12
  13 #include <linux/interrupt.h>
  14 #include <linux/kvm_host.h>
  15 #include <linux/hrtimer.h>
  16 #include <linux/mmu_context.h>
  17 #include <linux/nospec.h>
  18 #include <linux/signal.h>
  19 #include <linux/slab.h>
  20 #include <linux/bitmap.h>
  21 #include <linux/vmalloc.h>
  22 #include <asm/asm-offsets.h>
  23 #include <asm/dis.h>
  24 #include <linux/uaccess.h>
  25 #include <asm/sclp.h>
  26 #include <asm/isc.h>
  27 #include <asm/gmap.h>
  28 #include <asm/switch_to.h>
  29 #include <asm/nmi.h>
  30 #include <asm/airq.h>
  31 #include <asm/tpi.h>
  32 #include "kvm-s390.h"
  33 #include "gaccess.h"
  34 #include "trace-s390.h"
  35 #include "pci.h"
  36
  37 #define PFAULT_INIT 0x0600
  38 #define PFAULT_DONE 0x0680
  39 #define VIRTIO_PARAM 0x0d00
  40
  41 static struct kvm_s390_gib *gib;
  42
  43 /* handle external calls via sigp interpretation facility */
  44 static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id)
  45 {
  46         int c, scn;
  47
  48         if (!kvm_s390_test_cpuflags(vcpu, CPUSTAT_ECALL_PEND))
  49                 return 0;
  50
  51         BUG_ON(!kvm_s390_use_sca_entries());
  52         read_lock(&vcpu->kvm->arch.sca_lock);
  53         if (vcpu->kvm->arch.use_esca) {
  54                 struct esca_block *sca = vcpu->kvm->arch.sca;
  55                 union esca_sigp_ctrl sigp_ctrl =
  56                         sca->cpu[vcpu->vcpu_id].sigp_ctrl;
  57
  58                 c = sigp_ctrl.c;
  59                 scn = sigp_ctrl.scn;
  60         } else {
  61                 struct bsca_block *sca = vcpu->kvm->arch.sca;
  62                 union bsca_sigp_ctrl sigp_ctrl =
  63                         sca->cpu[vcpu->vcpu_id].sigp_ctrl;
  64
  65                 c = sigp_ctrl.c;
  66                 scn = sigp_ctrl.scn;
  67         }
  68         read_unlock(&vcpu->kvm->arch.sca_lock);
  69
  70         if (src_id)
  71                 *src_id = scn;
  72
  73         return c;
  74 }
  75
  76 static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id)
  77 {
  78         int expect, rc;
  79
  80         BUG_ON(!kvm_s390_use_sca_entries());
  81         read_lock(&vcpu->kvm->arch.sca_lock);
  82         if (vcpu->kvm->arch.use_esca) {
  83                 struct esca_block *sca = vcpu->kvm->arch.sca;
  84                 union esca_sigp_ctrl *sigp_ctrl =
  85                         &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
  86                 union esca_sigp_ctrl new_val = {0}, old_val;
  87
  88                 old_val = READ_ONCE(*sigp_ctrl);
  89                 new_val.scn = src_id;
  90                 new_val.c = 1;
  91                 old_val.c = 0;
  92
  93                 expect = old_val.value;
  94                 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
  95         } else {
  96                 struct bsca_block *sca = vcpu->kvm->arch.sca;
  97                 union bsca_sigp_ctrl *sigp_ctrl =
  98                         &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
  99                 union bsca_sigp_ctrl new_val = {0}, old_val;
 100
 101                 old_val = READ_ONCE(*sigp_ctrl);
 102                 new_val.scn = src_id;
 103                 new_val.c = 1;
 104                 old_val.c = 0;
 105
 106                 expect = old_val.value;
 107                 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
 108         }
 109         read_unlock(&vcpu->kvm->arch.sca_lock);
 110
 111         if (rc != expect) {
 112                 /* another external call is pending */
 113                 return -EBUSY;
 114         }
 115         kvm_s390_set_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
 116         return 0;
 117 }
 118
 119 static void sca_clear_ext_call(struct kvm_vcpu *vcpu)
 120 {
 121         int rc, expect;
 122
 123         if (!kvm_s390_use_sca_entries())
 124                 return;
 125         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
 126         read_lock(&vcpu->kvm->arch.sca_lock);
 127         if (vcpu->kvm->arch.use_esca) {
 128                 struct esca_block *sca = vcpu->kvm->arch.sca;
 129                 union esca_sigp_ctrl *sigp_ctrl =
 130                         &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
 131                 union esca_sigp_ctrl old;
 132
 133                 old = READ_ONCE(*sigp_ctrl);
 134                 expect = old.value;
 135                 rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
 136         } else {
 137                 struct bsca_block *sca = vcpu->kvm->arch.sca;
 138                 union bsca_sigp_ctrl *sigp_ctrl =
 139                         &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
 140                 union bsca_sigp_ctrl old;
 141
 142                 old = READ_ONCE(*sigp_ctrl);
 143                 expect = old.value;
 144                 rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
 145         }
 146         read_unlock(&vcpu->kvm->arch.sca_lock);
 147         WARN_ON(rc != expect); /* cannot clear? */
 148 }
 149
 150 int psw_extint_disabled(struct kvm_vcpu *vcpu)
 151 {
 152         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT);
 153 }
 154
 155 static int psw_ioint_disabled(struct kvm_vcpu *vcpu)
 156 {
 157         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO);
 158 }
 159
 160 static int psw_mchk_disabled(struct kvm_vcpu *vcpu)
 161 {
 162         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_MCHECK);
 163 }
 164
 165 static int psw_interrupts_disabled(struct kvm_vcpu *vcpu)
 166 {
 167         return psw_extint_disabled(vcpu) &&
 168                psw_ioint_disabled(vcpu) &&
 169                psw_mchk_disabled(vcpu);
 170 }
 171
 172 static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu)
 173 {
 174         if (psw_extint_disabled(vcpu) ||
 175             !(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
 176                 return 0;
 177         if (guestdbg_enabled(vcpu) && guestdbg_sstep_enabled(vcpu))
 178                 /* No timer interrupts when single stepping */
 179                 return 0;
 180         return 1;
 181 }
 182
 183 static int ckc_irq_pending(struct kvm_vcpu *vcpu)
 184 {
 185         const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
 186         const u64 ckc = vcpu->arch.sie_block->ckc;
 187
 188         if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
 189                 if ((s64)ckc >= (s64)now)
 190                         return 0;
 191         } else if (ckc >= now) {
 192                 return 0;
 193         }
 194         return ckc_interrupts_enabled(vcpu);
 195 }
 196
 197 static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu)
 198 {
 199         return !psw_extint_disabled(vcpu) &&
 200                (vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK);
 201 }
 202
 203 static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu)
 204 {
 205         if (!cpu_timer_interrupts_enabled(vcpu))
 206                 return 0;
 207         return kvm_s390_get_cpu_timer(vcpu) >> 63;
 208 }
 209
 210 static uint64_t isc_to_isc_bits(int isc)
 211 {
 212         return (0x80 >> isc) << 24;
 213 }
 214
 215 static inline u32 isc_to_int_word(u8 isc)
 216 {
 217         return ((u32)isc << 27) | 0x80000000;
 218 }
 219
 220 static inline u8 int_word_to_isc(u32 int_word)
 221 {
 222         return (int_word & 0x38000000) >> 27;
 223 }
 224
 225 /*
 226  * To use atomic bitmap functions, we have to provide a bitmap address
 227  * that is u64 aligned. However, the ipm might be u32 aligned.
 228  * Therefore, we logically start the bitmap at the very beginning of the
 229  * struct and fixup the bit number.
 230  */
 231 #define IPM_BIT_OFFSET (offsetof(struct kvm_s390_gisa, ipm) * BITS_PER_BYTE)
 232
 233 /**
 234  * gisa_set_iam - change the GISA interruption alert mask
 235  *
 236  * @gisa: gisa to operate on
 237  * @iam: new IAM value to use
 238  *
 239  * Change the IAM atomically with the next alert address and the IPM
 240  * of the GISA if the GISA is not part of the GIB alert list. All three
 241  * fields are located in the first long word of the GISA.
 242  *
 243  * Returns: 0 on success
 244  *          -EBUSY in case the gisa is part of the alert list
 245  */
 246 static inline int gisa_set_iam(struct kvm_s390_gisa *gisa, u8 iam)
 247 {
 248         u64 word, _word;
 249
 250         do {
 251                 word = READ_ONCE(gisa->u64.word[0]);
 252                 if ((u64)gisa != word >> 32)
 253                         return -EBUSY;
 254                 _word = (word & ~0xffUL) | iam;
 255         } while (cmpxchg(&gisa->u64.word[0], word, _word) != word);
 256
 257         return 0;
 258 }
 259
 260 /**
 261  * gisa_clear_ipm - clear the GISA interruption pending mask
 262  *
 263  * @gisa: gisa to operate on
 264  *
 265  * Clear the IPM atomically with the next alert address and the IAM
 266  * of the GISA unconditionally. All three fields are located in the
 267  * first long word of the GISA.
 268  */
 269 static inline void gisa_clear_ipm(struct kvm_s390_gisa *gisa)
 270 {
 271         u64 word, _word;
 272
 273         do {
 274                 word = READ_ONCE(gisa->u64.word[0]);
 275                 _word = word & ~(0xffUL << 24);
 276         } while (cmpxchg(&gisa->u64.word[0], word, _word) != word);
 277 }
 278
 279 /**
 280  * gisa_get_ipm_or_restore_iam - return IPM or restore GISA IAM
 281  *
 282  * @gi: gisa interrupt struct to work on
 283  *
 284  * Atomically restores the interruption alert mask if none of the
 285  * relevant ISCs are pending and return the IPM.
 286  *
 287  * Returns: the relevant pending ISCs
 288  */
 289 static inline u8 gisa_get_ipm_or_restore_iam(struct kvm_s390_gisa_interrupt *gi)
 290 {
 291         u8 pending_mask, alert_mask;
 292         u64 word, _word;
 293
 294         do {
 295                 word = READ_ONCE(gi->origin->u64.word[0]);
 296                 alert_mask = READ_ONCE(gi->alert.mask);
 297                 pending_mask = (u8)(word >> 24) & alert_mask;
 298                 if (pending_mask)
 299                         return pending_mask;
 300                 _word = (word & ~0xffUL) | alert_mask;
 301         } while (cmpxchg(&gi->origin->u64.word[0], word, _word) != word);
 302
 303         return 0;
 304 }
 305
 306 static inline int gisa_in_alert_list(struct kvm_s390_gisa *gisa)
 307 {
 308         return READ_ONCE(gisa->next_alert) != (u32)(u64)gisa;
 309 }
 310
 311 static inline void gisa_set_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
 312 {
 313         set_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
 314 }
 315
 316 static inline u8 gisa_get_ipm(struct kvm_s390_gisa *gisa)
 317 {
 318         return READ_ONCE(gisa->ipm);
 319 }
 320
 321 static inline void gisa_clear_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
 322 {
 323         clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
 324 }
 325
 326 static inline int gisa_tac_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
 327 {
 328         return test_and_clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
 329 }
 330
 331 static inline unsigned long pending_irqs_no_gisa(struct kvm_vcpu *vcpu)
 332 {
 333         unsigned long pending = vcpu->kvm->arch.float_int.pending_irqs |
 334                                 vcpu->arch.local_int.pending_irqs;
 335
 336         pending &= ~vcpu->kvm->arch.float_int.masked_irqs;
 337         return pending;
 338 }
 339
 340 static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu)
 341 {
 342         struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
 343         unsigned long pending_mask;
 344
 345         pending_mask = pending_irqs_no_gisa(vcpu);
 346         if (gi->origin)
 347                 pending_mask |= gisa_get_ipm(gi->origin) << IRQ_PEND_IO_ISC_7;
 348         return pending_mask;
 349 }
 350
 351 static inline int isc_to_irq_type(unsigned long isc)
 352 {
 353         return IRQ_PEND_IO_ISC_0 - isc;
 354 }
 355
 356 static inline int irq_type_to_isc(unsigned long irq_type)
 357 {
 358         return IRQ_PEND_IO_ISC_0 - irq_type;
 359 }
 360
 361 static unsigned long disable_iscs(struct kvm_vcpu *vcpu,
 362                                    unsigned long active_mask)
 363 {
 364         int i;
 365
 366         for (i = 0; i <= MAX_ISC; i++)
 367                 if (!(vcpu->arch.sie_block->gcr[6] & isc_to_isc_bits(i)))
 368                         active_mask &= ~(1UL << (isc_to_irq_type(i)));
 369
 370         return active_mask;
 371 }
 372
 373 static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu)
 374 {
 375         unsigned long active_mask;
 376
 377         active_mask = pending_irqs(vcpu);
 378         if (!active_mask)
 379                 return 0;
 380
 381         if (psw_extint_disabled(vcpu))
 382                 active_mask &= ~IRQ_PEND_EXT_MASK;
 383         if (psw_ioint_disabled(vcpu))
 384                 active_mask &= ~IRQ_PEND_IO_MASK;
 385         else
 386                 active_mask = disable_iscs(vcpu, active_mask);
 387         if (!(vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
 388                 __clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask);
 389         if (!(vcpu->arch.sie_block->gcr[0] & CR0_EMERGENCY_SIGNAL_SUBMASK))
 390                 __clear_bit(IRQ_PEND_EXT_EMERGENCY, &active_mask);
 391         if (!(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
 392                 __clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask);
 393         if (!(vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK))
 394                 __clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask);
 395         if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK)) {
 396                 __clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask);
 397                 __clear_bit(IRQ_PEND_EXT_SERVICE_EV, &active_mask);
 398         }
 399         if (psw_mchk_disabled(vcpu))
 400                 active_mask &= ~IRQ_PEND_MCHK_MASK;
 401         /* PV guest cpus can have a single interruption injected at a time. */
 402         if (kvm_s390_pv_cpu_get_handle(vcpu) &&
 403             vcpu->arch.sie_block->iictl != IICTL_CODE_NONE)
 404                 active_mask &= ~(IRQ_PEND_EXT_II_MASK |
 405                                  IRQ_PEND_IO_MASK |
 406                                  IRQ_PEND_MCHK_MASK);
 407         /*
 408          * Check both floating and local interrupt's cr14 because
 409          * bit IRQ_PEND_MCHK_REP could be set in both cases.
 410          */
 411         if (!(vcpu->arch.sie_block->gcr[14] &
 412            (vcpu->kvm->arch.float_int.mchk.cr14 |
 413            vcpu->arch.local_int.irq.mchk.cr14)))
 414                 __clear_bit(IRQ_PEND_MCHK_REP, &active_mask);
 415
 416         /*
 417          * STOP irqs will never be actively delivered. They are triggered via
 418          * intercept requests and cleared when the stop intercept is performed.
 419          */
 420         __clear_bit(IRQ_PEND_SIGP_STOP, &active_mask);
 421
 422         return active_mask;
 423 }
 424
 425 static void __set_cpu_idle(struct kvm_vcpu *vcpu)
 426 {
 427         kvm_s390_set_cpuflags(vcpu, CPUSTAT_WAIT);
 428         set_bit(vcpu->vcpu_idx, vcpu->kvm->arch.idle_mask);
 429 }
 430
 431 static void __unset_cpu_idle(struct kvm_vcpu *vcpu)
 432 {
 433         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_WAIT);
 434         clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.idle_mask);
 435 }
 436
 437 static void __reset_intercept_indicators(struct kvm_vcpu *vcpu)
 438 {
 439         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IO_INT | CPUSTAT_EXT_INT |
 440                                       CPUSTAT_STOP_INT);
 441         vcpu->arch.sie_block->lctl = 0x0000;
 442         vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT);
 443
 444         if (guestdbg_enabled(vcpu)) {
 445                 vcpu->arch.sie_block->lctl |= (LCTL_CR0 | LCTL_CR9 |
 446                                                LCTL_CR10 | LCTL_CR11);
 447                 vcpu->arch.sie_block->ictl |= (ICTL_STCTL | ICTL_PINT);
 448         }
 449 }
 450
 451 static void set_intercept_indicators_io(struct kvm_vcpu *vcpu)
 452 {
 453         if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_IO_MASK))
 454                 return;
 455         if (psw_ioint_disabled(vcpu))
 456                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IO_INT);
 457         else
 458                 vcpu->arch.sie_block->lctl |= LCTL_CR6;
 459 }
 460
 461 static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu)
 462 {
 463         if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_EXT_MASK))
 464                 return;
 465         if (psw_extint_disabled(vcpu))
 466                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
 467         else
 468                 vcpu->arch.sie_block->lctl |= LCTL_CR0;
 469 }
 470
 471 static void set_intercept_indicators_mchk(struct kvm_vcpu *vcpu)
 472 {
 473         if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_MCHK_MASK))
 474                 return;
 475         if (psw_mchk_disabled(vcpu))
 476                 vcpu->arch.sie_block->ictl |= ICTL_LPSW;
 477         else
 478                 vcpu->arch.sie_block->lctl |= LCTL_CR14;
 479 }
 480
 481 static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu)
 482 {
 483         if (kvm_s390_is_stop_irq_pending(vcpu))
 484                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
 485 }
 486
 487 /* Set interception request for non-deliverable interrupts */
 488 static void set_intercept_indicators(struct kvm_vcpu *vcpu)
 489 {
 490         set_intercept_indicators_io(vcpu);
 491         set_intercept_indicators_ext(vcpu);
 492         set_intercept_indicators_mchk(vcpu);
 493         set_intercept_indicators_stop(vcpu);
 494 }
 495
 496 static int __must_check __deliver_cpu_timer(struct kvm_vcpu *vcpu)
 497 {
 498         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 499         int rc = 0;
 500
 501         vcpu->stat.deliver_cputm++;
 502         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
 503                                          0, 0);
 504         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 505                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 506                 vcpu->arch.sie_block->eic = EXT_IRQ_CPU_TIMER;
 507         } else {
 508                 rc  = put_guest_lc(vcpu, EXT_IRQ_CPU_TIMER,
 509                                    (u16 *)__LC_EXT_INT_CODE);
 510                 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
 511                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 512                                      &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 513                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 514                                     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 515         }
 516         clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
 517         return rc ? -EFAULT : 0;
 518 }
 519
 520 static int __must_check __deliver_ckc(struct kvm_vcpu *vcpu)
 521 {
 522         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 523         int rc = 0;
 524
 525         vcpu->stat.deliver_ckc++;
 526         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
 527                                          0, 0);
 528         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 529                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 530                 vcpu->arch.sie_block->eic = EXT_IRQ_CLK_COMP;
 531         } else {
 532                 rc  = put_guest_lc(vcpu, EXT_IRQ_CLK_COMP,
 533                                    (u16 __user *)__LC_EXT_INT_CODE);
 534                 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
 535                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 536                                      &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 537                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 538                                     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 539         }
 540         clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
 541         return rc ? -EFAULT : 0;
 542 }
 543
 544 static int __must_check __deliver_pfault_init(struct kvm_vcpu *vcpu)
 545 {
 546         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 547         struct kvm_s390_ext_info ext;
 548         int rc;
 549
 550         spin_lock(&li->lock);
 551         ext = li->irq.ext;
 552         clear_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
 553         li->irq.ext.ext_params2 = 0;
 554         spin_unlock(&li->lock);
 555
 556         VCPU_EVENT(vcpu, 4, "deliver: pfault init token 0x%llx",
 557                    ext.ext_params2);
 558         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 559                                          KVM_S390_INT_PFAULT_INIT,
 560                                          0, ext.ext_params2);
 561
 562         rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *) __LC_EXT_INT_CODE);
 563         rc |= put_guest_lc(vcpu, PFAULT_INIT, (u16 *) __LC_EXT_CPU_ADDR);
 564         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 565                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 566         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 567                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 568         rc |= put_guest_lc(vcpu, ext.ext_params2, (u64 *) __LC_EXT_PARAMS2);
 569         return rc ? -EFAULT : 0;
 570 }
 571
 572 static int __write_machine_check(struct kvm_vcpu *vcpu,
 573                                  struct kvm_s390_mchk_info *mchk)
 574 {
 575         unsigned long ext_sa_addr;
 576         unsigned long lc;
 577         freg_t fprs[NUM_FPRS];
 578         union mci mci;
 579         int rc;
 580
 581         /*
 582          * All other possible payload for a machine check (e.g. the register
 583          * contents in the save area) will be handled by the ultravisor, as
 584          * the hypervisor does not not have the needed information for
 585          * protected guests.
 586          */
 587         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 588                 vcpu->arch.sie_block->iictl = IICTL_CODE_MCHK;
 589                 vcpu->arch.sie_block->mcic = mchk->mcic;
 590                 vcpu->arch.sie_block->faddr = mchk->failing_storage_address;
 591                 vcpu->arch.sie_block->edc = mchk->ext_damage_code;
 592                 return 0;
 593         }
 594
 595         mci.val = mchk->mcic;
 596         /* take care of lazy register loading */
 597         save_fpu_regs();
 598         save_access_regs(vcpu->run->s.regs.acrs);
 599         if (MACHINE_HAS_GS && vcpu->arch.gs_enabled)
 600                 save_gs_cb(current->thread.gs_cb);
 601
 602         /* Extended save area */
 603         rc = read_guest_lc(vcpu, __LC_MCESAD, &ext_sa_addr,
 604                            sizeof(unsigned long));
 605         /* Only bits 0 through 63-LC are used for address formation */
 606         lc = ext_sa_addr & MCESA_LC_MASK;
 607         if (test_kvm_facility(vcpu->kvm, 133)) {
 608                 switch (lc) {
 609                 case 0:
 610                 case 10:
 611                         ext_sa_addr &= ~0x3ffUL;
 612                         break;
 613                 case 11:
 614                         ext_sa_addr &= ~0x7ffUL;
 615                         break;
 616                 case 12:
 617                         ext_sa_addr &= ~0xfffUL;
 618                         break;
 619                 default:
 620                         ext_sa_addr = 0;
 621                         break;
 622                 }
 623         } else {
 624                 ext_sa_addr &= ~0x3ffUL;
 625         }
 626
 627         if (!rc && mci.vr && ext_sa_addr && test_kvm_facility(vcpu->kvm, 129)) {
 628                 if (write_guest_abs(vcpu, ext_sa_addr, vcpu->run->s.regs.vrs,
 629                                     512))
 630                         mci.vr = 0;
 631         } else {
 632                 mci.vr = 0;
 633         }
 634         if (!rc && mci.gs && ext_sa_addr && test_kvm_facility(vcpu->kvm, 133)
 635             && (lc == 11 || lc == 12)) {
 636                 if (write_guest_abs(vcpu, ext_sa_addr + 1024,
 637                                     &vcpu->run->s.regs.gscb, 32))
 638                         mci.gs = 0;
 639         } else {
 640                 mci.gs = 0;
 641         }
 642
 643         /* General interruption information */
 644         rc |= put_guest_lc(vcpu, 1, (u8 __user *) __LC_AR_MODE_ID);
 645         rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW,
 646                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 647         rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW,
 648                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 649         rc |= put_guest_lc(vcpu, mci.val, (u64 __user *) __LC_MCCK_CODE);
 650
 651         /* Register-save areas */
 652         if (MACHINE_HAS_VX) {
 653                 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
 654                 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, fprs, 128);
 655         } else {
 656                 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA,
 657                                      vcpu->run->s.regs.fprs, 128);
 658         }
 659         rc |= write_guest_lc(vcpu, __LC_GPREGS_SAVE_AREA,
 660                              vcpu->run->s.regs.gprs, 128);
 661         rc |= put_guest_lc(vcpu, current->thread.fpu.fpc,
 662                            (u32 __user *) __LC_FP_CREG_SAVE_AREA);
 663         rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->todpr,
 664                            (u32 __user *) __LC_TOD_PROGREG_SAVE_AREA);
 665         rc |= put_guest_lc(vcpu, kvm_s390_get_cpu_timer(vcpu),
 666                            (u64 __user *) __LC_CPU_TIMER_SAVE_AREA);
 667         rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->ckc >> 8,
 668                            (u64 __user *) __LC_CLOCK_COMP_SAVE_AREA);
 669         rc |= write_guest_lc(vcpu, __LC_AREGS_SAVE_AREA,
 670                              &vcpu->run->s.regs.acrs, 64);
 671         rc |= write_guest_lc(vcpu, __LC_CREGS_SAVE_AREA,
 672                              &vcpu->arch.sie_block->gcr, 128);
 673
 674         /* Extended interruption information */
 675         rc |= put_guest_lc(vcpu, mchk->ext_damage_code,
 676                            (u32 __user *) __LC_EXT_DAMAGE_CODE);
 677         rc |= put_guest_lc(vcpu, mchk->failing_storage_address,
 678                            (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR);
 679         rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA, &mchk->fixed_logout,
 680                              sizeof(mchk->fixed_logout));
 681         return rc ? -EFAULT : 0;
 682 }
 683
 684 static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu)
 685 {
 686         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
 687         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 688         struct kvm_s390_mchk_info mchk = {};
 689         int deliver = 0;
 690         int rc = 0;
 691
 692         spin_lock(&fi->lock);
 693         spin_lock(&li->lock);
 694         if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) ||
 695             test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) {
 696                 /*
 697                  * If there was an exigent machine check pending, then any
 698                  * repressible machine checks that might have been pending
 699                  * are indicated along with it, so always clear bits for
 700                  * repressible and exigent interrupts
 701                  */
 702                 mchk = li->irq.mchk;
 703                 clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
 704                 clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs);
 705                 memset(&li->irq.mchk, 0, sizeof(mchk));
 706                 deliver = 1;
 707         }
 708         /*
 709          * We indicate floating repressible conditions along with
 710          * other pending conditions. Channel Report Pending and Channel
 711          * Subsystem damage are the only two and are indicated by
 712          * bits in mcic and masked in cr14.
 713          */
 714         if (test_and_clear_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
 715                 mchk.mcic |= fi->mchk.mcic;
 716                 mchk.cr14 |= fi->mchk.cr14;
 717                 memset(&fi->mchk, 0, sizeof(mchk));
 718                 deliver = 1;
 719         }
 720         spin_unlock(&li->lock);
 721         spin_unlock(&fi->lock);
 722
 723         if (deliver) {
 724                 VCPU_EVENT(vcpu, 3, "deliver: machine check mcic 0x%llx",
 725                            mchk.mcic);
 726                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 727                                                  KVM_S390_MCHK,
 728                                                  mchk.cr14, mchk.mcic);
 729                 vcpu->stat.deliver_machine_check++;
 730                 rc = __write_machine_check(vcpu, &mchk);
 731         }
 732         return rc;
 733 }
 734
 735 static int __must_check __deliver_restart(struct kvm_vcpu *vcpu)
 736 {
 737         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 738         int rc = 0;
 739
 740         VCPU_EVENT(vcpu, 3, "%s", "deliver: cpu restart");
 741         vcpu->stat.deliver_restart_signal++;
 742         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);
 743
 744         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 745                 vcpu->arch.sie_block->iictl = IICTL_CODE_RESTART;
 746         } else {
 747                 rc  = write_guest_lc(vcpu,
 748                                      offsetof(struct lowcore, restart_old_psw),
 749                                      &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 750                 rc |= read_guest_lc(vcpu, offsetof(struct lowcore, restart_psw),
 751                                     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 752         }
 753         clear_bit(IRQ_PEND_RESTART, &li->pending_irqs);
 754         return rc ? -EFAULT : 0;
 755 }
 756
 757 static int __must_check __deliver_set_prefix(struct kvm_vcpu *vcpu)
 758 {
 759         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 760         struct kvm_s390_prefix_info prefix;
 761
 762         spin_lock(&li->lock);
 763         prefix = li->irq.prefix;
 764         li->irq.prefix.address = 0;
 765         clear_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
 766         spin_unlock(&li->lock);
 767
 768         vcpu->stat.deliver_prefix_signal++;
 769         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 770                                          KVM_S390_SIGP_SET_PREFIX,
 771                                          prefix.address, 0);
 772
 773         kvm_s390_set_prefix(vcpu, prefix.address);
 774         return 0;
 775 }
 776
 777 static int __must_check __deliver_emergency_signal(struct kvm_vcpu *vcpu)
 778 {
 779         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 780         int rc;
 781         int cpu_addr;
 782
 783         spin_lock(&li->lock);
 784         cpu_addr = find_first_bit(li->sigp_emerg_pending, KVM_MAX_VCPUS);
 785         clear_bit(cpu_addr, li->sigp_emerg_pending);
 786         if (bitmap_empty(li->sigp_emerg_pending, KVM_MAX_VCPUS))
 787                 clear_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
 788         spin_unlock(&li->lock);
 789
 790         VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp emerg");
 791         vcpu->stat.deliver_emergency_signal++;
 792         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
 793                                          cpu_addr, 0);
 794         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 795                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 796                 vcpu->arch.sie_block->eic = EXT_IRQ_EMERGENCY_SIG;
 797                 vcpu->arch.sie_block->extcpuaddr = cpu_addr;
 798                 return 0;
 799         }
 800
 801         rc  = put_guest_lc(vcpu, EXT_IRQ_EMERGENCY_SIG,
 802                            (u16 *)__LC_EXT_INT_CODE);
 803         rc |= put_guest_lc(vcpu, cpu_addr, (u16 *)__LC_EXT_CPU_ADDR);
 804         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 805                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 806         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 807                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 808         return rc ? -EFAULT : 0;
 809 }
 810
 811 static int __must_check __deliver_external_call(struct kvm_vcpu *vcpu)
 812 {
 813         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 814         struct kvm_s390_extcall_info extcall;
 815         int rc;
 816
 817         spin_lock(&li->lock);
 818         extcall = li->irq.extcall;
 819         li->irq.extcall.code = 0;
 820         clear_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
 821         spin_unlock(&li->lock);
 822
 823         VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp ext call");
 824         vcpu->stat.deliver_external_call++;
 825         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 826                                          KVM_S390_INT_EXTERNAL_CALL,
 827                                          extcall.code, 0);
 828         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 829                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 830                 vcpu->arch.sie_block->eic = EXT_IRQ_EXTERNAL_CALL;
 831                 vcpu->arch.sie_block->extcpuaddr = extcall.code;
 832                 return 0;
 833         }
 834
 835         rc  = put_guest_lc(vcpu, EXT_IRQ_EXTERNAL_CALL,
 836                            (u16 *)__LC_EXT_INT_CODE);
 837         rc |= put_guest_lc(vcpu, extcall.code, (u16 *)__LC_EXT_CPU_ADDR);
 838         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 839                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 840         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, &vcpu->arch.sie_block->gpsw,
 841                             sizeof(psw_t));
 842         return rc ? -EFAULT : 0;
 843 }
 844
 845 static int __deliver_prog_pv(struct kvm_vcpu *vcpu, u16 code)
 846 {
 847         switch (code) {
 848         case PGM_SPECIFICATION:
 849                 vcpu->arch.sie_block->iictl = IICTL_CODE_SPECIFICATION;
 850                 break;
 851         case PGM_OPERAND:
 852                 vcpu->arch.sie_block->iictl = IICTL_CODE_OPERAND;
 853                 break;
 854         default:
 855                 return -EINVAL;
 856         }
 857         return 0;
 858 }
 859
 860 static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
 861 {
 862         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 863         struct kvm_s390_pgm_info pgm_info;
 864         int rc = 0, nullifying = false;
 865         u16 ilen;
 866
 867         spin_lock(&li->lock);
 868         pgm_info = li->irq.pgm;
 869         clear_bit(IRQ_PEND_PROG, &li->pending_irqs);
 870         memset(&li->irq.pgm, 0, sizeof(pgm_info));
 871         spin_unlock(&li->lock);
 872
 873         ilen = pgm_info.flags & KVM_S390_PGM_FLAGS_ILC_MASK;
 874         VCPU_EVENT(vcpu, 3, "deliver: program irq code 0x%x, ilen:%d",
 875                    pgm_info.code, ilen);
 876         vcpu->stat.deliver_program++;
 877         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
 878                                          pgm_info.code, 0);
 879
 880         /* PER is handled by the ultravisor */
 881         if (kvm_s390_pv_cpu_is_protected(vcpu))
 882                 return __deliver_prog_pv(vcpu, pgm_info.code & ~PGM_PER);
 883
 884         switch (pgm_info.code & ~PGM_PER) {
 885         case PGM_AFX_TRANSLATION:
 886         case PGM_ASX_TRANSLATION:
 887         case PGM_EX_TRANSLATION:
 888         case PGM_LFX_TRANSLATION:
 889         case PGM_LSTE_SEQUENCE:
 890         case PGM_LSX_TRANSLATION:
 891         case PGM_LX_TRANSLATION:
 892         case PGM_PRIMARY_AUTHORITY:
 893         case PGM_SECONDARY_AUTHORITY:
 894                 nullifying = true;
 895                 fallthrough;
 896         case PGM_SPACE_SWITCH:
 897                 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
 898                                   (u64 *)__LC_TRANS_EXC_CODE);
 899                 break;
 900         case PGM_ALEN_TRANSLATION:
 901         case PGM_ALE_SEQUENCE:
 902         case PGM_ASTE_INSTANCE:
 903         case PGM_ASTE_SEQUENCE:
 904         case PGM_ASTE_VALIDITY:
 905         case PGM_EXTENDED_AUTHORITY:
 906                 rc = put_guest_lc(vcpu, pgm_info.exc_access_id,
 907                                   (u8 *)__LC_EXC_ACCESS_ID);
 908                 nullifying = true;
 909                 break;
 910         case PGM_ASCE_TYPE:
 911         case PGM_PAGE_TRANSLATION:
 912         case PGM_REGION_FIRST_TRANS:
 913         case PGM_REGION_SECOND_TRANS:
 914         case PGM_REGION_THIRD_TRANS:
 915         case PGM_SEGMENT_TRANSLATION:
 916                 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
 917                                   (u64 *)__LC_TRANS_EXC_CODE);
 918                 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
 919                                    (u8 *)__LC_EXC_ACCESS_ID);
 920                 rc |= put_guest_lc(vcpu, pgm_info.op_access_id,
 921                                    (u8 *)__LC_OP_ACCESS_ID);
 922                 nullifying = true;
 923                 break;
 924         case PGM_MONITOR:
 925                 rc = put_guest_lc(vcpu, pgm_info.mon_class_nr,
 926                                   (u16 *)__LC_MON_CLASS_NR);
 927                 rc |= put_guest_lc(vcpu, pgm_info.mon_code,
 928                                    (u64 *)__LC_MON_CODE);
 929                 break;
 930         case PGM_VECTOR_PROCESSING:
 931         case PGM_DATA:
 932                 rc = put_guest_lc(vcpu, pgm_info.data_exc_code,
 933                                   (u32 *)__LC_DATA_EXC_CODE);
 934                 break;
 935         case PGM_PROTECTION:
 936                 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
 937                                   (u64 *)__LC_TRANS_EXC_CODE);
 938                 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
 939                                    (u8 *)__LC_EXC_ACCESS_ID);
 940                 break;
 941         case PGM_STACK_FULL:
 942         case PGM_STACK_EMPTY:
 943         case PGM_STACK_SPECIFICATION:
 944         case PGM_STACK_TYPE:
 945         case PGM_STACK_OPERATION:
 946         case PGM_TRACE_TABEL:
 947         case PGM_CRYPTO_OPERATION:
 948                 nullifying = true;
 949                 break;
 950         }
 951
 952         if (pgm_info.code & PGM_PER) {
 953                 rc |= put_guest_lc(vcpu, pgm_info.per_code,
 954                                    (u8 *) __LC_PER_CODE);
 955                 rc |= put_guest_lc(vcpu, pgm_info.per_atmid,
 956                                    (u8 *)__LC_PER_ATMID);
 957                 rc |= put_guest_lc(vcpu, pgm_info.per_address,
 958                                    (u64 *) __LC_PER_ADDRESS);
 959                 rc |= put_guest_lc(vcpu, pgm_info.per_access_id,
 960                                    (u8 *) __LC_PER_ACCESS_ID);
 961         }
 962
 963         if (nullifying && !(pgm_info.flags & KVM_S390_PGM_FLAGS_NO_REWIND))
 964                 kvm_s390_rewind_psw(vcpu, ilen);
 965
 966         /* bit 1+2 of the target are the ilc, so we can directly use ilen */
 967         rc |= put_guest_lc(vcpu, ilen, (u16 *) __LC_PGM_ILC);
 968         rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->gbea,
 969                                  (u64 *) __LC_PGM_LAST_BREAK);
 970         rc |= put_guest_lc(vcpu, pgm_info.code,
 971                            (u16 *)__LC_PGM_INT_CODE);
 972         rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW,
 973                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 974         rc |= read_guest_lc(vcpu, __LC_PGM_NEW_PSW,
 975                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 976         return rc ? -EFAULT : 0;
 977 }
 978
 979 #define SCCB_MASK 0xFFFFFFF8
 980 #define SCCB_EVENT_PENDING 0x3
 981
 982 static int write_sclp(struct kvm_vcpu *vcpu, u32 parm)
 983 {
 984         int rc;
 985
 986         if (kvm_s390_pv_cpu_get_handle(vcpu)) {
 987                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 988                 vcpu->arch.sie_block->eic = EXT_IRQ_SERVICE_SIG;
 989                 vcpu->arch.sie_block->eiparams = parm;
 990                 return 0;
 991         }
 992
 993         rc  = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE);
 994         rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
 995         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 996                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 997         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 998                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 999         rc |= put_guest_lc(vcpu, parm,
1000                            (u32 *)__LC_EXT_PARAMS);
1001
1002         return rc ? -EFAULT : 0;
1003 }
1004
1005 static int __must_check __deliver_service(struct kvm_vcpu *vcpu)
1006 {
1007         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1008         struct kvm_s390_ext_info ext;
1009
1010         spin_lock(&fi->lock);
1011         if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs) ||
1012             !(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) {
1013                 spin_unlock(&fi->lock);
1014                 return 0;
1015         }
1016         ext = fi->srv_signal;
1017         memset(&fi->srv_signal, 0, sizeof(ext));
1018         clear_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
1019         clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
1020         if (kvm_s390_pv_cpu_is_protected(vcpu))
1021                 set_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs);
1022         spin_unlock(&fi->lock);
1023
1024         VCPU_EVENT(vcpu, 4, "deliver: sclp parameter 0x%x",
1025                    ext.ext_params);
1026         vcpu->stat.deliver_service_signal++;
1027         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
1028                                          ext.ext_params, 0);
1029
1030         return write_sclp(vcpu, ext.ext_params);
1031 }
1032
1033 static int __must_check __deliver_service_ev(struct kvm_vcpu *vcpu)
1034 {
1035         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1036         struct kvm_s390_ext_info ext;
1037
1038         spin_lock(&fi->lock);
1039         if (!(test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs))) {
1040                 spin_unlock(&fi->lock);
1041                 return 0;
1042         }
1043         ext = fi->srv_signal;
1044         /* only clear the event bit */
1045         fi->srv_signal.ext_params &= ~SCCB_EVENT_PENDING;
1046         clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
1047         spin_unlock(&fi->lock);
1048
1049         VCPU_EVENT(vcpu, 4, "%s", "deliver: sclp parameter event");
1050         vcpu->stat.deliver_service_signal++;
1051         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
1052                                          ext.ext_params, 0);
1053
1054         return write_sclp(vcpu, SCCB_EVENT_PENDING);
1055 }
1056
1057 static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu)
1058 {
1059         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1060         struct kvm_s390_interrupt_info *inti;
1061         int rc = 0;
1062
1063         spin_lock(&fi->lock);
1064         inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT],
1065                                         struct kvm_s390_interrupt_info,
1066                                         list);
1067         if (inti) {
1068                 list_del(&inti->list);
1069                 fi->counters[FIRQ_CNTR_PFAULT] -= 1;
1070         }
1071         if (list_empty(&fi->lists[FIRQ_LIST_PFAULT]))
1072                 clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
1073         spin_unlock(&fi->lock);
1074
1075         if (inti) {
1076                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1077                                                  KVM_S390_INT_PFAULT_DONE, 0,
1078                                                  inti->ext.ext_params2);
1079                 VCPU_EVENT(vcpu, 4, "deliver: pfault done token 0x%llx",
1080                            inti->ext.ext_params2);
1081
1082                 rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
1083                                 (u16 *)__LC_EXT_INT_CODE);
1084                 rc |= put_guest_lc(vcpu, PFAULT_DONE,
1085                                 (u16 *)__LC_EXT_CPU_ADDR);
1086                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
1087                                 &vcpu->arch.sie_block->gpsw,
1088                                 sizeof(psw_t));
1089                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
1090                                 &vcpu->arch.sie_block->gpsw,
1091                                 sizeof(psw_t));
1092                 rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
1093                                 (u64 *)__LC_EXT_PARAMS2);
1094                 kfree(inti);
1095         }
1096         return rc ? -EFAULT : 0;
1097 }
1098
1099 static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu)
1100 {
1101         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1102         struct kvm_s390_interrupt_info *inti;
1103         int rc = 0;
1104
1105         spin_lock(&fi->lock);
1106         inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO],
1107                                         struct kvm_s390_interrupt_info,
1108                                         list);
1109         if (inti) {
1110                 VCPU_EVENT(vcpu, 4,
1111                            "deliver: virtio parm: 0x%x,parm64: 0x%llx",
1112                            inti->ext.ext_params, inti->ext.ext_params2);
1113                 vcpu->stat.deliver_virtio++;
1114                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1115                                 inti->type,
1116                                 inti->ext.ext_params,
1117                                 inti->ext.ext_params2);
1118                 list_del(&inti->list);
1119                 fi->counters[FIRQ_CNTR_VIRTIO] -= 1;
1120         }
1121         if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO]))
1122                 clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
1123         spin_unlock(&fi->lock);
1124
1125         if (inti) {
1126                 rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
1127                                 (u16 *)__LC_EXT_INT_CODE);
1128                 rc |= put_guest_lc(vcpu, VIRTIO_PARAM,
1129                                 (u16 *)__LC_EXT_CPU_ADDR);
1130                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
1131                                 &vcpu->arch.sie_block->gpsw,
1132                                 sizeof(psw_t));
1133                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
1134                                 &vcpu->arch.sie_block->gpsw,
1135                                 sizeof(psw_t));
1136                 rc |= put_guest_lc(vcpu, inti->ext.ext_params,
1137                                 (u32 *)__LC_EXT_PARAMS);
1138                 rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
1139                                 (u64 *)__LC_EXT_PARAMS2);
1140                 kfree(inti);
1141         }
1142         return rc ? -EFAULT : 0;
1143 }
1144
1145 static int __do_deliver_io(struct kvm_vcpu *vcpu, struct kvm_s390_io_info *io)
1146 {
1147         int rc;
1148
1149         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
1150                 vcpu->arch.sie_block->iictl = IICTL_CODE_IO;
1151                 vcpu->arch.sie_block->subchannel_id = io->subchannel_id;
1152                 vcpu->arch.sie_block->subchannel_nr = io->subchannel_nr;
1153                 vcpu->arch.sie_block->io_int_parm = io->io_int_parm;
1154                 vcpu->arch.sie_block->io_int_word = io->io_int_word;
1155                 return 0;
1156         }
1157
1158         rc  = put_guest_lc(vcpu, io->subchannel_id, (u16 *)__LC_SUBCHANNEL_ID);
1159         rc |= put_guest_lc(vcpu, io->subchannel_nr, (u16 *)__LC_SUBCHANNEL_NR);
1160         rc |= put_guest_lc(vcpu, io->io_int_parm, (u32 *)__LC_IO_INT_PARM);
1161         rc |= put_guest_lc(vcpu, io->io_int_word, (u32 *)__LC_IO_INT_WORD);
1162         rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW,
1163                              &vcpu->arch.sie_block->gpsw,
1164                              sizeof(psw_t));
1165         rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW,
1166                             &vcpu->arch.sie_block->gpsw,
1167                             sizeof(psw_t));
1168         return rc ? -EFAULT : 0;
1169 }
1170
1171 static int __must_check __deliver_io(struct kvm_vcpu *vcpu,
1172                                      unsigned long irq_type)
1173 {
1174         struct list_head *isc_list;
1175         struct kvm_s390_float_interrupt *fi;
1176         struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
1177         struct kvm_s390_interrupt_info *inti = NULL;
1178         struct kvm_s390_io_info io;
1179         u32 isc;
1180         int rc = 0;
1181
1182         fi = &vcpu->kvm->arch.float_int;
1183
1184         spin_lock(&fi->lock);
1185         isc = irq_type_to_isc(irq_type);
1186         isc_list = &fi->lists[isc];
1187         inti = list_first_entry_or_null(isc_list,
1188                                         struct kvm_s390_interrupt_info,
1189                                         list);
1190         if (inti) {
1191                 if (inti->type & KVM_S390_INT_IO_AI_MASK)
1192                         VCPU_EVENT(vcpu, 4, "%s", "deliver: I/O (AI)");
1193                 else
1194                         VCPU_EVENT(vcpu, 4, "deliver: I/O %x ss %x schid %04x",
1195                         inti->io.subchannel_id >> 8,
1196                         inti->io.subchannel_id >> 1 & 0x3,
1197                         inti->io.subchannel_nr);
1198
1199                 vcpu->stat.deliver_io++;
1200                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1201                                 inti->type,
1202                                 ((__u32)inti->io.subchannel_id << 16) |
1203                                 inti->io.subchannel_nr,
1204                                 ((__u64)inti->io.io_int_parm << 32) |
1205                                 inti->io.io_int_word);
1206                 list_del(&inti->list);
1207                 fi->counters[FIRQ_CNTR_IO] -= 1;
1208         }
1209         if (list_empty(isc_list))
1210                 clear_bit(irq_type, &fi->pending_irqs);
1211         spin_unlock(&fi->lock);
1212
1213         if (inti) {
1214                 rc = __do_deliver_io(vcpu, &(inti->io));
1215                 kfree(inti);
1216                 goto out;
1217         }
1218
1219         if (gi->origin && gisa_tac_ipm_gisc(gi->origin, isc)) {
1220                 /*
1221                  * in case an adapter interrupt was not delivered
1222                  * in SIE context KVM will handle the delivery
1223                  */
1224                 VCPU_EVENT(vcpu, 4, "%s isc %u", "deliver: I/O (AI/gisa)", isc);
1225                 memset(&io, 0, sizeof(io));
1226                 io.io_int_word = isc_to_int_word(isc);
1227                 vcpu->stat.deliver_io++;
1228                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1229                         KVM_S390_INT_IO(1, 0, 0, 0),
1230                         ((__u32)io.subchannel_id << 16) |
1231                         io.subchannel_nr,
1232                         ((__u64)io.io_int_parm << 32) |
1233                         io.io_int_word);
1234                 rc = __do_deliver_io(vcpu, &io);
1235         }
1236 out:
1237         return rc;
1238 }
1239
1240 /* Check whether an external call is pending (deliverable or not) */
1241 int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu)
1242 {
1243         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1244
1245         if (!sclp.has_sigpif)
1246                 return test_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
1247
1248         return sca_ext_call_pending(vcpu, NULL);
1249 }
1250
1251 int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop)
1252 {
1253         if (deliverable_irqs(vcpu))
1254                 return 1;
1255
1256         if (kvm_cpu_has_pending_timer(vcpu))
1257                 return 1;
1258
1259         /* external call pending and deliverable */
1260         if (kvm_s390_ext_call_pending(vcpu) &&
1261             !psw_extint_disabled(vcpu) &&
1262             (vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
1263                 return 1;
1264
1265         if (!exclude_stop && kvm_s390_is_stop_irq_pending(vcpu))
1266                 return 1;
1267         return 0;
1268 }
1269
1270 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
1271 {
1272         return ckc_irq_pending(vcpu) || cpu_timer_irq_pending(vcpu);
1273 }
1274
1275 static u64 __calculate_sltime(struct kvm_vcpu *vcpu)
1276 {
1277         const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
1278         const u64 ckc = vcpu->arch.sie_block->ckc;
1279         u64 cputm, sltime = 0;
1280
1281         if (ckc_interrupts_enabled(vcpu)) {
1282                 if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
1283                         if ((s64)now < (s64)ckc)
1284                                 sltime = tod_to_ns((s64)ckc - (s64)now);
1285                 } else if (now < ckc) {
1286                         sltime = tod_to_ns(ckc - now);
1287                 }
1288                 /* already expired */
1289                 if (!sltime)
1290                         return 0;
1291                 if (cpu_timer_interrupts_enabled(vcpu)) {
1292                         cputm = kvm_s390_get_cpu_timer(vcpu);
1293                         /* already expired? */
1294                         if (cputm >> 63)
1295                                 return 0;
1296                         return min_t(u64, sltime, tod_to_ns(cputm));
1297                 }
1298         } else if (cpu_timer_interrupts_enabled(vcpu)) {
1299                 sltime = kvm_s390_get_cpu_timer(vcpu);
1300                 /* already expired? */
1301                 if (sltime >> 63)
1302                         return 0;
1303         }
1304         return sltime;
1305 }
1306
1307 int kvm_s390_handle_wait(struct kvm_vcpu *vcpu)
1308 {
1309         struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
1310         u64 sltime;
1311
1312         vcpu->stat.exit_wait_state++;
1313
1314         /* fast path */
1315         if (kvm_arch_vcpu_runnable(vcpu))
1316                 return 0;
1317
1318         if (psw_interrupts_disabled(vcpu)) {
1319                 VCPU_EVENT(vcpu, 3, "%s", "disabled wait");
1320                 return -EOPNOTSUPP; /* disabled wait */
1321         }
1322
1323         if (gi->origin &&
1324             (gisa_get_ipm_or_restore_iam(gi) &
1325              vcpu->arch.sie_block->gcr[6] >> 24))
1326                 return 0;
1327
1328         if (!ckc_interrupts_enabled(vcpu) &&
1329             !cpu_timer_interrupts_enabled(vcpu)) {
1330                 VCPU_EVENT(vcpu, 3, "%s", "enabled wait w/o timer");
1331                 __set_cpu_idle(vcpu);
1332                 goto no_timer;
1333         }
1334
1335         sltime = __calculate_sltime(vcpu);
1336         if (!sltime)
1337                 return 0;
1338
1339         __set_cpu_idle(vcpu);
1340         hrtimer_start(&vcpu->arch.ckc_timer, sltime, HRTIMER_MODE_REL);
1341         VCPU_EVENT(vcpu, 4, "enabled wait: %llu ns", sltime);
1342 no_timer:
1343         kvm_vcpu_srcu_read_unlock(vcpu);
1344         kvm_vcpu_halt(vcpu);
1345         vcpu->valid_wakeup = false;
1346         __unset_cpu_idle(vcpu);
1347         kvm_vcpu_srcu_read_lock(vcpu);
1348
1349         hrtimer_cancel(&vcpu->arch.ckc_timer);
1350         return 0;
1351 }
1352
1353 void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu)
1354 {
1355         vcpu->valid_wakeup = true;
1356         kvm_vcpu_wake_up(vcpu);
1357
1358         /*
1359          * The VCPU might not be sleeping but rather executing VSIE. Let's
1360          * kick it, so it leaves the SIE to process the request.
1361          */
1362         kvm_s390_vsie_kick(vcpu);
1363 }
1364
1365 enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer)
1366 {
1367         struct kvm_vcpu *vcpu;
1368         u64 sltime;
1369
1370         vcpu = container_of(timer, struct kvm_vcpu, arch.ckc_timer);
1371         sltime = __calculate_sltime(vcpu);
1372
1373         /*
1374          * If the monotonic clock runs faster than the tod clock we might be
1375          * woken up too early and have to go back to sleep to avoid deadlocks.
1376          */
1377         if (sltime && hrtimer_forward_now(timer, ns_to_ktime(sltime)))
1378                 return HRTIMER_RESTART;
1379         kvm_s390_vcpu_wakeup(vcpu);
1380         return HRTIMER_NORESTART;
1381 }
1382
1383 void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu)
1384 {
1385         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1386
1387         spin_lock(&li->lock);
1388         li->pending_irqs = 0;
1389         bitmap_zero(li->sigp_emerg_pending, KVM_MAX_VCPUS);
1390         memset(&li->irq, 0, sizeof(li->irq));
1391         spin_unlock(&li->lock);
1392
1393         sca_clear_ext_call(vcpu);
1394 }
1395
1396 int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
1397 {
1398         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1399         int rc = 0;
1400         unsigned long irq_type;
1401         unsigned long irqs;
1402
1403         __reset_intercept_indicators(vcpu);
1404
1405         /* pending ckc conditions might have been invalidated */
1406         clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1407         if (ckc_irq_pending(vcpu))
1408                 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1409
1410         /* pending cpu timer conditions might have been invalidated */
1411         clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1412         if (cpu_timer_irq_pending(vcpu))
1413                 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1414
1415         while ((irqs = deliverable_irqs(vcpu)) && !rc) {
1416                 /* bits are in the reverse order of interrupt priority */
1417                 irq_type = find_last_bit(&irqs, IRQ_PEND_COUNT);
1418                 switch (irq_type) {
1419                 case IRQ_PEND_IO_ISC_0:
1420                 case IRQ_PEND_IO_ISC_1:
1421                 case IRQ_PEND_IO_ISC_2:
1422                 case IRQ_PEND_IO_ISC_3:
1423                 case IRQ_PEND_IO_ISC_4:
1424                 case IRQ_PEND_IO_ISC_5:
1425                 case IRQ_PEND_IO_ISC_6:
1426                 case IRQ_PEND_IO_ISC_7:
1427                         rc = __deliver_io(vcpu, irq_type);
1428                         break;
1429                 case IRQ_PEND_MCHK_EX:
1430                 case IRQ_PEND_MCHK_REP:
1431                         rc = __deliver_machine_check(vcpu);
1432                         break;
1433                 case IRQ_PEND_PROG:
1434                         rc = __deliver_prog(vcpu);
1435                         break;
1436                 case IRQ_PEND_EXT_EMERGENCY:
1437                         rc = __deliver_emergency_signal(vcpu);
1438                         break;
1439                 case IRQ_PEND_EXT_EXTERNAL:
1440                         rc = __deliver_external_call(vcpu);
1441                         break;
1442                 case IRQ_PEND_EXT_CLOCK_COMP:
1443                         rc = __deliver_ckc(vcpu);
1444                         break;
1445                 case IRQ_PEND_EXT_CPU_TIMER:
1446                         rc = __deliver_cpu_timer(vcpu);
1447                         break;
1448                 case IRQ_PEND_RESTART:
1449                         rc = __deliver_restart(vcpu);
1450                         break;
1451                 case IRQ_PEND_SET_PREFIX:
1452                         rc = __deliver_set_prefix(vcpu);
1453                         break;
1454                 case IRQ_PEND_PFAULT_INIT:
1455                         rc = __deliver_pfault_init(vcpu);
1456                         break;
1457                 case IRQ_PEND_EXT_SERVICE:
1458                         rc = __deliver_service(vcpu);
1459                         break;
1460                 case IRQ_PEND_EXT_SERVICE_EV:
1461                         rc = __deliver_service_ev(vcpu);
1462                         break;
1463                 case IRQ_PEND_PFAULT_DONE:
1464                         rc = __deliver_pfault_done(vcpu);
1465                         break;
1466                 case IRQ_PEND_VIRTIO:
1467                         rc = __deliver_virtio(vcpu);
1468                         break;
1469                 default:
1470                         WARN_ONCE(1, "Unknown pending irq type %ld", irq_type);
1471                         clear_bit(irq_type, &li->pending_irqs);
1472                 }
1473         }
1474
1475         set_intercept_indicators(vcpu);
1476
1477         return rc;
1478 }
1479
1480 static int __inject_prog(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1481 {
1482         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1483
1484         vcpu->stat.inject_program++;
1485         VCPU_EVENT(vcpu, 3, "inject: program irq code 0x%x", irq->u.pgm.code);
1486         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
1487                                    irq->u.pgm.code, 0);
1488
1489         if (!(irq->u.pgm.flags & KVM_S390_PGM_FLAGS_ILC_VALID)) {
1490                 /* auto detection if no valid ILC was given */
1491                 irq->u.pgm.flags &= ~KVM_S390_PGM_FLAGS_ILC_MASK;
1492                 irq->u.pgm.flags |= kvm_s390_get_ilen(vcpu);
1493                 irq->u.pgm.flags |= KVM_S390_PGM_FLAGS_ILC_VALID;
1494         }
1495
1496         if (irq->u.pgm.code == PGM_PER) {
1497                 li->irq.pgm.code |= PGM_PER;
1498                 li->irq.pgm.flags = irq->u.pgm.flags;
1499                 /* only modify PER related information */
1500                 li->irq.pgm.per_address = irq->u.pgm.per_address;
1501                 li->irq.pgm.per_code = irq->u.pgm.per_code;
1502                 li->irq.pgm.per_atmid = irq->u.pgm.per_atmid;
1503                 li->irq.pgm.per_access_id = irq->u.pgm.per_access_id;
1504         } else if (!(irq->u.pgm.code & PGM_PER)) {
1505                 li->irq.pgm.code = (li->irq.pgm.code & PGM_PER) |
1506                                    irq->u.pgm.code;
1507                 li->irq.pgm.flags = irq->u.pgm.flags;
1508                 /* only modify non-PER information */
1509                 li->irq.pgm.trans_exc_code = irq->u.pgm.trans_exc_code;
1510                 li->irq.pgm.mon_code = irq->u.pgm.mon_code;
1511                 li->irq.pgm.data_exc_code = irq->u.pgm.data_exc_code;
1512                 li->irq.pgm.mon_class_nr = irq->u.pgm.mon_class_nr;
1513                 li->irq.pgm.exc_access_id = irq->u.pgm.exc_access_id;
1514                 li->irq.pgm.op_access_id = irq->u.pgm.op_access_id;
1515         } else {
1516                 li->irq.pgm = irq->u.pgm;
1517         }
1518         set_bit(IRQ_PEND_PROG, &li->pending_irqs);
1519         return 0;
1520 }
1521
1522 static int __inject_pfault_init(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1523 {
1524         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1525
1526         vcpu->stat.inject_pfault_init++;
1527         VCPU_EVENT(vcpu, 4, "inject: pfault init parameter block at 0x%llx",
1528                    irq->u.ext.ext_params2);
1529         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_PFAULT_INIT,
1530                                    irq->u.ext.ext_params,
1531                                    irq->u.ext.ext_params2);
1532
1533         li->irq.ext = irq->u.ext;
1534         set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
1535         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1536         return 0;
1537 }
1538
1539 static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1540 {
1541         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1542         struct kvm_s390_extcall_info *extcall = &li->irq.extcall;
1543         uint16_t src_id = irq->u.extcall.code;
1544
1545         vcpu->stat.inject_external_call++;
1546         VCPU_EVENT(vcpu, 4, "inject: external call source-cpu:%u",
1547                    src_id);
1548         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EXTERNAL_CALL,
1549                                    src_id, 0);
1550
1551         /* sending vcpu invalid */
1552         if (kvm_get_vcpu_by_id(vcpu->kvm, src_id) == NULL)
1553                 return -EINVAL;
1554
1555         if (sclp.has_sigpif && !kvm_s390_pv_cpu_get_handle(vcpu))
1556                 return sca_inject_ext_call(vcpu, src_id);
1557
1558         if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs))
1559                 return -EBUSY;
1560         *extcall = irq->u.extcall;
1561         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1562         return 0;
1563 }
1564
1565 static int __inject_set_prefix(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1566 {
1567         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1568         struct kvm_s390_prefix_info *prefix = &li->irq.prefix;
1569
1570         vcpu->stat.inject_set_prefix++;
1571         VCPU_EVENT(vcpu, 3, "inject: set prefix to %x",
1572                    irq->u.prefix.address);
1573         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_SET_PREFIX,
1574                                    irq->u.prefix.address, 0);
1575
1576         if (!is_vcpu_stopped(vcpu))
1577                 return -EBUSY;
1578
1579         *prefix = irq->u.prefix;
1580         set_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
1581         return 0;
1582 }
1583
1584 #define KVM_S390_STOP_SUPP_FLAGS (KVM_S390_STOP_FLAG_STORE_STATUS)
1585 static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1586 {
1587         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1588         struct kvm_s390_stop_info *stop = &li->irq.stop;
1589         int rc = 0;
1590
1591         vcpu->stat.inject_stop_signal++;
1592         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_STOP, 0, 0);
1593
1594         if (irq->u.stop.flags & ~KVM_S390_STOP_SUPP_FLAGS)
1595                 return -EINVAL;
1596
1597         if (is_vcpu_stopped(vcpu)) {
1598                 if (irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS)
1599                         rc = kvm_s390_store_status_unloaded(vcpu,
1600                                                 KVM_S390_STORE_STATUS_NOADDR);
1601                 return rc;
1602         }
1603
1604         if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs))
1605                 return -EBUSY;
1606         stop->flags = irq->u.stop.flags;
1607         kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
1608         return 0;
1609 }
1610
1611 static int __inject_sigp_restart(struct kvm_vcpu *vcpu)
1612 {
1613         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1614
1615         vcpu->stat.inject_restart++;
1616         VCPU_EVENT(vcpu, 3, "%s", "inject: restart int");
1617         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);
1618
1619         set_bit(IRQ_PEND_RESTART, &li->pending_irqs);
1620         return 0;
1621 }
1622
1623 static int __inject_sigp_emergency(struct kvm_vcpu *vcpu,
1624                                    struct kvm_s390_irq *irq)
1625 {
1626         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1627
1628         vcpu->stat.inject_emergency_signal++;
1629         VCPU_EVENT(vcpu, 4, "inject: emergency from cpu %u",
1630                    irq->u.emerg.code);
1631         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
1632                                    irq->u.emerg.code, 0);
1633
1634         /* sending vcpu invalid */
1635         if (kvm_get_vcpu_by_id(vcpu->kvm, irq->u.emerg.code) == NULL)
1636                 return -EINVAL;
1637
1638         set_bit(irq->u.emerg.code, li->sigp_emerg_pending);
1639         set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
1640         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1641         return 0;
1642 }
1643
1644 static int __inject_mchk(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1645 {
1646         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1647         struct kvm_s390_mchk_info *mchk = &li->irq.mchk;
1648
1649         vcpu->stat.inject_mchk++;
1650         VCPU_EVENT(vcpu, 3, "inject: machine check mcic 0x%llx",
1651                    irq->u.mchk.mcic);
1652         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_MCHK, 0,
1653                                    irq->u.mchk.mcic);
1654
1655         /*
1656          * Because repressible machine checks can be indicated along with
1657          * exigent machine checks (PoP, Chapter 11, Interruption action)
1658          * we need to combine cr14, mcic and external damage code.
1659          * Failing storage address and the logout area should not be or'ed
1660          * together, we just indicate the last occurrence of the corresponding
1661          * machine check
1662          */
1663         mchk->cr14 |= irq->u.mchk.cr14;
1664         mchk->mcic |= irq->u.mchk.mcic;
1665         mchk->ext_damage_code |= irq->u.mchk.ext_damage_code;
1666         mchk->failing_storage_address = irq->u.mchk.failing_storage_address;
1667         memcpy(&mchk->fixed_logout, &irq->u.mchk.fixed_logout,
1668                sizeof(mchk->fixed_logout));
1669         if (mchk->mcic & MCHK_EX_MASK)
1670                 set_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
1671         else if (mchk->mcic & MCHK_REP_MASK)
1672                 set_bit(IRQ_PEND_MCHK_REP,  &li->pending_irqs);
1673         return 0;
1674 }
1675
1676 static int __inject_ckc(struct kvm_vcpu *vcpu)
1677 {
1678         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1679
1680         vcpu->stat.inject_ckc++;
1681         VCPU_EVENT(vcpu, 3, "%s", "inject: clock comparator external");
1682         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
1683                                    0, 0);
1684
1685         set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1686         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1687         return 0;
1688 }
1689
1690 static int __inject_cpu_timer(struct kvm_vcpu *vcpu)
1691 {
1692         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1693
1694         vcpu->stat.inject_cputm++;
1695         VCPU_EVENT(vcpu, 3, "%s", "inject: cpu timer external");
1696         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
1697                                    0, 0);
1698
1699         set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1700         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1701         return 0;
1702 }
1703
1704 static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm,
1705                                                   int isc, u32 schid)
1706 {
1707         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1708         struct list_head *isc_list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1709         struct kvm_s390_interrupt_info *iter;
1710         u16 id = (schid & 0xffff0000U) >> 16;
1711         u16 nr = schid & 0x0000ffffU;
1712
1713         spin_lock(&fi->lock);
1714         list_for_each_entry(iter, isc_list, list) {
1715                 if (schid && (id != iter->io.subchannel_id ||
1716                               nr != iter->io.subchannel_nr))
1717                         continue;
1718                 /* found an appropriate entry */
1719                 list_del_init(&iter->list);
1720                 fi->counters[FIRQ_CNTR_IO] -= 1;
1721                 if (list_empty(isc_list))
1722                         clear_bit(isc_to_irq_type(isc), &fi->pending_irqs);
1723                 spin_unlock(&fi->lock);
1724                 return iter;
1725         }
1726         spin_unlock(&fi->lock);
1727         return NULL;
1728 }
1729
1730 static struct kvm_s390_interrupt_info *get_top_io_int(struct kvm *kvm,
1731                                                       u64 isc_mask, u32 schid)
1732 {
1733         struct kvm_s390_interrupt_info *inti = NULL;
1734         int isc;
1735
1736         for (isc = 0; isc <= MAX_ISC && !inti; isc++) {
1737                 if (isc_mask & isc_to_isc_bits(isc))
1738                         inti = get_io_int(kvm, isc, schid);
1739         }
1740         return inti;
1741 }
1742
1743 static int get_top_gisa_isc(struct kvm *kvm, u64 isc_mask, u32 schid)
1744 {
1745         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1746         unsigned long active_mask;
1747         int isc;
1748
1749         if (schid)
1750                 goto out;
1751         if (!gi->origin)
1752                 goto out;
1753
1754         active_mask = (isc_mask & gisa_get_ipm(gi->origin) << 24) << 32;
1755         while (active_mask) {
1756                 isc = __fls(active_mask) ^ (BITS_PER_LONG - 1);
1757                 if (gisa_tac_ipm_gisc(gi->origin, isc))
1758                         return isc;
1759                 clear_bit_inv(isc, &active_mask);
1760         }
1761 out:
1762         return -EINVAL;
1763 }
1764
1765 /*
1766  * Dequeue and return an I/O interrupt matching any of the interruption
1767  * subclasses as designated by the isc mask in cr6 and the schid (if != 0).
1768  * Take into account the interrupts pending in the interrupt list and in GISA.
1769  *
1770  * Note that for a guest that does not enable I/O interrupts
1771  * but relies on TPI, a flood of classic interrupts may starve
1772  * out adapter interrupts on the same isc. Linux does not do
1773  * that, and it is possible to work around the issue by configuring
1774  * different iscs for classic and adapter interrupts in the guest,
1775  * but we may want to revisit this in the future.
1776  */
1777 struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
1778                                                     u64 isc_mask, u32 schid)
1779 {
1780         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1781         struct kvm_s390_interrupt_info *inti, *tmp_inti;
1782         int isc;
1783
1784         inti = get_top_io_int(kvm, isc_mask, schid);
1785
1786         isc = get_top_gisa_isc(kvm, isc_mask, schid);
1787         if (isc < 0)
1788                 /* no AI in GISA */
1789                 goto out;
1790
1791         if (!inti)
1792                 /* AI in GISA but no classical IO int */
1793                 goto gisa_out;
1794
1795         /* both types of interrupts present */
1796         if (int_word_to_isc(inti->io.io_int_word) <= isc) {
1797                 /* classical IO int with higher priority */
1798                 gisa_set_ipm_gisc(gi->origin, isc);
1799                 goto out;
1800         }
1801 gisa_out:
1802         tmp_inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
1803         if (tmp_inti) {
1804                 tmp_inti->type = KVM_S390_INT_IO(1, 0, 0, 0);
1805                 tmp_inti->io.io_int_word = isc_to_int_word(isc);
1806                 if (inti)
1807                         kvm_s390_reinject_io_int(kvm, inti);
1808                 inti = tmp_inti;
1809         } else
1810                 gisa_set_ipm_gisc(gi->origin, isc);
1811 out:
1812         return inti;
1813 }
1814
1815 static int __inject_service(struct kvm *kvm,
1816                              struct kvm_s390_interrupt_info *inti)
1817 {
1818         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1819
1820         kvm->stat.inject_service_signal++;
1821         spin_lock(&fi->lock);
1822         fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING;
1823
1824         /* We always allow events, track them separately from the sccb ints */
1825         if (fi->srv_signal.ext_params & SCCB_EVENT_PENDING)
1826                 set_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
1827
1828         /*
1829          * Early versions of the QEMU s390 bios will inject several
1830          * service interrupts after another without handling a
1831          * condition code indicating busy.
1832          * We will silently ignore those superfluous sccb values.
1833          * A future version of QEMU will take care of serialization
1834          * of servc requests
1835          */
1836         if (fi->srv_signal.ext_params & SCCB_MASK)
1837                 goto out;
1838         fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK;
1839         set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
1840 out:
1841         spin_unlock(&fi->lock);
1842         kfree(inti);
1843         return 0;
1844 }
1845
1846 static int __inject_virtio(struct kvm *kvm,
1847                             struct kvm_s390_interrupt_info *inti)
1848 {
1849         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1850
1851         kvm->stat.inject_virtio++;
1852         spin_lock(&fi->lock);
1853         if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) {
1854                 spin_unlock(&fi->lock);
1855                 return -EBUSY;
1856         }
1857         fi->counters[FIRQ_CNTR_VIRTIO] += 1;
1858         list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]);
1859         set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
1860         spin_unlock(&fi->lock);
1861         return 0;
1862 }
1863
1864 static int __inject_pfault_done(struct kvm *kvm,
1865                                  struct kvm_s390_interrupt_info *inti)
1866 {
1867         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1868
1869         kvm->stat.inject_pfault_done++;
1870         spin_lock(&fi->lock);
1871         if (fi->counters[FIRQ_CNTR_PFAULT] >=
1872                 (ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) {
1873                 spin_unlock(&fi->lock);
1874                 return -EBUSY;
1875         }
1876         fi->counters[FIRQ_CNTR_PFAULT] += 1;
1877         list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]);
1878         set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
1879         spin_unlock(&fi->lock);
1880         return 0;
1881 }
1882
1883 #define CR_PENDING_SUBCLASS 28
1884 static int __inject_float_mchk(struct kvm *kvm,
1885                                 struct kvm_s390_interrupt_info *inti)
1886 {
1887         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1888
1889         kvm->stat.inject_float_mchk++;
1890         spin_lock(&fi->lock);
1891         fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS);
1892         fi->mchk.mcic |= inti->mchk.mcic;
1893         set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs);
1894         spin_unlock(&fi->lock);
1895         kfree(inti);
1896         return 0;
1897 }
1898
1899 static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1900 {
1901         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1902         struct kvm_s390_float_interrupt *fi;
1903         struct list_head *list;
1904         int isc;
1905
1906         kvm->stat.inject_io++;
1907         isc = int_word_to_isc(inti->io.io_int_word);
1908
1909         /*
1910          * We do not use the lock checking variant as this is just a
1911          * performance optimization and we do not hold the lock here.
1912          * This is ok as the code will pick interrupts from both "lists"
1913          * for delivery.
1914          */
1915         if (gi->origin && inti->type & KVM_S390_INT_IO_AI_MASK) {
1916                 VM_EVENT(kvm, 4, "%s isc %1u", "inject: I/O (AI/gisa)", isc);
1917                 gisa_set_ipm_gisc(gi->origin, isc);
1918                 kfree(inti);
1919                 return 0;
1920         }
1921
1922         fi = &kvm->arch.float_int;
1923         spin_lock(&fi->lock);
1924         if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) {
1925                 spin_unlock(&fi->lock);
1926                 return -EBUSY;
1927         }
1928         fi->counters[FIRQ_CNTR_IO] += 1;
1929
1930         if (inti->type & KVM_S390_INT_IO_AI_MASK)
1931                 VM_EVENT(kvm, 4, "%s", "inject: I/O (AI)");
1932         else
1933                 VM_EVENT(kvm, 4, "inject: I/O %x ss %x schid %04x",
1934                         inti->io.subchannel_id >> 8,
1935                         inti->io.subchannel_id >> 1 & 0x3,
1936                         inti->io.subchannel_nr);
1937         list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1938         list_add_tail(&inti->list, list);
1939         set_bit(isc_to_irq_type(isc), &fi->pending_irqs);
1940         spin_unlock(&fi->lock);
1941         return 0;
1942 }
1943
1944 /*
1945  * Find a destination VCPU for a floating irq and kick it.
1946  */
1947 static void __floating_irq_kick(struct kvm *kvm, u64 type)
1948 {
1949         struct kvm_vcpu *dst_vcpu;
1950         int sigcpu, online_vcpus, nr_tries = 0;
1951
1952         online_vcpus = atomic_read(&kvm->online_vcpus);
1953         if (!online_vcpus)
1954                 return;
1955
1956         /* find idle VCPUs first, then round robin */
1957         sigcpu = find_first_bit(kvm->arch.idle_mask, online_vcpus);
1958         if (sigcpu == online_vcpus) {
1959                 do {
1960                         sigcpu = kvm->arch.float_int.next_rr_cpu++;
1961                         kvm->arch.float_int.next_rr_cpu %= online_vcpus;
1962                         /* avoid endless loops if all vcpus are stopped */
1963                         if (nr_tries++ >= online_vcpus)
1964                                 return;
1965                 } while (is_vcpu_stopped(kvm_get_vcpu(kvm, sigcpu)));
1966         }
1967         dst_vcpu = kvm_get_vcpu(kvm, sigcpu);
1968
1969         /* make the VCPU drop out of the SIE, or wake it up if sleeping */
1970         switch (type) {
1971         case KVM_S390_MCHK:
1972                 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_STOP_INT);
1973                 break;
1974         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1975                 if (!(type & KVM_S390_INT_IO_AI_MASK &&
1976                       kvm->arch.gisa_int.origin) ||
1977                       kvm_s390_pv_cpu_get_handle(dst_vcpu))
1978                         kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_IO_INT);
1979                 break;
1980         default:
1981                 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_EXT_INT);
1982                 break;
1983         }
1984         kvm_s390_vcpu_wakeup(dst_vcpu);
1985 }
1986
1987 static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1988 {
1989         u64 type = READ_ONCE(inti->type);
1990         int rc;
1991
1992         switch (type) {
1993         case KVM_S390_MCHK:
1994                 rc = __inject_float_mchk(kvm, inti);
1995                 break;
1996         case KVM_S390_INT_VIRTIO:
1997                 rc = __inject_virtio(kvm, inti);
1998                 break;
1999         case KVM_S390_INT_SERVICE:
2000                 rc = __inject_service(kvm, inti);
2001                 break;
2002         case KVM_S390_INT_PFAULT_DONE:
2003                 rc = __inject_pfault_done(kvm, inti);
2004                 break;
2005         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2006                 rc = __inject_io(kvm, inti);
2007                 break;
2008         default:
2009                 rc = -EINVAL;
2010         }
2011         if (rc)
2012                 return rc;
2013
2014         __floating_irq_kick(kvm, type);
2015         return 0;
2016 }
2017
2018 int kvm_s390_inject_vm(struct kvm *kvm,
2019                        struct kvm_s390_interrupt *s390int)
2020 {
2021         struct kvm_s390_interrupt_info *inti;
2022         int rc;
2023
2024         inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
2025         if (!inti)
2026                 return -ENOMEM;
2027
2028         inti->type = s390int->type;
2029         switch (inti->type) {
2030         case KVM_S390_INT_VIRTIO:
2031                 VM_EVENT(kvm, 5, "inject: virtio parm:%x,parm64:%llx",
2032                          s390int->parm, s390int->parm64);
2033                 inti->ext.ext_params = s390int->parm;
2034                 inti->ext.ext_params2 = s390int->parm64;
2035                 break;
2036         case KVM_S390_INT_SERVICE:
2037                 VM_EVENT(kvm, 4, "inject: sclp parm:%x", s390int->parm);
2038                 inti->ext.ext_params = s390int->parm;
2039                 break;
2040         case KVM_S390_INT_PFAULT_DONE:
2041                 inti->ext.ext_params2 = s390int->parm64;
2042                 break;
2043         case KVM_S390_MCHK:
2044                 VM_EVENT(kvm, 3, "inject: machine check mcic 0x%llx",
2045                          s390int->parm64);
2046                 inti->mchk.cr14 = s390int->parm; /* upper bits are not used */
2047                 inti->mchk.mcic = s390int->parm64;
2048                 break;
2049         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2050                 inti->io.subchannel_id = s390int->parm >> 16;
2051                 inti->io.subchannel_nr = s390int->parm & 0x0000ffffu;
2052                 inti->io.io_int_parm = s390int->parm64 >> 32;
2053                 inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull;
2054                 break;
2055         default:
2056                 kfree(inti);
2057                 return -EINVAL;
2058         }
2059         trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64,
2060                                  2);
2061
2062         rc = __inject_vm(kvm, inti);
2063         if (rc)
2064                 kfree(inti);
2065         return rc;
2066 }
2067
2068 int kvm_s390_reinject_io_int(struct kvm *kvm,
2069                               struct kvm_s390_interrupt_info *inti)
2070 {
2071         return __inject_vm(kvm, inti);
2072 }
2073
2074 int s390int_to_s390irq(struct kvm_s390_interrupt *s390int,
2075                        struct kvm_s390_irq *irq)
2076 {
2077         irq->type = s390int->type;
2078         switch (irq->type) {
2079         case KVM_S390_PROGRAM_INT:
2080                 if (s390int->parm & 0xffff0000)
2081                         return -EINVAL;
2082                 irq->u.pgm.code = s390int->parm;
2083                 break;
2084         case KVM_S390_SIGP_SET_PREFIX:
2085                 irq->u.prefix.address = s390int->parm;
2086                 break;
2087         case KVM_S390_SIGP_STOP:
2088                 irq->u.stop.flags = s390int->parm;
2089                 break;
2090         case KVM_S390_INT_EXTERNAL_CALL:
2091                 if (s390int->parm & 0xffff0000)
2092                         return -EINVAL;
2093                 irq->u.extcall.code = s390int->parm;
2094                 break;
2095         case KVM_S390_INT_EMERGENCY:
2096                 if (s390int->parm & 0xffff0000)
2097                         return -EINVAL;
2098                 irq->u.emerg.code = s390int->parm;
2099                 break;
2100         case KVM_S390_MCHK:
2101                 irq->u.mchk.mcic = s390int->parm64;
2102                 break;
2103         case KVM_S390_INT_PFAULT_INIT:
2104                 irq->u.ext.ext_params = s390int->parm;
2105                 irq->u.ext.ext_params2 = s390int->parm64;
2106                 break;
2107         case KVM_S390_RESTART:
2108         case KVM_S390_INT_CLOCK_COMP:
2109         case KVM_S390_INT_CPU_TIMER:
2110                 break;
2111         default:
2112                 return -EINVAL;
2113         }
2114         return 0;
2115 }
2116
2117 int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu)
2118 {
2119         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2120
2121         return test_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
2122 }
2123
2124 int kvm_s390_is_restart_irq_pending(struct kvm_vcpu *vcpu)
2125 {
2126         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2127
2128         return test_bit(IRQ_PEND_RESTART, &li->pending_irqs);
2129 }
2130
2131 void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu)
2132 {
2133         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2134
2135         spin_lock(&li->lock);
2136         li->irq.stop.flags = 0;
2137         clear_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
2138         spin_unlock(&li->lock);
2139 }
2140
2141 static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
2142 {
2143         int rc;
2144
2145         switch (irq->type) {
2146         case KVM_S390_PROGRAM_INT:
2147                 rc = __inject_prog(vcpu, irq);
2148                 break;
2149         case KVM_S390_SIGP_SET_PREFIX:
2150                 rc = __inject_set_prefix(vcpu, irq);
2151                 break;
2152         case KVM_S390_SIGP_STOP:
2153                 rc = __inject_sigp_stop(vcpu, irq);
2154                 break;
2155         case KVM_S390_RESTART:
2156                 rc = __inject_sigp_restart(vcpu);
2157                 break;
2158         case KVM_S390_INT_CLOCK_COMP:
2159                 rc = __inject_ckc(vcpu);
2160                 break;
2161         case KVM_S390_INT_CPU_TIMER:
2162                 rc = __inject_cpu_timer(vcpu);
2163                 break;
2164         case KVM_S390_INT_EXTERNAL_CALL:
2165                 rc = __inject_extcall(vcpu, irq);
2166                 break;
2167         case KVM_S390_INT_EMERGENCY:
2168                 rc = __inject_sigp_emergency(vcpu, irq);
2169                 break;
2170         case KVM_S390_MCHK:
2171                 rc = __inject_mchk(vcpu, irq);
2172                 break;
2173         case KVM_S390_INT_PFAULT_INIT:
2174                 rc = __inject_pfault_init(vcpu, irq);
2175                 break;
2176         case KVM_S390_INT_VIRTIO:
2177         case KVM_S390_INT_SERVICE:
2178         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2179         default:
2180                 rc = -EINVAL;
2181         }
2182
2183         return rc;
2184 }
2185
2186 int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
2187 {
2188         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2189         int rc;
2190
2191         spin_lock(&li->lock);
2192         rc = do_inject_vcpu(vcpu, irq);
2193         spin_unlock(&li->lock);
2194         if (!rc)
2195                 kvm_s390_vcpu_wakeup(vcpu);
2196         return rc;
2197 }
2198
2199 static inline void clear_irq_list(struct list_head *_list)
2200 {
2201         struct kvm_s390_interrupt_info *inti, *n;
2202
2203         list_for_each_entry_safe(inti, n, _list, list) {
2204                 list_del(&inti->list);
2205                 kfree(inti);
2206         }
2207 }
2208
2209 static void inti_to_irq(struct kvm_s390_interrupt_info *inti,
2210                        struct kvm_s390_irq *irq)
2211 {
2212         irq->type = inti->type;
2213         switch (inti->type) {
2214         case KVM_S390_INT_PFAULT_INIT:
2215         case KVM_S390_INT_PFAULT_DONE:
2216         case KVM_S390_INT_VIRTIO:
2217                 irq->u.ext = inti->ext;
2218                 break;
2219         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2220                 irq->u.io = inti->io;
2221                 break;
2222         }
2223 }
2224
2225 void kvm_s390_clear_float_irqs(struct kvm *kvm)
2226 {
2227         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2228         int i;
2229
2230         mutex_lock(&kvm->lock);
2231         if (!kvm_s390_pv_is_protected(kvm))
2232                 fi->masked_irqs = 0;
2233         mutex_unlock(&kvm->lock);
2234         spin_lock(&fi->lock);
2235         fi->pending_irqs = 0;
2236         memset(&fi->srv_signal, 0, sizeof(fi->srv_signal));
2237         memset(&fi->mchk, 0, sizeof(fi->mchk));
2238         for (i = 0; i < FIRQ_LIST_COUNT; i++)
2239                 clear_irq_list(&fi->lists[i]);
2240         for (i = 0; i < FIRQ_MAX_COUNT; i++)
2241                 fi->counters[i] = 0;
2242         spin_unlock(&fi->lock);
2243         kvm_s390_gisa_clear(kvm);
2244 };
2245
2246 static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len)
2247 {
2248         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
2249         struct kvm_s390_interrupt_info *inti;
2250         struct kvm_s390_float_interrupt *fi;
2251         struct kvm_s390_irq *buf;
2252         struct kvm_s390_irq *irq;
2253         int max_irqs;
2254         int ret = 0;
2255         int n = 0;
2256         int i;
2257
2258         if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0)
2259                 return -EINVAL;
2260
2261         /*
2262          * We are already using -ENOMEM to signal
2263          * userspace it may retry with a bigger buffer,
2264          * so we need to use something else for this case
2265          */
2266         buf = vzalloc(len);
2267         if (!buf)
2268                 return -ENOBUFS;
2269
2270         max_irqs = len / sizeof(struct kvm_s390_irq);
2271
2272         if (gi->origin && gisa_get_ipm(gi->origin)) {
2273                 for (i = 0; i <= MAX_ISC; i++) {
2274                         if (n == max_irqs) {
2275                                 /* signal userspace to try again */
2276                                 ret = -ENOMEM;
2277                                 goto out_nolock;
2278                         }
2279                         if (gisa_tac_ipm_gisc(gi->origin, i)) {
2280                                 irq = (struct kvm_s390_irq *) &buf[n];
2281                                 irq->type = KVM_S390_INT_IO(1, 0, 0, 0);
2282                                 irq->u.io.io_int_word = isc_to_int_word(i);
2283                                 n++;
2284                         }
2285                 }
2286         }
2287         fi = &kvm->arch.float_int;
2288         spin_lock(&fi->lock);
2289         for (i = 0; i < FIRQ_LIST_COUNT; i++) {
2290                 list_for_each_entry(inti, &fi->lists[i], list) {
2291                         if (n == max_irqs) {
2292                                 /* signal userspace to try again */
2293                                 ret = -ENOMEM;
2294                                 goto out;
2295                         }
2296                         inti_to_irq(inti, &buf[n]);
2297                         n++;
2298                 }
2299         }
2300         if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs) ||
2301             test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs)) {
2302                 if (n == max_irqs) {
2303                         /* signal userspace to try again */
2304                         ret = -ENOMEM;
2305                         goto out;
2306                 }
2307                 irq = (struct kvm_s390_irq *) &buf[n];
2308                 irq->type = KVM_S390_INT_SERVICE;
2309                 irq->u.ext = fi->srv_signal;
2310                 n++;
2311         }
2312         if (test_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
2313                 if (n == max_irqs) {
2314                                 /* signal userspace to try again */
2315                                 ret = -ENOMEM;
2316                                 goto out;
2317                 }
2318                 irq = (struct kvm_s390_irq *) &buf[n];
2319                 irq->type = KVM_S390_MCHK;
2320                 irq->u.mchk = fi->mchk;
2321                 n++;
2322 }
2323
2324 out:
2325         spin_unlock(&fi->lock);
2326 out_nolock:
2327         if (!ret && n > 0) {
2328                 if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n))
2329                         ret = -EFAULT;
2330         }
2331         vfree(buf);
2332
2333         return ret < 0 ? ret : n;
2334 }
2335
2336 static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr)
2337 {
2338         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2339         struct kvm_s390_ais_all ais;
2340
2341         if (attr->attr < sizeof(ais))
2342                 return -EINVAL;
2343
2344         if (!test_kvm_facility(kvm, 72))
2345                 return -EOPNOTSUPP;
2346
2347         mutex_lock(&fi->ais_lock);
2348         ais.simm = fi->simm;
2349         ais.nimm = fi->nimm;
2350         mutex_unlock(&fi->ais_lock);
2351
2352         if (copy_to_user((void __user *)attr->addr, &ais, sizeof(ais)))
2353                 return -EFAULT;
2354
2355         return 0;
2356 }
2357
2358 static int flic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2359 {
2360         int r;
2361
2362         switch (attr->group) {
2363         case KVM_DEV_FLIC_GET_ALL_IRQS:
2364                 r = get_all_floating_irqs(dev->kvm, (u8 __user *) attr->addr,
2365                                           attr->attr);
2366                 break;
2367         case KVM_DEV_FLIC_AISM_ALL:
2368                 r = flic_ais_mode_get_all(dev->kvm, attr);
2369                 break;
2370         default:
2371                 r = -EINVAL;
2372         }
2373
2374         return r;
2375 }
2376
2377 static inline int copy_irq_from_user(struct kvm_s390_interrupt_info *inti,
2378                                      u64 addr)
2379 {
2380         struct kvm_s390_irq __user *uptr = (struct kvm_s390_irq __user *) addr;
2381         void *target = NULL;
2382         void __user *source;
2383         u64 size;
2384
2385         if (get_user(inti->type, (u64 __user *)addr))
2386                 return -EFAULT;
2387
2388         switch (inti->type) {
2389         case KVM_S390_INT_PFAULT_INIT:
2390         case KVM_S390_INT_PFAULT_DONE:
2391         case KVM_S390_INT_VIRTIO:
2392         case KVM_S390_INT_SERVICE:
2393                 target = (void *) &inti->ext;
2394                 source = &uptr->u.ext;
2395                 size = sizeof(inti->ext);
2396                 break;
2397         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2398                 target = (void *) &inti->io;
2399                 source = &uptr->u.io;
2400                 size = sizeof(inti->io);
2401                 break;
2402         case KVM_S390_MCHK:
2403                 target = (void *) &inti->mchk;
2404                 source = &uptr->u.mchk;
2405                 size = sizeof(inti->mchk);
2406                 break;
2407         default:
2408                 return -EINVAL;
2409         }
2410
2411         if (copy_from_user(target, source, size))
2412                 return -EFAULT;
2413
2414         return 0;
2415 }
2416
2417 static int enqueue_floating_irq(struct kvm_device *dev,
2418                                 struct kvm_device_attr *attr)
2419 {
2420         struct kvm_s390_interrupt_info *inti = NULL;
2421         int r = 0;
2422         int len = attr->attr;
2423
2424         if (len % sizeof(struct kvm_s390_irq) != 0)
2425                 return -EINVAL;
2426         else if (len > KVM_S390_FLIC_MAX_BUFFER)
2427                 return -EINVAL;
2428
2429         while (len >= sizeof(struct kvm_s390_irq)) {
2430                 inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
2431                 if (!inti)
2432                         return -ENOMEM;
2433
2434                 r = copy_irq_from_user(inti, attr->addr);
2435                 if (r) {
2436                         kfree(inti);
2437                         return r;
2438                 }
2439                 r = __inject_vm(dev->kvm, inti);
2440                 if (r) {
2441                         kfree(inti);
2442                         return r;
2443                 }
2444                 len -= sizeof(struct kvm_s390_irq);
2445                 attr->addr += sizeof(struct kvm_s390_irq);
2446         }
2447
2448         return r;
2449 }
2450
2451 static struct s390_io_adapter *get_io_adapter(struct kvm *kvm, unsigned int id)
2452 {
2453         if (id >= MAX_S390_IO_ADAPTERS)
2454                 return NULL;
2455         id = array_index_nospec(id, MAX_S390_IO_ADAPTERS);
2456         return kvm->arch.adapters[id];
2457 }
2458
2459 static int register_io_adapter(struct kvm_device *dev,
2460                                struct kvm_device_attr *attr)
2461 {
2462         struct s390_io_adapter *adapter;
2463         struct kvm_s390_io_adapter adapter_info;
2464
2465         if (copy_from_user(&adapter_info,
2466                            (void __user *)attr->addr, sizeof(adapter_info)))
2467                 return -EFAULT;
2468
2469         if (adapter_info.id >= MAX_S390_IO_ADAPTERS)
2470                 return -EINVAL;
2471
2472         adapter_info.id = array_index_nospec(adapter_info.id,
2473                                              MAX_S390_IO_ADAPTERS);
2474
2475         if (dev->kvm->arch.adapters[adapter_info.id] != NULL)
2476                 return -EINVAL;
2477
2478         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL_ACCOUNT);
2479         if (!adapter)
2480                 return -ENOMEM;
2481
2482         adapter->id = adapter_info.id;
2483         adapter->isc = adapter_info.isc;
2484         adapter->maskable = adapter_info.maskable;
2485         adapter->masked = false;
2486         adapter->swap = adapter_info.swap;
2487         adapter->suppressible = (adapter_info.flags) &
2488                                 KVM_S390_ADAPTER_SUPPRESSIBLE;
2489         dev->kvm->arch.adapters[adapter->id] = adapter;
2490
2491         return 0;
2492 }
2493
2494 int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked)
2495 {
2496         int ret;
2497         struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
2498
2499         if (!adapter || !adapter->maskable)
2500                 return -EINVAL;
2501         ret = adapter->masked;
2502         adapter->masked = masked;
2503         return ret;
2504 }
2505
2506 void kvm_s390_destroy_adapters(struct kvm *kvm)
2507 {
2508         int i;
2509
2510         for (i = 0; i < MAX_S390_IO_ADAPTERS; i++)
2511                 kfree(kvm->arch.adapters[i]);
2512 }
2513
2514 static int modify_io_adapter(struct kvm_device *dev,
2515                              struct kvm_device_attr *attr)
2516 {
2517         struct kvm_s390_io_adapter_req req;
2518         struct s390_io_adapter *adapter;
2519         int ret;
2520
2521         if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req)))
2522                 return -EFAULT;
2523
2524         adapter = get_io_adapter(dev->kvm, req.id);
2525         if (!adapter)
2526                 return -EINVAL;
2527         switch (req.type) {
2528         case KVM_S390_IO_ADAPTER_MASK:
2529                 ret = kvm_s390_mask_adapter(dev->kvm, req.id, req.mask);
2530                 if (ret > 0)
2531                         ret = 0;
2532                 break;
2533         /*
2534          * The following operations are no longer needed and therefore no-ops.
2535          * The gpa to hva translation is done when an IRQ route is set up. The
2536          * set_irq code uses get_user_pages_remote() to do the actual write.
2537          */
2538         case KVM_S390_IO_ADAPTER_MAP:
2539         case KVM_S390_IO_ADAPTER_UNMAP:
2540                 ret = 0;
2541                 break;
2542         default:
2543                 ret = -EINVAL;
2544         }
2545
2546         return ret;
2547 }
2548
2549 static int clear_io_irq(struct kvm *kvm, struct kvm_device_attr *attr)
2550
2551 {
2552         const u64 isc_mask = 0xffUL << 24; /* all iscs set */
2553         u32 schid;
2554
2555         if (attr->flags)
2556                 return -EINVAL;
2557         if (attr->attr != sizeof(schid))
2558                 return -EINVAL;
2559         if (copy_from_user(&schid, (void __user *) attr->addr, sizeof(schid)))
2560                 return -EFAULT;
2561         if (!schid)
2562                 return -EINVAL;
2563         kfree(kvm_s390_get_io_int(kvm, isc_mask, schid));
2564         /*
2565          * If userspace is conforming to the architecture, we can have at most
2566          * one pending I/O interrupt per subchannel, so this is effectively a
2567          * clear all.
2568          */
2569         return 0;
2570 }
2571
2572 static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr)
2573 {
2574         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2575         struct kvm_s390_ais_req req;
2576         int ret = 0;
2577
2578         if (!test_kvm_facility(kvm, 72))
2579                 return -EOPNOTSUPP;
2580
2581         if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req)))
2582                 return -EFAULT;
2583
2584         if (req.isc > MAX_ISC)
2585                 return -EINVAL;
2586
2587         trace_kvm_s390_modify_ais_mode(req.isc,
2588                                        (fi->simm & AIS_MODE_MASK(req.isc)) ?
2589                                        (fi->nimm & AIS_MODE_MASK(req.isc)) ?
2590                                        2 : KVM_S390_AIS_MODE_SINGLE :
2591                                        KVM_S390_AIS_MODE_ALL, req.mode);
2592
2593         mutex_lock(&fi->ais_lock);
2594         switch (req.mode) {
2595         case KVM_S390_AIS_MODE_ALL:
2596                 fi->simm &= ~AIS_MODE_MASK(req.isc);
2597                 fi->nimm &= ~AIS_MODE_MASK(req.isc);
2598                 break;
2599         case KVM_S390_AIS_MODE_SINGLE:
2600                 fi->simm |= AIS_MODE_MASK(req.isc);
2601                 fi->nimm &= ~AIS_MODE_MASK(req.isc);
2602                 break;
2603         default:
2604                 ret = -EINVAL;
2605         }
2606         mutex_unlock(&fi->ais_lock);
2607
2608         return ret;
2609 }
2610
2611 static int kvm_s390_inject_airq(struct kvm *kvm,
2612                                 struct s390_io_adapter *adapter)
2613 {
2614         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2615         struct kvm_s390_interrupt s390int = {
2616                 .type = KVM_S390_INT_IO(1, 0, 0, 0),
2617                 .parm = 0,
2618                 .parm64 = isc_to_int_word(adapter->isc),
2619         };
2620         int ret = 0;
2621
2622         if (!test_kvm_facility(kvm, 72) || !adapter->suppressible)
2623                 return kvm_s390_inject_vm(kvm, &s390int);
2624
2625         mutex_lock(&fi->ais_lock);
2626         if (fi->nimm & AIS_MODE_MASK(adapter->isc)) {
2627                 trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc);
2628                 goto out;
2629         }
2630
2631         ret = kvm_s390_inject_vm(kvm, &s390int);
2632         if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) {
2633                 fi->nimm |= AIS_MODE_MASK(adapter->isc);
2634                 trace_kvm_s390_modify_ais_mode(adapter->isc,
2635                                                KVM_S390_AIS_MODE_SINGLE, 2);
2636         }
2637 out:
2638         mutex_unlock(&fi->ais_lock);
2639         return ret;
2640 }
2641
2642 static int flic_inject_airq(struct kvm *kvm, struct kvm_device_attr *attr)
2643 {
2644         unsigned int id = attr->attr;
2645         struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
2646
2647         if (!adapter)
2648                 return -EINVAL;
2649
2650         return kvm_s390_inject_airq(kvm, adapter);
2651 }
2652
2653 static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr)
2654 {
2655         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2656         struct kvm_s390_ais_all ais;
2657
2658         if (!test_kvm_facility(kvm, 72))
2659                 return -EOPNOTSUPP;
2660
2661         if (copy_from_user(&ais, (void __user *)attr->addr, sizeof(ais)))
2662                 return -EFAULT;
2663
2664         mutex_lock(&fi->ais_lock);
2665         fi->simm = ais.simm;
2666         fi->nimm = ais.nimm;
2667         mutex_unlock(&fi->ais_lock);
2668
2669         return 0;
2670 }
2671
2672 static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2673 {
2674         int r = 0;
2675         unsigned long i;
2676         struct kvm_vcpu *vcpu;
2677
2678         switch (attr->group) {
2679         case KVM_DEV_FLIC_ENQUEUE:
2680                 r = enqueue_floating_irq(dev, attr);
2681                 break;
2682         case KVM_DEV_FLIC_CLEAR_IRQS:
2683                 kvm_s390_clear_float_irqs(dev->kvm);
2684                 break;
2685         case KVM_DEV_FLIC_APF_ENABLE:
2686                 dev->kvm->arch.gmap->pfault_enabled = 1;
2687                 break;
2688         case KVM_DEV_FLIC_APF_DISABLE_WAIT:
2689                 dev->kvm->arch.gmap->pfault_enabled = 0;
2690                 /*
2691                  * Make sure no async faults are in transition when
2692                  * clearing the queues. So we don't need to worry
2693                  * about late coming workers.
2694                  */
2695                 synchronize_srcu(&dev->kvm->srcu);
2696                 kvm_for_each_vcpu(i, vcpu, dev->kvm)
2697                         kvm_clear_async_pf_completion_queue(vcpu);
2698                 break;
2699         case KVM_DEV_FLIC_ADAPTER_REGISTER:
2700                 r = register_io_adapter(dev, attr);
2701                 break;
2702         case KVM_DEV_FLIC_ADAPTER_MODIFY:
2703                 r = modify_io_adapter(dev, attr);
2704                 break;
2705         case KVM_DEV_FLIC_CLEAR_IO_IRQ:
2706                 r = clear_io_irq(dev->kvm, attr);
2707                 break;
2708         case KVM_DEV_FLIC_AISM:
2709                 r = modify_ais_mode(dev->kvm, attr);
2710                 break;
2711         case KVM_DEV_FLIC_AIRQ_INJECT:
2712                 r = flic_inject_airq(dev->kvm, attr);
2713                 break;
2714         case KVM_DEV_FLIC_AISM_ALL:
2715                 r = flic_ais_mode_set_all(dev->kvm, attr);
2716                 break;
2717         default:
2718                 r = -EINVAL;
2719         }
2720
2721         return r;
2722 }
2723
2724 static int flic_has_attr(struct kvm_device *dev,
2725                              struct kvm_device_attr *attr)
2726 {
2727         switch (attr->group) {
2728         case KVM_DEV_FLIC_GET_ALL_IRQS:
2729         case KVM_DEV_FLIC_ENQUEUE:
2730         case KVM_DEV_FLIC_CLEAR_IRQS:
2731         case KVM_DEV_FLIC_APF_ENABLE:
2732         case KVM_DEV_FLIC_APF_DISABLE_WAIT:
2733         case KVM_DEV_FLIC_ADAPTER_REGISTER:
2734         case KVM_DEV_FLIC_ADAPTER_MODIFY:
2735         case KVM_DEV_FLIC_CLEAR_IO_IRQ:
2736         case KVM_DEV_FLIC_AISM:
2737         case KVM_DEV_FLIC_AIRQ_INJECT:
2738         case KVM_DEV_FLIC_AISM_ALL:
2739                 return 0;
2740         }
2741         return -ENXIO;
2742 }
2743
2744 static int flic_create(struct kvm_device *dev, u32 type)
2745 {
2746         if (!dev)
2747                 return -EINVAL;
2748         if (dev->kvm->arch.flic)
2749                 return -EINVAL;
2750         dev->kvm->arch.flic = dev;
2751         return 0;
2752 }
2753
2754 static void flic_destroy(struct kvm_device *dev)
2755 {
2756         dev->kvm->arch.flic = NULL;
2757         kfree(dev);
2758 }
2759
2760 /* s390 floating irq controller (flic) */
2761 struct kvm_device_ops kvm_flic_ops = {
2762         .name = "kvm-flic",
2763         .get_attr = flic_get_attr,
2764         .set_attr = flic_set_attr,
2765         .has_attr = flic_has_attr,
2766         .create = flic_create,
2767         .destroy = flic_destroy,
2768 };
2769
2770 static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap)
2771 {
2772         unsigned long bit;
2773
2774         bit = bit_nr + (addr % PAGE_SIZE) * 8;
2775
2776         return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit;
2777 }
2778
2779 static struct page *get_map_page(struct kvm *kvm, u64 uaddr)
2780 {
2781         struct page *page = NULL;
2782
2783         mmap_read_lock(kvm->mm);
2784         get_user_pages_remote(kvm->mm, uaddr, 1, FOLL_WRITE,
2785                               &page, NULL, NULL);
2786         mmap_read_unlock(kvm->mm);
2787         return page;
2788 }
2789
2790 static int adapter_indicators_set(struct kvm *kvm,
2791                                   struct s390_io_adapter *adapter,
2792                                   struct kvm_s390_adapter_int *adapter_int)
2793 {
2794         unsigned long bit;
2795         int summary_set, idx;
2796         struct page *ind_page, *summary_page;
2797         void *map;
2798
2799         ind_page = get_map_page(kvm, adapter_int->ind_addr);
2800         if (!ind_page)
2801                 return -1;
2802         summary_page = get_map_page(kvm, adapter_int->summary_addr);
2803         if (!summary_page) {
2804                 put_page(ind_page);
2805                 return -1;
2806         }
2807
2808         idx = srcu_read_lock(&kvm->srcu);
2809         map = page_address(ind_page);
2810         bit = get_ind_bit(adapter_int->ind_addr,
2811                           adapter_int->ind_offset, adapter->swap);
2812         set_bit(bit, map);
2813         mark_page_dirty(kvm, adapter_int->ind_addr >> PAGE_SHIFT);
2814         set_page_dirty_lock(ind_page);
2815         map = page_address(summary_page);
2816         bit = get_ind_bit(adapter_int->summary_addr,
2817                           adapter_int->summary_offset, adapter->swap);
2818         summary_set = test_and_set_bit(bit, map);
2819         mark_page_dirty(kvm, adapter_int->summary_addr >> PAGE_SHIFT);
2820         set_page_dirty_lock(summary_page);
2821         srcu_read_unlock(&kvm->srcu, idx);
2822
2823         put_page(ind_page);
2824         put_page(summary_page);
2825         return summary_set ? 0 : 1;
2826 }
2827
2828 /*
2829  * < 0 - not injected due to error
2830  * = 0 - coalesced, summary indicator already active
2831  * > 0 - injected interrupt
2832  */
2833 static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e,
2834                            struct kvm *kvm, int irq_source_id, int level,
2835                            bool line_status)
2836 {
2837         int ret;
2838         struct s390_io_adapter *adapter;
2839
2840         /* We're only interested in the 0->1 transition. */
2841         if (!level)
2842                 return 0;
2843         adapter = get_io_adapter(kvm, e->adapter.adapter_id);
2844         if (!adapter)
2845                 return -1;
2846         ret = adapter_indicators_set(kvm, adapter, &e->adapter);
2847         if ((ret > 0) && !adapter->masked) {
2848                 ret = kvm_s390_inject_airq(kvm, adapter);
2849                 if (ret == 0)
2850                         ret = 1;
2851         }
2852         return ret;
2853 }
2854
2855 /*
2856  * Inject the machine check to the guest.
2857  */
2858 void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu,
2859                                      struct mcck_volatile_info *mcck_info)
2860 {
2861         struct kvm_s390_interrupt_info inti;
2862         struct kvm_s390_irq irq;
2863         struct kvm_s390_mchk_info *mchk;
2864         union mci mci;
2865         __u64 cr14 = 0;         /* upper bits are not used */
2866         int rc;
2867
2868         mci.val = mcck_info->mcic;
2869         if (mci.sr)
2870                 cr14 |= CR14_RECOVERY_SUBMASK;
2871         if (mci.dg)
2872                 cr14 |= CR14_DEGRADATION_SUBMASK;
2873         if (mci.w)
2874                 cr14 |= CR14_WARNING_SUBMASK;
2875
2876         mchk = mci.ck ? &inti.mchk : &irq.u.mchk;
2877         mchk->cr14 = cr14;
2878         mchk->mcic = mcck_info->mcic;
2879         mchk->ext_damage_code = mcck_info->ext_damage_code;
2880         mchk->failing_storage_address = mcck_info->failing_storage_address;
2881         if (mci.ck) {
2882                 /* Inject the floating machine check */
2883                 inti.type = KVM_S390_MCHK;
2884                 rc = __inject_vm(vcpu->kvm, &inti);
2885         } else {
2886                 /* Inject the machine check to specified vcpu */
2887                 irq.type = KVM_S390_MCHK;
2888                 rc = kvm_s390_inject_vcpu(vcpu, &irq);
2889         }
2890         WARN_ON_ONCE(rc);
2891 }
2892
2893 int kvm_set_routing_entry(struct kvm *kvm,
2894                           struct kvm_kernel_irq_routing_entry *e,
2895                           const struct kvm_irq_routing_entry *ue)
2896 {
2897         u64 uaddr;
2898
2899         switch (ue->type) {
2900         /* we store the userspace addresses instead of the guest addresses */
2901         case KVM_IRQ_ROUTING_S390_ADAPTER:
2902                 e->set = set_adapter_int;
2903                 uaddr =  gmap_translate(kvm->arch.gmap, ue->u.adapter.summary_addr);
2904                 if (uaddr == -EFAULT)
2905                         return -EFAULT;
2906                 e->adapter.summary_addr = uaddr;
2907                 uaddr =  gmap_translate(kvm->arch.gmap, ue->u.adapter.ind_addr);
2908                 if (uaddr == -EFAULT)
2909                         return -EFAULT;
2910                 e->adapter.ind_addr = uaddr;
2911                 e->adapter.summary_offset = ue->u.adapter.summary_offset;
2912                 e->adapter.ind_offset = ue->u.adapter.ind_offset;
2913                 e->adapter.adapter_id = ue->u.adapter.adapter_id;
2914                 return 0;
2915         default:
2916                 return -EINVAL;
2917         }
2918 }
2919
2920 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
2921                 int irq_source_id, int level, bool line_status)
2922 {
2923         return -EINVAL;
2924 }
2925
2926 int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len)
2927 {
2928         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2929         struct kvm_s390_irq *buf;
2930         int r = 0;
2931         int n;
2932
2933         buf = vmalloc(len);
2934         if (!buf)
2935                 return -ENOMEM;
2936
2937         if (copy_from_user((void *) buf, irqstate, len)) {
2938                 r = -EFAULT;
2939                 goto out_free;
2940         }
2941
2942         /*
2943          * Don't allow setting the interrupt state
2944          * when there are already interrupts pending
2945          */
2946         spin_lock(&li->lock);
2947         if (li->pending_irqs) {
2948                 r = -EBUSY;
2949                 goto out_unlock;
2950         }
2951
2952         for (n = 0; n < len / sizeof(*buf); n++) {
2953                 r = do_inject_vcpu(vcpu, &buf[n]);
2954                 if (r)
2955                         break;
2956         }
2957
2958 out_unlock:
2959         spin_unlock(&li->lock);
2960 out_free:
2961         vfree(buf);
2962
2963         return r;
2964 }
2965
2966 static void store_local_irq(struct kvm_s390_local_interrupt *li,
2967                             struct kvm_s390_irq *irq,
2968                             unsigned long irq_type)
2969 {
2970         switch (irq_type) {
2971         case IRQ_PEND_MCHK_EX:
2972         case IRQ_PEND_MCHK_REP:
2973                 irq->type = KVM_S390_MCHK;
2974                 irq->u.mchk = li->irq.mchk;
2975                 break;
2976         case IRQ_PEND_PROG:
2977                 irq->type = KVM_S390_PROGRAM_INT;
2978                 irq->u.pgm = li->irq.pgm;
2979                 break;
2980         case IRQ_PEND_PFAULT_INIT:
2981                 irq->type = KVM_S390_INT_PFAULT_INIT;
2982                 irq->u.ext = li->irq.ext;
2983                 break;
2984         case IRQ_PEND_EXT_EXTERNAL:
2985                 irq->type = KVM_S390_INT_EXTERNAL_CALL;
2986                 irq->u.extcall = li->irq.extcall;
2987                 break;
2988         case IRQ_PEND_EXT_CLOCK_COMP:
2989                 irq->type = KVM_S390_INT_CLOCK_COMP;
2990                 break;
2991         case IRQ_PEND_EXT_CPU_TIMER:
2992                 irq->type = KVM_S390_INT_CPU_TIMER;
2993                 break;
2994         case IRQ_PEND_SIGP_STOP:
2995                 irq->type = KVM_S390_SIGP_STOP;
2996                 irq->u.stop = li->irq.stop;
2997                 break;
2998         case IRQ_PEND_RESTART:
2999                 irq->type = KVM_S390_RESTART;
3000                 break;
3001         case IRQ_PEND_SET_PREFIX:
3002                 irq->type = KVM_S390_SIGP_SET_PREFIX;
3003                 irq->u.prefix = li->irq.prefix;
3004                 break;
3005         }
3006 }
3007
3008 int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len)
3009 {
3010         int scn;
3011         DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS);
3012         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
3013         unsigned long pending_irqs;
3014         struct kvm_s390_irq irq;
3015         unsigned long irq_type;
3016         int cpuaddr;
3017         int n = 0;
3018
3019         spin_lock(&li->lock);
3020         pending_irqs = li->pending_irqs;
3021         memcpy(&sigp_emerg_pending, &li->sigp_emerg_pending,
3022                sizeof(sigp_emerg_pending));
3023         spin_unlock(&li->lock);
3024
3025         for_each_set_bit(irq_type, &pending_irqs, IRQ_PEND_COUNT) {
3026                 memset(&irq, 0, sizeof(irq));
3027                 if (irq_type == IRQ_PEND_EXT_EMERGENCY)
3028                         continue;
3029                 if (n + sizeof(irq) > len)
3030                         return -ENOBUFS;
3031                 store_local_irq(&vcpu->arch.local_int, &irq, irq_type);
3032                 if (copy_to_user(&buf[n], &irq, sizeof(irq)))
3033                         return -EFAULT;
3034                 n += sizeof(irq);
3035         }
3036
3037         if (test_bit(IRQ_PEND_EXT_EMERGENCY, &pending_irqs)) {
3038                 for_each_set_bit(cpuaddr, sigp_emerg_pending, KVM_MAX_VCPUS) {
3039                         memset(&irq, 0, sizeof(irq));
3040                         if (n + sizeof(irq) > len)
3041                                 return -ENOBUFS;
3042                         irq.type = KVM_S390_INT_EMERGENCY;
3043                         irq.u.emerg.code = cpuaddr;
3044                         if (copy_to_user(&buf[n], &irq, sizeof(irq)))
3045                                 return -EFAULT;
3046                         n += sizeof(irq);
3047                 }
3048         }
3049
3050         if (sca_ext_call_pending(vcpu, &scn)) {
3051                 if (n + sizeof(irq) > len)
3052                         return -ENOBUFS;
3053                 memset(&irq, 0, sizeof(irq));
3054                 irq.type = KVM_S390_INT_EXTERNAL_CALL;
3055                 irq.u.extcall.code = scn;
3056                 if (copy_to_user(&buf[n], &irq, sizeof(irq)))
3057                         return -EFAULT;
3058                 n += sizeof(irq);
3059         }
3060
3061         return n;
3062 }
3063
3064 static void __airqs_kick_single_vcpu(struct kvm *kvm, u8 deliverable_mask)
3065 {
3066         int vcpu_idx, online_vcpus = atomic_read(&kvm->online_vcpus);
3067         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3068         struct kvm_vcpu *vcpu;
3069         u8 vcpu_isc_mask;
3070
3071         for_each_set_bit(vcpu_idx, kvm->arch.idle_mask, online_vcpus) {
3072                 vcpu = kvm_get_vcpu(kvm, vcpu_idx);
3073                 if (psw_ioint_disabled(vcpu))
3074                         continue;
3075                 vcpu_isc_mask = (u8)(vcpu->arch.sie_block->gcr[6] >> 24);
3076                 if (deliverable_mask & vcpu_isc_mask) {
3077                         /* lately kicked but not yet running */
3078                         if (test_and_set_bit(vcpu_idx, gi->kicked_mask))
3079                                 return;
3080                         kvm_s390_vcpu_wakeup(vcpu);
3081                         return;
3082                 }
3083         }
3084 }
3085
3086 static enum hrtimer_restart gisa_vcpu_kicker(struct hrtimer *timer)
3087 {
3088         struct kvm_s390_gisa_interrupt *gi =
3089                 container_of(timer, struct kvm_s390_gisa_interrupt, timer);
3090         struct kvm *kvm =
3091                 container_of(gi->origin, struct sie_page2, gisa)->kvm;
3092         u8 pending_mask;
3093
3094         pending_mask = gisa_get_ipm_or_restore_iam(gi);
3095         if (pending_mask) {
3096                 __airqs_kick_single_vcpu(kvm, pending_mask);
3097                 hrtimer_forward_now(timer, ns_to_ktime(gi->expires));
3098                 return HRTIMER_RESTART;
3099         }
3100
3101         return HRTIMER_NORESTART;
3102 }
3103
3104 #define NULL_GISA_ADDR 0x00000000UL
3105 #define NONE_GISA_ADDR 0x00000001UL
3106 #define GISA_ADDR_MASK 0xfffff000UL
3107
3108 static void process_gib_alert_list(void)
3109 {
3110         struct kvm_s390_gisa_interrupt *gi;
3111         struct kvm_s390_gisa *gisa;
3112         struct kvm *kvm;
3113         u32 final, origin = 0UL;
3114
3115         do {
3116                 /*
3117                  * If the NONE_GISA_ADDR is still stored in the alert list
3118                  * origin, we will leave the outer loop. No further GISA has
3119                  * been added to the alert list by millicode while processing
3120                  * the current alert list.
3121                  */
3122                 final = (origin & NONE_GISA_ADDR);
3123                 /*
3124                  * Cut off the alert list and store the NONE_GISA_ADDR in the
3125                  * alert list origin to avoid further GAL interruptions.
3126                  * A new alert list can be build up by millicode in parallel
3127                  * for guests not in the yet cut-off alert list. When in the
3128                  * final loop, store the NULL_GISA_ADDR instead. This will re-
3129                  * enable GAL interruptions on the host again.
3130                  */
3131                 origin = xchg(&gib->alert_list_origin,
3132                               (!final) ? NONE_GISA_ADDR : NULL_GISA_ADDR);
3133                 /*
3134                  * Loop through the just cut-off alert list and start the
3135                  * gisa timers to kick idle vcpus to consume the pending
3136                  * interruptions asap.
3137                  */
3138                 while (origin & GISA_ADDR_MASK) {
3139                         gisa = (struct kvm_s390_gisa *)(u64)origin;
3140                         origin = gisa->next_alert;
3141                         gisa->next_alert = (u32)(u64)gisa;
3142                         kvm = container_of(gisa, struct sie_page2, gisa)->kvm;
3143                         gi = &kvm->arch.gisa_int;
3144                         if (hrtimer_active(&gi->timer))
3145                                 hrtimer_cancel(&gi->timer);
3146                         hrtimer_start(&gi->timer, 0, HRTIMER_MODE_REL);
3147                 }
3148         } while (!final);
3149
3150 }
3151
3152 void kvm_s390_gisa_clear(struct kvm *kvm)
3153 {
3154         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3155
3156         if (!gi->origin)
3157                 return;
3158         gisa_clear_ipm(gi->origin);
3159         VM_EVENT(kvm, 3, "gisa 0x%pK cleared", gi->origin);
3160 }
3161
3162 void kvm_s390_gisa_init(struct kvm *kvm)
3163 {
3164         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3165
3166         if (!css_general_characteristics.aiv)
3167                 return;
3168         gi->origin = &kvm->arch.sie_page2->gisa;
3169         gi->alert.mask = 0;
3170         spin_lock_init(&gi->alert.ref_lock);
3171         gi->expires = 50 * 1000; /* 50 usec */
3172         hrtimer_init(&gi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3173         gi->timer.function = gisa_vcpu_kicker;
3174         memset(gi->origin, 0, sizeof(struct kvm_s390_gisa));
3175         gi->origin->next_alert = (u32)(u64)gi->origin;
3176         VM_EVENT(kvm, 3, "gisa 0x%pK initialized", gi->origin);
3177 }
3178
3179 void kvm_s390_gisa_enable(struct kvm *kvm)
3180 {
3181         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3182         struct kvm_vcpu *vcpu;
3183         unsigned long i;
3184         u32 gisa_desc;
3185
3186         if (gi->origin)
3187                 return;
3188         kvm_s390_gisa_init(kvm);
3189         gisa_desc = kvm_s390_get_gisa_desc(kvm);
3190         if (!gisa_desc)
3191                 return;
3192         kvm_for_each_vcpu(i, vcpu, kvm) {
3193                 mutex_lock(&vcpu->mutex);
3194                 vcpu->arch.sie_block->gd = gisa_desc;
3195                 vcpu->arch.sie_block->eca |= ECA_AIV;
3196                 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
3197                            vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
3198                 mutex_unlock(&vcpu->mutex);
3199         }
3200 }
3201
3202 void kvm_s390_gisa_destroy(struct kvm *kvm)
3203 {
3204         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3205         struct kvm_s390_gisa *gisa = gi->origin;
3206
3207         if (!gi->origin)
3208                 return;
3209         if (gi->alert.mask)
3210                 KVM_EVENT(3, "vm 0x%pK has unexpected iam 0x%02x",
3211                           kvm, gi->alert.mask);
3212         while (gisa_in_alert_list(gi->origin))
3213                 cpu_relax();
3214         hrtimer_cancel(&gi->timer);
3215         gi->origin = NULL;
3216         VM_EVENT(kvm, 3, "gisa 0x%pK destroyed", gisa);
3217 }
3218
3219 void kvm_s390_gisa_disable(struct kvm *kvm)
3220 {
3221         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3222         struct kvm_vcpu *vcpu;
3223         unsigned long i;
3224
3225         if (!gi->origin)
3226                 return;
3227         kvm_for_each_vcpu(i, vcpu, kvm) {
3228                 mutex_lock(&vcpu->mutex);
3229                 vcpu->arch.sie_block->eca &= ~ECA_AIV;
3230                 vcpu->arch.sie_block->gd = 0U;
3231                 mutex_unlock(&vcpu->mutex);
3232                 VCPU_EVENT(vcpu, 3, "AIV disabled for cpu %03u", vcpu->vcpu_id);
3233         }
3234         kvm_s390_gisa_destroy(kvm);
3235 }
3236
3237 /**
3238  * kvm_s390_gisc_register - register a guest ISC
3239  *
3240  * @kvm:  the kernel vm to work with
3241  * @gisc: the guest interruption sub class to register
3242  *
3243  * The function extends the vm specific alert mask to use.
3244  * The effective IAM mask in the GISA is updated as well
3245  * in case the GISA is not part of the GIB alert list.
3246  * It will be updated latest when the IAM gets restored
3247  * by gisa_get_ipm_or_restore_iam().
3248  *
3249  * Returns: the nonspecific ISC (NISC) the gib alert mechanism
3250  *          has registered with the channel subsystem.
3251  *          -ENODEV in case the vm uses no GISA
3252  *          -ERANGE in case the guest ISC is invalid
3253  */
3254 int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc)
3255 {
3256         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3257
3258         if (!gi->origin)
3259                 return -ENODEV;
3260         if (gisc > MAX_ISC)
3261                 return -ERANGE;
3262
3263         spin_lock(&gi->alert.ref_lock);
3264         gi->alert.ref_count[gisc]++;
3265         if (gi->alert.ref_count[gisc] == 1) {
3266                 gi->alert.mask |= 0x80 >> gisc;
3267                 gisa_set_iam(gi->origin, gi->alert.mask);
3268         }
3269         spin_unlock(&gi->alert.ref_lock);
3270
3271         return gib->nisc;
3272 }
3273 EXPORT_SYMBOL_GPL(kvm_s390_gisc_register);
3274
3275 /**
3276  * kvm_s390_gisc_unregister - unregister a guest ISC
3277  *
3278  * @kvm:  the kernel vm to work with
3279  * @gisc: the guest interruption sub class to register
3280  *
3281  * The function reduces the vm specific alert mask to use.
3282  * The effective IAM mask in the GISA is updated as well
3283  * in case the GISA is not part of the GIB alert list.
3284  * It will be updated latest when the IAM gets restored
3285  * by gisa_get_ipm_or_restore_iam().
3286  *
3287  * Returns: the nonspecific ISC (NISC) the gib alert mechanism
3288  *          has registered with the channel subsystem.
3289  *          -ENODEV in case the vm uses no GISA
3290  *          -ERANGE in case the guest ISC is invalid
3291  *          -EINVAL in case the guest ISC is not registered
3292  */
3293 int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc)
3294 {
3295         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3296         int rc = 0;
3297
3298         if (!gi->origin)
3299                 return -ENODEV;
3300         if (gisc > MAX_ISC)
3301                 return -ERANGE;
3302
3303         spin_lock(&gi->alert.ref_lock);
3304         if (gi->alert.ref_count[gisc] == 0) {
3305                 rc = -EINVAL;
3306                 goto out;
3307         }
3308         gi->alert.ref_count[gisc]--;
3309         if (gi->alert.ref_count[gisc] == 0) {
3310                 gi->alert.mask &= ~(0x80 >> gisc);
3311                 gisa_set_iam(gi->origin, gi->alert.mask);
3312         }
3313 out:
3314         spin_unlock(&gi->alert.ref_lock);
3315
3316         return rc;
3317 }
3318 EXPORT_SYMBOL_GPL(kvm_s390_gisc_unregister);
3319
3320 static void aen_host_forward(unsigned long si)
3321 {
3322         struct kvm_s390_gisa_interrupt *gi;
3323         struct zpci_gaite *gaite;
3324         struct kvm *kvm;
3325
3326         gaite = (struct zpci_gaite *)aift->gait +
3327                 (si * sizeof(struct zpci_gaite));
3328         if (gaite->count == 0)
3329                 return;
3330         if (gaite->aisb != 0)
3331                 set_bit_inv(gaite->aisbo, phys_to_virt(gaite->aisb));
3332
3333         kvm = kvm_s390_pci_si_to_kvm(aift, si);
3334         if (!kvm)
3335                 return;
3336         gi = &kvm->arch.gisa_int;
3337
3338         if (!(gi->origin->g1.simm & AIS_MODE_MASK(gaite->gisc)) ||
3339             !(gi->origin->g1.nimm & AIS_MODE_MASK(gaite->gisc))) {
3340                 gisa_set_ipm_gisc(gi->origin, gaite->gisc);
3341                 if (hrtimer_active(&gi->timer))
3342                         hrtimer_cancel(&gi->timer);
3343                 hrtimer_start(&gi->timer, 0, HRTIMER_MODE_REL);
3344                 kvm->stat.aen_forward++;
3345         }
3346 }
3347
3348 static void aen_process_gait(u8 isc)
3349 {
3350         bool found = false, first = true;
3351         union zpci_sic_iib iib = {{0}};
3352         unsigned long si, flags;
3353
3354         spin_lock_irqsave(&aift->gait_lock, flags);
3355
3356         if (!aift->gait) {
3357                 spin_unlock_irqrestore(&aift->gait_lock, flags);
3358                 return;
3359         }
3360
3361         for (si = 0;;) {
3362                 /* Scan adapter summary indicator bit vector */
3363                 si = airq_iv_scan(aift->sbv, si, airq_iv_end(aift->sbv));
3364                 if (si == -1UL) {
3365                         if (first || found) {
3366                                 /* Re-enable interrupts. */
3367                                 zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, isc,
3368                                                   &iib);
3369                                 first = found = false;
3370                         } else {
3371                                 /* Interrupts on and all bits processed */
3372                                 break;
3373                         }
3374                         found = false;
3375                         si = 0;
3376                         /* Scan again after re-enabling interrupts */
3377                         continue;
3378                 }
3379                 found = true;
3380                 aen_host_forward(si);
3381         }
3382
3383         spin_unlock_irqrestore(&aift->gait_lock, flags);
3384 }
3385
3386 static void gib_alert_irq_handler(struct airq_struct *airq,
3387                                   struct tpi_info *tpi_info)
3388 {
3389         struct tpi_adapter_info *info = (struct tpi_adapter_info *)tpi_info;
3390
3391         inc_irq_stat(IRQIO_GAL);
3392
3393         if ((info->forward || info->error) &&
3394             IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) {
3395                 aen_process_gait(info->isc);
3396                 if (info->aism != 0)
3397                         process_gib_alert_list();
3398         } else {
3399                 process_gib_alert_list();
3400         }
3401 }
3402
3403 static struct airq_struct gib_alert_irq = {
3404         .handler = gib_alert_irq_handler,
3405         .lsi_ptr = &gib_alert_irq.lsi_mask,
3406 };
3407
3408 void kvm_s390_gib_destroy(void)
3409 {
3410         if (!gib)
3411                 return;
3412         if (kvm_s390_pci_interp_allowed() && aift) {
3413                 mutex_lock(&aift->aift_lock);
3414                 kvm_s390_pci_aen_exit();
3415                 mutex_unlock(&aift->aift_lock);
3416         }
3417         chsc_sgib(0);
3418         unregister_adapter_interrupt(&gib_alert_irq);
3419         free_page((unsigned long)gib);
3420         gib = NULL;
3421 }
3422
3423 int kvm_s390_gib_init(u8 nisc)
3424 {
3425         int rc = 0;
3426
3427         if (!css_general_characteristics.aiv) {
3428                 KVM_EVENT(3, "%s", "gib not initialized, no AIV facility");
3429                 goto out;
3430         }
3431
3432         gib = (struct kvm_s390_gib *)get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA);
3433         if (!gib) {
3434                 rc = -ENOMEM;
3435                 goto out;
3436         }
3437
3438         gib_alert_irq.isc = nisc;
3439         if (register_adapter_interrupt(&gib_alert_irq)) {
3440                 pr_err("Registering the GIB alert interruption handler failed\n");
3441                 rc = -EIO;
3442                 goto out_free_gib;
3443         }
3444
3445         gib->nisc = nisc;
3446         if (chsc_sgib((u32)(u64)gib)) {
3447                 pr_err("Associating the GIB with the AIV facility failed\n");
3448                 free_page((unsigned long)gib);
3449                 gib = NULL;
3450                 rc = -EIO;
3451                 goto out_unreg_gal;
3452         }
3453
3454         if (kvm_s390_pci_interp_allowed()) {
3455                 if (kvm_s390_pci_aen_init(nisc)) {
3456                         pr_err("Initializing AEN for PCI failed\n");
3457                         rc = -EIO;
3458                         goto out_unreg_gal;
3459                 }
3460         }
3461
3462         KVM_EVENT(3, "gib 0x%pK (nisc=%d) initialized", gib, gib->nisc);
3463         goto out;
3464
3465 out_unreg_gal:
3466         unregister_adapter_interrupt(&gib_alert_irq);
3467 out_free_gib:
3468         free_page((unsigned long)gib);
3469         gib = NULL;
3470 out:
3471         return rc;
3472 }