1 // SPDX-License-Identifier: GPL-2.0
3 * handling kvm guest interrupts
5 * Copyright IBM Corp. 2008, 2015
7 * Author(s): Carsten Otte <cotte@de.ibm.com>
10 #define KMSG_COMPONENT "kvm-s390"
11 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
13 #include <linux/interrupt.h>
14 #include <linux/kvm_host.h>
15 #include <linux/hrtimer.h>
16 #include <linux/mmu_context.h>
17 #include <linux/nospec.h>
18 #include <linux/signal.h>
19 #include <linux/slab.h>
20 #include <linux/bitmap.h>
21 #include <linux/vmalloc.h>
22 #include <asm/asm-offsets.h>
24 #include <linux/uaccess.h>
28 #include <asm/switch_to.h>
33 #include "trace-s390.h"
35 #define PFAULT_INIT 0x0600
36 #define PFAULT_DONE 0x0680
37 #define VIRTIO_PARAM 0x0d00
39 static struct kvm_s390_gib *gib;
41 /* handle external calls via sigp interpretation facility */
42 static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id)
46 if (!kvm_s390_test_cpuflags(vcpu, CPUSTAT_ECALL_PEND))
49 BUG_ON(!kvm_s390_use_sca_entries());
50 read_lock(&vcpu->kvm->arch.sca_lock);
51 if (vcpu->kvm->arch.use_esca) {
52 struct esca_block *sca = vcpu->kvm->arch.sca;
53 union esca_sigp_ctrl sigp_ctrl =
54 sca->cpu[vcpu->vcpu_id].sigp_ctrl;
59 struct bsca_block *sca = vcpu->kvm->arch.sca;
60 union bsca_sigp_ctrl sigp_ctrl =
61 sca->cpu[vcpu->vcpu_id].sigp_ctrl;
66 read_unlock(&vcpu->kvm->arch.sca_lock);
74 static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id)
78 BUG_ON(!kvm_s390_use_sca_entries());
79 read_lock(&vcpu->kvm->arch.sca_lock);
80 if (vcpu->kvm->arch.use_esca) {
81 struct esca_block *sca = vcpu->kvm->arch.sca;
82 union esca_sigp_ctrl *sigp_ctrl =
83 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
84 union esca_sigp_ctrl new_val = {0}, old_val;
86 old_val = READ_ONCE(*sigp_ctrl);
91 expect = old_val.value;
92 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
94 struct bsca_block *sca = vcpu->kvm->arch.sca;
95 union bsca_sigp_ctrl *sigp_ctrl =
96 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
97 union bsca_sigp_ctrl new_val = {0}, old_val;
99 old_val = READ_ONCE(*sigp_ctrl);
100 new_val.scn = src_id;
104 expect = old_val.value;
105 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
107 read_unlock(&vcpu->kvm->arch.sca_lock);
110 /* another external call is pending */
113 kvm_s390_set_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
117 static void sca_clear_ext_call(struct kvm_vcpu *vcpu)
121 if (!kvm_s390_use_sca_entries())
123 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
124 read_lock(&vcpu->kvm->arch.sca_lock);
125 if (vcpu->kvm->arch.use_esca) {
126 struct esca_block *sca = vcpu->kvm->arch.sca;
127 union esca_sigp_ctrl *sigp_ctrl =
128 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
129 union esca_sigp_ctrl old;
131 old = READ_ONCE(*sigp_ctrl);
133 rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
135 struct bsca_block *sca = vcpu->kvm->arch.sca;
136 union bsca_sigp_ctrl *sigp_ctrl =
137 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
138 union bsca_sigp_ctrl old;
140 old = READ_ONCE(*sigp_ctrl);
142 rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
144 read_unlock(&vcpu->kvm->arch.sca_lock);
145 WARN_ON(rc != expect); /* cannot clear? */
148 int psw_extint_disabled(struct kvm_vcpu *vcpu)
150 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT);
153 static int psw_ioint_disabled(struct kvm_vcpu *vcpu)
155 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO);
158 static int psw_mchk_disabled(struct kvm_vcpu *vcpu)
160 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_MCHECK);
163 static int psw_interrupts_disabled(struct kvm_vcpu *vcpu)
165 return psw_extint_disabled(vcpu) &&
166 psw_ioint_disabled(vcpu) &&
167 psw_mchk_disabled(vcpu);
170 static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu)
172 if (psw_extint_disabled(vcpu) ||
173 !(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
175 if (guestdbg_enabled(vcpu) && guestdbg_sstep_enabled(vcpu))
176 /* No timer interrupts when single stepping */
181 static int ckc_irq_pending(struct kvm_vcpu *vcpu)
183 const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
184 const u64 ckc = vcpu->arch.sie_block->ckc;
186 if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
187 if ((s64)ckc >= (s64)now)
189 } else if (ckc >= now) {
192 return ckc_interrupts_enabled(vcpu);
195 static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu)
197 return !psw_extint_disabled(vcpu) &&
198 (vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK);
201 static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu)
203 if (!cpu_timer_interrupts_enabled(vcpu))
205 return kvm_s390_get_cpu_timer(vcpu) >> 63;
208 static uint64_t isc_to_isc_bits(int isc)
210 return (0x80 >> isc) << 24;
213 static inline u32 isc_to_int_word(u8 isc)
215 return ((u32)isc << 27) | 0x80000000;
218 static inline u8 int_word_to_isc(u32 int_word)
220 return (int_word & 0x38000000) >> 27;
224 * To use atomic bitmap functions, we have to provide a bitmap address
225 * that is u64 aligned. However, the ipm might be u32 aligned.
226 * Therefore, we logically start the bitmap at the very beginning of the
227 * struct and fixup the bit number.
229 #define IPM_BIT_OFFSET (offsetof(struct kvm_s390_gisa, ipm) * BITS_PER_BYTE)
232 * gisa_set_iam - change the GISA interruption alert mask
234 * @gisa: gisa to operate on
235 * @iam: new IAM value to use
237 * Change the IAM atomically with the next alert address and the IPM
238 * of the GISA if the GISA is not part of the GIB alert list. All three
239 * fields are located in the first long word of the GISA.
241 * Returns: 0 on success
242 * -EBUSY in case the gisa is part of the alert list
244 static inline int gisa_set_iam(struct kvm_s390_gisa *gisa, u8 iam)
249 word = READ_ONCE(gisa->u64.word[0]);
250 if ((u64)gisa != word >> 32)
252 _word = (word & ~0xffUL) | iam;
253 } while (cmpxchg(&gisa->u64.word[0], word, _word) != word);
259 * gisa_clear_ipm - clear the GISA interruption pending mask
261 * @gisa: gisa to operate on
263 * Clear the IPM atomically with the next alert address and the IAM
264 * of the GISA unconditionally. All three fields are located in the
265 * first long word of the GISA.
267 static inline void gisa_clear_ipm(struct kvm_s390_gisa *gisa)
272 word = READ_ONCE(gisa->u64.word[0]);
273 _word = word & ~(0xffUL << 24);
274 } while (cmpxchg(&gisa->u64.word[0], word, _word) != word);
278 * gisa_get_ipm_or_restore_iam - return IPM or restore GISA IAM
280 * @gi: gisa interrupt struct to work on
282 * Atomically restores the interruption alert mask if none of the
283 * relevant ISCs are pending and return the IPM.
285 * Returns: the relevant pending ISCs
287 static inline u8 gisa_get_ipm_or_restore_iam(struct kvm_s390_gisa_interrupt *gi)
289 u8 pending_mask, alert_mask;
293 word = READ_ONCE(gi->origin->u64.word[0]);
294 alert_mask = READ_ONCE(gi->alert.mask);
295 pending_mask = (u8)(word >> 24) & alert_mask;
298 _word = (word & ~0xffUL) | alert_mask;
299 } while (cmpxchg(&gi->origin->u64.word[0], word, _word) != word);
304 static inline int gisa_in_alert_list(struct kvm_s390_gisa *gisa)
306 return READ_ONCE(gisa->next_alert) != (u32)(u64)gisa;
309 static inline void gisa_set_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
311 set_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
314 static inline u8 gisa_get_ipm(struct kvm_s390_gisa *gisa)
316 return READ_ONCE(gisa->ipm);
319 static inline void gisa_clear_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
321 clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
324 static inline int gisa_tac_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
326 return test_and_clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
329 static inline unsigned long pending_irqs_no_gisa(struct kvm_vcpu *vcpu)
331 return vcpu->kvm->arch.float_int.pending_irqs |
332 vcpu->arch.local_int.pending_irqs;
335 static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu)
337 struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
338 unsigned long pending_mask;
340 pending_mask = pending_irqs_no_gisa(vcpu);
342 pending_mask |= gisa_get_ipm(gi->origin) << IRQ_PEND_IO_ISC_7;
346 static inline int isc_to_irq_type(unsigned long isc)
348 return IRQ_PEND_IO_ISC_0 - isc;
351 static inline int irq_type_to_isc(unsigned long irq_type)
353 return IRQ_PEND_IO_ISC_0 - irq_type;
356 static unsigned long disable_iscs(struct kvm_vcpu *vcpu,
357 unsigned long active_mask)
361 for (i = 0; i <= MAX_ISC; i++)
362 if (!(vcpu->arch.sie_block->gcr[6] & isc_to_isc_bits(i)))
363 active_mask &= ~(1UL << (isc_to_irq_type(i)));
368 static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu)
370 unsigned long active_mask;
372 active_mask = pending_irqs(vcpu);
376 if (psw_extint_disabled(vcpu))
377 active_mask &= ~IRQ_PEND_EXT_MASK;
378 if (psw_ioint_disabled(vcpu))
379 active_mask &= ~IRQ_PEND_IO_MASK;
381 active_mask = disable_iscs(vcpu, active_mask);
382 if (!(vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
383 __clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask);
384 if (!(vcpu->arch.sie_block->gcr[0] & CR0_EMERGENCY_SIGNAL_SUBMASK))
385 __clear_bit(IRQ_PEND_EXT_EMERGENCY, &active_mask);
386 if (!(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
387 __clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask);
388 if (!(vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK))
389 __clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask);
390 if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
391 __clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask);
392 if (psw_mchk_disabled(vcpu))
393 active_mask &= ~IRQ_PEND_MCHK_MASK;
395 * Check both floating and local interrupt's cr14 because
396 * bit IRQ_PEND_MCHK_REP could be set in both cases.
398 if (!(vcpu->arch.sie_block->gcr[14] &
399 (vcpu->kvm->arch.float_int.mchk.cr14 |
400 vcpu->arch.local_int.irq.mchk.cr14)))
401 __clear_bit(IRQ_PEND_MCHK_REP, &active_mask);
404 * STOP irqs will never be actively delivered. They are triggered via
405 * intercept requests and cleared when the stop intercept is performed.
407 __clear_bit(IRQ_PEND_SIGP_STOP, &active_mask);
412 static void __set_cpu_idle(struct kvm_vcpu *vcpu)
414 kvm_s390_set_cpuflags(vcpu, CPUSTAT_WAIT);
415 set_bit(kvm_vcpu_get_idx(vcpu), vcpu->kvm->arch.idle_mask);
418 static void __unset_cpu_idle(struct kvm_vcpu *vcpu)
420 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_WAIT);
421 clear_bit(kvm_vcpu_get_idx(vcpu), vcpu->kvm->arch.idle_mask);
424 static void __reset_intercept_indicators(struct kvm_vcpu *vcpu)
426 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IO_INT | CPUSTAT_EXT_INT |
428 vcpu->arch.sie_block->lctl = 0x0000;
429 vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT);
431 if (guestdbg_enabled(vcpu)) {
432 vcpu->arch.sie_block->lctl |= (LCTL_CR0 | LCTL_CR9 |
433 LCTL_CR10 | LCTL_CR11);
434 vcpu->arch.sie_block->ictl |= (ICTL_STCTL | ICTL_PINT);
438 static void set_intercept_indicators_io(struct kvm_vcpu *vcpu)
440 if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_IO_MASK))
442 if (psw_ioint_disabled(vcpu))
443 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IO_INT);
445 vcpu->arch.sie_block->lctl |= LCTL_CR6;
448 static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu)
450 if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_EXT_MASK))
452 if (psw_extint_disabled(vcpu))
453 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
455 vcpu->arch.sie_block->lctl |= LCTL_CR0;
458 static void set_intercept_indicators_mchk(struct kvm_vcpu *vcpu)
460 if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_MCHK_MASK))
462 if (psw_mchk_disabled(vcpu))
463 vcpu->arch.sie_block->ictl |= ICTL_LPSW;
465 vcpu->arch.sie_block->lctl |= LCTL_CR14;
468 static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu)
470 if (kvm_s390_is_stop_irq_pending(vcpu))
471 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
474 /* Set interception request for non-deliverable interrupts */
475 static void set_intercept_indicators(struct kvm_vcpu *vcpu)
477 set_intercept_indicators_io(vcpu);
478 set_intercept_indicators_ext(vcpu);
479 set_intercept_indicators_mchk(vcpu);
480 set_intercept_indicators_stop(vcpu);
483 static int __must_check __deliver_cpu_timer(struct kvm_vcpu *vcpu)
485 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
488 vcpu->stat.deliver_cputm++;
489 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
492 rc = put_guest_lc(vcpu, EXT_IRQ_CPU_TIMER,
493 (u16 *)__LC_EXT_INT_CODE);
494 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
495 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
496 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
497 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
498 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
499 clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
500 return rc ? -EFAULT : 0;
503 static int __must_check __deliver_ckc(struct kvm_vcpu *vcpu)
505 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
508 vcpu->stat.deliver_ckc++;
509 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
512 rc = put_guest_lc(vcpu, EXT_IRQ_CLK_COMP,
513 (u16 __user *)__LC_EXT_INT_CODE);
514 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
515 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
516 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
517 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
518 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
519 clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
520 return rc ? -EFAULT : 0;
523 static int __must_check __deliver_pfault_init(struct kvm_vcpu *vcpu)
525 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
526 struct kvm_s390_ext_info ext;
529 spin_lock(&li->lock);
531 clear_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
532 li->irq.ext.ext_params2 = 0;
533 spin_unlock(&li->lock);
535 VCPU_EVENT(vcpu, 4, "deliver: pfault init token 0x%llx",
537 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
538 KVM_S390_INT_PFAULT_INIT,
541 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *) __LC_EXT_INT_CODE);
542 rc |= put_guest_lc(vcpu, PFAULT_INIT, (u16 *) __LC_EXT_CPU_ADDR);
543 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
544 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
545 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
546 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
547 rc |= put_guest_lc(vcpu, ext.ext_params2, (u64 *) __LC_EXT_PARAMS2);
548 return rc ? -EFAULT : 0;
551 static int __write_machine_check(struct kvm_vcpu *vcpu,
552 struct kvm_s390_mchk_info *mchk)
554 unsigned long ext_sa_addr;
556 freg_t fprs[NUM_FPRS];
560 mci.val = mchk->mcic;
561 /* take care of lazy register loading */
563 save_access_regs(vcpu->run->s.regs.acrs);
564 if (MACHINE_HAS_GS && vcpu->arch.gs_enabled)
565 save_gs_cb(current->thread.gs_cb);
567 /* Extended save area */
568 rc = read_guest_lc(vcpu, __LC_MCESAD, &ext_sa_addr,
569 sizeof(unsigned long));
570 /* Only bits 0 through 63-LC are used for address formation */
571 lc = ext_sa_addr & MCESA_LC_MASK;
572 if (test_kvm_facility(vcpu->kvm, 133)) {
576 ext_sa_addr &= ~0x3ffUL;
579 ext_sa_addr &= ~0x7ffUL;
582 ext_sa_addr &= ~0xfffUL;
589 ext_sa_addr &= ~0x3ffUL;
592 if (!rc && mci.vr && ext_sa_addr && test_kvm_facility(vcpu->kvm, 129)) {
593 if (write_guest_abs(vcpu, ext_sa_addr, vcpu->run->s.regs.vrs,
599 if (!rc && mci.gs && ext_sa_addr && test_kvm_facility(vcpu->kvm, 133)
600 && (lc == 11 || lc == 12)) {
601 if (write_guest_abs(vcpu, ext_sa_addr + 1024,
602 &vcpu->run->s.regs.gscb, 32))
608 /* General interruption information */
609 rc |= put_guest_lc(vcpu, 1, (u8 __user *) __LC_AR_MODE_ID);
610 rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW,
611 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
612 rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW,
613 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
614 rc |= put_guest_lc(vcpu, mci.val, (u64 __user *) __LC_MCCK_CODE);
616 /* Register-save areas */
617 if (MACHINE_HAS_VX) {
618 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
619 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, fprs, 128);
621 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA,
622 vcpu->run->s.regs.fprs, 128);
624 rc |= write_guest_lc(vcpu, __LC_GPREGS_SAVE_AREA,
625 vcpu->run->s.regs.gprs, 128);
626 rc |= put_guest_lc(vcpu, current->thread.fpu.fpc,
627 (u32 __user *) __LC_FP_CREG_SAVE_AREA);
628 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->todpr,
629 (u32 __user *) __LC_TOD_PROGREG_SAVE_AREA);
630 rc |= put_guest_lc(vcpu, kvm_s390_get_cpu_timer(vcpu),
631 (u64 __user *) __LC_CPU_TIMER_SAVE_AREA);
632 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->ckc >> 8,
633 (u64 __user *) __LC_CLOCK_COMP_SAVE_AREA);
634 rc |= write_guest_lc(vcpu, __LC_AREGS_SAVE_AREA,
635 &vcpu->run->s.regs.acrs, 64);
636 rc |= write_guest_lc(vcpu, __LC_CREGS_SAVE_AREA,
637 &vcpu->arch.sie_block->gcr, 128);
639 /* Extended interruption information */
640 rc |= put_guest_lc(vcpu, mchk->ext_damage_code,
641 (u32 __user *) __LC_EXT_DAMAGE_CODE);
642 rc |= put_guest_lc(vcpu, mchk->failing_storage_address,
643 (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR);
644 rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA, &mchk->fixed_logout,
645 sizeof(mchk->fixed_logout));
646 return rc ? -EFAULT : 0;
649 static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu)
651 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
652 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
653 struct kvm_s390_mchk_info mchk = {};
657 spin_lock(&fi->lock);
658 spin_lock(&li->lock);
659 if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) ||
660 test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) {
662 * If there was an exigent machine check pending, then any
663 * repressible machine checks that might have been pending
664 * are indicated along with it, so always clear bits for
665 * repressible and exigent interrupts
668 clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
669 clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs);
670 memset(&li->irq.mchk, 0, sizeof(mchk));
674 * We indicate floating repressible conditions along with
675 * other pending conditions. Channel Report Pending and Channel
676 * Subsystem damage are the only two and and are indicated by
677 * bits in mcic and masked in cr14.
679 if (test_and_clear_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
680 mchk.mcic |= fi->mchk.mcic;
681 mchk.cr14 |= fi->mchk.cr14;
682 memset(&fi->mchk, 0, sizeof(mchk));
685 spin_unlock(&li->lock);
686 spin_unlock(&fi->lock);
689 VCPU_EVENT(vcpu, 3, "deliver: machine check mcic 0x%llx",
691 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
693 mchk.cr14, mchk.mcic);
694 vcpu->stat.deliver_machine_check++;
695 rc = __write_machine_check(vcpu, &mchk);
700 static int __must_check __deliver_restart(struct kvm_vcpu *vcpu)
702 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
705 VCPU_EVENT(vcpu, 3, "%s", "deliver: cpu restart");
706 vcpu->stat.deliver_restart_signal++;
707 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);
709 rc = write_guest_lc(vcpu,
710 offsetof(struct lowcore, restart_old_psw),
711 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
712 rc |= read_guest_lc(vcpu, offsetof(struct lowcore, restart_psw),
713 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
714 clear_bit(IRQ_PEND_RESTART, &li->pending_irqs);
715 return rc ? -EFAULT : 0;
718 static int __must_check __deliver_set_prefix(struct kvm_vcpu *vcpu)
720 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
721 struct kvm_s390_prefix_info prefix;
723 spin_lock(&li->lock);
724 prefix = li->irq.prefix;
725 li->irq.prefix.address = 0;
726 clear_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
727 spin_unlock(&li->lock);
729 vcpu->stat.deliver_prefix_signal++;
730 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
731 KVM_S390_SIGP_SET_PREFIX,
734 kvm_s390_set_prefix(vcpu, prefix.address);
738 static int __must_check __deliver_emergency_signal(struct kvm_vcpu *vcpu)
740 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
744 spin_lock(&li->lock);
745 cpu_addr = find_first_bit(li->sigp_emerg_pending, KVM_MAX_VCPUS);
746 clear_bit(cpu_addr, li->sigp_emerg_pending);
747 if (bitmap_empty(li->sigp_emerg_pending, KVM_MAX_VCPUS))
748 clear_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
749 spin_unlock(&li->lock);
751 VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp emerg");
752 vcpu->stat.deliver_emergency_signal++;
753 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
756 rc = put_guest_lc(vcpu, EXT_IRQ_EMERGENCY_SIG,
757 (u16 *)__LC_EXT_INT_CODE);
758 rc |= put_guest_lc(vcpu, cpu_addr, (u16 *)__LC_EXT_CPU_ADDR);
759 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
760 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
761 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
762 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
763 return rc ? -EFAULT : 0;
766 static int __must_check __deliver_external_call(struct kvm_vcpu *vcpu)
768 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
769 struct kvm_s390_extcall_info extcall;
772 spin_lock(&li->lock);
773 extcall = li->irq.extcall;
774 li->irq.extcall.code = 0;
775 clear_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
776 spin_unlock(&li->lock);
778 VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp ext call");
779 vcpu->stat.deliver_external_call++;
780 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
781 KVM_S390_INT_EXTERNAL_CALL,
784 rc = put_guest_lc(vcpu, EXT_IRQ_EXTERNAL_CALL,
785 (u16 *)__LC_EXT_INT_CODE);
786 rc |= put_guest_lc(vcpu, extcall.code, (u16 *)__LC_EXT_CPU_ADDR);
787 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
788 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
789 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, &vcpu->arch.sie_block->gpsw,
791 return rc ? -EFAULT : 0;
794 static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
796 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
797 struct kvm_s390_pgm_info pgm_info;
798 int rc = 0, nullifying = false;
801 spin_lock(&li->lock);
802 pgm_info = li->irq.pgm;
803 clear_bit(IRQ_PEND_PROG, &li->pending_irqs);
804 memset(&li->irq.pgm, 0, sizeof(pgm_info));
805 spin_unlock(&li->lock);
807 ilen = pgm_info.flags & KVM_S390_PGM_FLAGS_ILC_MASK;
808 VCPU_EVENT(vcpu, 3, "deliver: program irq code 0x%x, ilen:%d",
809 pgm_info.code, ilen);
810 vcpu->stat.deliver_program++;
811 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
814 switch (pgm_info.code & ~PGM_PER) {
815 case PGM_AFX_TRANSLATION:
816 case PGM_ASX_TRANSLATION:
817 case PGM_EX_TRANSLATION:
818 case PGM_LFX_TRANSLATION:
819 case PGM_LSTE_SEQUENCE:
820 case PGM_LSX_TRANSLATION:
821 case PGM_LX_TRANSLATION:
822 case PGM_PRIMARY_AUTHORITY:
823 case PGM_SECONDARY_AUTHORITY:
826 case PGM_SPACE_SWITCH:
827 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
828 (u64 *)__LC_TRANS_EXC_CODE);
830 case PGM_ALEN_TRANSLATION:
831 case PGM_ALE_SEQUENCE:
832 case PGM_ASTE_INSTANCE:
833 case PGM_ASTE_SEQUENCE:
834 case PGM_ASTE_VALIDITY:
835 case PGM_EXTENDED_AUTHORITY:
836 rc = put_guest_lc(vcpu, pgm_info.exc_access_id,
837 (u8 *)__LC_EXC_ACCESS_ID);
841 case PGM_PAGE_TRANSLATION:
842 case PGM_REGION_FIRST_TRANS:
843 case PGM_REGION_SECOND_TRANS:
844 case PGM_REGION_THIRD_TRANS:
845 case PGM_SEGMENT_TRANSLATION:
846 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
847 (u64 *)__LC_TRANS_EXC_CODE);
848 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
849 (u8 *)__LC_EXC_ACCESS_ID);
850 rc |= put_guest_lc(vcpu, pgm_info.op_access_id,
851 (u8 *)__LC_OP_ACCESS_ID);
855 rc = put_guest_lc(vcpu, pgm_info.mon_class_nr,
856 (u16 *)__LC_MON_CLASS_NR);
857 rc |= put_guest_lc(vcpu, pgm_info.mon_code,
858 (u64 *)__LC_MON_CODE);
860 case PGM_VECTOR_PROCESSING:
862 rc = put_guest_lc(vcpu, pgm_info.data_exc_code,
863 (u32 *)__LC_DATA_EXC_CODE);
866 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
867 (u64 *)__LC_TRANS_EXC_CODE);
868 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
869 (u8 *)__LC_EXC_ACCESS_ID);
872 case PGM_STACK_EMPTY:
873 case PGM_STACK_SPECIFICATION:
875 case PGM_STACK_OPERATION:
876 case PGM_TRACE_TABEL:
877 case PGM_CRYPTO_OPERATION:
882 if (pgm_info.code & PGM_PER) {
883 rc |= put_guest_lc(vcpu, pgm_info.per_code,
884 (u8 *) __LC_PER_CODE);
885 rc |= put_guest_lc(vcpu, pgm_info.per_atmid,
886 (u8 *)__LC_PER_ATMID);
887 rc |= put_guest_lc(vcpu, pgm_info.per_address,
888 (u64 *) __LC_PER_ADDRESS);
889 rc |= put_guest_lc(vcpu, pgm_info.per_access_id,
890 (u8 *) __LC_PER_ACCESS_ID);
893 if (nullifying && !(pgm_info.flags & KVM_S390_PGM_FLAGS_NO_REWIND))
894 kvm_s390_rewind_psw(vcpu, ilen);
896 /* bit 1+2 of the target are the ilc, so we can directly use ilen */
897 rc |= put_guest_lc(vcpu, ilen, (u16 *) __LC_PGM_ILC);
898 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->gbea,
899 (u64 *) __LC_LAST_BREAK);
900 rc |= put_guest_lc(vcpu, pgm_info.code,
901 (u16 *)__LC_PGM_INT_CODE);
902 rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW,
903 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
904 rc |= read_guest_lc(vcpu, __LC_PGM_NEW_PSW,
905 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
906 return rc ? -EFAULT : 0;
909 static int __must_check __deliver_service(struct kvm_vcpu *vcpu)
911 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
912 struct kvm_s390_ext_info ext;
915 spin_lock(&fi->lock);
916 if (!(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) {
917 spin_unlock(&fi->lock);
920 ext = fi->srv_signal;
921 memset(&fi->srv_signal, 0, sizeof(ext));
922 clear_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
923 spin_unlock(&fi->lock);
925 VCPU_EVENT(vcpu, 4, "deliver: sclp parameter 0x%x",
927 vcpu->stat.deliver_service_signal++;
928 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
931 rc = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE);
932 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
933 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
934 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
935 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
936 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
937 rc |= put_guest_lc(vcpu, ext.ext_params,
938 (u32 *)__LC_EXT_PARAMS);
940 return rc ? -EFAULT : 0;
943 static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu)
945 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
946 struct kvm_s390_interrupt_info *inti;
949 spin_lock(&fi->lock);
950 inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT],
951 struct kvm_s390_interrupt_info,
954 list_del(&inti->list);
955 fi->counters[FIRQ_CNTR_PFAULT] -= 1;
957 if (list_empty(&fi->lists[FIRQ_LIST_PFAULT]))
958 clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
959 spin_unlock(&fi->lock);
962 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
963 KVM_S390_INT_PFAULT_DONE, 0,
964 inti->ext.ext_params2);
965 VCPU_EVENT(vcpu, 4, "deliver: pfault done token 0x%llx",
966 inti->ext.ext_params2);
968 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
969 (u16 *)__LC_EXT_INT_CODE);
970 rc |= put_guest_lc(vcpu, PFAULT_DONE,
971 (u16 *)__LC_EXT_CPU_ADDR);
972 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
973 &vcpu->arch.sie_block->gpsw,
975 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
976 &vcpu->arch.sie_block->gpsw,
978 rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
979 (u64 *)__LC_EXT_PARAMS2);
982 return rc ? -EFAULT : 0;
985 static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu)
987 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
988 struct kvm_s390_interrupt_info *inti;
991 spin_lock(&fi->lock);
992 inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO],
993 struct kvm_s390_interrupt_info,
997 "deliver: virtio parm: 0x%x,parm64: 0x%llx",
998 inti->ext.ext_params, inti->ext.ext_params2);
999 vcpu->stat.deliver_virtio++;
1000 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1002 inti->ext.ext_params,
1003 inti->ext.ext_params2);
1004 list_del(&inti->list);
1005 fi->counters[FIRQ_CNTR_VIRTIO] -= 1;
1007 if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO]))
1008 clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
1009 spin_unlock(&fi->lock);
1012 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
1013 (u16 *)__LC_EXT_INT_CODE);
1014 rc |= put_guest_lc(vcpu, VIRTIO_PARAM,
1015 (u16 *)__LC_EXT_CPU_ADDR);
1016 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
1017 &vcpu->arch.sie_block->gpsw,
1019 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
1020 &vcpu->arch.sie_block->gpsw,
1022 rc |= put_guest_lc(vcpu, inti->ext.ext_params,
1023 (u32 *)__LC_EXT_PARAMS);
1024 rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
1025 (u64 *)__LC_EXT_PARAMS2);
1028 return rc ? -EFAULT : 0;
1031 static int __do_deliver_io(struct kvm_vcpu *vcpu, struct kvm_s390_io_info *io)
1035 rc = put_guest_lc(vcpu, io->subchannel_id, (u16 *)__LC_SUBCHANNEL_ID);
1036 rc |= put_guest_lc(vcpu, io->subchannel_nr, (u16 *)__LC_SUBCHANNEL_NR);
1037 rc |= put_guest_lc(vcpu, io->io_int_parm, (u32 *)__LC_IO_INT_PARM);
1038 rc |= put_guest_lc(vcpu, io->io_int_word, (u32 *)__LC_IO_INT_WORD);
1039 rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW,
1040 &vcpu->arch.sie_block->gpsw,
1042 rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW,
1043 &vcpu->arch.sie_block->gpsw,
1045 return rc ? -EFAULT : 0;
1048 static int __must_check __deliver_io(struct kvm_vcpu *vcpu,
1049 unsigned long irq_type)
1051 struct list_head *isc_list;
1052 struct kvm_s390_float_interrupt *fi;
1053 struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
1054 struct kvm_s390_interrupt_info *inti = NULL;
1055 struct kvm_s390_io_info io;
1059 fi = &vcpu->kvm->arch.float_int;
1061 spin_lock(&fi->lock);
1062 isc = irq_type_to_isc(irq_type);
1063 isc_list = &fi->lists[isc];
1064 inti = list_first_entry_or_null(isc_list,
1065 struct kvm_s390_interrupt_info,
1068 if (inti->type & KVM_S390_INT_IO_AI_MASK)
1069 VCPU_EVENT(vcpu, 4, "%s", "deliver: I/O (AI)");
1071 VCPU_EVENT(vcpu, 4, "deliver: I/O %x ss %x schid %04x",
1072 inti->io.subchannel_id >> 8,
1073 inti->io.subchannel_id >> 1 & 0x3,
1074 inti->io.subchannel_nr);
1076 vcpu->stat.deliver_io++;
1077 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1079 ((__u32)inti->io.subchannel_id << 16) |
1080 inti->io.subchannel_nr,
1081 ((__u64)inti->io.io_int_parm << 32) |
1082 inti->io.io_int_word);
1083 list_del(&inti->list);
1084 fi->counters[FIRQ_CNTR_IO] -= 1;
1086 if (list_empty(isc_list))
1087 clear_bit(irq_type, &fi->pending_irqs);
1088 spin_unlock(&fi->lock);
1091 rc = __do_deliver_io(vcpu, &(inti->io));
1096 if (gi->origin && gisa_tac_ipm_gisc(gi->origin, isc)) {
1098 * in case an adapter interrupt was not delivered
1099 * in SIE context KVM will handle the delivery
1101 VCPU_EVENT(vcpu, 4, "%s isc %u", "deliver: I/O (AI/gisa)", isc);
1102 memset(&io, 0, sizeof(io));
1103 io.io_int_word = isc_to_int_word(isc);
1104 vcpu->stat.deliver_io++;
1105 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1106 KVM_S390_INT_IO(1, 0, 0, 0),
1107 ((__u32)io.subchannel_id << 16) |
1109 ((__u64)io.io_int_parm << 32) |
1111 rc = __do_deliver_io(vcpu, &io);
1117 /* Check whether an external call is pending (deliverable or not) */
1118 int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu)
1120 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1122 if (!sclp.has_sigpif)
1123 return test_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
1125 return sca_ext_call_pending(vcpu, NULL);
1128 int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop)
1130 if (deliverable_irqs(vcpu))
1133 if (kvm_cpu_has_pending_timer(vcpu))
1136 /* external call pending and deliverable */
1137 if (kvm_s390_ext_call_pending(vcpu) &&
1138 !psw_extint_disabled(vcpu) &&
1139 (vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
1142 if (!exclude_stop && kvm_s390_is_stop_irq_pending(vcpu))
1147 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
1149 return ckc_irq_pending(vcpu) || cpu_timer_irq_pending(vcpu);
1152 static u64 __calculate_sltime(struct kvm_vcpu *vcpu)
1154 const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
1155 const u64 ckc = vcpu->arch.sie_block->ckc;
1156 u64 cputm, sltime = 0;
1158 if (ckc_interrupts_enabled(vcpu)) {
1159 if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
1160 if ((s64)now < (s64)ckc)
1161 sltime = tod_to_ns((s64)ckc - (s64)now);
1162 } else if (now < ckc) {
1163 sltime = tod_to_ns(ckc - now);
1165 /* already expired */
1168 if (cpu_timer_interrupts_enabled(vcpu)) {
1169 cputm = kvm_s390_get_cpu_timer(vcpu);
1170 /* already expired? */
1173 return min(sltime, tod_to_ns(cputm));
1175 } else if (cpu_timer_interrupts_enabled(vcpu)) {
1176 sltime = kvm_s390_get_cpu_timer(vcpu);
1177 /* already expired? */
1184 int kvm_s390_handle_wait(struct kvm_vcpu *vcpu)
1186 struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
1189 vcpu->stat.exit_wait_state++;
1192 if (kvm_arch_vcpu_runnable(vcpu))
1195 if (psw_interrupts_disabled(vcpu)) {
1196 VCPU_EVENT(vcpu, 3, "%s", "disabled wait");
1197 return -EOPNOTSUPP; /* disabled wait */
1201 (gisa_get_ipm_or_restore_iam(gi) &
1202 vcpu->arch.sie_block->gcr[6] >> 24))
1205 if (!ckc_interrupts_enabled(vcpu) &&
1206 !cpu_timer_interrupts_enabled(vcpu)) {
1207 VCPU_EVENT(vcpu, 3, "%s", "enabled wait w/o timer");
1208 __set_cpu_idle(vcpu);
1212 sltime = __calculate_sltime(vcpu);
1216 __set_cpu_idle(vcpu);
1217 hrtimer_start(&vcpu->arch.ckc_timer, sltime, HRTIMER_MODE_REL);
1218 VCPU_EVENT(vcpu, 4, "enabled wait: %llu ns", sltime);
1220 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1221 kvm_vcpu_block(vcpu);
1222 __unset_cpu_idle(vcpu);
1223 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1225 hrtimer_cancel(&vcpu->arch.ckc_timer);
1229 void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu)
1231 vcpu->valid_wakeup = true;
1232 kvm_vcpu_wake_up(vcpu);
1235 * The VCPU might not be sleeping but rather executing VSIE. Let's
1236 * kick it, so it leaves the SIE to process the request.
1238 kvm_s390_vsie_kick(vcpu);
1241 enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer)
1243 struct kvm_vcpu *vcpu;
1246 vcpu = container_of(timer, struct kvm_vcpu, arch.ckc_timer);
1247 sltime = __calculate_sltime(vcpu);
1250 * If the monotonic clock runs faster than the tod clock we might be
1251 * woken up too early and have to go back to sleep to avoid deadlocks.
1253 if (sltime && hrtimer_forward_now(timer, ns_to_ktime(sltime)))
1254 return HRTIMER_RESTART;
1255 kvm_s390_vcpu_wakeup(vcpu);
1256 return HRTIMER_NORESTART;
1259 void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu)
1261 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1263 spin_lock(&li->lock);
1264 li->pending_irqs = 0;
1265 bitmap_zero(li->sigp_emerg_pending, KVM_MAX_VCPUS);
1266 memset(&li->irq, 0, sizeof(li->irq));
1267 spin_unlock(&li->lock);
1269 sca_clear_ext_call(vcpu);
1272 int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
1274 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1276 unsigned long irq_type;
1279 __reset_intercept_indicators(vcpu);
1281 /* pending ckc conditions might have been invalidated */
1282 clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1283 if (ckc_irq_pending(vcpu))
1284 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1286 /* pending cpu timer conditions might have been invalidated */
1287 clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1288 if (cpu_timer_irq_pending(vcpu))
1289 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1291 while ((irqs = deliverable_irqs(vcpu)) && !rc) {
1292 /* bits are in the reverse order of interrupt priority */
1293 irq_type = find_last_bit(&irqs, IRQ_PEND_COUNT);
1295 case IRQ_PEND_IO_ISC_0:
1296 case IRQ_PEND_IO_ISC_1:
1297 case IRQ_PEND_IO_ISC_2:
1298 case IRQ_PEND_IO_ISC_3:
1299 case IRQ_PEND_IO_ISC_4:
1300 case IRQ_PEND_IO_ISC_5:
1301 case IRQ_PEND_IO_ISC_6:
1302 case IRQ_PEND_IO_ISC_7:
1303 rc = __deliver_io(vcpu, irq_type);
1305 case IRQ_PEND_MCHK_EX:
1306 case IRQ_PEND_MCHK_REP:
1307 rc = __deliver_machine_check(vcpu);
1310 rc = __deliver_prog(vcpu);
1312 case IRQ_PEND_EXT_EMERGENCY:
1313 rc = __deliver_emergency_signal(vcpu);
1315 case IRQ_PEND_EXT_EXTERNAL:
1316 rc = __deliver_external_call(vcpu);
1318 case IRQ_PEND_EXT_CLOCK_COMP:
1319 rc = __deliver_ckc(vcpu);
1321 case IRQ_PEND_EXT_CPU_TIMER:
1322 rc = __deliver_cpu_timer(vcpu);
1324 case IRQ_PEND_RESTART:
1325 rc = __deliver_restart(vcpu);
1327 case IRQ_PEND_SET_PREFIX:
1328 rc = __deliver_set_prefix(vcpu);
1330 case IRQ_PEND_PFAULT_INIT:
1331 rc = __deliver_pfault_init(vcpu);
1333 case IRQ_PEND_EXT_SERVICE:
1334 rc = __deliver_service(vcpu);
1336 case IRQ_PEND_PFAULT_DONE:
1337 rc = __deliver_pfault_done(vcpu);
1339 case IRQ_PEND_VIRTIO:
1340 rc = __deliver_virtio(vcpu);
1343 WARN_ONCE(1, "Unknown pending irq type %ld", irq_type);
1344 clear_bit(irq_type, &li->pending_irqs);
1348 set_intercept_indicators(vcpu);
1353 static int __inject_prog(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1355 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1357 vcpu->stat.inject_program++;
1358 VCPU_EVENT(vcpu, 3, "inject: program irq code 0x%x", irq->u.pgm.code);
1359 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
1360 irq->u.pgm.code, 0);
1362 if (!(irq->u.pgm.flags & KVM_S390_PGM_FLAGS_ILC_VALID)) {
1363 /* auto detection if no valid ILC was given */
1364 irq->u.pgm.flags &= ~KVM_S390_PGM_FLAGS_ILC_MASK;
1365 irq->u.pgm.flags |= kvm_s390_get_ilen(vcpu);
1366 irq->u.pgm.flags |= KVM_S390_PGM_FLAGS_ILC_VALID;
1369 if (irq->u.pgm.code == PGM_PER) {
1370 li->irq.pgm.code |= PGM_PER;
1371 li->irq.pgm.flags = irq->u.pgm.flags;
1372 /* only modify PER related information */
1373 li->irq.pgm.per_address = irq->u.pgm.per_address;
1374 li->irq.pgm.per_code = irq->u.pgm.per_code;
1375 li->irq.pgm.per_atmid = irq->u.pgm.per_atmid;
1376 li->irq.pgm.per_access_id = irq->u.pgm.per_access_id;
1377 } else if (!(irq->u.pgm.code & PGM_PER)) {
1378 li->irq.pgm.code = (li->irq.pgm.code & PGM_PER) |
1380 li->irq.pgm.flags = irq->u.pgm.flags;
1381 /* only modify non-PER information */
1382 li->irq.pgm.trans_exc_code = irq->u.pgm.trans_exc_code;
1383 li->irq.pgm.mon_code = irq->u.pgm.mon_code;
1384 li->irq.pgm.data_exc_code = irq->u.pgm.data_exc_code;
1385 li->irq.pgm.mon_class_nr = irq->u.pgm.mon_class_nr;
1386 li->irq.pgm.exc_access_id = irq->u.pgm.exc_access_id;
1387 li->irq.pgm.op_access_id = irq->u.pgm.op_access_id;
1389 li->irq.pgm = irq->u.pgm;
1391 set_bit(IRQ_PEND_PROG, &li->pending_irqs);
1395 static int __inject_pfault_init(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1397 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1399 vcpu->stat.inject_pfault_init++;
1400 VCPU_EVENT(vcpu, 4, "inject: pfault init parameter block at 0x%llx",
1401 irq->u.ext.ext_params2);
1402 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_PFAULT_INIT,
1403 irq->u.ext.ext_params,
1404 irq->u.ext.ext_params2);
1406 li->irq.ext = irq->u.ext;
1407 set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
1408 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1412 static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1414 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1415 struct kvm_s390_extcall_info *extcall = &li->irq.extcall;
1416 uint16_t src_id = irq->u.extcall.code;
1418 vcpu->stat.inject_external_call++;
1419 VCPU_EVENT(vcpu, 4, "inject: external call source-cpu:%u",
1421 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EXTERNAL_CALL,
1424 /* sending vcpu invalid */
1425 if (kvm_get_vcpu_by_id(vcpu->kvm, src_id) == NULL)
1428 if (sclp.has_sigpif)
1429 return sca_inject_ext_call(vcpu, src_id);
1431 if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs))
1433 *extcall = irq->u.extcall;
1434 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1438 static int __inject_set_prefix(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1440 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1441 struct kvm_s390_prefix_info *prefix = &li->irq.prefix;
1443 vcpu->stat.inject_set_prefix++;
1444 VCPU_EVENT(vcpu, 3, "inject: set prefix to %x",
1445 irq->u.prefix.address);
1446 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_SET_PREFIX,
1447 irq->u.prefix.address, 0);
1449 if (!is_vcpu_stopped(vcpu))
1452 *prefix = irq->u.prefix;
1453 set_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
1457 #define KVM_S390_STOP_SUPP_FLAGS (KVM_S390_STOP_FLAG_STORE_STATUS)
1458 static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1460 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1461 struct kvm_s390_stop_info *stop = &li->irq.stop;
1464 vcpu->stat.inject_stop_signal++;
1465 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_STOP, 0, 0);
1467 if (irq->u.stop.flags & ~KVM_S390_STOP_SUPP_FLAGS)
1470 if (is_vcpu_stopped(vcpu)) {
1471 if (irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS)
1472 rc = kvm_s390_store_status_unloaded(vcpu,
1473 KVM_S390_STORE_STATUS_NOADDR);
1477 if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs))
1479 stop->flags = irq->u.stop.flags;
1480 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
1484 static int __inject_sigp_restart(struct kvm_vcpu *vcpu,
1485 struct kvm_s390_irq *irq)
1487 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1489 vcpu->stat.inject_restart++;
1490 VCPU_EVENT(vcpu, 3, "%s", "inject: restart int");
1491 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);
1493 set_bit(IRQ_PEND_RESTART, &li->pending_irqs);
1497 static int __inject_sigp_emergency(struct kvm_vcpu *vcpu,
1498 struct kvm_s390_irq *irq)
1500 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1502 vcpu->stat.inject_emergency_signal++;
1503 VCPU_EVENT(vcpu, 4, "inject: emergency from cpu %u",
1505 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
1506 irq->u.emerg.code, 0);
1508 /* sending vcpu invalid */
1509 if (kvm_get_vcpu_by_id(vcpu->kvm, irq->u.emerg.code) == NULL)
1512 set_bit(irq->u.emerg.code, li->sigp_emerg_pending);
1513 set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
1514 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1518 static int __inject_mchk(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1520 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1521 struct kvm_s390_mchk_info *mchk = &li->irq.mchk;
1523 vcpu->stat.inject_mchk++;
1524 VCPU_EVENT(vcpu, 3, "inject: machine check mcic 0x%llx",
1526 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_MCHK, 0,
1530 * Because repressible machine checks can be indicated along with
1531 * exigent machine checks (PoP, Chapter 11, Interruption action)
1532 * we need to combine cr14, mcic and external damage code.
1533 * Failing storage address and the logout area should not be or'ed
1534 * together, we just indicate the last occurrence of the corresponding
1537 mchk->cr14 |= irq->u.mchk.cr14;
1538 mchk->mcic |= irq->u.mchk.mcic;
1539 mchk->ext_damage_code |= irq->u.mchk.ext_damage_code;
1540 mchk->failing_storage_address = irq->u.mchk.failing_storage_address;
1541 memcpy(&mchk->fixed_logout, &irq->u.mchk.fixed_logout,
1542 sizeof(mchk->fixed_logout));
1543 if (mchk->mcic & MCHK_EX_MASK)
1544 set_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
1545 else if (mchk->mcic & MCHK_REP_MASK)
1546 set_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs);
1550 static int __inject_ckc(struct kvm_vcpu *vcpu)
1552 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1554 vcpu->stat.inject_ckc++;
1555 VCPU_EVENT(vcpu, 3, "%s", "inject: clock comparator external");
1556 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
1559 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1560 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1564 static int __inject_cpu_timer(struct kvm_vcpu *vcpu)
1566 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1568 vcpu->stat.inject_cputm++;
1569 VCPU_EVENT(vcpu, 3, "%s", "inject: cpu timer external");
1570 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
1573 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1574 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1578 static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm,
1581 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1582 struct list_head *isc_list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1583 struct kvm_s390_interrupt_info *iter;
1584 u16 id = (schid & 0xffff0000U) >> 16;
1585 u16 nr = schid & 0x0000ffffU;
1587 spin_lock(&fi->lock);
1588 list_for_each_entry(iter, isc_list, list) {
1589 if (schid && (id != iter->io.subchannel_id ||
1590 nr != iter->io.subchannel_nr))
1592 /* found an appropriate entry */
1593 list_del_init(&iter->list);
1594 fi->counters[FIRQ_CNTR_IO] -= 1;
1595 if (list_empty(isc_list))
1596 clear_bit(isc_to_irq_type(isc), &fi->pending_irqs);
1597 spin_unlock(&fi->lock);
1600 spin_unlock(&fi->lock);
1604 static struct kvm_s390_interrupt_info *get_top_io_int(struct kvm *kvm,
1605 u64 isc_mask, u32 schid)
1607 struct kvm_s390_interrupt_info *inti = NULL;
1610 for (isc = 0; isc <= MAX_ISC && !inti; isc++) {
1611 if (isc_mask & isc_to_isc_bits(isc))
1612 inti = get_io_int(kvm, isc, schid);
1617 static int get_top_gisa_isc(struct kvm *kvm, u64 isc_mask, u32 schid)
1619 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1620 unsigned long active_mask;
1628 active_mask = (isc_mask & gisa_get_ipm(gi->origin) << 24) << 32;
1629 while (active_mask) {
1630 isc = __fls(active_mask) ^ (BITS_PER_LONG - 1);
1631 if (gisa_tac_ipm_gisc(gi->origin, isc))
1633 clear_bit_inv(isc, &active_mask);
1640 * Dequeue and return an I/O interrupt matching any of the interruption
1641 * subclasses as designated by the isc mask in cr6 and the schid (if != 0).
1642 * Take into account the interrupts pending in the interrupt list and in GISA.
1644 * Note that for a guest that does not enable I/O interrupts
1645 * but relies on TPI, a flood of classic interrupts may starve
1646 * out adapter interrupts on the same isc. Linux does not do
1647 * that, and it is possible to work around the issue by configuring
1648 * different iscs for classic and adapter interrupts in the guest,
1649 * but we may want to revisit this in the future.
1651 struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
1652 u64 isc_mask, u32 schid)
1654 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1655 struct kvm_s390_interrupt_info *inti, *tmp_inti;
1658 inti = get_top_io_int(kvm, isc_mask, schid);
1660 isc = get_top_gisa_isc(kvm, isc_mask, schid);
1666 /* AI in GISA but no classical IO int */
1669 /* both types of interrupts present */
1670 if (int_word_to_isc(inti->io.io_int_word) <= isc) {
1671 /* classical IO int with higher priority */
1672 gisa_set_ipm_gisc(gi->origin, isc);
1676 tmp_inti = kzalloc(sizeof(*inti), GFP_KERNEL);
1678 tmp_inti->type = KVM_S390_INT_IO(1, 0, 0, 0);
1679 tmp_inti->io.io_int_word = isc_to_int_word(isc);
1681 kvm_s390_reinject_io_int(kvm, inti);
1684 gisa_set_ipm_gisc(gi->origin, isc);
1689 #define SCCB_MASK 0xFFFFFFF8
1690 #define SCCB_EVENT_PENDING 0x3
1692 static int __inject_service(struct kvm *kvm,
1693 struct kvm_s390_interrupt_info *inti)
1695 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1697 kvm->stat.inject_service_signal++;
1698 spin_lock(&fi->lock);
1699 fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING;
1701 * Early versions of the QEMU s390 bios will inject several
1702 * service interrupts after another without handling a
1703 * condition code indicating busy.
1704 * We will silently ignore those superfluous sccb values.
1705 * A future version of QEMU will take care of serialization
1708 if (fi->srv_signal.ext_params & SCCB_MASK)
1710 fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK;
1711 set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
1713 spin_unlock(&fi->lock);
1718 static int __inject_virtio(struct kvm *kvm,
1719 struct kvm_s390_interrupt_info *inti)
1721 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1723 kvm->stat.inject_virtio++;
1724 spin_lock(&fi->lock);
1725 if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) {
1726 spin_unlock(&fi->lock);
1729 fi->counters[FIRQ_CNTR_VIRTIO] += 1;
1730 list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]);
1731 set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
1732 spin_unlock(&fi->lock);
1736 static int __inject_pfault_done(struct kvm *kvm,
1737 struct kvm_s390_interrupt_info *inti)
1739 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1741 kvm->stat.inject_pfault_done++;
1742 spin_lock(&fi->lock);
1743 if (fi->counters[FIRQ_CNTR_PFAULT] >=
1744 (ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) {
1745 spin_unlock(&fi->lock);
1748 fi->counters[FIRQ_CNTR_PFAULT] += 1;
1749 list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]);
1750 set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
1751 spin_unlock(&fi->lock);
1755 #define CR_PENDING_SUBCLASS 28
1756 static int __inject_float_mchk(struct kvm *kvm,
1757 struct kvm_s390_interrupt_info *inti)
1759 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1761 kvm->stat.inject_float_mchk++;
1762 spin_lock(&fi->lock);
1763 fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS);
1764 fi->mchk.mcic |= inti->mchk.mcic;
1765 set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs);
1766 spin_unlock(&fi->lock);
1771 static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1773 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1774 struct kvm_s390_float_interrupt *fi;
1775 struct list_head *list;
1778 kvm->stat.inject_io++;
1779 isc = int_word_to_isc(inti->io.io_int_word);
1781 if (gi->origin && inti->type & KVM_S390_INT_IO_AI_MASK) {
1782 VM_EVENT(kvm, 4, "%s isc %1u", "inject: I/O (AI/gisa)", isc);
1783 gisa_set_ipm_gisc(gi->origin, isc);
1788 fi = &kvm->arch.float_int;
1789 spin_lock(&fi->lock);
1790 if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) {
1791 spin_unlock(&fi->lock);
1794 fi->counters[FIRQ_CNTR_IO] += 1;
1796 if (inti->type & KVM_S390_INT_IO_AI_MASK)
1797 VM_EVENT(kvm, 4, "%s", "inject: I/O (AI)");
1799 VM_EVENT(kvm, 4, "inject: I/O %x ss %x schid %04x",
1800 inti->io.subchannel_id >> 8,
1801 inti->io.subchannel_id >> 1 & 0x3,
1802 inti->io.subchannel_nr);
1803 list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1804 list_add_tail(&inti->list, list);
1805 set_bit(isc_to_irq_type(isc), &fi->pending_irqs);
1806 spin_unlock(&fi->lock);
1811 * Find a destination VCPU for a floating irq and kick it.
1813 static void __floating_irq_kick(struct kvm *kvm, u64 type)
1815 struct kvm_vcpu *dst_vcpu;
1816 int sigcpu, online_vcpus, nr_tries = 0;
1818 online_vcpus = atomic_read(&kvm->online_vcpus);
1822 /* find idle VCPUs first, then round robin */
1823 sigcpu = find_first_bit(kvm->arch.idle_mask, online_vcpus);
1824 if (sigcpu == online_vcpus) {
1826 sigcpu = kvm->arch.float_int.next_rr_cpu++;
1827 kvm->arch.float_int.next_rr_cpu %= online_vcpus;
1828 /* avoid endless loops if all vcpus are stopped */
1829 if (nr_tries++ >= online_vcpus)
1831 } while (is_vcpu_stopped(kvm_get_vcpu(kvm, sigcpu)));
1833 dst_vcpu = kvm_get_vcpu(kvm, sigcpu);
1835 /* make the VCPU drop out of the SIE, or wake it up if sleeping */
1838 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_STOP_INT);
1840 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1841 if (!(type & KVM_S390_INT_IO_AI_MASK &&
1842 kvm->arch.gisa_int.origin))
1843 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_IO_INT);
1846 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_EXT_INT);
1849 kvm_s390_vcpu_wakeup(dst_vcpu);
1852 static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1854 u64 type = READ_ONCE(inti->type);
1859 rc = __inject_float_mchk(kvm, inti);
1861 case KVM_S390_INT_VIRTIO:
1862 rc = __inject_virtio(kvm, inti);
1864 case KVM_S390_INT_SERVICE:
1865 rc = __inject_service(kvm, inti);
1867 case KVM_S390_INT_PFAULT_DONE:
1868 rc = __inject_pfault_done(kvm, inti);
1870 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1871 rc = __inject_io(kvm, inti);
1879 __floating_irq_kick(kvm, type);
1883 int kvm_s390_inject_vm(struct kvm *kvm,
1884 struct kvm_s390_interrupt *s390int)
1886 struct kvm_s390_interrupt_info *inti;
1889 inti = kzalloc(sizeof(*inti), GFP_KERNEL);
1893 inti->type = s390int->type;
1894 switch (inti->type) {
1895 case KVM_S390_INT_VIRTIO:
1896 VM_EVENT(kvm, 5, "inject: virtio parm:%x,parm64:%llx",
1897 s390int->parm, s390int->parm64);
1898 inti->ext.ext_params = s390int->parm;
1899 inti->ext.ext_params2 = s390int->parm64;
1901 case KVM_S390_INT_SERVICE:
1902 VM_EVENT(kvm, 4, "inject: sclp parm:%x", s390int->parm);
1903 inti->ext.ext_params = s390int->parm;
1905 case KVM_S390_INT_PFAULT_DONE:
1906 inti->ext.ext_params2 = s390int->parm64;
1909 VM_EVENT(kvm, 3, "inject: machine check mcic 0x%llx",
1911 inti->mchk.cr14 = s390int->parm; /* upper bits are not used */
1912 inti->mchk.mcic = s390int->parm64;
1914 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1915 inti->io.subchannel_id = s390int->parm >> 16;
1916 inti->io.subchannel_nr = s390int->parm & 0x0000ffffu;
1917 inti->io.io_int_parm = s390int->parm64 >> 32;
1918 inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull;
1924 trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64,
1927 rc = __inject_vm(kvm, inti);
1933 int kvm_s390_reinject_io_int(struct kvm *kvm,
1934 struct kvm_s390_interrupt_info *inti)
1936 return __inject_vm(kvm, inti);
1939 int s390int_to_s390irq(struct kvm_s390_interrupt *s390int,
1940 struct kvm_s390_irq *irq)
1942 irq->type = s390int->type;
1943 switch (irq->type) {
1944 case KVM_S390_PROGRAM_INT:
1945 if (s390int->parm & 0xffff0000)
1947 irq->u.pgm.code = s390int->parm;
1949 case KVM_S390_SIGP_SET_PREFIX:
1950 irq->u.prefix.address = s390int->parm;
1952 case KVM_S390_SIGP_STOP:
1953 irq->u.stop.flags = s390int->parm;
1955 case KVM_S390_INT_EXTERNAL_CALL:
1956 if (s390int->parm & 0xffff0000)
1958 irq->u.extcall.code = s390int->parm;
1960 case KVM_S390_INT_EMERGENCY:
1961 if (s390int->parm & 0xffff0000)
1963 irq->u.emerg.code = s390int->parm;
1966 irq->u.mchk.mcic = s390int->parm64;
1968 case KVM_S390_INT_PFAULT_INIT:
1969 irq->u.ext.ext_params = s390int->parm;
1970 irq->u.ext.ext_params2 = s390int->parm64;
1972 case KVM_S390_RESTART:
1973 case KVM_S390_INT_CLOCK_COMP:
1974 case KVM_S390_INT_CPU_TIMER:
1982 int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu)
1984 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1986 return test_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
1989 int kvm_s390_is_restart_irq_pending(struct kvm_vcpu *vcpu)
1991 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1993 return test_bit(IRQ_PEND_RESTART, &li->pending_irqs);
1996 void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu)
1998 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2000 spin_lock(&li->lock);
2001 li->irq.stop.flags = 0;
2002 clear_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
2003 spin_unlock(&li->lock);
2006 static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
2010 switch (irq->type) {
2011 case KVM_S390_PROGRAM_INT:
2012 rc = __inject_prog(vcpu, irq);
2014 case KVM_S390_SIGP_SET_PREFIX:
2015 rc = __inject_set_prefix(vcpu, irq);
2017 case KVM_S390_SIGP_STOP:
2018 rc = __inject_sigp_stop(vcpu, irq);
2020 case KVM_S390_RESTART:
2021 rc = __inject_sigp_restart(vcpu, irq);
2023 case KVM_S390_INT_CLOCK_COMP:
2024 rc = __inject_ckc(vcpu);
2026 case KVM_S390_INT_CPU_TIMER:
2027 rc = __inject_cpu_timer(vcpu);
2029 case KVM_S390_INT_EXTERNAL_CALL:
2030 rc = __inject_extcall(vcpu, irq);
2032 case KVM_S390_INT_EMERGENCY:
2033 rc = __inject_sigp_emergency(vcpu, irq);
2036 rc = __inject_mchk(vcpu, irq);
2038 case KVM_S390_INT_PFAULT_INIT:
2039 rc = __inject_pfault_init(vcpu, irq);
2041 case KVM_S390_INT_VIRTIO:
2042 case KVM_S390_INT_SERVICE:
2043 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2051 int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
2053 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2056 spin_lock(&li->lock);
2057 rc = do_inject_vcpu(vcpu, irq);
2058 spin_unlock(&li->lock);
2060 kvm_s390_vcpu_wakeup(vcpu);
2064 static inline void clear_irq_list(struct list_head *_list)
2066 struct kvm_s390_interrupt_info *inti, *n;
2068 list_for_each_entry_safe(inti, n, _list, list) {
2069 list_del(&inti->list);
2074 static void inti_to_irq(struct kvm_s390_interrupt_info *inti,
2075 struct kvm_s390_irq *irq)
2077 irq->type = inti->type;
2078 switch (inti->type) {
2079 case KVM_S390_INT_PFAULT_INIT:
2080 case KVM_S390_INT_PFAULT_DONE:
2081 case KVM_S390_INT_VIRTIO:
2082 irq->u.ext = inti->ext;
2084 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2085 irq->u.io = inti->io;
2090 void kvm_s390_clear_float_irqs(struct kvm *kvm)
2092 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2095 spin_lock(&fi->lock);
2096 fi->pending_irqs = 0;
2097 memset(&fi->srv_signal, 0, sizeof(fi->srv_signal));
2098 memset(&fi->mchk, 0, sizeof(fi->mchk));
2099 for (i = 0; i < FIRQ_LIST_COUNT; i++)
2100 clear_irq_list(&fi->lists[i]);
2101 for (i = 0; i < FIRQ_MAX_COUNT; i++)
2102 fi->counters[i] = 0;
2103 spin_unlock(&fi->lock);
2104 kvm_s390_gisa_clear(kvm);
2107 static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len)
2109 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
2110 struct kvm_s390_interrupt_info *inti;
2111 struct kvm_s390_float_interrupt *fi;
2112 struct kvm_s390_irq *buf;
2113 struct kvm_s390_irq *irq;
2119 if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0)
2123 * We are already using -ENOMEM to signal
2124 * userspace it may retry with a bigger buffer,
2125 * so we need to use something else for this case
2131 max_irqs = len / sizeof(struct kvm_s390_irq);
2133 if (gi->origin && gisa_get_ipm(gi->origin)) {
2134 for (i = 0; i <= MAX_ISC; i++) {
2135 if (n == max_irqs) {
2136 /* signal userspace to try again */
2140 if (gisa_tac_ipm_gisc(gi->origin, i)) {
2141 irq = (struct kvm_s390_irq *) &buf[n];
2142 irq->type = KVM_S390_INT_IO(1, 0, 0, 0);
2143 irq->u.io.io_int_word = isc_to_int_word(i);
2148 fi = &kvm->arch.float_int;
2149 spin_lock(&fi->lock);
2150 for (i = 0; i < FIRQ_LIST_COUNT; i++) {
2151 list_for_each_entry(inti, &fi->lists[i], list) {
2152 if (n == max_irqs) {
2153 /* signal userspace to try again */
2157 inti_to_irq(inti, &buf[n]);
2161 if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs)) {
2162 if (n == max_irqs) {
2163 /* signal userspace to try again */
2167 irq = (struct kvm_s390_irq *) &buf[n];
2168 irq->type = KVM_S390_INT_SERVICE;
2169 irq->u.ext = fi->srv_signal;
2172 if (test_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
2173 if (n == max_irqs) {
2174 /* signal userspace to try again */
2178 irq = (struct kvm_s390_irq *) &buf[n];
2179 irq->type = KVM_S390_MCHK;
2180 irq->u.mchk = fi->mchk;
2185 spin_unlock(&fi->lock);
2187 if (!ret && n > 0) {
2188 if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n))
2193 return ret < 0 ? ret : n;
2196 static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr)
2198 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2199 struct kvm_s390_ais_all ais;
2201 if (attr->attr < sizeof(ais))
2204 if (!test_kvm_facility(kvm, 72))
2207 mutex_lock(&fi->ais_lock);
2208 ais.simm = fi->simm;
2209 ais.nimm = fi->nimm;
2210 mutex_unlock(&fi->ais_lock);
2212 if (copy_to_user((void __user *)attr->addr, &ais, sizeof(ais)))
2218 static int flic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2222 switch (attr->group) {
2223 case KVM_DEV_FLIC_GET_ALL_IRQS:
2224 r = get_all_floating_irqs(dev->kvm, (u8 __user *) attr->addr,
2227 case KVM_DEV_FLIC_AISM_ALL:
2228 r = flic_ais_mode_get_all(dev->kvm, attr);
2237 static inline int copy_irq_from_user(struct kvm_s390_interrupt_info *inti,
2240 struct kvm_s390_irq __user *uptr = (struct kvm_s390_irq __user *) addr;
2241 void *target = NULL;
2242 void __user *source;
2245 if (get_user(inti->type, (u64 __user *)addr))
2248 switch (inti->type) {
2249 case KVM_S390_INT_PFAULT_INIT:
2250 case KVM_S390_INT_PFAULT_DONE:
2251 case KVM_S390_INT_VIRTIO:
2252 case KVM_S390_INT_SERVICE:
2253 target = (void *) &inti->ext;
2254 source = &uptr->u.ext;
2255 size = sizeof(inti->ext);
2257 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2258 target = (void *) &inti->io;
2259 source = &uptr->u.io;
2260 size = sizeof(inti->io);
2263 target = (void *) &inti->mchk;
2264 source = &uptr->u.mchk;
2265 size = sizeof(inti->mchk);
2271 if (copy_from_user(target, source, size))
2277 static int enqueue_floating_irq(struct kvm_device *dev,
2278 struct kvm_device_attr *attr)
2280 struct kvm_s390_interrupt_info *inti = NULL;
2282 int len = attr->attr;
2284 if (len % sizeof(struct kvm_s390_irq) != 0)
2286 else if (len > KVM_S390_FLIC_MAX_BUFFER)
2289 while (len >= sizeof(struct kvm_s390_irq)) {
2290 inti = kzalloc(sizeof(*inti), GFP_KERNEL);
2294 r = copy_irq_from_user(inti, attr->addr);
2299 r = __inject_vm(dev->kvm, inti);
2304 len -= sizeof(struct kvm_s390_irq);
2305 attr->addr += sizeof(struct kvm_s390_irq);
2311 static struct s390_io_adapter *get_io_adapter(struct kvm *kvm, unsigned int id)
2313 if (id >= MAX_S390_IO_ADAPTERS)
2315 id = array_index_nospec(id, MAX_S390_IO_ADAPTERS);
2316 return kvm->arch.adapters[id];
2319 static int register_io_adapter(struct kvm_device *dev,
2320 struct kvm_device_attr *attr)
2322 struct s390_io_adapter *adapter;
2323 struct kvm_s390_io_adapter adapter_info;
2325 if (copy_from_user(&adapter_info,
2326 (void __user *)attr->addr, sizeof(adapter_info)))
2329 if (adapter_info.id >= MAX_S390_IO_ADAPTERS)
2332 adapter_info.id = array_index_nospec(adapter_info.id,
2333 MAX_S390_IO_ADAPTERS);
2335 if (dev->kvm->arch.adapters[adapter_info.id] != NULL)
2338 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
2342 INIT_LIST_HEAD(&adapter->maps);
2343 init_rwsem(&adapter->maps_lock);
2344 atomic_set(&adapter->nr_maps, 0);
2345 adapter->id = adapter_info.id;
2346 adapter->isc = adapter_info.isc;
2347 adapter->maskable = adapter_info.maskable;
2348 adapter->masked = false;
2349 adapter->swap = adapter_info.swap;
2350 adapter->suppressible = (adapter_info.flags) &
2351 KVM_S390_ADAPTER_SUPPRESSIBLE;
2352 dev->kvm->arch.adapters[adapter->id] = adapter;
2357 int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked)
2360 struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
2362 if (!adapter || !adapter->maskable)
2364 ret = adapter->masked;
2365 adapter->masked = masked;
2369 static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr)
2371 struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
2372 struct s390_map_info *map;
2375 if (!adapter || !addr)
2378 map = kzalloc(sizeof(*map), GFP_KERNEL);
2383 INIT_LIST_HEAD(&map->list);
2384 map->guest_addr = addr;
2385 map->addr = gmap_translate(kvm->arch.gmap, addr);
2386 if (map->addr == -EFAULT) {
2390 ret = get_user_pages_fast(map->addr, 1, FOLL_WRITE, &map->page);
2394 down_write(&adapter->maps_lock);
2395 if (atomic_inc_return(&adapter->nr_maps) < MAX_S390_ADAPTER_MAPS) {
2396 list_add_tail(&map->list, &adapter->maps);
2399 put_page(map->page);
2402 up_write(&adapter->maps_lock);
2409 static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr)
2411 struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
2412 struct s390_map_info *map, *tmp;
2415 if (!adapter || !addr)
2418 down_write(&adapter->maps_lock);
2419 list_for_each_entry_safe(map, tmp, &adapter->maps, list) {
2420 if (map->guest_addr == addr) {
2422 atomic_dec(&adapter->nr_maps);
2423 list_del(&map->list);
2424 put_page(map->page);
2429 up_write(&adapter->maps_lock);
2431 return found ? 0 : -EINVAL;
2434 void kvm_s390_destroy_adapters(struct kvm *kvm)
2437 struct s390_map_info *map, *tmp;
2439 for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) {
2440 if (!kvm->arch.adapters[i])
2442 list_for_each_entry_safe(map, tmp,
2443 &kvm->arch.adapters[i]->maps, list) {
2444 list_del(&map->list);
2445 put_page(map->page);
2448 kfree(kvm->arch.adapters[i]);
2452 static int modify_io_adapter(struct kvm_device *dev,
2453 struct kvm_device_attr *attr)
2455 struct kvm_s390_io_adapter_req req;
2456 struct s390_io_adapter *adapter;
2459 if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req)))
2462 adapter = get_io_adapter(dev->kvm, req.id);
2466 case KVM_S390_IO_ADAPTER_MASK:
2467 ret = kvm_s390_mask_adapter(dev->kvm, req.id, req.mask);
2471 case KVM_S390_IO_ADAPTER_MAP:
2472 ret = kvm_s390_adapter_map(dev->kvm, req.id, req.addr);
2474 case KVM_S390_IO_ADAPTER_UNMAP:
2475 ret = kvm_s390_adapter_unmap(dev->kvm, req.id, req.addr);
2484 static int clear_io_irq(struct kvm *kvm, struct kvm_device_attr *attr)
2487 const u64 isc_mask = 0xffUL << 24; /* all iscs set */
2492 if (attr->attr != sizeof(schid))
2494 if (copy_from_user(&schid, (void __user *) attr->addr, sizeof(schid)))
2498 kfree(kvm_s390_get_io_int(kvm, isc_mask, schid));
2500 * If userspace is conforming to the architecture, we can have at most
2501 * one pending I/O interrupt per subchannel, so this is effectively a
2507 static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr)
2509 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2510 struct kvm_s390_ais_req req;
2513 if (!test_kvm_facility(kvm, 72))
2516 if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req)))
2519 if (req.isc > MAX_ISC)
2522 trace_kvm_s390_modify_ais_mode(req.isc,
2523 (fi->simm & AIS_MODE_MASK(req.isc)) ?
2524 (fi->nimm & AIS_MODE_MASK(req.isc)) ?
2525 2 : KVM_S390_AIS_MODE_SINGLE :
2526 KVM_S390_AIS_MODE_ALL, req.mode);
2528 mutex_lock(&fi->ais_lock);
2530 case KVM_S390_AIS_MODE_ALL:
2531 fi->simm &= ~AIS_MODE_MASK(req.isc);
2532 fi->nimm &= ~AIS_MODE_MASK(req.isc);
2534 case KVM_S390_AIS_MODE_SINGLE:
2535 fi->simm |= AIS_MODE_MASK(req.isc);
2536 fi->nimm &= ~AIS_MODE_MASK(req.isc);
2541 mutex_unlock(&fi->ais_lock);
2546 static int kvm_s390_inject_airq(struct kvm *kvm,
2547 struct s390_io_adapter *adapter)
2549 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2550 struct kvm_s390_interrupt s390int = {
2551 .type = KVM_S390_INT_IO(1, 0, 0, 0),
2553 .parm64 = isc_to_int_word(adapter->isc),
2557 if (!test_kvm_facility(kvm, 72) || !adapter->suppressible)
2558 return kvm_s390_inject_vm(kvm, &s390int);
2560 mutex_lock(&fi->ais_lock);
2561 if (fi->nimm & AIS_MODE_MASK(adapter->isc)) {
2562 trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc);
2566 ret = kvm_s390_inject_vm(kvm, &s390int);
2567 if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) {
2568 fi->nimm |= AIS_MODE_MASK(adapter->isc);
2569 trace_kvm_s390_modify_ais_mode(adapter->isc,
2570 KVM_S390_AIS_MODE_SINGLE, 2);
2573 mutex_unlock(&fi->ais_lock);
2577 static int flic_inject_airq(struct kvm *kvm, struct kvm_device_attr *attr)
2579 unsigned int id = attr->attr;
2580 struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
2585 return kvm_s390_inject_airq(kvm, adapter);
2588 static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr)
2590 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2591 struct kvm_s390_ais_all ais;
2593 if (!test_kvm_facility(kvm, 72))
2596 if (copy_from_user(&ais, (void __user *)attr->addr, sizeof(ais)))
2599 mutex_lock(&fi->ais_lock);
2600 fi->simm = ais.simm;
2601 fi->nimm = ais.nimm;
2602 mutex_unlock(&fi->ais_lock);
2607 static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2611 struct kvm_vcpu *vcpu;
2613 switch (attr->group) {
2614 case KVM_DEV_FLIC_ENQUEUE:
2615 r = enqueue_floating_irq(dev, attr);
2617 case KVM_DEV_FLIC_CLEAR_IRQS:
2618 kvm_s390_clear_float_irqs(dev->kvm);
2620 case KVM_DEV_FLIC_APF_ENABLE:
2621 dev->kvm->arch.gmap->pfault_enabled = 1;
2623 case KVM_DEV_FLIC_APF_DISABLE_WAIT:
2624 dev->kvm->arch.gmap->pfault_enabled = 0;
2626 * Make sure no async faults are in transition when
2627 * clearing the queues. So we don't need to worry
2628 * about late coming workers.
2630 synchronize_srcu(&dev->kvm->srcu);
2631 kvm_for_each_vcpu(i, vcpu, dev->kvm)
2632 kvm_clear_async_pf_completion_queue(vcpu);
2634 case KVM_DEV_FLIC_ADAPTER_REGISTER:
2635 r = register_io_adapter(dev, attr);
2637 case KVM_DEV_FLIC_ADAPTER_MODIFY:
2638 r = modify_io_adapter(dev, attr);
2640 case KVM_DEV_FLIC_CLEAR_IO_IRQ:
2641 r = clear_io_irq(dev->kvm, attr);
2643 case KVM_DEV_FLIC_AISM:
2644 r = modify_ais_mode(dev->kvm, attr);
2646 case KVM_DEV_FLIC_AIRQ_INJECT:
2647 r = flic_inject_airq(dev->kvm, attr);
2649 case KVM_DEV_FLIC_AISM_ALL:
2650 r = flic_ais_mode_set_all(dev->kvm, attr);
2659 static int flic_has_attr(struct kvm_device *dev,
2660 struct kvm_device_attr *attr)
2662 switch (attr->group) {
2663 case KVM_DEV_FLIC_GET_ALL_IRQS:
2664 case KVM_DEV_FLIC_ENQUEUE:
2665 case KVM_DEV_FLIC_CLEAR_IRQS:
2666 case KVM_DEV_FLIC_APF_ENABLE:
2667 case KVM_DEV_FLIC_APF_DISABLE_WAIT:
2668 case KVM_DEV_FLIC_ADAPTER_REGISTER:
2669 case KVM_DEV_FLIC_ADAPTER_MODIFY:
2670 case KVM_DEV_FLIC_CLEAR_IO_IRQ:
2671 case KVM_DEV_FLIC_AISM:
2672 case KVM_DEV_FLIC_AIRQ_INJECT:
2673 case KVM_DEV_FLIC_AISM_ALL:
2679 static int flic_create(struct kvm_device *dev, u32 type)
2683 if (dev->kvm->arch.flic)
2685 dev->kvm->arch.flic = dev;
2689 static void flic_destroy(struct kvm_device *dev)
2691 dev->kvm->arch.flic = NULL;
2695 /* s390 floating irq controller (flic) */
2696 struct kvm_device_ops kvm_flic_ops = {
2698 .get_attr = flic_get_attr,
2699 .set_attr = flic_set_attr,
2700 .has_attr = flic_has_attr,
2701 .create = flic_create,
2702 .destroy = flic_destroy,
2705 static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap)
2709 bit = bit_nr + (addr % PAGE_SIZE) * 8;
2711 return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit;
2714 static struct s390_map_info *get_map_info(struct s390_io_adapter *adapter,
2717 struct s390_map_info *map;
2722 list_for_each_entry(map, &adapter->maps, list) {
2723 if (map->guest_addr == addr)
2729 static int adapter_indicators_set(struct kvm *kvm,
2730 struct s390_io_adapter *adapter,
2731 struct kvm_s390_adapter_int *adapter_int)
2734 int summary_set, idx;
2735 struct s390_map_info *info;
2738 info = get_map_info(adapter, adapter_int->ind_addr);
2741 map = page_address(info->page);
2742 bit = get_ind_bit(info->addr, adapter_int->ind_offset, adapter->swap);
2744 idx = srcu_read_lock(&kvm->srcu);
2745 mark_page_dirty(kvm, info->guest_addr >> PAGE_SHIFT);
2746 set_page_dirty_lock(info->page);
2747 info = get_map_info(adapter, adapter_int->summary_addr);
2749 srcu_read_unlock(&kvm->srcu, idx);
2752 map = page_address(info->page);
2753 bit = get_ind_bit(info->addr, adapter_int->summary_offset,
2755 summary_set = test_and_set_bit(bit, map);
2756 mark_page_dirty(kvm, info->guest_addr >> PAGE_SHIFT);
2757 set_page_dirty_lock(info->page);
2758 srcu_read_unlock(&kvm->srcu, idx);
2759 return summary_set ? 0 : 1;
2763 * < 0 - not injected due to error
2764 * = 0 - coalesced, summary indicator already active
2765 * > 0 - injected interrupt
2767 static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e,
2768 struct kvm *kvm, int irq_source_id, int level,
2772 struct s390_io_adapter *adapter;
2774 /* We're only interested in the 0->1 transition. */
2777 adapter = get_io_adapter(kvm, e->adapter.adapter_id);
2780 down_read(&adapter->maps_lock);
2781 ret = adapter_indicators_set(kvm, adapter, &e->adapter);
2782 up_read(&adapter->maps_lock);
2783 if ((ret > 0) && !adapter->masked) {
2784 ret = kvm_s390_inject_airq(kvm, adapter);
2792 * Inject the machine check to the guest.
2794 void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu,
2795 struct mcck_volatile_info *mcck_info)
2797 struct kvm_s390_interrupt_info inti;
2798 struct kvm_s390_irq irq;
2799 struct kvm_s390_mchk_info *mchk;
2801 __u64 cr14 = 0; /* upper bits are not used */
2804 mci.val = mcck_info->mcic;
2806 cr14 |= CR14_RECOVERY_SUBMASK;
2808 cr14 |= CR14_DEGRADATION_SUBMASK;
2810 cr14 |= CR14_WARNING_SUBMASK;
2812 mchk = mci.ck ? &inti.mchk : &irq.u.mchk;
2814 mchk->mcic = mcck_info->mcic;
2815 mchk->ext_damage_code = mcck_info->ext_damage_code;
2816 mchk->failing_storage_address = mcck_info->failing_storage_address;
2818 /* Inject the floating machine check */
2819 inti.type = KVM_S390_MCHK;
2820 rc = __inject_vm(vcpu->kvm, &inti);
2822 /* Inject the machine check to specified vcpu */
2823 irq.type = KVM_S390_MCHK;
2824 rc = kvm_s390_inject_vcpu(vcpu, &irq);
2829 int kvm_set_routing_entry(struct kvm *kvm,
2830 struct kvm_kernel_irq_routing_entry *e,
2831 const struct kvm_irq_routing_entry *ue)
2836 case KVM_IRQ_ROUTING_S390_ADAPTER:
2837 e->set = set_adapter_int;
2838 e->adapter.summary_addr = ue->u.adapter.summary_addr;
2839 e->adapter.ind_addr = ue->u.adapter.ind_addr;
2840 e->adapter.summary_offset = ue->u.adapter.summary_offset;
2841 e->adapter.ind_offset = ue->u.adapter.ind_offset;
2842 e->adapter.adapter_id = ue->u.adapter.adapter_id;
2852 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
2853 int irq_source_id, int level, bool line_status)
2858 int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len)
2860 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2861 struct kvm_s390_irq *buf;
2869 if (copy_from_user((void *) buf, irqstate, len)) {
2875 * Don't allow setting the interrupt state
2876 * when there are already interrupts pending
2878 spin_lock(&li->lock);
2879 if (li->pending_irqs) {
2884 for (n = 0; n < len / sizeof(*buf); n++) {
2885 r = do_inject_vcpu(vcpu, &buf[n]);
2891 spin_unlock(&li->lock);
2898 static void store_local_irq(struct kvm_s390_local_interrupt *li,
2899 struct kvm_s390_irq *irq,
2900 unsigned long irq_type)
2903 case IRQ_PEND_MCHK_EX:
2904 case IRQ_PEND_MCHK_REP:
2905 irq->type = KVM_S390_MCHK;
2906 irq->u.mchk = li->irq.mchk;
2909 irq->type = KVM_S390_PROGRAM_INT;
2910 irq->u.pgm = li->irq.pgm;
2912 case IRQ_PEND_PFAULT_INIT:
2913 irq->type = KVM_S390_INT_PFAULT_INIT;
2914 irq->u.ext = li->irq.ext;
2916 case IRQ_PEND_EXT_EXTERNAL:
2917 irq->type = KVM_S390_INT_EXTERNAL_CALL;
2918 irq->u.extcall = li->irq.extcall;
2920 case IRQ_PEND_EXT_CLOCK_COMP:
2921 irq->type = KVM_S390_INT_CLOCK_COMP;
2923 case IRQ_PEND_EXT_CPU_TIMER:
2924 irq->type = KVM_S390_INT_CPU_TIMER;
2926 case IRQ_PEND_SIGP_STOP:
2927 irq->type = KVM_S390_SIGP_STOP;
2928 irq->u.stop = li->irq.stop;
2930 case IRQ_PEND_RESTART:
2931 irq->type = KVM_S390_RESTART;
2933 case IRQ_PEND_SET_PREFIX:
2934 irq->type = KVM_S390_SIGP_SET_PREFIX;
2935 irq->u.prefix = li->irq.prefix;
2940 int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len)
2943 DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS);
2944 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2945 unsigned long pending_irqs;
2946 struct kvm_s390_irq irq;
2947 unsigned long irq_type;
2951 spin_lock(&li->lock);
2952 pending_irqs = li->pending_irqs;
2953 memcpy(&sigp_emerg_pending, &li->sigp_emerg_pending,
2954 sizeof(sigp_emerg_pending));
2955 spin_unlock(&li->lock);
2957 for_each_set_bit(irq_type, &pending_irqs, IRQ_PEND_COUNT) {
2958 memset(&irq, 0, sizeof(irq));
2959 if (irq_type == IRQ_PEND_EXT_EMERGENCY)
2961 if (n + sizeof(irq) > len)
2963 store_local_irq(&vcpu->arch.local_int, &irq, irq_type);
2964 if (copy_to_user(&buf[n], &irq, sizeof(irq)))
2969 if (test_bit(IRQ_PEND_EXT_EMERGENCY, &pending_irqs)) {
2970 for_each_set_bit(cpuaddr, sigp_emerg_pending, KVM_MAX_VCPUS) {
2971 memset(&irq, 0, sizeof(irq));
2972 if (n + sizeof(irq) > len)
2974 irq.type = KVM_S390_INT_EMERGENCY;
2975 irq.u.emerg.code = cpuaddr;
2976 if (copy_to_user(&buf[n], &irq, sizeof(irq)))
2982 if (sca_ext_call_pending(vcpu, &scn)) {
2983 if (n + sizeof(irq) > len)
2985 memset(&irq, 0, sizeof(irq));
2986 irq.type = KVM_S390_INT_EXTERNAL_CALL;
2987 irq.u.extcall.code = scn;
2988 if (copy_to_user(&buf[n], &irq, sizeof(irq)))
2996 static void __airqs_kick_single_vcpu(struct kvm *kvm, u8 deliverable_mask)
2998 int vcpu_idx, online_vcpus = atomic_read(&kvm->online_vcpus);
2999 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3000 struct kvm_vcpu *vcpu;
3003 for_each_set_bit(vcpu_idx, kvm->arch.idle_mask, online_vcpus) {
3004 vcpu = kvm_get_vcpu(kvm, vcpu_idx);
3005 if (psw_ioint_disabled(vcpu))
3007 vcpu_isc_mask = (u8)(vcpu->arch.sie_block->gcr[6] >> 24);
3008 if (deliverable_mask & vcpu_isc_mask) {
3009 /* lately kicked but not yet running */
3010 if (test_and_set_bit(vcpu_idx, gi->kicked_mask))
3012 kvm_s390_vcpu_wakeup(vcpu);
3018 static enum hrtimer_restart gisa_vcpu_kicker(struct hrtimer *timer)
3020 struct kvm_s390_gisa_interrupt *gi =
3021 container_of(timer, struct kvm_s390_gisa_interrupt, timer);
3023 container_of(gi->origin, struct sie_page2, gisa)->kvm;
3026 pending_mask = gisa_get_ipm_or_restore_iam(gi);
3028 __airqs_kick_single_vcpu(kvm, pending_mask);
3029 hrtimer_forward_now(timer, ns_to_ktime(gi->expires));
3030 return HRTIMER_RESTART;
3033 return HRTIMER_NORESTART;
3036 #define NULL_GISA_ADDR 0x00000000UL
3037 #define NONE_GISA_ADDR 0x00000001UL
3038 #define GISA_ADDR_MASK 0xfffff000UL
3040 static void process_gib_alert_list(void)
3042 struct kvm_s390_gisa_interrupt *gi;
3043 struct kvm_s390_gisa *gisa;
3045 u32 final, origin = 0UL;
3049 * If the NONE_GISA_ADDR is still stored in the alert list
3050 * origin, we will leave the outer loop. No further GISA has
3051 * been added to the alert list by millicode while processing
3052 * the current alert list.
3054 final = (origin & NONE_GISA_ADDR);
3056 * Cut off the alert list and store the NONE_GISA_ADDR in the
3057 * alert list origin to avoid further GAL interruptions.
3058 * A new alert list can be build up by millicode in parallel
3059 * for guests not in the yet cut-off alert list. When in the
3060 * final loop, store the NULL_GISA_ADDR instead. This will re-
3061 * enable GAL interruptions on the host again.
3063 origin = xchg(&gib->alert_list_origin,
3064 (!final) ? NONE_GISA_ADDR : NULL_GISA_ADDR);
3066 * Loop through the just cut-off alert list and start the
3067 * gisa timers to kick idle vcpus to consume the pending
3068 * interruptions asap.
3070 while (origin & GISA_ADDR_MASK) {
3071 gisa = (struct kvm_s390_gisa *)(u64)origin;
3072 origin = gisa->next_alert;
3073 gisa->next_alert = (u32)(u64)gisa;
3074 kvm = container_of(gisa, struct sie_page2, gisa)->kvm;
3075 gi = &kvm->arch.gisa_int;
3076 if (hrtimer_active(&gi->timer))
3077 hrtimer_cancel(&gi->timer);
3078 hrtimer_start(&gi->timer, 0, HRTIMER_MODE_REL);
3084 void kvm_s390_gisa_clear(struct kvm *kvm)
3086 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3090 gisa_clear_ipm(gi->origin);
3091 VM_EVENT(kvm, 3, "gisa 0x%pK cleared", gi->origin);
3094 void kvm_s390_gisa_init(struct kvm *kvm)
3096 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3098 if (!css_general_characteristics.aiv)
3100 gi->origin = &kvm->arch.sie_page2->gisa;
3102 spin_lock_init(&gi->alert.ref_lock);
3103 gi->expires = 50 * 1000; /* 50 usec */
3104 hrtimer_init(&gi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3105 gi->timer.function = gisa_vcpu_kicker;
3106 memset(gi->origin, 0, sizeof(struct kvm_s390_gisa));
3107 gi->origin->next_alert = (u32)(u64)gi->origin;
3108 VM_EVENT(kvm, 3, "gisa 0x%pK initialized", gi->origin);
3111 void kvm_s390_gisa_destroy(struct kvm *kvm)
3113 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3118 KVM_EVENT(3, "vm 0x%pK has unexpected iam 0x%02x",
3119 kvm, gi->alert.mask);
3120 while (gisa_in_alert_list(gi->origin))
3122 hrtimer_cancel(&gi->timer);
3127 * kvm_s390_gisc_register - register a guest ISC
3129 * @kvm: the kernel vm to work with
3130 * @gisc: the guest interruption sub class to register
3132 * The function extends the vm specific alert mask to use.
3133 * The effective IAM mask in the GISA is updated as well
3134 * in case the GISA is not part of the GIB alert list.
3135 * It will be updated latest when the IAM gets restored
3136 * by gisa_get_ipm_or_restore_iam().
3138 * Returns: the nonspecific ISC (NISC) the gib alert mechanism
3139 * has registered with the channel subsystem.
3140 * -ENODEV in case the vm uses no GISA
3141 * -ERANGE in case the guest ISC is invalid
3143 int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc)
3145 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3152 spin_lock(&gi->alert.ref_lock);
3153 gi->alert.ref_count[gisc]++;
3154 if (gi->alert.ref_count[gisc] == 1) {
3155 gi->alert.mask |= 0x80 >> gisc;
3156 gisa_set_iam(gi->origin, gi->alert.mask);
3158 spin_unlock(&gi->alert.ref_lock);
3162 EXPORT_SYMBOL_GPL(kvm_s390_gisc_register);
3165 * kvm_s390_gisc_unregister - unregister a guest ISC
3167 * @kvm: the kernel vm to work with
3168 * @gisc: the guest interruption sub class to register
3170 * The function reduces the vm specific alert mask to use.
3171 * The effective IAM mask in the GISA is updated as well
3172 * in case the GISA is not part of the GIB alert list.
3173 * It will be updated latest when the IAM gets restored
3174 * by gisa_get_ipm_or_restore_iam().
3176 * Returns: the nonspecific ISC (NISC) the gib alert mechanism
3177 * has registered with the channel subsystem.
3178 * -ENODEV in case the vm uses no GISA
3179 * -ERANGE in case the guest ISC is invalid
3180 * -EINVAL in case the guest ISC is not registered
3182 int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc)
3184 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3192 spin_lock(&gi->alert.ref_lock);
3193 if (gi->alert.ref_count[gisc] == 0) {
3197 gi->alert.ref_count[gisc]--;
3198 if (gi->alert.ref_count[gisc] == 0) {
3199 gi->alert.mask &= ~(0x80 >> gisc);
3200 gisa_set_iam(gi->origin, gi->alert.mask);
3203 spin_unlock(&gi->alert.ref_lock);
3207 EXPORT_SYMBOL_GPL(kvm_s390_gisc_unregister);
3209 static void gib_alert_irq_handler(struct airq_struct *airq, bool floating)
3211 inc_irq_stat(IRQIO_GAL);
3212 process_gib_alert_list();
3215 static struct airq_struct gib_alert_irq = {
3216 .handler = gib_alert_irq_handler,
3217 .lsi_ptr = &gib_alert_irq.lsi_mask,
3220 void kvm_s390_gib_destroy(void)
3225 unregister_adapter_interrupt(&gib_alert_irq);
3226 free_page((unsigned long)gib);
3230 int kvm_s390_gib_init(u8 nisc)
3234 if (!css_general_characteristics.aiv) {
3235 KVM_EVENT(3, "%s", "gib not initialized, no AIV facility");
3239 gib = (struct kvm_s390_gib *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
3245 gib_alert_irq.isc = nisc;
3246 if (register_adapter_interrupt(&gib_alert_irq)) {
3247 pr_err("Registering the GIB alert interruption handler failed\n");
3253 if (chsc_sgib((u32)(u64)gib)) {
3254 pr_err("Associating the GIB with the AIV facility failed\n");
3255 free_page((unsigned long)gib);
3261 KVM_EVENT(3, "gib 0x%pK (nisc=%d) initialized", gib, gib->nisc);
3265 unregister_adapter_interrupt(&gib_alert_irq);
3267 free_page((unsigned long)gib);