1 // SPDX-License-Identifier: GPL-2.0
3 * hosting IBM Z kernel virtual machines (s390x)
5 * Copyright IBM Corp. 2008, 2018
7 * Author(s): Carsten Otte <cotte@de.ibm.com>
8 * Christian Borntraeger <borntraeger@de.ibm.com>
9 * Heiko Carstens <heiko.carstens@de.ibm.com>
10 * Christian Ehrhardt <ehrhardt@de.ibm.com>
11 * Jason J. Herne <jjherne@us.ibm.com>
14 #include <linux/compiler.h>
15 #include <linux/err.h>
17 #include <linux/hrtimer.h>
18 #include <linux/init.h>
19 #include <linux/kvm.h>
20 #include <linux/kvm_host.h>
21 #include <linux/mman.h>
22 #include <linux/module.h>
23 #include <linux/moduleparam.h>
24 #include <linux/random.h>
25 #include <linux/slab.h>
26 #include <linux/timer.h>
27 #include <linux/vmalloc.h>
28 #include <linux/bitmap.h>
29 #include <linux/sched/signal.h>
30 #include <linux/string.h>
32 #include <asm/asm-offsets.h>
33 #include <asm/lowcore.h>
35 #include <asm/pgtable.h>
38 #include <asm/switch_to.h>
41 #include <asm/cpacf.h>
42 #include <asm/timex.h>
46 #define KMSG_COMPONENT "kvm-s390"
48 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
50 #define CREATE_TRACE_POINTS
52 #include "trace-s390.h"
54 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
56 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
57 (KVM_MAX_VCPUS + LOCAL_IRQS))
59 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
60 #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
62 struct kvm_stats_debugfs_item debugfs_entries[] = {
63 { "userspace_handled", VCPU_STAT(exit_userspace) },
64 { "exit_null", VCPU_STAT(exit_null) },
65 { "exit_validity", VCPU_STAT(exit_validity) },
66 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
67 { "exit_external_request", VCPU_STAT(exit_external_request) },
68 { "exit_io_request", VCPU_STAT(exit_io_request) },
69 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
70 { "exit_instruction", VCPU_STAT(exit_instruction) },
71 { "exit_pei", VCPU_STAT(exit_pei) },
72 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
73 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
74 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
75 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
76 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
77 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
78 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
79 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
80 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
81 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
82 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
83 { "deliver_ckc", VCPU_STAT(deliver_ckc) },
84 { "deliver_cputm", VCPU_STAT(deliver_cputm) },
85 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
86 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
87 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
88 { "deliver_virtio", VCPU_STAT(deliver_virtio) },
89 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
90 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
91 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
92 { "deliver_program", VCPU_STAT(deliver_program) },
93 { "deliver_io", VCPU_STAT(deliver_io) },
94 { "deliver_machine_check", VCPU_STAT(deliver_machine_check) },
95 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
96 { "inject_ckc", VCPU_STAT(inject_ckc) },
97 { "inject_cputm", VCPU_STAT(inject_cputm) },
98 { "inject_external_call", VCPU_STAT(inject_external_call) },
99 { "inject_float_mchk", VM_STAT(inject_float_mchk) },
100 { "inject_emergency_signal", VCPU_STAT(inject_emergency_signal) },
101 { "inject_io", VM_STAT(inject_io) },
102 { "inject_mchk", VCPU_STAT(inject_mchk) },
103 { "inject_pfault_done", VM_STAT(inject_pfault_done) },
104 { "inject_program", VCPU_STAT(inject_program) },
105 { "inject_restart", VCPU_STAT(inject_restart) },
106 { "inject_service_signal", VM_STAT(inject_service_signal) },
107 { "inject_set_prefix", VCPU_STAT(inject_set_prefix) },
108 { "inject_stop_signal", VCPU_STAT(inject_stop_signal) },
109 { "inject_pfault_init", VCPU_STAT(inject_pfault_init) },
110 { "inject_virtio", VM_STAT(inject_virtio) },
111 { "instruction_epsw", VCPU_STAT(instruction_epsw) },
112 { "instruction_gs", VCPU_STAT(instruction_gs) },
113 { "instruction_io_other", VCPU_STAT(instruction_io_other) },
114 { "instruction_lpsw", VCPU_STAT(instruction_lpsw) },
115 { "instruction_lpswe", VCPU_STAT(instruction_lpswe) },
116 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
117 { "instruction_ptff", VCPU_STAT(instruction_ptff) },
118 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
119 { "instruction_sck", VCPU_STAT(instruction_sck) },
120 { "instruction_sckpf", VCPU_STAT(instruction_sckpf) },
121 { "instruction_spx", VCPU_STAT(instruction_spx) },
122 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
123 { "instruction_stap", VCPU_STAT(instruction_stap) },
124 { "instruction_iske", VCPU_STAT(instruction_iske) },
125 { "instruction_ri", VCPU_STAT(instruction_ri) },
126 { "instruction_rrbe", VCPU_STAT(instruction_rrbe) },
127 { "instruction_sske", VCPU_STAT(instruction_sske) },
128 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
129 { "instruction_essa", VCPU_STAT(instruction_essa) },
130 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
131 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
132 { "instruction_tb", VCPU_STAT(instruction_tb) },
133 { "instruction_tpi", VCPU_STAT(instruction_tpi) },
134 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
135 { "instruction_tsch", VCPU_STAT(instruction_tsch) },
136 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
137 { "instruction_sie", VCPU_STAT(instruction_sie) },
138 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
139 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
140 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
141 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
142 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
143 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
144 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
145 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
146 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
147 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
148 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
149 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
150 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
151 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
152 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
153 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
154 { "instruction_diag_10", VCPU_STAT(diagnose_10) },
155 { "instruction_diag_44", VCPU_STAT(diagnose_44) },
156 { "instruction_diag_9c", VCPU_STAT(diagnose_9c) },
157 { "instruction_diag_258", VCPU_STAT(diagnose_258) },
158 { "instruction_diag_308", VCPU_STAT(diagnose_308) },
159 { "instruction_diag_500", VCPU_STAT(diagnose_500) },
160 { "instruction_diag_other", VCPU_STAT(diagnose_other) },
164 struct kvm_s390_tod_clock_ext {
170 /* allow nested virtualization in KVM (if enabled by user space) */
172 module_param(nested, int, S_IRUGO);
173 MODULE_PARM_DESC(nested, "Nested virtualization support");
175 /* allow 1m huge page guest backing, if !nested */
177 module_param(hpage, int, 0444);
178 MODULE_PARM_DESC(hpage, "1m huge page backing support");
181 * For now we handle at most 16 double words as this is what the s390 base
182 * kernel handles and stores in the prefix page. If we ever need to go beyond
183 * this, this requires changes to code, but the external uapi can stay.
185 #define SIZE_INTERNAL 16
188 * Base feature mask that defines default mask for facilities. Consists of the
189 * defines in FACILITIES_KVM and the non-hypervisor managed bits.
191 static unsigned long kvm_s390_fac_base[SIZE_INTERNAL] = { FACILITIES_KVM };
193 * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL
194 * and defines the facilities that can be enabled via a cpu model.
196 static unsigned long kvm_s390_fac_ext[SIZE_INTERNAL] = { FACILITIES_KVM_CPUMODEL };
198 static unsigned long kvm_s390_fac_size(void)
200 BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_MASK_SIZE_U64);
201 BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_LIST_SIZE_U64);
202 BUILD_BUG_ON(SIZE_INTERNAL * sizeof(unsigned long) >
203 sizeof(S390_lowcore.stfle_fac_list));
205 return SIZE_INTERNAL;
208 /* available cpu features supported by kvm */
209 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
210 /* available subfunctions indicated via query / "test bit" */
211 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
213 static struct gmap_notifier gmap_notifier;
214 static struct gmap_notifier vsie_gmap_notifier;
215 debug_info_t *kvm_s390_dbf;
217 /* Section: not file related */
218 int kvm_arch_hardware_enable(void)
220 /* every s390 is virtualization enabled ;-) */
224 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
227 static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
232 * The TOD jumps by delta, we have to compensate this by adding
233 * -delta to the epoch.
237 /* sign-extension - we're adding to signed values below */
242 if (scb->ecd & ECD_MEF) {
243 scb->epdx += delta_idx;
244 if (scb->epoch < delta)
250 * This callback is executed during stop_machine(). All CPUs are therefore
251 * temporarily stopped. In order not to change guest behavior, we have to
252 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
253 * so a CPU won't be stopped while calculating with the epoch.
255 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
259 struct kvm_vcpu *vcpu;
261 unsigned long long *delta = v;
263 list_for_each_entry(kvm, &vm_list, vm_list) {
264 kvm_for_each_vcpu(i, vcpu, kvm) {
265 kvm_clock_sync_scb(vcpu->arch.sie_block, *delta);
267 kvm->arch.epoch = vcpu->arch.sie_block->epoch;
268 kvm->arch.epdx = vcpu->arch.sie_block->epdx;
270 if (vcpu->arch.cputm_enabled)
271 vcpu->arch.cputm_start += *delta;
272 if (vcpu->arch.vsie_block)
273 kvm_clock_sync_scb(vcpu->arch.vsie_block,
280 static struct notifier_block kvm_clock_notifier = {
281 .notifier_call = kvm_clock_sync,
284 int kvm_arch_hardware_setup(void)
286 gmap_notifier.notifier_call = kvm_gmap_notifier;
287 gmap_register_pte_notifier(&gmap_notifier);
288 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
289 gmap_register_pte_notifier(&vsie_gmap_notifier);
290 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
291 &kvm_clock_notifier);
295 void kvm_arch_hardware_unsetup(void)
297 gmap_unregister_pte_notifier(&gmap_notifier);
298 gmap_unregister_pte_notifier(&vsie_gmap_notifier);
299 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
300 &kvm_clock_notifier);
303 static void allow_cpu_feat(unsigned long nr)
305 set_bit_inv(nr, kvm_s390_available_cpu_feat);
308 static inline int plo_test_bit(unsigned char nr)
310 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
314 /* Parameter registers are ignored for "test bit" */
324 static void kvm_s390_cpu_feat_init(void)
328 for (i = 0; i < 256; ++i) {
330 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
333 if (test_facility(28)) /* TOD-clock steering */
334 ptff(kvm_s390_available_subfunc.ptff,
335 sizeof(kvm_s390_available_subfunc.ptff),
338 if (test_facility(17)) { /* MSA */
339 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
340 kvm_s390_available_subfunc.kmac);
341 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
342 kvm_s390_available_subfunc.kmc);
343 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
344 kvm_s390_available_subfunc.km);
345 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
346 kvm_s390_available_subfunc.kimd);
347 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
348 kvm_s390_available_subfunc.klmd);
350 if (test_facility(76)) /* MSA3 */
351 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
352 kvm_s390_available_subfunc.pckmo);
353 if (test_facility(77)) { /* MSA4 */
354 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
355 kvm_s390_available_subfunc.kmctr);
356 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
357 kvm_s390_available_subfunc.kmf);
358 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
359 kvm_s390_available_subfunc.kmo);
360 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
361 kvm_s390_available_subfunc.pcc);
363 if (test_facility(57)) /* MSA5 */
364 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
365 kvm_s390_available_subfunc.ppno);
367 if (test_facility(146)) /* MSA8 */
368 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
369 kvm_s390_available_subfunc.kma);
371 if (MACHINE_HAS_ESOP)
372 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
374 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
375 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
377 if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
378 !test_facility(3) || !nested)
380 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
381 if (sclp.has_64bscao)
382 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
384 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
386 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
388 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
390 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
392 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
394 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
396 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
398 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
399 * all skey handling functions read/set the skey from the PGSTE
400 * instead of the real storage key.
402 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
403 * pages being detected as preserved although they are resident.
405 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
406 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
408 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
409 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
410 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
412 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
413 * cannot easily shadow the SCA because of the ipte lock.
417 int kvm_arch_init(void *opaque)
421 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
425 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
427 goto out_debug_unreg;
430 kvm_s390_cpu_feat_init();
432 /* Register floating interrupt controller interface. */
433 rc = kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
435 pr_err("Failed to register FLIC rc=%d\n", rc);
436 goto out_debug_unreg;
441 debug_unregister(kvm_s390_dbf);
445 void kvm_arch_exit(void)
447 debug_unregister(kvm_s390_dbf);
450 /* Section: device related */
451 long kvm_arch_dev_ioctl(struct file *filp,
452 unsigned int ioctl, unsigned long arg)
454 if (ioctl == KVM_S390_ENABLE_SIE)
455 return s390_enable_sie();
459 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
464 case KVM_CAP_S390_PSW:
465 case KVM_CAP_S390_GMAP:
466 case KVM_CAP_SYNC_MMU:
467 #ifdef CONFIG_KVM_S390_UCONTROL
468 case KVM_CAP_S390_UCONTROL:
470 case KVM_CAP_ASYNC_PF:
471 case KVM_CAP_SYNC_REGS:
472 case KVM_CAP_ONE_REG:
473 case KVM_CAP_ENABLE_CAP:
474 case KVM_CAP_S390_CSS_SUPPORT:
475 case KVM_CAP_IOEVENTFD:
476 case KVM_CAP_DEVICE_CTRL:
477 case KVM_CAP_ENABLE_CAP_VM:
478 case KVM_CAP_S390_IRQCHIP:
479 case KVM_CAP_VM_ATTRIBUTES:
480 case KVM_CAP_MP_STATE:
481 case KVM_CAP_IMMEDIATE_EXIT:
482 case KVM_CAP_S390_INJECT_IRQ:
483 case KVM_CAP_S390_USER_SIGP:
484 case KVM_CAP_S390_USER_STSI:
485 case KVM_CAP_S390_SKEYS:
486 case KVM_CAP_S390_IRQ_STATE:
487 case KVM_CAP_S390_USER_INSTR0:
488 case KVM_CAP_S390_CMMA_MIGRATION:
489 case KVM_CAP_S390_AIS:
490 case KVM_CAP_S390_AIS_MIGRATION:
493 case KVM_CAP_S390_HPAGE_1M:
495 if (hpage && !kvm_is_ucontrol(kvm))
498 case KVM_CAP_S390_MEM_OP:
501 case KVM_CAP_NR_VCPUS:
502 case KVM_CAP_MAX_VCPUS:
503 case KVM_CAP_MAX_VCPU_ID:
504 r = KVM_S390_BSCA_CPU_SLOTS;
505 if (!kvm_s390_use_sca_entries())
507 else if (sclp.has_esca && sclp.has_64bscao)
508 r = KVM_S390_ESCA_CPU_SLOTS;
510 case KVM_CAP_NR_MEMSLOTS:
511 r = KVM_USER_MEM_SLOTS;
513 case KVM_CAP_S390_COW:
514 r = MACHINE_HAS_ESOP;
516 case KVM_CAP_S390_VECTOR_REGISTERS:
519 case KVM_CAP_S390_RI:
520 r = test_facility(64);
522 case KVM_CAP_S390_GS:
523 r = test_facility(133);
525 case KVM_CAP_S390_BPB:
526 r = test_facility(82);
534 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
535 struct kvm_memory_slot *memslot)
538 gfn_t cur_gfn, last_gfn;
539 unsigned long gaddr, vmaddr;
540 struct gmap *gmap = kvm->arch.gmap;
541 DECLARE_BITMAP(bitmap, _PAGE_ENTRIES);
543 /* Loop over all guest segments */
544 cur_gfn = memslot->base_gfn;
545 last_gfn = memslot->base_gfn + memslot->npages;
546 for (; cur_gfn <= last_gfn; cur_gfn += _PAGE_ENTRIES) {
547 gaddr = gfn_to_gpa(cur_gfn);
548 vmaddr = gfn_to_hva_memslot(memslot, cur_gfn);
549 if (kvm_is_error_hva(vmaddr))
552 bitmap_zero(bitmap, _PAGE_ENTRIES);
553 gmap_sync_dirty_log_pmd(gmap, bitmap, gaddr, vmaddr);
554 for (i = 0; i < _PAGE_ENTRIES; i++) {
555 if (test_bit(i, bitmap))
556 mark_page_dirty(kvm, cur_gfn + i);
559 if (fatal_signal_pending(current))
565 /* Section: vm related */
566 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
569 * Get (and clear) the dirty memory log for a memory slot.
571 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
572 struct kvm_dirty_log *log)
576 struct kvm_memslots *slots;
577 struct kvm_memory_slot *memslot;
580 if (kvm_is_ucontrol(kvm))
583 mutex_lock(&kvm->slots_lock);
586 if (log->slot >= KVM_USER_MEM_SLOTS)
589 slots = kvm_memslots(kvm);
590 memslot = id_to_memslot(slots, log->slot);
592 if (!memslot->dirty_bitmap)
595 kvm_s390_sync_dirty_log(kvm, memslot);
596 r = kvm_get_dirty_log(kvm, log, &is_dirty);
600 /* Clear the dirty log */
602 n = kvm_dirty_bitmap_bytes(memslot);
603 memset(memslot->dirty_bitmap, 0, n);
607 mutex_unlock(&kvm->slots_lock);
611 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
614 struct kvm_vcpu *vcpu;
616 kvm_for_each_vcpu(i, vcpu, kvm) {
617 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
621 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
629 case KVM_CAP_S390_IRQCHIP:
630 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
631 kvm->arch.use_irqchip = 1;
634 case KVM_CAP_S390_USER_SIGP:
635 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
636 kvm->arch.user_sigp = 1;
639 case KVM_CAP_S390_VECTOR_REGISTERS:
640 mutex_lock(&kvm->lock);
641 if (kvm->created_vcpus) {
643 } else if (MACHINE_HAS_VX) {
644 set_kvm_facility(kvm->arch.model.fac_mask, 129);
645 set_kvm_facility(kvm->arch.model.fac_list, 129);
646 if (test_facility(134)) {
647 set_kvm_facility(kvm->arch.model.fac_mask, 134);
648 set_kvm_facility(kvm->arch.model.fac_list, 134);
650 if (test_facility(135)) {
651 set_kvm_facility(kvm->arch.model.fac_mask, 135);
652 set_kvm_facility(kvm->arch.model.fac_list, 135);
657 mutex_unlock(&kvm->lock);
658 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
659 r ? "(not available)" : "(success)");
661 case KVM_CAP_S390_RI:
663 mutex_lock(&kvm->lock);
664 if (kvm->created_vcpus) {
666 } else if (test_facility(64)) {
667 set_kvm_facility(kvm->arch.model.fac_mask, 64);
668 set_kvm_facility(kvm->arch.model.fac_list, 64);
671 mutex_unlock(&kvm->lock);
672 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
673 r ? "(not available)" : "(success)");
675 case KVM_CAP_S390_AIS:
676 mutex_lock(&kvm->lock);
677 if (kvm->created_vcpus) {
680 set_kvm_facility(kvm->arch.model.fac_mask, 72);
681 set_kvm_facility(kvm->arch.model.fac_list, 72);
684 mutex_unlock(&kvm->lock);
685 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
686 r ? "(not available)" : "(success)");
688 case KVM_CAP_S390_GS:
690 mutex_lock(&kvm->lock);
691 if (kvm->created_vcpus) {
693 } else if (test_facility(133)) {
694 set_kvm_facility(kvm->arch.model.fac_mask, 133);
695 set_kvm_facility(kvm->arch.model.fac_list, 133);
698 mutex_unlock(&kvm->lock);
699 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
700 r ? "(not available)" : "(success)");
702 case KVM_CAP_S390_HPAGE_1M:
703 mutex_lock(&kvm->lock);
704 if (kvm->created_vcpus)
706 else if (!hpage || kvm->arch.use_cmma || kvm_is_ucontrol(kvm))
710 down_write(&kvm->mm->mmap_sem);
711 kvm->mm->context.allow_gmap_hpage_1m = 1;
712 up_write(&kvm->mm->mmap_sem);
714 * We might have to create fake 4k page
715 * tables. To avoid that the hardware works on
716 * stale PGSTEs, we emulate these instructions.
718 kvm->arch.use_skf = 0;
719 kvm->arch.use_pfmfi = 0;
721 mutex_unlock(&kvm->lock);
722 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s",
723 r ? "(not available)" : "(success)");
725 case KVM_CAP_S390_USER_STSI:
726 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
727 kvm->arch.user_stsi = 1;
730 case KVM_CAP_S390_USER_INSTR0:
731 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
732 kvm->arch.user_instr0 = 1;
733 icpt_operexc_on_all_vcpus(kvm);
743 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
747 switch (attr->attr) {
748 case KVM_S390_VM_MEM_LIMIT_SIZE:
750 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
751 kvm->arch.mem_limit);
752 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
762 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
766 switch (attr->attr) {
767 case KVM_S390_VM_MEM_ENABLE_CMMA:
772 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
773 mutex_lock(&kvm->lock);
774 if (kvm->created_vcpus)
776 else if (kvm->mm->context.allow_gmap_hpage_1m)
779 kvm->arch.use_cmma = 1;
780 /* Not compatible with cmma. */
781 kvm->arch.use_pfmfi = 0;
784 mutex_unlock(&kvm->lock);
786 case KVM_S390_VM_MEM_CLR_CMMA:
791 if (!kvm->arch.use_cmma)
794 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
795 mutex_lock(&kvm->lock);
796 idx = srcu_read_lock(&kvm->srcu);
797 s390_reset_cmma(kvm->arch.gmap->mm);
798 srcu_read_unlock(&kvm->srcu, idx);
799 mutex_unlock(&kvm->lock);
802 case KVM_S390_VM_MEM_LIMIT_SIZE: {
803 unsigned long new_limit;
805 if (kvm_is_ucontrol(kvm))
808 if (get_user(new_limit, (u64 __user *)attr->addr))
811 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
812 new_limit > kvm->arch.mem_limit)
818 /* gmap_create takes last usable address */
819 if (new_limit != KVM_S390_NO_MEM_LIMIT)
823 mutex_lock(&kvm->lock);
824 if (!kvm->created_vcpus) {
825 /* gmap_create will round the limit up */
826 struct gmap *new = gmap_create(current->mm, new_limit);
831 gmap_remove(kvm->arch.gmap);
833 kvm->arch.gmap = new;
837 mutex_unlock(&kvm->lock);
838 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
839 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
840 (void *) kvm->arch.gmap->asce);
850 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
852 void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
854 struct kvm_vcpu *vcpu;
857 kvm_s390_vcpu_block_all(kvm);
859 kvm_for_each_vcpu(i, vcpu, kvm)
860 kvm_s390_vcpu_crypto_setup(vcpu);
862 kvm_s390_vcpu_unblock_all(kvm);
865 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
867 if (!test_kvm_facility(kvm, 76))
870 mutex_lock(&kvm->lock);
871 switch (attr->attr) {
872 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
874 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
875 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
876 kvm->arch.crypto.aes_kw = 1;
877 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
879 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
881 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
882 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
883 kvm->arch.crypto.dea_kw = 1;
884 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
886 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
887 kvm->arch.crypto.aes_kw = 0;
888 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
889 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
890 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
892 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
893 kvm->arch.crypto.dea_kw = 0;
894 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
895 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
896 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
899 mutex_unlock(&kvm->lock);
903 kvm_s390_vcpu_crypto_reset_all(kvm);
904 mutex_unlock(&kvm->lock);
908 static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
911 struct kvm_vcpu *vcpu;
913 kvm_for_each_vcpu(cx, vcpu, kvm)
914 kvm_s390_sync_request(req, vcpu);
918 * Must be called with kvm->srcu held to avoid races on memslots, and with
919 * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
921 static int kvm_s390_vm_start_migration(struct kvm *kvm)
923 struct kvm_memory_slot *ms;
924 struct kvm_memslots *slots;
925 unsigned long ram_pages = 0;
928 /* migration mode already enabled */
929 if (kvm->arch.migration_mode)
931 slots = kvm_memslots(kvm);
932 if (!slots || !slots->used_slots)
935 if (!kvm->arch.use_cmma) {
936 kvm->arch.migration_mode = 1;
939 /* mark all the pages in active slots as dirty */
940 for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
941 ms = slots->memslots + slotnr;
942 if (!ms->dirty_bitmap)
945 * The second half of the bitmap is only used on x86,
946 * and would be wasted otherwise, so we put it to good
947 * use here to keep track of the state of the storage
950 memset(kvm_second_dirty_bitmap(ms), 0xff, kvm_dirty_bitmap_bytes(ms));
951 ram_pages += ms->npages;
953 atomic64_set(&kvm->arch.cmma_dirty_pages, ram_pages);
954 kvm->arch.migration_mode = 1;
955 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
960 * Must be called with kvm->slots_lock to avoid races with ourselves and
961 * kvm_s390_vm_start_migration.
963 static int kvm_s390_vm_stop_migration(struct kvm *kvm)
965 /* migration mode already disabled */
966 if (!kvm->arch.migration_mode)
968 kvm->arch.migration_mode = 0;
969 if (kvm->arch.use_cmma)
970 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
974 static int kvm_s390_vm_set_migration(struct kvm *kvm,
975 struct kvm_device_attr *attr)
979 mutex_lock(&kvm->slots_lock);
980 switch (attr->attr) {
981 case KVM_S390_VM_MIGRATION_START:
982 res = kvm_s390_vm_start_migration(kvm);
984 case KVM_S390_VM_MIGRATION_STOP:
985 res = kvm_s390_vm_stop_migration(kvm);
990 mutex_unlock(&kvm->slots_lock);
995 static int kvm_s390_vm_get_migration(struct kvm *kvm,
996 struct kvm_device_attr *attr)
998 u64 mig = kvm->arch.migration_mode;
1000 if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
1003 if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
1008 static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1010 struct kvm_s390_vm_tod_clock gtod;
1012 if (copy_from_user(>od, (void __user *)attr->addr, sizeof(gtod)))
1015 if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
1017 kvm_s390_set_tod_clock(kvm, >od);
1019 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
1020 gtod.epoch_idx, gtod.tod);
1025 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1029 if (copy_from_user(>od_high, (void __user *)attr->addr,
1035 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
1040 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1042 struct kvm_s390_vm_tod_clock gtod = { 0 };
1044 if (copy_from_user(>od.tod, (void __user *)attr->addr,
1048 kvm_s390_set_tod_clock(kvm, >od);
1049 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
1053 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1060 switch (attr->attr) {
1061 case KVM_S390_VM_TOD_EXT:
1062 ret = kvm_s390_set_tod_ext(kvm, attr);
1064 case KVM_S390_VM_TOD_HIGH:
1065 ret = kvm_s390_set_tod_high(kvm, attr);
1067 case KVM_S390_VM_TOD_LOW:
1068 ret = kvm_s390_set_tod_low(kvm, attr);
1077 static void kvm_s390_get_tod_clock(struct kvm *kvm,
1078 struct kvm_s390_vm_tod_clock *gtod)
1080 struct kvm_s390_tod_clock_ext htod;
1084 get_tod_clock_ext((char *)&htod);
1086 gtod->tod = htod.tod + kvm->arch.epoch;
1087 gtod->epoch_idx = 0;
1088 if (test_kvm_facility(kvm, 139)) {
1089 gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
1090 if (gtod->tod < htod.tod)
1091 gtod->epoch_idx += 1;
1097 static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1099 struct kvm_s390_vm_tod_clock gtod;
1101 memset(>od, 0, sizeof(gtod));
1102 kvm_s390_get_tod_clock(kvm, >od);
1103 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod)))
1106 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
1107 gtod.epoch_idx, gtod.tod);
1111 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1115 if (copy_to_user((void __user *)attr->addr, >od_high,
1118 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
1123 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1127 gtod = kvm_s390_get_tod_clock_fast(kvm);
1128 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod)))
1130 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
1135 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1142 switch (attr->attr) {
1143 case KVM_S390_VM_TOD_EXT:
1144 ret = kvm_s390_get_tod_ext(kvm, attr);
1146 case KVM_S390_VM_TOD_HIGH:
1147 ret = kvm_s390_get_tod_high(kvm, attr);
1149 case KVM_S390_VM_TOD_LOW:
1150 ret = kvm_s390_get_tod_low(kvm, attr);
1159 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1161 struct kvm_s390_vm_cpu_processor *proc;
1162 u16 lowest_ibc, unblocked_ibc;
1165 mutex_lock(&kvm->lock);
1166 if (kvm->created_vcpus) {
1170 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1175 if (!copy_from_user(proc, (void __user *)attr->addr,
1177 kvm->arch.model.cpuid = proc->cpuid;
1178 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1179 unblocked_ibc = sclp.ibc & 0xfff;
1180 if (lowest_ibc && proc->ibc) {
1181 if (proc->ibc > unblocked_ibc)
1182 kvm->arch.model.ibc = unblocked_ibc;
1183 else if (proc->ibc < lowest_ibc)
1184 kvm->arch.model.ibc = lowest_ibc;
1186 kvm->arch.model.ibc = proc->ibc;
1188 memcpy(kvm->arch.model.fac_list, proc->fac_list,
1189 S390_ARCH_FAC_LIST_SIZE_BYTE);
1190 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1191 kvm->arch.model.ibc,
1192 kvm->arch.model.cpuid);
1193 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1194 kvm->arch.model.fac_list[0],
1195 kvm->arch.model.fac_list[1],
1196 kvm->arch.model.fac_list[2]);
1201 mutex_unlock(&kvm->lock);
1205 static int kvm_s390_set_processor_feat(struct kvm *kvm,
1206 struct kvm_device_attr *attr)
1208 struct kvm_s390_vm_cpu_feat data;
1210 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1212 if (!bitmap_subset((unsigned long *) data.feat,
1213 kvm_s390_available_cpu_feat,
1214 KVM_S390_VM_CPU_FEAT_NR_BITS))
1217 mutex_lock(&kvm->lock);
1218 if (kvm->created_vcpus) {
1219 mutex_unlock(&kvm->lock);
1222 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
1223 KVM_S390_VM_CPU_FEAT_NR_BITS);
1224 mutex_unlock(&kvm->lock);
1225 VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1232 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1233 struct kvm_device_attr *attr)
1236 * Once supported by kernel + hw, we have to store the subfunctions
1237 * in kvm->arch and remember that user space configured them.
1242 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1246 switch (attr->attr) {
1247 case KVM_S390_VM_CPU_PROCESSOR:
1248 ret = kvm_s390_set_processor(kvm, attr);
1250 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1251 ret = kvm_s390_set_processor_feat(kvm, attr);
1253 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1254 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1260 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1262 struct kvm_s390_vm_cpu_processor *proc;
1265 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1270 proc->cpuid = kvm->arch.model.cpuid;
1271 proc->ibc = kvm->arch.model.ibc;
1272 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1273 S390_ARCH_FAC_LIST_SIZE_BYTE);
1274 VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1275 kvm->arch.model.ibc,
1276 kvm->arch.model.cpuid);
1277 VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1278 kvm->arch.model.fac_list[0],
1279 kvm->arch.model.fac_list[1],
1280 kvm->arch.model.fac_list[2]);
1281 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1288 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1290 struct kvm_s390_vm_cpu_machine *mach;
1293 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
1298 get_cpu_id((struct cpuid *) &mach->cpuid);
1299 mach->ibc = sclp.ibc;
1300 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
1301 S390_ARCH_FAC_LIST_SIZE_BYTE);
1302 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
1303 sizeof(S390_lowcore.stfle_fac_list));
1304 VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx",
1305 kvm->arch.model.ibc,
1306 kvm->arch.model.cpuid);
1307 VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx",
1311 VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1315 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1322 static int kvm_s390_get_processor_feat(struct kvm *kvm,
1323 struct kvm_device_attr *attr)
1325 struct kvm_s390_vm_cpu_feat data;
1327 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1328 KVM_S390_VM_CPU_FEAT_NR_BITS);
1329 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1331 VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1338 static int kvm_s390_get_machine_feat(struct kvm *kvm,
1339 struct kvm_device_attr *attr)
1341 struct kvm_s390_vm_cpu_feat data;
1343 bitmap_copy((unsigned long *) data.feat,
1344 kvm_s390_available_cpu_feat,
1345 KVM_S390_VM_CPU_FEAT_NR_BITS);
1346 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1348 VM_EVENT(kvm, 3, "GET: host feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1355 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1356 struct kvm_device_attr *attr)
1359 * Once we can actually configure subfunctions (kernel + hw support),
1360 * we have to check if they were already set by user space, if so copy
1361 * them from kvm->arch.
1366 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1367 struct kvm_device_attr *attr)
1369 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1370 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1374 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1378 switch (attr->attr) {
1379 case KVM_S390_VM_CPU_PROCESSOR:
1380 ret = kvm_s390_get_processor(kvm, attr);
1382 case KVM_S390_VM_CPU_MACHINE:
1383 ret = kvm_s390_get_machine(kvm, attr);
1385 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1386 ret = kvm_s390_get_processor_feat(kvm, attr);
1388 case KVM_S390_VM_CPU_MACHINE_FEAT:
1389 ret = kvm_s390_get_machine_feat(kvm, attr);
1391 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1392 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1394 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1395 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1401 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1405 switch (attr->group) {
1406 case KVM_S390_VM_MEM_CTRL:
1407 ret = kvm_s390_set_mem_control(kvm, attr);
1409 case KVM_S390_VM_TOD:
1410 ret = kvm_s390_set_tod(kvm, attr);
1412 case KVM_S390_VM_CPU_MODEL:
1413 ret = kvm_s390_set_cpu_model(kvm, attr);
1415 case KVM_S390_VM_CRYPTO:
1416 ret = kvm_s390_vm_set_crypto(kvm, attr);
1418 case KVM_S390_VM_MIGRATION:
1419 ret = kvm_s390_vm_set_migration(kvm, attr);
1429 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1433 switch (attr->group) {
1434 case KVM_S390_VM_MEM_CTRL:
1435 ret = kvm_s390_get_mem_control(kvm, attr);
1437 case KVM_S390_VM_TOD:
1438 ret = kvm_s390_get_tod(kvm, attr);
1440 case KVM_S390_VM_CPU_MODEL:
1441 ret = kvm_s390_get_cpu_model(kvm, attr);
1443 case KVM_S390_VM_MIGRATION:
1444 ret = kvm_s390_vm_get_migration(kvm, attr);
1454 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1458 switch (attr->group) {
1459 case KVM_S390_VM_MEM_CTRL:
1460 switch (attr->attr) {
1461 case KVM_S390_VM_MEM_ENABLE_CMMA:
1462 case KVM_S390_VM_MEM_CLR_CMMA:
1463 ret = sclp.has_cmma ? 0 : -ENXIO;
1465 case KVM_S390_VM_MEM_LIMIT_SIZE:
1473 case KVM_S390_VM_TOD:
1474 switch (attr->attr) {
1475 case KVM_S390_VM_TOD_LOW:
1476 case KVM_S390_VM_TOD_HIGH:
1484 case KVM_S390_VM_CPU_MODEL:
1485 switch (attr->attr) {
1486 case KVM_S390_VM_CPU_PROCESSOR:
1487 case KVM_S390_VM_CPU_MACHINE:
1488 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1489 case KVM_S390_VM_CPU_MACHINE_FEAT:
1490 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1493 /* configuring subfunctions is not supported yet */
1494 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1500 case KVM_S390_VM_CRYPTO:
1501 switch (attr->attr) {
1502 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1503 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1504 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1505 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1513 case KVM_S390_VM_MIGRATION:
1524 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1528 int srcu_idx, i, r = 0;
1530 if (args->flags != 0)
1533 /* Is this guest using storage keys? */
1534 if (!mm_uses_skeys(current->mm))
1535 return KVM_S390_GET_SKEYS_NONE;
1537 /* Enforce sane limit on memory allocation */
1538 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1541 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1545 down_read(¤t->mm->mmap_sem);
1546 srcu_idx = srcu_read_lock(&kvm->srcu);
1547 for (i = 0; i < args->count; i++) {
1548 hva = gfn_to_hva(kvm, args->start_gfn + i);
1549 if (kvm_is_error_hva(hva)) {
1554 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1558 srcu_read_unlock(&kvm->srcu, srcu_idx);
1559 up_read(¤t->mm->mmap_sem);
1562 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1563 sizeof(uint8_t) * args->count);
1572 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1576 int srcu_idx, i, r = 0;
1579 if (args->flags != 0)
1582 /* Enforce sane limit on memory allocation */
1583 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1586 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1590 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1591 sizeof(uint8_t) * args->count);
1597 /* Enable storage key handling for the guest */
1598 r = s390_enable_skey();
1603 down_read(¤t->mm->mmap_sem);
1604 srcu_idx = srcu_read_lock(&kvm->srcu);
1605 while (i < args->count) {
1607 hva = gfn_to_hva(kvm, args->start_gfn + i);
1608 if (kvm_is_error_hva(hva)) {
1613 /* Lowest order bit is reserved */
1614 if (keys[i] & 0x01) {
1619 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1621 r = fixup_user_fault(current, current->mm, hva,
1622 FAULT_FLAG_WRITE, &unlocked);
1629 srcu_read_unlock(&kvm->srcu, srcu_idx);
1630 up_read(¤t->mm->mmap_sem);
1637 * Base address and length must be sent at the start of each block, therefore
1638 * it's cheaper to send some clean data, as long as it's less than the size of
1641 #define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
1642 /* for consistency */
1643 #define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1646 * Similar to gfn_to_memslot, but returns the index of a memslot also when the
1647 * address falls in a hole. In that case the index of one of the memslots
1648 * bordering the hole is returned.
1650 static int gfn_to_memslot_approx(struct kvm_memslots *slots, gfn_t gfn)
1652 int start = 0, end = slots->used_slots;
1653 int slot = atomic_read(&slots->lru_slot);
1654 struct kvm_memory_slot *memslots = slots->memslots;
1656 if (gfn >= memslots[slot].base_gfn &&
1657 gfn < memslots[slot].base_gfn + memslots[slot].npages)
1660 while (start < end) {
1661 slot = start + (end - start) / 2;
1663 if (gfn >= memslots[slot].base_gfn)
1669 if (start >= slots->used_slots)
1670 return slots->used_slots - 1;
1672 if (gfn >= memslots[start].base_gfn &&
1673 gfn < memslots[start].base_gfn + memslots[start].npages) {
1674 atomic_set(&slots->lru_slot, start);
1680 static int kvm_s390_peek_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
1681 u8 *res, unsigned long bufsize)
1683 unsigned long pgstev, hva, cur_gfn = args->start_gfn;
1686 while (args->count < bufsize) {
1687 hva = gfn_to_hva(kvm, cur_gfn);
1689 * We return an error if the first value was invalid, but we
1690 * return successfully if at least one value was copied.
1692 if (kvm_is_error_hva(hva))
1693 return args->count ? 0 : -EFAULT;
1694 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
1696 res[args->count++] = (pgstev >> 24) & 0x43;
1703 static unsigned long kvm_s390_next_dirty_cmma(struct kvm_memslots *slots,
1704 unsigned long cur_gfn)
1706 int slotidx = gfn_to_memslot_approx(slots, cur_gfn);
1707 struct kvm_memory_slot *ms = slots->memslots + slotidx;
1708 unsigned long ofs = cur_gfn - ms->base_gfn;
1710 if (ms->base_gfn + ms->npages <= cur_gfn) {
1712 /* If we are above the highest slot, wrap around */
1714 slotidx = slots->used_slots - 1;
1716 ms = slots->memslots + slotidx;
1719 ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, ofs);
1720 while ((slotidx > 0) && (ofs >= ms->npages)) {
1722 ms = slots->memslots + slotidx;
1723 ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, 0);
1725 return ms->base_gfn + ofs;
1728 static int kvm_s390_get_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
1729 u8 *res, unsigned long bufsize)
1731 unsigned long mem_end, cur_gfn, next_gfn, hva, pgstev;
1732 struct kvm_memslots *slots = kvm_memslots(kvm);
1733 struct kvm_memory_slot *ms;
1735 cur_gfn = kvm_s390_next_dirty_cmma(slots, args->start_gfn);
1736 ms = gfn_to_memslot(kvm, cur_gfn);
1738 args->start_gfn = cur_gfn;
1741 next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
1742 mem_end = slots->memslots[0].base_gfn + slots->memslots[0].npages;
1744 while (args->count < bufsize) {
1745 hva = gfn_to_hva(kvm, cur_gfn);
1746 if (kvm_is_error_hva(hva))
1748 /* Decrement only if we actually flipped the bit to 0 */
1749 if (test_and_clear_bit(cur_gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms)))
1750 atomic64_dec(&kvm->arch.cmma_dirty_pages);
1751 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
1753 /* Save the value */
1754 res[args->count++] = (pgstev >> 24) & 0x43;
1755 /* If the next bit is too far away, stop. */
1756 if (next_gfn > cur_gfn + KVM_S390_MAX_BIT_DISTANCE)
1758 /* If we reached the previous "next", find the next one */
1759 if (cur_gfn == next_gfn)
1760 next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
1761 /* Reached the end of memory or of the buffer, stop */
1762 if ((next_gfn >= mem_end) ||
1763 (next_gfn - args->start_gfn >= bufsize))
1766 /* Reached the end of the current memslot, take the next one. */
1767 if (cur_gfn - ms->base_gfn >= ms->npages) {
1768 ms = gfn_to_memslot(kvm, cur_gfn);
1777 * This function searches for the next page with dirty CMMA attributes, and
1778 * saves the attributes in the buffer up to either the end of the buffer or
1779 * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
1780 * no trailing clean bytes are saved.
1781 * In case no dirty bits were found, or if CMMA was not enabled or used, the
1782 * output buffer will indicate 0 as length.
1784 static int kvm_s390_get_cmma_bits(struct kvm *kvm,
1785 struct kvm_s390_cmma_log *args)
1787 unsigned long bufsize;
1788 int srcu_idx, peek, ret;
1791 if (!kvm->arch.use_cmma)
1793 /* Invalid/unsupported flags were specified */
1794 if (args->flags & ~KVM_S390_CMMA_PEEK)
1796 /* Migration mode query, and we are not doing a migration */
1797 peek = !!(args->flags & KVM_S390_CMMA_PEEK);
1798 if (!peek && !kvm->arch.migration_mode)
1800 /* CMMA is disabled or was not used, or the buffer has length zero */
1801 bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
1802 if (!bufsize || !kvm->mm->context.uses_cmm) {
1803 memset(args, 0, sizeof(*args));
1806 /* We are not peeking, and there are no dirty pages */
1807 if (!peek && !atomic64_read(&kvm->arch.cmma_dirty_pages)) {
1808 memset(args, 0, sizeof(*args));
1812 values = vmalloc(bufsize);
1816 down_read(&kvm->mm->mmap_sem);
1817 srcu_idx = srcu_read_lock(&kvm->srcu);
1819 ret = kvm_s390_peek_cmma(kvm, args, values, bufsize);
1821 ret = kvm_s390_get_cmma(kvm, args, values, bufsize);
1822 srcu_read_unlock(&kvm->srcu, srcu_idx);
1823 up_read(&kvm->mm->mmap_sem);
1825 if (kvm->arch.migration_mode)
1826 args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages);
1828 args->remaining = 0;
1830 if (copy_to_user((void __user *)args->values, values, args->count))
1838 * This function sets the CMMA attributes for the given pages. If the input
1839 * buffer has zero length, no action is taken, otherwise the attributes are
1840 * set and the mm->context.uses_cmm flag is set.
1842 static int kvm_s390_set_cmma_bits(struct kvm *kvm,
1843 const struct kvm_s390_cmma_log *args)
1845 unsigned long hva, mask, pgstev, i;
1847 int srcu_idx, r = 0;
1851 if (!kvm->arch.use_cmma)
1853 /* invalid/unsupported flags */
1854 if (args->flags != 0)
1856 /* Enforce sane limit on memory allocation */
1857 if (args->count > KVM_S390_CMMA_SIZE_MAX)
1860 if (args->count == 0)
1863 bits = vmalloc(array_size(sizeof(*bits), args->count));
1867 r = copy_from_user(bits, (void __user *)args->values, args->count);
1873 down_read(&kvm->mm->mmap_sem);
1874 srcu_idx = srcu_read_lock(&kvm->srcu);
1875 for (i = 0; i < args->count; i++) {
1876 hva = gfn_to_hva(kvm, args->start_gfn + i);
1877 if (kvm_is_error_hva(hva)) {
1883 pgstev = pgstev << 24;
1884 mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
1885 set_pgste_bits(kvm->mm, hva, mask, pgstev);
1887 srcu_read_unlock(&kvm->srcu, srcu_idx);
1888 up_read(&kvm->mm->mmap_sem);
1890 if (!kvm->mm->context.uses_cmm) {
1891 down_write(&kvm->mm->mmap_sem);
1892 kvm->mm->context.uses_cmm = 1;
1893 up_write(&kvm->mm->mmap_sem);
1900 long kvm_arch_vm_ioctl(struct file *filp,
1901 unsigned int ioctl, unsigned long arg)
1903 struct kvm *kvm = filp->private_data;
1904 void __user *argp = (void __user *)arg;
1905 struct kvm_device_attr attr;
1909 case KVM_S390_INTERRUPT: {
1910 struct kvm_s390_interrupt s390int;
1913 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1915 r = kvm_s390_inject_vm(kvm, &s390int);
1918 case KVM_ENABLE_CAP: {
1919 struct kvm_enable_cap cap;
1921 if (copy_from_user(&cap, argp, sizeof(cap)))
1923 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1926 case KVM_CREATE_IRQCHIP: {
1927 struct kvm_irq_routing_entry routing;
1930 if (kvm->arch.use_irqchip) {
1931 /* Set up dummy routing. */
1932 memset(&routing, 0, sizeof(routing));
1933 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
1937 case KVM_SET_DEVICE_ATTR: {
1939 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1941 r = kvm_s390_vm_set_attr(kvm, &attr);
1944 case KVM_GET_DEVICE_ATTR: {
1946 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1948 r = kvm_s390_vm_get_attr(kvm, &attr);
1951 case KVM_HAS_DEVICE_ATTR: {
1953 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1955 r = kvm_s390_vm_has_attr(kvm, &attr);
1958 case KVM_S390_GET_SKEYS: {
1959 struct kvm_s390_skeys args;
1962 if (copy_from_user(&args, argp,
1963 sizeof(struct kvm_s390_skeys)))
1965 r = kvm_s390_get_skeys(kvm, &args);
1968 case KVM_S390_SET_SKEYS: {
1969 struct kvm_s390_skeys args;
1972 if (copy_from_user(&args, argp,
1973 sizeof(struct kvm_s390_skeys)))
1975 r = kvm_s390_set_skeys(kvm, &args);
1978 case KVM_S390_GET_CMMA_BITS: {
1979 struct kvm_s390_cmma_log args;
1982 if (copy_from_user(&args, argp, sizeof(args)))
1984 mutex_lock(&kvm->slots_lock);
1985 r = kvm_s390_get_cmma_bits(kvm, &args);
1986 mutex_unlock(&kvm->slots_lock);
1988 r = copy_to_user(argp, &args, sizeof(args));
1994 case KVM_S390_SET_CMMA_BITS: {
1995 struct kvm_s390_cmma_log args;
1998 if (copy_from_user(&args, argp, sizeof(args)))
2000 mutex_lock(&kvm->slots_lock);
2001 r = kvm_s390_set_cmma_bits(kvm, &args);
2002 mutex_unlock(&kvm->slots_lock);
2012 static int kvm_s390_query_ap_config(u8 *config)
2014 u32 fcn_code = 0x04000000UL;
2017 memset(config, 0, 128);
2021 ".long 0xb2af0000\n" /* PQAP(QCI) */
2027 : "r" (fcn_code), "r" (config)
2028 : "cc", "0", "2", "memory"
2034 static int kvm_s390_apxa_installed(void)
2039 if (test_facility(12)) {
2040 cc = kvm_s390_query_ap_config(config);
2043 pr_err("PQAP(QCI) failed with cc=%d", cc);
2045 return config[0] & 0x40;
2051 static void kvm_s390_set_crycb_format(struct kvm *kvm)
2053 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
2055 if (kvm_s390_apxa_installed())
2056 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
2058 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
2061 static u64 kvm_s390_get_initial_cpuid(void)
2066 cpuid.version = 0xff;
2067 return *((u64 *) &cpuid);
2070 static void kvm_s390_crypto_init(struct kvm *kvm)
2072 if (!test_kvm_facility(kvm, 76))
2075 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
2076 kvm_s390_set_crycb_format(kvm);
2078 /* Enable AES/DEA protected key functions by default */
2079 kvm->arch.crypto.aes_kw = 1;
2080 kvm->arch.crypto.dea_kw = 1;
2081 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
2082 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
2083 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
2084 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
2087 static void sca_dispose(struct kvm *kvm)
2089 if (kvm->arch.use_esca)
2090 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
2092 free_page((unsigned long)(kvm->arch.sca));
2093 kvm->arch.sca = NULL;
2096 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
2098 gfp_t alloc_flags = GFP_KERNEL;
2100 char debug_name[16];
2101 static unsigned long sca_offset;
2104 #ifdef CONFIG_KVM_S390_UCONTROL
2105 if (type & ~KVM_VM_S390_UCONTROL)
2107 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
2114 rc = s390_enable_sie();
2120 if (!sclp.has_64bscao)
2121 alloc_flags |= GFP_DMA;
2122 rwlock_init(&kvm->arch.sca_lock);
2123 /* start with basic SCA */
2124 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
2127 mutex_lock(&kvm_lock);
2129 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
2131 kvm->arch.sca = (struct bsca_block *)
2132 ((char *) kvm->arch.sca + sca_offset);
2133 mutex_unlock(&kvm_lock);
2135 sprintf(debug_name, "kvm-%u", current->pid);
2137 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
2141 BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
2142 kvm->arch.sie_page2 =
2143 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
2144 if (!kvm->arch.sie_page2)
2147 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
2149 for (i = 0; i < kvm_s390_fac_size(); i++) {
2150 kvm->arch.model.fac_mask[i] = S390_lowcore.stfle_fac_list[i] &
2151 (kvm_s390_fac_base[i] |
2152 kvm_s390_fac_ext[i]);
2153 kvm->arch.model.fac_list[i] = S390_lowcore.stfle_fac_list[i] &
2154 kvm_s390_fac_base[i];
2157 /* we are always in czam mode - even on pre z14 machines */
2158 set_kvm_facility(kvm->arch.model.fac_mask, 138);
2159 set_kvm_facility(kvm->arch.model.fac_list, 138);
2160 /* we emulate STHYI in kvm */
2161 set_kvm_facility(kvm->arch.model.fac_mask, 74);
2162 set_kvm_facility(kvm->arch.model.fac_list, 74);
2163 if (MACHINE_HAS_TLB_GUEST) {
2164 set_kvm_facility(kvm->arch.model.fac_mask, 147);
2165 set_kvm_facility(kvm->arch.model.fac_list, 147);
2168 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
2169 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
2171 kvm_s390_crypto_init(kvm);
2173 mutex_init(&kvm->arch.float_int.ais_lock);
2174 spin_lock_init(&kvm->arch.float_int.lock);
2175 for (i = 0; i < FIRQ_LIST_COUNT; i++)
2176 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
2177 init_waitqueue_head(&kvm->arch.ipte_wq);
2178 mutex_init(&kvm->arch.ipte_mutex);
2180 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
2181 VM_EVENT(kvm, 3, "vm created with type %lu", type);
2183 if (type & KVM_VM_S390_UCONTROL) {
2184 kvm->arch.gmap = NULL;
2185 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
2187 if (sclp.hamax == U64_MAX)
2188 kvm->arch.mem_limit = TASK_SIZE_MAX;
2190 kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
2192 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
2193 if (!kvm->arch.gmap)
2195 kvm->arch.gmap->private = kvm;
2196 kvm->arch.gmap->pfault_enabled = 0;
2199 kvm->arch.use_pfmfi = sclp.has_pfmfi;
2200 kvm->arch.use_skf = sclp.has_skey;
2201 spin_lock_init(&kvm->arch.start_stop_lock);
2202 kvm_s390_vsie_init(kvm);
2203 kvm_s390_gisa_init(kvm);
2204 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
2208 free_page((unsigned long)kvm->arch.sie_page2);
2209 debug_unregister(kvm->arch.dbf);
2211 KVM_EVENT(3, "creation of vm failed: %d", rc);
2215 bool kvm_arch_has_vcpu_debugfs(void)
2220 int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
2225 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
2227 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
2228 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
2229 kvm_s390_clear_local_irqs(vcpu);
2230 kvm_clear_async_pf_completion_queue(vcpu);
2231 if (!kvm_is_ucontrol(vcpu->kvm))
2234 if (kvm_is_ucontrol(vcpu->kvm))
2235 gmap_remove(vcpu->arch.gmap);
2237 if (vcpu->kvm->arch.use_cmma)
2238 kvm_s390_vcpu_unsetup_cmma(vcpu);
2239 free_page((unsigned long)(vcpu->arch.sie_block));
2241 kvm_vcpu_uninit(vcpu);
2242 kmem_cache_free(kvm_vcpu_cache, vcpu);
2245 static void kvm_free_vcpus(struct kvm *kvm)
2248 struct kvm_vcpu *vcpu;
2250 kvm_for_each_vcpu(i, vcpu, kvm)
2251 kvm_arch_vcpu_destroy(vcpu);
2253 mutex_lock(&kvm->lock);
2254 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
2255 kvm->vcpus[i] = NULL;
2257 atomic_set(&kvm->online_vcpus, 0);
2258 mutex_unlock(&kvm->lock);
2261 void kvm_arch_destroy_vm(struct kvm *kvm)
2263 kvm_free_vcpus(kvm);
2265 debug_unregister(kvm->arch.dbf);
2266 kvm_s390_gisa_destroy(kvm);
2267 free_page((unsigned long)kvm->arch.sie_page2);
2268 if (!kvm_is_ucontrol(kvm))
2269 gmap_remove(kvm->arch.gmap);
2270 kvm_s390_destroy_adapters(kvm);
2271 kvm_s390_clear_float_irqs(kvm);
2272 kvm_s390_vsie_destroy(kvm);
2273 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
2276 /* Section: vcpu related */
2277 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
2279 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
2280 if (!vcpu->arch.gmap)
2282 vcpu->arch.gmap->private = vcpu->kvm;
2287 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
2289 if (!kvm_s390_use_sca_entries())
2291 read_lock(&vcpu->kvm->arch.sca_lock);
2292 if (vcpu->kvm->arch.use_esca) {
2293 struct esca_block *sca = vcpu->kvm->arch.sca;
2295 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2296 sca->cpu[vcpu->vcpu_id].sda = 0;
2298 struct bsca_block *sca = vcpu->kvm->arch.sca;
2300 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2301 sca->cpu[vcpu->vcpu_id].sda = 0;
2303 read_unlock(&vcpu->kvm->arch.sca_lock);
2306 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
2308 if (!kvm_s390_use_sca_entries()) {
2309 struct bsca_block *sca = vcpu->kvm->arch.sca;
2311 /* we still need the basic sca for the ipte control */
2312 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2313 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2316 read_lock(&vcpu->kvm->arch.sca_lock);
2317 if (vcpu->kvm->arch.use_esca) {
2318 struct esca_block *sca = vcpu->kvm->arch.sca;
2320 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2321 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2322 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
2323 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2324 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2326 struct bsca_block *sca = vcpu->kvm->arch.sca;
2328 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2329 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2330 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2331 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2333 read_unlock(&vcpu->kvm->arch.sca_lock);
2336 /* Basic SCA to Extended SCA data copy routines */
2337 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
2340 d->sigp_ctrl.c = s->sigp_ctrl.c;
2341 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
2344 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
2348 d->ipte_control = s->ipte_control;
2350 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
2351 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
2354 static int sca_switch_to_extended(struct kvm *kvm)
2356 struct bsca_block *old_sca = kvm->arch.sca;
2357 struct esca_block *new_sca;
2358 struct kvm_vcpu *vcpu;
2359 unsigned int vcpu_idx;
2362 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
2366 scaoh = (u32)((u64)(new_sca) >> 32);
2367 scaol = (u32)(u64)(new_sca) & ~0x3fU;
2369 kvm_s390_vcpu_block_all(kvm);
2370 write_lock(&kvm->arch.sca_lock);
2372 sca_copy_b_to_e(new_sca, old_sca);
2374 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
2375 vcpu->arch.sie_block->scaoh = scaoh;
2376 vcpu->arch.sie_block->scaol = scaol;
2377 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2379 kvm->arch.sca = new_sca;
2380 kvm->arch.use_esca = 1;
2382 write_unlock(&kvm->arch.sca_lock);
2383 kvm_s390_vcpu_unblock_all(kvm);
2385 free_page((unsigned long)old_sca);
2387 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
2388 old_sca, kvm->arch.sca);
2392 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
2396 if (!kvm_s390_use_sca_entries()) {
2397 if (id < KVM_MAX_VCPUS)
2401 if (id < KVM_S390_BSCA_CPU_SLOTS)
2403 if (!sclp.has_esca || !sclp.has_64bscao)
2406 mutex_lock(&kvm->lock);
2407 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
2408 mutex_unlock(&kvm->lock);
2410 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
2413 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
2415 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2416 kvm_clear_async_pf_completion_queue(vcpu);
2417 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
2423 kvm_s390_set_prefix(vcpu, 0);
2424 if (test_kvm_facility(vcpu->kvm, 64))
2425 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
2426 if (test_kvm_facility(vcpu->kvm, 82))
2427 vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
2428 if (test_kvm_facility(vcpu->kvm, 133))
2429 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
2430 if (test_kvm_facility(vcpu->kvm, 156))
2431 vcpu->run->kvm_valid_regs |= KVM_SYNC_ETOKEN;
2432 /* fprs can be synchronized via vrs, even if the guest has no vx. With
2433 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
2436 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
2438 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
2440 if (kvm_is_ucontrol(vcpu->kvm))
2441 return __kvm_ucontrol_vcpu_init(vcpu);
2446 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2447 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2449 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
2450 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2451 vcpu->arch.cputm_start = get_tod_clock_fast();
2452 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2455 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2456 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2458 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
2459 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2460 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2461 vcpu->arch.cputm_start = 0;
2462 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2465 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2466 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2468 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
2469 vcpu->arch.cputm_enabled = true;
2470 __start_cpu_timer_accounting(vcpu);
2473 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2474 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2476 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
2477 __stop_cpu_timer_accounting(vcpu);
2478 vcpu->arch.cputm_enabled = false;
2481 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2483 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2484 __enable_cpu_timer_accounting(vcpu);
2488 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2490 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2491 __disable_cpu_timer_accounting(vcpu);
2495 /* set the cpu timer - may only be called from the VCPU thread itself */
2496 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
2498 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2499 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2500 if (vcpu->arch.cputm_enabled)
2501 vcpu->arch.cputm_start = get_tod_clock_fast();
2502 vcpu->arch.sie_block->cputm = cputm;
2503 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2507 /* update and get the cpu timer - can also be called from other VCPU threads */
2508 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
2513 if (unlikely(!vcpu->arch.cputm_enabled))
2514 return vcpu->arch.sie_block->cputm;
2516 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2518 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
2520 * If the writer would ever execute a read in the critical
2521 * section, e.g. in irq context, we have a deadlock.
2523 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
2524 value = vcpu->arch.sie_block->cputm;
2525 /* if cputm_start is 0, accounting is being started/stopped */
2526 if (likely(vcpu->arch.cputm_start))
2527 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2528 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
2533 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2536 gmap_enable(vcpu->arch.enabled_gmap);
2537 kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
2538 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2539 __start_cpu_timer_accounting(vcpu);
2543 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
2546 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2547 __stop_cpu_timer_accounting(vcpu);
2548 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
2549 vcpu->arch.enabled_gmap = gmap_get_enabled();
2550 gmap_disable(vcpu->arch.enabled_gmap);
2554 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
2556 /* this equals initial cpu reset in pop, but we don't switch to ESA */
2557 vcpu->arch.sie_block->gpsw.mask = 0UL;
2558 vcpu->arch.sie_block->gpsw.addr = 0UL;
2559 kvm_s390_set_prefix(vcpu, 0);
2560 kvm_s390_set_cpu_timer(vcpu, 0);
2561 vcpu->arch.sie_block->ckc = 0UL;
2562 vcpu->arch.sie_block->todpr = 0;
2563 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
2564 vcpu->arch.sie_block->gcr[0] = CR0_UNUSED_56 |
2565 CR0_INTERRUPT_KEY_SUBMASK |
2566 CR0_MEASUREMENT_ALERT_SUBMASK;
2567 vcpu->arch.sie_block->gcr[14] = CR14_UNUSED_32 |
2569 CR14_EXTERNAL_DAMAGE_SUBMASK;
2570 vcpu->run->s.regs.fpc = 0;
2571 vcpu->arch.sie_block->gbea = 1;
2572 vcpu->arch.sie_block->pp = 0;
2573 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
2574 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2575 kvm_clear_async_pf_completion_queue(vcpu);
2576 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
2577 kvm_s390_vcpu_stop(vcpu);
2578 kvm_s390_clear_local_irqs(vcpu);
2581 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
2583 mutex_lock(&vcpu->kvm->lock);
2585 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
2586 vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
2588 mutex_unlock(&vcpu->kvm->lock);
2589 if (!kvm_is_ucontrol(vcpu->kvm)) {
2590 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
2593 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
2594 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2595 /* make vcpu_load load the right gmap on the first trigger */
2596 vcpu->arch.enabled_gmap = vcpu->arch.gmap;
2599 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
2601 if (!test_kvm_facility(vcpu->kvm, 76))
2604 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
2606 if (vcpu->kvm->arch.crypto.aes_kw)
2607 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
2608 if (vcpu->kvm->arch.crypto.dea_kw)
2609 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
2611 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
2614 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
2616 free_page(vcpu->arch.sie_block->cbrlo);
2617 vcpu->arch.sie_block->cbrlo = 0;
2620 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
2622 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
2623 if (!vcpu->arch.sie_block->cbrlo)
2628 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
2630 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
2632 vcpu->arch.sie_block->ibc = model->ibc;
2633 if (test_kvm_facility(vcpu->kvm, 7))
2634 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
2637 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
2641 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
2645 if (test_kvm_facility(vcpu->kvm, 78))
2646 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
2647 else if (test_kvm_facility(vcpu->kvm, 8))
2648 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
2650 kvm_s390_vcpu_setup_model(vcpu);
2652 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
2653 if (MACHINE_HAS_ESOP)
2654 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
2655 if (test_kvm_facility(vcpu->kvm, 9))
2656 vcpu->arch.sie_block->ecb |= ECB_SRSI;
2657 if (test_kvm_facility(vcpu->kvm, 73))
2658 vcpu->arch.sie_block->ecb |= ECB_TE;
2660 if (test_kvm_facility(vcpu->kvm, 8) && vcpu->kvm->arch.use_pfmfi)
2661 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
2662 if (test_kvm_facility(vcpu->kvm, 130))
2663 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
2664 vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
2666 vcpu->arch.sie_block->eca |= ECA_CEI;
2668 vcpu->arch.sie_block->eca |= ECA_IB;
2670 vcpu->arch.sie_block->eca |= ECA_SII;
2671 if (sclp.has_sigpif)
2672 vcpu->arch.sie_block->eca |= ECA_SIGPI;
2673 if (test_kvm_facility(vcpu->kvm, 129)) {
2674 vcpu->arch.sie_block->eca |= ECA_VX;
2675 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
2677 if (test_kvm_facility(vcpu->kvm, 139))
2678 vcpu->arch.sie_block->ecd |= ECD_MEF;
2679 if (test_kvm_facility(vcpu->kvm, 156))
2680 vcpu->arch.sie_block->ecd |= ECD_ETOKENF;
2681 if (vcpu->arch.sie_block->gd) {
2682 vcpu->arch.sie_block->eca |= ECA_AIV;
2683 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
2684 vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
2686 vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
2688 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
2691 kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
2693 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
2695 if (vcpu->kvm->arch.use_cmma) {
2696 rc = kvm_s390_vcpu_setup_cmma(vcpu);
2700 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2701 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
2703 kvm_s390_vcpu_crypto_setup(vcpu);
2708 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
2711 struct kvm_vcpu *vcpu;
2712 struct sie_page *sie_page;
2715 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
2720 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
2724 BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
2725 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
2729 vcpu->arch.sie_block = &sie_page->sie_block;
2730 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
2732 /* the real guest size will always be smaller than msl */
2733 vcpu->arch.sie_block->mso = 0;
2734 vcpu->arch.sie_block->msl = sclp.hamax;
2736 vcpu->arch.sie_block->icpua = id;
2737 spin_lock_init(&vcpu->arch.local_int.lock);
2738 vcpu->arch.sie_block->gd = (u32)(u64)kvm->arch.gisa;
2739 if (vcpu->arch.sie_block->gd && sclp.has_gisaf)
2740 vcpu->arch.sie_block->gd |= GISA_FORMAT1;
2741 seqcount_init(&vcpu->arch.cputm_seqcount);
2743 rc = kvm_vcpu_init(vcpu, kvm, id);
2745 goto out_free_sie_block;
2746 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
2747 vcpu->arch.sie_block);
2748 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
2752 free_page((unsigned long)(vcpu->arch.sie_block));
2754 kmem_cache_free(kvm_vcpu_cache, vcpu);
2759 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
2761 return kvm_s390_vcpu_has_irq(vcpu, 0);
2764 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
2766 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
2769 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
2771 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2775 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
2777 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2780 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
2782 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2786 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2788 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2792 * Kick a guest cpu out of SIE and wait until SIE is not running.
2793 * If the CPU is not running (e.g. waiting as idle) the function will
2794 * return immediately. */
2795 void exit_sie(struct kvm_vcpu *vcpu)
2797 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
2798 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2802 /* Kick a guest cpu out of SIE to process a request synchronously */
2803 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
2805 kvm_make_request(req, vcpu);
2806 kvm_s390_vcpu_request(vcpu);
2809 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
2812 struct kvm *kvm = gmap->private;
2813 struct kvm_vcpu *vcpu;
2814 unsigned long prefix;
2817 if (gmap_is_shadow(gmap))
2819 if (start >= 1UL << 31)
2820 /* We are only interested in prefix pages */
2822 kvm_for_each_vcpu(i, vcpu, kvm) {
2823 /* match against both prefix pages */
2824 prefix = kvm_s390_get_prefix(vcpu);
2825 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
2826 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
2828 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2833 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2835 /* kvm common code refers to this, but never calls it */
2840 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2841 struct kvm_one_reg *reg)
2846 case KVM_REG_S390_TODPR:
2847 r = put_user(vcpu->arch.sie_block->todpr,
2848 (u32 __user *)reg->addr);
2850 case KVM_REG_S390_EPOCHDIFF:
2851 r = put_user(vcpu->arch.sie_block->epoch,
2852 (u64 __user *)reg->addr);
2854 case KVM_REG_S390_CPU_TIMER:
2855 r = put_user(kvm_s390_get_cpu_timer(vcpu),
2856 (u64 __user *)reg->addr);
2858 case KVM_REG_S390_CLOCK_COMP:
2859 r = put_user(vcpu->arch.sie_block->ckc,
2860 (u64 __user *)reg->addr);
2862 case KVM_REG_S390_PFTOKEN:
2863 r = put_user(vcpu->arch.pfault_token,
2864 (u64 __user *)reg->addr);
2866 case KVM_REG_S390_PFCOMPARE:
2867 r = put_user(vcpu->arch.pfault_compare,
2868 (u64 __user *)reg->addr);
2870 case KVM_REG_S390_PFSELECT:
2871 r = put_user(vcpu->arch.pfault_select,
2872 (u64 __user *)reg->addr);
2874 case KVM_REG_S390_PP:
2875 r = put_user(vcpu->arch.sie_block->pp,
2876 (u64 __user *)reg->addr);
2878 case KVM_REG_S390_GBEA:
2879 r = put_user(vcpu->arch.sie_block->gbea,
2880 (u64 __user *)reg->addr);
2889 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2890 struct kvm_one_reg *reg)
2896 case KVM_REG_S390_TODPR:
2897 r = get_user(vcpu->arch.sie_block->todpr,
2898 (u32 __user *)reg->addr);
2900 case KVM_REG_S390_EPOCHDIFF:
2901 r = get_user(vcpu->arch.sie_block->epoch,
2902 (u64 __user *)reg->addr);
2904 case KVM_REG_S390_CPU_TIMER:
2905 r = get_user(val, (u64 __user *)reg->addr);
2907 kvm_s390_set_cpu_timer(vcpu, val);
2909 case KVM_REG_S390_CLOCK_COMP:
2910 r = get_user(vcpu->arch.sie_block->ckc,
2911 (u64 __user *)reg->addr);
2913 case KVM_REG_S390_PFTOKEN:
2914 r = get_user(vcpu->arch.pfault_token,
2915 (u64 __user *)reg->addr);
2916 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2917 kvm_clear_async_pf_completion_queue(vcpu);
2919 case KVM_REG_S390_PFCOMPARE:
2920 r = get_user(vcpu->arch.pfault_compare,
2921 (u64 __user *)reg->addr);
2923 case KVM_REG_S390_PFSELECT:
2924 r = get_user(vcpu->arch.pfault_select,
2925 (u64 __user *)reg->addr);
2927 case KVM_REG_S390_PP:
2928 r = get_user(vcpu->arch.sie_block->pp,
2929 (u64 __user *)reg->addr);
2931 case KVM_REG_S390_GBEA:
2932 r = get_user(vcpu->arch.sie_block->gbea,
2933 (u64 __user *)reg->addr);
2942 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2944 kvm_s390_vcpu_initial_reset(vcpu);
2948 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2951 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs));
2956 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2959 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
2964 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2965 struct kvm_sregs *sregs)
2969 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
2970 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
2976 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2977 struct kvm_sregs *sregs)
2981 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
2982 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
2988 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2994 if (test_fp_ctl(fpu->fpc)) {
2998 vcpu->run->s.regs.fpc = fpu->fpc;
3000 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
3001 (freg_t *) fpu->fprs);
3003 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
3010 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3014 /* make sure we have the latest values */
3017 convert_vx_to_fp((freg_t *) fpu->fprs,
3018 (__vector128 *) vcpu->run->s.regs.vrs);
3020 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
3021 fpu->fpc = vcpu->run->s.regs.fpc;
3027 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
3031 if (!is_vcpu_stopped(vcpu))
3034 vcpu->run->psw_mask = psw.mask;
3035 vcpu->run->psw_addr = psw.addr;
3040 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
3041 struct kvm_translation *tr)
3043 return -EINVAL; /* not implemented yet */
3046 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
3047 KVM_GUESTDBG_USE_HW_BP | \
3048 KVM_GUESTDBG_ENABLE)
3050 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
3051 struct kvm_guest_debug *dbg)
3057 vcpu->guest_debug = 0;
3058 kvm_s390_clear_bp_data(vcpu);
3060 if (dbg->control & ~VALID_GUESTDBG_FLAGS) {
3064 if (!sclp.has_gpere) {
3069 if (dbg->control & KVM_GUESTDBG_ENABLE) {
3070 vcpu->guest_debug = dbg->control;
3071 /* enforce guest PER */
3072 kvm_s390_set_cpuflags(vcpu, CPUSTAT_P);
3074 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
3075 rc = kvm_s390_import_bp_data(vcpu, dbg);
3077 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
3078 vcpu->arch.guestdbg.last_bp = 0;
3082 vcpu->guest_debug = 0;
3083 kvm_s390_clear_bp_data(vcpu);
3084 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
3092 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
3093 struct kvm_mp_state *mp_state)
3099 /* CHECK_STOP and LOAD are not supported yet */
3100 ret = is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
3101 KVM_MP_STATE_OPERATING;
3107 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
3108 struct kvm_mp_state *mp_state)
3114 /* user space knows about this interface - let it control the state */
3115 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
3117 switch (mp_state->mp_state) {
3118 case KVM_MP_STATE_STOPPED:
3119 kvm_s390_vcpu_stop(vcpu);
3121 case KVM_MP_STATE_OPERATING:
3122 kvm_s390_vcpu_start(vcpu);
3124 case KVM_MP_STATE_LOAD:
3125 case KVM_MP_STATE_CHECK_STOP:
3126 /* fall through - CHECK_STOP and LOAD are not supported yet */
3135 static bool ibs_enabled(struct kvm_vcpu *vcpu)
3137 return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS);
3140 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
3143 kvm_s390_vcpu_request_handled(vcpu);
3144 if (!kvm_request_pending(vcpu))
3147 * We use MMU_RELOAD just to re-arm the ipte notifier for the
3148 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
3149 * This ensures that the ipte instruction for this request has
3150 * already finished. We might race against a second unmapper that
3151 * wants to set the blocking bit. Lets just retry the request loop.
3153 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
3155 rc = gmap_mprotect_notify(vcpu->arch.gmap,
3156 kvm_s390_get_prefix(vcpu),
3157 PAGE_SIZE * 2, PROT_WRITE);
3159 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
3165 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
3166 vcpu->arch.sie_block->ihcpu = 0xffff;
3170 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
3171 if (!ibs_enabled(vcpu)) {
3172 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
3173 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS);
3178 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
3179 if (ibs_enabled(vcpu)) {
3180 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
3181 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS);
3186 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
3187 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
3191 if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
3193 * Disable CMM virtualization; we will emulate the ESSA
3194 * instruction manually, in order to provide additional
3195 * functionalities needed for live migration.
3197 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
3201 if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
3203 * Re-enable CMM virtualization if CMMA is available and
3204 * CMM has been used.
3206 if ((vcpu->kvm->arch.use_cmma) &&
3207 (vcpu->kvm->mm->context.uses_cmm))
3208 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
3212 /* nothing to do, just clear the request */
3213 kvm_clear_request(KVM_REQ_UNHALT, vcpu);
3218 void kvm_s390_set_tod_clock(struct kvm *kvm,
3219 const struct kvm_s390_vm_tod_clock *gtod)
3221 struct kvm_vcpu *vcpu;
3222 struct kvm_s390_tod_clock_ext htod;
3225 mutex_lock(&kvm->lock);
3228 get_tod_clock_ext((char *)&htod);
3230 kvm->arch.epoch = gtod->tod - htod.tod;
3232 if (test_kvm_facility(kvm, 139)) {
3233 kvm->arch.epdx = gtod->epoch_idx - htod.epoch_idx;
3234 if (kvm->arch.epoch > gtod->tod)
3235 kvm->arch.epdx -= 1;
3238 kvm_s390_vcpu_block_all(kvm);
3239 kvm_for_each_vcpu(i, vcpu, kvm) {
3240 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
3241 vcpu->arch.sie_block->epdx = kvm->arch.epdx;
3244 kvm_s390_vcpu_unblock_all(kvm);
3246 mutex_unlock(&kvm->lock);
3250 * kvm_arch_fault_in_page - fault-in guest page if necessary
3251 * @vcpu: The corresponding virtual cpu
3252 * @gpa: Guest physical address
3253 * @writable: Whether the page should be writable or not
3255 * Make sure that a guest page has been faulted-in on the host.
3257 * Return: Zero on success, negative error code otherwise.
3259 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
3261 return gmap_fault(vcpu->arch.gmap, gpa,
3262 writable ? FAULT_FLAG_WRITE : 0);
3265 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
3266 unsigned long token)
3268 struct kvm_s390_interrupt inti;
3269 struct kvm_s390_irq irq;
3272 irq.u.ext.ext_params2 = token;
3273 irq.type = KVM_S390_INT_PFAULT_INIT;
3274 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3276 inti.type = KVM_S390_INT_PFAULT_DONE;
3277 inti.parm64 = token;
3278 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
3282 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
3283 struct kvm_async_pf *work)
3285 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
3286 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
3289 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
3290 struct kvm_async_pf *work)
3292 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
3293 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
3296 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
3297 struct kvm_async_pf *work)
3299 /* s390 will always inject the page directly */
3302 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
3305 * s390 will always inject the page directly,
3306 * but we still want check_async_completion to cleanup
3311 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
3314 struct kvm_arch_async_pf arch;
3317 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3319 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
3320 vcpu->arch.pfault_compare)
3322 if (psw_extint_disabled(vcpu))
3324 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3326 if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
3328 if (!vcpu->arch.gmap->pfault_enabled)
3331 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
3332 hva += current->thread.gmap_addr & ~PAGE_MASK;
3333 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3336 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
3340 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
3345 * On s390 notifications for arriving pages will be delivered directly
3346 * to the guest but the house keeping for completed pfaults is
3347 * handled outside the worker.
3349 kvm_check_async_pf_completion(vcpu);
3351 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
3352 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
3357 if (test_cpu_flag(CIF_MCCK_PENDING))
3360 if (!kvm_is_ucontrol(vcpu->kvm)) {
3361 rc = kvm_s390_deliver_pending_interrupts(vcpu);
3366 rc = kvm_s390_handle_requests(vcpu);
3370 if (guestdbg_enabled(vcpu)) {
3371 kvm_s390_backup_guest_per_regs(vcpu);
3372 kvm_s390_patch_guest_per_regs(vcpu);
3375 vcpu->arch.sie_block->icptcode = 0;
3376 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
3377 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
3378 trace_kvm_s390_sie_enter(vcpu, cpuflags);
3383 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
3385 struct kvm_s390_pgm_info pgm_info = {
3386 .code = PGM_ADDRESSING,
3391 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
3392 trace_kvm_s390_sie_fault(vcpu);
3395 * We want to inject an addressing exception, which is defined as a
3396 * suppressing or terminating exception. However, since we came here
3397 * by a DAT access exception, the PSW still points to the faulting
3398 * instruction since DAT exceptions are nullifying. So we've got
3399 * to look up the current opcode to get the length of the instruction
3400 * to be able to forward the PSW.
3402 rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
3403 ilen = insn_length(opcode);
3407 /* Instruction-Fetching Exceptions - we can't detect the ilen.
3408 * Forward by arbitrary ilc, injection will take care of
3409 * nullification if necessary.
3411 pgm_info = vcpu->arch.pgm;
3414 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
3415 kvm_s390_forward_psw(vcpu, ilen);
3416 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
3419 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
3421 struct mcck_volatile_info *mcck_info;
3422 struct sie_page *sie_page;
3424 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
3425 vcpu->arch.sie_block->icptcode);
3426 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
3428 if (guestdbg_enabled(vcpu))
3429 kvm_s390_restore_guest_per_regs(vcpu);
3431 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
3432 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
3434 if (exit_reason == -EINTR) {
3435 VCPU_EVENT(vcpu, 3, "%s", "machine check");
3436 sie_page = container_of(vcpu->arch.sie_block,
3437 struct sie_page, sie_block);
3438 mcck_info = &sie_page->mcck_info;
3439 kvm_s390_reinject_machine_check(vcpu, mcck_info);
3443 if (vcpu->arch.sie_block->icptcode > 0) {
3444 int rc = kvm_handle_sie_intercept(vcpu);
3446 if (rc != -EOPNOTSUPP)
3448 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
3449 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
3450 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
3451 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
3453 } else if (exit_reason != -EFAULT) {
3454 vcpu->stat.exit_null++;
3456 } else if (kvm_is_ucontrol(vcpu->kvm)) {
3457 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
3458 vcpu->run->s390_ucontrol.trans_exc_code =
3459 current->thread.gmap_addr;
3460 vcpu->run->s390_ucontrol.pgm_code = 0x10;
3462 } else if (current->thread.gmap_pfault) {
3463 trace_kvm_s390_major_guest_pfault(vcpu);
3464 current->thread.gmap_pfault = 0;
3465 if (kvm_arch_setup_async_pf(vcpu))
3467 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
3469 return vcpu_post_run_fault_in_sie(vcpu);
3472 static int __vcpu_run(struct kvm_vcpu *vcpu)
3474 int rc, exit_reason;
3477 * We try to hold kvm->srcu during most of vcpu_run (except when run-
3478 * ning the guest), so that memslots (and other stuff) are protected
3480 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3483 rc = vcpu_pre_run(vcpu);
3487 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
3489 * As PF_VCPU will be used in fault handler, between
3490 * guest_enter and guest_exit should be no uaccess.
3492 local_irq_disable();
3493 guest_enter_irqoff();
3494 __disable_cpu_timer_accounting(vcpu);
3496 exit_reason = sie64a(vcpu->arch.sie_block,
3497 vcpu->run->s.regs.gprs);
3498 local_irq_disable();
3499 __enable_cpu_timer_accounting(vcpu);
3500 guest_exit_irqoff();
3502 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3504 rc = vcpu_post_run(vcpu, exit_reason);
3505 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3507 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
3511 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3513 struct runtime_instr_cb *riccb;
3516 riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
3517 gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
3518 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
3519 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
3520 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
3521 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
3522 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
3523 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
3524 /* some control register changes require a tlb flush */
3525 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3527 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
3528 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
3529 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
3530 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
3531 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
3532 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
3534 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
3535 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
3536 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
3537 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
3538 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3539 kvm_clear_async_pf_completion_queue(vcpu);
3542 * If userspace sets the riccb (e.g. after migration) to a valid state,
3543 * we should enable RI here instead of doing the lazy enablement.
3545 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
3546 test_kvm_facility(vcpu->kvm, 64) &&
3548 !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
3549 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
3550 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
3553 * If userspace sets the gscb (e.g. after migration) to non-zero,
3554 * we should enable GS here instead of doing the lazy enablement.
3556 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
3557 test_kvm_facility(vcpu->kvm, 133) &&
3559 !vcpu->arch.gs_enabled) {
3560 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
3561 vcpu->arch.sie_block->ecb |= ECB_GS;
3562 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3563 vcpu->arch.gs_enabled = 1;
3565 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) &&
3566 test_kvm_facility(vcpu->kvm, 82)) {
3567 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
3568 vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
3570 save_access_regs(vcpu->arch.host_acrs);
3571 restore_access_regs(vcpu->run->s.regs.acrs);
3572 /* save host (userspace) fprs/vrs */
3574 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
3575 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
3577 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
3579 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
3580 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
3581 if (test_fp_ctl(current->thread.fpu.fpc))
3582 /* User space provided an invalid FPC, let's clear it */
3583 current->thread.fpu.fpc = 0;
3584 if (MACHINE_HAS_GS) {
3586 __ctl_set_bit(2, 4);
3587 if (current->thread.gs_cb) {
3588 vcpu->arch.host_gscb = current->thread.gs_cb;
3589 save_gs_cb(vcpu->arch.host_gscb);
3591 if (vcpu->arch.gs_enabled) {
3592 current->thread.gs_cb = (struct gs_cb *)
3593 &vcpu->run->s.regs.gscb;
3594 restore_gs_cb(current->thread.gs_cb);
3598 /* SIE will load etoken directly from SDNX and therefore kvm_run */
3600 kvm_run->kvm_dirty_regs = 0;
3603 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3605 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
3606 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
3607 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
3608 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
3609 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
3610 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
3611 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
3612 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
3613 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
3614 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
3615 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
3616 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
3617 kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
3618 save_access_regs(vcpu->run->s.regs.acrs);
3619 restore_access_regs(vcpu->arch.host_acrs);
3620 /* Save guest register state */
3622 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3623 /* Restore will be done lazily at return */
3624 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
3625 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
3626 if (MACHINE_HAS_GS) {
3628 __ctl_set_bit(2, 4);
3629 if (vcpu->arch.gs_enabled)
3630 save_gs_cb(current->thread.gs_cb);
3631 current->thread.gs_cb = vcpu->arch.host_gscb;
3632 restore_gs_cb(vcpu->arch.host_gscb);
3633 if (!vcpu->arch.host_gscb)
3634 __ctl_clear_bit(2, 4);
3635 vcpu->arch.host_gscb = NULL;
3638 /* SIE will save etoken directly into SDNX and therefore kvm_run */
3641 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3645 if (kvm_run->immediate_exit)
3650 if (guestdbg_exit_pending(vcpu)) {
3651 kvm_s390_prepare_debug_exit(vcpu);
3656 kvm_sigset_activate(vcpu);
3658 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
3659 kvm_s390_vcpu_start(vcpu);
3660 } else if (is_vcpu_stopped(vcpu)) {
3661 pr_err_ratelimited("can't run stopped vcpu %d\n",
3667 sync_regs(vcpu, kvm_run);
3668 enable_cpu_timer_accounting(vcpu);
3671 rc = __vcpu_run(vcpu);
3673 if (signal_pending(current) && !rc) {
3674 kvm_run->exit_reason = KVM_EXIT_INTR;
3678 if (guestdbg_exit_pending(vcpu) && !rc) {
3679 kvm_s390_prepare_debug_exit(vcpu);
3683 if (rc == -EREMOTE) {
3684 /* userspace support is needed, kvm_run has been prepared */
3688 disable_cpu_timer_accounting(vcpu);
3689 store_regs(vcpu, kvm_run);
3691 kvm_sigset_deactivate(vcpu);
3693 vcpu->stat.exit_userspace++;
3700 * store status at address
3701 * we use have two special cases:
3702 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
3703 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
3705 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
3707 unsigned char archmode = 1;
3708 freg_t fprs[NUM_FPRS];
3713 px = kvm_s390_get_prefix(vcpu);
3714 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
3715 if (write_guest_abs(vcpu, 163, &archmode, 1))
3718 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
3719 if (write_guest_real(vcpu, 163, &archmode, 1))
3723 gpa -= __LC_FPREGS_SAVE_AREA;
3725 /* manually convert vector registers if necessary */
3726 if (MACHINE_HAS_VX) {
3727 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
3728 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
3731 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
3732 vcpu->run->s.regs.fprs, 128);
3734 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
3735 vcpu->run->s.regs.gprs, 128);
3736 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
3737 &vcpu->arch.sie_block->gpsw, 16);
3738 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
3740 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
3741 &vcpu->run->s.regs.fpc, 4);
3742 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
3743 &vcpu->arch.sie_block->todpr, 4);
3744 cputm = kvm_s390_get_cpu_timer(vcpu);
3745 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
3747 clkcomp = vcpu->arch.sie_block->ckc >> 8;
3748 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
3750 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
3751 &vcpu->run->s.regs.acrs, 64);
3752 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
3753 &vcpu->arch.sie_block->gcr, 128);
3754 return rc ? -EFAULT : 0;
3757 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
3760 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
3761 * switch in the run ioctl. Let's update our copies before we save
3762 * it into the save area
3765 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3766 save_access_regs(vcpu->run->s.regs.acrs);
3768 return kvm_s390_store_status_unloaded(vcpu, addr);
3771 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3773 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
3774 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
3777 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
3780 struct kvm_vcpu *vcpu;
3782 kvm_for_each_vcpu(i, vcpu, kvm) {
3783 __disable_ibs_on_vcpu(vcpu);
3787 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3791 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
3792 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
3795 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
3797 int i, online_vcpus, started_vcpus = 0;
3799 if (!is_vcpu_stopped(vcpu))
3802 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
3803 /* Only one cpu at a time may enter/leave the STOPPED state. */
3804 spin_lock(&vcpu->kvm->arch.start_stop_lock);
3805 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3807 for (i = 0; i < online_vcpus; i++) {
3808 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
3812 if (started_vcpus == 0) {
3813 /* we're the only active VCPU -> speed it up */
3814 __enable_ibs_on_vcpu(vcpu);
3815 } else if (started_vcpus == 1) {
3817 * As we are starting a second VCPU, we have to disable
3818 * the IBS facility on all VCPUs to remove potentially
3819 * oustanding ENABLE requests.
3821 __disable_ibs_on_all_vcpus(vcpu->kvm);
3824 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
3826 * Another VCPU might have used IBS while we were offline.
3827 * Let's play safe and flush the VCPU at startup.
3829 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3830 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
3834 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
3836 int i, online_vcpus, started_vcpus = 0;
3837 struct kvm_vcpu *started_vcpu = NULL;
3839 if (is_vcpu_stopped(vcpu))
3842 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
3843 /* Only one cpu at a time may enter/leave the STOPPED state. */
3844 spin_lock(&vcpu->kvm->arch.start_stop_lock);
3845 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3848 * Set the VCPU to STOPPED and THEN clear the interrupt flag,
3849 * now that the SIGP STOP and SIGP STOP AND STORE STATUS orders
3850 * have been fully processed. This will ensure that the VCPU
3851 * is kept BUSY if another VCPU is inquiring with SIGP SENSE.
3853 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
3854 kvm_s390_clear_stop_irq(vcpu);
3856 __disable_ibs_on_vcpu(vcpu);
3858 for (i = 0; i < online_vcpus; i++) {
3859 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
3861 started_vcpu = vcpu->kvm->vcpus[i];
3865 if (started_vcpus == 1) {
3867 * As we only have one VCPU left, we want to enable the
3868 * IBS facility for that VCPU to speed it up.
3870 __enable_ibs_on_vcpu(started_vcpu);
3873 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
3877 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
3878 struct kvm_enable_cap *cap)
3886 case KVM_CAP_S390_CSS_SUPPORT:
3887 if (!vcpu->kvm->arch.css_support) {
3888 vcpu->kvm->arch.css_support = 1;
3889 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
3890 trace_kvm_s390_enable_css(vcpu->kvm);
3901 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
3902 struct kvm_s390_mem_op *mop)
3904 void __user *uaddr = (void __user *)mop->buf;
3905 void *tmpbuf = NULL;
3907 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
3908 | KVM_S390_MEMOP_F_CHECK_ONLY;
3910 if (mop->flags & ~supported_flags || mop->ar >= NUM_ACRS || !mop->size)
3913 if (mop->size > MEM_OP_MAX_SIZE)
3916 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
3917 tmpbuf = vmalloc(mop->size);
3922 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3925 case KVM_S390_MEMOP_LOGICAL_READ:
3926 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
3927 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3928 mop->size, GACC_FETCH);
3931 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3933 if (copy_to_user(uaddr, tmpbuf, mop->size))
3937 case KVM_S390_MEMOP_LOGICAL_WRITE:
3938 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
3939 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3940 mop->size, GACC_STORE);
3943 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
3947 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3953 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
3955 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
3956 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
3962 long kvm_arch_vcpu_async_ioctl(struct file *filp,
3963 unsigned int ioctl, unsigned long arg)
3965 struct kvm_vcpu *vcpu = filp->private_data;
3966 void __user *argp = (void __user *)arg;
3969 case KVM_S390_IRQ: {
3970 struct kvm_s390_irq s390irq;
3972 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
3974 return kvm_s390_inject_vcpu(vcpu, &s390irq);
3976 case KVM_S390_INTERRUPT: {
3977 struct kvm_s390_interrupt s390int;
3978 struct kvm_s390_irq s390irq = {};
3980 if (copy_from_user(&s390int, argp, sizeof(s390int)))
3982 if (s390int_to_s390irq(&s390int, &s390irq))
3984 return kvm_s390_inject_vcpu(vcpu, &s390irq);
3987 return -ENOIOCTLCMD;
3990 long kvm_arch_vcpu_ioctl(struct file *filp,
3991 unsigned int ioctl, unsigned long arg)
3993 struct kvm_vcpu *vcpu = filp->private_data;
3994 void __user *argp = (void __user *)arg;
4001 case KVM_S390_STORE_STATUS:
4002 idx = srcu_read_lock(&vcpu->kvm->srcu);
4003 r = kvm_s390_store_status_unloaded(vcpu, arg);
4004 srcu_read_unlock(&vcpu->kvm->srcu, idx);
4006 case KVM_S390_SET_INITIAL_PSW: {
4010 if (copy_from_user(&psw, argp, sizeof(psw)))
4012 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
4015 case KVM_S390_INITIAL_RESET:
4016 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
4018 case KVM_SET_ONE_REG:
4019 case KVM_GET_ONE_REG: {
4020 struct kvm_one_reg reg;
4022 if (copy_from_user(®, argp, sizeof(reg)))
4024 if (ioctl == KVM_SET_ONE_REG)
4025 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®);
4027 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®);
4030 #ifdef CONFIG_KVM_S390_UCONTROL
4031 case KVM_S390_UCAS_MAP: {
4032 struct kvm_s390_ucas_mapping ucasmap;
4034 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4039 if (!kvm_is_ucontrol(vcpu->kvm)) {
4044 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
4045 ucasmap.vcpu_addr, ucasmap.length);
4048 case KVM_S390_UCAS_UNMAP: {
4049 struct kvm_s390_ucas_mapping ucasmap;
4051 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4056 if (!kvm_is_ucontrol(vcpu->kvm)) {
4061 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
4066 case KVM_S390_VCPU_FAULT: {
4067 r = gmap_fault(vcpu->arch.gmap, arg, 0);
4070 case KVM_ENABLE_CAP:
4072 struct kvm_enable_cap cap;
4074 if (copy_from_user(&cap, argp, sizeof(cap)))
4076 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
4079 case KVM_S390_MEM_OP: {
4080 struct kvm_s390_mem_op mem_op;
4082 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
4083 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
4088 case KVM_S390_SET_IRQ_STATE: {
4089 struct kvm_s390_irq_state irq_state;
4092 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
4094 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
4095 irq_state.len == 0 ||
4096 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
4100 /* do not use irq_state.flags, it will break old QEMUs */
4101 r = kvm_s390_set_irq_state(vcpu,
4102 (void __user *) irq_state.buf,
4106 case KVM_S390_GET_IRQ_STATE: {
4107 struct kvm_s390_irq_state irq_state;
4110 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
4112 if (irq_state.len == 0) {
4116 /* do not use irq_state.flags, it will break old QEMUs */
4117 r = kvm_s390_get_irq_state(vcpu,
4118 (__u8 __user *) irq_state.buf,
4130 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
4132 #ifdef CONFIG_KVM_S390_UCONTROL
4133 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
4134 && (kvm_is_ucontrol(vcpu->kvm))) {
4135 vmf->page = virt_to_page(vcpu->arch.sie_block);
4136 get_page(vmf->page);
4140 return VM_FAULT_SIGBUS;
4143 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
4144 unsigned long npages)
4149 /* Section: memory related */
4150 int kvm_arch_prepare_memory_region(struct kvm *kvm,
4151 struct kvm_memory_slot *memslot,
4152 const struct kvm_userspace_memory_region *mem,
4153 enum kvm_mr_change change)
4155 /* A few sanity checks. We can have memory slots which have to be
4156 located/ended at a segment boundary (1MB). The memory in userland is
4157 ok to be fragmented into various different vmas. It is okay to mmap()
4158 and munmap() stuff in this slot after doing this call at any time */
4160 if (mem->userspace_addr & 0xffffful)
4163 if (mem->memory_size & 0xffffful)
4166 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
4172 void kvm_arch_commit_memory_region(struct kvm *kvm,
4173 const struct kvm_userspace_memory_region *mem,
4174 const struct kvm_memory_slot *old,
4175 const struct kvm_memory_slot *new,
4176 enum kvm_mr_change change)
4182 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
4183 old->npages * PAGE_SIZE);
4186 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
4187 old->npages * PAGE_SIZE);
4192 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
4193 mem->guest_phys_addr, mem->memory_size);
4195 case KVM_MR_FLAGS_ONLY:
4198 WARN(1, "Unknown KVM MR CHANGE: %d\n", change);
4201 pr_warn("failed to commit memory region\n");
4205 static inline unsigned long nonhyp_mask(int i)
4207 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
4209 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
4212 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
4214 vcpu->valid_wakeup = false;
4217 static int __init kvm_s390_init(void)
4221 if (!sclp.has_sief2) {
4222 pr_info("SIE not available\n");
4226 if (nested && hpage) {
4227 pr_info("nested (vSIE) and hpage (huge page backing) can currently not be activated concurrently");
4231 for (i = 0; i < 16; i++)
4232 kvm_s390_fac_base[i] |=
4233 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
4235 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
4238 static void __exit kvm_s390_exit(void)
4243 module_init(kvm_s390_init);
4244 module_exit(kvm_s390_exit);
4247 * Enable autoloading of the kvm module.
4248 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
4249 * since x86 takes a different approach.
4251 #include <linux/miscdevice.h>
4252 MODULE_ALIAS_MISCDEV(KVM_MINOR);
4253 MODULE_ALIAS("devname:kvm");