arch/s390/kvm/kvm-s390.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * hosting IBM Z kernel virtual machines (s390x)
   4  *
   5  * Copyright IBM Corp. 2008, 2018
   6  *
   7  *    Author(s): Carsten Otte <cotte@de.ibm.com>
   8  *               Christian Borntraeger <borntraeger@de.ibm.com>
   9  *               Heiko Carstens <heiko.carstens@de.ibm.com>
  10  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
  11  *               Jason J. Herne <jjherne@us.ibm.com>
  12  */
  13
  14 #define KMSG_COMPONENT "kvm-s390"
  15 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  16
  17 #include <linux/compiler.h>
  18 #include <linux/err.h>
  19 #include <linux/fs.h>
  20 #include <linux/hrtimer.h>
  21 #include <linux/init.h>
  22 #include <linux/kvm.h>
  23 #include <linux/kvm_host.h>
  24 #include <linux/mman.h>
  25 #include <linux/module.h>
  26 #include <linux/moduleparam.h>
  27 #include <linux/random.h>
  28 #include <linux/slab.h>
  29 #include <linux/timer.h>
  30 #include <linux/vmalloc.h>
  31 #include <linux/bitmap.h>
  32 #include <linux/sched/signal.h>
  33 #include <linux/string.h>
  34
  35 #include <asm/asm-offsets.h>
  36 #include <asm/lowcore.h>
  37 #include <asm/stp.h>
  38 #include <asm/pgtable.h>
  39 #include <asm/gmap.h>
  40 #include <asm/nmi.h>
  41 #include <asm/switch_to.h>
  42 #include <asm/isc.h>
  43 #include <asm/sclp.h>
  44 #include <asm/cpacf.h>
  45 #include <asm/timex.h>
  46 #include <asm/ap.h>
  47 #include "kvm-s390.h"
  48 #include "gaccess.h"
  49
  50 #define CREATE_TRACE_POINTS
  51 #include "trace.h"
  52 #include "trace-s390.h"
  53
  54 #define MEM_OP_MAX_SIZE 65536   /* Maximum transfer size for KVM_S390_MEM_OP */
  55 #define LOCAL_IRQS 32
  56 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
  57                            (KVM_MAX_VCPUS + LOCAL_IRQS))
  58
  59 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  60 #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
  61
  62 struct kvm_stats_debugfs_item debugfs_entries[] = {
  63         { "userspace_handled", VCPU_STAT(exit_userspace) },
  64         { "exit_null", VCPU_STAT(exit_null) },
  65         { "exit_validity", VCPU_STAT(exit_validity) },
  66         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
  67         { "exit_external_request", VCPU_STAT(exit_external_request) },
  68         { "exit_io_request", VCPU_STAT(exit_io_request) },
  69         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
  70         { "exit_instruction", VCPU_STAT(exit_instruction) },
  71         { "exit_pei", VCPU_STAT(exit_pei) },
  72         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
  73         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
  74         { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
  75         { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
  76         { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
  77         { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
  78         { "halt_no_poll_steal", VCPU_STAT(halt_no_poll_steal) },
  79         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  80         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
  81         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
  82         { "instruction_stctl", VCPU_STAT(instruction_stctl) },
  83         { "instruction_stctg", VCPU_STAT(instruction_stctg) },
  84         { "deliver_ckc", VCPU_STAT(deliver_ckc) },
  85         { "deliver_cputm", VCPU_STAT(deliver_cputm) },
  86         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
  87         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
  88         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
  89         { "deliver_virtio", VCPU_STAT(deliver_virtio) },
  90         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
  91         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
  92         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
  93         { "deliver_program", VCPU_STAT(deliver_program) },
  94         { "deliver_io", VCPU_STAT(deliver_io) },
  95         { "deliver_machine_check", VCPU_STAT(deliver_machine_check) },
  96         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
  97         { "inject_ckc", VCPU_STAT(inject_ckc) },
  98         { "inject_cputm", VCPU_STAT(inject_cputm) },
  99         { "inject_external_call", VCPU_STAT(inject_external_call) },
 100         { "inject_float_mchk", VM_STAT(inject_float_mchk) },
 101         { "inject_emergency_signal", VCPU_STAT(inject_emergency_signal) },
 102         { "inject_io", VM_STAT(inject_io) },
 103         { "inject_mchk", VCPU_STAT(inject_mchk) },
 104         { "inject_pfault_done", VM_STAT(inject_pfault_done) },
 105         { "inject_program", VCPU_STAT(inject_program) },
 106         { "inject_restart", VCPU_STAT(inject_restart) },
 107         { "inject_service_signal", VM_STAT(inject_service_signal) },
 108         { "inject_set_prefix", VCPU_STAT(inject_set_prefix) },
 109         { "inject_stop_signal", VCPU_STAT(inject_stop_signal) },
 110         { "inject_pfault_init", VCPU_STAT(inject_pfault_init) },
 111         { "inject_virtio", VM_STAT(inject_virtio) },
 112         { "instruction_epsw", VCPU_STAT(instruction_epsw) },
 113         { "instruction_gs", VCPU_STAT(instruction_gs) },
 114         { "instruction_io_other", VCPU_STAT(instruction_io_other) },
 115         { "instruction_lpsw", VCPU_STAT(instruction_lpsw) },
 116         { "instruction_lpswe", VCPU_STAT(instruction_lpswe) },
 117         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
 118         { "instruction_ptff", VCPU_STAT(instruction_ptff) },
 119         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
 120         { "instruction_sck", VCPU_STAT(instruction_sck) },
 121         { "instruction_sckpf", VCPU_STAT(instruction_sckpf) },
 122         { "instruction_spx", VCPU_STAT(instruction_spx) },
 123         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
 124         { "instruction_stap", VCPU_STAT(instruction_stap) },
 125         { "instruction_iske", VCPU_STAT(instruction_iske) },
 126         { "instruction_ri", VCPU_STAT(instruction_ri) },
 127         { "instruction_rrbe", VCPU_STAT(instruction_rrbe) },
 128         { "instruction_sske", VCPU_STAT(instruction_sske) },
 129         { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
 130         { "instruction_essa", VCPU_STAT(instruction_essa) },
 131         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
 132         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
 133         { "instruction_tb", VCPU_STAT(instruction_tb) },
 134         { "instruction_tpi", VCPU_STAT(instruction_tpi) },
 135         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
 136         { "instruction_tsch", VCPU_STAT(instruction_tsch) },
 137         { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
 138         { "instruction_sie", VCPU_STAT(instruction_sie) },
 139         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
 140         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
 141         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
 142         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
 143         { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
 144         { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
 145         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
 146         { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
 147         { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
 148         { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
 149         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
 150         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
 151         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
 152         { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
 153         { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
 154         { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
 155         { "instruction_diag_10", VCPU_STAT(diagnose_10) },
 156         { "instruction_diag_44", VCPU_STAT(diagnose_44) },
 157         { "instruction_diag_9c", VCPU_STAT(diagnose_9c) },
 158         { "instruction_diag_258", VCPU_STAT(diagnose_258) },
 159         { "instruction_diag_308", VCPU_STAT(diagnose_308) },
 160         { "instruction_diag_500", VCPU_STAT(diagnose_500) },
 161         { "instruction_diag_other", VCPU_STAT(diagnose_other) },
 162         { NULL }
 163 };
 164
 165 struct kvm_s390_tod_clock_ext {
 166         __u8 epoch_idx;
 167         __u64 tod;
 168         __u8 reserved[7];
 169 } __packed;
 170
 171 /* allow nested virtualization in KVM (if enabled by user space) */
 172 static int nested;
 173 module_param(nested, int, S_IRUGO);
 174 MODULE_PARM_DESC(nested, "Nested virtualization support");
 175
 176 /* allow 1m huge page guest backing, if !nested */
 177 static int hpage;
 178 module_param(hpage, int, 0444);
 179 MODULE_PARM_DESC(hpage, "1m huge page backing support");
 180
 181 /* maximum percentage of steal time for polling.  >100 is treated like 100 */
 182 static u8 halt_poll_max_steal = 10;
 183 module_param(halt_poll_max_steal, byte, 0644);
 184 MODULE_PARM_DESC(halt_poll_max_steal, "Maximum percentage of steal time to allow polling");
 185
 186 /*
 187  * For now we handle at most 16 double words as this is what the s390 base
 188  * kernel handles and stores in the prefix page. If we ever need to go beyond
 189  * this, this requires changes to code, but the external uapi can stay.
 190  */
 191 #define SIZE_INTERNAL 16
 192
 193 /*
 194  * Base feature mask that defines default mask for facilities. Consists of the
 195  * defines in FACILITIES_KVM and the non-hypervisor managed bits.
 196  */
 197 static unsigned long kvm_s390_fac_base[SIZE_INTERNAL] = { FACILITIES_KVM };
 198 /*
 199  * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL
 200  * and defines the facilities that can be enabled via a cpu model.
 201  */
 202 static unsigned long kvm_s390_fac_ext[SIZE_INTERNAL] = { FACILITIES_KVM_CPUMODEL };
 203
 204 static unsigned long kvm_s390_fac_size(void)
 205 {
 206         BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_MASK_SIZE_U64);
 207         BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_LIST_SIZE_U64);
 208         BUILD_BUG_ON(SIZE_INTERNAL * sizeof(unsigned long) >
 209                 sizeof(S390_lowcore.stfle_fac_list));
 210
 211         return SIZE_INTERNAL;
 212 }
 213
 214 /* available cpu features supported by kvm */
 215 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
 216 /* available subfunctions indicated via query / "test bit" */
 217 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
 218
 219 static struct gmap_notifier gmap_notifier;
 220 static struct gmap_notifier vsie_gmap_notifier;
 221 debug_info_t *kvm_s390_dbf;
 222
 223 /* Section: not file related */
 224 int kvm_arch_hardware_enable(void)
 225 {
 226         /* every s390 is virtualization enabled ;-) */
 227         return 0;
 228 }
 229
 230 int kvm_arch_check_processor_compat(void)
 231 {
 232         return 0;
 233 }
 234
 235 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
 236                               unsigned long end);
 237
 238 static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
 239 {
 240         u8 delta_idx = 0;
 241
 242         /*
 243          * The TOD jumps by delta, we have to compensate this by adding
 244          * -delta to the epoch.
 245          */
 246         delta = -delta;
 247
 248         /* sign-extension - we're adding to signed values below */
 249         if ((s64)delta < 0)
 250                 delta_idx = -1;
 251
 252         scb->epoch += delta;
 253         if (scb->ecd & ECD_MEF) {
 254                 scb->epdx += delta_idx;
 255                 if (scb->epoch < delta)
 256                         scb->epdx += 1;
 257         }
 258 }
 259
 260 /*
 261  * This callback is executed during stop_machine(). All CPUs are therefore
 262  * temporarily stopped. In order not to change guest behavior, we have to
 263  * disable preemption whenever we touch the epoch of kvm and the VCPUs,
 264  * so a CPU won't be stopped while calculating with the epoch.
 265  */
 266 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
 267                           void *v)
 268 {
 269         struct kvm *kvm;
 270         struct kvm_vcpu *vcpu;
 271         int i;
 272         unsigned long long *delta = v;
 273
 274         list_for_each_entry(kvm, &vm_list, vm_list) {
 275                 kvm_for_each_vcpu(i, vcpu, kvm) {
 276                         kvm_clock_sync_scb(vcpu->arch.sie_block, *delta);
 277                         if (i == 0) {
 278                                 kvm->arch.epoch = vcpu->arch.sie_block->epoch;
 279                                 kvm->arch.epdx = vcpu->arch.sie_block->epdx;
 280                         }
 281                         if (vcpu->arch.cputm_enabled)
 282                                 vcpu->arch.cputm_start += *delta;
 283                         if (vcpu->arch.vsie_block)
 284                                 kvm_clock_sync_scb(vcpu->arch.vsie_block,
 285                                                    *delta);
 286                 }
 287         }
 288         return NOTIFY_OK;
 289 }
 290
 291 static struct notifier_block kvm_clock_notifier = {
 292         .notifier_call = kvm_clock_sync,
 293 };
 294
 295 int kvm_arch_hardware_setup(void)
 296 {
 297         gmap_notifier.notifier_call = kvm_gmap_notifier;
 298         gmap_register_pte_notifier(&gmap_notifier);
 299         vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
 300         gmap_register_pte_notifier(&vsie_gmap_notifier);
 301         atomic_notifier_chain_register(&s390_epoch_delta_notifier,
 302                                        &kvm_clock_notifier);
 303         return 0;
 304 }
 305
 306 void kvm_arch_hardware_unsetup(void)
 307 {
 308         gmap_unregister_pte_notifier(&gmap_notifier);
 309         gmap_unregister_pte_notifier(&vsie_gmap_notifier);
 310         atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
 311                                          &kvm_clock_notifier);
 312 }
 313
 314 static void allow_cpu_feat(unsigned long nr)
 315 {
 316         set_bit_inv(nr, kvm_s390_available_cpu_feat);
 317 }
 318
 319 static inline int plo_test_bit(unsigned char nr)
 320 {
 321         unsigned long function = (unsigned long)nr | 0x100;
 322         int cc;
 323
 324         asm volatile(
 325                 "       lgr     0,%[function]\n"
 326                 /* Parameter registers are ignored for "test bit" */
 327                 "       plo     0,0,0,0(0)\n"
 328                 "       ipm     %0\n"
 329                 "       srl     %0,28\n"
 330                 : "=d" (cc)
 331                 : [function] "d" (function)
 332                 : "cc", "0");
 333         return cc == 0;
 334 }
 335
 336 static __always_inline void __insn32_query(unsigned int opcode, u8 *query)
 337 {
 338         asm volatile(
 339                 "       lghi    0,0\n"
 340                 "       lgr     1,%[query]\n"
 341                 /* Parameter registers are ignored */
 342                 "       .insn   rrf,%[opc] << 16,2,4,6,0\n"
 343                 :
 344                 : [query] "d" ((unsigned long)query), [opc] "i" (opcode)
 345                 : "cc", "memory", "0", "1");
 346 }
 347
 348 #define INSN_SORTL 0xb938
 349 #define INSN_DFLTCC 0xb939
 350
 351 static void kvm_s390_cpu_feat_init(void)
 352 {
 353         int i;
 354
 355         for (i = 0; i < 256; ++i) {
 356                 if (plo_test_bit(i))
 357                         kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
 358         }
 359
 360         if (test_facility(28)) /* TOD-clock steering */
 361                 ptff(kvm_s390_available_subfunc.ptff,
 362                      sizeof(kvm_s390_available_subfunc.ptff),
 363                      PTFF_QAF);
 364
 365         if (test_facility(17)) { /* MSA */
 366                 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
 367                               kvm_s390_available_subfunc.kmac);
 368                 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
 369                               kvm_s390_available_subfunc.kmc);
 370                 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
 371                               kvm_s390_available_subfunc.km);
 372                 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
 373                               kvm_s390_available_subfunc.kimd);
 374                 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
 375                               kvm_s390_available_subfunc.klmd);
 376         }
 377         if (test_facility(76)) /* MSA3 */
 378                 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
 379                               kvm_s390_available_subfunc.pckmo);
 380         if (test_facility(77)) { /* MSA4 */
 381                 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
 382                               kvm_s390_available_subfunc.kmctr);
 383                 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
 384                               kvm_s390_available_subfunc.kmf);
 385                 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
 386                               kvm_s390_available_subfunc.kmo);
 387                 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
 388                               kvm_s390_available_subfunc.pcc);
 389         }
 390         if (test_facility(57)) /* MSA5 */
 391                 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
 392                               kvm_s390_available_subfunc.ppno);
 393
 394         if (test_facility(146)) /* MSA8 */
 395                 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
 396                               kvm_s390_available_subfunc.kma);
 397
 398         if (test_facility(155)) /* MSA9 */
 399                 __cpacf_query(CPACF_KDSA, (cpacf_mask_t *)
 400                               kvm_s390_available_subfunc.kdsa);
 401
 402         if (test_facility(150)) /* SORTL */
 403                 __insn32_query(INSN_SORTL, kvm_s390_available_subfunc.sortl);
 404
 405         if (test_facility(151)) /* DFLTCC */
 406                 __insn32_query(INSN_DFLTCC, kvm_s390_available_subfunc.dfltcc);
 407
 408         if (MACHINE_HAS_ESOP)
 409                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
 410         /*
 411          * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
 412          * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
 413          */
 414         if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
 415             !test_facility(3) || !nested)
 416                 return;
 417         allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
 418         if (sclp.has_64bscao)
 419                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
 420         if (sclp.has_siif)
 421                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
 422         if (sclp.has_gpere)
 423                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
 424         if (sclp.has_gsls)
 425                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
 426         if (sclp.has_ib)
 427                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
 428         if (sclp.has_cei)
 429                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
 430         if (sclp.has_ibs)
 431                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
 432         if (sclp.has_kss)
 433                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
 434         /*
 435          * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
 436          * all skey handling functions read/set the skey from the PGSTE
 437          * instead of the real storage key.
 438          *
 439          * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
 440          * pages being detected as preserved although they are resident.
 441          *
 442          * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
 443          * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
 444          *
 445          * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
 446          * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
 447          * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
 448          *
 449          * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
 450          * cannot easily shadow the SCA because of the ipte lock.
 451          */
 452 }
 453
 454 int kvm_arch_init(void *opaque)
 455 {
 456         int rc;
 457
 458         kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
 459         if (!kvm_s390_dbf)
 460                 return -ENOMEM;
 461
 462         if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
 463                 rc = -ENOMEM;
 464                 goto out_debug_unreg;
 465         }
 466
 467         kvm_s390_cpu_feat_init();
 468
 469         /* Register floating interrupt controller interface. */
 470         rc = kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
 471         if (rc) {
 472                 pr_err("A FLIC registration call failed with rc=%d\n", rc);
 473                 goto out_debug_unreg;
 474         }
 475
 476         rc = kvm_s390_gib_init(GAL_ISC);
 477         if (rc)
 478                 goto out_gib_destroy;
 479
 480         return 0;
 481
 482 out_gib_destroy:
 483         kvm_s390_gib_destroy();
 484 out_debug_unreg:
 485         debug_unregister(kvm_s390_dbf);
 486         return rc;
 487 }
 488
 489 void kvm_arch_exit(void)
 490 {
 491         kvm_s390_gib_destroy();
 492         debug_unregister(kvm_s390_dbf);
 493 }
 494
 495 /* Section: device related */
 496 long kvm_arch_dev_ioctl(struct file *filp,
 497                         unsigned int ioctl, unsigned long arg)
 498 {
 499         if (ioctl == KVM_S390_ENABLE_SIE)
 500                 return s390_enable_sie();
 501         return -EINVAL;
 502 }
 503
 504 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 505 {
 506         int r;
 507
 508         switch (ext) {
 509         case KVM_CAP_S390_PSW:
 510         case KVM_CAP_S390_GMAP:
 511         case KVM_CAP_SYNC_MMU:
 512 #ifdef CONFIG_KVM_S390_UCONTROL
 513         case KVM_CAP_S390_UCONTROL:
 514 #endif
 515         case KVM_CAP_ASYNC_PF:
 516         case KVM_CAP_SYNC_REGS:
 517         case KVM_CAP_ONE_REG:
 518         case KVM_CAP_ENABLE_CAP:
 519         case KVM_CAP_S390_CSS_SUPPORT:
 520         case KVM_CAP_IOEVENTFD:
 521         case KVM_CAP_DEVICE_CTRL:
 522         case KVM_CAP_S390_IRQCHIP:
 523         case KVM_CAP_VM_ATTRIBUTES:
 524         case KVM_CAP_MP_STATE:
 525         case KVM_CAP_IMMEDIATE_EXIT:
 526         case KVM_CAP_S390_INJECT_IRQ:
 527         case KVM_CAP_S390_USER_SIGP:
 528         case KVM_CAP_S390_USER_STSI:
 529         case KVM_CAP_S390_SKEYS:
 530         case KVM_CAP_S390_IRQ_STATE:
 531         case KVM_CAP_S390_USER_INSTR0:
 532         case KVM_CAP_S390_CMMA_MIGRATION:
 533         case KVM_CAP_S390_AIS:
 534         case KVM_CAP_S390_AIS_MIGRATION:
 535                 r = 1;
 536                 break;
 537         case KVM_CAP_S390_HPAGE_1M:
 538                 r = 0;
 539                 if (hpage && !kvm_is_ucontrol(kvm))
 540                         r = 1;
 541                 break;
 542         case KVM_CAP_S390_MEM_OP:
 543                 r = MEM_OP_MAX_SIZE;
 544                 break;
 545         case KVM_CAP_NR_VCPUS:
 546         case KVM_CAP_MAX_VCPUS:
 547         case KVM_CAP_MAX_VCPU_ID:
 548                 r = KVM_S390_BSCA_CPU_SLOTS;
 549                 if (!kvm_s390_use_sca_entries())
 550                         r = KVM_MAX_VCPUS;
 551                 else if (sclp.has_esca && sclp.has_64bscao)
 552                         r = KVM_S390_ESCA_CPU_SLOTS;
 553                 break;
 554         case KVM_CAP_S390_COW:
 555                 r = MACHINE_HAS_ESOP;
 556                 break;
 557         case KVM_CAP_S390_VECTOR_REGISTERS:
 558                 r = MACHINE_HAS_VX;
 559                 break;
 560         case KVM_CAP_S390_RI:
 561                 r = test_facility(64);
 562                 break;
 563         case KVM_CAP_S390_GS:
 564                 r = test_facility(133);
 565                 break;
 566         case KVM_CAP_S390_BPB:
 567                 r = test_facility(82);
 568                 break;
 569         default:
 570                 r = 0;
 571         }
 572         return r;
 573 }
 574
 575 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
 576                                     struct kvm_memory_slot *memslot)
 577 {
 578         int i;
 579         gfn_t cur_gfn, last_gfn;
 580         unsigned long gaddr, vmaddr;
 581         struct gmap *gmap = kvm->arch.gmap;
 582         DECLARE_BITMAP(bitmap, _PAGE_ENTRIES);
 583
 584         /* Loop over all guest segments */
 585         cur_gfn = memslot->base_gfn;
 586         last_gfn = memslot->base_gfn + memslot->npages;
 587         for (; cur_gfn <= last_gfn; cur_gfn += _PAGE_ENTRIES) {
 588                 gaddr = gfn_to_gpa(cur_gfn);
 589                 vmaddr = gfn_to_hva_memslot(memslot, cur_gfn);
 590                 if (kvm_is_error_hva(vmaddr))
 591                         continue;
 592
 593                 bitmap_zero(bitmap, _PAGE_ENTRIES);
 594                 gmap_sync_dirty_log_pmd(gmap, bitmap, gaddr, vmaddr);
 595                 for (i = 0; i < _PAGE_ENTRIES; i++) {
 596                         if (test_bit(i, bitmap))
 597                                 mark_page_dirty(kvm, cur_gfn + i);
 598                 }
 599
 600                 if (fatal_signal_pending(current))
 601                         return;
 602                 cond_resched();
 603         }
 604 }
 605
 606 /* Section: vm related */
 607 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
 608
 609 /*
 610  * Get (and clear) the dirty memory log for a memory slot.
 611  */
 612 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 613                                struct kvm_dirty_log *log)
 614 {
 615         int r;
 616         unsigned long n;
 617         struct kvm_memslots *slots;
 618         struct kvm_memory_slot *memslot;
 619         int is_dirty = 0;
 620
 621         if (kvm_is_ucontrol(kvm))
 622                 return -EINVAL;
 623
 624         mutex_lock(&kvm->slots_lock);
 625
 626         r = -EINVAL;
 627         if (log->slot >= KVM_USER_MEM_SLOTS)
 628                 goto out;
 629
 630         slots = kvm_memslots(kvm);
 631         memslot = id_to_memslot(slots, log->slot);
 632         r = -ENOENT;
 633         if (!memslot->dirty_bitmap)
 634                 goto out;
 635
 636         kvm_s390_sync_dirty_log(kvm, memslot);
 637         r = kvm_get_dirty_log(kvm, log, &is_dirty);
 638         if (r)
 639                 goto out;
 640
 641         /* Clear the dirty log */
 642         if (is_dirty) {
 643                 n = kvm_dirty_bitmap_bytes(memslot);
 644                 memset(memslot->dirty_bitmap, 0, n);
 645         }
 646         r = 0;
 647 out:
 648         mutex_unlock(&kvm->slots_lock);
 649         return r;
 650 }
 651
 652 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
 653 {
 654         unsigned int i;
 655         struct kvm_vcpu *vcpu;
 656
 657         kvm_for_each_vcpu(i, vcpu, kvm) {
 658                 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
 659         }
 660 }
 661
 662 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 663 {
 664         int r;
 665
 666         if (cap->flags)
 667                 return -EINVAL;
 668
 669         switch (cap->cap) {
 670         case KVM_CAP_S390_IRQCHIP:
 671                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
 672                 kvm->arch.use_irqchip = 1;
 673                 r = 0;
 674                 break;
 675         case KVM_CAP_S390_USER_SIGP:
 676                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
 677                 kvm->arch.user_sigp = 1;
 678                 r = 0;
 679                 break;
 680         case KVM_CAP_S390_VECTOR_REGISTERS:
 681                 mutex_lock(&kvm->lock);
 682                 if (kvm->created_vcpus) {
 683                         r = -EBUSY;
 684                 } else if (MACHINE_HAS_VX) {
 685                         set_kvm_facility(kvm->arch.model.fac_mask, 129);
 686                         set_kvm_facility(kvm->arch.model.fac_list, 129);
 687                         if (test_facility(134)) {
 688                                 set_kvm_facility(kvm->arch.model.fac_mask, 134);
 689                                 set_kvm_facility(kvm->arch.model.fac_list, 134);
 690                         }
 691                         if (test_facility(135)) {
 692                                 set_kvm_facility(kvm->arch.model.fac_mask, 135);
 693                                 set_kvm_facility(kvm->arch.model.fac_list, 135);
 694                         }
 695                         if (test_facility(148)) {
 696                                 set_kvm_facility(kvm->arch.model.fac_mask, 148);
 697                                 set_kvm_facility(kvm->arch.model.fac_list, 148);
 698                         }
 699                         if (test_facility(152)) {
 700                                 set_kvm_facility(kvm->arch.model.fac_mask, 152);
 701                                 set_kvm_facility(kvm->arch.model.fac_list, 152);
 702                         }
 703                         r = 0;
 704                 } else
 705                         r = -EINVAL;
 706                 mutex_unlock(&kvm->lock);
 707                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
 708                          r ? "(not available)" : "(success)");
 709                 break;
 710         case KVM_CAP_S390_RI:
 711                 r = -EINVAL;
 712                 mutex_lock(&kvm->lock);
 713                 if (kvm->created_vcpus) {
 714                         r = -EBUSY;
 715                 } else if (test_facility(64)) {
 716                         set_kvm_facility(kvm->arch.model.fac_mask, 64);
 717                         set_kvm_facility(kvm->arch.model.fac_list, 64);
 718                         r = 0;
 719                 }
 720                 mutex_unlock(&kvm->lock);
 721                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
 722                          r ? "(not available)" : "(success)");
 723                 break;
 724         case KVM_CAP_S390_AIS:
 725                 mutex_lock(&kvm->lock);
 726                 if (kvm->created_vcpus) {
 727                         r = -EBUSY;
 728                 } else {
 729                         set_kvm_facility(kvm->arch.model.fac_mask, 72);
 730                         set_kvm_facility(kvm->arch.model.fac_list, 72);
 731                         r = 0;
 732                 }
 733                 mutex_unlock(&kvm->lock);
 734                 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
 735                          r ? "(not available)" : "(success)");
 736                 break;
 737         case KVM_CAP_S390_GS:
 738                 r = -EINVAL;
 739                 mutex_lock(&kvm->lock);
 740                 if (kvm->created_vcpus) {
 741                         r = -EBUSY;
 742                 } else if (test_facility(133)) {
 743                         set_kvm_facility(kvm->arch.model.fac_mask, 133);
 744                         set_kvm_facility(kvm->arch.model.fac_list, 133);
 745                         r = 0;
 746                 }
 747                 mutex_unlock(&kvm->lock);
 748                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
 749                          r ? "(not available)" : "(success)");
 750                 break;
 751         case KVM_CAP_S390_HPAGE_1M:
 752                 mutex_lock(&kvm->lock);
 753                 if (kvm->created_vcpus)
 754                         r = -EBUSY;
 755                 else if (!hpage || kvm->arch.use_cmma || kvm_is_ucontrol(kvm))
 756                         r = -EINVAL;
 757                 else {
 758                         r = 0;
 759                         down_write(&kvm->mm->mmap_sem);
 760                         kvm->mm->context.allow_gmap_hpage_1m = 1;
 761                         up_write(&kvm->mm->mmap_sem);
 762                         /*
 763                          * We might have to create fake 4k page
 764                          * tables. To avoid that the hardware works on
 765                          * stale PGSTEs, we emulate these instructions.
 766                          */
 767                         kvm->arch.use_skf = 0;
 768                         kvm->arch.use_pfmfi = 0;
 769                 }
 770                 mutex_unlock(&kvm->lock);
 771                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s",
 772                          r ? "(not available)" : "(success)");
 773                 break;
 774         case KVM_CAP_S390_USER_STSI:
 775                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
 776                 kvm->arch.user_stsi = 1;
 777                 r = 0;
 778                 break;
 779         case KVM_CAP_S390_USER_INSTR0:
 780                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
 781                 kvm->arch.user_instr0 = 1;
 782                 icpt_operexc_on_all_vcpus(kvm);
 783                 r = 0;
 784                 break;
 785         default:
 786                 r = -EINVAL;
 787                 break;
 788         }
 789         return r;
 790 }
 791
 792 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 793 {
 794         int ret;
 795
 796         switch (attr->attr) {
 797         case KVM_S390_VM_MEM_LIMIT_SIZE:
 798                 ret = 0;
 799                 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
 800                          kvm->arch.mem_limit);
 801                 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
 802                         ret = -EFAULT;
 803                 break;
 804         default:
 805                 ret = -ENXIO;
 806                 break;
 807         }
 808         return ret;
 809 }
 810
 811 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 812 {
 813         int ret;
 814         unsigned int idx;
 815         switch (attr->attr) {
 816         case KVM_S390_VM_MEM_ENABLE_CMMA:
 817                 ret = -ENXIO;
 818                 if (!sclp.has_cmma)
 819                         break;
 820
 821                 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
 822                 mutex_lock(&kvm->lock);
 823                 if (kvm->created_vcpus)
 824                         ret = -EBUSY;
 825                 else if (kvm->mm->context.allow_gmap_hpage_1m)
 826                         ret = -EINVAL;
 827                 else {
 828                         kvm->arch.use_cmma = 1;
 829                         /* Not compatible with cmma. */
 830                         kvm->arch.use_pfmfi = 0;
 831                         ret = 0;
 832                 }
 833                 mutex_unlock(&kvm->lock);
 834                 break;
 835         case KVM_S390_VM_MEM_CLR_CMMA:
 836                 ret = -ENXIO;
 837                 if (!sclp.has_cmma)
 838                         break;
 839                 ret = -EINVAL;
 840                 if (!kvm->arch.use_cmma)
 841                         break;
 842
 843                 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
 844                 mutex_lock(&kvm->lock);
 845                 idx = srcu_read_lock(&kvm->srcu);
 846                 s390_reset_cmma(kvm->arch.gmap->mm);
 847                 srcu_read_unlock(&kvm->srcu, idx);
 848                 mutex_unlock(&kvm->lock);
 849                 ret = 0;
 850                 break;
 851         case KVM_S390_VM_MEM_LIMIT_SIZE: {
 852                 unsigned long new_limit;
 853
 854                 if (kvm_is_ucontrol(kvm))
 855                         return -EINVAL;
 856
 857                 if (get_user(new_limit, (u64 __user *)attr->addr))
 858                         return -EFAULT;
 859
 860                 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
 861                     new_limit > kvm->arch.mem_limit)
 862                         return -E2BIG;
 863
 864                 if (!new_limit)
 865                         return -EINVAL;
 866
 867                 /* gmap_create takes last usable address */
 868                 if (new_limit != KVM_S390_NO_MEM_LIMIT)
 869                         new_limit -= 1;
 870
 871                 ret = -EBUSY;
 872                 mutex_lock(&kvm->lock);
 873                 if (!kvm->created_vcpus) {
 874                         /* gmap_create will round the limit up */
 875                         struct gmap *new = gmap_create(current->mm, new_limit);
 876
 877                         if (!new) {
 878                                 ret = -ENOMEM;
 879                         } else {
 880                                 gmap_remove(kvm->arch.gmap);
 881                                 new->private = kvm;
 882                                 kvm->arch.gmap = new;
 883                                 ret = 0;
 884                         }
 885                 }
 886                 mutex_unlock(&kvm->lock);
 887                 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
 888                 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
 889                          (void *) kvm->arch.gmap->asce);
 890                 break;
 891         }
 892         default:
 893                 ret = -ENXIO;
 894                 break;
 895         }
 896         return ret;
 897 }
 898
 899 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
 900
 901 void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
 902 {
 903         struct kvm_vcpu *vcpu;
 904         int i;
 905
 906         kvm_s390_vcpu_block_all(kvm);
 907
 908         kvm_for_each_vcpu(i, vcpu, kvm) {
 909                 kvm_s390_vcpu_crypto_setup(vcpu);
 910                 /* recreate the shadow crycb by leaving the VSIE handler */
 911                 kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
 912         }
 913
 914         kvm_s390_vcpu_unblock_all(kvm);
 915 }
 916
 917 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
 918 {
 919         mutex_lock(&kvm->lock);
 920         switch (attr->attr) {
 921         case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
 922                 if (!test_kvm_facility(kvm, 76)) {
 923                         mutex_unlock(&kvm->lock);
 924                         return -EINVAL;
 925                 }
 926                 get_random_bytes(
 927                         kvm->arch.crypto.crycb->aes_wrapping_key_mask,
 928                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 929                 kvm->arch.crypto.aes_kw = 1;
 930                 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
 931                 break;
 932         case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
 933                 if (!test_kvm_facility(kvm, 76)) {
 934                         mutex_unlock(&kvm->lock);
 935                         return -EINVAL;
 936                 }
 937                 get_random_bytes(
 938                         kvm->arch.crypto.crycb->dea_wrapping_key_mask,
 939                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 940                 kvm->arch.crypto.dea_kw = 1;
 941                 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
 942                 break;
 943         case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
 944                 if (!test_kvm_facility(kvm, 76)) {
 945                         mutex_unlock(&kvm->lock);
 946                         return -EINVAL;
 947                 }
 948                 kvm->arch.crypto.aes_kw = 0;
 949                 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
 950                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 951                 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
 952                 break;
 953         case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
 954                 if (!test_kvm_facility(kvm, 76)) {
 955                         mutex_unlock(&kvm->lock);
 956                         return -EINVAL;
 957                 }
 958                 kvm->arch.crypto.dea_kw = 0;
 959                 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
 960                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 961                 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
 962                 break;
 963         case KVM_S390_VM_CRYPTO_ENABLE_APIE:
 964                 if (!ap_instructions_available()) {
 965                         mutex_unlock(&kvm->lock);
 966                         return -EOPNOTSUPP;
 967                 }
 968                 kvm->arch.crypto.apie = 1;
 969                 break;
 970         case KVM_S390_VM_CRYPTO_DISABLE_APIE:
 971                 if (!ap_instructions_available()) {
 972                         mutex_unlock(&kvm->lock);
 973                         return -EOPNOTSUPP;
 974                 }
 975                 kvm->arch.crypto.apie = 0;
 976                 break;
 977         default:
 978                 mutex_unlock(&kvm->lock);
 979                 return -ENXIO;
 980         }
 981
 982         kvm_s390_vcpu_crypto_reset_all(kvm);
 983         mutex_unlock(&kvm->lock);
 984         return 0;
 985 }
 986
 987 static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
 988 {
 989         int cx;
 990         struct kvm_vcpu *vcpu;
 991
 992         kvm_for_each_vcpu(cx, vcpu, kvm)
 993                 kvm_s390_sync_request(req, vcpu);
 994 }
 995
 996 /*
 997  * Must be called with kvm->srcu held to avoid races on memslots, and with
 998  * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
 999  */
1000 static int kvm_s390_vm_start_migration(struct kvm *kvm)
1001 {
1002         struct kvm_memory_slot *ms;
1003         struct kvm_memslots *slots;
1004         unsigned long ram_pages = 0;
1005         int slotnr;
1006
1007         /* migration mode already enabled */
1008         if (kvm->arch.migration_mode)
1009                 return 0;
1010         slots = kvm_memslots(kvm);
1011         if (!slots || !slots->used_slots)
1012                 return -EINVAL;
1013
1014         if (!kvm->arch.use_cmma) {
1015                 kvm->arch.migration_mode = 1;
1016                 return 0;
1017         }
1018         /* mark all the pages in active slots as dirty */
1019         for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
1020                 ms = slots->memslots + slotnr;
1021                 if (!ms->dirty_bitmap)
1022                         return -EINVAL;
1023                 /*
1024                  * The second half of the bitmap is only used on x86,
1025                  * and would be wasted otherwise, so we put it to good
1026                  * use here to keep track of the state of the storage
1027                  * attributes.
1028                  */
1029                 memset(kvm_second_dirty_bitmap(ms), 0xff, kvm_dirty_bitmap_bytes(ms));
1030                 ram_pages += ms->npages;
1031         }
1032         atomic64_set(&kvm->arch.cmma_dirty_pages, ram_pages);
1033         kvm->arch.migration_mode = 1;
1034         kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
1035         return 0;
1036 }
1037
1038 /*
1039  * Must be called with kvm->slots_lock to avoid races with ourselves and
1040  * kvm_s390_vm_start_migration.
1041  */
1042 static int kvm_s390_vm_stop_migration(struct kvm *kvm)
1043 {
1044         /* migration mode already disabled */
1045         if (!kvm->arch.migration_mode)
1046                 return 0;
1047         kvm->arch.migration_mode = 0;
1048         if (kvm->arch.use_cmma)
1049                 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
1050         return 0;
1051 }
1052
1053 static int kvm_s390_vm_set_migration(struct kvm *kvm,
1054                                      struct kvm_device_attr *attr)
1055 {
1056         int res = -ENXIO;
1057
1058         mutex_lock(&kvm->slots_lock);
1059         switch (attr->attr) {
1060         case KVM_S390_VM_MIGRATION_START:
1061                 res = kvm_s390_vm_start_migration(kvm);
1062                 break;
1063         case KVM_S390_VM_MIGRATION_STOP:
1064                 res = kvm_s390_vm_stop_migration(kvm);
1065                 break;
1066         default:
1067                 break;
1068         }
1069         mutex_unlock(&kvm->slots_lock);
1070
1071         return res;
1072 }
1073
1074 static int kvm_s390_vm_get_migration(struct kvm *kvm,
1075                                      struct kvm_device_attr *attr)
1076 {
1077         u64 mig = kvm->arch.migration_mode;
1078
1079         if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
1080                 return -ENXIO;
1081
1082         if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
1083                 return -EFAULT;
1084         return 0;
1085 }
1086
1087 static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1088 {
1089         struct kvm_s390_vm_tod_clock gtod;
1090
1091         if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
1092                 return -EFAULT;
1093
1094         if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
1095                 return -EINVAL;
1096         kvm_s390_set_tod_clock(kvm, &gtod);
1097
1098         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
1099                 gtod.epoch_idx, gtod.tod);
1100
1101         return 0;
1102 }
1103
1104 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1105 {
1106         u8 gtod_high;
1107
1108         if (copy_from_user(&gtod_high, (void __user *)attr->addr,
1109                                            sizeof(gtod_high)))
1110                 return -EFAULT;
1111
1112         if (gtod_high != 0)
1113                 return -EINVAL;
1114         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
1115
1116         return 0;
1117 }
1118
1119 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1120 {
1121         struct kvm_s390_vm_tod_clock gtod = { 0 };
1122
1123         if (copy_from_user(&gtod.tod, (void __user *)attr->addr,
1124                            sizeof(gtod.tod)))
1125                 return -EFAULT;
1126
1127         kvm_s390_set_tod_clock(kvm, &gtod);
1128         VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
1129         return 0;
1130 }
1131
1132 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1133 {
1134         int ret;
1135
1136         if (attr->flags)
1137                 return -EINVAL;
1138
1139         switch (attr->attr) {
1140         case KVM_S390_VM_TOD_EXT:
1141                 ret = kvm_s390_set_tod_ext(kvm, attr);
1142                 break;
1143         case KVM_S390_VM_TOD_HIGH:
1144                 ret = kvm_s390_set_tod_high(kvm, attr);
1145                 break;
1146         case KVM_S390_VM_TOD_LOW:
1147                 ret = kvm_s390_set_tod_low(kvm, attr);
1148                 break;
1149         default:
1150                 ret = -ENXIO;
1151                 break;
1152         }
1153         return ret;
1154 }
1155
1156 static void kvm_s390_get_tod_clock(struct kvm *kvm,
1157                                    struct kvm_s390_vm_tod_clock *gtod)
1158 {
1159         struct kvm_s390_tod_clock_ext htod;
1160
1161         preempt_disable();
1162
1163         get_tod_clock_ext((char *)&htod);
1164
1165         gtod->tod = htod.tod + kvm->arch.epoch;
1166         gtod->epoch_idx = 0;
1167         if (test_kvm_facility(kvm, 139)) {
1168                 gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
1169                 if (gtod->tod < htod.tod)
1170                         gtod->epoch_idx += 1;
1171         }
1172
1173         preempt_enable();
1174 }
1175
1176 static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1177 {
1178         struct kvm_s390_vm_tod_clock gtod;
1179
1180         memset(&gtod, 0, sizeof(gtod));
1181         kvm_s390_get_tod_clock(kvm, &gtod);
1182         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1183                 return -EFAULT;
1184
1185         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
1186                 gtod.epoch_idx, gtod.tod);
1187         return 0;
1188 }
1189
1190 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1191 {
1192         u8 gtod_high = 0;
1193
1194         if (copy_to_user((void __user *)attr->addr, &gtod_high,
1195                                          sizeof(gtod_high)))
1196                 return -EFAULT;
1197         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
1198
1199         return 0;
1200 }
1201
1202 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1203 {
1204         u64 gtod;
1205
1206         gtod = kvm_s390_get_tod_clock_fast(kvm);
1207         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1208                 return -EFAULT;
1209         VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
1210
1211         return 0;
1212 }
1213
1214 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1215 {
1216         int ret;
1217
1218         if (attr->flags)
1219                 return -EINVAL;
1220
1221         switch (attr->attr) {
1222         case KVM_S390_VM_TOD_EXT:
1223                 ret = kvm_s390_get_tod_ext(kvm, attr);
1224                 break;
1225         case KVM_S390_VM_TOD_HIGH:
1226                 ret = kvm_s390_get_tod_high(kvm, attr);
1227                 break;
1228         case KVM_S390_VM_TOD_LOW:
1229                 ret = kvm_s390_get_tod_low(kvm, attr);
1230                 break;
1231         default:
1232                 ret = -ENXIO;
1233                 break;
1234         }
1235         return ret;
1236 }
1237
1238 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1239 {
1240         struct kvm_s390_vm_cpu_processor *proc;
1241         u16 lowest_ibc, unblocked_ibc;
1242         int ret = 0;
1243
1244         mutex_lock(&kvm->lock);
1245         if (kvm->created_vcpus) {
1246                 ret = -EBUSY;
1247                 goto out;
1248         }
1249         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1250         if (!proc) {
1251                 ret = -ENOMEM;
1252                 goto out;
1253         }
1254         if (!copy_from_user(proc, (void __user *)attr->addr,
1255                             sizeof(*proc))) {
1256                 kvm->arch.model.cpuid = proc->cpuid;
1257                 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1258                 unblocked_ibc = sclp.ibc & 0xfff;
1259                 if (lowest_ibc && proc->ibc) {
1260                         if (proc->ibc > unblocked_ibc)
1261                                 kvm->arch.model.ibc = unblocked_ibc;
1262                         else if (proc->ibc < lowest_ibc)
1263                                 kvm->arch.model.ibc = lowest_ibc;
1264                         else
1265                                 kvm->arch.model.ibc = proc->ibc;
1266                 }
1267                 memcpy(kvm->arch.model.fac_list, proc->fac_list,
1268                        S390_ARCH_FAC_LIST_SIZE_BYTE);
1269                 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1270                          kvm->arch.model.ibc,
1271                          kvm->arch.model.cpuid);
1272                 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1273                          kvm->arch.model.fac_list[0],
1274                          kvm->arch.model.fac_list[1],
1275                          kvm->arch.model.fac_list[2]);
1276         } else
1277                 ret = -EFAULT;
1278         kfree(proc);
1279 out:
1280         mutex_unlock(&kvm->lock);
1281         return ret;
1282 }
1283
1284 static int kvm_s390_set_processor_feat(struct kvm *kvm,
1285                                        struct kvm_device_attr *attr)
1286 {
1287         struct kvm_s390_vm_cpu_feat data;
1288
1289         if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1290                 return -EFAULT;
1291         if (!bitmap_subset((unsigned long *) data.feat,
1292                            kvm_s390_available_cpu_feat,
1293                            KVM_S390_VM_CPU_FEAT_NR_BITS))
1294                 return -EINVAL;
1295
1296         mutex_lock(&kvm->lock);
1297         if (kvm->created_vcpus) {
1298                 mutex_unlock(&kvm->lock);
1299                 return -EBUSY;
1300         }
1301         bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
1302                     KVM_S390_VM_CPU_FEAT_NR_BITS);
1303         mutex_unlock(&kvm->lock);
1304         VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1305                          data.feat[0],
1306                          data.feat[1],
1307                          data.feat[2]);
1308         return 0;
1309 }
1310
1311 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1312                                           struct kvm_device_attr *attr)
1313 {
1314         mutex_lock(&kvm->lock);
1315         if (kvm->created_vcpus) {
1316                 mutex_unlock(&kvm->lock);
1317                 return -EBUSY;
1318         }
1319
1320         if (copy_from_user(&kvm->arch.model.subfuncs, (void __user *)attr->addr,
1321                            sizeof(struct kvm_s390_vm_cpu_subfunc))) {
1322                 mutex_unlock(&kvm->lock);
1323                 return -EFAULT;
1324         }
1325         mutex_unlock(&kvm->lock);
1326
1327         VM_EVENT(kvm, 3, "SET: guest PLO    subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1328                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1329                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1330                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1331                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1332         VM_EVENT(kvm, 3, "SET: guest PTFF   subfunc 0x%16.16lx.%16.16lx",
1333                  ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1334                  ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1335         VM_EVENT(kvm, 3, "SET: guest KMAC   subfunc 0x%16.16lx.%16.16lx",
1336                  ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1337                  ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1338         VM_EVENT(kvm, 3, "SET: guest KMC    subfunc 0x%16.16lx.%16.16lx",
1339                  ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1340                  ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1341         VM_EVENT(kvm, 3, "SET: guest KM     subfunc 0x%16.16lx.%16.16lx",
1342                  ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1343                  ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1344         VM_EVENT(kvm, 3, "SET: guest KIMD   subfunc 0x%16.16lx.%16.16lx",
1345                  ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1346                  ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1347         VM_EVENT(kvm, 3, "SET: guest KLMD   subfunc 0x%16.16lx.%16.16lx",
1348                  ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1349                  ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1350         VM_EVENT(kvm, 3, "SET: guest PCKMO  subfunc 0x%16.16lx.%16.16lx",
1351                  ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1352                  ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1353         VM_EVENT(kvm, 3, "SET: guest KMCTR  subfunc 0x%16.16lx.%16.16lx",
1354                  ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1355                  ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1356         VM_EVENT(kvm, 3, "SET: guest KMF    subfunc 0x%16.16lx.%16.16lx",
1357                  ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1358                  ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1359         VM_EVENT(kvm, 3, "SET: guest KMO    subfunc 0x%16.16lx.%16.16lx",
1360                  ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1361                  ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1362         VM_EVENT(kvm, 3, "SET: guest PCC    subfunc 0x%16.16lx.%16.16lx",
1363                  ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1364                  ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1365         VM_EVENT(kvm, 3, "SET: guest PPNO   subfunc 0x%16.16lx.%16.16lx",
1366                  ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1367                  ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1368         VM_EVENT(kvm, 3, "SET: guest KMA    subfunc 0x%16.16lx.%16.16lx",
1369                  ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1370                  ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1371         VM_EVENT(kvm, 3, "SET: guest KDSA   subfunc 0x%16.16lx.%16.16lx",
1372                  ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1373                  ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1374         VM_EVENT(kvm, 3, "SET: guest SORTL  subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1375                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1376                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1377                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1378                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1379         VM_EVENT(kvm, 3, "SET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1380                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1381                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1382                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1383                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1384
1385         return 0;
1386 }
1387
1388 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1389 {
1390         int ret = -ENXIO;
1391
1392         switch (attr->attr) {
1393         case KVM_S390_VM_CPU_PROCESSOR:
1394                 ret = kvm_s390_set_processor(kvm, attr);
1395                 break;
1396         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1397                 ret = kvm_s390_set_processor_feat(kvm, attr);
1398                 break;
1399         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1400                 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1401                 break;
1402         }
1403         return ret;
1404 }
1405
1406 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1407 {
1408         struct kvm_s390_vm_cpu_processor *proc;
1409         int ret = 0;
1410
1411         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1412         if (!proc) {
1413                 ret = -ENOMEM;
1414                 goto out;
1415         }
1416         proc->cpuid = kvm->arch.model.cpuid;
1417         proc->ibc = kvm->arch.model.ibc;
1418         memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1419                S390_ARCH_FAC_LIST_SIZE_BYTE);
1420         VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1421                  kvm->arch.model.ibc,
1422                  kvm->arch.model.cpuid);
1423         VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1424                  kvm->arch.model.fac_list[0],
1425                  kvm->arch.model.fac_list[1],
1426                  kvm->arch.model.fac_list[2]);
1427         if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1428                 ret = -EFAULT;
1429         kfree(proc);
1430 out:
1431         return ret;
1432 }
1433
1434 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1435 {
1436         struct kvm_s390_vm_cpu_machine *mach;
1437         int ret = 0;
1438
1439         mach = kzalloc(sizeof(*mach), GFP_KERNEL);
1440         if (!mach) {
1441                 ret = -ENOMEM;
1442                 goto out;
1443         }
1444         get_cpu_id((struct cpuid *) &mach->cpuid);
1445         mach->ibc = sclp.ibc;
1446         memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
1447                S390_ARCH_FAC_LIST_SIZE_BYTE);
1448         memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
1449                sizeof(S390_lowcore.stfle_fac_list));
1450         VM_EVENT(kvm, 3, "GET: host ibc:  0x%4.4x, host cpuid:  0x%16.16llx",
1451                  kvm->arch.model.ibc,
1452                  kvm->arch.model.cpuid);
1453         VM_EVENT(kvm, 3, "GET: host facmask:  0x%16.16llx.%16.16llx.%16.16llx",
1454                  mach->fac_mask[0],
1455                  mach->fac_mask[1],
1456                  mach->fac_mask[2]);
1457         VM_EVENT(kvm, 3, "GET: host faclist:  0x%16.16llx.%16.16llx.%16.16llx",
1458                  mach->fac_list[0],
1459                  mach->fac_list[1],
1460                  mach->fac_list[2]);
1461         if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1462                 ret = -EFAULT;
1463         kfree(mach);
1464 out:
1465         return ret;
1466 }
1467
1468 static int kvm_s390_get_processor_feat(struct kvm *kvm,
1469                                        struct kvm_device_attr *attr)
1470 {
1471         struct kvm_s390_vm_cpu_feat data;
1472
1473         bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1474                     KVM_S390_VM_CPU_FEAT_NR_BITS);
1475         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1476                 return -EFAULT;
1477         VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1478                          data.feat[0],
1479                          data.feat[1],
1480                          data.feat[2]);
1481         return 0;
1482 }
1483
1484 static int kvm_s390_get_machine_feat(struct kvm *kvm,
1485                                      struct kvm_device_attr *attr)
1486 {
1487         struct kvm_s390_vm_cpu_feat data;
1488
1489         bitmap_copy((unsigned long *) data.feat,
1490                     kvm_s390_available_cpu_feat,
1491                     KVM_S390_VM_CPU_FEAT_NR_BITS);
1492         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1493                 return -EFAULT;
1494         VM_EVENT(kvm, 3, "GET: host feat:  0x%16.16llx.0x%16.16llx.0x%16.16llx",
1495                          data.feat[0],
1496                          data.feat[1],
1497                          data.feat[2]);
1498         return 0;
1499 }
1500
1501 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1502                                           struct kvm_device_attr *attr)
1503 {
1504         if (copy_to_user((void __user *)attr->addr, &kvm->arch.model.subfuncs,
1505             sizeof(struct kvm_s390_vm_cpu_subfunc)))
1506                 return -EFAULT;
1507
1508         VM_EVENT(kvm, 3, "GET: guest PLO    subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1509                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1510                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1511                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1512                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1513         VM_EVENT(kvm, 3, "GET: guest PTFF   subfunc 0x%16.16lx.%16.16lx",
1514                  ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1515                  ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1516         VM_EVENT(kvm, 3, "GET: guest KMAC   subfunc 0x%16.16lx.%16.16lx",
1517                  ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1518                  ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1519         VM_EVENT(kvm, 3, "GET: guest KMC    subfunc 0x%16.16lx.%16.16lx",
1520                  ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1521                  ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1522         VM_EVENT(kvm, 3, "GET: guest KM     subfunc 0x%16.16lx.%16.16lx",
1523                  ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1524                  ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1525         VM_EVENT(kvm, 3, "GET: guest KIMD   subfunc 0x%16.16lx.%16.16lx",
1526                  ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1527                  ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1528         VM_EVENT(kvm, 3, "GET: guest KLMD   subfunc 0x%16.16lx.%16.16lx",
1529                  ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1530                  ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1531         VM_EVENT(kvm, 3, "GET: guest PCKMO  subfunc 0x%16.16lx.%16.16lx",
1532                  ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1533                  ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1534         VM_EVENT(kvm, 3, "GET: guest KMCTR  subfunc 0x%16.16lx.%16.16lx",
1535                  ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1536                  ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1537         VM_EVENT(kvm, 3, "GET: guest KMF    subfunc 0x%16.16lx.%16.16lx",
1538                  ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1539                  ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1540         VM_EVENT(kvm, 3, "GET: guest KMO    subfunc 0x%16.16lx.%16.16lx",
1541                  ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1542                  ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1543         VM_EVENT(kvm, 3, "GET: guest PCC    subfunc 0x%16.16lx.%16.16lx",
1544                  ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1545                  ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1546         VM_EVENT(kvm, 3, "GET: guest PPNO   subfunc 0x%16.16lx.%16.16lx",
1547                  ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1548                  ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1549         VM_EVENT(kvm, 3, "GET: guest KMA    subfunc 0x%16.16lx.%16.16lx",
1550                  ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1551                  ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1552         VM_EVENT(kvm, 3, "GET: guest KDSA   subfunc 0x%16.16lx.%16.16lx",
1553                  ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1554                  ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1555         VM_EVENT(kvm, 3, "GET: guest SORTL  subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1556                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1557                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1558                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1559                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1560         VM_EVENT(kvm, 3, "GET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1561                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1562                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1563                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1564                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1565
1566         return 0;
1567 }
1568
1569 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1570                                         struct kvm_device_attr *attr)
1571 {
1572         if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1573             sizeof(struct kvm_s390_vm_cpu_subfunc)))
1574                 return -EFAULT;
1575
1576         VM_EVENT(kvm, 3, "GET: host  PLO    subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1577                  ((unsigned long *) &kvm_s390_available_subfunc.plo)[0],
1578                  ((unsigned long *) &kvm_s390_available_subfunc.plo)[1],
1579                  ((unsigned long *) &kvm_s390_available_subfunc.plo)[2],
1580                  ((unsigned long *) &kvm_s390_available_subfunc.plo)[3]);
1581         VM_EVENT(kvm, 3, "GET: host  PTFF   subfunc 0x%16.16lx.%16.16lx",
1582                  ((unsigned long *) &kvm_s390_available_subfunc.ptff)[0],
1583                  ((unsigned long *) &kvm_s390_available_subfunc.ptff)[1]);
1584         VM_EVENT(kvm, 3, "GET: host  KMAC   subfunc 0x%16.16lx.%16.16lx",
1585                  ((unsigned long *) &kvm_s390_available_subfunc.kmac)[0],
1586                  ((unsigned long *) &kvm_s390_available_subfunc.kmac)[1]);
1587         VM_EVENT(kvm, 3, "GET: host  KMC    subfunc 0x%16.16lx.%16.16lx",
1588                  ((unsigned long *) &kvm_s390_available_subfunc.kmc)[0],
1589                  ((unsigned long *) &kvm_s390_available_subfunc.kmc)[1]);
1590         VM_EVENT(kvm, 3, "GET: host  KM     subfunc 0x%16.16lx.%16.16lx",
1591                  ((unsigned long *) &kvm_s390_available_subfunc.km)[0],
1592                  ((unsigned long *) &kvm_s390_available_subfunc.km)[1]);
1593         VM_EVENT(kvm, 3, "GET: host  KIMD   subfunc 0x%16.16lx.%16.16lx",
1594                  ((unsigned long *) &kvm_s390_available_subfunc.kimd)[0],
1595                  ((unsigned long *) &kvm_s390_available_subfunc.kimd)[1]);
1596         VM_EVENT(kvm, 3, "GET: host  KLMD   subfunc 0x%16.16lx.%16.16lx",
1597                  ((unsigned long *) &kvm_s390_available_subfunc.klmd)[0],
1598                  ((unsigned long *) &kvm_s390_available_subfunc.klmd)[1]);
1599         VM_EVENT(kvm, 3, "GET: host  PCKMO  subfunc 0x%16.16lx.%16.16lx",
1600                  ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[0],
1601                  ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[1]);
1602         VM_EVENT(kvm, 3, "GET: host  KMCTR  subfunc 0x%16.16lx.%16.16lx",
1603                  ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[0],
1604                  ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[1]);
1605         VM_EVENT(kvm, 3, "GET: host  KMF    subfunc 0x%16.16lx.%16.16lx",
1606                  ((unsigned long *) &kvm_s390_available_subfunc.kmf)[0],
1607                  ((unsigned long *) &kvm_s390_available_subfunc.kmf)[1]);
1608         VM_EVENT(kvm, 3, "GET: host  KMO    subfunc 0x%16.16lx.%16.16lx",
1609                  ((unsigned long *) &kvm_s390_available_subfunc.kmo)[0],
1610                  ((unsigned long *) &kvm_s390_available_subfunc.kmo)[1]);
1611         VM_EVENT(kvm, 3, "GET: host  PCC    subfunc 0x%16.16lx.%16.16lx",
1612                  ((unsigned long *) &kvm_s390_available_subfunc.pcc)[0],
1613                  ((unsigned long *) &kvm_s390_available_subfunc.pcc)[1]);
1614         VM_EVENT(kvm, 3, "GET: host  PPNO   subfunc 0x%16.16lx.%16.16lx",
1615                  ((unsigned long *) &kvm_s390_available_subfunc.ppno)[0],
1616                  ((unsigned long *) &kvm_s390_available_subfunc.ppno)[1]);
1617         VM_EVENT(kvm, 3, "GET: host  KMA    subfunc 0x%16.16lx.%16.16lx",
1618                  ((unsigned long *) &kvm_s390_available_subfunc.kma)[0],
1619                  ((unsigned long *) &kvm_s390_available_subfunc.kma)[1]);
1620         VM_EVENT(kvm, 3, "GET: host  KDSA   subfunc 0x%16.16lx.%16.16lx",
1621                  ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[0],
1622                  ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[1]);
1623         VM_EVENT(kvm, 3, "GET: host  SORTL  subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1624                  ((unsigned long *) &kvm_s390_available_subfunc.sortl)[0],
1625                  ((unsigned long *) &kvm_s390_available_subfunc.sortl)[1],
1626                  ((unsigned long *) &kvm_s390_available_subfunc.sortl)[2],
1627                  ((unsigned long *) &kvm_s390_available_subfunc.sortl)[3]);
1628         VM_EVENT(kvm, 3, "GET: host  DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1629                  ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[0],
1630                  ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[1],
1631                  ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[2],
1632                  ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[3]);
1633
1634         return 0;
1635 }
1636
1637 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1638 {
1639         int ret = -ENXIO;
1640
1641         switch (attr->attr) {
1642         case KVM_S390_VM_CPU_PROCESSOR:
1643                 ret = kvm_s390_get_processor(kvm, attr);
1644                 break;
1645         case KVM_S390_VM_CPU_MACHINE:
1646                 ret = kvm_s390_get_machine(kvm, attr);
1647                 break;
1648         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1649                 ret = kvm_s390_get_processor_feat(kvm, attr);
1650                 break;
1651         case KVM_S390_VM_CPU_MACHINE_FEAT:
1652                 ret = kvm_s390_get_machine_feat(kvm, attr);
1653                 break;
1654         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1655                 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1656                 break;
1657         case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1658                 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1659                 break;
1660         }
1661         return ret;
1662 }
1663
1664 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1665 {
1666         int ret;
1667
1668         switch (attr->group) {
1669         case KVM_S390_VM_MEM_CTRL:
1670                 ret = kvm_s390_set_mem_control(kvm, attr);
1671                 break;
1672         case KVM_S390_VM_TOD:
1673                 ret = kvm_s390_set_tod(kvm, attr);
1674                 break;
1675         case KVM_S390_VM_CPU_MODEL:
1676                 ret = kvm_s390_set_cpu_model(kvm, attr);
1677                 break;
1678         case KVM_S390_VM_CRYPTO:
1679                 ret = kvm_s390_vm_set_crypto(kvm, attr);
1680                 break;
1681         case KVM_S390_VM_MIGRATION:
1682                 ret = kvm_s390_vm_set_migration(kvm, attr);
1683                 break;
1684         default:
1685                 ret = -ENXIO;
1686                 break;
1687         }
1688
1689         return ret;
1690 }
1691
1692 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1693 {
1694         int ret;
1695
1696         switch (attr->group) {
1697         case KVM_S390_VM_MEM_CTRL:
1698                 ret = kvm_s390_get_mem_control(kvm, attr);
1699                 break;
1700         case KVM_S390_VM_TOD:
1701                 ret = kvm_s390_get_tod(kvm, attr);
1702                 break;
1703         case KVM_S390_VM_CPU_MODEL:
1704                 ret = kvm_s390_get_cpu_model(kvm, attr);
1705                 break;
1706         case KVM_S390_VM_MIGRATION:
1707                 ret = kvm_s390_vm_get_migration(kvm, attr);
1708                 break;
1709         default:
1710                 ret = -ENXIO;
1711                 break;
1712         }
1713
1714         return ret;
1715 }
1716
1717 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1718 {
1719         int ret;
1720
1721         switch (attr->group) {
1722         case KVM_S390_VM_MEM_CTRL:
1723                 switch (attr->attr) {
1724                 case KVM_S390_VM_MEM_ENABLE_CMMA:
1725                 case KVM_S390_VM_MEM_CLR_CMMA:
1726                         ret = sclp.has_cmma ? 0 : -ENXIO;
1727                         break;
1728                 case KVM_S390_VM_MEM_LIMIT_SIZE:
1729                         ret = 0;
1730                         break;
1731                 default:
1732                         ret = -ENXIO;
1733                         break;
1734                 }
1735                 break;
1736         case KVM_S390_VM_TOD:
1737                 switch (attr->attr) {
1738                 case KVM_S390_VM_TOD_LOW:
1739                 case KVM_S390_VM_TOD_HIGH:
1740                         ret = 0;
1741                         break;
1742                 default:
1743                         ret = -ENXIO;
1744                         break;
1745                 }
1746                 break;
1747         case KVM_S390_VM_CPU_MODEL:
1748                 switch (attr->attr) {
1749                 case KVM_S390_VM_CPU_PROCESSOR:
1750                 case KVM_S390_VM_CPU_MACHINE:
1751                 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1752                 case KVM_S390_VM_CPU_MACHINE_FEAT:
1753                 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1754                 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1755                         ret = 0;
1756                         break;
1757                 default:
1758                         ret = -ENXIO;
1759                         break;
1760                 }
1761                 break;
1762         case KVM_S390_VM_CRYPTO:
1763                 switch (attr->attr) {
1764                 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1765                 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1766                 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1767                 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1768                         ret = 0;
1769                         break;
1770                 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
1771                 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
1772                         ret = ap_instructions_available() ? 0 : -ENXIO;
1773                         break;
1774                 default:
1775                         ret = -ENXIO;
1776                         break;
1777                 }
1778                 break;
1779         case KVM_S390_VM_MIGRATION:
1780                 ret = 0;
1781                 break;
1782         default:
1783                 ret = -ENXIO;
1784                 break;
1785         }
1786
1787         return ret;
1788 }
1789
1790 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1791 {
1792         uint8_t *keys;
1793         uint64_t hva;
1794         int srcu_idx, i, r = 0;
1795
1796         if (args->flags != 0)
1797                 return -EINVAL;
1798
1799         /* Is this guest using storage keys? */
1800         if (!mm_uses_skeys(current->mm))
1801                 return KVM_S390_GET_SKEYS_NONE;
1802
1803         /* Enforce sane limit on memory allocation */
1804         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1805                 return -EINVAL;
1806
1807         keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1808         if (!keys)
1809                 return -ENOMEM;
1810
1811         down_read(&current->mm->mmap_sem);
1812         srcu_idx = srcu_read_lock(&kvm->srcu);
1813         for (i = 0; i < args->count; i++) {
1814                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1815                 if (kvm_is_error_hva(hva)) {
1816                         r = -EFAULT;
1817                         break;
1818                 }
1819
1820                 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1821                 if (r)
1822                         break;
1823         }
1824         srcu_read_unlock(&kvm->srcu, srcu_idx);
1825         up_read(&current->mm->mmap_sem);
1826
1827         if (!r) {
1828                 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1829                                  sizeof(uint8_t) * args->count);
1830                 if (r)
1831                         r = -EFAULT;
1832         }
1833
1834         kvfree(keys);
1835         return r;
1836 }
1837
1838 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1839 {
1840         uint8_t *keys;
1841         uint64_t hva;
1842         int srcu_idx, i, r = 0;
1843         bool unlocked;
1844
1845         if (args->flags != 0)
1846                 return -EINVAL;
1847
1848         /* Enforce sane limit on memory allocation */
1849         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1850                 return -EINVAL;
1851
1852         keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1853         if (!keys)
1854                 return -ENOMEM;
1855
1856         r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1857                            sizeof(uint8_t) * args->count);
1858         if (r) {
1859                 r = -EFAULT;
1860                 goto out;
1861         }
1862
1863         /* Enable storage key handling for the guest */
1864         r = s390_enable_skey();
1865         if (r)
1866                 goto out;
1867
1868         i = 0;
1869         down_read(&current->mm->mmap_sem);
1870         srcu_idx = srcu_read_lock(&kvm->srcu);
1871         while (i < args->count) {
1872                 unlocked = false;
1873                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1874                 if (kvm_is_error_hva(hva)) {
1875                         r = -EFAULT;
1876                         break;
1877                 }
1878
1879                 /* Lowest order bit is reserved */
1880                 if (keys[i] & 0x01) {
1881                         r = -EINVAL;
1882                         break;
1883                 }
1884
1885                 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1886                 if (r) {
1887                         r = fixup_user_fault(current, current->mm, hva,
1888                                              FAULT_FLAG_WRITE, &unlocked);
1889                         if (r)
1890                                 break;
1891                 }
1892                 if (!r)
1893                         i++;
1894         }
1895         srcu_read_unlock(&kvm->srcu, srcu_idx);
1896         up_read(&current->mm->mmap_sem);
1897 out:
1898         kvfree(keys);
1899         return r;
1900 }
1901
1902 /*
1903  * Base address and length must be sent at the start of each block, therefore
1904  * it's cheaper to send some clean data, as long as it's less than the size of
1905  * two longs.
1906  */
1907 #define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
1908 /* for consistency */
1909 #define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1910
1911 /*
1912  * Similar to gfn_to_memslot, but returns the index of a memslot also when the
1913  * address falls in a hole. In that case the index of one of the memslots
1914  * bordering the hole is returned.
1915  */
1916 static int gfn_to_memslot_approx(struct kvm_memslots *slots, gfn_t gfn)
1917 {
1918         int start = 0, end = slots->used_slots;
1919         int slot = atomic_read(&slots->lru_slot);
1920         struct kvm_memory_slot *memslots = slots->memslots;
1921
1922         if (gfn >= memslots[slot].base_gfn &&
1923             gfn < memslots[slot].base_gfn + memslots[slot].npages)
1924                 return slot;
1925
1926         while (start < end) {
1927                 slot = start + (end - start) / 2;
1928
1929                 if (gfn >= memslots[slot].base_gfn)
1930                         end = slot;
1931                 else
1932                         start = slot + 1;
1933         }
1934
1935         if (start >= slots->used_slots)
1936                 return slots->used_slots - 1;
1937
1938         if (gfn >= memslots[start].base_gfn &&
1939             gfn < memslots[start].base_gfn + memslots[start].npages) {
1940                 atomic_set(&slots->lru_slot, start);
1941         }
1942
1943         return start;
1944 }
1945
1946 static int kvm_s390_peek_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
1947                               u8 *res, unsigned long bufsize)
1948 {
1949         unsigned long pgstev, hva, cur_gfn = args->start_gfn;
1950
1951         args->count = 0;
1952         while (args->count < bufsize) {
1953                 hva = gfn_to_hva(kvm, cur_gfn);
1954                 /*
1955                  * We return an error if the first value was invalid, but we
1956                  * return successfully if at least one value was copied.
1957                  */
1958                 if (kvm_is_error_hva(hva))
1959                         return args->count ? 0 : -EFAULT;
1960                 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
1961                         pgstev = 0;
1962                 res[args->count++] = (pgstev >> 24) & 0x43;
1963                 cur_gfn++;
1964         }
1965
1966         return 0;
1967 }
1968
1969 static unsigned long kvm_s390_next_dirty_cmma(struct kvm_memslots *slots,
1970                                               unsigned long cur_gfn)
1971 {
1972         int slotidx = gfn_to_memslot_approx(slots, cur_gfn);
1973         struct kvm_memory_slot *ms = slots->memslots + slotidx;
1974         unsigned long ofs = cur_gfn - ms->base_gfn;
1975
1976         if (ms->base_gfn + ms->npages <= cur_gfn) {
1977                 slotidx--;
1978                 /* If we are above the highest slot, wrap around */
1979                 if (slotidx < 0)
1980                         slotidx = slots->used_slots - 1;
1981
1982                 ms = slots->memslots + slotidx;
1983                 ofs = 0;
1984         }
1985
1986         if (cur_gfn < ms->base_gfn)
1987                 ofs = 0;
1988
1989         ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, ofs);
1990         while ((slotidx > 0) && (ofs >= ms->npages)) {
1991                 slotidx--;
1992                 ms = slots->memslots + slotidx;
1993                 ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, 0);
1994         }
1995         return ms->base_gfn + ofs;
1996 }
1997
1998 static int kvm_s390_get_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
1999                              u8 *res, unsigned long bufsize)
2000 {
2001         unsigned long mem_end, cur_gfn, next_gfn, hva, pgstev;
2002         struct kvm_memslots *slots = kvm_memslots(kvm);
2003         struct kvm_memory_slot *ms;
2004
2005         cur_gfn = kvm_s390_next_dirty_cmma(slots, args->start_gfn);
2006         ms = gfn_to_memslot(kvm, cur_gfn);
2007         args->count = 0;
2008         args->start_gfn = cur_gfn;
2009         if (!ms)
2010                 return 0;
2011         next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2012         mem_end = slots->memslots[0].base_gfn + slots->memslots[0].npages;
2013
2014         while (args->count < bufsize) {
2015                 hva = gfn_to_hva(kvm, cur_gfn);
2016                 if (kvm_is_error_hva(hva))
2017                         return 0;
2018                 /* Decrement only if we actually flipped the bit to 0 */
2019                 if (test_and_clear_bit(cur_gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms)))
2020                         atomic64_dec(&kvm->arch.cmma_dirty_pages);
2021                 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
2022                         pgstev = 0;
2023                 /* Save the value */
2024                 res[args->count++] = (pgstev >> 24) & 0x43;
2025                 /* If the next bit is too far away, stop. */
2026                 if (next_gfn > cur_gfn + KVM_S390_MAX_BIT_DISTANCE)
2027                         return 0;
2028                 /* If we reached the previous "next", find the next one */
2029                 if (cur_gfn == next_gfn)
2030                         next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2031                 /* Reached the end of memory or of the buffer, stop */
2032                 if ((next_gfn >= mem_end) ||
2033                     (next_gfn - args->start_gfn >= bufsize))
2034                         return 0;
2035                 cur_gfn++;
2036                 /* Reached the end of the current memslot, take the next one. */
2037                 if (cur_gfn - ms->base_gfn >= ms->npages) {
2038                         ms = gfn_to_memslot(kvm, cur_gfn);
2039                         if (!ms)
2040                                 return 0;
2041                 }
2042         }
2043         return 0;
2044 }
2045
2046 /*
2047  * This function searches for the next page with dirty CMMA attributes, and
2048  * saves the attributes in the buffer up to either the end of the buffer or
2049  * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
2050  * no trailing clean bytes are saved.
2051  * In case no dirty bits were found, or if CMMA was not enabled or used, the
2052  * output buffer will indicate 0 as length.
2053  */
2054 static int kvm_s390_get_cmma_bits(struct kvm *kvm,
2055                                   struct kvm_s390_cmma_log *args)
2056 {
2057         unsigned long bufsize;
2058         int srcu_idx, peek, ret;
2059         u8 *values;
2060
2061         if (!kvm->arch.use_cmma)
2062                 return -ENXIO;
2063         /* Invalid/unsupported flags were specified */
2064         if (args->flags & ~KVM_S390_CMMA_PEEK)
2065                 return -EINVAL;
2066         /* Migration mode query, and we are not doing a migration */
2067         peek = !!(args->flags & KVM_S390_CMMA_PEEK);
2068         if (!peek && !kvm->arch.migration_mode)
2069                 return -EINVAL;
2070         /* CMMA is disabled or was not used, or the buffer has length zero */
2071         bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
2072         if (!bufsize || !kvm->mm->context.uses_cmm) {
2073                 memset(args, 0, sizeof(*args));
2074                 return 0;
2075         }
2076         /* We are not peeking, and there are no dirty pages */
2077         if (!peek && !atomic64_read(&kvm->arch.cmma_dirty_pages)) {
2078                 memset(args, 0, sizeof(*args));
2079                 return 0;
2080         }
2081
2082         values = vmalloc(bufsize);
2083         if (!values)
2084                 return -ENOMEM;
2085
2086         down_read(&kvm->mm->mmap_sem);
2087         srcu_idx = srcu_read_lock(&kvm->srcu);
2088         if (peek)
2089                 ret = kvm_s390_peek_cmma(kvm, args, values, bufsize);
2090         else
2091                 ret = kvm_s390_get_cmma(kvm, args, values, bufsize);
2092         srcu_read_unlock(&kvm->srcu, srcu_idx);
2093         up_read(&kvm->mm->mmap_sem);
2094
2095         if (kvm->arch.migration_mode)
2096                 args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages);
2097         else
2098                 args->remaining = 0;
2099
2100         if (copy_to_user((void __user *)args->values, values, args->count))
2101                 ret = -EFAULT;
2102
2103         vfree(values);
2104         return ret;
2105 }
2106
2107 /*
2108  * This function sets the CMMA attributes for the given pages. If the input
2109  * buffer has zero length, no action is taken, otherwise the attributes are
2110  * set and the mm->context.uses_cmm flag is set.
2111  */
2112 static int kvm_s390_set_cmma_bits(struct kvm *kvm,
2113                                   const struct kvm_s390_cmma_log *args)
2114 {
2115         unsigned long hva, mask, pgstev, i;
2116         uint8_t *bits;
2117         int srcu_idx, r = 0;
2118
2119         mask = args->mask;
2120
2121         if (!kvm->arch.use_cmma)
2122                 return -ENXIO;
2123         /* invalid/unsupported flags */
2124         if (args->flags != 0)
2125                 return -EINVAL;
2126         /* Enforce sane limit on memory allocation */
2127         if (args->count > KVM_S390_CMMA_SIZE_MAX)
2128                 return -EINVAL;
2129         /* Nothing to do */
2130         if (args->count == 0)
2131                 return 0;
2132
2133         bits = vmalloc(array_size(sizeof(*bits), args->count));
2134         if (!bits)
2135                 return -ENOMEM;
2136
2137         r = copy_from_user(bits, (void __user *)args->values, args->count);
2138         if (r) {
2139                 r = -EFAULT;
2140                 goto out;
2141         }
2142
2143         down_read(&kvm->mm->mmap_sem);
2144         srcu_idx = srcu_read_lock(&kvm->srcu);
2145         for (i = 0; i < args->count; i++) {
2146                 hva = gfn_to_hva(kvm, args->start_gfn + i);
2147                 if (kvm_is_error_hva(hva)) {
2148                         r = -EFAULT;
2149                         break;
2150                 }
2151
2152                 pgstev = bits[i];
2153                 pgstev = pgstev << 24;
2154                 mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
2155                 set_pgste_bits(kvm->mm, hva, mask, pgstev);
2156         }
2157         srcu_read_unlock(&kvm->srcu, srcu_idx);
2158         up_read(&kvm->mm->mmap_sem);
2159
2160         if (!kvm->mm->context.uses_cmm) {
2161                 down_write(&kvm->mm->mmap_sem);
2162                 kvm->mm->context.uses_cmm = 1;
2163                 up_write(&kvm->mm->mmap_sem);
2164         }
2165 out:
2166         vfree(bits);
2167         return r;
2168 }
2169
2170 long kvm_arch_vm_ioctl(struct file *filp,
2171                        unsigned int ioctl, unsigned long arg)
2172 {
2173         struct kvm *kvm = filp->private_data;
2174         void __user *argp = (void __user *)arg;
2175         struct kvm_device_attr attr;
2176         int r;
2177
2178         switch (ioctl) {
2179         case KVM_S390_INTERRUPT: {
2180                 struct kvm_s390_interrupt s390int;
2181
2182                 r = -EFAULT;
2183                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2184                         break;
2185                 r = kvm_s390_inject_vm(kvm, &s390int);
2186                 break;
2187         }
2188         case KVM_CREATE_IRQCHIP: {
2189                 struct kvm_irq_routing_entry routing;
2190
2191                 r = -EINVAL;
2192                 if (kvm->arch.use_irqchip) {
2193                         /* Set up dummy routing. */
2194                         memset(&routing, 0, sizeof(routing));
2195                         r = kvm_set_irq_routing(kvm, &routing, 0, 0);
2196                 }
2197                 break;
2198         }
2199         case KVM_SET_DEVICE_ATTR: {
2200                 r = -EFAULT;
2201                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2202                         break;
2203                 r = kvm_s390_vm_set_attr(kvm, &attr);
2204                 break;
2205         }
2206         case KVM_GET_DEVICE_ATTR: {
2207                 r = -EFAULT;
2208                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2209                         break;
2210                 r = kvm_s390_vm_get_attr(kvm, &attr);
2211                 break;
2212         }
2213         case KVM_HAS_DEVICE_ATTR: {
2214                 r = -EFAULT;
2215                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2216                         break;
2217                 r = kvm_s390_vm_has_attr(kvm, &attr);
2218                 break;
2219         }
2220         case KVM_S390_GET_SKEYS: {
2221                 struct kvm_s390_skeys args;
2222
2223                 r = -EFAULT;
2224                 if (copy_from_user(&args, argp,
2225                                    sizeof(struct kvm_s390_skeys)))
2226                         break;
2227                 r = kvm_s390_get_skeys(kvm, &args);
2228                 break;
2229         }
2230         case KVM_S390_SET_SKEYS: {
2231                 struct kvm_s390_skeys args;
2232
2233                 r = -EFAULT;
2234                 if (copy_from_user(&args, argp,
2235                                    sizeof(struct kvm_s390_skeys)))
2236                         break;
2237                 r = kvm_s390_set_skeys(kvm, &args);
2238                 break;
2239         }
2240         case KVM_S390_GET_CMMA_BITS: {
2241                 struct kvm_s390_cmma_log args;
2242
2243                 r = -EFAULT;
2244                 if (copy_from_user(&args, argp, sizeof(args)))
2245                         break;
2246                 mutex_lock(&kvm->slots_lock);
2247                 r = kvm_s390_get_cmma_bits(kvm, &args);
2248                 mutex_unlock(&kvm->slots_lock);
2249                 if (!r) {
2250                         r = copy_to_user(argp, &args, sizeof(args));
2251                         if (r)
2252                                 r = -EFAULT;
2253                 }
2254                 break;
2255         }
2256         case KVM_S390_SET_CMMA_BITS: {
2257                 struct kvm_s390_cmma_log args;
2258
2259                 r = -EFAULT;
2260                 if (copy_from_user(&args, argp, sizeof(args)))
2261                         break;
2262                 mutex_lock(&kvm->slots_lock);
2263                 r = kvm_s390_set_cmma_bits(kvm, &args);
2264                 mutex_unlock(&kvm->slots_lock);
2265                 break;
2266         }
2267         default:
2268                 r = -ENOTTY;
2269         }
2270
2271         return r;
2272 }
2273
2274 static int kvm_s390_apxa_installed(void)
2275 {
2276         struct ap_config_info info;
2277
2278         if (ap_instructions_available()) {
2279                 if (ap_qci(&info) == 0)
2280                         return info.apxa;
2281         }
2282
2283         return 0;
2284 }
2285
2286 /*
2287  * The format of the crypto control block (CRYCB) is specified in the 3 low
2288  * order bits of the CRYCB designation (CRYCBD) field as follows:
2289  * Format 0: Neither the message security assist extension 3 (MSAX3) nor the
2290  *           AP extended addressing (APXA) facility are installed.
2291  * Format 1: The APXA facility is not installed but the MSAX3 facility is.
2292  * Format 2: Both the APXA and MSAX3 facilities are installed
2293  */
2294 static void kvm_s390_set_crycb_format(struct kvm *kvm)
2295 {
2296         kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
2297
2298         /* Clear the CRYCB format bits - i.e., set format 0 by default */
2299         kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
2300
2301         /* Check whether MSAX3 is installed */
2302         if (!test_kvm_facility(kvm, 76))
2303                 return;
2304
2305         if (kvm_s390_apxa_installed())
2306                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
2307         else
2308                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
2309 }
2310
2311 void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
2312                                unsigned long *aqm, unsigned long *adm)
2313 {
2314         struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
2315
2316         mutex_lock(&kvm->lock);
2317         kvm_s390_vcpu_block_all(kvm);
2318
2319         switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
2320         case CRYCB_FORMAT2: /* APCB1 use 256 bits */
2321                 memcpy(crycb->apcb1.apm, apm, 32);
2322                 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx %016lx %016lx %016lx",
2323                          apm[0], apm[1], apm[2], apm[3]);
2324                 memcpy(crycb->apcb1.aqm, aqm, 32);
2325                 VM_EVENT(kvm, 3, "SET CRYCB: aqm %016lx %016lx %016lx %016lx",
2326                          aqm[0], aqm[1], aqm[2], aqm[3]);
2327                 memcpy(crycb->apcb1.adm, adm, 32);
2328                 VM_EVENT(kvm, 3, "SET CRYCB: adm %016lx %016lx %016lx %016lx",
2329                          adm[0], adm[1], adm[2], adm[3]);
2330                 break;
2331         case CRYCB_FORMAT1:
2332         case CRYCB_FORMAT0: /* Fall through both use APCB0 */
2333                 memcpy(crycb->apcb0.apm, apm, 8);
2334                 memcpy(crycb->apcb0.aqm, aqm, 2);
2335                 memcpy(crycb->apcb0.adm, adm, 2);
2336                 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx aqm %04x adm %04x",
2337                          apm[0], *((unsigned short *)aqm),
2338                          *((unsigned short *)adm));
2339                 break;
2340         default:        /* Can not happen */
2341                 break;
2342         }
2343
2344         /* recreate the shadow crycb for each vcpu */
2345         kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2346         kvm_s390_vcpu_unblock_all(kvm);
2347         mutex_unlock(&kvm->lock);
2348 }
2349 EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks);
2350
2351 void kvm_arch_crypto_clear_masks(struct kvm *kvm)
2352 {
2353         mutex_lock(&kvm->lock);
2354         kvm_s390_vcpu_block_all(kvm);
2355
2356         memset(&kvm->arch.crypto.crycb->apcb0, 0,
2357                sizeof(kvm->arch.crypto.crycb->apcb0));
2358         memset(&kvm->arch.crypto.crycb->apcb1, 0,
2359                sizeof(kvm->arch.crypto.crycb->apcb1));
2360
2361         VM_EVENT(kvm, 3, "%s", "CLR CRYCB:");
2362         /* recreate the shadow crycb for each vcpu */
2363         kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2364         kvm_s390_vcpu_unblock_all(kvm);
2365         mutex_unlock(&kvm->lock);
2366 }
2367 EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks);
2368
2369 static u64 kvm_s390_get_initial_cpuid(void)
2370 {
2371         struct cpuid cpuid;
2372
2373         get_cpu_id(&cpuid);
2374         cpuid.version = 0xff;
2375         return *((u64 *) &cpuid);
2376 }
2377
2378 static void kvm_s390_crypto_init(struct kvm *kvm)
2379 {
2380         kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
2381         kvm_s390_set_crycb_format(kvm);
2382
2383         if (!test_kvm_facility(kvm, 76))
2384                 return;
2385
2386         /* Enable AES/DEA protected key functions by default */
2387         kvm->arch.crypto.aes_kw = 1;
2388         kvm->arch.crypto.dea_kw = 1;
2389         get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
2390                          sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
2391         get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
2392                          sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
2393 }
2394
2395 static void sca_dispose(struct kvm *kvm)
2396 {
2397         if (kvm->arch.use_esca)
2398                 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
2399         else
2400                 free_page((unsigned long)(kvm->arch.sca));
2401         kvm->arch.sca = NULL;
2402 }
2403
2404 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
2405 {
2406         gfp_t alloc_flags = GFP_KERNEL;
2407         int i, rc;
2408         char debug_name[16];
2409         static unsigned long sca_offset;
2410
2411         rc = -EINVAL;
2412 #ifdef CONFIG_KVM_S390_UCONTROL
2413         if (type & ~KVM_VM_S390_UCONTROL)
2414                 goto out_err;
2415         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
2416                 goto out_err;
2417 #else
2418         if (type)
2419                 goto out_err;
2420 #endif
2421
2422         rc = s390_enable_sie();
2423         if (rc)
2424                 goto out_err;
2425
2426         rc = -ENOMEM;
2427
2428         if (!sclp.has_64bscao)
2429                 alloc_flags |= GFP_DMA;
2430         rwlock_init(&kvm->arch.sca_lock);
2431         /* start with basic SCA */
2432         kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
2433         if (!kvm->arch.sca)
2434                 goto out_err;
2435         mutex_lock(&kvm_lock);
2436         sca_offset += 16;
2437         if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
2438                 sca_offset = 0;
2439         kvm->arch.sca = (struct bsca_block *)
2440                         ((char *) kvm->arch.sca + sca_offset);
2441         mutex_unlock(&kvm_lock);
2442
2443         sprintf(debug_name, "kvm-%u", current->pid);
2444
2445         kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
2446         if (!kvm->arch.dbf)
2447                 goto out_err;
2448
2449         BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
2450         kvm->arch.sie_page2 =
2451              (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
2452         if (!kvm->arch.sie_page2)
2453                 goto out_err;
2454
2455         kvm->arch.sie_page2->kvm = kvm;
2456         kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
2457
2458         for (i = 0; i < kvm_s390_fac_size(); i++) {
2459                 kvm->arch.model.fac_mask[i] = S390_lowcore.stfle_fac_list[i] &
2460                                               (kvm_s390_fac_base[i] |
2461                                                kvm_s390_fac_ext[i]);
2462                 kvm->arch.model.fac_list[i] = S390_lowcore.stfle_fac_list[i] &
2463                                               kvm_s390_fac_base[i];
2464         }
2465         kvm->arch.model.subfuncs = kvm_s390_available_subfunc;
2466
2467         /* we are always in czam mode - even on pre z14 machines */
2468         set_kvm_facility(kvm->arch.model.fac_mask, 138);
2469         set_kvm_facility(kvm->arch.model.fac_list, 138);
2470         /* we emulate STHYI in kvm */
2471         set_kvm_facility(kvm->arch.model.fac_mask, 74);
2472         set_kvm_facility(kvm->arch.model.fac_list, 74);
2473         if (MACHINE_HAS_TLB_GUEST) {
2474                 set_kvm_facility(kvm->arch.model.fac_mask, 147);
2475                 set_kvm_facility(kvm->arch.model.fac_list, 147);
2476         }
2477
2478         if (css_general_characteristics.aiv && test_facility(65))
2479                 set_kvm_facility(kvm->arch.model.fac_mask, 65);
2480
2481         kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
2482         kvm->arch.model.ibc = sclp.ibc & 0x0fff;
2483
2484         kvm_s390_crypto_init(kvm);
2485
2486         mutex_init(&kvm->arch.float_int.ais_lock);
2487         spin_lock_init(&kvm->arch.float_int.lock);
2488         for (i = 0; i < FIRQ_LIST_COUNT; i++)
2489                 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
2490         init_waitqueue_head(&kvm->arch.ipte_wq);
2491         mutex_init(&kvm->arch.ipte_mutex);
2492
2493         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
2494         VM_EVENT(kvm, 3, "vm created with type %lu", type);
2495
2496         if (type & KVM_VM_S390_UCONTROL) {
2497                 kvm->arch.gmap = NULL;
2498                 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
2499         } else {
2500                 if (sclp.hamax == U64_MAX)
2501                         kvm->arch.mem_limit = TASK_SIZE_MAX;
2502                 else
2503                         kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
2504                                                     sclp.hamax + 1);
2505                 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
2506                 if (!kvm->arch.gmap)
2507                         goto out_err;
2508                 kvm->arch.gmap->private = kvm;
2509                 kvm->arch.gmap->pfault_enabled = 0;
2510         }
2511
2512         kvm->arch.use_pfmfi = sclp.has_pfmfi;
2513         kvm->arch.use_skf = sclp.has_skey;
2514         spin_lock_init(&kvm->arch.start_stop_lock);
2515         kvm_s390_vsie_init(kvm);
2516         kvm_s390_gisa_init(kvm);
2517         KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
2518
2519         return 0;
2520 out_err:
2521         free_page((unsigned long)kvm->arch.sie_page2);
2522         debug_unregister(kvm->arch.dbf);
2523         sca_dispose(kvm);
2524         KVM_EVENT(3, "creation of vm failed: %d", rc);
2525         return rc;
2526 }
2527
2528 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
2529 {
2530         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
2531         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
2532         kvm_s390_clear_local_irqs(vcpu);
2533         kvm_clear_async_pf_completion_queue(vcpu);
2534         if (!kvm_is_ucontrol(vcpu->kvm))
2535                 sca_del_vcpu(vcpu);
2536
2537         if (kvm_is_ucontrol(vcpu->kvm))
2538                 gmap_remove(vcpu->arch.gmap);
2539
2540         if (vcpu->kvm->arch.use_cmma)
2541                 kvm_s390_vcpu_unsetup_cmma(vcpu);
2542         free_page((unsigned long)(vcpu->arch.sie_block));
2543
2544         kvm_vcpu_uninit(vcpu);
2545         kmem_cache_free(kvm_vcpu_cache, vcpu);
2546 }
2547
2548 static void kvm_free_vcpus(struct kvm *kvm)
2549 {
2550         unsigned int i;
2551         struct kvm_vcpu *vcpu;
2552
2553         kvm_for_each_vcpu(i, vcpu, kvm)
2554                 kvm_arch_vcpu_destroy(vcpu);
2555
2556         mutex_lock(&kvm->lock);
2557         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
2558                 kvm->vcpus[i] = NULL;
2559
2560         atomic_set(&kvm->online_vcpus, 0);
2561         mutex_unlock(&kvm->lock);
2562 }
2563
2564 void kvm_arch_destroy_vm(struct kvm *kvm)
2565 {
2566         kvm_free_vcpus(kvm);
2567         sca_dispose(kvm);
2568         debug_unregister(kvm->arch.dbf);
2569         kvm_s390_gisa_destroy(kvm);
2570         free_page((unsigned long)kvm->arch.sie_page2);
2571         if (!kvm_is_ucontrol(kvm))
2572                 gmap_remove(kvm->arch.gmap);
2573         kvm_s390_destroy_adapters(kvm);
2574         kvm_s390_clear_float_irqs(kvm);
2575         kvm_s390_vsie_destroy(kvm);
2576         KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
2577 }
2578
2579 /* Section: vcpu related */
2580 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
2581 {
2582         vcpu->arch.gmap = gmap_create(current->mm, -1UL);
2583         if (!vcpu->arch.gmap)
2584                 return -ENOMEM;
2585         vcpu->arch.gmap->private = vcpu->kvm;
2586
2587         return 0;
2588 }
2589
2590 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
2591 {
2592         if (!kvm_s390_use_sca_entries())
2593                 return;
2594         read_lock(&vcpu->kvm->arch.sca_lock);
2595         if (vcpu->kvm->arch.use_esca) {
2596                 struct esca_block *sca = vcpu->kvm->arch.sca;
2597
2598                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2599                 sca->cpu[vcpu->vcpu_id].sda = 0;
2600         } else {
2601                 struct bsca_block *sca = vcpu->kvm->arch.sca;
2602
2603                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2604                 sca->cpu[vcpu->vcpu_id].sda = 0;
2605         }
2606         read_unlock(&vcpu->kvm->arch.sca_lock);
2607 }
2608
2609 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
2610 {
2611         if (!kvm_s390_use_sca_entries()) {
2612                 struct bsca_block *sca = vcpu->kvm->arch.sca;
2613
2614                 /* we still need the basic sca for the ipte control */
2615                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2616                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2617                 return;
2618         }
2619         read_lock(&vcpu->kvm->arch.sca_lock);
2620         if (vcpu->kvm->arch.use_esca) {
2621                 struct esca_block *sca = vcpu->kvm->arch.sca;
2622
2623                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2624                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2625                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
2626                 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2627                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2628         } else {
2629                 struct bsca_block *sca = vcpu->kvm->arch.sca;
2630
2631                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2632                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2633                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2634                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2635         }
2636         read_unlock(&vcpu->kvm->arch.sca_lock);
2637 }
2638
2639 /* Basic SCA to Extended SCA data copy routines */
2640 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
2641 {
2642         d->sda = s->sda;
2643         d->sigp_ctrl.c = s->sigp_ctrl.c;
2644         d->sigp_ctrl.scn = s->sigp_ctrl.scn;
2645 }
2646
2647 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
2648 {
2649         int i;
2650
2651         d->ipte_control = s->ipte_control;
2652         d->mcn[0] = s->mcn;
2653         for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
2654                 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
2655 }
2656
2657 static int sca_switch_to_extended(struct kvm *kvm)
2658 {
2659         struct bsca_block *old_sca = kvm->arch.sca;
2660         struct esca_block *new_sca;
2661         struct kvm_vcpu *vcpu;
2662         unsigned int vcpu_idx;
2663         u32 scaol, scaoh;
2664
2665         new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
2666         if (!new_sca)
2667                 return -ENOMEM;
2668
2669         scaoh = (u32)((u64)(new_sca) >> 32);
2670         scaol = (u32)(u64)(new_sca) & ~0x3fU;
2671
2672         kvm_s390_vcpu_block_all(kvm);
2673         write_lock(&kvm->arch.sca_lock);
2674
2675         sca_copy_b_to_e(new_sca, old_sca);
2676
2677         kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
2678                 vcpu->arch.sie_block->scaoh = scaoh;
2679                 vcpu->arch.sie_block->scaol = scaol;
2680                 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2681         }
2682         kvm->arch.sca = new_sca;
2683         kvm->arch.use_esca = 1;
2684
2685         write_unlock(&kvm->arch.sca_lock);
2686         kvm_s390_vcpu_unblock_all(kvm);
2687
2688         free_page((unsigned long)old_sca);
2689
2690         VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
2691                  old_sca, kvm->arch.sca);
2692         return 0;
2693 }
2694
2695 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
2696 {
2697         int rc;
2698
2699         if (!kvm_s390_use_sca_entries()) {
2700                 if (id < KVM_MAX_VCPUS)
2701                         return true;
2702                 return false;
2703         }
2704         if (id < KVM_S390_BSCA_CPU_SLOTS)
2705                 return true;
2706         if (!sclp.has_esca || !sclp.has_64bscao)
2707                 return false;
2708
2709         mutex_lock(&kvm->lock);
2710         rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
2711         mutex_unlock(&kvm->lock);
2712
2713         return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
2714 }
2715
2716 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
2717 {
2718         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2719         kvm_clear_async_pf_completion_queue(vcpu);
2720         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
2721                                     KVM_SYNC_GPRS |
2722                                     KVM_SYNC_ACRS |
2723                                     KVM_SYNC_CRS |
2724                                     KVM_SYNC_ARCH0 |
2725                                     KVM_SYNC_PFAULT;
2726         kvm_s390_set_prefix(vcpu, 0);
2727         if (test_kvm_facility(vcpu->kvm, 64))
2728                 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
2729         if (test_kvm_facility(vcpu->kvm, 82))
2730                 vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
2731         if (test_kvm_facility(vcpu->kvm, 133))
2732                 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
2733         if (test_kvm_facility(vcpu->kvm, 156))
2734                 vcpu->run->kvm_valid_regs |= KVM_SYNC_ETOKEN;
2735         /* fprs can be synchronized via vrs, even if the guest has no vx. With
2736          * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
2737          */
2738         if (MACHINE_HAS_VX)
2739                 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
2740         else
2741                 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
2742
2743         if (kvm_is_ucontrol(vcpu->kvm))
2744                 return __kvm_ucontrol_vcpu_init(vcpu);
2745
2746         return 0;
2747 }
2748
2749 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2750 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2751 {
2752         WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
2753         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2754         vcpu->arch.cputm_start = get_tod_clock_fast();
2755         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2756 }
2757
2758 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2759 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2760 {
2761         WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
2762         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2763         vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2764         vcpu->arch.cputm_start = 0;
2765         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2766 }
2767
2768 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2769 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2770 {
2771         WARN_ON_ONCE(vcpu->arch.cputm_enabled);
2772         vcpu->arch.cputm_enabled = true;
2773         __start_cpu_timer_accounting(vcpu);
2774 }
2775
2776 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2777 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2778 {
2779         WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
2780         __stop_cpu_timer_accounting(vcpu);
2781         vcpu->arch.cputm_enabled = false;
2782 }
2783
2784 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2785 {
2786         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2787         __enable_cpu_timer_accounting(vcpu);
2788         preempt_enable();
2789 }
2790
2791 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2792 {
2793         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2794         __disable_cpu_timer_accounting(vcpu);
2795         preempt_enable();
2796 }
2797
2798 /* set the cpu timer - may only be called from the VCPU thread itself */
2799 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
2800 {
2801         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2802         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2803         if (vcpu->arch.cputm_enabled)
2804                 vcpu->arch.cputm_start = get_tod_clock_fast();
2805         vcpu->arch.sie_block->cputm = cputm;
2806         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2807         preempt_enable();
2808 }
2809
2810 /* update and get the cpu timer - can also be called from other VCPU threads */
2811 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
2812 {
2813         unsigned int seq;
2814         __u64 value;
2815
2816         if (unlikely(!vcpu->arch.cputm_enabled))
2817                 return vcpu->arch.sie_block->cputm;
2818
2819         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2820         do {
2821                 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
2822                 /*
2823                  * If the writer would ever execute a read in the critical
2824                  * section, e.g. in irq context, we have a deadlock.
2825                  */
2826                 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
2827                 value = vcpu->arch.sie_block->cputm;
2828                 /* if cputm_start is 0, accounting is being started/stopped */
2829                 if (likely(vcpu->arch.cputm_start))
2830                         value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2831         } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
2832         preempt_enable();
2833         return value;
2834 }
2835
2836 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2837 {
2838
2839         gmap_enable(vcpu->arch.enabled_gmap);
2840         kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
2841         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2842                 __start_cpu_timer_accounting(vcpu);
2843         vcpu->cpu = cpu;
2844 }
2845
2846 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
2847 {
2848         vcpu->cpu = -1;
2849         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2850                 __stop_cpu_timer_accounting(vcpu);
2851         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
2852         vcpu->arch.enabled_gmap = gmap_get_enabled();
2853         gmap_disable(vcpu->arch.enabled_gmap);
2854
2855 }
2856
2857 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
2858 {
2859         /* this equals initial cpu reset in pop, but we don't switch to ESA */
2860         vcpu->arch.sie_block->gpsw.mask = 0UL;
2861         vcpu->arch.sie_block->gpsw.addr = 0UL;
2862         kvm_s390_set_prefix(vcpu, 0);
2863         kvm_s390_set_cpu_timer(vcpu, 0);
2864         vcpu->arch.sie_block->ckc       = 0UL;
2865         vcpu->arch.sie_block->todpr     = 0;
2866         memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
2867         vcpu->arch.sie_block->gcr[0]  = CR0_UNUSED_56 |
2868                                         CR0_INTERRUPT_KEY_SUBMASK |
2869                                         CR0_MEASUREMENT_ALERT_SUBMASK;
2870         vcpu->arch.sie_block->gcr[14] = CR14_UNUSED_32 |
2871                                         CR14_UNUSED_33 |
2872                                         CR14_EXTERNAL_DAMAGE_SUBMASK;
2873         vcpu->run->s.regs.fpc = 0;
2874         vcpu->arch.sie_block->gbea = 1;
2875         vcpu->arch.sie_block->pp = 0;
2876         vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
2877         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2878         kvm_clear_async_pf_completion_queue(vcpu);
2879         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
2880                 kvm_s390_vcpu_stop(vcpu);
2881         kvm_s390_clear_local_irqs(vcpu);
2882 }
2883
2884 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
2885 {
2886         mutex_lock(&vcpu->kvm->lock);
2887         preempt_disable();
2888         vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
2889         vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
2890         preempt_enable();
2891         mutex_unlock(&vcpu->kvm->lock);
2892         if (!kvm_is_ucontrol(vcpu->kvm)) {
2893                 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
2894                 sca_add_vcpu(vcpu);
2895         }
2896         if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
2897                 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2898         /* make vcpu_load load the right gmap on the first trigger */
2899         vcpu->arch.enabled_gmap = vcpu->arch.gmap;
2900 }
2901
2902 static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr)
2903 {
2904         if (test_bit_inv(nr, (unsigned long *)&kvm->arch.model.subfuncs.pckmo) &&
2905             test_bit_inv(nr, (unsigned long *)&kvm_s390_available_subfunc.pckmo))
2906                 return true;
2907         return false;
2908 }
2909
2910 static bool kvm_has_pckmo_ecc(struct kvm *kvm)
2911 {
2912         /* At least one ECC subfunction must be present */
2913         return kvm_has_pckmo_subfunc(kvm, 32) ||
2914                kvm_has_pckmo_subfunc(kvm, 33) ||
2915                kvm_has_pckmo_subfunc(kvm, 34) ||
2916                kvm_has_pckmo_subfunc(kvm, 40) ||
2917                kvm_has_pckmo_subfunc(kvm, 41);
2918
2919 }
2920
2921 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
2922 {
2923         /*
2924          * If the AP instructions are not being interpreted and the MSAX3
2925          * facility is not configured for the guest, there is nothing to set up.
2926          */
2927         if (!vcpu->kvm->arch.crypto.apie && !test_kvm_facility(vcpu->kvm, 76))
2928                 return;
2929
2930         vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
2931         vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
2932         vcpu->arch.sie_block->eca &= ~ECA_APIE;
2933         vcpu->arch.sie_block->ecd &= ~ECD_ECC;
2934
2935         if (vcpu->kvm->arch.crypto.apie)
2936                 vcpu->arch.sie_block->eca |= ECA_APIE;
2937
2938         /* Set up protected key support */
2939         if (vcpu->kvm->arch.crypto.aes_kw) {
2940                 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
2941                 /* ecc is also wrapped with AES key */
2942                 if (kvm_has_pckmo_ecc(vcpu->kvm))
2943                         vcpu->arch.sie_block->ecd |= ECD_ECC;
2944         }
2945
2946         if (vcpu->kvm->arch.crypto.dea_kw)
2947                 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
2948 }
2949
2950 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
2951 {
2952         free_page(vcpu->arch.sie_block->cbrlo);
2953         vcpu->arch.sie_block->cbrlo = 0;
2954 }
2955
2956 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
2957 {
2958         vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
2959         if (!vcpu->arch.sie_block->cbrlo)
2960                 return -ENOMEM;
2961         return 0;
2962 }
2963
2964 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
2965 {
2966         struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
2967
2968         vcpu->arch.sie_block->ibc = model->ibc;
2969         if (test_kvm_facility(vcpu->kvm, 7))
2970                 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
2971 }
2972
2973 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
2974 {
2975         int rc = 0;
2976
2977         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
2978                                                     CPUSTAT_SM |
2979                                                     CPUSTAT_STOPPED);
2980
2981         if (test_kvm_facility(vcpu->kvm, 78))
2982                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
2983         else if (test_kvm_facility(vcpu->kvm, 8))
2984                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
2985
2986         kvm_s390_vcpu_setup_model(vcpu);
2987
2988         /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
2989         if (MACHINE_HAS_ESOP)
2990                 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
2991         if (test_kvm_facility(vcpu->kvm, 9))
2992                 vcpu->arch.sie_block->ecb |= ECB_SRSI;
2993         if (test_kvm_facility(vcpu->kvm, 73))
2994                 vcpu->arch.sie_block->ecb |= ECB_TE;
2995
2996         if (test_kvm_facility(vcpu->kvm, 8) && vcpu->kvm->arch.use_pfmfi)
2997                 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
2998         if (test_kvm_facility(vcpu->kvm, 130))
2999                 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
3000         vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
3001         if (sclp.has_cei)
3002                 vcpu->arch.sie_block->eca |= ECA_CEI;
3003         if (sclp.has_ib)
3004                 vcpu->arch.sie_block->eca |= ECA_IB;
3005         if (sclp.has_siif)
3006                 vcpu->arch.sie_block->eca |= ECA_SII;
3007         if (sclp.has_sigpif)
3008                 vcpu->arch.sie_block->eca |= ECA_SIGPI;
3009         if (test_kvm_facility(vcpu->kvm, 129)) {
3010                 vcpu->arch.sie_block->eca |= ECA_VX;
3011                 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3012         }
3013         if (test_kvm_facility(vcpu->kvm, 139))
3014                 vcpu->arch.sie_block->ecd |= ECD_MEF;
3015         if (test_kvm_facility(vcpu->kvm, 156))
3016                 vcpu->arch.sie_block->ecd |= ECD_ETOKENF;
3017         if (vcpu->arch.sie_block->gd) {
3018                 vcpu->arch.sie_block->eca |= ECA_AIV;
3019                 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
3020                            vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
3021         }
3022         vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
3023                                         | SDNXC;
3024         vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
3025
3026         if (sclp.has_kss)
3027                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
3028         else
3029                 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
3030
3031         if (vcpu->kvm->arch.use_cmma) {
3032                 rc = kvm_s390_vcpu_setup_cmma(vcpu);
3033                 if (rc)
3034                         return rc;
3035         }
3036         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3037         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
3038
3039         vcpu->arch.sie_block->hpid = HPID_KVM;
3040
3041         kvm_s390_vcpu_crypto_setup(vcpu);
3042
3043         return rc;
3044 }
3045
3046 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
3047                                       unsigned int id)
3048 {
3049         struct kvm_vcpu *vcpu;
3050         struct sie_page *sie_page;
3051         int rc = -EINVAL;
3052
3053         if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
3054                 goto out;
3055
3056         rc = -ENOMEM;
3057
3058         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
3059         if (!vcpu)
3060                 goto out;
3061
3062         BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
3063         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
3064         if (!sie_page)
3065                 goto out_free_cpu;
3066
3067         vcpu->arch.sie_block = &sie_page->sie_block;
3068         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
3069
3070         /* the real guest size will always be smaller than msl */
3071         vcpu->arch.sie_block->mso = 0;
3072         vcpu->arch.sie_block->msl = sclp.hamax;
3073
3074         vcpu->arch.sie_block->icpua = id;
3075         spin_lock_init(&vcpu->arch.local_int.lock);
3076         vcpu->arch.sie_block->gd = (u32)(u64)kvm->arch.gisa_int.origin;
3077         if (vcpu->arch.sie_block->gd && sclp.has_gisaf)
3078                 vcpu->arch.sie_block->gd |= GISA_FORMAT1;
3079         seqcount_init(&vcpu->arch.cputm_seqcount);
3080
3081         rc = kvm_vcpu_init(vcpu, kvm, id);
3082         if (rc)
3083                 goto out_free_sie_block;
3084         VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
3085                  vcpu->arch.sie_block);
3086         trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
3087
3088         return vcpu;
3089 out_free_sie_block:
3090         free_page((unsigned long)(vcpu->arch.sie_block));
3091 out_free_cpu:
3092         kmem_cache_free(kvm_vcpu_cache, vcpu);
3093 out:
3094         return ERR_PTR(rc);
3095 }
3096
3097 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
3098 {
3099         clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.gisa_int.kicked_mask);
3100         return kvm_s390_vcpu_has_irq(vcpu, 0);
3101 }
3102
3103 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
3104 {
3105         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
3106 }
3107
3108 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
3109 {
3110         atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3111         exit_sie(vcpu);
3112 }
3113
3114 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
3115 {
3116         atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3117 }
3118
3119 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
3120 {
3121         atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3122         exit_sie(vcpu);
3123 }
3124
3125 bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu)
3126 {
3127         return atomic_read(&vcpu->arch.sie_block->prog20) &
3128                (PROG_BLOCK_SIE | PROG_REQUEST);
3129 }
3130
3131 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
3132 {
3133         atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3134 }
3135
3136 /*
3137  * Kick a guest cpu out of (v)SIE and wait until (v)SIE is not running.
3138  * If the CPU is not running (e.g. waiting as idle) the function will
3139  * return immediately. */
3140 void exit_sie(struct kvm_vcpu *vcpu)
3141 {
3142         kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
3143         kvm_s390_vsie_kick(vcpu);
3144         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
3145                 cpu_relax();
3146 }
3147
3148 /* Kick a guest cpu out of SIE to process a request synchronously */
3149 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
3150 {
3151         kvm_make_request(req, vcpu);
3152         kvm_s390_vcpu_request(vcpu);
3153 }
3154
3155 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
3156                               unsigned long end)
3157 {
3158         struct kvm *kvm = gmap->private;
3159         struct kvm_vcpu *vcpu;
3160         unsigned long prefix;
3161         int i;
3162
3163         if (gmap_is_shadow(gmap))
3164                 return;
3165         if (start >= 1UL << 31)
3166                 /* We are only interested in prefix pages */
3167                 return;
3168         kvm_for_each_vcpu(i, vcpu, kvm) {
3169                 /* match against both prefix pages */
3170                 prefix = kvm_s390_get_prefix(vcpu);
3171                 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
3172                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
3173                                    start, end);
3174                         kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
3175                 }
3176         }
3177 }
3178
3179 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
3180 {
3181         /* do not poll with more than halt_poll_max_steal percent of steal time */
3182         if (S390_lowcore.avg_steal_timer * 100 / (TICK_USEC << 12) >=
3183             halt_poll_max_steal) {
3184                 vcpu->stat.halt_no_poll_steal++;
3185                 return true;
3186         }
3187         return false;
3188 }
3189
3190 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
3191 {
3192         /* kvm common code refers to this, but never calls it */
3193         BUG();
3194         return 0;
3195 }
3196
3197 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
3198                                            struct kvm_one_reg *reg)
3199 {
3200         int r = -EINVAL;
3201
3202         switch (reg->id) {
3203         case KVM_REG_S390_TODPR:
3204                 r = put_user(vcpu->arch.sie_block->todpr,
3205                              (u32 __user *)reg->addr);
3206                 break;
3207         case KVM_REG_S390_EPOCHDIFF:
3208                 r = put_user(vcpu->arch.sie_block->epoch,
3209                              (u64 __user *)reg->addr);
3210                 break;
3211         case KVM_REG_S390_CPU_TIMER:
3212                 r = put_user(kvm_s390_get_cpu_timer(vcpu),
3213                              (u64 __user *)reg->addr);
3214                 break;
3215         case KVM_REG_S390_CLOCK_COMP:
3216                 r = put_user(vcpu->arch.sie_block->ckc,
3217                              (u64 __user *)reg->addr);
3218                 break;
3219         case KVM_REG_S390_PFTOKEN:
3220                 r = put_user(vcpu->arch.pfault_token,
3221                              (u64 __user *)reg->addr);
3222                 break;
3223         case KVM_REG_S390_PFCOMPARE:
3224                 r = put_user(vcpu->arch.pfault_compare,
3225                              (u64 __user *)reg->addr);
3226                 break;
3227         case KVM_REG_S390_PFSELECT:
3228                 r = put_user(vcpu->arch.pfault_select,
3229                              (u64 __user *)reg->addr);
3230                 break;
3231         case KVM_REG_S390_PP:
3232                 r = put_user(vcpu->arch.sie_block->pp,
3233                              (u64 __user *)reg->addr);
3234                 break;
3235         case KVM_REG_S390_GBEA:
3236                 r = put_user(vcpu->arch.sie_block->gbea,
3237                              (u64 __user *)reg->addr);
3238                 break;
3239         default:
3240                 break;
3241         }
3242
3243         return r;
3244 }
3245
3246 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
3247                                            struct kvm_one_reg *reg)
3248 {
3249         int r = -EINVAL;
3250         __u64 val;
3251
3252         switch (reg->id) {
3253         case KVM_REG_S390_TODPR:
3254                 r = get_user(vcpu->arch.sie_block->todpr,
3255                              (u32 __user *)reg->addr);
3256                 break;
3257         case KVM_REG_S390_EPOCHDIFF:
3258                 r = get_user(vcpu->arch.sie_block->epoch,
3259                              (u64 __user *)reg->addr);
3260                 break;
3261         case KVM_REG_S390_CPU_TIMER:
3262                 r = get_user(val, (u64 __user *)reg->addr);
3263                 if (!r)
3264                         kvm_s390_set_cpu_timer(vcpu, val);
3265                 break;
3266         case KVM_REG_S390_CLOCK_COMP:
3267                 r = get_user(vcpu->arch.sie_block->ckc,
3268                              (u64 __user *)reg->addr);
3269                 break;
3270         case KVM_REG_S390_PFTOKEN:
3271                 r = get_user(vcpu->arch.pfault_token,
3272                              (u64 __user *)reg->addr);
3273                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3274                         kvm_clear_async_pf_completion_queue(vcpu);
3275                 break;
3276         case KVM_REG_S390_PFCOMPARE:
3277                 r = get_user(vcpu->arch.pfault_compare,
3278                              (u64 __user *)reg->addr);
3279                 break;
3280         case KVM_REG_S390_PFSELECT:
3281                 r = get_user(vcpu->arch.pfault_select,
3282                              (u64 __user *)reg->addr);
3283                 break;
3284         case KVM_REG_S390_PP:
3285                 r = get_user(vcpu->arch.sie_block->pp,
3286                              (u64 __user *)reg->addr);
3287                 break;
3288         case KVM_REG_S390_GBEA:
3289                 r = get_user(vcpu->arch.sie_block->gbea,
3290                              (u64 __user *)reg->addr);
3291                 break;
3292         default:
3293                 break;
3294         }
3295
3296         return r;
3297 }
3298
3299 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
3300 {
3301         kvm_s390_vcpu_initial_reset(vcpu);
3302         return 0;
3303 }
3304
3305 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3306 {
3307         vcpu_load(vcpu);
3308         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
3309         vcpu_put(vcpu);
3310         return 0;
3311 }
3312
3313 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3314 {
3315         vcpu_load(vcpu);
3316         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
3317         vcpu_put(vcpu);
3318         return 0;
3319 }
3320
3321 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
3322                                   struct kvm_sregs *sregs)
3323 {
3324         vcpu_load(vcpu);
3325
3326         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
3327         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
3328
3329         vcpu_put(vcpu);
3330         return 0;
3331 }
3332
3333 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
3334                                   struct kvm_sregs *sregs)
3335 {
3336         vcpu_load(vcpu);
3337
3338         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
3339         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
3340
3341         vcpu_put(vcpu);
3342         return 0;
3343 }
3344
3345 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3346 {
3347         int ret = 0;
3348
3349         vcpu_load(vcpu);
3350
3351         vcpu->run->s.regs.fpc = fpu->fpc;
3352         if (MACHINE_HAS_VX)
3353                 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
3354                                  (freg_t *) fpu->fprs);
3355         else
3356                 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
3357
3358         vcpu_put(vcpu);
3359         return ret;
3360 }
3361
3362 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3363 {
3364         vcpu_load(vcpu);
3365
3366         /* make sure we have the latest values */
3367         save_fpu_regs();
3368         if (MACHINE_HAS_VX)
3369                 convert_vx_to_fp((freg_t *) fpu->fprs,
3370                                  (__vector128 *) vcpu->run->s.regs.vrs);
3371         else
3372                 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
3373         fpu->fpc = vcpu->run->s.regs.fpc;
3374
3375         vcpu_put(vcpu);
3376         return 0;
3377 }
3378
3379 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
3380 {
3381         int rc = 0;
3382
3383         if (!is_vcpu_stopped(vcpu))
3384                 rc = -EBUSY;
3385         else {
3386                 vcpu->run->psw_mask = psw.mask;
3387                 vcpu->run->psw_addr = psw.addr;
3388         }
3389         return rc;
3390 }
3391
3392 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
3393                                   struct kvm_translation *tr)
3394 {
3395         return -EINVAL; /* not implemented yet */
3396 }
3397
3398 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
3399                               KVM_GUESTDBG_USE_HW_BP | \
3400                               KVM_GUESTDBG_ENABLE)
3401
3402 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
3403                                         struct kvm_guest_debug *dbg)
3404 {
3405         int rc = 0;
3406
3407         vcpu_load(vcpu);
3408
3409         vcpu->guest_debug = 0;
3410         kvm_s390_clear_bp_data(vcpu);
3411
3412         if (dbg->control & ~VALID_GUESTDBG_FLAGS) {
3413                 rc = -EINVAL;
3414                 goto out;
3415         }
3416         if (!sclp.has_gpere) {
3417                 rc = -EINVAL;
3418                 goto out;
3419         }
3420
3421         if (dbg->control & KVM_GUESTDBG_ENABLE) {
3422                 vcpu->guest_debug = dbg->control;
3423                 /* enforce guest PER */
3424                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_P);
3425
3426                 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
3427                         rc = kvm_s390_import_bp_data(vcpu, dbg);
3428         } else {
3429                 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
3430                 vcpu->arch.guestdbg.last_bp = 0;
3431         }
3432
3433         if (rc) {
3434                 vcpu->guest_debug = 0;
3435                 kvm_s390_clear_bp_data(vcpu);
3436                 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
3437         }
3438
3439 out:
3440         vcpu_put(vcpu);
3441         return rc;
3442 }
3443
3444 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
3445                                     struct kvm_mp_state *mp_state)
3446 {
3447         int ret;
3448
3449         vcpu_load(vcpu);
3450
3451         /* CHECK_STOP and LOAD are not supported yet */
3452         ret = is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
3453                                       KVM_MP_STATE_OPERATING;
3454
3455         vcpu_put(vcpu);
3456         return ret;
3457 }
3458
3459 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
3460                                     struct kvm_mp_state *mp_state)
3461 {
3462         int rc = 0;
3463
3464         vcpu_load(vcpu);
3465
3466         /* user space knows about this interface - let it control the state */
3467         vcpu->kvm->arch.user_cpu_state_ctrl = 1;
3468
3469         switch (mp_state->mp_state) {
3470         case KVM_MP_STATE_STOPPED:
3471                 kvm_s390_vcpu_stop(vcpu);
3472                 break;
3473         case KVM_MP_STATE_OPERATING:
3474                 kvm_s390_vcpu_start(vcpu);
3475                 break;
3476         case KVM_MP_STATE_LOAD:
3477         case KVM_MP_STATE_CHECK_STOP:
3478                 /* fall through - CHECK_STOP and LOAD are not supported yet */
3479         default:
3480                 rc = -ENXIO;
3481         }
3482
3483         vcpu_put(vcpu);
3484         return rc;
3485 }
3486
3487 static bool ibs_enabled(struct kvm_vcpu *vcpu)
3488 {
3489         return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS);
3490 }
3491
3492 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
3493 {
3494 retry:
3495         kvm_s390_vcpu_request_handled(vcpu);
3496         if (!kvm_request_pending(vcpu))
3497                 return 0;
3498         /*
3499          * We use MMU_RELOAD just to re-arm the ipte notifier for the
3500          * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
3501          * This ensures that the ipte instruction for this request has
3502          * already finished. We might race against a second unmapper that
3503          * wants to set the blocking bit. Lets just retry the request loop.
3504          */
3505         if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
3506                 int rc;
3507                 rc = gmap_mprotect_notify(vcpu->arch.gmap,
3508                                           kvm_s390_get_prefix(vcpu),
3509                                           PAGE_SIZE * 2, PROT_WRITE);
3510                 if (rc) {
3511                         kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
3512                         return rc;
3513                 }
3514                 goto retry;
3515         }
3516
3517         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
3518                 vcpu->arch.sie_block->ihcpu = 0xffff;
3519                 goto retry;
3520         }
3521
3522         if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
3523                 if (!ibs_enabled(vcpu)) {
3524                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
3525                         kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS);
3526                 }
3527                 goto retry;
3528         }
3529
3530         if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
3531                 if (ibs_enabled(vcpu)) {
3532                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
3533                         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS);
3534                 }
3535                 goto retry;
3536         }
3537
3538         if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
3539                 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
3540                 goto retry;
3541         }
3542
3543         if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
3544                 /*
3545                  * Disable CMM virtualization; we will emulate the ESSA
3546                  * instruction manually, in order to provide additional
3547                  * functionalities needed for live migration.
3548                  */
3549                 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
3550                 goto retry;
3551         }
3552
3553         if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
3554                 /*
3555                  * Re-enable CMM virtualization if CMMA is available and
3556                  * CMM has been used.
3557                  */
3558                 if ((vcpu->kvm->arch.use_cmma) &&
3559                     (vcpu->kvm->mm->context.uses_cmm))
3560                         vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
3561                 goto retry;
3562         }
3563
3564         /* nothing to do, just clear the request */
3565         kvm_clear_request(KVM_REQ_UNHALT, vcpu);
3566         /* we left the vsie handler, nothing to do, just clear the request */
3567         kvm_clear_request(KVM_REQ_VSIE_RESTART, vcpu);
3568
3569         return 0;
3570 }
3571
3572 void kvm_s390_set_tod_clock(struct kvm *kvm,
3573                             const struct kvm_s390_vm_tod_clock *gtod)
3574 {
3575         struct kvm_vcpu *vcpu;
3576         struct kvm_s390_tod_clock_ext htod;
3577         int i;
3578
3579         mutex_lock(&kvm->lock);
3580         preempt_disable();
3581
3582         get_tod_clock_ext((char *)&htod);
3583
3584         kvm->arch.epoch = gtod->tod - htod.tod;
3585         kvm->arch.epdx = 0;
3586         if (test_kvm_facility(kvm, 139)) {
3587                 kvm->arch.epdx = gtod->epoch_idx - htod.epoch_idx;
3588                 if (kvm->arch.epoch > gtod->tod)
3589                         kvm->arch.epdx -= 1;
3590         }
3591
3592         kvm_s390_vcpu_block_all(kvm);
3593         kvm_for_each_vcpu(i, vcpu, kvm) {
3594                 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
3595                 vcpu->arch.sie_block->epdx  = kvm->arch.epdx;
3596         }
3597
3598         kvm_s390_vcpu_unblock_all(kvm);
3599         preempt_enable();
3600         mutex_unlock(&kvm->lock);
3601 }
3602
3603 /**
3604  * kvm_arch_fault_in_page - fault-in guest page if necessary
3605  * @vcpu: The corresponding virtual cpu
3606  * @gpa: Guest physical address
3607  * @writable: Whether the page should be writable or not
3608  *
3609  * Make sure that a guest page has been faulted-in on the host.
3610  *
3611  * Return: Zero on success, negative error code otherwise.
3612  */
3613 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
3614 {
3615         return gmap_fault(vcpu->arch.gmap, gpa,
3616                           writable ? FAULT_FLAG_WRITE : 0);
3617 }
3618
3619 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
3620                                       unsigned long token)
3621 {
3622         struct kvm_s390_interrupt inti;
3623         struct kvm_s390_irq irq;
3624
3625         if (start_token) {
3626                 irq.u.ext.ext_params2 = token;
3627                 irq.type = KVM_S390_INT_PFAULT_INIT;
3628                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3629         } else {
3630                 inti.type = KVM_S390_INT_PFAULT_DONE;
3631                 inti.parm64 = token;
3632                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
3633         }
3634 }
3635
3636 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
3637                                      struct kvm_async_pf *work)
3638 {
3639         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
3640         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
3641 }
3642
3643 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
3644                                  struct kvm_async_pf *work)
3645 {
3646         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
3647         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
3648 }
3649
3650 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
3651                                struct kvm_async_pf *work)
3652 {
3653         /* s390 will always inject the page directly */
3654 }
3655
3656 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
3657 {
3658         /*
3659          * s390 will always inject the page directly,
3660          * but we still want check_async_completion to cleanup
3661          */
3662         return true;
3663 }
3664
3665 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
3666 {
3667         hva_t hva;
3668         struct kvm_arch_async_pf arch;
3669         int rc;
3670
3671         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3672                 return 0;
3673         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
3674             vcpu->arch.pfault_compare)
3675                 return 0;
3676         if (psw_extint_disabled(vcpu))
3677                 return 0;
3678         if (kvm_s390_vcpu_has_irq(vcpu, 0))
3679                 return 0;
3680         if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
3681                 return 0;
3682         if (!vcpu->arch.gmap->pfault_enabled)
3683                 return 0;
3684
3685         hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
3686         hva += current->thread.gmap_addr & ~PAGE_MASK;
3687         if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3688                 return 0;
3689
3690         rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
3691         return rc;
3692 }
3693
3694 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
3695 {
3696         int rc, cpuflags;
3697
3698         /*
3699          * On s390 notifications for arriving pages will be delivered directly
3700          * to the guest but the house keeping for completed pfaults is
3701          * handled outside the worker.
3702          */
3703         kvm_check_async_pf_completion(vcpu);
3704
3705         vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
3706         vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
3707
3708         if (need_resched())
3709                 schedule();
3710
3711         if (test_cpu_flag(CIF_MCCK_PENDING))
3712                 s390_handle_mcck();
3713
3714         if (!kvm_is_ucontrol(vcpu->kvm)) {
3715                 rc = kvm_s390_deliver_pending_interrupts(vcpu);
3716                 if (rc)
3717                         return rc;
3718         }
3719
3720         rc = kvm_s390_handle_requests(vcpu);
3721         if (rc)
3722                 return rc;
3723
3724         if (guestdbg_enabled(vcpu)) {
3725                 kvm_s390_backup_guest_per_regs(vcpu);
3726                 kvm_s390_patch_guest_per_regs(vcpu);
3727         }
3728
3729         clear_bit(kvm_vcpu_get_idx(vcpu), vcpu->kvm->arch.gisa_int.kicked_mask);
3730
3731         vcpu->arch.sie_block->icptcode = 0;
3732         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
3733         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
3734         trace_kvm_s390_sie_enter(vcpu, cpuflags);
3735
3736         return 0;
3737 }
3738
3739 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
3740 {
3741         struct kvm_s390_pgm_info pgm_info = {
3742                 .code = PGM_ADDRESSING,
3743         };
3744         u8 opcode, ilen;
3745         int rc;
3746
3747         VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
3748         trace_kvm_s390_sie_fault(vcpu);
3749
3750         /*
3751          * We want to inject an addressing exception, which is defined as a
3752          * suppressing or terminating exception. However, since we came here
3753          * by a DAT access exception, the PSW still points to the faulting
3754          * instruction since DAT exceptions are nullifying. So we've got
3755          * to look up the current opcode to get the length of the instruction
3756          * to be able to forward the PSW.
3757          */
3758         rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
3759         ilen = insn_length(opcode);
3760         if (rc < 0) {
3761                 return rc;
3762         } else if (rc) {
3763                 /* Instruction-Fetching Exceptions - we can't detect the ilen.
3764                  * Forward by arbitrary ilc, injection will take care of
3765                  * nullification if necessary.
3766                  */
3767                 pgm_info = vcpu->arch.pgm;
3768                 ilen = 4;
3769         }
3770         pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
3771         kvm_s390_forward_psw(vcpu, ilen);
3772         return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
3773 }
3774
3775 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
3776 {
3777         struct mcck_volatile_info *mcck_info;
3778         struct sie_page *sie_page;
3779
3780         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
3781                    vcpu->arch.sie_block->icptcode);
3782         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
3783
3784         if (guestdbg_enabled(vcpu))
3785                 kvm_s390_restore_guest_per_regs(vcpu);
3786
3787         vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
3788         vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
3789
3790         if (exit_reason == -EINTR) {
3791                 VCPU_EVENT(vcpu, 3, "%s", "machine check");
3792                 sie_page = container_of(vcpu->arch.sie_block,
3793                                         struct sie_page, sie_block);
3794                 mcck_info = &sie_page->mcck_info;
3795                 kvm_s390_reinject_machine_check(vcpu, mcck_info);
3796                 return 0;
3797         }
3798
3799         if (vcpu->arch.sie_block->icptcode > 0) {
3800                 int rc = kvm_handle_sie_intercept(vcpu);
3801
3802                 if (rc != -EOPNOTSUPP)
3803                         return rc;
3804                 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
3805                 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
3806                 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
3807                 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
3808                 return -EREMOTE;
3809         } else if (exit_reason != -EFAULT) {
3810                 vcpu->stat.exit_null++;
3811                 return 0;
3812         } else if (kvm_is_ucontrol(vcpu->kvm)) {
3813                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
3814                 vcpu->run->s390_ucontrol.trans_exc_code =
3815                                                 current->thread.gmap_addr;
3816                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
3817                 return -EREMOTE;
3818         } else if (current->thread.gmap_pfault) {
3819                 trace_kvm_s390_major_guest_pfault(vcpu);
3820                 current->thread.gmap_pfault = 0;
3821                 if (kvm_arch_setup_async_pf(vcpu))
3822                         return 0;
3823                 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
3824         }
3825         return vcpu_post_run_fault_in_sie(vcpu);
3826 }
3827
3828 static int __vcpu_run(struct kvm_vcpu *vcpu)
3829 {
3830         int rc, exit_reason;
3831
3832         /*
3833          * We try to hold kvm->srcu during most of vcpu_run (except when run-
3834          * ning the guest), so that memslots (and other stuff) are protected
3835          */
3836         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3837
3838         do {
3839                 rc = vcpu_pre_run(vcpu);
3840                 if (rc)
3841                         break;
3842
3843                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
3844                 /*
3845                  * As PF_VCPU will be used in fault handler, between
3846                  * guest_enter and guest_exit should be no uaccess.
3847                  */
3848                 local_irq_disable();
3849                 guest_enter_irqoff();
3850                 __disable_cpu_timer_accounting(vcpu);
3851                 local_irq_enable();
3852                 exit_reason = sie64a(vcpu->arch.sie_block,
3853                                      vcpu->run->s.regs.gprs);
3854                 local_irq_disable();
3855                 __enable_cpu_timer_accounting(vcpu);
3856                 guest_exit_irqoff();
3857                 local_irq_enable();
3858                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3859
3860                 rc = vcpu_post_run(vcpu, exit_reason);
3861         } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3862
3863         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
3864         return rc;
3865 }
3866
3867 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3868 {
3869         struct runtime_instr_cb *riccb;
3870         struct gs_cb *gscb;
3871
3872         riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
3873         gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
3874         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
3875         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
3876         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
3877                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
3878         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
3879                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
3880                 /* some control register changes require a tlb flush */
3881                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3882         }
3883         if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
3884                 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
3885                 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
3886                 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
3887                 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
3888                 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
3889         }
3890         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
3891                 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
3892                 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
3893                 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
3894                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3895                         kvm_clear_async_pf_completion_queue(vcpu);
3896         }
3897         /*
3898          * If userspace sets the riccb (e.g. after migration) to a valid state,
3899          * we should enable RI here instead of doing the lazy enablement.
3900          */
3901         if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
3902             test_kvm_facility(vcpu->kvm, 64) &&
3903             riccb->v &&
3904             !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
3905                 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
3906                 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
3907         }
3908         /*
3909          * If userspace sets the gscb (e.g. after migration) to non-zero,
3910          * we should enable GS here instead of doing the lazy enablement.
3911          */
3912         if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
3913             test_kvm_facility(vcpu->kvm, 133) &&
3914             gscb->gssm &&
3915             !vcpu->arch.gs_enabled) {
3916                 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
3917                 vcpu->arch.sie_block->ecb |= ECB_GS;
3918                 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3919                 vcpu->arch.gs_enabled = 1;
3920         }
3921         if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) &&
3922             test_kvm_facility(vcpu->kvm, 82)) {
3923                 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
3924                 vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
3925         }
3926         save_access_regs(vcpu->arch.host_acrs);
3927         restore_access_regs(vcpu->run->s.regs.acrs);
3928         /* save host (userspace) fprs/vrs */
3929         save_fpu_regs();
3930         vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
3931         vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
3932         if (MACHINE_HAS_VX)
3933                 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
3934         else
3935                 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
3936         current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
3937         if (test_fp_ctl(current->thread.fpu.fpc))
3938                 /* User space provided an invalid FPC, let's clear it */
3939                 current->thread.fpu.fpc = 0;
3940         if (MACHINE_HAS_GS) {
3941                 preempt_disable();
3942                 __ctl_set_bit(2, 4);
3943                 if (current->thread.gs_cb) {
3944                         vcpu->arch.host_gscb = current->thread.gs_cb;
3945                         save_gs_cb(vcpu->arch.host_gscb);
3946                 }
3947                 if (vcpu->arch.gs_enabled) {
3948                         current->thread.gs_cb = (struct gs_cb *)
3949                                                 &vcpu->run->s.regs.gscb;
3950                         restore_gs_cb(current->thread.gs_cb);
3951                 }
3952                 preempt_enable();
3953         }
3954         /* SIE will load etoken directly from SDNX and therefore kvm_run */
3955
3956         kvm_run->kvm_dirty_regs = 0;
3957 }
3958
3959 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3960 {
3961         kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
3962         kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
3963         kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
3964         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
3965         kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
3966         kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
3967         kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
3968         kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
3969         kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
3970         kvm_run->s.regs.pft = vcpu->arch.pfault_token;
3971         kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
3972         kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
3973         kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
3974         save_access_regs(vcpu->run->s.regs.acrs);
3975         restore_access_regs(vcpu->arch.host_acrs);
3976         /* Save guest register state */
3977         save_fpu_regs();
3978         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3979         /* Restore will be done lazily at return */
3980         current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
3981         current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
3982         if (MACHINE_HAS_GS) {
3983                 preempt_disable();
3984                 __ctl_set_bit(2, 4);
3985                 if (vcpu->arch.gs_enabled)
3986                         save_gs_cb(current->thread.gs_cb);
3987                 current->thread.gs_cb = vcpu->arch.host_gscb;
3988                 restore_gs_cb(vcpu->arch.host_gscb);
3989                 if (!vcpu->arch.host_gscb)
3990                         __ctl_clear_bit(2, 4);
3991                 vcpu->arch.host_gscb = NULL;
3992                 preempt_enable();
3993         }
3994         /* SIE will save etoken directly into SDNX and therefore kvm_run */
3995 }
3996
3997 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3998 {
3999         int rc;
4000
4001         if (kvm_run->immediate_exit)
4002                 return -EINTR;
4003
4004         if (kvm_run->kvm_valid_regs & ~KVM_SYNC_S390_VALID_FIELDS ||
4005             kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS)
4006                 return -EINVAL;
4007
4008         vcpu_load(vcpu);
4009
4010         if (guestdbg_exit_pending(vcpu)) {
4011                 kvm_s390_prepare_debug_exit(vcpu);
4012                 rc = 0;
4013                 goto out;
4014         }
4015
4016         kvm_sigset_activate(vcpu);
4017
4018         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
4019                 kvm_s390_vcpu_start(vcpu);
4020         } else if (is_vcpu_stopped(vcpu)) {
4021                 pr_err_ratelimited("can't run stopped vcpu %d\n",
4022                                    vcpu->vcpu_id);
4023                 rc = -EINVAL;
4024                 goto out;
4025         }
4026
4027         sync_regs(vcpu, kvm_run);
4028         enable_cpu_timer_accounting(vcpu);
4029
4030         might_fault();
4031         rc = __vcpu_run(vcpu);
4032
4033         if (signal_pending(current) && !rc) {
4034                 kvm_run->exit_reason = KVM_EXIT_INTR;
4035                 rc = -EINTR;
4036         }
4037
4038         if (guestdbg_exit_pending(vcpu) && !rc)  {
4039                 kvm_s390_prepare_debug_exit(vcpu);
4040                 rc = 0;
4041         }
4042
4043         if (rc == -EREMOTE) {
4044                 /* userspace support is needed, kvm_run has been prepared */
4045                 rc = 0;
4046         }
4047
4048         disable_cpu_timer_accounting(vcpu);
4049         store_regs(vcpu, kvm_run);
4050
4051         kvm_sigset_deactivate(vcpu);
4052
4053         vcpu->stat.exit_userspace++;
4054 out:
4055         vcpu_put(vcpu);
4056         return rc;
4057 }
4058
4059 /*
4060  * store status at address
4061  * we use have two special cases:
4062  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
4063  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
4064  */
4065 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
4066 {
4067         unsigned char archmode = 1;
4068         freg_t fprs[NUM_FPRS];
4069         unsigned int px;
4070         u64 clkcomp, cputm;
4071         int rc;
4072
4073         px = kvm_s390_get_prefix(vcpu);
4074         if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
4075                 if (write_guest_abs(vcpu, 163, &archmode, 1))
4076                         return -EFAULT;
4077                 gpa = 0;
4078         } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
4079                 if (write_guest_real(vcpu, 163, &archmode, 1))
4080                         return -EFAULT;
4081                 gpa = px;
4082         } else
4083                 gpa -= __LC_FPREGS_SAVE_AREA;
4084
4085         /* manually convert vector registers if necessary */
4086         if (MACHINE_HAS_VX) {
4087                 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
4088                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
4089                                      fprs, 128);
4090         } else {
4091                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
4092                                      vcpu->run->s.regs.fprs, 128);
4093         }
4094         rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
4095                               vcpu->run->s.regs.gprs, 128);
4096         rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
4097                               &vcpu->arch.sie_block->gpsw, 16);
4098         rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
4099                               &px, 4);
4100         rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
4101                               &vcpu->run->s.regs.fpc, 4);
4102         rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
4103                               &vcpu->arch.sie_block->todpr, 4);
4104         cputm = kvm_s390_get_cpu_timer(vcpu);
4105         rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4106                               &cputm, 8);
4107         clkcomp = vcpu->arch.sie_block->ckc >> 8;
4108         rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
4109                               &clkcomp, 8);
4110         rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
4111                               &vcpu->run->s.regs.acrs, 64);
4112         rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
4113                               &vcpu->arch.sie_block->gcr, 128);
4114         return rc ? -EFAULT : 0;
4115 }
4116
4117 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
4118 {
4119         /*
4120          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
4121          * switch in the run ioctl. Let's update our copies before we save
4122          * it into the save area
4123          */
4124         save_fpu_regs();
4125         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
4126         save_access_regs(vcpu->run->s.regs.acrs);
4127
4128         return kvm_s390_store_status_unloaded(vcpu, addr);
4129 }
4130
4131 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
4132 {
4133         kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
4134         kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
4135 }
4136
4137 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
4138 {
4139         unsigned int i;
4140         struct kvm_vcpu *vcpu;
4141
4142         kvm_for_each_vcpu(i, vcpu, kvm) {
4143                 __disable_ibs_on_vcpu(vcpu);
4144         }
4145 }
4146
4147 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
4148 {
4149         if (!sclp.has_ibs)
4150                 return;
4151         kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
4152         kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
4153 }
4154
4155 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
4156 {
4157         int i, online_vcpus, started_vcpus = 0;
4158
4159         if (!is_vcpu_stopped(vcpu))
4160                 return;
4161
4162         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
4163         /* Only one cpu at a time may enter/leave the STOPPED state. */
4164         spin_lock(&vcpu->kvm->arch.start_stop_lock);
4165         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
4166
4167         for (i = 0; i < online_vcpus; i++) {
4168                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
4169                         started_vcpus++;
4170         }
4171
4172         if (started_vcpus == 0) {
4173                 /* we're the only active VCPU -> speed it up */
4174                 __enable_ibs_on_vcpu(vcpu);
4175         } else if (started_vcpus == 1) {
4176                 /*
4177                  * As we are starting a second VCPU, we have to disable
4178                  * the IBS facility on all VCPUs to remove potentially
4179                  * oustanding ENABLE requests.
4180                  */
4181                 __disable_ibs_on_all_vcpus(vcpu->kvm);
4182         }
4183
4184         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
4185         /*
4186          * Another VCPU might have used IBS while we were offline.
4187          * Let's play safe and flush the VCPU at startup.
4188          */
4189         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
4190         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4191         return;
4192 }
4193
4194 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
4195 {
4196         int i, online_vcpus, started_vcpus = 0;
4197         struct kvm_vcpu *started_vcpu = NULL;
4198
4199         if (is_vcpu_stopped(vcpu))
4200                 return;
4201
4202         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
4203         /* Only one cpu at a time may enter/leave the STOPPED state. */
4204         spin_lock(&vcpu->kvm->arch.start_stop_lock);
4205         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
4206
4207         /*
4208          * Set the VCPU to STOPPED and THEN clear the interrupt flag,
4209          * now that the SIGP STOP and SIGP STOP AND STORE STATUS orders
4210          * have been fully processed. This will ensure that the VCPU
4211          * is kept BUSY if another VCPU is inquiring with SIGP SENSE.
4212          */
4213         kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
4214         kvm_s390_clear_stop_irq(vcpu);
4215
4216         __disable_ibs_on_vcpu(vcpu);
4217
4218         for (i = 0; i < online_vcpus; i++) {
4219                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
4220                         started_vcpus++;
4221                         started_vcpu = vcpu->kvm->vcpus[i];
4222                 }
4223         }
4224
4225         if (started_vcpus == 1) {
4226                 /*
4227                  * As we only have one VCPU left, we want to enable the
4228                  * IBS facility for that VCPU to speed it up.
4229                  */
4230                 __enable_ibs_on_vcpu(started_vcpu);
4231         }
4232
4233         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4234         return;
4235 }
4236
4237 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
4238                                      struct kvm_enable_cap *cap)
4239 {
4240         int r;
4241
4242         if (cap->flags)
4243                 return -EINVAL;
4244
4245         switch (cap->cap) {
4246         case KVM_CAP_S390_CSS_SUPPORT:
4247                 if (!vcpu->kvm->arch.css_support) {
4248                         vcpu->kvm->arch.css_support = 1;
4249                         VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
4250                         trace_kvm_s390_enable_css(vcpu->kvm);
4251                 }
4252                 r = 0;
4253                 break;
4254         default:
4255                 r = -EINVAL;
4256                 break;
4257         }
4258         return r;
4259 }
4260
4261 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
4262                                   struct kvm_s390_mem_op *mop)
4263 {
4264         void __user *uaddr = (void __user *)mop->buf;
4265         void *tmpbuf = NULL;
4266         int r, srcu_idx;
4267         const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
4268                                     | KVM_S390_MEMOP_F_CHECK_ONLY;
4269
4270         if (mop->flags & ~supported_flags || mop->ar >= NUM_ACRS || !mop->size)
4271                 return -EINVAL;
4272
4273         if (mop->size > MEM_OP_MAX_SIZE)
4274                 return -E2BIG;
4275
4276         if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
4277                 tmpbuf = vmalloc(mop->size);
4278                 if (!tmpbuf)
4279                         return -ENOMEM;
4280         }
4281
4282         srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
4283
4284         switch (mop->op) {
4285         case KVM_S390_MEMOP_LOGICAL_READ:
4286                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
4287                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
4288                                             mop->size, GACC_FETCH);
4289                         break;
4290                 }
4291                 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
4292                 if (r == 0) {
4293                         if (copy_to_user(uaddr, tmpbuf, mop->size))
4294                                 r = -EFAULT;
4295                 }
4296                 break;
4297         case KVM_S390_MEMOP_LOGICAL_WRITE:
4298                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
4299                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
4300                                             mop->size, GACC_STORE);
4301                         break;
4302                 }
4303                 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
4304                         r = -EFAULT;
4305                         break;
4306                 }
4307                 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
4308                 break;
4309         default:
4310                 r = -EINVAL;
4311         }
4312
4313         srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
4314
4315         if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
4316                 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
4317
4318         vfree(tmpbuf);
4319         return r;
4320 }
4321
4322 long kvm_arch_vcpu_async_ioctl(struct file *filp,
4323                                unsigned int ioctl, unsigned long arg)
4324 {
4325         struct kvm_vcpu *vcpu = filp->private_data;
4326         void __user *argp = (void __user *)arg;
4327
4328         switch (ioctl) {
4329         case KVM_S390_IRQ: {
4330                 struct kvm_s390_irq s390irq;
4331
4332                 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
4333                         return -EFAULT;
4334                 return kvm_s390_inject_vcpu(vcpu, &s390irq);
4335         }
4336         case KVM_S390_INTERRUPT: {
4337                 struct kvm_s390_interrupt s390int;
4338                 struct kvm_s390_irq s390irq = {};
4339
4340                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
4341                         return -EFAULT;
4342                 if (s390int_to_s390irq(&s390int, &s390irq))
4343                         return -EINVAL;
4344                 return kvm_s390_inject_vcpu(vcpu, &s390irq);
4345         }
4346         }
4347         return -ENOIOCTLCMD;
4348 }
4349
4350 long kvm_arch_vcpu_ioctl(struct file *filp,
4351                          unsigned int ioctl, unsigned long arg)
4352 {
4353         struct kvm_vcpu *vcpu = filp->private_data;
4354         void __user *argp = (void __user *)arg;
4355         int idx;
4356         long r;
4357
4358         vcpu_load(vcpu);
4359
4360         switch (ioctl) {
4361         case KVM_S390_STORE_STATUS:
4362                 idx = srcu_read_lock(&vcpu->kvm->srcu);
4363                 r = kvm_s390_store_status_unloaded(vcpu, arg);
4364                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
4365                 break;
4366         case KVM_S390_SET_INITIAL_PSW: {
4367                 psw_t psw;
4368
4369                 r = -EFAULT;
4370                 if (copy_from_user(&psw, argp, sizeof(psw)))
4371                         break;
4372                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
4373                 break;
4374         }
4375         case KVM_S390_INITIAL_RESET:
4376                 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
4377                 break;
4378         case KVM_SET_ONE_REG:
4379         case KVM_GET_ONE_REG: {
4380                 struct kvm_one_reg reg;
4381                 r = -EFAULT;
4382                 if (copy_from_user(&reg, argp, sizeof(reg)))
4383                         break;
4384                 if (ioctl == KVM_SET_ONE_REG)
4385                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
4386                 else
4387                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
4388                 break;
4389         }
4390 #ifdef CONFIG_KVM_S390_UCONTROL
4391         case KVM_S390_UCAS_MAP: {
4392                 struct kvm_s390_ucas_mapping ucasmap;
4393
4394                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4395                         r = -EFAULT;
4396                         break;
4397                 }
4398
4399                 if (!kvm_is_ucontrol(vcpu->kvm)) {
4400                         r = -EINVAL;
4401                         break;
4402                 }
4403
4404                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
4405                                      ucasmap.vcpu_addr, ucasmap.length);
4406                 break;
4407         }
4408         case KVM_S390_UCAS_UNMAP: {
4409                 struct kvm_s390_ucas_mapping ucasmap;
4410
4411                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4412                         r = -EFAULT;
4413                         break;
4414                 }
4415
4416                 if (!kvm_is_ucontrol(vcpu->kvm)) {
4417                         r = -EINVAL;
4418                         break;
4419                 }
4420
4421                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
4422                         ucasmap.length);
4423                 break;
4424         }
4425 #endif
4426         case KVM_S390_VCPU_FAULT: {
4427                 r = gmap_fault(vcpu->arch.gmap, arg, 0);
4428                 break;
4429         }
4430         case KVM_ENABLE_CAP:
4431         {
4432                 struct kvm_enable_cap cap;
4433                 r = -EFAULT;
4434                 if (copy_from_user(&cap, argp, sizeof(cap)))
4435                         break;
4436                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
4437                 break;
4438         }
4439         case KVM_S390_MEM_OP: {
4440                 struct kvm_s390_mem_op mem_op;
4441
4442                 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
4443                         r = kvm_s390_guest_mem_op(vcpu, &mem_op);
4444                 else
4445                         r = -EFAULT;
4446                 break;
4447         }
4448         case KVM_S390_SET_IRQ_STATE: {
4449                 struct kvm_s390_irq_state irq_state;
4450
4451                 r = -EFAULT;
4452                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
4453                         break;
4454                 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
4455                     irq_state.len == 0 ||
4456                     irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
4457                         r = -EINVAL;
4458                         break;
4459                 }
4460                 /* do not use irq_state.flags, it will break old QEMUs */
4461                 r = kvm_s390_set_irq_state(vcpu,
4462                                            (void __user *) irq_state.buf,
4463                                            irq_state.len);
4464                 break;
4465         }
4466         case KVM_S390_GET_IRQ_STATE: {
4467                 struct kvm_s390_irq_state irq_state;
4468
4469                 r = -EFAULT;
4470                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
4471                         break;
4472                 if (irq_state.len == 0) {
4473                         r = -EINVAL;
4474                         break;
4475                 }
4476                 /* do not use irq_state.flags, it will break old QEMUs */
4477                 r = kvm_s390_get_irq_state(vcpu,
4478                                            (__u8 __user *)  irq_state.buf,
4479                                            irq_state.len);
4480                 break;
4481         }
4482         default:
4483                 r = -ENOTTY;
4484         }
4485
4486         vcpu_put(vcpu);
4487         return r;
4488 }
4489
4490 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
4491 {
4492 #ifdef CONFIG_KVM_S390_UCONTROL
4493         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
4494                  && (kvm_is_ucontrol(vcpu->kvm))) {
4495                 vmf->page = virt_to_page(vcpu->arch.sie_block);
4496                 get_page(vmf->page);
4497                 return 0;
4498         }
4499 #endif
4500         return VM_FAULT_SIGBUS;
4501 }
4502
4503 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
4504                             unsigned long npages)
4505 {
4506         return 0;
4507 }
4508
4509 /* Section: memory related */
4510 int kvm_arch_prepare_memory_region(struct kvm *kvm,
4511                                    struct kvm_memory_slot *memslot,
4512                                    const struct kvm_userspace_memory_region *mem,
4513                                    enum kvm_mr_change change)
4514 {
4515         /* A few sanity checks. We can have memory slots which have to be
4516            located/ended at a segment boundary (1MB). The memory in userland is
4517            ok to be fragmented into various different vmas. It is okay to mmap()
4518            and munmap() stuff in this slot after doing this call at any time */
4519
4520         if (mem->userspace_addr & 0xffffful)
4521                 return -EINVAL;
4522
4523         if (mem->memory_size & 0xffffful)
4524                 return -EINVAL;
4525
4526         if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
4527                 return -EINVAL;
4528
4529         if (!kvm->arch.migration_mode)
4530                 return 0;
4531
4532         /*
4533          * Turn off migration mode when:
4534          * - userspace creates a new memslot with dirty logging off,
4535          * - userspace modifies an existing memslot (MOVE or FLAGS_ONLY) and
4536          *   dirty logging is turned off.
4537          * Migration mode expects dirty page logging being enabled to store
4538          * its dirty bitmap.
4539          */
4540         if (change != KVM_MR_DELETE &&
4541             !(mem->flags & KVM_MEM_LOG_DIRTY_PAGES))
4542                 WARN(kvm_s390_vm_stop_migration(kvm),
4543                      "Failed to stop migration mode");
4544
4545         return 0;
4546 }
4547
4548 void kvm_arch_commit_memory_region(struct kvm *kvm,
4549                                 const struct kvm_userspace_memory_region *mem,
4550                                 const struct kvm_memory_slot *old,
4551                                 const struct kvm_memory_slot *new,
4552                                 enum kvm_mr_change change)
4553 {
4554         int rc = 0;
4555
4556         switch (change) {
4557         case KVM_MR_DELETE:
4558                 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
4559                                         old->npages * PAGE_SIZE);
4560                 break;
4561         case KVM_MR_MOVE:
4562                 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
4563                                         old->npages * PAGE_SIZE);
4564                 if (rc)
4565                         break;
4566                 /* FALLTHROUGH */
4567         case KVM_MR_CREATE:
4568                 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
4569                                       mem->guest_phys_addr, mem->memory_size);
4570                 break;
4571         case KVM_MR_FLAGS_ONLY:
4572                 break;
4573         default:
4574                 WARN(1, "Unknown KVM MR CHANGE: %d\n", change);
4575         }
4576         if (rc)
4577                 pr_warn("failed to commit memory region\n");
4578         return;
4579 }
4580
4581 static inline unsigned long nonhyp_mask(int i)
4582 {
4583         unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
4584
4585         return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
4586 }
4587
4588 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
4589 {
4590         vcpu->valid_wakeup = false;
4591 }
4592
4593 static int __init kvm_s390_init(void)
4594 {
4595         int i;
4596
4597         if (!sclp.has_sief2) {
4598                 pr_info("SIE is not available\n");
4599                 return -ENODEV;
4600         }
4601
4602         if (nested && hpage) {
4603                 pr_info("A KVM host that supports nesting cannot back its KVM guests with huge pages\n");
4604                 return -EINVAL;
4605         }
4606
4607         for (i = 0; i < 16; i++)
4608                 kvm_s390_fac_base[i] |=
4609                         S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
4610
4611         return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
4612 }
4613
4614 static void __exit kvm_s390_exit(void)
4615 {
4616         kvm_exit();
4617 }
4618
4619 module_init(kvm_s390_init);
4620 module_exit(kvm_s390_exit);
4621
4622 /*
4623  * Enable autoloading of the kvm module.
4624  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
4625  * since x86 takes a different approach.
4626  */
4627 #include <linux/miscdevice.h>
4628 MODULE_ALIAS_MISCDEV(KVM_MINOR);
4629 MODULE_ALIAS("devname:kvm");