2 * hosting zSeries kernel virtual machines
4 * Copyright IBM Corp. 2008, 2009
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
17 #include <linux/compiler.h>
18 #include <linux/err.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/module.h>
25 #include <linux/random.h>
26 #include <linux/slab.h>
27 #include <linux/timer.h>
28 #include <linux/vmalloc.h>
29 #include <asm/asm-offsets.h>
30 #include <asm/lowcore.h>
31 #include <asm/pgtable.h>
33 #include <asm/switch_to.h>
39 #define KMSG_COMPONENT "kvm-s390"
41 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
43 #define CREATE_TRACE_POINTS
45 #include "trace-s390.h"
47 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
49 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
50 (KVM_MAX_VCPUS + LOCAL_IRQS))
52 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
54 struct kvm_stats_debugfs_item debugfs_entries[] = {
55 { "userspace_handled", VCPU_STAT(exit_userspace) },
56 { "exit_null", VCPU_STAT(exit_null) },
57 { "exit_validity", VCPU_STAT(exit_validity) },
58 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
59 { "exit_external_request", VCPU_STAT(exit_external_request) },
60 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
61 { "exit_instruction", VCPU_STAT(exit_instruction) },
62 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
63 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
64 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
65 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
66 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
67 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
68 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
69 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
70 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
71 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
72 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
73 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
74 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
75 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
76 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
77 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
78 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
79 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
80 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
81 { "instruction_spx", VCPU_STAT(instruction_spx) },
82 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
83 { "instruction_stap", VCPU_STAT(instruction_stap) },
84 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
85 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
86 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
87 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
88 { "instruction_essa", VCPU_STAT(instruction_essa) },
89 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
90 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
91 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
92 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
93 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
94 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
95 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
96 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
97 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
98 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
99 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
100 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
101 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
102 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
103 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
104 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
105 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
106 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
107 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
108 { "diagnose_10", VCPU_STAT(diagnose_10) },
109 { "diagnose_44", VCPU_STAT(diagnose_44) },
110 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
114 /* upper facilities limit for kvm */
115 unsigned long kvm_s390_fac_list_mask[] = {
116 0xffe6fffbfcfdfc40UL,
117 0x005e800000000000UL,
120 unsigned long kvm_s390_fac_list_mask_size(void)
122 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
123 return ARRAY_SIZE(kvm_s390_fac_list_mask);
126 static struct gmap_notifier gmap_notifier;
128 /* Section: not file related */
129 int kvm_arch_hardware_enable(void)
131 /* every s390 is virtualization enabled ;-) */
135 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
137 int kvm_arch_hardware_setup(void)
139 gmap_notifier.notifier_call = kvm_gmap_notifier;
140 gmap_register_ipte_notifier(&gmap_notifier);
144 void kvm_arch_hardware_unsetup(void)
146 gmap_unregister_ipte_notifier(&gmap_notifier);
149 int kvm_arch_init(void *opaque)
151 /* Register floating interrupt controller interface. */
152 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
155 /* Section: device related */
156 long kvm_arch_dev_ioctl(struct file *filp,
157 unsigned int ioctl, unsigned long arg)
159 if (ioctl == KVM_S390_ENABLE_SIE)
160 return s390_enable_sie();
164 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
169 case KVM_CAP_S390_PSW:
170 case KVM_CAP_S390_GMAP:
171 case KVM_CAP_SYNC_MMU:
172 #ifdef CONFIG_KVM_S390_UCONTROL
173 case KVM_CAP_S390_UCONTROL:
175 case KVM_CAP_ASYNC_PF:
176 case KVM_CAP_SYNC_REGS:
177 case KVM_CAP_ONE_REG:
178 case KVM_CAP_ENABLE_CAP:
179 case KVM_CAP_S390_CSS_SUPPORT:
180 case KVM_CAP_IOEVENTFD:
181 case KVM_CAP_DEVICE_CTRL:
182 case KVM_CAP_ENABLE_CAP_VM:
183 case KVM_CAP_S390_IRQCHIP:
184 case KVM_CAP_VM_ATTRIBUTES:
185 case KVM_CAP_MP_STATE:
186 case KVM_CAP_S390_INJECT_IRQ:
187 case KVM_CAP_S390_USER_SIGP:
188 case KVM_CAP_S390_USER_STSI:
189 case KVM_CAP_S390_SKEYS:
190 case KVM_CAP_S390_IRQ_STATE:
193 case KVM_CAP_S390_MEM_OP:
196 case KVM_CAP_NR_VCPUS:
197 case KVM_CAP_MAX_VCPUS:
200 case KVM_CAP_NR_MEMSLOTS:
201 r = KVM_USER_MEM_SLOTS;
203 case KVM_CAP_S390_COW:
204 r = MACHINE_HAS_ESOP;
206 case KVM_CAP_S390_VECTOR_REGISTERS:
215 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
216 struct kvm_memory_slot *memslot)
218 gfn_t cur_gfn, last_gfn;
219 unsigned long address;
220 struct gmap *gmap = kvm->arch.gmap;
222 down_read(&gmap->mm->mmap_sem);
223 /* Loop over all guest pages */
224 last_gfn = memslot->base_gfn + memslot->npages;
225 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
226 address = gfn_to_hva_memslot(memslot, cur_gfn);
228 if (gmap_test_and_clear_dirty(address, gmap))
229 mark_page_dirty(kvm, cur_gfn);
231 up_read(&gmap->mm->mmap_sem);
234 /* Section: vm related */
236 * Get (and clear) the dirty memory log for a memory slot.
238 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
239 struct kvm_dirty_log *log)
243 struct kvm_memslots *slots;
244 struct kvm_memory_slot *memslot;
247 mutex_lock(&kvm->slots_lock);
250 if (log->slot >= KVM_USER_MEM_SLOTS)
253 slots = kvm_memslots(kvm);
254 memslot = id_to_memslot(slots, log->slot);
256 if (!memslot->dirty_bitmap)
259 kvm_s390_sync_dirty_log(kvm, memslot);
260 r = kvm_get_dirty_log(kvm, log, &is_dirty);
264 /* Clear the dirty log */
266 n = kvm_dirty_bitmap_bytes(memslot);
267 memset(memslot->dirty_bitmap, 0, n);
271 mutex_unlock(&kvm->slots_lock);
275 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
283 case KVM_CAP_S390_IRQCHIP:
284 kvm->arch.use_irqchip = 1;
287 case KVM_CAP_S390_USER_SIGP:
288 kvm->arch.user_sigp = 1;
291 case KVM_CAP_S390_VECTOR_REGISTERS:
292 if (MACHINE_HAS_VX) {
293 set_kvm_facility(kvm->arch.model.fac->mask, 129);
294 set_kvm_facility(kvm->arch.model.fac->list, 129);
299 case KVM_CAP_S390_USER_STSI:
300 kvm->arch.user_stsi = 1;
310 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
314 switch (attr->attr) {
315 case KVM_S390_VM_MEM_LIMIT_SIZE:
317 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
327 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
331 switch (attr->attr) {
332 case KVM_S390_VM_MEM_ENABLE_CMMA:
334 mutex_lock(&kvm->lock);
335 if (atomic_read(&kvm->online_vcpus) == 0) {
336 kvm->arch.use_cmma = 1;
339 mutex_unlock(&kvm->lock);
341 case KVM_S390_VM_MEM_CLR_CMMA:
342 mutex_lock(&kvm->lock);
343 idx = srcu_read_lock(&kvm->srcu);
344 s390_reset_cmma(kvm->arch.gmap->mm);
345 srcu_read_unlock(&kvm->srcu, idx);
346 mutex_unlock(&kvm->lock);
349 case KVM_S390_VM_MEM_LIMIT_SIZE: {
350 unsigned long new_limit;
352 if (kvm_is_ucontrol(kvm))
355 if (get_user(new_limit, (u64 __user *)attr->addr))
358 if (new_limit > kvm->arch.gmap->asce_end)
362 mutex_lock(&kvm->lock);
363 if (atomic_read(&kvm->online_vcpus) == 0) {
364 /* gmap_alloc will round the limit up */
365 struct gmap *new = gmap_alloc(current->mm, new_limit);
370 gmap_free(kvm->arch.gmap);
372 kvm->arch.gmap = new;
376 mutex_unlock(&kvm->lock);
386 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
388 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
390 struct kvm_vcpu *vcpu;
393 if (!test_kvm_facility(kvm, 76))
396 mutex_lock(&kvm->lock);
397 switch (attr->attr) {
398 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
400 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
401 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
402 kvm->arch.crypto.aes_kw = 1;
404 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
406 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
407 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
408 kvm->arch.crypto.dea_kw = 1;
410 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
411 kvm->arch.crypto.aes_kw = 0;
412 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
413 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
415 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
416 kvm->arch.crypto.dea_kw = 0;
417 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
418 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
421 mutex_unlock(&kvm->lock);
425 kvm_for_each_vcpu(i, vcpu, kvm) {
426 kvm_s390_vcpu_crypto_setup(vcpu);
429 mutex_unlock(&kvm->lock);
433 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
437 if (copy_from_user(>od_high, (void __user *)attr->addr,
447 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
449 struct kvm_vcpu *cur_vcpu;
450 unsigned int vcpu_idx;
454 if (copy_from_user(>od, (void __user *)attr->addr, sizeof(gtod)))
457 r = store_tod_clock(&host_tod);
461 mutex_lock(&kvm->lock);
462 kvm->arch.epoch = gtod - host_tod;
463 kvm_s390_vcpu_block_all(kvm);
464 kvm_for_each_vcpu(vcpu_idx, cur_vcpu, kvm)
465 cur_vcpu->arch.sie_block->epoch = kvm->arch.epoch;
466 kvm_s390_vcpu_unblock_all(kvm);
467 mutex_unlock(&kvm->lock);
471 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
478 switch (attr->attr) {
479 case KVM_S390_VM_TOD_HIGH:
480 ret = kvm_s390_set_tod_high(kvm, attr);
482 case KVM_S390_VM_TOD_LOW:
483 ret = kvm_s390_set_tod_low(kvm, attr);
492 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
496 if (copy_to_user((void __user *)attr->addr, >od_high,
503 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
508 r = store_tod_clock(&host_tod);
512 gtod = host_tod + kvm->arch.epoch;
513 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod)))
519 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
526 switch (attr->attr) {
527 case KVM_S390_VM_TOD_HIGH:
528 ret = kvm_s390_get_tod_high(kvm, attr);
530 case KVM_S390_VM_TOD_LOW:
531 ret = kvm_s390_get_tod_low(kvm, attr);
540 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
542 struct kvm_s390_vm_cpu_processor *proc;
545 mutex_lock(&kvm->lock);
546 if (atomic_read(&kvm->online_vcpus)) {
550 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
555 if (!copy_from_user(proc, (void __user *)attr->addr,
557 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
558 sizeof(struct cpuid));
559 kvm->arch.model.ibc = proc->ibc;
560 memcpy(kvm->arch.model.fac->list, proc->fac_list,
561 S390_ARCH_FAC_LIST_SIZE_BYTE);
566 mutex_unlock(&kvm->lock);
570 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
574 switch (attr->attr) {
575 case KVM_S390_VM_CPU_PROCESSOR:
576 ret = kvm_s390_set_processor(kvm, attr);
582 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
584 struct kvm_s390_vm_cpu_processor *proc;
587 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
592 memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
593 proc->ibc = kvm->arch.model.ibc;
594 memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
595 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
602 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
604 struct kvm_s390_vm_cpu_machine *mach;
607 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
612 get_cpu_id((struct cpuid *) &mach->cpuid);
613 mach->ibc = sclp.ibc;
614 memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
615 S390_ARCH_FAC_LIST_SIZE_BYTE);
616 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
617 S390_ARCH_FAC_LIST_SIZE_BYTE);
618 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
625 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
629 switch (attr->attr) {
630 case KVM_S390_VM_CPU_PROCESSOR:
631 ret = kvm_s390_get_processor(kvm, attr);
633 case KVM_S390_VM_CPU_MACHINE:
634 ret = kvm_s390_get_machine(kvm, attr);
640 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
644 switch (attr->group) {
645 case KVM_S390_VM_MEM_CTRL:
646 ret = kvm_s390_set_mem_control(kvm, attr);
648 case KVM_S390_VM_TOD:
649 ret = kvm_s390_set_tod(kvm, attr);
651 case KVM_S390_VM_CPU_MODEL:
652 ret = kvm_s390_set_cpu_model(kvm, attr);
654 case KVM_S390_VM_CRYPTO:
655 ret = kvm_s390_vm_set_crypto(kvm, attr);
665 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
669 switch (attr->group) {
670 case KVM_S390_VM_MEM_CTRL:
671 ret = kvm_s390_get_mem_control(kvm, attr);
673 case KVM_S390_VM_TOD:
674 ret = kvm_s390_get_tod(kvm, attr);
676 case KVM_S390_VM_CPU_MODEL:
677 ret = kvm_s390_get_cpu_model(kvm, attr);
687 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
691 switch (attr->group) {
692 case KVM_S390_VM_MEM_CTRL:
693 switch (attr->attr) {
694 case KVM_S390_VM_MEM_ENABLE_CMMA:
695 case KVM_S390_VM_MEM_CLR_CMMA:
696 case KVM_S390_VM_MEM_LIMIT_SIZE:
704 case KVM_S390_VM_TOD:
705 switch (attr->attr) {
706 case KVM_S390_VM_TOD_LOW:
707 case KVM_S390_VM_TOD_HIGH:
715 case KVM_S390_VM_CPU_MODEL:
716 switch (attr->attr) {
717 case KVM_S390_VM_CPU_PROCESSOR:
718 case KVM_S390_VM_CPU_MACHINE:
726 case KVM_S390_VM_CRYPTO:
727 switch (attr->attr) {
728 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
729 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
730 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
731 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
747 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
751 unsigned long curkey;
754 if (args->flags != 0)
757 /* Is this guest using storage keys? */
758 if (!mm_use_skey(current->mm))
759 return KVM_S390_GET_SKEYS_NONE;
761 /* Enforce sane limit on memory allocation */
762 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
765 keys = kmalloc_array(args->count, sizeof(uint8_t),
766 GFP_KERNEL | __GFP_NOWARN);
768 keys = vmalloc(sizeof(uint8_t) * args->count);
772 for (i = 0; i < args->count; i++) {
773 hva = gfn_to_hva(kvm, args->start_gfn + i);
774 if (kvm_is_error_hva(hva)) {
779 curkey = get_guest_storage_key(current->mm, hva);
780 if (IS_ERR_VALUE(curkey)) {
787 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
788 sizeof(uint8_t) * args->count);
796 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
802 if (args->flags != 0)
805 /* Enforce sane limit on memory allocation */
806 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
809 keys = kmalloc_array(args->count, sizeof(uint8_t),
810 GFP_KERNEL | __GFP_NOWARN);
812 keys = vmalloc(sizeof(uint8_t) * args->count);
816 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
817 sizeof(uint8_t) * args->count);
823 /* Enable storage key handling for the guest */
826 for (i = 0; i < args->count; i++) {
827 hva = gfn_to_hva(kvm, args->start_gfn + i);
828 if (kvm_is_error_hva(hva)) {
833 /* Lowest order bit is reserved */
834 if (keys[i] & 0x01) {
839 r = set_guest_storage_key(current->mm, hva,
840 (unsigned long)keys[i], 0);
849 long kvm_arch_vm_ioctl(struct file *filp,
850 unsigned int ioctl, unsigned long arg)
852 struct kvm *kvm = filp->private_data;
853 void __user *argp = (void __user *)arg;
854 struct kvm_device_attr attr;
858 case KVM_S390_INTERRUPT: {
859 struct kvm_s390_interrupt s390int;
862 if (copy_from_user(&s390int, argp, sizeof(s390int)))
864 r = kvm_s390_inject_vm(kvm, &s390int);
867 case KVM_ENABLE_CAP: {
868 struct kvm_enable_cap cap;
870 if (copy_from_user(&cap, argp, sizeof(cap)))
872 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
875 case KVM_CREATE_IRQCHIP: {
876 struct kvm_irq_routing_entry routing;
879 if (kvm->arch.use_irqchip) {
880 /* Set up dummy routing. */
881 memset(&routing, 0, sizeof(routing));
882 kvm_set_irq_routing(kvm, &routing, 0, 0);
887 case KVM_SET_DEVICE_ATTR: {
889 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
891 r = kvm_s390_vm_set_attr(kvm, &attr);
894 case KVM_GET_DEVICE_ATTR: {
896 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
898 r = kvm_s390_vm_get_attr(kvm, &attr);
901 case KVM_HAS_DEVICE_ATTR: {
903 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
905 r = kvm_s390_vm_has_attr(kvm, &attr);
908 case KVM_S390_GET_SKEYS: {
909 struct kvm_s390_skeys args;
912 if (copy_from_user(&args, argp,
913 sizeof(struct kvm_s390_skeys)))
915 r = kvm_s390_get_skeys(kvm, &args);
918 case KVM_S390_SET_SKEYS: {
919 struct kvm_s390_skeys args;
922 if (copy_from_user(&args, argp,
923 sizeof(struct kvm_s390_skeys)))
925 r = kvm_s390_set_skeys(kvm, &args);
935 static int kvm_s390_query_ap_config(u8 *config)
937 u32 fcn_code = 0x04000000UL;
940 memset(config, 0, 128);
944 ".long 0xb2af0000\n" /* PQAP(QCI) */
950 : "r" (fcn_code), "r" (config)
951 : "cc", "0", "2", "memory"
957 static int kvm_s390_apxa_installed(void)
962 if (test_facility(2) && test_facility(12)) {
963 cc = kvm_s390_query_ap_config(config);
966 pr_err("PQAP(QCI) failed with cc=%d", cc);
968 return config[0] & 0x40;
974 static void kvm_s390_set_crycb_format(struct kvm *kvm)
976 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
978 if (kvm_s390_apxa_installed())
979 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
981 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
984 static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
987 cpu_id->version = 0xff;
990 static int kvm_s390_crypto_init(struct kvm *kvm)
992 if (!test_kvm_facility(kvm, 76))
995 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
996 GFP_KERNEL | GFP_DMA);
997 if (!kvm->arch.crypto.crycb)
1000 kvm_s390_set_crycb_format(kvm);
1002 /* Enable AES/DEA protected key functions by default */
1003 kvm->arch.crypto.aes_kw = 1;
1004 kvm->arch.crypto.dea_kw = 1;
1005 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1006 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1007 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1008 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1013 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1016 char debug_name[16];
1017 static unsigned long sca_offset;
1020 #ifdef CONFIG_KVM_S390_UCONTROL
1021 if (type & ~KVM_VM_S390_UCONTROL)
1023 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1030 rc = s390_enable_sie();
1036 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
1039 spin_lock(&kvm_lock);
1040 sca_offset = (sca_offset + 16) & 0x7f0;
1041 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
1042 spin_unlock(&kvm_lock);
1044 sprintf(debug_name, "kvm-%u", current->pid);
1046 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
1051 * The architectural maximum amount of facilities is 16 kbit. To store
1052 * this amount, 2 kbyte of memory is required. Thus we need a full
1053 * page to hold the guest facility list (arch.model.fac->list) and the
1054 * facility mask (arch.model.fac->mask). Its address size has to be
1055 * 31 bits and word aligned.
1057 kvm->arch.model.fac =
1058 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1059 if (!kvm->arch.model.fac)
1062 /* Populate the facility mask initially. */
1063 memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
1064 S390_ARCH_FAC_LIST_SIZE_BYTE);
1065 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1066 if (i < kvm_s390_fac_list_mask_size())
1067 kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
1069 kvm->arch.model.fac->mask[i] = 0UL;
1072 /* Populate the facility list initially. */
1073 memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
1074 S390_ARCH_FAC_LIST_SIZE_BYTE);
1076 kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
1077 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1079 if (kvm_s390_crypto_init(kvm) < 0)
1082 spin_lock_init(&kvm->arch.float_int.lock);
1083 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1084 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1085 init_waitqueue_head(&kvm->arch.ipte_wq);
1086 mutex_init(&kvm->arch.ipte_mutex);
1088 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1089 VM_EVENT(kvm, 3, "%s", "vm created");
1091 if (type & KVM_VM_S390_UCONTROL) {
1092 kvm->arch.gmap = NULL;
1094 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
1095 if (!kvm->arch.gmap)
1097 kvm->arch.gmap->private = kvm;
1098 kvm->arch.gmap->pfault_enabled = 0;
1101 kvm->arch.css_support = 0;
1102 kvm->arch.use_irqchip = 0;
1103 kvm->arch.epoch = 0;
1105 spin_lock_init(&kvm->arch.start_stop_lock);
1109 kfree(kvm->arch.crypto.crycb);
1110 free_page((unsigned long)kvm->arch.model.fac);
1111 debug_unregister(kvm->arch.dbf);
1112 free_page((unsigned long)(kvm->arch.sca));
1116 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1118 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1119 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1120 kvm_s390_clear_local_irqs(vcpu);
1121 kvm_clear_async_pf_completion_queue(vcpu);
1122 if (!kvm_is_ucontrol(vcpu->kvm)) {
1123 clear_bit(63 - vcpu->vcpu_id,
1124 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
1125 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
1126 (__u64) vcpu->arch.sie_block)
1127 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
1131 if (kvm_is_ucontrol(vcpu->kvm))
1132 gmap_free(vcpu->arch.gmap);
1134 if (kvm_s390_cmma_enabled(vcpu->kvm))
1135 kvm_s390_vcpu_unsetup_cmma(vcpu);
1136 free_page((unsigned long)(vcpu->arch.sie_block));
1138 kvm_vcpu_uninit(vcpu);
1139 kmem_cache_free(kvm_vcpu_cache, vcpu);
1142 static void kvm_free_vcpus(struct kvm *kvm)
1145 struct kvm_vcpu *vcpu;
1147 kvm_for_each_vcpu(i, vcpu, kvm)
1148 kvm_arch_vcpu_destroy(vcpu);
1150 mutex_lock(&kvm->lock);
1151 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1152 kvm->vcpus[i] = NULL;
1154 atomic_set(&kvm->online_vcpus, 0);
1155 mutex_unlock(&kvm->lock);
1158 void kvm_arch_destroy_vm(struct kvm *kvm)
1160 kvm_free_vcpus(kvm);
1161 free_page((unsigned long)kvm->arch.model.fac);
1162 free_page((unsigned long)(kvm->arch.sca));
1163 debug_unregister(kvm->arch.dbf);
1164 kfree(kvm->arch.crypto.crycb);
1165 if (!kvm_is_ucontrol(kvm))
1166 gmap_free(kvm->arch.gmap);
1167 kvm_s390_destroy_adapters(kvm);
1168 kvm_s390_clear_float_irqs(kvm);
1171 /* Section: vcpu related */
1172 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1174 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1175 if (!vcpu->arch.gmap)
1177 vcpu->arch.gmap->private = vcpu->kvm;
1182 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1184 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1185 kvm_clear_async_pf_completion_queue(vcpu);
1186 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1192 if (test_kvm_facility(vcpu->kvm, 129))
1193 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1195 if (kvm_is_ucontrol(vcpu->kvm))
1196 return __kvm_ucontrol_vcpu_init(vcpu);
1201 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1203 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1204 if (test_kvm_facility(vcpu->kvm, 129))
1205 save_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
1207 save_fp_regs(vcpu->arch.host_fpregs.fprs);
1208 save_access_regs(vcpu->arch.host_acrs);
1209 if (test_kvm_facility(vcpu->kvm, 129)) {
1210 restore_fp_ctl(&vcpu->run->s.regs.fpc);
1211 restore_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
1213 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1214 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1216 restore_access_regs(vcpu->run->s.regs.acrs);
1217 gmap_enable(vcpu->arch.gmap);
1218 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1221 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1223 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1224 gmap_disable(vcpu->arch.gmap);
1225 if (test_kvm_facility(vcpu->kvm, 129)) {
1226 save_fp_ctl(&vcpu->run->s.regs.fpc);
1227 save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
1229 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1230 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1232 save_access_regs(vcpu->run->s.regs.acrs);
1233 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1234 if (test_kvm_facility(vcpu->kvm, 129))
1235 restore_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
1237 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
1238 restore_access_regs(vcpu->arch.host_acrs);
1241 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1243 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1244 vcpu->arch.sie_block->gpsw.mask = 0UL;
1245 vcpu->arch.sie_block->gpsw.addr = 0UL;
1246 kvm_s390_set_prefix(vcpu, 0);
1247 vcpu->arch.sie_block->cputm = 0UL;
1248 vcpu->arch.sie_block->ckc = 0UL;
1249 vcpu->arch.sie_block->todpr = 0;
1250 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1251 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1252 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1253 vcpu->arch.guest_fpregs.fpc = 0;
1254 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
1255 vcpu->arch.sie_block->gbea = 1;
1256 vcpu->arch.sie_block->pp = 0;
1257 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1258 kvm_clear_async_pf_completion_queue(vcpu);
1259 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1260 kvm_s390_vcpu_stop(vcpu);
1261 kvm_s390_clear_local_irqs(vcpu);
1264 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1266 mutex_lock(&vcpu->kvm->lock);
1267 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1268 mutex_unlock(&vcpu->kvm->lock);
1269 if (!kvm_is_ucontrol(vcpu->kvm))
1270 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1273 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1275 if (!test_kvm_facility(vcpu->kvm, 76))
1278 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1280 if (vcpu->kvm->arch.crypto.aes_kw)
1281 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1282 if (vcpu->kvm->arch.crypto.dea_kw)
1283 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1285 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1288 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1290 free_page(vcpu->arch.sie_block->cbrlo);
1291 vcpu->arch.sie_block->cbrlo = 0;
1294 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1296 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1297 if (!vcpu->arch.sie_block->cbrlo)
1300 vcpu->arch.sie_block->ecb2 |= 0x80;
1301 vcpu->arch.sie_block->ecb2 &= ~0x08;
1305 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1307 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1309 vcpu->arch.cpu_id = model->cpu_id;
1310 vcpu->arch.sie_block->ibc = model->ibc;
1311 vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1314 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1318 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1322 if (test_kvm_facility(vcpu->kvm, 78))
1323 atomic_set_mask(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
1324 else if (test_kvm_facility(vcpu->kvm, 8))
1325 atomic_set_mask(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
1327 kvm_s390_vcpu_setup_model(vcpu);
1329 vcpu->arch.sie_block->ecb = 6;
1330 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1331 vcpu->arch.sie_block->ecb |= 0x10;
1333 vcpu->arch.sie_block->ecb2 = 8;
1334 vcpu->arch.sie_block->eca = 0xC1002000U;
1336 vcpu->arch.sie_block->eca |= 1;
1337 if (sclp.has_sigpif)
1338 vcpu->arch.sie_block->eca |= 0x10000000U;
1339 if (test_kvm_facility(vcpu->kvm, 129)) {
1340 vcpu->arch.sie_block->eca |= 0x00020000;
1341 vcpu->arch.sie_block->ecd |= 0x20000000;
1343 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1345 if (kvm_s390_cmma_enabled(vcpu->kvm)) {
1346 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1350 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1351 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1353 kvm_s390_vcpu_crypto_setup(vcpu);
1358 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1361 struct kvm_vcpu *vcpu;
1362 struct sie_page *sie_page;
1365 if (id >= KVM_MAX_VCPUS)
1370 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1374 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1378 vcpu->arch.sie_block = &sie_page->sie_block;
1379 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1380 vcpu->arch.host_vregs = &sie_page->vregs;
1382 vcpu->arch.sie_block->icpua = id;
1383 if (!kvm_is_ucontrol(kvm)) {
1384 if (!kvm->arch.sca) {
1388 if (!kvm->arch.sca->cpu[id].sda)
1389 kvm->arch.sca->cpu[id].sda =
1390 (__u64) vcpu->arch.sie_block;
1391 vcpu->arch.sie_block->scaoh =
1392 (__u32)(((__u64)kvm->arch.sca) >> 32);
1393 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
1394 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
1397 spin_lock_init(&vcpu->arch.local_int.lock);
1398 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1399 vcpu->arch.local_int.wq = &vcpu->wq;
1400 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1402 rc = kvm_vcpu_init(vcpu, kvm, id);
1404 goto out_free_sie_block;
1405 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
1406 vcpu->arch.sie_block);
1407 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1411 free_page((unsigned long)(vcpu->arch.sie_block));
1413 kmem_cache_free(kvm_vcpu_cache, vcpu);
1418 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1420 return kvm_s390_vcpu_has_irq(vcpu, 0);
1423 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
1425 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1429 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1431 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1434 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1436 atomic_set_mask(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1440 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1442 atomic_clear_mask(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1446 * Kick a guest cpu out of SIE and wait until SIE is not running.
1447 * If the CPU is not running (e.g. waiting as idle) the function will
1448 * return immediately. */
1449 void exit_sie(struct kvm_vcpu *vcpu)
1451 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1452 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1456 /* Kick a guest cpu out of SIE to process a request synchronously */
1457 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
1459 kvm_make_request(req, vcpu);
1460 kvm_s390_vcpu_request(vcpu);
1463 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1466 struct kvm *kvm = gmap->private;
1467 struct kvm_vcpu *vcpu;
1469 kvm_for_each_vcpu(i, vcpu, kvm) {
1470 /* match against both prefix pages */
1471 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
1472 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1473 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
1478 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1480 /* kvm common code refers to this, but never calls it */
1485 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1486 struct kvm_one_reg *reg)
1491 case KVM_REG_S390_TODPR:
1492 r = put_user(vcpu->arch.sie_block->todpr,
1493 (u32 __user *)reg->addr);
1495 case KVM_REG_S390_EPOCHDIFF:
1496 r = put_user(vcpu->arch.sie_block->epoch,
1497 (u64 __user *)reg->addr);
1499 case KVM_REG_S390_CPU_TIMER:
1500 r = put_user(vcpu->arch.sie_block->cputm,
1501 (u64 __user *)reg->addr);
1503 case KVM_REG_S390_CLOCK_COMP:
1504 r = put_user(vcpu->arch.sie_block->ckc,
1505 (u64 __user *)reg->addr);
1507 case KVM_REG_S390_PFTOKEN:
1508 r = put_user(vcpu->arch.pfault_token,
1509 (u64 __user *)reg->addr);
1511 case KVM_REG_S390_PFCOMPARE:
1512 r = put_user(vcpu->arch.pfault_compare,
1513 (u64 __user *)reg->addr);
1515 case KVM_REG_S390_PFSELECT:
1516 r = put_user(vcpu->arch.pfault_select,
1517 (u64 __user *)reg->addr);
1519 case KVM_REG_S390_PP:
1520 r = put_user(vcpu->arch.sie_block->pp,
1521 (u64 __user *)reg->addr);
1523 case KVM_REG_S390_GBEA:
1524 r = put_user(vcpu->arch.sie_block->gbea,
1525 (u64 __user *)reg->addr);
1534 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1535 struct kvm_one_reg *reg)
1540 case KVM_REG_S390_TODPR:
1541 r = get_user(vcpu->arch.sie_block->todpr,
1542 (u32 __user *)reg->addr);
1544 case KVM_REG_S390_EPOCHDIFF:
1545 r = get_user(vcpu->arch.sie_block->epoch,
1546 (u64 __user *)reg->addr);
1548 case KVM_REG_S390_CPU_TIMER:
1549 r = get_user(vcpu->arch.sie_block->cputm,
1550 (u64 __user *)reg->addr);
1552 case KVM_REG_S390_CLOCK_COMP:
1553 r = get_user(vcpu->arch.sie_block->ckc,
1554 (u64 __user *)reg->addr);
1556 case KVM_REG_S390_PFTOKEN:
1557 r = get_user(vcpu->arch.pfault_token,
1558 (u64 __user *)reg->addr);
1559 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1560 kvm_clear_async_pf_completion_queue(vcpu);
1562 case KVM_REG_S390_PFCOMPARE:
1563 r = get_user(vcpu->arch.pfault_compare,
1564 (u64 __user *)reg->addr);
1566 case KVM_REG_S390_PFSELECT:
1567 r = get_user(vcpu->arch.pfault_select,
1568 (u64 __user *)reg->addr);
1570 case KVM_REG_S390_PP:
1571 r = get_user(vcpu->arch.sie_block->pp,
1572 (u64 __user *)reg->addr);
1574 case KVM_REG_S390_GBEA:
1575 r = get_user(vcpu->arch.sie_block->gbea,
1576 (u64 __user *)reg->addr);
1585 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1587 kvm_s390_vcpu_initial_reset(vcpu);
1591 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1593 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs));
1597 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1599 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1603 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1604 struct kvm_sregs *sregs)
1606 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
1607 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
1608 restore_access_regs(vcpu->run->s.regs.acrs);
1612 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1613 struct kvm_sregs *sregs)
1615 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
1616 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
1620 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1622 if (test_fp_ctl(fpu->fpc))
1624 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
1625 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1626 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1627 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1631 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1633 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1634 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
1638 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1642 if (!is_vcpu_stopped(vcpu))
1645 vcpu->run->psw_mask = psw.mask;
1646 vcpu->run->psw_addr = psw.addr;
1651 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1652 struct kvm_translation *tr)
1654 return -EINVAL; /* not implemented yet */
1657 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1658 KVM_GUESTDBG_USE_HW_BP | \
1659 KVM_GUESTDBG_ENABLE)
1661 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1662 struct kvm_guest_debug *dbg)
1666 vcpu->guest_debug = 0;
1667 kvm_s390_clear_bp_data(vcpu);
1669 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
1672 if (dbg->control & KVM_GUESTDBG_ENABLE) {
1673 vcpu->guest_debug = dbg->control;
1674 /* enforce guest PER */
1675 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1677 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1678 rc = kvm_s390_import_bp_data(vcpu, dbg);
1680 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1681 vcpu->arch.guestdbg.last_bp = 0;
1685 vcpu->guest_debug = 0;
1686 kvm_s390_clear_bp_data(vcpu);
1687 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1693 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1694 struct kvm_mp_state *mp_state)
1696 /* CHECK_STOP and LOAD are not supported yet */
1697 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1698 KVM_MP_STATE_OPERATING;
1701 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1702 struct kvm_mp_state *mp_state)
1706 /* user space knows about this interface - let it control the state */
1707 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1709 switch (mp_state->mp_state) {
1710 case KVM_MP_STATE_STOPPED:
1711 kvm_s390_vcpu_stop(vcpu);
1713 case KVM_MP_STATE_OPERATING:
1714 kvm_s390_vcpu_start(vcpu);
1716 case KVM_MP_STATE_LOAD:
1717 case KVM_MP_STATE_CHECK_STOP:
1718 /* fall through - CHECK_STOP and LOAD are not supported yet */
1726 bool kvm_s390_cmma_enabled(struct kvm *kvm)
1728 if (!MACHINE_IS_LPAR)
1730 /* only enable for z10 and later */
1731 if (!MACHINE_HAS_EDAT1)
1733 if (!kvm->arch.use_cmma)
1738 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1740 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1743 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1745 if (!vcpu->requests)
1748 kvm_s390_vcpu_request_handled(vcpu);
1750 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1751 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1752 * This ensures that the ipte instruction for this request has
1753 * already finished. We might race against a second unmapper that
1754 * wants to set the blocking bit. Lets just retry the request loop.
1756 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1758 rc = gmap_ipte_notify(vcpu->arch.gmap,
1759 kvm_s390_get_prefix(vcpu),
1766 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1767 vcpu->arch.sie_block->ihcpu = 0xffff;
1771 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1772 if (!ibs_enabled(vcpu)) {
1773 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1774 atomic_set_mask(CPUSTAT_IBS,
1775 &vcpu->arch.sie_block->cpuflags);
1780 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1781 if (ibs_enabled(vcpu)) {
1782 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1783 atomic_clear_mask(CPUSTAT_IBS,
1784 &vcpu->arch.sie_block->cpuflags);
1789 /* nothing to do, just clear the request */
1790 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1796 * kvm_arch_fault_in_page - fault-in guest page if necessary
1797 * @vcpu: The corresponding virtual cpu
1798 * @gpa: Guest physical address
1799 * @writable: Whether the page should be writable or not
1801 * Make sure that a guest page has been faulted-in on the host.
1803 * Return: Zero on success, negative error code otherwise.
1805 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
1807 return gmap_fault(vcpu->arch.gmap, gpa,
1808 writable ? FAULT_FLAG_WRITE : 0);
1811 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1812 unsigned long token)
1814 struct kvm_s390_interrupt inti;
1815 struct kvm_s390_irq irq;
1818 irq.u.ext.ext_params2 = token;
1819 irq.type = KVM_S390_INT_PFAULT_INIT;
1820 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
1822 inti.type = KVM_S390_INT_PFAULT_DONE;
1823 inti.parm64 = token;
1824 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1828 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1829 struct kvm_async_pf *work)
1831 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1832 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1835 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1836 struct kvm_async_pf *work)
1838 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1839 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1842 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1843 struct kvm_async_pf *work)
1845 /* s390 will always inject the page directly */
1848 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1851 * s390 will always inject the page directly,
1852 * but we still want check_async_completion to cleanup
1857 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1860 struct kvm_arch_async_pf arch;
1863 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1865 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1866 vcpu->arch.pfault_compare)
1868 if (psw_extint_disabled(vcpu))
1870 if (kvm_s390_vcpu_has_irq(vcpu, 0))
1872 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1874 if (!vcpu->arch.gmap->pfault_enabled)
1877 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1878 hva += current->thread.gmap_addr & ~PAGE_MASK;
1879 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
1882 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1886 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
1891 * On s390 notifications for arriving pages will be delivered directly
1892 * to the guest but the house keeping for completed pfaults is
1893 * handled outside the worker.
1895 kvm_check_async_pf_completion(vcpu);
1897 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
1902 if (test_cpu_flag(CIF_MCCK_PENDING))
1905 if (!kvm_is_ucontrol(vcpu->kvm)) {
1906 rc = kvm_s390_deliver_pending_interrupts(vcpu);
1911 rc = kvm_s390_handle_requests(vcpu);
1915 if (guestdbg_enabled(vcpu)) {
1916 kvm_s390_backup_guest_per_regs(vcpu);
1917 kvm_s390_patch_guest_per_regs(vcpu);
1920 vcpu->arch.sie_block->icptcode = 0;
1921 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1922 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1923 trace_kvm_s390_sie_enter(vcpu, cpuflags);
1928 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
1930 psw_t *psw = &vcpu->arch.sie_block->gpsw;
1934 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1935 trace_kvm_s390_sie_fault(vcpu);
1938 * We want to inject an addressing exception, which is defined as a
1939 * suppressing or terminating exception. However, since we came here
1940 * by a DAT access exception, the PSW still points to the faulting
1941 * instruction since DAT exceptions are nullifying. So we've got
1942 * to look up the current opcode to get the length of the instruction
1943 * to be able to forward the PSW.
1945 rc = read_guest(vcpu, psw->addr, 0, &opcode, 1);
1947 return kvm_s390_inject_prog_cond(vcpu, rc);
1948 psw->addr = __rewind_psw(*psw, -insn_length(opcode));
1950 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1953 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1957 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1958 vcpu->arch.sie_block->icptcode);
1959 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1961 if (guestdbg_enabled(vcpu))
1962 kvm_s390_restore_guest_per_regs(vcpu);
1964 if (exit_reason >= 0) {
1966 } else if (kvm_is_ucontrol(vcpu->kvm)) {
1967 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1968 vcpu->run->s390_ucontrol.trans_exc_code =
1969 current->thread.gmap_addr;
1970 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1973 } else if (current->thread.gmap_pfault) {
1974 trace_kvm_s390_major_guest_pfault(vcpu);
1975 current->thread.gmap_pfault = 0;
1976 if (kvm_arch_setup_async_pf(vcpu)) {
1979 gpa_t gpa = current->thread.gmap_addr;
1980 rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1985 rc = vcpu_post_run_fault_in_sie(vcpu);
1987 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
1990 if (kvm_is_ucontrol(vcpu->kvm))
1991 /* Don't exit for host interrupts. */
1992 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
1994 rc = kvm_handle_sie_intercept(vcpu);
2000 static int __vcpu_run(struct kvm_vcpu *vcpu)
2002 int rc, exit_reason;
2005 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2006 * ning the guest), so that memslots (and other stuff) are protected
2008 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2011 rc = vcpu_pre_run(vcpu);
2015 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2017 * As PF_VCPU will be used in fault handler, between
2018 * guest_enter and guest_exit should be no uaccess.
2020 local_irq_disable();
2021 __kvm_guest_enter();
2023 exit_reason = sie64a(vcpu->arch.sie_block,
2024 vcpu->run->s.regs.gprs);
2025 local_irq_disable();
2028 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2030 rc = vcpu_post_run(vcpu, exit_reason);
2031 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2033 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2037 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2039 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2040 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2041 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2042 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2043 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2044 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2045 /* some control register changes require a tlb flush */
2046 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2048 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2049 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
2050 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2051 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2052 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2053 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2055 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2056 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2057 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2058 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2059 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2060 kvm_clear_async_pf_completion_queue(vcpu);
2062 kvm_run->kvm_dirty_regs = 0;
2065 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2067 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2068 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2069 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2070 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2071 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
2072 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2073 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2074 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2075 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2076 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2077 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2078 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2081 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2086 if (guestdbg_exit_pending(vcpu)) {
2087 kvm_s390_prepare_debug_exit(vcpu);
2091 if (vcpu->sigset_active)
2092 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2094 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2095 kvm_s390_vcpu_start(vcpu);
2096 } else if (is_vcpu_stopped(vcpu)) {
2097 pr_err_ratelimited("can't run stopped vcpu %d\n",
2102 sync_regs(vcpu, kvm_run);
2105 rc = __vcpu_run(vcpu);
2107 if (signal_pending(current) && !rc) {
2108 kvm_run->exit_reason = KVM_EXIT_INTR;
2112 if (guestdbg_exit_pending(vcpu) && !rc) {
2113 kvm_s390_prepare_debug_exit(vcpu);
2117 if (rc == -EOPNOTSUPP) {
2118 /* intercept cannot be handled in-kernel, prepare kvm-run */
2119 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
2120 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2121 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2122 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2126 if (rc == -EREMOTE) {
2127 /* intercept was handled, but userspace support is needed
2128 * kvm_run has been prepared by the handler */
2132 store_regs(vcpu, kvm_run);
2134 if (vcpu->sigset_active)
2135 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2137 vcpu->stat.exit_userspace++;
2142 * store status at address
2143 * we use have two special cases:
2144 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2145 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2147 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2149 unsigned char archmode = 1;
2154 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2155 if (write_guest_abs(vcpu, 163, &archmode, 1))
2157 gpa = SAVE_AREA_BASE;
2158 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2159 if (write_guest_real(vcpu, 163, &archmode, 1))
2161 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
2163 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
2164 vcpu->arch.guest_fpregs.fprs, 128);
2165 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
2166 vcpu->run->s.regs.gprs, 128);
2167 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
2168 &vcpu->arch.sie_block->gpsw, 16);
2169 px = kvm_s390_get_prefix(vcpu);
2170 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
2172 rc |= write_guest_abs(vcpu,
2173 gpa + offsetof(struct save_area, fp_ctrl_reg),
2174 &vcpu->arch.guest_fpregs.fpc, 4);
2175 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
2176 &vcpu->arch.sie_block->todpr, 4);
2177 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
2178 &vcpu->arch.sie_block->cputm, 8);
2179 clkcomp = vcpu->arch.sie_block->ckc >> 8;
2180 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
2182 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
2183 &vcpu->run->s.regs.acrs, 64);
2184 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
2185 &vcpu->arch.sie_block->gcr, 128);
2186 return rc ? -EFAULT : 0;
2189 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2192 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2193 * copying in vcpu load/put. Lets update our copies before we save
2194 * it into the save area
2196 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
2197 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
2198 save_access_regs(vcpu->run->s.regs.acrs);
2200 return kvm_s390_store_status_unloaded(vcpu, addr);
2204 * store additional status at address
2206 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2209 /* Only bits 0-53 are used for address formation */
2210 if (!(gpa & ~0x3ff))
2213 return write_guest_abs(vcpu, gpa & ~0x3ff,
2214 (void *)&vcpu->run->s.regs.vrs, 512);
2217 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2219 if (!test_kvm_facility(vcpu->kvm, 129))
2223 * The guest VXRS are in the host VXRs due to the lazy
2224 * copying in vcpu load/put. Let's update our copies before we save
2225 * it into the save area.
2227 save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
2229 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2232 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2234 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2235 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
2238 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2241 struct kvm_vcpu *vcpu;
2243 kvm_for_each_vcpu(i, vcpu, kvm) {
2244 __disable_ibs_on_vcpu(vcpu);
2248 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2250 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2251 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
2254 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2256 int i, online_vcpus, started_vcpus = 0;
2258 if (!is_vcpu_stopped(vcpu))
2261 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2262 /* Only one cpu at a time may enter/leave the STOPPED state. */
2263 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2264 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2266 for (i = 0; i < online_vcpus; i++) {
2267 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2271 if (started_vcpus == 0) {
2272 /* we're the only active VCPU -> speed it up */
2273 __enable_ibs_on_vcpu(vcpu);
2274 } else if (started_vcpus == 1) {
2276 * As we are starting a second VCPU, we have to disable
2277 * the IBS facility on all VCPUs to remove potentially
2278 * oustanding ENABLE requests.
2280 __disable_ibs_on_all_vcpus(vcpu->kvm);
2283 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2285 * Another VCPU might have used IBS while we were offline.
2286 * Let's play safe and flush the VCPU at startup.
2288 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2289 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2293 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2295 int i, online_vcpus, started_vcpus = 0;
2296 struct kvm_vcpu *started_vcpu = NULL;
2298 if (is_vcpu_stopped(vcpu))
2301 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2302 /* Only one cpu at a time may enter/leave the STOPPED state. */
2303 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2304 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2306 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2307 kvm_s390_clear_stop_irq(vcpu);
2309 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2310 __disable_ibs_on_vcpu(vcpu);
2312 for (i = 0; i < online_vcpus; i++) {
2313 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2315 started_vcpu = vcpu->kvm->vcpus[i];
2319 if (started_vcpus == 1) {
2321 * As we only have one VCPU left, we want to enable the
2322 * IBS facility for that VCPU to speed it up.
2324 __enable_ibs_on_vcpu(started_vcpu);
2327 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2331 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2332 struct kvm_enable_cap *cap)
2340 case KVM_CAP_S390_CSS_SUPPORT:
2341 if (!vcpu->kvm->arch.css_support) {
2342 vcpu->kvm->arch.css_support = 1;
2343 trace_kvm_s390_enable_css(vcpu->kvm);
2354 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2355 struct kvm_s390_mem_op *mop)
2357 void __user *uaddr = (void __user *)mop->buf;
2358 void *tmpbuf = NULL;
2360 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2361 | KVM_S390_MEMOP_F_CHECK_ONLY;
2363 if (mop->flags & ~supported_flags)
2366 if (mop->size > MEM_OP_MAX_SIZE)
2369 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2370 tmpbuf = vmalloc(mop->size);
2375 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2378 case KVM_S390_MEMOP_LOGICAL_READ:
2379 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2380 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, false);
2383 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2385 if (copy_to_user(uaddr, tmpbuf, mop->size))
2389 case KVM_S390_MEMOP_LOGICAL_WRITE:
2390 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2391 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, true);
2394 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2398 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2404 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2406 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2407 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2413 long kvm_arch_vcpu_ioctl(struct file *filp,
2414 unsigned int ioctl, unsigned long arg)
2416 struct kvm_vcpu *vcpu = filp->private_data;
2417 void __user *argp = (void __user *)arg;
2422 case KVM_S390_IRQ: {
2423 struct kvm_s390_irq s390irq;
2426 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2428 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2431 case KVM_S390_INTERRUPT: {
2432 struct kvm_s390_interrupt s390int;
2433 struct kvm_s390_irq s390irq;
2436 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2438 if (s390int_to_s390irq(&s390int, &s390irq))
2440 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2443 case KVM_S390_STORE_STATUS:
2444 idx = srcu_read_lock(&vcpu->kvm->srcu);
2445 r = kvm_s390_vcpu_store_status(vcpu, arg);
2446 srcu_read_unlock(&vcpu->kvm->srcu, idx);
2448 case KVM_S390_SET_INITIAL_PSW: {
2452 if (copy_from_user(&psw, argp, sizeof(psw)))
2454 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2457 case KVM_S390_INITIAL_RESET:
2458 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2460 case KVM_SET_ONE_REG:
2461 case KVM_GET_ONE_REG: {
2462 struct kvm_one_reg reg;
2464 if (copy_from_user(®, argp, sizeof(reg)))
2466 if (ioctl == KVM_SET_ONE_REG)
2467 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®);
2469 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®);
2472 #ifdef CONFIG_KVM_S390_UCONTROL
2473 case KVM_S390_UCAS_MAP: {
2474 struct kvm_s390_ucas_mapping ucasmap;
2476 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2481 if (!kvm_is_ucontrol(vcpu->kvm)) {
2486 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2487 ucasmap.vcpu_addr, ucasmap.length);
2490 case KVM_S390_UCAS_UNMAP: {
2491 struct kvm_s390_ucas_mapping ucasmap;
2493 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2498 if (!kvm_is_ucontrol(vcpu->kvm)) {
2503 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2508 case KVM_S390_VCPU_FAULT: {
2509 r = gmap_fault(vcpu->arch.gmap, arg, 0);
2512 case KVM_ENABLE_CAP:
2514 struct kvm_enable_cap cap;
2516 if (copy_from_user(&cap, argp, sizeof(cap)))
2518 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2521 case KVM_S390_MEM_OP: {
2522 struct kvm_s390_mem_op mem_op;
2524 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2525 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2530 case KVM_S390_SET_IRQ_STATE: {
2531 struct kvm_s390_irq_state irq_state;
2534 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2536 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
2537 irq_state.len == 0 ||
2538 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
2542 r = kvm_s390_set_irq_state(vcpu,
2543 (void __user *) irq_state.buf,
2547 case KVM_S390_GET_IRQ_STATE: {
2548 struct kvm_s390_irq_state irq_state;
2551 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2553 if (irq_state.len == 0) {
2557 r = kvm_s390_get_irq_state(vcpu,
2558 (__u8 __user *) irq_state.buf,
2568 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2570 #ifdef CONFIG_KVM_S390_UCONTROL
2571 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2572 && (kvm_is_ucontrol(vcpu->kvm))) {
2573 vmf->page = virt_to_page(vcpu->arch.sie_block);
2574 get_page(vmf->page);
2578 return VM_FAULT_SIGBUS;
2581 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2582 unsigned long npages)
2587 /* Section: memory related */
2588 int kvm_arch_prepare_memory_region(struct kvm *kvm,
2589 struct kvm_memory_slot *memslot,
2590 const struct kvm_userspace_memory_region *mem,
2591 enum kvm_mr_change change)
2593 /* A few sanity checks. We can have memory slots which have to be
2594 located/ended at a segment boundary (1MB). The memory in userland is
2595 ok to be fragmented into various different vmas. It is okay to mmap()
2596 and munmap() stuff in this slot after doing this call at any time */
2598 if (mem->userspace_addr & 0xffffful)
2601 if (mem->memory_size & 0xffffful)
2607 void kvm_arch_commit_memory_region(struct kvm *kvm,
2608 const struct kvm_userspace_memory_region *mem,
2609 const struct kvm_memory_slot *old,
2610 const struct kvm_memory_slot *new,
2611 enum kvm_mr_change change)
2615 /* If the basics of the memslot do not change, we do not want
2616 * to update the gmap. Every update causes several unnecessary
2617 * segment translation exceptions. This is usually handled just
2618 * fine by the normal fault handler + gmap, but it will also
2619 * cause faults on the prefix page of running guest CPUs.
2621 if (old->userspace_addr == mem->userspace_addr &&
2622 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2623 old->npages * PAGE_SIZE == mem->memory_size)
2626 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2627 mem->guest_phys_addr, mem->memory_size);
2629 pr_warn("failed to commit memory region\n");
2633 static int __init kvm_s390_init(void)
2635 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2638 static void __exit kvm_s390_exit(void)
2643 module_init(kvm_s390_init);
2644 module_exit(kvm_s390_exit);
2647 * Enable autoloading of the kvm module.
2648 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2649 * since x86 takes a different approach.
2651 #include <linux/miscdevice.h>
2652 MODULE_ALIAS_MISCDEV(KVM_MINOR);
2653 MODULE_ALIAS("devname:kvm");