2 * hosting zSeries kernel virtual machines
4 * Copyright IBM Corp. 2008, 2009
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
17 #include <linux/compiler.h>
18 #include <linux/err.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/module.h>
25 #include <linux/random.h>
26 #include <linux/slab.h>
27 #include <linux/timer.h>
28 #include <linux/vmalloc.h>
29 #include <asm/asm-offsets.h>
30 #include <asm/lowcore.h>
31 #include <asm/pgtable.h>
33 #include <asm/switch_to.h>
39 #define CREATE_TRACE_POINTS
41 #include "trace-s390.h"
43 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
45 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
46 (KVM_MAX_VCPUS + LOCAL_IRQS))
48 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
50 struct kvm_stats_debugfs_item debugfs_entries[] = {
51 { "userspace_handled", VCPU_STAT(exit_userspace) },
52 { "exit_null", VCPU_STAT(exit_null) },
53 { "exit_validity", VCPU_STAT(exit_validity) },
54 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
55 { "exit_external_request", VCPU_STAT(exit_external_request) },
56 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
57 { "exit_instruction", VCPU_STAT(exit_instruction) },
58 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
59 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
60 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
61 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
62 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
63 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
64 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
65 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
66 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
67 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
68 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
69 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
70 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
71 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
72 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
73 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
74 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
75 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
76 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
77 { "instruction_spx", VCPU_STAT(instruction_spx) },
78 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
79 { "instruction_stap", VCPU_STAT(instruction_stap) },
80 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
81 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
82 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
83 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
84 { "instruction_essa", VCPU_STAT(instruction_essa) },
85 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
86 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
87 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
88 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
89 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
90 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
91 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
92 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
93 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
94 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
95 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
96 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
97 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
98 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
99 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
100 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
101 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
102 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
103 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
104 { "diagnose_10", VCPU_STAT(diagnose_10) },
105 { "diagnose_44", VCPU_STAT(diagnose_44) },
106 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
110 /* upper facilities limit for kvm */
111 unsigned long kvm_s390_fac_list_mask[] = {
112 0xffe6fffbfcfdfc40UL,
113 0x205c800000000000UL,
116 unsigned long kvm_s390_fac_list_mask_size(void)
118 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
119 return ARRAY_SIZE(kvm_s390_fac_list_mask);
122 static struct gmap_notifier gmap_notifier;
124 /* Section: not file related */
125 int kvm_arch_hardware_enable(void)
127 /* every s390 is virtualization enabled ;-) */
131 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
133 int kvm_arch_hardware_setup(void)
135 gmap_notifier.notifier_call = kvm_gmap_notifier;
136 gmap_register_ipte_notifier(&gmap_notifier);
140 void kvm_arch_hardware_unsetup(void)
142 gmap_unregister_ipte_notifier(&gmap_notifier);
145 int kvm_arch_init(void *opaque)
147 /* Register floating interrupt controller interface. */
148 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
151 /* Section: device related */
152 long kvm_arch_dev_ioctl(struct file *filp,
153 unsigned int ioctl, unsigned long arg)
155 if (ioctl == KVM_S390_ENABLE_SIE)
156 return s390_enable_sie();
160 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
165 case KVM_CAP_S390_PSW:
166 case KVM_CAP_S390_GMAP:
167 case KVM_CAP_SYNC_MMU:
168 #ifdef CONFIG_KVM_S390_UCONTROL
169 case KVM_CAP_S390_UCONTROL:
171 case KVM_CAP_ASYNC_PF:
172 case KVM_CAP_SYNC_REGS:
173 case KVM_CAP_ONE_REG:
174 case KVM_CAP_ENABLE_CAP:
175 case KVM_CAP_S390_CSS_SUPPORT:
176 case KVM_CAP_IOEVENTFD:
177 case KVM_CAP_DEVICE_CTRL:
178 case KVM_CAP_ENABLE_CAP_VM:
179 case KVM_CAP_S390_IRQCHIP:
180 case KVM_CAP_VM_ATTRIBUTES:
181 case KVM_CAP_MP_STATE:
182 case KVM_CAP_S390_INJECT_IRQ:
183 case KVM_CAP_S390_USER_SIGP:
184 case KVM_CAP_S390_USER_STSI:
185 case KVM_CAP_S390_SKEYS:
186 case KVM_CAP_S390_IRQ_STATE:
189 case KVM_CAP_S390_MEM_OP:
192 case KVM_CAP_NR_VCPUS:
193 case KVM_CAP_MAX_VCPUS:
196 case KVM_CAP_NR_MEMSLOTS:
197 r = KVM_USER_MEM_SLOTS;
199 case KVM_CAP_S390_COW:
200 r = MACHINE_HAS_ESOP;
202 case KVM_CAP_S390_VECTOR_REGISTERS:
211 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
212 struct kvm_memory_slot *memslot)
214 gfn_t cur_gfn, last_gfn;
215 unsigned long address;
216 struct gmap *gmap = kvm->arch.gmap;
218 down_read(&gmap->mm->mmap_sem);
219 /* Loop over all guest pages */
220 last_gfn = memslot->base_gfn + memslot->npages;
221 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
222 address = gfn_to_hva_memslot(memslot, cur_gfn);
224 if (gmap_test_and_clear_dirty(address, gmap))
225 mark_page_dirty(kvm, cur_gfn);
227 up_read(&gmap->mm->mmap_sem);
230 /* Section: vm related */
232 * Get (and clear) the dirty memory log for a memory slot.
234 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
235 struct kvm_dirty_log *log)
239 struct kvm_memory_slot *memslot;
242 mutex_lock(&kvm->slots_lock);
245 if (log->slot >= KVM_USER_MEM_SLOTS)
248 memslot = id_to_memslot(kvm->memslots, log->slot);
250 if (!memslot->dirty_bitmap)
253 kvm_s390_sync_dirty_log(kvm, memslot);
254 r = kvm_get_dirty_log(kvm, log, &is_dirty);
258 /* Clear the dirty log */
260 n = kvm_dirty_bitmap_bytes(memslot);
261 memset(memslot->dirty_bitmap, 0, n);
265 mutex_unlock(&kvm->slots_lock);
269 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
277 case KVM_CAP_S390_IRQCHIP:
278 kvm->arch.use_irqchip = 1;
281 case KVM_CAP_S390_USER_SIGP:
282 kvm->arch.user_sigp = 1;
285 case KVM_CAP_S390_VECTOR_REGISTERS:
286 if (MACHINE_HAS_VX) {
287 set_kvm_facility(kvm->arch.model.fac->mask, 129);
288 set_kvm_facility(kvm->arch.model.fac->list, 129);
293 case KVM_CAP_S390_USER_STSI:
294 kvm->arch.user_stsi = 1;
304 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
308 switch (attr->attr) {
309 case KVM_S390_VM_MEM_LIMIT_SIZE:
311 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
321 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
325 switch (attr->attr) {
326 case KVM_S390_VM_MEM_ENABLE_CMMA:
328 mutex_lock(&kvm->lock);
329 if (atomic_read(&kvm->online_vcpus) == 0) {
330 kvm->arch.use_cmma = 1;
333 mutex_unlock(&kvm->lock);
335 case KVM_S390_VM_MEM_CLR_CMMA:
336 mutex_lock(&kvm->lock);
337 idx = srcu_read_lock(&kvm->srcu);
338 s390_reset_cmma(kvm->arch.gmap->mm);
339 srcu_read_unlock(&kvm->srcu, idx);
340 mutex_unlock(&kvm->lock);
343 case KVM_S390_VM_MEM_LIMIT_SIZE: {
344 unsigned long new_limit;
346 if (kvm_is_ucontrol(kvm))
349 if (get_user(new_limit, (u64 __user *)attr->addr))
352 if (new_limit > kvm->arch.gmap->asce_end)
356 mutex_lock(&kvm->lock);
357 if (atomic_read(&kvm->online_vcpus) == 0) {
358 /* gmap_alloc will round the limit up */
359 struct gmap *new = gmap_alloc(current->mm, new_limit);
364 gmap_free(kvm->arch.gmap);
366 kvm->arch.gmap = new;
370 mutex_unlock(&kvm->lock);
380 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
382 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
384 struct kvm_vcpu *vcpu;
387 if (!test_kvm_facility(kvm, 76))
390 mutex_lock(&kvm->lock);
391 switch (attr->attr) {
392 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
394 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
395 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
396 kvm->arch.crypto.aes_kw = 1;
398 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
400 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
401 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
402 kvm->arch.crypto.dea_kw = 1;
404 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
405 kvm->arch.crypto.aes_kw = 0;
406 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
407 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
409 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
410 kvm->arch.crypto.dea_kw = 0;
411 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
412 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
415 mutex_unlock(&kvm->lock);
419 kvm_for_each_vcpu(i, vcpu, kvm) {
420 kvm_s390_vcpu_crypto_setup(vcpu);
423 mutex_unlock(&kvm->lock);
427 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
431 if (copy_from_user(>od_high, (void __user *)attr->addr,
441 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
443 struct kvm_vcpu *cur_vcpu;
444 unsigned int vcpu_idx;
448 if (copy_from_user(>od, (void __user *)attr->addr, sizeof(gtod)))
451 r = store_tod_clock(&host_tod);
455 mutex_lock(&kvm->lock);
456 kvm->arch.epoch = gtod - host_tod;
457 kvm_for_each_vcpu(vcpu_idx, cur_vcpu, kvm) {
458 cur_vcpu->arch.sie_block->epoch = kvm->arch.epoch;
461 mutex_unlock(&kvm->lock);
465 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
472 switch (attr->attr) {
473 case KVM_S390_VM_TOD_HIGH:
474 ret = kvm_s390_set_tod_high(kvm, attr);
476 case KVM_S390_VM_TOD_LOW:
477 ret = kvm_s390_set_tod_low(kvm, attr);
486 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
490 if (copy_to_user((void __user *)attr->addr, >od_high,
497 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
502 r = store_tod_clock(&host_tod);
506 gtod = host_tod + kvm->arch.epoch;
507 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod)))
513 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
520 switch (attr->attr) {
521 case KVM_S390_VM_TOD_HIGH:
522 ret = kvm_s390_get_tod_high(kvm, attr);
524 case KVM_S390_VM_TOD_LOW:
525 ret = kvm_s390_get_tod_low(kvm, attr);
534 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
536 struct kvm_s390_vm_cpu_processor *proc;
539 mutex_lock(&kvm->lock);
540 if (atomic_read(&kvm->online_vcpus)) {
544 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
549 if (!copy_from_user(proc, (void __user *)attr->addr,
551 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
552 sizeof(struct cpuid));
553 kvm->arch.model.ibc = proc->ibc;
554 memcpy(kvm->arch.model.fac->list, proc->fac_list,
555 S390_ARCH_FAC_LIST_SIZE_BYTE);
560 mutex_unlock(&kvm->lock);
564 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
568 switch (attr->attr) {
569 case KVM_S390_VM_CPU_PROCESSOR:
570 ret = kvm_s390_set_processor(kvm, attr);
576 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
578 struct kvm_s390_vm_cpu_processor *proc;
581 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
586 memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
587 proc->ibc = kvm->arch.model.ibc;
588 memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
589 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
596 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
598 struct kvm_s390_vm_cpu_machine *mach;
601 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
606 get_cpu_id((struct cpuid *) &mach->cpuid);
607 mach->ibc = sclp_get_ibc();
608 memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
609 S390_ARCH_FAC_LIST_SIZE_BYTE);
610 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
611 S390_ARCH_FAC_LIST_SIZE_BYTE);
612 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
619 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
623 switch (attr->attr) {
624 case KVM_S390_VM_CPU_PROCESSOR:
625 ret = kvm_s390_get_processor(kvm, attr);
627 case KVM_S390_VM_CPU_MACHINE:
628 ret = kvm_s390_get_machine(kvm, attr);
634 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
638 switch (attr->group) {
639 case KVM_S390_VM_MEM_CTRL:
640 ret = kvm_s390_set_mem_control(kvm, attr);
642 case KVM_S390_VM_TOD:
643 ret = kvm_s390_set_tod(kvm, attr);
645 case KVM_S390_VM_CPU_MODEL:
646 ret = kvm_s390_set_cpu_model(kvm, attr);
648 case KVM_S390_VM_CRYPTO:
649 ret = kvm_s390_vm_set_crypto(kvm, attr);
659 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
663 switch (attr->group) {
664 case KVM_S390_VM_MEM_CTRL:
665 ret = kvm_s390_get_mem_control(kvm, attr);
667 case KVM_S390_VM_TOD:
668 ret = kvm_s390_get_tod(kvm, attr);
670 case KVM_S390_VM_CPU_MODEL:
671 ret = kvm_s390_get_cpu_model(kvm, attr);
681 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
685 switch (attr->group) {
686 case KVM_S390_VM_MEM_CTRL:
687 switch (attr->attr) {
688 case KVM_S390_VM_MEM_ENABLE_CMMA:
689 case KVM_S390_VM_MEM_CLR_CMMA:
690 case KVM_S390_VM_MEM_LIMIT_SIZE:
698 case KVM_S390_VM_TOD:
699 switch (attr->attr) {
700 case KVM_S390_VM_TOD_LOW:
701 case KVM_S390_VM_TOD_HIGH:
709 case KVM_S390_VM_CPU_MODEL:
710 switch (attr->attr) {
711 case KVM_S390_VM_CPU_PROCESSOR:
712 case KVM_S390_VM_CPU_MACHINE:
720 case KVM_S390_VM_CRYPTO:
721 switch (attr->attr) {
722 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
723 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
724 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
725 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
741 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
745 unsigned long curkey;
748 if (args->flags != 0)
751 /* Is this guest using storage keys? */
752 if (!mm_use_skey(current->mm))
753 return KVM_S390_GET_SKEYS_NONE;
755 /* Enforce sane limit on memory allocation */
756 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
759 keys = kmalloc_array(args->count, sizeof(uint8_t),
760 GFP_KERNEL | __GFP_NOWARN);
762 keys = vmalloc(sizeof(uint8_t) * args->count);
766 for (i = 0; i < args->count; i++) {
767 hva = gfn_to_hva(kvm, args->start_gfn + i);
768 if (kvm_is_error_hva(hva)) {
773 curkey = get_guest_storage_key(current->mm, hva);
774 if (IS_ERR_VALUE(curkey)) {
781 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
782 sizeof(uint8_t) * args->count);
790 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
796 if (args->flags != 0)
799 /* Enforce sane limit on memory allocation */
800 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
803 keys = kmalloc_array(args->count, sizeof(uint8_t),
804 GFP_KERNEL | __GFP_NOWARN);
806 keys = vmalloc(sizeof(uint8_t) * args->count);
810 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
811 sizeof(uint8_t) * args->count);
817 /* Enable storage key handling for the guest */
820 for (i = 0; i < args->count; i++) {
821 hva = gfn_to_hva(kvm, args->start_gfn + i);
822 if (kvm_is_error_hva(hva)) {
827 /* Lowest order bit is reserved */
828 if (keys[i] & 0x01) {
833 r = set_guest_storage_key(current->mm, hva,
834 (unsigned long)keys[i], 0);
843 long kvm_arch_vm_ioctl(struct file *filp,
844 unsigned int ioctl, unsigned long arg)
846 struct kvm *kvm = filp->private_data;
847 void __user *argp = (void __user *)arg;
848 struct kvm_device_attr attr;
852 case KVM_S390_INTERRUPT: {
853 struct kvm_s390_interrupt s390int;
856 if (copy_from_user(&s390int, argp, sizeof(s390int)))
858 r = kvm_s390_inject_vm(kvm, &s390int);
861 case KVM_ENABLE_CAP: {
862 struct kvm_enable_cap cap;
864 if (copy_from_user(&cap, argp, sizeof(cap)))
866 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
869 case KVM_CREATE_IRQCHIP: {
870 struct kvm_irq_routing_entry routing;
873 if (kvm->arch.use_irqchip) {
874 /* Set up dummy routing. */
875 memset(&routing, 0, sizeof(routing));
876 kvm_set_irq_routing(kvm, &routing, 0, 0);
881 case KVM_SET_DEVICE_ATTR: {
883 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
885 r = kvm_s390_vm_set_attr(kvm, &attr);
888 case KVM_GET_DEVICE_ATTR: {
890 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
892 r = kvm_s390_vm_get_attr(kvm, &attr);
895 case KVM_HAS_DEVICE_ATTR: {
897 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
899 r = kvm_s390_vm_has_attr(kvm, &attr);
902 case KVM_S390_GET_SKEYS: {
903 struct kvm_s390_skeys args;
906 if (copy_from_user(&args, argp,
907 sizeof(struct kvm_s390_skeys)))
909 r = kvm_s390_get_skeys(kvm, &args);
912 case KVM_S390_SET_SKEYS: {
913 struct kvm_s390_skeys args;
916 if (copy_from_user(&args, argp,
917 sizeof(struct kvm_s390_skeys)))
919 r = kvm_s390_set_skeys(kvm, &args);
929 static int kvm_s390_query_ap_config(u8 *config)
931 u32 fcn_code = 0x04000000UL;
934 memset(config, 0, 128);
938 ".long 0xb2af0000\n" /* PQAP(QCI) */
944 : "r" (fcn_code), "r" (config)
945 : "cc", "0", "2", "memory"
951 static int kvm_s390_apxa_installed(void)
956 if (test_facility(2) && test_facility(12)) {
957 cc = kvm_s390_query_ap_config(config);
960 pr_err("PQAP(QCI) failed with cc=%d", cc);
962 return config[0] & 0x40;
968 static void kvm_s390_set_crycb_format(struct kvm *kvm)
970 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
972 if (kvm_s390_apxa_installed())
973 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
975 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
978 static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
981 cpu_id->version = 0xff;
984 static int kvm_s390_crypto_init(struct kvm *kvm)
986 if (!test_kvm_facility(kvm, 76))
989 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
990 GFP_KERNEL | GFP_DMA);
991 if (!kvm->arch.crypto.crycb)
994 kvm_s390_set_crycb_format(kvm);
996 /* Enable AES/DEA protected key functions by default */
997 kvm->arch.crypto.aes_kw = 1;
998 kvm->arch.crypto.dea_kw = 1;
999 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1000 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1001 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1002 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1007 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1010 char debug_name[16];
1011 static unsigned long sca_offset;
1014 #ifdef CONFIG_KVM_S390_UCONTROL
1015 if (type & ~KVM_VM_S390_UCONTROL)
1017 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1024 rc = s390_enable_sie();
1030 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
1033 spin_lock(&kvm_lock);
1034 sca_offset = (sca_offset + 16) & 0x7f0;
1035 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
1036 spin_unlock(&kvm_lock);
1038 sprintf(debug_name, "kvm-%u", current->pid);
1040 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
1045 * The architectural maximum amount of facilities is 16 kbit. To store
1046 * this amount, 2 kbyte of memory is required. Thus we need a full
1047 * page to hold the guest facility list (arch.model.fac->list) and the
1048 * facility mask (arch.model.fac->mask). Its address size has to be
1049 * 31 bits and word aligned.
1051 kvm->arch.model.fac =
1052 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1053 if (!kvm->arch.model.fac)
1056 /* Populate the facility mask initially. */
1057 memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
1058 S390_ARCH_FAC_LIST_SIZE_BYTE);
1059 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1060 if (i < kvm_s390_fac_list_mask_size())
1061 kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
1063 kvm->arch.model.fac->mask[i] = 0UL;
1066 /* Populate the facility list initially. */
1067 memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
1068 S390_ARCH_FAC_LIST_SIZE_BYTE);
1070 kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
1071 kvm->arch.model.ibc = sclp_get_ibc() & 0x0fff;
1073 if (kvm_s390_crypto_init(kvm) < 0)
1076 spin_lock_init(&kvm->arch.float_int.lock);
1077 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1078 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1079 init_waitqueue_head(&kvm->arch.ipte_wq);
1080 mutex_init(&kvm->arch.ipte_mutex);
1082 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1083 VM_EVENT(kvm, 3, "%s", "vm created");
1085 if (type & KVM_VM_S390_UCONTROL) {
1086 kvm->arch.gmap = NULL;
1088 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
1089 if (!kvm->arch.gmap)
1091 kvm->arch.gmap->private = kvm;
1092 kvm->arch.gmap->pfault_enabled = 0;
1095 kvm->arch.css_support = 0;
1096 kvm->arch.use_irqchip = 0;
1097 kvm->arch.epoch = 0;
1099 spin_lock_init(&kvm->arch.start_stop_lock);
1103 kfree(kvm->arch.crypto.crycb);
1104 free_page((unsigned long)kvm->arch.model.fac);
1105 debug_unregister(kvm->arch.dbf);
1106 free_page((unsigned long)(kvm->arch.sca));
1110 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1112 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1113 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1114 kvm_s390_clear_local_irqs(vcpu);
1115 kvm_clear_async_pf_completion_queue(vcpu);
1116 if (!kvm_is_ucontrol(vcpu->kvm)) {
1117 clear_bit(63 - vcpu->vcpu_id,
1118 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
1119 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
1120 (__u64) vcpu->arch.sie_block)
1121 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
1125 if (kvm_is_ucontrol(vcpu->kvm))
1126 gmap_free(vcpu->arch.gmap);
1128 if (kvm_s390_cmma_enabled(vcpu->kvm))
1129 kvm_s390_vcpu_unsetup_cmma(vcpu);
1130 free_page((unsigned long)(vcpu->arch.sie_block));
1132 kvm_vcpu_uninit(vcpu);
1133 kmem_cache_free(kvm_vcpu_cache, vcpu);
1136 static void kvm_free_vcpus(struct kvm *kvm)
1139 struct kvm_vcpu *vcpu;
1141 kvm_for_each_vcpu(i, vcpu, kvm)
1142 kvm_arch_vcpu_destroy(vcpu);
1144 mutex_lock(&kvm->lock);
1145 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1146 kvm->vcpus[i] = NULL;
1148 atomic_set(&kvm->online_vcpus, 0);
1149 mutex_unlock(&kvm->lock);
1152 void kvm_arch_destroy_vm(struct kvm *kvm)
1154 kvm_free_vcpus(kvm);
1155 free_page((unsigned long)kvm->arch.model.fac);
1156 free_page((unsigned long)(kvm->arch.sca));
1157 debug_unregister(kvm->arch.dbf);
1158 kfree(kvm->arch.crypto.crycb);
1159 if (!kvm_is_ucontrol(kvm))
1160 gmap_free(kvm->arch.gmap);
1161 kvm_s390_destroy_adapters(kvm);
1162 kvm_s390_clear_float_irqs(kvm);
1165 /* Section: vcpu related */
1166 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1168 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1169 if (!vcpu->arch.gmap)
1171 vcpu->arch.gmap->private = vcpu->kvm;
1176 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1178 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1179 kvm_clear_async_pf_completion_queue(vcpu);
1180 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1186 if (test_kvm_facility(vcpu->kvm, 129))
1187 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1189 if (kvm_is_ucontrol(vcpu->kvm))
1190 return __kvm_ucontrol_vcpu_init(vcpu);
1195 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1197 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1198 if (test_kvm_facility(vcpu->kvm, 129))
1199 save_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
1201 save_fp_regs(vcpu->arch.host_fpregs.fprs);
1202 save_access_regs(vcpu->arch.host_acrs);
1203 if (test_kvm_facility(vcpu->kvm, 129)) {
1204 restore_fp_ctl(&vcpu->run->s.regs.fpc);
1205 restore_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
1207 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1208 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1210 restore_access_regs(vcpu->run->s.regs.acrs);
1211 gmap_enable(vcpu->arch.gmap);
1212 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1215 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1217 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1218 gmap_disable(vcpu->arch.gmap);
1219 if (test_kvm_facility(vcpu->kvm, 129)) {
1220 save_fp_ctl(&vcpu->run->s.regs.fpc);
1221 save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
1223 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1224 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1226 save_access_regs(vcpu->run->s.regs.acrs);
1227 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1228 if (test_kvm_facility(vcpu->kvm, 129))
1229 restore_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
1231 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
1232 restore_access_regs(vcpu->arch.host_acrs);
1235 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1237 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1238 vcpu->arch.sie_block->gpsw.mask = 0UL;
1239 vcpu->arch.sie_block->gpsw.addr = 0UL;
1240 kvm_s390_set_prefix(vcpu, 0);
1241 vcpu->arch.sie_block->cputm = 0UL;
1242 vcpu->arch.sie_block->ckc = 0UL;
1243 vcpu->arch.sie_block->todpr = 0;
1244 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1245 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1246 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1247 vcpu->arch.guest_fpregs.fpc = 0;
1248 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
1249 vcpu->arch.sie_block->gbea = 1;
1250 vcpu->arch.sie_block->pp = 0;
1251 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1252 kvm_clear_async_pf_completion_queue(vcpu);
1253 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1254 kvm_s390_vcpu_stop(vcpu);
1255 kvm_s390_clear_local_irqs(vcpu);
1258 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1260 mutex_lock(&vcpu->kvm->lock);
1261 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1262 mutex_unlock(&vcpu->kvm->lock);
1263 if (!kvm_is_ucontrol(vcpu->kvm))
1264 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1267 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1269 if (!test_kvm_facility(vcpu->kvm, 76))
1272 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1274 if (vcpu->kvm->arch.crypto.aes_kw)
1275 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1276 if (vcpu->kvm->arch.crypto.dea_kw)
1277 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1279 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1282 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1284 free_page(vcpu->arch.sie_block->cbrlo);
1285 vcpu->arch.sie_block->cbrlo = 0;
1288 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1290 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1291 if (!vcpu->arch.sie_block->cbrlo)
1294 vcpu->arch.sie_block->ecb2 |= 0x80;
1295 vcpu->arch.sie_block->ecb2 &= ~0x08;
1299 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1301 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1303 vcpu->arch.cpu_id = model->cpu_id;
1304 vcpu->arch.sie_block->ibc = model->ibc;
1305 vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1308 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1312 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1316 kvm_s390_vcpu_setup_model(vcpu);
1318 vcpu->arch.sie_block->ecb = 6;
1319 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1320 vcpu->arch.sie_block->ecb |= 0x10;
1322 vcpu->arch.sie_block->ecb2 = 8;
1323 vcpu->arch.sie_block->eca = 0xC1002000U;
1324 if (sclp_has_siif())
1325 vcpu->arch.sie_block->eca |= 1;
1326 if (sclp_has_sigpif())
1327 vcpu->arch.sie_block->eca |= 0x10000000U;
1328 if (test_kvm_facility(vcpu->kvm, 129)) {
1329 vcpu->arch.sie_block->eca |= 0x00020000;
1330 vcpu->arch.sie_block->ecd |= 0x20000000;
1332 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1334 if (kvm_s390_cmma_enabled(vcpu->kvm)) {
1335 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1339 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1340 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1342 kvm_s390_vcpu_crypto_setup(vcpu);
1347 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1350 struct kvm_vcpu *vcpu;
1351 struct sie_page *sie_page;
1354 if (id >= KVM_MAX_VCPUS)
1359 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1363 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1367 vcpu->arch.sie_block = &sie_page->sie_block;
1368 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1369 vcpu->arch.host_vregs = &sie_page->vregs;
1371 vcpu->arch.sie_block->icpua = id;
1372 if (!kvm_is_ucontrol(kvm)) {
1373 if (!kvm->arch.sca) {
1377 if (!kvm->arch.sca->cpu[id].sda)
1378 kvm->arch.sca->cpu[id].sda =
1379 (__u64) vcpu->arch.sie_block;
1380 vcpu->arch.sie_block->scaoh =
1381 (__u32)(((__u64)kvm->arch.sca) >> 32);
1382 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
1383 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
1386 spin_lock_init(&vcpu->arch.local_int.lock);
1387 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1388 vcpu->arch.local_int.wq = &vcpu->wq;
1389 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1391 rc = kvm_vcpu_init(vcpu, kvm, id);
1393 goto out_free_sie_block;
1394 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
1395 vcpu->arch.sie_block);
1396 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1400 free_page((unsigned long)(vcpu->arch.sie_block));
1402 kmem_cache_free(kvm_vcpu_cache, vcpu);
1407 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1409 return kvm_s390_vcpu_has_irq(vcpu, 0);
1412 void s390_vcpu_block(struct kvm_vcpu *vcpu)
1414 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1417 void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1419 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1423 * Kick a guest cpu out of SIE and wait until SIE is not running.
1424 * If the CPU is not running (e.g. waiting as idle) the function will
1425 * return immediately. */
1426 void exit_sie(struct kvm_vcpu *vcpu)
1428 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1429 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1433 /* Kick a guest cpu out of SIE and prevent SIE-reentry */
1434 void exit_sie_sync(struct kvm_vcpu *vcpu)
1436 s390_vcpu_block(vcpu);
1440 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1443 struct kvm *kvm = gmap->private;
1444 struct kvm_vcpu *vcpu;
1446 kvm_for_each_vcpu(i, vcpu, kvm) {
1447 /* match against both prefix pages */
1448 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
1449 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1450 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
1451 exit_sie_sync(vcpu);
1456 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1458 /* kvm common code refers to this, but never calls it */
1463 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1464 struct kvm_one_reg *reg)
1469 case KVM_REG_S390_TODPR:
1470 r = put_user(vcpu->arch.sie_block->todpr,
1471 (u32 __user *)reg->addr);
1473 case KVM_REG_S390_EPOCHDIFF:
1474 r = put_user(vcpu->arch.sie_block->epoch,
1475 (u64 __user *)reg->addr);
1477 case KVM_REG_S390_CPU_TIMER:
1478 r = put_user(vcpu->arch.sie_block->cputm,
1479 (u64 __user *)reg->addr);
1481 case KVM_REG_S390_CLOCK_COMP:
1482 r = put_user(vcpu->arch.sie_block->ckc,
1483 (u64 __user *)reg->addr);
1485 case KVM_REG_S390_PFTOKEN:
1486 r = put_user(vcpu->arch.pfault_token,
1487 (u64 __user *)reg->addr);
1489 case KVM_REG_S390_PFCOMPARE:
1490 r = put_user(vcpu->arch.pfault_compare,
1491 (u64 __user *)reg->addr);
1493 case KVM_REG_S390_PFSELECT:
1494 r = put_user(vcpu->arch.pfault_select,
1495 (u64 __user *)reg->addr);
1497 case KVM_REG_S390_PP:
1498 r = put_user(vcpu->arch.sie_block->pp,
1499 (u64 __user *)reg->addr);
1501 case KVM_REG_S390_GBEA:
1502 r = put_user(vcpu->arch.sie_block->gbea,
1503 (u64 __user *)reg->addr);
1512 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1513 struct kvm_one_reg *reg)
1518 case KVM_REG_S390_TODPR:
1519 r = get_user(vcpu->arch.sie_block->todpr,
1520 (u32 __user *)reg->addr);
1522 case KVM_REG_S390_EPOCHDIFF:
1523 r = get_user(vcpu->arch.sie_block->epoch,
1524 (u64 __user *)reg->addr);
1526 case KVM_REG_S390_CPU_TIMER:
1527 r = get_user(vcpu->arch.sie_block->cputm,
1528 (u64 __user *)reg->addr);
1530 case KVM_REG_S390_CLOCK_COMP:
1531 r = get_user(vcpu->arch.sie_block->ckc,
1532 (u64 __user *)reg->addr);
1534 case KVM_REG_S390_PFTOKEN:
1535 r = get_user(vcpu->arch.pfault_token,
1536 (u64 __user *)reg->addr);
1537 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1538 kvm_clear_async_pf_completion_queue(vcpu);
1540 case KVM_REG_S390_PFCOMPARE:
1541 r = get_user(vcpu->arch.pfault_compare,
1542 (u64 __user *)reg->addr);
1544 case KVM_REG_S390_PFSELECT:
1545 r = get_user(vcpu->arch.pfault_select,
1546 (u64 __user *)reg->addr);
1548 case KVM_REG_S390_PP:
1549 r = get_user(vcpu->arch.sie_block->pp,
1550 (u64 __user *)reg->addr);
1552 case KVM_REG_S390_GBEA:
1553 r = get_user(vcpu->arch.sie_block->gbea,
1554 (u64 __user *)reg->addr);
1563 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1565 kvm_s390_vcpu_initial_reset(vcpu);
1569 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1571 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs));
1575 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1577 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1581 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1582 struct kvm_sregs *sregs)
1584 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
1585 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
1586 restore_access_regs(vcpu->run->s.regs.acrs);
1590 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1591 struct kvm_sregs *sregs)
1593 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
1594 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
1598 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1600 if (test_fp_ctl(fpu->fpc))
1602 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
1603 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1604 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1605 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1609 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1611 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1612 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
1616 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1620 if (!is_vcpu_stopped(vcpu))
1623 vcpu->run->psw_mask = psw.mask;
1624 vcpu->run->psw_addr = psw.addr;
1629 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1630 struct kvm_translation *tr)
1632 return -EINVAL; /* not implemented yet */
1635 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1636 KVM_GUESTDBG_USE_HW_BP | \
1637 KVM_GUESTDBG_ENABLE)
1639 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1640 struct kvm_guest_debug *dbg)
1644 vcpu->guest_debug = 0;
1645 kvm_s390_clear_bp_data(vcpu);
1647 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
1650 if (dbg->control & KVM_GUESTDBG_ENABLE) {
1651 vcpu->guest_debug = dbg->control;
1652 /* enforce guest PER */
1653 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1655 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1656 rc = kvm_s390_import_bp_data(vcpu, dbg);
1658 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1659 vcpu->arch.guestdbg.last_bp = 0;
1663 vcpu->guest_debug = 0;
1664 kvm_s390_clear_bp_data(vcpu);
1665 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1671 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1672 struct kvm_mp_state *mp_state)
1674 /* CHECK_STOP and LOAD are not supported yet */
1675 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1676 KVM_MP_STATE_OPERATING;
1679 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1680 struct kvm_mp_state *mp_state)
1684 /* user space knows about this interface - let it control the state */
1685 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1687 switch (mp_state->mp_state) {
1688 case KVM_MP_STATE_STOPPED:
1689 kvm_s390_vcpu_stop(vcpu);
1691 case KVM_MP_STATE_OPERATING:
1692 kvm_s390_vcpu_start(vcpu);
1694 case KVM_MP_STATE_LOAD:
1695 case KVM_MP_STATE_CHECK_STOP:
1696 /* fall through - CHECK_STOP and LOAD are not supported yet */
1704 bool kvm_s390_cmma_enabled(struct kvm *kvm)
1706 if (!MACHINE_IS_LPAR)
1708 /* only enable for z10 and later */
1709 if (!MACHINE_HAS_EDAT1)
1711 if (!kvm->arch.use_cmma)
1716 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1718 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1721 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1724 s390_vcpu_unblock(vcpu);
1726 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1727 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1728 * This ensures that the ipte instruction for this request has
1729 * already finished. We might race against a second unmapper that
1730 * wants to set the blocking bit. Lets just retry the request loop.
1732 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1734 rc = gmap_ipte_notify(vcpu->arch.gmap,
1735 kvm_s390_get_prefix(vcpu),
1742 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1743 vcpu->arch.sie_block->ihcpu = 0xffff;
1747 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1748 if (!ibs_enabled(vcpu)) {
1749 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1750 atomic_set_mask(CPUSTAT_IBS,
1751 &vcpu->arch.sie_block->cpuflags);
1756 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1757 if (ibs_enabled(vcpu)) {
1758 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1759 atomic_clear_mask(CPUSTAT_IBS,
1760 &vcpu->arch.sie_block->cpuflags);
1765 /* nothing to do, just clear the request */
1766 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1772 * kvm_arch_fault_in_page - fault-in guest page if necessary
1773 * @vcpu: The corresponding virtual cpu
1774 * @gpa: Guest physical address
1775 * @writable: Whether the page should be writable or not
1777 * Make sure that a guest page has been faulted-in on the host.
1779 * Return: Zero on success, negative error code otherwise.
1781 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
1783 return gmap_fault(vcpu->arch.gmap, gpa,
1784 writable ? FAULT_FLAG_WRITE : 0);
1787 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1788 unsigned long token)
1790 struct kvm_s390_interrupt inti;
1791 struct kvm_s390_irq irq;
1794 irq.u.ext.ext_params2 = token;
1795 irq.type = KVM_S390_INT_PFAULT_INIT;
1796 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
1798 inti.type = KVM_S390_INT_PFAULT_DONE;
1799 inti.parm64 = token;
1800 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1804 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1805 struct kvm_async_pf *work)
1807 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1808 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1811 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1812 struct kvm_async_pf *work)
1814 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1815 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1818 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1819 struct kvm_async_pf *work)
1821 /* s390 will always inject the page directly */
1824 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1827 * s390 will always inject the page directly,
1828 * but we still want check_async_completion to cleanup
1833 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1836 struct kvm_arch_async_pf arch;
1839 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1841 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1842 vcpu->arch.pfault_compare)
1844 if (psw_extint_disabled(vcpu))
1846 if (kvm_s390_vcpu_has_irq(vcpu, 0))
1848 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1850 if (!vcpu->arch.gmap->pfault_enabled)
1853 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1854 hva += current->thread.gmap_addr & ~PAGE_MASK;
1855 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
1858 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1862 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
1867 * On s390 notifications for arriving pages will be delivered directly
1868 * to the guest but the house keeping for completed pfaults is
1869 * handled outside the worker.
1871 kvm_check_async_pf_completion(vcpu);
1873 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
1878 if (test_cpu_flag(CIF_MCCK_PENDING))
1881 if (!kvm_is_ucontrol(vcpu->kvm)) {
1882 rc = kvm_s390_deliver_pending_interrupts(vcpu);
1887 rc = kvm_s390_handle_requests(vcpu);
1891 if (guestdbg_enabled(vcpu)) {
1892 kvm_s390_backup_guest_per_regs(vcpu);
1893 kvm_s390_patch_guest_per_regs(vcpu);
1896 vcpu->arch.sie_block->icptcode = 0;
1897 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1898 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1899 trace_kvm_s390_sie_enter(vcpu, cpuflags);
1904 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
1906 psw_t *psw = &vcpu->arch.sie_block->gpsw;
1910 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1911 trace_kvm_s390_sie_fault(vcpu);
1914 * We want to inject an addressing exception, which is defined as a
1915 * suppressing or terminating exception. However, since we came here
1916 * by a DAT access exception, the PSW still points to the faulting
1917 * instruction since DAT exceptions are nullifying. So we've got
1918 * to look up the current opcode to get the length of the instruction
1919 * to be able to forward the PSW.
1921 rc = read_guest(vcpu, psw->addr, 0, &opcode, 1);
1923 return kvm_s390_inject_prog_cond(vcpu, rc);
1924 psw->addr = __rewind_psw(*psw, -insn_length(opcode));
1926 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1929 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1933 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1934 vcpu->arch.sie_block->icptcode);
1935 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1937 if (guestdbg_enabled(vcpu))
1938 kvm_s390_restore_guest_per_regs(vcpu);
1940 if (exit_reason >= 0) {
1942 } else if (kvm_is_ucontrol(vcpu->kvm)) {
1943 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1944 vcpu->run->s390_ucontrol.trans_exc_code =
1945 current->thread.gmap_addr;
1946 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1949 } else if (current->thread.gmap_pfault) {
1950 trace_kvm_s390_major_guest_pfault(vcpu);
1951 current->thread.gmap_pfault = 0;
1952 if (kvm_arch_setup_async_pf(vcpu)) {
1955 gpa_t gpa = current->thread.gmap_addr;
1956 rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1961 rc = vcpu_post_run_fault_in_sie(vcpu);
1963 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
1966 if (kvm_is_ucontrol(vcpu->kvm))
1967 /* Don't exit for host interrupts. */
1968 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
1970 rc = kvm_handle_sie_intercept(vcpu);
1976 static int __vcpu_run(struct kvm_vcpu *vcpu)
1978 int rc, exit_reason;
1981 * We try to hold kvm->srcu during most of vcpu_run (except when run-
1982 * ning the guest), so that memslots (and other stuff) are protected
1984 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1987 rc = vcpu_pre_run(vcpu);
1991 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1993 * As PF_VCPU will be used in fault handler, between
1994 * guest_enter and guest_exit should be no uaccess.
1999 exit_reason = sie64a(vcpu->arch.sie_block,
2000 vcpu->run->s.regs.gprs);
2002 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2004 rc = vcpu_post_run(vcpu, exit_reason);
2005 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2007 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2011 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2013 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2014 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2015 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2016 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2017 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2018 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2019 /* some control register changes require a tlb flush */
2020 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2022 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2023 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
2024 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2025 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2026 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2027 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2029 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2030 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2031 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2032 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2033 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2034 kvm_clear_async_pf_completion_queue(vcpu);
2036 kvm_run->kvm_dirty_regs = 0;
2039 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2041 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2042 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2043 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2044 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2045 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
2046 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2047 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2048 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2049 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2050 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2051 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2052 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2055 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2060 if (guestdbg_exit_pending(vcpu)) {
2061 kvm_s390_prepare_debug_exit(vcpu);
2065 if (vcpu->sigset_active)
2066 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2068 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2069 kvm_s390_vcpu_start(vcpu);
2070 } else if (is_vcpu_stopped(vcpu)) {
2071 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
2076 sync_regs(vcpu, kvm_run);
2079 rc = __vcpu_run(vcpu);
2081 if (signal_pending(current) && !rc) {
2082 kvm_run->exit_reason = KVM_EXIT_INTR;
2086 if (guestdbg_exit_pending(vcpu) && !rc) {
2087 kvm_s390_prepare_debug_exit(vcpu);
2091 if (rc == -EOPNOTSUPP) {
2092 /* intercept cannot be handled in-kernel, prepare kvm-run */
2093 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
2094 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2095 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2096 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2100 if (rc == -EREMOTE) {
2101 /* intercept was handled, but userspace support is needed
2102 * kvm_run has been prepared by the handler */
2106 store_regs(vcpu, kvm_run);
2108 if (vcpu->sigset_active)
2109 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2111 vcpu->stat.exit_userspace++;
2116 * store status at address
2117 * we use have two special cases:
2118 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2119 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2121 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2123 unsigned char archmode = 1;
2128 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2129 if (write_guest_abs(vcpu, 163, &archmode, 1))
2131 gpa = SAVE_AREA_BASE;
2132 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2133 if (write_guest_real(vcpu, 163, &archmode, 1))
2135 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
2137 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
2138 vcpu->arch.guest_fpregs.fprs, 128);
2139 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
2140 vcpu->run->s.regs.gprs, 128);
2141 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
2142 &vcpu->arch.sie_block->gpsw, 16);
2143 px = kvm_s390_get_prefix(vcpu);
2144 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
2146 rc |= write_guest_abs(vcpu,
2147 gpa + offsetof(struct save_area, fp_ctrl_reg),
2148 &vcpu->arch.guest_fpregs.fpc, 4);
2149 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
2150 &vcpu->arch.sie_block->todpr, 4);
2151 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
2152 &vcpu->arch.sie_block->cputm, 8);
2153 clkcomp = vcpu->arch.sie_block->ckc >> 8;
2154 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
2156 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
2157 &vcpu->run->s.regs.acrs, 64);
2158 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
2159 &vcpu->arch.sie_block->gcr, 128);
2160 return rc ? -EFAULT : 0;
2163 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2166 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2167 * copying in vcpu load/put. Lets update our copies before we save
2168 * it into the save area
2170 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
2171 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
2172 save_access_regs(vcpu->run->s.regs.acrs);
2174 return kvm_s390_store_status_unloaded(vcpu, addr);
2178 * store additional status at address
2180 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2183 /* Only bits 0-53 are used for address formation */
2184 if (!(gpa & ~0x3ff))
2187 return write_guest_abs(vcpu, gpa & ~0x3ff,
2188 (void *)&vcpu->run->s.regs.vrs, 512);
2191 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2193 if (!test_kvm_facility(vcpu->kvm, 129))
2197 * The guest VXRS are in the host VXRs due to the lazy
2198 * copying in vcpu load/put. Let's update our copies before we save
2199 * it into the save area.
2201 save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
2203 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2206 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2208 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2209 kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu);
2210 exit_sie_sync(vcpu);
2213 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2216 struct kvm_vcpu *vcpu;
2218 kvm_for_each_vcpu(i, vcpu, kvm) {
2219 __disable_ibs_on_vcpu(vcpu);
2223 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2225 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2226 kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu);
2227 exit_sie_sync(vcpu);
2230 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2232 int i, online_vcpus, started_vcpus = 0;
2234 if (!is_vcpu_stopped(vcpu))
2237 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2238 /* Only one cpu at a time may enter/leave the STOPPED state. */
2239 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2240 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2242 for (i = 0; i < online_vcpus; i++) {
2243 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2247 if (started_vcpus == 0) {
2248 /* we're the only active VCPU -> speed it up */
2249 __enable_ibs_on_vcpu(vcpu);
2250 } else if (started_vcpus == 1) {
2252 * As we are starting a second VCPU, we have to disable
2253 * the IBS facility on all VCPUs to remove potentially
2254 * oustanding ENABLE requests.
2256 __disable_ibs_on_all_vcpus(vcpu->kvm);
2259 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2261 * Another VCPU might have used IBS while we were offline.
2262 * Let's play safe and flush the VCPU at startup.
2264 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2265 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2269 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2271 int i, online_vcpus, started_vcpus = 0;
2272 struct kvm_vcpu *started_vcpu = NULL;
2274 if (is_vcpu_stopped(vcpu))
2277 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2278 /* Only one cpu at a time may enter/leave the STOPPED state. */
2279 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2280 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2282 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2283 kvm_s390_clear_stop_irq(vcpu);
2285 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2286 __disable_ibs_on_vcpu(vcpu);
2288 for (i = 0; i < online_vcpus; i++) {
2289 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2291 started_vcpu = vcpu->kvm->vcpus[i];
2295 if (started_vcpus == 1) {
2297 * As we only have one VCPU left, we want to enable the
2298 * IBS facility for that VCPU to speed it up.
2300 __enable_ibs_on_vcpu(started_vcpu);
2303 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2307 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2308 struct kvm_enable_cap *cap)
2316 case KVM_CAP_S390_CSS_SUPPORT:
2317 if (!vcpu->kvm->arch.css_support) {
2318 vcpu->kvm->arch.css_support = 1;
2319 trace_kvm_s390_enable_css(vcpu->kvm);
2330 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2331 struct kvm_s390_mem_op *mop)
2333 void __user *uaddr = (void __user *)mop->buf;
2334 void *tmpbuf = NULL;
2336 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2337 | KVM_S390_MEMOP_F_CHECK_ONLY;
2339 if (mop->flags & ~supported_flags)
2342 if (mop->size > MEM_OP_MAX_SIZE)
2345 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2346 tmpbuf = vmalloc(mop->size);
2351 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2354 case KVM_S390_MEMOP_LOGICAL_READ:
2355 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2356 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, false);
2359 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2361 if (copy_to_user(uaddr, tmpbuf, mop->size))
2365 case KVM_S390_MEMOP_LOGICAL_WRITE:
2366 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2367 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, true);
2370 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2374 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2380 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2382 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2383 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2389 long kvm_arch_vcpu_ioctl(struct file *filp,
2390 unsigned int ioctl, unsigned long arg)
2392 struct kvm_vcpu *vcpu = filp->private_data;
2393 void __user *argp = (void __user *)arg;
2398 case KVM_S390_IRQ: {
2399 struct kvm_s390_irq s390irq;
2402 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2404 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2407 case KVM_S390_INTERRUPT: {
2408 struct kvm_s390_interrupt s390int;
2409 struct kvm_s390_irq s390irq;
2412 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2414 if (s390int_to_s390irq(&s390int, &s390irq))
2416 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2419 case KVM_S390_STORE_STATUS:
2420 idx = srcu_read_lock(&vcpu->kvm->srcu);
2421 r = kvm_s390_vcpu_store_status(vcpu, arg);
2422 srcu_read_unlock(&vcpu->kvm->srcu, idx);
2424 case KVM_S390_SET_INITIAL_PSW: {
2428 if (copy_from_user(&psw, argp, sizeof(psw)))
2430 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2433 case KVM_S390_INITIAL_RESET:
2434 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2436 case KVM_SET_ONE_REG:
2437 case KVM_GET_ONE_REG: {
2438 struct kvm_one_reg reg;
2440 if (copy_from_user(®, argp, sizeof(reg)))
2442 if (ioctl == KVM_SET_ONE_REG)
2443 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®);
2445 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®);
2448 #ifdef CONFIG_KVM_S390_UCONTROL
2449 case KVM_S390_UCAS_MAP: {
2450 struct kvm_s390_ucas_mapping ucasmap;
2452 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2457 if (!kvm_is_ucontrol(vcpu->kvm)) {
2462 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2463 ucasmap.vcpu_addr, ucasmap.length);
2466 case KVM_S390_UCAS_UNMAP: {
2467 struct kvm_s390_ucas_mapping ucasmap;
2469 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2474 if (!kvm_is_ucontrol(vcpu->kvm)) {
2479 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2484 case KVM_S390_VCPU_FAULT: {
2485 r = gmap_fault(vcpu->arch.gmap, arg, 0);
2488 case KVM_ENABLE_CAP:
2490 struct kvm_enable_cap cap;
2492 if (copy_from_user(&cap, argp, sizeof(cap)))
2494 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2497 case KVM_S390_MEM_OP: {
2498 struct kvm_s390_mem_op mem_op;
2500 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2501 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2506 case KVM_S390_SET_IRQ_STATE: {
2507 struct kvm_s390_irq_state irq_state;
2510 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2512 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
2513 irq_state.len == 0 ||
2514 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
2518 r = kvm_s390_set_irq_state(vcpu,
2519 (void __user *) irq_state.buf,
2523 case KVM_S390_GET_IRQ_STATE: {
2524 struct kvm_s390_irq_state irq_state;
2527 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2529 if (irq_state.len == 0) {
2533 r = kvm_s390_get_irq_state(vcpu,
2534 (__u8 __user *) irq_state.buf,
2544 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2546 #ifdef CONFIG_KVM_S390_UCONTROL
2547 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2548 && (kvm_is_ucontrol(vcpu->kvm))) {
2549 vmf->page = virt_to_page(vcpu->arch.sie_block);
2550 get_page(vmf->page);
2554 return VM_FAULT_SIGBUS;
2557 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2558 unsigned long npages)
2563 /* Section: memory related */
2564 int kvm_arch_prepare_memory_region(struct kvm *kvm,
2565 struct kvm_memory_slot *memslot,
2566 struct kvm_userspace_memory_region *mem,
2567 enum kvm_mr_change change)
2569 /* A few sanity checks. We can have memory slots which have to be
2570 located/ended at a segment boundary (1MB). The memory in userland is
2571 ok to be fragmented into various different vmas. It is okay to mmap()
2572 and munmap() stuff in this slot after doing this call at any time */
2574 if (mem->userspace_addr & 0xffffful)
2577 if (mem->memory_size & 0xffffful)
2583 void kvm_arch_commit_memory_region(struct kvm *kvm,
2584 struct kvm_userspace_memory_region *mem,
2585 const struct kvm_memory_slot *old,
2586 enum kvm_mr_change change)
2590 /* If the basics of the memslot do not change, we do not want
2591 * to update the gmap. Every update causes several unnecessary
2592 * segment translation exceptions. This is usually handled just
2593 * fine by the normal fault handler + gmap, but it will also
2594 * cause faults on the prefix page of running guest CPUs.
2596 if (old->userspace_addr == mem->userspace_addr &&
2597 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2598 old->npages * PAGE_SIZE == mem->memory_size)
2601 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2602 mem->guest_phys_addr, mem->memory_size);
2604 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
2608 static int __init kvm_s390_init(void)
2610 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2613 static void __exit kvm_s390_exit(void)
2618 module_init(kvm_s390_init);
2619 module_exit(kvm_s390_exit);
2622 * Enable autoloading of the kvm module.
2623 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2624 * since x86 takes a different approach.
2626 #include <linux/miscdevice.h>
2627 MODULE_ALIAS_MISCDEV(KVM_MINOR);
2628 MODULE_ALIAS("devname:kvm");