2 * hosting zSeries kernel virtual machines
4 * Copyright IBM Corp. 2008, 2009
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
17 #include <linux/compiler.h>
18 #include <linux/err.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/module.h>
25 #include <linux/random.h>
26 #include <linux/slab.h>
27 #include <linux/timer.h>
28 #include <asm/asm-offsets.h>
29 #include <asm/lowcore.h>
30 #include <asm/pgtable.h>
32 #include <asm/switch_to.h>
37 #define CREATE_TRACE_POINTS
39 #include "trace-s390.h"
41 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
43 struct kvm_stats_debugfs_item debugfs_entries
[] = {
44 { "userspace_handled", VCPU_STAT(exit_userspace
) },
45 { "exit_null", VCPU_STAT(exit_null
) },
46 { "exit_validity", VCPU_STAT(exit_validity
) },
47 { "exit_stop_request", VCPU_STAT(exit_stop_request
) },
48 { "exit_external_request", VCPU_STAT(exit_external_request
) },
49 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt
) },
50 { "exit_instruction", VCPU_STAT(exit_instruction
) },
51 { "exit_program_interruption", VCPU_STAT(exit_program_interruption
) },
52 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program
) },
53 { "halt_successful_poll", VCPU_STAT(halt_successful_poll
) },
54 { "halt_wakeup", VCPU_STAT(halt_wakeup
) },
55 { "instruction_lctlg", VCPU_STAT(instruction_lctlg
) },
56 { "instruction_lctl", VCPU_STAT(instruction_lctl
) },
57 { "instruction_stctl", VCPU_STAT(instruction_stctl
) },
58 { "instruction_stctg", VCPU_STAT(instruction_stctg
) },
59 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal
) },
60 { "deliver_external_call", VCPU_STAT(deliver_external_call
) },
61 { "deliver_service_signal", VCPU_STAT(deliver_service_signal
) },
62 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt
) },
63 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal
) },
64 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal
) },
65 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal
) },
66 { "deliver_program_interruption", VCPU_STAT(deliver_program_int
) },
67 { "exit_wait_state", VCPU_STAT(exit_wait_state
) },
68 { "instruction_pfmf", VCPU_STAT(instruction_pfmf
) },
69 { "instruction_stidp", VCPU_STAT(instruction_stidp
) },
70 { "instruction_spx", VCPU_STAT(instruction_spx
) },
71 { "instruction_stpx", VCPU_STAT(instruction_stpx
) },
72 { "instruction_stap", VCPU_STAT(instruction_stap
) },
73 { "instruction_storage_key", VCPU_STAT(instruction_storage_key
) },
74 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock
) },
75 { "instruction_stsch", VCPU_STAT(instruction_stsch
) },
76 { "instruction_chsc", VCPU_STAT(instruction_chsc
) },
77 { "instruction_essa", VCPU_STAT(instruction_essa
) },
78 { "instruction_stsi", VCPU_STAT(instruction_stsi
) },
79 { "instruction_stfl", VCPU_STAT(instruction_stfl
) },
80 { "instruction_tprot", VCPU_STAT(instruction_tprot
) },
81 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense
) },
82 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running
) },
83 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call
) },
84 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency
) },
85 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency
) },
86 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start
) },
87 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop
) },
88 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status
) },
89 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status
) },
90 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status
) },
91 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch
) },
92 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix
) },
93 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart
) },
94 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset
) },
95 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset
) },
96 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown
) },
97 { "diagnose_10", VCPU_STAT(diagnose_10
) },
98 { "diagnose_44", VCPU_STAT(diagnose_44
) },
99 { "diagnose_9c", VCPU_STAT(diagnose_9c
) },
103 /* upper facilities limit for kvm */
104 unsigned long kvm_s390_fac_list_mask
[] = {
105 0xff82fffbf4fc2000UL
,
106 0x005c000000000000UL
,
109 unsigned long kvm_s390_fac_list_mask_size(void)
111 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask
) > S390_ARCH_FAC_MASK_SIZE_U64
);
112 return ARRAY_SIZE(kvm_s390_fac_list_mask
);
115 static struct gmap_notifier gmap_notifier
;
117 /* Section: not file related */
118 int kvm_arch_hardware_enable(void)
120 /* every s390 is virtualization enabled ;-) */
124 static void kvm_gmap_notifier(struct gmap
*gmap
, unsigned long address
);
126 int kvm_arch_hardware_setup(void)
128 gmap_notifier
.notifier_call
= kvm_gmap_notifier
;
129 gmap_register_ipte_notifier(&gmap_notifier
);
133 void kvm_arch_hardware_unsetup(void)
135 gmap_unregister_ipte_notifier(&gmap_notifier
);
138 int kvm_arch_init(void *opaque
)
140 /* Register floating interrupt controller interface. */
141 return kvm_register_device_ops(&kvm_flic_ops
, KVM_DEV_TYPE_FLIC
);
144 /* Section: device related */
145 long kvm_arch_dev_ioctl(struct file
*filp
,
146 unsigned int ioctl
, unsigned long arg
)
148 if (ioctl
== KVM_S390_ENABLE_SIE
)
149 return s390_enable_sie();
153 int kvm_vm_ioctl_check_extension(struct kvm
*kvm
, long ext
)
158 case KVM_CAP_S390_PSW
:
159 case KVM_CAP_S390_GMAP
:
160 case KVM_CAP_SYNC_MMU
:
161 #ifdef CONFIG_KVM_S390_UCONTROL
162 case KVM_CAP_S390_UCONTROL
:
164 case KVM_CAP_ASYNC_PF
:
165 case KVM_CAP_SYNC_REGS
:
166 case KVM_CAP_ONE_REG
:
167 case KVM_CAP_ENABLE_CAP
:
168 case KVM_CAP_S390_CSS_SUPPORT
:
170 case KVM_CAP_IOEVENTFD
:
171 case KVM_CAP_DEVICE_CTRL
:
172 case KVM_CAP_ENABLE_CAP_VM
:
173 case KVM_CAP_S390_IRQCHIP
:
174 case KVM_CAP_VM_ATTRIBUTES
:
175 case KVM_CAP_MP_STATE
:
176 case KVM_CAP_S390_USER_SIGP
:
179 case KVM_CAP_NR_VCPUS
:
180 case KVM_CAP_MAX_VCPUS
:
183 case KVM_CAP_NR_MEMSLOTS
:
184 r
= KVM_USER_MEM_SLOTS
;
186 case KVM_CAP_S390_COW
:
187 r
= MACHINE_HAS_ESOP
;
189 case KVM_CAP_S390_VECTOR_REGISTERS
:
198 static void kvm_s390_sync_dirty_log(struct kvm
*kvm
,
199 struct kvm_memory_slot
*memslot
)
201 gfn_t cur_gfn
, last_gfn
;
202 unsigned long address
;
203 struct gmap
*gmap
= kvm
->arch
.gmap
;
205 down_read(&gmap
->mm
->mmap_sem
);
206 /* Loop over all guest pages */
207 last_gfn
= memslot
->base_gfn
+ memslot
->npages
;
208 for (cur_gfn
= memslot
->base_gfn
; cur_gfn
<= last_gfn
; cur_gfn
++) {
209 address
= gfn_to_hva_memslot(memslot
, cur_gfn
);
211 if (gmap_test_and_clear_dirty(address
, gmap
))
212 mark_page_dirty(kvm
, cur_gfn
);
214 up_read(&gmap
->mm
->mmap_sem
);
217 /* Section: vm related */
219 * Get (and clear) the dirty memory log for a memory slot.
221 int kvm_vm_ioctl_get_dirty_log(struct kvm
*kvm
,
222 struct kvm_dirty_log
*log
)
226 struct kvm_memory_slot
*memslot
;
229 mutex_lock(&kvm
->slots_lock
);
232 if (log
->slot
>= KVM_USER_MEM_SLOTS
)
235 memslot
= id_to_memslot(kvm
->memslots
, log
->slot
);
237 if (!memslot
->dirty_bitmap
)
240 kvm_s390_sync_dirty_log(kvm
, memslot
);
241 r
= kvm_get_dirty_log(kvm
, log
, &is_dirty
);
245 /* Clear the dirty log */
247 n
= kvm_dirty_bitmap_bytes(memslot
);
248 memset(memslot
->dirty_bitmap
, 0, n
);
252 mutex_unlock(&kvm
->slots_lock
);
256 static int kvm_vm_ioctl_enable_cap(struct kvm
*kvm
, struct kvm_enable_cap
*cap
)
264 case KVM_CAP_S390_IRQCHIP
:
265 kvm
->arch
.use_irqchip
= 1;
268 case KVM_CAP_S390_USER_SIGP
:
269 kvm
->arch
.user_sigp
= 1;
272 case KVM_CAP_S390_VECTOR_REGISTERS
:
273 kvm
->arch
.use_vectors
= MACHINE_HAS_VX
;
274 r
= MACHINE_HAS_VX
? 0 : -EINVAL
;
283 static int kvm_s390_get_mem_control(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
287 switch (attr
->attr
) {
288 case KVM_S390_VM_MEM_LIMIT_SIZE
:
290 if (put_user(kvm
->arch
.gmap
->asce_end
, (u64 __user
*)attr
->addr
))
300 static int kvm_s390_set_mem_control(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
304 switch (attr
->attr
) {
305 case KVM_S390_VM_MEM_ENABLE_CMMA
:
307 mutex_lock(&kvm
->lock
);
308 if (atomic_read(&kvm
->online_vcpus
) == 0) {
309 kvm
->arch
.use_cmma
= 1;
312 mutex_unlock(&kvm
->lock
);
314 case KVM_S390_VM_MEM_CLR_CMMA
:
315 mutex_lock(&kvm
->lock
);
316 idx
= srcu_read_lock(&kvm
->srcu
);
317 s390_reset_cmma(kvm
->arch
.gmap
->mm
);
318 srcu_read_unlock(&kvm
->srcu
, idx
);
319 mutex_unlock(&kvm
->lock
);
322 case KVM_S390_VM_MEM_LIMIT_SIZE
: {
323 unsigned long new_limit
;
325 if (kvm_is_ucontrol(kvm
))
328 if (get_user(new_limit
, (u64 __user
*)attr
->addr
))
331 if (new_limit
> kvm
->arch
.gmap
->asce_end
)
335 mutex_lock(&kvm
->lock
);
336 if (atomic_read(&kvm
->online_vcpus
) == 0) {
337 /* gmap_alloc will round the limit up */
338 struct gmap
*new = gmap_alloc(current
->mm
, new_limit
);
343 gmap_free(kvm
->arch
.gmap
);
345 kvm
->arch
.gmap
= new;
349 mutex_unlock(&kvm
->lock
);
359 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu
*vcpu
);
361 static int kvm_s390_vm_set_crypto(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
363 struct kvm_vcpu
*vcpu
;
366 if (!test_kvm_facility(kvm
, 76))
369 mutex_lock(&kvm
->lock
);
370 switch (attr
->attr
) {
371 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW
:
373 kvm
->arch
.crypto
.crycb
->aes_wrapping_key_mask
,
374 sizeof(kvm
->arch
.crypto
.crycb
->aes_wrapping_key_mask
));
375 kvm
->arch
.crypto
.aes_kw
= 1;
377 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW
:
379 kvm
->arch
.crypto
.crycb
->dea_wrapping_key_mask
,
380 sizeof(kvm
->arch
.crypto
.crycb
->dea_wrapping_key_mask
));
381 kvm
->arch
.crypto
.dea_kw
= 1;
383 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW
:
384 kvm
->arch
.crypto
.aes_kw
= 0;
385 memset(kvm
->arch
.crypto
.crycb
->aes_wrapping_key_mask
, 0,
386 sizeof(kvm
->arch
.crypto
.crycb
->aes_wrapping_key_mask
));
388 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW
:
389 kvm
->arch
.crypto
.dea_kw
= 0;
390 memset(kvm
->arch
.crypto
.crycb
->dea_wrapping_key_mask
, 0,
391 sizeof(kvm
->arch
.crypto
.crycb
->dea_wrapping_key_mask
));
394 mutex_unlock(&kvm
->lock
);
398 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
399 kvm_s390_vcpu_crypto_setup(vcpu
);
402 mutex_unlock(&kvm
->lock
);
406 static int kvm_s390_set_tod_high(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
410 if (copy_from_user(>od_high
, (void __user
*)attr
->addr
,
420 static int kvm_s390_set_tod_low(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
422 struct kvm_vcpu
*cur_vcpu
;
423 unsigned int vcpu_idx
;
427 if (copy_from_user(>od
, (void __user
*)attr
->addr
, sizeof(gtod
)))
430 r
= store_tod_clock(&host_tod
);
434 mutex_lock(&kvm
->lock
);
435 kvm
->arch
.epoch
= gtod
- host_tod
;
436 kvm_for_each_vcpu(vcpu_idx
, cur_vcpu
, kvm
) {
437 cur_vcpu
->arch
.sie_block
->epoch
= kvm
->arch
.epoch
;
440 mutex_unlock(&kvm
->lock
);
444 static int kvm_s390_set_tod(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
451 switch (attr
->attr
) {
452 case KVM_S390_VM_TOD_HIGH
:
453 ret
= kvm_s390_set_tod_high(kvm
, attr
);
455 case KVM_S390_VM_TOD_LOW
:
456 ret
= kvm_s390_set_tod_low(kvm
, attr
);
465 static int kvm_s390_get_tod_high(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
469 if (copy_to_user((void __user
*)attr
->addr
, >od_high
,
476 static int kvm_s390_get_tod_low(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
481 r
= store_tod_clock(&host_tod
);
485 gtod
= host_tod
+ kvm
->arch
.epoch
;
486 if (copy_to_user((void __user
*)attr
->addr
, >od
, sizeof(gtod
)))
492 static int kvm_s390_get_tod(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
499 switch (attr
->attr
) {
500 case KVM_S390_VM_TOD_HIGH
:
501 ret
= kvm_s390_get_tod_high(kvm
, attr
);
503 case KVM_S390_VM_TOD_LOW
:
504 ret
= kvm_s390_get_tod_low(kvm
, attr
);
513 static int kvm_s390_set_processor(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
515 struct kvm_s390_vm_cpu_processor
*proc
;
518 mutex_lock(&kvm
->lock
);
519 if (atomic_read(&kvm
->online_vcpus
)) {
523 proc
= kzalloc(sizeof(*proc
), GFP_KERNEL
);
528 if (!copy_from_user(proc
, (void __user
*)attr
->addr
,
530 memcpy(&kvm
->arch
.model
.cpu_id
, &proc
->cpuid
,
531 sizeof(struct cpuid
));
532 kvm
->arch
.model
.ibc
= proc
->ibc
;
533 memcpy(kvm
->arch
.model
.fac
->list
, proc
->fac_list
,
534 S390_ARCH_FAC_LIST_SIZE_BYTE
);
539 mutex_unlock(&kvm
->lock
);
543 static int kvm_s390_set_cpu_model(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
547 switch (attr
->attr
) {
548 case KVM_S390_VM_CPU_PROCESSOR
:
549 ret
= kvm_s390_set_processor(kvm
, attr
);
555 static int kvm_s390_get_processor(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
557 struct kvm_s390_vm_cpu_processor
*proc
;
560 proc
= kzalloc(sizeof(*proc
), GFP_KERNEL
);
565 memcpy(&proc
->cpuid
, &kvm
->arch
.model
.cpu_id
, sizeof(struct cpuid
));
566 proc
->ibc
= kvm
->arch
.model
.ibc
;
567 memcpy(&proc
->fac_list
, kvm
->arch
.model
.fac
->list
, S390_ARCH_FAC_LIST_SIZE_BYTE
);
568 if (copy_to_user((void __user
*)attr
->addr
, proc
, sizeof(*proc
)))
575 static int kvm_s390_get_machine(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
577 struct kvm_s390_vm_cpu_machine
*mach
;
580 mach
= kzalloc(sizeof(*mach
), GFP_KERNEL
);
585 get_cpu_id((struct cpuid
*) &mach
->cpuid
);
586 mach
->ibc
= sclp_get_ibc();
587 memcpy(&mach
->fac_mask
, kvm
->arch
.model
.fac
->mask
,
588 S390_ARCH_FAC_LIST_SIZE_BYTE
);
589 memcpy((unsigned long *)&mach
->fac_list
, S390_lowcore
.stfle_fac_list
,
590 S390_ARCH_FAC_LIST_SIZE_BYTE
);
591 if (copy_to_user((void __user
*)attr
->addr
, mach
, sizeof(*mach
)))
598 static int kvm_s390_get_cpu_model(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
602 switch (attr
->attr
) {
603 case KVM_S390_VM_CPU_PROCESSOR
:
604 ret
= kvm_s390_get_processor(kvm
, attr
);
606 case KVM_S390_VM_CPU_MACHINE
:
607 ret
= kvm_s390_get_machine(kvm
, attr
);
613 static int kvm_s390_vm_set_attr(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
617 switch (attr
->group
) {
618 case KVM_S390_VM_MEM_CTRL
:
619 ret
= kvm_s390_set_mem_control(kvm
, attr
);
621 case KVM_S390_VM_TOD
:
622 ret
= kvm_s390_set_tod(kvm
, attr
);
624 case KVM_S390_VM_CPU_MODEL
:
625 ret
= kvm_s390_set_cpu_model(kvm
, attr
);
627 case KVM_S390_VM_CRYPTO
:
628 ret
= kvm_s390_vm_set_crypto(kvm
, attr
);
638 static int kvm_s390_vm_get_attr(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
642 switch (attr
->group
) {
643 case KVM_S390_VM_MEM_CTRL
:
644 ret
= kvm_s390_get_mem_control(kvm
, attr
);
646 case KVM_S390_VM_TOD
:
647 ret
= kvm_s390_get_tod(kvm
, attr
);
649 case KVM_S390_VM_CPU_MODEL
:
650 ret
= kvm_s390_get_cpu_model(kvm
, attr
);
660 static int kvm_s390_vm_has_attr(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
664 switch (attr
->group
) {
665 case KVM_S390_VM_MEM_CTRL
:
666 switch (attr
->attr
) {
667 case KVM_S390_VM_MEM_ENABLE_CMMA
:
668 case KVM_S390_VM_MEM_CLR_CMMA
:
669 case KVM_S390_VM_MEM_LIMIT_SIZE
:
677 case KVM_S390_VM_TOD
:
678 switch (attr
->attr
) {
679 case KVM_S390_VM_TOD_LOW
:
680 case KVM_S390_VM_TOD_HIGH
:
688 case KVM_S390_VM_CPU_MODEL
:
689 switch (attr
->attr
) {
690 case KVM_S390_VM_CPU_PROCESSOR
:
691 case KVM_S390_VM_CPU_MACHINE
:
699 case KVM_S390_VM_CRYPTO
:
700 switch (attr
->attr
) {
701 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW
:
702 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW
:
703 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW
:
704 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW
:
720 long kvm_arch_vm_ioctl(struct file
*filp
,
721 unsigned int ioctl
, unsigned long arg
)
723 struct kvm
*kvm
= filp
->private_data
;
724 void __user
*argp
= (void __user
*)arg
;
725 struct kvm_device_attr attr
;
729 case KVM_S390_INTERRUPT
: {
730 struct kvm_s390_interrupt s390int
;
733 if (copy_from_user(&s390int
, argp
, sizeof(s390int
)))
735 r
= kvm_s390_inject_vm(kvm
, &s390int
);
738 case KVM_ENABLE_CAP
: {
739 struct kvm_enable_cap cap
;
741 if (copy_from_user(&cap
, argp
, sizeof(cap
)))
743 r
= kvm_vm_ioctl_enable_cap(kvm
, &cap
);
746 case KVM_CREATE_IRQCHIP
: {
747 struct kvm_irq_routing_entry routing
;
750 if (kvm
->arch
.use_irqchip
) {
751 /* Set up dummy routing. */
752 memset(&routing
, 0, sizeof(routing
));
753 kvm_set_irq_routing(kvm
, &routing
, 0, 0);
758 case KVM_SET_DEVICE_ATTR
: {
760 if (copy_from_user(&attr
, (void __user
*)arg
, sizeof(attr
)))
762 r
= kvm_s390_vm_set_attr(kvm
, &attr
);
765 case KVM_GET_DEVICE_ATTR
: {
767 if (copy_from_user(&attr
, (void __user
*)arg
, sizeof(attr
)))
769 r
= kvm_s390_vm_get_attr(kvm
, &attr
);
772 case KVM_HAS_DEVICE_ATTR
: {
774 if (copy_from_user(&attr
, (void __user
*)arg
, sizeof(attr
)))
776 r
= kvm_s390_vm_has_attr(kvm
, &attr
);
786 static int kvm_s390_query_ap_config(u8
*config
)
788 u32 fcn_code
= 0x04000000UL
;
791 memset(config
, 0, 128);
795 ".long 0xb2af0000\n" /* PQAP(QCI) */
801 : "r" (fcn_code
), "r" (config
)
802 : "cc", "0", "2", "memory"
808 static int kvm_s390_apxa_installed(void)
813 if (test_facility(2) && test_facility(12)) {
814 cc
= kvm_s390_query_ap_config(config
);
817 pr_err("PQAP(QCI) failed with cc=%d", cc
);
819 return config
[0] & 0x40;
825 static void kvm_s390_set_crycb_format(struct kvm
*kvm
)
827 kvm
->arch
.crypto
.crycbd
= (__u32
)(unsigned long) kvm
->arch
.crypto
.crycb
;
829 if (kvm_s390_apxa_installed())
830 kvm
->arch
.crypto
.crycbd
|= CRYCB_FORMAT2
;
832 kvm
->arch
.crypto
.crycbd
|= CRYCB_FORMAT1
;
835 static void kvm_s390_get_cpu_id(struct cpuid
*cpu_id
)
838 cpu_id
->version
= 0xff;
841 static int kvm_s390_crypto_init(struct kvm
*kvm
)
843 if (!test_kvm_facility(kvm
, 76))
846 kvm
->arch
.crypto
.crycb
= kzalloc(sizeof(*kvm
->arch
.crypto
.crycb
),
847 GFP_KERNEL
| GFP_DMA
);
848 if (!kvm
->arch
.crypto
.crycb
)
851 kvm_s390_set_crycb_format(kvm
);
853 /* Enable AES/DEA protected key functions by default */
854 kvm
->arch
.crypto
.aes_kw
= 1;
855 kvm
->arch
.crypto
.dea_kw
= 1;
856 get_random_bytes(kvm
->arch
.crypto
.crycb
->aes_wrapping_key_mask
,
857 sizeof(kvm
->arch
.crypto
.crycb
->aes_wrapping_key_mask
));
858 get_random_bytes(kvm
->arch
.crypto
.crycb
->dea_wrapping_key_mask
,
859 sizeof(kvm
->arch
.crypto
.crycb
->dea_wrapping_key_mask
));
864 int kvm_arch_init_vm(struct kvm
*kvm
, unsigned long type
)
868 static unsigned long sca_offset
;
871 #ifdef CONFIG_KVM_S390_UCONTROL
872 if (type
& ~KVM_VM_S390_UCONTROL
)
874 if ((type
& KVM_VM_S390_UCONTROL
) && (!capable(CAP_SYS_ADMIN
)))
881 rc
= s390_enable_sie();
887 kvm
->arch
.sca
= (struct sca_block
*) get_zeroed_page(GFP_KERNEL
);
890 spin_lock(&kvm_lock
);
891 sca_offset
= (sca_offset
+ 16) & 0x7f0;
892 kvm
->arch
.sca
= (struct sca_block
*) ((char *) kvm
->arch
.sca
+ sca_offset
);
893 spin_unlock(&kvm_lock
);
895 sprintf(debug_name
, "kvm-%u", current
->pid
);
897 kvm
->arch
.dbf
= debug_register(debug_name
, 8, 2, 8 * sizeof(long));
902 * The architectural maximum amount of facilities is 16 kbit. To store
903 * this amount, 2 kbyte of memory is required. Thus we need a full
904 * page to hold the guest facility list (arch.model.fac->list) and the
905 * facility mask (arch.model.fac->mask). Its address size has to be
906 * 31 bits and word aligned.
908 kvm
->arch
.model
.fac
=
909 (struct kvm_s390_fac
*) get_zeroed_page(GFP_KERNEL
| GFP_DMA
);
910 if (!kvm
->arch
.model
.fac
)
913 /* Populate the facility mask initially. */
914 memcpy(kvm
->arch
.model
.fac
->mask
, S390_lowcore
.stfle_fac_list
,
915 S390_ARCH_FAC_LIST_SIZE_BYTE
);
916 for (i
= 0; i
< S390_ARCH_FAC_LIST_SIZE_U64
; i
++) {
917 if (i
< kvm_s390_fac_list_mask_size())
918 kvm
->arch
.model
.fac
->mask
[i
] &= kvm_s390_fac_list_mask
[i
];
920 kvm
->arch
.model
.fac
->mask
[i
] = 0UL;
923 /* Populate the facility list initially. */
924 memcpy(kvm
->arch
.model
.fac
->list
, kvm
->arch
.model
.fac
->mask
,
925 S390_ARCH_FAC_LIST_SIZE_BYTE
);
927 kvm_s390_get_cpu_id(&kvm
->arch
.model
.cpu_id
);
928 kvm
->arch
.model
.ibc
= sclp_get_ibc() & 0x0fff;
930 if (kvm_s390_crypto_init(kvm
) < 0)
933 spin_lock_init(&kvm
->arch
.float_int
.lock
);
934 INIT_LIST_HEAD(&kvm
->arch
.float_int
.list
);
935 init_waitqueue_head(&kvm
->arch
.ipte_wq
);
936 mutex_init(&kvm
->arch
.ipte_mutex
);
938 debug_register_view(kvm
->arch
.dbf
, &debug_sprintf_view
);
939 VM_EVENT(kvm
, 3, "%s", "vm created");
941 if (type
& KVM_VM_S390_UCONTROL
) {
942 kvm
->arch
.gmap
= NULL
;
944 kvm
->arch
.gmap
= gmap_alloc(current
->mm
, (1UL << 44) - 1);
947 kvm
->arch
.gmap
->private = kvm
;
948 kvm
->arch
.gmap
->pfault_enabled
= 0;
951 kvm
->arch
.css_support
= 0;
952 kvm
->arch
.use_irqchip
= 0;
953 kvm
->arch
.use_vectors
= 0;
956 spin_lock_init(&kvm
->arch
.start_stop_lock
);
960 kfree(kvm
->arch
.crypto
.crycb
);
962 free_page((unsigned long)kvm
->arch
.model
.fac
);
964 debug_unregister(kvm
->arch
.dbf
);
966 free_page((unsigned long)(kvm
->arch
.sca
));
971 void kvm_arch_vcpu_destroy(struct kvm_vcpu
*vcpu
)
973 VCPU_EVENT(vcpu
, 3, "%s", "free cpu");
974 trace_kvm_s390_destroy_vcpu(vcpu
->vcpu_id
);
975 kvm_s390_clear_local_irqs(vcpu
);
976 kvm_clear_async_pf_completion_queue(vcpu
);
977 if (!kvm_is_ucontrol(vcpu
->kvm
)) {
978 clear_bit(63 - vcpu
->vcpu_id
,
979 (unsigned long *) &vcpu
->kvm
->arch
.sca
->mcn
);
980 if (vcpu
->kvm
->arch
.sca
->cpu
[vcpu
->vcpu_id
].sda
==
981 (__u64
) vcpu
->arch
.sie_block
)
982 vcpu
->kvm
->arch
.sca
->cpu
[vcpu
->vcpu_id
].sda
= 0;
986 if (kvm_is_ucontrol(vcpu
->kvm
))
987 gmap_free(vcpu
->arch
.gmap
);
989 if (kvm_s390_cmma_enabled(vcpu
->kvm
))
990 kvm_s390_vcpu_unsetup_cmma(vcpu
);
991 free_page((unsigned long)(vcpu
->arch
.sie_block
));
993 kvm_vcpu_uninit(vcpu
);
994 kmem_cache_free(kvm_vcpu_cache
, vcpu
);
997 static void kvm_free_vcpus(struct kvm
*kvm
)
1000 struct kvm_vcpu
*vcpu
;
1002 kvm_for_each_vcpu(i
, vcpu
, kvm
)
1003 kvm_arch_vcpu_destroy(vcpu
);
1005 mutex_lock(&kvm
->lock
);
1006 for (i
= 0; i
< atomic_read(&kvm
->online_vcpus
); i
++)
1007 kvm
->vcpus
[i
] = NULL
;
1009 atomic_set(&kvm
->online_vcpus
, 0);
1010 mutex_unlock(&kvm
->lock
);
1013 void kvm_arch_destroy_vm(struct kvm
*kvm
)
1015 kvm_free_vcpus(kvm
);
1016 free_page((unsigned long)kvm
->arch
.model
.fac
);
1017 free_page((unsigned long)(kvm
->arch
.sca
));
1018 debug_unregister(kvm
->arch
.dbf
);
1019 kfree(kvm
->arch
.crypto
.crycb
);
1020 if (!kvm_is_ucontrol(kvm
))
1021 gmap_free(kvm
->arch
.gmap
);
1022 kvm_s390_destroy_adapters(kvm
);
1023 kvm_s390_clear_float_irqs(kvm
);
1026 /* Section: vcpu related */
1027 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu
*vcpu
)
1029 vcpu
->arch
.gmap
= gmap_alloc(current
->mm
, -1UL);
1030 if (!vcpu
->arch
.gmap
)
1032 vcpu
->arch
.gmap
->private = vcpu
->kvm
;
1037 int kvm_arch_vcpu_init(struct kvm_vcpu
*vcpu
)
1039 vcpu
->arch
.pfault_token
= KVM_S390_PFAULT_TOKEN_INVALID
;
1040 kvm_clear_async_pf_completion_queue(vcpu
);
1041 vcpu
->run
->kvm_valid_regs
= KVM_SYNC_PREFIX
|
1047 if (test_kvm_facility(vcpu
->kvm
, 129))
1048 vcpu
->run
->kvm_valid_regs
|= KVM_SYNC_VRS
;
1050 if (kvm_is_ucontrol(vcpu
->kvm
))
1051 return __kvm_ucontrol_vcpu_init(vcpu
);
1056 void kvm_arch_vcpu_load(struct kvm_vcpu
*vcpu
, int cpu
)
1058 save_fp_ctl(&vcpu
->arch
.host_fpregs
.fpc
);
1059 if (vcpu
->kvm
->arch
.use_vectors
)
1060 save_vx_regs((__vector128
*)&vcpu
->arch
.host_vregs
->vrs
);
1062 save_fp_regs(vcpu
->arch
.host_fpregs
.fprs
);
1063 save_access_regs(vcpu
->arch
.host_acrs
);
1064 if (vcpu
->kvm
->arch
.use_vectors
) {
1065 restore_fp_ctl(&vcpu
->run
->s
.regs
.fpc
);
1066 restore_vx_regs((__vector128
*)&vcpu
->run
->s
.regs
.vrs
);
1068 restore_fp_ctl(&vcpu
->arch
.guest_fpregs
.fpc
);
1069 restore_fp_regs(vcpu
->arch
.guest_fpregs
.fprs
);
1071 restore_access_regs(vcpu
->run
->s
.regs
.acrs
);
1072 gmap_enable(vcpu
->arch
.gmap
);
1073 atomic_set_mask(CPUSTAT_RUNNING
, &vcpu
->arch
.sie_block
->cpuflags
);
1076 void kvm_arch_vcpu_put(struct kvm_vcpu
*vcpu
)
1078 atomic_clear_mask(CPUSTAT_RUNNING
, &vcpu
->arch
.sie_block
->cpuflags
);
1079 gmap_disable(vcpu
->arch
.gmap
);
1080 if (vcpu
->kvm
->arch
.use_vectors
) {
1081 save_fp_ctl(&vcpu
->run
->s
.regs
.fpc
);
1082 save_vx_regs((__vector128
*)&vcpu
->run
->s
.regs
.vrs
);
1084 save_fp_ctl(&vcpu
->arch
.guest_fpregs
.fpc
);
1085 save_fp_regs(vcpu
->arch
.guest_fpregs
.fprs
);
1087 save_access_regs(vcpu
->run
->s
.regs
.acrs
);
1088 restore_fp_ctl(&vcpu
->arch
.host_fpregs
.fpc
);
1089 if (vcpu
->kvm
->arch
.use_vectors
)
1090 restore_vx_regs((__vector128
*)&vcpu
->arch
.host_vregs
->vrs
);
1092 restore_fp_regs(vcpu
->arch
.host_fpregs
.fprs
);
1093 restore_access_regs(vcpu
->arch
.host_acrs
);
1096 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu
*vcpu
)
1098 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1099 vcpu
->arch
.sie_block
->gpsw
.mask
= 0UL;
1100 vcpu
->arch
.sie_block
->gpsw
.addr
= 0UL;
1101 kvm_s390_set_prefix(vcpu
, 0);
1102 vcpu
->arch
.sie_block
->cputm
= 0UL;
1103 vcpu
->arch
.sie_block
->ckc
= 0UL;
1104 vcpu
->arch
.sie_block
->todpr
= 0;
1105 memset(vcpu
->arch
.sie_block
->gcr
, 0, 16 * sizeof(__u64
));
1106 vcpu
->arch
.sie_block
->gcr
[0] = 0xE0UL
;
1107 vcpu
->arch
.sie_block
->gcr
[14] = 0xC2000000UL
;
1108 vcpu
->arch
.guest_fpregs
.fpc
= 0;
1109 asm volatile("lfpc %0" : : "Q" (vcpu
->arch
.guest_fpregs
.fpc
));
1110 vcpu
->arch
.sie_block
->gbea
= 1;
1111 vcpu
->arch
.sie_block
->pp
= 0;
1112 vcpu
->arch
.pfault_token
= KVM_S390_PFAULT_TOKEN_INVALID
;
1113 kvm_clear_async_pf_completion_queue(vcpu
);
1114 if (!kvm_s390_user_cpu_state_ctrl(vcpu
->kvm
))
1115 kvm_s390_vcpu_stop(vcpu
);
1116 kvm_s390_clear_local_irqs(vcpu
);
1119 void kvm_arch_vcpu_postcreate(struct kvm_vcpu
*vcpu
)
1121 mutex_lock(&vcpu
->kvm
->lock
);
1122 vcpu
->arch
.sie_block
->epoch
= vcpu
->kvm
->arch
.epoch
;
1123 mutex_unlock(&vcpu
->kvm
->lock
);
1124 if (!kvm_is_ucontrol(vcpu
->kvm
))
1125 vcpu
->arch
.gmap
= vcpu
->kvm
->arch
.gmap
;
1128 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu
*vcpu
)
1130 if (!test_kvm_facility(vcpu
->kvm
, 76))
1133 vcpu
->arch
.sie_block
->ecb3
&= ~(ECB3_AES
| ECB3_DEA
);
1135 if (vcpu
->kvm
->arch
.crypto
.aes_kw
)
1136 vcpu
->arch
.sie_block
->ecb3
|= ECB3_AES
;
1137 if (vcpu
->kvm
->arch
.crypto
.dea_kw
)
1138 vcpu
->arch
.sie_block
->ecb3
|= ECB3_DEA
;
1140 vcpu
->arch
.sie_block
->crycbd
= vcpu
->kvm
->arch
.crypto
.crycbd
;
1143 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu
*vcpu
)
1145 free_page(vcpu
->arch
.sie_block
->cbrlo
);
1146 vcpu
->arch
.sie_block
->cbrlo
= 0;
1149 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu
*vcpu
)
1151 vcpu
->arch
.sie_block
->cbrlo
= get_zeroed_page(GFP_KERNEL
);
1152 if (!vcpu
->arch
.sie_block
->cbrlo
)
1155 vcpu
->arch
.sie_block
->ecb2
|= 0x80;
1156 vcpu
->arch
.sie_block
->ecb2
&= ~0x08;
1160 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu
*vcpu
)
1162 struct kvm_s390_cpu_model
*model
= &vcpu
->kvm
->arch
.model
;
1164 vcpu
->arch
.cpu_id
= model
->cpu_id
;
1165 vcpu
->arch
.sie_block
->ibc
= model
->ibc
;
1166 vcpu
->arch
.sie_block
->fac
= (int) (long) model
->fac
->list
;
1169 int kvm_arch_vcpu_setup(struct kvm_vcpu
*vcpu
)
1173 atomic_set(&vcpu
->arch
.sie_block
->cpuflags
, CPUSTAT_ZARCH
|
1177 kvm_s390_vcpu_setup_model(vcpu
);
1179 vcpu
->arch
.sie_block
->ecb
= 6;
1180 if (test_kvm_facility(vcpu
->kvm
, 50) && test_kvm_facility(vcpu
->kvm
, 73))
1181 vcpu
->arch
.sie_block
->ecb
|= 0x10;
1183 vcpu
->arch
.sie_block
->ecb2
= 8;
1184 vcpu
->arch
.sie_block
->eca
= 0xC1002000U
;
1185 if (sclp_has_siif())
1186 vcpu
->arch
.sie_block
->eca
|= 1;
1187 if (sclp_has_sigpif())
1188 vcpu
->arch
.sie_block
->eca
|= 0x10000000U
;
1189 vcpu
->arch
.sie_block
->ictl
|= ICTL_ISKE
| ICTL_SSKE
| ICTL_RRBE
;
1191 if (kvm_s390_cmma_enabled(vcpu
->kvm
)) {
1192 rc
= kvm_s390_vcpu_setup_cmma(vcpu
);
1196 hrtimer_init(&vcpu
->arch
.ckc_timer
, CLOCK_MONOTONIC
, HRTIMER_MODE_REL
);
1197 vcpu
->arch
.ckc_timer
.function
= kvm_s390_idle_wakeup
;
1199 kvm_s390_vcpu_crypto_setup(vcpu
);
1204 struct kvm_vcpu
*kvm_arch_vcpu_create(struct kvm
*kvm
,
1207 struct kvm_vcpu
*vcpu
;
1208 struct sie_page
*sie_page
;
1211 if (id
>= KVM_MAX_VCPUS
)
1216 vcpu
= kmem_cache_zalloc(kvm_vcpu_cache
, GFP_KERNEL
);
1220 sie_page
= (struct sie_page
*) get_zeroed_page(GFP_KERNEL
);
1224 vcpu
->arch
.sie_block
= &sie_page
->sie_block
;
1225 vcpu
->arch
.sie_block
->itdba
= (unsigned long) &sie_page
->itdb
;
1226 vcpu
->arch
.host_vregs
= &sie_page
->vregs
;
1228 vcpu
->arch
.sie_block
->icpua
= id
;
1229 if (!kvm_is_ucontrol(kvm
)) {
1230 if (!kvm
->arch
.sca
) {
1234 if (!kvm
->arch
.sca
->cpu
[id
].sda
)
1235 kvm
->arch
.sca
->cpu
[id
].sda
=
1236 (__u64
) vcpu
->arch
.sie_block
;
1237 vcpu
->arch
.sie_block
->scaoh
=
1238 (__u32
)(((__u64
)kvm
->arch
.sca
) >> 32);
1239 vcpu
->arch
.sie_block
->scaol
= (__u32
)(__u64
)kvm
->arch
.sca
;
1240 set_bit(63 - id
, (unsigned long *) &kvm
->arch
.sca
->mcn
);
1243 spin_lock_init(&vcpu
->arch
.local_int
.lock
);
1244 vcpu
->arch
.local_int
.float_int
= &kvm
->arch
.float_int
;
1245 vcpu
->arch
.local_int
.wq
= &vcpu
->wq
;
1246 vcpu
->arch
.local_int
.cpuflags
= &vcpu
->arch
.sie_block
->cpuflags
;
1248 rc
= kvm_vcpu_init(vcpu
, kvm
, id
);
1250 goto out_free_sie_block
;
1251 VM_EVENT(kvm
, 3, "create cpu %d at %p, sie block at %p", id
, vcpu
,
1252 vcpu
->arch
.sie_block
);
1253 trace_kvm_s390_create_vcpu(id
, vcpu
, vcpu
->arch
.sie_block
);
1257 free_page((unsigned long)(vcpu
->arch
.sie_block
));
1259 kmem_cache_free(kvm_vcpu_cache
, vcpu
);
1264 int kvm_arch_vcpu_runnable(struct kvm_vcpu
*vcpu
)
1266 return kvm_s390_vcpu_has_irq(vcpu
, 0);
1269 void s390_vcpu_block(struct kvm_vcpu
*vcpu
)
1271 atomic_set_mask(PROG_BLOCK_SIE
, &vcpu
->arch
.sie_block
->prog20
);
1274 void s390_vcpu_unblock(struct kvm_vcpu
*vcpu
)
1276 atomic_clear_mask(PROG_BLOCK_SIE
, &vcpu
->arch
.sie_block
->prog20
);
1280 * Kick a guest cpu out of SIE and wait until SIE is not running.
1281 * If the CPU is not running (e.g. waiting as idle) the function will
1282 * return immediately. */
1283 void exit_sie(struct kvm_vcpu
*vcpu
)
1285 atomic_set_mask(CPUSTAT_STOP_INT
, &vcpu
->arch
.sie_block
->cpuflags
);
1286 while (vcpu
->arch
.sie_block
->prog0c
& PROG_IN_SIE
)
1290 /* Kick a guest cpu out of SIE and prevent SIE-reentry */
1291 void exit_sie_sync(struct kvm_vcpu
*vcpu
)
1293 s390_vcpu_block(vcpu
);
1297 static void kvm_gmap_notifier(struct gmap
*gmap
, unsigned long address
)
1300 struct kvm
*kvm
= gmap
->private;
1301 struct kvm_vcpu
*vcpu
;
1303 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
1304 /* match against both prefix pages */
1305 if (kvm_s390_get_prefix(vcpu
) == (address
& ~0x1000UL
)) {
1306 VCPU_EVENT(vcpu
, 2, "gmap notifier for %lx", address
);
1307 kvm_make_request(KVM_REQ_MMU_RELOAD
, vcpu
);
1308 exit_sie_sync(vcpu
);
1313 int kvm_arch_vcpu_should_kick(struct kvm_vcpu
*vcpu
)
1315 /* kvm common code refers to this, but never calls it */
1320 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu
*vcpu
,
1321 struct kvm_one_reg
*reg
)
1326 case KVM_REG_S390_TODPR
:
1327 r
= put_user(vcpu
->arch
.sie_block
->todpr
,
1328 (u32 __user
*)reg
->addr
);
1330 case KVM_REG_S390_EPOCHDIFF
:
1331 r
= put_user(vcpu
->arch
.sie_block
->epoch
,
1332 (u64 __user
*)reg
->addr
);
1334 case KVM_REG_S390_CPU_TIMER
:
1335 r
= put_user(vcpu
->arch
.sie_block
->cputm
,
1336 (u64 __user
*)reg
->addr
);
1338 case KVM_REG_S390_CLOCK_COMP
:
1339 r
= put_user(vcpu
->arch
.sie_block
->ckc
,
1340 (u64 __user
*)reg
->addr
);
1342 case KVM_REG_S390_PFTOKEN
:
1343 r
= put_user(vcpu
->arch
.pfault_token
,
1344 (u64 __user
*)reg
->addr
);
1346 case KVM_REG_S390_PFCOMPARE
:
1347 r
= put_user(vcpu
->arch
.pfault_compare
,
1348 (u64 __user
*)reg
->addr
);
1350 case KVM_REG_S390_PFSELECT
:
1351 r
= put_user(vcpu
->arch
.pfault_select
,
1352 (u64 __user
*)reg
->addr
);
1354 case KVM_REG_S390_PP
:
1355 r
= put_user(vcpu
->arch
.sie_block
->pp
,
1356 (u64 __user
*)reg
->addr
);
1358 case KVM_REG_S390_GBEA
:
1359 r
= put_user(vcpu
->arch
.sie_block
->gbea
,
1360 (u64 __user
*)reg
->addr
);
1369 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu
*vcpu
,
1370 struct kvm_one_reg
*reg
)
1375 case KVM_REG_S390_TODPR
:
1376 r
= get_user(vcpu
->arch
.sie_block
->todpr
,
1377 (u32 __user
*)reg
->addr
);
1379 case KVM_REG_S390_EPOCHDIFF
:
1380 r
= get_user(vcpu
->arch
.sie_block
->epoch
,
1381 (u64 __user
*)reg
->addr
);
1383 case KVM_REG_S390_CPU_TIMER
:
1384 r
= get_user(vcpu
->arch
.sie_block
->cputm
,
1385 (u64 __user
*)reg
->addr
);
1387 case KVM_REG_S390_CLOCK_COMP
:
1388 r
= get_user(vcpu
->arch
.sie_block
->ckc
,
1389 (u64 __user
*)reg
->addr
);
1391 case KVM_REG_S390_PFTOKEN
:
1392 r
= get_user(vcpu
->arch
.pfault_token
,
1393 (u64 __user
*)reg
->addr
);
1394 if (vcpu
->arch
.pfault_token
== KVM_S390_PFAULT_TOKEN_INVALID
)
1395 kvm_clear_async_pf_completion_queue(vcpu
);
1397 case KVM_REG_S390_PFCOMPARE
:
1398 r
= get_user(vcpu
->arch
.pfault_compare
,
1399 (u64 __user
*)reg
->addr
);
1401 case KVM_REG_S390_PFSELECT
:
1402 r
= get_user(vcpu
->arch
.pfault_select
,
1403 (u64 __user
*)reg
->addr
);
1405 case KVM_REG_S390_PP
:
1406 r
= get_user(vcpu
->arch
.sie_block
->pp
,
1407 (u64 __user
*)reg
->addr
);
1409 case KVM_REG_S390_GBEA
:
1410 r
= get_user(vcpu
->arch
.sie_block
->gbea
,
1411 (u64 __user
*)reg
->addr
);
1420 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu
*vcpu
)
1422 kvm_s390_vcpu_initial_reset(vcpu
);
1426 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu
*vcpu
, struct kvm_regs
*regs
)
1428 memcpy(&vcpu
->run
->s
.regs
.gprs
, ®s
->gprs
, sizeof(regs
->gprs
));
1432 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu
*vcpu
, struct kvm_regs
*regs
)
1434 memcpy(®s
->gprs
, &vcpu
->run
->s
.regs
.gprs
, sizeof(regs
->gprs
));
1438 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu
*vcpu
,
1439 struct kvm_sregs
*sregs
)
1441 memcpy(&vcpu
->run
->s
.regs
.acrs
, &sregs
->acrs
, sizeof(sregs
->acrs
));
1442 memcpy(&vcpu
->arch
.sie_block
->gcr
, &sregs
->crs
, sizeof(sregs
->crs
));
1443 restore_access_regs(vcpu
->run
->s
.regs
.acrs
);
1447 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu
*vcpu
,
1448 struct kvm_sregs
*sregs
)
1450 memcpy(&sregs
->acrs
, &vcpu
->run
->s
.regs
.acrs
, sizeof(sregs
->acrs
));
1451 memcpy(&sregs
->crs
, &vcpu
->arch
.sie_block
->gcr
, sizeof(sregs
->crs
));
1455 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu
*vcpu
, struct kvm_fpu
*fpu
)
1457 if (test_fp_ctl(fpu
->fpc
))
1459 memcpy(&vcpu
->arch
.guest_fpregs
.fprs
, &fpu
->fprs
, sizeof(fpu
->fprs
));
1460 vcpu
->arch
.guest_fpregs
.fpc
= fpu
->fpc
;
1461 restore_fp_ctl(&vcpu
->arch
.guest_fpregs
.fpc
);
1462 restore_fp_regs(vcpu
->arch
.guest_fpregs
.fprs
);
1466 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu
*vcpu
, struct kvm_fpu
*fpu
)
1468 memcpy(&fpu
->fprs
, &vcpu
->arch
.guest_fpregs
.fprs
, sizeof(fpu
->fprs
));
1469 fpu
->fpc
= vcpu
->arch
.guest_fpregs
.fpc
;
1473 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu
*vcpu
, psw_t psw
)
1477 if (!is_vcpu_stopped(vcpu
))
1480 vcpu
->run
->psw_mask
= psw
.mask
;
1481 vcpu
->run
->psw_addr
= psw
.addr
;
1486 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu
*vcpu
,
1487 struct kvm_translation
*tr
)
1489 return -EINVAL
; /* not implemented yet */
1492 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1493 KVM_GUESTDBG_USE_HW_BP | \
1494 KVM_GUESTDBG_ENABLE)
1496 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu
*vcpu
,
1497 struct kvm_guest_debug
*dbg
)
1501 vcpu
->guest_debug
= 0;
1502 kvm_s390_clear_bp_data(vcpu
);
1504 if (dbg
->control
& ~VALID_GUESTDBG_FLAGS
)
1507 if (dbg
->control
& KVM_GUESTDBG_ENABLE
) {
1508 vcpu
->guest_debug
= dbg
->control
;
1509 /* enforce guest PER */
1510 atomic_set_mask(CPUSTAT_P
, &vcpu
->arch
.sie_block
->cpuflags
);
1512 if (dbg
->control
& KVM_GUESTDBG_USE_HW_BP
)
1513 rc
= kvm_s390_import_bp_data(vcpu
, dbg
);
1515 atomic_clear_mask(CPUSTAT_P
, &vcpu
->arch
.sie_block
->cpuflags
);
1516 vcpu
->arch
.guestdbg
.last_bp
= 0;
1520 vcpu
->guest_debug
= 0;
1521 kvm_s390_clear_bp_data(vcpu
);
1522 atomic_clear_mask(CPUSTAT_P
, &vcpu
->arch
.sie_block
->cpuflags
);
1528 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu
*vcpu
,
1529 struct kvm_mp_state
*mp_state
)
1531 /* CHECK_STOP and LOAD are not supported yet */
1532 return is_vcpu_stopped(vcpu
) ? KVM_MP_STATE_STOPPED
:
1533 KVM_MP_STATE_OPERATING
;
1536 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu
*vcpu
,
1537 struct kvm_mp_state
*mp_state
)
1541 /* user space knows about this interface - let it control the state */
1542 vcpu
->kvm
->arch
.user_cpu_state_ctrl
= 1;
1544 switch (mp_state
->mp_state
) {
1545 case KVM_MP_STATE_STOPPED
:
1546 kvm_s390_vcpu_stop(vcpu
);
1548 case KVM_MP_STATE_OPERATING
:
1549 kvm_s390_vcpu_start(vcpu
);
1551 case KVM_MP_STATE_LOAD
:
1552 case KVM_MP_STATE_CHECK_STOP
:
1553 /* fall through - CHECK_STOP and LOAD are not supported yet */
1561 bool kvm_s390_cmma_enabled(struct kvm
*kvm
)
1563 if (!MACHINE_IS_LPAR
)
1565 /* only enable for z10 and later */
1566 if (!MACHINE_HAS_EDAT1
)
1568 if (!kvm
->arch
.use_cmma
)
1573 static bool ibs_enabled(struct kvm_vcpu
*vcpu
)
1575 return atomic_read(&vcpu
->arch
.sie_block
->cpuflags
) & CPUSTAT_IBS
;
1578 static int kvm_s390_handle_requests(struct kvm_vcpu
*vcpu
)
1581 s390_vcpu_unblock(vcpu
);
1583 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1584 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1585 * This ensures that the ipte instruction for this request has
1586 * already finished. We might race against a second unmapper that
1587 * wants to set the blocking bit. Lets just retry the request loop.
1589 if (kvm_check_request(KVM_REQ_MMU_RELOAD
, vcpu
)) {
1591 rc
= gmap_ipte_notify(vcpu
->arch
.gmap
,
1592 kvm_s390_get_prefix(vcpu
),
1599 if (kvm_check_request(KVM_REQ_TLB_FLUSH
, vcpu
)) {
1600 vcpu
->arch
.sie_block
->ihcpu
= 0xffff;
1604 if (kvm_check_request(KVM_REQ_ENABLE_IBS
, vcpu
)) {
1605 if (!ibs_enabled(vcpu
)) {
1606 trace_kvm_s390_enable_disable_ibs(vcpu
->vcpu_id
, 1);
1607 atomic_set_mask(CPUSTAT_IBS
,
1608 &vcpu
->arch
.sie_block
->cpuflags
);
1613 if (kvm_check_request(KVM_REQ_DISABLE_IBS
, vcpu
)) {
1614 if (ibs_enabled(vcpu
)) {
1615 trace_kvm_s390_enable_disable_ibs(vcpu
->vcpu_id
, 0);
1616 atomic_clear_mask(CPUSTAT_IBS
,
1617 &vcpu
->arch
.sie_block
->cpuflags
);
1622 /* nothing to do, just clear the request */
1623 clear_bit(KVM_REQ_UNHALT
, &vcpu
->requests
);
1629 * kvm_arch_fault_in_page - fault-in guest page if necessary
1630 * @vcpu: The corresponding virtual cpu
1631 * @gpa: Guest physical address
1632 * @writable: Whether the page should be writable or not
1634 * Make sure that a guest page has been faulted-in on the host.
1636 * Return: Zero on success, negative error code otherwise.
1638 long kvm_arch_fault_in_page(struct kvm_vcpu
*vcpu
, gpa_t gpa
, int writable
)
1640 return gmap_fault(vcpu
->arch
.gmap
, gpa
,
1641 writable
? FAULT_FLAG_WRITE
: 0);
1644 static void __kvm_inject_pfault_token(struct kvm_vcpu
*vcpu
, bool start_token
,
1645 unsigned long token
)
1647 struct kvm_s390_interrupt inti
;
1648 struct kvm_s390_irq irq
;
1651 irq
.u
.ext
.ext_params2
= token
;
1652 irq
.type
= KVM_S390_INT_PFAULT_INIT
;
1653 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu
, &irq
));
1655 inti
.type
= KVM_S390_INT_PFAULT_DONE
;
1656 inti
.parm64
= token
;
1657 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu
->kvm
, &inti
));
1661 void kvm_arch_async_page_not_present(struct kvm_vcpu
*vcpu
,
1662 struct kvm_async_pf
*work
)
1664 trace_kvm_s390_pfault_init(vcpu
, work
->arch
.pfault_token
);
1665 __kvm_inject_pfault_token(vcpu
, true, work
->arch
.pfault_token
);
1668 void kvm_arch_async_page_present(struct kvm_vcpu
*vcpu
,
1669 struct kvm_async_pf
*work
)
1671 trace_kvm_s390_pfault_done(vcpu
, work
->arch
.pfault_token
);
1672 __kvm_inject_pfault_token(vcpu
, false, work
->arch
.pfault_token
);
1675 void kvm_arch_async_page_ready(struct kvm_vcpu
*vcpu
,
1676 struct kvm_async_pf
*work
)
1678 /* s390 will always inject the page directly */
1681 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu
*vcpu
)
1684 * s390 will always inject the page directly,
1685 * but we still want check_async_completion to cleanup
1690 static int kvm_arch_setup_async_pf(struct kvm_vcpu
*vcpu
)
1693 struct kvm_arch_async_pf arch
;
1696 if (vcpu
->arch
.pfault_token
== KVM_S390_PFAULT_TOKEN_INVALID
)
1698 if ((vcpu
->arch
.sie_block
->gpsw
.mask
& vcpu
->arch
.pfault_select
) !=
1699 vcpu
->arch
.pfault_compare
)
1701 if (psw_extint_disabled(vcpu
))
1703 if (kvm_s390_vcpu_has_irq(vcpu
, 0))
1705 if (!(vcpu
->arch
.sie_block
->gcr
[0] & 0x200ul
))
1707 if (!vcpu
->arch
.gmap
->pfault_enabled
)
1710 hva
= gfn_to_hva(vcpu
->kvm
, gpa_to_gfn(current
->thread
.gmap_addr
));
1711 hva
+= current
->thread
.gmap_addr
& ~PAGE_MASK
;
1712 if (read_guest_real(vcpu
, vcpu
->arch
.pfault_token
, &arch
.pfault_token
, 8))
1715 rc
= kvm_setup_async_pf(vcpu
, current
->thread
.gmap_addr
, hva
, &arch
);
1719 static int vcpu_pre_run(struct kvm_vcpu
*vcpu
)
1724 * On s390 notifications for arriving pages will be delivered directly
1725 * to the guest but the house keeping for completed pfaults is
1726 * handled outside the worker.
1728 kvm_check_async_pf_completion(vcpu
);
1730 memcpy(&vcpu
->arch
.sie_block
->gg14
, &vcpu
->run
->s
.regs
.gprs
[14], 16);
1735 if (test_cpu_flag(CIF_MCCK_PENDING
))
1738 if (!kvm_is_ucontrol(vcpu
->kvm
)) {
1739 rc
= kvm_s390_deliver_pending_interrupts(vcpu
);
1744 rc
= kvm_s390_handle_requests(vcpu
);
1748 if (guestdbg_enabled(vcpu
)) {
1749 kvm_s390_backup_guest_per_regs(vcpu
);
1750 kvm_s390_patch_guest_per_regs(vcpu
);
1753 vcpu
->arch
.sie_block
->icptcode
= 0;
1754 cpuflags
= atomic_read(&vcpu
->arch
.sie_block
->cpuflags
);
1755 VCPU_EVENT(vcpu
, 6, "entering sie flags %x", cpuflags
);
1756 trace_kvm_s390_sie_enter(vcpu
, cpuflags
);
1761 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu
*vcpu
)
1763 psw_t
*psw
= &vcpu
->arch
.sie_block
->gpsw
;
1767 VCPU_EVENT(vcpu
, 3, "%s", "fault in sie instruction");
1768 trace_kvm_s390_sie_fault(vcpu
);
1771 * We want to inject an addressing exception, which is defined as a
1772 * suppressing or terminating exception. However, since we came here
1773 * by a DAT access exception, the PSW still points to the faulting
1774 * instruction since DAT exceptions are nullifying. So we've got
1775 * to look up the current opcode to get the length of the instruction
1776 * to be able to forward the PSW.
1778 rc
= read_guest(vcpu
, psw
->addr
, &opcode
, 1);
1780 return kvm_s390_inject_prog_cond(vcpu
, rc
);
1781 psw
->addr
= __rewind_psw(*psw
, -insn_length(opcode
));
1783 return kvm_s390_inject_program_int(vcpu
, PGM_ADDRESSING
);
1786 static int vcpu_post_run(struct kvm_vcpu
*vcpu
, int exit_reason
)
1790 VCPU_EVENT(vcpu
, 6, "exit sie icptcode %d",
1791 vcpu
->arch
.sie_block
->icptcode
);
1792 trace_kvm_s390_sie_exit(vcpu
, vcpu
->arch
.sie_block
->icptcode
);
1794 if (guestdbg_enabled(vcpu
))
1795 kvm_s390_restore_guest_per_regs(vcpu
);
1797 if (exit_reason
>= 0) {
1799 } else if (kvm_is_ucontrol(vcpu
->kvm
)) {
1800 vcpu
->run
->exit_reason
= KVM_EXIT_S390_UCONTROL
;
1801 vcpu
->run
->s390_ucontrol
.trans_exc_code
=
1802 current
->thread
.gmap_addr
;
1803 vcpu
->run
->s390_ucontrol
.pgm_code
= 0x10;
1806 } else if (current
->thread
.gmap_pfault
) {
1807 trace_kvm_s390_major_guest_pfault(vcpu
);
1808 current
->thread
.gmap_pfault
= 0;
1809 if (kvm_arch_setup_async_pf(vcpu
)) {
1812 gpa_t gpa
= current
->thread
.gmap_addr
;
1813 rc
= kvm_arch_fault_in_page(vcpu
, gpa
, 1);
1818 rc
= vcpu_post_run_fault_in_sie(vcpu
);
1820 memcpy(&vcpu
->run
->s
.regs
.gprs
[14], &vcpu
->arch
.sie_block
->gg14
, 16);
1823 if (kvm_is_ucontrol(vcpu
->kvm
))
1824 /* Don't exit for host interrupts. */
1825 rc
= vcpu
->arch
.sie_block
->icptcode
? -EOPNOTSUPP
: 0;
1827 rc
= kvm_handle_sie_intercept(vcpu
);
1833 static int __vcpu_run(struct kvm_vcpu
*vcpu
)
1835 int rc
, exit_reason
;
1838 * We try to hold kvm->srcu during most of vcpu_run (except when run-
1839 * ning the guest), so that memslots (and other stuff) are protected
1841 vcpu
->srcu_idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
1844 rc
= vcpu_pre_run(vcpu
);
1848 srcu_read_unlock(&vcpu
->kvm
->srcu
, vcpu
->srcu_idx
);
1850 * As PF_VCPU will be used in fault handler, between
1851 * guest_enter and guest_exit should be no uaccess.
1856 exit_reason
= sie64a(vcpu
->arch
.sie_block
,
1857 vcpu
->run
->s
.regs
.gprs
);
1859 vcpu
->srcu_idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
1861 rc
= vcpu_post_run(vcpu
, exit_reason
);
1862 } while (!signal_pending(current
) && !guestdbg_exit_pending(vcpu
) && !rc
);
1864 srcu_read_unlock(&vcpu
->kvm
->srcu
, vcpu
->srcu_idx
);
1868 static void sync_regs(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
1870 vcpu
->arch
.sie_block
->gpsw
.mask
= kvm_run
->psw_mask
;
1871 vcpu
->arch
.sie_block
->gpsw
.addr
= kvm_run
->psw_addr
;
1872 if (kvm_run
->kvm_dirty_regs
& KVM_SYNC_PREFIX
)
1873 kvm_s390_set_prefix(vcpu
, kvm_run
->s
.regs
.prefix
);
1874 if (kvm_run
->kvm_dirty_regs
& KVM_SYNC_CRS
) {
1875 memcpy(&vcpu
->arch
.sie_block
->gcr
, &kvm_run
->s
.regs
.crs
, 128);
1876 /* some control register changes require a tlb flush */
1877 kvm_make_request(KVM_REQ_TLB_FLUSH
, vcpu
);
1879 if (kvm_run
->kvm_dirty_regs
& KVM_SYNC_ARCH0
) {
1880 vcpu
->arch
.sie_block
->cputm
= kvm_run
->s
.regs
.cputm
;
1881 vcpu
->arch
.sie_block
->ckc
= kvm_run
->s
.regs
.ckc
;
1882 vcpu
->arch
.sie_block
->todpr
= kvm_run
->s
.regs
.todpr
;
1883 vcpu
->arch
.sie_block
->pp
= kvm_run
->s
.regs
.pp
;
1884 vcpu
->arch
.sie_block
->gbea
= kvm_run
->s
.regs
.gbea
;
1886 if (kvm_run
->kvm_dirty_regs
& KVM_SYNC_PFAULT
) {
1887 vcpu
->arch
.pfault_token
= kvm_run
->s
.regs
.pft
;
1888 vcpu
->arch
.pfault_select
= kvm_run
->s
.regs
.pfs
;
1889 vcpu
->arch
.pfault_compare
= kvm_run
->s
.regs
.pfc
;
1890 if (vcpu
->arch
.pfault_token
== KVM_S390_PFAULT_TOKEN_INVALID
)
1891 kvm_clear_async_pf_completion_queue(vcpu
);
1893 kvm_run
->kvm_dirty_regs
= 0;
1896 static void store_regs(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
1898 kvm_run
->psw_mask
= vcpu
->arch
.sie_block
->gpsw
.mask
;
1899 kvm_run
->psw_addr
= vcpu
->arch
.sie_block
->gpsw
.addr
;
1900 kvm_run
->s
.regs
.prefix
= kvm_s390_get_prefix(vcpu
);
1901 memcpy(&kvm_run
->s
.regs
.crs
, &vcpu
->arch
.sie_block
->gcr
, 128);
1902 kvm_run
->s
.regs
.cputm
= vcpu
->arch
.sie_block
->cputm
;
1903 kvm_run
->s
.regs
.ckc
= vcpu
->arch
.sie_block
->ckc
;
1904 kvm_run
->s
.regs
.todpr
= vcpu
->arch
.sie_block
->todpr
;
1905 kvm_run
->s
.regs
.pp
= vcpu
->arch
.sie_block
->pp
;
1906 kvm_run
->s
.regs
.gbea
= vcpu
->arch
.sie_block
->gbea
;
1907 kvm_run
->s
.regs
.pft
= vcpu
->arch
.pfault_token
;
1908 kvm_run
->s
.regs
.pfs
= vcpu
->arch
.pfault_select
;
1909 kvm_run
->s
.regs
.pfc
= vcpu
->arch
.pfault_compare
;
1912 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
1917 if (guestdbg_exit_pending(vcpu
)) {
1918 kvm_s390_prepare_debug_exit(vcpu
);
1922 if (vcpu
->sigset_active
)
1923 sigprocmask(SIG_SETMASK
, &vcpu
->sigset
, &sigsaved
);
1925 if (!kvm_s390_user_cpu_state_ctrl(vcpu
->kvm
)) {
1926 kvm_s390_vcpu_start(vcpu
);
1927 } else if (is_vcpu_stopped(vcpu
)) {
1928 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
1933 sync_regs(vcpu
, kvm_run
);
1936 rc
= __vcpu_run(vcpu
);
1938 if (signal_pending(current
) && !rc
) {
1939 kvm_run
->exit_reason
= KVM_EXIT_INTR
;
1943 if (guestdbg_exit_pending(vcpu
) && !rc
) {
1944 kvm_s390_prepare_debug_exit(vcpu
);
1948 if (rc
== -EOPNOTSUPP
) {
1949 /* intercept cannot be handled in-kernel, prepare kvm-run */
1950 kvm_run
->exit_reason
= KVM_EXIT_S390_SIEIC
;
1951 kvm_run
->s390_sieic
.icptcode
= vcpu
->arch
.sie_block
->icptcode
;
1952 kvm_run
->s390_sieic
.ipa
= vcpu
->arch
.sie_block
->ipa
;
1953 kvm_run
->s390_sieic
.ipb
= vcpu
->arch
.sie_block
->ipb
;
1957 if (rc
== -EREMOTE
) {
1958 /* intercept was handled, but userspace support is needed
1959 * kvm_run has been prepared by the handler */
1963 store_regs(vcpu
, kvm_run
);
1965 if (vcpu
->sigset_active
)
1966 sigprocmask(SIG_SETMASK
, &sigsaved
, NULL
);
1968 vcpu
->stat
.exit_userspace
++;
1973 * store status at address
1974 * we use have two special cases:
1975 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1976 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1978 int kvm_s390_store_status_unloaded(struct kvm_vcpu
*vcpu
, unsigned long gpa
)
1980 unsigned char archmode
= 1;
1985 if (gpa
== KVM_S390_STORE_STATUS_NOADDR
) {
1986 if (write_guest_abs(vcpu
, 163, &archmode
, 1))
1988 gpa
= SAVE_AREA_BASE
;
1989 } else if (gpa
== KVM_S390_STORE_STATUS_PREFIXED
) {
1990 if (write_guest_real(vcpu
, 163, &archmode
, 1))
1992 gpa
= kvm_s390_real_to_abs(vcpu
, SAVE_AREA_BASE
);
1994 rc
= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, fp_regs
),
1995 vcpu
->arch
.guest_fpregs
.fprs
, 128);
1996 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, gp_regs
),
1997 vcpu
->run
->s
.regs
.gprs
, 128);
1998 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, psw
),
1999 &vcpu
->arch
.sie_block
->gpsw
, 16);
2000 px
= kvm_s390_get_prefix(vcpu
);
2001 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, pref_reg
),
2003 rc
|= write_guest_abs(vcpu
,
2004 gpa
+ offsetof(struct save_area
, fp_ctrl_reg
),
2005 &vcpu
->arch
.guest_fpregs
.fpc
, 4);
2006 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, tod_reg
),
2007 &vcpu
->arch
.sie_block
->todpr
, 4);
2008 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, timer
),
2009 &vcpu
->arch
.sie_block
->cputm
, 8);
2010 clkcomp
= vcpu
->arch
.sie_block
->ckc
>> 8;
2011 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, clk_cmp
),
2013 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, acc_regs
),
2014 &vcpu
->run
->s
.regs
.acrs
, 64);
2015 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, ctrl_regs
),
2016 &vcpu
->arch
.sie_block
->gcr
, 128);
2017 return rc
? -EFAULT
: 0;
2020 int kvm_s390_vcpu_store_status(struct kvm_vcpu
*vcpu
, unsigned long addr
)
2023 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2024 * copying in vcpu load/put. Lets update our copies before we save
2025 * it into the save area
2027 save_fp_ctl(&vcpu
->arch
.guest_fpregs
.fpc
);
2028 save_fp_regs(vcpu
->arch
.guest_fpregs
.fprs
);
2029 save_access_regs(vcpu
->run
->s
.regs
.acrs
);
2031 return kvm_s390_store_status_unloaded(vcpu
, addr
);
2035 * store additional status at address
2037 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu
*vcpu
,
2040 /* Only bits 0-53 are used for address formation */
2041 if (!(gpa
& ~0x3ff))
2044 return write_guest_abs(vcpu
, gpa
& ~0x3ff,
2045 (void *)&vcpu
->run
->s
.regs
.vrs
, 512);
2048 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu
*vcpu
, unsigned long addr
)
2050 if (!test_kvm_facility(vcpu
->kvm
, 129))
2054 * The guest VXRS are in the host VXRs due to the lazy
2055 * copying in vcpu load/put. Let's update our copies before we save
2056 * it into the save area.
2058 save_vx_regs((__vector128
*)&vcpu
->run
->s
.regs
.vrs
);
2060 return kvm_s390_store_adtl_status_unloaded(vcpu
, addr
);
2063 static void __disable_ibs_on_vcpu(struct kvm_vcpu
*vcpu
)
2065 kvm_check_request(KVM_REQ_ENABLE_IBS
, vcpu
);
2066 kvm_make_request(KVM_REQ_DISABLE_IBS
, vcpu
);
2067 exit_sie_sync(vcpu
);
2070 static void __disable_ibs_on_all_vcpus(struct kvm
*kvm
)
2073 struct kvm_vcpu
*vcpu
;
2075 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
2076 __disable_ibs_on_vcpu(vcpu
);
2080 static void __enable_ibs_on_vcpu(struct kvm_vcpu
*vcpu
)
2082 kvm_check_request(KVM_REQ_DISABLE_IBS
, vcpu
);
2083 kvm_make_request(KVM_REQ_ENABLE_IBS
, vcpu
);
2084 exit_sie_sync(vcpu
);
2087 void kvm_s390_vcpu_start(struct kvm_vcpu
*vcpu
)
2089 int i
, online_vcpus
, started_vcpus
= 0;
2091 if (!is_vcpu_stopped(vcpu
))
2094 trace_kvm_s390_vcpu_start_stop(vcpu
->vcpu_id
, 1);
2095 /* Only one cpu at a time may enter/leave the STOPPED state. */
2096 spin_lock(&vcpu
->kvm
->arch
.start_stop_lock
);
2097 online_vcpus
= atomic_read(&vcpu
->kvm
->online_vcpus
);
2099 for (i
= 0; i
< online_vcpus
; i
++) {
2100 if (!is_vcpu_stopped(vcpu
->kvm
->vcpus
[i
]))
2104 if (started_vcpus
== 0) {
2105 /* we're the only active VCPU -> speed it up */
2106 __enable_ibs_on_vcpu(vcpu
);
2107 } else if (started_vcpus
== 1) {
2109 * As we are starting a second VCPU, we have to disable
2110 * the IBS facility on all VCPUs to remove potentially
2111 * oustanding ENABLE requests.
2113 __disable_ibs_on_all_vcpus(vcpu
->kvm
);
2116 atomic_clear_mask(CPUSTAT_STOPPED
, &vcpu
->arch
.sie_block
->cpuflags
);
2118 * Another VCPU might have used IBS while we were offline.
2119 * Let's play safe and flush the VCPU at startup.
2121 kvm_make_request(KVM_REQ_TLB_FLUSH
, vcpu
);
2122 spin_unlock(&vcpu
->kvm
->arch
.start_stop_lock
);
2126 void kvm_s390_vcpu_stop(struct kvm_vcpu
*vcpu
)
2128 int i
, online_vcpus
, started_vcpus
= 0;
2129 struct kvm_vcpu
*started_vcpu
= NULL
;
2131 if (is_vcpu_stopped(vcpu
))
2134 trace_kvm_s390_vcpu_start_stop(vcpu
->vcpu_id
, 0);
2135 /* Only one cpu at a time may enter/leave the STOPPED state. */
2136 spin_lock(&vcpu
->kvm
->arch
.start_stop_lock
);
2137 online_vcpus
= atomic_read(&vcpu
->kvm
->online_vcpus
);
2139 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2140 kvm_s390_clear_stop_irq(vcpu
);
2142 atomic_set_mask(CPUSTAT_STOPPED
, &vcpu
->arch
.sie_block
->cpuflags
);
2143 __disable_ibs_on_vcpu(vcpu
);
2145 for (i
= 0; i
< online_vcpus
; i
++) {
2146 if (!is_vcpu_stopped(vcpu
->kvm
->vcpus
[i
])) {
2148 started_vcpu
= vcpu
->kvm
->vcpus
[i
];
2152 if (started_vcpus
== 1) {
2154 * As we only have one VCPU left, we want to enable the
2155 * IBS facility for that VCPU to speed it up.
2157 __enable_ibs_on_vcpu(started_vcpu
);
2160 spin_unlock(&vcpu
->kvm
->arch
.start_stop_lock
);
2164 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu
*vcpu
,
2165 struct kvm_enable_cap
*cap
)
2173 case KVM_CAP_S390_CSS_SUPPORT
:
2174 if (!vcpu
->kvm
->arch
.css_support
) {
2175 vcpu
->kvm
->arch
.css_support
= 1;
2176 trace_kvm_s390_enable_css(vcpu
->kvm
);
2187 long kvm_arch_vcpu_ioctl(struct file
*filp
,
2188 unsigned int ioctl
, unsigned long arg
)
2190 struct kvm_vcpu
*vcpu
= filp
->private_data
;
2191 void __user
*argp
= (void __user
*)arg
;
2196 case KVM_S390_INTERRUPT
: {
2197 struct kvm_s390_interrupt s390int
;
2198 struct kvm_s390_irq s390irq
;
2201 if (copy_from_user(&s390int
, argp
, sizeof(s390int
)))
2203 if (s390int_to_s390irq(&s390int
, &s390irq
))
2205 r
= kvm_s390_inject_vcpu(vcpu
, &s390irq
);
2208 case KVM_S390_STORE_STATUS
:
2209 idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
2210 r
= kvm_s390_vcpu_store_status(vcpu
, arg
);
2211 srcu_read_unlock(&vcpu
->kvm
->srcu
, idx
);
2213 case KVM_S390_SET_INITIAL_PSW
: {
2217 if (copy_from_user(&psw
, argp
, sizeof(psw
)))
2219 r
= kvm_arch_vcpu_ioctl_set_initial_psw(vcpu
, psw
);
2222 case KVM_S390_INITIAL_RESET
:
2223 r
= kvm_arch_vcpu_ioctl_initial_reset(vcpu
);
2225 case KVM_SET_ONE_REG
:
2226 case KVM_GET_ONE_REG
: {
2227 struct kvm_one_reg reg
;
2229 if (copy_from_user(®
, argp
, sizeof(reg
)))
2231 if (ioctl
== KVM_SET_ONE_REG
)
2232 r
= kvm_arch_vcpu_ioctl_set_one_reg(vcpu
, ®
);
2234 r
= kvm_arch_vcpu_ioctl_get_one_reg(vcpu
, ®
);
2237 #ifdef CONFIG_KVM_S390_UCONTROL
2238 case KVM_S390_UCAS_MAP
: {
2239 struct kvm_s390_ucas_mapping ucasmap
;
2241 if (copy_from_user(&ucasmap
, argp
, sizeof(ucasmap
))) {
2246 if (!kvm_is_ucontrol(vcpu
->kvm
)) {
2251 r
= gmap_map_segment(vcpu
->arch
.gmap
, ucasmap
.user_addr
,
2252 ucasmap
.vcpu_addr
, ucasmap
.length
);
2255 case KVM_S390_UCAS_UNMAP
: {
2256 struct kvm_s390_ucas_mapping ucasmap
;
2258 if (copy_from_user(&ucasmap
, argp
, sizeof(ucasmap
))) {
2263 if (!kvm_is_ucontrol(vcpu
->kvm
)) {
2268 r
= gmap_unmap_segment(vcpu
->arch
.gmap
, ucasmap
.vcpu_addr
,
2273 case KVM_S390_VCPU_FAULT
: {
2274 r
= gmap_fault(vcpu
->arch
.gmap
, arg
, 0);
2277 case KVM_ENABLE_CAP
:
2279 struct kvm_enable_cap cap
;
2281 if (copy_from_user(&cap
, argp
, sizeof(cap
)))
2283 r
= kvm_vcpu_ioctl_enable_cap(vcpu
, &cap
);
2292 int kvm_arch_vcpu_fault(struct kvm_vcpu
*vcpu
, struct vm_fault
*vmf
)
2294 #ifdef CONFIG_KVM_S390_UCONTROL
2295 if ((vmf
->pgoff
== KVM_S390_SIE_PAGE_OFFSET
)
2296 && (kvm_is_ucontrol(vcpu
->kvm
))) {
2297 vmf
->page
= virt_to_page(vcpu
->arch
.sie_block
);
2298 get_page(vmf
->page
);
2302 return VM_FAULT_SIGBUS
;
2305 int kvm_arch_create_memslot(struct kvm
*kvm
, struct kvm_memory_slot
*slot
,
2306 unsigned long npages
)
2311 /* Section: memory related */
2312 int kvm_arch_prepare_memory_region(struct kvm
*kvm
,
2313 struct kvm_memory_slot
*memslot
,
2314 struct kvm_userspace_memory_region
*mem
,
2315 enum kvm_mr_change change
)
2317 /* A few sanity checks. We can have memory slots which have to be
2318 located/ended at a segment boundary (1MB). The memory in userland is
2319 ok to be fragmented into various different vmas. It is okay to mmap()
2320 and munmap() stuff in this slot after doing this call at any time */
2322 if (mem
->userspace_addr
& 0xffffful
)
2325 if (mem
->memory_size
& 0xffffful
)
2331 void kvm_arch_commit_memory_region(struct kvm
*kvm
,
2332 struct kvm_userspace_memory_region
*mem
,
2333 const struct kvm_memory_slot
*old
,
2334 enum kvm_mr_change change
)
2338 /* If the basics of the memslot do not change, we do not want
2339 * to update the gmap. Every update causes several unnecessary
2340 * segment translation exceptions. This is usually handled just
2341 * fine by the normal fault handler + gmap, but it will also
2342 * cause faults on the prefix page of running guest CPUs.
2344 if (old
->userspace_addr
== mem
->userspace_addr
&&
2345 old
->base_gfn
* PAGE_SIZE
== mem
->guest_phys_addr
&&
2346 old
->npages
* PAGE_SIZE
== mem
->memory_size
)
2349 rc
= gmap_map_segment(kvm
->arch
.gmap
, mem
->userspace_addr
,
2350 mem
->guest_phys_addr
, mem
->memory_size
);
2352 printk(KERN_WARNING
"kvm-s390: failed to commit memory region\n");
2356 static int __init
kvm_s390_init(void)
2358 return kvm_init(NULL
, sizeof(struct kvm_vcpu
), 0, THIS_MODULE
);
2361 static void __exit
kvm_s390_exit(void)
2366 module_init(kvm_s390_init
);
2367 module_exit(kvm_s390_exit
);
2370 * Enable autoloading of the kvm module.
2371 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2372 * since x86 takes a different approach.
2374 #include <linux/miscdevice.h>
2375 MODULE_ALIAS_MISCDEV(KVM_MINOR
);
2376 MODULE_ALIAS("devname:kvm");