2 * hosting zSeries kernel virtual machines
4 * Copyright IBM Corp. 2008, 2009
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
17 #include <linux/compiler.h>
18 #include <linux/err.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/module.h>
25 #include <linux/random.h>
26 #include <linux/slab.h>
27 #include <linux/timer.h>
28 #include <linux/vmalloc.h>
29 #include <linux/bitmap.h>
30 #include <asm/asm-offsets.h>
31 #include <asm/lowcore.h>
33 #include <asm/pgtable.h>
36 #include <asm/switch_to.h>
39 #include <asm/cpacf.h>
44 #define KMSG_COMPONENT "kvm-s390"
46 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
48 #define CREATE_TRACE_POINTS
50 #include "trace-s390.h"
52 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
54 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
55 (KVM_MAX_VCPUS + LOCAL_IRQS))
57 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
59 struct kvm_stats_debugfs_item debugfs_entries
[] = {
60 { "userspace_handled", VCPU_STAT(exit_userspace
) },
61 { "exit_null", VCPU_STAT(exit_null
) },
62 { "exit_validity", VCPU_STAT(exit_validity
) },
63 { "exit_stop_request", VCPU_STAT(exit_stop_request
) },
64 { "exit_external_request", VCPU_STAT(exit_external_request
) },
65 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt
) },
66 { "exit_instruction", VCPU_STAT(exit_instruction
) },
67 { "exit_program_interruption", VCPU_STAT(exit_program_interruption
) },
68 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program
) },
69 { "exit_operation_exception", VCPU_STAT(exit_operation_exception
) },
70 { "halt_successful_poll", VCPU_STAT(halt_successful_poll
) },
71 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll
) },
72 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid
) },
73 { "halt_wakeup", VCPU_STAT(halt_wakeup
) },
74 { "instruction_lctlg", VCPU_STAT(instruction_lctlg
) },
75 { "instruction_lctl", VCPU_STAT(instruction_lctl
) },
76 { "instruction_stctl", VCPU_STAT(instruction_stctl
) },
77 { "instruction_stctg", VCPU_STAT(instruction_stctg
) },
78 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal
) },
79 { "deliver_external_call", VCPU_STAT(deliver_external_call
) },
80 { "deliver_service_signal", VCPU_STAT(deliver_service_signal
) },
81 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt
) },
82 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal
) },
83 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal
) },
84 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal
) },
85 { "deliver_program_interruption", VCPU_STAT(deliver_program_int
) },
86 { "exit_wait_state", VCPU_STAT(exit_wait_state
) },
87 { "instruction_pfmf", VCPU_STAT(instruction_pfmf
) },
88 { "instruction_stidp", VCPU_STAT(instruction_stidp
) },
89 { "instruction_spx", VCPU_STAT(instruction_spx
) },
90 { "instruction_stpx", VCPU_STAT(instruction_stpx
) },
91 { "instruction_stap", VCPU_STAT(instruction_stap
) },
92 { "instruction_storage_key", VCPU_STAT(instruction_storage_key
) },
93 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock
) },
94 { "instruction_stsch", VCPU_STAT(instruction_stsch
) },
95 { "instruction_chsc", VCPU_STAT(instruction_chsc
) },
96 { "instruction_essa", VCPU_STAT(instruction_essa
) },
97 { "instruction_stsi", VCPU_STAT(instruction_stsi
) },
98 { "instruction_stfl", VCPU_STAT(instruction_stfl
) },
99 { "instruction_tprot", VCPU_STAT(instruction_tprot
) },
100 { "instruction_sthyi", VCPU_STAT(instruction_sthyi
) },
101 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense
) },
102 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running
) },
103 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call
) },
104 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency
) },
105 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency
) },
106 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start
) },
107 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop
) },
108 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status
) },
109 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status
) },
110 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status
) },
111 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch
) },
112 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix
) },
113 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart
) },
114 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset
) },
115 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset
) },
116 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown
) },
117 { "diagnose_10", VCPU_STAT(diagnose_10
) },
118 { "diagnose_44", VCPU_STAT(diagnose_44
) },
119 { "diagnose_9c", VCPU_STAT(diagnose_9c
) },
120 { "diagnose_258", VCPU_STAT(diagnose_258
) },
121 { "diagnose_308", VCPU_STAT(diagnose_308
) },
122 { "diagnose_500", VCPU_STAT(diagnose_500
) },
126 /* upper facilities limit for kvm */
127 unsigned long kvm_s390_fac_list_mask
[16] = {
128 0xffe6000000000000UL
,
129 0x005e000000000000UL
,
132 unsigned long kvm_s390_fac_list_mask_size(void)
134 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask
) > S390_ARCH_FAC_MASK_SIZE_U64
);
135 return ARRAY_SIZE(kvm_s390_fac_list_mask
);
138 /* available cpu features supported by kvm */
139 static DECLARE_BITMAP(kvm_s390_available_cpu_feat
, KVM_S390_VM_CPU_FEAT_NR_BITS
);
140 /* available subfunctions indicated via query / "test bit" */
141 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc
;
143 static struct gmap_notifier gmap_notifier
;
144 debug_info_t
*kvm_s390_dbf
;
146 /* Section: not file related */
147 int kvm_arch_hardware_enable(void)
149 /* every s390 is virtualization enabled ;-) */
153 static void kvm_gmap_notifier(struct gmap
*gmap
, unsigned long address
);
156 * This callback is executed during stop_machine(). All CPUs are therefore
157 * temporarily stopped. In order not to change guest behavior, we have to
158 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
159 * so a CPU won't be stopped while calculating with the epoch.
161 static int kvm_clock_sync(struct notifier_block
*notifier
, unsigned long val
,
165 struct kvm_vcpu
*vcpu
;
167 unsigned long long *delta
= v
;
169 list_for_each_entry(kvm
, &vm_list
, vm_list
) {
170 kvm
->arch
.epoch
-= *delta
;
171 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
172 vcpu
->arch
.sie_block
->epoch
-= *delta
;
173 if (vcpu
->arch
.cputm_enabled
)
174 vcpu
->arch
.cputm_start
+= *delta
;
180 static struct notifier_block kvm_clock_notifier
= {
181 .notifier_call
= kvm_clock_sync
,
184 int kvm_arch_hardware_setup(void)
186 gmap_notifier
.notifier_call
= kvm_gmap_notifier
;
187 gmap_register_ipte_notifier(&gmap_notifier
);
188 atomic_notifier_chain_register(&s390_epoch_delta_notifier
,
189 &kvm_clock_notifier
);
193 void kvm_arch_hardware_unsetup(void)
195 gmap_unregister_ipte_notifier(&gmap_notifier
);
196 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier
,
197 &kvm_clock_notifier
);
200 static void allow_cpu_feat(unsigned long nr
)
202 set_bit_inv(nr
, kvm_s390_available_cpu_feat
);
205 static inline int plo_test_bit(unsigned char nr
)
207 register unsigned long r0
asm("0") = (unsigned long) nr
| 0x100;
208 int cc
= 3; /* subfunction not available */
211 /* Parameter registers are ignored for "test bit" */
221 static void kvm_s390_cpu_feat_init(void)
225 for (i
= 0; i
< 256; ++i
) {
227 kvm_s390_available_subfunc
.plo
[i
>> 3] |= 0x80 >> (i
& 7);
230 if (test_facility(28)) /* TOD-clock steering */
231 etr_ptff(kvm_s390_available_subfunc
.ptff
, ETR_PTFF_QAF
);
233 if (test_facility(17)) { /* MSA */
234 __cpacf_query(CPACF_KMAC
, kvm_s390_available_subfunc
.kmac
);
235 __cpacf_query(CPACF_KMC
, kvm_s390_available_subfunc
.kmc
);
236 __cpacf_query(CPACF_KM
, kvm_s390_available_subfunc
.km
);
237 __cpacf_query(CPACF_KIMD
, kvm_s390_available_subfunc
.kimd
);
238 __cpacf_query(CPACF_KLMD
, kvm_s390_available_subfunc
.klmd
);
240 if (test_facility(76)) /* MSA3 */
241 __cpacf_query(CPACF_PCKMO
, kvm_s390_available_subfunc
.pckmo
);
242 if (test_facility(77)) { /* MSA4 */
243 __cpacf_query(CPACF_KMCTR
, kvm_s390_available_subfunc
.kmctr
);
244 __cpacf_query(CPACF_KMF
, kvm_s390_available_subfunc
.kmf
);
245 __cpacf_query(CPACF_KMO
, kvm_s390_available_subfunc
.kmo
);
246 __cpacf_query(CPACF_PCC
, kvm_s390_available_subfunc
.pcc
);
248 if (test_facility(57)) /* MSA5 */
249 __cpacf_query(CPACF_PPNO
, kvm_s390_available_subfunc
.ppno
);
251 if (MACHINE_HAS_ESOP
)
252 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP
);
255 int kvm_arch_init(void *opaque
)
257 kvm_s390_dbf
= debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
261 if (debug_register_view(kvm_s390_dbf
, &debug_sprintf_view
)) {
262 debug_unregister(kvm_s390_dbf
);
266 kvm_s390_cpu_feat_init();
268 /* Register floating interrupt controller interface. */
269 return kvm_register_device_ops(&kvm_flic_ops
, KVM_DEV_TYPE_FLIC
);
272 void kvm_arch_exit(void)
274 debug_unregister(kvm_s390_dbf
);
277 /* Section: device related */
278 long kvm_arch_dev_ioctl(struct file
*filp
,
279 unsigned int ioctl
, unsigned long arg
)
281 if (ioctl
== KVM_S390_ENABLE_SIE
)
282 return s390_enable_sie();
286 int kvm_vm_ioctl_check_extension(struct kvm
*kvm
, long ext
)
291 case KVM_CAP_S390_PSW
:
292 case KVM_CAP_S390_GMAP
:
293 case KVM_CAP_SYNC_MMU
:
294 #ifdef CONFIG_KVM_S390_UCONTROL
295 case KVM_CAP_S390_UCONTROL
:
297 case KVM_CAP_ASYNC_PF
:
298 case KVM_CAP_SYNC_REGS
:
299 case KVM_CAP_ONE_REG
:
300 case KVM_CAP_ENABLE_CAP
:
301 case KVM_CAP_S390_CSS_SUPPORT
:
302 case KVM_CAP_IOEVENTFD
:
303 case KVM_CAP_DEVICE_CTRL
:
304 case KVM_CAP_ENABLE_CAP_VM
:
305 case KVM_CAP_S390_IRQCHIP
:
306 case KVM_CAP_VM_ATTRIBUTES
:
307 case KVM_CAP_MP_STATE
:
308 case KVM_CAP_S390_INJECT_IRQ
:
309 case KVM_CAP_S390_USER_SIGP
:
310 case KVM_CAP_S390_USER_STSI
:
311 case KVM_CAP_S390_SKEYS
:
312 case KVM_CAP_S390_IRQ_STATE
:
315 case KVM_CAP_S390_MEM_OP
:
318 case KVM_CAP_NR_VCPUS
:
319 case KVM_CAP_MAX_VCPUS
:
320 r
= KVM_S390_BSCA_CPU_SLOTS
;
321 if (sclp
.has_esca
&& sclp
.has_64bscao
)
322 r
= KVM_S390_ESCA_CPU_SLOTS
;
324 case KVM_CAP_NR_MEMSLOTS
:
325 r
= KVM_USER_MEM_SLOTS
;
327 case KVM_CAP_S390_COW
:
328 r
= MACHINE_HAS_ESOP
;
330 case KVM_CAP_S390_VECTOR_REGISTERS
:
333 case KVM_CAP_S390_RI
:
334 r
= test_facility(64);
342 static void kvm_s390_sync_dirty_log(struct kvm
*kvm
,
343 struct kvm_memory_slot
*memslot
)
345 gfn_t cur_gfn
, last_gfn
;
346 unsigned long address
;
347 struct gmap
*gmap
= kvm
->arch
.gmap
;
349 /* Loop over all guest pages */
350 last_gfn
= memslot
->base_gfn
+ memslot
->npages
;
351 for (cur_gfn
= memslot
->base_gfn
; cur_gfn
<= last_gfn
; cur_gfn
++) {
352 address
= gfn_to_hva_memslot(memslot
, cur_gfn
);
354 if (test_and_clear_guest_dirty(gmap
->mm
, address
))
355 mark_page_dirty(kvm
, cur_gfn
);
356 if (fatal_signal_pending(current
))
362 /* Section: vm related */
363 static void sca_del_vcpu(struct kvm_vcpu
*vcpu
);
366 * Get (and clear) the dirty memory log for a memory slot.
368 int kvm_vm_ioctl_get_dirty_log(struct kvm
*kvm
,
369 struct kvm_dirty_log
*log
)
373 struct kvm_memslots
*slots
;
374 struct kvm_memory_slot
*memslot
;
377 mutex_lock(&kvm
->slots_lock
);
380 if (log
->slot
>= KVM_USER_MEM_SLOTS
)
383 slots
= kvm_memslots(kvm
);
384 memslot
= id_to_memslot(slots
, log
->slot
);
386 if (!memslot
->dirty_bitmap
)
389 kvm_s390_sync_dirty_log(kvm
, memslot
);
390 r
= kvm_get_dirty_log(kvm
, log
, &is_dirty
);
394 /* Clear the dirty log */
396 n
= kvm_dirty_bitmap_bytes(memslot
);
397 memset(memslot
->dirty_bitmap
, 0, n
);
401 mutex_unlock(&kvm
->slots_lock
);
405 static int kvm_vm_ioctl_enable_cap(struct kvm
*kvm
, struct kvm_enable_cap
*cap
)
413 case KVM_CAP_S390_IRQCHIP
:
414 VM_EVENT(kvm
, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
415 kvm
->arch
.use_irqchip
= 1;
418 case KVM_CAP_S390_USER_SIGP
:
419 VM_EVENT(kvm
, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
420 kvm
->arch
.user_sigp
= 1;
423 case KVM_CAP_S390_VECTOR_REGISTERS
:
424 mutex_lock(&kvm
->lock
);
425 if (kvm
->created_vcpus
) {
427 } else if (MACHINE_HAS_VX
) {
428 set_kvm_facility(kvm
->arch
.model
.fac_mask
, 129);
429 set_kvm_facility(kvm
->arch
.model
.fac_list
, 129);
433 mutex_unlock(&kvm
->lock
);
434 VM_EVENT(kvm
, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
435 r
? "(not available)" : "(success)");
437 case KVM_CAP_S390_RI
:
439 mutex_lock(&kvm
->lock
);
440 if (kvm
->created_vcpus
) {
442 } else if (test_facility(64)) {
443 set_kvm_facility(kvm
->arch
.model
.fac_mask
, 64);
444 set_kvm_facility(kvm
->arch
.model
.fac_list
, 64);
447 mutex_unlock(&kvm
->lock
);
448 VM_EVENT(kvm
, 3, "ENABLE: CAP_S390_RI %s",
449 r
? "(not available)" : "(success)");
451 case KVM_CAP_S390_USER_STSI
:
452 VM_EVENT(kvm
, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
453 kvm
->arch
.user_stsi
= 1;
463 static int kvm_s390_get_mem_control(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
467 switch (attr
->attr
) {
468 case KVM_S390_VM_MEM_LIMIT_SIZE
:
470 VM_EVENT(kvm
, 3, "QUERY: max guest memory: %lu bytes",
471 kvm
->arch
.mem_limit
);
472 if (put_user(kvm
->arch
.mem_limit
, (u64 __user
*)attr
->addr
))
482 static int kvm_s390_set_mem_control(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
486 switch (attr
->attr
) {
487 case KVM_S390_VM_MEM_ENABLE_CMMA
:
493 VM_EVENT(kvm
, 3, "%s", "ENABLE: CMMA support");
494 mutex_lock(&kvm
->lock
);
495 if (!kvm
->created_vcpus
) {
496 kvm
->arch
.use_cmma
= 1;
499 mutex_unlock(&kvm
->lock
);
501 case KVM_S390_VM_MEM_CLR_CMMA
:
506 if (!kvm
->arch
.use_cmma
)
509 VM_EVENT(kvm
, 3, "%s", "RESET: CMMA states");
510 mutex_lock(&kvm
->lock
);
511 idx
= srcu_read_lock(&kvm
->srcu
);
512 s390_reset_cmma(kvm
->arch
.gmap
->mm
);
513 srcu_read_unlock(&kvm
->srcu
, idx
);
514 mutex_unlock(&kvm
->lock
);
517 case KVM_S390_VM_MEM_LIMIT_SIZE
: {
518 unsigned long new_limit
;
520 if (kvm_is_ucontrol(kvm
))
523 if (get_user(new_limit
, (u64 __user
*)attr
->addr
))
526 if (kvm
->arch
.mem_limit
!= KVM_S390_NO_MEM_LIMIT
&&
527 new_limit
> kvm
->arch
.mem_limit
)
533 /* gmap_alloc takes last usable address */
534 if (new_limit
!= KVM_S390_NO_MEM_LIMIT
)
538 mutex_lock(&kvm
->lock
);
539 if (!kvm
->created_vcpus
) {
540 /* gmap_alloc will round the limit up */
541 struct gmap
*new = gmap_alloc(current
->mm
, new_limit
);
546 gmap_free(kvm
->arch
.gmap
);
548 kvm
->arch
.gmap
= new;
552 mutex_unlock(&kvm
->lock
);
553 VM_EVENT(kvm
, 3, "SET: max guest address: %lu", new_limit
);
554 VM_EVENT(kvm
, 3, "New guest asce: 0x%pK",
555 (void *) kvm
->arch
.gmap
->asce
);
565 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu
*vcpu
);
567 static int kvm_s390_vm_set_crypto(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
569 struct kvm_vcpu
*vcpu
;
572 if (!test_kvm_facility(kvm
, 76))
575 mutex_lock(&kvm
->lock
);
576 switch (attr
->attr
) {
577 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW
:
579 kvm
->arch
.crypto
.crycb
->aes_wrapping_key_mask
,
580 sizeof(kvm
->arch
.crypto
.crycb
->aes_wrapping_key_mask
));
581 kvm
->arch
.crypto
.aes_kw
= 1;
582 VM_EVENT(kvm
, 3, "%s", "ENABLE: AES keywrapping support");
584 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW
:
586 kvm
->arch
.crypto
.crycb
->dea_wrapping_key_mask
,
587 sizeof(kvm
->arch
.crypto
.crycb
->dea_wrapping_key_mask
));
588 kvm
->arch
.crypto
.dea_kw
= 1;
589 VM_EVENT(kvm
, 3, "%s", "ENABLE: DEA keywrapping support");
591 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW
:
592 kvm
->arch
.crypto
.aes_kw
= 0;
593 memset(kvm
->arch
.crypto
.crycb
->aes_wrapping_key_mask
, 0,
594 sizeof(kvm
->arch
.crypto
.crycb
->aes_wrapping_key_mask
));
595 VM_EVENT(kvm
, 3, "%s", "DISABLE: AES keywrapping support");
597 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW
:
598 kvm
->arch
.crypto
.dea_kw
= 0;
599 memset(kvm
->arch
.crypto
.crycb
->dea_wrapping_key_mask
, 0,
600 sizeof(kvm
->arch
.crypto
.crycb
->dea_wrapping_key_mask
));
601 VM_EVENT(kvm
, 3, "%s", "DISABLE: DEA keywrapping support");
604 mutex_unlock(&kvm
->lock
);
608 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
609 kvm_s390_vcpu_crypto_setup(vcpu
);
612 mutex_unlock(&kvm
->lock
);
616 static int kvm_s390_set_tod_high(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
620 if (copy_from_user(>od_high
, (void __user
*)attr
->addr
,
626 VM_EVENT(kvm
, 3, "SET: TOD extension: 0x%x", gtod_high
);
631 static int kvm_s390_set_tod_low(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
635 if (copy_from_user(>od
, (void __user
*)attr
->addr
, sizeof(gtod
)))
638 kvm_s390_set_tod_clock(kvm
, gtod
);
639 VM_EVENT(kvm
, 3, "SET: TOD base: 0x%llx", gtod
);
643 static int kvm_s390_set_tod(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
650 switch (attr
->attr
) {
651 case KVM_S390_VM_TOD_HIGH
:
652 ret
= kvm_s390_set_tod_high(kvm
, attr
);
654 case KVM_S390_VM_TOD_LOW
:
655 ret
= kvm_s390_set_tod_low(kvm
, attr
);
664 static int kvm_s390_get_tod_high(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
668 if (copy_to_user((void __user
*)attr
->addr
, >od_high
,
671 VM_EVENT(kvm
, 3, "QUERY: TOD extension: 0x%x", gtod_high
);
676 static int kvm_s390_get_tod_low(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
680 gtod
= kvm_s390_get_tod_clock_fast(kvm
);
681 if (copy_to_user((void __user
*)attr
->addr
, >od
, sizeof(gtod
)))
683 VM_EVENT(kvm
, 3, "QUERY: TOD base: 0x%llx", gtod
);
688 static int kvm_s390_get_tod(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
695 switch (attr
->attr
) {
696 case KVM_S390_VM_TOD_HIGH
:
697 ret
= kvm_s390_get_tod_high(kvm
, attr
);
699 case KVM_S390_VM_TOD_LOW
:
700 ret
= kvm_s390_get_tod_low(kvm
, attr
);
709 static int kvm_s390_set_processor(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
711 struct kvm_s390_vm_cpu_processor
*proc
;
712 u16 lowest_ibc
, unblocked_ibc
;
715 mutex_lock(&kvm
->lock
);
716 if (kvm
->created_vcpus
) {
720 proc
= kzalloc(sizeof(*proc
), GFP_KERNEL
);
725 if (!copy_from_user(proc
, (void __user
*)attr
->addr
,
727 kvm
->arch
.model
.cpuid
= proc
->cpuid
;
728 lowest_ibc
= sclp
.ibc
>> 16 & 0xfff;
729 unblocked_ibc
= sclp
.ibc
& 0xfff;
731 if (proc
->ibc
> unblocked_ibc
)
732 kvm
->arch
.model
.ibc
= unblocked_ibc
;
733 else if (proc
->ibc
< lowest_ibc
)
734 kvm
->arch
.model
.ibc
= lowest_ibc
;
736 kvm
->arch
.model
.ibc
= proc
->ibc
;
738 memcpy(kvm
->arch
.model
.fac_list
, proc
->fac_list
,
739 S390_ARCH_FAC_LIST_SIZE_BYTE
);
744 mutex_unlock(&kvm
->lock
);
748 static int kvm_s390_set_processor_feat(struct kvm
*kvm
,
749 struct kvm_device_attr
*attr
)
751 struct kvm_s390_vm_cpu_feat data
;
754 if (copy_from_user(&data
, (void __user
*)attr
->addr
, sizeof(data
)))
756 if (!bitmap_subset((unsigned long *) data
.feat
,
757 kvm_s390_available_cpu_feat
,
758 KVM_S390_VM_CPU_FEAT_NR_BITS
))
761 mutex_lock(&kvm
->lock
);
762 if (!atomic_read(&kvm
->online_vcpus
)) {
763 bitmap_copy(kvm
->arch
.cpu_feat
, (unsigned long *) data
.feat
,
764 KVM_S390_VM_CPU_FEAT_NR_BITS
);
767 mutex_unlock(&kvm
->lock
);
771 static int kvm_s390_set_processor_subfunc(struct kvm
*kvm
,
772 struct kvm_device_attr
*attr
)
775 * Once supported by kernel + hw, we have to store the subfunctions
776 * in kvm->arch and remember that user space configured them.
781 static int kvm_s390_set_cpu_model(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
785 switch (attr
->attr
) {
786 case KVM_S390_VM_CPU_PROCESSOR
:
787 ret
= kvm_s390_set_processor(kvm
, attr
);
789 case KVM_S390_VM_CPU_PROCESSOR_FEAT
:
790 ret
= kvm_s390_set_processor_feat(kvm
, attr
);
792 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC
:
793 ret
= kvm_s390_set_processor_subfunc(kvm
, attr
);
799 static int kvm_s390_get_processor(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
801 struct kvm_s390_vm_cpu_processor
*proc
;
804 proc
= kzalloc(sizeof(*proc
), GFP_KERNEL
);
809 proc
->cpuid
= kvm
->arch
.model
.cpuid
;
810 proc
->ibc
= kvm
->arch
.model
.ibc
;
811 memcpy(&proc
->fac_list
, kvm
->arch
.model
.fac_list
,
812 S390_ARCH_FAC_LIST_SIZE_BYTE
);
813 if (copy_to_user((void __user
*)attr
->addr
, proc
, sizeof(*proc
)))
820 static int kvm_s390_get_machine(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
822 struct kvm_s390_vm_cpu_machine
*mach
;
825 mach
= kzalloc(sizeof(*mach
), GFP_KERNEL
);
830 get_cpu_id((struct cpuid
*) &mach
->cpuid
);
831 mach
->ibc
= sclp
.ibc
;
832 memcpy(&mach
->fac_mask
, kvm
->arch
.model
.fac_mask
,
833 S390_ARCH_FAC_LIST_SIZE_BYTE
);
834 memcpy((unsigned long *)&mach
->fac_list
, S390_lowcore
.stfle_fac_list
,
835 S390_ARCH_FAC_LIST_SIZE_BYTE
);
836 if (copy_to_user((void __user
*)attr
->addr
, mach
, sizeof(*mach
)))
843 static int kvm_s390_get_processor_feat(struct kvm
*kvm
,
844 struct kvm_device_attr
*attr
)
846 struct kvm_s390_vm_cpu_feat data
;
848 bitmap_copy((unsigned long *) data
.feat
, kvm
->arch
.cpu_feat
,
849 KVM_S390_VM_CPU_FEAT_NR_BITS
);
850 if (copy_to_user((void __user
*)attr
->addr
, &data
, sizeof(data
)))
855 static int kvm_s390_get_machine_feat(struct kvm
*kvm
,
856 struct kvm_device_attr
*attr
)
858 struct kvm_s390_vm_cpu_feat data
;
860 bitmap_copy((unsigned long *) data
.feat
,
861 kvm_s390_available_cpu_feat
,
862 KVM_S390_VM_CPU_FEAT_NR_BITS
);
863 if (copy_to_user((void __user
*)attr
->addr
, &data
, sizeof(data
)))
868 static int kvm_s390_get_processor_subfunc(struct kvm
*kvm
,
869 struct kvm_device_attr
*attr
)
872 * Once we can actually configure subfunctions (kernel + hw support),
873 * we have to check if they were already set by user space, if so copy
874 * them from kvm->arch.
879 static int kvm_s390_get_machine_subfunc(struct kvm
*kvm
,
880 struct kvm_device_attr
*attr
)
882 if (copy_to_user((void __user
*)attr
->addr
, &kvm_s390_available_subfunc
,
883 sizeof(struct kvm_s390_vm_cpu_subfunc
)))
887 static int kvm_s390_get_cpu_model(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
891 switch (attr
->attr
) {
892 case KVM_S390_VM_CPU_PROCESSOR
:
893 ret
= kvm_s390_get_processor(kvm
, attr
);
895 case KVM_S390_VM_CPU_MACHINE
:
896 ret
= kvm_s390_get_machine(kvm
, attr
);
898 case KVM_S390_VM_CPU_PROCESSOR_FEAT
:
899 ret
= kvm_s390_get_processor_feat(kvm
, attr
);
901 case KVM_S390_VM_CPU_MACHINE_FEAT
:
902 ret
= kvm_s390_get_machine_feat(kvm
, attr
);
904 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC
:
905 ret
= kvm_s390_get_processor_subfunc(kvm
, attr
);
907 case KVM_S390_VM_CPU_MACHINE_SUBFUNC
:
908 ret
= kvm_s390_get_machine_subfunc(kvm
, attr
);
914 static int kvm_s390_vm_set_attr(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
918 switch (attr
->group
) {
919 case KVM_S390_VM_MEM_CTRL
:
920 ret
= kvm_s390_set_mem_control(kvm
, attr
);
922 case KVM_S390_VM_TOD
:
923 ret
= kvm_s390_set_tod(kvm
, attr
);
925 case KVM_S390_VM_CPU_MODEL
:
926 ret
= kvm_s390_set_cpu_model(kvm
, attr
);
928 case KVM_S390_VM_CRYPTO
:
929 ret
= kvm_s390_vm_set_crypto(kvm
, attr
);
939 static int kvm_s390_vm_get_attr(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
943 switch (attr
->group
) {
944 case KVM_S390_VM_MEM_CTRL
:
945 ret
= kvm_s390_get_mem_control(kvm
, attr
);
947 case KVM_S390_VM_TOD
:
948 ret
= kvm_s390_get_tod(kvm
, attr
);
950 case KVM_S390_VM_CPU_MODEL
:
951 ret
= kvm_s390_get_cpu_model(kvm
, attr
);
961 static int kvm_s390_vm_has_attr(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
965 switch (attr
->group
) {
966 case KVM_S390_VM_MEM_CTRL
:
967 switch (attr
->attr
) {
968 case KVM_S390_VM_MEM_ENABLE_CMMA
:
969 case KVM_S390_VM_MEM_CLR_CMMA
:
970 ret
= sclp
.has_cmma
? 0 : -ENXIO
;
972 case KVM_S390_VM_MEM_LIMIT_SIZE
:
980 case KVM_S390_VM_TOD
:
981 switch (attr
->attr
) {
982 case KVM_S390_VM_TOD_LOW
:
983 case KVM_S390_VM_TOD_HIGH
:
991 case KVM_S390_VM_CPU_MODEL
:
992 switch (attr
->attr
) {
993 case KVM_S390_VM_CPU_PROCESSOR
:
994 case KVM_S390_VM_CPU_MACHINE
:
995 case KVM_S390_VM_CPU_PROCESSOR_FEAT
:
996 case KVM_S390_VM_CPU_MACHINE_FEAT
:
997 case KVM_S390_VM_CPU_MACHINE_SUBFUNC
:
1000 /* configuring subfunctions is not supported yet */
1001 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC
:
1007 case KVM_S390_VM_CRYPTO
:
1008 switch (attr
->attr
) {
1009 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW
:
1010 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW
:
1011 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW
:
1012 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW
:
1028 static long kvm_s390_get_skeys(struct kvm
*kvm
, struct kvm_s390_skeys
*args
)
1034 if (args
->flags
!= 0)
1037 /* Is this guest using storage keys? */
1038 if (!mm_use_skey(current
->mm
))
1039 return KVM_S390_GET_SKEYS_NONE
;
1041 /* Enforce sane limit on memory allocation */
1042 if (args
->count
< 1 || args
->count
> KVM_S390_SKEYS_MAX
)
1045 keys
= kmalloc_array(args
->count
, sizeof(uint8_t),
1046 GFP_KERNEL
| __GFP_NOWARN
);
1048 keys
= vmalloc(sizeof(uint8_t) * args
->count
);
1052 down_read(¤t
->mm
->mmap_sem
);
1053 for (i
= 0; i
< args
->count
; i
++) {
1054 hva
= gfn_to_hva(kvm
, args
->start_gfn
+ i
);
1055 if (kvm_is_error_hva(hva
)) {
1060 r
= get_guest_storage_key(current
->mm
, hva
, &keys
[i
]);
1064 up_read(¤t
->mm
->mmap_sem
);
1067 r
= copy_to_user((uint8_t __user
*)args
->skeydata_addr
, keys
,
1068 sizeof(uint8_t) * args
->count
);
1077 static long kvm_s390_set_skeys(struct kvm
*kvm
, struct kvm_s390_skeys
*args
)
1083 if (args
->flags
!= 0)
1086 /* Enforce sane limit on memory allocation */
1087 if (args
->count
< 1 || args
->count
> KVM_S390_SKEYS_MAX
)
1090 keys
= kmalloc_array(args
->count
, sizeof(uint8_t),
1091 GFP_KERNEL
| __GFP_NOWARN
);
1093 keys
= vmalloc(sizeof(uint8_t) * args
->count
);
1097 r
= copy_from_user(keys
, (uint8_t __user
*)args
->skeydata_addr
,
1098 sizeof(uint8_t) * args
->count
);
1104 /* Enable storage key handling for the guest */
1105 r
= s390_enable_skey();
1109 down_read(¤t
->mm
->mmap_sem
);
1110 for (i
= 0; i
< args
->count
; i
++) {
1111 hva
= gfn_to_hva(kvm
, args
->start_gfn
+ i
);
1112 if (kvm_is_error_hva(hva
)) {
1117 /* Lowest order bit is reserved */
1118 if (keys
[i
] & 0x01) {
1123 r
= set_guest_storage_key(current
->mm
, hva
, keys
[i
], 0);
1127 up_read(¤t
->mm
->mmap_sem
);
1133 long kvm_arch_vm_ioctl(struct file
*filp
,
1134 unsigned int ioctl
, unsigned long arg
)
1136 struct kvm
*kvm
= filp
->private_data
;
1137 void __user
*argp
= (void __user
*)arg
;
1138 struct kvm_device_attr attr
;
1142 case KVM_S390_INTERRUPT
: {
1143 struct kvm_s390_interrupt s390int
;
1146 if (copy_from_user(&s390int
, argp
, sizeof(s390int
)))
1148 r
= kvm_s390_inject_vm(kvm
, &s390int
);
1151 case KVM_ENABLE_CAP
: {
1152 struct kvm_enable_cap cap
;
1154 if (copy_from_user(&cap
, argp
, sizeof(cap
)))
1156 r
= kvm_vm_ioctl_enable_cap(kvm
, &cap
);
1159 case KVM_CREATE_IRQCHIP
: {
1160 struct kvm_irq_routing_entry routing
;
1163 if (kvm
->arch
.use_irqchip
) {
1164 /* Set up dummy routing. */
1165 memset(&routing
, 0, sizeof(routing
));
1166 r
= kvm_set_irq_routing(kvm
, &routing
, 0, 0);
1170 case KVM_SET_DEVICE_ATTR
: {
1172 if (copy_from_user(&attr
, (void __user
*)arg
, sizeof(attr
)))
1174 r
= kvm_s390_vm_set_attr(kvm
, &attr
);
1177 case KVM_GET_DEVICE_ATTR
: {
1179 if (copy_from_user(&attr
, (void __user
*)arg
, sizeof(attr
)))
1181 r
= kvm_s390_vm_get_attr(kvm
, &attr
);
1184 case KVM_HAS_DEVICE_ATTR
: {
1186 if (copy_from_user(&attr
, (void __user
*)arg
, sizeof(attr
)))
1188 r
= kvm_s390_vm_has_attr(kvm
, &attr
);
1191 case KVM_S390_GET_SKEYS
: {
1192 struct kvm_s390_skeys args
;
1195 if (copy_from_user(&args
, argp
,
1196 sizeof(struct kvm_s390_skeys
)))
1198 r
= kvm_s390_get_skeys(kvm
, &args
);
1201 case KVM_S390_SET_SKEYS
: {
1202 struct kvm_s390_skeys args
;
1205 if (copy_from_user(&args
, argp
,
1206 sizeof(struct kvm_s390_skeys
)))
1208 r
= kvm_s390_set_skeys(kvm
, &args
);
1218 static int kvm_s390_query_ap_config(u8
*config
)
1220 u32 fcn_code
= 0x04000000UL
;
1223 memset(config
, 0, 128);
1227 ".long 0xb2af0000\n" /* PQAP(QCI) */
1233 : "r" (fcn_code
), "r" (config
)
1234 : "cc", "0", "2", "memory"
1240 static int kvm_s390_apxa_installed(void)
1245 if (test_facility(12)) {
1246 cc
= kvm_s390_query_ap_config(config
);
1249 pr_err("PQAP(QCI) failed with cc=%d", cc
);
1251 return config
[0] & 0x40;
1257 static void kvm_s390_set_crycb_format(struct kvm
*kvm
)
1259 kvm
->arch
.crypto
.crycbd
= (__u32
)(unsigned long) kvm
->arch
.crypto
.crycb
;
1261 if (kvm_s390_apxa_installed())
1262 kvm
->arch
.crypto
.crycbd
|= CRYCB_FORMAT2
;
1264 kvm
->arch
.crypto
.crycbd
|= CRYCB_FORMAT1
;
1267 static u64
kvm_s390_get_initial_cpuid(void)
1272 cpuid
.version
= 0xff;
1273 return *((u64
*) &cpuid
);
1276 static void kvm_s390_crypto_init(struct kvm
*kvm
)
1278 if (!test_kvm_facility(kvm
, 76))
1281 kvm
->arch
.crypto
.crycb
= &kvm
->arch
.sie_page2
->crycb
;
1282 kvm_s390_set_crycb_format(kvm
);
1284 /* Enable AES/DEA protected key functions by default */
1285 kvm
->arch
.crypto
.aes_kw
= 1;
1286 kvm
->arch
.crypto
.dea_kw
= 1;
1287 get_random_bytes(kvm
->arch
.crypto
.crycb
->aes_wrapping_key_mask
,
1288 sizeof(kvm
->arch
.crypto
.crycb
->aes_wrapping_key_mask
));
1289 get_random_bytes(kvm
->arch
.crypto
.crycb
->dea_wrapping_key_mask
,
1290 sizeof(kvm
->arch
.crypto
.crycb
->dea_wrapping_key_mask
));
1293 static void sca_dispose(struct kvm
*kvm
)
1295 if (kvm
->arch
.use_esca
)
1296 free_pages_exact(kvm
->arch
.sca
, sizeof(struct esca_block
));
1298 free_page((unsigned long)(kvm
->arch
.sca
));
1299 kvm
->arch
.sca
= NULL
;
1302 int kvm_arch_init_vm(struct kvm
*kvm
, unsigned long type
)
1304 gfp_t alloc_flags
= GFP_KERNEL
;
1306 char debug_name
[16];
1307 static unsigned long sca_offset
;
1310 #ifdef CONFIG_KVM_S390_UCONTROL
1311 if (type
& ~KVM_VM_S390_UCONTROL
)
1313 if ((type
& KVM_VM_S390_UCONTROL
) && (!capable(CAP_SYS_ADMIN
)))
1320 rc
= s390_enable_sie();
1326 ratelimit_state_init(&kvm
->arch
.sthyi_limit
, 5 * HZ
, 500);
1328 kvm
->arch
.use_esca
= 0; /* start with basic SCA */
1329 if (!sclp
.has_64bscao
)
1330 alloc_flags
|= GFP_DMA
;
1331 rwlock_init(&kvm
->arch
.sca_lock
);
1332 kvm
->arch
.sca
= (struct bsca_block
*) get_zeroed_page(alloc_flags
);
1335 spin_lock(&kvm_lock
);
1337 if (sca_offset
+ sizeof(struct bsca_block
) > PAGE_SIZE
)
1339 kvm
->arch
.sca
= (struct bsca_block
*)
1340 ((char *) kvm
->arch
.sca
+ sca_offset
);
1341 spin_unlock(&kvm_lock
);
1343 sprintf(debug_name
, "kvm-%u", current
->pid
);
1345 kvm
->arch
.dbf
= debug_register(debug_name
, 32, 1, 7 * sizeof(long));
1349 kvm
->arch
.sie_page2
=
1350 (struct sie_page2
*) get_zeroed_page(GFP_KERNEL
| GFP_DMA
);
1351 if (!kvm
->arch
.sie_page2
)
1354 /* Populate the facility mask initially. */
1355 memcpy(kvm
->arch
.model
.fac_mask
, S390_lowcore
.stfle_fac_list
,
1356 S390_ARCH_FAC_LIST_SIZE_BYTE
);
1357 for (i
= 0; i
< S390_ARCH_FAC_LIST_SIZE_U64
; i
++) {
1358 if (i
< kvm_s390_fac_list_mask_size())
1359 kvm
->arch
.model
.fac_mask
[i
] &= kvm_s390_fac_list_mask
[i
];
1361 kvm
->arch
.model
.fac_mask
[i
] = 0UL;
1364 /* Populate the facility list initially. */
1365 kvm
->arch
.model
.fac_list
= kvm
->arch
.sie_page2
->fac_list
;
1366 memcpy(kvm
->arch
.model
.fac_list
, kvm
->arch
.model
.fac_mask
,
1367 S390_ARCH_FAC_LIST_SIZE_BYTE
);
1369 set_kvm_facility(kvm
->arch
.model
.fac_mask
, 74);
1370 set_kvm_facility(kvm
->arch
.model
.fac_list
, 74);
1372 kvm
->arch
.model
.cpuid
= kvm_s390_get_initial_cpuid();
1373 kvm
->arch
.model
.ibc
= sclp
.ibc
& 0x0fff;
1375 kvm_s390_crypto_init(kvm
);
1377 spin_lock_init(&kvm
->arch
.float_int
.lock
);
1378 for (i
= 0; i
< FIRQ_LIST_COUNT
; i
++)
1379 INIT_LIST_HEAD(&kvm
->arch
.float_int
.lists
[i
]);
1380 init_waitqueue_head(&kvm
->arch
.ipte_wq
);
1381 mutex_init(&kvm
->arch
.ipte_mutex
);
1383 debug_register_view(kvm
->arch
.dbf
, &debug_sprintf_view
);
1384 VM_EVENT(kvm
, 3, "vm created with type %lu", type
);
1386 if (type
& KVM_VM_S390_UCONTROL
) {
1387 kvm
->arch
.gmap
= NULL
;
1388 kvm
->arch
.mem_limit
= KVM_S390_NO_MEM_LIMIT
;
1390 if (sclp
.hamax
== U64_MAX
)
1391 kvm
->arch
.mem_limit
= TASK_MAX_SIZE
;
1393 kvm
->arch
.mem_limit
= min_t(unsigned long, TASK_MAX_SIZE
,
1395 kvm
->arch
.gmap
= gmap_alloc(current
->mm
, kvm
->arch
.mem_limit
- 1);
1396 if (!kvm
->arch
.gmap
)
1398 kvm
->arch
.gmap
->private = kvm
;
1399 kvm
->arch
.gmap
->pfault_enabled
= 0;
1402 kvm
->arch
.css_support
= 0;
1403 kvm
->arch
.use_irqchip
= 0;
1404 kvm
->arch
.epoch
= 0;
1406 spin_lock_init(&kvm
->arch
.start_stop_lock
);
1407 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm
, current
->pid
);
1411 free_page((unsigned long)kvm
->arch
.sie_page2
);
1412 debug_unregister(kvm
->arch
.dbf
);
1414 KVM_EVENT(3, "creation of vm failed: %d", rc
);
1418 void kvm_arch_vcpu_destroy(struct kvm_vcpu
*vcpu
)
1420 VCPU_EVENT(vcpu
, 3, "%s", "free cpu");
1421 trace_kvm_s390_destroy_vcpu(vcpu
->vcpu_id
);
1422 kvm_s390_clear_local_irqs(vcpu
);
1423 kvm_clear_async_pf_completion_queue(vcpu
);
1424 if (!kvm_is_ucontrol(vcpu
->kvm
))
1427 if (kvm_is_ucontrol(vcpu
->kvm
))
1428 gmap_free(vcpu
->arch
.gmap
);
1430 if (vcpu
->kvm
->arch
.use_cmma
)
1431 kvm_s390_vcpu_unsetup_cmma(vcpu
);
1432 free_page((unsigned long)(vcpu
->arch
.sie_block
));
1434 kvm_vcpu_uninit(vcpu
);
1435 kmem_cache_free(kvm_vcpu_cache
, vcpu
);
1438 static void kvm_free_vcpus(struct kvm
*kvm
)
1441 struct kvm_vcpu
*vcpu
;
1443 kvm_for_each_vcpu(i
, vcpu
, kvm
)
1444 kvm_arch_vcpu_destroy(vcpu
);
1446 mutex_lock(&kvm
->lock
);
1447 for (i
= 0; i
< atomic_read(&kvm
->online_vcpus
); i
++)
1448 kvm
->vcpus
[i
] = NULL
;
1450 atomic_set(&kvm
->online_vcpus
, 0);
1451 mutex_unlock(&kvm
->lock
);
1454 void kvm_arch_destroy_vm(struct kvm
*kvm
)
1456 kvm_free_vcpus(kvm
);
1458 debug_unregister(kvm
->arch
.dbf
);
1459 free_page((unsigned long)kvm
->arch
.sie_page2
);
1460 if (!kvm_is_ucontrol(kvm
))
1461 gmap_free(kvm
->arch
.gmap
);
1462 kvm_s390_destroy_adapters(kvm
);
1463 kvm_s390_clear_float_irqs(kvm
);
1464 KVM_EVENT(3, "vm 0x%pK destroyed", kvm
);
1467 /* Section: vcpu related */
1468 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu
*vcpu
)
1470 vcpu
->arch
.gmap
= gmap_alloc(current
->mm
, -1UL);
1471 if (!vcpu
->arch
.gmap
)
1473 vcpu
->arch
.gmap
->private = vcpu
->kvm
;
1478 static void sca_del_vcpu(struct kvm_vcpu
*vcpu
)
1480 read_lock(&vcpu
->kvm
->arch
.sca_lock
);
1481 if (vcpu
->kvm
->arch
.use_esca
) {
1482 struct esca_block
*sca
= vcpu
->kvm
->arch
.sca
;
1484 clear_bit_inv(vcpu
->vcpu_id
, (unsigned long *) sca
->mcn
);
1485 sca
->cpu
[vcpu
->vcpu_id
].sda
= 0;
1487 struct bsca_block
*sca
= vcpu
->kvm
->arch
.sca
;
1489 clear_bit_inv(vcpu
->vcpu_id
, (unsigned long *) &sca
->mcn
);
1490 sca
->cpu
[vcpu
->vcpu_id
].sda
= 0;
1492 read_unlock(&vcpu
->kvm
->arch
.sca_lock
);
1495 static void sca_add_vcpu(struct kvm_vcpu
*vcpu
)
1497 read_lock(&vcpu
->kvm
->arch
.sca_lock
);
1498 if (vcpu
->kvm
->arch
.use_esca
) {
1499 struct esca_block
*sca
= vcpu
->kvm
->arch
.sca
;
1501 sca
->cpu
[vcpu
->vcpu_id
].sda
= (__u64
) vcpu
->arch
.sie_block
;
1502 vcpu
->arch
.sie_block
->scaoh
= (__u32
)(((__u64
)sca
) >> 32);
1503 vcpu
->arch
.sie_block
->scaol
= (__u32
)(__u64
)sca
& ~0x3fU
;
1504 vcpu
->arch
.sie_block
->ecb2
|= 0x04U
;
1505 set_bit_inv(vcpu
->vcpu_id
, (unsigned long *) sca
->mcn
);
1507 struct bsca_block
*sca
= vcpu
->kvm
->arch
.sca
;
1509 sca
->cpu
[vcpu
->vcpu_id
].sda
= (__u64
) vcpu
->arch
.sie_block
;
1510 vcpu
->arch
.sie_block
->scaoh
= (__u32
)(((__u64
)sca
) >> 32);
1511 vcpu
->arch
.sie_block
->scaol
= (__u32
)(__u64
)sca
;
1512 set_bit_inv(vcpu
->vcpu_id
, (unsigned long *) &sca
->mcn
);
1514 read_unlock(&vcpu
->kvm
->arch
.sca_lock
);
1517 /* Basic SCA to Extended SCA data copy routines */
1518 static inline void sca_copy_entry(struct esca_entry
*d
, struct bsca_entry
*s
)
1521 d
->sigp_ctrl
.c
= s
->sigp_ctrl
.c
;
1522 d
->sigp_ctrl
.scn
= s
->sigp_ctrl
.scn
;
1525 static void sca_copy_b_to_e(struct esca_block
*d
, struct bsca_block
*s
)
1529 d
->ipte_control
= s
->ipte_control
;
1531 for (i
= 0; i
< KVM_S390_BSCA_CPU_SLOTS
; i
++)
1532 sca_copy_entry(&d
->cpu
[i
], &s
->cpu
[i
]);
1535 static int sca_switch_to_extended(struct kvm
*kvm
)
1537 struct bsca_block
*old_sca
= kvm
->arch
.sca
;
1538 struct esca_block
*new_sca
;
1539 struct kvm_vcpu
*vcpu
;
1540 unsigned int vcpu_idx
;
1543 new_sca
= alloc_pages_exact(sizeof(*new_sca
), GFP_KERNEL
|__GFP_ZERO
);
1547 scaoh
= (u32
)((u64
)(new_sca
) >> 32);
1548 scaol
= (u32
)(u64
)(new_sca
) & ~0x3fU
;
1550 kvm_s390_vcpu_block_all(kvm
);
1551 write_lock(&kvm
->arch
.sca_lock
);
1553 sca_copy_b_to_e(new_sca
, old_sca
);
1555 kvm_for_each_vcpu(vcpu_idx
, vcpu
, kvm
) {
1556 vcpu
->arch
.sie_block
->scaoh
= scaoh
;
1557 vcpu
->arch
.sie_block
->scaol
= scaol
;
1558 vcpu
->arch
.sie_block
->ecb2
|= 0x04U
;
1560 kvm
->arch
.sca
= new_sca
;
1561 kvm
->arch
.use_esca
= 1;
1563 write_unlock(&kvm
->arch
.sca_lock
);
1564 kvm_s390_vcpu_unblock_all(kvm
);
1566 free_page((unsigned long)old_sca
);
1568 VM_EVENT(kvm
, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1569 old_sca
, kvm
->arch
.sca
);
1573 static int sca_can_add_vcpu(struct kvm
*kvm
, unsigned int id
)
1577 if (id
< KVM_S390_BSCA_CPU_SLOTS
)
1579 if (!sclp
.has_esca
|| !sclp
.has_64bscao
)
1582 mutex_lock(&kvm
->lock
);
1583 rc
= kvm
->arch
.use_esca
? 0 : sca_switch_to_extended(kvm
);
1584 mutex_unlock(&kvm
->lock
);
1586 return rc
== 0 && id
< KVM_S390_ESCA_CPU_SLOTS
;
1589 int kvm_arch_vcpu_init(struct kvm_vcpu
*vcpu
)
1591 vcpu
->arch
.pfault_token
= KVM_S390_PFAULT_TOKEN_INVALID
;
1592 kvm_clear_async_pf_completion_queue(vcpu
);
1593 vcpu
->run
->kvm_valid_regs
= KVM_SYNC_PREFIX
|
1599 if (test_kvm_facility(vcpu
->kvm
, 64))
1600 vcpu
->run
->kvm_valid_regs
|= KVM_SYNC_RICCB
;
1601 /* fprs can be synchronized via vrs, even if the guest has no vx. With
1602 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1605 vcpu
->run
->kvm_valid_regs
|= KVM_SYNC_VRS
;
1607 vcpu
->run
->kvm_valid_regs
|= KVM_SYNC_FPRS
;
1609 if (kvm_is_ucontrol(vcpu
->kvm
))
1610 return __kvm_ucontrol_vcpu_init(vcpu
);
1615 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1616 static void __start_cpu_timer_accounting(struct kvm_vcpu
*vcpu
)
1618 WARN_ON_ONCE(vcpu
->arch
.cputm_start
!= 0);
1619 raw_write_seqcount_begin(&vcpu
->arch
.cputm_seqcount
);
1620 vcpu
->arch
.cputm_start
= get_tod_clock_fast();
1621 raw_write_seqcount_end(&vcpu
->arch
.cputm_seqcount
);
1624 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1625 static void __stop_cpu_timer_accounting(struct kvm_vcpu
*vcpu
)
1627 WARN_ON_ONCE(vcpu
->arch
.cputm_start
== 0);
1628 raw_write_seqcount_begin(&vcpu
->arch
.cputm_seqcount
);
1629 vcpu
->arch
.sie_block
->cputm
-= get_tod_clock_fast() - vcpu
->arch
.cputm_start
;
1630 vcpu
->arch
.cputm_start
= 0;
1631 raw_write_seqcount_end(&vcpu
->arch
.cputm_seqcount
);
1634 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1635 static void __enable_cpu_timer_accounting(struct kvm_vcpu
*vcpu
)
1637 WARN_ON_ONCE(vcpu
->arch
.cputm_enabled
);
1638 vcpu
->arch
.cputm_enabled
= true;
1639 __start_cpu_timer_accounting(vcpu
);
1642 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1643 static void __disable_cpu_timer_accounting(struct kvm_vcpu
*vcpu
)
1645 WARN_ON_ONCE(!vcpu
->arch
.cputm_enabled
);
1646 __stop_cpu_timer_accounting(vcpu
);
1647 vcpu
->arch
.cputm_enabled
= false;
1650 static void enable_cpu_timer_accounting(struct kvm_vcpu
*vcpu
)
1652 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1653 __enable_cpu_timer_accounting(vcpu
);
1657 static void disable_cpu_timer_accounting(struct kvm_vcpu
*vcpu
)
1659 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1660 __disable_cpu_timer_accounting(vcpu
);
1664 /* set the cpu timer - may only be called from the VCPU thread itself */
1665 void kvm_s390_set_cpu_timer(struct kvm_vcpu
*vcpu
, __u64 cputm
)
1667 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1668 raw_write_seqcount_begin(&vcpu
->arch
.cputm_seqcount
);
1669 if (vcpu
->arch
.cputm_enabled
)
1670 vcpu
->arch
.cputm_start
= get_tod_clock_fast();
1671 vcpu
->arch
.sie_block
->cputm
= cputm
;
1672 raw_write_seqcount_end(&vcpu
->arch
.cputm_seqcount
);
1676 /* update and get the cpu timer - can also be called from other VCPU threads */
1677 __u64
kvm_s390_get_cpu_timer(struct kvm_vcpu
*vcpu
)
1682 if (unlikely(!vcpu
->arch
.cputm_enabled
))
1683 return vcpu
->arch
.sie_block
->cputm
;
1685 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1687 seq
= raw_read_seqcount(&vcpu
->arch
.cputm_seqcount
);
1689 * If the writer would ever execute a read in the critical
1690 * section, e.g. in irq context, we have a deadlock.
1692 WARN_ON_ONCE((seq
& 1) && smp_processor_id() == vcpu
->cpu
);
1693 value
= vcpu
->arch
.sie_block
->cputm
;
1694 /* if cputm_start is 0, accounting is being started/stopped */
1695 if (likely(vcpu
->arch
.cputm_start
))
1696 value
-= get_tod_clock_fast() - vcpu
->arch
.cputm_start
;
1697 } while (read_seqcount_retry(&vcpu
->arch
.cputm_seqcount
, seq
& ~1));
1702 void kvm_arch_vcpu_load(struct kvm_vcpu
*vcpu
, int cpu
)
1704 /* Save host register state */
1706 vcpu
->arch
.host_fpregs
.fpc
= current
->thread
.fpu
.fpc
;
1707 vcpu
->arch
.host_fpregs
.regs
= current
->thread
.fpu
.regs
;
1710 current
->thread
.fpu
.regs
= vcpu
->run
->s
.regs
.vrs
;
1712 current
->thread
.fpu
.regs
= vcpu
->run
->s
.regs
.fprs
;
1713 current
->thread
.fpu
.fpc
= vcpu
->run
->s
.regs
.fpc
;
1714 if (test_fp_ctl(current
->thread
.fpu
.fpc
))
1715 /* User space provided an invalid FPC, let's clear it */
1716 current
->thread
.fpu
.fpc
= 0;
1718 save_access_regs(vcpu
->arch
.host_acrs
);
1719 restore_access_regs(vcpu
->run
->s
.regs
.acrs
);
1720 gmap_enable(vcpu
->arch
.gmap
);
1721 atomic_or(CPUSTAT_RUNNING
, &vcpu
->arch
.sie_block
->cpuflags
);
1722 if (vcpu
->arch
.cputm_enabled
&& !is_vcpu_idle(vcpu
))
1723 __start_cpu_timer_accounting(vcpu
);
1727 void kvm_arch_vcpu_put(struct kvm_vcpu
*vcpu
)
1730 if (vcpu
->arch
.cputm_enabled
&& !is_vcpu_idle(vcpu
))
1731 __stop_cpu_timer_accounting(vcpu
);
1732 atomic_andnot(CPUSTAT_RUNNING
, &vcpu
->arch
.sie_block
->cpuflags
);
1733 gmap_disable(vcpu
->arch
.gmap
);
1735 /* Save guest register state */
1737 vcpu
->run
->s
.regs
.fpc
= current
->thread
.fpu
.fpc
;
1739 /* Restore host register state */
1740 current
->thread
.fpu
.fpc
= vcpu
->arch
.host_fpregs
.fpc
;
1741 current
->thread
.fpu
.regs
= vcpu
->arch
.host_fpregs
.regs
;
1743 save_access_regs(vcpu
->run
->s
.regs
.acrs
);
1744 restore_access_regs(vcpu
->arch
.host_acrs
);
1747 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu
*vcpu
)
1749 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1750 vcpu
->arch
.sie_block
->gpsw
.mask
= 0UL;
1751 vcpu
->arch
.sie_block
->gpsw
.addr
= 0UL;
1752 kvm_s390_set_prefix(vcpu
, 0);
1753 kvm_s390_set_cpu_timer(vcpu
, 0);
1754 vcpu
->arch
.sie_block
->ckc
= 0UL;
1755 vcpu
->arch
.sie_block
->todpr
= 0;
1756 memset(vcpu
->arch
.sie_block
->gcr
, 0, 16 * sizeof(__u64
));
1757 vcpu
->arch
.sie_block
->gcr
[0] = 0xE0UL
;
1758 vcpu
->arch
.sie_block
->gcr
[14] = 0xC2000000UL
;
1759 /* make sure the new fpc will be lazily loaded */
1761 current
->thread
.fpu
.fpc
= 0;
1762 vcpu
->arch
.sie_block
->gbea
= 1;
1763 vcpu
->arch
.sie_block
->pp
= 0;
1764 vcpu
->arch
.pfault_token
= KVM_S390_PFAULT_TOKEN_INVALID
;
1765 kvm_clear_async_pf_completion_queue(vcpu
);
1766 if (!kvm_s390_user_cpu_state_ctrl(vcpu
->kvm
))
1767 kvm_s390_vcpu_stop(vcpu
);
1768 kvm_s390_clear_local_irqs(vcpu
);
1771 void kvm_arch_vcpu_postcreate(struct kvm_vcpu
*vcpu
)
1773 mutex_lock(&vcpu
->kvm
->lock
);
1775 vcpu
->arch
.sie_block
->epoch
= vcpu
->kvm
->arch
.epoch
;
1777 mutex_unlock(&vcpu
->kvm
->lock
);
1778 if (!kvm_is_ucontrol(vcpu
->kvm
)) {
1779 vcpu
->arch
.gmap
= vcpu
->kvm
->arch
.gmap
;
1785 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu
*vcpu
)
1787 if (!test_kvm_facility(vcpu
->kvm
, 76))
1790 vcpu
->arch
.sie_block
->ecb3
&= ~(ECB3_AES
| ECB3_DEA
);
1792 if (vcpu
->kvm
->arch
.crypto
.aes_kw
)
1793 vcpu
->arch
.sie_block
->ecb3
|= ECB3_AES
;
1794 if (vcpu
->kvm
->arch
.crypto
.dea_kw
)
1795 vcpu
->arch
.sie_block
->ecb3
|= ECB3_DEA
;
1797 vcpu
->arch
.sie_block
->crycbd
= vcpu
->kvm
->arch
.crypto
.crycbd
;
1800 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu
*vcpu
)
1802 free_page(vcpu
->arch
.sie_block
->cbrlo
);
1803 vcpu
->arch
.sie_block
->cbrlo
= 0;
1806 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu
*vcpu
)
1808 vcpu
->arch
.sie_block
->cbrlo
= get_zeroed_page(GFP_KERNEL
);
1809 if (!vcpu
->arch
.sie_block
->cbrlo
)
1812 vcpu
->arch
.sie_block
->ecb2
|= 0x80;
1813 vcpu
->arch
.sie_block
->ecb2
&= ~0x08;
1817 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu
*vcpu
)
1819 struct kvm_s390_cpu_model
*model
= &vcpu
->kvm
->arch
.model
;
1821 vcpu
->arch
.sie_block
->ibc
= model
->ibc
;
1822 if (test_kvm_facility(vcpu
->kvm
, 7))
1823 vcpu
->arch
.sie_block
->fac
= (u32
)(u64
) model
->fac_list
;
1826 int kvm_arch_vcpu_setup(struct kvm_vcpu
*vcpu
)
1830 atomic_set(&vcpu
->arch
.sie_block
->cpuflags
, CPUSTAT_ZARCH
|
1834 if (test_kvm_facility(vcpu
->kvm
, 78))
1835 atomic_or(CPUSTAT_GED2
, &vcpu
->arch
.sie_block
->cpuflags
);
1836 else if (test_kvm_facility(vcpu
->kvm
, 8))
1837 atomic_or(CPUSTAT_GED
, &vcpu
->arch
.sie_block
->cpuflags
);
1839 kvm_s390_vcpu_setup_model(vcpu
);
1841 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
1842 if (MACHINE_HAS_ESOP
)
1843 vcpu
->arch
.sie_block
->ecb
|= 0x02;
1844 if (test_kvm_facility(vcpu
->kvm
, 9))
1845 vcpu
->arch
.sie_block
->ecb
|= 0x04;
1846 if (test_kvm_facility(vcpu
->kvm
, 73))
1847 vcpu
->arch
.sie_block
->ecb
|= 0x10;
1849 if (test_kvm_facility(vcpu
->kvm
, 8) && sclp
.has_pfmfi
)
1850 vcpu
->arch
.sie_block
->ecb2
|= 0x08;
1851 vcpu
->arch
.sie_block
->eca
= 0x1002000U
;
1853 vcpu
->arch
.sie_block
->eca
|= 0x80000000U
;
1855 vcpu
->arch
.sie_block
->eca
|= 0x40000000U
;
1857 vcpu
->arch
.sie_block
->eca
|= 1;
1858 if (sclp
.has_sigpif
)
1859 vcpu
->arch
.sie_block
->eca
|= 0x10000000U
;
1860 if (test_kvm_facility(vcpu
->kvm
, 64))
1861 vcpu
->arch
.sie_block
->ecb3
|= 0x01;
1862 if (test_kvm_facility(vcpu
->kvm
, 129)) {
1863 vcpu
->arch
.sie_block
->eca
|= 0x00020000;
1864 vcpu
->arch
.sie_block
->ecd
|= 0x20000000;
1866 vcpu
->arch
.sie_block
->riccbd
= (unsigned long) &vcpu
->run
->s
.regs
.riccb
;
1867 vcpu
->arch
.sie_block
->ictl
|= ICTL_ISKE
| ICTL_SSKE
| ICTL_RRBE
;
1868 if (test_kvm_facility(vcpu
->kvm
, 74))
1869 vcpu
->arch
.sie_block
->ictl
|= ICTL_OPEREXC
;
1871 if (vcpu
->kvm
->arch
.use_cmma
) {
1872 rc
= kvm_s390_vcpu_setup_cmma(vcpu
);
1876 hrtimer_init(&vcpu
->arch
.ckc_timer
, CLOCK_MONOTONIC
, HRTIMER_MODE_REL
);
1877 vcpu
->arch
.ckc_timer
.function
= kvm_s390_idle_wakeup
;
1879 kvm_s390_vcpu_crypto_setup(vcpu
);
1884 struct kvm_vcpu
*kvm_arch_vcpu_create(struct kvm
*kvm
,
1887 struct kvm_vcpu
*vcpu
;
1888 struct sie_page
*sie_page
;
1891 if (!kvm_is_ucontrol(kvm
) && !sca_can_add_vcpu(kvm
, id
))
1896 vcpu
= kmem_cache_zalloc(kvm_vcpu_cache
, GFP_KERNEL
);
1900 sie_page
= (struct sie_page
*) get_zeroed_page(GFP_KERNEL
);
1904 vcpu
->arch
.sie_block
= &sie_page
->sie_block
;
1905 vcpu
->arch
.sie_block
->itdba
= (unsigned long) &sie_page
->itdb
;
1907 /* the real guest size will always be smaller than msl */
1908 vcpu
->arch
.sie_block
->mso
= 0;
1909 vcpu
->arch
.sie_block
->msl
= sclp
.hamax
;
1911 vcpu
->arch
.sie_block
->icpua
= id
;
1912 spin_lock_init(&vcpu
->arch
.local_int
.lock
);
1913 vcpu
->arch
.local_int
.float_int
= &kvm
->arch
.float_int
;
1914 vcpu
->arch
.local_int
.wq
= &vcpu
->wq
;
1915 vcpu
->arch
.local_int
.cpuflags
= &vcpu
->arch
.sie_block
->cpuflags
;
1916 seqcount_init(&vcpu
->arch
.cputm_seqcount
);
1918 rc
= kvm_vcpu_init(vcpu
, kvm
, id
);
1920 goto out_free_sie_block
;
1921 VM_EVENT(kvm
, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id
, vcpu
,
1922 vcpu
->arch
.sie_block
);
1923 trace_kvm_s390_create_vcpu(id
, vcpu
, vcpu
->arch
.sie_block
);
1927 free_page((unsigned long)(vcpu
->arch
.sie_block
));
1929 kmem_cache_free(kvm_vcpu_cache
, vcpu
);
1934 int kvm_arch_vcpu_runnable(struct kvm_vcpu
*vcpu
)
1936 return kvm_s390_vcpu_has_irq(vcpu
, 0);
1939 void kvm_s390_vcpu_block(struct kvm_vcpu
*vcpu
)
1941 atomic_or(PROG_BLOCK_SIE
, &vcpu
->arch
.sie_block
->prog20
);
1945 void kvm_s390_vcpu_unblock(struct kvm_vcpu
*vcpu
)
1947 atomic_andnot(PROG_BLOCK_SIE
, &vcpu
->arch
.sie_block
->prog20
);
1950 static void kvm_s390_vcpu_request(struct kvm_vcpu
*vcpu
)
1952 atomic_or(PROG_REQUEST
, &vcpu
->arch
.sie_block
->prog20
);
1956 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu
*vcpu
)
1958 atomic_andnot(PROG_REQUEST
, &vcpu
->arch
.sie_block
->prog20
);
1962 * Kick a guest cpu out of SIE and wait until SIE is not running.
1963 * If the CPU is not running (e.g. waiting as idle) the function will
1964 * return immediately. */
1965 void exit_sie(struct kvm_vcpu
*vcpu
)
1967 atomic_or(CPUSTAT_STOP_INT
, &vcpu
->arch
.sie_block
->cpuflags
);
1968 while (vcpu
->arch
.sie_block
->prog0c
& PROG_IN_SIE
)
1972 /* Kick a guest cpu out of SIE to process a request synchronously */
1973 void kvm_s390_sync_request(int req
, struct kvm_vcpu
*vcpu
)
1975 kvm_make_request(req
, vcpu
);
1976 kvm_s390_vcpu_request(vcpu
);
1979 static void kvm_gmap_notifier(struct gmap
*gmap
, unsigned long address
)
1982 struct kvm
*kvm
= gmap
->private;
1983 struct kvm_vcpu
*vcpu
;
1985 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
1986 /* match against both prefix pages */
1987 if (kvm_s390_get_prefix(vcpu
) == (address
& ~0x1000UL
)) {
1988 VCPU_EVENT(vcpu
, 2, "gmap notifier for %lx", address
);
1989 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD
, vcpu
);
1994 int kvm_arch_vcpu_should_kick(struct kvm_vcpu
*vcpu
)
1996 /* kvm common code refers to this, but never calls it */
2001 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu
*vcpu
,
2002 struct kvm_one_reg
*reg
)
2007 case KVM_REG_S390_TODPR
:
2008 r
= put_user(vcpu
->arch
.sie_block
->todpr
,
2009 (u32 __user
*)reg
->addr
);
2011 case KVM_REG_S390_EPOCHDIFF
:
2012 r
= put_user(vcpu
->arch
.sie_block
->epoch
,
2013 (u64 __user
*)reg
->addr
);
2015 case KVM_REG_S390_CPU_TIMER
:
2016 r
= put_user(kvm_s390_get_cpu_timer(vcpu
),
2017 (u64 __user
*)reg
->addr
);
2019 case KVM_REG_S390_CLOCK_COMP
:
2020 r
= put_user(vcpu
->arch
.sie_block
->ckc
,
2021 (u64 __user
*)reg
->addr
);
2023 case KVM_REG_S390_PFTOKEN
:
2024 r
= put_user(vcpu
->arch
.pfault_token
,
2025 (u64 __user
*)reg
->addr
);
2027 case KVM_REG_S390_PFCOMPARE
:
2028 r
= put_user(vcpu
->arch
.pfault_compare
,
2029 (u64 __user
*)reg
->addr
);
2031 case KVM_REG_S390_PFSELECT
:
2032 r
= put_user(vcpu
->arch
.pfault_select
,
2033 (u64 __user
*)reg
->addr
);
2035 case KVM_REG_S390_PP
:
2036 r
= put_user(vcpu
->arch
.sie_block
->pp
,
2037 (u64 __user
*)reg
->addr
);
2039 case KVM_REG_S390_GBEA
:
2040 r
= put_user(vcpu
->arch
.sie_block
->gbea
,
2041 (u64 __user
*)reg
->addr
);
2050 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu
*vcpu
,
2051 struct kvm_one_reg
*reg
)
2057 case KVM_REG_S390_TODPR
:
2058 r
= get_user(vcpu
->arch
.sie_block
->todpr
,
2059 (u32 __user
*)reg
->addr
);
2061 case KVM_REG_S390_EPOCHDIFF
:
2062 r
= get_user(vcpu
->arch
.sie_block
->epoch
,
2063 (u64 __user
*)reg
->addr
);
2065 case KVM_REG_S390_CPU_TIMER
:
2066 r
= get_user(val
, (u64 __user
*)reg
->addr
);
2068 kvm_s390_set_cpu_timer(vcpu
, val
);
2070 case KVM_REG_S390_CLOCK_COMP
:
2071 r
= get_user(vcpu
->arch
.sie_block
->ckc
,
2072 (u64 __user
*)reg
->addr
);
2074 case KVM_REG_S390_PFTOKEN
:
2075 r
= get_user(vcpu
->arch
.pfault_token
,
2076 (u64 __user
*)reg
->addr
);
2077 if (vcpu
->arch
.pfault_token
== KVM_S390_PFAULT_TOKEN_INVALID
)
2078 kvm_clear_async_pf_completion_queue(vcpu
);
2080 case KVM_REG_S390_PFCOMPARE
:
2081 r
= get_user(vcpu
->arch
.pfault_compare
,
2082 (u64 __user
*)reg
->addr
);
2084 case KVM_REG_S390_PFSELECT
:
2085 r
= get_user(vcpu
->arch
.pfault_select
,
2086 (u64 __user
*)reg
->addr
);
2088 case KVM_REG_S390_PP
:
2089 r
= get_user(vcpu
->arch
.sie_block
->pp
,
2090 (u64 __user
*)reg
->addr
);
2092 case KVM_REG_S390_GBEA
:
2093 r
= get_user(vcpu
->arch
.sie_block
->gbea
,
2094 (u64 __user
*)reg
->addr
);
2103 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu
*vcpu
)
2105 kvm_s390_vcpu_initial_reset(vcpu
);
2109 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu
*vcpu
, struct kvm_regs
*regs
)
2111 memcpy(&vcpu
->run
->s
.regs
.gprs
, ®s
->gprs
, sizeof(regs
->gprs
));
2115 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu
*vcpu
, struct kvm_regs
*regs
)
2117 memcpy(®s
->gprs
, &vcpu
->run
->s
.regs
.gprs
, sizeof(regs
->gprs
));
2121 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu
*vcpu
,
2122 struct kvm_sregs
*sregs
)
2124 memcpy(&vcpu
->run
->s
.regs
.acrs
, &sregs
->acrs
, sizeof(sregs
->acrs
));
2125 memcpy(&vcpu
->arch
.sie_block
->gcr
, &sregs
->crs
, sizeof(sregs
->crs
));
2126 restore_access_regs(vcpu
->run
->s
.regs
.acrs
);
2130 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu
*vcpu
,
2131 struct kvm_sregs
*sregs
)
2133 memcpy(&sregs
->acrs
, &vcpu
->run
->s
.regs
.acrs
, sizeof(sregs
->acrs
));
2134 memcpy(&sregs
->crs
, &vcpu
->arch
.sie_block
->gcr
, sizeof(sregs
->crs
));
2138 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu
*vcpu
, struct kvm_fpu
*fpu
)
2140 /* make sure the new values will be lazily loaded */
2142 if (test_fp_ctl(fpu
->fpc
))
2144 current
->thread
.fpu
.fpc
= fpu
->fpc
;
2146 convert_fp_to_vx(current
->thread
.fpu
.vxrs
, (freg_t
*)fpu
->fprs
);
2148 memcpy(current
->thread
.fpu
.fprs
, &fpu
->fprs
, sizeof(fpu
->fprs
));
2152 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu
*vcpu
, struct kvm_fpu
*fpu
)
2154 /* make sure we have the latest values */
2157 convert_vx_to_fp((freg_t
*)fpu
->fprs
, current
->thread
.fpu
.vxrs
);
2159 memcpy(fpu
->fprs
, current
->thread
.fpu
.fprs
, sizeof(fpu
->fprs
));
2160 fpu
->fpc
= current
->thread
.fpu
.fpc
;
2164 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu
*vcpu
, psw_t psw
)
2168 if (!is_vcpu_stopped(vcpu
))
2171 vcpu
->run
->psw_mask
= psw
.mask
;
2172 vcpu
->run
->psw_addr
= psw
.addr
;
2177 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu
*vcpu
,
2178 struct kvm_translation
*tr
)
2180 return -EINVAL
; /* not implemented yet */
2183 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2184 KVM_GUESTDBG_USE_HW_BP | \
2185 KVM_GUESTDBG_ENABLE)
2187 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu
*vcpu
,
2188 struct kvm_guest_debug
*dbg
)
2192 vcpu
->guest_debug
= 0;
2193 kvm_s390_clear_bp_data(vcpu
);
2195 if (dbg
->control
& ~VALID_GUESTDBG_FLAGS
)
2197 if (!sclp
.has_gpere
)
2200 if (dbg
->control
& KVM_GUESTDBG_ENABLE
) {
2201 vcpu
->guest_debug
= dbg
->control
;
2202 /* enforce guest PER */
2203 atomic_or(CPUSTAT_P
, &vcpu
->arch
.sie_block
->cpuflags
);
2205 if (dbg
->control
& KVM_GUESTDBG_USE_HW_BP
)
2206 rc
= kvm_s390_import_bp_data(vcpu
, dbg
);
2208 atomic_andnot(CPUSTAT_P
, &vcpu
->arch
.sie_block
->cpuflags
);
2209 vcpu
->arch
.guestdbg
.last_bp
= 0;
2213 vcpu
->guest_debug
= 0;
2214 kvm_s390_clear_bp_data(vcpu
);
2215 atomic_andnot(CPUSTAT_P
, &vcpu
->arch
.sie_block
->cpuflags
);
2221 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu
*vcpu
,
2222 struct kvm_mp_state
*mp_state
)
2224 /* CHECK_STOP and LOAD are not supported yet */
2225 return is_vcpu_stopped(vcpu
) ? KVM_MP_STATE_STOPPED
:
2226 KVM_MP_STATE_OPERATING
;
2229 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu
*vcpu
,
2230 struct kvm_mp_state
*mp_state
)
2234 /* user space knows about this interface - let it control the state */
2235 vcpu
->kvm
->arch
.user_cpu_state_ctrl
= 1;
2237 switch (mp_state
->mp_state
) {
2238 case KVM_MP_STATE_STOPPED
:
2239 kvm_s390_vcpu_stop(vcpu
);
2241 case KVM_MP_STATE_OPERATING
:
2242 kvm_s390_vcpu_start(vcpu
);
2244 case KVM_MP_STATE_LOAD
:
2245 case KVM_MP_STATE_CHECK_STOP
:
2246 /* fall through - CHECK_STOP and LOAD are not supported yet */
2254 static bool ibs_enabled(struct kvm_vcpu
*vcpu
)
2256 return atomic_read(&vcpu
->arch
.sie_block
->cpuflags
) & CPUSTAT_IBS
;
2259 static int kvm_s390_handle_requests(struct kvm_vcpu
*vcpu
)
2262 kvm_s390_vcpu_request_handled(vcpu
);
2263 if (!vcpu
->requests
)
2266 * We use MMU_RELOAD just to re-arm the ipte notifier for the
2267 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
2268 * This ensures that the ipte instruction for this request has
2269 * already finished. We might race against a second unmapper that
2270 * wants to set the blocking bit. Lets just retry the request loop.
2272 if (kvm_check_request(KVM_REQ_MMU_RELOAD
, vcpu
)) {
2274 rc
= gmap_ipte_notify(vcpu
->arch
.gmap
,
2275 kvm_s390_get_prefix(vcpu
),
2282 if (kvm_check_request(KVM_REQ_TLB_FLUSH
, vcpu
)) {
2283 vcpu
->arch
.sie_block
->ihcpu
= 0xffff;
2287 if (kvm_check_request(KVM_REQ_ENABLE_IBS
, vcpu
)) {
2288 if (!ibs_enabled(vcpu
)) {
2289 trace_kvm_s390_enable_disable_ibs(vcpu
->vcpu_id
, 1);
2290 atomic_or(CPUSTAT_IBS
,
2291 &vcpu
->arch
.sie_block
->cpuflags
);
2296 if (kvm_check_request(KVM_REQ_DISABLE_IBS
, vcpu
)) {
2297 if (ibs_enabled(vcpu
)) {
2298 trace_kvm_s390_enable_disable_ibs(vcpu
->vcpu_id
, 0);
2299 atomic_andnot(CPUSTAT_IBS
,
2300 &vcpu
->arch
.sie_block
->cpuflags
);
2305 /* nothing to do, just clear the request */
2306 clear_bit(KVM_REQ_UNHALT
, &vcpu
->requests
);
2311 void kvm_s390_set_tod_clock(struct kvm
*kvm
, u64 tod
)
2313 struct kvm_vcpu
*vcpu
;
2316 mutex_lock(&kvm
->lock
);
2318 kvm
->arch
.epoch
= tod
- get_tod_clock();
2319 kvm_s390_vcpu_block_all(kvm
);
2320 kvm_for_each_vcpu(i
, vcpu
, kvm
)
2321 vcpu
->arch
.sie_block
->epoch
= kvm
->arch
.epoch
;
2322 kvm_s390_vcpu_unblock_all(kvm
);
2324 mutex_unlock(&kvm
->lock
);
2328 * kvm_arch_fault_in_page - fault-in guest page if necessary
2329 * @vcpu: The corresponding virtual cpu
2330 * @gpa: Guest physical address
2331 * @writable: Whether the page should be writable or not
2333 * Make sure that a guest page has been faulted-in on the host.
2335 * Return: Zero on success, negative error code otherwise.
2337 long kvm_arch_fault_in_page(struct kvm_vcpu
*vcpu
, gpa_t gpa
, int writable
)
2339 return gmap_fault(vcpu
->arch
.gmap
, gpa
,
2340 writable
? FAULT_FLAG_WRITE
: 0);
2343 static void __kvm_inject_pfault_token(struct kvm_vcpu
*vcpu
, bool start_token
,
2344 unsigned long token
)
2346 struct kvm_s390_interrupt inti
;
2347 struct kvm_s390_irq irq
;
2350 irq
.u
.ext
.ext_params2
= token
;
2351 irq
.type
= KVM_S390_INT_PFAULT_INIT
;
2352 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu
, &irq
));
2354 inti
.type
= KVM_S390_INT_PFAULT_DONE
;
2355 inti
.parm64
= token
;
2356 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu
->kvm
, &inti
));
2360 void kvm_arch_async_page_not_present(struct kvm_vcpu
*vcpu
,
2361 struct kvm_async_pf
*work
)
2363 trace_kvm_s390_pfault_init(vcpu
, work
->arch
.pfault_token
);
2364 __kvm_inject_pfault_token(vcpu
, true, work
->arch
.pfault_token
);
2367 void kvm_arch_async_page_present(struct kvm_vcpu
*vcpu
,
2368 struct kvm_async_pf
*work
)
2370 trace_kvm_s390_pfault_done(vcpu
, work
->arch
.pfault_token
);
2371 __kvm_inject_pfault_token(vcpu
, false, work
->arch
.pfault_token
);
2374 void kvm_arch_async_page_ready(struct kvm_vcpu
*vcpu
,
2375 struct kvm_async_pf
*work
)
2377 /* s390 will always inject the page directly */
2380 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu
*vcpu
)
2383 * s390 will always inject the page directly,
2384 * but we still want check_async_completion to cleanup
2389 static int kvm_arch_setup_async_pf(struct kvm_vcpu
*vcpu
)
2392 struct kvm_arch_async_pf arch
;
2395 if (vcpu
->arch
.pfault_token
== KVM_S390_PFAULT_TOKEN_INVALID
)
2397 if ((vcpu
->arch
.sie_block
->gpsw
.mask
& vcpu
->arch
.pfault_select
) !=
2398 vcpu
->arch
.pfault_compare
)
2400 if (psw_extint_disabled(vcpu
))
2402 if (kvm_s390_vcpu_has_irq(vcpu
, 0))
2404 if (!(vcpu
->arch
.sie_block
->gcr
[0] & 0x200ul
))
2406 if (!vcpu
->arch
.gmap
->pfault_enabled
)
2409 hva
= gfn_to_hva(vcpu
->kvm
, gpa_to_gfn(current
->thread
.gmap_addr
));
2410 hva
+= current
->thread
.gmap_addr
& ~PAGE_MASK
;
2411 if (read_guest_real(vcpu
, vcpu
->arch
.pfault_token
, &arch
.pfault_token
, 8))
2414 rc
= kvm_setup_async_pf(vcpu
, current
->thread
.gmap_addr
, hva
, &arch
);
2418 static int vcpu_pre_run(struct kvm_vcpu
*vcpu
)
2423 * On s390 notifications for arriving pages will be delivered directly
2424 * to the guest but the house keeping for completed pfaults is
2425 * handled outside the worker.
2427 kvm_check_async_pf_completion(vcpu
);
2429 vcpu
->arch
.sie_block
->gg14
= vcpu
->run
->s
.regs
.gprs
[14];
2430 vcpu
->arch
.sie_block
->gg15
= vcpu
->run
->s
.regs
.gprs
[15];
2435 if (test_cpu_flag(CIF_MCCK_PENDING
))
2438 if (!kvm_is_ucontrol(vcpu
->kvm
)) {
2439 rc
= kvm_s390_deliver_pending_interrupts(vcpu
);
2444 rc
= kvm_s390_handle_requests(vcpu
);
2448 if (guestdbg_enabled(vcpu
)) {
2449 kvm_s390_backup_guest_per_regs(vcpu
);
2450 kvm_s390_patch_guest_per_regs(vcpu
);
2453 vcpu
->arch
.sie_block
->icptcode
= 0;
2454 cpuflags
= atomic_read(&vcpu
->arch
.sie_block
->cpuflags
);
2455 VCPU_EVENT(vcpu
, 6, "entering sie flags %x", cpuflags
);
2456 trace_kvm_s390_sie_enter(vcpu
, cpuflags
);
2461 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu
*vcpu
)
2463 struct kvm_s390_pgm_info pgm_info
= {
2464 .code
= PGM_ADDRESSING
,
2469 VCPU_EVENT(vcpu
, 3, "%s", "fault in sie instruction");
2470 trace_kvm_s390_sie_fault(vcpu
);
2473 * We want to inject an addressing exception, which is defined as a
2474 * suppressing or terminating exception. However, since we came here
2475 * by a DAT access exception, the PSW still points to the faulting
2476 * instruction since DAT exceptions are nullifying. So we've got
2477 * to look up the current opcode to get the length of the instruction
2478 * to be able to forward the PSW.
2480 rc
= read_guest_instr(vcpu
, &opcode
, 1);
2481 ilen
= insn_length(opcode
);
2485 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2486 * Forward by arbitrary ilc, injection will take care of
2487 * nullification if necessary.
2489 pgm_info
= vcpu
->arch
.pgm
;
2492 pgm_info
.flags
= ilen
| KVM_S390_PGM_FLAGS_ILC_VALID
;
2493 kvm_s390_forward_psw(vcpu
, ilen
);
2494 return kvm_s390_inject_prog_irq(vcpu
, &pgm_info
);
2497 static int vcpu_post_run(struct kvm_vcpu
*vcpu
, int exit_reason
)
2499 VCPU_EVENT(vcpu
, 6, "exit sie icptcode %d",
2500 vcpu
->arch
.sie_block
->icptcode
);
2501 trace_kvm_s390_sie_exit(vcpu
, vcpu
->arch
.sie_block
->icptcode
);
2503 if (guestdbg_enabled(vcpu
))
2504 kvm_s390_restore_guest_per_regs(vcpu
);
2506 vcpu
->run
->s
.regs
.gprs
[14] = vcpu
->arch
.sie_block
->gg14
;
2507 vcpu
->run
->s
.regs
.gprs
[15] = vcpu
->arch
.sie_block
->gg15
;
2509 if (vcpu
->arch
.sie_block
->icptcode
> 0) {
2510 int rc
= kvm_handle_sie_intercept(vcpu
);
2512 if (rc
!= -EOPNOTSUPP
)
2514 vcpu
->run
->exit_reason
= KVM_EXIT_S390_SIEIC
;
2515 vcpu
->run
->s390_sieic
.icptcode
= vcpu
->arch
.sie_block
->icptcode
;
2516 vcpu
->run
->s390_sieic
.ipa
= vcpu
->arch
.sie_block
->ipa
;
2517 vcpu
->run
->s390_sieic
.ipb
= vcpu
->arch
.sie_block
->ipb
;
2519 } else if (exit_reason
!= -EFAULT
) {
2520 vcpu
->stat
.exit_null
++;
2522 } else if (kvm_is_ucontrol(vcpu
->kvm
)) {
2523 vcpu
->run
->exit_reason
= KVM_EXIT_S390_UCONTROL
;
2524 vcpu
->run
->s390_ucontrol
.trans_exc_code
=
2525 current
->thread
.gmap_addr
;
2526 vcpu
->run
->s390_ucontrol
.pgm_code
= 0x10;
2528 } else if (current
->thread
.gmap_pfault
) {
2529 trace_kvm_s390_major_guest_pfault(vcpu
);
2530 current
->thread
.gmap_pfault
= 0;
2531 if (kvm_arch_setup_async_pf(vcpu
))
2533 return kvm_arch_fault_in_page(vcpu
, current
->thread
.gmap_addr
, 1);
2535 return vcpu_post_run_fault_in_sie(vcpu
);
2538 static int __vcpu_run(struct kvm_vcpu
*vcpu
)
2540 int rc
, exit_reason
;
2543 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2544 * ning the guest), so that memslots (and other stuff) are protected
2546 vcpu
->srcu_idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
2549 rc
= vcpu_pre_run(vcpu
);
2553 srcu_read_unlock(&vcpu
->kvm
->srcu
, vcpu
->srcu_idx
);
2555 * As PF_VCPU will be used in fault handler, between
2556 * guest_enter and guest_exit should be no uaccess.
2558 local_irq_disable();
2559 __kvm_guest_enter();
2560 __disable_cpu_timer_accounting(vcpu
);
2562 exit_reason
= sie64a(vcpu
->arch
.sie_block
,
2563 vcpu
->run
->s
.regs
.gprs
);
2564 local_irq_disable();
2565 __enable_cpu_timer_accounting(vcpu
);
2568 vcpu
->srcu_idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
2570 rc
= vcpu_post_run(vcpu
, exit_reason
);
2571 } while (!signal_pending(current
) && !guestdbg_exit_pending(vcpu
) && !rc
);
2573 srcu_read_unlock(&vcpu
->kvm
->srcu
, vcpu
->srcu_idx
);
2577 static void sync_regs(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
2579 vcpu
->arch
.sie_block
->gpsw
.mask
= kvm_run
->psw_mask
;
2580 vcpu
->arch
.sie_block
->gpsw
.addr
= kvm_run
->psw_addr
;
2581 if (kvm_run
->kvm_dirty_regs
& KVM_SYNC_PREFIX
)
2582 kvm_s390_set_prefix(vcpu
, kvm_run
->s
.regs
.prefix
);
2583 if (kvm_run
->kvm_dirty_regs
& KVM_SYNC_CRS
) {
2584 memcpy(&vcpu
->arch
.sie_block
->gcr
, &kvm_run
->s
.regs
.crs
, 128);
2585 /* some control register changes require a tlb flush */
2586 kvm_make_request(KVM_REQ_TLB_FLUSH
, vcpu
);
2588 if (kvm_run
->kvm_dirty_regs
& KVM_SYNC_ARCH0
) {
2589 kvm_s390_set_cpu_timer(vcpu
, kvm_run
->s
.regs
.cputm
);
2590 vcpu
->arch
.sie_block
->ckc
= kvm_run
->s
.regs
.ckc
;
2591 vcpu
->arch
.sie_block
->todpr
= kvm_run
->s
.regs
.todpr
;
2592 vcpu
->arch
.sie_block
->pp
= kvm_run
->s
.regs
.pp
;
2593 vcpu
->arch
.sie_block
->gbea
= kvm_run
->s
.regs
.gbea
;
2595 if (kvm_run
->kvm_dirty_regs
& KVM_SYNC_PFAULT
) {
2596 vcpu
->arch
.pfault_token
= kvm_run
->s
.regs
.pft
;
2597 vcpu
->arch
.pfault_select
= kvm_run
->s
.regs
.pfs
;
2598 vcpu
->arch
.pfault_compare
= kvm_run
->s
.regs
.pfc
;
2599 if (vcpu
->arch
.pfault_token
== KVM_S390_PFAULT_TOKEN_INVALID
)
2600 kvm_clear_async_pf_completion_queue(vcpu
);
2602 kvm_run
->kvm_dirty_regs
= 0;
2605 static void store_regs(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
2607 kvm_run
->psw_mask
= vcpu
->arch
.sie_block
->gpsw
.mask
;
2608 kvm_run
->psw_addr
= vcpu
->arch
.sie_block
->gpsw
.addr
;
2609 kvm_run
->s
.regs
.prefix
= kvm_s390_get_prefix(vcpu
);
2610 memcpy(&kvm_run
->s
.regs
.crs
, &vcpu
->arch
.sie_block
->gcr
, 128);
2611 kvm_run
->s
.regs
.cputm
= kvm_s390_get_cpu_timer(vcpu
);
2612 kvm_run
->s
.regs
.ckc
= vcpu
->arch
.sie_block
->ckc
;
2613 kvm_run
->s
.regs
.todpr
= vcpu
->arch
.sie_block
->todpr
;
2614 kvm_run
->s
.regs
.pp
= vcpu
->arch
.sie_block
->pp
;
2615 kvm_run
->s
.regs
.gbea
= vcpu
->arch
.sie_block
->gbea
;
2616 kvm_run
->s
.regs
.pft
= vcpu
->arch
.pfault_token
;
2617 kvm_run
->s
.regs
.pfs
= vcpu
->arch
.pfault_select
;
2618 kvm_run
->s
.regs
.pfc
= vcpu
->arch
.pfault_compare
;
2621 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
2626 if (guestdbg_exit_pending(vcpu
)) {
2627 kvm_s390_prepare_debug_exit(vcpu
);
2631 if (vcpu
->sigset_active
)
2632 sigprocmask(SIG_SETMASK
, &vcpu
->sigset
, &sigsaved
);
2634 if (!kvm_s390_user_cpu_state_ctrl(vcpu
->kvm
)) {
2635 kvm_s390_vcpu_start(vcpu
);
2636 } else if (is_vcpu_stopped(vcpu
)) {
2637 pr_err_ratelimited("can't run stopped vcpu %d\n",
2642 sync_regs(vcpu
, kvm_run
);
2643 enable_cpu_timer_accounting(vcpu
);
2646 rc
= __vcpu_run(vcpu
);
2648 if (signal_pending(current
) && !rc
) {
2649 kvm_run
->exit_reason
= KVM_EXIT_INTR
;
2653 if (guestdbg_exit_pending(vcpu
) && !rc
) {
2654 kvm_s390_prepare_debug_exit(vcpu
);
2658 if (rc
== -EREMOTE
) {
2659 /* userspace support is needed, kvm_run has been prepared */
2663 disable_cpu_timer_accounting(vcpu
);
2664 store_regs(vcpu
, kvm_run
);
2666 if (vcpu
->sigset_active
)
2667 sigprocmask(SIG_SETMASK
, &sigsaved
, NULL
);
2669 vcpu
->stat
.exit_userspace
++;
2674 * store status at address
2675 * we use have two special cases:
2676 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2677 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2679 int kvm_s390_store_status_unloaded(struct kvm_vcpu
*vcpu
, unsigned long gpa
)
2681 unsigned char archmode
= 1;
2682 freg_t fprs
[NUM_FPRS
];
2687 px
= kvm_s390_get_prefix(vcpu
);
2688 if (gpa
== KVM_S390_STORE_STATUS_NOADDR
) {
2689 if (write_guest_abs(vcpu
, 163, &archmode
, 1))
2692 } else if (gpa
== KVM_S390_STORE_STATUS_PREFIXED
) {
2693 if (write_guest_real(vcpu
, 163, &archmode
, 1))
2697 gpa
-= __LC_FPREGS_SAVE_AREA
;
2699 /* manually convert vector registers if necessary */
2700 if (MACHINE_HAS_VX
) {
2701 convert_vx_to_fp(fprs
, (__vector128
*) vcpu
->run
->s
.regs
.vrs
);
2702 rc
= write_guest_abs(vcpu
, gpa
+ __LC_FPREGS_SAVE_AREA
,
2705 rc
= write_guest_abs(vcpu
, gpa
+ __LC_FPREGS_SAVE_AREA
,
2706 vcpu
->run
->s
.regs
.fprs
, 128);
2708 rc
|= write_guest_abs(vcpu
, gpa
+ __LC_GPREGS_SAVE_AREA
,
2709 vcpu
->run
->s
.regs
.gprs
, 128);
2710 rc
|= write_guest_abs(vcpu
, gpa
+ __LC_PSW_SAVE_AREA
,
2711 &vcpu
->arch
.sie_block
->gpsw
, 16);
2712 rc
|= write_guest_abs(vcpu
, gpa
+ __LC_PREFIX_SAVE_AREA
,
2714 rc
|= write_guest_abs(vcpu
, gpa
+ __LC_FP_CREG_SAVE_AREA
,
2715 &vcpu
->run
->s
.regs
.fpc
, 4);
2716 rc
|= write_guest_abs(vcpu
, gpa
+ __LC_TOD_PROGREG_SAVE_AREA
,
2717 &vcpu
->arch
.sie_block
->todpr
, 4);
2718 cputm
= kvm_s390_get_cpu_timer(vcpu
);
2719 rc
|= write_guest_abs(vcpu
, gpa
+ __LC_CPU_TIMER_SAVE_AREA
,
2721 clkcomp
= vcpu
->arch
.sie_block
->ckc
>> 8;
2722 rc
|= write_guest_abs(vcpu
, gpa
+ __LC_CLOCK_COMP_SAVE_AREA
,
2724 rc
|= write_guest_abs(vcpu
, gpa
+ __LC_AREGS_SAVE_AREA
,
2725 &vcpu
->run
->s
.regs
.acrs
, 64);
2726 rc
|= write_guest_abs(vcpu
, gpa
+ __LC_CREGS_SAVE_AREA
,
2727 &vcpu
->arch
.sie_block
->gcr
, 128);
2728 return rc
? -EFAULT
: 0;
2731 int kvm_s390_vcpu_store_status(struct kvm_vcpu
*vcpu
, unsigned long addr
)
2734 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2735 * copying in vcpu load/put. Lets update our copies before we save
2736 * it into the save area
2739 vcpu
->run
->s
.regs
.fpc
= current
->thread
.fpu
.fpc
;
2740 save_access_regs(vcpu
->run
->s
.regs
.acrs
);
2742 return kvm_s390_store_status_unloaded(vcpu
, addr
);
2746 * store additional status at address
2748 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu
*vcpu
,
2751 /* Only bits 0-53 are used for address formation */
2752 if (!(gpa
& ~0x3ff))
2755 return write_guest_abs(vcpu
, gpa
& ~0x3ff,
2756 (void *)&vcpu
->run
->s
.regs
.vrs
, 512);
2759 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu
*vcpu
, unsigned long addr
)
2761 if (!test_kvm_facility(vcpu
->kvm
, 129))
2765 * The guest VXRS are in the host VXRs due to the lazy
2766 * copying in vcpu load/put. We can simply call save_fpu_regs()
2767 * to save the current register state because we are in the
2768 * middle of a load/put cycle.
2770 * Let's update our copies before we save it into the save area.
2774 return kvm_s390_store_adtl_status_unloaded(vcpu
, addr
);
2777 static void __disable_ibs_on_vcpu(struct kvm_vcpu
*vcpu
)
2779 kvm_check_request(KVM_REQ_ENABLE_IBS
, vcpu
);
2780 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS
, vcpu
);
2783 static void __disable_ibs_on_all_vcpus(struct kvm
*kvm
)
2786 struct kvm_vcpu
*vcpu
;
2788 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
2789 __disable_ibs_on_vcpu(vcpu
);
2793 static void __enable_ibs_on_vcpu(struct kvm_vcpu
*vcpu
)
2797 kvm_check_request(KVM_REQ_DISABLE_IBS
, vcpu
);
2798 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS
, vcpu
);
2801 void kvm_s390_vcpu_start(struct kvm_vcpu
*vcpu
)
2803 int i
, online_vcpus
, started_vcpus
= 0;
2805 if (!is_vcpu_stopped(vcpu
))
2808 trace_kvm_s390_vcpu_start_stop(vcpu
->vcpu_id
, 1);
2809 /* Only one cpu at a time may enter/leave the STOPPED state. */
2810 spin_lock(&vcpu
->kvm
->arch
.start_stop_lock
);
2811 online_vcpus
= atomic_read(&vcpu
->kvm
->online_vcpus
);
2813 for (i
= 0; i
< online_vcpus
; i
++) {
2814 if (!is_vcpu_stopped(vcpu
->kvm
->vcpus
[i
]))
2818 if (started_vcpus
== 0) {
2819 /* we're the only active VCPU -> speed it up */
2820 __enable_ibs_on_vcpu(vcpu
);
2821 } else if (started_vcpus
== 1) {
2823 * As we are starting a second VCPU, we have to disable
2824 * the IBS facility on all VCPUs to remove potentially
2825 * oustanding ENABLE requests.
2827 __disable_ibs_on_all_vcpus(vcpu
->kvm
);
2830 atomic_andnot(CPUSTAT_STOPPED
, &vcpu
->arch
.sie_block
->cpuflags
);
2832 * Another VCPU might have used IBS while we were offline.
2833 * Let's play safe and flush the VCPU at startup.
2835 kvm_make_request(KVM_REQ_TLB_FLUSH
, vcpu
);
2836 spin_unlock(&vcpu
->kvm
->arch
.start_stop_lock
);
2840 void kvm_s390_vcpu_stop(struct kvm_vcpu
*vcpu
)
2842 int i
, online_vcpus
, started_vcpus
= 0;
2843 struct kvm_vcpu
*started_vcpu
= NULL
;
2845 if (is_vcpu_stopped(vcpu
))
2848 trace_kvm_s390_vcpu_start_stop(vcpu
->vcpu_id
, 0);
2849 /* Only one cpu at a time may enter/leave the STOPPED state. */
2850 spin_lock(&vcpu
->kvm
->arch
.start_stop_lock
);
2851 online_vcpus
= atomic_read(&vcpu
->kvm
->online_vcpus
);
2853 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2854 kvm_s390_clear_stop_irq(vcpu
);
2856 atomic_or(CPUSTAT_STOPPED
, &vcpu
->arch
.sie_block
->cpuflags
);
2857 __disable_ibs_on_vcpu(vcpu
);
2859 for (i
= 0; i
< online_vcpus
; i
++) {
2860 if (!is_vcpu_stopped(vcpu
->kvm
->vcpus
[i
])) {
2862 started_vcpu
= vcpu
->kvm
->vcpus
[i
];
2866 if (started_vcpus
== 1) {
2868 * As we only have one VCPU left, we want to enable the
2869 * IBS facility for that VCPU to speed it up.
2871 __enable_ibs_on_vcpu(started_vcpu
);
2874 spin_unlock(&vcpu
->kvm
->arch
.start_stop_lock
);
2878 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu
*vcpu
,
2879 struct kvm_enable_cap
*cap
)
2887 case KVM_CAP_S390_CSS_SUPPORT
:
2888 if (!vcpu
->kvm
->arch
.css_support
) {
2889 vcpu
->kvm
->arch
.css_support
= 1;
2890 VM_EVENT(vcpu
->kvm
, 3, "%s", "ENABLE: CSS support");
2891 trace_kvm_s390_enable_css(vcpu
->kvm
);
2902 static long kvm_s390_guest_mem_op(struct kvm_vcpu
*vcpu
,
2903 struct kvm_s390_mem_op
*mop
)
2905 void __user
*uaddr
= (void __user
*)mop
->buf
;
2906 void *tmpbuf
= NULL
;
2908 const u64 supported_flags
= KVM_S390_MEMOP_F_INJECT_EXCEPTION
2909 | KVM_S390_MEMOP_F_CHECK_ONLY
;
2911 if (mop
->flags
& ~supported_flags
)
2914 if (mop
->size
> MEM_OP_MAX_SIZE
)
2917 if (!(mop
->flags
& KVM_S390_MEMOP_F_CHECK_ONLY
)) {
2918 tmpbuf
= vmalloc(mop
->size
);
2923 srcu_idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
2926 case KVM_S390_MEMOP_LOGICAL_READ
:
2927 if (mop
->flags
& KVM_S390_MEMOP_F_CHECK_ONLY
) {
2928 r
= check_gva_range(vcpu
, mop
->gaddr
, mop
->ar
,
2929 mop
->size
, GACC_FETCH
);
2932 r
= read_guest(vcpu
, mop
->gaddr
, mop
->ar
, tmpbuf
, mop
->size
);
2934 if (copy_to_user(uaddr
, tmpbuf
, mop
->size
))
2938 case KVM_S390_MEMOP_LOGICAL_WRITE
:
2939 if (mop
->flags
& KVM_S390_MEMOP_F_CHECK_ONLY
) {
2940 r
= check_gva_range(vcpu
, mop
->gaddr
, mop
->ar
,
2941 mop
->size
, GACC_STORE
);
2944 if (copy_from_user(tmpbuf
, uaddr
, mop
->size
)) {
2948 r
= write_guest(vcpu
, mop
->gaddr
, mop
->ar
, tmpbuf
, mop
->size
);
2954 srcu_read_unlock(&vcpu
->kvm
->srcu
, srcu_idx
);
2956 if (r
> 0 && (mop
->flags
& KVM_S390_MEMOP_F_INJECT_EXCEPTION
) != 0)
2957 kvm_s390_inject_prog_irq(vcpu
, &vcpu
->arch
.pgm
);
2963 long kvm_arch_vcpu_ioctl(struct file
*filp
,
2964 unsigned int ioctl
, unsigned long arg
)
2966 struct kvm_vcpu
*vcpu
= filp
->private_data
;
2967 void __user
*argp
= (void __user
*)arg
;
2972 case KVM_S390_IRQ
: {
2973 struct kvm_s390_irq s390irq
;
2976 if (copy_from_user(&s390irq
, argp
, sizeof(s390irq
)))
2978 r
= kvm_s390_inject_vcpu(vcpu
, &s390irq
);
2981 case KVM_S390_INTERRUPT
: {
2982 struct kvm_s390_interrupt s390int
;
2983 struct kvm_s390_irq s390irq
;
2986 if (copy_from_user(&s390int
, argp
, sizeof(s390int
)))
2988 if (s390int_to_s390irq(&s390int
, &s390irq
))
2990 r
= kvm_s390_inject_vcpu(vcpu
, &s390irq
);
2993 case KVM_S390_STORE_STATUS
:
2994 idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
2995 r
= kvm_s390_vcpu_store_status(vcpu
, arg
);
2996 srcu_read_unlock(&vcpu
->kvm
->srcu
, idx
);
2998 case KVM_S390_SET_INITIAL_PSW
: {
3002 if (copy_from_user(&psw
, argp
, sizeof(psw
)))
3004 r
= kvm_arch_vcpu_ioctl_set_initial_psw(vcpu
, psw
);
3007 case KVM_S390_INITIAL_RESET
:
3008 r
= kvm_arch_vcpu_ioctl_initial_reset(vcpu
);
3010 case KVM_SET_ONE_REG
:
3011 case KVM_GET_ONE_REG
: {
3012 struct kvm_one_reg reg
;
3014 if (copy_from_user(®
, argp
, sizeof(reg
)))
3016 if (ioctl
== KVM_SET_ONE_REG
)
3017 r
= kvm_arch_vcpu_ioctl_set_one_reg(vcpu
, ®
);
3019 r
= kvm_arch_vcpu_ioctl_get_one_reg(vcpu
, ®
);
3022 #ifdef CONFIG_KVM_S390_UCONTROL
3023 case KVM_S390_UCAS_MAP
: {
3024 struct kvm_s390_ucas_mapping ucasmap
;
3026 if (copy_from_user(&ucasmap
, argp
, sizeof(ucasmap
))) {
3031 if (!kvm_is_ucontrol(vcpu
->kvm
)) {
3036 r
= gmap_map_segment(vcpu
->arch
.gmap
, ucasmap
.user_addr
,
3037 ucasmap
.vcpu_addr
, ucasmap
.length
);
3040 case KVM_S390_UCAS_UNMAP
: {
3041 struct kvm_s390_ucas_mapping ucasmap
;
3043 if (copy_from_user(&ucasmap
, argp
, sizeof(ucasmap
))) {
3048 if (!kvm_is_ucontrol(vcpu
->kvm
)) {
3053 r
= gmap_unmap_segment(vcpu
->arch
.gmap
, ucasmap
.vcpu_addr
,
3058 case KVM_S390_VCPU_FAULT
: {
3059 r
= gmap_fault(vcpu
->arch
.gmap
, arg
, 0);
3062 case KVM_ENABLE_CAP
:
3064 struct kvm_enable_cap cap
;
3066 if (copy_from_user(&cap
, argp
, sizeof(cap
)))
3068 r
= kvm_vcpu_ioctl_enable_cap(vcpu
, &cap
);
3071 case KVM_S390_MEM_OP
: {
3072 struct kvm_s390_mem_op mem_op
;
3074 if (copy_from_user(&mem_op
, argp
, sizeof(mem_op
)) == 0)
3075 r
= kvm_s390_guest_mem_op(vcpu
, &mem_op
);
3080 case KVM_S390_SET_IRQ_STATE
: {
3081 struct kvm_s390_irq_state irq_state
;
3084 if (copy_from_user(&irq_state
, argp
, sizeof(irq_state
)))
3086 if (irq_state
.len
> VCPU_IRQS_MAX_BUF
||
3087 irq_state
.len
== 0 ||
3088 irq_state
.len
% sizeof(struct kvm_s390_irq
) > 0) {
3092 r
= kvm_s390_set_irq_state(vcpu
,
3093 (void __user
*) irq_state
.buf
,
3097 case KVM_S390_GET_IRQ_STATE
: {
3098 struct kvm_s390_irq_state irq_state
;
3101 if (copy_from_user(&irq_state
, argp
, sizeof(irq_state
)))
3103 if (irq_state
.len
== 0) {
3107 r
= kvm_s390_get_irq_state(vcpu
,
3108 (__u8 __user
*) irq_state
.buf
,
3118 int kvm_arch_vcpu_fault(struct kvm_vcpu
*vcpu
, struct vm_fault
*vmf
)
3120 #ifdef CONFIG_KVM_S390_UCONTROL
3121 if ((vmf
->pgoff
== KVM_S390_SIE_PAGE_OFFSET
)
3122 && (kvm_is_ucontrol(vcpu
->kvm
))) {
3123 vmf
->page
= virt_to_page(vcpu
->arch
.sie_block
);
3124 get_page(vmf
->page
);
3128 return VM_FAULT_SIGBUS
;
3131 int kvm_arch_create_memslot(struct kvm
*kvm
, struct kvm_memory_slot
*slot
,
3132 unsigned long npages
)
3137 /* Section: memory related */
3138 int kvm_arch_prepare_memory_region(struct kvm
*kvm
,
3139 struct kvm_memory_slot
*memslot
,
3140 const struct kvm_userspace_memory_region
*mem
,
3141 enum kvm_mr_change change
)
3143 /* A few sanity checks. We can have memory slots which have to be
3144 located/ended at a segment boundary (1MB). The memory in userland is
3145 ok to be fragmented into various different vmas. It is okay to mmap()
3146 and munmap() stuff in this slot after doing this call at any time */
3148 if (mem
->userspace_addr
& 0xffffful
)
3151 if (mem
->memory_size
& 0xffffful
)
3154 if (mem
->guest_phys_addr
+ mem
->memory_size
> kvm
->arch
.mem_limit
)
3160 void kvm_arch_commit_memory_region(struct kvm
*kvm
,
3161 const struct kvm_userspace_memory_region
*mem
,
3162 const struct kvm_memory_slot
*old
,
3163 const struct kvm_memory_slot
*new,
3164 enum kvm_mr_change change
)
3168 /* If the basics of the memslot do not change, we do not want
3169 * to update the gmap. Every update causes several unnecessary
3170 * segment translation exceptions. This is usually handled just
3171 * fine by the normal fault handler + gmap, but it will also
3172 * cause faults on the prefix page of running guest CPUs.
3174 if (old
->userspace_addr
== mem
->userspace_addr
&&
3175 old
->base_gfn
* PAGE_SIZE
== mem
->guest_phys_addr
&&
3176 old
->npages
* PAGE_SIZE
== mem
->memory_size
)
3179 rc
= gmap_map_segment(kvm
->arch
.gmap
, mem
->userspace_addr
,
3180 mem
->guest_phys_addr
, mem
->memory_size
);
3182 pr_warn("failed to commit memory region\n");
3186 static inline unsigned long nonhyp_mask(int i
)
3188 unsigned int nonhyp_fai
= (sclp
.hmfai
<< i
* 2) >> 30;
3190 return 0x0000ffffffffffffUL
>> (nonhyp_fai
<< 4);
3193 void kvm_arch_vcpu_block_finish(struct kvm_vcpu
*vcpu
)
3195 vcpu
->valid_wakeup
= false;
3198 static int __init
kvm_s390_init(void)
3202 if (!sclp
.has_sief2
) {
3203 pr_info("SIE not available\n");
3207 for (i
= 0; i
< 16; i
++)
3208 kvm_s390_fac_list_mask
[i
] |=
3209 S390_lowcore
.stfle_fac_list
[i
] & nonhyp_mask(i
);
3211 return kvm_init(NULL
, sizeof(struct kvm_vcpu
), 0, THIS_MODULE
);
3214 static void __exit
kvm_s390_exit(void)
3219 module_init(kvm_s390_init
);
3220 module_exit(kvm_s390_exit
);
3223 * Enable autoloading of the kvm module.
3224 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3225 * since x86 takes a different approach.
3227 #include <linux/miscdevice.h>
3228 MODULE_ALIAS_MISCDEV(KVM_MINOR
);
3229 MODULE_ALIAS("devname:kvm");