]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blob - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-hirsute-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'drm-misc-fixes-2020-03-18-1' of git://anongit.freedesktop.org/drm/drm...
[mirror_ubuntu-hirsute-kernel.git] / arch / s390 / kvm / kvm-s390.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * hosting IBM Z kernel virtual machines (s390x)
4 *
5 * Copyright IBM Corp. 2008, 2018
6 *
7 * Author(s): Carsten Otte <cotte@de.ibm.com>
8 * Christian Borntraeger <borntraeger@de.ibm.com>
9 * Heiko Carstens <heiko.carstens@de.ibm.com>
10 * Christian Ehrhardt <ehrhardt@de.ibm.com>
11 * Jason J. Herne <jjherne@us.ibm.com>
12 */
13
14 #define KMSG_COMPONENT "kvm-s390"
15 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
16
17 #include <linux/compiler.h>
18 #include <linux/err.h>
19 #include <linux/fs.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/mman.h>
25 #include <linux/module.h>
26 #include <linux/moduleparam.h>
27 #include <linux/random.h>
28 #include <linux/slab.h>
29 #include <linux/timer.h>
30 #include <linux/vmalloc.h>
31 #include <linux/bitmap.h>
32 #include <linux/sched/signal.h>
33 #include <linux/string.h>
34
35 #include <asm/asm-offsets.h>
36 #include <asm/lowcore.h>
37 #include <asm/stp.h>
38 #include <asm/pgtable.h>
39 #include <asm/gmap.h>
40 #include <asm/nmi.h>
41 #include <asm/switch_to.h>
42 #include <asm/isc.h>
43 #include <asm/sclp.h>
44 #include <asm/cpacf.h>
45 #include <asm/timex.h>
46 #include <asm/ap.h>
47 #include "kvm-s390.h"
48 #include "gaccess.h"
49
50 #define CREATE_TRACE_POINTS
51 #include "trace.h"
52 #include "trace-s390.h"
53
54 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
55 #define LOCAL_IRQS 32
56 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
57 (KVM_MAX_VCPUS + LOCAL_IRQS))
58
59 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
60 #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
61
62 struct kvm_stats_debugfs_item debugfs_entries[] = {
63 { "userspace_handled", VCPU_STAT(exit_userspace) },
64 { "exit_null", VCPU_STAT(exit_null) },
65 { "exit_validity", VCPU_STAT(exit_validity) },
66 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
67 { "exit_external_request", VCPU_STAT(exit_external_request) },
68 { "exit_io_request", VCPU_STAT(exit_io_request) },
69 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
70 { "exit_instruction", VCPU_STAT(exit_instruction) },
71 { "exit_pei", VCPU_STAT(exit_pei) },
72 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
73 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
74 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
75 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
76 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
77 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
78 { "halt_no_poll_steal", VCPU_STAT(halt_no_poll_steal) },
79 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
80 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
81 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
82 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
83 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
84 { "deliver_ckc", VCPU_STAT(deliver_ckc) },
85 { "deliver_cputm", VCPU_STAT(deliver_cputm) },
86 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
87 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
88 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
89 { "deliver_virtio", VCPU_STAT(deliver_virtio) },
90 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
91 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
92 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
93 { "deliver_program", VCPU_STAT(deliver_program) },
94 { "deliver_io", VCPU_STAT(deliver_io) },
95 { "deliver_machine_check", VCPU_STAT(deliver_machine_check) },
96 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
97 { "inject_ckc", VCPU_STAT(inject_ckc) },
98 { "inject_cputm", VCPU_STAT(inject_cputm) },
99 { "inject_external_call", VCPU_STAT(inject_external_call) },
100 { "inject_float_mchk", VM_STAT(inject_float_mchk) },
101 { "inject_emergency_signal", VCPU_STAT(inject_emergency_signal) },
102 { "inject_io", VM_STAT(inject_io) },
103 { "inject_mchk", VCPU_STAT(inject_mchk) },
104 { "inject_pfault_done", VM_STAT(inject_pfault_done) },
105 { "inject_program", VCPU_STAT(inject_program) },
106 { "inject_restart", VCPU_STAT(inject_restart) },
107 { "inject_service_signal", VM_STAT(inject_service_signal) },
108 { "inject_set_prefix", VCPU_STAT(inject_set_prefix) },
109 { "inject_stop_signal", VCPU_STAT(inject_stop_signal) },
110 { "inject_pfault_init", VCPU_STAT(inject_pfault_init) },
111 { "inject_virtio", VM_STAT(inject_virtio) },
112 { "instruction_epsw", VCPU_STAT(instruction_epsw) },
113 { "instruction_gs", VCPU_STAT(instruction_gs) },
114 { "instruction_io_other", VCPU_STAT(instruction_io_other) },
115 { "instruction_lpsw", VCPU_STAT(instruction_lpsw) },
116 { "instruction_lpswe", VCPU_STAT(instruction_lpswe) },
117 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
118 { "instruction_ptff", VCPU_STAT(instruction_ptff) },
119 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
120 { "instruction_sck", VCPU_STAT(instruction_sck) },
121 { "instruction_sckpf", VCPU_STAT(instruction_sckpf) },
122 { "instruction_spx", VCPU_STAT(instruction_spx) },
123 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
124 { "instruction_stap", VCPU_STAT(instruction_stap) },
125 { "instruction_iske", VCPU_STAT(instruction_iske) },
126 { "instruction_ri", VCPU_STAT(instruction_ri) },
127 { "instruction_rrbe", VCPU_STAT(instruction_rrbe) },
128 { "instruction_sske", VCPU_STAT(instruction_sske) },
129 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
130 { "instruction_essa", VCPU_STAT(instruction_essa) },
131 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
132 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
133 { "instruction_tb", VCPU_STAT(instruction_tb) },
134 { "instruction_tpi", VCPU_STAT(instruction_tpi) },
135 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
136 { "instruction_tsch", VCPU_STAT(instruction_tsch) },
137 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
138 { "instruction_sie", VCPU_STAT(instruction_sie) },
139 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
140 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
141 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
142 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
143 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
144 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
145 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
146 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
147 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
148 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
149 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
150 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
151 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
152 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
153 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
154 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
155 { "instruction_diag_10", VCPU_STAT(diagnose_10) },
156 { "instruction_diag_44", VCPU_STAT(diagnose_44) },
157 { "instruction_diag_9c", VCPU_STAT(diagnose_9c) },
158 { "diag_9c_ignored", VCPU_STAT(diagnose_9c_ignored) },
159 { "instruction_diag_258", VCPU_STAT(diagnose_258) },
160 { "instruction_diag_308", VCPU_STAT(diagnose_308) },
161 { "instruction_diag_500", VCPU_STAT(diagnose_500) },
162 { "instruction_diag_other", VCPU_STAT(diagnose_other) },
163 { NULL }
164 };
165
166 struct kvm_s390_tod_clock_ext {
167 __u8 epoch_idx;
168 __u64 tod;
169 __u8 reserved[7];
170 } __packed;
171
172 /* allow nested virtualization in KVM (if enabled by user space) */
173 static int nested;
174 module_param(nested, int, S_IRUGO);
175 MODULE_PARM_DESC(nested, "Nested virtualization support");
176
177 /* allow 1m huge page guest backing, if !nested */
178 static int hpage;
179 module_param(hpage, int, 0444);
180 MODULE_PARM_DESC(hpage, "1m huge page backing support");
181
182 /* maximum percentage of steal time for polling. >100 is treated like 100 */
183 static u8 halt_poll_max_steal = 10;
184 module_param(halt_poll_max_steal, byte, 0644);
185 MODULE_PARM_DESC(halt_poll_max_steal, "Maximum percentage of steal time to allow polling");
186
187 /*
188 * For now we handle at most 16 double words as this is what the s390 base
189 * kernel handles and stores in the prefix page. If we ever need to go beyond
190 * this, this requires changes to code, but the external uapi can stay.
191 */
192 #define SIZE_INTERNAL 16
193
194 /*
195 * Base feature mask that defines default mask for facilities. Consists of the
196 * defines in FACILITIES_KVM and the non-hypervisor managed bits.
197 */
198 static unsigned long kvm_s390_fac_base[SIZE_INTERNAL] = { FACILITIES_KVM };
199 /*
200 * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL
201 * and defines the facilities that can be enabled via a cpu model.
202 */
203 static unsigned long kvm_s390_fac_ext[SIZE_INTERNAL] = { FACILITIES_KVM_CPUMODEL };
204
205 static unsigned long kvm_s390_fac_size(void)
206 {
207 BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_MASK_SIZE_U64);
208 BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_LIST_SIZE_U64);
209 BUILD_BUG_ON(SIZE_INTERNAL * sizeof(unsigned long) >
210 sizeof(S390_lowcore.stfle_fac_list));
211
212 return SIZE_INTERNAL;
213 }
214
215 /* available cpu features supported by kvm */
216 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
217 /* available subfunctions indicated via query / "test bit" */
218 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
219
220 static struct gmap_notifier gmap_notifier;
221 static struct gmap_notifier vsie_gmap_notifier;
222 debug_info_t *kvm_s390_dbf;
223
224 /* Section: not file related */
225 int kvm_arch_hardware_enable(void)
226 {
227 /* every s390 is virtualization enabled ;-) */
228 return 0;
229 }
230
231 int kvm_arch_check_processor_compat(void)
232 {
233 return 0;
234 }
235
236 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
237 unsigned long end);
238
239 static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
240 {
241 u8 delta_idx = 0;
242
243 /*
244 * The TOD jumps by delta, we have to compensate this by adding
245 * -delta to the epoch.
246 */
247 delta = -delta;
248
249 /* sign-extension - we're adding to signed values below */
250 if ((s64)delta < 0)
251 delta_idx = -1;
252
253 scb->epoch += delta;
254 if (scb->ecd & ECD_MEF) {
255 scb->epdx += delta_idx;
256 if (scb->epoch < delta)
257 scb->epdx += 1;
258 }
259 }
260
261 /*
262 * This callback is executed during stop_machine(). All CPUs are therefore
263 * temporarily stopped. In order not to change guest behavior, we have to
264 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
265 * so a CPU won't be stopped while calculating with the epoch.
266 */
267 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
268 void *v)
269 {
270 struct kvm *kvm;
271 struct kvm_vcpu *vcpu;
272 int i;
273 unsigned long long *delta = v;
274
275 list_for_each_entry(kvm, &vm_list, vm_list) {
276 kvm_for_each_vcpu(i, vcpu, kvm) {
277 kvm_clock_sync_scb(vcpu->arch.sie_block, *delta);
278 if (i == 0) {
279 kvm->arch.epoch = vcpu->arch.sie_block->epoch;
280 kvm->arch.epdx = vcpu->arch.sie_block->epdx;
281 }
282 if (vcpu->arch.cputm_enabled)
283 vcpu->arch.cputm_start += *delta;
284 if (vcpu->arch.vsie_block)
285 kvm_clock_sync_scb(vcpu->arch.vsie_block,
286 *delta);
287 }
288 }
289 return NOTIFY_OK;
290 }
291
292 static struct notifier_block kvm_clock_notifier = {
293 .notifier_call = kvm_clock_sync,
294 };
295
296 int kvm_arch_hardware_setup(void)
297 {
298 gmap_notifier.notifier_call = kvm_gmap_notifier;
299 gmap_register_pte_notifier(&gmap_notifier);
300 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
301 gmap_register_pte_notifier(&vsie_gmap_notifier);
302 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
303 &kvm_clock_notifier);
304 return 0;
305 }
306
307 void kvm_arch_hardware_unsetup(void)
308 {
309 gmap_unregister_pte_notifier(&gmap_notifier);
310 gmap_unregister_pte_notifier(&vsie_gmap_notifier);
311 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
312 &kvm_clock_notifier);
313 }
314
315 static void allow_cpu_feat(unsigned long nr)
316 {
317 set_bit_inv(nr, kvm_s390_available_cpu_feat);
318 }
319
320 static inline int plo_test_bit(unsigned char nr)
321 {
322 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
323 int cc;
324
325 asm volatile(
326 /* Parameter registers are ignored for "test bit" */
327 " plo 0,0,0,0(0)\n"
328 " ipm %0\n"
329 " srl %0,28\n"
330 : "=d" (cc)
331 : "d" (r0)
332 : "cc");
333 return cc == 0;
334 }
335
336 static __always_inline void __insn32_query(unsigned int opcode, u8 *query)
337 {
338 register unsigned long r0 asm("0") = 0; /* query function */
339 register unsigned long r1 asm("1") = (unsigned long) query;
340
341 asm volatile(
342 /* Parameter regs are ignored */
343 " .insn rrf,%[opc] << 16,2,4,6,0\n"
344 :
345 : "d" (r0), "a" (r1), [opc] "i" (opcode)
346 : "cc", "memory");
347 }
348
349 #define INSN_SORTL 0xb938
350 #define INSN_DFLTCC 0xb939
351
352 static void kvm_s390_cpu_feat_init(void)
353 {
354 int i;
355
356 for (i = 0; i < 256; ++i) {
357 if (plo_test_bit(i))
358 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
359 }
360
361 if (test_facility(28)) /* TOD-clock steering */
362 ptff(kvm_s390_available_subfunc.ptff,
363 sizeof(kvm_s390_available_subfunc.ptff),
364 PTFF_QAF);
365
366 if (test_facility(17)) { /* MSA */
367 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
368 kvm_s390_available_subfunc.kmac);
369 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
370 kvm_s390_available_subfunc.kmc);
371 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
372 kvm_s390_available_subfunc.km);
373 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
374 kvm_s390_available_subfunc.kimd);
375 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
376 kvm_s390_available_subfunc.klmd);
377 }
378 if (test_facility(76)) /* MSA3 */
379 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
380 kvm_s390_available_subfunc.pckmo);
381 if (test_facility(77)) { /* MSA4 */
382 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
383 kvm_s390_available_subfunc.kmctr);
384 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
385 kvm_s390_available_subfunc.kmf);
386 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
387 kvm_s390_available_subfunc.kmo);
388 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
389 kvm_s390_available_subfunc.pcc);
390 }
391 if (test_facility(57)) /* MSA5 */
392 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
393 kvm_s390_available_subfunc.ppno);
394
395 if (test_facility(146)) /* MSA8 */
396 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
397 kvm_s390_available_subfunc.kma);
398
399 if (test_facility(155)) /* MSA9 */
400 __cpacf_query(CPACF_KDSA, (cpacf_mask_t *)
401 kvm_s390_available_subfunc.kdsa);
402
403 if (test_facility(150)) /* SORTL */
404 __insn32_query(INSN_SORTL, kvm_s390_available_subfunc.sortl);
405
406 if (test_facility(151)) /* DFLTCC */
407 __insn32_query(INSN_DFLTCC, kvm_s390_available_subfunc.dfltcc);
408
409 if (MACHINE_HAS_ESOP)
410 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
411 /*
412 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
413 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
414 */
415 if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
416 !test_facility(3) || !nested)
417 return;
418 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
419 if (sclp.has_64bscao)
420 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
421 if (sclp.has_siif)
422 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
423 if (sclp.has_gpere)
424 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
425 if (sclp.has_gsls)
426 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
427 if (sclp.has_ib)
428 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
429 if (sclp.has_cei)
430 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
431 if (sclp.has_ibs)
432 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
433 if (sclp.has_kss)
434 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
435 /*
436 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
437 * all skey handling functions read/set the skey from the PGSTE
438 * instead of the real storage key.
439 *
440 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
441 * pages being detected as preserved although they are resident.
442 *
443 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
444 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
445 *
446 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
447 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
448 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
449 *
450 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
451 * cannot easily shadow the SCA because of the ipte lock.
452 */
453 }
454
455 int kvm_arch_init(void *opaque)
456 {
457 int rc = -ENOMEM;
458
459 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
460 if (!kvm_s390_dbf)
461 return -ENOMEM;
462
463 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view))
464 goto out;
465
466 kvm_s390_cpu_feat_init();
467
468 /* Register floating interrupt controller interface. */
469 rc = kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
470 if (rc) {
471 pr_err("A FLIC registration call failed with rc=%d\n", rc);
472 goto out;
473 }
474
475 rc = kvm_s390_gib_init(GAL_ISC);
476 if (rc)
477 goto out;
478
479 return 0;
480
481 out:
482 kvm_arch_exit();
483 return rc;
484 }
485
486 void kvm_arch_exit(void)
487 {
488 kvm_s390_gib_destroy();
489 debug_unregister(kvm_s390_dbf);
490 }
491
492 /* Section: device related */
493 long kvm_arch_dev_ioctl(struct file *filp,
494 unsigned int ioctl, unsigned long arg)
495 {
496 if (ioctl == KVM_S390_ENABLE_SIE)
497 return s390_enable_sie();
498 return -EINVAL;
499 }
500
501 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
502 {
503 int r;
504
505 switch (ext) {
506 case KVM_CAP_S390_PSW:
507 case KVM_CAP_S390_GMAP:
508 case KVM_CAP_SYNC_MMU:
509 #ifdef CONFIG_KVM_S390_UCONTROL
510 case KVM_CAP_S390_UCONTROL:
511 #endif
512 case KVM_CAP_ASYNC_PF:
513 case KVM_CAP_SYNC_REGS:
514 case KVM_CAP_ONE_REG:
515 case KVM_CAP_ENABLE_CAP:
516 case KVM_CAP_S390_CSS_SUPPORT:
517 case KVM_CAP_IOEVENTFD:
518 case KVM_CAP_DEVICE_CTRL:
519 case KVM_CAP_S390_IRQCHIP:
520 case KVM_CAP_VM_ATTRIBUTES:
521 case KVM_CAP_MP_STATE:
522 case KVM_CAP_IMMEDIATE_EXIT:
523 case KVM_CAP_S390_INJECT_IRQ:
524 case KVM_CAP_S390_USER_SIGP:
525 case KVM_CAP_S390_USER_STSI:
526 case KVM_CAP_S390_SKEYS:
527 case KVM_CAP_S390_IRQ_STATE:
528 case KVM_CAP_S390_USER_INSTR0:
529 case KVM_CAP_S390_CMMA_MIGRATION:
530 case KVM_CAP_S390_AIS:
531 case KVM_CAP_S390_AIS_MIGRATION:
532 case KVM_CAP_S390_VCPU_RESETS:
533 r = 1;
534 break;
535 case KVM_CAP_S390_HPAGE_1M:
536 r = 0;
537 if (hpage && !kvm_is_ucontrol(kvm))
538 r = 1;
539 break;
540 case KVM_CAP_S390_MEM_OP:
541 r = MEM_OP_MAX_SIZE;
542 break;
543 case KVM_CAP_NR_VCPUS:
544 case KVM_CAP_MAX_VCPUS:
545 case KVM_CAP_MAX_VCPU_ID:
546 r = KVM_S390_BSCA_CPU_SLOTS;
547 if (!kvm_s390_use_sca_entries())
548 r = KVM_MAX_VCPUS;
549 else if (sclp.has_esca && sclp.has_64bscao)
550 r = KVM_S390_ESCA_CPU_SLOTS;
551 break;
552 case KVM_CAP_S390_COW:
553 r = MACHINE_HAS_ESOP;
554 break;
555 case KVM_CAP_S390_VECTOR_REGISTERS:
556 r = MACHINE_HAS_VX;
557 break;
558 case KVM_CAP_S390_RI:
559 r = test_facility(64);
560 break;
561 case KVM_CAP_S390_GS:
562 r = test_facility(133);
563 break;
564 case KVM_CAP_S390_BPB:
565 r = test_facility(82);
566 break;
567 default:
568 r = 0;
569 }
570 return r;
571 }
572
573 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
574 struct kvm_memory_slot *memslot)
575 {
576 int i;
577 gfn_t cur_gfn, last_gfn;
578 unsigned long gaddr, vmaddr;
579 struct gmap *gmap = kvm->arch.gmap;
580 DECLARE_BITMAP(bitmap, _PAGE_ENTRIES);
581
582 /* Loop over all guest segments */
583 cur_gfn = memslot->base_gfn;
584 last_gfn = memslot->base_gfn + memslot->npages;
585 for (; cur_gfn <= last_gfn; cur_gfn += _PAGE_ENTRIES) {
586 gaddr = gfn_to_gpa(cur_gfn);
587 vmaddr = gfn_to_hva_memslot(memslot, cur_gfn);
588 if (kvm_is_error_hva(vmaddr))
589 continue;
590
591 bitmap_zero(bitmap, _PAGE_ENTRIES);
592 gmap_sync_dirty_log_pmd(gmap, bitmap, gaddr, vmaddr);
593 for (i = 0; i < _PAGE_ENTRIES; i++) {
594 if (test_bit(i, bitmap))
595 mark_page_dirty(kvm, cur_gfn + i);
596 }
597
598 if (fatal_signal_pending(current))
599 return;
600 cond_resched();
601 }
602 }
603
604 /* Section: vm related */
605 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
606
607 /*
608 * Get (and clear) the dirty memory log for a memory slot.
609 */
610 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
611 struct kvm_dirty_log *log)
612 {
613 int r;
614 unsigned long n;
615 struct kvm_memslots *slots;
616 struct kvm_memory_slot *memslot;
617 int is_dirty = 0;
618
619 if (kvm_is_ucontrol(kvm))
620 return -EINVAL;
621
622 mutex_lock(&kvm->slots_lock);
623
624 r = -EINVAL;
625 if (log->slot >= KVM_USER_MEM_SLOTS)
626 goto out;
627
628 slots = kvm_memslots(kvm);
629 memslot = id_to_memslot(slots, log->slot);
630 r = -ENOENT;
631 if (!memslot->dirty_bitmap)
632 goto out;
633
634 kvm_s390_sync_dirty_log(kvm, memslot);
635 r = kvm_get_dirty_log(kvm, log, &is_dirty);
636 if (r)
637 goto out;
638
639 /* Clear the dirty log */
640 if (is_dirty) {
641 n = kvm_dirty_bitmap_bytes(memslot);
642 memset(memslot->dirty_bitmap, 0, n);
643 }
644 r = 0;
645 out:
646 mutex_unlock(&kvm->slots_lock);
647 return r;
648 }
649
650 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
651 {
652 unsigned int i;
653 struct kvm_vcpu *vcpu;
654
655 kvm_for_each_vcpu(i, vcpu, kvm) {
656 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
657 }
658 }
659
660 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
661 {
662 int r;
663
664 if (cap->flags)
665 return -EINVAL;
666
667 switch (cap->cap) {
668 case KVM_CAP_S390_IRQCHIP:
669 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
670 kvm->arch.use_irqchip = 1;
671 r = 0;
672 break;
673 case KVM_CAP_S390_USER_SIGP:
674 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
675 kvm->arch.user_sigp = 1;
676 r = 0;
677 break;
678 case KVM_CAP_S390_VECTOR_REGISTERS:
679 mutex_lock(&kvm->lock);
680 if (kvm->created_vcpus) {
681 r = -EBUSY;
682 } else if (MACHINE_HAS_VX) {
683 set_kvm_facility(kvm->arch.model.fac_mask, 129);
684 set_kvm_facility(kvm->arch.model.fac_list, 129);
685 if (test_facility(134)) {
686 set_kvm_facility(kvm->arch.model.fac_mask, 134);
687 set_kvm_facility(kvm->arch.model.fac_list, 134);
688 }
689 if (test_facility(135)) {
690 set_kvm_facility(kvm->arch.model.fac_mask, 135);
691 set_kvm_facility(kvm->arch.model.fac_list, 135);
692 }
693 if (test_facility(148)) {
694 set_kvm_facility(kvm->arch.model.fac_mask, 148);
695 set_kvm_facility(kvm->arch.model.fac_list, 148);
696 }
697 if (test_facility(152)) {
698 set_kvm_facility(kvm->arch.model.fac_mask, 152);
699 set_kvm_facility(kvm->arch.model.fac_list, 152);
700 }
701 r = 0;
702 } else
703 r = -EINVAL;
704 mutex_unlock(&kvm->lock);
705 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
706 r ? "(not available)" : "(success)");
707 break;
708 case KVM_CAP_S390_RI:
709 r = -EINVAL;
710 mutex_lock(&kvm->lock);
711 if (kvm->created_vcpus) {
712 r = -EBUSY;
713 } else if (test_facility(64)) {
714 set_kvm_facility(kvm->arch.model.fac_mask, 64);
715 set_kvm_facility(kvm->arch.model.fac_list, 64);
716 r = 0;
717 }
718 mutex_unlock(&kvm->lock);
719 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
720 r ? "(not available)" : "(success)");
721 break;
722 case KVM_CAP_S390_AIS:
723 mutex_lock(&kvm->lock);
724 if (kvm->created_vcpus) {
725 r = -EBUSY;
726 } else {
727 set_kvm_facility(kvm->arch.model.fac_mask, 72);
728 set_kvm_facility(kvm->arch.model.fac_list, 72);
729 r = 0;
730 }
731 mutex_unlock(&kvm->lock);
732 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
733 r ? "(not available)" : "(success)");
734 break;
735 case KVM_CAP_S390_GS:
736 r = -EINVAL;
737 mutex_lock(&kvm->lock);
738 if (kvm->created_vcpus) {
739 r = -EBUSY;
740 } else if (test_facility(133)) {
741 set_kvm_facility(kvm->arch.model.fac_mask, 133);
742 set_kvm_facility(kvm->arch.model.fac_list, 133);
743 r = 0;
744 }
745 mutex_unlock(&kvm->lock);
746 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
747 r ? "(not available)" : "(success)");
748 break;
749 case KVM_CAP_S390_HPAGE_1M:
750 mutex_lock(&kvm->lock);
751 if (kvm->created_vcpus)
752 r = -EBUSY;
753 else if (!hpage || kvm->arch.use_cmma || kvm_is_ucontrol(kvm))
754 r = -EINVAL;
755 else {
756 r = 0;
757 down_write(&kvm->mm->mmap_sem);
758 kvm->mm->context.allow_gmap_hpage_1m = 1;
759 up_write(&kvm->mm->mmap_sem);
760 /*
761 * We might have to create fake 4k page
762 * tables. To avoid that the hardware works on
763 * stale PGSTEs, we emulate these instructions.
764 */
765 kvm->arch.use_skf = 0;
766 kvm->arch.use_pfmfi = 0;
767 }
768 mutex_unlock(&kvm->lock);
769 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s",
770 r ? "(not available)" : "(success)");
771 break;
772 case KVM_CAP_S390_USER_STSI:
773 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
774 kvm->arch.user_stsi = 1;
775 r = 0;
776 break;
777 case KVM_CAP_S390_USER_INSTR0:
778 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
779 kvm->arch.user_instr0 = 1;
780 icpt_operexc_on_all_vcpus(kvm);
781 r = 0;
782 break;
783 default:
784 r = -EINVAL;
785 break;
786 }
787 return r;
788 }
789
790 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
791 {
792 int ret;
793
794 switch (attr->attr) {
795 case KVM_S390_VM_MEM_LIMIT_SIZE:
796 ret = 0;
797 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
798 kvm->arch.mem_limit);
799 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
800 ret = -EFAULT;
801 break;
802 default:
803 ret = -ENXIO;
804 break;
805 }
806 return ret;
807 }
808
809 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
810 {
811 int ret;
812 unsigned int idx;
813 switch (attr->attr) {
814 case KVM_S390_VM_MEM_ENABLE_CMMA:
815 ret = -ENXIO;
816 if (!sclp.has_cmma)
817 break;
818
819 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
820 mutex_lock(&kvm->lock);
821 if (kvm->created_vcpus)
822 ret = -EBUSY;
823 else if (kvm->mm->context.allow_gmap_hpage_1m)
824 ret = -EINVAL;
825 else {
826 kvm->arch.use_cmma = 1;
827 /* Not compatible with cmma. */
828 kvm->arch.use_pfmfi = 0;
829 ret = 0;
830 }
831 mutex_unlock(&kvm->lock);
832 break;
833 case KVM_S390_VM_MEM_CLR_CMMA:
834 ret = -ENXIO;
835 if (!sclp.has_cmma)
836 break;
837 ret = -EINVAL;
838 if (!kvm->arch.use_cmma)
839 break;
840
841 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
842 mutex_lock(&kvm->lock);
843 idx = srcu_read_lock(&kvm->srcu);
844 s390_reset_cmma(kvm->arch.gmap->mm);
845 srcu_read_unlock(&kvm->srcu, idx);
846 mutex_unlock(&kvm->lock);
847 ret = 0;
848 break;
849 case KVM_S390_VM_MEM_LIMIT_SIZE: {
850 unsigned long new_limit;
851
852 if (kvm_is_ucontrol(kvm))
853 return -EINVAL;
854
855 if (get_user(new_limit, (u64 __user *)attr->addr))
856 return -EFAULT;
857
858 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
859 new_limit > kvm->arch.mem_limit)
860 return -E2BIG;
861
862 if (!new_limit)
863 return -EINVAL;
864
865 /* gmap_create takes last usable address */
866 if (new_limit != KVM_S390_NO_MEM_LIMIT)
867 new_limit -= 1;
868
869 ret = -EBUSY;
870 mutex_lock(&kvm->lock);
871 if (!kvm->created_vcpus) {
872 /* gmap_create will round the limit up */
873 struct gmap *new = gmap_create(current->mm, new_limit);
874
875 if (!new) {
876 ret = -ENOMEM;
877 } else {
878 gmap_remove(kvm->arch.gmap);
879 new->private = kvm;
880 kvm->arch.gmap = new;
881 ret = 0;
882 }
883 }
884 mutex_unlock(&kvm->lock);
885 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
886 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
887 (void *) kvm->arch.gmap->asce);
888 break;
889 }
890 default:
891 ret = -ENXIO;
892 break;
893 }
894 return ret;
895 }
896
897 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
898
899 void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
900 {
901 struct kvm_vcpu *vcpu;
902 int i;
903
904 kvm_s390_vcpu_block_all(kvm);
905
906 kvm_for_each_vcpu(i, vcpu, kvm) {
907 kvm_s390_vcpu_crypto_setup(vcpu);
908 /* recreate the shadow crycb by leaving the VSIE handler */
909 kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
910 }
911
912 kvm_s390_vcpu_unblock_all(kvm);
913 }
914
915 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
916 {
917 mutex_lock(&kvm->lock);
918 switch (attr->attr) {
919 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
920 if (!test_kvm_facility(kvm, 76)) {
921 mutex_unlock(&kvm->lock);
922 return -EINVAL;
923 }
924 get_random_bytes(
925 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
926 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
927 kvm->arch.crypto.aes_kw = 1;
928 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
929 break;
930 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
931 if (!test_kvm_facility(kvm, 76)) {
932 mutex_unlock(&kvm->lock);
933 return -EINVAL;
934 }
935 get_random_bytes(
936 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
937 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
938 kvm->arch.crypto.dea_kw = 1;
939 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
940 break;
941 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
942 if (!test_kvm_facility(kvm, 76)) {
943 mutex_unlock(&kvm->lock);
944 return -EINVAL;
945 }
946 kvm->arch.crypto.aes_kw = 0;
947 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
948 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
949 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
950 break;
951 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
952 if (!test_kvm_facility(kvm, 76)) {
953 mutex_unlock(&kvm->lock);
954 return -EINVAL;
955 }
956 kvm->arch.crypto.dea_kw = 0;
957 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
958 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
959 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
960 break;
961 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
962 if (!ap_instructions_available()) {
963 mutex_unlock(&kvm->lock);
964 return -EOPNOTSUPP;
965 }
966 kvm->arch.crypto.apie = 1;
967 break;
968 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
969 if (!ap_instructions_available()) {
970 mutex_unlock(&kvm->lock);
971 return -EOPNOTSUPP;
972 }
973 kvm->arch.crypto.apie = 0;
974 break;
975 default:
976 mutex_unlock(&kvm->lock);
977 return -ENXIO;
978 }
979
980 kvm_s390_vcpu_crypto_reset_all(kvm);
981 mutex_unlock(&kvm->lock);
982 return 0;
983 }
984
985 static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
986 {
987 int cx;
988 struct kvm_vcpu *vcpu;
989
990 kvm_for_each_vcpu(cx, vcpu, kvm)
991 kvm_s390_sync_request(req, vcpu);
992 }
993
994 /*
995 * Must be called with kvm->srcu held to avoid races on memslots, and with
996 * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
997 */
998 static int kvm_s390_vm_start_migration(struct kvm *kvm)
999 {
1000 struct kvm_memory_slot *ms;
1001 struct kvm_memslots *slots;
1002 unsigned long ram_pages = 0;
1003 int slotnr;
1004
1005 /* migration mode already enabled */
1006 if (kvm->arch.migration_mode)
1007 return 0;
1008 slots = kvm_memslots(kvm);
1009 if (!slots || !slots->used_slots)
1010 return -EINVAL;
1011
1012 if (!kvm->arch.use_cmma) {
1013 kvm->arch.migration_mode = 1;
1014 return 0;
1015 }
1016 /* mark all the pages in active slots as dirty */
1017 for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
1018 ms = slots->memslots + slotnr;
1019 if (!ms->dirty_bitmap)
1020 return -EINVAL;
1021 /*
1022 * The second half of the bitmap is only used on x86,
1023 * and would be wasted otherwise, so we put it to good
1024 * use here to keep track of the state of the storage
1025 * attributes.
1026 */
1027 memset(kvm_second_dirty_bitmap(ms), 0xff, kvm_dirty_bitmap_bytes(ms));
1028 ram_pages += ms->npages;
1029 }
1030 atomic64_set(&kvm->arch.cmma_dirty_pages, ram_pages);
1031 kvm->arch.migration_mode = 1;
1032 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
1033 return 0;
1034 }
1035
1036 /*
1037 * Must be called with kvm->slots_lock to avoid races with ourselves and
1038 * kvm_s390_vm_start_migration.
1039 */
1040 static int kvm_s390_vm_stop_migration(struct kvm *kvm)
1041 {
1042 /* migration mode already disabled */
1043 if (!kvm->arch.migration_mode)
1044 return 0;
1045 kvm->arch.migration_mode = 0;
1046 if (kvm->arch.use_cmma)
1047 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
1048 return 0;
1049 }
1050
1051 static int kvm_s390_vm_set_migration(struct kvm *kvm,
1052 struct kvm_device_attr *attr)
1053 {
1054 int res = -ENXIO;
1055
1056 mutex_lock(&kvm->slots_lock);
1057 switch (attr->attr) {
1058 case KVM_S390_VM_MIGRATION_START:
1059 res = kvm_s390_vm_start_migration(kvm);
1060 break;
1061 case KVM_S390_VM_MIGRATION_STOP:
1062 res = kvm_s390_vm_stop_migration(kvm);
1063 break;
1064 default:
1065 break;
1066 }
1067 mutex_unlock(&kvm->slots_lock);
1068
1069 return res;
1070 }
1071
1072 static int kvm_s390_vm_get_migration(struct kvm *kvm,
1073 struct kvm_device_attr *attr)
1074 {
1075 u64 mig = kvm->arch.migration_mode;
1076
1077 if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
1078 return -ENXIO;
1079
1080 if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
1081 return -EFAULT;
1082 return 0;
1083 }
1084
1085 static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1086 {
1087 struct kvm_s390_vm_tod_clock gtod;
1088
1089 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
1090 return -EFAULT;
1091
1092 if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
1093 return -EINVAL;
1094 kvm_s390_set_tod_clock(kvm, &gtod);
1095
1096 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
1097 gtod.epoch_idx, gtod.tod);
1098
1099 return 0;
1100 }
1101
1102 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1103 {
1104 u8 gtod_high;
1105
1106 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
1107 sizeof(gtod_high)))
1108 return -EFAULT;
1109
1110 if (gtod_high != 0)
1111 return -EINVAL;
1112 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
1113
1114 return 0;
1115 }
1116
1117 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1118 {
1119 struct kvm_s390_vm_tod_clock gtod = { 0 };
1120
1121 if (copy_from_user(&gtod.tod, (void __user *)attr->addr,
1122 sizeof(gtod.tod)))
1123 return -EFAULT;
1124
1125 kvm_s390_set_tod_clock(kvm, &gtod);
1126 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
1127 return 0;
1128 }
1129
1130 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1131 {
1132 int ret;
1133
1134 if (attr->flags)
1135 return -EINVAL;
1136
1137 switch (attr->attr) {
1138 case KVM_S390_VM_TOD_EXT:
1139 ret = kvm_s390_set_tod_ext(kvm, attr);
1140 break;
1141 case KVM_S390_VM_TOD_HIGH:
1142 ret = kvm_s390_set_tod_high(kvm, attr);
1143 break;
1144 case KVM_S390_VM_TOD_LOW:
1145 ret = kvm_s390_set_tod_low(kvm, attr);
1146 break;
1147 default:
1148 ret = -ENXIO;
1149 break;
1150 }
1151 return ret;
1152 }
1153
1154 static void kvm_s390_get_tod_clock(struct kvm *kvm,
1155 struct kvm_s390_vm_tod_clock *gtod)
1156 {
1157 struct kvm_s390_tod_clock_ext htod;
1158
1159 preempt_disable();
1160
1161 get_tod_clock_ext((char *)&htod);
1162
1163 gtod->tod = htod.tod + kvm->arch.epoch;
1164 gtod->epoch_idx = 0;
1165 if (test_kvm_facility(kvm, 139)) {
1166 gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
1167 if (gtod->tod < htod.tod)
1168 gtod->epoch_idx += 1;
1169 }
1170
1171 preempt_enable();
1172 }
1173
1174 static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1175 {
1176 struct kvm_s390_vm_tod_clock gtod;
1177
1178 memset(&gtod, 0, sizeof(gtod));
1179 kvm_s390_get_tod_clock(kvm, &gtod);
1180 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1181 return -EFAULT;
1182
1183 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
1184 gtod.epoch_idx, gtod.tod);
1185 return 0;
1186 }
1187
1188 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1189 {
1190 u8 gtod_high = 0;
1191
1192 if (copy_to_user((void __user *)attr->addr, &gtod_high,
1193 sizeof(gtod_high)))
1194 return -EFAULT;
1195 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
1196
1197 return 0;
1198 }
1199
1200 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1201 {
1202 u64 gtod;
1203
1204 gtod = kvm_s390_get_tod_clock_fast(kvm);
1205 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1206 return -EFAULT;
1207 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
1208
1209 return 0;
1210 }
1211
1212 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1213 {
1214 int ret;
1215
1216 if (attr->flags)
1217 return -EINVAL;
1218
1219 switch (attr->attr) {
1220 case KVM_S390_VM_TOD_EXT:
1221 ret = kvm_s390_get_tod_ext(kvm, attr);
1222 break;
1223 case KVM_S390_VM_TOD_HIGH:
1224 ret = kvm_s390_get_tod_high(kvm, attr);
1225 break;
1226 case KVM_S390_VM_TOD_LOW:
1227 ret = kvm_s390_get_tod_low(kvm, attr);
1228 break;
1229 default:
1230 ret = -ENXIO;
1231 break;
1232 }
1233 return ret;
1234 }
1235
1236 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1237 {
1238 struct kvm_s390_vm_cpu_processor *proc;
1239 u16 lowest_ibc, unblocked_ibc;
1240 int ret = 0;
1241
1242 mutex_lock(&kvm->lock);
1243 if (kvm->created_vcpus) {
1244 ret = -EBUSY;
1245 goto out;
1246 }
1247 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1248 if (!proc) {
1249 ret = -ENOMEM;
1250 goto out;
1251 }
1252 if (!copy_from_user(proc, (void __user *)attr->addr,
1253 sizeof(*proc))) {
1254 kvm->arch.model.cpuid = proc->cpuid;
1255 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1256 unblocked_ibc = sclp.ibc & 0xfff;
1257 if (lowest_ibc && proc->ibc) {
1258 if (proc->ibc > unblocked_ibc)
1259 kvm->arch.model.ibc = unblocked_ibc;
1260 else if (proc->ibc < lowest_ibc)
1261 kvm->arch.model.ibc = lowest_ibc;
1262 else
1263 kvm->arch.model.ibc = proc->ibc;
1264 }
1265 memcpy(kvm->arch.model.fac_list, proc->fac_list,
1266 S390_ARCH_FAC_LIST_SIZE_BYTE);
1267 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1268 kvm->arch.model.ibc,
1269 kvm->arch.model.cpuid);
1270 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1271 kvm->arch.model.fac_list[0],
1272 kvm->arch.model.fac_list[1],
1273 kvm->arch.model.fac_list[2]);
1274 } else
1275 ret = -EFAULT;
1276 kfree(proc);
1277 out:
1278 mutex_unlock(&kvm->lock);
1279 return ret;
1280 }
1281
1282 static int kvm_s390_set_processor_feat(struct kvm *kvm,
1283 struct kvm_device_attr *attr)
1284 {
1285 struct kvm_s390_vm_cpu_feat data;
1286
1287 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1288 return -EFAULT;
1289 if (!bitmap_subset((unsigned long *) data.feat,
1290 kvm_s390_available_cpu_feat,
1291 KVM_S390_VM_CPU_FEAT_NR_BITS))
1292 return -EINVAL;
1293
1294 mutex_lock(&kvm->lock);
1295 if (kvm->created_vcpus) {
1296 mutex_unlock(&kvm->lock);
1297 return -EBUSY;
1298 }
1299 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
1300 KVM_S390_VM_CPU_FEAT_NR_BITS);
1301 mutex_unlock(&kvm->lock);
1302 VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1303 data.feat[0],
1304 data.feat[1],
1305 data.feat[2]);
1306 return 0;
1307 }
1308
1309 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1310 struct kvm_device_attr *attr)
1311 {
1312 mutex_lock(&kvm->lock);
1313 if (kvm->created_vcpus) {
1314 mutex_unlock(&kvm->lock);
1315 return -EBUSY;
1316 }
1317
1318 if (copy_from_user(&kvm->arch.model.subfuncs, (void __user *)attr->addr,
1319 sizeof(struct kvm_s390_vm_cpu_subfunc))) {
1320 mutex_unlock(&kvm->lock);
1321 return -EFAULT;
1322 }
1323 mutex_unlock(&kvm->lock);
1324
1325 VM_EVENT(kvm, 3, "SET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1326 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1327 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1328 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1329 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1330 VM_EVENT(kvm, 3, "SET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1331 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1332 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1333 VM_EVENT(kvm, 3, "SET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1334 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1335 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1336 VM_EVENT(kvm, 3, "SET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1337 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1338 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1339 VM_EVENT(kvm, 3, "SET: guest KM subfunc 0x%16.16lx.%16.16lx",
1340 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1341 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1342 VM_EVENT(kvm, 3, "SET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1343 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1344 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1345 VM_EVENT(kvm, 3, "SET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1346 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1347 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1348 VM_EVENT(kvm, 3, "SET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1349 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1350 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1351 VM_EVENT(kvm, 3, "SET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1352 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1353 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1354 VM_EVENT(kvm, 3, "SET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1355 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1356 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1357 VM_EVENT(kvm, 3, "SET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1358 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1359 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1360 VM_EVENT(kvm, 3, "SET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1361 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1362 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1363 VM_EVENT(kvm, 3, "SET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1364 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1365 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1366 VM_EVENT(kvm, 3, "SET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1367 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1368 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1369 VM_EVENT(kvm, 3, "SET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1370 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1371 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1372 VM_EVENT(kvm, 3, "SET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1373 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1374 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1375 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1376 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1377 VM_EVENT(kvm, 3, "SET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1378 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1379 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1380 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1381 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1382
1383 return 0;
1384 }
1385
1386 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1387 {
1388 int ret = -ENXIO;
1389
1390 switch (attr->attr) {
1391 case KVM_S390_VM_CPU_PROCESSOR:
1392 ret = kvm_s390_set_processor(kvm, attr);
1393 break;
1394 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1395 ret = kvm_s390_set_processor_feat(kvm, attr);
1396 break;
1397 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1398 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1399 break;
1400 }
1401 return ret;
1402 }
1403
1404 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1405 {
1406 struct kvm_s390_vm_cpu_processor *proc;
1407 int ret = 0;
1408
1409 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1410 if (!proc) {
1411 ret = -ENOMEM;
1412 goto out;
1413 }
1414 proc->cpuid = kvm->arch.model.cpuid;
1415 proc->ibc = kvm->arch.model.ibc;
1416 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1417 S390_ARCH_FAC_LIST_SIZE_BYTE);
1418 VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1419 kvm->arch.model.ibc,
1420 kvm->arch.model.cpuid);
1421 VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1422 kvm->arch.model.fac_list[0],
1423 kvm->arch.model.fac_list[1],
1424 kvm->arch.model.fac_list[2]);
1425 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1426 ret = -EFAULT;
1427 kfree(proc);
1428 out:
1429 return ret;
1430 }
1431
1432 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1433 {
1434 struct kvm_s390_vm_cpu_machine *mach;
1435 int ret = 0;
1436
1437 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
1438 if (!mach) {
1439 ret = -ENOMEM;
1440 goto out;
1441 }
1442 get_cpu_id((struct cpuid *) &mach->cpuid);
1443 mach->ibc = sclp.ibc;
1444 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
1445 S390_ARCH_FAC_LIST_SIZE_BYTE);
1446 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
1447 sizeof(S390_lowcore.stfle_fac_list));
1448 VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx",
1449 kvm->arch.model.ibc,
1450 kvm->arch.model.cpuid);
1451 VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx",
1452 mach->fac_mask[0],
1453 mach->fac_mask[1],
1454 mach->fac_mask[2]);
1455 VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1456 mach->fac_list[0],
1457 mach->fac_list[1],
1458 mach->fac_list[2]);
1459 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1460 ret = -EFAULT;
1461 kfree(mach);
1462 out:
1463 return ret;
1464 }
1465
1466 static int kvm_s390_get_processor_feat(struct kvm *kvm,
1467 struct kvm_device_attr *attr)
1468 {
1469 struct kvm_s390_vm_cpu_feat data;
1470
1471 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1472 KVM_S390_VM_CPU_FEAT_NR_BITS);
1473 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1474 return -EFAULT;
1475 VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1476 data.feat[0],
1477 data.feat[1],
1478 data.feat[2]);
1479 return 0;
1480 }
1481
1482 static int kvm_s390_get_machine_feat(struct kvm *kvm,
1483 struct kvm_device_attr *attr)
1484 {
1485 struct kvm_s390_vm_cpu_feat data;
1486
1487 bitmap_copy((unsigned long *) data.feat,
1488 kvm_s390_available_cpu_feat,
1489 KVM_S390_VM_CPU_FEAT_NR_BITS);
1490 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1491 return -EFAULT;
1492 VM_EVENT(kvm, 3, "GET: host feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1493 data.feat[0],
1494 data.feat[1],
1495 data.feat[2]);
1496 return 0;
1497 }
1498
1499 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1500 struct kvm_device_attr *attr)
1501 {
1502 if (copy_to_user((void __user *)attr->addr, &kvm->arch.model.subfuncs,
1503 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1504 return -EFAULT;
1505
1506 VM_EVENT(kvm, 3, "GET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1507 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1508 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1509 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1510 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1511 VM_EVENT(kvm, 3, "GET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1512 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1513 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1514 VM_EVENT(kvm, 3, "GET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1515 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1516 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1517 VM_EVENT(kvm, 3, "GET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1518 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1519 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1520 VM_EVENT(kvm, 3, "GET: guest KM subfunc 0x%16.16lx.%16.16lx",
1521 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1522 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1523 VM_EVENT(kvm, 3, "GET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1524 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1525 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1526 VM_EVENT(kvm, 3, "GET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1527 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1528 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1529 VM_EVENT(kvm, 3, "GET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1530 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1531 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1532 VM_EVENT(kvm, 3, "GET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1533 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1534 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1535 VM_EVENT(kvm, 3, "GET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1536 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1537 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1538 VM_EVENT(kvm, 3, "GET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1539 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1540 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1541 VM_EVENT(kvm, 3, "GET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1542 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1543 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1544 VM_EVENT(kvm, 3, "GET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1545 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1546 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1547 VM_EVENT(kvm, 3, "GET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1548 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1549 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1550 VM_EVENT(kvm, 3, "GET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1551 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1552 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1553 VM_EVENT(kvm, 3, "GET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1554 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1555 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1556 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1557 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1558 VM_EVENT(kvm, 3, "GET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1559 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1560 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1561 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1562 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1563
1564 return 0;
1565 }
1566
1567 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1568 struct kvm_device_attr *attr)
1569 {
1570 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1571 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1572 return -EFAULT;
1573
1574 VM_EVENT(kvm, 3, "GET: host PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1575 ((unsigned long *) &kvm_s390_available_subfunc.plo)[0],
1576 ((unsigned long *) &kvm_s390_available_subfunc.plo)[1],
1577 ((unsigned long *) &kvm_s390_available_subfunc.plo)[2],
1578 ((unsigned long *) &kvm_s390_available_subfunc.plo)[3]);
1579 VM_EVENT(kvm, 3, "GET: host PTFF subfunc 0x%16.16lx.%16.16lx",
1580 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[0],
1581 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[1]);
1582 VM_EVENT(kvm, 3, "GET: host KMAC subfunc 0x%16.16lx.%16.16lx",
1583 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[0],
1584 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[1]);
1585 VM_EVENT(kvm, 3, "GET: host KMC subfunc 0x%16.16lx.%16.16lx",
1586 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[0],
1587 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[1]);
1588 VM_EVENT(kvm, 3, "GET: host KM subfunc 0x%16.16lx.%16.16lx",
1589 ((unsigned long *) &kvm_s390_available_subfunc.km)[0],
1590 ((unsigned long *) &kvm_s390_available_subfunc.km)[1]);
1591 VM_EVENT(kvm, 3, "GET: host KIMD subfunc 0x%16.16lx.%16.16lx",
1592 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[0],
1593 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[1]);
1594 VM_EVENT(kvm, 3, "GET: host KLMD subfunc 0x%16.16lx.%16.16lx",
1595 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[0],
1596 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[1]);
1597 VM_EVENT(kvm, 3, "GET: host PCKMO subfunc 0x%16.16lx.%16.16lx",
1598 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[0],
1599 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[1]);
1600 VM_EVENT(kvm, 3, "GET: host KMCTR subfunc 0x%16.16lx.%16.16lx",
1601 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[0],
1602 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[1]);
1603 VM_EVENT(kvm, 3, "GET: host KMF subfunc 0x%16.16lx.%16.16lx",
1604 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[0],
1605 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[1]);
1606 VM_EVENT(kvm, 3, "GET: host KMO subfunc 0x%16.16lx.%16.16lx",
1607 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[0],
1608 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[1]);
1609 VM_EVENT(kvm, 3, "GET: host PCC subfunc 0x%16.16lx.%16.16lx",
1610 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[0],
1611 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[1]);
1612 VM_EVENT(kvm, 3, "GET: host PPNO subfunc 0x%16.16lx.%16.16lx",
1613 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[0],
1614 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[1]);
1615 VM_EVENT(kvm, 3, "GET: host KMA subfunc 0x%16.16lx.%16.16lx",
1616 ((unsigned long *) &kvm_s390_available_subfunc.kma)[0],
1617 ((unsigned long *) &kvm_s390_available_subfunc.kma)[1]);
1618 VM_EVENT(kvm, 3, "GET: host KDSA subfunc 0x%16.16lx.%16.16lx",
1619 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[0],
1620 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[1]);
1621 VM_EVENT(kvm, 3, "GET: host SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1622 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[0],
1623 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[1],
1624 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[2],
1625 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[3]);
1626 VM_EVENT(kvm, 3, "GET: host DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1627 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[0],
1628 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[1],
1629 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[2],
1630 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[3]);
1631
1632 return 0;
1633 }
1634
1635 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1636 {
1637 int ret = -ENXIO;
1638
1639 switch (attr->attr) {
1640 case KVM_S390_VM_CPU_PROCESSOR:
1641 ret = kvm_s390_get_processor(kvm, attr);
1642 break;
1643 case KVM_S390_VM_CPU_MACHINE:
1644 ret = kvm_s390_get_machine(kvm, attr);
1645 break;
1646 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1647 ret = kvm_s390_get_processor_feat(kvm, attr);
1648 break;
1649 case KVM_S390_VM_CPU_MACHINE_FEAT:
1650 ret = kvm_s390_get_machine_feat(kvm, attr);
1651 break;
1652 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1653 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1654 break;
1655 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1656 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1657 break;
1658 }
1659 return ret;
1660 }
1661
1662 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1663 {
1664 int ret;
1665
1666 switch (attr->group) {
1667 case KVM_S390_VM_MEM_CTRL:
1668 ret = kvm_s390_set_mem_control(kvm, attr);
1669 break;
1670 case KVM_S390_VM_TOD:
1671 ret = kvm_s390_set_tod(kvm, attr);
1672 break;
1673 case KVM_S390_VM_CPU_MODEL:
1674 ret = kvm_s390_set_cpu_model(kvm, attr);
1675 break;
1676 case KVM_S390_VM_CRYPTO:
1677 ret = kvm_s390_vm_set_crypto(kvm, attr);
1678 break;
1679 case KVM_S390_VM_MIGRATION:
1680 ret = kvm_s390_vm_set_migration(kvm, attr);
1681 break;
1682 default:
1683 ret = -ENXIO;
1684 break;
1685 }
1686
1687 return ret;
1688 }
1689
1690 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1691 {
1692 int ret;
1693
1694 switch (attr->group) {
1695 case KVM_S390_VM_MEM_CTRL:
1696 ret = kvm_s390_get_mem_control(kvm, attr);
1697 break;
1698 case KVM_S390_VM_TOD:
1699 ret = kvm_s390_get_tod(kvm, attr);
1700 break;
1701 case KVM_S390_VM_CPU_MODEL:
1702 ret = kvm_s390_get_cpu_model(kvm, attr);
1703 break;
1704 case KVM_S390_VM_MIGRATION:
1705 ret = kvm_s390_vm_get_migration(kvm, attr);
1706 break;
1707 default:
1708 ret = -ENXIO;
1709 break;
1710 }
1711
1712 return ret;
1713 }
1714
1715 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1716 {
1717 int ret;
1718
1719 switch (attr->group) {
1720 case KVM_S390_VM_MEM_CTRL:
1721 switch (attr->attr) {
1722 case KVM_S390_VM_MEM_ENABLE_CMMA:
1723 case KVM_S390_VM_MEM_CLR_CMMA:
1724 ret = sclp.has_cmma ? 0 : -ENXIO;
1725 break;
1726 case KVM_S390_VM_MEM_LIMIT_SIZE:
1727 ret = 0;
1728 break;
1729 default:
1730 ret = -ENXIO;
1731 break;
1732 }
1733 break;
1734 case KVM_S390_VM_TOD:
1735 switch (attr->attr) {
1736 case KVM_S390_VM_TOD_LOW:
1737 case KVM_S390_VM_TOD_HIGH:
1738 ret = 0;
1739 break;
1740 default:
1741 ret = -ENXIO;
1742 break;
1743 }
1744 break;
1745 case KVM_S390_VM_CPU_MODEL:
1746 switch (attr->attr) {
1747 case KVM_S390_VM_CPU_PROCESSOR:
1748 case KVM_S390_VM_CPU_MACHINE:
1749 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1750 case KVM_S390_VM_CPU_MACHINE_FEAT:
1751 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1752 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1753 ret = 0;
1754 break;
1755 default:
1756 ret = -ENXIO;
1757 break;
1758 }
1759 break;
1760 case KVM_S390_VM_CRYPTO:
1761 switch (attr->attr) {
1762 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1763 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1764 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1765 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1766 ret = 0;
1767 break;
1768 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
1769 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
1770 ret = ap_instructions_available() ? 0 : -ENXIO;
1771 break;
1772 default:
1773 ret = -ENXIO;
1774 break;
1775 }
1776 break;
1777 case KVM_S390_VM_MIGRATION:
1778 ret = 0;
1779 break;
1780 default:
1781 ret = -ENXIO;
1782 break;
1783 }
1784
1785 return ret;
1786 }
1787
1788 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1789 {
1790 uint8_t *keys;
1791 uint64_t hva;
1792 int srcu_idx, i, r = 0;
1793
1794 if (args->flags != 0)
1795 return -EINVAL;
1796
1797 /* Is this guest using storage keys? */
1798 if (!mm_uses_skeys(current->mm))
1799 return KVM_S390_GET_SKEYS_NONE;
1800
1801 /* Enforce sane limit on memory allocation */
1802 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1803 return -EINVAL;
1804
1805 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1806 if (!keys)
1807 return -ENOMEM;
1808
1809 down_read(&current->mm->mmap_sem);
1810 srcu_idx = srcu_read_lock(&kvm->srcu);
1811 for (i = 0; i < args->count; i++) {
1812 hva = gfn_to_hva(kvm, args->start_gfn + i);
1813 if (kvm_is_error_hva(hva)) {
1814 r = -EFAULT;
1815 break;
1816 }
1817
1818 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1819 if (r)
1820 break;
1821 }
1822 srcu_read_unlock(&kvm->srcu, srcu_idx);
1823 up_read(&current->mm->mmap_sem);
1824
1825 if (!r) {
1826 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1827 sizeof(uint8_t) * args->count);
1828 if (r)
1829 r = -EFAULT;
1830 }
1831
1832 kvfree(keys);
1833 return r;
1834 }
1835
1836 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1837 {
1838 uint8_t *keys;
1839 uint64_t hva;
1840 int srcu_idx, i, r = 0;
1841 bool unlocked;
1842
1843 if (args->flags != 0)
1844 return -EINVAL;
1845
1846 /* Enforce sane limit on memory allocation */
1847 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1848 return -EINVAL;
1849
1850 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1851 if (!keys)
1852 return -ENOMEM;
1853
1854 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1855 sizeof(uint8_t) * args->count);
1856 if (r) {
1857 r = -EFAULT;
1858 goto out;
1859 }
1860
1861 /* Enable storage key handling for the guest */
1862 r = s390_enable_skey();
1863 if (r)
1864 goto out;
1865
1866 i = 0;
1867 down_read(&current->mm->mmap_sem);
1868 srcu_idx = srcu_read_lock(&kvm->srcu);
1869 while (i < args->count) {
1870 unlocked = false;
1871 hva = gfn_to_hva(kvm, args->start_gfn + i);
1872 if (kvm_is_error_hva(hva)) {
1873 r = -EFAULT;
1874 break;
1875 }
1876
1877 /* Lowest order bit is reserved */
1878 if (keys[i] & 0x01) {
1879 r = -EINVAL;
1880 break;
1881 }
1882
1883 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1884 if (r) {
1885 r = fixup_user_fault(current, current->mm, hva,
1886 FAULT_FLAG_WRITE, &unlocked);
1887 if (r)
1888 break;
1889 }
1890 if (!r)
1891 i++;
1892 }
1893 srcu_read_unlock(&kvm->srcu, srcu_idx);
1894 up_read(&current->mm->mmap_sem);
1895 out:
1896 kvfree(keys);
1897 return r;
1898 }
1899
1900 /*
1901 * Base address and length must be sent at the start of each block, therefore
1902 * it's cheaper to send some clean data, as long as it's less than the size of
1903 * two longs.
1904 */
1905 #define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
1906 /* for consistency */
1907 #define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1908
1909 /*
1910 * Similar to gfn_to_memslot, but returns the index of a memslot also when the
1911 * address falls in a hole. In that case the index of one of the memslots
1912 * bordering the hole is returned.
1913 */
1914 static int gfn_to_memslot_approx(struct kvm_memslots *slots, gfn_t gfn)
1915 {
1916 int start = 0, end = slots->used_slots;
1917 int slot = atomic_read(&slots->lru_slot);
1918 struct kvm_memory_slot *memslots = slots->memslots;
1919
1920 if (gfn >= memslots[slot].base_gfn &&
1921 gfn < memslots[slot].base_gfn + memslots[slot].npages)
1922 return slot;
1923
1924 while (start < end) {
1925 slot = start + (end - start) / 2;
1926
1927 if (gfn >= memslots[slot].base_gfn)
1928 end = slot;
1929 else
1930 start = slot + 1;
1931 }
1932
1933 if (gfn >= memslots[start].base_gfn &&
1934 gfn < memslots[start].base_gfn + memslots[start].npages) {
1935 atomic_set(&slots->lru_slot, start);
1936 }
1937
1938 return start;
1939 }
1940
1941 static int kvm_s390_peek_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
1942 u8 *res, unsigned long bufsize)
1943 {
1944 unsigned long pgstev, hva, cur_gfn = args->start_gfn;
1945
1946 args->count = 0;
1947 while (args->count < bufsize) {
1948 hva = gfn_to_hva(kvm, cur_gfn);
1949 /*
1950 * We return an error if the first value was invalid, but we
1951 * return successfully if at least one value was copied.
1952 */
1953 if (kvm_is_error_hva(hva))
1954 return args->count ? 0 : -EFAULT;
1955 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
1956 pgstev = 0;
1957 res[args->count++] = (pgstev >> 24) & 0x43;
1958 cur_gfn++;
1959 }
1960
1961 return 0;
1962 }
1963
1964 static unsigned long kvm_s390_next_dirty_cmma(struct kvm_memslots *slots,
1965 unsigned long cur_gfn)
1966 {
1967 int slotidx = gfn_to_memslot_approx(slots, cur_gfn);
1968 struct kvm_memory_slot *ms = slots->memslots + slotidx;
1969 unsigned long ofs = cur_gfn - ms->base_gfn;
1970
1971 if (ms->base_gfn + ms->npages <= cur_gfn) {
1972 slotidx--;
1973 /* If we are above the highest slot, wrap around */
1974 if (slotidx < 0)
1975 slotidx = slots->used_slots - 1;
1976
1977 ms = slots->memslots + slotidx;
1978 ofs = 0;
1979 }
1980 ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, ofs);
1981 while ((slotidx > 0) && (ofs >= ms->npages)) {
1982 slotidx--;
1983 ms = slots->memslots + slotidx;
1984 ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, 0);
1985 }
1986 return ms->base_gfn + ofs;
1987 }
1988
1989 static int kvm_s390_get_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
1990 u8 *res, unsigned long bufsize)
1991 {
1992 unsigned long mem_end, cur_gfn, next_gfn, hva, pgstev;
1993 struct kvm_memslots *slots = kvm_memslots(kvm);
1994 struct kvm_memory_slot *ms;
1995
1996 cur_gfn = kvm_s390_next_dirty_cmma(slots, args->start_gfn);
1997 ms = gfn_to_memslot(kvm, cur_gfn);
1998 args->count = 0;
1999 args->start_gfn = cur_gfn;
2000 if (!ms)
2001 return 0;
2002 next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2003 mem_end = slots->memslots[0].base_gfn + slots->memslots[0].npages;
2004
2005 while (args->count < bufsize) {
2006 hva = gfn_to_hva(kvm, cur_gfn);
2007 if (kvm_is_error_hva(hva))
2008 return 0;
2009 /* Decrement only if we actually flipped the bit to 0 */
2010 if (test_and_clear_bit(cur_gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms)))
2011 atomic64_dec(&kvm->arch.cmma_dirty_pages);
2012 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
2013 pgstev = 0;
2014 /* Save the value */
2015 res[args->count++] = (pgstev >> 24) & 0x43;
2016 /* If the next bit is too far away, stop. */
2017 if (next_gfn > cur_gfn + KVM_S390_MAX_BIT_DISTANCE)
2018 return 0;
2019 /* If we reached the previous "next", find the next one */
2020 if (cur_gfn == next_gfn)
2021 next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2022 /* Reached the end of memory or of the buffer, stop */
2023 if ((next_gfn >= mem_end) ||
2024 (next_gfn - args->start_gfn >= bufsize))
2025 return 0;
2026 cur_gfn++;
2027 /* Reached the end of the current memslot, take the next one. */
2028 if (cur_gfn - ms->base_gfn >= ms->npages) {
2029 ms = gfn_to_memslot(kvm, cur_gfn);
2030 if (!ms)
2031 return 0;
2032 }
2033 }
2034 return 0;
2035 }
2036
2037 /*
2038 * This function searches for the next page with dirty CMMA attributes, and
2039 * saves the attributes in the buffer up to either the end of the buffer or
2040 * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
2041 * no trailing clean bytes are saved.
2042 * In case no dirty bits were found, or if CMMA was not enabled or used, the
2043 * output buffer will indicate 0 as length.
2044 */
2045 static int kvm_s390_get_cmma_bits(struct kvm *kvm,
2046 struct kvm_s390_cmma_log *args)
2047 {
2048 unsigned long bufsize;
2049 int srcu_idx, peek, ret;
2050 u8 *values;
2051
2052 if (!kvm->arch.use_cmma)
2053 return -ENXIO;
2054 /* Invalid/unsupported flags were specified */
2055 if (args->flags & ~KVM_S390_CMMA_PEEK)
2056 return -EINVAL;
2057 /* Migration mode query, and we are not doing a migration */
2058 peek = !!(args->flags & KVM_S390_CMMA_PEEK);
2059 if (!peek && !kvm->arch.migration_mode)
2060 return -EINVAL;
2061 /* CMMA is disabled or was not used, or the buffer has length zero */
2062 bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
2063 if (!bufsize || !kvm->mm->context.uses_cmm) {
2064 memset(args, 0, sizeof(*args));
2065 return 0;
2066 }
2067 /* We are not peeking, and there are no dirty pages */
2068 if (!peek && !atomic64_read(&kvm->arch.cmma_dirty_pages)) {
2069 memset(args, 0, sizeof(*args));
2070 return 0;
2071 }
2072
2073 values = vmalloc(bufsize);
2074 if (!values)
2075 return -ENOMEM;
2076
2077 down_read(&kvm->mm->mmap_sem);
2078 srcu_idx = srcu_read_lock(&kvm->srcu);
2079 if (peek)
2080 ret = kvm_s390_peek_cmma(kvm, args, values, bufsize);
2081 else
2082 ret = kvm_s390_get_cmma(kvm, args, values, bufsize);
2083 srcu_read_unlock(&kvm->srcu, srcu_idx);
2084 up_read(&kvm->mm->mmap_sem);
2085
2086 if (kvm->arch.migration_mode)
2087 args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages);
2088 else
2089 args->remaining = 0;
2090
2091 if (copy_to_user((void __user *)args->values, values, args->count))
2092 ret = -EFAULT;
2093
2094 vfree(values);
2095 return ret;
2096 }
2097
2098 /*
2099 * This function sets the CMMA attributes for the given pages. If the input
2100 * buffer has zero length, no action is taken, otherwise the attributes are
2101 * set and the mm->context.uses_cmm flag is set.
2102 */
2103 static int kvm_s390_set_cmma_bits(struct kvm *kvm,
2104 const struct kvm_s390_cmma_log *args)
2105 {
2106 unsigned long hva, mask, pgstev, i;
2107 uint8_t *bits;
2108 int srcu_idx, r = 0;
2109
2110 mask = args->mask;
2111
2112 if (!kvm->arch.use_cmma)
2113 return -ENXIO;
2114 /* invalid/unsupported flags */
2115 if (args->flags != 0)
2116 return -EINVAL;
2117 /* Enforce sane limit on memory allocation */
2118 if (args->count > KVM_S390_CMMA_SIZE_MAX)
2119 return -EINVAL;
2120 /* Nothing to do */
2121 if (args->count == 0)
2122 return 0;
2123
2124 bits = vmalloc(array_size(sizeof(*bits), args->count));
2125 if (!bits)
2126 return -ENOMEM;
2127
2128 r = copy_from_user(bits, (void __user *)args->values, args->count);
2129 if (r) {
2130 r = -EFAULT;
2131 goto out;
2132 }
2133
2134 down_read(&kvm->mm->mmap_sem);
2135 srcu_idx = srcu_read_lock(&kvm->srcu);
2136 for (i = 0; i < args->count; i++) {
2137 hva = gfn_to_hva(kvm, args->start_gfn + i);
2138 if (kvm_is_error_hva(hva)) {
2139 r = -EFAULT;
2140 break;
2141 }
2142
2143 pgstev = bits[i];
2144 pgstev = pgstev << 24;
2145 mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
2146 set_pgste_bits(kvm->mm, hva, mask, pgstev);
2147 }
2148 srcu_read_unlock(&kvm->srcu, srcu_idx);
2149 up_read(&kvm->mm->mmap_sem);
2150
2151 if (!kvm->mm->context.uses_cmm) {
2152 down_write(&kvm->mm->mmap_sem);
2153 kvm->mm->context.uses_cmm = 1;
2154 up_write(&kvm->mm->mmap_sem);
2155 }
2156 out:
2157 vfree(bits);
2158 return r;
2159 }
2160
2161 long kvm_arch_vm_ioctl(struct file *filp,
2162 unsigned int ioctl, unsigned long arg)
2163 {
2164 struct kvm *kvm = filp->private_data;
2165 void __user *argp = (void __user *)arg;
2166 struct kvm_device_attr attr;
2167 int r;
2168
2169 switch (ioctl) {
2170 case KVM_S390_INTERRUPT: {
2171 struct kvm_s390_interrupt s390int;
2172
2173 r = -EFAULT;
2174 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2175 break;
2176 r = kvm_s390_inject_vm(kvm, &s390int);
2177 break;
2178 }
2179 case KVM_CREATE_IRQCHIP: {
2180 struct kvm_irq_routing_entry routing;
2181
2182 r = -EINVAL;
2183 if (kvm->arch.use_irqchip) {
2184 /* Set up dummy routing. */
2185 memset(&routing, 0, sizeof(routing));
2186 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
2187 }
2188 break;
2189 }
2190 case KVM_SET_DEVICE_ATTR: {
2191 r = -EFAULT;
2192 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2193 break;
2194 r = kvm_s390_vm_set_attr(kvm, &attr);
2195 break;
2196 }
2197 case KVM_GET_DEVICE_ATTR: {
2198 r = -EFAULT;
2199 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2200 break;
2201 r = kvm_s390_vm_get_attr(kvm, &attr);
2202 break;
2203 }
2204 case KVM_HAS_DEVICE_ATTR: {
2205 r = -EFAULT;
2206 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2207 break;
2208 r = kvm_s390_vm_has_attr(kvm, &attr);
2209 break;
2210 }
2211 case KVM_S390_GET_SKEYS: {
2212 struct kvm_s390_skeys args;
2213
2214 r = -EFAULT;
2215 if (copy_from_user(&args, argp,
2216 sizeof(struct kvm_s390_skeys)))
2217 break;
2218 r = kvm_s390_get_skeys(kvm, &args);
2219 break;
2220 }
2221 case KVM_S390_SET_SKEYS: {
2222 struct kvm_s390_skeys args;
2223
2224 r = -EFAULT;
2225 if (copy_from_user(&args, argp,
2226 sizeof(struct kvm_s390_skeys)))
2227 break;
2228 r = kvm_s390_set_skeys(kvm, &args);
2229 break;
2230 }
2231 case KVM_S390_GET_CMMA_BITS: {
2232 struct kvm_s390_cmma_log args;
2233
2234 r = -EFAULT;
2235 if (copy_from_user(&args, argp, sizeof(args)))
2236 break;
2237 mutex_lock(&kvm->slots_lock);
2238 r = kvm_s390_get_cmma_bits(kvm, &args);
2239 mutex_unlock(&kvm->slots_lock);
2240 if (!r) {
2241 r = copy_to_user(argp, &args, sizeof(args));
2242 if (r)
2243 r = -EFAULT;
2244 }
2245 break;
2246 }
2247 case KVM_S390_SET_CMMA_BITS: {
2248 struct kvm_s390_cmma_log args;
2249
2250 r = -EFAULT;
2251 if (copy_from_user(&args, argp, sizeof(args)))
2252 break;
2253 mutex_lock(&kvm->slots_lock);
2254 r = kvm_s390_set_cmma_bits(kvm, &args);
2255 mutex_unlock(&kvm->slots_lock);
2256 break;
2257 }
2258 default:
2259 r = -ENOTTY;
2260 }
2261
2262 return r;
2263 }
2264
2265 static int kvm_s390_apxa_installed(void)
2266 {
2267 struct ap_config_info info;
2268
2269 if (ap_instructions_available()) {
2270 if (ap_qci(&info) == 0)
2271 return info.apxa;
2272 }
2273
2274 return 0;
2275 }
2276
2277 /*
2278 * The format of the crypto control block (CRYCB) is specified in the 3 low
2279 * order bits of the CRYCB designation (CRYCBD) field as follows:
2280 * Format 0: Neither the message security assist extension 3 (MSAX3) nor the
2281 * AP extended addressing (APXA) facility are installed.
2282 * Format 1: The APXA facility is not installed but the MSAX3 facility is.
2283 * Format 2: Both the APXA and MSAX3 facilities are installed
2284 */
2285 static void kvm_s390_set_crycb_format(struct kvm *kvm)
2286 {
2287 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
2288
2289 /* Clear the CRYCB format bits - i.e., set format 0 by default */
2290 kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
2291
2292 /* Check whether MSAX3 is installed */
2293 if (!test_kvm_facility(kvm, 76))
2294 return;
2295
2296 if (kvm_s390_apxa_installed())
2297 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
2298 else
2299 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
2300 }
2301
2302 void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
2303 unsigned long *aqm, unsigned long *adm)
2304 {
2305 struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
2306
2307 mutex_lock(&kvm->lock);
2308 kvm_s390_vcpu_block_all(kvm);
2309
2310 switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
2311 case CRYCB_FORMAT2: /* APCB1 use 256 bits */
2312 memcpy(crycb->apcb1.apm, apm, 32);
2313 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx %016lx %016lx %016lx",
2314 apm[0], apm[1], apm[2], apm[3]);
2315 memcpy(crycb->apcb1.aqm, aqm, 32);
2316 VM_EVENT(kvm, 3, "SET CRYCB: aqm %016lx %016lx %016lx %016lx",
2317 aqm[0], aqm[1], aqm[2], aqm[3]);
2318 memcpy(crycb->apcb1.adm, adm, 32);
2319 VM_EVENT(kvm, 3, "SET CRYCB: adm %016lx %016lx %016lx %016lx",
2320 adm[0], adm[1], adm[2], adm[3]);
2321 break;
2322 case CRYCB_FORMAT1:
2323 case CRYCB_FORMAT0: /* Fall through both use APCB0 */
2324 memcpy(crycb->apcb0.apm, apm, 8);
2325 memcpy(crycb->apcb0.aqm, aqm, 2);
2326 memcpy(crycb->apcb0.adm, adm, 2);
2327 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx aqm %04x adm %04x",
2328 apm[0], *((unsigned short *)aqm),
2329 *((unsigned short *)adm));
2330 break;
2331 default: /* Can not happen */
2332 break;
2333 }
2334
2335 /* recreate the shadow crycb for each vcpu */
2336 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2337 kvm_s390_vcpu_unblock_all(kvm);
2338 mutex_unlock(&kvm->lock);
2339 }
2340 EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks);
2341
2342 void kvm_arch_crypto_clear_masks(struct kvm *kvm)
2343 {
2344 mutex_lock(&kvm->lock);
2345 kvm_s390_vcpu_block_all(kvm);
2346
2347 memset(&kvm->arch.crypto.crycb->apcb0, 0,
2348 sizeof(kvm->arch.crypto.crycb->apcb0));
2349 memset(&kvm->arch.crypto.crycb->apcb1, 0,
2350 sizeof(kvm->arch.crypto.crycb->apcb1));
2351
2352 VM_EVENT(kvm, 3, "%s", "CLR CRYCB:");
2353 /* recreate the shadow crycb for each vcpu */
2354 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2355 kvm_s390_vcpu_unblock_all(kvm);
2356 mutex_unlock(&kvm->lock);
2357 }
2358 EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks);
2359
2360 static u64 kvm_s390_get_initial_cpuid(void)
2361 {
2362 struct cpuid cpuid;
2363
2364 get_cpu_id(&cpuid);
2365 cpuid.version = 0xff;
2366 return *((u64 *) &cpuid);
2367 }
2368
2369 static void kvm_s390_crypto_init(struct kvm *kvm)
2370 {
2371 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
2372 kvm_s390_set_crycb_format(kvm);
2373
2374 if (!test_kvm_facility(kvm, 76))
2375 return;
2376
2377 /* Enable AES/DEA protected key functions by default */
2378 kvm->arch.crypto.aes_kw = 1;
2379 kvm->arch.crypto.dea_kw = 1;
2380 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
2381 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
2382 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
2383 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
2384 }
2385
2386 static void sca_dispose(struct kvm *kvm)
2387 {
2388 if (kvm->arch.use_esca)
2389 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
2390 else
2391 free_page((unsigned long)(kvm->arch.sca));
2392 kvm->arch.sca = NULL;
2393 }
2394
2395 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
2396 {
2397 gfp_t alloc_flags = GFP_KERNEL;
2398 int i, rc;
2399 char debug_name[16];
2400 static unsigned long sca_offset;
2401
2402 rc = -EINVAL;
2403 #ifdef CONFIG_KVM_S390_UCONTROL
2404 if (type & ~KVM_VM_S390_UCONTROL)
2405 goto out_err;
2406 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
2407 goto out_err;
2408 #else
2409 if (type)
2410 goto out_err;
2411 #endif
2412
2413 rc = s390_enable_sie();
2414 if (rc)
2415 goto out_err;
2416
2417 rc = -ENOMEM;
2418
2419 if (!sclp.has_64bscao)
2420 alloc_flags |= GFP_DMA;
2421 rwlock_init(&kvm->arch.sca_lock);
2422 /* start with basic SCA */
2423 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
2424 if (!kvm->arch.sca)
2425 goto out_err;
2426 mutex_lock(&kvm_lock);
2427 sca_offset += 16;
2428 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
2429 sca_offset = 0;
2430 kvm->arch.sca = (struct bsca_block *)
2431 ((char *) kvm->arch.sca + sca_offset);
2432 mutex_unlock(&kvm_lock);
2433
2434 sprintf(debug_name, "kvm-%u", current->pid);
2435
2436 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
2437 if (!kvm->arch.dbf)
2438 goto out_err;
2439
2440 BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
2441 kvm->arch.sie_page2 =
2442 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
2443 if (!kvm->arch.sie_page2)
2444 goto out_err;
2445
2446 kvm->arch.sie_page2->kvm = kvm;
2447 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
2448
2449 for (i = 0; i < kvm_s390_fac_size(); i++) {
2450 kvm->arch.model.fac_mask[i] = S390_lowcore.stfle_fac_list[i] &
2451 (kvm_s390_fac_base[i] |
2452 kvm_s390_fac_ext[i]);
2453 kvm->arch.model.fac_list[i] = S390_lowcore.stfle_fac_list[i] &
2454 kvm_s390_fac_base[i];
2455 }
2456 kvm->arch.model.subfuncs = kvm_s390_available_subfunc;
2457
2458 /* we are always in czam mode - even on pre z14 machines */
2459 set_kvm_facility(kvm->arch.model.fac_mask, 138);
2460 set_kvm_facility(kvm->arch.model.fac_list, 138);
2461 /* we emulate STHYI in kvm */
2462 set_kvm_facility(kvm->arch.model.fac_mask, 74);
2463 set_kvm_facility(kvm->arch.model.fac_list, 74);
2464 if (MACHINE_HAS_TLB_GUEST) {
2465 set_kvm_facility(kvm->arch.model.fac_mask, 147);
2466 set_kvm_facility(kvm->arch.model.fac_list, 147);
2467 }
2468
2469 if (css_general_characteristics.aiv && test_facility(65))
2470 set_kvm_facility(kvm->arch.model.fac_mask, 65);
2471
2472 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
2473 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
2474
2475 kvm_s390_crypto_init(kvm);
2476
2477 mutex_init(&kvm->arch.float_int.ais_lock);
2478 spin_lock_init(&kvm->arch.float_int.lock);
2479 for (i = 0; i < FIRQ_LIST_COUNT; i++)
2480 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
2481 init_waitqueue_head(&kvm->arch.ipte_wq);
2482 mutex_init(&kvm->arch.ipte_mutex);
2483
2484 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
2485 VM_EVENT(kvm, 3, "vm created with type %lu", type);
2486
2487 if (type & KVM_VM_S390_UCONTROL) {
2488 kvm->arch.gmap = NULL;
2489 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
2490 } else {
2491 if (sclp.hamax == U64_MAX)
2492 kvm->arch.mem_limit = TASK_SIZE_MAX;
2493 else
2494 kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
2495 sclp.hamax + 1);
2496 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
2497 if (!kvm->arch.gmap)
2498 goto out_err;
2499 kvm->arch.gmap->private = kvm;
2500 kvm->arch.gmap->pfault_enabled = 0;
2501 }
2502
2503 kvm->arch.use_pfmfi = sclp.has_pfmfi;
2504 kvm->arch.use_skf = sclp.has_skey;
2505 spin_lock_init(&kvm->arch.start_stop_lock);
2506 kvm_s390_vsie_init(kvm);
2507 kvm_s390_gisa_init(kvm);
2508 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
2509
2510 return 0;
2511 out_err:
2512 free_page((unsigned long)kvm->arch.sie_page2);
2513 debug_unregister(kvm->arch.dbf);
2514 sca_dispose(kvm);
2515 KVM_EVENT(3, "creation of vm failed: %d", rc);
2516 return rc;
2517 }
2518
2519 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
2520 {
2521 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
2522 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
2523 kvm_s390_clear_local_irqs(vcpu);
2524 kvm_clear_async_pf_completion_queue(vcpu);
2525 if (!kvm_is_ucontrol(vcpu->kvm))
2526 sca_del_vcpu(vcpu);
2527
2528 if (kvm_is_ucontrol(vcpu->kvm))
2529 gmap_remove(vcpu->arch.gmap);
2530
2531 if (vcpu->kvm->arch.use_cmma)
2532 kvm_s390_vcpu_unsetup_cmma(vcpu);
2533 free_page((unsigned long)(vcpu->arch.sie_block));
2534 }
2535
2536 static void kvm_free_vcpus(struct kvm *kvm)
2537 {
2538 unsigned int i;
2539 struct kvm_vcpu *vcpu;
2540
2541 kvm_for_each_vcpu(i, vcpu, kvm)
2542 kvm_vcpu_destroy(vcpu);
2543
2544 mutex_lock(&kvm->lock);
2545 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
2546 kvm->vcpus[i] = NULL;
2547
2548 atomic_set(&kvm->online_vcpus, 0);
2549 mutex_unlock(&kvm->lock);
2550 }
2551
2552 void kvm_arch_destroy_vm(struct kvm *kvm)
2553 {
2554 kvm_free_vcpus(kvm);
2555 sca_dispose(kvm);
2556 debug_unregister(kvm->arch.dbf);
2557 kvm_s390_gisa_destroy(kvm);
2558 free_page((unsigned long)kvm->arch.sie_page2);
2559 if (!kvm_is_ucontrol(kvm))
2560 gmap_remove(kvm->arch.gmap);
2561 kvm_s390_destroy_adapters(kvm);
2562 kvm_s390_clear_float_irqs(kvm);
2563 kvm_s390_vsie_destroy(kvm);
2564 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
2565 }
2566
2567 /* Section: vcpu related */
2568 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
2569 {
2570 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
2571 if (!vcpu->arch.gmap)
2572 return -ENOMEM;
2573 vcpu->arch.gmap->private = vcpu->kvm;
2574
2575 return 0;
2576 }
2577
2578 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
2579 {
2580 if (!kvm_s390_use_sca_entries())
2581 return;
2582 read_lock(&vcpu->kvm->arch.sca_lock);
2583 if (vcpu->kvm->arch.use_esca) {
2584 struct esca_block *sca = vcpu->kvm->arch.sca;
2585
2586 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2587 sca->cpu[vcpu->vcpu_id].sda = 0;
2588 } else {
2589 struct bsca_block *sca = vcpu->kvm->arch.sca;
2590
2591 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2592 sca->cpu[vcpu->vcpu_id].sda = 0;
2593 }
2594 read_unlock(&vcpu->kvm->arch.sca_lock);
2595 }
2596
2597 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
2598 {
2599 if (!kvm_s390_use_sca_entries()) {
2600 struct bsca_block *sca = vcpu->kvm->arch.sca;
2601
2602 /* we still need the basic sca for the ipte control */
2603 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2604 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2605 return;
2606 }
2607 read_lock(&vcpu->kvm->arch.sca_lock);
2608 if (vcpu->kvm->arch.use_esca) {
2609 struct esca_block *sca = vcpu->kvm->arch.sca;
2610
2611 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2612 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2613 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
2614 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2615 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2616 } else {
2617 struct bsca_block *sca = vcpu->kvm->arch.sca;
2618
2619 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2620 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2621 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2622 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2623 }
2624 read_unlock(&vcpu->kvm->arch.sca_lock);
2625 }
2626
2627 /* Basic SCA to Extended SCA data copy routines */
2628 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
2629 {
2630 d->sda = s->sda;
2631 d->sigp_ctrl.c = s->sigp_ctrl.c;
2632 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
2633 }
2634
2635 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
2636 {
2637 int i;
2638
2639 d->ipte_control = s->ipte_control;
2640 d->mcn[0] = s->mcn;
2641 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
2642 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
2643 }
2644
2645 static int sca_switch_to_extended(struct kvm *kvm)
2646 {
2647 struct bsca_block *old_sca = kvm->arch.sca;
2648 struct esca_block *new_sca;
2649 struct kvm_vcpu *vcpu;
2650 unsigned int vcpu_idx;
2651 u32 scaol, scaoh;
2652
2653 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
2654 if (!new_sca)
2655 return -ENOMEM;
2656
2657 scaoh = (u32)((u64)(new_sca) >> 32);
2658 scaol = (u32)(u64)(new_sca) & ~0x3fU;
2659
2660 kvm_s390_vcpu_block_all(kvm);
2661 write_lock(&kvm->arch.sca_lock);
2662
2663 sca_copy_b_to_e(new_sca, old_sca);
2664
2665 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
2666 vcpu->arch.sie_block->scaoh = scaoh;
2667 vcpu->arch.sie_block->scaol = scaol;
2668 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2669 }
2670 kvm->arch.sca = new_sca;
2671 kvm->arch.use_esca = 1;
2672
2673 write_unlock(&kvm->arch.sca_lock);
2674 kvm_s390_vcpu_unblock_all(kvm);
2675
2676 free_page((unsigned long)old_sca);
2677
2678 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
2679 old_sca, kvm->arch.sca);
2680 return 0;
2681 }
2682
2683 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
2684 {
2685 int rc;
2686
2687 if (!kvm_s390_use_sca_entries()) {
2688 if (id < KVM_MAX_VCPUS)
2689 return true;
2690 return false;
2691 }
2692 if (id < KVM_S390_BSCA_CPU_SLOTS)
2693 return true;
2694 if (!sclp.has_esca || !sclp.has_64bscao)
2695 return false;
2696
2697 mutex_lock(&kvm->lock);
2698 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
2699 mutex_unlock(&kvm->lock);
2700
2701 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
2702 }
2703
2704 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2705 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2706 {
2707 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
2708 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2709 vcpu->arch.cputm_start = get_tod_clock_fast();
2710 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2711 }
2712
2713 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2714 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2715 {
2716 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
2717 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2718 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2719 vcpu->arch.cputm_start = 0;
2720 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2721 }
2722
2723 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2724 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2725 {
2726 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
2727 vcpu->arch.cputm_enabled = true;
2728 __start_cpu_timer_accounting(vcpu);
2729 }
2730
2731 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2732 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2733 {
2734 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
2735 __stop_cpu_timer_accounting(vcpu);
2736 vcpu->arch.cputm_enabled = false;
2737 }
2738
2739 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2740 {
2741 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2742 __enable_cpu_timer_accounting(vcpu);
2743 preempt_enable();
2744 }
2745
2746 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2747 {
2748 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2749 __disable_cpu_timer_accounting(vcpu);
2750 preempt_enable();
2751 }
2752
2753 /* set the cpu timer - may only be called from the VCPU thread itself */
2754 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
2755 {
2756 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2757 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2758 if (vcpu->arch.cputm_enabled)
2759 vcpu->arch.cputm_start = get_tod_clock_fast();
2760 vcpu->arch.sie_block->cputm = cputm;
2761 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2762 preempt_enable();
2763 }
2764
2765 /* update and get the cpu timer - can also be called from other VCPU threads */
2766 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
2767 {
2768 unsigned int seq;
2769 __u64 value;
2770
2771 if (unlikely(!vcpu->arch.cputm_enabled))
2772 return vcpu->arch.sie_block->cputm;
2773
2774 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2775 do {
2776 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
2777 /*
2778 * If the writer would ever execute a read in the critical
2779 * section, e.g. in irq context, we have a deadlock.
2780 */
2781 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
2782 value = vcpu->arch.sie_block->cputm;
2783 /* if cputm_start is 0, accounting is being started/stopped */
2784 if (likely(vcpu->arch.cputm_start))
2785 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2786 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
2787 preempt_enable();
2788 return value;
2789 }
2790
2791 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2792 {
2793
2794 gmap_enable(vcpu->arch.enabled_gmap);
2795 kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
2796 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2797 __start_cpu_timer_accounting(vcpu);
2798 vcpu->cpu = cpu;
2799 }
2800
2801 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
2802 {
2803 vcpu->cpu = -1;
2804 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2805 __stop_cpu_timer_accounting(vcpu);
2806 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
2807 vcpu->arch.enabled_gmap = gmap_get_enabled();
2808 gmap_disable(vcpu->arch.enabled_gmap);
2809
2810 }
2811
2812 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
2813 {
2814 mutex_lock(&vcpu->kvm->lock);
2815 preempt_disable();
2816 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
2817 vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
2818 preempt_enable();
2819 mutex_unlock(&vcpu->kvm->lock);
2820 if (!kvm_is_ucontrol(vcpu->kvm)) {
2821 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
2822 sca_add_vcpu(vcpu);
2823 }
2824 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
2825 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2826 /* make vcpu_load load the right gmap on the first trigger */
2827 vcpu->arch.enabled_gmap = vcpu->arch.gmap;
2828 }
2829
2830 static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr)
2831 {
2832 if (test_bit_inv(nr, (unsigned long *)&kvm->arch.model.subfuncs.pckmo) &&
2833 test_bit_inv(nr, (unsigned long *)&kvm_s390_available_subfunc.pckmo))
2834 return true;
2835 return false;
2836 }
2837
2838 static bool kvm_has_pckmo_ecc(struct kvm *kvm)
2839 {
2840 /* At least one ECC subfunction must be present */
2841 return kvm_has_pckmo_subfunc(kvm, 32) ||
2842 kvm_has_pckmo_subfunc(kvm, 33) ||
2843 kvm_has_pckmo_subfunc(kvm, 34) ||
2844 kvm_has_pckmo_subfunc(kvm, 40) ||
2845 kvm_has_pckmo_subfunc(kvm, 41);
2846
2847 }
2848
2849 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
2850 {
2851 /*
2852 * If the AP instructions are not being interpreted and the MSAX3
2853 * facility is not configured for the guest, there is nothing to set up.
2854 */
2855 if (!vcpu->kvm->arch.crypto.apie && !test_kvm_facility(vcpu->kvm, 76))
2856 return;
2857
2858 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
2859 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
2860 vcpu->arch.sie_block->eca &= ~ECA_APIE;
2861 vcpu->arch.sie_block->ecd &= ~ECD_ECC;
2862
2863 if (vcpu->kvm->arch.crypto.apie)
2864 vcpu->arch.sie_block->eca |= ECA_APIE;
2865
2866 /* Set up protected key support */
2867 if (vcpu->kvm->arch.crypto.aes_kw) {
2868 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
2869 /* ecc is also wrapped with AES key */
2870 if (kvm_has_pckmo_ecc(vcpu->kvm))
2871 vcpu->arch.sie_block->ecd |= ECD_ECC;
2872 }
2873
2874 if (vcpu->kvm->arch.crypto.dea_kw)
2875 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
2876 }
2877
2878 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
2879 {
2880 free_page(vcpu->arch.sie_block->cbrlo);
2881 vcpu->arch.sie_block->cbrlo = 0;
2882 }
2883
2884 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
2885 {
2886 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
2887 if (!vcpu->arch.sie_block->cbrlo)
2888 return -ENOMEM;
2889 return 0;
2890 }
2891
2892 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
2893 {
2894 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
2895
2896 vcpu->arch.sie_block->ibc = model->ibc;
2897 if (test_kvm_facility(vcpu->kvm, 7))
2898 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
2899 }
2900
2901 static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu)
2902 {
2903 int rc = 0;
2904
2905 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
2906 CPUSTAT_SM |
2907 CPUSTAT_STOPPED);
2908
2909 if (test_kvm_facility(vcpu->kvm, 78))
2910 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
2911 else if (test_kvm_facility(vcpu->kvm, 8))
2912 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
2913
2914 kvm_s390_vcpu_setup_model(vcpu);
2915
2916 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
2917 if (MACHINE_HAS_ESOP)
2918 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
2919 if (test_kvm_facility(vcpu->kvm, 9))
2920 vcpu->arch.sie_block->ecb |= ECB_SRSI;
2921 if (test_kvm_facility(vcpu->kvm, 73))
2922 vcpu->arch.sie_block->ecb |= ECB_TE;
2923
2924 if (test_kvm_facility(vcpu->kvm, 8) && vcpu->kvm->arch.use_pfmfi)
2925 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
2926 if (test_kvm_facility(vcpu->kvm, 130))
2927 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
2928 vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
2929 if (sclp.has_cei)
2930 vcpu->arch.sie_block->eca |= ECA_CEI;
2931 if (sclp.has_ib)
2932 vcpu->arch.sie_block->eca |= ECA_IB;
2933 if (sclp.has_siif)
2934 vcpu->arch.sie_block->eca |= ECA_SII;
2935 if (sclp.has_sigpif)
2936 vcpu->arch.sie_block->eca |= ECA_SIGPI;
2937 if (test_kvm_facility(vcpu->kvm, 129)) {
2938 vcpu->arch.sie_block->eca |= ECA_VX;
2939 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
2940 }
2941 if (test_kvm_facility(vcpu->kvm, 139))
2942 vcpu->arch.sie_block->ecd |= ECD_MEF;
2943 if (test_kvm_facility(vcpu->kvm, 156))
2944 vcpu->arch.sie_block->ecd |= ECD_ETOKENF;
2945 if (vcpu->arch.sie_block->gd) {
2946 vcpu->arch.sie_block->eca |= ECA_AIV;
2947 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
2948 vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
2949 }
2950 vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
2951 | SDNXC;
2952 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
2953
2954 if (sclp.has_kss)
2955 kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
2956 else
2957 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
2958
2959 if (vcpu->kvm->arch.use_cmma) {
2960 rc = kvm_s390_vcpu_setup_cmma(vcpu);
2961 if (rc)
2962 return rc;
2963 }
2964 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2965 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
2966
2967 vcpu->arch.sie_block->hpid = HPID_KVM;
2968
2969 kvm_s390_vcpu_crypto_setup(vcpu);
2970
2971 return rc;
2972 }
2973
2974 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
2975 {
2976 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
2977 return -EINVAL;
2978 return 0;
2979 }
2980
2981 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
2982 {
2983 struct sie_page *sie_page;
2984 int rc;
2985
2986 BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
2987 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
2988 if (!sie_page)
2989 return -ENOMEM;
2990
2991 vcpu->arch.sie_block = &sie_page->sie_block;
2992 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
2993
2994 /* the real guest size will always be smaller than msl */
2995 vcpu->arch.sie_block->mso = 0;
2996 vcpu->arch.sie_block->msl = sclp.hamax;
2997
2998 vcpu->arch.sie_block->icpua = vcpu->vcpu_id;
2999 spin_lock_init(&vcpu->arch.local_int.lock);
3000 vcpu->arch.sie_block->gd = (u32)(u64)vcpu->kvm->arch.gisa_int.origin;
3001 if (vcpu->arch.sie_block->gd && sclp.has_gisaf)
3002 vcpu->arch.sie_block->gd |= GISA_FORMAT1;
3003 seqcount_init(&vcpu->arch.cputm_seqcount);
3004
3005 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
3006 kvm_clear_async_pf_completion_queue(vcpu);
3007 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
3008 KVM_SYNC_GPRS |
3009 KVM_SYNC_ACRS |
3010 KVM_SYNC_CRS |
3011 KVM_SYNC_ARCH0 |
3012 KVM_SYNC_PFAULT;
3013 kvm_s390_set_prefix(vcpu, 0);
3014 if (test_kvm_facility(vcpu->kvm, 64))
3015 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
3016 if (test_kvm_facility(vcpu->kvm, 82))
3017 vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
3018 if (test_kvm_facility(vcpu->kvm, 133))
3019 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
3020 if (test_kvm_facility(vcpu->kvm, 156))
3021 vcpu->run->kvm_valid_regs |= KVM_SYNC_ETOKEN;
3022 /* fprs can be synchronized via vrs, even if the guest has no vx. With
3023 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
3024 */
3025 if (MACHINE_HAS_VX)
3026 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
3027 else
3028 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
3029
3030 if (kvm_is_ucontrol(vcpu->kvm)) {
3031 rc = __kvm_ucontrol_vcpu_init(vcpu);
3032 if (rc)
3033 goto out_free_sie_block;
3034 }
3035
3036 VM_EVENT(vcpu->kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK",
3037 vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3038 trace_kvm_s390_create_vcpu(vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3039
3040 rc = kvm_s390_vcpu_setup(vcpu);
3041 if (rc)
3042 goto out_ucontrol_uninit;
3043 return 0;
3044
3045 out_ucontrol_uninit:
3046 if (kvm_is_ucontrol(vcpu->kvm))
3047 gmap_remove(vcpu->arch.gmap);
3048 out_free_sie_block:
3049 free_page((unsigned long)(vcpu->arch.sie_block));
3050 return rc;
3051 }
3052
3053 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
3054 {
3055 return kvm_s390_vcpu_has_irq(vcpu, 0);
3056 }
3057
3058 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
3059 {
3060 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
3061 }
3062
3063 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
3064 {
3065 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3066 exit_sie(vcpu);
3067 }
3068
3069 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
3070 {
3071 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3072 }
3073
3074 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
3075 {
3076 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3077 exit_sie(vcpu);
3078 }
3079
3080 bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu)
3081 {
3082 return atomic_read(&vcpu->arch.sie_block->prog20) &
3083 (PROG_BLOCK_SIE | PROG_REQUEST);
3084 }
3085
3086 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
3087 {
3088 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3089 }
3090
3091 /*
3092 * Kick a guest cpu out of (v)SIE and wait until (v)SIE is not running.
3093 * If the CPU is not running (e.g. waiting as idle) the function will
3094 * return immediately. */
3095 void exit_sie(struct kvm_vcpu *vcpu)
3096 {
3097 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
3098 kvm_s390_vsie_kick(vcpu);
3099 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
3100 cpu_relax();
3101 }
3102
3103 /* Kick a guest cpu out of SIE to process a request synchronously */
3104 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
3105 {
3106 kvm_make_request(req, vcpu);
3107 kvm_s390_vcpu_request(vcpu);
3108 }
3109
3110 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
3111 unsigned long end)
3112 {
3113 struct kvm *kvm = gmap->private;
3114 struct kvm_vcpu *vcpu;
3115 unsigned long prefix;
3116 int i;
3117
3118 if (gmap_is_shadow(gmap))
3119 return;
3120 if (start >= 1UL << 31)
3121 /* We are only interested in prefix pages */
3122 return;
3123 kvm_for_each_vcpu(i, vcpu, kvm) {
3124 /* match against both prefix pages */
3125 prefix = kvm_s390_get_prefix(vcpu);
3126 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
3127 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
3128 start, end);
3129 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
3130 }
3131 }
3132 }
3133
3134 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
3135 {
3136 /* do not poll with more than halt_poll_max_steal percent of steal time */
3137 if (S390_lowcore.avg_steal_timer * 100 / (TICK_USEC << 12) >=
3138 halt_poll_max_steal) {
3139 vcpu->stat.halt_no_poll_steal++;
3140 return true;
3141 }
3142 return false;
3143 }
3144
3145 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
3146 {
3147 /* kvm common code refers to this, but never calls it */
3148 BUG();
3149 return 0;
3150 }
3151
3152 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
3153 struct kvm_one_reg *reg)
3154 {
3155 int r = -EINVAL;
3156
3157 switch (reg->id) {
3158 case KVM_REG_S390_TODPR:
3159 r = put_user(vcpu->arch.sie_block->todpr,
3160 (u32 __user *)reg->addr);
3161 break;
3162 case KVM_REG_S390_EPOCHDIFF:
3163 r = put_user(vcpu->arch.sie_block->epoch,
3164 (u64 __user *)reg->addr);
3165 break;
3166 case KVM_REG_S390_CPU_TIMER:
3167 r = put_user(kvm_s390_get_cpu_timer(vcpu),
3168 (u64 __user *)reg->addr);
3169 break;
3170 case KVM_REG_S390_CLOCK_COMP:
3171 r = put_user(vcpu->arch.sie_block->ckc,
3172 (u64 __user *)reg->addr);
3173 break;
3174 case KVM_REG_S390_PFTOKEN:
3175 r = put_user(vcpu->arch.pfault_token,
3176 (u64 __user *)reg->addr);
3177 break;
3178 case KVM_REG_S390_PFCOMPARE:
3179 r = put_user(vcpu->arch.pfault_compare,
3180 (u64 __user *)reg->addr);
3181 break;
3182 case KVM_REG_S390_PFSELECT:
3183 r = put_user(vcpu->arch.pfault_select,
3184 (u64 __user *)reg->addr);
3185 break;
3186 case KVM_REG_S390_PP:
3187 r = put_user(vcpu->arch.sie_block->pp,
3188 (u64 __user *)reg->addr);
3189 break;
3190 case KVM_REG_S390_GBEA:
3191 r = put_user(vcpu->arch.sie_block->gbea,
3192 (u64 __user *)reg->addr);
3193 break;
3194 default:
3195 break;
3196 }
3197
3198 return r;
3199 }
3200
3201 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
3202 struct kvm_one_reg *reg)
3203 {
3204 int r = -EINVAL;
3205 __u64 val;
3206
3207 switch (reg->id) {
3208 case KVM_REG_S390_TODPR:
3209 r = get_user(vcpu->arch.sie_block->todpr,
3210 (u32 __user *)reg->addr);
3211 break;
3212 case KVM_REG_S390_EPOCHDIFF:
3213 r = get_user(vcpu->arch.sie_block->epoch,
3214 (u64 __user *)reg->addr);
3215 break;
3216 case KVM_REG_S390_CPU_TIMER:
3217 r = get_user(val, (u64 __user *)reg->addr);
3218 if (!r)
3219 kvm_s390_set_cpu_timer(vcpu, val);
3220 break;
3221 case KVM_REG_S390_CLOCK_COMP:
3222 r = get_user(vcpu->arch.sie_block->ckc,
3223 (u64 __user *)reg->addr);
3224 break;
3225 case KVM_REG_S390_PFTOKEN:
3226 r = get_user(vcpu->arch.pfault_token,
3227 (u64 __user *)reg->addr);
3228 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3229 kvm_clear_async_pf_completion_queue(vcpu);
3230 break;
3231 case KVM_REG_S390_PFCOMPARE:
3232 r = get_user(vcpu->arch.pfault_compare,
3233 (u64 __user *)reg->addr);
3234 break;
3235 case KVM_REG_S390_PFSELECT:
3236 r = get_user(vcpu->arch.pfault_select,
3237 (u64 __user *)reg->addr);
3238 break;
3239 case KVM_REG_S390_PP:
3240 r = get_user(vcpu->arch.sie_block->pp,
3241 (u64 __user *)reg->addr);
3242 break;
3243 case KVM_REG_S390_GBEA:
3244 r = get_user(vcpu->arch.sie_block->gbea,
3245 (u64 __user *)reg->addr);
3246 break;
3247 default:
3248 break;
3249 }
3250
3251 return r;
3252 }
3253
3254 static void kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
3255 {
3256 vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_RI;
3257 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
3258 memset(vcpu->run->s.regs.riccb, 0, sizeof(vcpu->run->s.regs.riccb));
3259
3260 kvm_clear_async_pf_completion_queue(vcpu);
3261 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
3262 kvm_s390_vcpu_stop(vcpu);
3263 kvm_s390_clear_local_irqs(vcpu);
3264 }
3265
3266 static void kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
3267 {
3268 /* Initial reset is a superset of the normal reset */
3269 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
3270
3271 /*
3272 * This equals initial cpu reset in pop, but we don't switch to ESA.
3273 * We do not only reset the internal data, but also ...
3274 */
3275 vcpu->arch.sie_block->gpsw.mask = 0;
3276 vcpu->arch.sie_block->gpsw.addr = 0;
3277 kvm_s390_set_prefix(vcpu, 0);
3278 kvm_s390_set_cpu_timer(vcpu, 0);
3279 vcpu->arch.sie_block->ckc = 0;
3280 vcpu->arch.sie_block->todpr = 0;
3281 memset(vcpu->arch.sie_block->gcr, 0, sizeof(vcpu->arch.sie_block->gcr));
3282 vcpu->arch.sie_block->gcr[0] = CR0_INITIAL_MASK;
3283 vcpu->arch.sie_block->gcr[14] = CR14_INITIAL_MASK;
3284
3285 /* ... the data in sync regs */
3286 memset(vcpu->run->s.regs.crs, 0, sizeof(vcpu->run->s.regs.crs));
3287 vcpu->run->s.regs.ckc = 0;
3288 vcpu->run->s.regs.crs[0] = CR0_INITIAL_MASK;
3289 vcpu->run->s.regs.crs[14] = CR14_INITIAL_MASK;
3290 vcpu->run->psw_addr = 0;
3291 vcpu->run->psw_mask = 0;
3292 vcpu->run->s.regs.todpr = 0;
3293 vcpu->run->s.regs.cputm = 0;
3294 vcpu->run->s.regs.ckc = 0;
3295 vcpu->run->s.regs.pp = 0;
3296 vcpu->run->s.regs.gbea = 1;
3297 vcpu->run->s.regs.fpc = 0;
3298 vcpu->arch.sie_block->gbea = 1;
3299 vcpu->arch.sie_block->pp = 0;
3300 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
3301 }
3302
3303 static void kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
3304 {
3305 struct kvm_sync_regs *regs = &vcpu->run->s.regs;
3306
3307 /* Clear reset is a superset of the initial reset */
3308 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3309
3310 memset(&regs->gprs, 0, sizeof(regs->gprs));
3311 memset(&regs->vrs, 0, sizeof(regs->vrs));
3312 memset(&regs->acrs, 0, sizeof(regs->acrs));
3313 memset(&regs->gscb, 0, sizeof(regs->gscb));
3314
3315 regs->etoken = 0;
3316 regs->etoken_extension = 0;
3317 }
3318
3319 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3320 {
3321 vcpu_load(vcpu);
3322 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
3323 vcpu_put(vcpu);
3324 return 0;
3325 }
3326
3327 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3328 {
3329 vcpu_load(vcpu);
3330 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
3331 vcpu_put(vcpu);
3332 return 0;
3333 }
3334
3335 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
3336 struct kvm_sregs *sregs)
3337 {
3338 vcpu_load(vcpu);
3339
3340 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
3341 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
3342
3343 vcpu_put(vcpu);
3344 return 0;
3345 }
3346
3347 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
3348 struct kvm_sregs *sregs)
3349 {
3350 vcpu_load(vcpu);
3351
3352 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
3353 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
3354
3355 vcpu_put(vcpu);
3356 return 0;
3357 }
3358
3359 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3360 {
3361 int ret = 0;
3362
3363 vcpu_load(vcpu);
3364
3365 if (test_fp_ctl(fpu->fpc)) {
3366 ret = -EINVAL;
3367 goto out;
3368 }
3369 vcpu->run->s.regs.fpc = fpu->fpc;
3370 if (MACHINE_HAS_VX)
3371 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
3372 (freg_t *) fpu->fprs);
3373 else
3374 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
3375
3376 out:
3377 vcpu_put(vcpu);
3378 return ret;
3379 }
3380
3381 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3382 {
3383 vcpu_load(vcpu);
3384
3385 /* make sure we have the latest values */
3386 save_fpu_regs();
3387 if (MACHINE_HAS_VX)
3388 convert_vx_to_fp((freg_t *) fpu->fprs,
3389 (__vector128 *) vcpu->run->s.regs.vrs);
3390 else
3391 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
3392 fpu->fpc = vcpu->run->s.regs.fpc;
3393
3394 vcpu_put(vcpu);
3395 return 0;
3396 }
3397
3398 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
3399 {
3400 int rc = 0;
3401
3402 if (!is_vcpu_stopped(vcpu))
3403 rc = -EBUSY;
3404 else {
3405 vcpu->run->psw_mask = psw.mask;
3406 vcpu->run->psw_addr = psw.addr;
3407 }
3408 return rc;
3409 }
3410
3411 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
3412 struct kvm_translation *tr)
3413 {
3414 return -EINVAL; /* not implemented yet */
3415 }
3416
3417 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
3418 KVM_GUESTDBG_USE_HW_BP | \
3419 KVM_GUESTDBG_ENABLE)
3420
3421 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
3422 struct kvm_guest_debug *dbg)
3423 {
3424 int rc = 0;
3425
3426 vcpu_load(vcpu);
3427
3428 vcpu->guest_debug = 0;
3429 kvm_s390_clear_bp_data(vcpu);
3430
3431 if (dbg->control & ~VALID_GUESTDBG_FLAGS) {
3432 rc = -EINVAL;
3433 goto out;
3434 }
3435 if (!sclp.has_gpere) {
3436 rc = -EINVAL;
3437 goto out;
3438 }
3439
3440 if (dbg->control & KVM_GUESTDBG_ENABLE) {
3441 vcpu->guest_debug = dbg->control;
3442 /* enforce guest PER */
3443 kvm_s390_set_cpuflags(vcpu, CPUSTAT_P);
3444
3445 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
3446 rc = kvm_s390_import_bp_data(vcpu, dbg);
3447 } else {
3448 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
3449 vcpu->arch.guestdbg.last_bp = 0;
3450 }
3451
3452 if (rc) {
3453 vcpu->guest_debug = 0;
3454 kvm_s390_clear_bp_data(vcpu);
3455 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
3456 }
3457
3458 out:
3459 vcpu_put(vcpu);
3460 return rc;
3461 }
3462
3463 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
3464 struct kvm_mp_state *mp_state)
3465 {
3466 int ret;
3467
3468 vcpu_load(vcpu);
3469
3470 /* CHECK_STOP and LOAD are not supported yet */
3471 ret = is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
3472 KVM_MP_STATE_OPERATING;
3473
3474 vcpu_put(vcpu);
3475 return ret;
3476 }
3477
3478 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
3479 struct kvm_mp_state *mp_state)
3480 {
3481 int rc = 0;
3482
3483 vcpu_load(vcpu);
3484
3485 /* user space knows about this interface - let it control the state */
3486 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
3487
3488 switch (mp_state->mp_state) {
3489 case KVM_MP_STATE_STOPPED:
3490 kvm_s390_vcpu_stop(vcpu);
3491 break;
3492 case KVM_MP_STATE_OPERATING:
3493 kvm_s390_vcpu_start(vcpu);
3494 break;
3495 case KVM_MP_STATE_LOAD:
3496 case KVM_MP_STATE_CHECK_STOP:
3497 /* fall through - CHECK_STOP and LOAD are not supported yet */
3498 default:
3499 rc = -ENXIO;
3500 }
3501
3502 vcpu_put(vcpu);
3503 return rc;
3504 }
3505
3506 static bool ibs_enabled(struct kvm_vcpu *vcpu)
3507 {
3508 return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS);
3509 }
3510
3511 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
3512 {
3513 retry:
3514 kvm_s390_vcpu_request_handled(vcpu);
3515 if (!kvm_request_pending(vcpu))
3516 return 0;
3517 /*
3518 * We use MMU_RELOAD just to re-arm the ipte notifier for the
3519 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
3520 * This ensures that the ipte instruction for this request has
3521 * already finished. We might race against a second unmapper that
3522 * wants to set the blocking bit. Lets just retry the request loop.
3523 */
3524 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
3525 int rc;
3526 rc = gmap_mprotect_notify(vcpu->arch.gmap,
3527 kvm_s390_get_prefix(vcpu),
3528 PAGE_SIZE * 2, PROT_WRITE);
3529 if (rc) {
3530 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
3531 return rc;
3532 }
3533 goto retry;
3534 }
3535
3536 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
3537 vcpu->arch.sie_block->ihcpu = 0xffff;
3538 goto retry;
3539 }
3540
3541 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
3542 if (!ibs_enabled(vcpu)) {
3543 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
3544 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS);
3545 }
3546 goto retry;
3547 }
3548
3549 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
3550 if (ibs_enabled(vcpu)) {
3551 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
3552 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS);
3553 }
3554 goto retry;
3555 }
3556
3557 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
3558 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
3559 goto retry;
3560 }
3561
3562 if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
3563 /*
3564 * Disable CMM virtualization; we will emulate the ESSA
3565 * instruction manually, in order to provide additional
3566 * functionalities needed for live migration.
3567 */
3568 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
3569 goto retry;
3570 }
3571
3572 if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
3573 /*
3574 * Re-enable CMM virtualization if CMMA is available and
3575 * CMM has been used.
3576 */
3577 if ((vcpu->kvm->arch.use_cmma) &&
3578 (vcpu->kvm->mm->context.uses_cmm))
3579 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
3580 goto retry;
3581 }
3582
3583 /* nothing to do, just clear the request */
3584 kvm_clear_request(KVM_REQ_UNHALT, vcpu);
3585 /* we left the vsie handler, nothing to do, just clear the request */
3586 kvm_clear_request(KVM_REQ_VSIE_RESTART, vcpu);
3587
3588 return 0;
3589 }
3590
3591 void kvm_s390_set_tod_clock(struct kvm *kvm,
3592 const struct kvm_s390_vm_tod_clock *gtod)
3593 {
3594 struct kvm_vcpu *vcpu;
3595 struct kvm_s390_tod_clock_ext htod;
3596 int i;
3597
3598 mutex_lock(&kvm->lock);
3599 preempt_disable();
3600
3601 get_tod_clock_ext((char *)&htod);
3602
3603 kvm->arch.epoch = gtod->tod - htod.tod;
3604 kvm->arch.epdx = 0;
3605 if (test_kvm_facility(kvm, 139)) {
3606 kvm->arch.epdx = gtod->epoch_idx - htod.epoch_idx;
3607 if (kvm->arch.epoch > gtod->tod)
3608 kvm->arch.epdx -= 1;
3609 }
3610
3611 kvm_s390_vcpu_block_all(kvm);
3612 kvm_for_each_vcpu(i, vcpu, kvm) {
3613 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
3614 vcpu->arch.sie_block->epdx = kvm->arch.epdx;
3615 }
3616
3617 kvm_s390_vcpu_unblock_all(kvm);
3618 preempt_enable();
3619 mutex_unlock(&kvm->lock);
3620 }
3621
3622 /**
3623 * kvm_arch_fault_in_page - fault-in guest page if necessary
3624 * @vcpu: The corresponding virtual cpu
3625 * @gpa: Guest physical address
3626 * @writable: Whether the page should be writable or not
3627 *
3628 * Make sure that a guest page has been faulted-in on the host.
3629 *
3630 * Return: Zero on success, negative error code otherwise.
3631 */
3632 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
3633 {
3634 return gmap_fault(vcpu->arch.gmap, gpa,
3635 writable ? FAULT_FLAG_WRITE : 0);
3636 }
3637
3638 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
3639 unsigned long token)
3640 {
3641 struct kvm_s390_interrupt inti;
3642 struct kvm_s390_irq irq;
3643
3644 if (start_token) {
3645 irq.u.ext.ext_params2 = token;
3646 irq.type = KVM_S390_INT_PFAULT_INIT;
3647 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3648 } else {
3649 inti.type = KVM_S390_INT_PFAULT_DONE;
3650 inti.parm64 = token;
3651 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
3652 }
3653 }
3654
3655 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
3656 struct kvm_async_pf *work)
3657 {
3658 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
3659 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
3660 }
3661
3662 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
3663 struct kvm_async_pf *work)
3664 {
3665 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
3666 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
3667 }
3668
3669 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
3670 struct kvm_async_pf *work)
3671 {
3672 /* s390 will always inject the page directly */
3673 }
3674
3675 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
3676 {
3677 /*
3678 * s390 will always inject the page directly,
3679 * but we still want check_async_completion to cleanup
3680 */
3681 return true;
3682 }
3683
3684 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
3685 {
3686 hva_t hva;
3687 struct kvm_arch_async_pf arch;
3688 int rc;
3689
3690 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3691 return 0;
3692 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
3693 vcpu->arch.pfault_compare)
3694 return 0;
3695 if (psw_extint_disabled(vcpu))
3696 return 0;
3697 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3698 return 0;
3699 if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
3700 return 0;
3701 if (!vcpu->arch.gmap->pfault_enabled)
3702 return 0;
3703
3704 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
3705 hva += current->thread.gmap_addr & ~PAGE_MASK;
3706 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3707 return 0;
3708
3709 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
3710 return rc;
3711 }
3712
3713 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
3714 {
3715 int rc, cpuflags;
3716
3717 /*
3718 * On s390 notifications for arriving pages will be delivered directly
3719 * to the guest but the house keeping for completed pfaults is
3720 * handled outside the worker.
3721 */
3722 kvm_check_async_pf_completion(vcpu);
3723
3724 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
3725 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
3726
3727 if (need_resched())
3728 schedule();
3729
3730 if (test_cpu_flag(CIF_MCCK_PENDING))
3731 s390_handle_mcck();
3732
3733 if (!kvm_is_ucontrol(vcpu->kvm)) {
3734 rc = kvm_s390_deliver_pending_interrupts(vcpu);
3735 if (rc)
3736 return rc;
3737 }
3738
3739 rc = kvm_s390_handle_requests(vcpu);
3740 if (rc)
3741 return rc;
3742
3743 if (guestdbg_enabled(vcpu)) {
3744 kvm_s390_backup_guest_per_regs(vcpu);
3745 kvm_s390_patch_guest_per_regs(vcpu);
3746 }
3747
3748 clear_bit(vcpu->vcpu_id, vcpu->kvm->arch.gisa_int.kicked_mask);
3749
3750 vcpu->arch.sie_block->icptcode = 0;
3751 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
3752 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
3753 trace_kvm_s390_sie_enter(vcpu, cpuflags);
3754
3755 return 0;
3756 }
3757
3758 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
3759 {
3760 struct kvm_s390_pgm_info pgm_info = {
3761 .code = PGM_ADDRESSING,
3762 };
3763 u8 opcode, ilen;
3764 int rc;
3765
3766 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
3767 trace_kvm_s390_sie_fault(vcpu);
3768
3769 /*
3770 * We want to inject an addressing exception, which is defined as a
3771 * suppressing or terminating exception. However, since we came here
3772 * by a DAT access exception, the PSW still points to the faulting
3773 * instruction since DAT exceptions are nullifying. So we've got
3774 * to look up the current opcode to get the length of the instruction
3775 * to be able to forward the PSW.
3776 */
3777 rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
3778 ilen = insn_length(opcode);
3779 if (rc < 0) {
3780 return rc;
3781 } else if (rc) {
3782 /* Instruction-Fetching Exceptions - we can't detect the ilen.
3783 * Forward by arbitrary ilc, injection will take care of
3784 * nullification if necessary.
3785 */
3786 pgm_info = vcpu->arch.pgm;
3787 ilen = 4;
3788 }
3789 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
3790 kvm_s390_forward_psw(vcpu, ilen);
3791 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
3792 }
3793
3794 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
3795 {
3796 struct mcck_volatile_info *mcck_info;
3797 struct sie_page *sie_page;
3798
3799 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
3800 vcpu->arch.sie_block->icptcode);
3801 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
3802
3803 if (guestdbg_enabled(vcpu))
3804 kvm_s390_restore_guest_per_regs(vcpu);
3805
3806 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
3807 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
3808
3809 if (exit_reason == -EINTR) {
3810 VCPU_EVENT(vcpu, 3, "%s", "machine check");
3811 sie_page = container_of(vcpu->arch.sie_block,
3812 struct sie_page, sie_block);
3813 mcck_info = &sie_page->mcck_info;
3814 kvm_s390_reinject_machine_check(vcpu, mcck_info);
3815 return 0;
3816 }
3817
3818 if (vcpu->arch.sie_block->icptcode > 0) {
3819 int rc = kvm_handle_sie_intercept(vcpu);
3820
3821 if (rc != -EOPNOTSUPP)
3822 return rc;
3823 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
3824 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
3825 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
3826 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
3827 return -EREMOTE;
3828 } else if (exit_reason != -EFAULT) {
3829 vcpu->stat.exit_null++;
3830 return 0;
3831 } else if (kvm_is_ucontrol(vcpu->kvm)) {
3832 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
3833 vcpu->run->s390_ucontrol.trans_exc_code =
3834 current->thread.gmap_addr;
3835 vcpu->run->s390_ucontrol.pgm_code = 0x10;
3836 return -EREMOTE;
3837 } else if (current->thread.gmap_pfault) {
3838 trace_kvm_s390_major_guest_pfault(vcpu);
3839 current->thread.gmap_pfault = 0;
3840 if (kvm_arch_setup_async_pf(vcpu))
3841 return 0;
3842 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
3843 }
3844 return vcpu_post_run_fault_in_sie(vcpu);
3845 }
3846
3847 static int __vcpu_run(struct kvm_vcpu *vcpu)
3848 {
3849 int rc, exit_reason;
3850
3851 /*
3852 * We try to hold kvm->srcu during most of vcpu_run (except when run-
3853 * ning the guest), so that memslots (and other stuff) are protected
3854 */
3855 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3856
3857 do {
3858 rc = vcpu_pre_run(vcpu);
3859 if (rc)
3860 break;
3861
3862 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
3863 /*
3864 * As PF_VCPU will be used in fault handler, between
3865 * guest_enter and guest_exit should be no uaccess.
3866 */
3867 local_irq_disable();
3868 guest_enter_irqoff();
3869 __disable_cpu_timer_accounting(vcpu);
3870 local_irq_enable();
3871 exit_reason = sie64a(vcpu->arch.sie_block,
3872 vcpu->run->s.regs.gprs);
3873 local_irq_disable();
3874 __enable_cpu_timer_accounting(vcpu);
3875 guest_exit_irqoff();
3876 local_irq_enable();
3877 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3878
3879 rc = vcpu_post_run(vcpu, exit_reason);
3880 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3881
3882 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
3883 return rc;
3884 }
3885
3886 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3887 {
3888 struct runtime_instr_cb *riccb;
3889 struct gs_cb *gscb;
3890
3891 riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
3892 gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
3893 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
3894 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
3895 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
3896 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
3897 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
3898 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
3899 /* some control register changes require a tlb flush */
3900 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3901 }
3902 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
3903 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
3904 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
3905 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
3906 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
3907 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
3908 }
3909 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
3910 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
3911 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
3912 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
3913 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3914 kvm_clear_async_pf_completion_queue(vcpu);
3915 }
3916 /*
3917 * If userspace sets the riccb (e.g. after migration) to a valid state,
3918 * we should enable RI here instead of doing the lazy enablement.
3919 */
3920 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
3921 test_kvm_facility(vcpu->kvm, 64) &&
3922 riccb->v &&
3923 !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
3924 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
3925 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
3926 }
3927 /*
3928 * If userspace sets the gscb (e.g. after migration) to non-zero,
3929 * we should enable GS here instead of doing the lazy enablement.
3930 */
3931 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
3932 test_kvm_facility(vcpu->kvm, 133) &&
3933 gscb->gssm &&
3934 !vcpu->arch.gs_enabled) {
3935 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
3936 vcpu->arch.sie_block->ecb |= ECB_GS;
3937 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3938 vcpu->arch.gs_enabled = 1;
3939 }
3940 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) &&
3941 test_kvm_facility(vcpu->kvm, 82)) {
3942 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
3943 vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
3944 }
3945 save_access_regs(vcpu->arch.host_acrs);
3946 restore_access_regs(vcpu->run->s.regs.acrs);
3947 /* save host (userspace) fprs/vrs */
3948 save_fpu_regs();
3949 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
3950 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
3951 if (MACHINE_HAS_VX)
3952 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
3953 else
3954 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
3955 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
3956 if (test_fp_ctl(current->thread.fpu.fpc))
3957 /* User space provided an invalid FPC, let's clear it */
3958 current->thread.fpu.fpc = 0;
3959 if (MACHINE_HAS_GS) {
3960 preempt_disable();
3961 __ctl_set_bit(2, 4);
3962 if (current->thread.gs_cb) {
3963 vcpu->arch.host_gscb = current->thread.gs_cb;
3964 save_gs_cb(vcpu->arch.host_gscb);
3965 }
3966 if (vcpu->arch.gs_enabled) {
3967 current->thread.gs_cb = (struct gs_cb *)
3968 &vcpu->run->s.regs.gscb;
3969 restore_gs_cb(current->thread.gs_cb);
3970 }
3971 preempt_enable();
3972 }
3973 /* SIE will load etoken directly from SDNX and therefore kvm_run */
3974
3975 kvm_run->kvm_dirty_regs = 0;
3976 }
3977
3978 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3979 {
3980 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
3981 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
3982 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
3983 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
3984 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
3985 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
3986 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
3987 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
3988 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
3989 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
3990 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
3991 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
3992 kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
3993 save_access_regs(vcpu->run->s.regs.acrs);
3994 restore_access_regs(vcpu->arch.host_acrs);
3995 /* Save guest register state */
3996 save_fpu_regs();
3997 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3998 /* Restore will be done lazily at return */
3999 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
4000 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
4001 if (MACHINE_HAS_GS) {
4002 __ctl_set_bit(2, 4);
4003 if (vcpu->arch.gs_enabled)
4004 save_gs_cb(current->thread.gs_cb);
4005 preempt_disable();
4006 current->thread.gs_cb = vcpu->arch.host_gscb;
4007 restore_gs_cb(vcpu->arch.host_gscb);
4008 preempt_enable();
4009 if (!vcpu->arch.host_gscb)
4010 __ctl_clear_bit(2, 4);
4011 vcpu->arch.host_gscb = NULL;
4012 }
4013 /* SIE will save etoken directly into SDNX and therefore kvm_run */
4014 }
4015
4016 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
4017 {
4018 int rc;
4019
4020 if (kvm_run->immediate_exit)
4021 return -EINTR;
4022
4023 if (kvm_run->kvm_valid_regs & ~KVM_SYNC_S390_VALID_FIELDS ||
4024 kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS)
4025 return -EINVAL;
4026
4027 vcpu_load(vcpu);
4028
4029 if (guestdbg_exit_pending(vcpu)) {
4030 kvm_s390_prepare_debug_exit(vcpu);
4031 rc = 0;
4032 goto out;
4033 }
4034
4035 kvm_sigset_activate(vcpu);
4036
4037 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
4038 kvm_s390_vcpu_start(vcpu);
4039 } else if (is_vcpu_stopped(vcpu)) {
4040 pr_err_ratelimited("can't run stopped vcpu %d\n",
4041 vcpu->vcpu_id);
4042 rc = -EINVAL;
4043 goto out;
4044 }
4045
4046 sync_regs(vcpu, kvm_run);
4047 enable_cpu_timer_accounting(vcpu);
4048
4049 might_fault();
4050 rc = __vcpu_run(vcpu);
4051
4052 if (signal_pending(current) && !rc) {
4053 kvm_run->exit_reason = KVM_EXIT_INTR;
4054 rc = -EINTR;
4055 }
4056
4057 if (guestdbg_exit_pending(vcpu) && !rc) {
4058 kvm_s390_prepare_debug_exit(vcpu);
4059 rc = 0;
4060 }
4061
4062 if (rc == -EREMOTE) {
4063 /* userspace support is needed, kvm_run has been prepared */
4064 rc = 0;
4065 }
4066
4067 disable_cpu_timer_accounting(vcpu);
4068 store_regs(vcpu, kvm_run);
4069
4070 kvm_sigset_deactivate(vcpu);
4071
4072 vcpu->stat.exit_userspace++;
4073 out:
4074 vcpu_put(vcpu);
4075 return rc;
4076 }
4077
4078 /*
4079 * store status at address
4080 * we use have two special cases:
4081 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
4082 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
4083 */
4084 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
4085 {
4086 unsigned char archmode = 1;
4087 freg_t fprs[NUM_FPRS];
4088 unsigned int px;
4089 u64 clkcomp, cputm;
4090 int rc;
4091
4092 px = kvm_s390_get_prefix(vcpu);
4093 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
4094 if (write_guest_abs(vcpu, 163, &archmode, 1))
4095 return -EFAULT;
4096 gpa = 0;
4097 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
4098 if (write_guest_real(vcpu, 163, &archmode, 1))
4099 return -EFAULT;
4100 gpa = px;
4101 } else
4102 gpa -= __LC_FPREGS_SAVE_AREA;
4103
4104 /* manually convert vector registers if necessary */
4105 if (MACHINE_HAS_VX) {
4106 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
4107 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
4108 fprs, 128);
4109 } else {
4110 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
4111 vcpu->run->s.regs.fprs, 128);
4112 }
4113 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
4114 vcpu->run->s.regs.gprs, 128);
4115 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
4116 &vcpu->arch.sie_block->gpsw, 16);
4117 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
4118 &px, 4);
4119 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
4120 &vcpu->run->s.regs.fpc, 4);
4121 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
4122 &vcpu->arch.sie_block->todpr, 4);
4123 cputm = kvm_s390_get_cpu_timer(vcpu);
4124 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4125 &cputm, 8);
4126 clkcomp = vcpu->arch.sie_block->ckc >> 8;
4127 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
4128 &clkcomp, 8);
4129 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
4130 &vcpu->run->s.regs.acrs, 64);
4131 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
4132 &vcpu->arch.sie_block->gcr, 128);
4133 return rc ? -EFAULT : 0;
4134 }
4135
4136 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
4137 {
4138 /*
4139 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
4140 * switch in the run ioctl. Let's update our copies before we save
4141 * it into the save area
4142 */
4143 save_fpu_regs();
4144 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
4145 save_access_regs(vcpu->run->s.regs.acrs);
4146
4147 return kvm_s390_store_status_unloaded(vcpu, addr);
4148 }
4149
4150 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
4151 {
4152 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
4153 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
4154 }
4155
4156 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
4157 {
4158 unsigned int i;
4159 struct kvm_vcpu *vcpu;
4160
4161 kvm_for_each_vcpu(i, vcpu, kvm) {
4162 __disable_ibs_on_vcpu(vcpu);
4163 }
4164 }
4165
4166 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
4167 {
4168 if (!sclp.has_ibs)
4169 return;
4170 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
4171 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
4172 }
4173
4174 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
4175 {
4176 int i, online_vcpus, started_vcpus = 0;
4177
4178 if (!is_vcpu_stopped(vcpu))
4179 return;
4180
4181 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
4182 /* Only one cpu at a time may enter/leave the STOPPED state. */
4183 spin_lock(&vcpu->kvm->arch.start_stop_lock);
4184 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
4185
4186 for (i = 0; i < online_vcpus; i++) {
4187 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
4188 started_vcpus++;
4189 }
4190
4191 if (started_vcpus == 0) {
4192 /* we're the only active VCPU -> speed it up */
4193 __enable_ibs_on_vcpu(vcpu);
4194 } else if (started_vcpus == 1) {
4195 /*
4196 * As we are starting a second VCPU, we have to disable
4197 * the IBS facility on all VCPUs to remove potentially
4198 * oustanding ENABLE requests.
4199 */
4200 __disable_ibs_on_all_vcpus(vcpu->kvm);
4201 }
4202
4203 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
4204 /*
4205 * Another VCPU might have used IBS while we were offline.
4206 * Let's play safe and flush the VCPU at startup.
4207 */
4208 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
4209 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4210 return;
4211 }
4212
4213 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
4214 {
4215 int i, online_vcpus, started_vcpus = 0;
4216 struct kvm_vcpu *started_vcpu = NULL;
4217
4218 if (is_vcpu_stopped(vcpu))
4219 return;
4220
4221 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
4222 /* Only one cpu at a time may enter/leave the STOPPED state. */
4223 spin_lock(&vcpu->kvm->arch.start_stop_lock);
4224 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
4225
4226 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
4227 kvm_s390_clear_stop_irq(vcpu);
4228
4229 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
4230 __disable_ibs_on_vcpu(vcpu);
4231
4232 for (i = 0; i < online_vcpus; i++) {
4233 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
4234 started_vcpus++;
4235 started_vcpu = vcpu->kvm->vcpus[i];
4236 }
4237 }
4238
4239 if (started_vcpus == 1) {
4240 /*
4241 * As we only have one VCPU left, we want to enable the
4242 * IBS facility for that VCPU to speed it up.
4243 */
4244 __enable_ibs_on_vcpu(started_vcpu);
4245 }
4246
4247 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4248 return;
4249 }
4250
4251 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
4252 struct kvm_enable_cap *cap)
4253 {
4254 int r;
4255
4256 if (cap->flags)
4257 return -EINVAL;
4258
4259 switch (cap->cap) {
4260 case KVM_CAP_S390_CSS_SUPPORT:
4261 if (!vcpu->kvm->arch.css_support) {
4262 vcpu->kvm->arch.css_support = 1;
4263 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
4264 trace_kvm_s390_enable_css(vcpu->kvm);
4265 }
4266 r = 0;
4267 break;
4268 default:
4269 r = -EINVAL;
4270 break;
4271 }
4272 return r;
4273 }
4274
4275 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
4276 struct kvm_s390_mem_op *mop)
4277 {
4278 void __user *uaddr = (void __user *)mop->buf;
4279 void *tmpbuf = NULL;
4280 int r, srcu_idx;
4281 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
4282 | KVM_S390_MEMOP_F_CHECK_ONLY;
4283
4284 if (mop->flags & ~supported_flags || mop->ar >= NUM_ACRS || !mop->size)
4285 return -EINVAL;
4286
4287 if (mop->size > MEM_OP_MAX_SIZE)
4288 return -E2BIG;
4289
4290 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
4291 tmpbuf = vmalloc(mop->size);
4292 if (!tmpbuf)
4293 return -ENOMEM;
4294 }
4295
4296 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
4297
4298 switch (mop->op) {
4299 case KVM_S390_MEMOP_LOGICAL_READ:
4300 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
4301 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
4302 mop->size, GACC_FETCH);
4303 break;
4304 }
4305 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
4306 if (r == 0) {
4307 if (copy_to_user(uaddr, tmpbuf, mop->size))
4308 r = -EFAULT;
4309 }
4310 break;
4311 case KVM_S390_MEMOP_LOGICAL_WRITE:
4312 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
4313 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
4314 mop->size, GACC_STORE);
4315 break;
4316 }
4317 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
4318 r = -EFAULT;
4319 break;
4320 }
4321 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
4322 break;
4323 default:
4324 r = -EINVAL;
4325 }
4326
4327 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
4328
4329 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
4330 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
4331
4332 vfree(tmpbuf);
4333 return r;
4334 }
4335
4336 long kvm_arch_vcpu_async_ioctl(struct file *filp,
4337 unsigned int ioctl, unsigned long arg)
4338 {
4339 struct kvm_vcpu *vcpu = filp->private_data;
4340 void __user *argp = (void __user *)arg;
4341
4342 switch (ioctl) {
4343 case KVM_S390_IRQ: {
4344 struct kvm_s390_irq s390irq;
4345
4346 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
4347 return -EFAULT;
4348 return kvm_s390_inject_vcpu(vcpu, &s390irq);
4349 }
4350 case KVM_S390_INTERRUPT: {
4351 struct kvm_s390_interrupt s390int;
4352 struct kvm_s390_irq s390irq = {};
4353
4354 if (copy_from_user(&s390int, argp, sizeof(s390int)))
4355 return -EFAULT;
4356 if (s390int_to_s390irq(&s390int, &s390irq))
4357 return -EINVAL;
4358 return kvm_s390_inject_vcpu(vcpu, &s390irq);
4359 }
4360 }
4361 return -ENOIOCTLCMD;
4362 }
4363
4364 long kvm_arch_vcpu_ioctl(struct file *filp,
4365 unsigned int ioctl, unsigned long arg)
4366 {
4367 struct kvm_vcpu *vcpu = filp->private_data;
4368 void __user *argp = (void __user *)arg;
4369 int idx;
4370 long r;
4371
4372 vcpu_load(vcpu);
4373
4374 switch (ioctl) {
4375 case KVM_S390_STORE_STATUS:
4376 idx = srcu_read_lock(&vcpu->kvm->srcu);
4377 r = kvm_s390_store_status_unloaded(vcpu, arg);
4378 srcu_read_unlock(&vcpu->kvm->srcu, idx);
4379 break;
4380 case KVM_S390_SET_INITIAL_PSW: {
4381 psw_t psw;
4382
4383 r = -EFAULT;
4384 if (copy_from_user(&psw, argp, sizeof(psw)))
4385 break;
4386 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
4387 break;
4388 }
4389 case KVM_S390_CLEAR_RESET:
4390 r = 0;
4391 kvm_arch_vcpu_ioctl_clear_reset(vcpu);
4392 break;
4393 case KVM_S390_INITIAL_RESET:
4394 r = 0;
4395 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
4396 break;
4397 case KVM_S390_NORMAL_RESET:
4398 r = 0;
4399 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
4400 break;
4401 case KVM_SET_ONE_REG:
4402 case KVM_GET_ONE_REG: {
4403 struct kvm_one_reg reg;
4404 r = -EFAULT;
4405 if (copy_from_user(&reg, argp, sizeof(reg)))
4406 break;
4407 if (ioctl == KVM_SET_ONE_REG)
4408 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
4409 else
4410 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
4411 break;
4412 }
4413 #ifdef CONFIG_KVM_S390_UCONTROL
4414 case KVM_S390_UCAS_MAP: {
4415 struct kvm_s390_ucas_mapping ucasmap;
4416
4417 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4418 r = -EFAULT;
4419 break;
4420 }
4421
4422 if (!kvm_is_ucontrol(vcpu->kvm)) {
4423 r = -EINVAL;
4424 break;
4425 }
4426
4427 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
4428 ucasmap.vcpu_addr, ucasmap.length);
4429 break;
4430 }
4431 case KVM_S390_UCAS_UNMAP: {
4432 struct kvm_s390_ucas_mapping ucasmap;
4433
4434 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4435 r = -EFAULT;
4436 break;
4437 }
4438
4439 if (!kvm_is_ucontrol(vcpu->kvm)) {
4440 r = -EINVAL;
4441 break;
4442 }
4443
4444 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
4445 ucasmap.length);
4446 break;
4447 }
4448 #endif
4449 case KVM_S390_VCPU_FAULT: {
4450 r = gmap_fault(vcpu->arch.gmap, arg, 0);
4451 break;
4452 }
4453 case KVM_ENABLE_CAP:
4454 {
4455 struct kvm_enable_cap cap;
4456 r = -EFAULT;
4457 if (copy_from_user(&cap, argp, sizeof(cap)))
4458 break;
4459 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
4460 break;
4461 }
4462 case KVM_S390_MEM_OP: {
4463 struct kvm_s390_mem_op mem_op;
4464
4465 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
4466 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
4467 else
4468 r = -EFAULT;
4469 break;
4470 }
4471 case KVM_S390_SET_IRQ_STATE: {
4472 struct kvm_s390_irq_state irq_state;
4473
4474 r = -EFAULT;
4475 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
4476 break;
4477 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
4478 irq_state.len == 0 ||
4479 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
4480 r = -EINVAL;
4481 break;
4482 }
4483 /* do not use irq_state.flags, it will break old QEMUs */
4484 r = kvm_s390_set_irq_state(vcpu,
4485 (void __user *) irq_state.buf,
4486 irq_state.len);
4487 break;
4488 }
4489 case KVM_S390_GET_IRQ_STATE: {
4490 struct kvm_s390_irq_state irq_state;
4491
4492 r = -EFAULT;
4493 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
4494 break;
4495 if (irq_state.len == 0) {
4496 r = -EINVAL;
4497 break;
4498 }
4499 /* do not use irq_state.flags, it will break old QEMUs */
4500 r = kvm_s390_get_irq_state(vcpu,
4501 (__u8 __user *) irq_state.buf,
4502 irq_state.len);
4503 break;
4504 }
4505 default:
4506 r = -ENOTTY;
4507 }
4508
4509 vcpu_put(vcpu);
4510 return r;
4511 }
4512
4513 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
4514 {
4515 #ifdef CONFIG_KVM_S390_UCONTROL
4516 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
4517 && (kvm_is_ucontrol(vcpu->kvm))) {
4518 vmf->page = virt_to_page(vcpu->arch.sie_block);
4519 get_page(vmf->page);
4520 return 0;
4521 }
4522 #endif
4523 return VM_FAULT_SIGBUS;
4524 }
4525
4526 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
4527 unsigned long npages)
4528 {
4529 return 0;
4530 }
4531
4532 /* Section: memory related */
4533 int kvm_arch_prepare_memory_region(struct kvm *kvm,
4534 struct kvm_memory_slot *memslot,
4535 const struct kvm_userspace_memory_region *mem,
4536 enum kvm_mr_change change)
4537 {
4538 /* A few sanity checks. We can have memory slots which have to be
4539 located/ended at a segment boundary (1MB). The memory in userland is
4540 ok to be fragmented into various different vmas. It is okay to mmap()
4541 and munmap() stuff in this slot after doing this call at any time */
4542
4543 if (mem->userspace_addr & 0xffffful)
4544 return -EINVAL;
4545
4546 if (mem->memory_size & 0xffffful)
4547 return -EINVAL;
4548
4549 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
4550 return -EINVAL;
4551
4552 return 0;
4553 }
4554
4555 void kvm_arch_commit_memory_region(struct kvm *kvm,
4556 const struct kvm_userspace_memory_region *mem,
4557 const struct kvm_memory_slot *old,
4558 const struct kvm_memory_slot *new,
4559 enum kvm_mr_change change)
4560 {
4561 int rc = 0;
4562
4563 switch (change) {
4564 case KVM_MR_DELETE:
4565 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
4566 old->npages * PAGE_SIZE);
4567 break;
4568 case KVM_MR_MOVE:
4569 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
4570 old->npages * PAGE_SIZE);
4571 if (rc)
4572 break;
4573 /* FALLTHROUGH */
4574 case KVM_MR_CREATE:
4575 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
4576 mem->guest_phys_addr, mem->memory_size);
4577 break;
4578 case KVM_MR_FLAGS_ONLY:
4579 break;
4580 default:
4581 WARN(1, "Unknown KVM MR CHANGE: %d\n", change);
4582 }
4583 if (rc)
4584 pr_warn("failed to commit memory region\n");
4585 return;
4586 }
4587
4588 static inline unsigned long nonhyp_mask(int i)
4589 {
4590 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
4591
4592 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
4593 }
4594
4595 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
4596 {
4597 vcpu->valid_wakeup = false;
4598 }
4599
4600 static int __init kvm_s390_init(void)
4601 {
4602 int i;
4603
4604 if (!sclp.has_sief2) {
4605 pr_info("SIE is not available\n");
4606 return -ENODEV;
4607 }
4608
4609 if (nested && hpage) {
4610 pr_info("A KVM host that supports nesting cannot back its KVM guests with huge pages\n");
4611 return -EINVAL;
4612 }
4613
4614 for (i = 0; i < 16; i++)
4615 kvm_s390_fac_base[i] |=
4616 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
4617
4618 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
4619 }
4620
4621 static void __exit kvm_s390_exit(void)
4622 {
4623 kvm_exit();
4624 }
4625
4626 module_init(kvm_s390_init);
4627 module_exit(kvm_s390_exit);
4628
4629 /*
4630 * Enable autoloading of the kvm module.
4631 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
4632 * since x86 takes a different approach.
4633 */
4634 #include <linux/miscdevice.h>
4635 MODULE_ALIAS_MISCDEV(KVM_MINOR);
4636 MODULE_ALIAS("devname:kvm");
Ubuntu Hirsute Linux kernel mirror