]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame_incremental - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-bionic-kernel.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
UBUNTU: Ubuntu-4.15.0-96.97
[mirror_ubuntu-bionic-kernel.git] / arch / s390 / kvm / kvm-s390.c
... / ...
CommitLineData
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * hosting IBM Z kernel virtual machines (s390x)
4 *
5 * Copyright IBM Corp. 2008, 2018
6 *
7 * Author(s): Carsten Otte <cotte@de.ibm.com>
8 * Christian Borntraeger <borntraeger@de.ibm.com>
9 * Heiko Carstens <heiko.carstens@de.ibm.com>
10 * Christian Ehrhardt <ehrhardt@de.ibm.com>
11 * Jason J. Herne <jjherne@us.ibm.com>
12 */
13
14#include <linux/compiler.h>
15#include <linux/err.h>
16#include <linux/fs.h>
17#include <linux/hrtimer.h>
18#include <linux/init.h>
19#include <linux/kvm.h>
20#include <linux/kvm_host.h>
21#include <linux/mman.h>
22#include <linux/module.h>
23#include <linux/moduleparam.h>
24#include <linux/random.h>
25#include <linux/slab.h>
26#include <linux/timer.h>
27#include <linux/vmalloc.h>
28#include <linux/bitmap.h>
29#include <linux/sched/signal.h>
30#include <linux/string.h>
31
32#include <asm/asm-offsets.h>
33#include <asm/lowcore.h>
34#include <asm/stp.h>
35#include <asm/pgtable.h>
36#include <asm/gmap.h>
37#include <asm/nmi.h>
38#include <asm/switch_to.h>
39#include <asm/isc.h>
40#include <asm/sclp.h>
41#include <asm/cpacf.h>
42#include <asm/timex.h>
43#include <asm/ap.h>
44#include "kvm-s390.h"
45#include "gaccess.h"
46
47#define KMSG_COMPONENT "kvm-s390"
48#undef pr_fmt
49#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
50
51#define CREATE_TRACE_POINTS
52#include "trace.h"
53#include "trace-s390.h"
54
55#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
56#define LOCAL_IRQS 32
57#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
58 (KVM_MAX_VCPUS + LOCAL_IRQS))
59
60#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
61
62struct kvm_stats_debugfs_item debugfs_entries[] = {
63 { "userspace_handled", VCPU_STAT(exit_userspace) },
64 { "exit_null", VCPU_STAT(exit_null) },
65 { "exit_validity", VCPU_STAT(exit_validity) },
66 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
67 { "exit_external_request", VCPU_STAT(exit_external_request) },
68 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
69 { "exit_instruction", VCPU_STAT(exit_instruction) },
70 { "exit_pei", VCPU_STAT(exit_pei) },
71 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
72 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
73 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
74 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
75 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
76 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
77 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
78 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
79 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
80 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
81 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
82 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
83 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
84 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
85 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
86 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
87 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
88 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
89 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
90 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
91 { "instruction_epsw", VCPU_STAT(instruction_epsw) },
92 { "instruction_gs", VCPU_STAT(instruction_gs) },
93 { "instruction_io_other", VCPU_STAT(instruction_io_other) },
94 { "instruction_lpsw", VCPU_STAT(instruction_lpsw) },
95 { "instruction_lpswe", VCPU_STAT(instruction_lpswe) },
96 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
97 { "instruction_ptff", VCPU_STAT(instruction_ptff) },
98 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
99 { "instruction_sck", VCPU_STAT(instruction_sck) },
100 { "instruction_sckpf", VCPU_STAT(instruction_sckpf) },
101 { "instruction_spx", VCPU_STAT(instruction_spx) },
102 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
103 { "instruction_stap", VCPU_STAT(instruction_stap) },
104 { "instruction_iske", VCPU_STAT(instruction_iske) },
105 { "instruction_ri", VCPU_STAT(instruction_ri) },
106 { "instruction_rrbe", VCPU_STAT(instruction_rrbe) },
107 { "instruction_sske", VCPU_STAT(instruction_sske) },
108 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
109 { "instruction_essa", VCPU_STAT(instruction_essa) },
110 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
111 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
112 { "instruction_tb", VCPU_STAT(instruction_tb) },
113 { "instruction_tpi", VCPU_STAT(instruction_tpi) },
114 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
115 { "instruction_tsch", VCPU_STAT(instruction_tsch) },
116 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
117 { "instruction_sie", VCPU_STAT(instruction_sie) },
118 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
119 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
120 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
121 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
122 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
123 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
124 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
125 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
126 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
127 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
128 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
129 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
130 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
131 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
132 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
133 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
134 { "instruction_diag_10", VCPU_STAT(diagnose_10) },
135 { "instruction_diag_44", VCPU_STAT(diagnose_44) },
136 { "instruction_diag_9c", VCPU_STAT(diagnose_9c) },
137 { "instruction_diag_258", VCPU_STAT(diagnose_258) },
138 { "instruction_diag_308", VCPU_STAT(diagnose_308) },
139 { "instruction_diag_500", VCPU_STAT(diagnose_500) },
140 { "instruction_diag_other", VCPU_STAT(diagnose_other) },
141 { NULL }
142};
143
144struct kvm_s390_tod_clock_ext {
145 __u8 epoch_idx;
146 __u64 tod;
147 __u8 reserved[7];
148} __packed;
149
150/* allow nested virtualization in KVM (if enabled by user space) */
151static int nested;
152module_param(nested, int, S_IRUGO);
153MODULE_PARM_DESC(nested, "Nested virtualization support");
154
155
156/*
157 * For now we handle at most 16 double words as this is what the s390 base
158 * kernel handles and stores in the prefix page. If we ever need to go beyond
159 * this, this requires changes to code, but the external uapi can stay.
160 */
161#define SIZE_INTERNAL 16
162
163/*
164 * Base feature mask that defines default mask for facilities. Consists of the
165 * defines in FACILITIES_KVM and the non-hypervisor managed bits.
166 */
167static unsigned long kvm_s390_fac_base[SIZE_INTERNAL] = { FACILITIES_KVM };
168/*
169 * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL
170 * and defines the facilities that can be enabled via a cpu model.
171 */
172static unsigned long kvm_s390_fac_ext[SIZE_INTERNAL] = { FACILITIES_KVM_CPUMODEL };
173
174static unsigned long kvm_s390_fac_size(void)
175{
176 BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_MASK_SIZE_U64);
177 BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_LIST_SIZE_U64);
178 BUILD_BUG_ON(SIZE_INTERNAL * sizeof(unsigned long) >
179 sizeof(S390_lowcore.stfle_fac_list));
180
181 return SIZE_INTERNAL;
182}
183
184/* available cpu features supported by kvm */
185static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
186/* available subfunctions indicated via query / "test bit" */
187static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
188
189static struct gmap_notifier gmap_notifier;
190static struct gmap_notifier vsie_gmap_notifier;
191debug_info_t *kvm_s390_dbf;
192
193/* Section: not file related */
194int kvm_arch_hardware_enable(void)
195{
196 /* every s390 is virtualization enabled ;-) */
197 return 0;
198}
199
200static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
201 unsigned long end);
202
203static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
204{
205 u8 delta_idx = 0;
206
207 /*
208 * The TOD jumps by delta, we have to compensate this by adding
209 * -delta to the epoch.
210 */
211 delta = -delta;
212
213 /* sign-extension - we're adding to signed values below */
214 if ((s64)delta < 0)
215 delta_idx = -1;
216
217 scb->epoch += delta;
218 if (scb->ecd & ECD_MEF) {
219 scb->epdx += delta_idx;
220 if (scb->epoch < delta)
221 scb->epdx += 1;
222 }
223}
224
225/*
226 * This callback is executed during stop_machine(). All CPUs are therefore
227 * temporarily stopped. In order not to change guest behavior, we have to
228 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
229 * so a CPU won't be stopped while calculating with the epoch.
230 */
231static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
232 void *v)
233{
234 struct kvm *kvm;
235 struct kvm_vcpu *vcpu;
236 int i;
237 unsigned long long *delta = v;
238
239 list_for_each_entry(kvm, &vm_list, vm_list) {
240 kvm_for_each_vcpu(i, vcpu, kvm) {
241 kvm_clock_sync_scb(vcpu->arch.sie_block, *delta);
242 if (i == 0) {
243 kvm->arch.epoch = vcpu->arch.sie_block->epoch;
244 kvm->arch.epdx = vcpu->arch.sie_block->epdx;
245 }
246 if (vcpu->arch.cputm_enabled)
247 vcpu->arch.cputm_start += *delta;
248 if (vcpu->arch.vsie_block)
249 kvm_clock_sync_scb(vcpu->arch.vsie_block,
250 *delta);
251 }
252 }
253 return NOTIFY_OK;
254}
255
256static struct notifier_block kvm_clock_notifier = {
257 .notifier_call = kvm_clock_sync,
258};
259
260int kvm_arch_hardware_setup(void)
261{
262 gmap_notifier.notifier_call = kvm_gmap_notifier;
263 gmap_register_pte_notifier(&gmap_notifier);
264 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
265 gmap_register_pte_notifier(&vsie_gmap_notifier);
266 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
267 &kvm_clock_notifier);
268 return 0;
269}
270
271void kvm_arch_hardware_unsetup(void)
272{
273 gmap_unregister_pte_notifier(&gmap_notifier);
274 gmap_unregister_pte_notifier(&vsie_gmap_notifier);
275 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
276 &kvm_clock_notifier);
277}
278
279static void allow_cpu_feat(unsigned long nr)
280{
281 set_bit_inv(nr, kvm_s390_available_cpu_feat);
282}
283
284static inline int plo_test_bit(unsigned char nr)
285{
286 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
287 int cc;
288
289 asm volatile(
290 /* Parameter registers are ignored for "test bit" */
291 " plo 0,0,0,0(0)\n"
292 " ipm %0\n"
293 " srl %0,28\n"
294 : "=d" (cc)
295 : "d" (r0)
296 : "cc");
297 return cc == 0;
298}
299
300static inline void __insn32_query(unsigned int opcode, u8 query[32])
301{
302 register unsigned long r0 asm("0") = 0; /* query function */
303 register unsigned long r1 asm("1") = (unsigned long) query;
304
305 asm volatile(
306 /* Parameter regs are ignored */
307 " .insn rrf,%[opc] << 16,2,4,6,0\n"
308 : "=m" (*query)
309 : "d" (r0), "a" (r1), [opc] "i" (opcode)
310 : "cc");
311}
312
313#define INSN_SORTL 0xb938
314#define INSN_DFLTCC 0xb939
315
316static void kvm_s390_cpu_feat_init(void)
317{
318 int i;
319
320 for (i = 0; i < 256; ++i) {
321 if (plo_test_bit(i))
322 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
323 }
324
325 if (test_facility(28)) /* TOD-clock steering */
326 ptff(kvm_s390_available_subfunc.ptff,
327 sizeof(kvm_s390_available_subfunc.ptff),
328 PTFF_QAF);
329
330 if (test_facility(17)) { /* MSA */
331 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
332 kvm_s390_available_subfunc.kmac);
333 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
334 kvm_s390_available_subfunc.kmc);
335 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
336 kvm_s390_available_subfunc.km);
337 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
338 kvm_s390_available_subfunc.kimd);
339 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
340 kvm_s390_available_subfunc.klmd);
341 }
342 if (test_facility(76)) /* MSA3 */
343 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
344 kvm_s390_available_subfunc.pckmo);
345 if (test_facility(77)) { /* MSA4 */
346 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
347 kvm_s390_available_subfunc.kmctr);
348 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
349 kvm_s390_available_subfunc.kmf);
350 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
351 kvm_s390_available_subfunc.kmo);
352 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
353 kvm_s390_available_subfunc.pcc);
354 }
355 if (test_facility(57)) /* MSA5 */
356 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
357 kvm_s390_available_subfunc.ppno);
358
359 if (test_facility(146)) /* MSA8 */
360 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
361 kvm_s390_available_subfunc.kma);
362
363 if (test_facility(155)) /* MSA9 */
364 __cpacf_query(CPACF_KDSA, (cpacf_mask_t *)
365 kvm_s390_available_subfunc.kdsa);
366
367 if (test_facility(150)) /* SORTL */
368 __insn32_query(INSN_SORTL, kvm_s390_available_subfunc.sortl);
369
370 if (test_facility(151)) /* DFLTCC */
371 __insn32_query(INSN_DFLTCC, kvm_s390_available_subfunc.dfltcc);
372
373 if (MACHINE_HAS_ESOP)
374 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
375 /*
376 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
377 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
378 */
379 if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
380 !test_facility(3) || !nested)
381 return;
382 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
383 if (sclp.has_64bscao)
384 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
385 if (sclp.has_siif)
386 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
387 if (sclp.has_gpere)
388 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
389 if (sclp.has_gsls)
390 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
391 if (sclp.has_ib)
392 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
393 if (sclp.has_cei)
394 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
395 if (sclp.has_ibs)
396 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
397 if (sclp.has_kss)
398 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
399 /*
400 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
401 * all skey handling functions read/set the skey from the PGSTE
402 * instead of the real storage key.
403 *
404 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
405 * pages being detected as preserved although they are resident.
406 *
407 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
408 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
409 *
410 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
411 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
412 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
413 *
414 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
415 * cannot easily shadow the SCA because of the ipte lock.
416 */
417}
418
419int kvm_arch_init(void *opaque)
420{
421 int rc;
422
423 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
424 if (!kvm_s390_dbf)
425 return -ENOMEM;
426
427 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
428 rc = -ENOMEM;
429 goto out_debug_unreg;
430 }
431
432 kvm_s390_cpu_feat_init();
433
434 /* Register floating interrupt controller interface. */
435 rc = kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
436 if (rc) {
437 pr_err("Failed to register FLIC rc=%d\n", rc);
438 goto out_debug_unreg;
439 }
440 return 0;
441
442out_debug_unreg:
443 debug_unregister(kvm_s390_dbf);
444 return rc;
445}
446
447void kvm_arch_exit(void)
448{
449 debug_unregister(kvm_s390_dbf);
450}
451
452/* Section: device related */
453long kvm_arch_dev_ioctl(struct file *filp,
454 unsigned int ioctl, unsigned long arg)
455{
456 if (ioctl == KVM_S390_ENABLE_SIE)
457 return s390_enable_sie();
458 return -EINVAL;
459}
460
461int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
462{
463 int r;
464
465 switch (ext) {
466 case KVM_CAP_S390_PSW:
467 case KVM_CAP_S390_GMAP:
468 case KVM_CAP_SYNC_MMU:
469#ifdef CONFIG_KVM_S390_UCONTROL
470 case KVM_CAP_S390_UCONTROL:
471#endif
472 case KVM_CAP_ASYNC_PF:
473 case KVM_CAP_SYNC_REGS:
474 case KVM_CAP_ONE_REG:
475 case KVM_CAP_ENABLE_CAP:
476 case KVM_CAP_S390_CSS_SUPPORT:
477 case KVM_CAP_IOEVENTFD:
478 case KVM_CAP_DEVICE_CTRL:
479 case KVM_CAP_ENABLE_CAP_VM:
480 case KVM_CAP_S390_IRQCHIP:
481 case KVM_CAP_VM_ATTRIBUTES:
482 case KVM_CAP_MP_STATE:
483 case KVM_CAP_IMMEDIATE_EXIT:
484 case KVM_CAP_S390_INJECT_IRQ:
485 case KVM_CAP_S390_USER_SIGP:
486 case KVM_CAP_S390_USER_STSI:
487 case KVM_CAP_S390_SKEYS:
488 case KVM_CAP_S390_IRQ_STATE:
489 case KVM_CAP_S390_USER_INSTR0:
490 case KVM_CAP_S390_CMMA_MIGRATION:
491 case KVM_CAP_S390_AIS:
492 case KVM_CAP_S390_AIS_MIGRATION:
493 r = 1;
494 break;
495 case KVM_CAP_S390_MEM_OP:
496 r = MEM_OP_MAX_SIZE;
497 break;
498 case KVM_CAP_NR_VCPUS:
499 case KVM_CAP_MAX_VCPUS:
500 case KVM_CAP_MAX_VCPU_ID:
501 r = KVM_S390_BSCA_CPU_SLOTS;
502 if (!kvm_s390_use_sca_entries())
503 r = KVM_MAX_VCPUS;
504 else if (sclp.has_esca && sclp.has_64bscao)
505 r = KVM_S390_ESCA_CPU_SLOTS;
506 break;
507 case KVM_CAP_NR_MEMSLOTS:
508 r = KVM_USER_MEM_SLOTS;
509 break;
510 case KVM_CAP_S390_COW:
511 r = MACHINE_HAS_ESOP;
512 break;
513 case KVM_CAP_S390_VECTOR_REGISTERS:
514 r = MACHINE_HAS_VX;
515 break;
516 case KVM_CAP_S390_RI:
517 r = test_facility(64);
518 break;
519 case KVM_CAP_S390_GS:
520 r = test_facility(133);
521 break;
522 case KVM_CAP_S390_BPB:
523 r = test_facility(82);
524 break;
525 default:
526 r = 0;
527 }
528 return r;
529}
530
531static void kvm_s390_sync_dirty_log(struct kvm *kvm,
532 struct kvm_memory_slot *memslot)
533{
534 gfn_t cur_gfn, last_gfn;
535 unsigned long address;
536 struct gmap *gmap = kvm->arch.gmap;
537
538 /* Loop over all guest pages */
539 last_gfn = memslot->base_gfn + memslot->npages;
540 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
541 address = gfn_to_hva_memslot(memslot, cur_gfn);
542
543 if (test_and_clear_guest_dirty(gmap->mm, address))
544 mark_page_dirty(kvm, cur_gfn);
545 if (fatal_signal_pending(current))
546 return;
547 cond_resched();
548 }
549}
550
551/* Section: vm related */
552static void sca_del_vcpu(struct kvm_vcpu *vcpu);
553
554/*
555 * Get (and clear) the dirty memory log for a memory slot.
556 */
557int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
558 struct kvm_dirty_log *log)
559{
560 int r;
561 unsigned long n;
562 struct kvm_memslots *slots;
563 struct kvm_memory_slot *memslot;
564 int is_dirty = 0;
565
566 if (kvm_is_ucontrol(kvm))
567 return -EINVAL;
568
569 mutex_lock(&kvm->slots_lock);
570
571 r = -EINVAL;
572 if (log->slot >= KVM_USER_MEM_SLOTS)
573 goto out;
574
575 slots = kvm_memslots(kvm);
576 memslot = id_to_memslot(slots, log->slot);
577 r = -ENOENT;
578 if (!memslot->dirty_bitmap)
579 goto out;
580
581 kvm_s390_sync_dirty_log(kvm, memslot);
582 r = kvm_get_dirty_log(kvm, log, &is_dirty);
583 if (r)
584 goto out;
585
586 /* Clear the dirty log */
587 if (is_dirty) {
588 n = kvm_dirty_bitmap_bytes(memslot);
589 memset(memslot->dirty_bitmap, 0, n);
590 }
591 r = 0;
592out:
593 mutex_unlock(&kvm->slots_lock);
594 return r;
595}
596
597static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
598{
599 unsigned int i;
600 struct kvm_vcpu *vcpu;
601
602 kvm_for_each_vcpu(i, vcpu, kvm) {
603 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
604 }
605}
606
607static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
608{
609 int r;
610
611 if (cap->flags)
612 return -EINVAL;
613
614 switch (cap->cap) {
615 case KVM_CAP_S390_IRQCHIP:
616 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
617 kvm->arch.use_irqchip = 1;
618 r = 0;
619 break;
620 case KVM_CAP_S390_USER_SIGP:
621 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
622 kvm->arch.user_sigp = 1;
623 r = 0;
624 break;
625 case KVM_CAP_S390_VECTOR_REGISTERS:
626 mutex_lock(&kvm->lock);
627 if (kvm->created_vcpus) {
628 r = -EBUSY;
629 } else if (MACHINE_HAS_VX) {
630 set_kvm_facility(kvm->arch.model.fac_mask, 129);
631 set_kvm_facility(kvm->arch.model.fac_list, 129);
632 if (test_facility(134)) {
633 set_kvm_facility(kvm->arch.model.fac_mask, 134);
634 set_kvm_facility(kvm->arch.model.fac_list, 134);
635 }
636 if (test_facility(135)) {
637 set_kvm_facility(kvm->arch.model.fac_mask, 135);
638 set_kvm_facility(kvm->arch.model.fac_list, 135);
639 }
640 if (test_facility(148)) {
641 set_kvm_facility(kvm->arch.model.fac_mask, 148);
642 set_kvm_facility(kvm->arch.model.fac_list, 148);
643 }
644 if (test_facility(152)) {
645 set_kvm_facility(kvm->arch.model.fac_mask, 152);
646 set_kvm_facility(kvm->arch.model.fac_list, 152);
647 }
648 r = 0;
649 } else
650 r = -EINVAL;
651 mutex_unlock(&kvm->lock);
652 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
653 r ? "(not available)" : "(success)");
654 break;
655 case KVM_CAP_S390_RI:
656 r = -EINVAL;
657 mutex_lock(&kvm->lock);
658 if (kvm->created_vcpus) {
659 r = -EBUSY;
660 } else if (test_facility(64)) {
661 set_kvm_facility(kvm->arch.model.fac_mask, 64);
662 set_kvm_facility(kvm->arch.model.fac_list, 64);
663 r = 0;
664 }
665 mutex_unlock(&kvm->lock);
666 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
667 r ? "(not available)" : "(success)");
668 break;
669 case KVM_CAP_S390_AIS:
670 mutex_lock(&kvm->lock);
671 if (kvm->created_vcpus) {
672 r = -EBUSY;
673 } else {
674 set_kvm_facility(kvm->arch.model.fac_mask, 72);
675 set_kvm_facility(kvm->arch.model.fac_list, 72);
676 r = 0;
677 }
678 mutex_unlock(&kvm->lock);
679 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
680 r ? "(not available)" : "(success)");
681 break;
682 case KVM_CAP_S390_GS:
683 r = -EINVAL;
684 mutex_lock(&kvm->lock);
685 if (kvm->created_vcpus) {
686 r = -EBUSY;
687 } else if (test_facility(133)) {
688 set_kvm_facility(kvm->arch.model.fac_mask, 133);
689 set_kvm_facility(kvm->arch.model.fac_list, 133);
690 r = 0;
691 }
692 mutex_unlock(&kvm->lock);
693 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
694 r ? "(not available)" : "(success)");
695 break;
696 case KVM_CAP_S390_USER_STSI:
697 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
698 kvm->arch.user_stsi = 1;
699 r = 0;
700 break;
701 case KVM_CAP_S390_USER_INSTR0:
702 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
703 kvm->arch.user_instr0 = 1;
704 icpt_operexc_on_all_vcpus(kvm);
705 r = 0;
706 break;
707 default:
708 r = -EINVAL;
709 break;
710 }
711 return r;
712}
713
714static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
715{
716 int ret;
717
718 switch (attr->attr) {
719 case KVM_S390_VM_MEM_LIMIT_SIZE:
720 ret = 0;
721 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
722 kvm->arch.mem_limit);
723 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
724 ret = -EFAULT;
725 break;
726 default:
727 ret = -ENXIO;
728 break;
729 }
730 return ret;
731}
732
733static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
734{
735 int ret;
736 unsigned int idx;
737 switch (attr->attr) {
738 case KVM_S390_VM_MEM_ENABLE_CMMA:
739 ret = -ENXIO;
740 if (!sclp.has_cmma)
741 break;
742
743 ret = -EBUSY;
744 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
745 mutex_lock(&kvm->lock);
746 if (!kvm->created_vcpus) {
747 kvm->arch.use_cmma = 1;
748 ret = 0;
749 }
750 mutex_unlock(&kvm->lock);
751 break;
752 case KVM_S390_VM_MEM_CLR_CMMA:
753 ret = -ENXIO;
754 if (!sclp.has_cmma)
755 break;
756 ret = -EINVAL;
757 if (!kvm->arch.use_cmma)
758 break;
759
760 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
761 mutex_lock(&kvm->lock);
762 idx = srcu_read_lock(&kvm->srcu);
763 s390_reset_cmma(kvm->arch.gmap->mm);
764 srcu_read_unlock(&kvm->srcu, idx);
765 mutex_unlock(&kvm->lock);
766 ret = 0;
767 break;
768 case KVM_S390_VM_MEM_LIMIT_SIZE: {
769 unsigned long new_limit;
770
771 if (kvm_is_ucontrol(kvm))
772 return -EINVAL;
773
774 if (get_user(new_limit, (u64 __user *)attr->addr))
775 return -EFAULT;
776
777 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
778 new_limit > kvm->arch.mem_limit)
779 return -E2BIG;
780
781 if (!new_limit)
782 return -EINVAL;
783
784 /* gmap_create takes last usable address */
785 if (new_limit != KVM_S390_NO_MEM_LIMIT)
786 new_limit -= 1;
787
788 ret = -EBUSY;
789 mutex_lock(&kvm->lock);
790 if (!kvm->created_vcpus) {
791 /* gmap_create will round the limit up */
792 struct gmap *new = gmap_create(current->mm, new_limit);
793
794 if (!new) {
795 ret = -ENOMEM;
796 } else {
797 gmap_remove(kvm->arch.gmap);
798 new->private = kvm;
799 kvm->arch.gmap = new;
800 ret = 0;
801 }
802 }
803 mutex_unlock(&kvm->lock);
804 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
805 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
806 (void *) kvm->arch.gmap->asce);
807 break;
808 }
809 default:
810 ret = -ENXIO;
811 break;
812 }
813 return ret;
814}
815
816static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
817
818void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
819{
820 struct kvm_vcpu *vcpu;
821 int i;
822
823 kvm_s390_vcpu_block_all(kvm);
824
825 kvm_for_each_vcpu(i, vcpu, kvm) {
826 kvm_s390_vcpu_crypto_setup(vcpu);
827 /* recreate the shadow crycb by leaving the VSIE handler */
828 kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
829 }
830
831 kvm_s390_vcpu_unblock_all(kvm);
832}
833
834static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
835{
836 mutex_lock(&kvm->lock);
837 switch (attr->attr) {
838 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
839 if (!test_kvm_facility(kvm, 76)) {
840 mutex_unlock(&kvm->lock);
841 return -EINVAL;
842 }
843 get_random_bytes(
844 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
845 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
846 kvm->arch.crypto.aes_kw = 1;
847 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
848 break;
849 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
850 if (!test_kvm_facility(kvm, 76)) {
851 mutex_unlock(&kvm->lock);
852 return -EINVAL;
853 }
854 get_random_bytes(
855 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
856 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
857 kvm->arch.crypto.dea_kw = 1;
858 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
859 break;
860 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
861 if (!test_kvm_facility(kvm, 76)) {
862 mutex_unlock(&kvm->lock);
863 return -EINVAL;
864 }
865 kvm->arch.crypto.aes_kw = 0;
866 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
867 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
868 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
869 break;
870 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
871 if (!test_kvm_facility(kvm, 76)) {
872 mutex_unlock(&kvm->lock);
873 return -EINVAL;
874 }
875 kvm->arch.crypto.dea_kw = 0;
876 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
877 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
878 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
879 break;
880 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
881 if (!ap_instructions_available()) {
882 mutex_unlock(&kvm->lock);
883 return -EOPNOTSUPP;
884 }
885 kvm->arch.crypto.apie = 1;
886 break;
887 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
888 if (!ap_instructions_available()) {
889 mutex_unlock(&kvm->lock);
890 return -EOPNOTSUPP;
891 }
892 kvm->arch.crypto.apie = 0;
893 break;
894 default:
895 mutex_unlock(&kvm->lock);
896 return -ENXIO;
897 }
898
899 kvm_s390_vcpu_crypto_reset_all(kvm);
900 mutex_unlock(&kvm->lock);
901 return 0;
902}
903
904static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
905{
906 int cx;
907 struct kvm_vcpu *vcpu;
908
909 kvm_for_each_vcpu(cx, vcpu, kvm)
910 kvm_s390_sync_request(req, vcpu);
911}
912
913/*
914 * Must be called with kvm->srcu held to avoid races on memslots, and with
915 * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
916 */
917static int kvm_s390_vm_start_migration(struct kvm *kvm)
918{
919 struct kvm_s390_migration_state *mgs;
920 struct kvm_memory_slot *ms;
921 /* should be the only one */
922 struct kvm_memslots *slots;
923 unsigned long ram_pages;
924 int slotnr;
925
926 /* migration mode already enabled */
927 if (kvm->arch.migration_state)
928 return 0;
929
930 slots = kvm_memslots(kvm);
931 if (!slots || !slots->used_slots)
932 return -EINVAL;
933
934 mgs = kzalloc(sizeof(*mgs), GFP_KERNEL);
935 if (!mgs)
936 return -ENOMEM;
937 kvm->arch.migration_state = mgs;
938
939 if (kvm->arch.use_cmma) {
940 /*
941 * Get the first slot. They are reverse sorted by base_gfn, so
942 * the first slot is also the one at the end of the address
943 * space. We have verified above that at least one slot is
944 * present.
945 */
946 ms = slots->memslots;
947 /* round up so we only use full longs */
948 ram_pages = roundup(ms->base_gfn + ms->npages, BITS_PER_LONG);
949 /* allocate enough bytes to store all the bits */
950 mgs->pgste_bitmap = vmalloc(ram_pages / 8);
951 if (!mgs->pgste_bitmap) {
952 kfree(mgs);
953 kvm->arch.migration_state = NULL;
954 return -ENOMEM;
955 }
956
957 mgs->bitmap_size = ram_pages;
958 atomic64_set(&mgs->dirty_pages, ram_pages);
959 /* mark all the pages in active slots as dirty */
960 for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
961 ms = slots->memslots + slotnr;
962 bitmap_set(mgs->pgste_bitmap, ms->base_gfn, ms->npages);
963 }
964
965 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
966 }
967 return 0;
968}
969
970/*
971 * Must be called with kvm->slots_lock to avoid races with ourselves and
972 * kvm_s390_vm_start_migration.
973 */
974static int kvm_s390_vm_stop_migration(struct kvm *kvm)
975{
976 struct kvm_s390_migration_state *mgs;
977
978 /* migration mode already disabled */
979 if (!kvm->arch.migration_state)
980 return 0;
981 mgs = kvm->arch.migration_state;
982 kvm->arch.migration_state = NULL;
983
984 if (kvm->arch.use_cmma) {
985 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
986 /* We have to wait for the essa emulation to finish */
987 synchronize_srcu(&kvm->srcu);
988 vfree(mgs->pgste_bitmap);
989 }
990 kfree(mgs);
991 return 0;
992}
993
994static int kvm_s390_vm_set_migration(struct kvm *kvm,
995 struct kvm_device_attr *attr)
996{
997 int res = -ENXIO;
998
999 mutex_lock(&kvm->slots_lock);
1000 switch (attr->attr) {
1001 case KVM_S390_VM_MIGRATION_START:
1002 res = kvm_s390_vm_start_migration(kvm);
1003 break;
1004 case KVM_S390_VM_MIGRATION_STOP:
1005 res = kvm_s390_vm_stop_migration(kvm);
1006 break;
1007 default:
1008 break;
1009 }
1010 mutex_unlock(&kvm->slots_lock);
1011
1012 return res;
1013}
1014
1015static int kvm_s390_vm_get_migration(struct kvm *kvm,
1016 struct kvm_device_attr *attr)
1017{
1018 u64 mig = (kvm->arch.migration_state != NULL);
1019
1020 if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
1021 return -ENXIO;
1022
1023 if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
1024 return -EFAULT;
1025 return 0;
1026}
1027
1028static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1029{
1030 struct kvm_s390_vm_tod_clock gtod;
1031
1032 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
1033 return -EFAULT;
1034
1035 if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
1036 return -EINVAL;
1037 kvm_s390_set_tod_clock(kvm, &gtod);
1038
1039 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
1040 gtod.epoch_idx, gtod.tod);
1041
1042 return 0;
1043}
1044
1045static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1046{
1047 u8 gtod_high;
1048
1049 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
1050 sizeof(gtod_high)))
1051 return -EFAULT;
1052
1053 if (gtod_high != 0)
1054 return -EINVAL;
1055 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
1056
1057 return 0;
1058}
1059
1060static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1061{
1062 struct kvm_s390_vm_tod_clock gtod = { 0 };
1063
1064 if (copy_from_user(&gtod.tod, (void __user *)attr->addr,
1065 sizeof(gtod.tod)))
1066 return -EFAULT;
1067
1068 kvm_s390_set_tod_clock(kvm, &gtod);
1069 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
1070 return 0;
1071}
1072
1073static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1074{
1075 int ret;
1076
1077 if (attr->flags)
1078 return -EINVAL;
1079
1080 switch (attr->attr) {
1081 case KVM_S390_VM_TOD_EXT:
1082 ret = kvm_s390_set_tod_ext(kvm, attr);
1083 break;
1084 case KVM_S390_VM_TOD_HIGH:
1085 ret = kvm_s390_set_tod_high(kvm, attr);
1086 break;
1087 case KVM_S390_VM_TOD_LOW:
1088 ret = kvm_s390_set_tod_low(kvm, attr);
1089 break;
1090 default:
1091 ret = -ENXIO;
1092 break;
1093 }
1094 return ret;
1095}
1096
1097static void kvm_s390_get_tod_clock_ext(struct kvm *kvm,
1098 struct kvm_s390_vm_tod_clock *gtod)
1099{
1100 struct kvm_s390_tod_clock_ext htod;
1101
1102 preempt_disable();
1103
1104 get_tod_clock_ext((char *)&htod);
1105
1106 gtod->tod = htod.tod + kvm->arch.epoch;
1107 gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
1108
1109 if (gtod->tod < htod.tod)
1110 gtod->epoch_idx += 1;
1111
1112 preempt_enable();
1113}
1114
1115static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1116{
1117 struct kvm_s390_vm_tod_clock gtod;
1118
1119 memset(&gtod, 0, sizeof(gtod));
1120
1121 if (test_kvm_facility(kvm, 139))
1122 kvm_s390_get_tod_clock_ext(kvm, &gtod);
1123 else
1124 gtod.tod = kvm_s390_get_tod_clock_fast(kvm);
1125
1126 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1127 return -EFAULT;
1128
1129 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
1130 gtod.epoch_idx, gtod.tod);
1131 return 0;
1132}
1133
1134static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1135{
1136 u8 gtod_high = 0;
1137
1138 if (copy_to_user((void __user *)attr->addr, &gtod_high,
1139 sizeof(gtod_high)))
1140 return -EFAULT;
1141 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
1142
1143 return 0;
1144}
1145
1146static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1147{
1148 u64 gtod;
1149
1150 gtod = kvm_s390_get_tod_clock_fast(kvm);
1151 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1152 return -EFAULT;
1153 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
1154
1155 return 0;
1156}
1157
1158static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1159{
1160 int ret;
1161
1162 if (attr->flags)
1163 return -EINVAL;
1164
1165 switch (attr->attr) {
1166 case KVM_S390_VM_TOD_EXT:
1167 ret = kvm_s390_get_tod_ext(kvm, attr);
1168 break;
1169 case KVM_S390_VM_TOD_HIGH:
1170 ret = kvm_s390_get_tod_high(kvm, attr);
1171 break;
1172 case KVM_S390_VM_TOD_LOW:
1173 ret = kvm_s390_get_tod_low(kvm, attr);
1174 break;
1175 default:
1176 ret = -ENXIO;
1177 break;
1178 }
1179 return ret;
1180}
1181
1182static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1183{
1184 struct kvm_s390_vm_cpu_processor *proc;
1185 u16 lowest_ibc, unblocked_ibc;
1186 int ret = 0;
1187
1188 mutex_lock(&kvm->lock);
1189 if (kvm->created_vcpus) {
1190 ret = -EBUSY;
1191 goto out;
1192 }
1193 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1194 if (!proc) {
1195 ret = -ENOMEM;
1196 goto out;
1197 }
1198 if (!copy_from_user(proc, (void __user *)attr->addr,
1199 sizeof(*proc))) {
1200 kvm->arch.model.cpuid = proc->cpuid;
1201 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1202 unblocked_ibc = sclp.ibc & 0xfff;
1203 if (lowest_ibc && proc->ibc) {
1204 if (proc->ibc > unblocked_ibc)
1205 kvm->arch.model.ibc = unblocked_ibc;
1206 else if (proc->ibc < lowest_ibc)
1207 kvm->arch.model.ibc = lowest_ibc;
1208 else
1209 kvm->arch.model.ibc = proc->ibc;
1210 }
1211 memcpy(kvm->arch.model.fac_list, proc->fac_list,
1212 S390_ARCH_FAC_LIST_SIZE_BYTE);
1213 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1214 kvm->arch.model.ibc,
1215 kvm->arch.model.cpuid);
1216 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1217 kvm->arch.model.fac_list[0],
1218 kvm->arch.model.fac_list[1],
1219 kvm->arch.model.fac_list[2]);
1220 } else
1221 ret = -EFAULT;
1222 kfree(proc);
1223out:
1224 mutex_unlock(&kvm->lock);
1225 return ret;
1226}
1227
1228static int kvm_s390_set_processor_feat(struct kvm *kvm,
1229 struct kvm_device_attr *attr)
1230{
1231 struct kvm_s390_vm_cpu_feat data;
1232 int ret = -EBUSY;
1233
1234 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1235 return -EFAULT;
1236 if (!bitmap_subset((unsigned long *) data.feat,
1237 kvm_s390_available_cpu_feat,
1238 KVM_S390_VM_CPU_FEAT_NR_BITS))
1239 return -EINVAL;
1240
1241 mutex_lock(&kvm->lock);
1242 if (!kvm->created_vcpus) {
1243 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
1244 KVM_S390_VM_CPU_FEAT_NR_BITS);
1245 ret = 0;
1246 }
1247 mutex_unlock(&kvm->lock);
1248 return ret;
1249}
1250
1251static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1252 struct kvm_device_attr *attr)
1253{
1254 mutex_lock(&kvm->lock);
1255 if (kvm->created_vcpus) {
1256 mutex_unlock(&kvm->lock);
1257 return -EBUSY;
1258 }
1259
1260 if (copy_from_user(&kvm->arch.model.subfuncs, (void __user *)attr->addr,
1261 sizeof(struct kvm_s390_vm_cpu_subfunc))) {
1262 mutex_unlock(&kvm->lock);
1263 return -EFAULT;
1264 }
1265 mutex_unlock(&kvm->lock);
1266
1267 VM_EVENT(kvm, 3, "SET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1268 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1269 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1270 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1271 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1272 VM_EVENT(kvm, 3, "SET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1273 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1274 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1275 VM_EVENT(kvm, 3, "SET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1276 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1277 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1278 VM_EVENT(kvm, 3, "SET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1279 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1280 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1281 VM_EVENT(kvm, 3, "SET: guest KM subfunc 0x%16.16lx.%16.16lx",
1282 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1283 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1284 VM_EVENT(kvm, 3, "SET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1285 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1286 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1287 VM_EVENT(kvm, 3, "SET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1288 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1289 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1290 VM_EVENT(kvm, 3, "SET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1291 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1292 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1293 VM_EVENT(kvm, 3, "SET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1294 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1295 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1296 VM_EVENT(kvm, 3, "SET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1297 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1298 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1299 VM_EVENT(kvm, 3, "SET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1300 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1301 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1302 VM_EVENT(kvm, 3, "SET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1303 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1304 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1305 VM_EVENT(kvm, 3, "SET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1306 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1307 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1308 VM_EVENT(kvm, 3, "SET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1309 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1310 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1311 VM_EVENT(kvm, 3, "SET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1312 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1313 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1314 VM_EVENT(kvm, 3, "SET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1315 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1316 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1317 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1318 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1319 VM_EVENT(kvm, 3, "SET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1320 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1321 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1322 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1323 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1324
1325 return 0;
1326}
1327
1328static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1329{
1330 int ret = -ENXIO;
1331
1332 switch (attr->attr) {
1333 case KVM_S390_VM_CPU_PROCESSOR:
1334 ret = kvm_s390_set_processor(kvm, attr);
1335 break;
1336 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1337 ret = kvm_s390_set_processor_feat(kvm, attr);
1338 break;
1339 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1340 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1341 break;
1342 }
1343 return ret;
1344}
1345
1346static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1347{
1348 struct kvm_s390_vm_cpu_processor *proc;
1349 int ret = 0;
1350
1351 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1352 if (!proc) {
1353 ret = -ENOMEM;
1354 goto out;
1355 }
1356 proc->cpuid = kvm->arch.model.cpuid;
1357 proc->ibc = kvm->arch.model.ibc;
1358 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1359 S390_ARCH_FAC_LIST_SIZE_BYTE);
1360 VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1361 kvm->arch.model.ibc,
1362 kvm->arch.model.cpuid);
1363 VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1364 kvm->arch.model.fac_list[0],
1365 kvm->arch.model.fac_list[1],
1366 kvm->arch.model.fac_list[2]);
1367 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1368 ret = -EFAULT;
1369 kfree(proc);
1370out:
1371 return ret;
1372}
1373
1374static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1375{
1376 struct kvm_s390_vm_cpu_machine *mach;
1377 int ret = 0;
1378
1379 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
1380 if (!mach) {
1381 ret = -ENOMEM;
1382 goto out;
1383 }
1384 get_cpu_id((struct cpuid *) &mach->cpuid);
1385 mach->ibc = sclp.ibc;
1386 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
1387 S390_ARCH_FAC_LIST_SIZE_BYTE);
1388 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
1389 sizeof(S390_lowcore.stfle_fac_list));
1390 VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx",
1391 kvm->arch.model.ibc,
1392 kvm->arch.model.cpuid);
1393 VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx",
1394 mach->fac_mask[0],
1395 mach->fac_mask[1],
1396 mach->fac_mask[2]);
1397 VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1398 mach->fac_list[0],
1399 mach->fac_list[1],
1400 mach->fac_list[2]);
1401 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1402 ret = -EFAULT;
1403 kfree(mach);
1404out:
1405 return ret;
1406}
1407
1408static int kvm_s390_get_processor_feat(struct kvm *kvm,
1409 struct kvm_device_attr *attr)
1410{
1411 struct kvm_s390_vm_cpu_feat data;
1412
1413 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1414 KVM_S390_VM_CPU_FEAT_NR_BITS);
1415 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1416 return -EFAULT;
1417 return 0;
1418}
1419
1420static int kvm_s390_get_machine_feat(struct kvm *kvm,
1421 struct kvm_device_attr *attr)
1422{
1423 struct kvm_s390_vm_cpu_feat data;
1424
1425 bitmap_copy((unsigned long *) data.feat,
1426 kvm_s390_available_cpu_feat,
1427 KVM_S390_VM_CPU_FEAT_NR_BITS);
1428 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1429 return -EFAULT;
1430 return 0;
1431}
1432
1433static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1434 struct kvm_device_attr *attr)
1435{
1436 if (copy_to_user((void __user *)attr->addr, &kvm->arch.model.subfuncs,
1437 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1438 return -EFAULT;
1439
1440 VM_EVENT(kvm, 3, "GET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1441 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1442 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1443 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1444 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1445 VM_EVENT(kvm, 3, "GET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1446 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1447 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1448 VM_EVENT(kvm, 3, "GET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1449 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1450 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1451 VM_EVENT(kvm, 3, "GET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1452 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1453 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1454 VM_EVENT(kvm, 3, "GET: guest KM subfunc 0x%16.16lx.%16.16lx",
1455 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1456 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1457 VM_EVENT(kvm, 3, "GET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1458 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1459 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1460 VM_EVENT(kvm, 3, "GET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1461 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1462 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1463 VM_EVENT(kvm, 3, "GET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1464 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1465 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1466 VM_EVENT(kvm, 3, "GET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1467 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1468 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1469 VM_EVENT(kvm, 3, "GET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1470 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1471 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1472 VM_EVENT(kvm, 3, "GET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1473 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1474 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1475 VM_EVENT(kvm, 3, "GET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1476 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1477 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1478 VM_EVENT(kvm, 3, "GET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1479 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1480 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1481 VM_EVENT(kvm, 3, "GET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1482 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1483 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1484 VM_EVENT(kvm, 3, "GET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1485 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1486 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1487 VM_EVENT(kvm, 3, "GET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1488 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1489 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1490 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1491 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1492 VM_EVENT(kvm, 3, "GET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1493 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1494 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1495 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1496 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1497
1498 return 0;
1499}
1500
1501static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1502 struct kvm_device_attr *attr)
1503{
1504 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1505 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1506 return -EFAULT;
1507
1508 VM_EVENT(kvm, 3, "GET: host PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1509 ((unsigned long *) &kvm_s390_available_subfunc.plo)[0],
1510 ((unsigned long *) &kvm_s390_available_subfunc.plo)[1],
1511 ((unsigned long *) &kvm_s390_available_subfunc.plo)[2],
1512 ((unsigned long *) &kvm_s390_available_subfunc.plo)[3]);
1513 VM_EVENT(kvm, 3, "GET: host PTFF subfunc 0x%16.16lx.%16.16lx",
1514 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[0],
1515 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[1]);
1516 VM_EVENT(kvm, 3, "GET: host KMAC subfunc 0x%16.16lx.%16.16lx",
1517 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[0],
1518 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[1]);
1519 VM_EVENT(kvm, 3, "GET: host KMC subfunc 0x%16.16lx.%16.16lx",
1520 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[0],
1521 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[1]);
1522 VM_EVENT(kvm, 3, "GET: host KM subfunc 0x%16.16lx.%16.16lx",
1523 ((unsigned long *) &kvm_s390_available_subfunc.km)[0],
1524 ((unsigned long *) &kvm_s390_available_subfunc.km)[1]);
1525 VM_EVENT(kvm, 3, "GET: host KIMD subfunc 0x%16.16lx.%16.16lx",
1526 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[0],
1527 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[1]);
1528 VM_EVENT(kvm, 3, "GET: host KLMD subfunc 0x%16.16lx.%16.16lx",
1529 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[0],
1530 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[1]);
1531 VM_EVENT(kvm, 3, "GET: host PCKMO subfunc 0x%16.16lx.%16.16lx",
1532 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[0],
1533 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[1]);
1534 VM_EVENT(kvm, 3, "GET: host KMCTR subfunc 0x%16.16lx.%16.16lx",
1535 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[0],
1536 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[1]);
1537 VM_EVENT(kvm, 3, "GET: host KMF subfunc 0x%16.16lx.%16.16lx",
1538 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[0],
1539 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[1]);
1540 VM_EVENT(kvm, 3, "GET: host KMO subfunc 0x%16.16lx.%16.16lx",
1541 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[0],
1542 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[1]);
1543 VM_EVENT(kvm, 3, "GET: host PCC subfunc 0x%16.16lx.%16.16lx",
1544 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[0],
1545 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[1]);
1546 VM_EVENT(kvm, 3, "GET: host PPNO subfunc 0x%16.16lx.%16.16lx",
1547 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[0],
1548 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[1]);
1549 VM_EVENT(kvm, 3, "GET: host KMA subfunc 0x%16.16lx.%16.16lx",
1550 ((unsigned long *) &kvm_s390_available_subfunc.kma)[0],
1551 ((unsigned long *) &kvm_s390_available_subfunc.kma)[1]);
1552 VM_EVENT(kvm, 3, "GET: host KDSA subfunc 0x%16.16lx.%16.16lx",
1553 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[0],
1554 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[1]);
1555 VM_EVENT(kvm, 3, "GET: host SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1556 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[0],
1557 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[1],
1558 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[2],
1559 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[3]);
1560 VM_EVENT(kvm, 3, "GET: host DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1561 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[0],
1562 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[1],
1563 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[2],
1564 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[3]);
1565
1566 return 0;
1567}
1568
1569static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1570{
1571 int ret = -ENXIO;
1572
1573 switch (attr->attr) {
1574 case KVM_S390_VM_CPU_PROCESSOR:
1575 ret = kvm_s390_get_processor(kvm, attr);
1576 break;
1577 case KVM_S390_VM_CPU_MACHINE:
1578 ret = kvm_s390_get_machine(kvm, attr);
1579 break;
1580 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1581 ret = kvm_s390_get_processor_feat(kvm, attr);
1582 break;
1583 case KVM_S390_VM_CPU_MACHINE_FEAT:
1584 ret = kvm_s390_get_machine_feat(kvm, attr);
1585 break;
1586 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1587 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1588 break;
1589 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1590 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1591 break;
1592 }
1593 return ret;
1594}
1595
1596static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1597{
1598 int ret;
1599
1600 switch (attr->group) {
1601 case KVM_S390_VM_MEM_CTRL:
1602 ret = kvm_s390_set_mem_control(kvm, attr);
1603 break;
1604 case KVM_S390_VM_TOD:
1605 ret = kvm_s390_set_tod(kvm, attr);
1606 break;
1607 case KVM_S390_VM_CPU_MODEL:
1608 ret = kvm_s390_set_cpu_model(kvm, attr);
1609 break;
1610 case KVM_S390_VM_CRYPTO:
1611 ret = kvm_s390_vm_set_crypto(kvm, attr);
1612 break;
1613 case KVM_S390_VM_MIGRATION:
1614 ret = kvm_s390_vm_set_migration(kvm, attr);
1615 break;
1616 default:
1617 ret = -ENXIO;
1618 break;
1619 }
1620
1621 return ret;
1622}
1623
1624static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1625{
1626 int ret;
1627
1628 switch (attr->group) {
1629 case KVM_S390_VM_MEM_CTRL:
1630 ret = kvm_s390_get_mem_control(kvm, attr);
1631 break;
1632 case KVM_S390_VM_TOD:
1633 ret = kvm_s390_get_tod(kvm, attr);
1634 break;
1635 case KVM_S390_VM_CPU_MODEL:
1636 ret = kvm_s390_get_cpu_model(kvm, attr);
1637 break;
1638 case KVM_S390_VM_MIGRATION:
1639 ret = kvm_s390_vm_get_migration(kvm, attr);
1640 break;
1641 default:
1642 ret = -ENXIO;
1643 break;
1644 }
1645
1646 return ret;
1647}
1648
1649static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1650{
1651 int ret;
1652
1653 switch (attr->group) {
1654 case KVM_S390_VM_MEM_CTRL:
1655 switch (attr->attr) {
1656 case KVM_S390_VM_MEM_ENABLE_CMMA:
1657 case KVM_S390_VM_MEM_CLR_CMMA:
1658 ret = sclp.has_cmma ? 0 : -ENXIO;
1659 break;
1660 case KVM_S390_VM_MEM_LIMIT_SIZE:
1661 ret = 0;
1662 break;
1663 default:
1664 ret = -ENXIO;
1665 break;
1666 }
1667 break;
1668 case KVM_S390_VM_TOD:
1669 switch (attr->attr) {
1670 case KVM_S390_VM_TOD_LOW:
1671 case KVM_S390_VM_TOD_HIGH:
1672 ret = 0;
1673 break;
1674 default:
1675 ret = -ENXIO;
1676 break;
1677 }
1678 break;
1679 case KVM_S390_VM_CPU_MODEL:
1680 switch (attr->attr) {
1681 case KVM_S390_VM_CPU_PROCESSOR:
1682 case KVM_S390_VM_CPU_MACHINE:
1683 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1684 case KVM_S390_VM_CPU_MACHINE_FEAT:
1685 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1686 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1687 ret = 0;
1688 break;
1689 default:
1690 ret = -ENXIO;
1691 break;
1692 }
1693 break;
1694 case KVM_S390_VM_CRYPTO:
1695 switch (attr->attr) {
1696 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1697 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1698 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1699 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1700 ret = 0;
1701 break;
1702 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
1703 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
1704 ret = ap_instructions_available() ? 0 : -ENXIO;
1705 break;
1706 default:
1707 ret = -ENXIO;
1708 break;
1709 }
1710 break;
1711 case KVM_S390_VM_MIGRATION:
1712 ret = 0;
1713 break;
1714 default:
1715 ret = -ENXIO;
1716 break;
1717 }
1718
1719 return ret;
1720}
1721
1722static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1723{
1724 uint8_t *keys;
1725 uint64_t hva;
1726 int srcu_idx, i, r = 0;
1727
1728 if (args->flags != 0)
1729 return -EINVAL;
1730
1731 /* Is this guest using storage keys? */
1732 if (!mm_use_skey(current->mm))
1733 return KVM_S390_GET_SKEYS_NONE;
1734
1735 /* Enforce sane limit on memory allocation */
1736 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1737 return -EINVAL;
1738
1739 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1740 if (!keys)
1741 return -ENOMEM;
1742
1743 down_read(&current->mm->mmap_sem);
1744 srcu_idx = srcu_read_lock(&kvm->srcu);
1745 for (i = 0; i < args->count; i++) {
1746 hva = gfn_to_hva(kvm, args->start_gfn + i);
1747 if (kvm_is_error_hva(hva)) {
1748 r = -EFAULT;
1749 break;
1750 }
1751
1752 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1753 if (r)
1754 break;
1755 }
1756 srcu_read_unlock(&kvm->srcu, srcu_idx);
1757 up_read(&current->mm->mmap_sem);
1758
1759 if (!r) {
1760 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1761 sizeof(uint8_t) * args->count);
1762 if (r)
1763 r = -EFAULT;
1764 }
1765
1766 kvfree(keys);
1767 return r;
1768}
1769
1770static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1771{
1772 uint8_t *keys;
1773 uint64_t hva;
1774 int srcu_idx, i, r = 0;
1775
1776 if (args->flags != 0)
1777 return -EINVAL;
1778
1779 /* Enforce sane limit on memory allocation */
1780 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1781 return -EINVAL;
1782
1783 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1784 if (!keys)
1785 return -ENOMEM;
1786
1787 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1788 sizeof(uint8_t) * args->count);
1789 if (r) {
1790 r = -EFAULT;
1791 goto out;
1792 }
1793
1794 /* Enable storage key handling for the guest */
1795 r = s390_enable_skey();
1796 if (r)
1797 goto out;
1798
1799 down_read(&current->mm->mmap_sem);
1800 srcu_idx = srcu_read_lock(&kvm->srcu);
1801 for (i = 0; i < args->count; i++) {
1802 hva = gfn_to_hva(kvm, args->start_gfn + i);
1803 if (kvm_is_error_hva(hva)) {
1804 r = -EFAULT;
1805 break;
1806 }
1807
1808 /* Lowest order bit is reserved */
1809 if (keys[i] & 0x01) {
1810 r = -EINVAL;
1811 break;
1812 }
1813
1814 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1815 if (r)
1816 break;
1817 }
1818 srcu_read_unlock(&kvm->srcu, srcu_idx);
1819 up_read(&current->mm->mmap_sem);
1820out:
1821 kvfree(keys);
1822 return r;
1823}
1824
1825/*
1826 * Base address and length must be sent at the start of each block, therefore
1827 * it's cheaper to send some clean data, as long as it's less than the size of
1828 * two longs.
1829 */
1830#define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
1831/* for consistency */
1832#define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1833
1834/*
1835 * This function searches for the next page with dirty CMMA attributes, and
1836 * saves the attributes in the buffer up to either the end of the buffer or
1837 * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
1838 * no trailing clean bytes are saved.
1839 * In case no dirty bits were found, or if CMMA was not enabled or used, the
1840 * output buffer will indicate 0 as length.
1841 */
1842static int kvm_s390_get_cmma_bits(struct kvm *kvm,
1843 struct kvm_s390_cmma_log *args)
1844{
1845 struct kvm_s390_migration_state *s = kvm->arch.migration_state;
1846 unsigned long bufsize, hva, pgstev, i, next, cur;
1847 int srcu_idx, peek, r = 0, rr;
1848 u8 *res;
1849
1850 cur = args->start_gfn;
1851 i = next = pgstev = 0;
1852
1853 if (unlikely(!kvm->arch.use_cmma))
1854 return -ENXIO;
1855 /* Invalid/unsupported flags were specified */
1856 if (args->flags & ~KVM_S390_CMMA_PEEK)
1857 return -EINVAL;
1858 /* Migration mode query, and we are not doing a migration */
1859 peek = !!(args->flags & KVM_S390_CMMA_PEEK);
1860 if (!peek && !s)
1861 return -EINVAL;
1862 /* CMMA is disabled or was not used, or the buffer has length zero */
1863 bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
1864 if (!bufsize || !kvm->mm->context.use_cmma) {
1865 memset(args, 0, sizeof(*args));
1866 return 0;
1867 }
1868
1869 if (!peek) {
1870 /* We are not peeking, and there are no dirty pages */
1871 if (!atomic64_read(&s->dirty_pages)) {
1872 memset(args, 0, sizeof(*args));
1873 return 0;
1874 }
1875 cur = find_next_bit(s->pgste_bitmap, s->bitmap_size,
1876 args->start_gfn);
1877 if (cur >= s->bitmap_size) /* nothing found, loop back */
1878 cur = find_next_bit(s->pgste_bitmap, s->bitmap_size, 0);
1879 if (cur >= s->bitmap_size) { /* again! (very unlikely) */
1880 memset(args, 0, sizeof(*args));
1881 return 0;
1882 }
1883 next = find_next_bit(s->pgste_bitmap, s->bitmap_size, cur + 1);
1884 }
1885
1886 res = vmalloc(bufsize);
1887 if (!res)
1888 return -ENOMEM;
1889
1890 args->start_gfn = cur;
1891
1892 down_read(&kvm->mm->mmap_sem);
1893 srcu_idx = srcu_read_lock(&kvm->srcu);
1894 while (i < bufsize) {
1895 hva = gfn_to_hva(kvm, cur);
1896 if (kvm_is_error_hva(hva)) {
1897 r = -EFAULT;
1898 break;
1899 }
1900 /* decrement only if we actually flipped the bit to 0 */
1901 if (!peek && test_and_clear_bit(cur, s->pgste_bitmap))
1902 atomic64_dec(&s->dirty_pages);
1903 r = get_pgste(kvm->mm, hva, &pgstev);
1904 if (r < 0)
1905 pgstev = 0;
1906 /* save the value */
1907 res[i++] = (pgstev >> 24) & 0x43;
1908 /*
1909 * if the next bit is too far away, stop.
1910 * if we reached the previous "next", find the next one
1911 */
1912 if (!peek) {
1913 if (next > cur + KVM_S390_MAX_BIT_DISTANCE)
1914 break;
1915 if (cur == next)
1916 next = find_next_bit(s->pgste_bitmap,
1917 s->bitmap_size, cur + 1);
1918 /* reached the end of the bitmap or of the buffer, stop */
1919 if ((next >= s->bitmap_size) ||
1920 (next >= args->start_gfn + bufsize))
1921 break;
1922 }
1923 cur++;
1924 }
1925 srcu_read_unlock(&kvm->srcu, srcu_idx);
1926 up_read(&kvm->mm->mmap_sem);
1927 args->count = i;
1928 args->remaining = s ? atomic64_read(&s->dirty_pages) : 0;
1929
1930 rr = copy_to_user((void __user *)args->values, res, args->count);
1931 if (rr)
1932 r = -EFAULT;
1933
1934 vfree(res);
1935 return r;
1936}
1937
1938/*
1939 * This function sets the CMMA attributes for the given pages. If the input
1940 * buffer has zero length, no action is taken, otherwise the attributes are
1941 * set and the mm->context.use_cmma flag is set.
1942 */
1943static int kvm_s390_set_cmma_bits(struct kvm *kvm,
1944 const struct kvm_s390_cmma_log *args)
1945{
1946 unsigned long hva, mask, pgstev, i;
1947 uint8_t *bits;
1948 int srcu_idx, r = 0;
1949
1950 mask = args->mask;
1951
1952 if (!kvm->arch.use_cmma)
1953 return -ENXIO;
1954 /* invalid/unsupported flags */
1955 if (args->flags != 0)
1956 return -EINVAL;
1957 /* Enforce sane limit on memory allocation */
1958 if (args->count > KVM_S390_CMMA_SIZE_MAX)
1959 return -EINVAL;
1960 /* Nothing to do */
1961 if (args->count == 0)
1962 return 0;
1963
1964 bits = vmalloc(sizeof(*bits) * args->count);
1965 if (!bits)
1966 return -ENOMEM;
1967
1968 r = copy_from_user(bits, (void __user *)args->values, args->count);
1969 if (r) {
1970 r = -EFAULT;
1971 goto out;
1972 }
1973
1974 down_read(&kvm->mm->mmap_sem);
1975 srcu_idx = srcu_read_lock(&kvm->srcu);
1976 for (i = 0; i < args->count; i++) {
1977 hva = gfn_to_hva(kvm, args->start_gfn + i);
1978 if (kvm_is_error_hva(hva)) {
1979 r = -EFAULT;
1980 break;
1981 }
1982
1983 pgstev = bits[i];
1984 pgstev = pgstev << 24;
1985 mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
1986 set_pgste_bits(kvm->mm, hva, mask, pgstev);
1987 }
1988 srcu_read_unlock(&kvm->srcu, srcu_idx);
1989 up_read(&kvm->mm->mmap_sem);
1990
1991 if (!kvm->mm->context.use_cmma) {
1992 down_write(&kvm->mm->mmap_sem);
1993 kvm->mm->context.use_cmma = 1;
1994 up_write(&kvm->mm->mmap_sem);
1995 }
1996out:
1997 vfree(bits);
1998 return r;
1999}
2000
2001long kvm_arch_vm_ioctl(struct file *filp,
2002 unsigned int ioctl, unsigned long arg)
2003{
2004 struct kvm *kvm = filp->private_data;
2005 void __user *argp = (void __user *)arg;
2006 struct kvm_device_attr attr;
2007 int r;
2008
2009 switch (ioctl) {
2010 case KVM_S390_INTERRUPT: {
2011 struct kvm_s390_interrupt s390int;
2012
2013 r = -EFAULT;
2014 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2015 break;
2016 r = kvm_s390_inject_vm(kvm, &s390int);
2017 break;
2018 }
2019 case KVM_ENABLE_CAP: {
2020 struct kvm_enable_cap cap;
2021 r = -EFAULT;
2022 if (copy_from_user(&cap, argp, sizeof(cap)))
2023 break;
2024 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
2025 break;
2026 }
2027 case KVM_CREATE_IRQCHIP: {
2028 struct kvm_irq_routing_entry routing;
2029
2030 r = -EINVAL;
2031 if (kvm->arch.use_irqchip) {
2032 /* Set up dummy routing. */
2033 memset(&routing, 0, sizeof(routing));
2034 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
2035 }
2036 break;
2037 }
2038 case KVM_SET_DEVICE_ATTR: {
2039 r = -EFAULT;
2040 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2041 break;
2042 r = kvm_s390_vm_set_attr(kvm, &attr);
2043 break;
2044 }
2045 case KVM_GET_DEVICE_ATTR: {
2046 r = -EFAULT;
2047 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2048 break;
2049 r = kvm_s390_vm_get_attr(kvm, &attr);
2050 break;
2051 }
2052 case KVM_HAS_DEVICE_ATTR: {
2053 r = -EFAULT;
2054 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2055 break;
2056 r = kvm_s390_vm_has_attr(kvm, &attr);
2057 break;
2058 }
2059 case KVM_S390_GET_SKEYS: {
2060 struct kvm_s390_skeys args;
2061
2062 r = -EFAULT;
2063 if (copy_from_user(&args, argp,
2064 sizeof(struct kvm_s390_skeys)))
2065 break;
2066 r = kvm_s390_get_skeys(kvm, &args);
2067 break;
2068 }
2069 case KVM_S390_SET_SKEYS: {
2070 struct kvm_s390_skeys args;
2071
2072 r = -EFAULT;
2073 if (copy_from_user(&args, argp,
2074 sizeof(struct kvm_s390_skeys)))
2075 break;
2076 r = kvm_s390_set_skeys(kvm, &args);
2077 break;
2078 }
2079 case KVM_S390_GET_CMMA_BITS: {
2080 struct kvm_s390_cmma_log args;
2081
2082 r = -EFAULT;
2083 if (copy_from_user(&args, argp, sizeof(args)))
2084 break;
2085 mutex_lock(&kvm->slots_lock);
2086 r = kvm_s390_get_cmma_bits(kvm, &args);
2087 mutex_unlock(&kvm->slots_lock);
2088 if (!r) {
2089 r = copy_to_user(argp, &args, sizeof(args));
2090 if (r)
2091 r = -EFAULT;
2092 }
2093 break;
2094 }
2095 case KVM_S390_SET_CMMA_BITS: {
2096 struct kvm_s390_cmma_log args;
2097
2098 r = -EFAULT;
2099 if (copy_from_user(&args, argp, sizeof(args)))
2100 break;
2101 mutex_lock(&kvm->slots_lock);
2102 r = kvm_s390_set_cmma_bits(kvm, &args);
2103 mutex_unlock(&kvm->slots_lock);
2104 break;
2105 }
2106 default:
2107 r = -ENOTTY;
2108 }
2109
2110 return r;
2111}
2112
2113static int kvm_s390_apxa_installed(void)
2114{
2115 struct ap_config_info info;
2116
2117 if (ap_instructions_available()) {
2118 if (ap_qci(&info) == 0)
2119 return info.apxa;
2120 }
2121
2122 return 0;
2123}
2124
2125/*
2126 * The format of the crypto control block (CRYCB) is specified in the 3 low
2127 * order bits of the CRYCB designation (CRYCBD) field as follows:
2128 * Format 0: Neither the message security assist extension 3 (MSAX3) nor the
2129 * AP extended addressing (APXA) facility are installed.
2130 * Format 1: The APXA facility is not installed but the MSAX3 facility is.
2131 * Format 2: Both the APXA and MSAX3 facilities are installed
2132 */
2133static void kvm_s390_set_crycb_format(struct kvm *kvm)
2134{
2135 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
2136
2137 /* Clear the CRYCB format bits - i.e., set format 0 by default */
2138 kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
2139
2140 /* Check whether MSAX3 is installed */
2141 if (!test_kvm_facility(kvm, 76))
2142 return;
2143
2144 if (kvm_s390_apxa_installed())
2145 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
2146 else
2147 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
2148}
2149
2150void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
2151 unsigned long *aqm, unsigned long *adm)
2152{
2153 struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
2154
2155 mutex_lock(&kvm->lock);
2156 kvm_s390_vcpu_block_all(kvm);
2157
2158 switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
2159 case CRYCB_FORMAT2: /* APCB1 use 256 bits */
2160 memcpy(crycb->apcb1.apm, apm, 32);
2161 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx %016lx %016lx %016lx",
2162 apm[0], apm[1], apm[2], apm[3]);
2163 memcpy(crycb->apcb1.aqm, aqm, 32);
2164 VM_EVENT(kvm, 3, "SET CRYCB: aqm %016lx %016lx %016lx %016lx",
2165 aqm[0], aqm[1], aqm[2], aqm[3]);
2166 memcpy(crycb->apcb1.adm, adm, 32);
2167 VM_EVENT(kvm, 3, "SET CRYCB: adm %016lx %016lx %016lx %016lx",
2168 adm[0], adm[1], adm[2], adm[3]);
2169 break;
2170 case CRYCB_FORMAT1:
2171 case CRYCB_FORMAT0: /* Fall through both use APCB0 */
2172 memcpy(crycb->apcb0.apm, apm, 8);
2173 memcpy(crycb->apcb0.aqm, aqm, 2);
2174 memcpy(crycb->apcb0.adm, adm, 2);
2175 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx aqm %04x adm %04x",
2176 apm[0], *((unsigned short *)aqm),
2177 *((unsigned short *)adm));
2178 break;
2179 default: /* Can not happen */
2180 break;
2181 }
2182
2183 /* recreate the shadow crycb for each vcpu */
2184 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2185 kvm_s390_vcpu_unblock_all(kvm);
2186 mutex_unlock(&kvm->lock);
2187}
2188EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks);
2189
2190void kvm_arch_crypto_clear_masks(struct kvm *kvm)
2191{
2192 mutex_lock(&kvm->lock);
2193 kvm_s390_vcpu_block_all(kvm);
2194
2195 memset(&kvm->arch.crypto.crycb->apcb0, 0,
2196 sizeof(kvm->arch.crypto.crycb->apcb0));
2197 memset(&kvm->arch.crypto.crycb->apcb1, 0,
2198 sizeof(kvm->arch.crypto.crycb->apcb1));
2199
2200 VM_EVENT(kvm, 3, "%s", "CLR CRYCB:");
2201 /* recreate the shadow crycb for each vcpu */
2202 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2203 kvm_s390_vcpu_unblock_all(kvm);
2204 mutex_unlock(&kvm->lock);
2205}
2206EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks);
2207
2208static u64 kvm_s390_get_initial_cpuid(void)
2209{
2210 struct cpuid cpuid;
2211
2212 get_cpu_id(&cpuid);
2213 cpuid.version = 0xff;
2214 return *((u64 *) &cpuid);
2215}
2216
2217static void kvm_s390_crypto_init(struct kvm *kvm)
2218{
2219 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
2220 kvm_s390_set_crycb_format(kvm);
2221
2222 if (!test_kvm_facility(kvm, 76))
2223 return;
2224
2225 /* Enable AES/DEA protected key functions by default */
2226 kvm->arch.crypto.aes_kw = 1;
2227 kvm->arch.crypto.dea_kw = 1;
2228 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
2229 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
2230 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
2231 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
2232}
2233
2234static void sca_dispose(struct kvm *kvm)
2235{
2236 if (kvm->arch.use_esca)
2237 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
2238 else
2239 free_page((unsigned long)(kvm->arch.sca));
2240 kvm->arch.sca = NULL;
2241}
2242
2243int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
2244{
2245 gfp_t alloc_flags = GFP_KERNEL;
2246 int i, rc;
2247 char debug_name[16];
2248 static unsigned long sca_offset;
2249
2250 rc = -EINVAL;
2251#ifdef CONFIG_KVM_S390_UCONTROL
2252 if (type & ~KVM_VM_S390_UCONTROL)
2253 goto out_err;
2254 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
2255 goto out_err;
2256#else
2257 if (type)
2258 goto out_err;
2259#endif
2260
2261 rc = s390_enable_sie();
2262 if (rc)
2263 goto out_err;
2264
2265 rc = -ENOMEM;
2266
2267 kvm->arch.use_esca = 0; /* start with basic SCA */
2268 if (!sclp.has_64bscao)
2269 alloc_flags |= GFP_DMA;
2270 rwlock_init(&kvm->arch.sca_lock);
2271 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
2272 if (!kvm->arch.sca)
2273 goto out_err;
2274 mutex_lock(&kvm_lock);
2275 sca_offset += 16;
2276 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
2277 sca_offset = 0;
2278 kvm->arch.sca = (struct bsca_block *)
2279 ((char *) kvm->arch.sca + sca_offset);
2280 mutex_unlock(&kvm_lock);
2281
2282 sprintf(debug_name, "kvm-%u", current->pid);
2283
2284 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
2285 if (!kvm->arch.dbf)
2286 goto out_err;
2287
2288 kvm->arch.sie_page2 =
2289 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
2290 if (!kvm->arch.sie_page2)
2291 goto out_err;
2292
2293 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
2294
2295 for (i = 0; i < kvm_s390_fac_size(); i++) {
2296 kvm->arch.model.fac_mask[i] = S390_lowcore.stfle_fac_list[i] &
2297 (kvm_s390_fac_base[i] |
2298 kvm_s390_fac_ext[i]);
2299 kvm->arch.model.fac_list[i] = S390_lowcore.stfle_fac_list[i] &
2300 kvm_s390_fac_base[i];
2301 }
2302 kvm->arch.model.subfuncs = kvm_s390_available_subfunc;
2303
2304 /* we are always in czam mode - even on pre z14 machines */
2305 set_kvm_facility(kvm->arch.model.fac_mask, 138);
2306 set_kvm_facility(kvm->arch.model.fac_list, 138);
2307 /* we emulate STHYI in kvm */
2308 set_kvm_facility(kvm->arch.model.fac_mask, 74);
2309 set_kvm_facility(kvm->arch.model.fac_list, 74);
2310 if (MACHINE_HAS_TLB_GUEST) {
2311 set_kvm_facility(kvm->arch.model.fac_mask, 147);
2312 set_kvm_facility(kvm->arch.model.fac_list, 147);
2313 }
2314
2315 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
2316 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
2317
2318 kvm_s390_crypto_init(kvm);
2319
2320 mutex_init(&kvm->arch.float_int.ais_lock);
2321 kvm->arch.float_int.simm = 0;
2322 kvm->arch.float_int.nimm = 0;
2323 spin_lock_init(&kvm->arch.float_int.lock);
2324 for (i = 0; i < FIRQ_LIST_COUNT; i++)
2325 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
2326 init_waitqueue_head(&kvm->arch.ipte_wq);
2327 mutex_init(&kvm->arch.ipte_mutex);
2328
2329 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
2330 VM_EVENT(kvm, 3, "vm created with type %lu", type);
2331
2332 if (type & KVM_VM_S390_UCONTROL) {
2333 kvm->arch.gmap = NULL;
2334 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
2335 } else {
2336 if (sclp.hamax == U64_MAX)
2337 kvm->arch.mem_limit = TASK_SIZE_MAX;
2338 else
2339 kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
2340 sclp.hamax + 1);
2341 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
2342 if (!kvm->arch.gmap)
2343 goto out_err;
2344 kvm->arch.gmap->private = kvm;
2345 kvm->arch.gmap->pfault_enabled = 0;
2346 }
2347
2348 kvm->arch.css_support = 0;
2349 kvm->arch.use_irqchip = 0;
2350 kvm->arch.epoch = 0;
2351
2352 spin_lock_init(&kvm->arch.start_stop_lock);
2353 kvm_s390_vsie_init(kvm);
2354 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
2355
2356 return 0;
2357out_err:
2358 free_page((unsigned long)kvm->arch.sie_page2);
2359 debug_unregister(kvm->arch.dbf);
2360 sca_dispose(kvm);
2361 KVM_EVENT(3, "creation of vm failed: %d", rc);
2362 return rc;
2363}
2364
2365bool kvm_arch_has_vcpu_debugfs(void)
2366{
2367 return false;
2368}
2369
2370int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
2371{
2372 return 0;
2373}
2374
2375void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
2376{
2377 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
2378 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
2379 kvm_s390_clear_local_irqs(vcpu);
2380 kvm_clear_async_pf_completion_queue(vcpu);
2381 if (!kvm_is_ucontrol(vcpu->kvm))
2382 sca_del_vcpu(vcpu);
2383
2384 if (kvm_is_ucontrol(vcpu->kvm))
2385 gmap_remove(vcpu->arch.gmap);
2386
2387 if (vcpu->kvm->arch.use_cmma)
2388 kvm_s390_vcpu_unsetup_cmma(vcpu);
2389 free_page((unsigned long)(vcpu->arch.sie_block));
2390
2391 kvm_vcpu_uninit(vcpu);
2392 kmem_cache_free(kvm_vcpu_cache, vcpu);
2393}
2394
2395static void kvm_free_vcpus(struct kvm *kvm)
2396{
2397 unsigned int i;
2398 struct kvm_vcpu *vcpu;
2399
2400 kvm_for_each_vcpu(i, vcpu, kvm)
2401 kvm_arch_vcpu_destroy(vcpu);
2402
2403 mutex_lock(&kvm->lock);
2404 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
2405 kvm->vcpus[i] = NULL;
2406
2407 atomic_set(&kvm->online_vcpus, 0);
2408 mutex_unlock(&kvm->lock);
2409}
2410
2411void kvm_arch_destroy_vm(struct kvm *kvm)
2412{
2413 kvm_free_vcpus(kvm);
2414 sca_dispose(kvm);
2415 debug_unregister(kvm->arch.dbf);
2416 free_page((unsigned long)kvm->arch.sie_page2);
2417 if (!kvm_is_ucontrol(kvm))
2418 gmap_remove(kvm->arch.gmap);
2419 kvm_s390_destroy_adapters(kvm);
2420 kvm_s390_clear_float_irqs(kvm);
2421 kvm_s390_vsie_destroy(kvm);
2422 if (kvm->arch.migration_state) {
2423 vfree(kvm->arch.migration_state->pgste_bitmap);
2424 kfree(kvm->arch.migration_state);
2425 }
2426 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
2427}
2428
2429/* Section: vcpu related */
2430static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
2431{
2432 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
2433 if (!vcpu->arch.gmap)
2434 return -ENOMEM;
2435 vcpu->arch.gmap->private = vcpu->kvm;
2436
2437 return 0;
2438}
2439
2440static void sca_del_vcpu(struct kvm_vcpu *vcpu)
2441{
2442 if (!kvm_s390_use_sca_entries())
2443 return;
2444 read_lock(&vcpu->kvm->arch.sca_lock);
2445 if (vcpu->kvm->arch.use_esca) {
2446 struct esca_block *sca = vcpu->kvm->arch.sca;
2447
2448 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2449 sca->cpu[vcpu->vcpu_id].sda = 0;
2450 } else {
2451 struct bsca_block *sca = vcpu->kvm->arch.sca;
2452
2453 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2454 sca->cpu[vcpu->vcpu_id].sda = 0;
2455 }
2456 read_unlock(&vcpu->kvm->arch.sca_lock);
2457}
2458
2459static void sca_add_vcpu(struct kvm_vcpu *vcpu)
2460{
2461 if (!kvm_s390_use_sca_entries()) {
2462 struct bsca_block *sca = vcpu->kvm->arch.sca;
2463
2464 /* we still need the basic sca for the ipte control */
2465 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2466 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2467 return;
2468 }
2469 read_lock(&vcpu->kvm->arch.sca_lock);
2470 if (vcpu->kvm->arch.use_esca) {
2471 struct esca_block *sca = vcpu->kvm->arch.sca;
2472
2473 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2474 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2475 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
2476 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2477 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2478 } else {
2479 struct bsca_block *sca = vcpu->kvm->arch.sca;
2480
2481 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2482 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2483 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2484 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2485 }
2486 read_unlock(&vcpu->kvm->arch.sca_lock);
2487}
2488
2489/* Basic SCA to Extended SCA data copy routines */
2490static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
2491{
2492 d->sda = s->sda;
2493 d->sigp_ctrl.c = s->sigp_ctrl.c;
2494 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
2495}
2496
2497static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
2498{
2499 int i;
2500
2501 d->ipte_control = s->ipte_control;
2502 d->mcn[0] = s->mcn;
2503 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
2504 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
2505}
2506
2507static int sca_switch_to_extended(struct kvm *kvm)
2508{
2509 struct bsca_block *old_sca = kvm->arch.sca;
2510 struct esca_block *new_sca;
2511 struct kvm_vcpu *vcpu;
2512 unsigned int vcpu_idx;
2513 u32 scaol, scaoh;
2514
2515 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
2516 if (!new_sca)
2517 return -ENOMEM;
2518
2519 scaoh = (u32)((u64)(new_sca) >> 32);
2520 scaol = (u32)(u64)(new_sca) & ~0x3fU;
2521
2522 kvm_s390_vcpu_block_all(kvm);
2523 write_lock(&kvm->arch.sca_lock);
2524
2525 sca_copy_b_to_e(new_sca, old_sca);
2526
2527 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
2528 vcpu->arch.sie_block->scaoh = scaoh;
2529 vcpu->arch.sie_block->scaol = scaol;
2530 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2531 }
2532 kvm->arch.sca = new_sca;
2533 kvm->arch.use_esca = 1;
2534
2535 write_unlock(&kvm->arch.sca_lock);
2536 kvm_s390_vcpu_unblock_all(kvm);
2537
2538 free_page((unsigned long)old_sca);
2539
2540 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
2541 old_sca, kvm->arch.sca);
2542 return 0;
2543}
2544
2545static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
2546{
2547 int rc;
2548
2549 if (!kvm_s390_use_sca_entries()) {
2550 if (id < KVM_MAX_VCPUS)
2551 return true;
2552 return false;
2553 }
2554 if (id < KVM_S390_BSCA_CPU_SLOTS)
2555 return true;
2556 if (!sclp.has_esca || !sclp.has_64bscao)
2557 return false;
2558
2559 mutex_lock(&kvm->lock);
2560 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
2561 mutex_unlock(&kvm->lock);
2562
2563 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
2564}
2565
2566int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
2567{
2568 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2569 kvm_clear_async_pf_completion_queue(vcpu);
2570 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
2571 KVM_SYNC_GPRS |
2572 KVM_SYNC_ACRS |
2573 KVM_SYNC_CRS |
2574 KVM_SYNC_ARCH0 |
2575 KVM_SYNC_PFAULT;
2576 kvm_s390_set_prefix(vcpu, 0);
2577 if (test_kvm_facility(vcpu->kvm, 64))
2578 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
2579 if (test_kvm_facility(vcpu->kvm, 82))
2580 vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
2581 if (test_kvm_facility(vcpu->kvm, 133))
2582 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
2583 if (test_kvm_facility(vcpu->kvm, 156))
2584 vcpu->run->kvm_valid_regs |= KVM_SYNC_ETOKEN;
2585 /* fprs can be synchronized via vrs, even if the guest has no vx. With
2586 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
2587 */
2588 if (MACHINE_HAS_VX)
2589 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
2590 else
2591 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
2592
2593 if (kvm_is_ucontrol(vcpu->kvm))
2594 return __kvm_ucontrol_vcpu_init(vcpu);
2595
2596 return 0;
2597}
2598
2599/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2600static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2601{
2602 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
2603 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2604 vcpu->arch.cputm_start = get_tod_clock_fast();
2605 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2606}
2607
2608/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2609static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2610{
2611 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
2612 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2613 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2614 vcpu->arch.cputm_start = 0;
2615 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2616}
2617
2618/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2619static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2620{
2621 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
2622 vcpu->arch.cputm_enabled = true;
2623 __start_cpu_timer_accounting(vcpu);
2624}
2625
2626/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2627static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2628{
2629 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
2630 __stop_cpu_timer_accounting(vcpu);
2631 vcpu->arch.cputm_enabled = false;
2632}
2633
2634static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2635{
2636 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2637 __enable_cpu_timer_accounting(vcpu);
2638 preempt_enable();
2639}
2640
2641static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2642{
2643 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2644 __disable_cpu_timer_accounting(vcpu);
2645 preempt_enable();
2646}
2647
2648/* set the cpu timer - may only be called from the VCPU thread itself */
2649void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
2650{
2651 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2652 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2653 if (vcpu->arch.cputm_enabled)
2654 vcpu->arch.cputm_start = get_tod_clock_fast();
2655 vcpu->arch.sie_block->cputm = cputm;
2656 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2657 preempt_enable();
2658}
2659
2660/* update and get the cpu timer - can also be called from other VCPU threads */
2661__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
2662{
2663 unsigned int seq;
2664 __u64 value;
2665
2666 if (unlikely(!vcpu->arch.cputm_enabled))
2667 return vcpu->arch.sie_block->cputm;
2668
2669 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2670 do {
2671 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
2672 /*
2673 * If the writer would ever execute a read in the critical
2674 * section, e.g. in irq context, we have a deadlock.
2675 */
2676 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
2677 value = vcpu->arch.sie_block->cputm;
2678 /* if cputm_start is 0, accounting is being started/stopped */
2679 if (likely(vcpu->arch.cputm_start))
2680 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2681 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
2682 preempt_enable();
2683 return value;
2684}
2685
2686void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2687{
2688
2689 gmap_enable(vcpu->arch.enabled_gmap);
2690 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
2691 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2692 __start_cpu_timer_accounting(vcpu);
2693 vcpu->cpu = cpu;
2694}
2695
2696void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
2697{
2698 vcpu->cpu = -1;
2699 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2700 __stop_cpu_timer_accounting(vcpu);
2701 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
2702 vcpu->arch.enabled_gmap = gmap_get_enabled();
2703 gmap_disable(vcpu->arch.enabled_gmap);
2704
2705}
2706
2707static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
2708{
2709 /* this equals initial cpu reset in pop, but we don't switch to ESA */
2710 vcpu->arch.sie_block->gpsw.mask = 0UL;
2711 vcpu->arch.sie_block->gpsw.addr = 0UL;
2712 kvm_s390_set_prefix(vcpu, 0);
2713 kvm_s390_set_cpu_timer(vcpu, 0);
2714 vcpu->arch.sie_block->ckc = 0UL;
2715 vcpu->arch.sie_block->todpr = 0;
2716 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
2717 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
2718 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
2719 vcpu->run->s.regs.fpc = 0;
2720 vcpu->arch.sie_block->gbea = 1;
2721 vcpu->arch.sie_block->pp = 0;
2722 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
2723 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2724 kvm_clear_async_pf_completion_queue(vcpu);
2725 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
2726 kvm_s390_vcpu_stop(vcpu);
2727 kvm_s390_clear_local_irqs(vcpu);
2728}
2729
2730void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
2731{
2732 mutex_lock(&vcpu->kvm->lock);
2733 preempt_disable();
2734 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
2735 vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
2736 preempt_enable();
2737 mutex_unlock(&vcpu->kvm->lock);
2738 if (!kvm_is_ucontrol(vcpu->kvm)) {
2739 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
2740 sca_add_vcpu(vcpu);
2741 }
2742 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
2743 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2744 /* make vcpu_load load the right gmap on the first trigger */
2745 vcpu->arch.enabled_gmap = vcpu->arch.gmap;
2746}
2747
2748static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr)
2749{
2750 if (test_bit_inv(nr, (unsigned long *)&kvm->arch.model.subfuncs.pckmo) &&
2751 test_bit_inv(nr, (unsigned long *)&kvm_s390_available_subfunc.pckmo))
2752 return true;
2753 return false;
2754}
2755
2756static bool kvm_has_pckmo_ecc(struct kvm *kvm)
2757{
2758 /* At least one ECC subfunction must be present */
2759 return kvm_has_pckmo_subfunc(kvm, 32) ||
2760 kvm_has_pckmo_subfunc(kvm, 33) ||
2761 kvm_has_pckmo_subfunc(kvm, 34) ||
2762 kvm_has_pckmo_subfunc(kvm, 40) ||
2763 kvm_has_pckmo_subfunc(kvm, 41);
2764
2765}
2766
2767static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
2768{
2769 /*
2770 * If the AP instructions are not being interpreted and the MSAX3
2771 * facility is not configured for the guest, there is nothing to set up.
2772 */
2773 if (!vcpu->kvm->arch.crypto.apie && !test_kvm_facility(vcpu->kvm, 76))
2774 return;
2775
2776 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
2777 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
2778 vcpu->arch.sie_block->eca &= ~ECA_APIE;
2779 vcpu->arch.sie_block->ecd &= ~ECD_ECC;
2780
2781 if (vcpu->kvm->arch.crypto.apie)
2782 vcpu->arch.sie_block->eca |= ECA_APIE;
2783
2784 /* Set up protected key support */
2785 if (vcpu->kvm->arch.crypto.aes_kw) {
2786 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
2787 /* ecc is also wrapped with AES key */
2788 if (kvm_has_pckmo_ecc(vcpu->kvm))
2789 vcpu->arch.sie_block->ecd |= ECD_ECC;
2790 }
2791
2792 if (vcpu->kvm->arch.crypto.dea_kw)
2793 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
2794}
2795
2796void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
2797{
2798 free_page(vcpu->arch.sie_block->cbrlo);
2799 vcpu->arch.sie_block->cbrlo = 0;
2800}
2801
2802int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
2803{
2804 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
2805 if (!vcpu->arch.sie_block->cbrlo)
2806 return -ENOMEM;
2807
2808 vcpu->arch.sie_block->ecb2 &= ~ECB2_PFMFI;
2809 return 0;
2810}
2811
2812static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
2813{
2814 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
2815
2816 vcpu->arch.sie_block->ibc = model->ibc;
2817 if (test_kvm_facility(vcpu->kvm, 7))
2818 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
2819}
2820
2821int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
2822{
2823 int rc = 0;
2824
2825 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
2826 CPUSTAT_SM |
2827 CPUSTAT_STOPPED);
2828
2829 if (test_kvm_facility(vcpu->kvm, 78))
2830 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
2831 else if (test_kvm_facility(vcpu->kvm, 8))
2832 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
2833
2834 kvm_s390_vcpu_setup_model(vcpu);
2835
2836 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
2837 if (MACHINE_HAS_ESOP)
2838 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
2839 if (test_kvm_facility(vcpu->kvm, 9))
2840 vcpu->arch.sie_block->ecb |= ECB_SRSI;
2841 if (test_kvm_facility(vcpu->kvm, 73))
2842 vcpu->arch.sie_block->ecb |= ECB_TE;
2843
2844 if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
2845 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
2846 if (test_kvm_facility(vcpu->kvm, 130))
2847 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
2848 vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
2849 if (sclp.has_cei)
2850 vcpu->arch.sie_block->eca |= ECA_CEI;
2851 if (sclp.has_ib)
2852 vcpu->arch.sie_block->eca |= ECA_IB;
2853 if (sclp.has_siif)
2854 vcpu->arch.sie_block->eca |= ECA_SII;
2855 if (sclp.has_sigpif)
2856 vcpu->arch.sie_block->eca |= ECA_SIGPI;
2857 if (test_kvm_facility(vcpu->kvm, 129)) {
2858 vcpu->arch.sie_block->eca |= ECA_VX;
2859 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
2860 }
2861 if (test_kvm_facility(vcpu->kvm, 139))
2862 vcpu->arch.sie_block->ecd |= ECD_MEF;
2863 if (test_kvm_facility(vcpu->kvm, 156))
2864 vcpu->arch.sie_block->ecd |= ECD_ETOKENF;
2865 vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
2866 | SDNXC;
2867 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
2868
2869 if (sclp.has_kss)
2870 atomic_or(CPUSTAT_KSS, &vcpu->arch.sie_block->cpuflags);
2871 else
2872 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
2873
2874 if (vcpu->kvm->arch.use_cmma) {
2875 rc = kvm_s390_vcpu_setup_cmma(vcpu);
2876 if (rc)
2877 return rc;
2878 }
2879 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2880 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
2881
2882 kvm_s390_vcpu_crypto_setup(vcpu);
2883
2884 return rc;
2885}
2886
2887struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
2888 unsigned int id)
2889{
2890 struct kvm_vcpu *vcpu;
2891 struct sie_page *sie_page;
2892 int rc = -EINVAL;
2893
2894 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
2895 goto out;
2896
2897 rc = -ENOMEM;
2898
2899 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
2900 if (!vcpu)
2901 goto out;
2902
2903 BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
2904 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
2905 if (!sie_page)
2906 goto out_free_cpu;
2907
2908 vcpu->arch.sie_block = &sie_page->sie_block;
2909 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
2910
2911 /* the real guest size will always be smaller than msl */
2912 vcpu->arch.sie_block->mso = 0;
2913 vcpu->arch.sie_block->msl = sclp.hamax;
2914
2915 vcpu->arch.sie_block->icpua = id;
2916 spin_lock_init(&vcpu->arch.local_int.lock);
2917 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
2918 vcpu->arch.local_int.wq = &vcpu->wq;
2919 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
2920 seqcount_init(&vcpu->arch.cputm_seqcount);
2921
2922 rc = kvm_vcpu_init(vcpu, kvm, id);
2923 if (rc)
2924 goto out_free_sie_block;
2925 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
2926 vcpu->arch.sie_block);
2927 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
2928
2929 return vcpu;
2930out_free_sie_block:
2931 free_page((unsigned long)(vcpu->arch.sie_block));
2932out_free_cpu:
2933 kmem_cache_free(kvm_vcpu_cache, vcpu);
2934out:
2935 return ERR_PTR(rc);
2936}
2937
2938int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
2939{
2940 return kvm_s390_vcpu_has_irq(vcpu, 0);
2941}
2942
2943bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
2944{
2945 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
2946}
2947
2948void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
2949{
2950 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2951 exit_sie(vcpu);
2952}
2953
2954void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
2955{
2956 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2957}
2958
2959static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
2960{
2961 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2962 exit_sie(vcpu);
2963}
2964
2965bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu)
2966{
2967 return atomic_read(&vcpu->arch.sie_block->prog20) &
2968 (PROG_BLOCK_SIE | PROG_REQUEST);
2969}
2970
2971static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2972{
2973 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2974}
2975
2976/*
2977 * Kick a guest cpu out of (v)SIE and wait until (v)SIE is not running.
2978 * If the CPU is not running (e.g. waiting as idle) the function will
2979 * return immediately. */
2980void exit_sie(struct kvm_vcpu *vcpu)
2981{
2982 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
2983 kvm_s390_vsie_kick(vcpu);
2984 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2985 cpu_relax();
2986}
2987
2988/* Kick a guest cpu out of SIE to process a request synchronously */
2989void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
2990{
2991 kvm_make_request(req, vcpu);
2992 kvm_s390_vcpu_request(vcpu);
2993}
2994
2995static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
2996 unsigned long end)
2997{
2998 struct kvm *kvm = gmap->private;
2999 struct kvm_vcpu *vcpu;
3000 unsigned long prefix;
3001 int i;
3002
3003 if (gmap_is_shadow(gmap))
3004 return;
3005 if (start >= 1UL << 31)
3006 /* We are only interested in prefix pages */
3007 return;
3008 kvm_for_each_vcpu(i, vcpu, kvm) {
3009 /* match against both prefix pages */
3010 prefix = kvm_s390_get_prefix(vcpu);
3011 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
3012 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
3013 start, end);
3014 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
3015 }
3016 }
3017}
3018
3019int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
3020{
3021 /* kvm common code refers to this, but never calls it */
3022 BUG();
3023 return 0;
3024}
3025
3026static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
3027 struct kvm_one_reg *reg)
3028{
3029 int r = -EINVAL;
3030
3031 switch (reg->id) {
3032 case KVM_REG_S390_TODPR:
3033 r = put_user(vcpu->arch.sie_block->todpr,
3034 (u32 __user *)reg->addr);
3035 break;
3036 case KVM_REG_S390_EPOCHDIFF:
3037 r = put_user(vcpu->arch.sie_block->epoch,
3038 (u64 __user *)reg->addr);
3039 break;
3040 case KVM_REG_S390_CPU_TIMER:
3041 r = put_user(kvm_s390_get_cpu_timer(vcpu),
3042 (u64 __user *)reg->addr);
3043 break;
3044 case KVM_REG_S390_CLOCK_COMP:
3045 r = put_user(vcpu->arch.sie_block->ckc,
3046 (u64 __user *)reg->addr);
3047 break;
3048 case KVM_REG_S390_PFTOKEN:
3049 r = put_user(vcpu->arch.pfault_token,
3050 (u64 __user *)reg->addr);
3051 break;
3052 case KVM_REG_S390_PFCOMPARE:
3053 r = put_user(vcpu->arch.pfault_compare,
3054 (u64 __user *)reg->addr);
3055 break;
3056 case KVM_REG_S390_PFSELECT:
3057 r = put_user(vcpu->arch.pfault_select,
3058 (u64 __user *)reg->addr);
3059 break;
3060 case KVM_REG_S390_PP:
3061 r = put_user(vcpu->arch.sie_block->pp,
3062 (u64 __user *)reg->addr);
3063 break;
3064 case KVM_REG_S390_GBEA:
3065 r = put_user(vcpu->arch.sie_block->gbea,
3066 (u64 __user *)reg->addr);
3067 break;
3068 default:
3069 break;
3070 }
3071
3072 return r;
3073}
3074
3075static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
3076 struct kvm_one_reg *reg)
3077{
3078 int r = -EINVAL;
3079 __u64 val;
3080
3081 switch (reg->id) {
3082 case KVM_REG_S390_TODPR:
3083 r = get_user(vcpu->arch.sie_block->todpr,
3084 (u32 __user *)reg->addr);
3085 break;
3086 case KVM_REG_S390_EPOCHDIFF:
3087 r = get_user(vcpu->arch.sie_block->epoch,
3088 (u64 __user *)reg->addr);
3089 break;
3090 case KVM_REG_S390_CPU_TIMER:
3091 r = get_user(val, (u64 __user *)reg->addr);
3092 if (!r)
3093 kvm_s390_set_cpu_timer(vcpu, val);
3094 break;
3095 case KVM_REG_S390_CLOCK_COMP:
3096 r = get_user(vcpu->arch.sie_block->ckc,
3097 (u64 __user *)reg->addr);
3098 break;
3099 case KVM_REG_S390_PFTOKEN:
3100 r = get_user(vcpu->arch.pfault_token,
3101 (u64 __user *)reg->addr);
3102 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3103 kvm_clear_async_pf_completion_queue(vcpu);
3104 break;
3105 case KVM_REG_S390_PFCOMPARE:
3106 r = get_user(vcpu->arch.pfault_compare,
3107 (u64 __user *)reg->addr);
3108 break;
3109 case KVM_REG_S390_PFSELECT:
3110 r = get_user(vcpu->arch.pfault_select,
3111 (u64 __user *)reg->addr);
3112 break;
3113 case KVM_REG_S390_PP:
3114 r = get_user(vcpu->arch.sie_block->pp,
3115 (u64 __user *)reg->addr);
3116 break;
3117 case KVM_REG_S390_GBEA:
3118 r = get_user(vcpu->arch.sie_block->gbea,
3119 (u64 __user *)reg->addr);
3120 break;
3121 default:
3122 break;
3123 }
3124
3125 return r;
3126}
3127
3128static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
3129{
3130 kvm_s390_vcpu_initial_reset(vcpu);
3131 return 0;
3132}
3133
3134int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3135{
3136 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
3137 return 0;
3138}
3139
3140int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3141{
3142 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
3143 return 0;
3144}
3145
3146int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
3147 struct kvm_sregs *sregs)
3148{
3149 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
3150 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
3151 return 0;
3152}
3153
3154int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
3155 struct kvm_sregs *sregs)
3156{
3157 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
3158 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
3159 return 0;
3160}
3161
3162int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3163{
3164 if (test_fp_ctl(fpu->fpc))
3165 return -EINVAL;
3166 vcpu->run->s.regs.fpc = fpu->fpc;
3167 if (MACHINE_HAS_VX)
3168 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
3169 (freg_t *) fpu->fprs);
3170 else
3171 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
3172 return 0;
3173}
3174
3175int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3176{
3177 /* make sure we have the latest values */
3178 save_fpu_regs();
3179 if (MACHINE_HAS_VX)
3180 convert_vx_to_fp((freg_t *) fpu->fprs,
3181 (__vector128 *) vcpu->run->s.regs.vrs);
3182 else
3183 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
3184 fpu->fpc = vcpu->run->s.regs.fpc;
3185 return 0;
3186}
3187
3188static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
3189{
3190 int rc = 0;
3191
3192 if (!is_vcpu_stopped(vcpu))
3193 rc = -EBUSY;
3194 else {
3195 vcpu->run->psw_mask = psw.mask;
3196 vcpu->run->psw_addr = psw.addr;
3197 }
3198 return rc;
3199}
3200
3201int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
3202 struct kvm_translation *tr)
3203{
3204 return -EINVAL; /* not implemented yet */
3205}
3206
3207#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
3208 KVM_GUESTDBG_USE_HW_BP | \
3209 KVM_GUESTDBG_ENABLE)
3210
3211int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
3212 struct kvm_guest_debug *dbg)
3213{
3214 int rc = 0;
3215
3216 vcpu->guest_debug = 0;
3217 kvm_s390_clear_bp_data(vcpu);
3218
3219 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
3220 return -EINVAL;
3221 if (!sclp.has_gpere)
3222 return -EINVAL;
3223
3224 if (dbg->control & KVM_GUESTDBG_ENABLE) {
3225 vcpu->guest_debug = dbg->control;
3226 /* enforce guest PER */
3227 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
3228
3229 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
3230 rc = kvm_s390_import_bp_data(vcpu, dbg);
3231 } else {
3232 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
3233 vcpu->arch.guestdbg.last_bp = 0;
3234 }
3235
3236 if (rc) {
3237 vcpu->guest_debug = 0;
3238 kvm_s390_clear_bp_data(vcpu);
3239 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
3240 }
3241
3242 return rc;
3243}
3244
3245int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
3246 struct kvm_mp_state *mp_state)
3247{
3248 /* CHECK_STOP and LOAD are not supported yet */
3249 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
3250 KVM_MP_STATE_OPERATING;
3251}
3252
3253int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
3254 struct kvm_mp_state *mp_state)
3255{
3256 int rc = 0;
3257
3258 /* user space knows about this interface - let it control the state */
3259 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
3260
3261 switch (mp_state->mp_state) {
3262 case KVM_MP_STATE_STOPPED:
3263 kvm_s390_vcpu_stop(vcpu);
3264 break;
3265 case KVM_MP_STATE_OPERATING:
3266 kvm_s390_vcpu_start(vcpu);
3267 break;
3268 case KVM_MP_STATE_LOAD:
3269 case KVM_MP_STATE_CHECK_STOP:
3270 /* fall through - CHECK_STOP and LOAD are not supported yet */
3271 default:
3272 rc = -ENXIO;
3273 }
3274
3275 return rc;
3276}
3277
3278static bool ibs_enabled(struct kvm_vcpu *vcpu)
3279{
3280 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
3281}
3282
3283static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
3284{
3285retry:
3286 kvm_s390_vcpu_request_handled(vcpu);
3287 if (!kvm_request_pending(vcpu))
3288 return 0;
3289 /*
3290 * We use MMU_RELOAD just to re-arm the ipte notifier for the
3291 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
3292 * This ensures that the ipte instruction for this request has
3293 * already finished. We might race against a second unmapper that
3294 * wants to set the blocking bit. Lets just retry the request loop.
3295 */
3296 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
3297 int rc;
3298 rc = gmap_mprotect_notify(vcpu->arch.gmap,
3299 kvm_s390_get_prefix(vcpu),
3300 PAGE_SIZE * 2, PROT_WRITE);
3301 if (rc) {
3302 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
3303 return rc;
3304 }
3305 goto retry;
3306 }
3307
3308 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
3309 vcpu->arch.sie_block->ihcpu = 0xffff;
3310 goto retry;
3311 }
3312
3313 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
3314 if (!ibs_enabled(vcpu)) {
3315 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
3316 atomic_or(CPUSTAT_IBS,
3317 &vcpu->arch.sie_block->cpuflags);
3318 }
3319 goto retry;
3320 }
3321
3322 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
3323 if (ibs_enabled(vcpu)) {
3324 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
3325 atomic_andnot(CPUSTAT_IBS,
3326 &vcpu->arch.sie_block->cpuflags);
3327 }
3328 goto retry;
3329 }
3330
3331 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
3332 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
3333 goto retry;
3334 }
3335
3336 if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
3337 /*
3338 * Disable CMMA virtualization; we will emulate the ESSA
3339 * instruction manually, in order to provide additional
3340 * functionalities needed for live migration.
3341 */
3342 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
3343 goto retry;
3344 }
3345
3346 if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
3347 /*
3348 * Re-enable CMMA virtualization if CMMA is available and
3349 * was used.
3350 */
3351 if ((vcpu->kvm->arch.use_cmma) &&
3352 (vcpu->kvm->mm->context.use_cmma))
3353 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
3354 goto retry;
3355 }
3356
3357 /* nothing to do, just clear the request */
3358 kvm_clear_request(KVM_REQ_UNHALT, vcpu);
3359 /* we left the vsie handler, nothing to do, just clear the request */
3360 kvm_clear_request(KVM_REQ_VSIE_RESTART, vcpu);
3361
3362 return 0;
3363}
3364
3365void kvm_s390_set_tod_clock(struct kvm *kvm,
3366 const struct kvm_s390_vm_tod_clock *gtod)
3367{
3368 struct kvm_vcpu *vcpu;
3369 struct kvm_s390_tod_clock_ext htod;
3370 int i;
3371
3372 mutex_lock(&kvm->lock);
3373 preempt_disable();
3374
3375 get_tod_clock_ext((char *)&htod);
3376
3377 kvm->arch.epoch = gtod->tod - htod.tod;
3378 kvm->arch.epdx = 0;
3379 if (test_kvm_facility(kvm, 139)) {
3380 kvm->arch.epdx = gtod->epoch_idx - htod.epoch_idx;
3381 if (kvm->arch.epoch > gtod->tod)
3382 kvm->arch.epdx -= 1;
3383 }
3384
3385 kvm_s390_vcpu_block_all(kvm);
3386 kvm_for_each_vcpu(i, vcpu, kvm) {
3387 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
3388 vcpu->arch.sie_block->epdx = kvm->arch.epdx;
3389 }
3390
3391 kvm_s390_vcpu_unblock_all(kvm);
3392 preempt_enable();
3393 mutex_unlock(&kvm->lock);
3394}
3395
3396/**
3397 * kvm_arch_fault_in_page - fault-in guest page if necessary
3398 * @vcpu: The corresponding virtual cpu
3399 * @gpa: Guest physical address
3400 * @writable: Whether the page should be writable or not
3401 *
3402 * Make sure that a guest page has been faulted-in on the host.
3403 *
3404 * Return: Zero on success, negative error code otherwise.
3405 */
3406long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
3407{
3408 return gmap_fault(vcpu->arch.gmap, gpa,
3409 writable ? FAULT_FLAG_WRITE : 0);
3410}
3411
3412static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
3413 unsigned long token)
3414{
3415 struct kvm_s390_interrupt inti;
3416 struct kvm_s390_irq irq;
3417
3418 if (start_token) {
3419 irq.u.ext.ext_params2 = token;
3420 irq.type = KVM_S390_INT_PFAULT_INIT;
3421 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3422 } else {
3423 inti.type = KVM_S390_INT_PFAULT_DONE;
3424 inti.parm64 = token;
3425 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
3426 }
3427}
3428
3429void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
3430 struct kvm_async_pf *work)
3431{
3432 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
3433 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
3434}
3435
3436void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
3437 struct kvm_async_pf *work)
3438{
3439 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
3440 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
3441}
3442
3443void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
3444 struct kvm_async_pf *work)
3445{
3446 /* s390 will always inject the page directly */
3447}
3448
3449bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
3450{
3451 /*
3452 * s390 will always inject the page directly,
3453 * but we still want check_async_completion to cleanup
3454 */
3455 return true;
3456}
3457
3458static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
3459{
3460 hva_t hva;
3461 struct kvm_arch_async_pf arch;
3462 int rc;
3463
3464 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3465 return 0;
3466 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
3467 vcpu->arch.pfault_compare)
3468 return 0;
3469 if (psw_extint_disabled(vcpu))
3470 return 0;
3471 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3472 return 0;
3473 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
3474 return 0;
3475 if (!vcpu->arch.gmap->pfault_enabled)
3476 return 0;
3477
3478 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
3479 hva += current->thread.gmap_addr & ~PAGE_MASK;
3480 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3481 return 0;
3482
3483 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
3484 return rc;
3485}
3486
3487static int vcpu_pre_run(struct kvm_vcpu *vcpu)
3488{
3489 int rc, cpuflags;
3490
3491 /*
3492 * On s390 notifications for arriving pages will be delivered directly
3493 * to the guest but the house keeping for completed pfaults is
3494 * handled outside the worker.
3495 */
3496 kvm_check_async_pf_completion(vcpu);
3497
3498 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
3499 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
3500
3501 if (need_resched())
3502 schedule();
3503
3504 if (test_cpu_flag(CIF_MCCK_PENDING))
3505 s390_handle_mcck();
3506
3507 if (!kvm_is_ucontrol(vcpu->kvm)) {
3508 rc = kvm_s390_deliver_pending_interrupts(vcpu);
3509 if (rc)
3510 return rc;
3511 }
3512
3513 rc = kvm_s390_handle_requests(vcpu);
3514 if (rc)
3515 return rc;
3516
3517 if (guestdbg_enabled(vcpu)) {
3518 kvm_s390_backup_guest_per_regs(vcpu);
3519 kvm_s390_patch_guest_per_regs(vcpu);
3520 }
3521
3522 vcpu->arch.sie_block->icptcode = 0;
3523 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
3524 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
3525 trace_kvm_s390_sie_enter(vcpu, cpuflags);
3526
3527 return 0;
3528}
3529
3530static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
3531{
3532 struct kvm_s390_pgm_info pgm_info = {
3533 .code = PGM_ADDRESSING,
3534 };
3535 u8 opcode, ilen;
3536 int rc;
3537
3538 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
3539 trace_kvm_s390_sie_fault(vcpu);
3540
3541 /*
3542 * We want to inject an addressing exception, which is defined as a
3543 * suppressing or terminating exception. However, since we came here
3544 * by a DAT access exception, the PSW still points to the faulting
3545 * instruction since DAT exceptions are nullifying. So we've got
3546 * to look up the current opcode to get the length of the instruction
3547 * to be able to forward the PSW.
3548 */
3549 rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
3550 ilen = insn_length(opcode);
3551 if (rc < 0) {
3552 return rc;
3553 } else if (rc) {
3554 /* Instruction-Fetching Exceptions - we can't detect the ilen.
3555 * Forward by arbitrary ilc, injection will take care of
3556 * nullification if necessary.
3557 */
3558 pgm_info = vcpu->arch.pgm;
3559 ilen = 4;
3560 }
3561 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
3562 kvm_s390_forward_psw(vcpu, ilen);
3563 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
3564}
3565
3566static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
3567{
3568 struct mcck_volatile_info *mcck_info;
3569 struct sie_page *sie_page;
3570
3571 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
3572 vcpu->arch.sie_block->icptcode);
3573 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
3574
3575 if (guestdbg_enabled(vcpu))
3576 kvm_s390_restore_guest_per_regs(vcpu);
3577
3578 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
3579 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
3580
3581 if (exit_reason == -EINTR) {
3582 VCPU_EVENT(vcpu, 3, "%s", "machine check");
3583 sie_page = container_of(vcpu->arch.sie_block,
3584 struct sie_page, sie_block);
3585 mcck_info = &sie_page->mcck_info;
3586 kvm_s390_reinject_machine_check(vcpu, mcck_info);
3587 return 0;
3588 }
3589
3590 if (vcpu->arch.sie_block->icptcode > 0) {
3591 int rc = kvm_handle_sie_intercept(vcpu);
3592
3593 if (rc != -EOPNOTSUPP)
3594 return rc;
3595 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
3596 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
3597 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
3598 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
3599 return -EREMOTE;
3600 } else if (exit_reason != -EFAULT) {
3601 vcpu->stat.exit_null++;
3602 return 0;
3603 } else if (kvm_is_ucontrol(vcpu->kvm)) {
3604 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
3605 vcpu->run->s390_ucontrol.trans_exc_code =
3606 current->thread.gmap_addr;
3607 vcpu->run->s390_ucontrol.pgm_code = 0x10;
3608 return -EREMOTE;
3609 } else if (current->thread.gmap_pfault) {
3610 trace_kvm_s390_major_guest_pfault(vcpu);
3611 current->thread.gmap_pfault = 0;
3612 if (kvm_arch_setup_async_pf(vcpu))
3613 return 0;
3614 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
3615 }
3616 return vcpu_post_run_fault_in_sie(vcpu);
3617}
3618
3619static int __vcpu_run(struct kvm_vcpu *vcpu)
3620{
3621 int rc, exit_reason;
3622
3623 /*
3624 * We try to hold kvm->srcu during most of vcpu_run (except when run-
3625 * ning the guest), so that memslots (and other stuff) are protected
3626 */
3627 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3628
3629 do {
3630 rc = vcpu_pre_run(vcpu);
3631 if (rc)
3632 break;
3633
3634 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
3635 /*
3636 * As PF_VCPU will be used in fault handler, between
3637 * guest_enter and guest_exit should be no uaccess.
3638 */
3639 local_irq_disable();
3640 guest_enter_irqoff();
3641 __disable_cpu_timer_accounting(vcpu);
3642 local_irq_enable();
3643 exit_reason = sie64a(vcpu->arch.sie_block,
3644 vcpu->run->s.regs.gprs);
3645 local_irq_disable();
3646 __enable_cpu_timer_accounting(vcpu);
3647 guest_exit_irqoff();
3648 local_irq_enable();
3649 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3650
3651 rc = vcpu_post_run(vcpu, exit_reason);
3652 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3653
3654 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
3655 return rc;
3656}
3657
3658static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3659{
3660 struct runtime_instr_cb *riccb;
3661 struct gs_cb *gscb;
3662
3663 riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
3664 gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
3665 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
3666 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
3667 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
3668 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
3669 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
3670 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
3671 /* some control register changes require a tlb flush */
3672 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3673 }
3674 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
3675 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
3676 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
3677 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
3678 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
3679 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
3680 }
3681 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
3682 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
3683 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
3684 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
3685 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3686 kvm_clear_async_pf_completion_queue(vcpu);
3687 }
3688 /*
3689 * If userspace sets the riccb (e.g. after migration) to a valid state,
3690 * we should enable RI here instead of doing the lazy enablement.
3691 */
3692 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
3693 test_kvm_facility(vcpu->kvm, 64) &&
3694 riccb->v &&
3695 !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
3696 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
3697 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
3698 }
3699 /*
3700 * If userspace sets the gscb (e.g. after migration) to non-zero,
3701 * we should enable GS here instead of doing the lazy enablement.
3702 */
3703 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
3704 test_kvm_facility(vcpu->kvm, 133) &&
3705 gscb->gssm &&
3706 !vcpu->arch.gs_enabled) {
3707 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
3708 vcpu->arch.sie_block->ecb |= ECB_GS;
3709 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3710 vcpu->arch.gs_enabled = 1;
3711 }
3712 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) &&
3713 test_kvm_facility(vcpu->kvm, 82)) {
3714 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
3715 vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
3716 }
3717 save_access_regs(vcpu->arch.host_acrs);
3718 restore_access_regs(vcpu->run->s.regs.acrs);
3719 /* save host (userspace) fprs/vrs */
3720 save_fpu_regs();
3721 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
3722 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
3723 if (MACHINE_HAS_VX)
3724 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
3725 else
3726 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
3727 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
3728 if (test_fp_ctl(current->thread.fpu.fpc))
3729 /* User space provided an invalid FPC, let's clear it */
3730 current->thread.fpu.fpc = 0;
3731 if (MACHINE_HAS_GS) {
3732 preempt_disable();
3733 __ctl_set_bit(2, 4);
3734 if (current->thread.gs_cb) {
3735 vcpu->arch.host_gscb = current->thread.gs_cb;
3736 save_gs_cb(vcpu->arch.host_gscb);
3737 }
3738 if (vcpu->arch.gs_enabled) {
3739 current->thread.gs_cb = (struct gs_cb *)
3740 &vcpu->run->s.regs.gscb;
3741 restore_gs_cb(current->thread.gs_cb);
3742 }
3743 preempt_enable();
3744 }
3745 /* SIE will load etoken directly from SDNX and therefore kvm_run */
3746
3747 kvm_run->kvm_dirty_regs = 0;
3748}
3749
3750static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3751{
3752 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
3753 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
3754 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
3755 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
3756 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
3757 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
3758 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
3759 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
3760 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
3761 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
3762 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
3763 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
3764 kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
3765 save_access_regs(vcpu->run->s.regs.acrs);
3766 restore_access_regs(vcpu->arch.host_acrs);
3767 /* Save guest register state */
3768 save_fpu_regs();
3769 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3770 /* Restore will be done lazily at return */
3771 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
3772 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
3773 if (MACHINE_HAS_GS) {
3774 __ctl_set_bit(2, 4);
3775 if (vcpu->arch.gs_enabled)
3776 save_gs_cb(current->thread.gs_cb);
3777 preempt_disable();
3778 current->thread.gs_cb = vcpu->arch.host_gscb;
3779 restore_gs_cb(vcpu->arch.host_gscb);
3780 preempt_enable();
3781 if (!vcpu->arch.host_gscb)
3782 __ctl_clear_bit(2, 4);
3783 vcpu->arch.host_gscb = NULL;
3784 }
3785 /* SIE will save etoken directly into SDNX and therefore kvm_run */
3786}
3787
3788int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3789{
3790 int rc;
3791
3792 if (kvm_run->immediate_exit)
3793 return -EINTR;
3794
3795 if (guestdbg_exit_pending(vcpu)) {
3796 kvm_s390_prepare_debug_exit(vcpu);
3797 return 0;
3798 }
3799
3800 kvm_sigset_activate(vcpu);
3801
3802 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
3803 kvm_s390_vcpu_start(vcpu);
3804 } else if (is_vcpu_stopped(vcpu)) {
3805 pr_err_ratelimited("can't run stopped vcpu %d\n",
3806 vcpu->vcpu_id);
3807 return -EINVAL;
3808 }
3809
3810 sync_regs(vcpu, kvm_run);
3811 enable_cpu_timer_accounting(vcpu);
3812
3813 might_fault();
3814 rc = __vcpu_run(vcpu);
3815
3816 if (signal_pending(current) && !rc) {
3817 kvm_run->exit_reason = KVM_EXIT_INTR;
3818 rc = -EINTR;
3819 }
3820
3821 if (guestdbg_exit_pending(vcpu) && !rc) {
3822 kvm_s390_prepare_debug_exit(vcpu);
3823 rc = 0;
3824 }
3825
3826 if (rc == -EREMOTE) {
3827 /* userspace support is needed, kvm_run has been prepared */
3828 rc = 0;
3829 }
3830
3831 disable_cpu_timer_accounting(vcpu);
3832 store_regs(vcpu, kvm_run);
3833
3834 kvm_sigset_deactivate(vcpu);
3835
3836 vcpu->stat.exit_userspace++;
3837 return rc;
3838}
3839
3840/*
3841 * store status at address
3842 * we use have two special cases:
3843 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
3844 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
3845 */
3846int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
3847{
3848 unsigned char archmode = 1;
3849 freg_t fprs[NUM_FPRS];
3850 unsigned int px;
3851 u64 clkcomp, cputm;
3852 int rc;
3853
3854 px = kvm_s390_get_prefix(vcpu);
3855 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
3856 if (write_guest_abs(vcpu, 163, &archmode, 1))
3857 return -EFAULT;
3858 gpa = 0;
3859 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
3860 if (write_guest_real(vcpu, 163, &archmode, 1))
3861 return -EFAULT;
3862 gpa = px;
3863 } else
3864 gpa -= __LC_FPREGS_SAVE_AREA;
3865
3866 /* manually convert vector registers if necessary */
3867 if (MACHINE_HAS_VX) {
3868 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
3869 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
3870 fprs, 128);
3871 } else {
3872 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
3873 vcpu->run->s.regs.fprs, 128);
3874 }
3875 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
3876 vcpu->run->s.regs.gprs, 128);
3877 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
3878 &vcpu->arch.sie_block->gpsw, 16);
3879 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
3880 &px, 4);
3881 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
3882 &vcpu->run->s.regs.fpc, 4);
3883 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
3884 &vcpu->arch.sie_block->todpr, 4);
3885 cputm = kvm_s390_get_cpu_timer(vcpu);
3886 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
3887 &cputm, 8);
3888 clkcomp = vcpu->arch.sie_block->ckc >> 8;
3889 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
3890 &clkcomp, 8);
3891 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
3892 &vcpu->run->s.regs.acrs, 64);
3893 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
3894 &vcpu->arch.sie_block->gcr, 128);
3895 return rc ? -EFAULT : 0;
3896}
3897
3898int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
3899{
3900 /*
3901 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
3902 * switch in the run ioctl. Let's update our copies before we save
3903 * it into the save area
3904 */
3905 save_fpu_regs();
3906 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3907 save_access_regs(vcpu->run->s.regs.acrs);
3908
3909 return kvm_s390_store_status_unloaded(vcpu, addr);
3910}
3911
3912static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3913{
3914 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
3915 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
3916}
3917
3918static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
3919{
3920 unsigned int i;
3921 struct kvm_vcpu *vcpu;
3922
3923 kvm_for_each_vcpu(i, vcpu, kvm) {
3924 __disable_ibs_on_vcpu(vcpu);
3925 }
3926}
3927
3928static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3929{
3930 if (!sclp.has_ibs)
3931 return;
3932 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
3933 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
3934}
3935
3936void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
3937{
3938 int i, online_vcpus, started_vcpus = 0;
3939
3940 if (!is_vcpu_stopped(vcpu))
3941 return;
3942
3943 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
3944 /* Only one cpu at a time may enter/leave the STOPPED state. */
3945 spin_lock(&vcpu->kvm->arch.start_stop_lock);
3946 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3947
3948 for (i = 0; i < online_vcpus; i++) {
3949 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
3950 started_vcpus++;
3951 }
3952
3953 if (started_vcpus == 0) {
3954 /* we're the only active VCPU -> speed it up */
3955 __enable_ibs_on_vcpu(vcpu);
3956 } else if (started_vcpus == 1) {
3957 /*
3958 * As we are starting a second VCPU, we have to disable
3959 * the IBS facility on all VCPUs to remove potentially
3960 * oustanding ENABLE requests.
3961 */
3962 __disable_ibs_on_all_vcpus(vcpu->kvm);
3963 }
3964
3965 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
3966 /*
3967 * Another VCPU might have used IBS while we were offline.
3968 * Let's play safe and flush the VCPU at startup.
3969 */
3970 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3971 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
3972 return;
3973}
3974
3975void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
3976{
3977 int i, online_vcpus, started_vcpus = 0;
3978 struct kvm_vcpu *started_vcpu = NULL;
3979
3980 if (is_vcpu_stopped(vcpu))
3981 return;
3982
3983 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
3984 /* Only one cpu at a time may enter/leave the STOPPED state. */
3985 spin_lock(&vcpu->kvm->arch.start_stop_lock);
3986 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3987
3988 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
3989 kvm_s390_clear_stop_irq(vcpu);
3990
3991 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
3992 __disable_ibs_on_vcpu(vcpu);
3993
3994 for (i = 0; i < online_vcpus; i++) {
3995 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
3996 started_vcpus++;
3997 started_vcpu = vcpu->kvm->vcpus[i];
3998 }
3999 }
4000
4001 if (started_vcpus == 1) {
4002 /*
4003 * As we only have one VCPU left, we want to enable the
4004 * IBS facility for that VCPU to speed it up.
4005 */
4006 __enable_ibs_on_vcpu(started_vcpu);
4007 }
4008
4009 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4010 return;
4011}
4012
4013static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
4014 struct kvm_enable_cap *cap)
4015{
4016 int r;
4017
4018 if (cap->flags)
4019 return -EINVAL;
4020
4021 switch (cap->cap) {
4022 case KVM_CAP_S390_CSS_SUPPORT:
4023 if (!vcpu->kvm->arch.css_support) {
4024 vcpu->kvm->arch.css_support = 1;
4025 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
4026 trace_kvm_s390_enable_css(vcpu->kvm);
4027 }
4028 r = 0;
4029 break;
4030 default:
4031 r = -EINVAL;
4032 break;
4033 }
4034 return r;
4035}
4036
4037static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
4038 struct kvm_s390_mem_op *mop)
4039{
4040 void __user *uaddr = (void __user *)mop->buf;
4041 void *tmpbuf = NULL;
4042 int r, srcu_idx;
4043 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
4044 | KVM_S390_MEMOP_F_CHECK_ONLY;
4045
4046 if (mop->flags & ~supported_flags || mop->ar >= NUM_ACRS || !mop->size)
4047 return -EINVAL;
4048
4049 if (mop->size > MEM_OP_MAX_SIZE)
4050 return -E2BIG;
4051
4052 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
4053 tmpbuf = vmalloc(mop->size);
4054 if (!tmpbuf)
4055 return -ENOMEM;
4056 }
4057
4058 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
4059
4060 switch (mop->op) {
4061 case KVM_S390_MEMOP_LOGICAL_READ:
4062 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
4063 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
4064 mop->size, GACC_FETCH);
4065 break;
4066 }
4067 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
4068 if (r == 0) {
4069 if (copy_to_user(uaddr, tmpbuf, mop->size))
4070 r = -EFAULT;
4071 }
4072 break;
4073 case KVM_S390_MEMOP_LOGICAL_WRITE:
4074 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
4075 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
4076 mop->size, GACC_STORE);
4077 break;
4078 }
4079 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
4080 r = -EFAULT;
4081 break;
4082 }
4083 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
4084 break;
4085 default:
4086 r = -EINVAL;
4087 }
4088
4089 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
4090
4091 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
4092 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
4093
4094 vfree(tmpbuf);
4095 return r;
4096}
4097
4098long kvm_arch_vcpu_ioctl(struct file *filp,
4099 unsigned int ioctl, unsigned long arg)
4100{
4101 struct kvm_vcpu *vcpu = filp->private_data;
4102 void __user *argp = (void __user *)arg;
4103 int idx;
4104 long r;
4105
4106 switch (ioctl) {
4107 case KVM_S390_IRQ: {
4108 struct kvm_s390_irq s390irq;
4109
4110 r = -EFAULT;
4111 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
4112 break;
4113 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
4114 break;
4115 }
4116 case KVM_S390_INTERRUPT: {
4117 struct kvm_s390_interrupt s390int;
4118 struct kvm_s390_irq s390irq = {};
4119
4120 r = -EFAULT;
4121 if (copy_from_user(&s390int, argp, sizeof(s390int)))
4122 break;
4123 if (s390int_to_s390irq(&s390int, &s390irq))
4124 return -EINVAL;
4125 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
4126 break;
4127 }
4128 case KVM_S390_STORE_STATUS:
4129 idx = srcu_read_lock(&vcpu->kvm->srcu);
4130 r = kvm_s390_store_status_unloaded(vcpu, arg);
4131 srcu_read_unlock(&vcpu->kvm->srcu, idx);
4132 break;
4133 case KVM_S390_SET_INITIAL_PSW: {
4134 psw_t psw;
4135
4136 r = -EFAULT;
4137 if (copy_from_user(&psw, argp, sizeof(psw)))
4138 break;
4139 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
4140 break;
4141 }
4142 case KVM_S390_INITIAL_RESET:
4143 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
4144 break;
4145 case KVM_SET_ONE_REG:
4146 case KVM_GET_ONE_REG: {
4147 struct kvm_one_reg reg;
4148 r = -EFAULT;
4149 if (copy_from_user(&reg, argp, sizeof(reg)))
4150 break;
4151 if (ioctl == KVM_SET_ONE_REG)
4152 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
4153 else
4154 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
4155 break;
4156 }
4157#ifdef CONFIG_KVM_S390_UCONTROL
4158 case KVM_S390_UCAS_MAP: {
4159 struct kvm_s390_ucas_mapping ucasmap;
4160
4161 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4162 r = -EFAULT;
4163 break;
4164 }
4165
4166 if (!kvm_is_ucontrol(vcpu->kvm)) {
4167 r = -EINVAL;
4168 break;
4169 }
4170
4171 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
4172 ucasmap.vcpu_addr, ucasmap.length);
4173 break;
4174 }
4175 case KVM_S390_UCAS_UNMAP: {
4176 struct kvm_s390_ucas_mapping ucasmap;
4177
4178 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4179 r = -EFAULT;
4180 break;
4181 }
4182
4183 if (!kvm_is_ucontrol(vcpu->kvm)) {
4184 r = -EINVAL;
4185 break;
4186 }
4187
4188 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
4189 ucasmap.length);
4190 break;
4191 }
4192#endif
4193 case KVM_S390_VCPU_FAULT: {
4194 r = gmap_fault(vcpu->arch.gmap, arg, 0);
4195 break;
4196 }
4197 case KVM_ENABLE_CAP:
4198 {
4199 struct kvm_enable_cap cap;
4200 r = -EFAULT;
4201 if (copy_from_user(&cap, argp, sizeof(cap)))
4202 break;
4203 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
4204 break;
4205 }
4206 case KVM_S390_MEM_OP: {
4207 struct kvm_s390_mem_op mem_op;
4208
4209 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
4210 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
4211 else
4212 r = -EFAULT;
4213 break;
4214 }
4215 case KVM_S390_SET_IRQ_STATE: {
4216 struct kvm_s390_irq_state irq_state;
4217
4218 r = -EFAULT;
4219 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
4220 break;
4221 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
4222 irq_state.len == 0 ||
4223 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
4224 r = -EINVAL;
4225 break;
4226 }
4227 /* do not use irq_state.flags, it will break old QEMUs */
4228 r = kvm_s390_set_irq_state(vcpu,
4229 (void __user *) irq_state.buf,
4230 irq_state.len);
4231 break;
4232 }
4233 case KVM_S390_GET_IRQ_STATE: {
4234 struct kvm_s390_irq_state irq_state;
4235
4236 r = -EFAULT;
4237 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
4238 break;
4239 if (irq_state.len == 0) {
4240 r = -EINVAL;
4241 break;
4242 }
4243 /* do not use irq_state.flags, it will break old QEMUs */
4244 r = kvm_s390_get_irq_state(vcpu,
4245 (__u8 __user *) irq_state.buf,
4246 irq_state.len);
4247 break;
4248 }
4249 default:
4250 r = -ENOTTY;
4251 }
4252 return r;
4253}
4254
4255int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
4256{
4257#ifdef CONFIG_KVM_S390_UCONTROL
4258 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
4259 && (kvm_is_ucontrol(vcpu->kvm))) {
4260 vmf->page = virt_to_page(vcpu->arch.sie_block);
4261 get_page(vmf->page);
4262 return 0;
4263 }
4264#endif
4265 return VM_FAULT_SIGBUS;
4266}
4267
4268int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
4269 unsigned long npages)
4270{
4271 return 0;
4272}
4273
4274/* Section: memory related */
4275int kvm_arch_prepare_memory_region(struct kvm *kvm,
4276 struct kvm_memory_slot *memslot,
4277 const struct kvm_userspace_memory_region *mem,
4278 enum kvm_mr_change change)
4279{
4280 /* A few sanity checks. We can have memory slots which have to be
4281 located/ended at a segment boundary (1MB). The memory in userland is
4282 ok to be fragmented into various different vmas. It is okay to mmap()
4283 and munmap() stuff in this slot after doing this call at any time */
4284
4285 if (mem->userspace_addr & 0xffffful)
4286 return -EINVAL;
4287
4288 if (mem->memory_size & 0xffffful)
4289 return -EINVAL;
4290
4291 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
4292 return -EINVAL;
4293
4294 return 0;
4295}
4296
4297void kvm_arch_commit_memory_region(struct kvm *kvm,
4298 const struct kvm_userspace_memory_region *mem,
4299 const struct kvm_memory_slot *old,
4300 const struct kvm_memory_slot *new,
4301 enum kvm_mr_change change)
4302{
4303 int rc = 0;
4304
4305 switch (change) {
4306 case KVM_MR_DELETE:
4307 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
4308 old->npages * PAGE_SIZE);
4309 break;
4310 case KVM_MR_MOVE:
4311 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
4312 old->npages * PAGE_SIZE);
4313 if (rc)
4314 break;
4315 /* FALLTHROUGH */
4316 case KVM_MR_CREATE:
4317 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
4318 mem->guest_phys_addr, mem->memory_size);
4319 break;
4320 case KVM_MR_FLAGS_ONLY:
4321 break;
4322 default:
4323 WARN(1, "Unknown KVM MR CHANGE: %d\n", change);
4324 }
4325 if (rc)
4326 pr_warn("failed to commit memory region\n");
4327 return;
4328}
4329
4330static inline unsigned long nonhyp_mask(int i)
4331{
4332 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
4333
4334 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
4335}
4336
4337void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
4338{
4339 vcpu->valid_wakeup = false;
4340}
4341
4342static int __init kvm_s390_init(void)
4343{
4344 int i;
4345
4346 if (!sclp.has_sief2) {
4347 pr_info("SIE not available\n");
4348 return -ENODEV;
4349 }
4350
4351 for (i = 0; i < 16; i++)
4352 kvm_s390_fac_base[i] |=
4353 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
4354
4355 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
4356}
4357
4358static void __exit kvm_s390_exit(void)
4359{
4360 kvm_exit();
4361}
4362
4363module_init(kvm_s390_init);
4364module_exit(kvm_s390_exit);
4365
4366/*
4367 * Enable autoloading of the kvm module.
4368 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
4369 * since x86 takes a different approach.
4370 */
4371#include <linux/miscdevice.h>
4372MODULE_ALIAS_MISCDEV(KVM_MINOR);
4373MODULE_ALIAS("devname:kvm");
ubuntu-bionic-kernel mirror