]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blob - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-artful-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux
[mirror_ubuntu-artful-kernel.git] / arch / s390 / kvm / kvm-s390.c
1 /*
2 * hosting zSeries kernel virtual machines
3 *
4 * Copyright IBM Corp. 2008, 2009
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
15 */
16
17 #include <linux/compiler.h>
18 #include <linux/err.h>
19 #include <linux/fs.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/mman.h>
25 #include <linux/module.h>
26 #include <linux/moduleparam.h>
27 #include <linux/random.h>
28 #include <linux/slab.h>
29 #include <linux/timer.h>
30 #include <linux/vmalloc.h>
31 #include <linux/bitmap.h>
32 #include <asm/asm-offsets.h>
33 #include <asm/lowcore.h>
34 #include <asm/stp.h>
35 #include <asm/pgtable.h>
36 #include <asm/gmap.h>
37 #include <asm/nmi.h>
38 #include <asm/switch_to.h>
39 #include <asm/isc.h>
40 #include <asm/sclp.h>
41 #include <asm/cpacf.h>
42 #include <asm/timex.h>
43 #include "kvm-s390.h"
44 #include "gaccess.h"
45
46 #define KMSG_COMPONENT "kvm-s390"
47 #undef pr_fmt
48 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
49
50 #define CREATE_TRACE_POINTS
51 #include "trace.h"
52 #include "trace-s390.h"
53
54 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
55 #define LOCAL_IRQS 32
56 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
57 (KVM_MAX_VCPUS + LOCAL_IRQS))
58
59 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
60
61 struct kvm_stats_debugfs_item debugfs_entries[] = {
62 { "userspace_handled", VCPU_STAT(exit_userspace) },
63 { "exit_null", VCPU_STAT(exit_null) },
64 { "exit_validity", VCPU_STAT(exit_validity) },
65 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
66 { "exit_external_request", VCPU_STAT(exit_external_request) },
67 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
68 { "exit_instruction", VCPU_STAT(exit_instruction) },
69 { "exit_pei", VCPU_STAT(exit_pei) },
70 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
71 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
72 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
73 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
74 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
75 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
76 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
77 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
78 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
79 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
80 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
81 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
82 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
83 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
84 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
85 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
86 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
87 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
88 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
89 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
90 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
91 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
92 { "instruction_spx", VCPU_STAT(instruction_spx) },
93 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
94 { "instruction_stap", VCPU_STAT(instruction_stap) },
95 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
96 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
97 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
98 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
99 { "instruction_essa", VCPU_STAT(instruction_essa) },
100 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
101 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
102 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
103 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
104 { "instruction_sie", VCPU_STAT(instruction_sie) },
105 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
106 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
107 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
108 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
109 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
110 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
111 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
112 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
113 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
114 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
115 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
116 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
117 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
118 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
119 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
120 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
121 { "diagnose_10", VCPU_STAT(diagnose_10) },
122 { "diagnose_44", VCPU_STAT(diagnose_44) },
123 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
124 { "diagnose_258", VCPU_STAT(diagnose_258) },
125 { "diagnose_308", VCPU_STAT(diagnose_308) },
126 { "diagnose_500", VCPU_STAT(diagnose_500) },
127 { NULL }
128 };
129
130 /* allow nested virtualization in KVM (if enabled by user space) */
131 static int nested;
132 module_param(nested, int, S_IRUGO);
133 MODULE_PARM_DESC(nested, "Nested virtualization support");
134
135 /* upper facilities limit for kvm */
136 unsigned long kvm_s390_fac_list_mask[16] = { FACILITIES_KVM };
137
138 unsigned long kvm_s390_fac_list_mask_size(void)
139 {
140 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
141 return ARRAY_SIZE(kvm_s390_fac_list_mask);
142 }
143
144 /* available cpu features supported by kvm */
145 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
146 /* available subfunctions indicated via query / "test bit" */
147 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
148
149 static struct gmap_notifier gmap_notifier;
150 static struct gmap_notifier vsie_gmap_notifier;
151 debug_info_t *kvm_s390_dbf;
152
153 /* Section: not file related */
154 int kvm_arch_hardware_enable(void)
155 {
156 /* every s390 is virtualization enabled ;-) */
157 return 0;
158 }
159
160 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
161 unsigned long end);
162
163 /*
164 * This callback is executed during stop_machine(). All CPUs are therefore
165 * temporarily stopped. In order not to change guest behavior, we have to
166 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
167 * so a CPU won't be stopped while calculating with the epoch.
168 */
169 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
170 void *v)
171 {
172 struct kvm *kvm;
173 struct kvm_vcpu *vcpu;
174 int i;
175 unsigned long long *delta = v;
176
177 list_for_each_entry(kvm, &vm_list, vm_list) {
178 kvm->arch.epoch -= *delta;
179 kvm_for_each_vcpu(i, vcpu, kvm) {
180 vcpu->arch.sie_block->epoch -= *delta;
181 if (vcpu->arch.cputm_enabled)
182 vcpu->arch.cputm_start += *delta;
183 if (vcpu->arch.vsie_block)
184 vcpu->arch.vsie_block->epoch -= *delta;
185 }
186 }
187 return NOTIFY_OK;
188 }
189
190 static struct notifier_block kvm_clock_notifier = {
191 .notifier_call = kvm_clock_sync,
192 };
193
194 int kvm_arch_hardware_setup(void)
195 {
196 gmap_notifier.notifier_call = kvm_gmap_notifier;
197 gmap_register_pte_notifier(&gmap_notifier);
198 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
199 gmap_register_pte_notifier(&vsie_gmap_notifier);
200 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
201 &kvm_clock_notifier);
202 return 0;
203 }
204
205 void kvm_arch_hardware_unsetup(void)
206 {
207 gmap_unregister_pte_notifier(&gmap_notifier);
208 gmap_unregister_pte_notifier(&vsie_gmap_notifier);
209 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
210 &kvm_clock_notifier);
211 }
212
213 static void allow_cpu_feat(unsigned long nr)
214 {
215 set_bit_inv(nr, kvm_s390_available_cpu_feat);
216 }
217
218 static inline int plo_test_bit(unsigned char nr)
219 {
220 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
221 int cc = 3; /* subfunction not available */
222
223 asm volatile(
224 /* Parameter registers are ignored for "test bit" */
225 " plo 0,0,0,0(0)\n"
226 " ipm %0\n"
227 " srl %0,28\n"
228 : "=d" (cc)
229 : "d" (r0)
230 : "cc");
231 return cc == 0;
232 }
233
234 static void kvm_s390_cpu_feat_init(void)
235 {
236 int i;
237
238 for (i = 0; i < 256; ++i) {
239 if (plo_test_bit(i))
240 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
241 }
242
243 if (test_facility(28)) /* TOD-clock steering */
244 ptff(kvm_s390_available_subfunc.ptff,
245 sizeof(kvm_s390_available_subfunc.ptff),
246 PTFF_QAF);
247
248 if (test_facility(17)) { /* MSA */
249 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
250 kvm_s390_available_subfunc.kmac);
251 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
252 kvm_s390_available_subfunc.kmc);
253 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
254 kvm_s390_available_subfunc.km);
255 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
256 kvm_s390_available_subfunc.kimd);
257 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
258 kvm_s390_available_subfunc.klmd);
259 }
260 if (test_facility(76)) /* MSA3 */
261 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
262 kvm_s390_available_subfunc.pckmo);
263 if (test_facility(77)) { /* MSA4 */
264 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
265 kvm_s390_available_subfunc.kmctr);
266 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
267 kvm_s390_available_subfunc.kmf);
268 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
269 kvm_s390_available_subfunc.kmo);
270 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
271 kvm_s390_available_subfunc.pcc);
272 }
273 if (test_facility(57)) /* MSA5 */
274 __cpacf_query(CPACF_PPNO, (cpacf_mask_t *)
275 kvm_s390_available_subfunc.ppno);
276
277 if (MACHINE_HAS_ESOP)
278 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
279 /*
280 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
281 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
282 */
283 if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
284 !test_facility(3) || !nested)
285 return;
286 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
287 if (sclp.has_64bscao)
288 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
289 if (sclp.has_siif)
290 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
291 if (sclp.has_gpere)
292 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
293 if (sclp.has_gsls)
294 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
295 if (sclp.has_ib)
296 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
297 if (sclp.has_cei)
298 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
299 if (sclp.has_ibs)
300 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
301 /*
302 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
303 * all skey handling functions read/set the skey from the PGSTE
304 * instead of the real storage key.
305 *
306 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
307 * pages being detected as preserved although they are resident.
308 *
309 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
310 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
311 *
312 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
313 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
314 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
315 *
316 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
317 * cannot easily shadow the SCA because of the ipte lock.
318 */
319 }
320
321 int kvm_arch_init(void *opaque)
322 {
323 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
324 if (!kvm_s390_dbf)
325 return -ENOMEM;
326
327 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
328 debug_unregister(kvm_s390_dbf);
329 return -ENOMEM;
330 }
331
332 kvm_s390_cpu_feat_init();
333
334 /* Register floating interrupt controller interface. */
335 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
336 }
337
338 void kvm_arch_exit(void)
339 {
340 debug_unregister(kvm_s390_dbf);
341 }
342
343 /* Section: device related */
344 long kvm_arch_dev_ioctl(struct file *filp,
345 unsigned int ioctl, unsigned long arg)
346 {
347 if (ioctl == KVM_S390_ENABLE_SIE)
348 return s390_enable_sie();
349 return -EINVAL;
350 }
351
352 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
353 {
354 int r;
355
356 switch (ext) {
357 case KVM_CAP_S390_PSW:
358 case KVM_CAP_S390_GMAP:
359 case KVM_CAP_SYNC_MMU:
360 #ifdef CONFIG_KVM_S390_UCONTROL
361 case KVM_CAP_S390_UCONTROL:
362 #endif
363 case KVM_CAP_ASYNC_PF:
364 case KVM_CAP_SYNC_REGS:
365 case KVM_CAP_ONE_REG:
366 case KVM_CAP_ENABLE_CAP:
367 case KVM_CAP_S390_CSS_SUPPORT:
368 case KVM_CAP_IOEVENTFD:
369 case KVM_CAP_DEVICE_CTRL:
370 case KVM_CAP_ENABLE_CAP_VM:
371 case KVM_CAP_S390_IRQCHIP:
372 case KVM_CAP_VM_ATTRIBUTES:
373 case KVM_CAP_MP_STATE:
374 case KVM_CAP_S390_INJECT_IRQ:
375 case KVM_CAP_S390_USER_SIGP:
376 case KVM_CAP_S390_USER_STSI:
377 case KVM_CAP_S390_SKEYS:
378 case KVM_CAP_S390_IRQ_STATE:
379 case KVM_CAP_S390_USER_INSTR0:
380 r = 1;
381 break;
382 case KVM_CAP_S390_MEM_OP:
383 r = MEM_OP_MAX_SIZE;
384 break;
385 case KVM_CAP_NR_VCPUS:
386 case KVM_CAP_MAX_VCPUS:
387 r = KVM_S390_BSCA_CPU_SLOTS;
388 if (!kvm_s390_use_sca_entries())
389 r = KVM_MAX_VCPUS;
390 else if (sclp.has_esca && sclp.has_64bscao)
391 r = KVM_S390_ESCA_CPU_SLOTS;
392 break;
393 case KVM_CAP_NR_MEMSLOTS:
394 r = KVM_USER_MEM_SLOTS;
395 break;
396 case KVM_CAP_S390_COW:
397 r = MACHINE_HAS_ESOP;
398 break;
399 case KVM_CAP_S390_VECTOR_REGISTERS:
400 r = MACHINE_HAS_VX;
401 break;
402 case KVM_CAP_S390_RI:
403 r = test_facility(64);
404 break;
405 default:
406 r = 0;
407 }
408 return r;
409 }
410
411 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
412 struct kvm_memory_slot *memslot)
413 {
414 gfn_t cur_gfn, last_gfn;
415 unsigned long address;
416 struct gmap *gmap = kvm->arch.gmap;
417
418 /* Loop over all guest pages */
419 last_gfn = memslot->base_gfn + memslot->npages;
420 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
421 address = gfn_to_hva_memslot(memslot, cur_gfn);
422
423 if (test_and_clear_guest_dirty(gmap->mm, address))
424 mark_page_dirty(kvm, cur_gfn);
425 if (fatal_signal_pending(current))
426 return;
427 cond_resched();
428 }
429 }
430
431 /* Section: vm related */
432 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
433
434 /*
435 * Get (and clear) the dirty memory log for a memory slot.
436 */
437 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
438 struct kvm_dirty_log *log)
439 {
440 int r;
441 unsigned long n;
442 struct kvm_memslots *slots;
443 struct kvm_memory_slot *memslot;
444 int is_dirty = 0;
445
446 mutex_lock(&kvm->slots_lock);
447
448 r = -EINVAL;
449 if (log->slot >= KVM_USER_MEM_SLOTS)
450 goto out;
451
452 slots = kvm_memslots(kvm);
453 memslot = id_to_memslot(slots, log->slot);
454 r = -ENOENT;
455 if (!memslot->dirty_bitmap)
456 goto out;
457
458 kvm_s390_sync_dirty_log(kvm, memslot);
459 r = kvm_get_dirty_log(kvm, log, &is_dirty);
460 if (r)
461 goto out;
462
463 /* Clear the dirty log */
464 if (is_dirty) {
465 n = kvm_dirty_bitmap_bytes(memslot);
466 memset(memslot->dirty_bitmap, 0, n);
467 }
468 r = 0;
469 out:
470 mutex_unlock(&kvm->slots_lock);
471 return r;
472 }
473
474 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
475 {
476 unsigned int i;
477 struct kvm_vcpu *vcpu;
478
479 kvm_for_each_vcpu(i, vcpu, kvm) {
480 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
481 }
482 }
483
484 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
485 {
486 int r;
487
488 if (cap->flags)
489 return -EINVAL;
490
491 switch (cap->cap) {
492 case KVM_CAP_S390_IRQCHIP:
493 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
494 kvm->arch.use_irqchip = 1;
495 r = 0;
496 break;
497 case KVM_CAP_S390_USER_SIGP:
498 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
499 kvm->arch.user_sigp = 1;
500 r = 0;
501 break;
502 case KVM_CAP_S390_VECTOR_REGISTERS:
503 mutex_lock(&kvm->lock);
504 if (kvm->created_vcpus) {
505 r = -EBUSY;
506 } else if (MACHINE_HAS_VX) {
507 set_kvm_facility(kvm->arch.model.fac_mask, 129);
508 set_kvm_facility(kvm->arch.model.fac_list, 129);
509 r = 0;
510 } else
511 r = -EINVAL;
512 mutex_unlock(&kvm->lock);
513 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
514 r ? "(not available)" : "(success)");
515 break;
516 case KVM_CAP_S390_RI:
517 r = -EINVAL;
518 mutex_lock(&kvm->lock);
519 if (kvm->created_vcpus) {
520 r = -EBUSY;
521 } else if (test_facility(64)) {
522 set_kvm_facility(kvm->arch.model.fac_mask, 64);
523 set_kvm_facility(kvm->arch.model.fac_list, 64);
524 r = 0;
525 }
526 mutex_unlock(&kvm->lock);
527 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
528 r ? "(not available)" : "(success)");
529 break;
530 case KVM_CAP_S390_USER_STSI:
531 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
532 kvm->arch.user_stsi = 1;
533 r = 0;
534 break;
535 case KVM_CAP_S390_USER_INSTR0:
536 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
537 kvm->arch.user_instr0 = 1;
538 icpt_operexc_on_all_vcpus(kvm);
539 r = 0;
540 break;
541 default:
542 r = -EINVAL;
543 break;
544 }
545 return r;
546 }
547
548 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
549 {
550 int ret;
551
552 switch (attr->attr) {
553 case KVM_S390_VM_MEM_LIMIT_SIZE:
554 ret = 0;
555 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
556 kvm->arch.mem_limit);
557 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
558 ret = -EFAULT;
559 break;
560 default:
561 ret = -ENXIO;
562 break;
563 }
564 return ret;
565 }
566
567 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
568 {
569 int ret;
570 unsigned int idx;
571 switch (attr->attr) {
572 case KVM_S390_VM_MEM_ENABLE_CMMA:
573 ret = -ENXIO;
574 if (!sclp.has_cmma)
575 break;
576
577 ret = -EBUSY;
578 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
579 mutex_lock(&kvm->lock);
580 if (!kvm->created_vcpus) {
581 kvm->arch.use_cmma = 1;
582 ret = 0;
583 }
584 mutex_unlock(&kvm->lock);
585 break;
586 case KVM_S390_VM_MEM_CLR_CMMA:
587 ret = -ENXIO;
588 if (!sclp.has_cmma)
589 break;
590 ret = -EINVAL;
591 if (!kvm->arch.use_cmma)
592 break;
593
594 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
595 mutex_lock(&kvm->lock);
596 idx = srcu_read_lock(&kvm->srcu);
597 s390_reset_cmma(kvm->arch.gmap->mm);
598 srcu_read_unlock(&kvm->srcu, idx);
599 mutex_unlock(&kvm->lock);
600 ret = 0;
601 break;
602 case KVM_S390_VM_MEM_LIMIT_SIZE: {
603 unsigned long new_limit;
604
605 if (kvm_is_ucontrol(kvm))
606 return -EINVAL;
607
608 if (get_user(new_limit, (u64 __user *)attr->addr))
609 return -EFAULT;
610
611 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
612 new_limit > kvm->arch.mem_limit)
613 return -E2BIG;
614
615 if (!new_limit)
616 return -EINVAL;
617
618 /* gmap_create takes last usable address */
619 if (new_limit != KVM_S390_NO_MEM_LIMIT)
620 new_limit -= 1;
621
622 ret = -EBUSY;
623 mutex_lock(&kvm->lock);
624 if (!kvm->created_vcpus) {
625 /* gmap_create will round the limit up */
626 struct gmap *new = gmap_create(current->mm, new_limit);
627
628 if (!new) {
629 ret = -ENOMEM;
630 } else {
631 gmap_remove(kvm->arch.gmap);
632 new->private = kvm;
633 kvm->arch.gmap = new;
634 ret = 0;
635 }
636 }
637 mutex_unlock(&kvm->lock);
638 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
639 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
640 (void *) kvm->arch.gmap->asce);
641 break;
642 }
643 default:
644 ret = -ENXIO;
645 break;
646 }
647 return ret;
648 }
649
650 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
651
652 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
653 {
654 struct kvm_vcpu *vcpu;
655 int i;
656
657 if (!test_kvm_facility(kvm, 76))
658 return -EINVAL;
659
660 mutex_lock(&kvm->lock);
661 switch (attr->attr) {
662 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
663 get_random_bytes(
664 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
665 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
666 kvm->arch.crypto.aes_kw = 1;
667 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
668 break;
669 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
670 get_random_bytes(
671 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
672 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
673 kvm->arch.crypto.dea_kw = 1;
674 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
675 break;
676 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
677 kvm->arch.crypto.aes_kw = 0;
678 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
679 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
680 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
681 break;
682 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
683 kvm->arch.crypto.dea_kw = 0;
684 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
685 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
686 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
687 break;
688 default:
689 mutex_unlock(&kvm->lock);
690 return -ENXIO;
691 }
692
693 kvm_for_each_vcpu(i, vcpu, kvm) {
694 kvm_s390_vcpu_crypto_setup(vcpu);
695 exit_sie(vcpu);
696 }
697 mutex_unlock(&kvm->lock);
698 return 0;
699 }
700
701 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
702 {
703 u8 gtod_high;
704
705 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
706 sizeof(gtod_high)))
707 return -EFAULT;
708
709 if (gtod_high != 0)
710 return -EINVAL;
711 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
712
713 return 0;
714 }
715
716 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
717 {
718 u64 gtod;
719
720 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
721 return -EFAULT;
722
723 kvm_s390_set_tod_clock(kvm, gtod);
724 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
725 return 0;
726 }
727
728 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
729 {
730 int ret;
731
732 if (attr->flags)
733 return -EINVAL;
734
735 switch (attr->attr) {
736 case KVM_S390_VM_TOD_HIGH:
737 ret = kvm_s390_set_tod_high(kvm, attr);
738 break;
739 case KVM_S390_VM_TOD_LOW:
740 ret = kvm_s390_set_tod_low(kvm, attr);
741 break;
742 default:
743 ret = -ENXIO;
744 break;
745 }
746 return ret;
747 }
748
749 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
750 {
751 u8 gtod_high = 0;
752
753 if (copy_to_user((void __user *)attr->addr, &gtod_high,
754 sizeof(gtod_high)))
755 return -EFAULT;
756 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
757
758 return 0;
759 }
760
761 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
762 {
763 u64 gtod;
764
765 gtod = kvm_s390_get_tod_clock_fast(kvm);
766 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
767 return -EFAULT;
768 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
769
770 return 0;
771 }
772
773 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
774 {
775 int ret;
776
777 if (attr->flags)
778 return -EINVAL;
779
780 switch (attr->attr) {
781 case KVM_S390_VM_TOD_HIGH:
782 ret = kvm_s390_get_tod_high(kvm, attr);
783 break;
784 case KVM_S390_VM_TOD_LOW:
785 ret = kvm_s390_get_tod_low(kvm, attr);
786 break;
787 default:
788 ret = -ENXIO;
789 break;
790 }
791 return ret;
792 }
793
794 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
795 {
796 struct kvm_s390_vm_cpu_processor *proc;
797 u16 lowest_ibc, unblocked_ibc;
798 int ret = 0;
799
800 mutex_lock(&kvm->lock);
801 if (kvm->created_vcpus) {
802 ret = -EBUSY;
803 goto out;
804 }
805 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
806 if (!proc) {
807 ret = -ENOMEM;
808 goto out;
809 }
810 if (!copy_from_user(proc, (void __user *)attr->addr,
811 sizeof(*proc))) {
812 kvm->arch.model.cpuid = proc->cpuid;
813 lowest_ibc = sclp.ibc >> 16 & 0xfff;
814 unblocked_ibc = sclp.ibc & 0xfff;
815 if (lowest_ibc && proc->ibc) {
816 if (proc->ibc > unblocked_ibc)
817 kvm->arch.model.ibc = unblocked_ibc;
818 else if (proc->ibc < lowest_ibc)
819 kvm->arch.model.ibc = lowest_ibc;
820 else
821 kvm->arch.model.ibc = proc->ibc;
822 }
823 memcpy(kvm->arch.model.fac_list, proc->fac_list,
824 S390_ARCH_FAC_LIST_SIZE_BYTE);
825 } else
826 ret = -EFAULT;
827 kfree(proc);
828 out:
829 mutex_unlock(&kvm->lock);
830 return ret;
831 }
832
833 static int kvm_s390_set_processor_feat(struct kvm *kvm,
834 struct kvm_device_attr *attr)
835 {
836 struct kvm_s390_vm_cpu_feat data;
837 int ret = -EBUSY;
838
839 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
840 return -EFAULT;
841 if (!bitmap_subset((unsigned long *) data.feat,
842 kvm_s390_available_cpu_feat,
843 KVM_S390_VM_CPU_FEAT_NR_BITS))
844 return -EINVAL;
845
846 mutex_lock(&kvm->lock);
847 if (!atomic_read(&kvm->online_vcpus)) {
848 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
849 KVM_S390_VM_CPU_FEAT_NR_BITS);
850 ret = 0;
851 }
852 mutex_unlock(&kvm->lock);
853 return ret;
854 }
855
856 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
857 struct kvm_device_attr *attr)
858 {
859 /*
860 * Once supported by kernel + hw, we have to store the subfunctions
861 * in kvm->arch and remember that user space configured them.
862 */
863 return -ENXIO;
864 }
865
866 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
867 {
868 int ret = -ENXIO;
869
870 switch (attr->attr) {
871 case KVM_S390_VM_CPU_PROCESSOR:
872 ret = kvm_s390_set_processor(kvm, attr);
873 break;
874 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
875 ret = kvm_s390_set_processor_feat(kvm, attr);
876 break;
877 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
878 ret = kvm_s390_set_processor_subfunc(kvm, attr);
879 break;
880 }
881 return ret;
882 }
883
884 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
885 {
886 struct kvm_s390_vm_cpu_processor *proc;
887 int ret = 0;
888
889 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
890 if (!proc) {
891 ret = -ENOMEM;
892 goto out;
893 }
894 proc->cpuid = kvm->arch.model.cpuid;
895 proc->ibc = kvm->arch.model.ibc;
896 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
897 S390_ARCH_FAC_LIST_SIZE_BYTE);
898 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
899 ret = -EFAULT;
900 kfree(proc);
901 out:
902 return ret;
903 }
904
905 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
906 {
907 struct kvm_s390_vm_cpu_machine *mach;
908 int ret = 0;
909
910 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
911 if (!mach) {
912 ret = -ENOMEM;
913 goto out;
914 }
915 get_cpu_id((struct cpuid *) &mach->cpuid);
916 mach->ibc = sclp.ibc;
917 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
918 S390_ARCH_FAC_LIST_SIZE_BYTE);
919 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
920 sizeof(S390_lowcore.stfle_fac_list));
921 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
922 ret = -EFAULT;
923 kfree(mach);
924 out:
925 return ret;
926 }
927
928 static int kvm_s390_get_processor_feat(struct kvm *kvm,
929 struct kvm_device_attr *attr)
930 {
931 struct kvm_s390_vm_cpu_feat data;
932
933 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
934 KVM_S390_VM_CPU_FEAT_NR_BITS);
935 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
936 return -EFAULT;
937 return 0;
938 }
939
940 static int kvm_s390_get_machine_feat(struct kvm *kvm,
941 struct kvm_device_attr *attr)
942 {
943 struct kvm_s390_vm_cpu_feat data;
944
945 bitmap_copy((unsigned long *) data.feat,
946 kvm_s390_available_cpu_feat,
947 KVM_S390_VM_CPU_FEAT_NR_BITS);
948 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
949 return -EFAULT;
950 return 0;
951 }
952
953 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
954 struct kvm_device_attr *attr)
955 {
956 /*
957 * Once we can actually configure subfunctions (kernel + hw support),
958 * we have to check if they were already set by user space, if so copy
959 * them from kvm->arch.
960 */
961 return -ENXIO;
962 }
963
964 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
965 struct kvm_device_attr *attr)
966 {
967 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
968 sizeof(struct kvm_s390_vm_cpu_subfunc)))
969 return -EFAULT;
970 return 0;
971 }
972 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
973 {
974 int ret = -ENXIO;
975
976 switch (attr->attr) {
977 case KVM_S390_VM_CPU_PROCESSOR:
978 ret = kvm_s390_get_processor(kvm, attr);
979 break;
980 case KVM_S390_VM_CPU_MACHINE:
981 ret = kvm_s390_get_machine(kvm, attr);
982 break;
983 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
984 ret = kvm_s390_get_processor_feat(kvm, attr);
985 break;
986 case KVM_S390_VM_CPU_MACHINE_FEAT:
987 ret = kvm_s390_get_machine_feat(kvm, attr);
988 break;
989 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
990 ret = kvm_s390_get_processor_subfunc(kvm, attr);
991 break;
992 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
993 ret = kvm_s390_get_machine_subfunc(kvm, attr);
994 break;
995 }
996 return ret;
997 }
998
999 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1000 {
1001 int ret;
1002
1003 switch (attr->group) {
1004 case KVM_S390_VM_MEM_CTRL:
1005 ret = kvm_s390_set_mem_control(kvm, attr);
1006 break;
1007 case KVM_S390_VM_TOD:
1008 ret = kvm_s390_set_tod(kvm, attr);
1009 break;
1010 case KVM_S390_VM_CPU_MODEL:
1011 ret = kvm_s390_set_cpu_model(kvm, attr);
1012 break;
1013 case KVM_S390_VM_CRYPTO:
1014 ret = kvm_s390_vm_set_crypto(kvm, attr);
1015 break;
1016 default:
1017 ret = -ENXIO;
1018 break;
1019 }
1020
1021 return ret;
1022 }
1023
1024 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1025 {
1026 int ret;
1027
1028 switch (attr->group) {
1029 case KVM_S390_VM_MEM_CTRL:
1030 ret = kvm_s390_get_mem_control(kvm, attr);
1031 break;
1032 case KVM_S390_VM_TOD:
1033 ret = kvm_s390_get_tod(kvm, attr);
1034 break;
1035 case KVM_S390_VM_CPU_MODEL:
1036 ret = kvm_s390_get_cpu_model(kvm, attr);
1037 break;
1038 default:
1039 ret = -ENXIO;
1040 break;
1041 }
1042
1043 return ret;
1044 }
1045
1046 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1047 {
1048 int ret;
1049
1050 switch (attr->group) {
1051 case KVM_S390_VM_MEM_CTRL:
1052 switch (attr->attr) {
1053 case KVM_S390_VM_MEM_ENABLE_CMMA:
1054 case KVM_S390_VM_MEM_CLR_CMMA:
1055 ret = sclp.has_cmma ? 0 : -ENXIO;
1056 break;
1057 case KVM_S390_VM_MEM_LIMIT_SIZE:
1058 ret = 0;
1059 break;
1060 default:
1061 ret = -ENXIO;
1062 break;
1063 }
1064 break;
1065 case KVM_S390_VM_TOD:
1066 switch (attr->attr) {
1067 case KVM_S390_VM_TOD_LOW:
1068 case KVM_S390_VM_TOD_HIGH:
1069 ret = 0;
1070 break;
1071 default:
1072 ret = -ENXIO;
1073 break;
1074 }
1075 break;
1076 case KVM_S390_VM_CPU_MODEL:
1077 switch (attr->attr) {
1078 case KVM_S390_VM_CPU_PROCESSOR:
1079 case KVM_S390_VM_CPU_MACHINE:
1080 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1081 case KVM_S390_VM_CPU_MACHINE_FEAT:
1082 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1083 ret = 0;
1084 break;
1085 /* configuring subfunctions is not supported yet */
1086 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1087 default:
1088 ret = -ENXIO;
1089 break;
1090 }
1091 break;
1092 case KVM_S390_VM_CRYPTO:
1093 switch (attr->attr) {
1094 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1095 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1096 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1097 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1098 ret = 0;
1099 break;
1100 default:
1101 ret = -ENXIO;
1102 break;
1103 }
1104 break;
1105 default:
1106 ret = -ENXIO;
1107 break;
1108 }
1109
1110 return ret;
1111 }
1112
1113 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1114 {
1115 uint8_t *keys;
1116 uint64_t hva;
1117 int i, r = 0;
1118
1119 if (args->flags != 0)
1120 return -EINVAL;
1121
1122 /* Is this guest using storage keys? */
1123 if (!mm_use_skey(current->mm))
1124 return KVM_S390_GET_SKEYS_NONE;
1125
1126 /* Enforce sane limit on memory allocation */
1127 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1128 return -EINVAL;
1129
1130 keys = kmalloc_array(args->count, sizeof(uint8_t),
1131 GFP_KERNEL | __GFP_NOWARN);
1132 if (!keys)
1133 keys = vmalloc(sizeof(uint8_t) * args->count);
1134 if (!keys)
1135 return -ENOMEM;
1136
1137 down_read(&current->mm->mmap_sem);
1138 for (i = 0; i < args->count; i++) {
1139 hva = gfn_to_hva(kvm, args->start_gfn + i);
1140 if (kvm_is_error_hva(hva)) {
1141 r = -EFAULT;
1142 break;
1143 }
1144
1145 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1146 if (r)
1147 break;
1148 }
1149 up_read(&current->mm->mmap_sem);
1150
1151 if (!r) {
1152 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1153 sizeof(uint8_t) * args->count);
1154 if (r)
1155 r = -EFAULT;
1156 }
1157
1158 kvfree(keys);
1159 return r;
1160 }
1161
1162 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1163 {
1164 uint8_t *keys;
1165 uint64_t hva;
1166 int i, r = 0;
1167
1168 if (args->flags != 0)
1169 return -EINVAL;
1170
1171 /* Enforce sane limit on memory allocation */
1172 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1173 return -EINVAL;
1174
1175 keys = kmalloc_array(args->count, sizeof(uint8_t),
1176 GFP_KERNEL | __GFP_NOWARN);
1177 if (!keys)
1178 keys = vmalloc(sizeof(uint8_t) * args->count);
1179 if (!keys)
1180 return -ENOMEM;
1181
1182 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1183 sizeof(uint8_t) * args->count);
1184 if (r) {
1185 r = -EFAULT;
1186 goto out;
1187 }
1188
1189 /* Enable storage key handling for the guest */
1190 r = s390_enable_skey();
1191 if (r)
1192 goto out;
1193
1194 down_read(&current->mm->mmap_sem);
1195 for (i = 0; i < args->count; i++) {
1196 hva = gfn_to_hva(kvm, args->start_gfn + i);
1197 if (kvm_is_error_hva(hva)) {
1198 r = -EFAULT;
1199 break;
1200 }
1201
1202 /* Lowest order bit is reserved */
1203 if (keys[i] & 0x01) {
1204 r = -EINVAL;
1205 break;
1206 }
1207
1208 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1209 if (r)
1210 break;
1211 }
1212 up_read(&current->mm->mmap_sem);
1213 out:
1214 kvfree(keys);
1215 return r;
1216 }
1217
1218 long kvm_arch_vm_ioctl(struct file *filp,
1219 unsigned int ioctl, unsigned long arg)
1220 {
1221 struct kvm *kvm = filp->private_data;
1222 void __user *argp = (void __user *)arg;
1223 struct kvm_device_attr attr;
1224 int r;
1225
1226 switch (ioctl) {
1227 case KVM_S390_INTERRUPT: {
1228 struct kvm_s390_interrupt s390int;
1229
1230 r = -EFAULT;
1231 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1232 break;
1233 r = kvm_s390_inject_vm(kvm, &s390int);
1234 break;
1235 }
1236 case KVM_ENABLE_CAP: {
1237 struct kvm_enable_cap cap;
1238 r = -EFAULT;
1239 if (copy_from_user(&cap, argp, sizeof(cap)))
1240 break;
1241 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1242 break;
1243 }
1244 case KVM_CREATE_IRQCHIP: {
1245 struct kvm_irq_routing_entry routing;
1246
1247 r = -EINVAL;
1248 if (kvm->arch.use_irqchip) {
1249 /* Set up dummy routing. */
1250 memset(&routing, 0, sizeof(routing));
1251 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
1252 }
1253 break;
1254 }
1255 case KVM_SET_DEVICE_ATTR: {
1256 r = -EFAULT;
1257 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1258 break;
1259 r = kvm_s390_vm_set_attr(kvm, &attr);
1260 break;
1261 }
1262 case KVM_GET_DEVICE_ATTR: {
1263 r = -EFAULT;
1264 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1265 break;
1266 r = kvm_s390_vm_get_attr(kvm, &attr);
1267 break;
1268 }
1269 case KVM_HAS_DEVICE_ATTR: {
1270 r = -EFAULT;
1271 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1272 break;
1273 r = kvm_s390_vm_has_attr(kvm, &attr);
1274 break;
1275 }
1276 case KVM_S390_GET_SKEYS: {
1277 struct kvm_s390_skeys args;
1278
1279 r = -EFAULT;
1280 if (copy_from_user(&args, argp,
1281 sizeof(struct kvm_s390_skeys)))
1282 break;
1283 r = kvm_s390_get_skeys(kvm, &args);
1284 break;
1285 }
1286 case KVM_S390_SET_SKEYS: {
1287 struct kvm_s390_skeys args;
1288
1289 r = -EFAULT;
1290 if (copy_from_user(&args, argp,
1291 sizeof(struct kvm_s390_skeys)))
1292 break;
1293 r = kvm_s390_set_skeys(kvm, &args);
1294 break;
1295 }
1296 default:
1297 r = -ENOTTY;
1298 }
1299
1300 return r;
1301 }
1302
1303 static int kvm_s390_query_ap_config(u8 *config)
1304 {
1305 u32 fcn_code = 0x04000000UL;
1306 u32 cc = 0;
1307
1308 memset(config, 0, 128);
1309 asm volatile(
1310 "lgr 0,%1\n"
1311 "lgr 2,%2\n"
1312 ".long 0xb2af0000\n" /* PQAP(QCI) */
1313 "0: ipm %0\n"
1314 "srl %0,28\n"
1315 "1:\n"
1316 EX_TABLE(0b, 1b)
1317 : "+r" (cc)
1318 : "r" (fcn_code), "r" (config)
1319 : "cc", "0", "2", "memory"
1320 );
1321
1322 return cc;
1323 }
1324
1325 static int kvm_s390_apxa_installed(void)
1326 {
1327 u8 config[128];
1328 int cc;
1329
1330 if (test_facility(12)) {
1331 cc = kvm_s390_query_ap_config(config);
1332
1333 if (cc)
1334 pr_err("PQAP(QCI) failed with cc=%d", cc);
1335 else
1336 return config[0] & 0x40;
1337 }
1338
1339 return 0;
1340 }
1341
1342 static void kvm_s390_set_crycb_format(struct kvm *kvm)
1343 {
1344 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1345
1346 if (kvm_s390_apxa_installed())
1347 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1348 else
1349 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1350 }
1351
1352 static u64 kvm_s390_get_initial_cpuid(void)
1353 {
1354 struct cpuid cpuid;
1355
1356 get_cpu_id(&cpuid);
1357 cpuid.version = 0xff;
1358 return *((u64 *) &cpuid);
1359 }
1360
1361 static void kvm_s390_crypto_init(struct kvm *kvm)
1362 {
1363 if (!test_kvm_facility(kvm, 76))
1364 return;
1365
1366 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
1367 kvm_s390_set_crycb_format(kvm);
1368
1369 /* Enable AES/DEA protected key functions by default */
1370 kvm->arch.crypto.aes_kw = 1;
1371 kvm->arch.crypto.dea_kw = 1;
1372 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1373 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1374 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1375 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1376 }
1377
1378 static void sca_dispose(struct kvm *kvm)
1379 {
1380 if (kvm->arch.use_esca)
1381 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
1382 else
1383 free_page((unsigned long)(kvm->arch.sca));
1384 kvm->arch.sca = NULL;
1385 }
1386
1387 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1388 {
1389 gfp_t alloc_flags = GFP_KERNEL;
1390 int i, rc;
1391 char debug_name[16];
1392 static unsigned long sca_offset;
1393
1394 rc = -EINVAL;
1395 #ifdef CONFIG_KVM_S390_UCONTROL
1396 if (type & ~KVM_VM_S390_UCONTROL)
1397 goto out_err;
1398 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1399 goto out_err;
1400 #else
1401 if (type)
1402 goto out_err;
1403 #endif
1404
1405 rc = s390_enable_sie();
1406 if (rc)
1407 goto out_err;
1408
1409 rc = -ENOMEM;
1410
1411 ratelimit_state_init(&kvm->arch.sthyi_limit, 5 * HZ, 500);
1412
1413 kvm->arch.use_esca = 0; /* start with basic SCA */
1414 if (!sclp.has_64bscao)
1415 alloc_flags |= GFP_DMA;
1416 rwlock_init(&kvm->arch.sca_lock);
1417 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
1418 if (!kvm->arch.sca)
1419 goto out_err;
1420 spin_lock(&kvm_lock);
1421 sca_offset += 16;
1422 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
1423 sca_offset = 0;
1424 kvm->arch.sca = (struct bsca_block *)
1425 ((char *) kvm->arch.sca + sca_offset);
1426 spin_unlock(&kvm_lock);
1427
1428 sprintf(debug_name, "kvm-%u", current->pid);
1429
1430 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1431 if (!kvm->arch.dbf)
1432 goto out_err;
1433
1434 kvm->arch.sie_page2 =
1435 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1436 if (!kvm->arch.sie_page2)
1437 goto out_err;
1438
1439 /* Populate the facility mask initially. */
1440 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
1441 sizeof(S390_lowcore.stfle_fac_list));
1442 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1443 if (i < kvm_s390_fac_list_mask_size())
1444 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
1445 else
1446 kvm->arch.model.fac_mask[i] = 0UL;
1447 }
1448
1449 /* Populate the facility list initially. */
1450 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1451 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
1452 S390_ARCH_FAC_LIST_SIZE_BYTE);
1453
1454 set_kvm_facility(kvm->arch.model.fac_mask, 74);
1455 set_kvm_facility(kvm->arch.model.fac_list, 74);
1456
1457 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
1458 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1459
1460 kvm_s390_crypto_init(kvm);
1461
1462 spin_lock_init(&kvm->arch.float_int.lock);
1463 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1464 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1465 init_waitqueue_head(&kvm->arch.ipte_wq);
1466 mutex_init(&kvm->arch.ipte_mutex);
1467
1468 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1469 VM_EVENT(kvm, 3, "vm created with type %lu", type);
1470
1471 if (type & KVM_VM_S390_UCONTROL) {
1472 kvm->arch.gmap = NULL;
1473 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
1474 } else {
1475 if (sclp.hamax == U64_MAX)
1476 kvm->arch.mem_limit = TASK_MAX_SIZE;
1477 else
1478 kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
1479 sclp.hamax + 1);
1480 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
1481 if (!kvm->arch.gmap)
1482 goto out_err;
1483 kvm->arch.gmap->private = kvm;
1484 kvm->arch.gmap->pfault_enabled = 0;
1485 }
1486
1487 kvm->arch.css_support = 0;
1488 kvm->arch.use_irqchip = 0;
1489 kvm->arch.epoch = 0;
1490
1491 spin_lock_init(&kvm->arch.start_stop_lock);
1492 kvm_s390_vsie_init(kvm);
1493 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
1494
1495 return 0;
1496 out_err:
1497 free_page((unsigned long)kvm->arch.sie_page2);
1498 debug_unregister(kvm->arch.dbf);
1499 sca_dispose(kvm);
1500 KVM_EVENT(3, "creation of vm failed: %d", rc);
1501 return rc;
1502 }
1503
1504 bool kvm_arch_has_vcpu_debugfs(void)
1505 {
1506 return false;
1507 }
1508
1509 int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
1510 {
1511 return 0;
1512 }
1513
1514 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1515 {
1516 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1517 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1518 kvm_s390_clear_local_irqs(vcpu);
1519 kvm_clear_async_pf_completion_queue(vcpu);
1520 if (!kvm_is_ucontrol(vcpu->kvm))
1521 sca_del_vcpu(vcpu);
1522
1523 if (kvm_is_ucontrol(vcpu->kvm))
1524 gmap_remove(vcpu->arch.gmap);
1525
1526 if (vcpu->kvm->arch.use_cmma)
1527 kvm_s390_vcpu_unsetup_cmma(vcpu);
1528 free_page((unsigned long)(vcpu->arch.sie_block));
1529
1530 kvm_vcpu_uninit(vcpu);
1531 kmem_cache_free(kvm_vcpu_cache, vcpu);
1532 }
1533
1534 static void kvm_free_vcpus(struct kvm *kvm)
1535 {
1536 unsigned int i;
1537 struct kvm_vcpu *vcpu;
1538
1539 kvm_for_each_vcpu(i, vcpu, kvm)
1540 kvm_arch_vcpu_destroy(vcpu);
1541
1542 mutex_lock(&kvm->lock);
1543 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1544 kvm->vcpus[i] = NULL;
1545
1546 atomic_set(&kvm->online_vcpus, 0);
1547 mutex_unlock(&kvm->lock);
1548 }
1549
1550 void kvm_arch_destroy_vm(struct kvm *kvm)
1551 {
1552 kvm_free_vcpus(kvm);
1553 sca_dispose(kvm);
1554 debug_unregister(kvm->arch.dbf);
1555 free_page((unsigned long)kvm->arch.sie_page2);
1556 if (!kvm_is_ucontrol(kvm))
1557 gmap_remove(kvm->arch.gmap);
1558 kvm_s390_destroy_adapters(kvm);
1559 kvm_s390_clear_float_irqs(kvm);
1560 kvm_s390_vsie_destroy(kvm);
1561 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
1562 }
1563
1564 /* Section: vcpu related */
1565 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1566 {
1567 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
1568 if (!vcpu->arch.gmap)
1569 return -ENOMEM;
1570 vcpu->arch.gmap->private = vcpu->kvm;
1571
1572 return 0;
1573 }
1574
1575 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1576 {
1577 if (!kvm_s390_use_sca_entries())
1578 return;
1579 read_lock(&vcpu->kvm->arch.sca_lock);
1580 if (vcpu->kvm->arch.use_esca) {
1581 struct esca_block *sca = vcpu->kvm->arch.sca;
1582
1583 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1584 sca->cpu[vcpu->vcpu_id].sda = 0;
1585 } else {
1586 struct bsca_block *sca = vcpu->kvm->arch.sca;
1587
1588 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1589 sca->cpu[vcpu->vcpu_id].sda = 0;
1590 }
1591 read_unlock(&vcpu->kvm->arch.sca_lock);
1592 }
1593
1594 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
1595 {
1596 if (!kvm_s390_use_sca_entries()) {
1597 struct bsca_block *sca = vcpu->kvm->arch.sca;
1598
1599 /* we still need the basic sca for the ipte control */
1600 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1601 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
1602 }
1603 read_lock(&vcpu->kvm->arch.sca_lock);
1604 if (vcpu->kvm->arch.use_esca) {
1605 struct esca_block *sca = vcpu->kvm->arch.sca;
1606
1607 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1608 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1609 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
1610 vcpu->arch.sie_block->ecb2 |= 0x04U;
1611 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1612 } else {
1613 struct bsca_block *sca = vcpu->kvm->arch.sca;
1614
1615 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1616 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1617 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
1618 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1619 }
1620 read_unlock(&vcpu->kvm->arch.sca_lock);
1621 }
1622
1623 /* Basic SCA to Extended SCA data copy routines */
1624 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1625 {
1626 d->sda = s->sda;
1627 d->sigp_ctrl.c = s->sigp_ctrl.c;
1628 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1629 }
1630
1631 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1632 {
1633 int i;
1634
1635 d->ipte_control = s->ipte_control;
1636 d->mcn[0] = s->mcn;
1637 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1638 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1639 }
1640
1641 static int sca_switch_to_extended(struct kvm *kvm)
1642 {
1643 struct bsca_block *old_sca = kvm->arch.sca;
1644 struct esca_block *new_sca;
1645 struct kvm_vcpu *vcpu;
1646 unsigned int vcpu_idx;
1647 u32 scaol, scaoh;
1648
1649 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1650 if (!new_sca)
1651 return -ENOMEM;
1652
1653 scaoh = (u32)((u64)(new_sca) >> 32);
1654 scaol = (u32)(u64)(new_sca) & ~0x3fU;
1655
1656 kvm_s390_vcpu_block_all(kvm);
1657 write_lock(&kvm->arch.sca_lock);
1658
1659 sca_copy_b_to_e(new_sca, old_sca);
1660
1661 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1662 vcpu->arch.sie_block->scaoh = scaoh;
1663 vcpu->arch.sie_block->scaol = scaol;
1664 vcpu->arch.sie_block->ecb2 |= 0x04U;
1665 }
1666 kvm->arch.sca = new_sca;
1667 kvm->arch.use_esca = 1;
1668
1669 write_unlock(&kvm->arch.sca_lock);
1670 kvm_s390_vcpu_unblock_all(kvm);
1671
1672 free_page((unsigned long)old_sca);
1673
1674 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1675 old_sca, kvm->arch.sca);
1676 return 0;
1677 }
1678
1679 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1680 {
1681 int rc;
1682
1683 if (!kvm_s390_use_sca_entries()) {
1684 if (id < KVM_MAX_VCPUS)
1685 return true;
1686 return false;
1687 }
1688 if (id < KVM_S390_BSCA_CPU_SLOTS)
1689 return true;
1690 if (!sclp.has_esca || !sclp.has_64bscao)
1691 return false;
1692
1693 mutex_lock(&kvm->lock);
1694 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1695 mutex_unlock(&kvm->lock);
1696
1697 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
1698 }
1699
1700 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1701 {
1702 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1703 kvm_clear_async_pf_completion_queue(vcpu);
1704 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1705 KVM_SYNC_GPRS |
1706 KVM_SYNC_ACRS |
1707 KVM_SYNC_CRS |
1708 KVM_SYNC_ARCH0 |
1709 KVM_SYNC_PFAULT;
1710 kvm_s390_set_prefix(vcpu, 0);
1711 if (test_kvm_facility(vcpu->kvm, 64))
1712 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
1713 /* fprs can be synchronized via vrs, even if the guest has no vx. With
1714 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1715 */
1716 if (MACHINE_HAS_VX)
1717 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1718 else
1719 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
1720
1721 if (kvm_is_ucontrol(vcpu->kvm))
1722 return __kvm_ucontrol_vcpu_init(vcpu);
1723
1724 return 0;
1725 }
1726
1727 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1728 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1729 {
1730 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
1731 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1732 vcpu->arch.cputm_start = get_tod_clock_fast();
1733 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1734 }
1735
1736 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1737 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1738 {
1739 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
1740 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1741 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1742 vcpu->arch.cputm_start = 0;
1743 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1744 }
1745
1746 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1747 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1748 {
1749 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
1750 vcpu->arch.cputm_enabled = true;
1751 __start_cpu_timer_accounting(vcpu);
1752 }
1753
1754 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1755 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1756 {
1757 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
1758 __stop_cpu_timer_accounting(vcpu);
1759 vcpu->arch.cputm_enabled = false;
1760 }
1761
1762 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1763 {
1764 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1765 __enable_cpu_timer_accounting(vcpu);
1766 preempt_enable();
1767 }
1768
1769 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1770 {
1771 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1772 __disable_cpu_timer_accounting(vcpu);
1773 preempt_enable();
1774 }
1775
1776 /* set the cpu timer - may only be called from the VCPU thread itself */
1777 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
1778 {
1779 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1780 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1781 if (vcpu->arch.cputm_enabled)
1782 vcpu->arch.cputm_start = get_tod_clock_fast();
1783 vcpu->arch.sie_block->cputm = cputm;
1784 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1785 preempt_enable();
1786 }
1787
1788 /* update and get the cpu timer - can also be called from other VCPU threads */
1789 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
1790 {
1791 unsigned int seq;
1792 __u64 value;
1793
1794 if (unlikely(!vcpu->arch.cputm_enabled))
1795 return vcpu->arch.sie_block->cputm;
1796
1797 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1798 do {
1799 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
1800 /*
1801 * If the writer would ever execute a read in the critical
1802 * section, e.g. in irq context, we have a deadlock.
1803 */
1804 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
1805 value = vcpu->arch.sie_block->cputm;
1806 /* if cputm_start is 0, accounting is being started/stopped */
1807 if (likely(vcpu->arch.cputm_start))
1808 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1809 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
1810 preempt_enable();
1811 return value;
1812 }
1813
1814 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1815 {
1816
1817 gmap_enable(vcpu->arch.enabled_gmap);
1818 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1819 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
1820 __start_cpu_timer_accounting(vcpu);
1821 vcpu->cpu = cpu;
1822 }
1823
1824 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1825 {
1826 vcpu->cpu = -1;
1827 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
1828 __stop_cpu_timer_accounting(vcpu);
1829 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1830 vcpu->arch.enabled_gmap = gmap_get_enabled();
1831 gmap_disable(vcpu->arch.enabled_gmap);
1832
1833 }
1834
1835 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1836 {
1837 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1838 vcpu->arch.sie_block->gpsw.mask = 0UL;
1839 vcpu->arch.sie_block->gpsw.addr = 0UL;
1840 kvm_s390_set_prefix(vcpu, 0);
1841 kvm_s390_set_cpu_timer(vcpu, 0);
1842 vcpu->arch.sie_block->ckc = 0UL;
1843 vcpu->arch.sie_block->todpr = 0;
1844 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1845 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1846 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1847 /* make sure the new fpc will be lazily loaded */
1848 save_fpu_regs();
1849 current->thread.fpu.fpc = 0;
1850 vcpu->arch.sie_block->gbea = 1;
1851 vcpu->arch.sie_block->pp = 0;
1852 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1853 kvm_clear_async_pf_completion_queue(vcpu);
1854 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1855 kvm_s390_vcpu_stop(vcpu);
1856 kvm_s390_clear_local_irqs(vcpu);
1857 }
1858
1859 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1860 {
1861 mutex_lock(&vcpu->kvm->lock);
1862 preempt_disable();
1863 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1864 preempt_enable();
1865 mutex_unlock(&vcpu->kvm->lock);
1866 if (!kvm_is_ucontrol(vcpu->kvm)) {
1867 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1868 sca_add_vcpu(vcpu);
1869 }
1870 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
1871 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
1872 /* make vcpu_load load the right gmap on the first trigger */
1873 vcpu->arch.enabled_gmap = vcpu->arch.gmap;
1874 }
1875
1876 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1877 {
1878 if (!test_kvm_facility(vcpu->kvm, 76))
1879 return;
1880
1881 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1882
1883 if (vcpu->kvm->arch.crypto.aes_kw)
1884 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1885 if (vcpu->kvm->arch.crypto.dea_kw)
1886 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1887
1888 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1889 }
1890
1891 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1892 {
1893 free_page(vcpu->arch.sie_block->cbrlo);
1894 vcpu->arch.sie_block->cbrlo = 0;
1895 }
1896
1897 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1898 {
1899 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1900 if (!vcpu->arch.sie_block->cbrlo)
1901 return -ENOMEM;
1902
1903 vcpu->arch.sie_block->ecb2 |= 0x80;
1904 vcpu->arch.sie_block->ecb2 &= ~0x08;
1905 return 0;
1906 }
1907
1908 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1909 {
1910 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1911
1912 vcpu->arch.sie_block->ibc = model->ibc;
1913 if (test_kvm_facility(vcpu->kvm, 7))
1914 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
1915 }
1916
1917 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1918 {
1919 int rc = 0;
1920
1921 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1922 CPUSTAT_SM |
1923 CPUSTAT_STOPPED);
1924
1925 if (test_kvm_facility(vcpu->kvm, 78))
1926 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
1927 else if (test_kvm_facility(vcpu->kvm, 8))
1928 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
1929
1930 kvm_s390_vcpu_setup_model(vcpu);
1931
1932 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
1933 if (MACHINE_HAS_ESOP)
1934 vcpu->arch.sie_block->ecb |= 0x02;
1935 if (test_kvm_facility(vcpu->kvm, 9))
1936 vcpu->arch.sie_block->ecb |= 0x04;
1937 if (test_kvm_facility(vcpu->kvm, 73))
1938 vcpu->arch.sie_block->ecb |= 0x10;
1939
1940 if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
1941 vcpu->arch.sie_block->ecb2 |= 0x08;
1942 vcpu->arch.sie_block->eca = 0x1002000U;
1943 if (sclp.has_cei)
1944 vcpu->arch.sie_block->eca |= 0x80000000U;
1945 if (sclp.has_ib)
1946 vcpu->arch.sie_block->eca |= 0x40000000U;
1947 if (sclp.has_siif)
1948 vcpu->arch.sie_block->eca |= 1;
1949 if (sclp.has_sigpif)
1950 vcpu->arch.sie_block->eca |= 0x10000000U;
1951 if (test_kvm_facility(vcpu->kvm, 129)) {
1952 vcpu->arch.sie_block->eca |= 0x00020000;
1953 vcpu->arch.sie_block->ecd |= 0x20000000;
1954 }
1955 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
1956 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1957
1958 if (vcpu->kvm->arch.use_cmma) {
1959 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1960 if (rc)
1961 return rc;
1962 }
1963 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1964 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1965
1966 kvm_s390_vcpu_crypto_setup(vcpu);
1967
1968 return rc;
1969 }
1970
1971 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1972 unsigned int id)
1973 {
1974 struct kvm_vcpu *vcpu;
1975 struct sie_page *sie_page;
1976 int rc = -EINVAL;
1977
1978 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
1979 goto out;
1980
1981 rc = -ENOMEM;
1982
1983 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1984 if (!vcpu)
1985 goto out;
1986
1987 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1988 if (!sie_page)
1989 goto out_free_cpu;
1990
1991 vcpu->arch.sie_block = &sie_page->sie_block;
1992 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1993
1994 /* the real guest size will always be smaller than msl */
1995 vcpu->arch.sie_block->mso = 0;
1996 vcpu->arch.sie_block->msl = sclp.hamax;
1997
1998 vcpu->arch.sie_block->icpua = id;
1999 spin_lock_init(&vcpu->arch.local_int.lock);
2000 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
2001 vcpu->arch.local_int.wq = &vcpu->wq;
2002 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
2003 seqcount_init(&vcpu->arch.cputm_seqcount);
2004
2005 rc = kvm_vcpu_init(vcpu, kvm, id);
2006 if (rc)
2007 goto out_free_sie_block;
2008 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
2009 vcpu->arch.sie_block);
2010 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
2011
2012 return vcpu;
2013 out_free_sie_block:
2014 free_page((unsigned long)(vcpu->arch.sie_block));
2015 out_free_cpu:
2016 kmem_cache_free(kvm_vcpu_cache, vcpu);
2017 out:
2018 return ERR_PTR(rc);
2019 }
2020
2021 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
2022 {
2023 return kvm_s390_vcpu_has_irq(vcpu, 0);
2024 }
2025
2026 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
2027 {
2028 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2029 exit_sie(vcpu);
2030 }
2031
2032 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
2033 {
2034 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2035 }
2036
2037 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
2038 {
2039 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2040 exit_sie(vcpu);
2041 }
2042
2043 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2044 {
2045 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2046 }
2047
2048 /*
2049 * Kick a guest cpu out of SIE and wait until SIE is not running.
2050 * If the CPU is not running (e.g. waiting as idle) the function will
2051 * return immediately. */
2052 void exit_sie(struct kvm_vcpu *vcpu)
2053 {
2054 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
2055 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2056 cpu_relax();
2057 }
2058
2059 /* Kick a guest cpu out of SIE to process a request synchronously */
2060 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
2061 {
2062 kvm_make_request(req, vcpu);
2063 kvm_s390_vcpu_request(vcpu);
2064 }
2065
2066 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
2067 unsigned long end)
2068 {
2069 struct kvm *kvm = gmap->private;
2070 struct kvm_vcpu *vcpu;
2071 unsigned long prefix;
2072 int i;
2073
2074 if (gmap_is_shadow(gmap))
2075 return;
2076 if (start >= 1UL << 31)
2077 /* We are only interested in prefix pages */
2078 return;
2079 kvm_for_each_vcpu(i, vcpu, kvm) {
2080 /* match against both prefix pages */
2081 prefix = kvm_s390_get_prefix(vcpu);
2082 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
2083 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
2084 start, end);
2085 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2086 }
2087 }
2088 }
2089
2090 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2091 {
2092 /* kvm common code refers to this, but never calls it */
2093 BUG();
2094 return 0;
2095 }
2096
2097 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2098 struct kvm_one_reg *reg)
2099 {
2100 int r = -EINVAL;
2101
2102 switch (reg->id) {
2103 case KVM_REG_S390_TODPR:
2104 r = put_user(vcpu->arch.sie_block->todpr,
2105 (u32 __user *)reg->addr);
2106 break;
2107 case KVM_REG_S390_EPOCHDIFF:
2108 r = put_user(vcpu->arch.sie_block->epoch,
2109 (u64 __user *)reg->addr);
2110 break;
2111 case KVM_REG_S390_CPU_TIMER:
2112 r = put_user(kvm_s390_get_cpu_timer(vcpu),
2113 (u64 __user *)reg->addr);
2114 break;
2115 case KVM_REG_S390_CLOCK_COMP:
2116 r = put_user(vcpu->arch.sie_block->ckc,
2117 (u64 __user *)reg->addr);
2118 break;
2119 case KVM_REG_S390_PFTOKEN:
2120 r = put_user(vcpu->arch.pfault_token,
2121 (u64 __user *)reg->addr);
2122 break;
2123 case KVM_REG_S390_PFCOMPARE:
2124 r = put_user(vcpu->arch.pfault_compare,
2125 (u64 __user *)reg->addr);
2126 break;
2127 case KVM_REG_S390_PFSELECT:
2128 r = put_user(vcpu->arch.pfault_select,
2129 (u64 __user *)reg->addr);
2130 break;
2131 case KVM_REG_S390_PP:
2132 r = put_user(vcpu->arch.sie_block->pp,
2133 (u64 __user *)reg->addr);
2134 break;
2135 case KVM_REG_S390_GBEA:
2136 r = put_user(vcpu->arch.sie_block->gbea,
2137 (u64 __user *)reg->addr);
2138 break;
2139 default:
2140 break;
2141 }
2142
2143 return r;
2144 }
2145
2146 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2147 struct kvm_one_reg *reg)
2148 {
2149 int r = -EINVAL;
2150 __u64 val;
2151
2152 switch (reg->id) {
2153 case KVM_REG_S390_TODPR:
2154 r = get_user(vcpu->arch.sie_block->todpr,
2155 (u32 __user *)reg->addr);
2156 break;
2157 case KVM_REG_S390_EPOCHDIFF:
2158 r = get_user(vcpu->arch.sie_block->epoch,
2159 (u64 __user *)reg->addr);
2160 break;
2161 case KVM_REG_S390_CPU_TIMER:
2162 r = get_user(val, (u64 __user *)reg->addr);
2163 if (!r)
2164 kvm_s390_set_cpu_timer(vcpu, val);
2165 break;
2166 case KVM_REG_S390_CLOCK_COMP:
2167 r = get_user(vcpu->arch.sie_block->ckc,
2168 (u64 __user *)reg->addr);
2169 break;
2170 case KVM_REG_S390_PFTOKEN:
2171 r = get_user(vcpu->arch.pfault_token,
2172 (u64 __user *)reg->addr);
2173 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2174 kvm_clear_async_pf_completion_queue(vcpu);
2175 break;
2176 case KVM_REG_S390_PFCOMPARE:
2177 r = get_user(vcpu->arch.pfault_compare,
2178 (u64 __user *)reg->addr);
2179 break;
2180 case KVM_REG_S390_PFSELECT:
2181 r = get_user(vcpu->arch.pfault_select,
2182 (u64 __user *)reg->addr);
2183 break;
2184 case KVM_REG_S390_PP:
2185 r = get_user(vcpu->arch.sie_block->pp,
2186 (u64 __user *)reg->addr);
2187 break;
2188 case KVM_REG_S390_GBEA:
2189 r = get_user(vcpu->arch.sie_block->gbea,
2190 (u64 __user *)reg->addr);
2191 break;
2192 default:
2193 break;
2194 }
2195
2196 return r;
2197 }
2198
2199 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2200 {
2201 kvm_s390_vcpu_initial_reset(vcpu);
2202 return 0;
2203 }
2204
2205 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2206 {
2207 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
2208 return 0;
2209 }
2210
2211 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2212 {
2213 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
2214 return 0;
2215 }
2216
2217 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2218 struct kvm_sregs *sregs)
2219 {
2220 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
2221 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
2222 return 0;
2223 }
2224
2225 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2226 struct kvm_sregs *sregs)
2227 {
2228 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
2229 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
2230 return 0;
2231 }
2232
2233 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2234 {
2235 if (test_fp_ctl(fpu->fpc))
2236 return -EINVAL;
2237 vcpu->run->s.regs.fpc = fpu->fpc;
2238 if (MACHINE_HAS_VX)
2239 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
2240 (freg_t *) fpu->fprs);
2241 else
2242 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
2243 return 0;
2244 }
2245
2246 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2247 {
2248 /* make sure we have the latest values */
2249 save_fpu_regs();
2250 if (MACHINE_HAS_VX)
2251 convert_vx_to_fp((freg_t *) fpu->fprs,
2252 (__vector128 *) vcpu->run->s.regs.vrs);
2253 else
2254 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
2255 fpu->fpc = vcpu->run->s.regs.fpc;
2256 return 0;
2257 }
2258
2259 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2260 {
2261 int rc = 0;
2262
2263 if (!is_vcpu_stopped(vcpu))
2264 rc = -EBUSY;
2265 else {
2266 vcpu->run->psw_mask = psw.mask;
2267 vcpu->run->psw_addr = psw.addr;
2268 }
2269 return rc;
2270 }
2271
2272 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2273 struct kvm_translation *tr)
2274 {
2275 return -EINVAL; /* not implemented yet */
2276 }
2277
2278 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2279 KVM_GUESTDBG_USE_HW_BP | \
2280 KVM_GUESTDBG_ENABLE)
2281
2282 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2283 struct kvm_guest_debug *dbg)
2284 {
2285 int rc = 0;
2286
2287 vcpu->guest_debug = 0;
2288 kvm_s390_clear_bp_data(vcpu);
2289
2290 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
2291 return -EINVAL;
2292 if (!sclp.has_gpere)
2293 return -EINVAL;
2294
2295 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2296 vcpu->guest_debug = dbg->control;
2297 /* enforce guest PER */
2298 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2299
2300 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2301 rc = kvm_s390_import_bp_data(vcpu, dbg);
2302 } else {
2303 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2304 vcpu->arch.guestdbg.last_bp = 0;
2305 }
2306
2307 if (rc) {
2308 vcpu->guest_debug = 0;
2309 kvm_s390_clear_bp_data(vcpu);
2310 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2311 }
2312
2313 return rc;
2314 }
2315
2316 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2317 struct kvm_mp_state *mp_state)
2318 {
2319 /* CHECK_STOP and LOAD are not supported yet */
2320 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2321 KVM_MP_STATE_OPERATING;
2322 }
2323
2324 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2325 struct kvm_mp_state *mp_state)
2326 {
2327 int rc = 0;
2328
2329 /* user space knows about this interface - let it control the state */
2330 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2331
2332 switch (mp_state->mp_state) {
2333 case KVM_MP_STATE_STOPPED:
2334 kvm_s390_vcpu_stop(vcpu);
2335 break;
2336 case KVM_MP_STATE_OPERATING:
2337 kvm_s390_vcpu_start(vcpu);
2338 break;
2339 case KVM_MP_STATE_LOAD:
2340 case KVM_MP_STATE_CHECK_STOP:
2341 /* fall through - CHECK_STOP and LOAD are not supported yet */
2342 default:
2343 rc = -ENXIO;
2344 }
2345
2346 return rc;
2347 }
2348
2349 static bool ibs_enabled(struct kvm_vcpu *vcpu)
2350 {
2351 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2352 }
2353
2354 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2355 {
2356 retry:
2357 kvm_s390_vcpu_request_handled(vcpu);
2358 if (!vcpu->requests)
2359 return 0;
2360 /*
2361 * We use MMU_RELOAD just to re-arm the ipte notifier for the
2362 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2363 * This ensures that the ipte instruction for this request has
2364 * already finished. We might race against a second unmapper that
2365 * wants to set the blocking bit. Lets just retry the request loop.
2366 */
2367 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2368 int rc;
2369 rc = gmap_mprotect_notify(vcpu->arch.gmap,
2370 kvm_s390_get_prefix(vcpu),
2371 PAGE_SIZE * 2, PROT_WRITE);
2372 if (rc) {
2373 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
2374 return rc;
2375 }
2376 goto retry;
2377 }
2378
2379 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2380 vcpu->arch.sie_block->ihcpu = 0xffff;
2381 goto retry;
2382 }
2383
2384 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2385 if (!ibs_enabled(vcpu)) {
2386 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
2387 atomic_or(CPUSTAT_IBS,
2388 &vcpu->arch.sie_block->cpuflags);
2389 }
2390 goto retry;
2391 }
2392
2393 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2394 if (ibs_enabled(vcpu)) {
2395 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
2396 atomic_andnot(CPUSTAT_IBS,
2397 &vcpu->arch.sie_block->cpuflags);
2398 }
2399 goto retry;
2400 }
2401
2402 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
2403 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2404 goto retry;
2405 }
2406
2407 /* nothing to do, just clear the request */
2408 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2409
2410 return 0;
2411 }
2412
2413 void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2414 {
2415 struct kvm_vcpu *vcpu;
2416 int i;
2417
2418 mutex_lock(&kvm->lock);
2419 preempt_disable();
2420 kvm->arch.epoch = tod - get_tod_clock();
2421 kvm_s390_vcpu_block_all(kvm);
2422 kvm_for_each_vcpu(i, vcpu, kvm)
2423 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2424 kvm_s390_vcpu_unblock_all(kvm);
2425 preempt_enable();
2426 mutex_unlock(&kvm->lock);
2427 }
2428
2429 /**
2430 * kvm_arch_fault_in_page - fault-in guest page if necessary
2431 * @vcpu: The corresponding virtual cpu
2432 * @gpa: Guest physical address
2433 * @writable: Whether the page should be writable or not
2434 *
2435 * Make sure that a guest page has been faulted-in on the host.
2436 *
2437 * Return: Zero on success, negative error code otherwise.
2438 */
2439 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
2440 {
2441 return gmap_fault(vcpu->arch.gmap, gpa,
2442 writable ? FAULT_FLAG_WRITE : 0);
2443 }
2444
2445 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2446 unsigned long token)
2447 {
2448 struct kvm_s390_interrupt inti;
2449 struct kvm_s390_irq irq;
2450
2451 if (start_token) {
2452 irq.u.ext.ext_params2 = token;
2453 irq.type = KVM_S390_INT_PFAULT_INIT;
2454 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
2455 } else {
2456 inti.type = KVM_S390_INT_PFAULT_DONE;
2457 inti.parm64 = token;
2458 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2459 }
2460 }
2461
2462 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2463 struct kvm_async_pf *work)
2464 {
2465 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2466 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2467 }
2468
2469 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2470 struct kvm_async_pf *work)
2471 {
2472 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2473 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2474 }
2475
2476 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2477 struct kvm_async_pf *work)
2478 {
2479 /* s390 will always inject the page directly */
2480 }
2481
2482 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2483 {
2484 /*
2485 * s390 will always inject the page directly,
2486 * but we still want check_async_completion to cleanup
2487 */
2488 return true;
2489 }
2490
2491 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2492 {
2493 hva_t hva;
2494 struct kvm_arch_async_pf arch;
2495 int rc;
2496
2497 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2498 return 0;
2499 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2500 vcpu->arch.pfault_compare)
2501 return 0;
2502 if (psw_extint_disabled(vcpu))
2503 return 0;
2504 if (kvm_s390_vcpu_has_irq(vcpu, 0))
2505 return 0;
2506 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2507 return 0;
2508 if (!vcpu->arch.gmap->pfault_enabled)
2509 return 0;
2510
2511 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2512 hva += current->thread.gmap_addr & ~PAGE_MASK;
2513 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
2514 return 0;
2515
2516 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2517 return rc;
2518 }
2519
2520 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
2521 {
2522 int rc, cpuflags;
2523
2524 /*
2525 * On s390 notifications for arriving pages will be delivered directly
2526 * to the guest but the house keeping for completed pfaults is
2527 * handled outside the worker.
2528 */
2529 kvm_check_async_pf_completion(vcpu);
2530
2531 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2532 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
2533
2534 if (need_resched())
2535 schedule();
2536
2537 if (test_cpu_flag(CIF_MCCK_PENDING))
2538 s390_handle_mcck();
2539
2540 if (!kvm_is_ucontrol(vcpu->kvm)) {
2541 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2542 if (rc)
2543 return rc;
2544 }
2545
2546 rc = kvm_s390_handle_requests(vcpu);
2547 if (rc)
2548 return rc;
2549
2550 if (guestdbg_enabled(vcpu)) {
2551 kvm_s390_backup_guest_per_regs(vcpu);
2552 kvm_s390_patch_guest_per_regs(vcpu);
2553 }
2554
2555 vcpu->arch.sie_block->icptcode = 0;
2556 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2557 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2558 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2559
2560 return 0;
2561 }
2562
2563 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2564 {
2565 struct kvm_s390_pgm_info pgm_info = {
2566 .code = PGM_ADDRESSING,
2567 };
2568 u8 opcode, ilen;
2569 int rc;
2570
2571 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2572 trace_kvm_s390_sie_fault(vcpu);
2573
2574 /*
2575 * We want to inject an addressing exception, which is defined as a
2576 * suppressing or terminating exception. However, since we came here
2577 * by a DAT access exception, the PSW still points to the faulting
2578 * instruction since DAT exceptions are nullifying. So we've got
2579 * to look up the current opcode to get the length of the instruction
2580 * to be able to forward the PSW.
2581 */
2582 rc = read_guest_instr(vcpu, &opcode, 1);
2583 ilen = insn_length(opcode);
2584 if (rc < 0) {
2585 return rc;
2586 } else if (rc) {
2587 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2588 * Forward by arbitrary ilc, injection will take care of
2589 * nullification if necessary.
2590 */
2591 pgm_info = vcpu->arch.pgm;
2592 ilen = 4;
2593 }
2594 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2595 kvm_s390_forward_psw(vcpu, ilen);
2596 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
2597 }
2598
2599 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2600 {
2601 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2602 vcpu->arch.sie_block->icptcode);
2603 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2604
2605 if (guestdbg_enabled(vcpu))
2606 kvm_s390_restore_guest_per_regs(vcpu);
2607
2608 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2609 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
2610
2611 if (vcpu->arch.sie_block->icptcode > 0) {
2612 int rc = kvm_handle_sie_intercept(vcpu);
2613
2614 if (rc != -EOPNOTSUPP)
2615 return rc;
2616 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2617 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2618 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2619 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2620 return -EREMOTE;
2621 } else if (exit_reason != -EFAULT) {
2622 vcpu->stat.exit_null++;
2623 return 0;
2624 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2625 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2626 vcpu->run->s390_ucontrol.trans_exc_code =
2627 current->thread.gmap_addr;
2628 vcpu->run->s390_ucontrol.pgm_code = 0x10;
2629 return -EREMOTE;
2630 } else if (current->thread.gmap_pfault) {
2631 trace_kvm_s390_major_guest_pfault(vcpu);
2632 current->thread.gmap_pfault = 0;
2633 if (kvm_arch_setup_async_pf(vcpu))
2634 return 0;
2635 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
2636 }
2637 return vcpu_post_run_fault_in_sie(vcpu);
2638 }
2639
2640 static int __vcpu_run(struct kvm_vcpu *vcpu)
2641 {
2642 int rc, exit_reason;
2643
2644 /*
2645 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2646 * ning the guest), so that memslots (and other stuff) are protected
2647 */
2648 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2649
2650 do {
2651 rc = vcpu_pre_run(vcpu);
2652 if (rc)
2653 break;
2654
2655 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2656 /*
2657 * As PF_VCPU will be used in fault handler, between
2658 * guest_enter and guest_exit should be no uaccess.
2659 */
2660 local_irq_disable();
2661 guest_enter_irqoff();
2662 __disable_cpu_timer_accounting(vcpu);
2663 local_irq_enable();
2664 exit_reason = sie64a(vcpu->arch.sie_block,
2665 vcpu->run->s.regs.gprs);
2666 local_irq_disable();
2667 __enable_cpu_timer_accounting(vcpu);
2668 guest_exit_irqoff();
2669 local_irq_enable();
2670 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2671
2672 rc = vcpu_post_run(vcpu, exit_reason);
2673 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2674
2675 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2676 return rc;
2677 }
2678
2679 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2680 {
2681 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2682 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2683 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2684 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2685 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2686 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2687 /* some control register changes require a tlb flush */
2688 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2689 }
2690 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2691 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
2692 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2693 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2694 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2695 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2696 }
2697 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2698 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2699 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2700 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2701 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2702 kvm_clear_async_pf_completion_queue(vcpu);
2703 }
2704 /*
2705 * If userspace sets the riccb (e.g. after migration) to a valid state,
2706 * we should enable RI here instead of doing the lazy enablement.
2707 */
2708 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
2709 test_kvm_facility(vcpu->kvm, 64)) {
2710 struct runtime_instr_cb *riccb =
2711 (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
2712
2713 if (riccb->valid)
2714 vcpu->arch.sie_block->ecb3 |= 0x01;
2715 }
2716 save_access_regs(vcpu->arch.host_acrs);
2717 restore_access_regs(vcpu->run->s.regs.acrs);
2718 /* save host (userspace) fprs/vrs */
2719 save_fpu_regs();
2720 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
2721 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
2722 if (MACHINE_HAS_VX)
2723 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
2724 else
2725 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
2726 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
2727 if (test_fp_ctl(current->thread.fpu.fpc))
2728 /* User space provided an invalid FPC, let's clear it */
2729 current->thread.fpu.fpc = 0;
2730
2731 kvm_run->kvm_dirty_regs = 0;
2732 }
2733
2734 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2735 {
2736 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2737 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2738 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2739 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2740 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
2741 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2742 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2743 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2744 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2745 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2746 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2747 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2748 save_access_regs(vcpu->run->s.regs.acrs);
2749 restore_access_regs(vcpu->arch.host_acrs);
2750 /* Save guest register state */
2751 save_fpu_regs();
2752 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
2753 /* Restore will be done lazily at return */
2754 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
2755 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
2756
2757 }
2758
2759 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2760 {
2761 int rc;
2762 sigset_t sigsaved;
2763
2764 if (guestdbg_exit_pending(vcpu)) {
2765 kvm_s390_prepare_debug_exit(vcpu);
2766 return 0;
2767 }
2768
2769 if (vcpu->sigset_active)
2770 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2771
2772 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2773 kvm_s390_vcpu_start(vcpu);
2774 } else if (is_vcpu_stopped(vcpu)) {
2775 pr_err_ratelimited("can't run stopped vcpu %d\n",
2776 vcpu->vcpu_id);
2777 return -EINVAL;
2778 }
2779
2780 sync_regs(vcpu, kvm_run);
2781 enable_cpu_timer_accounting(vcpu);
2782
2783 might_fault();
2784 rc = __vcpu_run(vcpu);
2785
2786 if (signal_pending(current) && !rc) {
2787 kvm_run->exit_reason = KVM_EXIT_INTR;
2788 rc = -EINTR;
2789 }
2790
2791 if (guestdbg_exit_pending(vcpu) && !rc) {
2792 kvm_s390_prepare_debug_exit(vcpu);
2793 rc = 0;
2794 }
2795
2796 if (rc == -EREMOTE) {
2797 /* userspace support is needed, kvm_run has been prepared */
2798 rc = 0;
2799 }
2800
2801 disable_cpu_timer_accounting(vcpu);
2802 store_regs(vcpu, kvm_run);
2803
2804 if (vcpu->sigset_active)
2805 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2806
2807 vcpu->stat.exit_userspace++;
2808 return rc;
2809 }
2810
2811 /*
2812 * store status at address
2813 * we use have two special cases:
2814 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2815 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2816 */
2817 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2818 {
2819 unsigned char archmode = 1;
2820 freg_t fprs[NUM_FPRS];
2821 unsigned int px;
2822 u64 clkcomp, cputm;
2823 int rc;
2824
2825 px = kvm_s390_get_prefix(vcpu);
2826 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2827 if (write_guest_abs(vcpu, 163, &archmode, 1))
2828 return -EFAULT;
2829 gpa = 0;
2830 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2831 if (write_guest_real(vcpu, 163, &archmode, 1))
2832 return -EFAULT;
2833 gpa = px;
2834 } else
2835 gpa -= __LC_FPREGS_SAVE_AREA;
2836
2837 /* manually convert vector registers if necessary */
2838 if (MACHINE_HAS_VX) {
2839 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
2840 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2841 fprs, 128);
2842 } else {
2843 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2844 vcpu->run->s.regs.fprs, 128);
2845 }
2846 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
2847 vcpu->run->s.regs.gprs, 128);
2848 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
2849 &vcpu->arch.sie_block->gpsw, 16);
2850 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
2851 &px, 4);
2852 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
2853 &vcpu->run->s.regs.fpc, 4);
2854 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
2855 &vcpu->arch.sie_block->todpr, 4);
2856 cputm = kvm_s390_get_cpu_timer(vcpu);
2857 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
2858 &cputm, 8);
2859 clkcomp = vcpu->arch.sie_block->ckc >> 8;
2860 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
2861 &clkcomp, 8);
2862 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
2863 &vcpu->run->s.regs.acrs, 64);
2864 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
2865 &vcpu->arch.sie_block->gcr, 128);
2866 return rc ? -EFAULT : 0;
2867 }
2868
2869 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2870 {
2871 /*
2872 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2873 * switch in the run ioctl. Let's update our copies before we save
2874 * it into the save area
2875 */
2876 save_fpu_regs();
2877 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
2878 save_access_regs(vcpu->run->s.regs.acrs);
2879
2880 return kvm_s390_store_status_unloaded(vcpu, addr);
2881 }
2882
2883 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2884 {
2885 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2886 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
2887 }
2888
2889 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2890 {
2891 unsigned int i;
2892 struct kvm_vcpu *vcpu;
2893
2894 kvm_for_each_vcpu(i, vcpu, kvm) {
2895 __disable_ibs_on_vcpu(vcpu);
2896 }
2897 }
2898
2899 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2900 {
2901 if (!sclp.has_ibs)
2902 return;
2903 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2904 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
2905 }
2906
2907 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2908 {
2909 int i, online_vcpus, started_vcpus = 0;
2910
2911 if (!is_vcpu_stopped(vcpu))
2912 return;
2913
2914 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2915 /* Only one cpu at a time may enter/leave the STOPPED state. */
2916 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2917 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2918
2919 for (i = 0; i < online_vcpus; i++) {
2920 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2921 started_vcpus++;
2922 }
2923
2924 if (started_vcpus == 0) {
2925 /* we're the only active VCPU -> speed it up */
2926 __enable_ibs_on_vcpu(vcpu);
2927 } else if (started_vcpus == 1) {
2928 /*
2929 * As we are starting a second VCPU, we have to disable
2930 * the IBS facility on all VCPUs to remove potentially
2931 * oustanding ENABLE requests.
2932 */
2933 __disable_ibs_on_all_vcpus(vcpu->kvm);
2934 }
2935
2936 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2937 /*
2938 * Another VCPU might have used IBS while we were offline.
2939 * Let's play safe and flush the VCPU at startup.
2940 */
2941 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2942 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2943 return;
2944 }
2945
2946 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2947 {
2948 int i, online_vcpus, started_vcpus = 0;
2949 struct kvm_vcpu *started_vcpu = NULL;
2950
2951 if (is_vcpu_stopped(vcpu))
2952 return;
2953
2954 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2955 /* Only one cpu at a time may enter/leave the STOPPED state. */
2956 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2957 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2958
2959 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2960 kvm_s390_clear_stop_irq(vcpu);
2961
2962 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2963 __disable_ibs_on_vcpu(vcpu);
2964
2965 for (i = 0; i < online_vcpus; i++) {
2966 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2967 started_vcpus++;
2968 started_vcpu = vcpu->kvm->vcpus[i];
2969 }
2970 }
2971
2972 if (started_vcpus == 1) {
2973 /*
2974 * As we only have one VCPU left, we want to enable the
2975 * IBS facility for that VCPU to speed it up.
2976 */
2977 __enable_ibs_on_vcpu(started_vcpu);
2978 }
2979
2980 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2981 return;
2982 }
2983
2984 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2985 struct kvm_enable_cap *cap)
2986 {
2987 int r;
2988
2989 if (cap->flags)
2990 return -EINVAL;
2991
2992 switch (cap->cap) {
2993 case KVM_CAP_S390_CSS_SUPPORT:
2994 if (!vcpu->kvm->arch.css_support) {
2995 vcpu->kvm->arch.css_support = 1;
2996 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
2997 trace_kvm_s390_enable_css(vcpu->kvm);
2998 }
2999 r = 0;
3000 break;
3001 default:
3002 r = -EINVAL;
3003 break;
3004 }
3005 return r;
3006 }
3007
3008 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
3009 struct kvm_s390_mem_op *mop)
3010 {
3011 void __user *uaddr = (void __user *)mop->buf;
3012 void *tmpbuf = NULL;
3013 int r, srcu_idx;
3014 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
3015 | KVM_S390_MEMOP_F_CHECK_ONLY;
3016
3017 if (mop->flags & ~supported_flags)
3018 return -EINVAL;
3019
3020 if (mop->size > MEM_OP_MAX_SIZE)
3021 return -E2BIG;
3022
3023 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
3024 tmpbuf = vmalloc(mop->size);
3025 if (!tmpbuf)
3026 return -ENOMEM;
3027 }
3028
3029 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3030
3031 switch (mop->op) {
3032 case KVM_S390_MEMOP_LOGICAL_READ:
3033 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
3034 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3035 mop->size, GACC_FETCH);
3036 break;
3037 }
3038 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3039 if (r == 0) {
3040 if (copy_to_user(uaddr, tmpbuf, mop->size))
3041 r = -EFAULT;
3042 }
3043 break;
3044 case KVM_S390_MEMOP_LOGICAL_WRITE:
3045 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
3046 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3047 mop->size, GACC_STORE);
3048 break;
3049 }
3050 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
3051 r = -EFAULT;
3052 break;
3053 }
3054 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3055 break;
3056 default:
3057 r = -EINVAL;
3058 }
3059
3060 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
3061
3062 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
3063 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
3064
3065 vfree(tmpbuf);
3066 return r;
3067 }
3068
3069 long kvm_arch_vcpu_ioctl(struct file *filp,
3070 unsigned int ioctl, unsigned long arg)
3071 {
3072 struct kvm_vcpu *vcpu = filp->private_data;
3073 void __user *argp = (void __user *)arg;
3074 int idx;
3075 long r;
3076
3077 switch (ioctl) {
3078 case KVM_S390_IRQ: {
3079 struct kvm_s390_irq s390irq;
3080
3081 r = -EFAULT;
3082 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
3083 break;
3084 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3085 break;
3086 }
3087 case KVM_S390_INTERRUPT: {
3088 struct kvm_s390_interrupt s390int;
3089 struct kvm_s390_irq s390irq;
3090
3091 r = -EFAULT;
3092 if (copy_from_user(&s390int, argp, sizeof(s390int)))
3093 break;
3094 if (s390int_to_s390irq(&s390int, &s390irq))
3095 return -EINVAL;
3096 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3097 break;
3098 }
3099 case KVM_S390_STORE_STATUS:
3100 idx = srcu_read_lock(&vcpu->kvm->srcu);
3101 r = kvm_s390_vcpu_store_status(vcpu, arg);
3102 srcu_read_unlock(&vcpu->kvm->srcu, idx);
3103 break;
3104 case KVM_S390_SET_INITIAL_PSW: {
3105 psw_t psw;
3106
3107 r = -EFAULT;
3108 if (copy_from_user(&psw, argp, sizeof(psw)))
3109 break;
3110 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3111 break;
3112 }
3113 case KVM_S390_INITIAL_RESET:
3114 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3115 break;
3116 case KVM_SET_ONE_REG:
3117 case KVM_GET_ONE_REG: {
3118 struct kvm_one_reg reg;
3119 r = -EFAULT;
3120 if (copy_from_user(&reg, argp, sizeof(reg)))
3121 break;
3122 if (ioctl == KVM_SET_ONE_REG)
3123 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3124 else
3125 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3126 break;
3127 }
3128 #ifdef CONFIG_KVM_S390_UCONTROL
3129 case KVM_S390_UCAS_MAP: {
3130 struct kvm_s390_ucas_mapping ucasmap;
3131
3132 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3133 r = -EFAULT;
3134 break;
3135 }
3136
3137 if (!kvm_is_ucontrol(vcpu->kvm)) {
3138 r = -EINVAL;
3139 break;
3140 }
3141
3142 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3143 ucasmap.vcpu_addr, ucasmap.length);
3144 break;
3145 }
3146 case KVM_S390_UCAS_UNMAP: {
3147 struct kvm_s390_ucas_mapping ucasmap;
3148
3149 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3150 r = -EFAULT;
3151 break;
3152 }
3153
3154 if (!kvm_is_ucontrol(vcpu->kvm)) {
3155 r = -EINVAL;
3156 break;
3157 }
3158
3159 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3160 ucasmap.length);
3161 break;
3162 }
3163 #endif
3164 case KVM_S390_VCPU_FAULT: {
3165 r = gmap_fault(vcpu->arch.gmap, arg, 0);
3166 break;
3167 }
3168 case KVM_ENABLE_CAP:
3169 {
3170 struct kvm_enable_cap cap;
3171 r = -EFAULT;
3172 if (copy_from_user(&cap, argp, sizeof(cap)))
3173 break;
3174 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3175 break;
3176 }
3177 case KVM_S390_MEM_OP: {
3178 struct kvm_s390_mem_op mem_op;
3179
3180 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3181 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3182 else
3183 r = -EFAULT;
3184 break;
3185 }
3186 case KVM_S390_SET_IRQ_STATE: {
3187 struct kvm_s390_irq_state irq_state;
3188
3189 r = -EFAULT;
3190 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3191 break;
3192 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3193 irq_state.len == 0 ||
3194 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3195 r = -EINVAL;
3196 break;
3197 }
3198 r = kvm_s390_set_irq_state(vcpu,
3199 (void __user *) irq_state.buf,
3200 irq_state.len);
3201 break;
3202 }
3203 case KVM_S390_GET_IRQ_STATE: {
3204 struct kvm_s390_irq_state irq_state;
3205
3206 r = -EFAULT;
3207 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3208 break;
3209 if (irq_state.len == 0) {
3210 r = -EINVAL;
3211 break;
3212 }
3213 r = kvm_s390_get_irq_state(vcpu,
3214 (__u8 __user *) irq_state.buf,
3215 irq_state.len);
3216 break;
3217 }
3218 default:
3219 r = -ENOTTY;
3220 }
3221 return r;
3222 }
3223
3224 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3225 {
3226 #ifdef CONFIG_KVM_S390_UCONTROL
3227 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3228 && (kvm_is_ucontrol(vcpu->kvm))) {
3229 vmf->page = virt_to_page(vcpu->arch.sie_block);
3230 get_page(vmf->page);
3231 return 0;
3232 }
3233 #endif
3234 return VM_FAULT_SIGBUS;
3235 }
3236
3237 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3238 unsigned long npages)
3239 {
3240 return 0;
3241 }
3242
3243 /* Section: memory related */
3244 int kvm_arch_prepare_memory_region(struct kvm *kvm,
3245 struct kvm_memory_slot *memslot,
3246 const struct kvm_userspace_memory_region *mem,
3247 enum kvm_mr_change change)
3248 {
3249 /* A few sanity checks. We can have memory slots which have to be
3250 located/ended at a segment boundary (1MB). The memory in userland is
3251 ok to be fragmented into various different vmas. It is okay to mmap()
3252 and munmap() stuff in this slot after doing this call at any time */
3253
3254 if (mem->userspace_addr & 0xffffful)
3255 return -EINVAL;
3256
3257 if (mem->memory_size & 0xffffful)
3258 return -EINVAL;
3259
3260 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3261 return -EINVAL;
3262
3263 return 0;
3264 }
3265
3266 void kvm_arch_commit_memory_region(struct kvm *kvm,
3267 const struct kvm_userspace_memory_region *mem,
3268 const struct kvm_memory_slot *old,
3269 const struct kvm_memory_slot *new,
3270 enum kvm_mr_change change)
3271 {
3272 int rc;
3273
3274 /* If the basics of the memslot do not change, we do not want
3275 * to update the gmap. Every update causes several unnecessary
3276 * segment translation exceptions. This is usually handled just
3277 * fine by the normal fault handler + gmap, but it will also
3278 * cause faults on the prefix page of running guest CPUs.
3279 */
3280 if (old->userspace_addr == mem->userspace_addr &&
3281 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3282 old->npages * PAGE_SIZE == mem->memory_size)
3283 return;
3284
3285 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3286 mem->guest_phys_addr, mem->memory_size);
3287 if (rc)
3288 pr_warn("failed to commit memory region\n");
3289 return;
3290 }
3291
3292 static inline unsigned long nonhyp_mask(int i)
3293 {
3294 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3295
3296 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3297 }
3298
3299 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3300 {
3301 vcpu->valid_wakeup = false;
3302 }
3303
3304 static int __init kvm_s390_init(void)
3305 {
3306 int i;
3307
3308 if (!sclp.has_sief2) {
3309 pr_info("SIE not available\n");
3310 return -ENODEV;
3311 }
3312
3313 for (i = 0; i < 16; i++)
3314 kvm_s390_fac_list_mask[i] |=
3315 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3316
3317 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
3318 }
3319
3320 static void __exit kvm_s390_exit(void)
3321 {
3322 kvm_exit();
3323 }
3324
3325 module_init(kvm_s390_init);
3326 module_exit(kvm_s390_exit);
3327
3328 /*
3329 * Enable autoloading of the kvm module.
3330 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3331 * since x86 takes a different approach.
3332 */
3333 #include <linux/miscdevice.h>
3334 MODULE_ALIAS_MISCDEV(KVM_MINOR);
3335 MODULE_ALIAS("devname:kvm");
Ubuntu Artful kernel mirror