]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame_incremental - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-jammy-kernel.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
s390/mm: don't fault everything in read-write in gmap_pte_op_fixup()
[mirror_ubuntu-jammy-kernel.git] / arch / s390 / kvm / kvm-s390.c
... / ...
CommitLineData
1/*
2 * hosting zSeries kernel virtual machines
3 *
4 * Copyright IBM Corp. 2008, 2009
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
20#include <linux/hrtimer.h>
21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
24#include <linux/mman.h>
25#include <linux/module.h>
26#include <linux/random.h>
27#include <linux/slab.h>
28#include <linux/timer.h>
29#include <linux/vmalloc.h>
30#include <linux/bitmap.h>
31#include <asm/asm-offsets.h>
32#include <asm/lowcore.h>
33#include <asm/etr.h>
34#include <asm/pgtable.h>
35#include <asm/gmap.h>
36#include <asm/nmi.h>
37#include <asm/switch_to.h>
38#include <asm/isc.h>
39#include <asm/sclp.h>
40#include <asm/cpacf.h>
41#include <asm/etr.h>
42#include "kvm-s390.h"
43#include "gaccess.h"
44
45#define KMSG_COMPONENT "kvm-s390"
46#undef pr_fmt
47#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
48
49#define CREATE_TRACE_POINTS
50#include "trace.h"
51#include "trace-s390.h"
52
53#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
54#define LOCAL_IRQS 32
55#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
56 (KVM_MAX_VCPUS + LOCAL_IRQS))
57
58#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
59
60struct kvm_stats_debugfs_item debugfs_entries[] = {
61 { "userspace_handled", VCPU_STAT(exit_userspace) },
62 { "exit_null", VCPU_STAT(exit_null) },
63 { "exit_validity", VCPU_STAT(exit_validity) },
64 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
65 { "exit_external_request", VCPU_STAT(exit_external_request) },
66 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
67 { "exit_instruction", VCPU_STAT(exit_instruction) },
68 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
69 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
70 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
71 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
72 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
73 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
74 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
75 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
76 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
77 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
78 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
79 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
80 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
81 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
82 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
83 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
84 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
85 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
86 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
87 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
88 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
89 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
90 { "instruction_spx", VCPU_STAT(instruction_spx) },
91 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
92 { "instruction_stap", VCPU_STAT(instruction_stap) },
93 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
94 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
95 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
96 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
97 { "instruction_essa", VCPU_STAT(instruction_essa) },
98 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
99 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
100 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
101 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
102 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
103 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
104 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
105 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
106 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
107 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
108 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
109 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
110 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
111 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
112 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
113 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
114 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
115 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
116 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
117 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
118 { "diagnose_10", VCPU_STAT(diagnose_10) },
119 { "diagnose_44", VCPU_STAT(diagnose_44) },
120 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
121 { "diagnose_258", VCPU_STAT(diagnose_258) },
122 { "diagnose_308", VCPU_STAT(diagnose_308) },
123 { "diagnose_500", VCPU_STAT(diagnose_500) },
124 { NULL }
125};
126
127/* upper facilities limit for kvm */
128unsigned long kvm_s390_fac_list_mask[16] = {
129 0xffe6000000000000UL,
130 0x005e000000000000UL,
131};
132
133unsigned long kvm_s390_fac_list_mask_size(void)
134{
135 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
136 return ARRAY_SIZE(kvm_s390_fac_list_mask);
137}
138
139/* available cpu features supported by kvm */
140static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
141/* available subfunctions indicated via query / "test bit" */
142static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
143
144static struct gmap_notifier gmap_notifier;
145debug_info_t *kvm_s390_dbf;
146
147/* Section: not file related */
148int kvm_arch_hardware_enable(void)
149{
150 /* every s390 is virtualization enabled ;-) */
151 return 0;
152}
153
154static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
155 unsigned long end);
156
157/*
158 * This callback is executed during stop_machine(). All CPUs are therefore
159 * temporarily stopped. In order not to change guest behavior, we have to
160 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
161 * so a CPU won't be stopped while calculating with the epoch.
162 */
163static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
164 void *v)
165{
166 struct kvm *kvm;
167 struct kvm_vcpu *vcpu;
168 int i;
169 unsigned long long *delta = v;
170
171 list_for_each_entry(kvm, &vm_list, vm_list) {
172 kvm->arch.epoch -= *delta;
173 kvm_for_each_vcpu(i, vcpu, kvm) {
174 vcpu->arch.sie_block->epoch -= *delta;
175 if (vcpu->arch.cputm_enabled)
176 vcpu->arch.cputm_start += *delta;
177 }
178 }
179 return NOTIFY_OK;
180}
181
182static struct notifier_block kvm_clock_notifier = {
183 .notifier_call = kvm_clock_sync,
184};
185
186int kvm_arch_hardware_setup(void)
187{
188 gmap_notifier.notifier_call = kvm_gmap_notifier;
189 gmap_register_pte_notifier(&gmap_notifier);
190 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
191 &kvm_clock_notifier);
192 return 0;
193}
194
195void kvm_arch_hardware_unsetup(void)
196{
197 gmap_unregister_pte_notifier(&gmap_notifier);
198 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
199 &kvm_clock_notifier);
200}
201
202static void allow_cpu_feat(unsigned long nr)
203{
204 set_bit_inv(nr, kvm_s390_available_cpu_feat);
205}
206
207static inline int plo_test_bit(unsigned char nr)
208{
209 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
210 int cc = 3; /* subfunction not available */
211
212 asm volatile(
213 /* Parameter registers are ignored for "test bit" */
214 " plo 0,0,0,0(0)\n"
215 " ipm %0\n"
216 " srl %0,28\n"
217 : "=d" (cc)
218 : "d" (r0)
219 : "cc");
220 return cc == 0;
221}
222
223static void kvm_s390_cpu_feat_init(void)
224{
225 int i;
226
227 for (i = 0; i < 256; ++i) {
228 if (plo_test_bit(i))
229 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
230 }
231
232 if (test_facility(28)) /* TOD-clock steering */
233 etr_ptff(kvm_s390_available_subfunc.ptff, ETR_PTFF_QAF);
234
235 if (test_facility(17)) { /* MSA */
236 __cpacf_query(CPACF_KMAC, kvm_s390_available_subfunc.kmac);
237 __cpacf_query(CPACF_KMC, kvm_s390_available_subfunc.kmc);
238 __cpacf_query(CPACF_KM, kvm_s390_available_subfunc.km);
239 __cpacf_query(CPACF_KIMD, kvm_s390_available_subfunc.kimd);
240 __cpacf_query(CPACF_KLMD, kvm_s390_available_subfunc.klmd);
241 }
242 if (test_facility(76)) /* MSA3 */
243 __cpacf_query(CPACF_PCKMO, kvm_s390_available_subfunc.pckmo);
244 if (test_facility(77)) { /* MSA4 */
245 __cpacf_query(CPACF_KMCTR, kvm_s390_available_subfunc.kmctr);
246 __cpacf_query(CPACF_KMF, kvm_s390_available_subfunc.kmf);
247 __cpacf_query(CPACF_KMO, kvm_s390_available_subfunc.kmo);
248 __cpacf_query(CPACF_PCC, kvm_s390_available_subfunc.pcc);
249 }
250 if (test_facility(57)) /* MSA5 */
251 __cpacf_query(CPACF_PPNO, kvm_s390_available_subfunc.ppno);
252
253 if (MACHINE_HAS_ESOP)
254 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
255}
256
257int kvm_arch_init(void *opaque)
258{
259 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
260 if (!kvm_s390_dbf)
261 return -ENOMEM;
262
263 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
264 debug_unregister(kvm_s390_dbf);
265 return -ENOMEM;
266 }
267
268 kvm_s390_cpu_feat_init();
269
270 /* Register floating interrupt controller interface. */
271 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
272}
273
274void kvm_arch_exit(void)
275{
276 debug_unregister(kvm_s390_dbf);
277}
278
279/* Section: device related */
280long kvm_arch_dev_ioctl(struct file *filp,
281 unsigned int ioctl, unsigned long arg)
282{
283 if (ioctl == KVM_S390_ENABLE_SIE)
284 return s390_enable_sie();
285 return -EINVAL;
286}
287
288int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
289{
290 int r;
291
292 switch (ext) {
293 case KVM_CAP_S390_PSW:
294 case KVM_CAP_S390_GMAP:
295 case KVM_CAP_SYNC_MMU:
296#ifdef CONFIG_KVM_S390_UCONTROL
297 case KVM_CAP_S390_UCONTROL:
298#endif
299 case KVM_CAP_ASYNC_PF:
300 case KVM_CAP_SYNC_REGS:
301 case KVM_CAP_ONE_REG:
302 case KVM_CAP_ENABLE_CAP:
303 case KVM_CAP_S390_CSS_SUPPORT:
304 case KVM_CAP_IOEVENTFD:
305 case KVM_CAP_DEVICE_CTRL:
306 case KVM_CAP_ENABLE_CAP_VM:
307 case KVM_CAP_S390_IRQCHIP:
308 case KVM_CAP_VM_ATTRIBUTES:
309 case KVM_CAP_MP_STATE:
310 case KVM_CAP_S390_INJECT_IRQ:
311 case KVM_CAP_S390_USER_SIGP:
312 case KVM_CAP_S390_USER_STSI:
313 case KVM_CAP_S390_SKEYS:
314 case KVM_CAP_S390_IRQ_STATE:
315 r = 1;
316 break;
317 case KVM_CAP_S390_MEM_OP:
318 r = MEM_OP_MAX_SIZE;
319 break;
320 case KVM_CAP_NR_VCPUS:
321 case KVM_CAP_MAX_VCPUS:
322 r = KVM_S390_BSCA_CPU_SLOTS;
323 if (sclp.has_esca && sclp.has_64bscao)
324 r = KVM_S390_ESCA_CPU_SLOTS;
325 break;
326 case KVM_CAP_NR_MEMSLOTS:
327 r = KVM_USER_MEM_SLOTS;
328 break;
329 case KVM_CAP_S390_COW:
330 r = MACHINE_HAS_ESOP;
331 break;
332 case KVM_CAP_S390_VECTOR_REGISTERS:
333 r = MACHINE_HAS_VX;
334 break;
335 case KVM_CAP_S390_RI:
336 r = test_facility(64);
337 break;
338 default:
339 r = 0;
340 }
341 return r;
342}
343
344static void kvm_s390_sync_dirty_log(struct kvm *kvm,
345 struct kvm_memory_slot *memslot)
346{
347 gfn_t cur_gfn, last_gfn;
348 unsigned long address;
349 struct gmap *gmap = kvm->arch.gmap;
350
351 /* Loop over all guest pages */
352 last_gfn = memslot->base_gfn + memslot->npages;
353 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
354 address = gfn_to_hva_memslot(memslot, cur_gfn);
355
356 if (test_and_clear_guest_dirty(gmap->mm, address))
357 mark_page_dirty(kvm, cur_gfn);
358 if (fatal_signal_pending(current))
359 return;
360 cond_resched();
361 }
362}
363
364/* Section: vm related */
365static void sca_del_vcpu(struct kvm_vcpu *vcpu);
366
367/*
368 * Get (and clear) the dirty memory log for a memory slot.
369 */
370int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
371 struct kvm_dirty_log *log)
372{
373 int r;
374 unsigned long n;
375 struct kvm_memslots *slots;
376 struct kvm_memory_slot *memslot;
377 int is_dirty = 0;
378
379 mutex_lock(&kvm->slots_lock);
380
381 r = -EINVAL;
382 if (log->slot >= KVM_USER_MEM_SLOTS)
383 goto out;
384
385 slots = kvm_memslots(kvm);
386 memslot = id_to_memslot(slots, log->slot);
387 r = -ENOENT;
388 if (!memslot->dirty_bitmap)
389 goto out;
390
391 kvm_s390_sync_dirty_log(kvm, memslot);
392 r = kvm_get_dirty_log(kvm, log, &is_dirty);
393 if (r)
394 goto out;
395
396 /* Clear the dirty log */
397 if (is_dirty) {
398 n = kvm_dirty_bitmap_bytes(memslot);
399 memset(memslot->dirty_bitmap, 0, n);
400 }
401 r = 0;
402out:
403 mutex_unlock(&kvm->slots_lock);
404 return r;
405}
406
407static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
408{
409 int r;
410
411 if (cap->flags)
412 return -EINVAL;
413
414 switch (cap->cap) {
415 case KVM_CAP_S390_IRQCHIP:
416 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
417 kvm->arch.use_irqchip = 1;
418 r = 0;
419 break;
420 case KVM_CAP_S390_USER_SIGP:
421 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
422 kvm->arch.user_sigp = 1;
423 r = 0;
424 break;
425 case KVM_CAP_S390_VECTOR_REGISTERS:
426 mutex_lock(&kvm->lock);
427 if (kvm->created_vcpus) {
428 r = -EBUSY;
429 } else if (MACHINE_HAS_VX) {
430 set_kvm_facility(kvm->arch.model.fac_mask, 129);
431 set_kvm_facility(kvm->arch.model.fac_list, 129);
432 r = 0;
433 } else
434 r = -EINVAL;
435 mutex_unlock(&kvm->lock);
436 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
437 r ? "(not available)" : "(success)");
438 break;
439 case KVM_CAP_S390_RI:
440 r = -EINVAL;
441 mutex_lock(&kvm->lock);
442 if (kvm->created_vcpus) {
443 r = -EBUSY;
444 } else if (test_facility(64)) {
445 set_kvm_facility(kvm->arch.model.fac_mask, 64);
446 set_kvm_facility(kvm->arch.model.fac_list, 64);
447 r = 0;
448 }
449 mutex_unlock(&kvm->lock);
450 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
451 r ? "(not available)" : "(success)");
452 break;
453 case KVM_CAP_S390_USER_STSI:
454 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
455 kvm->arch.user_stsi = 1;
456 r = 0;
457 break;
458 default:
459 r = -EINVAL;
460 break;
461 }
462 return r;
463}
464
465static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
466{
467 int ret;
468
469 switch (attr->attr) {
470 case KVM_S390_VM_MEM_LIMIT_SIZE:
471 ret = 0;
472 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
473 kvm->arch.mem_limit);
474 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
475 ret = -EFAULT;
476 break;
477 default:
478 ret = -ENXIO;
479 break;
480 }
481 return ret;
482}
483
484static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
485{
486 int ret;
487 unsigned int idx;
488 switch (attr->attr) {
489 case KVM_S390_VM_MEM_ENABLE_CMMA:
490 ret = -ENXIO;
491 if (!sclp.has_cmma)
492 break;
493
494 ret = -EBUSY;
495 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
496 mutex_lock(&kvm->lock);
497 if (!kvm->created_vcpus) {
498 kvm->arch.use_cmma = 1;
499 ret = 0;
500 }
501 mutex_unlock(&kvm->lock);
502 break;
503 case KVM_S390_VM_MEM_CLR_CMMA:
504 ret = -ENXIO;
505 if (!sclp.has_cmma)
506 break;
507 ret = -EINVAL;
508 if (!kvm->arch.use_cmma)
509 break;
510
511 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
512 mutex_lock(&kvm->lock);
513 idx = srcu_read_lock(&kvm->srcu);
514 s390_reset_cmma(kvm->arch.gmap->mm);
515 srcu_read_unlock(&kvm->srcu, idx);
516 mutex_unlock(&kvm->lock);
517 ret = 0;
518 break;
519 case KVM_S390_VM_MEM_LIMIT_SIZE: {
520 unsigned long new_limit;
521
522 if (kvm_is_ucontrol(kvm))
523 return -EINVAL;
524
525 if (get_user(new_limit, (u64 __user *)attr->addr))
526 return -EFAULT;
527
528 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
529 new_limit > kvm->arch.mem_limit)
530 return -E2BIG;
531
532 if (!new_limit)
533 return -EINVAL;
534
535 /* gmap_create takes last usable address */
536 if (new_limit != KVM_S390_NO_MEM_LIMIT)
537 new_limit -= 1;
538
539 ret = -EBUSY;
540 mutex_lock(&kvm->lock);
541 if (!kvm->created_vcpus) {
542 /* gmap_create will round the limit up */
543 struct gmap *new = gmap_create(current->mm, new_limit);
544
545 if (!new) {
546 ret = -ENOMEM;
547 } else {
548 gmap_remove(kvm->arch.gmap);
549 new->private = kvm;
550 kvm->arch.gmap = new;
551 ret = 0;
552 }
553 }
554 mutex_unlock(&kvm->lock);
555 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
556 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
557 (void *) kvm->arch.gmap->asce);
558 break;
559 }
560 default:
561 ret = -ENXIO;
562 break;
563 }
564 return ret;
565}
566
567static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
568
569static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
570{
571 struct kvm_vcpu *vcpu;
572 int i;
573
574 if (!test_kvm_facility(kvm, 76))
575 return -EINVAL;
576
577 mutex_lock(&kvm->lock);
578 switch (attr->attr) {
579 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
580 get_random_bytes(
581 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
582 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
583 kvm->arch.crypto.aes_kw = 1;
584 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
585 break;
586 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
587 get_random_bytes(
588 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
589 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
590 kvm->arch.crypto.dea_kw = 1;
591 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
592 break;
593 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
594 kvm->arch.crypto.aes_kw = 0;
595 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
596 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
597 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
598 break;
599 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
600 kvm->arch.crypto.dea_kw = 0;
601 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
602 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
603 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
604 break;
605 default:
606 mutex_unlock(&kvm->lock);
607 return -ENXIO;
608 }
609
610 kvm_for_each_vcpu(i, vcpu, kvm) {
611 kvm_s390_vcpu_crypto_setup(vcpu);
612 exit_sie(vcpu);
613 }
614 mutex_unlock(&kvm->lock);
615 return 0;
616}
617
618static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
619{
620 u8 gtod_high;
621
622 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
623 sizeof(gtod_high)))
624 return -EFAULT;
625
626 if (gtod_high != 0)
627 return -EINVAL;
628 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
629
630 return 0;
631}
632
633static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
634{
635 u64 gtod;
636
637 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
638 return -EFAULT;
639
640 kvm_s390_set_tod_clock(kvm, gtod);
641 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
642 return 0;
643}
644
645static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
646{
647 int ret;
648
649 if (attr->flags)
650 return -EINVAL;
651
652 switch (attr->attr) {
653 case KVM_S390_VM_TOD_HIGH:
654 ret = kvm_s390_set_tod_high(kvm, attr);
655 break;
656 case KVM_S390_VM_TOD_LOW:
657 ret = kvm_s390_set_tod_low(kvm, attr);
658 break;
659 default:
660 ret = -ENXIO;
661 break;
662 }
663 return ret;
664}
665
666static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
667{
668 u8 gtod_high = 0;
669
670 if (copy_to_user((void __user *)attr->addr, &gtod_high,
671 sizeof(gtod_high)))
672 return -EFAULT;
673 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
674
675 return 0;
676}
677
678static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
679{
680 u64 gtod;
681
682 gtod = kvm_s390_get_tod_clock_fast(kvm);
683 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
684 return -EFAULT;
685 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
686
687 return 0;
688}
689
690static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
691{
692 int ret;
693
694 if (attr->flags)
695 return -EINVAL;
696
697 switch (attr->attr) {
698 case KVM_S390_VM_TOD_HIGH:
699 ret = kvm_s390_get_tod_high(kvm, attr);
700 break;
701 case KVM_S390_VM_TOD_LOW:
702 ret = kvm_s390_get_tod_low(kvm, attr);
703 break;
704 default:
705 ret = -ENXIO;
706 break;
707 }
708 return ret;
709}
710
711static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
712{
713 struct kvm_s390_vm_cpu_processor *proc;
714 u16 lowest_ibc, unblocked_ibc;
715 int ret = 0;
716
717 mutex_lock(&kvm->lock);
718 if (kvm->created_vcpus) {
719 ret = -EBUSY;
720 goto out;
721 }
722 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
723 if (!proc) {
724 ret = -ENOMEM;
725 goto out;
726 }
727 if (!copy_from_user(proc, (void __user *)attr->addr,
728 sizeof(*proc))) {
729 kvm->arch.model.cpuid = proc->cpuid;
730 lowest_ibc = sclp.ibc >> 16 & 0xfff;
731 unblocked_ibc = sclp.ibc & 0xfff;
732 if (lowest_ibc) {
733 if (proc->ibc > unblocked_ibc)
734 kvm->arch.model.ibc = unblocked_ibc;
735 else if (proc->ibc < lowest_ibc)
736 kvm->arch.model.ibc = lowest_ibc;
737 else
738 kvm->arch.model.ibc = proc->ibc;
739 }
740 memcpy(kvm->arch.model.fac_list, proc->fac_list,
741 S390_ARCH_FAC_LIST_SIZE_BYTE);
742 } else
743 ret = -EFAULT;
744 kfree(proc);
745out:
746 mutex_unlock(&kvm->lock);
747 return ret;
748}
749
750static int kvm_s390_set_processor_feat(struct kvm *kvm,
751 struct kvm_device_attr *attr)
752{
753 struct kvm_s390_vm_cpu_feat data;
754 int ret = -EBUSY;
755
756 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
757 return -EFAULT;
758 if (!bitmap_subset((unsigned long *) data.feat,
759 kvm_s390_available_cpu_feat,
760 KVM_S390_VM_CPU_FEAT_NR_BITS))
761 return -EINVAL;
762
763 mutex_lock(&kvm->lock);
764 if (!atomic_read(&kvm->online_vcpus)) {
765 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
766 KVM_S390_VM_CPU_FEAT_NR_BITS);
767 ret = 0;
768 }
769 mutex_unlock(&kvm->lock);
770 return ret;
771}
772
773static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
774 struct kvm_device_attr *attr)
775{
776 /*
777 * Once supported by kernel + hw, we have to store the subfunctions
778 * in kvm->arch and remember that user space configured them.
779 */
780 return -ENXIO;
781}
782
783static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
784{
785 int ret = -ENXIO;
786
787 switch (attr->attr) {
788 case KVM_S390_VM_CPU_PROCESSOR:
789 ret = kvm_s390_set_processor(kvm, attr);
790 break;
791 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
792 ret = kvm_s390_set_processor_feat(kvm, attr);
793 break;
794 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
795 ret = kvm_s390_set_processor_subfunc(kvm, attr);
796 break;
797 }
798 return ret;
799}
800
801static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
802{
803 struct kvm_s390_vm_cpu_processor *proc;
804 int ret = 0;
805
806 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
807 if (!proc) {
808 ret = -ENOMEM;
809 goto out;
810 }
811 proc->cpuid = kvm->arch.model.cpuid;
812 proc->ibc = kvm->arch.model.ibc;
813 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
814 S390_ARCH_FAC_LIST_SIZE_BYTE);
815 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
816 ret = -EFAULT;
817 kfree(proc);
818out:
819 return ret;
820}
821
822static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
823{
824 struct kvm_s390_vm_cpu_machine *mach;
825 int ret = 0;
826
827 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
828 if (!mach) {
829 ret = -ENOMEM;
830 goto out;
831 }
832 get_cpu_id((struct cpuid *) &mach->cpuid);
833 mach->ibc = sclp.ibc;
834 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
835 S390_ARCH_FAC_LIST_SIZE_BYTE);
836 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
837 S390_ARCH_FAC_LIST_SIZE_BYTE);
838 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
839 ret = -EFAULT;
840 kfree(mach);
841out:
842 return ret;
843}
844
845static int kvm_s390_get_processor_feat(struct kvm *kvm,
846 struct kvm_device_attr *attr)
847{
848 struct kvm_s390_vm_cpu_feat data;
849
850 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
851 KVM_S390_VM_CPU_FEAT_NR_BITS);
852 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
853 return -EFAULT;
854 return 0;
855}
856
857static int kvm_s390_get_machine_feat(struct kvm *kvm,
858 struct kvm_device_attr *attr)
859{
860 struct kvm_s390_vm_cpu_feat data;
861
862 bitmap_copy((unsigned long *) data.feat,
863 kvm_s390_available_cpu_feat,
864 KVM_S390_VM_CPU_FEAT_NR_BITS);
865 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
866 return -EFAULT;
867 return 0;
868}
869
870static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
871 struct kvm_device_attr *attr)
872{
873 /*
874 * Once we can actually configure subfunctions (kernel + hw support),
875 * we have to check if they were already set by user space, if so copy
876 * them from kvm->arch.
877 */
878 return -ENXIO;
879}
880
881static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
882 struct kvm_device_attr *attr)
883{
884 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
885 sizeof(struct kvm_s390_vm_cpu_subfunc)))
886 return -EFAULT;
887 return 0;
888}
889static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
890{
891 int ret = -ENXIO;
892
893 switch (attr->attr) {
894 case KVM_S390_VM_CPU_PROCESSOR:
895 ret = kvm_s390_get_processor(kvm, attr);
896 break;
897 case KVM_S390_VM_CPU_MACHINE:
898 ret = kvm_s390_get_machine(kvm, attr);
899 break;
900 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
901 ret = kvm_s390_get_processor_feat(kvm, attr);
902 break;
903 case KVM_S390_VM_CPU_MACHINE_FEAT:
904 ret = kvm_s390_get_machine_feat(kvm, attr);
905 break;
906 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
907 ret = kvm_s390_get_processor_subfunc(kvm, attr);
908 break;
909 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
910 ret = kvm_s390_get_machine_subfunc(kvm, attr);
911 break;
912 }
913 return ret;
914}
915
916static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
917{
918 int ret;
919
920 switch (attr->group) {
921 case KVM_S390_VM_MEM_CTRL:
922 ret = kvm_s390_set_mem_control(kvm, attr);
923 break;
924 case KVM_S390_VM_TOD:
925 ret = kvm_s390_set_tod(kvm, attr);
926 break;
927 case KVM_S390_VM_CPU_MODEL:
928 ret = kvm_s390_set_cpu_model(kvm, attr);
929 break;
930 case KVM_S390_VM_CRYPTO:
931 ret = kvm_s390_vm_set_crypto(kvm, attr);
932 break;
933 default:
934 ret = -ENXIO;
935 break;
936 }
937
938 return ret;
939}
940
941static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
942{
943 int ret;
944
945 switch (attr->group) {
946 case KVM_S390_VM_MEM_CTRL:
947 ret = kvm_s390_get_mem_control(kvm, attr);
948 break;
949 case KVM_S390_VM_TOD:
950 ret = kvm_s390_get_tod(kvm, attr);
951 break;
952 case KVM_S390_VM_CPU_MODEL:
953 ret = kvm_s390_get_cpu_model(kvm, attr);
954 break;
955 default:
956 ret = -ENXIO;
957 break;
958 }
959
960 return ret;
961}
962
963static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
964{
965 int ret;
966
967 switch (attr->group) {
968 case KVM_S390_VM_MEM_CTRL:
969 switch (attr->attr) {
970 case KVM_S390_VM_MEM_ENABLE_CMMA:
971 case KVM_S390_VM_MEM_CLR_CMMA:
972 ret = sclp.has_cmma ? 0 : -ENXIO;
973 break;
974 case KVM_S390_VM_MEM_LIMIT_SIZE:
975 ret = 0;
976 break;
977 default:
978 ret = -ENXIO;
979 break;
980 }
981 break;
982 case KVM_S390_VM_TOD:
983 switch (attr->attr) {
984 case KVM_S390_VM_TOD_LOW:
985 case KVM_S390_VM_TOD_HIGH:
986 ret = 0;
987 break;
988 default:
989 ret = -ENXIO;
990 break;
991 }
992 break;
993 case KVM_S390_VM_CPU_MODEL:
994 switch (attr->attr) {
995 case KVM_S390_VM_CPU_PROCESSOR:
996 case KVM_S390_VM_CPU_MACHINE:
997 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
998 case KVM_S390_VM_CPU_MACHINE_FEAT:
999 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1000 ret = 0;
1001 break;
1002 /* configuring subfunctions is not supported yet */
1003 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1004 default:
1005 ret = -ENXIO;
1006 break;
1007 }
1008 break;
1009 case KVM_S390_VM_CRYPTO:
1010 switch (attr->attr) {
1011 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1012 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1013 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1014 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1015 ret = 0;
1016 break;
1017 default:
1018 ret = -ENXIO;
1019 break;
1020 }
1021 break;
1022 default:
1023 ret = -ENXIO;
1024 break;
1025 }
1026
1027 return ret;
1028}
1029
1030static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1031{
1032 uint8_t *keys;
1033 uint64_t hva;
1034 int i, r = 0;
1035
1036 if (args->flags != 0)
1037 return -EINVAL;
1038
1039 /* Is this guest using storage keys? */
1040 if (!mm_use_skey(current->mm))
1041 return KVM_S390_GET_SKEYS_NONE;
1042
1043 /* Enforce sane limit on memory allocation */
1044 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1045 return -EINVAL;
1046
1047 keys = kmalloc_array(args->count, sizeof(uint8_t),
1048 GFP_KERNEL | __GFP_NOWARN);
1049 if (!keys)
1050 keys = vmalloc(sizeof(uint8_t) * args->count);
1051 if (!keys)
1052 return -ENOMEM;
1053
1054 down_read(&current->mm->mmap_sem);
1055 for (i = 0; i < args->count; i++) {
1056 hva = gfn_to_hva(kvm, args->start_gfn + i);
1057 if (kvm_is_error_hva(hva)) {
1058 r = -EFAULT;
1059 break;
1060 }
1061
1062 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1063 if (r)
1064 break;
1065 }
1066 up_read(&current->mm->mmap_sem);
1067
1068 if (!r) {
1069 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1070 sizeof(uint8_t) * args->count);
1071 if (r)
1072 r = -EFAULT;
1073 }
1074
1075 kvfree(keys);
1076 return r;
1077}
1078
1079static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1080{
1081 uint8_t *keys;
1082 uint64_t hva;
1083 int i, r = 0;
1084
1085 if (args->flags != 0)
1086 return -EINVAL;
1087
1088 /* Enforce sane limit on memory allocation */
1089 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1090 return -EINVAL;
1091
1092 keys = kmalloc_array(args->count, sizeof(uint8_t),
1093 GFP_KERNEL | __GFP_NOWARN);
1094 if (!keys)
1095 keys = vmalloc(sizeof(uint8_t) * args->count);
1096 if (!keys)
1097 return -ENOMEM;
1098
1099 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1100 sizeof(uint8_t) * args->count);
1101 if (r) {
1102 r = -EFAULT;
1103 goto out;
1104 }
1105
1106 /* Enable storage key handling for the guest */
1107 r = s390_enable_skey();
1108 if (r)
1109 goto out;
1110
1111 down_read(&current->mm->mmap_sem);
1112 for (i = 0; i < args->count; i++) {
1113 hva = gfn_to_hva(kvm, args->start_gfn + i);
1114 if (kvm_is_error_hva(hva)) {
1115 r = -EFAULT;
1116 break;
1117 }
1118
1119 /* Lowest order bit is reserved */
1120 if (keys[i] & 0x01) {
1121 r = -EINVAL;
1122 break;
1123 }
1124
1125 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1126 if (r)
1127 break;
1128 }
1129 up_read(&current->mm->mmap_sem);
1130out:
1131 kvfree(keys);
1132 return r;
1133}
1134
1135long kvm_arch_vm_ioctl(struct file *filp,
1136 unsigned int ioctl, unsigned long arg)
1137{
1138 struct kvm *kvm = filp->private_data;
1139 void __user *argp = (void __user *)arg;
1140 struct kvm_device_attr attr;
1141 int r;
1142
1143 switch (ioctl) {
1144 case KVM_S390_INTERRUPT: {
1145 struct kvm_s390_interrupt s390int;
1146
1147 r = -EFAULT;
1148 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1149 break;
1150 r = kvm_s390_inject_vm(kvm, &s390int);
1151 break;
1152 }
1153 case KVM_ENABLE_CAP: {
1154 struct kvm_enable_cap cap;
1155 r = -EFAULT;
1156 if (copy_from_user(&cap, argp, sizeof(cap)))
1157 break;
1158 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1159 break;
1160 }
1161 case KVM_CREATE_IRQCHIP: {
1162 struct kvm_irq_routing_entry routing;
1163
1164 r = -EINVAL;
1165 if (kvm->arch.use_irqchip) {
1166 /* Set up dummy routing. */
1167 memset(&routing, 0, sizeof(routing));
1168 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
1169 }
1170 break;
1171 }
1172 case KVM_SET_DEVICE_ATTR: {
1173 r = -EFAULT;
1174 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1175 break;
1176 r = kvm_s390_vm_set_attr(kvm, &attr);
1177 break;
1178 }
1179 case KVM_GET_DEVICE_ATTR: {
1180 r = -EFAULT;
1181 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1182 break;
1183 r = kvm_s390_vm_get_attr(kvm, &attr);
1184 break;
1185 }
1186 case KVM_HAS_DEVICE_ATTR: {
1187 r = -EFAULT;
1188 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1189 break;
1190 r = kvm_s390_vm_has_attr(kvm, &attr);
1191 break;
1192 }
1193 case KVM_S390_GET_SKEYS: {
1194 struct kvm_s390_skeys args;
1195
1196 r = -EFAULT;
1197 if (copy_from_user(&args, argp,
1198 sizeof(struct kvm_s390_skeys)))
1199 break;
1200 r = kvm_s390_get_skeys(kvm, &args);
1201 break;
1202 }
1203 case KVM_S390_SET_SKEYS: {
1204 struct kvm_s390_skeys args;
1205
1206 r = -EFAULT;
1207 if (copy_from_user(&args, argp,
1208 sizeof(struct kvm_s390_skeys)))
1209 break;
1210 r = kvm_s390_set_skeys(kvm, &args);
1211 break;
1212 }
1213 default:
1214 r = -ENOTTY;
1215 }
1216
1217 return r;
1218}
1219
1220static int kvm_s390_query_ap_config(u8 *config)
1221{
1222 u32 fcn_code = 0x04000000UL;
1223 u32 cc = 0;
1224
1225 memset(config, 0, 128);
1226 asm volatile(
1227 "lgr 0,%1\n"
1228 "lgr 2,%2\n"
1229 ".long 0xb2af0000\n" /* PQAP(QCI) */
1230 "0: ipm %0\n"
1231 "srl %0,28\n"
1232 "1:\n"
1233 EX_TABLE(0b, 1b)
1234 : "+r" (cc)
1235 : "r" (fcn_code), "r" (config)
1236 : "cc", "0", "2", "memory"
1237 );
1238
1239 return cc;
1240}
1241
1242static int kvm_s390_apxa_installed(void)
1243{
1244 u8 config[128];
1245 int cc;
1246
1247 if (test_facility(12)) {
1248 cc = kvm_s390_query_ap_config(config);
1249
1250 if (cc)
1251 pr_err("PQAP(QCI) failed with cc=%d", cc);
1252 else
1253 return config[0] & 0x40;
1254 }
1255
1256 return 0;
1257}
1258
1259static void kvm_s390_set_crycb_format(struct kvm *kvm)
1260{
1261 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1262
1263 if (kvm_s390_apxa_installed())
1264 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1265 else
1266 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1267}
1268
1269static u64 kvm_s390_get_initial_cpuid(void)
1270{
1271 struct cpuid cpuid;
1272
1273 get_cpu_id(&cpuid);
1274 cpuid.version = 0xff;
1275 return *((u64 *) &cpuid);
1276}
1277
1278static void kvm_s390_crypto_init(struct kvm *kvm)
1279{
1280 if (!test_kvm_facility(kvm, 76))
1281 return;
1282
1283 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
1284 kvm_s390_set_crycb_format(kvm);
1285
1286 /* Enable AES/DEA protected key functions by default */
1287 kvm->arch.crypto.aes_kw = 1;
1288 kvm->arch.crypto.dea_kw = 1;
1289 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1290 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1291 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1292 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1293}
1294
1295static void sca_dispose(struct kvm *kvm)
1296{
1297 if (kvm->arch.use_esca)
1298 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
1299 else
1300 free_page((unsigned long)(kvm->arch.sca));
1301 kvm->arch.sca = NULL;
1302}
1303
1304int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1305{
1306 gfp_t alloc_flags = GFP_KERNEL;
1307 int i, rc;
1308 char debug_name[16];
1309 static unsigned long sca_offset;
1310
1311 rc = -EINVAL;
1312#ifdef CONFIG_KVM_S390_UCONTROL
1313 if (type & ~KVM_VM_S390_UCONTROL)
1314 goto out_err;
1315 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1316 goto out_err;
1317#else
1318 if (type)
1319 goto out_err;
1320#endif
1321
1322 rc = s390_enable_sie();
1323 if (rc)
1324 goto out_err;
1325
1326 rc = -ENOMEM;
1327
1328 ratelimit_state_init(&kvm->arch.sthyi_limit, 5 * HZ, 500);
1329
1330 kvm->arch.use_esca = 0; /* start with basic SCA */
1331 if (!sclp.has_64bscao)
1332 alloc_flags |= GFP_DMA;
1333 rwlock_init(&kvm->arch.sca_lock);
1334 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
1335 if (!kvm->arch.sca)
1336 goto out_err;
1337 spin_lock(&kvm_lock);
1338 sca_offset += 16;
1339 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
1340 sca_offset = 0;
1341 kvm->arch.sca = (struct bsca_block *)
1342 ((char *) kvm->arch.sca + sca_offset);
1343 spin_unlock(&kvm_lock);
1344
1345 sprintf(debug_name, "kvm-%u", current->pid);
1346
1347 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1348 if (!kvm->arch.dbf)
1349 goto out_err;
1350
1351 kvm->arch.sie_page2 =
1352 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1353 if (!kvm->arch.sie_page2)
1354 goto out_err;
1355
1356 /* Populate the facility mask initially. */
1357 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
1358 S390_ARCH_FAC_LIST_SIZE_BYTE);
1359 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1360 if (i < kvm_s390_fac_list_mask_size())
1361 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
1362 else
1363 kvm->arch.model.fac_mask[i] = 0UL;
1364 }
1365
1366 /* Populate the facility list initially. */
1367 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1368 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
1369 S390_ARCH_FAC_LIST_SIZE_BYTE);
1370
1371 set_kvm_facility(kvm->arch.model.fac_mask, 74);
1372 set_kvm_facility(kvm->arch.model.fac_list, 74);
1373
1374 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
1375 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1376
1377 kvm_s390_crypto_init(kvm);
1378
1379 spin_lock_init(&kvm->arch.float_int.lock);
1380 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1381 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1382 init_waitqueue_head(&kvm->arch.ipte_wq);
1383 mutex_init(&kvm->arch.ipte_mutex);
1384
1385 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1386 VM_EVENT(kvm, 3, "vm created with type %lu", type);
1387
1388 if (type & KVM_VM_S390_UCONTROL) {
1389 kvm->arch.gmap = NULL;
1390 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
1391 } else {
1392 if (sclp.hamax == U64_MAX)
1393 kvm->arch.mem_limit = TASK_MAX_SIZE;
1394 else
1395 kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
1396 sclp.hamax + 1);
1397 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
1398 if (!kvm->arch.gmap)
1399 goto out_err;
1400 kvm->arch.gmap->private = kvm;
1401 kvm->arch.gmap->pfault_enabled = 0;
1402 }
1403
1404 kvm->arch.css_support = 0;
1405 kvm->arch.use_irqchip = 0;
1406 kvm->arch.epoch = 0;
1407
1408 spin_lock_init(&kvm->arch.start_stop_lock);
1409 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
1410
1411 return 0;
1412out_err:
1413 free_page((unsigned long)kvm->arch.sie_page2);
1414 debug_unregister(kvm->arch.dbf);
1415 sca_dispose(kvm);
1416 KVM_EVENT(3, "creation of vm failed: %d", rc);
1417 return rc;
1418}
1419
1420void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1421{
1422 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1423 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1424 kvm_s390_clear_local_irqs(vcpu);
1425 kvm_clear_async_pf_completion_queue(vcpu);
1426 if (!kvm_is_ucontrol(vcpu->kvm))
1427 sca_del_vcpu(vcpu);
1428
1429 if (kvm_is_ucontrol(vcpu->kvm))
1430 gmap_remove(vcpu->arch.gmap);
1431
1432 if (vcpu->kvm->arch.use_cmma)
1433 kvm_s390_vcpu_unsetup_cmma(vcpu);
1434 free_page((unsigned long)(vcpu->arch.sie_block));
1435
1436 kvm_vcpu_uninit(vcpu);
1437 kmem_cache_free(kvm_vcpu_cache, vcpu);
1438}
1439
1440static void kvm_free_vcpus(struct kvm *kvm)
1441{
1442 unsigned int i;
1443 struct kvm_vcpu *vcpu;
1444
1445 kvm_for_each_vcpu(i, vcpu, kvm)
1446 kvm_arch_vcpu_destroy(vcpu);
1447
1448 mutex_lock(&kvm->lock);
1449 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1450 kvm->vcpus[i] = NULL;
1451
1452 atomic_set(&kvm->online_vcpus, 0);
1453 mutex_unlock(&kvm->lock);
1454}
1455
1456void kvm_arch_destroy_vm(struct kvm *kvm)
1457{
1458 kvm_free_vcpus(kvm);
1459 sca_dispose(kvm);
1460 debug_unregister(kvm->arch.dbf);
1461 free_page((unsigned long)kvm->arch.sie_page2);
1462 if (!kvm_is_ucontrol(kvm))
1463 gmap_remove(kvm->arch.gmap);
1464 kvm_s390_destroy_adapters(kvm);
1465 kvm_s390_clear_float_irqs(kvm);
1466 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
1467}
1468
1469/* Section: vcpu related */
1470static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1471{
1472 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
1473 if (!vcpu->arch.gmap)
1474 return -ENOMEM;
1475 vcpu->arch.gmap->private = vcpu->kvm;
1476
1477 return 0;
1478}
1479
1480static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1481{
1482 read_lock(&vcpu->kvm->arch.sca_lock);
1483 if (vcpu->kvm->arch.use_esca) {
1484 struct esca_block *sca = vcpu->kvm->arch.sca;
1485
1486 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1487 sca->cpu[vcpu->vcpu_id].sda = 0;
1488 } else {
1489 struct bsca_block *sca = vcpu->kvm->arch.sca;
1490
1491 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1492 sca->cpu[vcpu->vcpu_id].sda = 0;
1493 }
1494 read_unlock(&vcpu->kvm->arch.sca_lock);
1495}
1496
1497static void sca_add_vcpu(struct kvm_vcpu *vcpu)
1498{
1499 read_lock(&vcpu->kvm->arch.sca_lock);
1500 if (vcpu->kvm->arch.use_esca) {
1501 struct esca_block *sca = vcpu->kvm->arch.sca;
1502
1503 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1504 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1505 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
1506 vcpu->arch.sie_block->ecb2 |= 0x04U;
1507 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1508 } else {
1509 struct bsca_block *sca = vcpu->kvm->arch.sca;
1510
1511 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1512 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1513 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
1514 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1515 }
1516 read_unlock(&vcpu->kvm->arch.sca_lock);
1517}
1518
1519/* Basic SCA to Extended SCA data copy routines */
1520static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1521{
1522 d->sda = s->sda;
1523 d->sigp_ctrl.c = s->sigp_ctrl.c;
1524 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1525}
1526
1527static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1528{
1529 int i;
1530
1531 d->ipte_control = s->ipte_control;
1532 d->mcn[0] = s->mcn;
1533 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1534 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1535}
1536
1537static int sca_switch_to_extended(struct kvm *kvm)
1538{
1539 struct bsca_block *old_sca = kvm->arch.sca;
1540 struct esca_block *new_sca;
1541 struct kvm_vcpu *vcpu;
1542 unsigned int vcpu_idx;
1543 u32 scaol, scaoh;
1544
1545 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1546 if (!new_sca)
1547 return -ENOMEM;
1548
1549 scaoh = (u32)((u64)(new_sca) >> 32);
1550 scaol = (u32)(u64)(new_sca) & ~0x3fU;
1551
1552 kvm_s390_vcpu_block_all(kvm);
1553 write_lock(&kvm->arch.sca_lock);
1554
1555 sca_copy_b_to_e(new_sca, old_sca);
1556
1557 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1558 vcpu->arch.sie_block->scaoh = scaoh;
1559 vcpu->arch.sie_block->scaol = scaol;
1560 vcpu->arch.sie_block->ecb2 |= 0x04U;
1561 }
1562 kvm->arch.sca = new_sca;
1563 kvm->arch.use_esca = 1;
1564
1565 write_unlock(&kvm->arch.sca_lock);
1566 kvm_s390_vcpu_unblock_all(kvm);
1567
1568 free_page((unsigned long)old_sca);
1569
1570 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1571 old_sca, kvm->arch.sca);
1572 return 0;
1573}
1574
1575static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1576{
1577 int rc;
1578
1579 if (id < KVM_S390_BSCA_CPU_SLOTS)
1580 return true;
1581 if (!sclp.has_esca || !sclp.has_64bscao)
1582 return false;
1583
1584 mutex_lock(&kvm->lock);
1585 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1586 mutex_unlock(&kvm->lock);
1587
1588 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
1589}
1590
1591int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1592{
1593 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1594 kvm_clear_async_pf_completion_queue(vcpu);
1595 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1596 KVM_SYNC_GPRS |
1597 KVM_SYNC_ACRS |
1598 KVM_SYNC_CRS |
1599 KVM_SYNC_ARCH0 |
1600 KVM_SYNC_PFAULT;
1601 if (test_kvm_facility(vcpu->kvm, 64))
1602 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
1603 /* fprs can be synchronized via vrs, even if the guest has no vx. With
1604 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1605 */
1606 if (MACHINE_HAS_VX)
1607 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1608 else
1609 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
1610
1611 if (kvm_is_ucontrol(vcpu->kvm))
1612 return __kvm_ucontrol_vcpu_init(vcpu);
1613
1614 return 0;
1615}
1616
1617/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1618static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1619{
1620 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
1621 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1622 vcpu->arch.cputm_start = get_tod_clock_fast();
1623 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1624}
1625
1626/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1627static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1628{
1629 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
1630 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1631 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1632 vcpu->arch.cputm_start = 0;
1633 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1634}
1635
1636/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1637static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1638{
1639 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
1640 vcpu->arch.cputm_enabled = true;
1641 __start_cpu_timer_accounting(vcpu);
1642}
1643
1644/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1645static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1646{
1647 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
1648 __stop_cpu_timer_accounting(vcpu);
1649 vcpu->arch.cputm_enabled = false;
1650}
1651
1652static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1653{
1654 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1655 __enable_cpu_timer_accounting(vcpu);
1656 preempt_enable();
1657}
1658
1659static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1660{
1661 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1662 __disable_cpu_timer_accounting(vcpu);
1663 preempt_enable();
1664}
1665
1666/* set the cpu timer - may only be called from the VCPU thread itself */
1667void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
1668{
1669 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1670 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1671 if (vcpu->arch.cputm_enabled)
1672 vcpu->arch.cputm_start = get_tod_clock_fast();
1673 vcpu->arch.sie_block->cputm = cputm;
1674 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1675 preempt_enable();
1676}
1677
1678/* update and get the cpu timer - can also be called from other VCPU threads */
1679__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
1680{
1681 unsigned int seq;
1682 __u64 value;
1683
1684 if (unlikely(!vcpu->arch.cputm_enabled))
1685 return vcpu->arch.sie_block->cputm;
1686
1687 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1688 do {
1689 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
1690 /*
1691 * If the writer would ever execute a read in the critical
1692 * section, e.g. in irq context, we have a deadlock.
1693 */
1694 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
1695 value = vcpu->arch.sie_block->cputm;
1696 /* if cputm_start is 0, accounting is being started/stopped */
1697 if (likely(vcpu->arch.cputm_start))
1698 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1699 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
1700 preempt_enable();
1701 return value;
1702}
1703
1704void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1705{
1706 /* Save host register state */
1707 save_fpu_regs();
1708 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
1709 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
1710
1711 if (MACHINE_HAS_VX)
1712 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
1713 else
1714 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
1715 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
1716 if (test_fp_ctl(current->thread.fpu.fpc))
1717 /* User space provided an invalid FPC, let's clear it */
1718 current->thread.fpu.fpc = 0;
1719
1720 save_access_regs(vcpu->arch.host_acrs);
1721 restore_access_regs(vcpu->run->s.regs.acrs);
1722 gmap_enable(vcpu->arch.gmap);
1723 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1724 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
1725 __start_cpu_timer_accounting(vcpu);
1726 vcpu->cpu = cpu;
1727}
1728
1729void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1730{
1731 vcpu->cpu = -1;
1732 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
1733 __stop_cpu_timer_accounting(vcpu);
1734 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1735 gmap_disable(vcpu->arch.gmap);
1736
1737 /* Save guest register state */
1738 save_fpu_regs();
1739 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
1740
1741 /* Restore host register state */
1742 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
1743 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
1744
1745 save_access_regs(vcpu->run->s.regs.acrs);
1746 restore_access_regs(vcpu->arch.host_acrs);
1747}
1748
1749static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1750{
1751 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1752 vcpu->arch.sie_block->gpsw.mask = 0UL;
1753 vcpu->arch.sie_block->gpsw.addr = 0UL;
1754 kvm_s390_set_prefix(vcpu, 0);
1755 kvm_s390_set_cpu_timer(vcpu, 0);
1756 vcpu->arch.sie_block->ckc = 0UL;
1757 vcpu->arch.sie_block->todpr = 0;
1758 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1759 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1760 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1761 /* make sure the new fpc will be lazily loaded */
1762 save_fpu_regs();
1763 current->thread.fpu.fpc = 0;
1764 vcpu->arch.sie_block->gbea = 1;
1765 vcpu->arch.sie_block->pp = 0;
1766 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1767 kvm_clear_async_pf_completion_queue(vcpu);
1768 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1769 kvm_s390_vcpu_stop(vcpu);
1770 kvm_s390_clear_local_irqs(vcpu);
1771}
1772
1773void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1774{
1775 mutex_lock(&vcpu->kvm->lock);
1776 preempt_disable();
1777 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1778 preempt_enable();
1779 mutex_unlock(&vcpu->kvm->lock);
1780 if (!kvm_is_ucontrol(vcpu->kvm)) {
1781 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1782 sca_add_vcpu(vcpu);
1783 }
1784
1785}
1786
1787static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1788{
1789 if (!test_kvm_facility(vcpu->kvm, 76))
1790 return;
1791
1792 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1793
1794 if (vcpu->kvm->arch.crypto.aes_kw)
1795 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1796 if (vcpu->kvm->arch.crypto.dea_kw)
1797 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1798
1799 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1800}
1801
1802void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1803{
1804 free_page(vcpu->arch.sie_block->cbrlo);
1805 vcpu->arch.sie_block->cbrlo = 0;
1806}
1807
1808int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1809{
1810 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1811 if (!vcpu->arch.sie_block->cbrlo)
1812 return -ENOMEM;
1813
1814 vcpu->arch.sie_block->ecb2 |= 0x80;
1815 vcpu->arch.sie_block->ecb2 &= ~0x08;
1816 return 0;
1817}
1818
1819static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1820{
1821 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1822
1823 vcpu->arch.sie_block->ibc = model->ibc;
1824 if (test_kvm_facility(vcpu->kvm, 7))
1825 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
1826}
1827
1828int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1829{
1830 int rc = 0;
1831
1832 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1833 CPUSTAT_SM |
1834 CPUSTAT_STOPPED);
1835
1836 if (test_kvm_facility(vcpu->kvm, 78))
1837 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
1838 else if (test_kvm_facility(vcpu->kvm, 8))
1839 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
1840
1841 kvm_s390_vcpu_setup_model(vcpu);
1842
1843 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
1844 if (MACHINE_HAS_ESOP)
1845 vcpu->arch.sie_block->ecb |= 0x02;
1846 if (test_kvm_facility(vcpu->kvm, 9))
1847 vcpu->arch.sie_block->ecb |= 0x04;
1848 if (test_kvm_facility(vcpu->kvm, 73))
1849 vcpu->arch.sie_block->ecb |= 0x10;
1850
1851 if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
1852 vcpu->arch.sie_block->ecb2 |= 0x08;
1853 vcpu->arch.sie_block->eca = 0x1002000U;
1854 if (sclp.has_cei)
1855 vcpu->arch.sie_block->eca |= 0x80000000U;
1856 if (sclp.has_ib)
1857 vcpu->arch.sie_block->eca |= 0x40000000U;
1858 if (sclp.has_siif)
1859 vcpu->arch.sie_block->eca |= 1;
1860 if (sclp.has_sigpif)
1861 vcpu->arch.sie_block->eca |= 0x10000000U;
1862 if (test_kvm_facility(vcpu->kvm, 64))
1863 vcpu->arch.sie_block->ecb3 |= 0x01;
1864 if (test_kvm_facility(vcpu->kvm, 129)) {
1865 vcpu->arch.sie_block->eca |= 0x00020000;
1866 vcpu->arch.sie_block->ecd |= 0x20000000;
1867 }
1868 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
1869 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1870 if (test_kvm_facility(vcpu->kvm, 74))
1871 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
1872
1873 if (vcpu->kvm->arch.use_cmma) {
1874 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1875 if (rc)
1876 return rc;
1877 }
1878 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1879 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1880
1881 kvm_s390_vcpu_crypto_setup(vcpu);
1882
1883 return rc;
1884}
1885
1886struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1887 unsigned int id)
1888{
1889 struct kvm_vcpu *vcpu;
1890 struct sie_page *sie_page;
1891 int rc = -EINVAL;
1892
1893 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
1894 goto out;
1895
1896 rc = -ENOMEM;
1897
1898 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1899 if (!vcpu)
1900 goto out;
1901
1902 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1903 if (!sie_page)
1904 goto out_free_cpu;
1905
1906 vcpu->arch.sie_block = &sie_page->sie_block;
1907 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1908
1909 /* the real guest size will always be smaller than msl */
1910 vcpu->arch.sie_block->mso = 0;
1911 vcpu->arch.sie_block->msl = sclp.hamax;
1912
1913 vcpu->arch.sie_block->icpua = id;
1914 spin_lock_init(&vcpu->arch.local_int.lock);
1915 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1916 vcpu->arch.local_int.wq = &vcpu->wq;
1917 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1918 seqcount_init(&vcpu->arch.cputm_seqcount);
1919
1920 rc = kvm_vcpu_init(vcpu, kvm, id);
1921 if (rc)
1922 goto out_free_sie_block;
1923 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
1924 vcpu->arch.sie_block);
1925 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1926
1927 return vcpu;
1928out_free_sie_block:
1929 free_page((unsigned long)(vcpu->arch.sie_block));
1930out_free_cpu:
1931 kmem_cache_free(kvm_vcpu_cache, vcpu);
1932out:
1933 return ERR_PTR(rc);
1934}
1935
1936int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1937{
1938 return kvm_s390_vcpu_has_irq(vcpu, 0);
1939}
1940
1941void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
1942{
1943 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1944 exit_sie(vcpu);
1945}
1946
1947void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1948{
1949 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1950}
1951
1952static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1953{
1954 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1955 exit_sie(vcpu);
1956}
1957
1958static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1959{
1960 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1961}
1962
1963/*
1964 * Kick a guest cpu out of SIE and wait until SIE is not running.
1965 * If the CPU is not running (e.g. waiting as idle) the function will
1966 * return immediately. */
1967void exit_sie(struct kvm_vcpu *vcpu)
1968{
1969 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1970 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1971 cpu_relax();
1972}
1973
1974/* Kick a guest cpu out of SIE to process a request synchronously */
1975void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
1976{
1977 kvm_make_request(req, vcpu);
1978 kvm_s390_vcpu_request(vcpu);
1979}
1980
1981static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
1982 unsigned long end)
1983{
1984 struct kvm *kvm = gmap->private;
1985 struct kvm_vcpu *vcpu;
1986 unsigned long prefix;
1987 int i;
1988
1989 if (start >= 1UL << 31)
1990 /* We are only interested in prefix pages */
1991 return;
1992 kvm_for_each_vcpu(i, vcpu, kvm) {
1993 /* match against both prefix pages */
1994 prefix = kvm_s390_get_prefix(vcpu);
1995 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
1996 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
1997 start, end);
1998 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
1999 }
2000 }
2001}
2002
2003int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2004{
2005 /* kvm common code refers to this, but never calls it */
2006 BUG();
2007 return 0;
2008}
2009
2010static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2011 struct kvm_one_reg *reg)
2012{
2013 int r = -EINVAL;
2014
2015 switch (reg->id) {
2016 case KVM_REG_S390_TODPR:
2017 r = put_user(vcpu->arch.sie_block->todpr,
2018 (u32 __user *)reg->addr);
2019 break;
2020 case KVM_REG_S390_EPOCHDIFF:
2021 r = put_user(vcpu->arch.sie_block->epoch,
2022 (u64 __user *)reg->addr);
2023 break;
2024 case KVM_REG_S390_CPU_TIMER:
2025 r = put_user(kvm_s390_get_cpu_timer(vcpu),
2026 (u64 __user *)reg->addr);
2027 break;
2028 case KVM_REG_S390_CLOCK_COMP:
2029 r = put_user(vcpu->arch.sie_block->ckc,
2030 (u64 __user *)reg->addr);
2031 break;
2032 case KVM_REG_S390_PFTOKEN:
2033 r = put_user(vcpu->arch.pfault_token,
2034 (u64 __user *)reg->addr);
2035 break;
2036 case KVM_REG_S390_PFCOMPARE:
2037 r = put_user(vcpu->arch.pfault_compare,
2038 (u64 __user *)reg->addr);
2039 break;
2040 case KVM_REG_S390_PFSELECT:
2041 r = put_user(vcpu->arch.pfault_select,
2042 (u64 __user *)reg->addr);
2043 break;
2044 case KVM_REG_S390_PP:
2045 r = put_user(vcpu->arch.sie_block->pp,
2046 (u64 __user *)reg->addr);
2047 break;
2048 case KVM_REG_S390_GBEA:
2049 r = put_user(vcpu->arch.sie_block->gbea,
2050 (u64 __user *)reg->addr);
2051 break;
2052 default:
2053 break;
2054 }
2055
2056 return r;
2057}
2058
2059static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2060 struct kvm_one_reg *reg)
2061{
2062 int r = -EINVAL;
2063 __u64 val;
2064
2065 switch (reg->id) {
2066 case KVM_REG_S390_TODPR:
2067 r = get_user(vcpu->arch.sie_block->todpr,
2068 (u32 __user *)reg->addr);
2069 break;
2070 case KVM_REG_S390_EPOCHDIFF:
2071 r = get_user(vcpu->arch.sie_block->epoch,
2072 (u64 __user *)reg->addr);
2073 break;
2074 case KVM_REG_S390_CPU_TIMER:
2075 r = get_user(val, (u64 __user *)reg->addr);
2076 if (!r)
2077 kvm_s390_set_cpu_timer(vcpu, val);
2078 break;
2079 case KVM_REG_S390_CLOCK_COMP:
2080 r = get_user(vcpu->arch.sie_block->ckc,
2081 (u64 __user *)reg->addr);
2082 break;
2083 case KVM_REG_S390_PFTOKEN:
2084 r = get_user(vcpu->arch.pfault_token,
2085 (u64 __user *)reg->addr);
2086 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2087 kvm_clear_async_pf_completion_queue(vcpu);
2088 break;
2089 case KVM_REG_S390_PFCOMPARE:
2090 r = get_user(vcpu->arch.pfault_compare,
2091 (u64 __user *)reg->addr);
2092 break;
2093 case KVM_REG_S390_PFSELECT:
2094 r = get_user(vcpu->arch.pfault_select,
2095 (u64 __user *)reg->addr);
2096 break;
2097 case KVM_REG_S390_PP:
2098 r = get_user(vcpu->arch.sie_block->pp,
2099 (u64 __user *)reg->addr);
2100 break;
2101 case KVM_REG_S390_GBEA:
2102 r = get_user(vcpu->arch.sie_block->gbea,
2103 (u64 __user *)reg->addr);
2104 break;
2105 default:
2106 break;
2107 }
2108
2109 return r;
2110}
2111
2112static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2113{
2114 kvm_s390_vcpu_initial_reset(vcpu);
2115 return 0;
2116}
2117
2118int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2119{
2120 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
2121 return 0;
2122}
2123
2124int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2125{
2126 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
2127 return 0;
2128}
2129
2130int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2131 struct kvm_sregs *sregs)
2132{
2133 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
2134 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
2135 restore_access_regs(vcpu->run->s.regs.acrs);
2136 return 0;
2137}
2138
2139int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2140 struct kvm_sregs *sregs)
2141{
2142 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
2143 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
2144 return 0;
2145}
2146
2147int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2148{
2149 /* make sure the new values will be lazily loaded */
2150 save_fpu_regs();
2151 if (test_fp_ctl(fpu->fpc))
2152 return -EINVAL;
2153 current->thread.fpu.fpc = fpu->fpc;
2154 if (MACHINE_HAS_VX)
2155 convert_fp_to_vx(current->thread.fpu.vxrs, (freg_t *)fpu->fprs);
2156 else
2157 memcpy(current->thread.fpu.fprs, &fpu->fprs, sizeof(fpu->fprs));
2158 return 0;
2159}
2160
2161int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2162{
2163 /* make sure we have the latest values */
2164 save_fpu_regs();
2165 if (MACHINE_HAS_VX)
2166 convert_vx_to_fp((freg_t *)fpu->fprs, current->thread.fpu.vxrs);
2167 else
2168 memcpy(fpu->fprs, current->thread.fpu.fprs, sizeof(fpu->fprs));
2169 fpu->fpc = current->thread.fpu.fpc;
2170 return 0;
2171}
2172
2173static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2174{
2175 int rc = 0;
2176
2177 if (!is_vcpu_stopped(vcpu))
2178 rc = -EBUSY;
2179 else {
2180 vcpu->run->psw_mask = psw.mask;
2181 vcpu->run->psw_addr = psw.addr;
2182 }
2183 return rc;
2184}
2185
2186int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2187 struct kvm_translation *tr)
2188{
2189 return -EINVAL; /* not implemented yet */
2190}
2191
2192#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2193 KVM_GUESTDBG_USE_HW_BP | \
2194 KVM_GUESTDBG_ENABLE)
2195
2196int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2197 struct kvm_guest_debug *dbg)
2198{
2199 int rc = 0;
2200
2201 vcpu->guest_debug = 0;
2202 kvm_s390_clear_bp_data(vcpu);
2203
2204 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
2205 return -EINVAL;
2206 if (!sclp.has_gpere)
2207 return -EINVAL;
2208
2209 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2210 vcpu->guest_debug = dbg->control;
2211 /* enforce guest PER */
2212 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2213
2214 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2215 rc = kvm_s390_import_bp_data(vcpu, dbg);
2216 } else {
2217 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2218 vcpu->arch.guestdbg.last_bp = 0;
2219 }
2220
2221 if (rc) {
2222 vcpu->guest_debug = 0;
2223 kvm_s390_clear_bp_data(vcpu);
2224 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2225 }
2226
2227 return rc;
2228}
2229
2230int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2231 struct kvm_mp_state *mp_state)
2232{
2233 /* CHECK_STOP and LOAD are not supported yet */
2234 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2235 KVM_MP_STATE_OPERATING;
2236}
2237
2238int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2239 struct kvm_mp_state *mp_state)
2240{
2241 int rc = 0;
2242
2243 /* user space knows about this interface - let it control the state */
2244 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2245
2246 switch (mp_state->mp_state) {
2247 case KVM_MP_STATE_STOPPED:
2248 kvm_s390_vcpu_stop(vcpu);
2249 break;
2250 case KVM_MP_STATE_OPERATING:
2251 kvm_s390_vcpu_start(vcpu);
2252 break;
2253 case KVM_MP_STATE_LOAD:
2254 case KVM_MP_STATE_CHECK_STOP:
2255 /* fall through - CHECK_STOP and LOAD are not supported yet */
2256 default:
2257 rc = -ENXIO;
2258 }
2259
2260 return rc;
2261}
2262
2263static bool ibs_enabled(struct kvm_vcpu *vcpu)
2264{
2265 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2266}
2267
2268static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2269{
2270retry:
2271 kvm_s390_vcpu_request_handled(vcpu);
2272 if (!vcpu->requests)
2273 return 0;
2274 /*
2275 * We use MMU_RELOAD just to re-arm the ipte notifier for the
2276 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2277 * This ensures that the ipte instruction for this request has
2278 * already finished. We might race against a second unmapper that
2279 * wants to set the blocking bit. Lets just retry the request loop.
2280 */
2281 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2282 int rc;
2283 rc = gmap_mprotect_notify(vcpu->arch.gmap,
2284 kvm_s390_get_prefix(vcpu),
2285 PAGE_SIZE * 2, PROT_WRITE);
2286 if (rc)
2287 return rc;
2288 goto retry;
2289 }
2290
2291 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2292 vcpu->arch.sie_block->ihcpu = 0xffff;
2293 goto retry;
2294 }
2295
2296 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2297 if (!ibs_enabled(vcpu)) {
2298 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
2299 atomic_or(CPUSTAT_IBS,
2300 &vcpu->arch.sie_block->cpuflags);
2301 }
2302 goto retry;
2303 }
2304
2305 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2306 if (ibs_enabled(vcpu)) {
2307 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
2308 atomic_andnot(CPUSTAT_IBS,
2309 &vcpu->arch.sie_block->cpuflags);
2310 }
2311 goto retry;
2312 }
2313
2314 /* nothing to do, just clear the request */
2315 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2316
2317 return 0;
2318}
2319
2320void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2321{
2322 struct kvm_vcpu *vcpu;
2323 int i;
2324
2325 mutex_lock(&kvm->lock);
2326 preempt_disable();
2327 kvm->arch.epoch = tod - get_tod_clock();
2328 kvm_s390_vcpu_block_all(kvm);
2329 kvm_for_each_vcpu(i, vcpu, kvm)
2330 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2331 kvm_s390_vcpu_unblock_all(kvm);
2332 preempt_enable();
2333 mutex_unlock(&kvm->lock);
2334}
2335
2336/**
2337 * kvm_arch_fault_in_page - fault-in guest page if necessary
2338 * @vcpu: The corresponding virtual cpu
2339 * @gpa: Guest physical address
2340 * @writable: Whether the page should be writable or not
2341 *
2342 * Make sure that a guest page has been faulted-in on the host.
2343 *
2344 * Return: Zero on success, negative error code otherwise.
2345 */
2346long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
2347{
2348 return gmap_fault(vcpu->arch.gmap, gpa,
2349 writable ? FAULT_FLAG_WRITE : 0);
2350}
2351
2352static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2353 unsigned long token)
2354{
2355 struct kvm_s390_interrupt inti;
2356 struct kvm_s390_irq irq;
2357
2358 if (start_token) {
2359 irq.u.ext.ext_params2 = token;
2360 irq.type = KVM_S390_INT_PFAULT_INIT;
2361 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
2362 } else {
2363 inti.type = KVM_S390_INT_PFAULT_DONE;
2364 inti.parm64 = token;
2365 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2366 }
2367}
2368
2369void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2370 struct kvm_async_pf *work)
2371{
2372 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2373 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2374}
2375
2376void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2377 struct kvm_async_pf *work)
2378{
2379 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2380 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2381}
2382
2383void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2384 struct kvm_async_pf *work)
2385{
2386 /* s390 will always inject the page directly */
2387}
2388
2389bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2390{
2391 /*
2392 * s390 will always inject the page directly,
2393 * but we still want check_async_completion to cleanup
2394 */
2395 return true;
2396}
2397
2398static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2399{
2400 hva_t hva;
2401 struct kvm_arch_async_pf arch;
2402 int rc;
2403
2404 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2405 return 0;
2406 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2407 vcpu->arch.pfault_compare)
2408 return 0;
2409 if (psw_extint_disabled(vcpu))
2410 return 0;
2411 if (kvm_s390_vcpu_has_irq(vcpu, 0))
2412 return 0;
2413 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2414 return 0;
2415 if (!vcpu->arch.gmap->pfault_enabled)
2416 return 0;
2417
2418 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2419 hva += current->thread.gmap_addr & ~PAGE_MASK;
2420 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
2421 return 0;
2422
2423 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2424 return rc;
2425}
2426
2427static int vcpu_pre_run(struct kvm_vcpu *vcpu)
2428{
2429 int rc, cpuflags;
2430
2431 /*
2432 * On s390 notifications for arriving pages will be delivered directly
2433 * to the guest but the house keeping for completed pfaults is
2434 * handled outside the worker.
2435 */
2436 kvm_check_async_pf_completion(vcpu);
2437
2438 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2439 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
2440
2441 if (need_resched())
2442 schedule();
2443
2444 if (test_cpu_flag(CIF_MCCK_PENDING))
2445 s390_handle_mcck();
2446
2447 if (!kvm_is_ucontrol(vcpu->kvm)) {
2448 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2449 if (rc)
2450 return rc;
2451 }
2452
2453 rc = kvm_s390_handle_requests(vcpu);
2454 if (rc)
2455 return rc;
2456
2457 if (guestdbg_enabled(vcpu)) {
2458 kvm_s390_backup_guest_per_regs(vcpu);
2459 kvm_s390_patch_guest_per_regs(vcpu);
2460 }
2461
2462 vcpu->arch.sie_block->icptcode = 0;
2463 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2464 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2465 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2466
2467 return 0;
2468}
2469
2470static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2471{
2472 struct kvm_s390_pgm_info pgm_info = {
2473 .code = PGM_ADDRESSING,
2474 };
2475 u8 opcode, ilen;
2476 int rc;
2477
2478 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2479 trace_kvm_s390_sie_fault(vcpu);
2480
2481 /*
2482 * We want to inject an addressing exception, which is defined as a
2483 * suppressing or terminating exception. However, since we came here
2484 * by a DAT access exception, the PSW still points to the faulting
2485 * instruction since DAT exceptions are nullifying. So we've got
2486 * to look up the current opcode to get the length of the instruction
2487 * to be able to forward the PSW.
2488 */
2489 rc = read_guest_instr(vcpu, &opcode, 1);
2490 ilen = insn_length(opcode);
2491 if (rc < 0) {
2492 return rc;
2493 } else if (rc) {
2494 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2495 * Forward by arbitrary ilc, injection will take care of
2496 * nullification if necessary.
2497 */
2498 pgm_info = vcpu->arch.pgm;
2499 ilen = 4;
2500 }
2501 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2502 kvm_s390_forward_psw(vcpu, ilen);
2503 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
2504}
2505
2506static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2507{
2508 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2509 vcpu->arch.sie_block->icptcode);
2510 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2511
2512 if (guestdbg_enabled(vcpu))
2513 kvm_s390_restore_guest_per_regs(vcpu);
2514
2515 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2516 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
2517
2518 if (vcpu->arch.sie_block->icptcode > 0) {
2519 int rc = kvm_handle_sie_intercept(vcpu);
2520
2521 if (rc != -EOPNOTSUPP)
2522 return rc;
2523 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2524 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2525 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2526 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2527 return -EREMOTE;
2528 } else if (exit_reason != -EFAULT) {
2529 vcpu->stat.exit_null++;
2530 return 0;
2531 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2532 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2533 vcpu->run->s390_ucontrol.trans_exc_code =
2534 current->thread.gmap_addr;
2535 vcpu->run->s390_ucontrol.pgm_code = 0x10;
2536 return -EREMOTE;
2537 } else if (current->thread.gmap_pfault) {
2538 trace_kvm_s390_major_guest_pfault(vcpu);
2539 current->thread.gmap_pfault = 0;
2540 if (kvm_arch_setup_async_pf(vcpu))
2541 return 0;
2542 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
2543 }
2544 return vcpu_post_run_fault_in_sie(vcpu);
2545}
2546
2547static int __vcpu_run(struct kvm_vcpu *vcpu)
2548{
2549 int rc, exit_reason;
2550
2551 /*
2552 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2553 * ning the guest), so that memslots (and other stuff) are protected
2554 */
2555 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2556
2557 do {
2558 rc = vcpu_pre_run(vcpu);
2559 if (rc)
2560 break;
2561
2562 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2563 /*
2564 * As PF_VCPU will be used in fault handler, between
2565 * guest_enter and guest_exit should be no uaccess.
2566 */
2567 local_irq_disable();
2568 __kvm_guest_enter();
2569 __disable_cpu_timer_accounting(vcpu);
2570 local_irq_enable();
2571 exit_reason = sie64a(vcpu->arch.sie_block,
2572 vcpu->run->s.regs.gprs);
2573 local_irq_disable();
2574 __enable_cpu_timer_accounting(vcpu);
2575 __kvm_guest_exit();
2576 local_irq_enable();
2577 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2578
2579 rc = vcpu_post_run(vcpu, exit_reason);
2580 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2581
2582 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2583 return rc;
2584}
2585
2586static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2587{
2588 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2589 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2590 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2591 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2592 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2593 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2594 /* some control register changes require a tlb flush */
2595 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2596 }
2597 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2598 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
2599 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2600 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2601 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2602 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2603 }
2604 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2605 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2606 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2607 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2608 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2609 kvm_clear_async_pf_completion_queue(vcpu);
2610 }
2611 kvm_run->kvm_dirty_regs = 0;
2612}
2613
2614static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2615{
2616 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2617 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2618 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2619 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2620 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
2621 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2622 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2623 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2624 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2625 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2626 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2627 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2628}
2629
2630int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2631{
2632 int rc;
2633 sigset_t sigsaved;
2634
2635 if (guestdbg_exit_pending(vcpu)) {
2636 kvm_s390_prepare_debug_exit(vcpu);
2637 return 0;
2638 }
2639
2640 if (vcpu->sigset_active)
2641 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2642
2643 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2644 kvm_s390_vcpu_start(vcpu);
2645 } else if (is_vcpu_stopped(vcpu)) {
2646 pr_err_ratelimited("can't run stopped vcpu %d\n",
2647 vcpu->vcpu_id);
2648 return -EINVAL;
2649 }
2650
2651 sync_regs(vcpu, kvm_run);
2652 enable_cpu_timer_accounting(vcpu);
2653
2654 might_fault();
2655 rc = __vcpu_run(vcpu);
2656
2657 if (signal_pending(current) && !rc) {
2658 kvm_run->exit_reason = KVM_EXIT_INTR;
2659 rc = -EINTR;
2660 }
2661
2662 if (guestdbg_exit_pending(vcpu) && !rc) {
2663 kvm_s390_prepare_debug_exit(vcpu);
2664 rc = 0;
2665 }
2666
2667 if (rc == -EREMOTE) {
2668 /* userspace support is needed, kvm_run has been prepared */
2669 rc = 0;
2670 }
2671
2672 disable_cpu_timer_accounting(vcpu);
2673 store_regs(vcpu, kvm_run);
2674
2675 if (vcpu->sigset_active)
2676 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2677
2678 vcpu->stat.exit_userspace++;
2679 return rc;
2680}
2681
2682/*
2683 * store status at address
2684 * we use have two special cases:
2685 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2686 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2687 */
2688int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2689{
2690 unsigned char archmode = 1;
2691 freg_t fprs[NUM_FPRS];
2692 unsigned int px;
2693 u64 clkcomp, cputm;
2694 int rc;
2695
2696 px = kvm_s390_get_prefix(vcpu);
2697 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2698 if (write_guest_abs(vcpu, 163, &archmode, 1))
2699 return -EFAULT;
2700 gpa = 0;
2701 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2702 if (write_guest_real(vcpu, 163, &archmode, 1))
2703 return -EFAULT;
2704 gpa = px;
2705 } else
2706 gpa -= __LC_FPREGS_SAVE_AREA;
2707
2708 /* manually convert vector registers if necessary */
2709 if (MACHINE_HAS_VX) {
2710 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
2711 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2712 fprs, 128);
2713 } else {
2714 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2715 vcpu->run->s.regs.fprs, 128);
2716 }
2717 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
2718 vcpu->run->s.regs.gprs, 128);
2719 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
2720 &vcpu->arch.sie_block->gpsw, 16);
2721 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
2722 &px, 4);
2723 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
2724 &vcpu->run->s.regs.fpc, 4);
2725 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
2726 &vcpu->arch.sie_block->todpr, 4);
2727 cputm = kvm_s390_get_cpu_timer(vcpu);
2728 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
2729 &cputm, 8);
2730 clkcomp = vcpu->arch.sie_block->ckc >> 8;
2731 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
2732 &clkcomp, 8);
2733 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
2734 &vcpu->run->s.regs.acrs, 64);
2735 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
2736 &vcpu->arch.sie_block->gcr, 128);
2737 return rc ? -EFAULT : 0;
2738}
2739
2740int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2741{
2742 /*
2743 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2744 * copying in vcpu load/put. Lets update our copies before we save
2745 * it into the save area
2746 */
2747 save_fpu_regs();
2748 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
2749 save_access_regs(vcpu->run->s.regs.acrs);
2750
2751 return kvm_s390_store_status_unloaded(vcpu, addr);
2752}
2753
2754/*
2755 * store additional status at address
2756 */
2757int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2758 unsigned long gpa)
2759{
2760 /* Only bits 0-53 are used for address formation */
2761 if (!(gpa & ~0x3ff))
2762 return 0;
2763
2764 return write_guest_abs(vcpu, gpa & ~0x3ff,
2765 (void *)&vcpu->run->s.regs.vrs, 512);
2766}
2767
2768int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2769{
2770 if (!test_kvm_facility(vcpu->kvm, 129))
2771 return 0;
2772
2773 /*
2774 * The guest VXRS are in the host VXRs due to the lazy
2775 * copying in vcpu load/put. We can simply call save_fpu_regs()
2776 * to save the current register state because we are in the
2777 * middle of a load/put cycle.
2778 *
2779 * Let's update our copies before we save it into the save area.
2780 */
2781 save_fpu_regs();
2782
2783 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2784}
2785
2786static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2787{
2788 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2789 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
2790}
2791
2792static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2793{
2794 unsigned int i;
2795 struct kvm_vcpu *vcpu;
2796
2797 kvm_for_each_vcpu(i, vcpu, kvm) {
2798 __disable_ibs_on_vcpu(vcpu);
2799 }
2800}
2801
2802static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2803{
2804 if (!sclp.has_ibs)
2805 return;
2806 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2807 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
2808}
2809
2810void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2811{
2812 int i, online_vcpus, started_vcpus = 0;
2813
2814 if (!is_vcpu_stopped(vcpu))
2815 return;
2816
2817 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2818 /* Only one cpu at a time may enter/leave the STOPPED state. */
2819 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2820 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2821
2822 for (i = 0; i < online_vcpus; i++) {
2823 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2824 started_vcpus++;
2825 }
2826
2827 if (started_vcpus == 0) {
2828 /* we're the only active VCPU -> speed it up */
2829 __enable_ibs_on_vcpu(vcpu);
2830 } else if (started_vcpus == 1) {
2831 /*
2832 * As we are starting a second VCPU, we have to disable
2833 * the IBS facility on all VCPUs to remove potentially
2834 * oustanding ENABLE requests.
2835 */
2836 __disable_ibs_on_all_vcpus(vcpu->kvm);
2837 }
2838
2839 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2840 /*
2841 * Another VCPU might have used IBS while we were offline.
2842 * Let's play safe and flush the VCPU at startup.
2843 */
2844 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2845 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2846 return;
2847}
2848
2849void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2850{
2851 int i, online_vcpus, started_vcpus = 0;
2852 struct kvm_vcpu *started_vcpu = NULL;
2853
2854 if (is_vcpu_stopped(vcpu))
2855 return;
2856
2857 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2858 /* Only one cpu at a time may enter/leave the STOPPED state. */
2859 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2860 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2861
2862 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2863 kvm_s390_clear_stop_irq(vcpu);
2864
2865 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2866 __disable_ibs_on_vcpu(vcpu);
2867
2868 for (i = 0; i < online_vcpus; i++) {
2869 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2870 started_vcpus++;
2871 started_vcpu = vcpu->kvm->vcpus[i];
2872 }
2873 }
2874
2875 if (started_vcpus == 1) {
2876 /*
2877 * As we only have one VCPU left, we want to enable the
2878 * IBS facility for that VCPU to speed it up.
2879 */
2880 __enable_ibs_on_vcpu(started_vcpu);
2881 }
2882
2883 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2884 return;
2885}
2886
2887static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2888 struct kvm_enable_cap *cap)
2889{
2890 int r;
2891
2892 if (cap->flags)
2893 return -EINVAL;
2894
2895 switch (cap->cap) {
2896 case KVM_CAP_S390_CSS_SUPPORT:
2897 if (!vcpu->kvm->arch.css_support) {
2898 vcpu->kvm->arch.css_support = 1;
2899 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
2900 trace_kvm_s390_enable_css(vcpu->kvm);
2901 }
2902 r = 0;
2903 break;
2904 default:
2905 r = -EINVAL;
2906 break;
2907 }
2908 return r;
2909}
2910
2911static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2912 struct kvm_s390_mem_op *mop)
2913{
2914 void __user *uaddr = (void __user *)mop->buf;
2915 void *tmpbuf = NULL;
2916 int r, srcu_idx;
2917 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2918 | KVM_S390_MEMOP_F_CHECK_ONLY;
2919
2920 if (mop->flags & ~supported_flags)
2921 return -EINVAL;
2922
2923 if (mop->size > MEM_OP_MAX_SIZE)
2924 return -E2BIG;
2925
2926 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2927 tmpbuf = vmalloc(mop->size);
2928 if (!tmpbuf)
2929 return -ENOMEM;
2930 }
2931
2932 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2933
2934 switch (mop->op) {
2935 case KVM_S390_MEMOP_LOGICAL_READ:
2936 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2937 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2938 mop->size, GACC_FETCH);
2939 break;
2940 }
2941 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2942 if (r == 0) {
2943 if (copy_to_user(uaddr, tmpbuf, mop->size))
2944 r = -EFAULT;
2945 }
2946 break;
2947 case KVM_S390_MEMOP_LOGICAL_WRITE:
2948 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2949 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2950 mop->size, GACC_STORE);
2951 break;
2952 }
2953 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2954 r = -EFAULT;
2955 break;
2956 }
2957 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2958 break;
2959 default:
2960 r = -EINVAL;
2961 }
2962
2963 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2964
2965 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2966 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2967
2968 vfree(tmpbuf);
2969 return r;
2970}
2971
2972long kvm_arch_vcpu_ioctl(struct file *filp,
2973 unsigned int ioctl, unsigned long arg)
2974{
2975 struct kvm_vcpu *vcpu = filp->private_data;
2976 void __user *argp = (void __user *)arg;
2977 int idx;
2978 long r;
2979
2980 switch (ioctl) {
2981 case KVM_S390_IRQ: {
2982 struct kvm_s390_irq s390irq;
2983
2984 r = -EFAULT;
2985 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2986 break;
2987 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2988 break;
2989 }
2990 case KVM_S390_INTERRUPT: {
2991 struct kvm_s390_interrupt s390int;
2992 struct kvm_s390_irq s390irq;
2993
2994 r = -EFAULT;
2995 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2996 break;
2997 if (s390int_to_s390irq(&s390int, &s390irq))
2998 return -EINVAL;
2999 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3000 break;
3001 }
3002 case KVM_S390_STORE_STATUS:
3003 idx = srcu_read_lock(&vcpu->kvm->srcu);
3004 r = kvm_s390_vcpu_store_status(vcpu, arg);
3005 srcu_read_unlock(&vcpu->kvm->srcu, idx);
3006 break;
3007 case KVM_S390_SET_INITIAL_PSW: {
3008 psw_t psw;
3009
3010 r = -EFAULT;
3011 if (copy_from_user(&psw, argp, sizeof(psw)))
3012 break;
3013 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3014 break;
3015 }
3016 case KVM_S390_INITIAL_RESET:
3017 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3018 break;
3019 case KVM_SET_ONE_REG:
3020 case KVM_GET_ONE_REG: {
3021 struct kvm_one_reg reg;
3022 r = -EFAULT;
3023 if (copy_from_user(&reg, argp, sizeof(reg)))
3024 break;
3025 if (ioctl == KVM_SET_ONE_REG)
3026 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3027 else
3028 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3029 break;
3030 }
3031#ifdef CONFIG_KVM_S390_UCONTROL
3032 case KVM_S390_UCAS_MAP: {
3033 struct kvm_s390_ucas_mapping ucasmap;
3034
3035 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3036 r = -EFAULT;
3037 break;
3038 }
3039
3040 if (!kvm_is_ucontrol(vcpu->kvm)) {
3041 r = -EINVAL;
3042 break;
3043 }
3044
3045 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3046 ucasmap.vcpu_addr, ucasmap.length);
3047 break;
3048 }
3049 case KVM_S390_UCAS_UNMAP: {
3050 struct kvm_s390_ucas_mapping ucasmap;
3051
3052 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3053 r = -EFAULT;
3054 break;
3055 }
3056
3057 if (!kvm_is_ucontrol(vcpu->kvm)) {
3058 r = -EINVAL;
3059 break;
3060 }
3061
3062 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3063 ucasmap.length);
3064 break;
3065 }
3066#endif
3067 case KVM_S390_VCPU_FAULT: {
3068 r = gmap_fault(vcpu->arch.gmap, arg, 0);
3069 break;
3070 }
3071 case KVM_ENABLE_CAP:
3072 {
3073 struct kvm_enable_cap cap;
3074 r = -EFAULT;
3075 if (copy_from_user(&cap, argp, sizeof(cap)))
3076 break;
3077 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3078 break;
3079 }
3080 case KVM_S390_MEM_OP: {
3081 struct kvm_s390_mem_op mem_op;
3082
3083 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3084 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3085 else
3086 r = -EFAULT;
3087 break;
3088 }
3089 case KVM_S390_SET_IRQ_STATE: {
3090 struct kvm_s390_irq_state irq_state;
3091
3092 r = -EFAULT;
3093 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3094 break;
3095 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3096 irq_state.len == 0 ||
3097 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3098 r = -EINVAL;
3099 break;
3100 }
3101 r = kvm_s390_set_irq_state(vcpu,
3102 (void __user *) irq_state.buf,
3103 irq_state.len);
3104 break;
3105 }
3106 case KVM_S390_GET_IRQ_STATE: {
3107 struct kvm_s390_irq_state irq_state;
3108
3109 r = -EFAULT;
3110 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3111 break;
3112 if (irq_state.len == 0) {
3113 r = -EINVAL;
3114 break;
3115 }
3116 r = kvm_s390_get_irq_state(vcpu,
3117 (__u8 __user *) irq_state.buf,
3118 irq_state.len);
3119 break;
3120 }
3121 default:
3122 r = -ENOTTY;
3123 }
3124 return r;
3125}
3126
3127int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3128{
3129#ifdef CONFIG_KVM_S390_UCONTROL
3130 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3131 && (kvm_is_ucontrol(vcpu->kvm))) {
3132 vmf->page = virt_to_page(vcpu->arch.sie_block);
3133 get_page(vmf->page);
3134 return 0;
3135 }
3136#endif
3137 return VM_FAULT_SIGBUS;
3138}
3139
3140int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3141 unsigned long npages)
3142{
3143 return 0;
3144}
3145
3146/* Section: memory related */
3147int kvm_arch_prepare_memory_region(struct kvm *kvm,
3148 struct kvm_memory_slot *memslot,
3149 const struct kvm_userspace_memory_region *mem,
3150 enum kvm_mr_change change)
3151{
3152 /* A few sanity checks. We can have memory slots which have to be
3153 located/ended at a segment boundary (1MB). The memory in userland is
3154 ok to be fragmented into various different vmas. It is okay to mmap()
3155 and munmap() stuff in this slot after doing this call at any time */
3156
3157 if (mem->userspace_addr & 0xffffful)
3158 return -EINVAL;
3159
3160 if (mem->memory_size & 0xffffful)
3161 return -EINVAL;
3162
3163 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3164 return -EINVAL;
3165
3166 return 0;
3167}
3168
3169void kvm_arch_commit_memory_region(struct kvm *kvm,
3170 const struct kvm_userspace_memory_region *mem,
3171 const struct kvm_memory_slot *old,
3172 const struct kvm_memory_slot *new,
3173 enum kvm_mr_change change)
3174{
3175 int rc;
3176
3177 /* If the basics of the memslot do not change, we do not want
3178 * to update the gmap. Every update causes several unnecessary
3179 * segment translation exceptions. This is usually handled just
3180 * fine by the normal fault handler + gmap, but it will also
3181 * cause faults on the prefix page of running guest CPUs.
3182 */
3183 if (old->userspace_addr == mem->userspace_addr &&
3184 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3185 old->npages * PAGE_SIZE == mem->memory_size)
3186 return;
3187
3188 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3189 mem->guest_phys_addr, mem->memory_size);
3190 if (rc)
3191 pr_warn("failed to commit memory region\n");
3192 return;
3193}
3194
3195static inline unsigned long nonhyp_mask(int i)
3196{
3197 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3198
3199 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3200}
3201
3202void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3203{
3204 vcpu->valid_wakeup = false;
3205}
3206
3207static int __init kvm_s390_init(void)
3208{
3209 int i;
3210
3211 if (!sclp.has_sief2) {
3212 pr_info("SIE not available\n");
3213 return -ENODEV;
3214 }
3215
3216 for (i = 0; i < 16; i++)
3217 kvm_s390_fac_list_mask[i] |=
3218 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3219
3220 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
3221}
3222
3223static void __exit kvm_s390_exit(void)
3224{
3225 kvm_exit();
3226}
3227
3228module_init(kvm_s390_init);
3229module_exit(kvm_s390_exit);
3230
3231/*
3232 * Enable autoloading of the kvm module.
3233 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3234 * since x86 takes a different approach.
3235 */
3236#include <linux/miscdevice.h>
3237MODULE_ALIAS_MISCDEV(KVM_MINOR);
3238MODULE_ALIAS("devname:kvm");
Ubuntu Jammy Linux kernel mirror