]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-jammy-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
KVM: s390: vsie: correctly set and handle guest TOD
[mirror_ubuntu-jammy-kernel.git] / arch / s390 / kvm / kvm-s390.c
1 /*
2 * hosting zSeries kernel virtual machines
3 *
4 * Copyright IBM Corp. 2008, 2009
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
15 */
16
17 #include <linux/compiler.h>
18 #include <linux/err.h>
19 #include <linux/fs.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/mman.h>
25 #include <linux/module.h>
26 #include <linux/random.h>
27 #include <linux/slab.h>
28 #include <linux/timer.h>
29 #include <linux/vmalloc.h>
30 #include <linux/bitmap.h>
31 #include <asm/asm-offsets.h>
32 #include <asm/lowcore.h>
33 #include <asm/etr.h>
34 #include <asm/pgtable.h>
35 #include <asm/gmap.h>
36 #include <asm/nmi.h>
37 #include <asm/switch_to.h>
38 #include <asm/isc.h>
39 #include <asm/sclp.h>
40 #include <asm/cpacf.h>
41 #include <asm/etr.h>
42 #include "kvm-s390.h"
43 #include "gaccess.h"
44
45 #define KMSG_COMPONENT "kvm-s390"
46 #undef pr_fmt
47 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
48
49 #define CREATE_TRACE_POINTS
50 #include "trace.h"
51 #include "trace-s390.h"
52
53 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
54 #define LOCAL_IRQS 32
55 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
56 (KVM_MAX_VCPUS + LOCAL_IRQS))
57
58 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
59
60 struct kvm_stats_debugfs_item debugfs_entries[] = {
61 { "userspace_handled", VCPU_STAT(exit_userspace) },
62 { "exit_null", VCPU_STAT(exit_null) },
63 { "exit_validity", VCPU_STAT(exit_validity) },
64 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
65 { "exit_external_request", VCPU_STAT(exit_external_request) },
66 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
67 { "exit_instruction", VCPU_STAT(exit_instruction) },
68 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
69 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
70 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
71 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
72 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
73 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
74 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
75 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
76 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
77 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
78 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
79 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
80 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
81 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
82 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
83 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
84 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
85 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
86 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
87 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
88 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
89 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
90 { "instruction_spx", VCPU_STAT(instruction_spx) },
91 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
92 { "instruction_stap", VCPU_STAT(instruction_stap) },
93 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
94 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
95 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
96 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
97 { "instruction_essa", VCPU_STAT(instruction_essa) },
98 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
99 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
100 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
101 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
102 { "instruction_sie", VCPU_STAT(instruction_sie) },
103 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
104 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
105 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
106 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
107 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
108 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
109 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
110 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
111 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
112 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
113 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
114 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
115 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
116 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
117 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
118 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
119 { "diagnose_10", VCPU_STAT(diagnose_10) },
120 { "diagnose_44", VCPU_STAT(diagnose_44) },
121 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
122 { "diagnose_258", VCPU_STAT(diagnose_258) },
123 { "diagnose_308", VCPU_STAT(diagnose_308) },
124 { "diagnose_500", VCPU_STAT(diagnose_500) },
125 { NULL }
126 };
127
128 /* upper facilities limit for kvm */
129 unsigned long kvm_s390_fac_list_mask[16] = {
130 0xffe6000000000000UL,
131 0x005e000000000000UL,
132 };
133
134 unsigned long kvm_s390_fac_list_mask_size(void)
135 {
136 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
137 return ARRAY_SIZE(kvm_s390_fac_list_mask);
138 }
139
140 /* available cpu features supported by kvm */
141 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
142 /* available subfunctions indicated via query / "test bit" */
143 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
144
145 static struct gmap_notifier gmap_notifier;
146 static struct gmap_notifier vsie_gmap_notifier;
147 debug_info_t *kvm_s390_dbf;
148
149 /* Section: not file related */
150 int kvm_arch_hardware_enable(void)
151 {
152 /* every s390 is virtualization enabled ;-) */
153 return 0;
154 }
155
156 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
157 unsigned long end);
158
159 /*
160 * This callback is executed during stop_machine(). All CPUs are therefore
161 * temporarily stopped. In order not to change guest behavior, we have to
162 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
163 * so a CPU won't be stopped while calculating with the epoch.
164 */
165 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
166 void *v)
167 {
168 struct kvm *kvm;
169 struct kvm_vcpu *vcpu;
170 int i;
171 unsigned long long *delta = v;
172
173 list_for_each_entry(kvm, &vm_list, vm_list) {
174 kvm->arch.epoch -= *delta;
175 kvm_for_each_vcpu(i, vcpu, kvm) {
176 vcpu->arch.sie_block->epoch -= *delta;
177 if (vcpu->arch.cputm_enabled)
178 vcpu->arch.cputm_start += *delta;
179 if (vcpu->arch.vsie_block)
180 vcpu->arch.vsie_block->epoch -= *delta;
181 }
182 }
183 return NOTIFY_OK;
184 }
185
186 static struct notifier_block kvm_clock_notifier = {
187 .notifier_call = kvm_clock_sync,
188 };
189
190 int kvm_arch_hardware_setup(void)
191 {
192 gmap_notifier.notifier_call = kvm_gmap_notifier;
193 gmap_register_pte_notifier(&gmap_notifier);
194 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
195 gmap_register_pte_notifier(&vsie_gmap_notifier);
196 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
197 &kvm_clock_notifier);
198 return 0;
199 }
200
201 void kvm_arch_hardware_unsetup(void)
202 {
203 gmap_unregister_pte_notifier(&gmap_notifier);
204 gmap_unregister_pte_notifier(&vsie_gmap_notifier);
205 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
206 &kvm_clock_notifier);
207 }
208
209 static void allow_cpu_feat(unsigned long nr)
210 {
211 set_bit_inv(nr, kvm_s390_available_cpu_feat);
212 }
213
214 static inline int plo_test_bit(unsigned char nr)
215 {
216 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
217 int cc = 3; /* subfunction not available */
218
219 asm volatile(
220 /* Parameter registers are ignored for "test bit" */
221 " plo 0,0,0,0(0)\n"
222 " ipm %0\n"
223 " srl %0,28\n"
224 : "=d" (cc)
225 : "d" (r0)
226 : "cc");
227 return cc == 0;
228 }
229
230 static void kvm_s390_cpu_feat_init(void)
231 {
232 int i;
233
234 for (i = 0; i < 256; ++i) {
235 if (plo_test_bit(i))
236 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
237 }
238
239 if (test_facility(28)) /* TOD-clock steering */
240 etr_ptff(kvm_s390_available_subfunc.ptff, ETR_PTFF_QAF);
241
242 if (test_facility(17)) { /* MSA */
243 __cpacf_query(CPACF_KMAC, kvm_s390_available_subfunc.kmac);
244 __cpacf_query(CPACF_KMC, kvm_s390_available_subfunc.kmc);
245 __cpacf_query(CPACF_KM, kvm_s390_available_subfunc.km);
246 __cpacf_query(CPACF_KIMD, kvm_s390_available_subfunc.kimd);
247 __cpacf_query(CPACF_KLMD, kvm_s390_available_subfunc.klmd);
248 }
249 if (test_facility(76)) /* MSA3 */
250 __cpacf_query(CPACF_PCKMO, kvm_s390_available_subfunc.pckmo);
251 if (test_facility(77)) { /* MSA4 */
252 __cpacf_query(CPACF_KMCTR, kvm_s390_available_subfunc.kmctr);
253 __cpacf_query(CPACF_KMF, kvm_s390_available_subfunc.kmf);
254 __cpacf_query(CPACF_KMO, kvm_s390_available_subfunc.kmo);
255 __cpacf_query(CPACF_PCC, kvm_s390_available_subfunc.pcc);
256 }
257 if (test_facility(57)) /* MSA5 */
258 __cpacf_query(CPACF_PPNO, kvm_s390_available_subfunc.ppno);
259
260 if (MACHINE_HAS_ESOP)
261 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
262 /*
263 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
264 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
265 */
266 if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
267 !test_facility(3))
268 return;
269 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
270 if (sclp.has_64bscao)
271 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
272 if (sclp.has_siif)
273 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
274 if (sclp.has_gpere)
275 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
276 if (sclp.has_gsls)
277 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
278 if (sclp.has_ib)
279 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
280 if (sclp.has_cei)
281 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
282 if (sclp.has_ibs)
283 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
284 }
285
286 int kvm_arch_init(void *opaque)
287 {
288 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
289 if (!kvm_s390_dbf)
290 return -ENOMEM;
291
292 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
293 debug_unregister(kvm_s390_dbf);
294 return -ENOMEM;
295 }
296
297 kvm_s390_cpu_feat_init();
298
299 /* Register floating interrupt controller interface. */
300 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
301 }
302
303 void kvm_arch_exit(void)
304 {
305 debug_unregister(kvm_s390_dbf);
306 }
307
308 /* Section: device related */
309 long kvm_arch_dev_ioctl(struct file *filp,
310 unsigned int ioctl, unsigned long arg)
311 {
312 if (ioctl == KVM_S390_ENABLE_SIE)
313 return s390_enable_sie();
314 return -EINVAL;
315 }
316
317 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
318 {
319 int r;
320
321 switch (ext) {
322 case KVM_CAP_S390_PSW:
323 case KVM_CAP_S390_GMAP:
324 case KVM_CAP_SYNC_MMU:
325 #ifdef CONFIG_KVM_S390_UCONTROL
326 case KVM_CAP_S390_UCONTROL:
327 #endif
328 case KVM_CAP_ASYNC_PF:
329 case KVM_CAP_SYNC_REGS:
330 case KVM_CAP_ONE_REG:
331 case KVM_CAP_ENABLE_CAP:
332 case KVM_CAP_S390_CSS_SUPPORT:
333 case KVM_CAP_IOEVENTFD:
334 case KVM_CAP_DEVICE_CTRL:
335 case KVM_CAP_ENABLE_CAP_VM:
336 case KVM_CAP_S390_IRQCHIP:
337 case KVM_CAP_VM_ATTRIBUTES:
338 case KVM_CAP_MP_STATE:
339 case KVM_CAP_S390_INJECT_IRQ:
340 case KVM_CAP_S390_USER_SIGP:
341 case KVM_CAP_S390_USER_STSI:
342 case KVM_CAP_S390_SKEYS:
343 case KVM_CAP_S390_IRQ_STATE:
344 r = 1;
345 break;
346 case KVM_CAP_S390_MEM_OP:
347 r = MEM_OP_MAX_SIZE;
348 break;
349 case KVM_CAP_NR_VCPUS:
350 case KVM_CAP_MAX_VCPUS:
351 r = KVM_S390_BSCA_CPU_SLOTS;
352 if (sclp.has_esca && sclp.has_64bscao)
353 r = KVM_S390_ESCA_CPU_SLOTS;
354 break;
355 case KVM_CAP_NR_MEMSLOTS:
356 r = KVM_USER_MEM_SLOTS;
357 break;
358 case KVM_CAP_S390_COW:
359 r = MACHINE_HAS_ESOP;
360 break;
361 case KVM_CAP_S390_VECTOR_REGISTERS:
362 r = MACHINE_HAS_VX;
363 break;
364 case KVM_CAP_S390_RI:
365 r = test_facility(64);
366 break;
367 default:
368 r = 0;
369 }
370 return r;
371 }
372
373 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
374 struct kvm_memory_slot *memslot)
375 {
376 gfn_t cur_gfn, last_gfn;
377 unsigned long address;
378 struct gmap *gmap = kvm->arch.gmap;
379
380 /* Loop over all guest pages */
381 last_gfn = memslot->base_gfn + memslot->npages;
382 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
383 address = gfn_to_hva_memslot(memslot, cur_gfn);
384
385 if (test_and_clear_guest_dirty(gmap->mm, address))
386 mark_page_dirty(kvm, cur_gfn);
387 if (fatal_signal_pending(current))
388 return;
389 cond_resched();
390 }
391 }
392
393 /* Section: vm related */
394 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
395
396 /*
397 * Get (and clear) the dirty memory log for a memory slot.
398 */
399 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
400 struct kvm_dirty_log *log)
401 {
402 int r;
403 unsigned long n;
404 struct kvm_memslots *slots;
405 struct kvm_memory_slot *memslot;
406 int is_dirty = 0;
407
408 mutex_lock(&kvm->slots_lock);
409
410 r = -EINVAL;
411 if (log->slot >= KVM_USER_MEM_SLOTS)
412 goto out;
413
414 slots = kvm_memslots(kvm);
415 memslot = id_to_memslot(slots, log->slot);
416 r = -ENOENT;
417 if (!memslot->dirty_bitmap)
418 goto out;
419
420 kvm_s390_sync_dirty_log(kvm, memslot);
421 r = kvm_get_dirty_log(kvm, log, &is_dirty);
422 if (r)
423 goto out;
424
425 /* Clear the dirty log */
426 if (is_dirty) {
427 n = kvm_dirty_bitmap_bytes(memslot);
428 memset(memslot->dirty_bitmap, 0, n);
429 }
430 r = 0;
431 out:
432 mutex_unlock(&kvm->slots_lock);
433 return r;
434 }
435
436 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
437 {
438 int r;
439
440 if (cap->flags)
441 return -EINVAL;
442
443 switch (cap->cap) {
444 case KVM_CAP_S390_IRQCHIP:
445 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
446 kvm->arch.use_irqchip = 1;
447 r = 0;
448 break;
449 case KVM_CAP_S390_USER_SIGP:
450 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
451 kvm->arch.user_sigp = 1;
452 r = 0;
453 break;
454 case KVM_CAP_S390_VECTOR_REGISTERS:
455 mutex_lock(&kvm->lock);
456 if (kvm->created_vcpus) {
457 r = -EBUSY;
458 } else if (MACHINE_HAS_VX) {
459 set_kvm_facility(kvm->arch.model.fac_mask, 129);
460 set_kvm_facility(kvm->arch.model.fac_list, 129);
461 r = 0;
462 } else
463 r = -EINVAL;
464 mutex_unlock(&kvm->lock);
465 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
466 r ? "(not available)" : "(success)");
467 break;
468 case KVM_CAP_S390_RI:
469 r = -EINVAL;
470 mutex_lock(&kvm->lock);
471 if (kvm->created_vcpus) {
472 r = -EBUSY;
473 } else if (test_facility(64)) {
474 set_kvm_facility(kvm->arch.model.fac_mask, 64);
475 set_kvm_facility(kvm->arch.model.fac_list, 64);
476 r = 0;
477 }
478 mutex_unlock(&kvm->lock);
479 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
480 r ? "(not available)" : "(success)");
481 break;
482 case KVM_CAP_S390_USER_STSI:
483 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
484 kvm->arch.user_stsi = 1;
485 r = 0;
486 break;
487 default:
488 r = -EINVAL;
489 break;
490 }
491 return r;
492 }
493
494 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
495 {
496 int ret;
497
498 switch (attr->attr) {
499 case KVM_S390_VM_MEM_LIMIT_SIZE:
500 ret = 0;
501 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
502 kvm->arch.mem_limit);
503 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
504 ret = -EFAULT;
505 break;
506 default:
507 ret = -ENXIO;
508 break;
509 }
510 return ret;
511 }
512
513 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
514 {
515 int ret;
516 unsigned int idx;
517 switch (attr->attr) {
518 case KVM_S390_VM_MEM_ENABLE_CMMA:
519 ret = -ENXIO;
520 if (!sclp.has_cmma)
521 break;
522
523 ret = -EBUSY;
524 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
525 mutex_lock(&kvm->lock);
526 if (!kvm->created_vcpus) {
527 kvm->arch.use_cmma = 1;
528 ret = 0;
529 }
530 mutex_unlock(&kvm->lock);
531 break;
532 case KVM_S390_VM_MEM_CLR_CMMA:
533 ret = -ENXIO;
534 if (!sclp.has_cmma)
535 break;
536 ret = -EINVAL;
537 if (!kvm->arch.use_cmma)
538 break;
539
540 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
541 mutex_lock(&kvm->lock);
542 idx = srcu_read_lock(&kvm->srcu);
543 s390_reset_cmma(kvm->arch.gmap->mm);
544 srcu_read_unlock(&kvm->srcu, idx);
545 mutex_unlock(&kvm->lock);
546 ret = 0;
547 break;
548 case KVM_S390_VM_MEM_LIMIT_SIZE: {
549 unsigned long new_limit;
550
551 if (kvm_is_ucontrol(kvm))
552 return -EINVAL;
553
554 if (get_user(new_limit, (u64 __user *)attr->addr))
555 return -EFAULT;
556
557 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
558 new_limit > kvm->arch.mem_limit)
559 return -E2BIG;
560
561 if (!new_limit)
562 return -EINVAL;
563
564 /* gmap_create takes last usable address */
565 if (new_limit != KVM_S390_NO_MEM_LIMIT)
566 new_limit -= 1;
567
568 ret = -EBUSY;
569 mutex_lock(&kvm->lock);
570 if (!kvm->created_vcpus) {
571 /* gmap_create will round the limit up */
572 struct gmap *new = gmap_create(current->mm, new_limit);
573
574 if (!new) {
575 ret = -ENOMEM;
576 } else {
577 gmap_remove(kvm->arch.gmap);
578 new->private = kvm;
579 kvm->arch.gmap = new;
580 ret = 0;
581 }
582 }
583 mutex_unlock(&kvm->lock);
584 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
585 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
586 (void *) kvm->arch.gmap->asce);
587 break;
588 }
589 default:
590 ret = -ENXIO;
591 break;
592 }
593 return ret;
594 }
595
596 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
597
598 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
599 {
600 struct kvm_vcpu *vcpu;
601 int i;
602
603 if (!test_kvm_facility(kvm, 76))
604 return -EINVAL;
605
606 mutex_lock(&kvm->lock);
607 switch (attr->attr) {
608 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
609 get_random_bytes(
610 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
611 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
612 kvm->arch.crypto.aes_kw = 1;
613 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
614 break;
615 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
616 get_random_bytes(
617 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
618 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
619 kvm->arch.crypto.dea_kw = 1;
620 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
621 break;
622 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
623 kvm->arch.crypto.aes_kw = 0;
624 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
625 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
626 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
627 break;
628 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
629 kvm->arch.crypto.dea_kw = 0;
630 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
631 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
632 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
633 break;
634 default:
635 mutex_unlock(&kvm->lock);
636 return -ENXIO;
637 }
638
639 kvm_for_each_vcpu(i, vcpu, kvm) {
640 kvm_s390_vcpu_crypto_setup(vcpu);
641 exit_sie(vcpu);
642 }
643 mutex_unlock(&kvm->lock);
644 return 0;
645 }
646
647 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
648 {
649 u8 gtod_high;
650
651 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
652 sizeof(gtod_high)))
653 return -EFAULT;
654
655 if (gtod_high != 0)
656 return -EINVAL;
657 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
658
659 return 0;
660 }
661
662 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
663 {
664 u64 gtod;
665
666 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
667 return -EFAULT;
668
669 kvm_s390_set_tod_clock(kvm, gtod);
670 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
671 return 0;
672 }
673
674 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
675 {
676 int ret;
677
678 if (attr->flags)
679 return -EINVAL;
680
681 switch (attr->attr) {
682 case KVM_S390_VM_TOD_HIGH:
683 ret = kvm_s390_set_tod_high(kvm, attr);
684 break;
685 case KVM_S390_VM_TOD_LOW:
686 ret = kvm_s390_set_tod_low(kvm, attr);
687 break;
688 default:
689 ret = -ENXIO;
690 break;
691 }
692 return ret;
693 }
694
695 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
696 {
697 u8 gtod_high = 0;
698
699 if (copy_to_user((void __user *)attr->addr, &gtod_high,
700 sizeof(gtod_high)))
701 return -EFAULT;
702 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
703
704 return 0;
705 }
706
707 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
708 {
709 u64 gtod;
710
711 gtod = kvm_s390_get_tod_clock_fast(kvm);
712 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
713 return -EFAULT;
714 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
715
716 return 0;
717 }
718
719 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
720 {
721 int ret;
722
723 if (attr->flags)
724 return -EINVAL;
725
726 switch (attr->attr) {
727 case KVM_S390_VM_TOD_HIGH:
728 ret = kvm_s390_get_tod_high(kvm, attr);
729 break;
730 case KVM_S390_VM_TOD_LOW:
731 ret = kvm_s390_get_tod_low(kvm, attr);
732 break;
733 default:
734 ret = -ENXIO;
735 break;
736 }
737 return ret;
738 }
739
740 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
741 {
742 struct kvm_s390_vm_cpu_processor *proc;
743 u16 lowest_ibc, unblocked_ibc;
744 int ret = 0;
745
746 mutex_lock(&kvm->lock);
747 if (kvm->created_vcpus) {
748 ret = -EBUSY;
749 goto out;
750 }
751 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
752 if (!proc) {
753 ret = -ENOMEM;
754 goto out;
755 }
756 if (!copy_from_user(proc, (void __user *)attr->addr,
757 sizeof(*proc))) {
758 kvm->arch.model.cpuid = proc->cpuid;
759 lowest_ibc = sclp.ibc >> 16 & 0xfff;
760 unblocked_ibc = sclp.ibc & 0xfff;
761 if (lowest_ibc) {
762 if (proc->ibc > unblocked_ibc)
763 kvm->arch.model.ibc = unblocked_ibc;
764 else if (proc->ibc < lowest_ibc)
765 kvm->arch.model.ibc = lowest_ibc;
766 else
767 kvm->arch.model.ibc = proc->ibc;
768 }
769 memcpy(kvm->arch.model.fac_list, proc->fac_list,
770 S390_ARCH_FAC_LIST_SIZE_BYTE);
771 } else
772 ret = -EFAULT;
773 kfree(proc);
774 out:
775 mutex_unlock(&kvm->lock);
776 return ret;
777 }
778
779 static int kvm_s390_set_processor_feat(struct kvm *kvm,
780 struct kvm_device_attr *attr)
781 {
782 struct kvm_s390_vm_cpu_feat data;
783 int ret = -EBUSY;
784
785 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
786 return -EFAULT;
787 if (!bitmap_subset((unsigned long *) data.feat,
788 kvm_s390_available_cpu_feat,
789 KVM_S390_VM_CPU_FEAT_NR_BITS))
790 return -EINVAL;
791
792 mutex_lock(&kvm->lock);
793 if (!atomic_read(&kvm->online_vcpus)) {
794 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
795 KVM_S390_VM_CPU_FEAT_NR_BITS);
796 ret = 0;
797 }
798 mutex_unlock(&kvm->lock);
799 return ret;
800 }
801
802 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
803 struct kvm_device_attr *attr)
804 {
805 /*
806 * Once supported by kernel + hw, we have to store the subfunctions
807 * in kvm->arch and remember that user space configured them.
808 */
809 return -ENXIO;
810 }
811
812 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
813 {
814 int ret = -ENXIO;
815
816 switch (attr->attr) {
817 case KVM_S390_VM_CPU_PROCESSOR:
818 ret = kvm_s390_set_processor(kvm, attr);
819 break;
820 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
821 ret = kvm_s390_set_processor_feat(kvm, attr);
822 break;
823 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
824 ret = kvm_s390_set_processor_subfunc(kvm, attr);
825 break;
826 }
827 return ret;
828 }
829
830 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
831 {
832 struct kvm_s390_vm_cpu_processor *proc;
833 int ret = 0;
834
835 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
836 if (!proc) {
837 ret = -ENOMEM;
838 goto out;
839 }
840 proc->cpuid = kvm->arch.model.cpuid;
841 proc->ibc = kvm->arch.model.ibc;
842 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
843 S390_ARCH_FAC_LIST_SIZE_BYTE);
844 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
845 ret = -EFAULT;
846 kfree(proc);
847 out:
848 return ret;
849 }
850
851 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
852 {
853 struct kvm_s390_vm_cpu_machine *mach;
854 int ret = 0;
855
856 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
857 if (!mach) {
858 ret = -ENOMEM;
859 goto out;
860 }
861 get_cpu_id((struct cpuid *) &mach->cpuid);
862 mach->ibc = sclp.ibc;
863 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
864 S390_ARCH_FAC_LIST_SIZE_BYTE);
865 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
866 S390_ARCH_FAC_LIST_SIZE_BYTE);
867 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
868 ret = -EFAULT;
869 kfree(mach);
870 out:
871 return ret;
872 }
873
874 static int kvm_s390_get_processor_feat(struct kvm *kvm,
875 struct kvm_device_attr *attr)
876 {
877 struct kvm_s390_vm_cpu_feat data;
878
879 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
880 KVM_S390_VM_CPU_FEAT_NR_BITS);
881 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
882 return -EFAULT;
883 return 0;
884 }
885
886 static int kvm_s390_get_machine_feat(struct kvm *kvm,
887 struct kvm_device_attr *attr)
888 {
889 struct kvm_s390_vm_cpu_feat data;
890
891 bitmap_copy((unsigned long *) data.feat,
892 kvm_s390_available_cpu_feat,
893 KVM_S390_VM_CPU_FEAT_NR_BITS);
894 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
895 return -EFAULT;
896 return 0;
897 }
898
899 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
900 struct kvm_device_attr *attr)
901 {
902 /*
903 * Once we can actually configure subfunctions (kernel + hw support),
904 * we have to check if they were already set by user space, if so copy
905 * them from kvm->arch.
906 */
907 return -ENXIO;
908 }
909
910 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
911 struct kvm_device_attr *attr)
912 {
913 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
914 sizeof(struct kvm_s390_vm_cpu_subfunc)))
915 return -EFAULT;
916 return 0;
917 }
918 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
919 {
920 int ret = -ENXIO;
921
922 switch (attr->attr) {
923 case KVM_S390_VM_CPU_PROCESSOR:
924 ret = kvm_s390_get_processor(kvm, attr);
925 break;
926 case KVM_S390_VM_CPU_MACHINE:
927 ret = kvm_s390_get_machine(kvm, attr);
928 break;
929 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
930 ret = kvm_s390_get_processor_feat(kvm, attr);
931 break;
932 case KVM_S390_VM_CPU_MACHINE_FEAT:
933 ret = kvm_s390_get_machine_feat(kvm, attr);
934 break;
935 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
936 ret = kvm_s390_get_processor_subfunc(kvm, attr);
937 break;
938 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
939 ret = kvm_s390_get_machine_subfunc(kvm, attr);
940 break;
941 }
942 return ret;
943 }
944
945 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
946 {
947 int ret;
948
949 switch (attr->group) {
950 case KVM_S390_VM_MEM_CTRL:
951 ret = kvm_s390_set_mem_control(kvm, attr);
952 break;
953 case KVM_S390_VM_TOD:
954 ret = kvm_s390_set_tod(kvm, attr);
955 break;
956 case KVM_S390_VM_CPU_MODEL:
957 ret = kvm_s390_set_cpu_model(kvm, attr);
958 break;
959 case KVM_S390_VM_CRYPTO:
960 ret = kvm_s390_vm_set_crypto(kvm, attr);
961 break;
962 default:
963 ret = -ENXIO;
964 break;
965 }
966
967 return ret;
968 }
969
970 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
971 {
972 int ret;
973
974 switch (attr->group) {
975 case KVM_S390_VM_MEM_CTRL:
976 ret = kvm_s390_get_mem_control(kvm, attr);
977 break;
978 case KVM_S390_VM_TOD:
979 ret = kvm_s390_get_tod(kvm, attr);
980 break;
981 case KVM_S390_VM_CPU_MODEL:
982 ret = kvm_s390_get_cpu_model(kvm, attr);
983 break;
984 default:
985 ret = -ENXIO;
986 break;
987 }
988
989 return ret;
990 }
991
992 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
993 {
994 int ret;
995
996 switch (attr->group) {
997 case KVM_S390_VM_MEM_CTRL:
998 switch (attr->attr) {
999 case KVM_S390_VM_MEM_ENABLE_CMMA:
1000 case KVM_S390_VM_MEM_CLR_CMMA:
1001 ret = sclp.has_cmma ? 0 : -ENXIO;
1002 break;
1003 case KVM_S390_VM_MEM_LIMIT_SIZE:
1004 ret = 0;
1005 break;
1006 default:
1007 ret = -ENXIO;
1008 break;
1009 }
1010 break;
1011 case KVM_S390_VM_TOD:
1012 switch (attr->attr) {
1013 case KVM_S390_VM_TOD_LOW:
1014 case KVM_S390_VM_TOD_HIGH:
1015 ret = 0;
1016 break;
1017 default:
1018 ret = -ENXIO;
1019 break;
1020 }
1021 break;
1022 case KVM_S390_VM_CPU_MODEL:
1023 switch (attr->attr) {
1024 case KVM_S390_VM_CPU_PROCESSOR:
1025 case KVM_S390_VM_CPU_MACHINE:
1026 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1027 case KVM_S390_VM_CPU_MACHINE_FEAT:
1028 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1029 ret = 0;
1030 break;
1031 /* configuring subfunctions is not supported yet */
1032 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1033 default:
1034 ret = -ENXIO;
1035 break;
1036 }
1037 break;
1038 case KVM_S390_VM_CRYPTO:
1039 switch (attr->attr) {
1040 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1041 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1042 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1043 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1044 ret = 0;
1045 break;
1046 default:
1047 ret = -ENXIO;
1048 break;
1049 }
1050 break;
1051 default:
1052 ret = -ENXIO;
1053 break;
1054 }
1055
1056 return ret;
1057 }
1058
1059 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1060 {
1061 uint8_t *keys;
1062 uint64_t hva;
1063 int i, r = 0;
1064
1065 if (args->flags != 0)
1066 return -EINVAL;
1067
1068 /* Is this guest using storage keys? */
1069 if (!mm_use_skey(current->mm))
1070 return KVM_S390_GET_SKEYS_NONE;
1071
1072 /* Enforce sane limit on memory allocation */
1073 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1074 return -EINVAL;
1075
1076 keys = kmalloc_array(args->count, sizeof(uint8_t),
1077 GFP_KERNEL | __GFP_NOWARN);
1078 if (!keys)
1079 keys = vmalloc(sizeof(uint8_t) * args->count);
1080 if (!keys)
1081 return -ENOMEM;
1082
1083 down_read(&current->mm->mmap_sem);
1084 for (i = 0; i < args->count; i++) {
1085 hva = gfn_to_hva(kvm, args->start_gfn + i);
1086 if (kvm_is_error_hva(hva)) {
1087 r = -EFAULT;
1088 break;
1089 }
1090
1091 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1092 if (r)
1093 break;
1094 }
1095 up_read(&current->mm->mmap_sem);
1096
1097 if (!r) {
1098 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1099 sizeof(uint8_t) * args->count);
1100 if (r)
1101 r = -EFAULT;
1102 }
1103
1104 kvfree(keys);
1105 return r;
1106 }
1107
1108 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1109 {
1110 uint8_t *keys;
1111 uint64_t hva;
1112 int i, r = 0;
1113
1114 if (args->flags != 0)
1115 return -EINVAL;
1116
1117 /* Enforce sane limit on memory allocation */
1118 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1119 return -EINVAL;
1120
1121 keys = kmalloc_array(args->count, sizeof(uint8_t),
1122 GFP_KERNEL | __GFP_NOWARN);
1123 if (!keys)
1124 keys = vmalloc(sizeof(uint8_t) * args->count);
1125 if (!keys)
1126 return -ENOMEM;
1127
1128 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1129 sizeof(uint8_t) * args->count);
1130 if (r) {
1131 r = -EFAULT;
1132 goto out;
1133 }
1134
1135 /* Enable storage key handling for the guest */
1136 r = s390_enable_skey();
1137 if (r)
1138 goto out;
1139
1140 down_read(&current->mm->mmap_sem);
1141 for (i = 0; i < args->count; i++) {
1142 hva = gfn_to_hva(kvm, args->start_gfn + i);
1143 if (kvm_is_error_hva(hva)) {
1144 r = -EFAULT;
1145 break;
1146 }
1147
1148 /* Lowest order bit is reserved */
1149 if (keys[i] & 0x01) {
1150 r = -EINVAL;
1151 break;
1152 }
1153
1154 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1155 if (r)
1156 break;
1157 }
1158 up_read(&current->mm->mmap_sem);
1159 out:
1160 kvfree(keys);
1161 return r;
1162 }
1163
1164 long kvm_arch_vm_ioctl(struct file *filp,
1165 unsigned int ioctl, unsigned long arg)
1166 {
1167 struct kvm *kvm = filp->private_data;
1168 void __user *argp = (void __user *)arg;
1169 struct kvm_device_attr attr;
1170 int r;
1171
1172 switch (ioctl) {
1173 case KVM_S390_INTERRUPT: {
1174 struct kvm_s390_interrupt s390int;
1175
1176 r = -EFAULT;
1177 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1178 break;
1179 r = kvm_s390_inject_vm(kvm, &s390int);
1180 break;
1181 }
1182 case KVM_ENABLE_CAP: {
1183 struct kvm_enable_cap cap;
1184 r = -EFAULT;
1185 if (copy_from_user(&cap, argp, sizeof(cap)))
1186 break;
1187 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1188 break;
1189 }
1190 case KVM_CREATE_IRQCHIP: {
1191 struct kvm_irq_routing_entry routing;
1192
1193 r = -EINVAL;
1194 if (kvm->arch.use_irqchip) {
1195 /* Set up dummy routing. */
1196 memset(&routing, 0, sizeof(routing));
1197 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
1198 }
1199 break;
1200 }
1201 case KVM_SET_DEVICE_ATTR: {
1202 r = -EFAULT;
1203 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1204 break;
1205 r = kvm_s390_vm_set_attr(kvm, &attr);
1206 break;
1207 }
1208 case KVM_GET_DEVICE_ATTR: {
1209 r = -EFAULT;
1210 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1211 break;
1212 r = kvm_s390_vm_get_attr(kvm, &attr);
1213 break;
1214 }
1215 case KVM_HAS_DEVICE_ATTR: {
1216 r = -EFAULT;
1217 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1218 break;
1219 r = kvm_s390_vm_has_attr(kvm, &attr);
1220 break;
1221 }
1222 case KVM_S390_GET_SKEYS: {
1223 struct kvm_s390_skeys args;
1224
1225 r = -EFAULT;
1226 if (copy_from_user(&args, argp,
1227 sizeof(struct kvm_s390_skeys)))
1228 break;
1229 r = kvm_s390_get_skeys(kvm, &args);
1230 break;
1231 }
1232 case KVM_S390_SET_SKEYS: {
1233 struct kvm_s390_skeys args;
1234
1235 r = -EFAULT;
1236 if (copy_from_user(&args, argp,
1237 sizeof(struct kvm_s390_skeys)))
1238 break;
1239 r = kvm_s390_set_skeys(kvm, &args);
1240 break;
1241 }
1242 default:
1243 r = -ENOTTY;
1244 }
1245
1246 return r;
1247 }
1248
1249 static int kvm_s390_query_ap_config(u8 *config)
1250 {
1251 u32 fcn_code = 0x04000000UL;
1252 u32 cc = 0;
1253
1254 memset(config, 0, 128);
1255 asm volatile(
1256 "lgr 0,%1\n"
1257 "lgr 2,%2\n"
1258 ".long 0xb2af0000\n" /* PQAP(QCI) */
1259 "0: ipm %0\n"
1260 "srl %0,28\n"
1261 "1:\n"
1262 EX_TABLE(0b, 1b)
1263 : "+r" (cc)
1264 : "r" (fcn_code), "r" (config)
1265 : "cc", "0", "2", "memory"
1266 );
1267
1268 return cc;
1269 }
1270
1271 static int kvm_s390_apxa_installed(void)
1272 {
1273 u8 config[128];
1274 int cc;
1275
1276 if (test_facility(12)) {
1277 cc = kvm_s390_query_ap_config(config);
1278
1279 if (cc)
1280 pr_err("PQAP(QCI) failed with cc=%d", cc);
1281 else
1282 return config[0] & 0x40;
1283 }
1284
1285 return 0;
1286 }
1287
1288 static void kvm_s390_set_crycb_format(struct kvm *kvm)
1289 {
1290 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1291
1292 if (kvm_s390_apxa_installed())
1293 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1294 else
1295 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1296 }
1297
1298 static u64 kvm_s390_get_initial_cpuid(void)
1299 {
1300 struct cpuid cpuid;
1301
1302 get_cpu_id(&cpuid);
1303 cpuid.version = 0xff;
1304 return *((u64 *) &cpuid);
1305 }
1306
1307 static void kvm_s390_crypto_init(struct kvm *kvm)
1308 {
1309 if (!test_kvm_facility(kvm, 76))
1310 return;
1311
1312 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
1313 kvm_s390_set_crycb_format(kvm);
1314
1315 /* Enable AES/DEA protected key functions by default */
1316 kvm->arch.crypto.aes_kw = 1;
1317 kvm->arch.crypto.dea_kw = 1;
1318 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1319 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1320 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1321 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1322 }
1323
1324 static void sca_dispose(struct kvm *kvm)
1325 {
1326 if (kvm->arch.use_esca)
1327 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
1328 else
1329 free_page((unsigned long)(kvm->arch.sca));
1330 kvm->arch.sca = NULL;
1331 }
1332
1333 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1334 {
1335 gfp_t alloc_flags = GFP_KERNEL;
1336 int i, rc;
1337 char debug_name[16];
1338 static unsigned long sca_offset;
1339
1340 rc = -EINVAL;
1341 #ifdef CONFIG_KVM_S390_UCONTROL
1342 if (type & ~KVM_VM_S390_UCONTROL)
1343 goto out_err;
1344 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1345 goto out_err;
1346 #else
1347 if (type)
1348 goto out_err;
1349 #endif
1350
1351 rc = s390_enable_sie();
1352 if (rc)
1353 goto out_err;
1354
1355 rc = -ENOMEM;
1356
1357 ratelimit_state_init(&kvm->arch.sthyi_limit, 5 * HZ, 500);
1358
1359 kvm->arch.use_esca = 0; /* start with basic SCA */
1360 if (!sclp.has_64bscao)
1361 alloc_flags |= GFP_DMA;
1362 rwlock_init(&kvm->arch.sca_lock);
1363 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
1364 if (!kvm->arch.sca)
1365 goto out_err;
1366 spin_lock(&kvm_lock);
1367 sca_offset += 16;
1368 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
1369 sca_offset = 0;
1370 kvm->arch.sca = (struct bsca_block *)
1371 ((char *) kvm->arch.sca + sca_offset);
1372 spin_unlock(&kvm_lock);
1373
1374 sprintf(debug_name, "kvm-%u", current->pid);
1375
1376 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1377 if (!kvm->arch.dbf)
1378 goto out_err;
1379
1380 kvm->arch.sie_page2 =
1381 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1382 if (!kvm->arch.sie_page2)
1383 goto out_err;
1384
1385 /* Populate the facility mask initially. */
1386 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
1387 S390_ARCH_FAC_LIST_SIZE_BYTE);
1388 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1389 if (i < kvm_s390_fac_list_mask_size())
1390 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
1391 else
1392 kvm->arch.model.fac_mask[i] = 0UL;
1393 }
1394
1395 /* Populate the facility list initially. */
1396 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1397 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
1398 S390_ARCH_FAC_LIST_SIZE_BYTE);
1399
1400 set_kvm_facility(kvm->arch.model.fac_mask, 74);
1401 set_kvm_facility(kvm->arch.model.fac_list, 74);
1402
1403 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
1404 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1405
1406 kvm_s390_crypto_init(kvm);
1407
1408 spin_lock_init(&kvm->arch.float_int.lock);
1409 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1410 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1411 init_waitqueue_head(&kvm->arch.ipte_wq);
1412 mutex_init(&kvm->arch.ipte_mutex);
1413
1414 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1415 VM_EVENT(kvm, 3, "vm created with type %lu", type);
1416
1417 if (type & KVM_VM_S390_UCONTROL) {
1418 kvm->arch.gmap = NULL;
1419 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
1420 } else {
1421 if (sclp.hamax == U64_MAX)
1422 kvm->arch.mem_limit = TASK_MAX_SIZE;
1423 else
1424 kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
1425 sclp.hamax + 1);
1426 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
1427 if (!kvm->arch.gmap)
1428 goto out_err;
1429 kvm->arch.gmap->private = kvm;
1430 kvm->arch.gmap->pfault_enabled = 0;
1431 }
1432
1433 kvm->arch.css_support = 0;
1434 kvm->arch.use_irqchip = 0;
1435 kvm->arch.epoch = 0;
1436
1437 spin_lock_init(&kvm->arch.start_stop_lock);
1438 kvm_s390_vsie_init(kvm);
1439 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
1440
1441 return 0;
1442 out_err:
1443 free_page((unsigned long)kvm->arch.sie_page2);
1444 debug_unregister(kvm->arch.dbf);
1445 sca_dispose(kvm);
1446 KVM_EVENT(3, "creation of vm failed: %d", rc);
1447 return rc;
1448 }
1449
1450 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1451 {
1452 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1453 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1454 kvm_s390_clear_local_irqs(vcpu);
1455 kvm_clear_async_pf_completion_queue(vcpu);
1456 if (!kvm_is_ucontrol(vcpu->kvm))
1457 sca_del_vcpu(vcpu);
1458
1459 if (kvm_is_ucontrol(vcpu->kvm))
1460 gmap_remove(vcpu->arch.gmap);
1461
1462 if (vcpu->kvm->arch.use_cmma)
1463 kvm_s390_vcpu_unsetup_cmma(vcpu);
1464 free_page((unsigned long)(vcpu->arch.sie_block));
1465
1466 kvm_vcpu_uninit(vcpu);
1467 kmem_cache_free(kvm_vcpu_cache, vcpu);
1468 }
1469
1470 static void kvm_free_vcpus(struct kvm *kvm)
1471 {
1472 unsigned int i;
1473 struct kvm_vcpu *vcpu;
1474
1475 kvm_for_each_vcpu(i, vcpu, kvm)
1476 kvm_arch_vcpu_destroy(vcpu);
1477
1478 mutex_lock(&kvm->lock);
1479 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1480 kvm->vcpus[i] = NULL;
1481
1482 atomic_set(&kvm->online_vcpus, 0);
1483 mutex_unlock(&kvm->lock);
1484 }
1485
1486 void kvm_arch_destroy_vm(struct kvm *kvm)
1487 {
1488 kvm_free_vcpus(kvm);
1489 sca_dispose(kvm);
1490 debug_unregister(kvm->arch.dbf);
1491 free_page((unsigned long)kvm->arch.sie_page2);
1492 if (!kvm_is_ucontrol(kvm))
1493 gmap_remove(kvm->arch.gmap);
1494 kvm_s390_destroy_adapters(kvm);
1495 kvm_s390_clear_float_irqs(kvm);
1496 kvm_s390_vsie_destroy(kvm);
1497 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
1498 }
1499
1500 /* Section: vcpu related */
1501 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1502 {
1503 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
1504 if (!vcpu->arch.gmap)
1505 return -ENOMEM;
1506 vcpu->arch.gmap->private = vcpu->kvm;
1507
1508 return 0;
1509 }
1510
1511 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1512 {
1513 read_lock(&vcpu->kvm->arch.sca_lock);
1514 if (vcpu->kvm->arch.use_esca) {
1515 struct esca_block *sca = vcpu->kvm->arch.sca;
1516
1517 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1518 sca->cpu[vcpu->vcpu_id].sda = 0;
1519 } else {
1520 struct bsca_block *sca = vcpu->kvm->arch.sca;
1521
1522 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1523 sca->cpu[vcpu->vcpu_id].sda = 0;
1524 }
1525 read_unlock(&vcpu->kvm->arch.sca_lock);
1526 }
1527
1528 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
1529 {
1530 read_lock(&vcpu->kvm->arch.sca_lock);
1531 if (vcpu->kvm->arch.use_esca) {
1532 struct esca_block *sca = vcpu->kvm->arch.sca;
1533
1534 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1535 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1536 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
1537 vcpu->arch.sie_block->ecb2 |= 0x04U;
1538 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1539 } else {
1540 struct bsca_block *sca = vcpu->kvm->arch.sca;
1541
1542 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1543 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1544 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
1545 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1546 }
1547 read_unlock(&vcpu->kvm->arch.sca_lock);
1548 }
1549
1550 /* Basic SCA to Extended SCA data copy routines */
1551 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1552 {
1553 d->sda = s->sda;
1554 d->sigp_ctrl.c = s->sigp_ctrl.c;
1555 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1556 }
1557
1558 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1559 {
1560 int i;
1561
1562 d->ipte_control = s->ipte_control;
1563 d->mcn[0] = s->mcn;
1564 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1565 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1566 }
1567
1568 static int sca_switch_to_extended(struct kvm *kvm)
1569 {
1570 struct bsca_block *old_sca = kvm->arch.sca;
1571 struct esca_block *new_sca;
1572 struct kvm_vcpu *vcpu;
1573 unsigned int vcpu_idx;
1574 u32 scaol, scaoh;
1575
1576 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1577 if (!new_sca)
1578 return -ENOMEM;
1579
1580 scaoh = (u32)((u64)(new_sca) >> 32);
1581 scaol = (u32)(u64)(new_sca) & ~0x3fU;
1582
1583 kvm_s390_vcpu_block_all(kvm);
1584 write_lock(&kvm->arch.sca_lock);
1585
1586 sca_copy_b_to_e(new_sca, old_sca);
1587
1588 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1589 vcpu->arch.sie_block->scaoh = scaoh;
1590 vcpu->arch.sie_block->scaol = scaol;
1591 vcpu->arch.sie_block->ecb2 |= 0x04U;
1592 }
1593 kvm->arch.sca = new_sca;
1594 kvm->arch.use_esca = 1;
1595
1596 write_unlock(&kvm->arch.sca_lock);
1597 kvm_s390_vcpu_unblock_all(kvm);
1598
1599 free_page((unsigned long)old_sca);
1600
1601 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1602 old_sca, kvm->arch.sca);
1603 return 0;
1604 }
1605
1606 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1607 {
1608 int rc;
1609
1610 if (id < KVM_S390_BSCA_CPU_SLOTS)
1611 return true;
1612 if (!sclp.has_esca || !sclp.has_64bscao)
1613 return false;
1614
1615 mutex_lock(&kvm->lock);
1616 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1617 mutex_unlock(&kvm->lock);
1618
1619 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
1620 }
1621
1622 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1623 {
1624 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1625 kvm_clear_async_pf_completion_queue(vcpu);
1626 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1627 KVM_SYNC_GPRS |
1628 KVM_SYNC_ACRS |
1629 KVM_SYNC_CRS |
1630 KVM_SYNC_ARCH0 |
1631 KVM_SYNC_PFAULT;
1632 if (test_kvm_facility(vcpu->kvm, 64))
1633 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
1634 /* fprs can be synchronized via vrs, even if the guest has no vx. With
1635 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1636 */
1637 if (MACHINE_HAS_VX)
1638 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1639 else
1640 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
1641
1642 if (kvm_is_ucontrol(vcpu->kvm))
1643 return __kvm_ucontrol_vcpu_init(vcpu);
1644
1645 return 0;
1646 }
1647
1648 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1649 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1650 {
1651 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
1652 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1653 vcpu->arch.cputm_start = get_tod_clock_fast();
1654 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1655 }
1656
1657 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1658 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1659 {
1660 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
1661 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1662 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1663 vcpu->arch.cputm_start = 0;
1664 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1665 }
1666
1667 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1668 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1669 {
1670 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
1671 vcpu->arch.cputm_enabled = true;
1672 __start_cpu_timer_accounting(vcpu);
1673 }
1674
1675 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1676 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1677 {
1678 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
1679 __stop_cpu_timer_accounting(vcpu);
1680 vcpu->arch.cputm_enabled = false;
1681 }
1682
1683 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1684 {
1685 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1686 __enable_cpu_timer_accounting(vcpu);
1687 preempt_enable();
1688 }
1689
1690 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1691 {
1692 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1693 __disable_cpu_timer_accounting(vcpu);
1694 preempt_enable();
1695 }
1696
1697 /* set the cpu timer - may only be called from the VCPU thread itself */
1698 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
1699 {
1700 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1701 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1702 if (vcpu->arch.cputm_enabled)
1703 vcpu->arch.cputm_start = get_tod_clock_fast();
1704 vcpu->arch.sie_block->cputm = cputm;
1705 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1706 preempt_enable();
1707 }
1708
1709 /* update and get the cpu timer - can also be called from other VCPU threads */
1710 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
1711 {
1712 unsigned int seq;
1713 __u64 value;
1714
1715 if (unlikely(!vcpu->arch.cputm_enabled))
1716 return vcpu->arch.sie_block->cputm;
1717
1718 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1719 do {
1720 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
1721 /*
1722 * If the writer would ever execute a read in the critical
1723 * section, e.g. in irq context, we have a deadlock.
1724 */
1725 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
1726 value = vcpu->arch.sie_block->cputm;
1727 /* if cputm_start is 0, accounting is being started/stopped */
1728 if (likely(vcpu->arch.cputm_start))
1729 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1730 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
1731 preempt_enable();
1732 return value;
1733 }
1734
1735 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1736 {
1737 /* Save host register state */
1738 save_fpu_regs();
1739 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
1740 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
1741
1742 if (MACHINE_HAS_VX)
1743 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
1744 else
1745 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
1746 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
1747 if (test_fp_ctl(current->thread.fpu.fpc))
1748 /* User space provided an invalid FPC, let's clear it */
1749 current->thread.fpu.fpc = 0;
1750
1751 save_access_regs(vcpu->arch.host_acrs);
1752 restore_access_regs(vcpu->run->s.regs.acrs);
1753 gmap_enable(vcpu->arch.enabled_gmap);
1754 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1755 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
1756 __start_cpu_timer_accounting(vcpu);
1757 vcpu->cpu = cpu;
1758 }
1759
1760 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1761 {
1762 vcpu->cpu = -1;
1763 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
1764 __stop_cpu_timer_accounting(vcpu);
1765 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1766 vcpu->arch.enabled_gmap = gmap_get_enabled();
1767 gmap_disable(vcpu->arch.enabled_gmap);
1768
1769 /* Save guest register state */
1770 save_fpu_regs();
1771 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
1772
1773 /* Restore host register state */
1774 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
1775 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
1776
1777 save_access_regs(vcpu->run->s.regs.acrs);
1778 restore_access_regs(vcpu->arch.host_acrs);
1779 }
1780
1781 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1782 {
1783 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1784 vcpu->arch.sie_block->gpsw.mask = 0UL;
1785 vcpu->arch.sie_block->gpsw.addr = 0UL;
1786 kvm_s390_set_prefix(vcpu, 0);
1787 kvm_s390_set_cpu_timer(vcpu, 0);
1788 vcpu->arch.sie_block->ckc = 0UL;
1789 vcpu->arch.sie_block->todpr = 0;
1790 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1791 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1792 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1793 /* make sure the new fpc will be lazily loaded */
1794 save_fpu_regs();
1795 current->thread.fpu.fpc = 0;
1796 vcpu->arch.sie_block->gbea = 1;
1797 vcpu->arch.sie_block->pp = 0;
1798 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1799 kvm_clear_async_pf_completion_queue(vcpu);
1800 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1801 kvm_s390_vcpu_stop(vcpu);
1802 kvm_s390_clear_local_irqs(vcpu);
1803 }
1804
1805 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1806 {
1807 mutex_lock(&vcpu->kvm->lock);
1808 preempt_disable();
1809 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1810 preempt_enable();
1811 mutex_unlock(&vcpu->kvm->lock);
1812 if (!kvm_is_ucontrol(vcpu->kvm)) {
1813 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1814 sca_add_vcpu(vcpu);
1815 }
1816 /* make vcpu_load load the right gmap on the first trigger */
1817 vcpu->arch.enabled_gmap = vcpu->arch.gmap;
1818 }
1819
1820 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1821 {
1822 if (!test_kvm_facility(vcpu->kvm, 76))
1823 return;
1824
1825 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1826
1827 if (vcpu->kvm->arch.crypto.aes_kw)
1828 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1829 if (vcpu->kvm->arch.crypto.dea_kw)
1830 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1831
1832 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1833 }
1834
1835 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1836 {
1837 free_page(vcpu->arch.sie_block->cbrlo);
1838 vcpu->arch.sie_block->cbrlo = 0;
1839 }
1840
1841 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1842 {
1843 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1844 if (!vcpu->arch.sie_block->cbrlo)
1845 return -ENOMEM;
1846
1847 vcpu->arch.sie_block->ecb2 |= 0x80;
1848 vcpu->arch.sie_block->ecb2 &= ~0x08;
1849 return 0;
1850 }
1851
1852 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1853 {
1854 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1855
1856 vcpu->arch.sie_block->ibc = model->ibc;
1857 if (test_kvm_facility(vcpu->kvm, 7))
1858 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
1859 }
1860
1861 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1862 {
1863 int rc = 0;
1864
1865 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1866 CPUSTAT_SM |
1867 CPUSTAT_STOPPED);
1868
1869 if (test_kvm_facility(vcpu->kvm, 78))
1870 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
1871 else if (test_kvm_facility(vcpu->kvm, 8))
1872 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
1873
1874 kvm_s390_vcpu_setup_model(vcpu);
1875
1876 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
1877 if (MACHINE_HAS_ESOP)
1878 vcpu->arch.sie_block->ecb |= 0x02;
1879 if (test_kvm_facility(vcpu->kvm, 9))
1880 vcpu->arch.sie_block->ecb |= 0x04;
1881 if (test_kvm_facility(vcpu->kvm, 73))
1882 vcpu->arch.sie_block->ecb |= 0x10;
1883
1884 if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
1885 vcpu->arch.sie_block->ecb2 |= 0x08;
1886 vcpu->arch.sie_block->eca = 0x1002000U;
1887 if (sclp.has_cei)
1888 vcpu->arch.sie_block->eca |= 0x80000000U;
1889 if (sclp.has_ib)
1890 vcpu->arch.sie_block->eca |= 0x40000000U;
1891 if (sclp.has_siif)
1892 vcpu->arch.sie_block->eca |= 1;
1893 if (sclp.has_sigpif)
1894 vcpu->arch.sie_block->eca |= 0x10000000U;
1895 if (test_kvm_facility(vcpu->kvm, 64))
1896 vcpu->arch.sie_block->ecb3 |= 0x01;
1897 if (test_kvm_facility(vcpu->kvm, 129)) {
1898 vcpu->arch.sie_block->eca |= 0x00020000;
1899 vcpu->arch.sie_block->ecd |= 0x20000000;
1900 }
1901 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
1902 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1903 if (test_kvm_facility(vcpu->kvm, 74))
1904 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
1905
1906 if (vcpu->kvm->arch.use_cmma) {
1907 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1908 if (rc)
1909 return rc;
1910 }
1911 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1912 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1913
1914 kvm_s390_vcpu_crypto_setup(vcpu);
1915
1916 return rc;
1917 }
1918
1919 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1920 unsigned int id)
1921 {
1922 struct kvm_vcpu *vcpu;
1923 struct sie_page *sie_page;
1924 int rc = -EINVAL;
1925
1926 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
1927 goto out;
1928
1929 rc = -ENOMEM;
1930
1931 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1932 if (!vcpu)
1933 goto out;
1934
1935 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1936 if (!sie_page)
1937 goto out_free_cpu;
1938
1939 vcpu->arch.sie_block = &sie_page->sie_block;
1940 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1941
1942 /* the real guest size will always be smaller than msl */
1943 vcpu->arch.sie_block->mso = 0;
1944 vcpu->arch.sie_block->msl = sclp.hamax;
1945
1946 vcpu->arch.sie_block->icpua = id;
1947 spin_lock_init(&vcpu->arch.local_int.lock);
1948 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1949 vcpu->arch.local_int.wq = &vcpu->wq;
1950 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1951 seqcount_init(&vcpu->arch.cputm_seqcount);
1952
1953 rc = kvm_vcpu_init(vcpu, kvm, id);
1954 if (rc)
1955 goto out_free_sie_block;
1956 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
1957 vcpu->arch.sie_block);
1958 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1959
1960 return vcpu;
1961 out_free_sie_block:
1962 free_page((unsigned long)(vcpu->arch.sie_block));
1963 out_free_cpu:
1964 kmem_cache_free(kvm_vcpu_cache, vcpu);
1965 out:
1966 return ERR_PTR(rc);
1967 }
1968
1969 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1970 {
1971 return kvm_s390_vcpu_has_irq(vcpu, 0);
1972 }
1973
1974 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
1975 {
1976 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1977 exit_sie(vcpu);
1978 }
1979
1980 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1981 {
1982 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1983 }
1984
1985 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1986 {
1987 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1988 exit_sie(vcpu);
1989 }
1990
1991 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1992 {
1993 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1994 }
1995
1996 /*
1997 * Kick a guest cpu out of SIE and wait until SIE is not running.
1998 * If the CPU is not running (e.g. waiting as idle) the function will
1999 * return immediately. */
2000 void exit_sie(struct kvm_vcpu *vcpu)
2001 {
2002 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
2003 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2004 cpu_relax();
2005 }
2006
2007 /* Kick a guest cpu out of SIE to process a request synchronously */
2008 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
2009 {
2010 kvm_make_request(req, vcpu);
2011 kvm_s390_vcpu_request(vcpu);
2012 }
2013
2014 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
2015 unsigned long end)
2016 {
2017 struct kvm *kvm = gmap->private;
2018 struct kvm_vcpu *vcpu;
2019 unsigned long prefix;
2020 int i;
2021
2022 if (gmap_is_shadow(gmap))
2023 return;
2024 if (start >= 1UL << 31)
2025 /* We are only interested in prefix pages */
2026 return;
2027 kvm_for_each_vcpu(i, vcpu, kvm) {
2028 /* match against both prefix pages */
2029 prefix = kvm_s390_get_prefix(vcpu);
2030 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
2031 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
2032 start, end);
2033 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2034 }
2035 }
2036 }
2037
2038 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2039 {
2040 /* kvm common code refers to this, but never calls it */
2041 BUG();
2042 return 0;
2043 }
2044
2045 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2046 struct kvm_one_reg *reg)
2047 {
2048 int r = -EINVAL;
2049
2050 switch (reg->id) {
2051 case KVM_REG_S390_TODPR:
2052 r = put_user(vcpu->arch.sie_block->todpr,
2053 (u32 __user *)reg->addr);
2054 break;
2055 case KVM_REG_S390_EPOCHDIFF:
2056 r = put_user(vcpu->arch.sie_block->epoch,
2057 (u64 __user *)reg->addr);
2058 break;
2059 case KVM_REG_S390_CPU_TIMER:
2060 r = put_user(kvm_s390_get_cpu_timer(vcpu),
2061 (u64 __user *)reg->addr);
2062 break;
2063 case KVM_REG_S390_CLOCK_COMP:
2064 r = put_user(vcpu->arch.sie_block->ckc,
2065 (u64 __user *)reg->addr);
2066 break;
2067 case KVM_REG_S390_PFTOKEN:
2068 r = put_user(vcpu->arch.pfault_token,
2069 (u64 __user *)reg->addr);
2070 break;
2071 case KVM_REG_S390_PFCOMPARE:
2072 r = put_user(vcpu->arch.pfault_compare,
2073 (u64 __user *)reg->addr);
2074 break;
2075 case KVM_REG_S390_PFSELECT:
2076 r = put_user(vcpu->arch.pfault_select,
2077 (u64 __user *)reg->addr);
2078 break;
2079 case KVM_REG_S390_PP:
2080 r = put_user(vcpu->arch.sie_block->pp,
2081 (u64 __user *)reg->addr);
2082 break;
2083 case KVM_REG_S390_GBEA:
2084 r = put_user(vcpu->arch.sie_block->gbea,
2085 (u64 __user *)reg->addr);
2086 break;
2087 default:
2088 break;
2089 }
2090
2091 return r;
2092 }
2093
2094 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2095 struct kvm_one_reg *reg)
2096 {
2097 int r = -EINVAL;
2098 __u64 val;
2099
2100 switch (reg->id) {
2101 case KVM_REG_S390_TODPR:
2102 r = get_user(vcpu->arch.sie_block->todpr,
2103 (u32 __user *)reg->addr);
2104 break;
2105 case KVM_REG_S390_EPOCHDIFF:
2106 r = get_user(vcpu->arch.sie_block->epoch,
2107 (u64 __user *)reg->addr);
2108 break;
2109 case KVM_REG_S390_CPU_TIMER:
2110 r = get_user(val, (u64 __user *)reg->addr);
2111 if (!r)
2112 kvm_s390_set_cpu_timer(vcpu, val);
2113 break;
2114 case KVM_REG_S390_CLOCK_COMP:
2115 r = get_user(vcpu->arch.sie_block->ckc,
2116 (u64 __user *)reg->addr);
2117 break;
2118 case KVM_REG_S390_PFTOKEN:
2119 r = get_user(vcpu->arch.pfault_token,
2120 (u64 __user *)reg->addr);
2121 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2122 kvm_clear_async_pf_completion_queue(vcpu);
2123 break;
2124 case KVM_REG_S390_PFCOMPARE:
2125 r = get_user(vcpu->arch.pfault_compare,
2126 (u64 __user *)reg->addr);
2127 break;
2128 case KVM_REG_S390_PFSELECT:
2129 r = get_user(vcpu->arch.pfault_select,
2130 (u64 __user *)reg->addr);
2131 break;
2132 case KVM_REG_S390_PP:
2133 r = get_user(vcpu->arch.sie_block->pp,
2134 (u64 __user *)reg->addr);
2135 break;
2136 case KVM_REG_S390_GBEA:
2137 r = get_user(vcpu->arch.sie_block->gbea,
2138 (u64 __user *)reg->addr);
2139 break;
2140 default:
2141 break;
2142 }
2143
2144 return r;
2145 }
2146
2147 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2148 {
2149 kvm_s390_vcpu_initial_reset(vcpu);
2150 return 0;
2151 }
2152
2153 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2154 {
2155 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
2156 return 0;
2157 }
2158
2159 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2160 {
2161 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
2162 return 0;
2163 }
2164
2165 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2166 struct kvm_sregs *sregs)
2167 {
2168 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
2169 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
2170 restore_access_regs(vcpu->run->s.regs.acrs);
2171 return 0;
2172 }
2173
2174 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2175 struct kvm_sregs *sregs)
2176 {
2177 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
2178 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
2179 return 0;
2180 }
2181
2182 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2183 {
2184 /* make sure the new values will be lazily loaded */
2185 save_fpu_regs();
2186 if (test_fp_ctl(fpu->fpc))
2187 return -EINVAL;
2188 current->thread.fpu.fpc = fpu->fpc;
2189 if (MACHINE_HAS_VX)
2190 convert_fp_to_vx(current->thread.fpu.vxrs, (freg_t *)fpu->fprs);
2191 else
2192 memcpy(current->thread.fpu.fprs, &fpu->fprs, sizeof(fpu->fprs));
2193 return 0;
2194 }
2195
2196 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2197 {
2198 /* make sure we have the latest values */
2199 save_fpu_regs();
2200 if (MACHINE_HAS_VX)
2201 convert_vx_to_fp((freg_t *)fpu->fprs, current->thread.fpu.vxrs);
2202 else
2203 memcpy(fpu->fprs, current->thread.fpu.fprs, sizeof(fpu->fprs));
2204 fpu->fpc = current->thread.fpu.fpc;
2205 return 0;
2206 }
2207
2208 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2209 {
2210 int rc = 0;
2211
2212 if (!is_vcpu_stopped(vcpu))
2213 rc = -EBUSY;
2214 else {
2215 vcpu->run->psw_mask = psw.mask;
2216 vcpu->run->psw_addr = psw.addr;
2217 }
2218 return rc;
2219 }
2220
2221 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2222 struct kvm_translation *tr)
2223 {
2224 return -EINVAL; /* not implemented yet */
2225 }
2226
2227 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2228 KVM_GUESTDBG_USE_HW_BP | \
2229 KVM_GUESTDBG_ENABLE)
2230
2231 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2232 struct kvm_guest_debug *dbg)
2233 {
2234 int rc = 0;
2235
2236 vcpu->guest_debug = 0;
2237 kvm_s390_clear_bp_data(vcpu);
2238
2239 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
2240 return -EINVAL;
2241 if (!sclp.has_gpere)
2242 return -EINVAL;
2243
2244 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2245 vcpu->guest_debug = dbg->control;
2246 /* enforce guest PER */
2247 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2248
2249 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2250 rc = kvm_s390_import_bp_data(vcpu, dbg);
2251 } else {
2252 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2253 vcpu->arch.guestdbg.last_bp = 0;
2254 }
2255
2256 if (rc) {
2257 vcpu->guest_debug = 0;
2258 kvm_s390_clear_bp_data(vcpu);
2259 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2260 }
2261
2262 return rc;
2263 }
2264
2265 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2266 struct kvm_mp_state *mp_state)
2267 {
2268 /* CHECK_STOP and LOAD are not supported yet */
2269 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2270 KVM_MP_STATE_OPERATING;
2271 }
2272
2273 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2274 struct kvm_mp_state *mp_state)
2275 {
2276 int rc = 0;
2277
2278 /* user space knows about this interface - let it control the state */
2279 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2280
2281 switch (mp_state->mp_state) {
2282 case KVM_MP_STATE_STOPPED:
2283 kvm_s390_vcpu_stop(vcpu);
2284 break;
2285 case KVM_MP_STATE_OPERATING:
2286 kvm_s390_vcpu_start(vcpu);
2287 break;
2288 case KVM_MP_STATE_LOAD:
2289 case KVM_MP_STATE_CHECK_STOP:
2290 /* fall through - CHECK_STOP and LOAD are not supported yet */
2291 default:
2292 rc = -ENXIO;
2293 }
2294
2295 return rc;
2296 }
2297
2298 static bool ibs_enabled(struct kvm_vcpu *vcpu)
2299 {
2300 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2301 }
2302
2303 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2304 {
2305 retry:
2306 kvm_s390_vcpu_request_handled(vcpu);
2307 if (!vcpu->requests)
2308 return 0;
2309 /*
2310 * We use MMU_RELOAD just to re-arm the ipte notifier for the
2311 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2312 * This ensures that the ipte instruction for this request has
2313 * already finished. We might race against a second unmapper that
2314 * wants to set the blocking bit. Lets just retry the request loop.
2315 */
2316 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2317 int rc;
2318 rc = gmap_mprotect_notify(vcpu->arch.gmap,
2319 kvm_s390_get_prefix(vcpu),
2320 PAGE_SIZE * 2, PROT_WRITE);
2321 if (rc)
2322 return rc;
2323 goto retry;
2324 }
2325
2326 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2327 vcpu->arch.sie_block->ihcpu = 0xffff;
2328 goto retry;
2329 }
2330
2331 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2332 if (!ibs_enabled(vcpu)) {
2333 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
2334 atomic_or(CPUSTAT_IBS,
2335 &vcpu->arch.sie_block->cpuflags);
2336 }
2337 goto retry;
2338 }
2339
2340 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2341 if (ibs_enabled(vcpu)) {
2342 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
2343 atomic_andnot(CPUSTAT_IBS,
2344 &vcpu->arch.sie_block->cpuflags);
2345 }
2346 goto retry;
2347 }
2348
2349 /* nothing to do, just clear the request */
2350 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2351
2352 return 0;
2353 }
2354
2355 void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2356 {
2357 struct kvm_vcpu *vcpu;
2358 int i;
2359
2360 mutex_lock(&kvm->lock);
2361 preempt_disable();
2362 kvm->arch.epoch = tod - get_tod_clock();
2363 kvm_s390_vcpu_block_all(kvm);
2364 kvm_for_each_vcpu(i, vcpu, kvm)
2365 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2366 kvm_s390_vcpu_unblock_all(kvm);
2367 preempt_enable();
2368 mutex_unlock(&kvm->lock);
2369 }
2370
2371 /**
2372 * kvm_arch_fault_in_page - fault-in guest page if necessary
2373 * @vcpu: The corresponding virtual cpu
2374 * @gpa: Guest physical address
2375 * @writable: Whether the page should be writable or not
2376 *
2377 * Make sure that a guest page has been faulted-in on the host.
2378 *
2379 * Return: Zero on success, negative error code otherwise.
2380 */
2381 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
2382 {
2383 return gmap_fault(vcpu->arch.gmap, gpa,
2384 writable ? FAULT_FLAG_WRITE : 0);
2385 }
2386
2387 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2388 unsigned long token)
2389 {
2390 struct kvm_s390_interrupt inti;
2391 struct kvm_s390_irq irq;
2392
2393 if (start_token) {
2394 irq.u.ext.ext_params2 = token;
2395 irq.type = KVM_S390_INT_PFAULT_INIT;
2396 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
2397 } else {
2398 inti.type = KVM_S390_INT_PFAULT_DONE;
2399 inti.parm64 = token;
2400 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2401 }
2402 }
2403
2404 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2405 struct kvm_async_pf *work)
2406 {
2407 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2408 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2409 }
2410
2411 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2412 struct kvm_async_pf *work)
2413 {
2414 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2415 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2416 }
2417
2418 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2419 struct kvm_async_pf *work)
2420 {
2421 /* s390 will always inject the page directly */
2422 }
2423
2424 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2425 {
2426 /*
2427 * s390 will always inject the page directly,
2428 * but we still want check_async_completion to cleanup
2429 */
2430 return true;
2431 }
2432
2433 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2434 {
2435 hva_t hva;
2436 struct kvm_arch_async_pf arch;
2437 int rc;
2438
2439 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2440 return 0;
2441 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2442 vcpu->arch.pfault_compare)
2443 return 0;
2444 if (psw_extint_disabled(vcpu))
2445 return 0;
2446 if (kvm_s390_vcpu_has_irq(vcpu, 0))
2447 return 0;
2448 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2449 return 0;
2450 if (!vcpu->arch.gmap->pfault_enabled)
2451 return 0;
2452
2453 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2454 hva += current->thread.gmap_addr & ~PAGE_MASK;
2455 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
2456 return 0;
2457
2458 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2459 return rc;
2460 }
2461
2462 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
2463 {
2464 int rc, cpuflags;
2465
2466 /*
2467 * On s390 notifications for arriving pages will be delivered directly
2468 * to the guest but the house keeping for completed pfaults is
2469 * handled outside the worker.
2470 */
2471 kvm_check_async_pf_completion(vcpu);
2472
2473 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2474 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
2475
2476 if (need_resched())
2477 schedule();
2478
2479 if (test_cpu_flag(CIF_MCCK_PENDING))
2480 s390_handle_mcck();
2481
2482 if (!kvm_is_ucontrol(vcpu->kvm)) {
2483 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2484 if (rc)
2485 return rc;
2486 }
2487
2488 rc = kvm_s390_handle_requests(vcpu);
2489 if (rc)
2490 return rc;
2491
2492 if (guestdbg_enabled(vcpu)) {
2493 kvm_s390_backup_guest_per_regs(vcpu);
2494 kvm_s390_patch_guest_per_regs(vcpu);
2495 }
2496
2497 vcpu->arch.sie_block->icptcode = 0;
2498 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2499 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2500 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2501
2502 return 0;
2503 }
2504
2505 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2506 {
2507 struct kvm_s390_pgm_info pgm_info = {
2508 .code = PGM_ADDRESSING,
2509 };
2510 u8 opcode, ilen;
2511 int rc;
2512
2513 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2514 trace_kvm_s390_sie_fault(vcpu);
2515
2516 /*
2517 * We want to inject an addressing exception, which is defined as a
2518 * suppressing or terminating exception. However, since we came here
2519 * by a DAT access exception, the PSW still points to the faulting
2520 * instruction since DAT exceptions are nullifying. So we've got
2521 * to look up the current opcode to get the length of the instruction
2522 * to be able to forward the PSW.
2523 */
2524 rc = read_guest_instr(vcpu, &opcode, 1);
2525 ilen = insn_length(opcode);
2526 if (rc < 0) {
2527 return rc;
2528 } else if (rc) {
2529 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2530 * Forward by arbitrary ilc, injection will take care of
2531 * nullification if necessary.
2532 */
2533 pgm_info = vcpu->arch.pgm;
2534 ilen = 4;
2535 }
2536 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2537 kvm_s390_forward_psw(vcpu, ilen);
2538 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
2539 }
2540
2541 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2542 {
2543 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2544 vcpu->arch.sie_block->icptcode);
2545 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2546
2547 if (guestdbg_enabled(vcpu))
2548 kvm_s390_restore_guest_per_regs(vcpu);
2549
2550 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2551 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
2552
2553 if (vcpu->arch.sie_block->icptcode > 0) {
2554 int rc = kvm_handle_sie_intercept(vcpu);
2555
2556 if (rc != -EOPNOTSUPP)
2557 return rc;
2558 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2559 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2560 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2561 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2562 return -EREMOTE;
2563 } else if (exit_reason != -EFAULT) {
2564 vcpu->stat.exit_null++;
2565 return 0;
2566 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2567 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2568 vcpu->run->s390_ucontrol.trans_exc_code =
2569 current->thread.gmap_addr;
2570 vcpu->run->s390_ucontrol.pgm_code = 0x10;
2571 return -EREMOTE;
2572 } else if (current->thread.gmap_pfault) {
2573 trace_kvm_s390_major_guest_pfault(vcpu);
2574 current->thread.gmap_pfault = 0;
2575 if (kvm_arch_setup_async_pf(vcpu))
2576 return 0;
2577 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
2578 }
2579 return vcpu_post_run_fault_in_sie(vcpu);
2580 }
2581
2582 static int __vcpu_run(struct kvm_vcpu *vcpu)
2583 {
2584 int rc, exit_reason;
2585
2586 /*
2587 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2588 * ning the guest), so that memslots (and other stuff) are protected
2589 */
2590 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2591
2592 do {
2593 rc = vcpu_pre_run(vcpu);
2594 if (rc)
2595 break;
2596
2597 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2598 /*
2599 * As PF_VCPU will be used in fault handler, between
2600 * guest_enter and guest_exit should be no uaccess.
2601 */
2602 local_irq_disable();
2603 __kvm_guest_enter();
2604 __disable_cpu_timer_accounting(vcpu);
2605 local_irq_enable();
2606 exit_reason = sie64a(vcpu->arch.sie_block,
2607 vcpu->run->s.regs.gprs);
2608 local_irq_disable();
2609 __enable_cpu_timer_accounting(vcpu);
2610 __kvm_guest_exit();
2611 local_irq_enable();
2612 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2613
2614 rc = vcpu_post_run(vcpu, exit_reason);
2615 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2616
2617 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2618 return rc;
2619 }
2620
2621 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2622 {
2623 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2624 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2625 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2626 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2627 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2628 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2629 /* some control register changes require a tlb flush */
2630 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2631 }
2632 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2633 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
2634 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2635 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2636 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2637 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2638 }
2639 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2640 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2641 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2642 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2643 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2644 kvm_clear_async_pf_completion_queue(vcpu);
2645 }
2646 kvm_run->kvm_dirty_regs = 0;
2647 }
2648
2649 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2650 {
2651 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2652 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2653 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2654 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2655 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
2656 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2657 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2658 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2659 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2660 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2661 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2662 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2663 }
2664
2665 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2666 {
2667 int rc;
2668 sigset_t sigsaved;
2669
2670 if (guestdbg_exit_pending(vcpu)) {
2671 kvm_s390_prepare_debug_exit(vcpu);
2672 return 0;
2673 }
2674
2675 if (vcpu->sigset_active)
2676 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2677
2678 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2679 kvm_s390_vcpu_start(vcpu);
2680 } else if (is_vcpu_stopped(vcpu)) {
2681 pr_err_ratelimited("can't run stopped vcpu %d\n",
2682 vcpu->vcpu_id);
2683 return -EINVAL;
2684 }
2685
2686 sync_regs(vcpu, kvm_run);
2687 enable_cpu_timer_accounting(vcpu);
2688
2689 might_fault();
2690 rc = __vcpu_run(vcpu);
2691
2692 if (signal_pending(current) && !rc) {
2693 kvm_run->exit_reason = KVM_EXIT_INTR;
2694 rc = -EINTR;
2695 }
2696
2697 if (guestdbg_exit_pending(vcpu) && !rc) {
2698 kvm_s390_prepare_debug_exit(vcpu);
2699 rc = 0;
2700 }
2701
2702 if (rc == -EREMOTE) {
2703 /* userspace support is needed, kvm_run has been prepared */
2704 rc = 0;
2705 }
2706
2707 disable_cpu_timer_accounting(vcpu);
2708 store_regs(vcpu, kvm_run);
2709
2710 if (vcpu->sigset_active)
2711 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2712
2713 vcpu->stat.exit_userspace++;
2714 return rc;
2715 }
2716
2717 /*
2718 * store status at address
2719 * we use have two special cases:
2720 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2721 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2722 */
2723 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2724 {
2725 unsigned char archmode = 1;
2726 freg_t fprs[NUM_FPRS];
2727 unsigned int px;
2728 u64 clkcomp, cputm;
2729 int rc;
2730
2731 px = kvm_s390_get_prefix(vcpu);
2732 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2733 if (write_guest_abs(vcpu, 163, &archmode, 1))
2734 return -EFAULT;
2735 gpa = 0;
2736 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2737 if (write_guest_real(vcpu, 163, &archmode, 1))
2738 return -EFAULT;
2739 gpa = px;
2740 } else
2741 gpa -= __LC_FPREGS_SAVE_AREA;
2742
2743 /* manually convert vector registers if necessary */
2744 if (MACHINE_HAS_VX) {
2745 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
2746 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2747 fprs, 128);
2748 } else {
2749 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2750 vcpu->run->s.regs.fprs, 128);
2751 }
2752 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
2753 vcpu->run->s.regs.gprs, 128);
2754 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
2755 &vcpu->arch.sie_block->gpsw, 16);
2756 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
2757 &px, 4);
2758 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
2759 &vcpu->run->s.regs.fpc, 4);
2760 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
2761 &vcpu->arch.sie_block->todpr, 4);
2762 cputm = kvm_s390_get_cpu_timer(vcpu);
2763 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
2764 &cputm, 8);
2765 clkcomp = vcpu->arch.sie_block->ckc >> 8;
2766 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
2767 &clkcomp, 8);
2768 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
2769 &vcpu->run->s.regs.acrs, 64);
2770 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
2771 &vcpu->arch.sie_block->gcr, 128);
2772 return rc ? -EFAULT : 0;
2773 }
2774
2775 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2776 {
2777 /*
2778 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2779 * copying in vcpu load/put. Lets update our copies before we save
2780 * it into the save area
2781 */
2782 save_fpu_regs();
2783 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
2784 save_access_regs(vcpu->run->s.regs.acrs);
2785
2786 return kvm_s390_store_status_unloaded(vcpu, addr);
2787 }
2788
2789 /*
2790 * store additional status at address
2791 */
2792 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2793 unsigned long gpa)
2794 {
2795 /* Only bits 0-53 are used for address formation */
2796 if (!(gpa & ~0x3ff))
2797 return 0;
2798
2799 return write_guest_abs(vcpu, gpa & ~0x3ff,
2800 (void *)&vcpu->run->s.regs.vrs, 512);
2801 }
2802
2803 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2804 {
2805 if (!test_kvm_facility(vcpu->kvm, 129))
2806 return 0;
2807
2808 /*
2809 * The guest VXRS are in the host VXRs due to the lazy
2810 * copying in vcpu load/put. We can simply call save_fpu_regs()
2811 * to save the current register state because we are in the
2812 * middle of a load/put cycle.
2813 *
2814 * Let's update our copies before we save it into the save area.
2815 */
2816 save_fpu_regs();
2817
2818 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2819 }
2820
2821 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2822 {
2823 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2824 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
2825 }
2826
2827 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2828 {
2829 unsigned int i;
2830 struct kvm_vcpu *vcpu;
2831
2832 kvm_for_each_vcpu(i, vcpu, kvm) {
2833 __disable_ibs_on_vcpu(vcpu);
2834 }
2835 }
2836
2837 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2838 {
2839 if (!sclp.has_ibs)
2840 return;
2841 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2842 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
2843 }
2844
2845 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2846 {
2847 int i, online_vcpus, started_vcpus = 0;
2848
2849 if (!is_vcpu_stopped(vcpu))
2850 return;
2851
2852 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2853 /* Only one cpu at a time may enter/leave the STOPPED state. */
2854 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2855 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2856
2857 for (i = 0; i < online_vcpus; i++) {
2858 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2859 started_vcpus++;
2860 }
2861
2862 if (started_vcpus == 0) {
2863 /* we're the only active VCPU -> speed it up */
2864 __enable_ibs_on_vcpu(vcpu);
2865 } else if (started_vcpus == 1) {
2866 /*
2867 * As we are starting a second VCPU, we have to disable
2868 * the IBS facility on all VCPUs to remove potentially
2869 * oustanding ENABLE requests.
2870 */
2871 __disable_ibs_on_all_vcpus(vcpu->kvm);
2872 }
2873
2874 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2875 /*
2876 * Another VCPU might have used IBS while we were offline.
2877 * Let's play safe and flush the VCPU at startup.
2878 */
2879 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2880 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2881 return;
2882 }
2883
2884 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2885 {
2886 int i, online_vcpus, started_vcpus = 0;
2887 struct kvm_vcpu *started_vcpu = NULL;
2888
2889 if (is_vcpu_stopped(vcpu))
2890 return;
2891
2892 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2893 /* Only one cpu at a time may enter/leave the STOPPED state. */
2894 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2895 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2896
2897 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2898 kvm_s390_clear_stop_irq(vcpu);
2899
2900 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2901 __disable_ibs_on_vcpu(vcpu);
2902
2903 for (i = 0; i < online_vcpus; i++) {
2904 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2905 started_vcpus++;
2906 started_vcpu = vcpu->kvm->vcpus[i];
2907 }
2908 }
2909
2910 if (started_vcpus == 1) {
2911 /*
2912 * As we only have one VCPU left, we want to enable the
2913 * IBS facility for that VCPU to speed it up.
2914 */
2915 __enable_ibs_on_vcpu(started_vcpu);
2916 }
2917
2918 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2919 return;
2920 }
2921
2922 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2923 struct kvm_enable_cap *cap)
2924 {
2925 int r;
2926
2927 if (cap->flags)
2928 return -EINVAL;
2929
2930 switch (cap->cap) {
2931 case KVM_CAP_S390_CSS_SUPPORT:
2932 if (!vcpu->kvm->arch.css_support) {
2933 vcpu->kvm->arch.css_support = 1;
2934 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
2935 trace_kvm_s390_enable_css(vcpu->kvm);
2936 }
2937 r = 0;
2938 break;
2939 default:
2940 r = -EINVAL;
2941 break;
2942 }
2943 return r;
2944 }
2945
2946 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2947 struct kvm_s390_mem_op *mop)
2948 {
2949 void __user *uaddr = (void __user *)mop->buf;
2950 void *tmpbuf = NULL;
2951 int r, srcu_idx;
2952 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2953 | KVM_S390_MEMOP_F_CHECK_ONLY;
2954
2955 if (mop->flags & ~supported_flags)
2956 return -EINVAL;
2957
2958 if (mop->size > MEM_OP_MAX_SIZE)
2959 return -E2BIG;
2960
2961 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2962 tmpbuf = vmalloc(mop->size);
2963 if (!tmpbuf)
2964 return -ENOMEM;
2965 }
2966
2967 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2968
2969 switch (mop->op) {
2970 case KVM_S390_MEMOP_LOGICAL_READ:
2971 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2972 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2973 mop->size, GACC_FETCH);
2974 break;
2975 }
2976 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2977 if (r == 0) {
2978 if (copy_to_user(uaddr, tmpbuf, mop->size))
2979 r = -EFAULT;
2980 }
2981 break;
2982 case KVM_S390_MEMOP_LOGICAL_WRITE:
2983 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2984 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2985 mop->size, GACC_STORE);
2986 break;
2987 }
2988 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2989 r = -EFAULT;
2990 break;
2991 }
2992 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2993 break;
2994 default:
2995 r = -EINVAL;
2996 }
2997
2998 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2999
3000 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
3001 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
3002
3003 vfree(tmpbuf);
3004 return r;
3005 }
3006
3007 long kvm_arch_vcpu_ioctl(struct file *filp,
3008 unsigned int ioctl, unsigned long arg)
3009 {
3010 struct kvm_vcpu *vcpu = filp->private_data;
3011 void __user *argp = (void __user *)arg;
3012 int idx;
3013 long r;
3014
3015 switch (ioctl) {
3016 case KVM_S390_IRQ: {
3017 struct kvm_s390_irq s390irq;
3018
3019 r = -EFAULT;
3020 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
3021 break;
3022 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3023 break;
3024 }
3025 case KVM_S390_INTERRUPT: {
3026 struct kvm_s390_interrupt s390int;
3027 struct kvm_s390_irq s390irq;
3028
3029 r = -EFAULT;
3030 if (copy_from_user(&s390int, argp, sizeof(s390int)))
3031 break;
3032 if (s390int_to_s390irq(&s390int, &s390irq))
3033 return -EINVAL;
3034 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3035 break;
3036 }
3037 case KVM_S390_STORE_STATUS:
3038 idx = srcu_read_lock(&vcpu->kvm->srcu);
3039 r = kvm_s390_vcpu_store_status(vcpu, arg);
3040 srcu_read_unlock(&vcpu->kvm->srcu, idx);
3041 break;
3042 case KVM_S390_SET_INITIAL_PSW: {
3043 psw_t psw;
3044
3045 r = -EFAULT;
3046 if (copy_from_user(&psw, argp, sizeof(psw)))
3047 break;
3048 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3049 break;
3050 }
3051 case KVM_S390_INITIAL_RESET:
3052 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3053 break;
3054 case KVM_SET_ONE_REG:
3055 case KVM_GET_ONE_REG: {
3056 struct kvm_one_reg reg;
3057 r = -EFAULT;
3058 if (copy_from_user(&reg, argp, sizeof(reg)))
3059 break;
3060 if (ioctl == KVM_SET_ONE_REG)
3061 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3062 else
3063 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3064 break;
3065 }
3066 #ifdef CONFIG_KVM_S390_UCONTROL
3067 case KVM_S390_UCAS_MAP: {
3068 struct kvm_s390_ucas_mapping ucasmap;
3069
3070 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3071 r = -EFAULT;
3072 break;
3073 }
3074
3075 if (!kvm_is_ucontrol(vcpu->kvm)) {
3076 r = -EINVAL;
3077 break;
3078 }
3079
3080 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3081 ucasmap.vcpu_addr, ucasmap.length);
3082 break;
3083 }
3084 case KVM_S390_UCAS_UNMAP: {
3085 struct kvm_s390_ucas_mapping ucasmap;
3086
3087 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3088 r = -EFAULT;
3089 break;
3090 }
3091
3092 if (!kvm_is_ucontrol(vcpu->kvm)) {
3093 r = -EINVAL;
3094 break;
3095 }
3096
3097 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3098 ucasmap.length);
3099 break;
3100 }
3101 #endif
3102 case KVM_S390_VCPU_FAULT: {
3103 r = gmap_fault(vcpu->arch.gmap, arg, 0);
3104 break;
3105 }
3106 case KVM_ENABLE_CAP:
3107 {
3108 struct kvm_enable_cap cap;
3109 r = -EFAULT;
3110 if (copy_from_user(&cap, argp, sizeof(cap)))
3111 break;
3112 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3113 break;
3114 }
3115 case KVM_S390_MEM_OP: {
3116 struct kvm_s390_mem_op mem_op;
3117
3118 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3119 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3120 else
3121 r = -EFAULT;
3122 break;
3123 }
3124 case KVM_S390_SET_IRQ_STATE: {
3125 struct kvm_s390_irq_state irq_state;
3126
3127 r = -EFAULT;
3128 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3129 break;
3130 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3131 irq_state.len == 0 ||
3132 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3133 r = -EINVAL;
3134 break;
3135 }
3136 r = kvm_s390_set_irq_state(vcpu,
3137 (void __user *) irq_state.buf,
3138 irq_state.len);
3139 break;
3140 }
3141 case KVM_S390_GET_IRQ_STATE: {
3142 struct kvm_s390_irq_state irq_state;
3143
3144 r = -EFAULT;
3145 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3146 break;
3147 if (irq_state.len == 0) {
3148 r = -EINVAL;
3149 break;
3150 }
3151 r = kvm_s390_get_irq_state(vcpu,
3152 (__u8 __user *) irq_state.buf,
3153 irq_state.len);
3154 break;
3155 }
3156 default:
3157 r = -ENOTTY;
3158 }
3159 return r;
3160 }
3161
3162 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3163 {
3164 #ifdef CONFIG_KVM_S390_UCONTROL
3165 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3166 && (kvm_is_ucontrol(vcpu->kvm))) {
3167 vmf->page = virt_to_page(vcpu->arch.sie_block);
3168 get_page(vmf->page);
3169 return 0;
3170 }
3171 #endif
3172 return VM_FAULT_SIGBUS;
3173 }
3174
3175 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3176 unsigned long npages)
3177 {
3178 return 0;
3179 }
3180
3181 /* Section: memory related */
3182 int kvm_arch_prepare_memory_region(struct kvm *kvm,
3183 struct kvm_memory_slot *memslot,
3184 const struct kvm_userspace_memory_region *mem,
3185 enum kvm_mr_change change)
3186 {
3187 /* A few sanity checks. We can have memory slots which have to be
3188 located/ended at a segment boundary (1MB). The memory in userland is
3189 ok to be fragmented into various different vmas. It is okay to mmap()
3190 and munmap() stuff in this slot after doing this call at any time */
3191
3192 if (mem->userspace_addr & 0xffffful)
3193 return -EINVAL;
3194
3195 if (mem->memory_size & 0xffffful)
3196 return -EINVAL;
3197
3198 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3199 return -EINVAL;
3200
3201 return 0;
3202 }
3203
3204 void kvm_arch_commit_memory_region(struct kvm *kvm,
3205 const struct kvm_userspace_memory_region *mem,
3206 const struct kvm_memory_slot *old,
3207 const struct kvm_memory_slot *new,
3208 enum kvm_mr_change change)
3209 {
3210 int rc;
3211
3212 /* If the basics of the memslot do not change, we do not want
3213 * to update the gmap. Every update causes several unnecessary
3214 * segment translation exceptions. This is usually handled just
3215 * fine by the normal fault handler + gmap, but it will also
3216 * cause faults on the prefix page of running guest CPUs.
3217 */
3218 if (old->userspace_addr == mem->userspace_addr &&
3219 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3220 old->npages * PAGE_SIZE == mem->memory_size)
3221 return;
3222
3223 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3224 mem->guest_phys_addr, mem->memory_size);
3225 if (rc)
3226 pr_warn("failed to commit memory region\n");
3227 return;
3228 }
3229
3230 static inline unsigned long nonhyp_mask(int i)
3231 {
3232 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3233
3234 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3235 }
3236
3237 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3238 {
3239 vcpu->valid_wakeup = false;
3240 }
3241
3242 static int __init kvm_s390_init(void)
3243 {
3244 int i;
3245
3246 if (!sclp.has_sief2) {
3247 pr_info("SIE not available\n");
3248 return -ENODEV;
3249 }
3250
3251 for (i = 0; i < 16; i++)
3252 kvm_s390_fac_list_mask[i] |=
3253 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3254
3255 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
3256 }
3257
3258 static void __exit kvm_s390_exit(void)
3259 {
3260 kvm_exit();
3261 }
3262
3263 module_init(kvm_s390_init);
3264 module_exit(kvm_s390_exit);
3265
3266 /*
3267 * Enable autoloading of the kvm module.
3268 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3269 * since x86 takes a different approach.
3270 */
3271 #include <linux/miscdevice.h>
3272 MODULE_ALIAS_MISCDEV(KVM_MINOR);
3273 MODULE_ALIAS("devname:kvm");
Ubuntu Jammy Linux kernel mirror