]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blob - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-artful-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'ntb-4.13-bugfixes' of git://github.com/jonmason/ntb
[mirror_ubuntu-artful-kernel.git] / arch / s390 / kvm / kvm-s390.c
1 /*
2 * hosting zSeries kernel virtual machines
3 *
4 * Copyright IBM Corp. 2008, 2009
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
15 */
16
17 #include <linux/compiler.h>
18 #include <linux/err.h>
19 #include <linux/fs.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/mman.h>
25 #include <linux/module.h>
26 #include <linux/moduleparam.h>
27 #include <linux/random.h>
28 #include <linux/slab.h>
29 #include <linux/timer.h>
30 #include <linux/vmalloc.h>
31 #include <linux/bitmap.h>
32 #include <linux/sched/signal.h>
33 #include <linux/string.h>
34
35 #include <asm/asm-offsets.h>
36 #include <asm/lowcore.h>
37 #include <asm/stp.h>
38 #include <asm/pgtable.h>
39 #include <asm/gmap.h>
40 #include <asm/nmi.h>
41 #include <asm/switch_to.h>
42 #include <asm/isc.h>
43 #include <asm/sclp.h>
44 #include <asm/cpacf.h>
45 #include <asm/timex.h>
46 #include "kvm-s390.h"
47 #include "gaccess.h"
48
49 #define KMSG_COMPONENT "kvm-s390"
50 #undef pr_fmt
51 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
52
53 #define CREATE_TRACE_POINTS
54 #include "trace.h"
55 #include "trace-s390.h"
56
57 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
58 #define LOCAL_IRQS 32
59 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
60 (KVM_MAX_VCPUS + LOCAL_IRQS))
61
62 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
63
64 struct kvm_stats_debugfs_item debugfs_entries[] = {
65 { "userspace_handled", VCPU_STAT(exit_userspace) },
66 { "exit_null", VCPU_STAT(exit_null) },
67 { "exit_validity", VCPU_STAT(exit_validity) },
68 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
69 { "exit_external_request", VCPU_STAT(exit_external_request) },
70 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
71 { "exit_instruction", VCPU_STAT(exit_instruction) },
72 { "exit_pei", VCPU_STAT(exit_pei) },
73 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
74 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
75 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
76 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
77 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
78 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
79 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
80 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
81 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
82 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
83 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
84 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
85 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
86 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
87 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
88 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
89 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
90 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
91 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
92 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
93 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
94 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
95 { "instruction_spx", VCPU_STAT(instruction_spx) },
96 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
97 { "instruction_stap", VCPU_STAT(instruction_stap) },
98 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
99 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
100 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
101 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
102 { "instruction_essa", VCPU_STAT(instruction_essa) },
103 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
104 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
105 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
106 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
107 { "instruction_sie", VCPU_STAT(instruction_sie) },
108 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
109 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
110 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
111 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
112 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
113 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
114 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
115 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
116 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
117 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
118 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
119 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
120 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
121 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
122 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
123 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
124 { "diagnose_10", VCPU_STAT(diagnose_10) },
125 { "diagnose_44", VCPU_STAT(diagnose_44) },
126 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
127 { "diagnose_258", VCPU_STAT(diagnose_258) },
128 { "diagnose_308", VCPU_STAT(diagnose_308) },
129 { "diagnose_500", VCPU_STAT(diagnose_500) },
130 { NULL }
131 };
132
133 /* allow nested virtualization in KVM (if enabled by user space) */
134 static int nested;
135 module_param(nested, int, S_IRUGO);
136 MODULE_PARM_DESC(nested, "Nested virtualization support");
137
138 /* upper facilities limit for kvm */
139 unsigned long kvm_s390_fac_list_mask[16] = { FACILITIES_KVM };
140
141 unsigned long kvm_s390_fac_list_mask_size(void)
142 {
143 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
144 return ARRAY_SIZE(kvm_s390_fac_list_mask);
145 }
146
147 /* available cpu features supported by kvm */
148 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
149 /* available subfunctions indicated via query / "test bit" */
150 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
151
152 static struct gmap_notifier gmap_notifier;
153 static struct gmap_notifier vsie_gmap_notifier;
154 debug_info_t *kvm_s390_dbf;
155
156 /* Section: not file related */
157 int kvm_arch_hardware_enable(void)
158 {
159 /* every s390 is virtualization enabled ;-) */
160 return 0;
161 }
162
163 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
164 unsigned long end);
165
166 /*
167 * This callback is executed during stop_machine(). All CPUs are therefore
168 * temporarily stopped. In order not to change guest behavior, we have to
169 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
170 * so a CPU won't be stopped while calculating with the epoch.
171 */
172 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
173 void *v)
174 {
175 struct kvm *kvm;
176 struct kvm_vcpu *vcpu;
177 int i;
178 unsigned long long *delta = v;
179
180 list_for_each_entry(kvm, &vm_list, vm_list) {
181 kvm->arch.epoch -= *delta;
182 kvm_for_each_vcpu(i, vcpu, kvm) {
183 vcpu->arch.sie_block->epoch -= *delta;
184 if (vcpu->arch.cputm_enabled)
185 vcpu->arch.cputm_start += *delta;
186 if (vcpu->arch.vsie_block)
187 vcpu->arch.vsie_block->epoch -= *delta;
188 }
189 }
190 return NOTIFY_OK;
191 }
192
193 static struct notifier_block kvm_clock_notifier = {
194 .notifier_call = kvm_clock_sync,
195 };
196
197 int kvm_arch_hardware_setup(void)
198 {
199 gmap_notifier.notifier_call = kvm_gmap_notifier;
200 gmap_register_pte_notifier(&gmap_notifier);
201 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
202 gmap_register_pte_notifier(&vsie_gmap_notifier);
203 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
204 &kvm_clock_notifier);
205 return 0;
206 }
207
208 void kvm_arch_hardware_unsetup(void)
209 {
210 gmap_unregister_pte_notifier(&gmap_notifier);
211 gmap_unregister_pte_notifier(&vsie_gmap_notifier);
212 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
213 &kvm_clock_notifier);
214 }
215
216 static void allow_cpu_feat(unsigned long nr)
217 {
218 set_bit_inv(nr, kvm_s390_available_cpu_feat);
219 }
220
221 static inline int plo_test_bit(unsigned char nr)
222 {
223 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
224 int cc;
225
226 asm volatile(
227 /* Parameter registers are ignored for "test bit" */
228 " plo 0,0,0,0(0)\n"
229 " ipm %0\n"
230 " srl %0,28\n"
231 : "=d" (cc)
232 : "d" (r0)
233 : "cc");
234 return cc == 0;
235 }
236
237 static void kvm_s390_cpu_feat_init(void)
238 {
239 int i;
240
241 for (i = 0; i < 256; ++i) {
242 if (plo_test_bit(i))
243 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
244 }
245
246 if (test_facility(28)) /* TOD-clock steering */
247 ptff(kvm_s390_available_subfunc.ptff,
248 sizeof(kvm_s390_available_subfunc.ptff),
249 PTFF_QAF);
250
251 if (test_facility(17)) { /* MSA */
252 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
253 kvm_s390_available_subfunc.kmac);
254 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
255 kvm_s390_available_subfunc.kmc);
256 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
257 kvm_s390_available_subfunc.km);
258 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
259 kvm_s390_available_subfunc.kimd);
260 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
261 kvm_s390_available_subfunc.klmd);
262 }
263 if (test_facility(76)) /* MSA3 */
264 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
265 kvm_s390_available_subfunc.pckmo);
266 if (test_facility(77)) { /* MSA4 */
267 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
268 kvm_s390_available_subfunc.kmctr);
269 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
270 kvm_s390_available_subfunc.kmf);
271 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
272 kvm_s390_available_subfunc.kmo);
273 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
274 kvm_s390_available_subfunc.pcc);
275 }
276 if (test_facility(57)) /* MSA5 */
277 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
278 kvm_s390_available_subfunc.ppno);
279
280 if (test_facility(146)) /* MSA8 */
281 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
282 kvm_s390_available_subfunc.kma);
283
284 if (MACHINE_HAS_ESOP)
285 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
286 /*
287 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
288 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
289 */
290 if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
291 !test_facility(3) || !nested)
292 return;
293 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
294 if (sclp.has_64bscao)
295 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
296 if (sclp.has_siif)
297 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
298 if (sclp.has_gpere)
299 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
300 if (sclp.has_gsls)
301 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
302 if (sclp.has_ib)
303 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
304 if (sclp.has_cei)
305 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
306 if (sclp.has_ibs)
307 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
308 if (sclp.has_kss)
309 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
310 /*
311 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
312 * all skey handling functions read/set the skey from the PGSTE
313 * instead of the real storage key.
314 *
315 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
316 * pages being detected as preserved although they are resident.
317 *
318 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
319 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
320 *
321 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
322 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
323 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
324 *
325 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
326 * cannot easily shadow the SCA because of the ipte lock.
327 */
328 }
329
330 int kvm_arch_init(void *opaque)
331 {
332 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
333 if (!kvm_s390_dbf)
334 return -ENOMEM;
335
336 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
337 debug_unregister(kvm_s390_dbf);
338 return -ENOMEM;
339 }
340
341 kvm_s390_cpu_feat_init();
342
343 /* Register floating interrupt controller interface. */
344 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
345 }
346
347 void kvm_arch_exit(void)
348 {
349 debug_unregister(kvm_s390_dbf);
350 }
351
352 /* Section: device related */
353 long kvm_arch_dev_ioctl(struct file *filp,
354 unsigned int ioctl, unsigned long arg)
355 {
356 if (ioctl == KVM_S390_ENABLE_SIE)
357 return s390_enable_sie();
358 return -EINVAL;
359 }
360
361 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
362 {
363 int r;
364
365 switch (ext) {
366 case KVM_CAP_S390_PSW:
367 case KVM_CAP_S390_GMAP:
368 case KVM_CAP_SYNC_MMU:
369 #ifdef CONFIG_KVM_S390_UCONTROL
370 case KVM_CAP_S390_UCONTROL:
371 #endif
372 case KVM_CAP_ASYNC_PF:
373 case KVM_CAP_SYNC_REGS:
374 case KVM_CAP_ONE_REG:
375 case KVM_CAP_ENABLE_CAP:
376 case KVM_CAP_S390_CSS_SUPPORT:
377 case KVM_CAP_IOEVENTFD:
378 case KVM_CAP_DEVICE_CTRL:
379 case KVM_CAP_ENABLE_CAP_VM:
380 case KVM_CAP_S390_IRQCHIP:
381 case KVM_CAP_VM_ATTRIBUTES:
382 case KVM_CAP_MP_STATE:
383 case KVM_CAP_IMMEDIATE_EXIT:
384 case KVM_CAP_S390_INJECT_IRQ:
385 case KVM_CAP_S390_USER_SIGP:
386 case KVM_CAP_S390_USER_STSI:
387 case KVM_CAP_S390_SKEYS:
388 case KVM_CAP_S390_IRQ_STATE:
389 case KVM_CAP_S390_USER_INSTR0:
390 case KVM_CAP_S390_CMMA_MIGRATION:
391 case KVM_CAP_S390_AIS:
392 r = 1;
393 break;
394 case KVM_CAP_S390_MEM_OP:
395 r = MEM_OP_MAX_SIZE;
396 break;
397 case KVM_CAP_NR_VCPUS:
398 case KVM_CAP_MAX_VCPUS:
399 r = KVM_S390_BSCA_CPU_SLOTS;
400 if (!kvm_s390_use_sca_entries())
401 r = KVM_MAX_VCPUS;
402 else if (sclp.has_esca && sclp.has_64bscao)
403 r = KVM_S390_ESCA_CPU_SLOTS;
404 break;
405 case KVM_CAP_NR_MEMSLOTS:
406 r = KVM_USER_MEM_SLOTS;
407 break;
408 case KVM_CAP_S390_COW:
409 r = MACHINE_HAS_ESOP;
410 break;
411 case KVM_CAP_S390_VECTOR_REGISTERS:
412 r = MACHINE_HAS_VX;
413 break;
414 case KVM_CAP_S390_RI:
415 r = test_facility(64);
416 break;
417 case KVM_CAP_S390_GS:
418 r = test_facility(133);
419 break;
420 default:
421 r = 0;
422 }
423 return r;
424 }
425
426 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
427 struct kvm_memory_slot *memslot)
428 {
429 gfn_t cur_gfn, last_gfn;
430 unsigned long address;
431 struct gmap *gmap = kvm->arch.gmap;
432
433 /* Loop over all guest pages */
434 last_gfn = memslot->base_gfn + memslot->npages;
435 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
436 address = gfn_to_hva_memslot(memslot, cur_gfn);
437
438 if (test_and_clear_guest_dirty(gmap->mm, address))
439 mark_page_dirty(kvm, cur_gfn);
440 if (fatal_signal_pending(current))
441 return;
442 cond_resched();
443 }
444 }
445
446 /* Section: vm related */
447 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
448
449 /*
450 * Get (and clear) the dirty memory log for a memory slot.
451 */
452 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
453 struct kvm_dirty_log *log)
454 {
455 int r;
456 unsigned long n;
457 struct kvm_memslots *slots;
458 struct kvm_memory_slot *memslot;
459 int is_dirty = 0;
460
461 if (kvm_is_ucontrol(kvm))
462 return -EINVAL;
463
464 mutex_lock(&kvm->slots_lock);
465
466 r = -EINVAL;
467 if (log->slot >= KVM_USER_MEM_SLOTS)
468 goto out;
469
470 slots = kvm_memslots(kvm);
471 memslot = id_to_memslot(slots, log->slot);
472 r = -ENOENT;
473 if (!memslot->dirty_bitmap)
474 goto out;
475
476 kvm_s390_sync_dirty_log(kvm, memslot);
477 r = kvm_get_dirty_log(kvm, log, &is_dirty);
478 if (r)
479 goto out;
480
481 /* Clear the dirty log */
482 if (is_dirty) {
483 n = kvm_dirty_bitmap_bytes(memslot);
484 memset(memslot->dirty_bitmap, 0, n);
485 }
486 r = 0;
487 out:
488 mutex_unlock(&kvm->slots_lock);
489 return r;
490 }
491
492 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
493 {
494 unsigned int i;
495 struct kvm_vcpu *vcpu;
496
497 kvm_for_each_vcpu(i, vcpu, kvm) {
498 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
499 }
500 }
501
502 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
503 {
504 int r;
505
506 if (cap->flags)
507 return -EINVAL;
508
509 switch (cap->cap) {
510 case KVM_CAP_S390_IRQCHIP:
511 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
512 kvm->arch.use_irqchip = 1;
513 r = 0;
514 break;
515 case KVM_CAP_S390_USER_SIGP:
516 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
517 kvm->arch.user_sigp = 1;
518 r = 0;
519 break;
520 case KVM_CAP_S390_VECTOR_REGISTERS:
521 mutex_lock(&kvm->lock);
522 if (kvm->created_vcpus) {
523 r = -EBUSY;
524 } else if (MACHINE_HAS_VX) {
525 set_kvm_facility(kvm->arch.model.fac_mask, 129);
526 set_kvm_facility(kvm->arch.model.fac_list, 129);
527 if (test_facility(134)) {
528 set_kvm_facility(kvm->arch.model.fac_mask, 134);
529 set_kvm_facility(kvm->arch.model.fac_list, 134);
530 }
531 if (test_facility(135)) {
532 set_kvm_facility(kvm->arch.model.fac_mask, 135);
533 set_kvm_facility(kvm->arch.model.fac_list, 135);
534 }
535 r = 0;
536 } else
537 r = -EINVAL;
538 mutex_unlock(&kvm->lock);
539 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
540 r ? "(not available)" : "(success)");
541 break;
542 case KVM_CAP_S390_RI:
543 r = -EINVAL;
544 mutex_lock(&kvm->lock);
545 if (kvm->created_vcpus) {
546 r = -EBUSY;
547 } else if (test_facility(64)) {
548 set_kvm_facility(kvm->arch.model.fac_mask, 64);
549 set_kvm_facility(kvm->arch.model.fac_list, 64);
550 r = 0;
551 }
552 mutex_unlock(&kvm->lock);
553 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
554 r ? "(not available)" : "(success)");
555 break;
556 case KVM_CAP_S390_AIS:
557 mutex_lock(&kvm->lock);
558 if (kvm->created_vcpus) {
559 r = -EBUSY;
560 } else {
561 set_kvm_facility(kvm->arch.model.fac_mask, 72);
562 set_kvm_facility(kvm->arch.model.fac_list, 72);
563 r = 0;
564 }
565 mutex_unlock(&kvm->lock);
566 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
567 r ? "(not available)" : "(success)");
568 break;
569 case KVM_CAP_S390_GS:
570 r = -EINVAL;
571 mutex_lock(&kvm->lock);
572 if (atomic_read(&kvm->online_vcpus)) {
573 r = -EBUSY;
574 } else if (test_facility(133)) {
575 set_kvm_facility(kvm->arch.model.fac_mask, 133);
576 set_kvm_facility(kvm->arch.model.fac_list, 133);
577 r = 0;
578 }
579 mutex_unlock(&kvm->lock);
580 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
581 r ? "(not available)" : "(success)");
582 break;
583 case KVM_CAP_S390_USER_STSI:
584 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
585 kvm->arch.user_stsi = 1;
586 r = 0;
587 break;
588 case KVM_CAP_S390_USER_INSTR0:
589 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
590 kvm->arch.user_instr0 = 1;
591 icpt_operexc_on_all_vcpus(kvm);
592 r = 0;
593 break;
594 default:
595 r = -EINVAL;
596 break;
597 }
598 return r;
599 }
600
601 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
602 {
603 int ret;
604
605 switch (attr->attr) {
606 case KVM_S390_VM_MEM_LIMIT_SIZE:
607 ret = 0;
608 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
609 kvm->arch.mem_limit);
610 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
611 ret = -EFAULT;
612 break;
613 default:
614 ret = -ENXIO;
615 break;
616 }
617 return ret;
618 }
619
620 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
621 {
622 int ret;
623 unsigned int idx;
624 switch (attr->attr) {
625 case KVM_S390_VM_MEM_ENABLE_CMMA:
626 ret = -ENXIO;
627 if (!sclp.has_cmma)
628 break;
629
630 ret = -EBUSY;
631 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
632 mutex_lock(&kvm->lock);
633 if (!kvm->created_vcpus) {
634 kvm->arch.use_cmma = 1;
635 ret = 0;
636 }
637 mutex_unlock(&kvm->lock);
638 break;
639 case KVM_S390_VM_MEM_CLR_CMMA:
640 ret = -ENXIO;
641 if (!sclp.has_cmma)
642 break;
643 ret = -EINVAL;
644 if (!kvm->arch.use_cmma)
645 break;
646
647 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
648 mutex_lock(&kvm->lock);
649 idx = srcu_read_lock(&kvm->srcu);
650 s390_reset_cmma(kvm->arch.gmap->mm);
651 srcu_read_unlock(&kvm->srcu, idx);
652 mutex_unlock(&kvm->lock);
653 ret = 0;
654 break;
655 case KVM_S390_VM_MEM_LIMIT_SIZE: {
656 unsigned long new_limit;
657
658 if (kvm_is_ucontrol(kvm))
659 return -EINVAL;
660
661 if (get_user(new_limit, (u64 __user *)attr->addr))
662 return -EFAULT;
663
664 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
665 new_limit > kvm->arch.mem_limit)
666 return -E2BIG;
667
668 if (!new_limit)
669 return -EINVAL;
670
671 /* gmap_create takes last usable address */
672 if (new_limit != KVM_S390_NO_MEM_LIMIT)
673 new_limit -= 1;
674
675 ret = -EBUSY;
676 mutex_lock(&kvm->lock);
677 if (!kvm->created_vcpus) {
678 /* gmap_create will round the limit up */
679 struct gmap *new = gmap_create(current->mm, new_limit);
680
681 if (!new) {
682 ret = -ENOMEM;
683 } else {
684 gmap_remove(kvm->arch.gmap);
685 new->private = kvm;
686 kvm->arch.gmap = new;
687 ret = 0;
688 }
689 }
690 mutex_unlock(&kvm->lock);
691 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
692 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
693 (void *) kvm->arch.gmap->asce);
694 break;
695 }
696 default:
697 ret = -ENXIO;
698 break;
699 }
700 return ret;
701 }
702
703 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
704
705 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
706 {
707 struct kvm_vcpu *vcpu;
708 int i;
709
710 if (!test_kvm_facility(kvm, 76))
711 return -EINVAL;
712
713 mutex_lock(&kvm->lock);
714 switch (attr->attr) {
715 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
716 get_random_bytes(
717 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
718 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
719 kvm->arch.crypto.aes_kw = 1;
720 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
721 break;
722 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
723 get_random_bytes(
724 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
725 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
726 kvm->arch.crypto.dea_kw = 1;
727 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
728 break;
729 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
730 kvm->arch.crypto.aes_kw = 0;
731 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
732 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
733 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
734 break;
735 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
736 kvm->arch.crypto.dea_kw = 0;
737 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
738 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
739 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
740 break;
741 default:
742 mutex_unlock(&kvm->lock);
743 return -ENXIO;
744 }
745
746 kvm_for_each_vcpu(i, vcpu, kvm) {
747 kvm_s390_vcpu_crypto_setup(vcpu);
748 exit_sie(vcpu);
749 }
750 mutex_unlock(&kvm->lock);
751 return 0;
752 }
753
754 static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
755 {
756 int cx;
757 struct kvm_vcpu *vcpu;
758
759 kvm_for_each_vcpu(cx, vcpu, kvm)
760 kvm_s390_sync_request(req, vcpu);
761 }
762
763 /*
764 * Must be called with kvm->srcu held to avoid races on memslots, and with
765 * kvm->lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
766 */
767 static int kvm_s390_vm_start_migration(struct kvm *kvm)
768 {
769 struct kvm_s390_migration_state *mgs;
770 struct kvm_memory_slot *ms;
771 /* should be the only one */
772 struct kvm_memslots *slots;
773 unsigned long ram_pages;
774 int slotnr;
775
776 /* migration mode already enabled */
777 if (kvm->arch.migration_state)
778 return 0;
779
780 slots = kvm_memslots(kvm);
781 if (!slots || !slots->used_slots)
782 return -EINVAL;
783
784 mgs = kzalloc(sizeof(*mgs), GFP_KERNEL);
785 if (!mgs)
786 return -ENOMEM;
787 kvm->arch.migration_state = mgs;
788
789 if (kvm->arch.use_cmma) {
790 /*
791 * Get the last slot. They should be sorted by base_gfn, so the
792 * last slot is also the one at the end of the address space.
793 * We have verified above that at least one slot is present.
794 */
795 ms = slots->memslots + slots->used_slots - 1;
796 /* round up so we only use full longs */
797 ram_pages = roundup(ms->base_gfn + ms->npages, BITS_PER_LONG);
798 /* allocate enough bytes to store all the bits */
799 mgs->pgste_bitmap = vmalloc(ram_pages / 8);
800 if (!mgs->pgste_bitmap) {
801 kfree(mgs);
802 kvm->arch.migration_state = NULL;
803 return -ENOMEM;
804 }
805
806 mgs->bitmap_size = ram_pages;
807 atomic64_set(&mgs->dirty_pages, ram_pages);
808 /* mark all the pages in active slots as dirty */
809 for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
810 ms = slots->memslots + slotnr;
811 bitmap_set(mgs->pgste_bitmap, ms->base_gfn, ms->npages);
812 }
813
814 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
815 }
816 return 0;
817 }
818
819 /*
820 * Must be called with kvm->lock to avoid races with ourselves and
821 * kvm_s390_vm_start_migration.
822 */
823 static int kvm_s390_vm_stop_migration(struct kvm *kvm)
824 {
825 struct kvm_s390_migration_state *mgs;
826
827 /* migration mode already disabled */
828 if (!kvm->arch.migration_state)
829 return 0;
830 mgs = kvm->arch.migration_state;
831 kvm->arch.migration_state = NULL;
832
833 if (kvm->arch.use_cmma) {
834 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
835 vfree(mgs->pgste_bitmap);
836 }
837 kfree(mgs);
838 return 0;
839 }
840
841 static int kvm_s390_vm_set_migration(struct kvm *kvm,
842 struct kvm_device_attr *attr)
843 {
844 int idx, res = -ENXIO;
845
846 mutex_lock(&kvm->lock);
847 switch (attr->attr) {
848 case KVM_S390_VM_MIGRATION_START:
849 idx = srcu_read_lock(&kvm->srcu);
850 res = kvm_s390_vm_start_migration(kvm);
851 srcu_read_unlock(&kvm->srcu, idx);
852 break;
853 case KVM_S390_VM_MIGRATION_STOP:
854 res = kvm_s390_vm_stop_migration(kvm);
855 break;
856 default:
857 break;
858 }
859 mutex_unlock(&kvm->lock);
860
861 return res;
862 }
863
864 static int kvm_s390_vm_get_migration(struct kvm *kvm,
865 struct kvm_device_attr *attr)
866 {
867 u64 mig = (kvm->arch.migration_state != NULL);
868
869 if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
870 return -ENXIO;
871
872 if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
873 return -EFAULT;
874 return 0;
875 }
876
877 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
878 {
879 u8 gtod_high;
880
881 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
882 sizeof(gtod_high)))
883 return -EFAULT;
884
885 if (gtod_high != 0)
886 return -EINVAL;
887 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
888
889 return 0;
890 }
891
892 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
893 {
894 u64 gtod;
895
896 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
897 return -EFAULT;
898
899 kvm_s390_set_tod_clock(kvm, gtod);
900 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
901 return 0;
902 }
903
904 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
905 {
906 int ret;
907
908 if (attr->flags)
909 return -EINVAL;
910
911 switch (attr->attr) {
912 case KVM_S390_VM_TOD_HIGH:
913 ret = kvm_s390_set_tod_high(kvm, attr);
914 break;
915 case KVM_S390_VM_TOD_LOW:
916 ret = kvm_s390_set_tod_low(kvm, attr);
917 break;
918 default:
919 ret = -ENXIO;
920 break;
921 }
922 return ret;
923 }
924
925 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
926 {
927 u8 gtod_high = 0;
928
929 if (copy_to_user((void __user *)attr->addr, &gtod_high,
930 sizeof(gtod_high)))
931 return -EFAULT;
932 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
933
934 return 0;
935 }
936
937 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
938 {
939 u64 gtod;
940
941 gtod = kvm_s390_get_tod_clock_fast(kvm);
942 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
943 return -EFAULT;
944 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
945
946 return 0;
947 }
948
949 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
950 {
951 int ret;
952
953 if (attr->flags)
954 return -EINVAL;
955
956 switch (attr->attr) {
957 case KVM_S390_VM_TOD_HIGH:
958 ret = kvm_s390_get_tod_high(kvm, attr);
959 break;
960 case KVM_S390_VM_TOD_LOW:
961 ret = kvm_s390_get_tod_low(kvm, attr);
962 break;
963 default:
964 ret = -ENXIO;
965 break;
966 }
967 return ret;
968 }
969
970 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
971 {
972 struct kvm_s390_vm_cpu_processor *proc;
973 u16 lowest_ibc, unblocked_ibc;
974 int ret = 0;
975
976 mutex_lock(&kvm->lock);
977 if (kvm->created_vcpus) {
978 ret = -EBUSY;
979 goto out;
980 }
981 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
982 if (!proc) {
983 ret = -ENOMEM;
984 goto out;
985 }
986 if (!copy_from_user(proc, (void __user *)attr->addr,
987 sizeof(*proc))) {
988 kvm->arch.model.cpuid = proc->cpuid;
989 lowest_ibc = sclp.ibc >> 16 & 0xfff;
990 unblocked_ibc = sclp.ibc & 0xfff;
991 if (lowest_ibc && proc->ibc) {
992 if (proc->ibc > unblocked_ibc)
993 kvm->arch.model.ibc = unblocked_ibc;
994 else if (proc->ibc < lowest_ibc)
995 kvm->arch.model.ibc = lowest_ibc;
996 else
997 kvm->arch.model.ibc = proc->ibc;
998 }
999 memcpy(kvm->arch.model.fac_list, proc->fac_list,
1000 S390_ARCH_FAC_LIST_SIZE_BYTE);
1001 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1002 kvm->arch.model.ibc,
1003 kvm->arch.model.cpuid);
1004 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1005 kvm->arch.model.fac_list[0],
1006 kvm->arch.model.fac_list[1],
1007 kvm->arch.model.fac_list[2]);
1008 } else
1009 ret = -EFAULT;
1010 kfree(proc);
1011 out:
1012 mutex_unlock(&kvm->lock);
1013 return ret;
1014 }
1015
1016 static int kvm_s390_set_processor_feat(struct kvm *kvm,
1017 struct kvm_device_attr *attr)
1018 {
1019 struct kvm_s390_vm_cpu_feat data;
1020 int ret = -EBUSY;
1021
1022 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1023 return -EFAULT;
1024 if (!bitmap_subset((unsigned long *) data.feat,
1025 kvm_s390_available_cpu_feat,
1026 KVM_S390_VM_CPU_FEAT_NR_BITS))
1027 return -EINVAL;
1028
1029 mutex_lock(&kvm->lock);
1030 if (!atomic_read(&kvm->online_vcpus)) {
1031 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
1032 KVM_S390_VM_CPU_FEAT_NR_BITS);
1033 ret = 0;
1034 }
1035 mutex_unlock(&kvm->lock);
1036 return ret;
1037 }
1038
1039 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1040 struct kvm_device_attr *attr)
1041 {
1042 /*
1043 * Once supported by kernel + hw, we have to store the subfunctions
1044 * in kvm->arch and remember that user space configured them.
1045 */
1046 return -ENXIO;
1047 }
1048
1049 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1050 {
1051 int ret = -ENXIO;
1052
1053 switch (attr->attr) {
1054 case KVM_S390_VM_CPU_PROCESSOR:
1055 ret = kvm_s390_set_processor(kvm, attr);
1056 break;
1057 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1058 ret = kvm_s390_set_processor_feat(kvm, attr);
1059 break;
1060 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1061 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1062 break;
1063 }
1064 return ret;
1065 }
1066
1067 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1068 {
1069 struct kvm_s390_vm_cpu_processor *proc;
1070 int ret = 0;
1071
1072 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1073 if (!proc) {
1074 ret = -ENOMEM;
1075 goto out;
1076 }
1077 proc->cpuid = kvm->arch.model.cpuid;
1078 proc->ibc = kvm->arch.model.ibc;
1079 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1080 S390_ARCH_FAC_LIST_SIZE_BYTE);
1081 VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1082 kvm->arch.model.ibc,
1083 kvm->arch.model.cpuid);
1084 VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1085 kvm->arch.model.fac_list[0],
1086 kvm->arch.model.fac_list[1],
1087 kvm->arch.model.fac_list[2]);
1088 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1089 ret = -EFAULT;
1090 kfree(proc);
1091 out:
1092 return ret;
1093 }
1094
1095 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1096 {
1097 struct kvm_s390_vm_cpu_machine *mach;
1098 int ret = 0;
1099
1100 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
1101 if (!mach) {
1102 ret = -ENOMEM;
1103 goto out;
1104 }
1105 get_cpu_id((struct cpuid *) &mach->cpuid);
1106 mach->ibc = sclp.ibc;
1107 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
1108 S390_ARCH_FAC_LIST_SIZE_BYTE);
1109 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
1110 sizeof(S390_lowcore.stfle_fac_list));
1111 VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx",
1112 kvm->arch.model.ibc,
1113 kvm->arch.model.cpuid);
1114 VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx",
1115 mach->fac_mask[0],
1116 mach->fac_mask[1],
1117 mach->fac_mask[2]);
1118 VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1119 mach->fac_list[0],
1120 mach->fac_list[1],
1121 mach->fac_list[2]);
1122 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1123 ret = -EFAULT;
1124 kfree(mach);
1125 out:
1126 return ret;
1127 }
1128
1129 static int kvm_s390_get_processor_feat(struct kvm *kvm,
1130 struct kvm_device_attr *attr)
1131 {
1132 struct kvm_s390_vm_cpu_feat data;
1133
1134 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1135 KVM_S390_VM_CPU_FEAT_NR_BITS);
1136 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1137 return -EFAULT;
1138 return 0;
1139 }
1140
1141 static int kvm_s390_get_machine_feat(struct kvm *kvm,
1142 struct kvm_device_attr *attr)
1143 {
1144 struct kvm_s390_vm_cpu_feat data;
1145
1146 bitmap_copy((unsigned long *) data.feat,
1147 kvm_s390_available_cpu_feat,
1148 KVM_S390_VM_CPU_FEAT_NR_BITS);
1149 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1150 return -EFAULT;
1151 return 0;
1152 }
1153
1154 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1155 struct kvm_device_attr *attr)
1156 {
1157 /*
1158 * Once we can actually configure subfunctions (kernel + hw support),
1159 * we have to check if they were already set by user space, if so copy
1160 * them from kvm->arch.
1161 */
1162 return -ENXIO;
1163 }
1164
1165 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1166 struct kvm_device_attr *attr)
1167 {
1168 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1169 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1170 return -EFAULT;
1171 return 0;
1172 }
1173 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1174 {
1175 int ret = -ENXIO;
1176
1177 switch (attr->attr) {
1178 case KVM_S390_VM_CPU_PROCESSOR:
1179 ret = kvm_s390_get_processor(kvm, attr);
1180 break;
1181 case KVM_S390_VM_CPU_MACHINE:
1182 ret = kvm_s390_get_machine(kvm, attr);
1183 break;
1184 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1185 ret = kvm_s390_get_processor_feat(kvm, attr);
1186 break;
1187 case KVM_S390_VM_CPU_MACHINE_FEAT:
1188 ret = kvm_s390_get_machine_feat(kvm, attr);
1189 break;
1190 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1191 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1192 break;
1193 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1194 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1195 break;
1196 }
1197 return ret;
1198 }
1199
1200 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1201 {
1202 int ret;
1203
1204 switch (attr->group) {
1205 case KVM_S390_VM_MEM_CTRL:
1206 ret = kvm_s390_set_mem_control(kvm, attr);
1207 break;
1208 case KVM_S390_VM_TOD:
1209 ret = kvm_s390_set_tod(kvm, attr);
1210 break;
1211 case KVM_S390_VM_CPU_MODEL:
1212 ret = kvm_s390_set_cpu_model(kvm, attr);
1213 break;
1214 case KVM_S390_VM_CRYPTO:
1215 ret = kvm_s390_vm_set_crypto(kvm, attr);
1216 break;
1217 case KVM_S390_VM_MIGRATION:
1218 ret = kvm_s390_vm_set_migration(kvm, attr);
1219 break;
1220 default:
1221 ret = -ENXIO;
1222 break;
1223 }
1224
1225 return ret;
1226 }
1227
1228 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1229 {
1230 int ret;
1231
1232 switch (attr->group) {
1233 case KVM_S390_VM_MEM_CTRL:
1234 ret = kvm_s390_get_mem_control(kvm, attr);
1235 break;
1236 case KVM_S390_VM_TOD:
1237 ret = kvm_s390_get_tod(kvm, attr);
1238 break;
1239 case KVM_S390_VM_CPU_MODEL:
1240 ret = kvm_s390_get_cpu_model(kvm, attr);
1241 break;
1242 case KVM_S390_VM_MIGRATION:
1243 ret = kvm_s390_vm_get_migration(kvm, attr);
1244 break;
1245 default:
1246 ret = -ENXIO;
1247 break;
1248 }
1249
1250 return ret;
1251 }
1252
1253 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1254 {
1255 int ret;
1256
1257 switch (attr->group) {
1258 case KVM_S390_VM_MEM_CTRL:
1259 switch (attr->attr) {
1260 case KVM_S390_VM_MEM_ENABLE_CMMA:
1261 case KVM_S390_VM_MEM_CLR_CMMA:
1262 ret = sclp.has_cmma ? 0 : -ENXIO;
1263 break;
1264 case KVM_S390_VM_MEM_LIMIT_SIZE:
1265 ret = 0;
1266 break;
1267 default:
1268 ret = -ENXIO;
1269 break;
1270 }
1271 break;
1272 case KVM_S390_VM_TOD:
1273 switch (attr->attr) {
1274 case KVM_S390_VM_TOD_LOW:
1275 case KVM_S390_VM_TOD_HIGH:
1276 ret = 0;
1277 break;
1278 default:
1279 ret = -ENXIO;
1280 break;
1281 }
1282 break;
1283 case KVM_S390_VM_CPU_MODEL:
1284 switch (attr->attr) {
1285 case KVM_S390_VM_CPU_PROCESSOR:
1286 case KVM_S390_VM_CPU_MACHINE:
1287 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1288 case KVM_S390_VM_CPU_MACHINE_FEAT:
1289 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1290 ret = 0;
1291 break;
1292 /* configuring subfunctions is not supported yet */
1293 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1294 default:
1295 ret = -ENXIO;
1296 break;
1297 }
1298 break;
1299 case KVM_S390_VM_CRYPTO:
1300 switch (attr->attr) {
1301 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1302 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1303 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1304 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1305 ret = 0;
1306 break;
1307 default:
1308 ret = -ENXIO;
1309 break;
1310 }
1311 break;
1312 case KVM_S390_VM_MIGRATION:
1313 ret = 0;
1314 break;
1315 default:
1316 ret = -ENXIO;
1317 break;
1318 }
1319
1320 return ret;
1321 }
1322
1323 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1324 {
1325 uint8_t *keys;
1326 uint64_t hva;
1327 int srcu_idx, i, r = 0;
1328
1329 if (args->flags != 0)
1330 return -EINVAL;
1331
1332 /* Is this guest using storage keys? */
1333 if (!mm_use_skey(current->mm))
1334 return KVM_S390_GET_SKEYS_NONE;
1335
1336 /* Enforce sane limit on memory allocation */
1337 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1338 return -EINVAL;
1339
1340 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1341 if (!keys)
1342 return -ENOMEM;
1343
1344 down_read(&current->mm->mmap_sem);
1345 srcu_idx = srcu_read_lock(&kvm->srcu);
1346 for (i = 0; i < args->count; i++) {
1347 hva = gfn_to_hva(kvm, args->start_gfn + i);
1348 if (kvm_is_error_hva(hva)) {
1349 r = -EFAULT;
1350 break;
1351 }
1352
1353 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1354 if (r)
1355 break;
1356 }
1357 srcu_read_unlock(&kvm->srcu, srcu_idx);
1358 up_read(&current->mm->mmap_sem);
1359
1360 if (!r) {
1361 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1362 sizeof(uint8_t) * args->count);
1363 if (r)
1364 r = -EFAULT;
1365 }
1366
1367 kvfree(keys);
1368 return r;
1369 }
1370
1371 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1372 {
1373 uint8_t *keys;
1374 uint64_t hva;
1375 int srcu_idx, i, r = 0;
1376
1377 if (args->flags != 0)
1378 return -EINVAL;
1379
1380 /* Enforce sane limit on memory allocation */
1381 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1382 return -EINVAL;
1383
1384 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1385 if (!keys)
1386 return -ENOMEM;
1387
1388 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1389 sizeof(uint8_t) * args->count);
1390 if (r) {
1391 r = -EFAULT;
1392 goto out;
1393 }
1394
1395 /* Enable storage key handling for the guest */
1396 r = s390_enable_skey();
1397 if (r)
1398 goto out;
1399
1400 down_read(&current->mm->mmap_sem);
1401 srcu_idx = srcu_read_lock(&kvm->srcu);
1402 for (i = 0; i < args->count; i++) {
1403 hva = gfn_to_hva(kvm, args->start_gfn + i);
1404 if (kvm_is_error_hva(hva)) {
1405 r = -EFAULT;
1406 break;
1407 }
1408
1409 /* Lowest order bit is reserved */
1410 if (keys[i] & 0x01) {
1411 r = -EINVAL;
1412 break;
1413 }
1414
1415 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1416 if (r)
1417 break;
1418 }
1419 srcu_read_unlock(&kvm->srcu, srcu_idx);
1420 up_read(&current->mm->mmap_sem);
1421 out:
1422 kvfree(keys);
1423 return r;
1424 }
1425
1426 /*
1427 * Base address and length must be sent at the start of each block, therefore
1428 * it's cheaper to send some clean data, as long as it's less than the size of
1429 * two longs.
1430 */
1431 #define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
1432 /* for consistency */
1433 #define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1434
1435 /*
1436 * This function searches for the next page with dirty CMMA attributes, and
1437 * saves the attributes in the buffer up to either the end of the buffer or
1438 * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
1439 * no trailing clean bytes are saved.
1440 * In case no dirty bits were found, or if CMMA was not enabled or used, the
1441 * output buffer will indicate 0 as length.
1442 */
1443 static int kvm_s390_get_cmma_bits(struct kvm *kvm,
1444 struct kvm_s390_cmma_log *args)
1445 {
1446 struct kvm_s390_migration_state *s = kvm->arch.migration_state;
1447 unsigned long bufsize, hva, pgstev, i, next, cur;
1448 int srcu_idx, peek, r = 0, rr;
1449 u8 *res;
1450
1451 cur = args->start_gfn;
1452 i = next = pgstev = 0;
1453
1454 if (unlikely(!kvm->arch.use_cmma))
1455 return -ENXIO;
1456 /* Invalid/unsupported flags were specified */
1457 if (args->flags & ~KVM_S390_CMMA_PEEK)
1458 return -EINVAL;
1459 /* Migration mode query, and we are not doing a migration */
1460 peek = !!(args->flags & KVM_S390_CMMA_PEEK);
1461 if (!peek && !s)
1462 return -EINVAL;
1463 /* CMMA is disabled or was not used, or the buffer has length zero */
1464 bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
1465 if (!bufsize || !kvm->mm->context.use_cmma) {
1466 memset(args, 0, sizeof(*args));
1467 return 0;
1468 }
1469
1470 if (!peek) {
1471 /* We are not peeking, and there are no dirty pages */
1472 if (!atomic64_read(&s->dirty_pages)) {
1473 memset(args, 0, sizeof(*args));
1474 return 0;
1475 }
1476 cur = find_next_bit(s->pgste_bitmap, s->bitmap_size,
1477 args->start_gfn);
1478 if (cur >= s->bitmap_size) /* nothing found, loop back */
1479 cur = find_next_bit(s->pgste_bitmap, s->bitmap_size, 0);
1480 if (cur >= s->bitmap_size) { /* again! (very unlikely) */
1481 memset(args, 0, sizeof(*args));
1482 return 0;
1483 }
1484 next = find_next_bit(s->pgste_bitmap, s->bitmap_size, cur + 1);
1485 }
1486
1487 res = vmalloc(bufsize);
1488 if (!res)
1489 return -ENOMEM;
1490
1491 args->start_gfn = cur;
1492
1493 down_read(&kvm->mm->mmap_sem);
1494 srcu_idx = srcu_read_lock(&kvm->srcu);
1495 while (i < bufsize) {
1496 hva = gfn_to_hva(kvm, cur);
1497 if (kvm_is_error_hva(hva)) {
1498 r = -EFAULT;
1499 break;
1500 }
1501 /* decrement only if we actually flipped the bit to 0 */
1502 if (!peek && test_and_clear_bit(cur, s->pgste_bitmap))
1503 atomic64_dec(&s->dirty_pages);
1504 r = get_pgste(kvm->mm, hva, &pgstev);
1505 if (r < 0)
1506 pgstev = 0;
1507 /* save the value */
1508 res[i++] = (pgstev >> 24) & 0x3;
1509 /*
1510 * if the next bit is too far away, stop.
1511 * if we reached the previous "next", find the next one
1512 */
1513 if (!peek) {
1514 if (next > cur + KVM_S390_MAX_BIT_DISTANCE)
1515 break;
1516 if (cur == next)
1517 next = find_next_bit(s->pgste_bitmap,
1518 s->bitmap_size, cur + 1);
1519 /* reached the end of the bitmap or of the buffer, stop */
1520 if ((next >= s->bitmap_size) ||
1521 (next >= args->start_gfn + bufsize))
1522 break;
1523 }
1524 cur++;
1525 }
1526 srcu_read_unlock(&kvm->srcu, srcu_idx);
1527 up_read(&kvm->mm->mmap_sem);
1528 args->count = i;
1529 args->remaining = s ? atomic64_read(&s->dirty_pages) : 0;
1530
1531 rr = copy_to_user((void __user *)args->values, res, args->count);
1532 if (rr)
1533 r = -EFAULT;
1534
1535 vfree(res);
1536 return r;
1537 }
1538
1539 /*
1540 * This function sets the CMMA attributes for the given pages. If the input
1541 * buffer has zero length, no action is taken, otherwise the attributes are
1542 * set and the mm->context.use_cmma flag is set.
1543 */
1544 static int kvm_s390_set_cmma_bits(struct kvm *kvm,
1545 const struct kvm_s390_cmma_log *args)
1546 {
1547 unsigned long hva, mask, pgstev, i;
1548 uint8_t *bits;
1549 int srcu_idx, r = 0;
1550
1551 mask = args->mask;
1552
1553 if (!kvm->arch.use_cmma)
1554 return -ENXIO;
1555 /* invalid/unsupported flags */
1556 if (args->flags != 0)
1557 return -EINVAL;
1558 /* Enforce sane limit on memory allocation */
1559 if (args->count > KVM_S390_CMMA_SIZE_MAX)
1560 return -EINVAL;
1561 /* Nothing to do */
1562 if (args->count == 0)
1563 return 0;
1564
1565 bits = vmalloc(sizeof(*bits) * args->count);
1566 if (!bits)
1567 return -ENOMEM;
1568
1569 r = copy_from_user(bits, (void __user *)args->values, args->count);
1570 if (r) {
1571 r = -EFAULT;
1572 goto out;
1573 }
1574
1575 down_read(&kvm->mm->mmap_sem);
1576 srcu_idx = srcu_read_lock(&kvm->srcu);
1577 for (i = 0; i < args->count; i++) {
1578 hva = gfn_to_hva(kvm, args->start_gfn + i);
1579 if (kvm_is_error_hva(hva)) {
1580 r = -EFAULT;
1581 break;
1582 }
1583
1584 pgstev = bits[i];
1585 pgstev = pgstev << 24;
1586 mask &= _PGSTE_GPS_USAGE_MASK;
1587 set_pgste_bits(kvm->mm, hva, mask, pgstev);
1588 }
1589 srcu_read_unlock(&kvm->srcu, srcu_idx);
1590 up_read(&kvm->mm->mmap_sem);
1591
1592 if (!kvm->mm->context.use_cmma) {
1593 down_write(&kvm->mm->mmap_sem);
1594 kvm->mm->context.use_cmma = 1;
1595 up_write(&kvm->mm->mmap_sem);
1596 }
1597 out:
1598 vfree(bits);
1599 return r;
1600 }
1601
1602 long kvm_arch_vm_ioctl(struct file *filp,
1603 unsigned int ioctl, unsigned long arg)
1604 {
1605 struct kvm *kvm = filp->private_data;
1606 void __user *argp = (void __user *)arg;
1607 struct kvm_device_attr attr;
1608 int r;
1609
1610 switch (ioctl) {
1611 case KVM_S390_INTERRUPT: {
1612 struct kvm_s390_interrupt s390int;
1613
1614 r = -EFAULT;
1615 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1616 break;
1617 r = kvm_s390_inject_vm(kvm, &s390int);
1618 break;
1619 }
1620 case KVM_ENABLE_CAP: {
1621 struct kvm_enable_cap cap;
1622 r = -EFAULT;
1623 if (copy_from_user(&cap, argp, sizeof(cap)))
1624 break;
1625 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1626 break;
1627 }
1628 case KVM_CREATE_IRQCHIP: {
1629 struct kvm_irq_routing_entry routing;
1630
1631 r = -EINVAL;
1632 if (kvm->arch.use_irqchip) {
1633 /* Set up dummy routing. */
1634 memset(&routing, 0, sizeof(routing));
1635 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
1636 }
1637 break;
1638 }
1639 case KVM_SET_DEVICE_ATTR: {
1640 r = -EFAULT;
1641 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1642 break;
1643 r = kvm_s390_vm_set_attr(kvm, &attr);
1644 break;
1645 }
1646 case KVM_GET_DEVICE_ATTR: {
1647 r = -EFAULT;
1648 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1649 break;
1650 r = kvm_s390_vm_get_attr(kvm, &attr);
1651 break;
1652 }
1653 case KVM_HAS_DEVICE_ATTR: {
1654 r = -EFAULT;
1655 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1656 break;
1657 r = kvm_s390_vm_has_attr(kvm, &attr);
1658 break;
1659 }
1660 case KVM_S390_GET_SKEYS: {
1661 struct kvm_s390_skeys args;
1662
1663 r = -EFAULT;
1664 if (copy_from_user(&args, argp,
1665 sizeof(struct kvm_s390_skeys)))
1666 break;
1667 r = kvm_s390_get_skeys(kvm, &args);
1668 break;
1669 }
1670 case KVM_S390_SET_SKEYS: {
1671 struct kvm_s390_skeys args;
1672
1673 r = -EFAULT;
1674 if (copy_from_user(&args, argp,
1675 sizeof(struct kvm_s390_skeys)))
1676 break;
1677 r = kvm_s390_set_skeys(kvm, &args);
1678 break;
1679 }
1680 case KVM_S390_GET_CMMA_BITS: {
1681 struct kvm_s390_cmma_log args;
1682
1683 r = -EFAULT;
1684 if (copy_from_user(&args, argp, sizeof(args)))
1685 break;
1686 r = kvm_s390_get_cmma_bits(kvm, &args);
1687 if (!r) {
1688 r = copy_to_user(argp, &args, sizeof(args));
1689 if (r)
1690 r = -EFAULT;
1691 }
1692 break;
1693 }
1694 case KVM_S390_SET_CMMA_BITS: {
1695 struct kvm_s390_cmma_log args;
1696
1697 r = -EFAULT;
1698 if (copy_from_user(&args, argp, sizeof(args)))
1699 break;
1700 r = kvm_s390_set_cmma_bits(kvm, &args);
1701 break;
1702 }
1703 default:
1704 r = -ENOTTY;
1705 }
1706
1707 return r;
1708 }
1709
1710 static int kvm_s390_query_ap_config(u8 *config)
1711 {
1712 u32 fcn_code = 0x04000000UL;
1713 u32 cc = 0;
1714
1715 memset(config, 0, 128);
1716 asm volatile(
1717 "lgr 0,%1\n"
1718 "lgr 2,%2\n"
1719 ".long 0xb2af0000\n" /* PQAP(QCI) */
1720 "0: ipm %0\n"
1721 "srl %0,28\n"
1722 "1:\n"
1723 EX_TABLE(0b, 1b)
1724 : "+r" (cc)
1725 : "r" (fcn_code), "r" (config)
1726 : "cc", "0", "2", "memory"
1727 );
1728
1729 return cc;
1730 }
1731
1732 static int kvm_s390_apxa_installed(void)
1733 {
1734 u8 config[128];
1735 int cc;
1736
1737 if (test_facility(12)) {
1738 cc = kvm_s390_query_ap_config(config);
1739
1740 if (cc)
1741 pr_err("PQAP(QCI) failed with cc=%d", cc);
1742 else
1743 return config[0] & 0x40;
1744 }
1745
1746 return 0;
1747 }
1748
1749 static void kvm_s390_set_crycb_format(struct kvm *kvm)
1750 {
1751 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1752
1753 if (kvm_s390_apxa_installed())
1754 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1755 else
1756 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1757 }
1758
1759 static u64 kvm_s390_get_initial_cpuid(void)
1760 {
1761 struct cpuid cpuid;
1762
1763 get_cpu_id(&cpuid);
1764 cpuid.version = 0xff;
1765 return *((u64 *) &cpuid);
1766 }
1767
1768 static void kvm_s390_crypto_init(struct kvm *kvm)
1769 {
1770 if (!test_kvm_facility(kvm, 76))
1771 return;
1772
1773 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
1774 kvm_s390_set_crycb_format(kvm);
1775
1776 /* Enable AES/DEA protected key functions by default */
1777 kvm->arch.crypto.aes_kw = 1;
1778 kvm->arch.crypto.dea_kw = 1;
1779 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1780 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1781 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1782 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1783 }
1784
1785 static void sca_dispose(struct kvm *kvm)
1786 {
1787 if (kvm->arch.use_esca)
1788 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
1789 else
1790 free_page((unsigned long)(kvm->arch.sca));
1791 kvm->arch.sca = NULL;
1792 }
1793
1794 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1795 {
1796 gfp_t alloc_flags = GFP_KERNEL;
1797 int i, rc;
1798 char debug_name[16];
1799 static unsigned long sca_offset;
1800
1801 rc = -EINVAL;
1802 #ifdef CONFIG_KVM_S390_UCONTROL
1803 if (type & ~KVM_VM_S390_UCONTROL)
1804 goto out_err;
1805 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1806 goto out_err;
1807 #else
1808 if (type)
1809 goto out_err;
1810 #endif
1811
1812 rc = s390_enable_sie();
1813 if (rc)
1814 goto out_err;
1815
1816 rc = -ENOMEM;
1817
1818 ratelimit_state_init(&kvm->arch.sthyi_limit, 5 * HZ, 500);
1819
1820 kvm->arch.use_esca = 0; /* start with basic SCA */
1821 if (!sclp.has_64bscao)
1822 alloc_flags |= GFP_DMA;
1823 rwlock_init(&kvm->arch.sca_lock);
1824 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
1825 if (!kvm->arch.sca)
1826 goto out_err;
1827 spin_lock(&kvm_lock);
1828 sca_offset += 16;
1829 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
1830 sca_offset = 0;
1831 kvm->arch.sca = (struct bsca_block *)
1832 ((char *) kvm->arch.sca + sca_offset);
1833 spin_unlock(&kvm_lock);
1834
1835 sprintf(debug_name, "kvm-%u", current->pid);
1836
1837 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1838 if (!kvm->arch.dbf)
1839 goto out_err;
1840
1841 kvm->arch.sie_page2 =
1842 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1843 if (!kvm->arch.sie_page2)
1844 goto out_err;
1845
1846 /* Populate the facility mask initially. */
1847 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
1848 sizeof(S390_lowcore.stfle_fac_list));
1849 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1850 if (i < kvm_s390_fac_list_mask_size())
1851 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
1852 else
1853 kvm->arch.model.fac_mask[i] = 0UL;
1854 }
1855
1856 /* Populate the facility list initially. */
1857 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1858 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
1859 S390_ARCH_FAC_LIST_SIZE_BYTE);
1860
1861 set_kvm_facility(kvm->arch.model.fac_mask, 74);
1862 set_kvm_facility(kvm->arch.model.fac_list, 74);
1863
1864 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
1865 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1866
1867 kvm_s390_crypto_init(kvm);
1868
1869 mutex_init(&kvm->arch.float_int.ais_lock);
1870 kvm->arch.float_int.simm = 0;
1871 kvm->arch.float_int.nimm = 0;
1872 spin_lock_init(&kvm->arch.float_int.lock);
1873 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1874 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1875 init_waitqueue_head(&kvm->arch.ipte_wq);
1876 mutex_init(&kvm->arch.ipte_mutex);
1877
1878 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1879 VM_EVENT(kvm, 3, "vm created with type %lu", type);
1880
1881 if (type & KVM_VM_S390_UCONTROL) {
1882 kvm->arch.gmap = NULL;
1883 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
1884 } else {
1885 if (sclp.hamax == U64_MAX)
1886 kvm->arch.mem_limit = TASK_SIZE_MAX;
1887 else
1888 kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
1889 sclp.hamax + 1);
1890 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
1891 if (!kvm->arch.gmap)
1892 goto out_err;
1893 kvm->arch.gmap->private = kvm;
1894 kvm->arch.gmap->pfault_enabled = 0;
1895 }
1896
1897 kvm->arch.css_support = 0;
1898 kvm->arch.use_irqchip = 0;
1899 kvm->arch.epoch = 0;
1900
1901 spin_lock_init(&kvm->arch.start_stop_lock);
1902 kvm_s390_vsie_init(kvm);
1903 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
1904
1905 return 0;
1906 out_err:
1907 free_page((unsigned long)kvm->arch.sie_page2);
1908 debug_unregister(kvm->arch.dbf);
1909 sca_dispose(kvm);
1910 KVM_EVENT(3, "creation of vm failed: %d", rc);
1911 return rc;
1912 }
1913
1914 bool kvm_arch_has_vcpu_debugfs(void)
1915 {
1916 return false;
1917 }
1918
1919 int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
1920 {
1921 return 0;
1922 }
1923
1924 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1925 {
1926 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1927 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1928 kvm_s390_clear_local_irqs(vcpu);
1929 kvm_clear_async_pf_completion_queue(vcpu);
1930 if (!kvm_is_ucontrol(vcpu->kvm))
1931 sca_del_vcpu(vcpu);
1932
1933 if (kvm_is_ucontrol(vcpu->kvm))
1934 gmap_remove(vcpu->arch.gmap);
1935
1936 if (vcpu->kvm->arch.use_cmma)
1937 kvm_s390_vcpu_unsetup_cmma(vcpu);
1938 free_page((unsigned long)(vcpu->arch.sie_block));
1939
1940 kvm_vcpu_uninit(vcpu);
1941 kmem_cache_free(kvm_vcpu_cache, vcpu);
1942 }
1943
1944 static void kvm_free_vcpus(struct kvm *kvm)
1945 {
1946 unsigned int i;
1947 struct kvm_vcpu *vcpu;
1948
1949 kvm_for_each_vcpu(i, vcpu, kvm)
1950 kvm_arch_vcpu_destroy(vcpu);
1951
1952 mutex_lock(&kvm->lock);
1953 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1954 kvm->vcpus[i] = NULL;
1955
1956 atomic_set(&kvm->online_vcpus, 0);
1957 mutex_unlock(&kvm->lock);
1958 }
1959
1960 void kvm_arch_destroy_vm(struct kvm *kvm)
1961 {
1962 kvm_free_vcpus(kvm);
1963 sca_dispose(kvm);
1964 debug_unregister(kvm->arch.dbf);
1965 free_page((unsigned long)kvm->arch.sie_page2);
1966 if (!kvm_is_ucontrol(kvm))
1967 gmap_remove(kvm->arch.gmap);
1968 kvm_s390_destroy_adapters(kvm);
1969 kvm_s390_clear_float_irqs(kvm);
1970 kvm_s390_vsie_destroy(kvm);
1971 if (kvm->arch.migration_state) {
1972 vfree(kvm->arch.migration_state->pgste_bitmap);
1973 kfree(kvm->arch.migration_state);
1974 }
1975 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
1976 }
1977
1978 /* Section: vcpu related */
1979 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1980 {
1981 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
1982 if (!vcpu->arch.gmap)
1983 return -ENOMEM;
1984 vcpu->arch.gmap->private = vcpu->kvm;
1985
1986 return 0;
1987 }
1988
1989 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1990 {
1991 if (!kvm_s390_use_sca_entries())
1992 return;
1993 read_lock(&vcpu->kvm->arch.sca_lock);
1994 if (vcpu->kvm->arch.use_esca) {
1995 struct esca_block *sca = vcpu->kvm->arch.sca;
1996
1997 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1998 sca->cpu[vcpu->vcpu_id].sda = 0;
1999 } else {
2000 struct bsca_block *sca = vcpu->kvm->arch.sca;
2001
2002 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2003 sca->cpu[vcpu->vcpu_id].sda = 0;
2004 }
2005 read_unlock(&vcpu->kvm->arch.sca_lock);
2006 }
2007
2008 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
2009 {
2010 if (!kvm_s390_use_sca_entries()) {
2011 struct bsca_block *sca = vcpu->kvm->arch.sca;
2012
2013 /* we still need the basic sca for the ipte control */
2014 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2015 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2016 }
2017 read_lock(&vcpu->kvm->arch.sca_lock);
2018 if (vcpu->kvm->arch.use_esca) {
2019 struct esca_block *sca = vcpu->kvm->arch.sca;
2020
2021 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2022 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2023 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
2024 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2025 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2026 } else {
2027 struct bsca_block *sca = vcpu->kvm->arch.sca;
2028
2029 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2030 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2031 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2032 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2033 }
2034 read_unlock(&vcpu->kvm->arch.sca_lock);
2035 }
2036
2037 /* Basic SCA to Extended SCA data copy routines */
2038 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
2039 {
2040 d->sda = s->sda;
2041 d->sigp_ctrl.c = s->sigp_ctrl.c;
2042 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
2043 }
2044
2045 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
2046 {
2047 int i;
2048
2049 d->ipte_control = s->ipte_control;
2050 d->mcn[0] = s->mcn;
2051 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
2052 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
2053 }
2054
2055 static int sca_switch_to_extended(struct kvm *kvm)
2056 {
2057 struct bsca_block *old_sca = kvm->arch.sca;
2058 struct esca_block *new_sca;
2059 struct kvm_vcpu *vcpu;
2060 unsigned int vcpu_idx;
2061 u32 scaol, scaoh;
2062
2063 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
2064 if (!new_sca)
2065 return -ENOMEM;
2066
2067 scaoh = (u32)((u64)(new_sca) >> 32);
2068 scaol = (u32)(u64)(new_sca) & ~0x3fU;
2069
2070 kvm_s390_vcpu_block_all(kvm);
2071 write_lock(&kvm->arch.sca_lock);
2072
2073 sca_copy_b_to_e(new_sca, old_sca);
2074
2075 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
2076 vcpu->arch.sie_block->scaoh = scaoh;
2077 vcpu->arch.sie_block->scaol = scaol;
2078 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2079 }
2080 kvm->arch.sca = new_sca;
2081 kvm->arch.use_esca = 1;
2082
2083 write_unlock(&kvm->arch.sca_lock);
2084 kvm_s390_vcpu_unblock_all(kvm);
2085
2086 free_page((unsigned long)old_sca);
2087
2088 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
2089 old_sca, kvm->arch.sca);
2090 return 0;
2091 }
2092
2093 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
2094 {
2095 int rc;
2096
2097 if (!kvm_s390_use_sca_entries()) {
2098 if (id < KVM_MAX_VCPUS)
2099 return true;
2100 return false;
2101 }
2102 if (id < KVM_S390_BSCA_CPU_SLOTS)
2103 return true;
2104 if (!sclp.has_esca || !sclp.has_64bscao)
2105 return false;
2106
2107 mutex_lock(&kvm->lock);
2108 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
2109 mutex_unlock(&kvm->lock);
2110
2111 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
2112 }
2113
2114 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
2115 {
2116 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2117 kvm_clear_async_pf_completion_queue(vcpu);
2118 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
2119 KVM_SYNC_GPRS |
2120 KVM_SYNC_ACRS |
2121 KVM_SYNC_CRS |
2122 KVM_SYNC_ARCH0 |
2123 KVM_SYNC_PFAULT;
2124 kvm_s390_set_prefix(vcpu, 0);
2125 if (test_kvm_facility(vcpu->kvm, 64))
2126 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
2127 if (test_kvm_facility(vcpu->kvm, 133))
2128 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
2129 /* fprs can be synchronized via vrs, even if the guest has no vx. With
2130 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
2131 */
2132 if (MACHINE_HAS_VX)
2133 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
2134 else
2135 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
2136
2137 if (kvm_is_ucontrol(vcpu->kvm))
2138 return __kvm_ucontrol_vcpu_init(vcpu);
2139
2140 return 0;
2141 }
2142
2143 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2144 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2145 {
2146 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
2147 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2148 vcpu->arch.cputm_start = get_tod_clock_fast();
2149 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2150 }
2151
2152 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2153 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2154 {
2155 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
2156 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2157 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2158 vcpu->arch.cputm_start = 0;
2159 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2160 }
2161
2162 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2163 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2164 {
2165 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
2166 vcpu->arch.cputm_enabled = true;
2167 __start_cpu_timer_accounting(vcpu);
2168 }
2169
2170 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2171 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2172 {
2173 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
2174 __stop_cpu_timer_accounting(vcpu);
2175 vcpu->arch.cputm_enabled = false;
2176 }
2177
2178 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2179 {
2180 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2181 __enable_cpu_timer_accounting(vcpu);
2182 preempt_enable();
2183 }
2184
2185 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2186 {
2187 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2188 __disable_cpu_timer_accounting(vcpu);
2189 preempt_enable();
2190 }
2191
2192 /* set the cpu timer - may only be called from the VCPU thread itself */
2193 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
2194 {
2195 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2196 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2197 if (vcpu->arch.cputm_enabled)
2198 vcpu->arch.cputm_start = get_tod_clock_fast();
2199 vcpu->arch.sie_block->cputm = cputm;
2200 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2201 preempt_enable();
2202 }
2203
2204 /* update and get the cpu timer - can also be called from other VCPU threads */
2205 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
2206 {
2207 unsigned int seq;
2208 __u64 value;
2209
2210 if (unlikely(!vcpu->arch.cputm_enabled))
2211 return vcpu->arch.sie_block->cputm;
2212
2213 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2214 do {
2215 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
2216 /*
2217 * If the writer would ever execute a read in the critical
2218 * section, e.g. in irq context, we have a deadlock.
2219 */
2220 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
2221 value = vcpu->arch.sie_block->cputm;
2222 /* if cputm_start is 0, accounting is being started/stopped */
2223 if (likely(vcpu->arch.cputm_start))
2224 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2225 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
2226 preempt_enable();
2227 return value;
2228 }
2229
2230 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2231 {
2232
2233 gmap_enable(vcpu->arch.enabled_gmap);
2234 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
2235 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2236 __start_cpu_timer_accounting(vcpu);
2237 vcpu->cpu = cpu;
2238 }
2239
2240 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
2241 {
2242 vcpu->cpu = -1;
2243 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2244 __stop_cpu_timer_accounting(vcpu);
2245 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
2246 vcpu->arch.enabled_gmap = gmap_get_enabled();
2247 gmap_disable(vcpu->arch.enabled_gmap);
2248
2249 }
2250
2251 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
2252 {
2253 /* this equals initial cpu reset in pop, but we don't switch to ESA */
2254 vcpu->arch.sie_block->gpsw.mask = 0UL;
2255 vcpu->arch.sie_block->gpsw.addr = 0UL;
2256 kvm_s390_set_prefix(vcpu, 0);
2257 kvm_s390_set_cpu_timer(vcpu, 0);
2258 vcpu->arch.sie_block->ckc = 0UL;
2259 vcpu->arch.sie_block->todpr = 0;
2260 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
2261 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
2262 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
2263 /* make sure the new fpc will be lazily loaded */
2264 save_fpu_regs();
2265 current->thread.fpu.fpc = 0;
2266 vcpu->arch.sie_block->gbea = 1;
2267 vcpu->arch.sie_block->pp = 0;
2268 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2269 kvm_clear_async_pf_completion_queue(vcpu);
2270 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
2271 kvm_s390_vcpu_stop(vcpu);
2272 kvm_s390_clear_local_irqs(vcpu);
2273 }
2274
2275 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
2276 {
2277 mutex_lock(&vcpu->kvm->lock);
2278 preempt_disable();
2279 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
2280 preempt_enable();
2281 mutex_unlock(&vcpu->kvm->lock);
2282 if (!kvm_is_ucontrol(vcpu->kvm)) {
2283 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
2284 sca_add_vcpu(vcpu);
2285 }
2286 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
2287 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2288 /* make vcpu_load load the right gmap on the first trigger */
2289 vcpu->arch.enabled_gmap = vcpu->arch.gmap;
2290 }
2291
2292 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
2293 {
2294 if (!test_kvm_facility(vcpu->kvm, 76))
2295 return;
2296
2297 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
2298
2299 if (vcpu->kvm->arch.crypto.aes_kw)
2300 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
2301 if (vcpu->kvm->arch.crypto.dea_kw)
2302 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
2303
2304 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
2305 }
2306
2307 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
2308 {
2309 free_page(vcpu->arch.sie_block->cbrlo);
2310 vcpu->arch.sie_block->cbrlo = 0;
2311 }
2312
2313 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
2314 {
2315 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
2316 if (!vcpu->arch.sie_block->cbrlo)
2317 return -ENOMEM;
2318
2319 vcpu->arch.sie_block->ecb2 &= ~ECB2_PFMFI;
2320 return 0;
2321 }
2322
2323 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
2324 {
2325 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
2326
2327 vcpu->arch.sie_block->ibc = model->ibc;
2328 if (test_kvm_facility(vcpu->kvm, 7))
2329 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
2330 }
2331
2332 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
2333 {
2334 int rc = 0;
2335
2336 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
2337 CPUSTAT_SM |
2338 CPUSTAT_STOPPED);
2339
2340 if (test_kvm_facility(vcpu->kvm, 78))
2341 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
2342 else if (test_kvm_facility(vcpu->kvm, 8))
2343 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
2344
2345 kvm_s390_vcpu_setup_model(vcpu);
2346
2347 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
2348 if (MACHINE_HAS_ESOP)
2349 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
2350 if (test_kvm_facility(vcpu->kvm, 9))
2351 vcpu->arch.sie_block->ecb |= ECB_SRSI;
2352 if (test_kvm_facility(vcpu->kvm, 73))
2353 vcpu->arch.sie_block->ecb |= ECB_TE;
2354
2355 if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
2356 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
2357 if (test_kvm_facility(vcpu->kvm, 130))
2358 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
2359 vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
2360 if (sclp.has_cei)
2361 vcpu->arch.sie_block->eca |= ECA_CEI;
2362 if (sclp.has_ib)
2363 vcpu->arch.sie_block->eca |= ECA_IB;
2364 if (sclp.has_siif)
2365 vcpu->arch.sie_block->eca |= ECA_SII;
2366 if (sclp.has_sigpif)
2367 vcpu->arch.sie_block->eca |= ECA_SIGPI;
2368 if (test_kvm_facility(vcpu->kvm, 129)) {
2369 vcpu->arch.sie_block->eca |= ECA_VX;
2370 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
2371 }
2372 vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
2373 | SDNXC;
2374 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
2375
2376 if (sclp.has_kss)
2377 atomic_or(CPUSTAT_KSS, &vcpu->arch.sie_block->cpuflags);
2378 else
2379 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
2380
2381 if (vcpu->kvm->arch.use_cmma) {
2382 rc = kvm_s390_vcpu_setup_cmma(vcpu);
2383 if (rc)
2384 return rc;
2385 }
2386 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2387 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
2388
2389 kvm_s390_vcpu_crypto_setup(vcpu);
2390
2391 return rc;
2392 }
2393
2394 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
2395 unsigned int id)
2396 {
2397 struct kvm_vcpu *vcpu;
2398 struct sie_page *sie_page;
2399 int rc = -EINVAL;
2400
2401 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
2402 goto out;
2403
2404 rc = -ENOMEM;
2405
2406 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
2407 if (!vcpu)
2408 goto out;
2409
2410 BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
2411 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
2412 if (!sie_page)
2413 goto out_free_cpu;
2414
2415 vcpu->arch.sie_block = &sie_page->sie_block;
2416 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
2417
2418 /* the real guest size will always be smaller than msl */
2419 vcpu->arch.sie_block->mso = 0;
2420 vcpu->arch.sie_block->msl = sclp.hamax;
2421
2422 vcpu->arch.sie_block->icpua = id;
2423 spin_lock_init(&vcpu->arch.local_int.lock);
2424 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
2425 vcpu->arch.local_int.wq = &vcpu->wq;
2426 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
2427 seqcount_init(&vcpu->arch.cputm_seqcount);
2428
2429 rc = kvm_vcpu_init(vcpu, kvm, id);
2430 if (rc)
2431 goto out_free_sie_block;
2432 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
2433 vcpu->arch.sie_block);
2434 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
2435
2436 return vcpu;
2437 out_free_sie_block:
2438 free_page((unsigned long)(vcpu->arch.sie_block));
2439 out_free_cpu:
2440 kmem_cache_free(kvm_vcpu_cache, vcpu);
2441 out:
2442 return ERR_PTR(rc);
2443 }
2444
2445 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
2446 {
2447 return kvm_s390_vcpu_has_irq(vcpu, 0);
2448 }
2449
2450 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
2451 {
2452 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2453 exit_sie(vcpu);
2454 }
2455
2456 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
2457 {
2458 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2459 }
2460
2461 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
2462 {
2463 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2464 exit_sie(vcpu);
2465 }
2466
2467 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2468 {
2469 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2470 }
2471
2472 /*
2473 * Kick a guest cpu out of SIE and wait until SIE is not running.
2474 * If the CPU is not running (e.g. waiting as idle) the function will
2475 * return immediately. */
2476 void exit_sie(struct kvm_vcpu *vcpu)
2477 {
2478 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
2479 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2480 cpu_relax();
2481 }
2482
2483 /* Kick a guest cpu out of SIE to process a request synchronously */
2484 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
2485 {
2486 kvm_make_request(req, vcpu);
2487 kvm_s390_vcpu_request(vcpu);
2488 }
2489
2490 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
2491 unsigned long end)
2492 {
2493 struct kvm *kvm = gmap->private;
2494 struct kvm_vcpu *vcpu;
2495 unsigned long prefix;
2496 int i;
2497
2498 if (gmap_is_shadow(gmap))
2499 return;
2500 if (start >= 1UL << 31)
2501 /* We are only interested in prefix pages */
2502 return;
2503 kvm_for_each_vcpu(i, vcpu, kvm) {
2504 /* match against both prefix pages */
2505 prefix = kvm_s390_get_prefix(vcpu);
2506 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
2507 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
2508 start, end);
2509 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2510 }
2511 }
2512 }
2513
2514 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2515 {
2516 /* kvm common code refers to this, but never calls it */
2517 BUG();
2518 return 0;
2519 }
2520
2521 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2522 struct kvm_one_reg *reg)
2523 {
2524 int r = -EINVAL;
2525
2526 switch (reg->id) {
2527 case KVM_REG_S390_TODPR:
2528 r = put_user(vcpu->arch.sie_block->todpr,
2529 (u32 __user *)reg->addr);
2530 break;
2531 case KVM_REG_S390_EPOCHDIFF:
2532 r = put_user(vcpu->arch.sie_block->epoch,
2533 (u64 __user *)reg->addr);
2534 break;
2535 case KVM_REG_S390_CPU_TIMER:
2536 r = put_user(kvm_s390_get_cpu_timer(vcpu),
2537 (u64 __user *)reg->addr);
2538 break;
2539 case KVM_REG_S390_CLOCK_COMP:
2540 r = put_user(vcpu->arch.sie_block->ckc,
2541 (u64 __user *)reg->addr);
2542 break;
2543 case KVM_REG_S390_PFTOKEN:
2544 r = put_user(vcpu->arch.pfault_token,
2545 (u64 __user *)reg->addr);
2546 break;
2547 case KVM_REG_S390_PFCOMPARE:
2548 r = put_user(vcpu->arch.pfault_compare,
2549 (u64 __user *)reg->addr);
2550 break;
2551 case KVM_REG_S390_PFSELECT:
2552 r = put_user(vcpu->arch.pfault_select,
2553 (u64 __user *)reg->addr);
2554 break;
2555 case KVM_REG_S390_PP:
2556 r = put_user(vcpu->arch.sie_block->pp,
2557 (u64 __user *)reg->addr);
2558 break;
2559 case KVM_REG_S390_GBEA:
2560 r = put_user(vcpu->arch.sie_block->gbea,
2561 (u64 __user *)reg->addr);
2562 break;
2563 default:
2564 break;
2565 }
2566
2567 return r;
2568 }
2569
2570 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2571 struct kvm_one_reg *reg)
2572 {
2573 int r = -EINVAL;
2574 __u64 val;
2575
2576 switch (reg->id) {
2577 case KVM_REG_S390_TODPR:
2578 r = get_user(vcpu->arch.sie_block->todpr,
2579 (u32 __user *)reg->addr);
2580 break;
2581 case KVM_REG_S390_EPOCHDIFF:
2582 r = get_user(vcpu->arch.sie_block->epoch,
2583 (u64 __user *)reg->addr);
2584 break;
2585 case KVM_REG_S390_CPU_TIMER:
2586 r = get_user(val, (u64 __user *)reg->addr);
2587 if (!r)
2588 kvm_s390_set_cpu_timer(vcpu, val);
2589 break;
2590 case KVM_REG_S390_CLOCK_COMP:
2591 r = get_user(vcpu->arch.sie_block->ckc,
2592 (u64 __user *)reg->addr);
2593 break;
2594 case KVM_REG_S390_PFTOKEN:
2595 r = get_user(vcpu->arch.pfault_token,
2596 (u64 __user *)reg->addr);
2597 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2598 kvm_clear_async_pf_completion_queue(vcpu);
2599 break;
2600 case KVM_REG_S390_PFCOMPARE:
2601 r = get_user(vcpu->arch.pfault_compare,
2602 (u64 __user *)reg->addr);
2603 break;
2604 case KVM_REG_S390_PFSELECT:
2605 r = get_user(vcpu->arch.pfault_select,
2606 (u64 __user *)reg->addr);
2607 break;
2608 case KVM_REG_S390_PP:
2609 r = get_user(vcpu->arch.sie_block->pp,
2610 (u64 __user *)reg->addr);
2611 break;
2612 case KVM_REG_S390_GBEA:
2613 r = get_user(vcpu->arch.sie_block->gbea,
2614 (u64 __user *)reg->addr);
2615 break;
2616 default:
2617 break;
2618 }
2619
2620 return r;
2621 }
2622
2623 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2624 {
2625 kvm_s390_vcpu_initial_reset(vcpu);
2626 return 0;
2627 }
2628
2629 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2630 {
2631 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
2632 return 0;
2633 }
2634
2635 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2636 {
2637 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
2638 return 0;
2639 }
2640
2641 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2642 struct kvm_sregs *sregs)
2643 {
2644 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
2645 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
2646 return 0;
2647 }
2648
2649 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2650 struct kvm_sregs *sregs)
2651 {
2652 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
2653 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
2654 return 0;
2655 }
2656
2657 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2658 {
2659 if (test_fp_ctl(fpu->fpc))
2660 return -EINVAL;
2661 vcpu->run->s.regs.fpc = fpu->fpc;
2662 if (MACHINE_HAS_VX)
2663 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
2664 (freg_t *) fpu->fprs);
2665 else
2666 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
2667 return 0;
2668 }
2669
2670 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2671 {
2672 /* make sure we have the latest values */
2673 save_fpu_regs();
2674 if (MACHINE_HAS_VX)
2675 convert_vx_to_fp((freg_t *) fpu->fprs,
2676 (__vector128 *) vcpu->run->s.regs.vrs);
2677 else
2678 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
2679 fpu->fpc = vcpu->run->s.regs.fpc;
2680 return 0;
2681 }
2682
2683 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2684 {
2685 int rc = 0;
2686
2687 if (!is_vcpu_stopped(vcpu))
2688 rc = -EBUSY;
2689 else {
2690 vcpu->run->psw_mask = psw.mask;
2691 vcpu->run->psw_addr = psw.addr;
2692 }
2693 return rc;
2694 }
2695
2696 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2697 struct kvm_translation *tr)
2698 {
2699 return -EINVAL; /* not implemented yet */
2700 }
2701
2702 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2703 KVM_GUESTDBG_USE_HW_BP | \
2704 KVM_GUESTDBG_ENABLE)
2705
2706 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2707 struct kvm_guest_debug *dbg)
2708 {
2709 int rc = 0;
2710
2711 vcpu->guest_debug = 0;
2712 kvm_s390_clear_bp_data(vcpu);
2713
2714 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
2715 return -EINVAL;
2716 if (!sclp.has_gpere)
2717 return -EINVAL;
2718
2719 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2720 vcpu->guest_debug = dbg->control;
2721 /* enforce guest PER */
2722 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2723
2724 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2725 rc = kvm_s390_import_bp_data(vcpu, dbg);
2726 } else {
2727 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2728 vcpu->arch.guestdbg.last_bp = 0;
2729 }
2730
2731 if (rc) {
2732 vcpu->guest_debug = 0;
2733 kvm_s390_clear_bp_data(vcpu);
2734 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2735 }
2736
2737 return rc;
2738 }
2739
2740 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2741 struct kvm_mp_state *mp_state)
2742 {
2743 /* CHECK_STOP and LOAD are not supported yet */
2744 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2745 KVM_MP_STATE_OPERATING;
2746 }
2747
2748 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2749 struct kvm_mp_state *mp_state)
2750 {
2751 int rc = 0;
2752
2753 /* user space knows about this interface - let it control the state */
2754 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2755
2756 switch (mp_state->mp_state) {
2757 case KVM_MP_STATE_STOPPED:
2758 kvm_s390_vcpu_stop(vcpu);
2759 break;
2760 case KVM_MP_STATE_OPERATING:
2761 kvm_s390_vcpu_start(vcpu);
2762 break;
2763 case KVM_MP_STATE_LOAD:
2764 case KVM_MP_STATE_CHECK_STOP:
2765 /* fall through - CHECK_STOP and LOAD are not supported yet */
2766 default:
2767 rc = -ENXIO;
2768 }
2769
2770 return rc;
2771 }
2772
2773 static bool ibs_enabled(struct kvm_vcpu *vcpu)
2774 {
2775 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2776 }
2777
2778 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2779 {
2780 retry:
2781 kvm_s390_vcpu_request_handled(vcpu);
2782 if (!kvm_request_pending(vcpu))
2783 return 0;
2784 /*
2785 * We use MMU_RELOAD just to re-arm the ipte notifier for the
2786 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2787 * This ensures that the ipte instruction for this request has
2788 * already finished. We might race against a second unmapper that
2789 * wants to set the blocking bit. Lets just retry the request loop.
2790 */
2791 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2792 int rc;
2793 rc = gmap_mprotect_notify(vcpu->arch.gmap,
2794 kvm_s390_get_prefix(vcpu),
2795 PAGE_SIZE * 2, PROT_WRITE);
2796 if (rc) {
2797 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
2798 return rc;
2799 }
2800 goto retry;
2801 }
2802
2803 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2804 vcpu->arch.sie_block->ihcpu = 0xffff;
2805 goto retry;
2806 }
2807
2808 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2809 if (!ibs_enabled(vcpu)) {
2810 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
2811 atomic_or(CPUSTAT_IBS,
2812 &vcpu->arch.sie_block->cpuflags);
2813 }
2814 goto retry;
2815 }
2816
2817 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2818 if (ibs_enabled(vcpu)) {
2819 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
2820 atomic_andnot(CPUSTAT_IBS,
2821 &vcpu->arch.sie_block->cpuflags);
2822 }
2823 goto retry;
2824 }
2825
2826 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
2827 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2828 goto retry;
2829 }
2830
2831 if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
2832 /*
2833 * Disable CMMA virtualization; we will emulate the ESSA
2834 * instruction manually, in order to provide additional
2835 * functionalities needed for live migration.
2836 */
2837 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
2838 goto retry;
2839 }
2840
2841 if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
2842 /*
2843 * Re-enable CMMA virtualization if CMMA is available and
2844 * was used.
2845 */
2846 if ((vcpu->kvm->arch.use_cmma) &&
2847 (vcpu->kvm->mm->context.use_cmma))
2848 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
2849 goto retry;
2850 }
2851
2852 /* nothing to do, just clear the request */
2853 kvm_clear_request(KVM_REQ_UNHALT, vcpu);
2854
2855 return 0;
2856 }
2857
2858 void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2859 {
2860 struct kvm_vcpu *vcpu;
2861 int i;
2862
2863 mutex_lock(&kvm->lock);
2864 preempt_disable();
2865 kvm->arch.epoch = tod - get_tod_clock();
2866 kvm_s390_vcpu_block_all(kvm);
2867 kvm_for_each_vcpu(i, vcpu, kvm)
2868 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2869 kvm_s390_vcpu_unblock_all(kvm);
2870 preempt_enable();
2871 mutex_unlock(&kvm->lock);
2872 }
2873
2874 /**
2875 * kvm_arch_fault_in_page - fault-in guest page if necessary
2876 * @vcpu: The corresponding virtual cpu
2877 * @gpa: Guest physical address
2878 * @writable: Whether the page should be writable or not
2879 *
2880 * Make sure that a guest page has been faulted-in on the host.
2881 *
2882 * Return: Zero on success, negative error code otherwise.
2883 */
2884 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
2885 {
2886 return gmap_fault(vcpu->arch.gmap, gpa,
2887 writable ? FAULT_FLAG_WRITE : 0);
2888 }
2889
2890 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2891 unsigned long token)
2892 {
2893 struct kvm_s390_interrupt inti;
2894 struct kvm_s390_irq irq;
2895
2896 if (start_token) {
2897 irq.u.ext.ext_params2 = token;
2898 irq.type = KVM_S390_INT_PFAULT_INIT;
2899 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
2900 } else {
2901 inti.type = KVM_S390_INT_PFAULT_DONE;
2902 inti.parm64 = token;
2903 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2904 }
2905 }
2906
2907 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2908 struct kvm_async_pf *work)
2909 {
2910 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2911 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2912 }
2913
2914 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2915 struct kvm_async_pf *work)
2916 {
2917 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2918 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2919 }
2920
2921 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2922 struct kvm_async_pf *work)
2923 {
2924 /* s390 will always inject the page directly */
2925 }
2926
2927 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2928 {
2929 /*
2930 * s390 will always inject the page directly,
2931 * but we still want check_async_completion to cleanup
2932 */
2933 return true;
2934 }
2935
2936 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2937 {
2938 hva_t hva;
2939 struct kvm_arch_async_pf arch;
2940 int rc;
2941
2942 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2943 return 0;
2944 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2945 vcpu->arch.pfault_compare)
2946 return 0;
2947 if (psw_extint_disabled(vcpu))
2948 return 0;
2949 if (kvm_s390_vcpu_has_irq(vcpu, 0))
2950 return 0;
2951 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2952 return 0;
2953 if (!vcpu->arch.gmap->pfault_enabled)
2954 return 0;
2955
2956 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2957 hva += current->thread.gmap_addr & ~PAGE_MASK;
2958 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
2959 return 0;
2960
2961 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2962 return rc;
2963 }
2964
2965 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
2966 {
2967 int rc, cpuflags;
2968
2969 /*
2970 * On s390 notifications for arriving pages will be delivered directly
2971 * to the guest but the house keeping for completed pfaults is
2972 * handled outside the worker.
2973 */
2974 kvm_check_async_pf_completion(vcpu);
2975
2976 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2977 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
2978
2979 if (need_resched())
2980 schedule();
2981
2982 if (test_cpu_flag(CIF_MCCK_PENDING))
2983 s390_handle_mcck();
2984
2985 if (!kvm_is_ucontrol(vcpu->kvm)) {
2986 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2987 if (rc)
2988 return rc;
2989 }
2990
2991 rc = kvm_s390_handle_requests(vcpu);
2992 if (rc)
2993 return rc;
2994
2995 if (guestdbg_enabled(vcpu)) {
2996 kvm_s390_backup_guest_per_regs(vcpu);
2997 kvm_s390_patch_guest_per_regs(vcpu);
2998 }
2999
3000 vcpu->arch.sie_block->icptcode = 0;
3001 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
3002 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
3003 trace_kvm_s390_sie_enter(vcpu, cpuflags);
3004
3005 return 0;
3006 }
3007
3008 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
3009 {
3010 struct kvm_s390_pgm_info pgm_info = {
3011 .code = PGM_ADDRESSING,
3012 };
3013 u8 opcode, ilen;
3014 int rc;
3015
3016 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
3017 trace_kvm_s390_sie_fault(vcpu);
3018
3019 /*
3020 * We want to inject an addressing exception, which is defined as a
3021 * suppressing or terminating exception. However, since we came here
3022 * by a DAT access exception, the PSW still points to the faulting
3023 * instruction since DAT exceptions are nullifying. So we've got
3024 * to look up the current opcode to get the length of the instruction
3025 * to be able to forward the PSW.
3026 */
3027 rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
3028 ilen = insn_length(opcode);
3029 if (rc < 0) {
3030 return rc;
3031 } else if (rc) {
3032 /* Instruction-Fetching Exceptions - we can't detect the ilen.
3033 * Forward by arbitrary ilc, injection will take care of
3034 * nullification if necessary.
3035 */
3036 pgm_info = vcpu->arch.pgm;
3037 ilen = 4;
3038 }
3039 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
3040 kvm_s390_forward_psw(vcpu, ilen);
3041 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
3042 }
3043
3044 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
3045 {
3046 struct mcck_volatile_info *mcck_info;
3047 struct sie_page *sie_page;
3048
3049 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
3050 vcpu->arch.sie_block->icptcode);
3051 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
3052
3053 if (guestdbg_enabled(vcpu))
3054 kvm_s390_restore_guest_per_regs(vcpu);
3055
3056 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
3057 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
3058
3059 if (exit_reason == -EINTR) {
3060 VCPU_EVENT(vcpu, 3, "%s", "machine check");
3061 sie_page = container_of(vcpu->arch.sie_block,
3062 struct sie_page, sie_block);
3063 mcck_info = &sie_page->mcck_info;
3064 kvm_s390_reinject_machine_check(vcpu, mcck_info);
3065 return 0;
3066 }
3067
3068 if (vcpu->arch.sie_block->icptcode > 0) {
3069 int rc = kvm_handle_sie_intercept(vcpu);
3070
3071 if (rc != -EOPNOTSUPP)
3072 return rc;
3073 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
3074 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
3075 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
3076 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
3077 return -EREMOTE;
3078 } else if (exit_reason != -EFAULT) {
3079 vcpu->stat.exit_null++;
3080 return 0;
3081 } else if (kvm_is_ucontrol(vcpu->kvm)) {
3082 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
3083 vcpu->run->s390_ucontrol.trans_exc_code =
3084 current->thread.gmap_addr;
3085 vcpu->run->s390_ucontrol.pgm_code = 0x10;
3086 return -EREMOTE;
3087 } else if (current->thread.gmap_pfault) {
3088 trace_kvm_s390_major_guest_pfault(vcpu);
3089 current->thread.gmap_pfault = 0;
3090 if (kvm_arch_setup_async_pf(vcpu))
3091 return 0;
3092 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
3093 }
3094 return vcpu_post_run_fault_in_sie(vcpu);
3095 }
3096
3097 static int __vcpu_run(struct kvm_vcpu *vcpu)
3098 {
3099 int rc, exit_reason;
3100
3101 /*
3102 * We try to hold kvm->srcu during most of vcpu_run (except when run-
3103 * ning the guest), so that memslots (and other stuff) are protected
3104 */
3105 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3106
3107 do {
3108 rc = vcpu_pre_run(vcpu);
3109 if (rc)
3110 break;
3111
3112 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
3113 /*
3114 * As PF_VCPU will be used in fault handler, between
3115 * guest_enter and guest_exit should be no uaccess.
3116 */
3117 local_irq_disable();
3118 guest_enter_irqoff();
3119 __disable_cpu_timer_accounting(vcpu);
3120 local_irq_enable();
3121 exit_reason = sie64a(vcpu->arch.sie_block,
3122 vcpu->run->s.regs.gprs);
3123 local_irq_disable();
3124 __enable_cpu_timer_accounting(vcpu);
3125 guest_exit_irqoff();
3126 local_irq_enable();
3127 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3128
3129 rc = vcpu_post_run(vcpu, exit_reason);
3130 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3131
3132 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
3133 return rc;
3134 }
3135
3136 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3137 {
3138 struct runtime_instr_cb *riccb;
3139 struct gs_cb *gscb;
3140
3141 riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
3142 gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
3143 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
3144 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
3145 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
3146 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
3147 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
3148 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
3149 /* some control register changes require a tlb flush */
3150 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3151 }
3152 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
3153 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
3154 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
3155 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
3156 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
3157 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
3158 }
3159 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
3160 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
3161 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
3162 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
3163 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3164 kvm_clear_async_pf_completion_queue(vcpu);
3165 }
3166 /*
3167 * If userspace sets the riccb (e.g. after migration) to a valid state,
3168 * we should enable RI here instead of doing the lazy enablement.
3169 */
3170 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
3171 test_kvm_facility(vcpu->kvm, 64) &&
3172 riccb->valid &&
3173 !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
3174 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
3175 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
3176 }
3177 /*
3178 * If userspace sets the gscb (e.g. after migration) to non-zero,
3179 * we should enable GS here instead of doing the lazy enablement.
3180 */
3181 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
3182 test_kvm_facility(vcpu->kvm, 133) &&
3183 gscb->gssm &&
3184 !vcpu->arch.gs_enabled) {
3185 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
3186 vcpu->arch.sie_block->ecb |= ECB_GS;
3187 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3188 vcpu->arch.gs_enabled = 1;
3189 }
3190 save_access_regs(vcpu->arch.host_acrs);
3191 restore_access_regs(vcpu->run->s.regs.acrs);
3192 /* save host (userspace) fprs/vrs */
3193 save_fpu_regs();
3194 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
3195 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
3196 if (MACHINE_HAS_VX)
3197 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
3198 else
3199 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
3200 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
3201 if (test_fp_ctl(current->thread.fpu.fpc))
3202 /* User space provided an invalid FPC, let's clear it */
3203 current->thread.fpu.fpc = 0;
3204 if (MACHINE_HAS_GS) {
3205 preempt_disable();
3206 __ctl_set_bit(2, 4);
3207 if (current->thread.gs_cb) {
3208 vcpu->arch.host_gscb = current->thread.gs_cb;
3209 save_gs_cb(vcpu->arch.host_gscb);
3210 }
3211 if (vcpu->arch.gs_enabled) {
3212 current->thread.gs_cb = (struct gs_cb *)
3213 &vcpu->run->s.regs.gscb;
3214 restore_gs_cb(current->thread.gs_cb);
3215 }
3216 preempt_enable();
3217 }
3218
3219 kvm_run->kvm_dirty_regs = 0;
3220 }
3221
3222 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3223 {
3224 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
3225 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
3226 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
3227 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
3228 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
3229 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
3230 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
3231 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
3232 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
3233 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
3234 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
3235 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
3236 save_access_regs(vcpu->run->s.regs.acrs);
3237 restore_access_regs(vcpu->arch.host_acrs);
3238 /* Save guest register state */
3239 save_fpu_regs();
3240 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3241 /* Restore will be done lazily at return */
3242 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
3243 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
3244 if (MACHINE_HAS_GS) {
3245 __ctl_set_bit(2, 4);
3246 if (vcpu->arch.gs_enabled)
3247 save_gs_cb(current->thread.gs_cb);
3248 preempt_disable();
3249 current->thread.gs_cb = vcpu->arch.host_gscb;
3250 restore_gs_cb(vcpu->arch.host_gscb);
3251 preempt_enable();
3252 if (!vcpu->arch.host_gscb)
3253 __ctl_clear_bit(2, 4);
3254 vcpu->arch.host_gscb = NULL;
3255 }
3256
3257 }
3258
3259 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3260 {
3261 int rc;
3262 sigset_t sigsaved;
3263
3264 if (kvm_run->immediate_exit)
3265 return -EINTR;
3266
3267 if (guestdbg_exit_pending(vcpu)) {
3268 kvm_s390_prepare_debug_exit(vcpu);
3269 return 0;
3270 }
3271
3272 if (vcpu->sigset_active)
3273 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
3274
3275 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
3276 kvm_s390_vcpu_start(vcpu);
3277 } else if (is_vcpu_stopped(vcpu)) {
3278 pr_err_ratelimited("can't run stopped vcpu %d\n",
3279 vcpu->vcpu_id);
3280 return -EINVAL;
3281 }
3282
3283 sync_regs(vcpu, kvm_run);
3284 enable_cpu_timer_accounting(vcpu);
3285
3286 might_fault();
3287 rc = __vcpu_run(vcpu);
3288
3289 if (signal_pending(current) && !rc) {
3290 kvm_run->exit_reason = KVM_EXIT_INTR;
3291 rc = -EINTR;
3292 }
3293
3294 if (guestdbg_exit_pending(vcpu) && !rc) {
3295 kvm_s390_prepare_debug_exit(vcpu);
3296 rc = 0;
3297 }
3298
3299 if (rc == -EREMOTE) {
3300 /* userspace support is needed, kvm_run has been prepared */
3301 rc = 0;
3302 }
3303
3304 disable_cpu_timer_accounting(vcpu);
3305 store_regs(vcpu, kvm_run);
3306
3307 if (vcpu->sigset_active)
3308 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
3309
3310 vcpu->stat.exit_userspace++;
3311 return rc;
3312 }
3313
3314 /*
3315 * store status at address
3316 * we use have two special cases:
3317 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
3318 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
3319 */
3320 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
3321 {
3322 unsigned char archmode = 1;
3323 freg_t fprs[NUM_FPRS];
3324 unsigned int px;
3325 u64 clkcomp, cputm;
3326 int rc;
3327
3328 px = kvm_s390_get_prefix(vcpu);
3329 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
3330 if (write_guest_abs(vcpu, 163, &archmode, 1))
3331 return -EFAULT;
3332 gpa = 0;
3333 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
3334 if (write_guest_real(vcpu, 163, &archmode, 1))
3335 return -EFAULT;
3336 gpa = px;
3337 } else
3338 gpa -= __LC_FPREGS_SAVE_AREA;
3339
3340 /* manually convert vector registers if necessary */
3341 if (MACHINE_HAS_VX) {
3342 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
3343 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
3344 fprs, 128);
3345 } else {
3346 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
3347 vcpu->run->s.regs.fprs, 128);
3348 }
3349 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
3350 vcpu->run->s.regs.gprs, 128);
3351 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
3352 &vcpu->arch.sie_block->gpsw, 16);
3353 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
3354 &px, 4);
3355 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
3356 &vcpu->run->s.regs.fpc, 4);
3357 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
3358 &vcpu->arch.sie_block->todpr, 4);
3359 cputm = kvm_s390_get_cpu_timer(vcpu);
3360 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
3361 &cputm, 8);
3362 clkcomp = vcpu->arch.sie_block->ckc >> 8;
3363 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
3364 &clkcomp, 8);
3365 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
3366 &vcpu->run->s.regs.acrs, 64);
3367 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
3368 &vcpu->arch.sie_block->gcr, 128);
3369 return rc ? -EFAULT : 0;
3370 }
3371
3372 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
3373 {
3374 /*
3375 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
3376 * switch in the run ioctl. Let's update our copies before we save
3377 * it into the save area
3378 */
3379 save_fpu_regs();
3380 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3381 save_access_regs(vcpu->run->s.regs.acrs);
3382
3383 return kvm_s390_store_status_unloaded(vcpu, addr);
3384 }
3385
3386 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3387 {
3388 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
3389 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
3390 }
3391
3392 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
3393 {
3394 unsigned int i;
3395 struct kvm_vcpu *vcpu;
3396
3397 kvm_for_each_vcpu(i, vcpu, kvm) {
3398 __disable_ibs_on_vcpu(vcpu);
3399 }
3400 }
3401
3402 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3403 {
3404 if (!sclp.has_ibs)
3405 return;
3406 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
3407 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
3408 }
3409
3410 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
3411 {
3412 int i, online_vcpus, started_vcpus = 0;
3413
3414 if (!is_vcpu_stopped(vcpu))
3415 return;
3416
3417 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
3418 /* Only one cpu at a time may enter/leave the STOPPED state. */
3419 spin_lock(&vcpu->kvm->arch.start_stop_lock);
3420 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3421
3422 for (i = 0; i < online_vcpus; i++) {
3423 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
3424 started_vcpus++;
3425 }
3426
3427 if (started_vcpus == 0) {
3428 /* we're the only active VCPU -> speed it up */
3429 __enable_ibs_on_vcpu(vcpu);
3430 } else if (started_vcpus == 1) {
3431 /*
3432 * As we are starting a second VCPU, we have to disable
3433 * the IBS facility on all VCPUs to remove potentially
3434 * oustanding ENABLE requests.
3435 */
3436 __disable_ibs_on_all_vcpus(vcpu->kvm);
3437 }
3438
3439 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
3440 /*
3441 * Another VCPU might have used IBS while we were offline.
3442 * Let's play safe and flush the VCPU at startup.
3443 */
3444 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3445 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
3446 return;
3447 }
3448
3449 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
3450 {
3451 int i, online_vcpus, started_vcpus = 0;
3452 struct kvm_vcpu *started_vcpu = NULL;
3453
3454 if (is_vcpu_stopped(vcpu))
3455 return;
3456
3457 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
3458 /* Only one cpu at a time may enter/leave the STOPPED state. */
3459 spin_lock(&vcpu->kvm->arch.start_stop_lock);
3460 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3461
3462 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
3463 kvm_s390_clear_stop_irq(vcpu);
3464
3465 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
3466 __disable_ibs_on_vcpu(vcpu);
3467
3468 for (i = 0; i < online_vcpus; i++) {
3469 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
3470 started_vcpus++;
3471 started_vcpu = vcpu->kvm->vcpus[i];
3472 }
3473 }
3474
3475 if (started_vcpus == 1) {
3476 /*
3477 * As we only have one VCPU left, we want to enable the
3478 * IBS facility for that VCPU to speed it up.
3479 */
3480 __enable_ibs_on_vcpu(started_vcpu);
3481 }
3482
3483 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
3484 return;
3485 }
3486
3487 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
3488 struct kvm_enable_cap *cap)
3489 {
3490 int r;
3491
3492 if (cap->flags)
3493 return -EINVAL;
3494
3495 switch (cap->cap) {
3496 case KVM_CAP_S390_CSS_SUPPORT:
3497 if (!vcpu->kvm->arch.css_support) {
3498 vcpu->kvm->arch.css_support = 1;
3499 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
3500 trace_kvm_s390_enable_css(vcpu->kvm);
3501 }
3502 r = 0;
3503 break;
3504 default:
3505 r = -EINVAL;
3506 break;
3507 }
3508 return r;
3509 }
3510
3511 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
3512 struct kvm_s390_mem_op *mop)
3513 {
3514 void __user *uaddr = (void __user *)mop->buf;
3515 void *tmpbuf = NULL;
3516 int r, srcu_idx;
3517 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
3518 | KVM_S390_MEMOP_F_CHECK_ONLY;
3519
3520 if (mop->flags & ~supported_flags)
3521 return -EINVAL;
3522
3523 if (mop->size > MEM_OP_MAX_SIZE)
3524 return -E2BIG;
3525
3526 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
3527 tmpbuf = vmalloc(mop->size);
3528 if (!tmpbuf)
3529 return -ENOMEM;
3530 }
3531
3532 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3533
3534 switch (mop->op) {
3535 case KVM_S390_MEMOP_LOGICAL_READ:
3536 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
3537 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3538 mop->size, GACC_FETCH);
3539 break;
3540 }
3541 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3542 if (r == 0) {
3543 if (copy_to_user(uaddr, tmpbuf, mop->size))
3544 r = -EFAULT;
3545 }
3546 break;
3547 case KVM_S390_MEMOP_LOGICAL_WRITE:
3548 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
3549 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3550 mop->size, GACC_STORE);
3551 break;
3552 }
3553 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
3554 r = -EFAULT;
3555 break;
3556 }
3557 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3558 break;
3559 default:
3560 r = -EINVAL;
3561 }
3562
3563 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
3564
3565 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
3566 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
3567
3568 vfree(tmpbuf);
3569 return r;
3570 }
3571
3572 long kvm_arch_vcpu_ioctl(struct file *filp,
3573 unsigned int ioctl, unsigned long arg)
3574 {
3575 struct kvm_vcpu *vcpu = filp->private_data;
3576 void __user *argp = (void __user *)arg;
3577 int idx;
3578 long r;
3579
3580 switch (ioctl) {
3581 case KVM_S390_IRQ: {
3582 struct kvm_s390_irq s390irq;
3583
3584 r = -EFAULT;
3585 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
3586 break;
3587 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3588 break;
3589 }
3590 case KVM_S390_INTERRUPT: {
3591 struct kvm_s390_interrupt s390int;
3592 struct kvm_s390_irq s390irq;
3593
3594 r = -EFAULT;
3595 if (copy_from_user(&s390int, argp, sizeof(s390int)))
3596 break;
3597 if (s390int_to_s390irq(&s390int, &s390irq))
3598 return -EINVAL;
3599 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3600 break;
3601 }
3602 case KVM_S390_STORE_STATUS:
3603 idx = srcu_read_lock(&vcpu->kvm->srcu);
3604 r = kvm_s390_vcpu_store_status(vcpu, arg);
3605 srcu_read_unlock(&vcpu->kvm->srcu, idx);
3606 break;
3607 case KVM_S390_SET_INITIAL_PSW: {
3608 psw_t psw;
3609
3610 r = -EFAULT;
3611 if (copy_from_user(&psw, argp, sizeof(psw)))
3612 break;
3613 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3614 break;
3615 }
3616 case KVM_S390_INITIAL_RESET:
3617 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3618 break;
3619 case KVM_SET_ONE_REG:
3620 case KVM_GET_ONE_REG: {
3621 struct kvm_one_reg reg;
3622 r = -EFAULT;
3623 if (copy_from_user(&reg, argp, sizeof(reg)))
3624 break;
3625 if (ioctl == KVM_SET_ONE_REG)
3626 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3627 else
3628 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3629 break;
3630 }
3631 #ifdef CONFIG_KVM_S390_UCONTROL
3632 case KVM_S390_UCAS_MAP: {
3633 struct kvm_s390_ucas_mapping ucasmap;
3634
3635 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3636 r = -EFAULT;
3637 break;
3638 }
3639
3640 if (!kvm_is_ucontrol(vcpu->kvm)) {
3641 r = -EINVAL;
3642 break;
3643 }
3644
3645 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3646 ucasmap.vcpu_addr, ucasmap.length);
3647 break;
3648 }
3649 case KVM_S390_UCAS_UNMAP: {
3650 struct kvm_s390_ucas_mapping ucasmap;
3651
3652 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3653 r = -EFAULT;
3654 break;
3655 }
3656
3657 if (!kvm_is_ucontrol(vcpu->kvm)) {
3658 r = -EINVAL;
3659 break;
3660 }
3661
3662 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3663 ucasmap.length);
3664 break;
3665 }
3666 #endif
3667 case KVM_S390_VCPU_FAULT: {
3668 r = gmap_fault(vcpu->arch.gmap, arg, 0);
3669 break;
3670 }
3671 case KVM_ENABLE_CAP:
3672 {
3673 struct kvm_enable_cap cap;
3674 r = -EFAULT;
3675 if (copy_from_user(&cap, argp, sizeof(cap)))
3676 break;
3677 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3678 break;
3679 }
3680 case KVM_S390_MEM_OP: {
3681 struct kvm_s390_mem_op mem_op;
3682
3683 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3684 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3685 else
3686 r = -EFAULT;
3687 break;
3688 }
3689 case KVM_S390_SET_IRQ_STATE: {
3690 struct kvm_s390_irq_state irq_state;
3691
3692 r = -EFAULT;
3693 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3694 break;
3695 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3696 irq_state.len == 0 ||
3697 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3698 r = -EINVAL;
3699 break;
3700 }
3701 r = kvm_s390_set_irq_state(vcpu,
3702 (void __user *) irq_state.buf,
3703 irq_state.len);
3704 break;
3705 }
3706 case KVM_S390_GET_IRQ_STATE: {
3707 struct kvm_s390_irq_state irq_state;
3708
3709 r = -EFAULT;
3710 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3711 break;
3712 if (irq_state.len == 0) {
3713 r = -EINVAL;
3714 break;
3715 }
3716 r = kvm_s390_get_irq_state(vcpu,
3717 (__u8 __user *) irq_state.buf,
3718 irq_state.len);
3719 break;
3720 }
3721 default:
3722 r = -ENOTTY;
3723 }
3724 return r;
3725 }
3726
3727 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3728 {
3729 #ifdef CONFIG_KVM_S390_UCONTROL
3730 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3731 && (kvm_is_ucontrol(vcpu->kvm))) {
3732 vmf->page = virt_to_page(vcpu->arch.sie_block);
3733 get_page(vmf->page);
3734 return 0;
3735 }
3736 #endif
3737 return VM_FAULT_SIGBUS;
3738 }
3739
3740 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3741 unsigned long npages)
3742 {
3743 return 0;
3744 }
3745
3746 /* Section: memory related */
3747 int kvm_arch_prepare_memory_region(struct kvm *kvm,
3748 struct kvm_memory_slot *memslot,
3749 const struct kvm_userspace_memory_region *mem,
3750 enum kvm_mr_change change)
3751 {
3752 /* A few sanity checks. We can have memory slots which have to be
3753 located/ended at a segment boundary (1MB). The memory in userland is
3754 ok to be fragmented into various different vmas. It is okay to mmap()
3755 and munmap() stuff in this slot after doing this call at any time */
3756
3757 if (mem->userspace_addr & 0xffffful)
3758 return -EINVAL;
3759
3760 if (mem->memory_size & 0xffffful)
3761 return -EINVAL;
3762
3763 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3764 return -EINVAL;
3765
3766 return 0;
3767 }
3768
3769 void kvm_arch_commit_memory_region(struct kvm *kvm,
3770 const struct kvm_userspace_memory_region *mem,
3771 const struct kvm_memory_slot *old,
3772 const struct kvm_memory_slot *new,
3773 enum kvm_mr_change change)
3774 {
3775 int rc;
3776
3777 /* If the basics of the memslot do not change, we do not want
3778 * to update the gmap. Every update causes several unnecessary
3779 * segment translation exceptions. This is usually handled just
3780 * fine by the normal fault handler + gmap, but it will also
3781 * cause faults on the prefix page of running guest CPUs.
3782 */
3783 if (old->userspace_addr == mem->userspace_addr &&
3784 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3785 old->npages * PAGE_SIZE == mem->memory_size)
3786 return;
3787
3788 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3789 mem->guest_phys_addr, mem->memory_size);
3790 if (rc)
3791 pr_warn("failed to commit memory region\n");
3792 return;
3793 }
3794
3795 static inline unsigned long nonhyp_mask(int i)
3796 {
3797 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3798
3799 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3800 }
3801
3802 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3803 {
3804 vcpu->valid_wakeup = false;
3805 }
3806
3807 static int __init kvm_s390_init(void)
3808 {
3809 int i;
3810
3811 if (!sclp.has_sief2) {
3812 pr_info("SIE not available\n");
3813 return -ENODEV;
3814 }
3815
3816 for (i = 0; i < 16; i++)
3817 kvm_s390_fac_list_mask[i] |=
3818 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3819
3820 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
3821 }
3822
3823 static void __exit kvm_s390_exit(void)
3824 {
3825 kvm_exit();
3826 }
3827
3828 module_init(kvm_s390_init);
3829 module_exit(kvm_s390_exit);
3830
3831 /*
3832 * Enable autoloading of the kvm module.
3833 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3834 * since x86 takes a different approach.
3835 */
3836 #include <linux/miscdevice.h>
3837 MODULE_ALIAS_MISCDEV(KVM_MINOR);
3838 MODULE_ALIAS("devname:kvm");
Ubuntu Artful kernel mirror