]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-jammy-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
KVM: s390: Machine Check
[mirror_ubuntu-jammy-kernel.git] / arch / s390 / kvm / kvm-s390.c
1 /*
2 * hosting zSeries kernel virtual machines
3 *
4 * Copyright IBM Corp. 2008, 2009
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
15 */
16
17 #include <linux/compiler.h>
18 #include <linux/err.h>
19 #include <linux/fs.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/module.h>
25 #include <linux/random.h>
26 #include <linux/slab.h>
27 #include <linux/timer.h>
28 #include <asm/asm-offsets.h>
29 #include <asm/lowcore.h>
30 #include <asm/pgtable.h>
31 #include <asm/nmi.h>
32 #include <asm/switch_to.h>
33 #include <asm/sclp.h>
34 #include "kvm-s390.h"
35 #include "gaccess.h"
36
37 #define CREATE_TRACE_POINTS
38 #include "trace.h"
39 #include "trace-s390.h"
40
41 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
42
43 struct kvm_stats_debugfs_item debugfs_entries[] = {
44 { "userspace_handled", VCPU_STAT(exit_userspace) },
45 { "exit_null", VCPU_STAT(exit_null) },
46 { "exit_validity", VCPU_STAT(exit_validity) },
47 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
48 { "exit_external_request", VCPU_STAT(exit_external_request) },
49 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
50 { "exit_instruction", VCPU_STAT(exit_instruction) },
51 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
52 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
53 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
54 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
55 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
56 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
57 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
58 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
59 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
60 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
61 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
62 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
63 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
64 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
65 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
66 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
67 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
68 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
69 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
70 { "instruction_spx", VCPU_STAT(instruction_spx) },
71 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
72 { "instruction_stap", VCPU_STAT(instruction_stap) },
73 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
74 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
75 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
76 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
77 { "instruction_essa", VCPU_STAT(instruction_essa) },
78 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
79 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
80 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
81 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
82 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
83 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
84 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
85 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
86 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
87 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
88 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
89 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
90 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
91 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
92 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
93 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
94 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
95 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
96 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
97 { "diagnose_10", VCPU_STAT(diagnose_10) },
98 { "diagnose_44", VCPU_STAT(diagnose_44) },
99 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
100 { NULL }
101 };
102
103 /* upper facilities limit for kvm */
104 unsigned long kvm_s390_fac_list_mask[] = {
105 0xff82fffbf4fc2000UL,
106 0x005c000000000000UL,
107 };
108
109 unsigned long kvm_s390_fac_list_mask_size(void)
110 {
111 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
112 return ARRAY_SIZE(kvm_s390_fac_list_mask);
113 }
114
115 static struct gmap_notifier gmap_notifier;
116
117 /* Section: not file related */
118 int kvm_arch_hardware_enable(void)
119 {
120 /* every s390 is virtualization enabled ;-) */
121 return 0;
122 }
123
124 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
125
126 int kvm_arch_hardware_setup(void)
127 {
128 gmap_notifier.notifier_call = kvm_gmap_notifier;
129 gmap_register_ipte_notifier(&gmap_notifier);
130 return 0;
131 }
132
133 void kvm_arch_hardware_unsetup(void)
134 {
135 gmap_unregister_ipte_notifier(&gmap_notifier);
136 }
137
138 int kvm_arch_init(void *opaque)
139 {
140 /* Register floating interrupt controller interface. */
141 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
142 }
143
144 /* Section: device related */
145 long kvm_arch_dev_ioctl(struct file *filp,
146 unsigned int ioctl, unsigned long arg)
147 {
148 if (ioctl == KVM_S390_ENABLE_SIE)
149 return s390_enable_sie();
150 return -EINVAL;
151 }
152
153 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
154 {
155 int r;
156
157 switch (ext) {
158 case KVM_CAP_S390_PSW:
159 case KVM_CAP_S390_GMAP:
160 case KVM_CAP_SYNC_MMU:
161 #ifdef CONFIG_KVM_S390_UCONTROL
162 case KVM_CAP_S390_UCONTROL:
163 #endif
164 case KVM_CAP_ASYNC_PF:
165 case KVM_CAP_SYNC_REGS:
166 case KVM_CAP_ONE_REG:
167 case KVM_CAP_ENABLE_CAP:
168 case KVM_CAP_S390_CSS_SUPPORT:
169 case KVM_CAP_IRQFD:
170 case KVM_CAP_IOEVENTFD:
171 case KVM_CAP_DEVICE_CTRL:
172 case KVM_CAP_ENABLE_CAP_VM:
173 case KVM_CAP_S390_IRQCHIP:
174 case KVM_CAP_VM_ATTRIBUTES:
175 case KVM_CAP_MP_STATE:
176 case KVM_CAP_S390_USER_SIGP:
177 r = 1;
178 break;
179 case KVM_CAP_NR_VCPUS:
180 case KVM_CAP_MAX_VCPUS:
181 r = KVM_MAX_VCPUS;
182 break;
183 case KVM_CAP_NR_MEMSLOTS:
184 r = KVM_USER_MEM_SLOTS;
185 break;
186 case KVM_CAP_S390_COW:
187 r = MACHINE_HAS_ESOP;
188 break;
189 case KVM_CAP_S390_VECTOR_REGISTERS:
190 r = MACHINE_HAS_VX;
191 break;
192 default:
193 r = 0;
194 }
195 return r;
196 }
197
198 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
199 struct kvm_memory_slot *memslot)
200 {
201 gfn_t cur_gfn, last_gfn;
202 unsigned long address;
203 struct gmap *gmap = kvm->arch.gmap;
204
205 down_read(&gmap->mm->mmap_sem);
206 /* Loop over all guest pages */
207 last_gfn = memslot->base_gfn + memslot->npages;
208 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
209 address = gfn_to_hva_memslot(memslot, cur_gfn);
210
211 if (gmap_test_and_clear_dirty(address, gmap))
212 mark_page_dirty(kvm, cur_gfn);
213 }
214 up_read(&gmap->mm->mmap_sem);
215 }
216
217 /* Section: vm related */
218 /*
219 * Get (and clear) the dirty memory log for a memory slot.
220 */
221 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
222 struct kvm_dirty_log *log)
223 {
224 int r;
225 unsigned long n;
226 struct kvm_memory_slot *memslot;
227 int is_dirty = 0;
228
229 mutex_lock(&kvm->slots_lock);
230
231 r = -EINVAL;
232 if (log->slot >= KVM_USER_MEM_SLOTS)
233 goto out;
234
235 memslot = id_to_memslot(kvm->memslots, log->slot);
236 r = -ENOENT;
237 if (!memslot->dirty_bitmap)
238 goto out;
239
240 kvm_s390_sync_dirty_log(kvm, memslot);
241 r = kvm_get_dirty_log(kvm, log, &is_dirty);
242 if (r)
243 goto out;
244
245 /* Clear the dirty log */
246 if (is_dirty) {
247 n = kvm_dirty_bitmap_bytes(memslot);
248 memset(memslot->dirty_bitmap, 0, n);
249 }
250 r = 0;
251 out:
252 mutex_unlock(&kvm->slots_lock);
253 return r;
254 }
255
256 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
257 {
258 int r;
259
260 if (cap->flags)
261 return -EINVAL;
262
263 switch (cap->cap) {
264 case KVM_CAP_S390_IRQCHIP:
265 kvm->arch.use_irqchip = 1;
266 r = 0;
267 break;
268 case KVM_CAP_S390_USER_SIGP:
269 kvm->arch.user_sigp = 1;
270 r = 0;
271 break;
272 case KVM_CAP_S390_VECTOR_REGISTERS:
273 kvm->arch.use_vectors = MACHINE_HAS_VX;
274 r = MACHINE_HAS_VX ? 0 : -EINVAL;
275 break;
276 default:
277 r = -EINVAL;
278 break;
279 }
280 return r;
281 }
282
283 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
284 {
285 int ret;
286
287 switch (attr->attr) {
288 case KVM_S390_VM_MEM_LIMIT_SIZE:
289 ret = 0;
290 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
291 ret = -EFAULT;
292 break;
293 default:
294 ret = -ENXIO;
295 break;
296 }
297 return ret;
298 }
299
300 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
301 {
302 int ret;
303 unsigned int idx;
304 switch (attr->attr) {
305 case KVM_S390_VM_MEM_ENABLE_CMMA:
306 ret = -EBUSY;
307 mutex_lock(&kvm->lock);
308 if (atomic_read(&kvm->online_vcpus) == 0) {
309 kvm->arch.use_cmma = 1;
310 ret = 0;
311 }
312 mutex_unlock(&kvm->lock);
313 break;
314 case KVM_S390_VM_MEM_CLR_CMMA:
315 mutex_lock(&kvm->lock);
316 idx = srcu_read_lock(&kvm->srcu);
317 s390_reset_cmma(kvm->arch.gmap->mm);
318 srcu_read_unlock(&kvm->srcu, idx);
319 mutex_unlock(&kvm->lock);
320 ret = 0;
321 break;
322 case KVM_S390_VM_MEM_LIMIT_SIZE: {
323 unsigned long new_limit;
324
325 if (kvm_is_ucontrol(kvm))
326 return -EINVAL;
327
328 if (get_user(new_limit, (u64 __user *)attr->addr))
329 return -EFAULT;
330
331 if (new_limit > kvm->arch.gmap->asce_end)
332 return -E2BIG;
333
334 ret = -EBUSY;
335 mutex_lock(&kvm->lock);
336 if (atomic_read(&kvm->online_vcpus) == 0) {
337 /* gmap_alloc will round the limit up */
338 struct gmap *new = gmap_alloc(current->mm, new_limit);
339
340 if (!new) {
341 ret = -ENOMEM;
342 } else {
343 gmap_free(kvm->arch.gmap);
344 new->private = kvm;
345 kvm->arch.gmap = new;
346 ret = 0;
347 }
348 }
349 mutex_unlock(&kvm->lock);
350 break;
351 }
352 default:
353 ret = -ENXIO;
354 break;
355 }
356 return ret;
357 }
358
359 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
360
361 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
362 {
363 struct kvm_vcpu *vcpu;
364 int i;
365
366 if (!test_kvm_facility(kvm, 76))
367 return -EINVAL;
368
369 mutex_lock(&kvm->lock);
370 switch (attr->attr) {
371 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
372 get_random_bytes(
373 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
374 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
375 kvm->arch.crypto.aes_kw = 1;
376 break;
377 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
378 get_random_bytes(
379 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
380 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
381 kvm->arch.crypto.dea_kw = 1;
382 break;
383 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
384 kvm->arch.crypto.aes_kw = 0;
385 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
386 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
387 break;
388 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
389 kvm->arch.crypto.dea_kw = 0;
390 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
391 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
392 break;
393 default:
394 mutex_unlock(&kvm->lock);
395 return -ENXIO;
396 }
397
398 kvm_for_each_vcpu(i, vcpu, kvm) {
399 kvm_s390_vcpu_crypto_setup(vcpu);
400 exit_sie(vcpu);
401 }
402 mutex_unlock(&kvm->lock);
403 return 0;
404 }
405
406 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
407 {
408 u8 gtod_high;
409
410 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
411 sizeof(gtod_high)))
412 return -EFAULT;
413
414 if (gtod_high != 0)
415 return -EINVAL;
416
417 return 0;
418 }
419
420 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
421 {
422 struct kvm_vcpu *cur_vcpu;
423 unsigned int vcpu_idx;
424 u64 host_tod, gtod;
425 int r;
426
427 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
428 return -EFAULT;
429
430 r = store_tod_clock(&host_tod);
431 if (r)
432 return r;
433
434 mutex_lock(&kvm->lock);
435 kvm->arch.epoch = gtod - host_tod;
436 kvm_for_each_vcpu(vcpu_idx, cur_vcpu, kvm) {
437 cur_vcpu->arch.sie_block->epoch = kvm->arch.epoch;
438 exit_sie(cur_vcpu);
439 }
440 mutex_unlock(&kvm->lock);
441 return 0;
442 }
443
444 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
445 {
446 int ret;
447
448 if (attr->flags)
449 return -EINVAL;
450
451 switch (attr->attr) {
452 case KVM_S390_VM_TOD_HIGH:
453 ret = kvm_s390_set_tod_high(kvm, attr);
454 break;
455 case KVM_S390_VM_TOD_LOW:
456 ret = kvm_s390_set_tod_low(kvm, attr);
457 break;
458 default:
459 ret = -ENXIO;
460 break;
461 }
462 return ret;
463 }
464
465 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
466 {
467 u8 gtod_high = 0;
468
469 if (copy_to_user((void __user *)attr->addr, &gtod_high,
470 sizeof(gtod_high)))
471 return -EFAULT;
472
473 return 0;
474 }
475
476 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
477 {
478 u64 host_tod, gtod;
479 int r;
480
481 r = store_tod_clock(&host_tod);
482 if (r)
483 return r;
484
485 gtod = host_tod + kvm->arch.epoch;
486 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
487 return -EFAULT;
488
489 return 0;
490 }
491
492 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
493 {
494 int ret;
495
496 if (attr->flags)
497 return -EINVAL;
498
499 switch (attr->attr) {
500 case KVM_S390_VM_TOD_HIGH:
501 ret = kvm_s390_get_tod_high(kvm, attr);
502 break;
503 case KVM_S390_VM_TOD_LOW:
504 ret = kvm_s390_get_tod_low(kvm, attr);
505 break;
506 default:
507 ret = -ENXIO;
508 break;
509 }
510 return ret;
511 }
512
513 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
514 {
515 struct kvm_s390_vm_cpu_processor *proc;
516 int ret = 0;
517
518 mutex_lock(&kvm->lock);
519 if (atomic_read(&kvm->online_vcpus)) {
520 ret = -EBUSY;
521 goto out;
522 }
523 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
524 if (!proc) {
525 ret = -ENOMEM;
526 goto out;
527 }
528 if (!copy_from_user(proc, (void __user *)attr->addr,
529 sizeof(*proc))) {
530 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
531 sizeof(struct cpuid));
532 kvm->arch.model.ibc = proc->ibc;
533 memcpy(kvm->arch.model.fac->list, proc->fac_list,
534 S390_ARCH_FAC_LIST_SIZE_BYTE);
535 } else
536 ret = -EFAULT;
537 kfree(proc);
538 out:
539 mutex_unlock(&kvm->lock);
540 return ret;
541 }
542
543 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
544 {
545 int ret = -ENXIO;
546
547 switch (attr->attr) {
548 case KVM_S390_VM_CPU_PROCESSOR:
549 ret = kvm_s390_set_processor(kvm, attr);
550 break;
551 }
552 return ret;
553 }
554
555 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
556 {
557 struct kvm_s390_vm_cpu_processor *proc;
558 int ret = 0;
559
560 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
561 if (!proc) {
562 ret = -ENOMEM;
563 goto out;
564 }
565 memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
566 proc->ibc = kvm->arch.model.ibc;
567 memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
568 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
569 ret = -EFAULT;
570 kfree(proc);
571 out:
572 return ret;
573 }
574
575 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
576 {
577 struct kvm_s390_vm_cpu_machine *mach;
578 int ret = 0;
579
580 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
581 if (!mach) {
582 ret = -ENOMEM;
583 goto out;
584 }
585 get_cpu_id((struct cpuid *) &mach->cpuid);
586 mach->ibc = sclp_get_ibc();
587 memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
588 S390_ARCH_FAC_LIST_SIZE_BYTE);
589 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
590 S390_ARCH_FAC_LIST_SIZE_BYTE);
591 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
592 ret = -EFAULT;
593 kfree(mach);
594 out:
595 return ret;
596 }
597
598 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
599 {
600 int ret = -ENXIO;
601
602 switch (attr->attr) {
603 case KVM_S390_VM_CPU_PROCESSOR:
604 ret = kvm_s390_get_processor(kvm, attr);
605 break;
606 case KVM_S390_VM_CPU_MACHINE:
607 ret = kvm_s390_get_machine(kvm, attr);
608 break;
609 }
610 return ret;
611 }
612
613 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
614 {
615 int ret;
616
617 switch (attr->group) {
618 case KVM_S390_VM_MEM_CTRL:
619 ret = kvm_s390_set_mem_control(kvm, attr);
620 break;
621 case KVM_S390_VM_TOD:
622 ret = kvm_s390_set_tod(kvm, attr);
623 break;
624 case KVM_S390_VM_CPU_MODEL:
625 ret = kvm_s390_set_cpu_model(kvm, attr);
626 break;
627 case KVM_S390_VM_CRYPTO:
628 ret = kvm_s390_vm_set_crypto(kvm, attr);
629 break;
630 default:
631 ret = -ENXIO;
632 break;
633 }
634
635 return ret;
636 }
637
638 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
639 {
640 int ret;
641
642 switch (attr->group) {
643 case KVM_S390_VM_MEM_CTRL:
644 ret = kvm_s390_get_mem_control(kvm, attr);
645 break;
646 case KVM_S390_VM_TOD:
647 ret = kvm_s390_get_tod(kvm, attr);
648 break;
649 case KVM_S390_VM_CPU_MODEL:
650 ret = kvm_s390_get_cpu_model(kvm, attr);
651 break;
652 default:
653 ret = -ENXIO;
654 break;
655 }
656
657 return ret;
658 }
659
660 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
661 {
662 int ret;
663
664 switch (attr->group) {
665 case KVM_S390_VM_MEM_CTRL:
666 switch (attr->attr) {
667 case KVM_S390_VM_MEM_ENABLE_CMMA:
668 case KVM_S390_VM_MEM_CLR_CMMA:
669 case KVM_S390_VM_MEM_LIMIT_SIZE:
670 ret = 0;
671 break;
672 default:
673 ret = -ENXIO;
674 break;
675 }
676 break;
677 case KVM_S390_VM_TOD:
678 switch (attr->attr) {
679 case KVM_S390_VM_TOD_LOW:
680 case KVM_S390_VM_TOD_HIGH:
681 ret = 0;
682 break;
683 default:
684 ret = -ENXIO;
685 break;
686 }
687 break;
688 case KVM_S390_VM_CPU_MODEL:
689 switch (attr->attr) {
690 case KVM_S390_VM_CPU_PROCESSOR:
691 case KVM_S390_VM_CPU_MACHINE:
692 ret = 0;
693 break;
694 default:
695 ret = -ENXIO;
696 break;
697 }
698 break;
699 case KVM_S390_VM_CRYPTO:
700 switch (attr->attr) {
701 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
702 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
703 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
704 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
705 ret = 0;
706 break;
707 default:
708 ret = -ENXIO;
709 break;
710 }
711 break;
712 default:
713 ret = -ENXIO;
714 break;
715 }
716
717 return ret;
718 }
719
720 long kvm_arch_vm_ioctl(struct file *filp,
721 unsigned int ioctl, unsigned long arg)
722 {
723 struct kvm *kvm = filp->private_data;
724 void __user *argp = (void __user *)arg;
725 struct kvm_device_attr attr;
726 int r;
727
728 switch (ioctl) {
729 case KVM_S390_INTERRUPT: {
730 struct kvm_s390_interrupt s390int;
731
732 r = -EFAULT;
733 if (copy_from_user(&s390int, argp, sizeof(s390int)))
734 break;
735 r = kvm_s390_inject_vm(kvm, &s390int);
736 break;
737 }
738 case KVM_ENABLE_CAP: {
739 struct kvm_enable_cap cap;
740 r = -EFAULT;
741 if (copy_from_user(&cap, argp, sizeof(cap)))
742 break;
743 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
744 break;
745 }
746 case KVM_CREATE_IRQCHIP: {
747 struct kvm_irq_routing_entry routing;
748
749 r = -EINVAL;
750 if (kvm->arch.use_irqchip) {
751 /* Set up dummy routing. */
752 memset(&routing, 0, sizeof(routing));
753 kvm_set_irq_routing(kvm, &routing, 0, 0);
754 r = 0;
755 }
756 break;
757 }
758 case KVM_SET_DEVICE_ATTR: {
759 r = -EFAULT;
760 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
761 break;
762 r = kvm_s390_vm_set_attr(kvm, &attr);
763 break;
764 }
765 case KVM_GET_DEVICE_ATTR: {
766 r = -EFAULT;
767 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
768 break;
769 r = kvm_s390_vm_get_attr(kvm, &attr);
770 break;
771 }
772 case KVM_HAS_DEVICE_ATTR: {
773 r = -EFAULT;
774 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
775 break;
776 r = kvm_s390_vm_has_attr(kvm, &attr);
777 break;
778 }
779 default:
780 r = -ENOTTY;
781 }
782
783 return r;
784 }
785
786 static int kvm_s390_query_ap_config(u8 *config)
787 {
788 u32 fcn_code = 0x04000000UL;
789 u32 cc = 0;
790
791 memset(config, 0, 128);
792 asm volatile(
793 "lgr 0,%1\n"
794 "lgr 2,%2\n"
795 ".long 0xb2af0000\n" /* PQAP(QCI) */
796 "0: ipm %0\n"
797 "srl %0,28\n"
798 "1:\n"
799 EX_TABLE(0b, 1b)
800 : "+r" (cc)
801 : "r" (fcn_code), "r" (config)
802 : "cc", "0", "2", "memory"
803 );
804
805 return cc;
806 }
807
808 static int kvm_s390_apxa_installed(void)
809 {
810 u8 config[128];
811 int cc;
812
813 if (test_facility(2) && test_facility(12)) {
814 cc = kvm_s390_query_ap_config(config);
815
816 if (cc)
817 pr_err("PQAP(QCI) failed with cc=%d", cc);
818 else
819 return config[0] & 0x40;
820 }
821
822 return 0;
823 }
824
825 static void kvm_s390_set_crycb_format(struct kvm *kvm)
826 {
827 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
828
829 if (kvm_s390_apxa_installed())
830 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
831 else
832 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
833 }
834
835 static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
836 {
837 get_cpu_id(cpu_id);
838 cpu_id->version = 0xff;
839 }
840
841 static int kvm_s390_crypto_init(struct kvm *kvm)
842 {
843 if (!test_kvm_facility(kvm, 76))
844 return 0;
845
846 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
847 GFP_KERNEL | GFP_DMA);
848 if (!kvm->arch.crypto.crycb)
849 return -ENOMEM;
850
851 kvm_s390_set_crycb_format(kvm);
852
853 /* Enable AES/DEA protected key functions by default */
854 kvm->arch.crypto.aes_kw = 1;
855 kvm->arch.crypto.dea_kw = 1;
856 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
857 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
858 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
859 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
860
861 return 0;
862 }
863
864 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
865 {
866 int i, rc;
867 char debug_name[16];
868 static unsigned long sca_offset;
869
870 rc = -EINVAL;
871 #ifdef CONFIG_KVM_S390_UCONTROL
872 if (type & ~KVM_VM_S390_UCONTROL)
873 goto out_err;
874 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
875 goto out_err;
876 #else
877 if (type)
878 goto out_err;
879 #endif
880
881 rc = s390_enable_sie();
882 if (rc)
883 goto out_err;
884
885 rc = -ENOMEM;
886
887 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
888 if (!kvm->arch.sca)
889 goto out_err;
890 spin_lock(&kvm_lock);
891 sca_offset = (sca_offset + 16) & 0x7f0;
892 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
893 spin_unlock(&kvm_lock);
894
895 sprintf(debug_name, "kvm-%u", current->pid);
896
897 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
898 if (!kvm->arch.dbf)
899 goto out_nodbf;
900
901 /*
902 * The architectural maximum amount of facilities is 16 kbit. To store
903 * this amount, 2 kbyte of memory is required. Thus we need a full
904 * page to hold the guest facility list (arch.model.fac->list) and the
905 * facility mask (arch.model.fac->mask). Its address size has to be
906 * 31 bits and word aligned.
907 */
908 kvm->arch.model.fac =
909 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
910 if (!kvm->arch.model.fac)
911 goto out_nofac;
912
913 /* Populate the facility mask initially. */
914 memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
915 S390_ARCH_FAC_LIST_SIZE_BYTE);
916 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
917 if (i < kvm_s390_fac_list_mask_size())
918 kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
919 else
920 kvm->arch.model.fac->mask[i] = 0UL;
921 }
922
923 /* Populate the facility list initially. */
924 memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
925 S390_ARCH_FAC_LIST_SIZE_BYTE);
926
927 kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
928 kvm->arch.model.ibc = sclp_get_ibc() & 0x0fff;
929
930 if (kvm_s390_crypto_init(kvm) < 0)
931 goto out_crypto;
932
933 spin_lock_init(&kvm->arch.float_int.lock);
934 INIT_LIST_HEAD(&kvm->arch.float_int.list);
935 init_waitqueue_head(&kvm->arch.ipte_wq);
936 mutex_init(&kvm->arch.ipte_mutex);
937
938 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
939 VM_EVENT(kvm, 3, "%s", "vm created");
940
941 if (type & KVM_VM_S390_UCONTROL) {
942 kvm->arch.gmap = NULL;
943 } else {
944 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
945 if (!kvm->arch.gmap)
946 goto out_nogmap;
947 kvm->arch.gmap->private = kvm;
948 kvm->arch.gmap->pfault_enabled = 0;
949 }
950
951 kvm->arch.css_support = 0;
952 kvm->arch.use_irqchip = 0;
953 kvm->arch.use_vectors = 0;
954 kvm->arch.epoch = 0;
955
956 spin_lock_init(&kvm->arch.start_stop_lock);
957
958 return 0;
959 out_nogmap:
960 kfree(kvm->arch.crypto.crycb);
961 out_crypto:
962 free_page((unsigned long)kvm->arch.model.fac);
963 out_nofac:
964 debug_unregister(kvm->arch.dbf);
965 out_nodbf:
966 free_page((unsigned long)(kvm->arch.sca));
967 out_err:
968 return rc;
969 }
970
971 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
972 {
973 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
974 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
975 kvm_s390_clear_local_irqs(vcpu);
976 kvm_clear_async_pf_completion_queue(vcpu);
977 if (!kvm_is_ucontrol(vcpu->kvm)) {
978 clear_bit(63 - vcpu->vcpu_id,
979 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
980 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
981 (__u64) vcpu->arch.sie_block)
982 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
983 }
984 smp_mb();
985
986 if (kvm_is_ucontrol(vcpu->kvm))
987 gmap_free(vcpu->arch.gmap);
988
989 if (kvm_s390_cmma_enabled(vcpu->kvm))
990 kvm_s390_vcpu_unsetup_cmma(vcpu);
991 free_page((unsigned long)(vcpu->arch.sie_block));
992
993 kvm_vcpu_uninit(vcpu);
994 kmem_cache_free(kvm_vcpu_cache, vcpu);
995 }
996
997 static void kvm_free_vcpus(struct kvm *kvm)
998 {
999 unsigned int i;
1000 struct kvm_vcpu *vcpu;
1001
1002 kvm_for_each_vcpu(i, vcpu, kvm)
1003 kvm_arch_vcpu_destroy(vcpu);
1004
1005 mutex_lock(&kvm->lock);
1006 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1007 kvm->vcpus[i] = NULL;
1008
1009 atomic_set(&kvm->online_vcpus, 0);
1010 mutex_unlock(&kvm->lock);
1011 }
1012
1013 void kvm_arch_destroy_vm(struct kvm *kvm)
1014 {
1015 kvm_free_vcpus(kvm);
1016 free_page((unsigned long)kvm->arch.model.fac);
1017 free_page((unsigned long)(kvm->arch.sca));
1018 debug_unregister(kvm->arch.dbf);
1019 kfree(kvm->arch.crypto.crycb);
1020 if (!kvm_is_ucontrol(kvm))
1021 gmap_free(kvm->arch.gmap);
1022 kvm_s390_destroy_adapters(kvm);
1023 kvm_s390_clear_float_irqs(kvm);
1024 }
1025
1026 /* Section: vcpu related */
1027 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1028 {
1029 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1030 if (!vcpu->arch.gmap)
1031 return -ENOMEM;
1032 vcpu->arch.gmap->private = vcpu->kvm;
1033
1034 return 0;
1035 }
1036
1037 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1038 {
1039 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1040 kvm_clear_async_pf_completion_queue(vcpu);
1041 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1042 KVM_SYNC_GPRS |
1043 KVM_SYNC_ACRS |
1044 KVM_SYNC_CRS |
1045 KVM_SYNC_ARCH0 |
1046 KVM_SYNC_PFAULT;
1047 if (test_kvm_facility(vcpu->kvm, 129))
1048 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1049
1050 if (kvm_is_ucontrol(vcpu->kvm))
1051 return __kvm_ucontrol_vcpu_init(vcpu);
1052
1053 return 0;
1054 }
1055
1056 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1057 {
1058 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1059 if (vcpu->kvm->arch.use_vectors)
1060 save_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
1061 else
1062 save_fp_regs(vcpu->arch.host_fpregs.fprs);
1063 save_access_regs(vcpu->arch.host_acrs);
1064 if (vcpu->kvm->arch.use_vectors) {
1065 restore_fp_ctl(&vcpu->run->s.regs.fpc);
1066 restore_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
1067 } else {
1068 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1069 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1070 }
1071 restore_access_regs(vcpu->run->s.regs.acrs);
1072 gmap_enable(vcpu->arch.gmap);
1073 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1074 }
1075
1076 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1077 {
1078 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1079 gmap_disable(vcpu->arch.gmap);
1080 if (vcpu->kvm->arch.use_vectors) {
1081 save_fp_ctl(&vcpu->run->s.regs.fpc);
1082 save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
1083 } else {
1084 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1085 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1086 }
1087 save_access_regs(vcpu->run->s.regs.acrs);
1088 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1089 if (vcpu->kvm->arch.use_vectors)
1090 restore_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
1091 else
1092 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
1093 restore_access_regs(vcpu->arch.host_acrs);
1094 }
1095
1096 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1097 {
1098 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1099 vcpu->arch.sie_block->gpsw.mask = 0UL;
1100 vcpu->arch.sie_block->gpsw.addr = 0UL;
1101 kvm_s390_set_prefix(vcpu, 0);
1102 vcpu->arch.sie_block->cputm = 0UL;
1103 vcpu->arch.sie_block->ckc = 0UL;
1104 vcpu->arch.sie_block->todpr = 0;
1105 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1106 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1107 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1108 vcpu->arch.guest_fpregs.fpc = 0;
1109 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
1110 vcpu->arch.sie_block->gbea = 1;
1111 vcpu->arch.sie_block->pp = 0;
1112 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1113 kvm_clear_async_pf_completion_queue(vcpu);
1114 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1115 kvm_s390_vcpu_stop(vcpu);
1116 kvm_s390_clear_local_irqs(vcpu);
1117 }
1118
1119 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1120 {
1121 mutex_lock(&vcpu->kvm->lock);
1122 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1123 mutex_unlock(&vcpu->kvm->lock);
1124 if (!kvm_is_ucontrol(vcpu->kvm))
1125 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1126 }
1127
1128 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1129 {
1130 if (!test_kvm_facility(vcpu->kvm, 76))
1131 return;
1132
1133 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1134
1135 if (vcpu->kvm->arch.crypto.aes_kw)
1136 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1137 if (vcpu->kvm->arch.crypto.dea_kw)
1138 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1139
1140 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1141 }
1142
1143 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1144 {
1145 free_page(vcpu->arch.sie_block->cbrlo);
1146 vcpu->arch.sie_block->cbrlo = 0;
1147 }
1148
1149 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1150 {
1151 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1152 if (!vcpu->arch.sie_block->cbrlo)
1153 return -ENOMEM;
1154
1155 vcpu->arch.sie_block->ecb2 |= 0x80;
1156 vcpu->arch.sie_block->ecb2 &= ~0x08;
1157 return 0;
1158 }
1159
1160 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1161 {
1162 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1163
1164 vcpu->arch.cpu_id = model->cpu_id;
1165 vcpu->arch.sie_block->ibc = model->ibc;
1166 vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1167 }
1168
1169 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1170 {
1171 int rc = 0;
1172
1173 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1174 CPUSTAT_SM |
1175 CPUSTAT_STOPPED |
1176 CPUSTAT_GED);
1177 kvm_s390_vcpu_setup_model(vcpu);
1178
1179 vcpu->arch.sie_block->ecb = 6;
1180 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1181 vcpu->arch.sie_block->ecb |= 0x10;
1182
1183 vcpu->arch.sie_block->ecb2 = 8;
1184 vcpu->arch.sie_block->eca = 0xC1002000U;
1185 if (sclp_has_siif())
1186 vcpu->arch.sie_block->eca |= 1;
1187 if (sclp_has_sigpif())
1188 vcpu->arch.sie_block->eca |= 0x10000000U;
1189 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1190
1191 if (kvm_s390_cmma_enabled(vcpu->kvm)) {
1192 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1193 if (rc)
1194 return rc;
1195 }
1196 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1197 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1198
1199 kvm_s390_vcpu_crypto_setup(vcpu);
1200
1201 return rc;
1202 }
1203
1204 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1205 unsigned int id)
1206 {
1207 struct kvm_vcpu *vcpu;
1208 struct sie_page *sie_page;
1209 int rc = -EINVAL;
1210
1211 if (id >= KVM_MAX_VCPUS)
1212 goto out;
1213
1214 rc = -ENOMEM;
1215
1216 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1217 if (!vcpu)
1218 goto out;
1219
1220 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1221 if (!sie_page)
1222 goto out_free_cpu;
1223
1224 vcpu->arch.sie_block = &sie_page->sie_block;
1225 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1226 vcpu->arch.host_vregs = &sie_page->vregs;
1227
1228 vcpu->arch.sie_block->icpua = id;
1229 if (!kvm_is_ucontrol(kvm)) {
1230 if (!kvm->arch.sca) {
1231 WARN_ON_ONCE(1);
1232 goto out_free_cpu;
1233 }
1234 if (!kvm->arch.sca->cpu[id].sda)
1235 kvm->arch.sca->cpu[id].sda =
1236 (__u64) vcpu->arch.sie_block;
1237 vcpu->arch.sie_block->scaoh =
1238 (__u32)(((__u64)kvm->arch.sca) >> 32);
1239 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
1240 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
1241 }
1242
1243 spin_lock_init(&vcpu->arch.local_int.lock);
1244 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1245 vcpu->arch.local_int.wq = &vcpu->wq;
1246 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1247
1248 rc = kvm_vcpu_init(vcpu, kvm, id);
1249 if (rc)
1250 goto out_free_sie_block;
1251 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
1252 vcpu->arch.sie_block);
1253 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1254
1255 return vcpu;
1256 out_free_sie_block:
1257 free_page((unsigned long)(vcpu->arch.sie_block));
1258 out_free_cpu:
1259 kmem_cache_free(kvm_vcpu_cache, vcpu);
1260 out:
1261 return ERR_PTR(rc);
1262 }
1263
1264 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1265 {
1266 return kvm_s390_vcpu_has_irq(vcpu, 0);
1267 }
1268
1269 void s390_vcpu_block(struct kvm_vcpu *vcpu)
1270 {
1271 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1272 }
1273
1274 void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1275 {
1276 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1277 }
1278
1279 /*
1280 * Kick a guest cpu out of SIE and wait until SIE is not running.
1281 * If the CPU is not running (e.g. waiting as idle) the function will
1282 * return immediately. */
1283 void exit_sie(struct kvm_vcpu *vcpu)
1284 {
1285 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1286 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1287 cpu_relax();
1288 }
1289
1290 /* Kick a guest cpu out of SIE and prevent SIE-reentry */
1291 void exit_sie_sync(struct kvm_vcpu *vcpu)
1292 {
1293 s390_vcpu_block(vcpu);
1294 exit_sie(vcpu);
1295 }
1296
1297 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1298 {
1299 int i;
1300 struct kvm *kvm = gmap->private;
1301 struct kvm_vcpu *vcpu;
1302
1303 kvm_for_each_vcpu(i, vcpu, kvm) {
1304 /* match against both prefix pages */
1305 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
1306 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1307 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
1308 exit_sie_sync(vcpu);
1309 }
1310 }
1311 }
1312
1313 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1314 {
1315 /* kvm common code refers to this, but never calls it */
1316 BUG();
1317 return 0;
1318 }
1319
1320 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1321 struct kvm_one_reg *reg)
1322 {
1323 int r = -EINVAL;
1324
1325 switch (reg->id) {
1326 case KVM_REG_S390_TODPR:
1327 r = put_user(vcpu->arch.sie_block->todpr,
1328 (u32 __user *)reg->addr);
1329 break;
1330 case KVM_REG_S390_EPOCHDIFF:
1331 r = put_user(vcpu->arch.sie_block->epoch,
1332 (u64 __user *)reg->addr);
1333 break;
1334 case KVM_REG_S390_CPU_TIMER:
1335 r = put_user(vcpu->arch.sie_block->cputm,
1336 (u64 __user *)reg->addr);
1337 break;
1338 case KVM_REG_S390_CLOCK_COMP:
1339 r = put_user(vcpu->arch.sie_block->ckc,
1340 (u64 __user *)reg->addr);
1341 break;
1342 case KVM_REG_S390_PFTOKEN:
1343 r = put_user(vcpu->arch.pfault_token,
1344 (u64 __user *)reg->addr);
1345 break;
1346 case KVM_REG_S390_PFCOMPARE:
1347 r = put_user(vcpu->arch.pfault_compare,
1348 (u64 __user *)reg->addr);
1349 break;
1350 case KVM_REG_S390_PFSELECT:
1351 r = put_user(vcpu->arch.pfault_select,
1352 (u64 __user *)reg->addr);
1353 break;
1354 case KVM_REG_S390_PP:
1355 r = put_user(vcpu->arch.sie_block->pp,
1356 (u64 __user *)reg->addr);
1357 break;
1358 case KVM_REG_S390_GBEA:
1359 r = put_user(vcpu->arch.sie_block->gbea,
1360 (u64 __user *)reg->addr);
1361 break;
1362 default:
1363 break;
1364 }
1365
1366 return r;
1367 }
1368
1369 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1370 struct kvm_one_reg *reg)
1371 {
1372 int r = -EINVAL;
1373
1374 switch (reg->id) {
1375 case KVM_REG_S390_TODPR:
1376 r = get_user(vcpu->arch.sie_block->todpr,
1377 (u32 __user *)reg->addr);
1378 break;
1379 case KVM_REG_S390_EPOCHDIFF:
1380 r = get_user(vcpu->arch.sie_block->epoch,
1381 (u64 __user *)reg->addr);
1382 break;
1383 case KVM_REG_S390_CPU_TIMER:
1384 r = get_user(vcpu->arch.sie_block->cputm,
1385 (u64 __user *)reg->addr);
1386 break;
1387 case KVM_REG_S390_CLOCK_COMP:
1388 r = get_user(vcpu->arch.sie_block->ckc,
1389 (u64 __user *)reg->addr);
1390 break;
1391 case KVM_REG_S390_PFTOKEN:
1392 r = get_user(vcpu->arch.pfault_token,
1393 (u64 __user *)reg->addr);
1394 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1395 kvm_clear_async_pf_completion_queue(vcpu);
1396 break;
1397 case KVM_REG_S390_PFCOMPARE:
1398 r = get_user(vcpu->arch.pfault_compare,
1399 (u64 __user *)reg->addr);
1400 break;
1401 case KVM_REG_S390_PFSELECT:
1402 r = get_user(vcpu->arch.pfault_select,
1403 (u64 __user *)reg->addr);
1404 break;
1405 case KVM_REG_S390_PP:
1406 r = get_user(vcpu->arch.sie_block->pp,
1407 (u64 __user *)reg->addr);
1408 break;
1409 case KVM_REG_S390_GBEA:
1410 r = get_user(vcpu->arch.sie_block->gbea,
1411 (u64 __user *)reg->addr);
1412 break;
1413 default:
1414 break;
1415 }
1416
1417 return r;
1418 }
1419
1420 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1421 {
1422 kvm_s390_vcpu_initial_reset(vcpu);
1423 return 0;
1424 }
1425
1426 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1427 {
1428 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
1429 return 0;
1430 }
1431
1432 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1433 {
1434 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1435 return 0;
1436 }
1437
1438 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1439 struct kvm_sregs *sregs)
1440 {
1441 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
1442 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
1443 restore_access_regs(vcpu->run->s.regs.acrs);
1444 return 0;
1445 }
1446
1447 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1448 struct kvm_sregs *sregs)
1449 {
1450 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
1451 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
1452 return 0;
1453 }
1454
1455 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1456 {
1457 if (test_fp_ctl(fpu->fpc))
1458 return -EINVAL;
1459 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
1460 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1461 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1462 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1463 return 0;
1464 }
1465
1466 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1467 {
1468 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1469 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
1470 return 0;
1471 }
1472
1473 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1474 {
1475 int rc = 0;
1476
1477 if (!is_vcpu_stopped(vcpu))
1478 rc = -EBUSY;
1479 else {
1480 vcpu->run->psw_mask = psw.mask;
1481 vcpu->run->psw_addr = psw.addr;
1482 }
1483 return rc;
1484 }
1485
1486 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1487 struct kvm_translation *tr)
1488 {
1489 return -EINVAL; /* not implemented yet */
1490 }
1491
1492 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1493 KVM_GUESTDBG_USE_HW_BP | \
1494 KVM_GUESTDBG_ENABLE)
1495
1496 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1497 struct kvm_guest_debug *dbg)
1498 {
1499 int rc = 0;
1500
1501 vcpu->guest_debug = 0;
1502 kvm_s390_clear_bp_data(vcpu);
1503
1504 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
1505 return -EINVAL;
1506
1507 if (dbg->control & KVM_GUESTDBG_ENABLE) {
1508 vcpu->guest_debug = dbg->control;
1509 /* enforce guest PER */
1510 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1511
1512 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1513 rc = kvm_s390_import_bp_data(vcpu, dbg);
1514 } else {
1515 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1516 vcpu->arch.guestdbg.last_bp = 0;
1517 }
1518
1519 if (rc) {
1520 vcpu->guest_debug = 0;
1521 kvm_s390_clear_bp_data(vcpu);
1522 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1523 }
1524
1525 return rc;
1526 }
1527
1528 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1529 struct kvm_mp_state *mp_state)
1530 {
1531 /* CHECK_STOP and LOAD are not supported yet */
1532 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1533 KVM_MP_STATE_OPERATING;
1534 }
1535
1536 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1537 struct kvm_mp_state *mp_state)
1538 {
1539 int rc = 0;
1540
1541 /* user space knows about this interface - let it control the state */
1542 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1543
1544 switch (mp_state->mp_state) {
1545 case KVM_MP_STATE_STOPPED:
1546 kvm_s390_vcpu_stop(vcpu);
1547 break;
1548 case KVM_MP_STATE_OPERATING:
1549 kvm_s390_vcpu_start(vcpu);
1550 break;
1551 case KVM_MP_STATE_LOAD:
1552 case KVM_MP_STATE_CHECK_STOP:
1553 /* fall through - CHECK_STOP and LOAD are not supported yet */
1554 default:
1555 rc = -ENXIO;
1556 }
1557
1558 return rc;
1559 }
1560
1561 bool kvm_s390_cmma_enabled(struct kvm *kvm)
1562 {
1563 if (!MACHINE_IS_LPAR)
1564 return false;
1565 /* only enable for z10 and later */
1566 if (!MACHINE_HAS_EDAT1)
1567 return false;
1568 if (!kvm->arch.use_cmma)
1569 return false;
1570 return true;
1571 }
1572
1573 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1574 {
1575 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1576 }
1577
1578 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1579 {
1580 retry:
1581 s390_vcpu_unblock(vcpu);
1582 /*
1583 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1584 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1585 * This ensures that the ipte instruction for this request has
1586 * already finished. We might race against a second unmapper that
1587 * wants to set the blocking bit. Lets just retry the request loop.
1588 */
1589 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1590 int rc;
1591 rc = gmap_ipte_notify(vcpu->arch.gmap,
1592 kvm_s390_get_prefix(vcpu),
1593 PAGE_SIZE * 2);
1594 if (rc)
1595 return rc;
1596 goto retry;
1597 }
1598
1599 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1600 vcpu->arch.sie_block->ihcpu = 0xffff;
1601 goto retry;
1602 }
1603
1604 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1605 if (!ibs_enabled(vcpu)) {
1606 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1607 atomic_set_mask(CPUSTAT_IBS,
1608 &vcpu->arch.sie_block->cpuflags);
1609 }
1610 goto retry;
1611 }
1612
1613 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1614 if (ibs_enabled(vcpu)) {
1615 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1616 atomic_clear_mask(CPUSTAT_IBS,
1617 &vcpu->arch.sie_block->cpuflags);
1618 }
1619 goto retry;
1620 }
1621
1622 /* nothing to do, just clear the request */
1623 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1624
1625 return 0;
1626 }
1627
1628 /**
1629 * kvm_arch_fault_in_page - fault-in guest page if necessary
1630 * @vcpu: The corresponding virtual cpu
1631 * @gpa: Guest physical address
1632 * @writable: Whether the page should be writable or not
1633 *
1634 * Make sure that a guest page has been faulted-in on the host.
1635 *
1636 * Return: Zero on success, negative error code otherwise.
1637 */
1638 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
1639 {
1640 return gmap_fault(vcpu->arch.gmap, gpa,
1641 writable ? FAULT_FLAG_WRITE : 0);
1642 }
1643
1644 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1645 unsigned long token)
1646 {
1647 struct kvm_s390_interrupt inti;
1648 struct kvm_s390_irq irq;
1649
1650 if (start_token) {
1651 irq.u.ext.ext_params2 = token;
1652 irq.type = KVM_S390_INT_PFAULT_INIT;
1653 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
1654 } else {
1655 inti.type = KVM_S390_INT_PFAULT_DONE;
1656 inti.parm64 = token;
1657 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1658 }
1659 }
1660
1661 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1662 struct kvm_async_pf *work)
1663 {
1664 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1665 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1666 }
1667
1668 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1669 struct kvm_async_pf *work)
1670 {
1671 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1672 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1673 }
1674
1675 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1676 struct kvm_async_pf *work)
1677 {
1678 /* s390 will always inject the page directly */
1679 }
1680
1681 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1682 {
1683 /*
1684 * s390 will always inject the page directly,
1685 * but we still want check_async_completion to cleanup
1686 */
1687 return true;
1688 }
1689
1690 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1691 {
1692 hva_t hva;
1693 struct kvm_arch_async_pf arch;
1694 int rc;
1695
1696 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1697 return 0;
1698 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1699 vcpu->arch.pfault_compare)
1700 return 0;
1701 if (psw_extint_disabled(vcpu))
1702 return 0;
1703 if (kvm_s390_vcpu_has_irq(vcpu, 0))
1704 return 0;
1705 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1706 return 0;
1707 if (!vcpu->arch.gmap->pfault_enabled)
1708 return 0;
1709
1710 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1711 hva += current->thread.gmap_addr & ~PAGE_MASK;
1712 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
1713 return 0;
1714
1715 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1716 return rc;
1717 }
1718
1719 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
1720 {
1721 int rc, cpuflags;
1722
1723 /*
1724 * On s390 notifications for arriving pages will be delivered directly
1725 * to the guest but the house keeping for completed pfaults is
1726 * handled outside the worker.
1727 */
1728 kvm_check_async_pf_completion(vcpu);
1729
1730 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
1731
1732 if (need_resched())
1733 schedule();
1734
1735 if (test_cpu_flag(CIF_MCCK_PENDING))
1736 s390_handle_mcck();
1737
1738 if (!kvm_is_ucontrol(vcpu->kvm)) {
1739 rc = kvm_s390_deliver_pending_interrupts(vcpu);
1740 if (rc)
1741 return rc;
1742 }
1743
1744 rc = kvm_s390_handle_requests(vcpu);
1745 if (rc)
1746 return rc;
1747
1748 if (guestdbg_enabled(vcpu)) {
1749 kvm_s390_backup_guest_per_regs(vcpu);
1750 kvm_s390_patch_guest_per_regs(vcpu);
1751 }
1752
1753 vcpu->arch.sie_block->icptcode = 0;
1754 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1755 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1756 trace_kvm_s390_sie_enter(vcpu, cpuflags);
1757
1758 return 0;
1759 }
1760
1761 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
1762 {
1763 psw_t *psw = &vcpu->arch.sie_block->gpsw;
1764 u8 opcode;
1765 int rc;
1766
1767 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1768 trace_kvm_s390_sie_fault(vcpu);
1769
1770 /*
1771 * We want to inject an addressing exception, which is defined as a
1772 * suppressing or terminating exception. However, since we came here
1773 * by a DAT access exception, the PSW still points to the faulting
1774 * instruction since DAT exceptions are nullifying. So we've got
1775 * to look up the current opcode to get the length of the instruction
1776 * to be able to forward the PSW.
1777 */
1778 rc = read_guest(vcpu, psw->addr, &opcode, 1);
1779 if (rc)
1780 return kvm_s390_inject_prog_cond(vcpu, rc);
1781 psw->addr = __rewind_psw(*psw, -insn_length(opcode));
1782
1783 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1784 }
1785
1786 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1787 {
1788 int rc = -1;
1789
1790 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1791 vcpu->arch.sie_block->icptcode);
1792 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1793
1794 if (guestdbg_enabled(vcpu))
1795 kvm_s390_restore_guest_per_regs(vcpu);
1796
1797 if (exit_reason >= 0) {
1798 rc = 0;
1799 } else if (kvm_is_ucontrol(vcpu->kvm)) {
1800 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1801 vcpu->run->s390_ucontrol.trans_exc_code =
1802 current->thread.gmap_addr;
1803 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1804 rc = -EREMOTE;
1805
1806 } else if (current->thread.gmap_pfault) {
1807 trace_kvm_s390_major_guest_pfault(vcpu);
1808 current->thread.gmap_pfault = 0;
1809 if (kvm_arch_setup_async_pf(vcpu)) {
1810 rc = 0;
1811 } else {
1812 gpa_t gpa = current->thread.gmap_addr;
1813 rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1814 }
1815 }
1816
1817 if (rc == -1)
1818 rc = vcpu_post_run_fault_in_sie(vcpu);
1819
1820 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
1821
1822 if (rc == 0) {
1823 if (kvm_is_ucontrol(vcpu->kvm))
1824 /* Don't exit for host interrupts. */
1825 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
1826 else
1827 rc = kvm_handle_sie_intercept(vcpu);
1828 }
1829
1830 return rc;
1831 }
1832
1833 static int __vcpu_run(struct kvm_vcpu *vcpu)
1834 {
1835 int rc, exit_reason;
1836
1837 /*
1838 * We try to hold kvm->srcu during most of vcpu_run (except when run-
1839 * ning the guest), so that memslots (and other stuff) are protected
1840 */
1841 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1842
1843 do {
1844 rc = vcpu_pre_run(vcpu);
1845 if (rc)
1846 break;
1847
1848 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1849 /*
1850 * As PF_VCPU will be used in fault handler, between
1851 * guest_enter and guest_exit should be no uaccess.
1852 */
1853 preempt_disable();
1854 kvm_guest_enter();
1855 preempt_enable();
1856 exit_reason = sie64a(vcpu->arch.sie_block,
1857 vcpu->run->s.regs.gprs);
1858 kvm_guest_exit();
1859 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1860
1861 rc = vcpu_post_run(vcpu, exit_reason);
1862 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
1863
1864 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1865 return rc;
1866 }
1867
1868 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1869 {
1870 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1871 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
1872 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
1873 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1874 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1875 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
1876 /* some control register changes require a tlb flush */
1877 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1878 }
1879 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
1880 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
1881 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
1882 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
1883 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
1884 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
1885 }
1886 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
1887 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
1888 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
1889 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
1890 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1891 kvm_clear_async_pf_completion_queue(vcpu);
1892 }
1893 kvm_run->kvm_dirty_regs = 0;
1894 }
1895
1896 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1897 {
1898 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
1899 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
1900 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
1901 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
1902 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
1903 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
1904 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
1905 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
1906 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
1907 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
1908 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
1909 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
1910 }
1911
1912 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1913 {
1914 int rc;
1915 sigset_t sigsaved;
1916
1917 if (guestdbg_exit_pending(vcpu)) {
1918 kvm_s390_prepare_debug_exit(vcpu);
1919 return 0;
1920 }
1921
1922 if (vcpu->sigset_active)
1923 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1924
1925 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
1926 kvm_s390_vcpu_start(vcpu);
1927 } else if (is_vcpu_stopped(vcpu)) {
1928 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
1929 vcpu->vcpu_id);
1930 return -EINVAL;
1931 }
1932
1933 sync_regs(vcpu, kvm_run);
1934
1935 might_fault();
1936 rc = __vcpu_run(vcpu);
1937
1938 if (signal_pending(current) && !rc) {
1939 kvm_run->exit_reason = KVM_EXIT_INTR;
1940 rc = -EINTR;
1941 }
1942
1943 if (guestdbg_exit_pending(vcpu) && !rc) {
1944 kvm_s390_prepare_debug_exit(vcpu);
1945 rc = 0;
1946 }
1947
1948 if (rc == -EOPNOTSUPP) {
1949 /* intercept cannot be handled in-kernel, prepare kvm-run */
1950 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
1951 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
1952 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
1953 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
1954 rc = 0;
1955 }
1956
1957 if (rc == -EREMOTE) {
1958 /* intercept was handled, but userspace support is needed
1959 * kvm_run has been prepared by the handler */
1960 rc = 0;
1961 }
1962
1963 store_regs(vcpu, kvm_run);
1964
1965 if (vcpu->sigset_active)
1966 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1967
1968 vcpu->stat.exit_userspace++;
1969 return rc;
1970 }
1971
1972 /*
1973 * store status at address
1974 * we use have two special cases:
1975 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1976 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1977 */
1978 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
1979 {
1980 unsigned char archmode = 1;
1981 unsigned int px;
1982 u64 clkcomp;
1983 int rc;
1984
1985 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
1986 if (write_guest_abs(vcpu, 163, &archmode, 1))
1987 return -EFAULT;
1988 gpa = SAVE_AREA_BASE;
1989 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
1990 if (write_guest_real(vcpu, 163, &archmode, 1))
1991 return -EFAULT;
1992 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
1993 }
1994 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
1995 vcpu->arch.guest_fpregs.fprs, 128);
1996 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
1997 vcpu->run->s.regs.gprs, 128);
1998 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
1999 &vcpu->arch.sie_block->gpsw, 16);
2000 px = kvm_s390_get_prefix(vcpu);
2001 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
2002 &px, 4);
2003 rc |= write_guest_abs(vcpu,
2004 gpa + offsetof(struct save_area, fp_ctrl_reg),
2005 &vcpu->arch.guest_fpregs.fpc, 4);
2006 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
2007 &vcpu->arch.sie_block->todpr, 4);
2008 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
2009 &vcpu->arch.sie_block->cputm, 8);
2010 clkcomp = vcpu->arch.sie_block->ckc >> 8;
2011 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
2012 &clkcomp, 8);
2013 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
2014 &vcpu->run->s.regs.acrs, 64);
2015 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
2016 &vcpu->arch.sie_block->gcr, 128);
2017 return rc ? -EFAULT : 0;
2018 }
2019
2020 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2021 {
2022 /*
2023 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2024 * copying in vcpu load/put. Lets update our copies before we save
2025 * it into the save area
2026 */
2027 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
2028 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
2029 save_access_regs(vcpu->run->s.regs.acrs);
2030
2031 return kvm_s390_store_status_unloaded(vcpu, addr);
2032 }
2033
2034 /*
2035 * store additional status at address
2036 */
2037 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2038 unsigned long gpa)
2039 {
2040 /* Only bits 0-53 are used for address formation */
2041 if (!(gpa & ~0x3ff))
2042 return 0;
2043
2044 return write_guest_abs(vcpu, gpa & ~0x3ff,
2045 (void *)&vcpu->run->s.regs.vrs, 512);
2046 }
2047
2048 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2049 {
2050 if (!test_kvm_facility(vcpu->kvm, 129))
2051 return 0;
2052
2053 /*
2054 * The guest VXRS are in the host VXRs due to the lazy
2055 * copying in vcpu load/put. Let's update our copies before we save
2056 * it into the save area.
2057 */
2058 save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
2059
2060 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2061 }
2062
2063 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2064 {
2065 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2066 kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu);
2067 exit_sie_sync(vcpu);
2068 }
2069
2070 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2071 {
2072 unsigned int i;
2073 struct kvm_vcpu *vcpu;
2074
2075 kvm_for_each_vcpu(i, vcpu, kvm) {
2076 __disable_ibs_on_vcpu(vcpu);
2077 }
2078 }
2079
2080 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2081 {
2082 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2083 kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu);
2084 exit_sie_sync(vcpu);
2085 }
2086
2087 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2088 {
2089 int i, online_vcpus, started_vcpus = 0;
2090
2091 if (!is_vcpu_stopped(vcpu))
2092 return;
2093
2094 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2095 /* Only one cpu at a time may enter/leave the STOPPED state. */
2096 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2097 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2098
2099 for (i = 0; i < online_vcpus; i++) {
2100 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2101 started_vcpus++;
2102 }
2103
2104 if (started_vcpus == 0) {
2105 /* we're the only active VCPU -> speed it up */
2106 __enable_ibs_on_vcpu(vcpu);
2107 } else if (started_vcpus == 1) {
2108 /*
2109 * As we are starting a second VCPU, we have to disable
2110 * the IBS facility on all VCPUs to remove potentially
2111 * oustanding ENABLE requests.
2112 */
2113 __disable_ibs_on_all_vcpus(vcpu->kvm);
2114 }
2115
2116 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2117 /*
2118 * Another VCPU might have used IBS while we were offline.
2119 * Let's play safe and flush the VCPU at startup.
2120 */
2121 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2122 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2123 return;
2124 }
2125
2126 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2127 {
2128 int i, online_vcpus, started_vcpus = 0;
2129 struct kvm_vcpu *started_vcpu = NULL;
2130
2131 if (is_vcpu_stopped(vcpu))
2132 return;
2133
2134 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2135 /* Only one cpu at a time may enter/leave the STOPPED state. */
2136 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2137 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2138
2139 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2140 kvm_s390_clear_stop_irq(vcpu);
2141
2142 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2143 __disable_ibs_on_vcpu(vcpu);
2144
2145 for (i = 0; i < online_vcpus; i++) {
2146 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2147 started_vcpus++;
2148 started_vcpu = vcpu->kvm->vcpus[i];
2149 }
2150 }
2151
2152 if (started_vcpus == 1) {
2153 /*
2154 * As we only have one VCPU left, we want to enable the
2155 * IBS facility for that VCPU to speed it up.
2156 */
2157 __enable_ibs_on_vcpu(started_vcpu);
2158 }
2159
2160 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2161 return;
2162 }
2163
2164 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2165 struct kvm_enable_cap *cap)
2166 {
2167 int r;
2168
2169 if (cap->flags)
2170 return -EINVAL;
2171
2172 switch (cap->cap) {
2173 case KVM_CAP_S390_CSS_SUPPORT:
2174 if (!vcpu->kvm->arch.css_support) {
2175 vcpu->kvm->arch.css_support = 1;
2176 trace_kvm_s390_enable_css(vcpu->kvm);
2177 }
2178 r = 0;
2179 break;
2180 default:
2181 r = -EINVAL;
2182 break;
2183 }
2184 return r;
2185 }
2186
2187 long kvm_arch_vcpu_ioctl(struct file *filp,
2188 unsigned int ioctl, unsigned long arg)
2189 {
2190 struct kvm_vcpu *vcpu = filp->private_data;
2191 void __user *argp = (void __user *)arg;
2192 int idx;
2193 long r;
2194
2195 switch (ioctl) {
2196 case KVM_S390_INTERRUPT: {
2197 struct kvm_s390_interrupt s390int;
2198 struct kvm_s390_irq s390irq;
2199
2200 r = -EFAULT;
2201 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2202 break;
2203 if (s390int_to_s390irq(&s390int, &s390irq))
2204 return -EINVAL;
2205 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2206 break;
2207 }
2208 case KVM_S390_STORE_STATUS:
2209 idx = srcu_read_lock(&vcpu->kvm->srcu);
2210 r = kvm_s390_vcpu_store_status(vcpu, arg);
2211 srcu_read_unlock(&vcpu->kvm->srcu, idx);
2212 break;
2213 case KVM_S390_SET_INITIAL_PSW: {
2214 psw_t psw;
2215
2216 r = -EFAULT;
2217 if (copy_from_user(&psw, argp, sizeof(psw)))
2218 break;
2219 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2220 break;
2221 }
2222 case KVM_S390_INITIAL_RESET:
2223 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2224 break;
2225 case KVM_SET_ONE_REG:
2226 case KVM_GET_ONE_REG: {
2227 struct kvm_one_reg reg;
2228 r = -EFAULT;
2229 if (copy_from_user(&reg, argp, sizeof(reg)))
2230 break;
2231 if (ioctl == KVM_SET_ONE_REG)
2232 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
2233 else
2234 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
2235 break;
2236 }
2237 #ifdef CONFIG_KVM_S390_UCONTROL
2238 case KVM_S390_UCAS_MAP: {
2239 struct kvm_s390_ucas_mapping ucasmap;
2240
2241 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2242 r = -EFAULT;
2243 break;
2244 }
2245
2246 if (!kvm_is_ucontrol(vcpu->kvm)) {
2247 r = -EINVAL;
2248 break;
2249 }
2250
2251 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2252 ucasmap.vcpu_addr, ucasmap.length);
2253 break;
2254 }
2255 case KVM_S390_UCAS_UNMAP: {
2256 struct kvm_s390_ucas_mapping ucasmap;
2257
2258 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2259 r = -EFAULT;
2260 break;
2261 }
2262
2263 if (!kvm_is_ucontrol(vcpu->kvm)) {
2264 r = -EINVAL;
2265 break;
2266 }
2267
2268 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2269 ucasmap.length);
2270 break;
2271 }
2272 #endif
2273 case KVM_S390_VCPU_FAULT: {
2274 r = gmap_fault(vcpu->arch.gmap, arg, 0);
2275 break;
2276 }
2277 case KVM_ENABLE_CAP:
2278 {
2279 struct kvm_enable_cap cap;
2280 r = -EFAULT;
2281 if (copy_from_user(&cap, argp, sizeof(cap)))
2282 break;
2283 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2284 break;
2285 }
2286 default:
2287 r = -ENOTTY;
2288 }
2289 return r;
2290 }
2291
2292 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2293 {
2294 #ifdef CONFIG_KVM_S390_UCONTROL
2295 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2296 && (kvm_is_ucontrol(vcpu->kvm))) {
2297 vmf->page = virt_to_page(vcpu->arch.sie_block);
2298 get_page(vmf->page);
2299 return 0;
2300 }
2301 #endif
2302 return VM_FAULT_SIGBUS;
2303 }
2304
2305 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2306 unsigned long npages)
2307 {
2308 return 0;
2309 }
2310
2311 /* Section: memory related */
2312 int kvm_arch_prepare_memory_region(struct kvm *kvm,
2313 struct kvm_memory_slot *memslot,
2314 struct kvm_userspace_memory_region *mem,
2315 enum kvm_mr_change change)
2316 {
2317 /* A few sanity checks. We can have memory slots which have to be
2318 located/ended at a segment boundary (1MB). The memory in userland is
2319 ok to be fragmented into various different vmas. It is okay to mmap()
2320 and munmap() stuff in this slot after doing this call at any time */
2321
2322 if (mem->userspace_addr & 0xffffful)
2323 return -EINVAL;
2324
2325 if (mem->memory_size & 0xffffful)
2326 return -EINVAL;
2327
2328 return 0;
2329 }
2330
2331 void kvm_arch_commit_memory_region(struct kvm *kvm,
2332 struct kvm_userspace_memory_region *mem,
2333 const struct kvm_memory_slot *old,
2334 enum kvm_mr_change change)
2335 {
2336 int rc;
2337
2338 /* If the basics of the memslot do not change, we do not want
2339 * to update the gmap. Every update causes several unnecessary
2340 * segment translation exceptions. This is usually handled just
2341 * fine by the normal fault handler + gmap, but it will also
2342 * cause faults on the prefix page of running guest CPUs.
2343 */
2344 if (old->userspace_addr == mem->userspace_addr &&
2345 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2346 old->npages * PAGE_SIZE == mem->memory_size)
2347 return;
2348
2349 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2350 mem->guest_phys_addr, mem->memory_size);
2351 if (rc)
2352 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
2353 return;
2354 }
2355
2356 static int __init kvm_s390_init(void)
2357 {
2358 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2359 }
2360
2361 static void __exit kvm_s390_exit(void)
2362 {
2363 kvm_exit();
2364 }
2365
2366 module_init(kvm_s390_init);
2367 module_exit(kvm_s390_exit);
2368
2369 /*
2370 * Enable autoloading of the kvm module.
2371 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2372 * since x86 takes a different approach.
2373 */
2374 #include <linux/miscdevice.h>
2375 MODULE_ALIAS_MISCDEV(KVM_MINOR);
2376 MODULE_ALIAS("devname:kvm");
Ubuntu Jammy Linux kernel mirror