]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: s390: add bitmap for handling cpu-local interrupts
[mirror_ubuntu-jammy-kernel.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
24#include <linux/module.h>
25#include <linux/slab.h>
ba5c1e9b 26#include <linux/timer.h>
cbb870c8 27#include <asm/asm-offsets.h>
b0c632db
HC		 28#include <asm/lowcore.h>
29#include <asm/pgtable.h>
f5daba1d 30#include <asm/nmi.h>
a0616cde 31#include <asm/switch_to.h>
78c4b59f 32#include <asm/facility.h>
1526bf9c 33#include <asm/sclp.h>
8f2abe6a 34#include "kvm-s390.h"
b0c632db
HC		 35#include "gaccess.h"
36
5786fffa
CH		 37#define CREATE_TRACE_POINTS
38#include "trace.h"
ade38c31 39#include "trace-s390.h"
5786fffa 40
b0c632db
HC		 41#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
42
43struct kvm_stats_debugfs_item debugfs_entries[] = {
44 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 45 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 46 { "exit_validity", VCPU_STAT(exit_validity) },
47 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
48 { "exit_external_request", VCPU_STAT(exit_external_request) },
49 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b
CO		 50 { "exit_instruction", VCPU_STAT(exit_instruction) },
51 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
52 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
ce2e4f0b 53 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 54 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 55 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 56 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
57 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 58 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 59 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 60 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
61 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
62 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
63 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
64 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
65 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
66 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 67 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 68 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
69 { "instruction_spx", VCPU_STAT(instruction_spx) },
70 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
71 { "instruction_stap", VCPU_STAT(instruction_stap) },
72 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 73 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 74 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
75 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 76 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 77 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
78 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 79 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
5288fbf0 80 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 81 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 82 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 83 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 84 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
85 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 86 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 87 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
88 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
5288fbf0
CB		 89 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
90 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
91 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 92 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
93 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
94 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 95 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 96 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 97 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
b0c632db
HC		 98 { NULL }
99};
100
78c4b59f 101unsigned long *vfacilities;
2c70fe44 102static struct gmap_notifier gmap_notifier;
b0c632db 103
78c4b59f 104/* test availability of vfacility */
280ef0f1 105int test_vfacility(unsigned long nr)
78c4b59f
MM		 106{
107 return __test_facility(nr, (void *) vfacilities);
108}
109
b0c632db 110/* Section: not file related */
13a34e06 111int kvm_arch_hardware_enable(void)
b0c632db
HC		 112{
113 /* every s390 is virtualization enabled ;-) */
10474ae8 114 return 0;
b0c632db
HC		 115}
116
2c70fe44
CB		 117static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
118
b0c632db
HC		 119int kvm_arch_hardware_setup(void)
120{
2c70fe44
CB		 121 gmap_notifier.notifier_call = kvm_gmap_notifier;
122 gmap_register_ipte_notifier(&gmap_notifier);
b0c632db
HC		 123 return 0;
124}
125
126void kvm_arch_hardware_unsetup(void)
127{
2c70fe44 128 gmap_unregister_ipte_notifier(&gmap_notifier);
b0c632db
HC		 129}
130
b0c632db
HC		 131int kvm_arch_init(void *opaque)
132{
84877d93
CH		 133 /* Register floating interrupt controller interface. */
134 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 135}
136
b0c632db
HC		 137/* Section: device related */
138long kvm_arch_dev_ioctl(struct file *filp,
139 unsigned int ioctl, unsigned long arg)
140{
141 if (ioctl == KVM_S390_ENABLE_SIE)
142 return s390_enable_sie();
143 return -EINVAL;
144}
145
784aa3d7 146int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 147{
d7b0b5eb
CO		 148 int r;
149
2bd0ac4e 150 switch (ext) {
d7b0b5eb 151 case KVM_CAP_S390_PSW:
b6cf8788 152 case KVM_CAP_S390_GMAP:
52e16b18 153 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 154#ifdef CONFIG_KVM_S390_UCONTROL
155 case KVM_CAP_S390_UCONTROL:
156#endif
3c038e6b 157 case KVM_CAP_ASYNC_PF:
60b413c9 158 case KVM_CAP_SYNC_REGS:
14eebd91 159 case KVM_CAP_ONE_REG:
d6712df9 160 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 161 case KVM_CAP_S390_CSS_SUPPORT:
ebc32262 162 case KVM_CAP_IRQFD:
10ccaa1e 163 case KVM_CAP_IOEVENTFD:
c05c4186 164 case KVM_CAP_DEVICE_CTRL:
d938dc55 165 case KVM_CAP_ENABLE_CAP_VM:
78599d90 166 case KVM_CAP_S390_IRQCHIP:
f2061656 167 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 168 case KVM_CAP_MP_STATE:
d7b0b5eb
CO		 169 r = 1;
170 break;
e726b1bd
CB		 171 case KVM_CAP_NR_VCPUS:
172 case KVM_CAP_MAX_VCPUS:
173 r = KVM_MAX_VCPUS;
174 break;
e1e2e605
NW		 175 case KVM_CAP_NR_MEMSLOTS:
176 r = KVM_USER_MEM_SLOTS;
177 break;
1526bf9c 178 case KVM_CAP_S390_COW:
abf09bed 179 r = MACHINE_HAS_ESOP;
1526bf9c 180 break;
2bd0ac4e 181 default:
d7b0b5eb 182 r = 0;
2bd0ac4e 183 }
d7b0b5eb 184 return r;
b0c632db
HC		 185}
186
15f36ebd
JH		 187static void kvm_s390_sync_dirty_log(struct kvm *kvm,
188 struct kvm_memory_slot *memslot)
189{
190 gfn_t cur_gfn, last_gfn;
191 unsigned long address;
192 struct gmap *gmap = kvm->arch.gmap;
193
194 down_read(&gmap->mm->mmap_sem);
195 /* Loop over all guest pages */
196 last_gfn = memslot->base_gfn + memslot->npages;
197 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
198 address = gfn_to_hva_memslot(memslot, cur_gfn);
199
200 if (gmap_test_and_clear_dirty(address, gmap))
201 mark_page_dirty(kvm, cur_gfn);
202 }
203 up_read(&gmap->mm->mmap_sem);
204}
205
b0c632db
HC		 206/* Section: vm related */
207/*
208 * Get (and clear) the dirty memory log for a memory slot.
209 */
210int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
211 struct kvm_dirty_log *log)
212{
15f36ebd
JH		 213 int r;
214 unsigned long n;
215 struct kvm_memory_slot *memslot;
216 int is_dirty = 0;
217
218 mutex_lock(&kvm->slots_lock);
219
220 r = -EINVAL;
221 if (log->slot >= KVM_USER_MEM_SLOTS)
222 goto out;
223
224 memslot = id_to_memslot(kvm->memslots, log->slot);
225 r = -ENOENT;
226 if (!memslot->dirty_bitmap)
227 goto out;
228
229 kvm_s390_sync_dirty_log(kvm, memslot);
230 r = kvm_get_dirty_log(kvm, log, &is_dirty);
231 if (r)
232 goto out;
233
234 /* Clear the dirty log */
235 if (is_dirty) {
236 n = kvm_dirty_bitmap_bytes(memslot);
237 memset(memslot->dirty_bitmap, 0, n);
238 }
239 r = 0;
240out:
241 mutex_unlock(&kvm->slots_lock);
242 return r;
b0c632db
HC		 243}
244
d938dc55
CH		 245static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
246{
247 int r;
248
249 if (cap->flags)
250 return -EINVAL;
251
252 switch (cap->cap) {
84223598
CH		 253 case KVM_CAP_S390_IRQCHIP:
254 kvm->arch.use_irqchip = 1;
255 r = 0;
256 break;
d938dc55
CH		 257 default:
258 r = -EINVAL;
259 break;
260 }
261 return r;
262}
263
4f718eab
DD		 264static int kvm_s390_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
265{
266 int ret;
267 unsigned int idx;
268 switch (attr->attr) {
269 case KVM_S390_VM_MEM_ENABLE_CMMA:
270 ret = -EBUSY;
271 mutex_lock(&kvm->lock);
272 if (atomic_read(&kvm->online_vcpus) == 0) {
273 kvm->arch.use_cmma = 1;
274 ret = 0;
275 }
276 mutex_unlock(&kvm->lock);
277 break;
278 case KVM_S390_VM_MEM_CLR_CMMA:
279 mutex_lock(&kvm->lock);
280 idx = srcu_read_lock(&kvm->srcu);
281 page_table_reset_pgste(kvm->arch.gmap->mm, 0, TASK_SIZE, false);
282 srcu_read_unlock(&kvm->srcu, idx);
283 mutex_unlock(&kvm->lock);
284 ret = 0;
285 break;
286 default:
287 ret = -ENXIO;
288 break;
289 }
290 return ret;
291}
292
f2061656
DD		 293static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
294{
295 int ret;
296
297 switch (attr->group) {
4f718eab
DD		 298 case KVM_S390_VM_MEM_CTRL:
299 ret = kvm_s390_mem_control(kvm, attr);
300 break;
f2061656
DD		 301 default:
302 ret = -ENXIO;
303 break;
304 }
305
306 return ret;
307}
308
309static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
310{
311 return -ENXIO;
312}
313
314static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
315{
316 int ret;
317
318 switch (attr->group) {
4f718eab
DD		 319 case KVM_S390_VM_MEM_CTRL:
320 switch (attr->attr) {
321 case KVM_S390_VM_MEM_ENABLE_CMMA:
322 case KVM_S390_VM_MEM_CLR_CMMA:
323 ret = 0;
324 break;
325 default:
326 ret = -ENXIO;
327 break;
328 }
329 break;
f2061656
DD		 330 default:
331 ret = -ENXIO;
332 break;
333 }
334
335 return ret;
336}
337
b0c632db
HC		 338long kvm_arch_vm_ioctl(struct file *filp,
339 unsigned int ioctl, unsigned long arg)
340{
341 struct kvm *kvm = filp->private_data;
342 void __user *argp = (void __user *)arg;
f2061656 343 struct kvm_device_attr attr;
b0c632db
HC		 344 int r;
345
346 switch (ioctl) {
ba5c1e9b
CO		 347 case KVM_S390_INTERRUPT: {
348 struct kvm_s390_interrupt s390int;
349
350 r = -EFAULT;
351 if (copy_from_user(&s390int, argp, sizeof(s390int)))
352 break;
353 r = kvm_s390_inject_vm(kvm, &s390int);
354 break;
355 }
d938dc55
CH		 356 case KVM_ENABLE_CAP: {
357 struct kvm_enable_cap cap;
358 r = -EFAULT;
359 if (copy_from_user(&cap, argp, sizeof(cap)))
360 break;
361 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
362 break;
363 }
84223598
CH		 364 case KVM_CREATE_IRQCHIP: {
365 struct kvm_irq_routing_entry routing;
366
367 r = -EINVAL;
368 if (kvm->arch.use_irqchip) {
369 /* Set up dummy routing. */
370 memset(&routing, 0, sizeof(routing));
371 kvm_set_irq_routing(kvm, &routing, 0, 0);
372 r = 0;
373 }
374 break;
375 }
f2061656
DD		 376 case KVM_SET_DEVICE_ATTR: {
377 r = -EFAULT;
378 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
379 break;
380 r = kvm_s390_vm_set_attr(kvm, &attr);
381 break;
382 }
383 case KVM_GET_DEVICE_ATTR: {
384 r = -EFAULT;
385 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
386 break;
387 r = kvm_s390_vm_get_attr(kvm, &attr);
388 break;
389 }
390 case KVM_HAS_DEVICE_ATTR: {
391 r = -EFAULT;
392 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
393 break;
394 r = kvm_s390_vm_has_attr(kvm, &attr);
395 break;
396 }
b0c632db 397 default:
367e1319 398 r = -ENOTTY;
b0c632db
HC		 399 }
400
401 return r;
402}
403
5102ee87
TK		 404static int kvm_s390_crypto_init(struct kvm *kvm)
405{
406 if (!test_vfacility(76))
407 return 0;
408
409 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
410 GFP_KERNEL | GFP_DMA);
411 if (!kvm->arch.crypto.crycb)
412 return -ENOMEM;
413
414 kvm->arch.crypto.crycbd = (__u32) (unsigned long) kvm->arch.crypto.crycb |
415 CRYCB_FORMAT1;
416
417 return 0;
418}
419
e08b9637 420int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 421{
b0c632db
HC		 422 int rc;
423 char debug_name[16];
f6c137ff 424 static unsigned long sca_offset;
b0c632db 425
e08b9637
CO		 426 rc = -EINVAL;
427#ifdef CONFIG_KVM_S390_UCONTROL
428 if (type & ~KVM_VM_S390_UCONTROL)
429 goto out_err;
430 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
431 goto out_err;
432#else
433 if (type)
434 goto out_err;
435#endif
436
b0c632db
HC		 437 rc = s390_enable_sie();
438 if (rc)
d89f5eff 439 goto out_err;
b0c632db 440
b290411a
CO		 441 rc = -ENOMEM;
442
b0c632db
HC		 443 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
444 if (!kvm->arch.sca)
d89f5eff 445 goto out_err;
f6c137ff
CB		 446 spin_lock(&kvm_lock);
447 sca_offset = (sca_offset + 16) & 0x7f0;
448 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
449 spin_unlock(&kvm_lock);
b0c632db
HC		 450
451 sprintf(debug_name, "kvm-%u", current->pid);
452
453 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
454 if (!kvm->arch.dbf)
455 goto out_nodbf;
456
5102ee87
TK		 457 if (kvm_s390_crypto_init(kvm) < 0)
458 goto out_crypto;
459
ba5c1e9b
CO		 460 spin_lock_init(&kvm->arch.float_int.lock);
461 INIT_LIST_HEAD(&kvm->arch.float_int.list);
8a242234 462 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 463 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 464
b0c632db
HC		 465 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
466 VM_EVENT(kvm, 3, "%s", "vm created");
467
e08b9637
CO		 468 if (type & KVM_VM_S390_UCONTROL) {
469 kvm->arch.gmap = NULL;
470 } else {
0349985a 471 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
e08b9637
CO		 472 if (!kvm->arch.gmap)
473 goto out_nogmap;
2c70fe44 474 kvm->arch.gmap->private = kvm;
24eb3a82 475 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 476 }
fa6b7fe9
CH		 477
478 kvm->arch.css_support = 0;
84223598 479 kvm->arch.use_irqchip = 0;
fa6b7fe9 480
8ad35755
DH		 481 spin_lock_init(&kvm->arch.start_stop_lock);
482
d89f5eff 483 return 0;
598841ca 484out_nogmap:
5102ee87
TK		 485 kfree(kvm->arch.crypto.crycb);
486out_crypto:
598841ca 487 debug_unregister(kvm->arch.dbf);
b0c632db
HC		 488out_nodbf:
489 free_page((unsigned long)(kvm->arch.sca));
d89f5eff
JK		 490out_err:
491 return rc;
b0c632db
HC		 492}
493
d329c035
CB		 494void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
495{
496 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 497 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 498 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 499 kvm_clear_async_pf_completion_queue(vcpu);
58f9460b
CO		 500 if (!kvm_is_ucontrol(vcpu->kvm)) {
501 clear_bit(63 - vcpu->vcpu_id,
502 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
503 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
504 (__u64) vcpu->arch.sie_block)
505 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
506 }
abf4a71e 507 smp_mb();
27e0393f
CO		 508
509 if (kvm_is_ucontrol(vcpu->kvm))
510 gmap_free(vcpu->arch.gmap);
511
b31605c1
DD		 512 if (kvm_s390_cmma_enabled(vcpu->kvm))
513 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 514 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 515
6692cef3 516 kvm_vcpu_uninit(vcpu);
b110feaf 517 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 518}
519
520static void kvm_free_vcpus(struct kvm *kvm)
521{
522 unsigned int i;
988a2cae 523 struct kvm_vcpu *vcpu;
d329c035 524
988a2cae
GN		 525 kvm_for_each_vcpu(i, vcpu, kvm)
526 kvm_arch_vcpu_destroy(vcpu);
527
528 mutex_lock(&kvm->lock);
529 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
530 kvm->vcpus[i] = NULL;
531
532 atomic_set(&kvm->online_vcpus, 0);
533 mutex_unlock(&kvm->lock);
d329c035
CB		 534}
535
b0c632db
HC		 536void kvm_arch_destroy_vm(struct kvm *kvm)
537{
d329c035 538 kvm_free_vcpus(kvm);
b0c632db 539 free_page((unsigned long)(kvm->arch.sca));
d329c035 540 debug_unregister(kvm->arch.dbf);
5102ee87 541 kfree(kvm->arch.crypto.crycb);
27e0393f
CO		 542 if (!kvm_is_ucontrol(kvm))
543 gmap_free(kvm->arch.gmap);
841b91c5 544 kvm_s390_destroy_adapters(kvm);
67335e63 545 kvm_s390_clear_float_irqs(kvm);
b0c632db
HC		 546}
547
548/* Section: vcpu related */
549int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
550{
3c038e6b
DD		 551 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
552 kvm_clear_async_pf_completion_queue(vcpu);
27e0393f 553 if (kvm_is_ucontrol(vcpu->kvm)) {
c6c956b8 554 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
27e0393f
CO		 555 if (!vcpu->arch.gmap)
556 return -ENOMEM;
2c70fe44 557 vcpu->arch.gmap->private = vcpu->kvm;
27e0393f
CO		 558 return 0;
559 }
560
598841ca 561 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
59674c1a
CB		 562 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
563 KVM_SYNC_GPRS |
9eed0735 564 KVM_SYNC_ACRS |
b028ee3e
DH		 565 KVM_SYNC_CRS |
566 KVM_SYNC_ARCH0 |
567 KVM_SYNC_PFAULT;
b0c632db
HC		 568 return 0;
569}
570
b0c632db
HC		 571void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
572{
4725c860
MS		 573 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
574 save_fp_regs(vcpu->arch.host_fpregs.fprs);
b0c632db 575 save_access_regs(vcpu->arch.host_acrs);
4725c860
MS		 576 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
577 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
59674c1a 578 restore_access_regs(vcpu->run->s.regs.acrs);
480e5926 579 gmap_enable(vcpu->arch.gmap);
9e6dabef 580 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
b0c632db
HC		 581}
582
583void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
584{
9e6dabef 585 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
480e5926 586 gmap_disable(vcpu->arch.gmap);
4725c860
MS		 587 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
588 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
59674c1a 589 save_access_regs(vcpu->run->s.regs.acrs);
4725c860
MS		 590 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
591 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
b0c632db
HC		 592 restore_access_regs(vcpu->arch.host_acrs);
593}
594
595static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
596{
597 /* this equals initial cpu reset in pop, but we don't switch to ESA */
598 vcpu->arch.sie_block->gpsw.mask = 0UL;
599 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 600 kvm_s390_set_prefix(vcpu, 0);
b0c632db
HC		 601 vcpu->arch.sie_block->cputm = 0UL;
602 vcpu->arch.sie_block->ckc = 0UL;
603 vcpu->arch.sie_block->todpr = 0;
604 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
605 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
606 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
607 vcpu->arch.guest_fpregs.fpc = 0;
608 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
609 vcpu->arch.sie_block->gbea = 1;
672550fb 610 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 611 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
612 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 613 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
614 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 615 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 616}
617
42897d86
MT		 618int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
619{
620 return 0;
621}
622
5102ee87
TK		 623static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
624{
625 if (!test_vfacility(76))
626 return;
627
628 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
629}
630
b31605c1
DD		 631void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
632{
633 free_page(vcpu->arch.sie_block->cbrlo);
634 vcpu->arch.sie_block->cbrlo = 0;
635}
636
637int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
638{
639 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
640 if (!vcpu->arch.sie_block->cbrlo)
641 return -ENOMEM;
642
643 vcpu->arch.sie_block->ecb2 |= 0x80;
644 vcpu->arch.sie_block->ecb2 &= ~0x08;
645 return 0;
646}
647
b0c632db
HC		 648int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
649{
b31605c1 650 int rc = 0;
b31288fa 651
9e6dabef
CH		 652 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
653 CPUSTAT_SM |
69d0d3a3
CB		 654 CPUSTAT_STOPPED |
655 CPUSTAT_GED);
fc34531d 656 vcpu->arch.sie_block->ecb = 6;
7feb6bb8
MM		 657 if (test_vfacility(50) && test_vfacility(73))
658 vcpu->arch.sie_block->ecb |= 0x10;
659
69d0d3a3 660 vcpu->arch.sie_block->ecb2 = 8;
4953919f 661 vcpu->arch.sie_block->eca = 0xD1002000U;
217a4406
HC		 662 if (sclp_has_siif())
663 vcpu->arch.sie_block->eca |= 1;
78c4b59f 664 vcpu->arch.sie_block->fac = (int) (long) vfacilities;
5a5e6536
MR		 665 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE |
666 ICTL_TPROT;
667
b31605c1
DD		 668 if (kvm_s390_cmma_enabled(vcpu->kvm)) {
669 rc = kvm_s390_vcpu_setup_cmma(vcpu);
670 if (rc)
671 return rc;
b31288fa 672 }
ca872302 673 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
ca872302 674 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
453423dc 675 get_cpu_id(&vcpu->arch.cpu_id);
92e6ecf3 676 vcpu->arch.cpu_id.version = 0xff;
5102ee87
TK		 677
678 kvm_s390_vcpu_crypto_setup(vcpu);
679
b31605c1 680 return rc;
b0c632db
HC		 681}
682
683struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
684 unsigned int id)
685{
4d47555a 686 struct kvm_vcpu *vcpu;
7feb6bb8 687 struct sie_page *sie_page;
4d47555a
CO		 688 int rc = -EINVAL;
689
690 if (id >= KVM_MAX_VCPUS)
691 goto out;
692
693 rc = -ENOMEM;
b0c632db 694
b110feaf 695 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 696 if (!vcpu)
4d47555a 697 goto out;
b0c632db 698
7feb6bb8
MM		 699 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
700 if (!sie_page)
b0c632db
HC		 701 goto out_free_cpu;
702
7feb6bb8
MM		 703 vcpu->arch.sie_block = &sie_page->sie_block;
704 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
705
b0c632db 706 vcpu->arch.sie_block->icpua = id;
58f9460b
CO		 707 if (!kvm_is_ucontrol(kvm)) {
708 if (!kvm->arch.sca) {
709 WARN_ON_ONCE(1);
710 goto out_free_cpu;
711 }
712 if (!kvm->arch.sca->cpu[id].sda)
713 kvm->arch.sca->cpu[id].sda =
714 (__u64) vcpu->arch.sie_block;
715 vcpu->arch.sie_block->scaoh =
716 (__u32)(((__u64)kvm->arch.sca) >> 32);
717 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
718 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
719 }
b0c632db 720
ba5c1e9b
CO		 721 spin_lock_init(&vcpu->arch.local_int.lock);
722 INIT_LIST_HEAD(&vcpu->arch.local_int.list);
723 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 724 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 725 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
ba5c1e9b 726
b0c632db
HC		 727 rc = kvm_vcpu_init(vcpu, kvm, id);
728 if (rc)
7b06bf2f 729 goto out_free_sie_block;
b0c632db
HC		 730 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
731 vcpu->arch.sie_block);
ade38c31 732 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 733
b0c632db 734 return vcpu;
7b06bf2f
WY		 735out_free_sie_block:
736 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 737out_free_cpu:
b110feaf 738 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 739out:
b0c632db
HC		 740 return ERR_PTR(rc);
741}
742
b0c632db
HC		 743int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
744{
f87618e8 745 return kvm_cpu_has_interrupt(vcpu);
b0c632db
HC		 746}
747
49b99e1e
CB		 748void s390_vcpu_block(struct kvm_vcpu *vcpu)
749{
750 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
751}
752
753void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
754{
755 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
756}
757
758/*
759 * Kick a guest cpu out of SIE and wait until SIE is not running.
760 * If the CPU is not running (e.g. waiting as idle) the function will
761 * return immediately. */
762void exit_sie(struct kvm_vcpu *vcpu)
763{
764 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
765 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
766 cpu_relax();
767}
768
769/* Kick a guest cpu out of SIE and prevent SIE-reentry */
770void exit_sie_sync(struct kvm_vcpu *vcpu)
771{
772 s390_vcpu_block(vcpu);
773 exit_sie(vcpu);
774}
775
2c70fe44
CB		 776static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
777{
778 int i;
779 struct kvm *kvm = gmap->private;
780 struct kvm_vcpu *vcpu;
781
782 kvm_for_each_vcpu(i, vcpu, kvm) {
783 /* match against both prefix pages */
fda902cb 784 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
2c70fe44
CB		 785 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
786 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
787 exit_sie_sync(vcpu);
788 }
789 }
790}
791
b6d33834
CD		 792int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
793{
794 /* kvm common code refers to this, but never calls it */
795 BUG();
796 return 0;
797}
798
14eebd91
CO		 799static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
800 struct kvm_one_reg *reg)
801{
802 int r = -EINVAL;
803
804 switch (reg->id) {
29b7c71b
CO		 805 case KVM_REG_S390_TODPR:
806 r = put_user(vcpu->arch.sie_block->todpr,
807 (u32 __user *)reg->addr);
808 break;
809 case KVM_REG_S390_EPOCHDIFF:
810 r = put_user(vcpu->arch.sie_block->epoch,
811 (u64 __user *)reg->addr);
812 break;
46a6dd1c
J		 813 case KVM_REG_S390_CPU_TIMER:
814 r = put_user(vcpu->arch.sie_block->cputm,
815 (u64 __user *)reg->addr);
816 break;
817 case KVM_REG_S390_CLOCK_COMP:
818 r = put_user(vcpu->arch.sie_block->ckc,
819 (u64 __user *)reg->addr);
820 break;
536336c2
DD		 821 case KVM_REG_S390_PFTOKEN:
822 r = put_user(vcpu->arch.pfault_token,
823 (u64 __user *)reg->addr);
824 break;
825 case KVM_REG_S390_PFCOMPARE:
826 r = put_user(vcpu->arch.pfault_compare,
827 (u64 __user *)reg->addr);
828 break;
829 case KVM_REG_S390_PFSELECT:
830 r = put_user(vcpu->arch.pfault_select,
831 (u64 __user *)reg->addr);
832 break;
672550fb
CB		 833 case KVM_REG_S390_PP:
834 r = put_user(vcpu->arch.sie_block->pp,
835 (u64 __user *)reg->addr);
836 break;
afa45ff5
CB		 837 case KVM_REG_S390_GBEA:
838 r = put_user(vcpu->arch.sie_block->gbea,
839 (u64 __user *)reg->addr);
840 break;
14eebd91
CO		 841 default:
842 break;
843 }
844
845 return r;
846}
847
848static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
849 struct kvm_one_reg *reg)
850{
851 int r = -EINVAL;
852
853 switch (reg->id) {
29b7c71b
CO		 854 case KVM_REG_S390_TODPR:
855 r = get_user(vcpu->arch.sie_block->todpr,
856 (u32 __user *)reg->addr);
857 break;
858 case KVM_REG_S390_EPOCHDIFF:
859 r = get_user(vcpu->arch.sie_block->epoch,
860 (u64 __user *)reg->addr);
861 break;
46a6dd1c
J		 862 case KVM_REG_S390_CPU_TIMER:
863 r = get_user(vcpu->arch.sie_block->cputm,
864 (u64 __user *)reg->addr);
865 break;
866 case KVM_REG_S390_CLOCK_COMP:
867 r = get_user(vcpu->arch.sie_block->ckc,
868 (u64 __user *)reg->addr);
869 break;
536336c2
DD		 870 case KVM_REG_S390_PFTOKEN:
871 r = get_user(vcpu->arch.pfault_token,
872 (u64 __user *)reg->addr);
873 break;
874 case KVM_REG_S390_PFCOMPARE:
875 r = get_user(vcpu->arch.pfault_compare,
876 (u64 __user *)reg->addr);
877 break;
878 case KVM_REG_S390_PFSELECT:
879 r = get_user(vcpu->arch.pfault_select,
880 (u64 __user *)reg->addr);
881 break;
672550fb
CB		 882 case KVM_REG_S390_PP:
883 r = get_user(vcpu->arch.sie_block->pp,
884 (u64 __user *)reg->addr);
885 break;
afa45ff5
CB		 886 case KVM_REG_S390_GBEA:
887 r = get_user(vcpu->arch.sie_block->gbea,
888 (u64 __user *)reg->addr);
889 break;
14eebd91
CO		 890 default:
891 break;
892 }
893
894 return r;
895}
b6d33834 896
b0c632db
HC		 897static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
898{
b0c632db 899 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 900 return 0;
901}
902
903int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
904{
5a32c1af 905 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 906 return 0;
907}
908
909int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
910{
5a32c1af 911 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 912 return 0;
913}
914
915int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
916 struct kvm_sregs *sregs)
917{
59674c1a 918 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 919 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
59674c1a 920 restore_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 921 return 0;
922}
923
924int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
925 struct kvm_sregs *sregs)
926{
59674c1a 927 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 928 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 929 return 0;
930}
931
932int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
933{
4725c860
MS		 934 if (test_fp_ctl(fpu->fpc))
935 return -EINVAL;
b0c632db 936 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
4725c860
MS		 937 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
938 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
939 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
b0c632db
HC		 940 return 0;
941}
942
943int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
944{
b0c632db
HC		 945 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
946 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
b0c632db
HC		 947 return 0;
948}
949
950static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
951{
952 int rc = 0;
953
7a42fdc2 954 if (!is_vcpu_stopped(vcpu))
b0c632db 955 rc = -EBUSY;
d7b0b5eb
CO		 956 else {
957 vcpu->run->psw_mask = psw.mask;
958 vcpu->run->psw_addr = psw.addr;
959 }
b0c632db
HC		 960 return rc;
961}
962
963int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
964 struct kvm_translation *tr)
965{
966 return -EINVAL; /* not implemented yet */
967}
968
27291e21
DH		 969#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
970 KVM_GUESTDBG_USE_HW_BP | \
971 KVM_GUESTDBG_ENABLE)
972
d0bfb940
JK		 973int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
974 struct kvm_guest_debug *dbg)
b0c632db 975{
27291e21
DH		 976 int rc = 0;
977
978 vcpu->guest_debug = 0;
979 kvm_s390_clear_bp_data(vcpu);
980
2de3bfc2 981 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21
DH		 982 return -EINVAL;
983
984 if (dbg->control & KVM_GUESTDBG_ENABLE) {
985 vcpu->guest_debug = dbg->control;
986 /* enforce guest PER */
987 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
988
989 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
990 rc = kvm_s390_import_bp_data(vcpu, dbg);
991 } else {
992 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
993 vcpu->arch.guestdbg.last_bp = 0;
994 }
995
996 if (rc) {
997 vcpu->guest_debug = 0;
998 kvm_s390_clear_bp_data(vcpu);
999 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1000 }
1001
1002 return rc;
b0c632db
HC		 1003}
1004
62d9f0db
MT		 1005int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1006 struct kvm_mp_state *mp_state)
1007{
6352e4d2
DH		 1008 /* CHECK_STOP and LOAD are not supported yet */
1009 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1010 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 1011}
1012
1013int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1014 struct kvm_mp_state *mp_state)
1015{
6352e4d2
DH		 1016 int rc = 0;
1017
1018 /* user space knows about this interface - let it control the state */
1019 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1020
1021 switch (mp_state->mp_state) {
1022 case KVM_MP_STATE_STOPPED:
1023 kvm_s390_vcpu_stop(vcpu);
1024 break;
1025 case KVM_MP_STATE_OPERATING:
1026 kvm_s390_vcpu_start(vcpu);
1027 break;
1028 case KVM_MP_STATE_LOAD:
1029 case KVM_MP_STATE_CHECK_STOP:
1030 /* fall through - CHECK_STOP and LOAD are not supported yet */
1031 default:
1032 rc = -ENXIO;
1033 }
1034
1035 return rc;
62d9f0db
MT		 1036}
1037
b31605c1
DD		 1038bool kvm_s390_cmma_enabled(struct kvm *kvm)
1039{
1040 if (!MACHINE_IS_LPAR)
1041 return false;
1042 /* only enable for z10 and later */
1043 if (!MACHINE_HAS_EDAT1)
1044 return false;
1045 if (!kvm->arch.use_cmma)
1046 return false;
1047 return true;
1048}
1049
8ad35755
DH		 1050static bool ibs_enabled(struct kvm_vcpu *vcpu)
1051{
1052 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1053}
1054
2c70fe44
CB		 1055static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1056{
8ad35755
DH		 1057retry:
1058 s390_vcpu_unblock(vcpu);
2c70fe44
CB		 1059 /*
1060 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1061 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1062 * This ensures that the ipte instruction for this request has
1063 * already finished. We might race against a second unmapper that
1064 * wants to set the blocking bit. Lets just retry the request loop.
1065 */
8ad35755 1066 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44
CB		 1067 int rc;
1068 rc = gmap_ipte_notify(vcpu->arch.gmap,
fda902cb 1069 kvm_s390_get_prefix(vcpu),
2c70fe44
CB		 1070 PAGE_SIZE * 2);
1071 if (rc)
1072 return rc;
8ad35755 1073 goto retry;
2c70fe44 1074 }
8ad35755 1075
d3d692c8
DH		 1076 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1077 vcpu->arch.sie_block->ihcpu = 0xffff;
1078 goto retry;
1079 }
1080
8ad35755
DH		 1081 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1082 if (!ibs_enabled(vcpu)) {
1083 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1084 atomic_set_mask(CPUSTAT_IBS,
1085 &vcpu->arch.sie_block->cpuflags);
1086 }
1087 goto retry;
2c70fe44 1088 }
8ad35755
DH		 1089
1090 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1091 if (ibs_enabled(vcpu)) {
1092 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1093 atomic_clear_mask(CPUSTAT_IBS,
1094 &vcpu->arch.sie_block->cpuflags);
1095 }
1096 goto retry;
1097 }
1098
0759d068
DH		 1099 /* nothing to do, just clear the request */
1100 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1101
2c70fe44
CB		 1102 return 0;
1103}
1104
fa576c58
TH		 1105/**
1106 * kvm_arch_fault_in_page - fault-in guest page if necessary
1107 * @vcpu: The corresponding virtual cpu
1108 * @gpa: Guest physical address
1109 * @writable: Whether the page should be writable or not
1110 *
1111 * Make sure that a guest page has been faulted-in on the host.
1112 *
1113 * Return: Zero on success, negative error code otherwise.
1114 */
1115long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 1116{
527e30b4
MS		 1117 return gmap_fault(vcpu->arch.gmap, gpa,
1118 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 1119}
1120
3c038e6b
DD		 1121static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1122 unsigned long token)
1123{
1124 struct kvm_s390_interrupt inti;
1125 inti.parm64 = token;
1126
1127 if (start_token) {
1128 inti.type = KVM_S390_INT_PFAULT_INIT;
1129 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &inti));
1130 } else {
1131 inti.type = KVM_S390_INT_PFAULT_DONE;
1132 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1133 }
1134}
1135
1136void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1137 struct kvm_async_pf *work)
1138{
1139 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1140 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1141}
1142
1143void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1144 struct kvm_async_pf *work)
1145{
1146 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1147 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1148}
1149
1150void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1151 struct kvm_async_pf *work)
1152{
1153 /* s390 will always inject the page directly */
1154}
1155
1156bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1157{
1158 /*
1159 * s390 will always inject the page directly,
1160 * but we still want check_async_completion to cleanup
1161 */
1162 return true;
1163}
1164
1165static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1166{
1167 hva_t hva;
1168 struct kvm_arch_async_pf arch;
1169 int rc;
1170
1171 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1172 return 0;
1173 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1174 vcpu->arch.pfault_compare)
1175 return 0;
1176 if (psw_extint_disabled(vcpu))
1177 return 0;
1178 if (kvm_cpu_has_interrupt(vcpu))
1179 return 0;
1180 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1181 return 0;
1182 if (!vcpu->arch.gmap->pfault_enabled)
1183 return 0;
1184
81480cc1
HC		 1185 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1186 hva += current->thread.gmap_addr & ~PAGE_MASK;
1187 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 1188 return 0;
1189
1190 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1191 return rc;
1192}
1193
3fb4c40f 1194static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 1195{
3fb4c40f 1196 int rc, cpuflags;
e168bf8d 1197
3c038e6b
DD		 1198 /*
1199 * On s390 notifications for arriving pages will be delivered directly
1200 * to the guest but the house keeping for completed pfaults is
1201 * handled outside the worker.
1202 */
1203 kvm_check_async_pf_completion(vcpu);
1204
5a32c1af 1205 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
b0c632db
HC		 1206
1207 if (need_resched())
1208 schedule();
1209
d3a73acb 1210 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 1211 s390_handle_mcck();
1212
79395031
JF		 1213 if (!kvm_is_ucontrol(vcpu->kvm)) {
1214 rc = kvm_s390_deliver_pending_interrupts(vcpu);
1215 if (rc)
1216 return rc;
1217 }
0ff31867 1218
2c70fe44
CB		 1219 rc = kvm_s390_handle_requests(vcpu);
1220 if (rc)
1221 return rc;
1222
27291e21
DH		 1223 if (guestdbg_enabled(vcpu)) {
1224 kvm_s390_backup_guest_per_regs(vcpu);
1225 kvm_s390_patch_guest_per_regs(vcpu);
1226 }
1227
b0c632db 1228 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 1229 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1230 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1231 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 1232
3fb4c40f
TH		 1233 return 0;
1234}
1235
1236static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1237{
24eb3a82 1238 int rc = -1;
2b29a9fd
DD		 1239
1240 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1241 vcpu->arch.sie_block->icptcode);
1242 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1243
27291e21
DH		 1244 if (guestdbg_enabled(vcpu))
1245 kvm_s390_restore_guest_per_regs(vcpu);
1246
3fb4c40f 1247 if (exit_reason >= 0) {
7c470539 1248 rc = 0;
210b1607
TH		 1249 } else if (kvm_is_ucontrol(vcpu->kvm)) {
1250 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1251 vcpu->run->s390_ucontrol.trans_exc_code =
1252 current->thread.gmap_addr;
1253 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1254 rc = -EREMOTE;
24eb3a82
DD		 1255
1256 } else if (current->thread.gmap_pfault) {
3c038e6b 1257 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 1258 current->thread.gmap_pfault = 0;
fa576c58 1259 if (kvm_arch_setup_async_pf(vcpu)) {
24eb3a82 1260 rc = 0;
fa576c58
TH		 1261 } else {
1262 gpa_t gpa = current->thread.gmap_addr;
1263 rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1264 }
24eb3a82
DD		 1265 }
1266
1267 if (rc == -1) {
699bde3b
CB		 1268 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1269 trace_kvm_s390_sie_fault(vcpu);
1270 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1f0d0f09 1271 }
b0c632db 1272
5a32c1af 1273 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
3fb4c40f 1274
a76ccff6
TH		 1275 if (rc == 0) {
1276 if (kvm_is_ucontrol(vcpu->kvm))
2955c83f
CB		 1277 /* Don't exit for host interrupts. */
1278 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
a76ccff6
TH		 1279 else
1280 rc = kvm_handle_sie_intercept(vcpu);
1281 }
1282
3fb4c40f
TH		 1283 return rc;
1284}
1285
1286static int __vcpu_run(struct kvm_vcpu *vcpu)
1287{
1288 int rc, exit_reason;
1289
800c1065
TH		 1290 /*
1291 * We try to hold kvm->srcu during most of vcpu_run (except when run-
1292 * ning the guest), so that memslots (and other stuff) are protected
1293 */
1294 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1295
a76ccff6
TH		 1296 do {
1297 rc = vcpu_pre_run(vcpu);
1298 if (rc)
1299 break;
3fb4c40f 1300
800c1065 1301 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 1302 /*
1303 * As PF_VCPU will be used in fault handler, between
1304 * guest_enter and guest_exit should be no uaccess.
1305 */
1306 preempt_disable();
1307 kvm_guest_enter();
1308 preempt_enable();
1309 exit_reason = sie64a(vcpu->arch.sie_block,
1310 vcpu->run->s.regs.gprs);
1311 kvm_guest_exit();
800c1065 1312 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 1313
1314 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 1315 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 1316
800c1065 1317 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 1318 return rc;
b0c632db
HC		 1319}
1320
b028ee3e
DH		 1321static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1322{
1323 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1324 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
1325 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
1326 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1327 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1328 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 1329 /* some control register changes require a tlb flush */
1330 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 1331 }
1332 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
1333 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
1334 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
1335 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
1336 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
1337 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
1338 }
1339 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
1340 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
1341 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
1342 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
1343 }
1344 kvm_run->kvm_dirty_regs = 0;
1345}
1346
1347static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1348{
1349 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
1350 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
1351 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
1352 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
1353 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
1354 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
1355 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
1356 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
1357 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
1358 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
1359 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
1360 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
1361}
1362
b0c632db
HC		 1363int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1364{
8f2abe6a 1365 int rc;
b0c632db
HC		 1366 sigset_t sigsaved;
1367
27291e21
DH		 1368 if (guestdbg_exit_pending(vcpu)) {
1369 kvm_s390_prepare_debug_exit(vcpu);
1370 return 0;
1371 }
1372
b0c632db
HC		 1373 if (vcpu->sigset_active)
1374 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1375
6352e4d2
DH		 1376 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
1377 kvm_s390_vcpu_start(vcpu);
1378 } else if (is_vcpu_stopped(vcpu)) {
1379 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
1380 vcpu->vcpu_id);
1381 return -EINVAL;
1382 }
b0c632db 1383
b028ee3e 1384 sync_regs(vcpu, kvm_run);
d7b0b5eb 1385
dab4079d 1386 might_fault();
a76ccff6 1387 rc = __vcpu_run(vcpu);
9ace903d 1388
b1d16c49
CE		 1389 if (signal_pending(current) && !rc) {
1390 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 1391 rc = -EINTR;
b1d16c49 1392 }
8f2abe6a 1393
27291e21
DH		 1394 if (guestdbg_exit_pending(vcpu) && !rc) {
1395 kvm_s390_prepare_debug_exit(vcpu);
1396 rc = 0;
1397 }
1398
b8e660b8 1399 if (rc == -EOPNOTSUPP) {
8f2abe6a
CB		 1400 /* intercept cannot be handled in-kernel, prepare kvm-run */
1401 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
1402 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
8f2abe6a
CB		 1403 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
1404 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
1405 rc = 0;
1406 }
1407
1408 if (rc == -EREMOTE) {
1409 /* intercept was handled, but userspace support is needed
1410 * kvm_run has been prepared by the handler */
1411 rc = 0;
1412 }
b0c632db 1413
b028ee3e 1414 store_regs(vcpu, kvm_run);
d7b0b5eb 1415
b0c632db
HC		 1416 if (vcpu->sigset_active)
1417 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1418
b0c632db 1419 vcpu->stat.exit_userspace++;
7e8e6ab4 1420 return rc;
b0c632db
HC		 1421}
1422
b0c632db
HC		 1423/*
1424 * store status at address
1425 * we use have two special cases:
1426 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1427 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1428 */
d0bce605 1429int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 1430{
092670cd 1431 unsigned char archmode = 1;
fda902cb 1432 unsigned int px;
178bd789 1433 u64 clkcomp;
d0bce605 1434 int rc;
b0c632db 1435
d0bce605
HC		 1436 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
1437 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 1438 return -EFAULT;
d0bce605
HC		 1439 gpa = SAVE_AREA_BASE;
1440 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
1441 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 1442 return -EFAULT;
d0bce605
HC		 1443 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
1444 }
1445 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
1446 vcpu->arch.guest_fpregs.fprs, 128);
1447 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
1448 vcpu->run->s.regs.gprs, 128);
1449 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
1450 &vcpu->arch.sie_block->gpsw, 16);
fda902cb 1451 px = kvm_s390_get_prefix(vcpu);
d0bce605 1452 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
fda902cb 1453 &px, 4);
d0bce605
HC		 1454 rc |= write_guest_abs(vcpu,
1455 gpa + offsetof(struct save_area, fp_ctrl_reg),
1456 &vcpu->arch.guest_fpregs.fpc, 4);
1457 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
1458 &vcpu->arch.sie_block->todpr, 4);
1459 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
1460 &vcpu->arch.sie_block->cputm, 8);
178bd789 1461 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d0bce605
HC		 1462 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
1463 &clkcomp, 8);
1464 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
1465 &vcpu->run->s.regs.acrs, 64);
1466 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
1467 &vcpu->arch.sie_block->gcr, 128);
1468 return rc ? -EFAULT : 0;
b0c632db
HC		 1469}
1470
e879892c
TH		 1471int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1472{
1473 /*
1474 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1475 * copying in vcpu load/put. Lets update our copies before we save
1476 * it into the save area
1477 */
1478 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1479 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1480 save_access_regs(vcpu->run->s.regs.acrs);
1481
1482 return kvm_s390_store_status_unloaded(vcpu, addr);
1483}
1484
8ad35755
DH		 1485static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1486{
1487 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
1488 kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu);
1489 exit_sie_sync(vcpu);
1490}
1491
1492static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
1493{
1494 unsigned int i;
1495 struct kvm_vcpu *vcpu;
1496
1497 kvm_for_each_vcpu(i, vcpu, kvm) {
1498 __disable_ibs_on_vcpu(vcpu);
1499 }
1500}
1501
1502static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1503{
1504 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
1505 kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu);
1506 exit_sie_sync(vcpu);
1507}
1508
6852d7b6
DH		 1509void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
1510{
8ad35755
DH		 1511 int i, online_vcpus, started_vcpus = 0;
1512
1513 if (!is_vcpu_stopped(vcpu))
1514 return;
1515
6852d7b6 1516 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 1517 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 1518 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 1519 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1520
1521 for (i = 0; i < online_vcpus; i++) {
1522 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
1523 started_vcpus++;
1524 }
1525
1526 if (started_vcpus == 0) {
1527 /* we're the only active VCPU -> speed it up */
1528 __enable_ibs_on_vcpu(vcpu);
1529 } else if (started_vcpus == 1) {
1530 /*
1531 * As we are starting a second VCPU, we have to disable
1532 * the IBS facility on all VCPUs to remove potentially
1533 * oustanding ENABLE requests.
1534 */
1535 __disable_ibs_on_all_vcpus(vcpu->kvm);
1536 }
1537
6852d7b6 1538 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 1539 /*
1540 * Another VCPU might have used IBS while we were offline.
1541 * Let's play safe and flush the VCPU at startup.
1542 */
d3d692c8 1543 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 1544 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 1545 return;
6852d7b6
DH		 1546}
1547
1548void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
1549{
8ad35755
DH		 1550 int i, online_vcpus, started_vcpus = 0;
1551 struct kvm_vcpu *started_vcpu = NULL;
1552
1553 if (is_vcpu_stopped(vcpu))
1554 return;
1555
6852d7b6 1556 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 1557 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 1558 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 1559 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1560
32f5ff63 1561 /* Need to lock access to action_bits to avoid a SIGP race condition */
4ae3c081 1562 spin_lock(&vcpu->arch.local_int.lock);
6852d7b6 1563 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
32f5ff63
DH		 1564
1565 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
1566 vcpu->arch.local_int.action_bits &=
1567 ~(ACTION_STOP_ON_STOP | ACTION_STORE_ON_STOP);
4ae3c081 1568 spin_unlock(&vcpu->arch.local_int.lock);
32f5ff63 1569
8ad35755
DH		 1570 __disable_ibs_on_vcpu(vcpu);
1571
1572 for (i = 0; i < online_vcpus; i++) {
1573 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
1574 started_vcpus++;
1575 started_vcpu = vcpu->kvm->vcpus[i];
1576 }
1577 }
1578
1579 if (started_vcpus == 1) {
1580 /*
1581 * As we only have one VCPU left, we want to enable the
1582 * IBS facility for that VCPU to speed it up.
1583 */
1584 __enable_ibs_on_vcpu(started_vcpu);
1585 }
1586
433b9ee4 1587 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 1588 return;
6852d7b6
DH		 1589}
1590
d6712df9
CH		 1591static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1592 struct kvm_enable_cap *cap)
1593{
1594 int r;
1595
1596 if (cap->flags)
1597 return -EINVAL;
1598
1599 switch (cap->cap) {
fa6b7fe9
CH		 1600 case KVM_CAP_S390_CSS_SUPPORT:
1601 if (!vcpu->kvm->arch.css_support) {
1602 vcpu->kvm->arch.css_support = 1;
1603 trace_kvm_s390_enable_css(vcpu->kvm);
1604 }
1605 r = 0;
1606 break;
d6712df9
CH		 1607 default:
1608 r = -EINVAL;
1609 break;
1610 }
1611 return r;
1612}
1613
b0c632db
HC		 1614long kvm_arch_vcpu_ioctl(struct file *filp,
1615 unsigned int ioctl, unsigned long arg)
1616{
1617 struct kvm_vcpu *vcpu = filp->private_data;
1618 void __user *argp = (void __user *)arg;
800c1065 1619 int idx;
bc923cc9 1620 long r;
b0c632db 1621
93736624
AK		 1622 switch (ioctl) {
1623 case KVM_S390_INTERRUPT: {
ba5c1e9b
CO		 1624 struct kvm_s390_interrupt s390int;
1625
93736624 1626 r = -EFAULT;
ba5c1e9b 1627 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624
AK		 1628 break;
1629 r = kvm_s390_inject_vcpu(vcpu, &s390int);
1630 break;
ba5c1e9b 1631 }
b0c632db 1632 case KVM_S390_STORE_STATUS:
800c1065 1633 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 1634 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 1635 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 1636 break;
b0c632db
HC		 1637 case KVM_S390_SET_INITIAL_PSW: {
1638 psw_t psw;
1639
bc923cc9 1640 r = -EFAULT;
b0c632db 1641 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 1642 break;
1643 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
1644 break;
b0c632db
HC		 1645 }
1646 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 1647 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
1648 break;
14eebd91
CO		 1649 case KVM_SET_ONE_REG:
1650 case KVM_GET_ONE_REG: {
1651 struct kvm_one_reg reg;
1652 r = -EFAULT;
1653 if (copy_from_user(&reg, argp, sizeof(reg)))
1654 break;
1655 if (ioctl == KVM_SET_ONE_REG)
1656 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
1657 else
1658 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
1659 break;
1660 }
27e0393f
CO		 1661#ifdef CONFIG_KVM_S390_UCONTROL
1662 case KVM_S390_UCAS_MAP: {
1663 struct kvm_s390_ucas_mapping ucasmap;
1664
1665 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1666 r = -EFAULT;
1667 break;
1668 }
1669
1670 if (!kvm_is_ucontrol(vcpu->kvm)) {
1671 r = -EINVAL;
1672 break;
1673 }
1674
1675 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
1676 ucasmap.vcpu_addr, ucasmap.length);
1677 break;
1678 }
1679 case KVM_S390_UCAS_UNMAP: {
1680 struct kvm_s390_ucas_mapping ucasmap;
1681
1682 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1683 r = -EFAULT;
1684 break;
1685 }
1686
1687 if (!kvm_is_ucontrol(vcpu->kvm)) {
1688 r = -EINVAL;
1689 break;
1690 }
1691
1692 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
1693 ucasmap.length);
1694 break;
1695 }
1696#endif
ccc7910f 1697 case KVM_S390_VCPU_FAULT: {
527e30b4 1698 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 1699 break;
1700 }
d6712df9
CH		 1701 case KVM_ENABLE_CAP:
1702 {
1703 struct kvm_enable_cap cap;
1704 r = -EFAULT;
1705 if (copy_from_user(&cap, argp, sizeof(cap)))
1706 break;
1707 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1708 break;
1709 }
b0c632db 1710 default:
3e6afcf1 1711 r = -ENOTTY;
b0c632db 1712 }
bc923cc9 1713 return r;
b0c632db
HC		 1714}
1715
5b1c1493
CO		 1716int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
1717{
1718#ifdef CONFIG_KVM_S390_UCONTROL
1719 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
1720 && (kvm_is_ucontrol(vcpu->kvm))) {
1721 vmf->page = virt_to_page(vcpu->arch.sie_block);
1722 get_page(vmf->page);
1723 return 0;
1724 }
1725#endif
1726 return VM_FAULT_SIGBUS;
1727}
1728
5587027c
AK		 1729int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
1730 unsigned long npages)
db3fe4eb
TY		 1731{
1732 return 0;
1733}
1734
b0c632db 1735/* Section: memory related */
f7784b8e
MT		 1736int kvm_arch_prepare_memory_region(struct kvm *kvm,
1737 struct kvm_memory_slot *memslot,
7b6195a9
TY		 1738 struct kvm_userspace_memory_region *mem,
1739 enum kvm_mr_change change)
b0c632db 1740{
dd2887e7
NW		 1741 /* A few sanity checks. We can have memory slots which have to be
1742 located/ended at a segment boundary (1MB). The memory in userland is
1743 ok to be fragmented into various different vmas. It is okay to mmap()
1744 and munmap() stuff in this slot after doing this call at any time */
b0c632db 1745
598841ca 1746 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 1747 return -EINVAL;
1748
598841ca 1749 if (mem->memory_size & 0xffffful)
b0c632db
HC		 1750 return -EINVAL;
1751
f7784b8e
MT		 1752 return 0;
1753}
1754
1755void kvm_arch_commit_memory_region(struct kvm *kvm,
1756 struct kvm_userspace_memory_region *mem,
8482644a
TY		 1757 const struct kvm_memory_slot *old,
1758 enum kvm_mr_change change)
f7784b8e 1759{
f7850c92 1760 int rc;
f7784b8e 1761
2cef4deb
CB		 1762 /* If the basics of the memslot do not change, we do not want
1763 * to update the gmap. Every update causes several unnecessary
1764 * segment translation exceptions. This is usually handled just
1765 * fine by the normal fault handler + gmap, but it will also
1766 * cause faults on the prefix page of running guest CPUs.
1767 */
1768 if (old->userspace_addr == mem->userspace_addr &&
1769 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
1770 old->npages * PAGE_SIZE == mem->memory_size)
1771 return;
598841ca
CO		 1772
1773 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
1774 mem->guest_phys_addr, mem->memory_size);
1775 if (rc)
f7850c92 1776 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
598841ca 1777 return;
b0c632db
HC		 1778}
1779
b0c632db
HC		 1780static int __init kvm_s390_init(void)
1781{
ef50f7ac 1782 int ret;
0ee75bea 1783 ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
ef50f7ac
CB		 1784 if (ret)
1785 return ret;
1786
1787 /*
1788 * guests can ask for up to 255+1 double words, we need a full page
25985edc 1789 * to hold the maximum amount of facilities. On the other hand, we
ef50f7ac
CB		 1790 * only set facilities that are known to work in KVM.
1791 */
78c4b59f
MM		 1792 vfacilities = (unsigned long *) get_zeroed_page(GFP_KERNEL|GFP_DMA);
1793 if (!vfacilities) {
ef50f7ac
CB		 1794 kvm_exit();
1795 return -ENOMEM;
1796 }
78c4b59f 1797 memcpy(vfacilities, S390_lowcore.stfle_fac_list, 16);
7be81a46 1798 vfacilities[0] &= 0xff82fffbf47c2000UL;
7feb6bb8 1799 vfacilities[1] &= 0x005c000000000000UL;
ef50f7ac 1800 return 0;
b0c632db
HC		 1801}
1802
1803static void __exit kvm_s390_exit(void)
1804{
78c4b59f 1805 free_page((unsigned long) vfacilities);
b0c632db
HC		 1806 kvm_exit();
1807}
1808
1809module_init(kvm_s390_init);
1810module_exit(kvm_s390_exit);
566af940
CH		 1811
1812/*
1813 * Enable autoloading of the kvm module.
1814 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
1815 * since x86 takes a different approach.
1816 */
1817#include <linux/miscdevice.h>
1818MODULE_ALIAS_MISCDEV(KVM_MINOR);
1819MODULE_ALIAS("devname:kvm");
Ubuntu Jammy Linux kernel mirror