]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: s390/facilities: allow TOD-CLOCK steering facility bit
[mirror_ubuntu-jammy-kernel.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
24#include <linux/module.h>
25#include <linux/slab.h>
ba5c1e9b 26#include <linux/timer.h>
cbb870c8 27#include <asm/asm-offsets.h>
b0c632db
HC		 28#include <asm/lowcore.h>
29#include <asm/pgtable.h>
f5daba1d 30#include <asm/nmi.h>
a0616cde 31#include <asm/switch_to.h>
78c4b59f 32#include <asm/facility.h>
1526bf9c 33#include <asm/sclp.h>
8f2abe6a 34#include "kvm-s390.h"
b0c632db
HC		 35#include "gaccess.h"
36
5786fffa
CH		 37#define CREATE_TRACE_POINTS
38#include "trace.h"
ade38c31 39#include "trace-s390.h"
5786fffa 40
b0c632db
HC		 41#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
42
43struct kvm_stats_debugfs_item debugfs_entries[] = {
44 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 45 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 46 { "exit_validity", VCPU_STAT(exit_validity) },
47 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
48 { "exit_external_request", VCPU_STAT(exit_external_request) },
49 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b
CO		 50 { "exit_instruction", VCPU_STAT(exit_instruction) },
51 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
52 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
f5e10b09 53 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 54 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 55 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
56 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 57 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 58 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 59 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
60 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
61 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
62 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
63 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
64 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
65 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 66 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 67 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
68 { "instruction_spx", VCPU_STAT(instruction_spx) },
69 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
70 { "instruction_stap", VCPU_STAT(instruction_stap) },
71 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 72 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 73 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
74 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 75 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 76 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
77 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 78 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
5288fbf0 79 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 80 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 81 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0
CB		 82 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
83 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
84 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
85 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
86 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
388186bc 87 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 88 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 89 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
b0c632db
HC		 90 { NULL }
91};
92
78c4b59f 93unsigned long *vfacilities;
2c70fe44 94static struct gmap_notifier gmap_notifier;
b0c632db 95
78c4b59f 96/* test availability of vfacility */
280ef0f1 97int test_vfacility(unsigned long nr)
78c4b59f
MM		 98{
99 return __test_facility(nr, (void *) vfacilities);
100}
101
b0c632db 102/* Section: not file related */
13a34e06 103int kvm_arch_hardware_enable(void)
b0c632db
HC		 104{
105 /* every s390 is virtualization enabled ;-) */
10474ae8 106 return 0;
b0c632db
HC		 107}
108
2c70fe44
CB		 109static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
110
b0c632db
HC		 111int kvm_arch_hardware_setup(void)
112{
2c70fe44
CB		 113 gmap_notifier.notifier_call = kvm_gmap_notifier;
114 gmap_register_ipte_notifier(&gmap_notifier);
b0c632db
HC		 115 return 0;
116}
117
118void kvm_arch_hardware_unsetup(void)
119{
2c70fe44 120 gmap_unregister_ipte_notifier(&gmap_notifier);
b0c632db
HC		 121}
122
b0c632db
HC		 123int kvm_arch_init(void *opaque)
124{
84877d93
CH		 125 /* Register floating interrupt controller interface. */
126 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 127}
128
b0c632db
HC		 129/* Section: device related */
130long kvm_arch_dev_ioctl(struct file *filp,
131 unsigned int ioctl, unsigned long arg)
132{
133 if (ioctl == KVM_S390_ENABLE_SIE)
134 return s390_enable_sie();
135 return -EINVAL;
136}
137
784aa3d7 138int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 139{
d7b0b5eb
CO		 140 int r;
141
2bd0ac4e 142 switch (ext) {
d7b0b5eb 143 case KVM_CAP_S390_PSW:
b6cf8788 144 case KVM_CAP_S390_GMAP:
52e16b18 145 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 146#ifdef CONFIG_KVM_S390_UCONTROL
147 case KVM_CAP_S390_UCONTROL:
148#endif
3c038e6b 149 case KVM_CAP_ASYNC_PF:
60b413c9 150 case KVM_CAP_SYNC_REGS:
14eebd91 151 case KVM_CAP_ONE_REG:
d6712df9 152 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 153 case KVM_CAP_S390_CSS_SUPPORT:
ebc32262 154 case KVM_CAP_IRQFD:
10ccaa1e 155 case KVM_CAP_IOEVENTFD:
c05c4186 156 case KVM_CAP_DEVICE_CTRL:
d938dc55 157 case KVM_CAP_ENABLE_CAP_VM:
78599d90 158 case KVM_CAP_S390_IRQCHIP:
f2061656 159 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 160 case KVM_CAP_MP_STATE:
d7b0b5eb
CO		 161 r = 1;
162 break;
e726b1bd
CB		 163 case KVM_CAP_NR_VCPUS:
164 case KVM_CAP_MAX_VCPUS:
165 r = KVM_MAX_VCPUS;
166 break;
e1e2e605
NW		 167 case KVM_CAP_NR_MEMSLOTS:
168 r = KVM_USER_MEM_SLOTS;
169 break;
1526bf9c 170 case KVM_CAP_S390_COW:
abf09bed 171 r = MACHINE_HAS_ESOP;
1526bf9c 172 break;
2bd0ac4e 173 default:
d7b0b5eb 174 r = 0;
2bd0ac4e 175 }
d7b0b5eb 176 return r;
b0c632db
HC		 177}
178
15f36ebd
JH		 179static void kvm_s390_sync_dirty_log(struct kvm *kvm,
180 struct kvm_memory_slot *memslot)
181{
182 gfn_t cur_gfn, last_gfn;
183 unsigned long address;
184 struct gmap *gmap = kvm->arch.gmap;
185
186 down_read(&gmap->mm->mmap_sem);
187 /* Loop over all guest pages */
188 last_gfn = memslot->base_gfn + memslot->npages;
189 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
190 address = gfn_to_hva_memslot(memslot, cur_gfn);
191
192 if (gmap_test_and_clear_dirty(address, gmap))
193 mark_page_dirty(kvm, cur_gfn);
194 }
195 up_read(&gmap->mm->mmap_sem);
196}
197
b0c632db
HC		 198/* Section: vm related */
199/*
200 * Get (and clear) the dirty memory log for a memory slot.
201 */
202int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
203 struct kvm_dirty_log *log)
204{
15f36ebd
JH		 205 int r;
206 unsigned long n;
207 struct kvm_memory_slot *memslot;
208 int is_dirty = 0;
209
210 mutex_lock(&kvm->slots_lock);
211
212 r = -EINVAL;
213 if (log->slot >= KVM_USER_MEM_SLOTS)
214 goto out;
215
216 memslot = id_to_memslot(kvm->memslots, log->slot);
217 r = -ENOENT;
218 if (!memslot->dirty_bitmap)
219 goto out;
220
221 kvm_s390_sync_dirty_log(kvm, memslot);
222 r = kvm_get_dirty_log(kvm, log, &is_dirty);
223 if (r)
224 goto out;
225
226 /* Clear the dirty log */
227 if (is_dirty) {
228 n = kvm_dirty_bitmap_bytes(memslot);
229 memset(memslot->dirty_bitmap, 0, n);
230 }
231 r = 0;
232out:
233 mutex_unlock(&kvm->slots_lock);
234 return r;
b0c632db
HC		 235}
236
d938dc55
CH		 237static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
238{
239 int r;
240
241 if (cap->flags)
242 return -EINVAL;
243
244 switch (cap->cap) {
84223598
CH		 245 case KVM_CAP_S390_IRQCHIP:
246 kvm->arch.use_irqchip = 1;
247 r = 0;
248 break;
d938dc55
CH		 249 default:
250 r = -EINVAL;
251 break;
252 }
253 return r;
254}
255
4f718eab
DD		 256static int kvm_s390_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
257{
258 int ret;
259 unsigned int idx;
260 switch (attr->attr) {
261 case KVM_S390_VM_MEM_ENABLE_CMMA:
262 ret = -EBUSY;
263 mutex_lock(&kvm->lock);
264 if (atomic_read(&kvm->online_vcpus) == 0) {
265 kvm->arch.use_cmma = 1;
266 ret = 0;
267 }
268 mutex_unlock(&kvm->lock);
269 break;
270 case KVM_S390_VM_MEM_CLR_CMMA:
271 mutex_lock(&kvm->lock);
272 idx = srcu_read_lock(&kvm->srcu);
273 page_table_reset_pgste(kvm->arch.gmap->mm, 0, TASK_SIZE, false);
274 srcu_read_unlock(&kvm->srcu, idx);
275 mutex_unlock(&kvm->lock);
276 ret = 0;
277 break;
278 default:
279 ret = -ENXIO;
280 break;
281 }
282 return ret;
283}
284
f2061656
DD		 285static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
286{
287 int ret;
288
289 switch (attr->group) {
4f718eab
DD		 290 case KVM_S390_VM_MEM_CTRL:
291 ret = kvm_s390_mem_control(kvm, attr);
292 break;
f2061656
DD		 293 default:
294 ret = -ENXIO;
295 break;
296 }
297
298 return ret;
299}
300
301static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
302{
303 return -ENXIO;
304}
305
306static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
307{
308 int ret;
309
310 switch (attr->group) {
4f718eab
DD		 311 case KVM_S390_VM_MEM_CTRL:
312 switch (attr->attr) {
313 case KVM_S390_VM_MEM_ENABLE_CMMA:
314 case KVM_S390_VM_MEM_CLR_CMMA:
315 ret = 0;
316 break;
317 default:
318 ret = -ENXIO;
319 break;
320 }
321 break;
f2061656
DD		 322 default:
323 ret = -ENXIO;
324 break;
325 }
326
327 return ret;
328}
329
b0c632db
HC		 330long kvm_arch_vm_ioctl(struct file *filp,
331 unsigned int ioctl, unsigned long arg)
332{
333 struct kvm *kvm = filp->private_data;
334 void __user *argp = (void __user *)arg;
f2061656 335 struct kvm_device_attr attr;
b0c632db
HC		 336 int r;
337
338 switch (ioctl) {
ba5c1e9b
CO		 339 case KVM_S390_INTERRUPT: {
340 struct kvm_s390_interrupt s390int;
341
342 r = -EFAULT;
343 if (copy_from_user(&s390int, argp, sizeof(s390int)))
344 break;
345 r = kvm_s390_inject_vm(kvm, &s390int);
346 break;
347 }
d938dc55
CH		 348 case KVM_ENABLE_CAP: {
349 struct kvm_enable_cap cap;
350 r = -EFAULT;
351 if (copy_from_user(&cap, argp, sizeof(cap)))
352 break;
353 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
354 break;
355 }
84223598
CH		 356 case KVM_CREATE_IRQCHIP: {
357 struct kvm_irq_routing_entry routing;
358
359 r = -EINVAL;
360 if (kvm->arch.use_irqchip) {
361 /* Set up dummy routing. */
362 memset(&routing, 0, sizeof(routing));
363 kvm_set_irq_routing(kvm, &routing, 0, 0);
364 r = 0;
365 }
366 break;
367 }
f2061656
DD		 368 case KVM_SET_DEVICE_ATTR: {
369 r = -EFAULT;
370 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
371 break;
372 r = kvm_s390_vm_set_attr(kvm, &attr);
373 break;
374 }
375 case KVM_GET_DEVICE_ATTR: {
376 r = -EFAULT;
377 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
378 break;
379 r = kvm_s390_vm_get_attr(kvm, &attr);
380 break;
381 }
382 case KVM_HAS_DEVICE_ATTR: {
383 r = -EFAULT;
384 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
385 break;
386 r = kvm_s390_vm_has_attr(kvm, &attr);
387 break;
388 }
b0c632db 389 default:
367e1319 390 r = -ENOTTY;
b0c632db
HC		 391 }
392
393 return r;
394}
395
5102ee87
TK		 396static int kvm_s390_crypto_init(struct kvm *kvm)
397{
398 if (!test_vfacility(76))
399 return 0;
400
401 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
402 GFP_KERNEL | GFP_DMA);
403 if (!kvm->arch.crypto.crycb)
404 return -ENOMEM;
405
406 kvm->arch.crypto.crycbd = (__u32) (unsigned long) kvm->arch.crypto.crycb |
407 CRYCB_FORMAT1;
408
409 return 0;
410}
411
e08b9637 412int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 413{
b0c632db
HC		 414 int rc;
415 char debug_name[16];
f6c137ff 416 static unsigned long sca_offset;
b0c632db 417
e08b9637
CO		 418 rc = -EINVAL;
419#ifdef CONFIG_KVM_S390_UCONTROL
420 if (type & ~KVM_VM_S390_UCONTROL)
421 goto out_err;
422 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
423 goto out_err;
424#else
425 if (type)
426 goto out_err;
427#endif
428
b0c632db
HC		 429 rc = s390_enable_sie();
430 if (rc)
d89f5eff 431 goto out_err;
b0c632db 432
b290411a
CO		 433 rc = -ENOMEM;
434
b0c632db
HC		 435 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
436 if (!kvm->arch.sca)
d89f5eff 437 goto out_err;
f6c137ff
CB		 438 spin_lock(&kvm_lock);
439 sca_offset = (sca_offset + 16) & 0x7f0;
440 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
441 spin_unlock(&kvm_lock);
b0c632db
HC		 442
443 sprintf(debug_name, "kvm-%u", current->pid);
444
445 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
446 if (!kvm->arch.dbf)
447 goto out_nodbf;
448
5102ee87
TK		 449 if (kvm_s390_crypto_init(kvm) < 0)
450 goto out_crypto;
451
ba5c1e9b
CO		 452 spin_lock_init(&kvm->arch.float_int.lock);
453 INIT_LIST_HEAD(&kvm->arch.float_int.list);
8a242234 454 init_waitqueue_head(&kvm->arch.ipte_wq);
ba5c1e9b 455
b0c632db
HC		 456 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
457 VM_EVENT(kvm, 3, "%s", "vm created");
458
e08b9637
CO		 459 if (type & KVM_VM_S390_UCONTROL) {
460 kvm->arch.gmap = NULL;
461 } else {
0349985a 462 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
e08b9637
CO		 463 if (!kvm->arch.gmap)
464 goto out_nogmap;
2c70fe44 465 kvm->arch.gmap->private = kvm;
24eb3a82 466 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 467 }
fa6b7fe9
CH		 468
469 kvm->arch.css_support = 0;
84223598 470 kvm->arch.use_irqchip = 0;
fa6b7fe9 471
8ad35755
DH		 472 spin_lock_init(&kvm->arch.start_stop_lock);
473
d89f5eff 474 return 0;
598841ca 475out_nogmap:
5102ee87
TK		 476 kfree(kvm->arch.crypto.crycb);
477out_crypto:
598841ca 478 debug_unregister(kvm->arch.dbf);
b0c632db
HC		 479out_nodbf:
480 free_page((unsigned long)(kvm->arch.sca));
d89f5eff
JK		 481out_err:
482 return rc;
b0c632db
HC		 483}
484
d329c035
CB		 485void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
486{
487 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 488 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 489 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 490 kvm_clear_async_pf_completion_queue(vcpu);
58f9460b
CO		 491 if (!kvm_is_ucontrol(vcpu->kvm)) {
492 clear_bit(63 - vcpu->vcpu_id,
493 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
494 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
495 (__u64) vcpu->arch.sie_block)
496 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
497 }
abf4a71e 498 smp_mb();
27e0393f
CO		 499
500 if (kvm_is_ucontrol(vcpu->kvm))
501 gmap_free(vcpu->arch.gmap);
502
b31605c1
DD		 503 if (kvm_s390_cmma_enabled(vcpu->kvm))
504 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 505 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 506
6692cef3 507 kvm_vcpu_uninit(vcpu);
b110feaf 508 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 509}
510
511static void kvm_free_vcpus(struct kvm *kvm)
512{
513 unsigned int i;
988a2cae 514 struct kvm_vcpu *vcpu;
d329c035 515
988a2cae
GN		 516 kvm_for_each_vcpu(i, vcpu, kvm)
517 kvm_arch_vcpu_destroy(vcpu);
518
519 mutex_lock(&kvm->lock);
520 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
521 kvm->vcpus[i] = NULL;
522
523 atomic_set(&kvm->online_vcpus, 0);
524 mutex_unlock(&kvm->lock);
d329c035
CB		 525}
526
b0c632db
HC		 527void kvm_arch_destroy_vm(struct kvm *kvm)
528{
d329c035 529 kvm_free_vcpus(kvm);
b0c632db 530 free_page((unsigned long)(kvm->arch.sca));
d329c035 531 debug_unregister(kvm->arch.dbf);
5102ee87 532 kfree(kvm->arch.crypto.crycb);
27e0393f
CO		 533 if (!kvm_is_ucontrol(kvm))
534 gmap_free(kvm->arch.gmap);
841b91c5 535 kvm_s390_destroy_adapters(kvm);
67335e63 536 kvm_s390_clear_float_irqs(kvm);
b0c632db
HC		 537}
538
539/* Section: vcpu related */
540int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
541{
3c038e6b
DD		 542 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
543 kvm_clear_async_pf_completion_queue(vcpu);
27e0393f 544 if (kvm_is_ucontrol(vcpu->kvm)) {
c6c956b8 545 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
27e0393f
CO		 546 if (!vcpu->arch.gmap)
547 return -ENOMEM;
2c70fe44 548 vcpu->arch.gmap->private = vcpu->kvm;
27e0393f
CO		 549 return 0;
550 }
551
598841ca 552 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
59674c1a
CB		 553 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
554 KVM_SYNC_GPRS |
9eed0735 555 KVM_SYNC_ACRS |
b028ee3e
DH		 556 KVM_SYNC_CRS |
557 KVM_SYNC_ARCH0 |
558 KVM_SYNC_PFAULT;
b0c632db
HC		 559 return 0;
560}
561
b0c632db
HC		 562void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
563{
4725c860
MS		 564 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
565 save_fp_regs(vcpu->arch.host_fpregs.fprs);
b0c632db 566 save_access_regs(vcpu->arch.host_acrs);
4725c860
MS		 567 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
568 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
59674c1a 569 restore_access_regs(vcpu->run->s.regs.acrs);
480e5926 570 gmap_enable(vcpu->arch.gmap);
9e6dabef 571 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
b0c632db
HC		 572}
573
574void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
575{
9e6dabef 576 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
480e5926 577 gmap_disable(vcpu->arch.gmap);
4725c860
MS		 578 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
579 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
59674c1a 580 save_access_regs(vcpu->run->s.regs.acrs);
4725c860
MS		 581 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
582 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
b0c632db
HC		 583 restore_access_regs(vcpu->arch.host_acrs);
584}
585
586static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
587{
588 /* this equals initial cpu reset in pop, but we don't switch to ESA */
589 vcpu->arch.sie_block->gpsw.mask = 0UL;
590 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 591 kvm_s390_set_prefix(vcpu, 0);
b0c632db
HC		 592 vcpu->arch.sie_block->cputm = 0UL;
593 vcpu->arch.sie_block->ckc = 0UL;
594 vcpu->arch.sie_block->todpr = 0;
595 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
596 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
597 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
598 vcpu->arch.guest_fpregs.fpc = 0;
599 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
600 vcpu->arch.sie_block->gbea = 1;
672550fb 601 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 602 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
603 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 604 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
605 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 606 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 607}
608
42897d86
MT		 609int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
610{
611 return 0;
612}
613
5102ee87
TK		 614static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
615{
616 if (!test_vfacility(76))
617 return;
618
619 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
620}
621
b31605c1
DD		 622void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
623{
624 free_page(vcpu->arch.sie_block->cbrlo);
625 vcpu->arch.sie_block->cbrlo = 0;
626}
627
628int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
629{
630 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
631 if (!vcpu->arch.sie_block->cbrlo)
632 return -ENOMEM;
633
634 vcpu->arch.sie_block->ecb2 |= 0x80;
635 vcpu->arch.sie_block->ecb2 &= ~0x08;
636 return 0;
637}
638
b0c632db
HC		 639int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
640{
b31605c1 641 int rc = 0;
b31288fa 642
9e6dabef
CH		 643 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
644 CPUSTAT_SM |
69d0d3a3
CB		 645 CPUSTAT_STOPPED |
646 CPUSTAT_GED);
fc34531d 647 vcpu->arch.sie_block->ecb = 6;
7feb6bb8
MM		 648 if (test_vfacility(50) && test_vfacility(73))
649 vcpu->arch.sie_block->ecb |= 0x10;
650
69d0d3a3 651 vcpu->arch.sie_block->ecb2 = 8;
4953919f 652 vcpu->arch.sie_block->eca = 0xD1002000U;
217a4406
HC		 653 if (sclp_has_siif())
654 vcpu->arch.sie_block->eca |= 1;
78c4b59f 655 vcpu->arch.sie_block->fac = (int) (long) vfacilities;
5a5e6536
MR		 656 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE |
657 ICTL_TPROT;
658
b31605c1
DD		 659 if (kvm_s390_cmma_enabled(vcpu->kvm)) {
660 rc = kvm_s390_vcpu_setup_cmma(vcpu);
661 if (rc)
662 return rc;
b31288fa 663 }
ca872302 664 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
ca872302 665 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
453423dc 666 get_cpu_id(&vcpu->arch.cpu_id);
92e6ecf3 667 vcpu->arch.cpu_id.version = 0xff;
5102ee87
TK		 668
669 kvm_s390_vcpu_crypto_setup(vcpu);
670
b31605c1 671 return rc;
b0c632db
HC		 672}
673
674struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
675 unsigned int id)
676{
4d47555a 677 struct kvm_vcpu *vcpu;
7feb6bb8 678 struct sie_page *sie_page;
4d47555a
CO		 679 int rc = -EINVAL;
680
681 if (id >= KVM_MAX_VCPUS)
682 goto out;
683
684 rc = -ENOMEM;
b0c632db 685
b110feaf 686 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 687 if (!vcpu)
4d47555a 688 goto out;
b0c632db 689
7feb6bb8
MM		 690 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
691 if (!sie_page)
b0c632db
HC		 692 goto out_free_cpu;
693
7feb6bb8
MM		 694 vcpu->arch.sie_block = &sie_page->sie_block;
695 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
696
b0c632db 697 vcpu->arch.sie_block->icpua = id;
58f9460b
CO		 698 if (!kvm_is_ucontrol(kvm)) {
699 if (!kvm->arch.sca) {
700 WARN_ON_ONCE(1);
701 goto out_free_cpu;
702 }
703 if (!kvm->arch.sca->cpu[id].sda)
704 kvm->arch.sca->cpu[id].sda =
705 (__u64) vcpu->arch.sie_block;
706 vcpu->arch.sie_block->scaoh =
707 (__u32)(((__u64)kvm->arch.sca) >> 32);
708 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
709 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
710 }
b0c632db 711
ba5c1e9b
CO		 712 spin_lock_init(&vcpu->arch.local_int.lock);
713 INIT_LIST_HEAD(&vcpu->arch.local_int.list);
714 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 715 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 716 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
ba5c1e9b 717
b0c632db
HC		 718 rc = kvm_vcpu_init(vcpu, kvm, id);
719 if (rc)
7b06bf2f 720 goto out_free_sie_block;
b0c632db
HC		 721 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
722 vcpu->arch.sie_block);
ade38c31 723 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 724
b0c632db 725 return vcpu;
7b06bf2f
WY		 726out_free_sie_block:
727 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 728out_free_cpu:
b110feaf 729 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 730out:
b0c632db
HC		 731 return ERR_PTR(rc);
732}
733
b0c632db
HC		 734int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
735{
f87618e8 736 return kvm_cpu_has_interrupt(vcpu);
b0c632db
HC		 737}
738
49b99e1e
CB		 739void s390_vcpu_block(struct kvm_vcpu *vcpu)
740{
741 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
742}
743
744void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
745{
746 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
747}
748
749/*
750 * Kick a guest cpu out of SIE and wait until SIE is not running.
751 * If the CPU is not running (e.g. waiting as idle) the function will
752 * return immediately. */
753void exit_sie(struct kvm_vcpu *vcpu)
754{
755 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
756 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
757 cpu_relax();
758}
759
760/* Kick a guest cpu out of SIE and prevent SIE-reentry */
761void exit_sie_sync(struct kvm_vcpu *vcpu)
762{
763 s390_vcpu_block(vcpu);
764 exit_sie(vcpu);
765}
766
2c70fe44
CB		 767static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
768{
769 int i;
770 struct kvm *kvm = gmap->private;
771 struct kvm_vcpu *vcpu;
772
773 kvm_for_each_vcpu(i, vcpu, kvm) {
774 /* match against both prefix pages */
fda902cb 775 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
2c70fe44
CB		 776 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
777 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
778 exit_sie_sync(vcpu);
779 }
780 }
781}
782
b6d33834
CD		 783int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
784{
785 /* kvm common code refers to this, but never calls it */
786 BUG();
787 return 0;
788}
789
14eebd91
CO		 790static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
791 struct kvm_one_reg *reg)
792{
793 int r = -EINVAL;
794
795 switch (reg->id) {
29b7c71b
CO		 796 case KVM_REG_S390_TODPR:
797 r = put_user(vcpu->arch.sie_block->todpr,
798 (u32 __user *)reg->addr);
799 break;
800 case KVM_REG_S390_EPOCHDIFF:
801 r = put_user(vcpu->arch.sie_block->epoch,
802 (u64 __user *)reg->addr);
803 break;
46a6dd1c
J		 804 case KVM_REG_S390_CPU_TIMER:
805 r = put_user(vcpu->arch.sie_block->cputm,
806 (u64 __user *)reg->addr);
807 break;
808 case KVM_REG_S390_CLOCK_COMP:
809 r = put_user(vcpu->arch.sie_block->ckc,
810 (u64 __user *)reg->addr);
811 break;
536336c2
DD		 812 case KVM_REG_S390_PFTOKEN:
813 r = put_user(vcpu->arch.pfault_token,
814 (u64 __user *)reg->addr);
815 break;
816 case KVM_REG_S390_PFCOMPARE:
817 r = put_user(vcpu->arch.pfault_compare,
818 (u64 __user *)reg->addr);
819 break;
820 case KVM_REG_S390_PFSELECT:
821 r = put_user(vcpu->arch.pfault_select,
822 (u64 __user *)reg->addr);
823 break;
672550fb
CB		 824 case KVM_REG_S390_PP:
825 r = put_user(vcpu->arch.sie_block->pp,
826 (u64 __user *)reg->addr);
827 break;
afa45ff5
CB		 828 case KVM_REG_S390_GBEA:
829 r = put_user(vcpu->arch.sie_block->gbea,
830 (u64 __user *)reg->addr);
831 break;
14eebd91
CO		 832 default:
833 break;
834 }
835
836 return r;
837}
838
839static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
840 struct kvm_one_reg *reg)
841{
842 int r = -EINVAL;
843
844 switch (reg->id) {
29b7c71b
CO		 845 case KVM_REG_S390_TODPR:
846 r = get_user(vcpu->arch.sie_block->todpr,
847 (u32 __user *)reg->addr);
848 break;
849 case KVM_REG_S390_EPOCHDIFF:
850 r = get_user(vcpu->arch.sie_block->epoch,
851 (u64 __user *)reg->addr);
852 break;
46a6dd1c
J		 853 case KVM_REG_S390_CPU_TIMER:
854 r = get_user(vcpu->arch.sie_block->cputm,
855 (u64 __user *)reg->addr);
856 break;
857 case KVM_REG_S390_CLOCK_COMP:
858 r = get_user(vcpu->arch.sie_block->ckc,
859 (u64 __user *)reg->addr);
860 break;
536336c2
DD		 861 case KVM_REG_S390_PFTOKEN:
862 r = get_user(vcpu->arch.pfault_token,
863 (u64 __user *)reg->addr);
864 break;
865 case KVM_REG_S390_PFCOMPARE:
866 r = get_user(vcpu->arch.pfault_compare,
867 (u64 __user *)reg->addr);
868 break;
869 case KVM_REG_S390_PFSELECT:
870 r = get_user(vcpu->arch.pfault_select,
871 (u64 __user *)reg->addr);
872 break;
672550fb
CB		 873 case KVM_REG_S390_PP:
874 r = get_user(vcpu->arch.sie_block->pp,
875 (u64 __user *)reg->addr);
876 break;
afa45ff5
CB		 877 case KVM_REG_S390_GBEA:
878 r = get_user(vcpu->arch.sie_block->gbea,
879 (u64 __user *)reg->addr);
880 break;
14eebd91
CO		 881 default:
882 break;
883 }
884
885 return r;
886}
b6d33834 887
b0c632db
HC		 888static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
889{
b0c632db 890 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 891 return 0;
892}
893
894int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
895{
5a32c1af 896 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 897 return 0;
898}
899
900int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
901{
5a32c1af 902 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 903 return 0;
904}
905
906int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
907 struct kvm_sregs *sregs)
908{
59674c1a 909 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 910 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
59674c1a 911 restore_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 912 return 0;
913}
914
915int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
916 struct kvm_sregs *sregs)
917{
59674c1a 918 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 919 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 920 return 0;
921}
922
923int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
924{
4725c860
MS		 925 if (test_fp_ctl(fpu->fpc))
926 return -EINVAL;
b0c632db 927 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
4725c860
MS		 928 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
929 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
930 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
b0c632db
HC		 931 return 0;
932}
933
934int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
935{
b0c632db
HC		 936 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
937 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
b0c632db
HC		 938 return 0;
939}
940
941static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
942{
943 int rc = 0;
944
7a42fdc2 945 if (!is_vcpu_stopped(vcpu))
b0c632db 946 rc = -EBUSY;
d7b0b5eb
CO		 947 else {
948 vcpu->run->psw_mask = psw.mask;
949 vcpu->run->psw_addr = psw.addr;
950 }
b0c632db
HC		 951 return rc;
952}
953
954int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
955 struct kvm_translation *tr)
956{
957 return -EINVAL; /* not implemented yet */
958}
959
27291e21
DH		 960#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
961 KVM_GUESTDBG_USE_HW_BP | \
962 KVM_GUESTDBG_ENABLE)
963
d0bfb940
JK		 964int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
965 struct kvm_guest_debug *dbg)
b0c632db 966{
27291e21
DH		 967 int rc = 0;
968
969 vcpu->guest_debug = 0;
970 kvm_s390_clear_bp_data(vcpu);
971
2de3bfc2 972 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21
DH		 973 return -EINVAL;
974
975 if (dbg->control & KVM_GUESTDBG_ENABLE) {
976 vcpu->guest_debug = dbg->control;
977 /* enforce guest PER */
978 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
979
980 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
981 rc = kvm_s390_import_bp_data(vcpu, dbg);
982 } else {
983 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
984 vcpu->arch.guestdbg.last_bp = 0;
985 }
986
987 if (rc) {
988 vcpu->guest_debug = 0;
989 kvm_s390_clear_bp_data(vcpu);
990 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
991 }
992
993 return rc;
b0c632db
HC		 994}
995
62d9f0db
MT		 996int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
997 struct kvm_mp_state *mp_state)
998{
6352e4d2
DH		 999 /* CHECK_STOP and LOAD are not supported yet */
1000 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1001 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 1002}
1003
1004int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1005 struct kvm_mp_state *mp_state)
1006{
6352e4d2
DH		 1007 int rc = 0;
1008
1009 /* user space knows about this interface - let it control the state */
1010 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1011
1012 switch (mp_state->mp_state) {
1013 case KVM_MP_STATE_STOPPED:
1014 kvm_s390_vcpu_stop(vcpu);
1015 break;
1016 case KVM_MP_STATE_OPERATING:
1017 kvm_s390_vcpu_start(vcpu);
1018 break;
1019 case KVM_MP_STATE_LOAD:
1020 case KVM_MP_STATE_CHECK_STOP:
1021 /* fall through - CHECK_STOP and LOAD are not supported yet */
1022 default:
1023 rc = -ENXIO;
1024 }
1025
1026 return rc;
62d9f0db
MT		 1027}
1028
b31605c1
DD		 1029bool kvm_s390_cmma_enabled(struct kvm *kvm)
1030{
1031 if (!MACHINE_IS_LPAR)
1032 return false;
1033 /* only enable for z10 and later */
1034 if (!MACHINE_HAS_EDAT1)
1035 return false;
1036 if (!kvm->arch.use_cmma)
1037 return false;
1038 return true;
1039}
1040
8ad35755
DH		 1041static bool ibs_enabled(struct kvm_vcpu *vcpu)
1042{
1043 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1044}
1045
2c70fe44
CB		 1046static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1047{
8ad35755
DH		 1048retry:
1049 s390_vcpu_unblock(vcpu);
2c70fe44
CB		 1050 /*
1051 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1052 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1053 * This ensures that the ipte instruction for this request has
1054 * already finished. We might race against a second unmapper that
1055 * wants to set the blocking bit. Lets just retry the request loop.
1056 */
8ad35755 1057 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44
CB		 1058 int rc;
1059 rc = gmap_ipte_notify(vcpu->arch.gmap,
fda902cb 1060 kvm_s390_get_prefix(vcpu),
2c70fe44
CB		 1061 PAGE_SIZE * 2);
1062 if (rc)
1063 return rc;
8ad35755 1064 goto retry;
2c70fe44 1065 }
8ad35755 1066
d3d692c8
DH		 1067 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1068 vcpu->arch.sie_block->ihcpu = 0xffff;
1069 goto retry;
1070 }
1071
8ad35755
DH		 1072 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1073 if (!ibs_enabled(vcpu)) {
1074 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1075 atomic_set_mask(CPUSTAT_IBS,
1076 &vcpu->arch.sie_block->cpuflags);
1077 }
1078 goto retry;
2c70fe44 1079 }
8ad35755
DH		 1080
1081 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1082 if (ibs_enabled(vcpu)) {
1083 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1084 atomic_clear_mask(CPUSTAT_IBS,
1085 &vcpu->arch.sie_block->cpuflags);
1086 }
1087 goto retry;
1088 }
1089
0759d068
DH		 1090 /* nothing to do, just clear the request */
1091 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1092
2c70fe44
CB		 1093 return 0;
1094}
1095
fa576c58
TH		 1096/**
1097 * kvm_arch_fault_in_page - fault-in guest page if necessary
1098 * @vcpu: The corresponding virtual cpu
1099 * @gpa: Guest physical address
1100 * @writable: Whether the page should be writable or not
1101 *
1102 * Make sure that a guest page has been faulted-in on the host.
1103 *
1104 * Return: Zero on success, negative error code otherwise.
1105 */
1106long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 1107{
527e30b4
MS		 1108 return gmap_fault(vcpu->arch.gmap, gpa,
1109 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 1110}
1111
3c038e6b
DD		 1112static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1113 unsigned long token)
1114{
1115 struct kvm_s390_interrupt inti;
1116 inti.parm64 = token;
1117
1118 if (start_token) {
1119 inti.type = KVM_S390_INT_PFAULT_INIT;
1120 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &inti));
1121 } else {
1122 inti.type = KVM_S390_INT_PFAULT_DONE;
1123 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1124 }
1125}
1126
1127void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1128 struct kvm_async_pf *work)
1129{
1130 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1131 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1132}
1133
1134void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1135 struct kvm_async_pf *work)
1136{
1137 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1138 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1139}
1140
1141void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1142 struct kvm_async_pf *work)
1143{
1144 /* s390 will always inject the page directly */
1145}
1146
1147bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1148{
1149 /*
1150 * s390 will always inject the page directly,
1151 * but we still want check_async_completion to cleanup
1152 */
1153 return true;
1154}
1155
1156static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1157{
1158 hva_t hva;
1159 struct kvm_arch_async_pf arch;
1160 int rc;
1161
1162 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1163 return 0;
1164 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1165 vcpu->arch.pfault_compare)
1166 return 0;
1167 if (psw_extint_disabled(vcpu))
1168 return 0;
1169 if (kvm_cpu_has_interrupt(vcpu))
1170 return 0;
1171 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1172 return 0;
1173 if (!vcpu->arch.gmap->pfault_enabled)
1174 return 0;
1175
81480cc1
HC		 1176 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1177 hva += current->thread.gmap_addr & ~PAGE_MASK;
1178 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 1179 return 0;
1180
1181 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1182 return rc;
1183}
1184
3fb4c40f 1185static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 1186{
3fb4c40f 1187 int rc, cpuflags;
e168bf8d 1188
3c038e6b
DD		 1189 /*
1190 * On s390 notifications for arriving pages will be delivered directly
1191 * to the guest but the house keeping for completed pfaults is
1192 * handled outside the worker.
1193 */
1194 kvm_check_async_pf_completion(vcpu);
1195
5a32c1af 1196 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
b0c632db
HC		 1197
1198 if (need_resched())
1199 schedule();
1200
d3a73acb 1201 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 1202 s390_handle_mcck();
1203
79395031
JF		 1204 if (!kvm_is_ucontrol(vcpu->kvm)) {
1205 rc = kvm_s390_deliver_pending_interrupts(vcpu);
1206 if (rc)
1207 return rc;
1208 }
0ff31867 1209
2c70fe44
CB		 1210 rc = kvm_s390_handle_requests(vcpu);
1211 if (rc)
1212 return rc;
1213
27291e21
DH		 1214 if (guestdbg_enabled(vcpu)) {
1215 kvm_s390_backup_guest_per_regs(vcpu);
1216 kvm_s390_patch_guest_per_regs(vcpu);
1217 }
1218
b0c632db 1219 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 1220 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1221 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1222 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 1223
3fb4c40f
TH		 1224 return 0;
1225}
1226
1227static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1228{
24eb3a82 1229 int rc = -1;
2b29a9fd
DD		 1230
1231 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1232 vcpu->arch.sie_block->icptcode);
1233 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1234
27291e21
DH		 1235 if (guestdbg_enabled(vcpu))
1236 kvm_s390_restore_guest_per_regs(vcpu);
1237
3fb4c40f 1238 if (exit_reason >= 0) {
7c470539 1239 rc = 0;
210b1607
TH		 1240 } else if (kvm_is_ucontrol(vcpu->kvm)) {
1241 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1242 vcpu->run->s390_ucontrol.trans_exc_code =
1243 current->thread.gmap_addr;
1244 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1245 rc = -EREMOTE;
24eb3a82
DD		 1246
1247 } else if (current->thread.gmap_pfault) {
3c038e6b 1248 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 1249 current->thread.gmap_pfault = 0;
fa576c58 1250 if (kvm_arch_setup_async_pf(vcpu)) {
24eb3a82 1251 rc = 0;
fa576c58
TH		 1252 } else {
1253 gpa_t gpa = current->thread.gmap_addr;
1254 rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1255 }
24eb3a82
DD		 1256 }
1257
1258 if (rc == -1) {
699bde3b
CB		 1259 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1260 trace_kvm_s390_sie_fault(vcpu);
1261 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1f0d0f09 1262 }
b0c632db 1263
5a32c1af 1264 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
3fb4c40f 1265
a76ccff6
TH		 1266 if (rc == 0) {
1267 if (kvm_is_ucontrol(vcpu->kvm))
2955c83f
CB		 1268 /* Don't exit for host interrupts. */
1269 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
a76ccff6
TH		 1270 else
1271 rc = kvm_handle_sie_intercept(vcpu);
1272 }
1273
3fb4c40f
TH		 1274 return rc;
1275}
1276
1277static int __vcpu_run(struct kvm_vcpu *vcpu)
1278{
1279 int rc, exit_reason;
1280
800c1065
TH		 1281 /*
1282 * We try to hold kvm->srcu during most of vcpu_run (except when run-
1283 * ning the guest), so that memslots (and other stuff) are protected
1284 */
1285 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1286
a76ccff6
TH		 1287 do {
1288 rc = vcpu_pre_run(vcpu);
1289 if (rc)
1290 break;
3fb4c40f 1291
800c1065 1292 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 1293 /*
1294 * As PF_VCPU will be used in fault handler, between
1295 * guest_enter and guest_exit should be no uaccess.
1296 */
1297 preempt_disable();
1298 kvm_guest_enter();
1299 preempt_enable();
1300 exit_reason = sie64a(vcpu->arch.sie_block,
1301 vcpu->run->s.regs.gprs);
1302 kvm_guest_exit();
800c1065 1303 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 1304
1305 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 1306 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 1307
800c1065 1308 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 1309 return rc;
b0c632db
HC		 1310}
1311
b028ee3e
DH		 1312static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1313{
1314 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1315 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
1316 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
1317 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1318 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1319 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 1320 /* some control register changes require a tlb flush */
1321 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 1322 }
1323 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
1324 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
1325 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
1326 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
1327 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
1328 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
1329 }
1330 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
1331 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
1332 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
1333 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
1334 }
1335 kvm_run->kvm_dirty_regs = 0;
1336}
1337
1338static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1339{
1340 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
1341 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
1342 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
1343 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
1344 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
1345 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
1346 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
1347 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
1348 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
1349 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
1350 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
1351 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
1352}
1353
b0c632db
HC		 1354int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1355{
8f2abe6a 1356 int rc;
b0c632db
HC		 1357 sigset_t sigsaved;
1358
27291e21
DH		 1359 if (guestdbg_exit_pending(vcpu)) {
1360 kvm_s390_prepare_debug_exit(vcpu);
1361 return 0;
1362 }
1363
b0c632db
HC		 1364 if (vcpu->sigset_active)
1365 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1366
6352e4d2
DH		 1367 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
1368 kvm_s390_vcpu_start(vcpu);
1369 } else if (is_vcpu_stopped(vcpu)) {
1370 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
1371 vcpu->vcpu_id);
1372 return -EINVAL;
1373 }
b0c632db 1374
b028ee3e 1375 sync_regs(vcpu, kvm_run);
d7b0b5eb 1376
dab4079d 1377 might_fault();
a76ccff6 1378 rc = __vcpu_run(vcpu);
9ace903d 1379
b1d16c49
CE		 1380 if (signal_pending(current) && !rc) {
1381 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 1382 rc = -EINTR;
b1d16c49 1383 }
8f2abe6a 1384
27291e21
DH		 1385 if (guestdbg_exit_pending(vcpu) && !rc) {
1386 kvm_s390_prepare_debug_exit(vcpu);
1387 rc = 0;
1388 }
1389
b8e660b8 1390 if (rc == -EOPNOTSUPP) {
8f2abe6a
CB		 1391 /* intercept cannot be handled in-kernel, prepare kvm-run */
1392 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
1393 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
8f2abe6a
CB		 1394 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
1395 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
1396 rc = 0;
1397 }
1398
1399 if (rc == -EREMOTE) {
1400 /* intercept was handled, but userspace support is needed
1401 * kvm_run has been prepared by the handler */
1402 rc = 0;
1403 }
b0c632db 1404
b028ee3e 1405 store_regs(vcpu, kvm_run);
d7b0b5eb 1406
b0c632db
HC		 1407 if (vcpu->sigset_active)
1408 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1409
b0c632db 1410 vcpu->stat.exit_userspace++;
7e8e6ab4 1411 return rc;
b0c632db
HC		 1412}
1413
b0c632db
HC		 1414/*
1415 * store status at address
1416 * we use have two special cases:
1417 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1418 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1419 */
d0bce605 1420int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 1421{
092670cd 1422 unsigned char archmode = 1;
fda902cb 1423 unsigned int px;
178bd789 1424 u64 clkcomp;
d0bce605 1425 int rc;
b0c632db 1426
d0bce605
HC		 1427 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
1428 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 1429 return -EFAULT;
d0bce605
HC		 1430 gpa = SAVE_AREA_BASE;
1431 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
1432 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 1433 return -EFAULT;
d0bce605
HC		 1434 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
1435 }
1436 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
1437 vcpu->arch.guest_fpregs.fprs, 128);
1438 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
1439 vcpu->run->s.regs.gprs, 128);
1440 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
1441 &vcpu->arch.sie_block->gpsw, 16);
fda902cb 1442 px = kvm_s390_get_prefix(vcpu);
d0bce605 1443 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
fda902cb 1444 &px, 4);
d0bce605
HC		 1445 rc |= write_guest_abs(vcpu,
1446 gpa + offsetof(struct save_area, fp_ctrl_reg),
1447 &vcpu->arch.guest_fpregs.fpc, 4);
1448 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
1449 &vcpu->arch.sie_block->todpr, 4);
1450 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
1451 &vcpu->arch.sie_block->cputm, 8);
178bd789 1452 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d0bce605
HC		 1453 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
1454 &clkcomp, 8);
1455 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
1456 &vcpu->run->s.regs.acrs, 64);
1457 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
1458 &vcpu->arch.sie_block->gcr, 128);
1459 return rc ? -EFAULT : 0;
b0c632db
HC		 1460}
1461
e879892c
TH		 1462int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1463{
1464 /*
1465 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1466 * copying in vcpu load/put. Lets update our copies before we save
1467 * it into the save area
1468 */
1469 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1470 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1471 save_access_regs(vcpu->run->s.regs.acrs);
1472
1473 return kvm_s390_store_status_unloaded(vcpu, addr);
1474}
1475
8ad35755
DH		 1476static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1477{
1478 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
1479 kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu);
1480 exit_sie_sync(vcpu);
1481}
1482
1483static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
1484{
1485 unsigned int i;
1486 struct kvm_vcpu *vcpu;
1487
1488 kvm_for_each_vcpu(i, vcpu, kvm) {
1489 __disable_ibs_on_vcpu(vcpu);
1490 }
1491}
1492
1493static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1494{
1495 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
1496 kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu);
1497 exit_sie_sync(vcpu);
1498}
1499
6852d7b6
DH		 1500void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
1501{
8ad35755
DH		 1502 int i, online_vcpus, started_vcpus = 0;
1503
1504 if (!is_vcpu_stopped(vcpu))
1505 return;
1506
6852d7b6 1507 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 1508 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 1509 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 1510 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1511
1512 for (i = 0; i < online_vcpus; i++) {
1513 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
1514 started_vcpus++;
1515 }
1516
1517 if (started_vcpus == 0) {
1518 /* we're the only active VCPU -> speed it up */
1519 __enable_ibs_on_vcpu(vcpu);
1520 } else if (started_vcpus == 1) {
1521 /*
1522 * As we are starting a second VCPU, we have to disable
1523 * the IBS facility on all VCPUs to remove potentially
1524 * oustanding ENABLE requests.
1525 */
1526 __disable_ibs_on_all_vcpus(vcpu->kvm);
1527 }
1528
6852d7b6 1529 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 1530 /*
1531 * Another VCPU might have used IBS while we were offline.
1532 * Let's play safe and flush the VCPU at startup.
1533 */
d3d692c8 1534 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 1535 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 1536 return;
6852d7b6
DH		 1537}
1538
1539void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
1540{
8ad35755
DH		 1541 int i, online_vcpus, started_vcpus = 0;
1542 struct kvm_vcpu *started_vcpu = NULL;
1543
1544 if (is_vcpu_stopped(vcpu))
1545 return;
1546
6852d7b6 1547 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 1548 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 1549 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 1550 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1551
32f5ff63 1552 /* Need to lock access to action_bits to avoid a SIGP race condition */
4ae3c081 1553 spin_lock(&vcpu->arch.local_int.lock);
6852d7b6 1554 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
32f5ff63
DH		 1555
1556 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
1557 vcpu->arch.local_int.action_bits &=
1558 ~(ACTION_STOP_ON_STOP | ACTION_STORE_ON_STOP);
4ae3c081 1559 spin_unlock(&vcpu->arch.local_int.lock);
32f5ff63 1560
8ad35755
DH		 1561 __disable_ibs_on_vcpu(vcpu);
1562
1563 for (i = 0; i < online_vcpus; i++) {
1564 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
1565 started_vcpus++;
1566 started_vcpu = vcpu->kvm->vcpus[i];
1567 }
1568 }
1569
1570 if (started_vcpus == 1) {
1571 /*
1572 * As we only have one VCPU left, we want to enable the
1573 * IBS facility for that VCPU to speed it up.
1574 */
1575 __enable_ibs_on_vcpu(started_vcpu);
1576 }
1577
433b9ee4 1578 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 1579 return;
6852d7b6
DH		 1580}
1581
d6712df9
CH		 1582static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1583 struct kvm_enable_cap *cap)
1584{
1585 int r;
1586
1587 if (cap->flags)
1588 return -EINVAL;
1589
1590 switch (cap->cap) {
fa6b7fe9
CH		 1591 case KVM_CAP_S390_CSS_SUPPORT:
1592 if (!vcpu->kvm->arch.css_support) {
1593 vcpu->kvm->arch.css_support = 1;
1594 trace_kvm_s390_enable_css(vcpu->kvm);
1595 }
1596 r = 0;
1597 break;
d6712df9
CH		 1598 default:
1599 r = -EINVAL;
1600 break;
1601 }
1602 return r;
1603}
1604
b0c632db
HC		 1605long kvm_arch_vcpu_ioctl(struct file *filp,
1606 unsigned int ioctl, unsigned long arg)
1607{
1608 struct kvm_vcpu *vcpu = filp->private_data;
1609 void __user *argp = (void __user *)arg;
800c1065 1610 int idx;
bc923cc9 1611 long r;
b0c632db 1612
93736624
AK		 1613 switch (ioctl) {
1614 case KVM_S390_INTERRUPT: {
ba5c1e9b
CO		 1615 struct kvm_s390_interrupt s390int;
1616
93736624 1617 r = -EFAULT;
ba5c1e9b 1618 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624
AK		 1619 break;
1620 r = kvm_s390_inject_vcpu(vcpu, &s390int);
1621 break;
ba5c1e9b 1622 }
b0c632db 1623 case KVM_S390_STORE_STATUS:
800c1065 1624 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 1625 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 1626 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 1627 break;
b0c632db
HC		 1628 case KVM_S390_SET_INITIAL_PSW: {
1629 psw_t psw;
1630
bc923cc9 1631 r = -EFAULT;
b0c632db 1632 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 1633 break;
1634 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
1635 break;
b0c632db
HC		 1636 }
1637 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 1638 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
1639 break;
14eebd91
CO		 1640 case KVM_SET_ONE_REG:
1641 case KVM_GET_ONE_REG: {
1642 struct kvm_one_reg reg;
1643 r = -EFAULT;
1644 if (copy_from_user(&reg, argp, sizeof(reg)))
1645 break;
1646 if (ioctl == KVM_SET_ONE_REG)
1647 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
1648 else
1649 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
1650 break;
1651 }
27e0393f
CO		 1652#ifdef CONFIG_KVM_S390_UCONTROL
1653 case KVM_S390_UCAS_MAP: {
1654 struct kvm_s390_ucas_mapping ucasmap;
1655
1656 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1657 r = -EFAULT;
1658 break;
1659 }
1660
1661 if (!kvm_is_ucontrol(vcpu->kvm)) {
1662 r = -EINVAL;
1663 break;
1664 }
1665
1666 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
1667 ucasmap.vcpu_addr, ucasmap.length);
1668 break;
1669 }
1670 case KVM_S390_UCAS_UNMAP: {
1671 struct kvm_s390_ucas_mapping ucasmap;
1672
1673 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1674 r = -EFAULT;
1675 break;
1676 }
1677
1678 if (!kvm_is_ucontrol(vcpu->kvm)) {
1679 r = -EINVAL;
1680 break;
1681 }
1682
1683 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
1684 ucasmap.length);
1685 break;
1686 }
1687#endif
ccc7910f 1688 case KVM_S390_VCPU_FAULT: {
527e30b4 1689 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 1690 break;
1691 }
d6712df9
CH		 1692 case KVM_ENABLE_CAP:
1693 {
1694 struct kvm_enable_cap cap;
1695 r = -EFAULT;
1696 if (copy_from_user(&cap, argp, sizeof(cap)))
1697 break;
1698 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1699 break;
1700 }
b0c632db 1701 default:
3e6afcf1 1702 r = -ENOTTY;
b0c632db 1703 }
bc923cc9 1704 return r;
b0c632db
HC		 1705}
1706
5b1c1493
CO		 1707int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
1708{
1709#ifdef CONFIG_KVM_S390_UCONTROL
1710 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
1711 && (kvm_is_ucontrol(vcpu->kvm))) {
1712 vmf->page = virt_to_page(vcpu->arch.sie_block);
1713 get_page(vmf->page);
1714 return 0;
1715 }
1716#endif
1717 return VM_FAULT_SIGBUS;
1718}
1719
5587027c
AK		 1720int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
1721 unsigned long npages)
db3fe4eb
TY		 1722{
1723 return 0;
1724}
1725
b0c632db 1726/* Section: memory related */
f7784b8e
MT		 1727int kvm_arch_prepare_memory_region(struct kvm *kvm,
1728 struct kvm_memory_slot *memslot,
7b6195a9
TY		 1729 struct kvm_userspace_memory_region *mem,
1730 enum kvm_mr_change change)
b0c632db 1731{
dd2887e7
NW		 1732 /* A few sanity checks. We can have memory slots which have to be
1733 located/ended at a segment boundary (1MB). The memory in userland is
1734 ok to be fragmented into various different vmas. It is okay to mmap()
1735 and munmap() stuff in this slot after doing this call at any time */
b0c632db 1736
598841ca 1737 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 1738 return -EINVAL;
1739
598841ca 1740 if (mem->memory_size & 0xffffful)
b0c632db
HC		 1741 return -EINVAL;
1742
f7784b8e
MT		 1743 return 0;
1744}
1745
1746void kvm_arch_commit_memory_region(struct kvm *kvm,
1747 struct kvm_userspace_memory_region *mem,
8482644a
TY		 1748 const struct kvm_memory_slot *old,
1749 enum kvm_mr_change change)
f7784b8e 1750{
f7850c92 1751 int rc;
f7784b8e 1752
2cef4deb
CB		 1753 /* If the basics of the memslot do not change, we do not want
1754 * to update the gmap. Every update causes several unnecessary
1755 * segment translation exceptions. This is usually handled just
1756 * fine by the normal fault handler + gmap, but it will also
1757 * cause faults on the prefix page of running guest CPUs.
1758 */
1759 if (old->userspace_addr == mem->userspace_addr &&
1760 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
1761 old->npages * PAGE_SIZE == mem->memory_size)
1762 return;
598841ca
CO		 1763
1764 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
1765 mem->guest_phys_addr, mem->memory_size);
1766 if (rc)
f7850c92 1767 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
598841ca 1768 return;
b0c632db
HC		 1769}
1770
b0c632db
HC		 1771static int __init kvm_s390_init(void)
1772{
ef50f7ac 1773 int ret;
0ee75bea 1774 ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
ef50f7ac
CB		 1775 if (ret)
1776 return ret;
1777
1778 /*
1779 * guests can ask for up to 255+1 double words, we need a full page
25985edc 1780 * to hold the maximum amount of facilities. On the other hand, we
ef50f7ac
CB		 1781 * only set facilities that are known to work in KVM.
1782 */
78c4b59f
MM		 1783 vfacilities = (unsigned long *) get_zeroed_page(GFP_KERNEL|GFP_DMA);
1784 if (!vfacilities) {
ef50f7ac
CB		 1785 kvm_exit();
1786 return -ENOMEM;
1787 }
78c4b59f 1788 memcpy(vfacilities, S390_lowcore.stfle_fac_list, 16);
7be81a46 1789 vfacilities[0] &= 0xff82fffbf47c2000UL;
7feb6bb8 1790 vfacilities[1] &= 0x005c000000000000UL;
ef50f7ac 1791 return 0;
b0c632db
HC		 1792}
1793
1794static void __exit kvm_s390_exit(void)
1795{
78c4b59f 1796 free_page((unsigned long) vfacilities);
b0c632db
HC		 1797 kvm_exit();
1798}
1799
1800module_init(kvm_s390_init);
1801module_exit(kvm_s390_exit);
566af940
CH		 1802
1803/*
1804 * Enable autoloading of the kvm module.
1805 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
1806 * since x86 takes a different approach.
1807 */
1808#include <linux/miscdevice.h>
1809MODULE_ALIAS_MISCDEV(KVM_MINOR);
1810MODULE_ALIAS("devname:kvm");
Ubuntu Jammy Linux kernel mirror