]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: s390: only one external call may be pending at a time
[mirror_ubuntu-hirsute-kernel.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
24#include <linux/module.h>
25#include <linux/slab.h>
ba5c1e9b 26#include <linux/timer.h>
cbb870c8 27#include <asm/asm-offsets.h>
b0c632db
HC		 28#include <asm/lowcore.h>
29#include <asm/pgtable.h>
f5daba1d 30#include <asm/nmi.h>
a0616cde 31#include <asm/switch_to.h>
78c4b59f 32#include <asm/facility.h>
1526bf9c 33#include <asm/sclp.h>
8f2abe6a 34#include "kvm-s390.h"
b0c632db
HC		 35#include "gaccess.h"
36
5786fffa
CH		 37#define CREATE_TRACE_POINTS
38#include "trace.h"
ade38c31 39#include "trace-s390.h"
5786fffa 40
b0c632db
HC		 41#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
42
43struct kvm_stats_debugfs_item debugfs_entries[] = {
44 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 45 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 46 { "exit_validity", VCPU_STAT(exit_validity) },
47 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
48 { "exit_external_request", VCPU_STAT(exit_external_request) },
49 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b
CO		 50 { "exit_instruction", VCPU_STAT(exit_instruction) },
51 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
52 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
ce2e4f0b 53 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 54 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 55 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 56 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
57 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 58 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 59 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 60 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
61 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
62 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
63 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
64 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
65 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
66 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 67 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 68 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
69 { "instruction_spx", VCPU_STAT(instruction_spx) },
70 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
71 { "instruction_stap", VCPU_STAT(instruction_stap) },
72 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 73 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 74 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
75 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 76 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 77 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
78 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 79 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
5288fbf0 80 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 81 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 82 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 83 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 84 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
85 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 86 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 87 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
88 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
5288fbf0
CB		 89 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
90 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
91 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 92 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
93 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
94 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 95 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 96 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 97 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
b0c632db
HC		 98 { NULL }
99};
100
78c4b59f 101unsigned long *vfacilities;
2c70fe44 102static struct gmap_notifier gmap_notifier;
b0c632db 103
78c4b59f 104/* test availability of vfacility */
280ef0f1 105int test_vfacility(unsigned long nr)
78c4b59f
MM		 106{
107 return __test_facility(nr, (void *) vfacilities);
108}
109
b0c632db 110/* Section: not file related */
13a34e06 111int kvm_arch_hardware_enable(void)
b0c632db
HC		 112{
113 /* every s390 is virtualization enabled ;-) */
10474ae8 114 return 0;
b0c632db
HC		 115}
116
2c70fe44
CB		 117static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
118
b0c632db
HC		 119int kvm_arch_hardware_setup(void)
120{
2c70fe44
CB		 121 gmap_notifier.notifier_call = kvm_gmap_notifier;
122 gmap_register_ipte_notifier(&gmap_notifier);
b0c632db
HC		 123 return 0;
124}
125
126void kvm_arch_hardware_unsetup(void)
127{
2c70fe44 128 gmap_unregister_ipte_notifier(&gmap_notifier);
b0c632db
HC		 129}
130
b0c632db
HC		 131int kvm_arch_init(void *opaque)
132{
84877d93
CH		 133 /* Register floating interrupt controller interface. */
134 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 135}
136
b0c632db
HC		 137/* Section: device related */
138long kvm_arch_dev_ioctl(struct file *filp,
139 unsigned int ioctl, unsigned long arg)
140{
141 if (ioctl == KVM_S390_ENABLE_SIE)
142 return s390_enable_sie();
143 return -EINVAL;
144}
145
784aa3d7 146int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 147{
d7b0b5eb
CO		 148 int r;
149
2bd0ac4e 150 switch (ext) {
d7b0b5eb 151 case KVM_CAP_S390_PSW:
b6cf8788 152 case KVM_CAP_S390_GMAP:
52e16b18 153 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 154#ifdef CONFIG_KVM_S390_UCONTROL
155 case KVM_CAP_S390_UCONTROL:
156#endif
3c038e6b 157 case KVM_CAP_ASYNC_PF:
60b413c9 158 case KVM_CAP_SYNC_REGS:
14eebd91 159 case KVM_CAP_ONE_REG:
d6712df9 160 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 161 case KVM_CAP_S390_CSS_SUPPORT:
ebc32262 162 case KVM_CAP_IRQFD:
10ccaa1e 163 case KVM_CAP_IOEVENTFD:
c05c4186 164 case KVM_CAP_DEVICE_CTRL:
d938dc55 165 case KVM_CAP_ENABLE_CAP_VM:
78599d90 166 case KVM_CAP_S390_IRQCHIP:
f2061656 167 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 168 case KVM_CAP_MP_STATE:
d7b0b5eb
CO		 169 r = 1;
170 break;
e726b1bd
CB		 171 case KVM_CAP_NR_VCPUS:
172 case KVM_CAP_MAX_VCPUS:
173 r = KVM_MAX_VCPUS;
174 break;
e1e2e605
NW		 175 case KVM_CAP_NR_MEMSLOTS:
176 r = KVM_USER_MEM_SLOTS;
177 break;
1526bf9c 178 case KVM_CAP_S390_COW:
abf09bed 179 r = MACHINE_HAS_ESOP;
1526bf9c 180 break;
2bd0ac4e 181 default:
d7b0b5eb 182 r = 0;
2bd0ac4e 183 }
d7b0b5eb 184 return r;
b0c632db
HC		 185}
186
15f36ebd
JH		 187static void kvm_s390_sync_dirty_log(struct kvm *kvm,
188 struct kvm_memory_slot *memslot)
189{
190 gfn_t cur_gfn, last_gfn;
191 unsigned long address;
192 struct gmap *gmap = kvm->arch.gmap;
193
194 down_read(&gmap->mm->mmap_sem);
195 /* Loop over all guest pages */
196 last_gfn = memslot->base_gfn + memslot->npages;
197 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
198 address = gfn_to_hva_memslot(memslot, cur_gfn);
199
200 if (gmap_test_and_clear_dirty(address, gmap))
201 mark_page_dirty(kvm, cur_gfn);
202 }
203 up_read(&gmap->mm->mmap_sem);
204}
205
b0c632db
HC		 206/* Section: vm related */
207/*
208 * Get (and clear) the dirty memory log for a memory slot.
209 */
210int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
211 struct kvm_dirty_log *log)
212{
15f36ebd
JH		 213 int r;
214 unsigned long n;
215 struct kvm_memory_slot *memslot;
216 int is_dirty = 0;
217
218 mutex_lock(&kvm->slots_lock);
219
220 r = -EINVAL;
221 if (log->slot >= KVM_USER_MEM_SLOTS)
222 goto out;
223
224 memslot = id_to_memslot(kvm->memslots, log->slot);
225 r = -ENOENT;
226 if (!memslot->dirty_bitmap)
227 goto out;
228
229 kvm_s390_sync_dirty_log(kvm, memslot);
230 r = kvm_get_dirty_log(kvm, log, &is_dirty);
231 if (r)
232 goto out;
233
234 /* Clear the dirty log */
235 if (is_dirty) {
236 n = kvm_dirty_bitmap_bytes(memslot);
237 memset(memslot->dirty_bitmap, 0, n);
238 }
239 r = 0;
240out:
241 mutex_unlock(&kvm->slots_lock);
242 return r;
b0c632db
HC		 243}
244
d938dc55
CH		 245static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
246{
247 int r;
248
249 if (cap->flags)
250 return -EINVAL;
251
252 switch (cap->cap) {
84223598
CH		 253 case KVM_CAP_S390_IRQCHIP:
254 kvm->arch.use_irqchip = 1;
255 r = 0;
256 break;
d938dc55
CH		 257 default:
258 r = -EINVAL;
259 break;
260 }
261 return r;
262}
263
8c0a7ce6
DD		 264static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
265{
266 int ret;
267
268 switch (attr->attr) {
269 case KVM_S390_VM_MEM_LIMIT_SIZE:
270 ret = 0;
271 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
272 ret = -EFAULT;
273 break;
274 default:
275 ret = -ENXIO;
276 break;
277 }
278 return ret;
279}
280
281static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 282{
283 int ret;
284 unsigned int idx;
285 switch (attr->attr) {
286 case KVM_S390_VM_MEM_ENABLE_CMMA:
287 ret = -EBUSY;
288 mutex_lock(&kvm->lock);
289 if (atomic_read(&kvm->online_vcpus) == 0) {
290 kvm->arch.use_cmma = 1;
291 ret = 0;
292 }
293 mutex_unlock(&kvm->lock);
294 break;
295 case KVM_S390_VM_MEM_CLR_CMMA:
296 mutex_lock(&kvm->lock);
297 idx = srcu_read_lock(&kvm->srcu);
a13cff31 298 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 299 srcu_read_unlock(&kvm->srcu, idx);
300 mutex_unlock(&kvm->lock);
301 ret = 0;
302 break;
8c0a7ce6
DD		 303 case KVM_S390_VM_MEM_LIMIT_SIZE: {
304 unsigned long new_limit;
305
306 if (kvm_is_ucontrol(kvm))
307 return -EINVAL;
308
309 if (get_user(new_limit, (u64 __user *)attr->addr))
310 return -EFAULT;
311
312 if (new_limit > kvm->arch.gmap->asce_end)
313 return -E2BIG;
314
315 ret = -EBUSY;
316 mutex_lock(&kvm->lock);
317 if (atomic_read(&kvm->online_vcpus) == 0) {
318 /* gmap_alloc will round the limit up */
319 struct gmap *new = gmap_alloc(current->mm, new_limit);
320
321 if (!new) {
322 ret = -ENOMEM;
323 } else {
324 gmap_free(kvm->arch.gmap);
325 new->private = kvm;
326 kvm->arch.gmap = new;
327 ret = 0;
328 }
329 }
330 mutex_unlock(&kvm->lock);
331 break;
332 }
4f718eab
DD		 333 default:
334 ret = -ENXIO;
335 break;
336 }
337 return ret;
338}
339
f2061656
DD		 340static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
341{
342 int ret;
343
344 switch (attr->group) {
4f718eab 345 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 346 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 347 break;
f2061656
DD		 348 default:
349 ret = -ENXIO;
350 break;
351 }
352
353 return ret;
354}
355
356static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
357{
8c0a7ce6
DD		 358 int ret;
359
360 switch (attr->group) {
361 case KVM_S390_VM_MEM_CTRL:
362 ret = kvm_s390_get_mem_control(kvm, attr);
363 break;
364 default:
365 ret = -ENXIO;
366 break;
367 }
368
369 return ret;
f2061656
DD		 370}
371
372static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
373{
374 int ret;
375
376 switch (attr->group) {
4f718eab
DD		 377 case KVM_S390_VM_MEM_CTRL:
378 switch (attr->attr) {
379 case KVM_S390_VM_MEM_ENABLE_CMMA:
380 case KVM_S390_VM_MEM_CLR_CMMA:
8c0a7ce6 381 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 382 ret = 0;
383 break;
384 default:
385 ret = -ENXIO;
386 break;
387 }
388 break;
f2061656
DD		 389 default:
390 ret = -ENXIO;
391 break;
392 }
393
394 return ret;
395}
396
b0c632db
HC		 397long kvm_arch_vm_ioctl(struct file *filp,
398 unsigned int ioctl, unsigned long arg)
399{
400 struct kvm *kvm = filp->private_data;
401 void __user *argp = (void __user *)arg;
f2061656 402 struct kvm_device_attr attr;
b0c632db
HC		 403 int r;
404
405 switch (ioctl) {
ba5c1e9b
CO		 406 case KVM_S390_INTERRUPT: {
407 struct kvm_s390_interrupt s390int;
408
409 r = -EFAULT;
410 if (copy_from_user(&s390int, argp, sizeof(s390int)))
411 break;
412 r = kvm_s390_inject_vm(kvm, &s390int);
413 break;
414 }
d938dc55
CH		 415 case KVM_ENABLE_CAP: {
416 struct kvm_enable_cap cap;
417 r = -EFAULT;
418 if (copy_from_user(&cap, argp, sizeof(cap)))
419 break;
420 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
421 break;
422 }
84223598
CH		 423 case KVM_CREATE_IRQCHIP: {
424 struct kvm_irq_routing_entry routing;
425
426 r = -EINVAL;
427 if (kvm->arch.use_irqchip) {
428 /* Set up dummy routing. */
429 memset(&routing, 0, sizeof(routing));
430 kvm_set_irq_routing(kvm, &routing, 0, 0);
431 r = 0;
432 }
433 break;
434 }
f2061656
DD		 435 case KVM_SET_DEVICE_ATTR: {
436 r = -EFAULT;
437 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
438 break;
439 r = kvm_s390_vm_set_attr(kvm, &attr);
440 break;
441 }
442 case KVM_GET_DEVICE_ATTR: {
443 r = -EFAULT;
444 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
445 break;
446 r = kvm_s390_vm_get_attr(kvm, &attr);
447 break;
448 }
449 case KVM_HAS_DEVICE_ATTR: {
450 r = -EFAULT;
451 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
452 break;
453 r = kvm_s390_vm_has_attr(kvm, &attr);
454 break;
455 }
b0c632db 456 default:
367e1319 457 r = -ENOTTY;
b0c632db
HC		 458 }
459
460 return r;
461}
462
5102ee87
TK		 463static int kvm_s390_crypto_init(struct kvm *kvm)
464{
465 if (!test_vfacility(76))
466 return 0;
467
468 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
469 GFP_KERNEL | GFP_DMA);
470 if (!kvm->arch.crypto.crycb)
471 return -ENOMEM;
472
473 kvm->arch.crypto.crycbd = (__u32) (unsigned long) kvm->arch.crypto.crycb |
474 CRYCB_FORMAT1;
475
476 return 0;
477}
478
e08b9637 479int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 480{
b0c632db
HC		 481 int rc;
482 char debug_name[16];
f6c137ff 483 static unsigned long sca_offset;
b0c632db 484
e08b9637
CO		 485 rc = -EINVAL;
486#ifdef CONFIG_KVM_S390_UCONTROL
487 if (type & ~KVM_VM_S390_UCONTROL)
488 goto out_err;
489 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
490 goto out_err;
491#else
492 if (type)
493 goto out_err;
494#endif
495
b0c632db
HC		 496 rc = s390_enable_sie();
497 if (rc)
d89f5eff 498 goto out_err;
b0c632db 499
b290411a
CO		 500 rc = -ENOMEM;
501
b0c632db
HC		 502 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
503 if (!kvm->arch.sca)
d89f5eff 504 goto out_err;
f6c137ff
CB		 505 spin_lock(&kvm_lock);
506 sca_offset = (sca_offset + 16) & 0x7f0;
507 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
508 spin_unlock(&kvm_lock);
b0c632db
HC		 509
510 sprintf(debug_name, "kvm-%u", current->pid);
511
512 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
513 if (!kvm->arch.dbf)
514 goto out_nodbf;
515
5102ee87
TK		 516 if (kvm_s390_crypto_init(kvm) < 0)
517 goto out_crypto;
518
ba5c1e9b
CO		 519 spin_lock_init(&kvm->arch.float_int.lock);
520 INIT_LIST_HEAD(&kvm->arch.float_int.list);
8a242234 521 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 522 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 523
b0c632db
HC		 524 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
525 VM_EVENT(kvm, 3, "%s", "vm created");
526
e08b9637
CO		 527 if (type & KVM_VM_S390_UCONTROL) {
528 kvm->arch.gmap = NULL;
529 } else {
0349985a 530 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
e08b9637
CO		 531 if (!kvm->arch.gmap)
532 goto out_nogmap;
2c70fe44 533 kvm->arch.gmap->private = kvm;
24eb3a82 534 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 535 }
fa6b7fe9
CH		 536
537 kvm->arch.css_support = 0;
84223598 538 kvm->arch.use_irqchip = 0;
fa6b7fe9 539
8ad35755
DH		 540 spin_lock_init(&kvm->arch.start_stop_lock);
541
d89f5eff 542 return 0;
598841ca 543out_nogmap:
5102ee87
TK		 544 kfree(kvm->arch.crypto.crycb);
545out_crypto:
598841ca 546 debug_unregister(kvm->arch.dbf);
b0c632db
HC		 547out_nodbf:
548 free_page((unsigned long)(kvm->arch.sca));
d89f5eff
JK		 549out_err:
550 return rc;
b0c632db
HC		 551}
552
d329c035
CB		 553void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
554{
555 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 556 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 557 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 558 kvm_clear_async_pf_completion_queue(vcpu);
58f9460b
CO		 559 if (!kvm_is_ucontrol(vcpu->kvm)) {
560 clear_bit(63 - vcpu->vcpu_id,
561 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
562 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
563 (__u64) vcpu->arch.sie_block)
564 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
565 }
abf4a71e 566 smp_mb();
27e0393f
CO		 567
568 if (kvm_is_ucontrol(vcpu->kvm))
569 gmap_free(vcpu->arch.gmap);
570
b31605c1
DD		 571 if (kvm_s390_cmma_enabled(vcpu->kvm))
572 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 573 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 574
6692cef3 575 kvm_vcpu_uninit(vcpu);
b110feaf 576 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 577}
578
579static void kvm_free_vcpus(struct kvm *kvm)
580{
581 unsigned int i;
988a2cae 582 struct kvm_vcpu *vcpu;
d329c035 583
988a2cae
GN		 584 kvm_for_each_vcpu(i, vcpu, kvm)
585 kvm_arch_vcpu_destroy(vcpu);
586
587 mutex_lock(&kvm->lock);
588 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
589 kvm->vcpus[i] = NULL;
590
591 atomic_set(&kvm->online_vcpus, 0);
592 mutex_unlock(&kvm->lock);
d329c035
CB		 593}
594
b0c632db
HC		 595void kvm_arch_destroy_vm(struct kvm *kvm)
596{
d329c035 597 kvm_free_vcpus(kvm);
b0c632db 598 free_page((unsigned long)(kvm->arch.sca));
d329c035 599 debug_unregister(kvm->arch.dbf);
5102ee87 600 kfree(kvm->arch.crypto.crycb);
27e0393f
CO		 601 if (!kvm_is_ucontrol(kvm))
602 gmap_free(kvm->arch.gmap);
841b91c5 603 kvm_s390_destroy_adapters(kvm);
67335e63 604 kvm_s390_clear_float_irqs(kvm);
b0c632db
HC		 605}
606
607/* Section: vcpu related */
dafd032a
DD		 608static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
609{
610 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
611 if (!vcpu->arch.gmap)
612 return -ENOMEM;
613 vcpu->arch.gmap->private = vcpu->kvm;
614
615 return 0;
616}
617
b0c632db
HC		 618int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
619{
3c038e6b
DD		 620 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
621 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 622 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
623 KVM_SYNC_GPRS |
9eed0735 624 KVM_SYNC_ACRS |
b028ee3e
DH		 625 KVM_SYNC_CRS |
626 KVM_SYNC_ARCH0 |
627 KVM_SYNC_PFAULT;
dafd032a
DD		 628
629 if (kvm_is_ucontrol(vcpu->kvm))
630 return __kvm_ucontrol_vcpu_init(vcpu);
631
b0c632db
HC		 632 return 0;
633}
634
b0c632db
HC		 635void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
636{
4725c860
MS		 637 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
638 save_fp_regs(vcpu->arch.host_fpregs.fprs);
b0c632db 639 save_access_regs(vcpu->arch.host_acrs);
4725c860
MS		 640 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
641 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
59674c1a 642 restore_access_regs(vcpu->run->s.regs.acrs);
480e5926 643 gmap_enable(vcpu->arch.gmap);
9e6dabef 644 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
b0c632db
HC		 645}
646
647void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
648{
9e6dabef 649 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
480e5926 650 gmap_disable(vcpu->arch.gmap);
4725c860
MS		 651 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
652 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
59674c1a 653 save_access_regs(vcpu->run->s.regs.acrs);
4725c860
MS		 654 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
655 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
b0c632db
HC		 656 restore_access_regs(vcpu->arch.host_acrs);
657}
658
659static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
660{
661 /* this equals initial cpu reset in pop, but we don't switch to ESA */
662 vcpu->arch.sie_block->gpsw.mask = 0UL;
663 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 664 kvm_s390_set_prefix(vcpu, 0);
b0c632db
HC		 665 vcpu->arch.sie_block->cputm = 0UL;
666 vcpu->arch.sie_block->ckc = 0UL;
667 vcpu->arch.sie_block->todpr = 0;
668 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
669 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
670 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
671 vcpu->arch.guest_fpregs.fpc = 0;
672 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
673 vcpu->arch.sie_block->gbea = 1;
672550fb 674 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 675 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
676 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 677 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
678 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 679 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 680}
681
31928aa5 682void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 683{
dafd032a
DD		 684 if (!kvm_is_ucontrol(vcpu->kvm))
685 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
42897d86
MT		 686}
687
5102ee87
TK		 688static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
689{
690 if (!test_vfacility(76))
691 return;
692
693 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
694}
695
b31605c1
DD		 696void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
697{
698 free_page(vcpu->arch.sie_block->cbrlo);
699 vcpu->arch.sie_block->cbrlo = 0;
700}
701
702int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
703{
704 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
705 if (!vcpu->arch.sie_block->cbrlo)
706 return -ENOMEM;
707
708 vcpu->arch.sie_block->ecb2 |= 0x80;
709 vcpu->arch.sie_block->ecb2 &= ~0x08;
710 return 0;
711}
712
b0c632db
HC		 713int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
714{
b31605c1 715 int rc = 0;
b31288fa 716
9e6dabef
CH		 717 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
718 CPUSTAT_SM |
69d0d3a3
CB		 719 CPUSTAT_STOPPED |
720 CPUSTAT_GED);
fc34531d 721 vcpu->arch.sie_block->ecb = 6;
7feb6bb8
MM		 722 if (test_vfacility(50) && test_vfacility(73))
723 vcpu->arch.sie_block->ecb |= 0x10;
724
69d0d3a3 725 vcpu->arch.sie_block->ecb2 = 8;
ea5f4969 726 vcpu->arch.sie_block->eca = 0xC1002000U;
217a4406
HC		 727 if (sclp_has_siif())
728 vcpu->arch.sie_block->eca |= 1;
ea5f4969
DH		 729 if (sclp_has_sigpif())
730 vcpu->arch.sie_block->eca |= 0x10000000U;
78c4b59f 731 vcpu->arch.sie_block->fac = (int) (long) vfacilities;
5a5e6536
MR		 732 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE |
733 ICTL_TPROT;
734
b31605c1
DD		 735 if (kvm_s390_cmma_enabled(vcpu->kvm)) {
736 rc = kvm_s390_vcpu_setup_cmma(vcpu);
737 if (rc)
738 return rc;
b31288fa 739 }
0ac96caf 740 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 741 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
453423dc 742 get_cpu_id(&vcpu->arch.cpu_id);
92e6ecf3 743 vcpu->arch.cpu_id.version = 0xff;
5102ee87
TK		 744
745 kvm_s390_vcpu_crypto_setup(vcpu);
746
b31605c1 747 return rc;
b0c632db
HC		 748}
749
750struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
751 unsigned int id)
752{
4d47555a 753 struct kvm_vcpu *vcpu;
7feb6bb8 754 struct sie_page *sie_page;
4d47555a
CO		 755 int rc = -EINVAL;
756
757 if (id >= KVM_MAX_VCPUS)
758 goto out;
759
760 rc = -ENOMEM;
b0c632db 761
b110feaf 762 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 763 if (!vcpu)
4d47555a 764 goto out;
b0c632db 765
7feb6bb8
MM		 766 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
767 if (!sie_page)
b0c632db
HC		 768 goto out_free_cpu;
769
7feb6bb8
MM		 770 vcpu->arch.sie_block = &sie_page->sie_block;
771 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
772
b0c632db 773 vcpu->arch.sie_block->icpua = id;
58f9460b
CO		 774 if (!kvm_is_ucontrol(kvm)) {
775 if (!kvm->arch.sca) {
776 WARN_ON_ONCE(1);
777 goto out_free_cpu;
778 }
779 if (!kvm->arch.sca->cpu[id].sda)
780 kvm->arch.sca->cpu[id].sda =
781 (__u64) vcpu->arch.sie_block;
782 vcpu->arch.sie_block->scaoh =
783 (__u32)(((__u64)kvm->arch.sca) >> 32);
784 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
785 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
786 }
b0c632db 787
ba5c1e9b 788 spin_lock_init(&vcpu->arch.local_int.lock);
ba5c1e9b 789 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 790 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 791 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
ba5c1e9b 792
b0c632db
HC		 793 rc = kvm_vcpu_init(vcpu, kvm, id);
794 if (rc)
7b06bf2f 795 goto out_free_sie_block;
b0c632db
HC		 796 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
797 vcpu->arch.sie_block);
ade38c31 798 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 799
b0c632db 800 return vcpu;
7b06bf2f
WY		 801out_free_sie_block:
802 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 803out_free_cpu:
b110feaf 804 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 805out:
b0c632db
HC		 806 return ERR_PTR(rc);
807}
808
b0c632db
HC		 809int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
810{
9a022067 811 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 812}
813
49b99e1e
CB		 814void s390_vcpu_block(struct kvm_vcpu *vcpu)
815{
816 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
817}
818
819void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
820{
821 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
822}
823
824/*
825 * Kick a guest cpu out of SIE and wait until SIE is not running.
826 * If the CPU is not running (e.g. waiting as idle) the function will
827 * return immediately. */
828void exit_sie(struct kvm_vcpu *vcpu)
829{
830 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
831 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
832 cpu_relax();
833}
834
835/* Kick a guest cpu out of SIE and prevent SIE-reentry */
836void exit_sie_sync(struct kvm_vcpu *vcpu)
837{
838 s390_vcpu_block(vcpu);
839 exit_sie(vcpu);
840}
841
2c70fe44
CB		 842static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
843{
844 int i;
845 struct kvm *kvm = gmap->private;
846 struct kvm_vcpu *vcpu;
847
848 kvm_for_each_vcpu(i, vcpu, kvm) {
849 /* match against both prefix pages */
fda902cb 850 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
2c70fe44
CB		 851 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
852 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
853 exit_sie_sync(vcpu);
854 }
855 }
856}
857
b6d33834
CD		 858int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
859{
860 /* kvm common code refers to this, but never calls it */
861 BUG();
862 return 0;
863}
864
14eebd91
CO		 865static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
866 struct kvm_one_reg *reg)
867{
868 int r = -EINVAL;
869
870 switch (reg->id) {
29b7c71b
CO		 871 case KVM_REG_S390_TODPR:
872 r = put_user(vcpu->arch.sie_block->todpr,
873 (u32 __user *)reg->addr);
874 break;
875 case KVM_REG_S390_EPOCHDIFF:
876 r = put_user(vcpu->arch.sie_block->epoch,
877 (u64 __user *)reg->addr);
878 break;
46a6dd1c
J		 879 case KVM_REG_S390_CPU_TIMER:
880 r = put_user(vcpu->arch.sie_block->cputm,
881 (u64 __user *)reg->addr);
882 break;
883 case KVM_REG_S390_CLOCK_COMP:
884 r = put_user(vcpu->arch.sie_block->ckc,
885 (u64 __user *)reg->addr);
886 break;
536336c2
DD		 887 case KVM_REG_S390_PFTOKEN:
888 r = put_user(vcpu->arch.pfault_token,
889 (u64 __user *)reg->addr);
890 break;
891 case KVM_REG_S390_PFCOMPARE:
892 r = put_user(vcpu->arch.pfault_compare,
893 (u64 __user *)reg->addr);
894 break;
895 case KVM_REG_S390_PFSELECT:
896 r = put_user(vcpu->arch.pfault_select,
897 (u64 __user *)reg->addr);
898 break;
672550fb
CB		 899 case KVM_REG_S390_PP:
900 r = put_user(vcpu->arch.sie_block->pp,
901 (u64 __user *)reg->addr);
902 break;
afa45ff5
CB		 903 case KVM_REG_S390_GBEA:
904 r = put_user(vcpu->arch.sie_block->gbea,
905 (u64 __user *)reg->addr);
906 break;
14eebd91
CO		 907 default:
908 break;
909 }
910
911 return r;
912}
913
914static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
915 struct kvm_one_reg *reg)
916{
917 int r = -EINVAL;
918
919 switch (reg->id) {
29b7c71b
CO		 920 case KVM_REG_S390_TODPR:
921 r = get_user(vcpu->arch.sie_block->todpr,
922 (u32 __user *)reg->addr);
923 break;
924 case KVM_REG_S390_EPOCHDIFF:
925 r = get_user(vcpu->arch.sie_block->epoch,
926 (u64 __user *)reg->addr);
927 break;
46a6dd1c
J		 928 case KVM_REG_S390_CPU_TIMER:
929 r = get_user(vcpu->arch.sie_block->cputm,
930 (u64 __user *)reg->addr);
931 break;
932 case KVM_REG_S390_CLOCK_COMP:
933 r = get_user(vcpu->arch.sie_block->ckc,
934 (u64 __user *)reg->addr);
935 break;
536336c2
DD		 936 case KVM_REG_S390_PFTOKEN:
937 r = get_user(vcpu->arch.pfault_token,
938 (u64 __user *)reg->addr);
939 break;
940 case KVM_REG_S390_PFCOMPARE:
941 r = get_user(vcpu->arch.pfault_compare,
942 (u64 __user *)reg->addr);
943 break;
944 case KVM_REG_S390_PFSELECT:
945 r = get_user(vcpu->arch.pfault_select,
946 (u64 __user *)reg->addr);
947 break;
672550fb
CB		 948 case KVM_REG_S390_PP:
949 r = get_user(vcpu->arch.sie_block->pp,
950 (u64 __user *)reg->addr);
951 break;
afa45ff5
CB		 952 case KVM_REG_S390_GBEA:
953 r = get_user(vcpu->arch.sie_block->gbea,
954 (u64 __user *)reg->addr);
955 break;
14eebd91
CO		 956 default:
957 break;
958 }
959
960 return r;
961}
b6d33834 962
b0c632db
HC		 963static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
964{
b0c632db 965 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 966 return 0;
967}
968
969int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
970{
5a32c1af 971 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 972 return 0;
973}
974
975int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
976{
5a32c1af 977 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 978 return 0;
979}
980
981int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
982 struct kvm_sregs *sregs)
983{
59674c1a 984 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 985 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
59674c1a 986 restore_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 987 return 0;
988}
989
990int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
991 struct kvm_sregs *sregs)
992{
59674c1a 993 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 994 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 995 return 0;
996}
997
998int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
999{
4725c860
MS		 1000 if (test_fp_ctl(fpu->fpc))
1001 return -EINVAL;
b0c632db 1002 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
4725c860
MS		 1003 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1004 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1005 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
b0c632db
HC		 1006 return 0;
1007}
1008
1009int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1010{
b0c632db
HC		 1011 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1012 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
b0c632db
HC		 1013 return 0;
1014}
1015
1016static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1017{
1018 int rc = 0;
1019
7a42fdc2 1020 if (!is_vcpu_stopped(vcpu))
b0c632db 1021 rc = -EBUSY;
d7b0b5eb
CO		 1022 else {
1023 vcpu->run->psw_mask = psw.mask;
1024 vcpu->run->psw_addr = psw.addr;
1025 }
b0c632db
HC		 1026 return rc;
1027}
1028
1029int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1030 struct kvm_translation *tr)
1031{
1032 return -EINVAL; /* not implemented yet */
1033}
1034
27291e21
DH		 1035#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1036 KVM_GUESTDBG_USE_HW_BP | \
1037 KVM_GUESTDBG_ENABLE)
1038
d0bfb940
JK		 1039int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1040 struct kvm_guest_debug *dbg)
b0c632db 1041{
27291e21
DH		 1042 int rc = 0;
1043
1044 vcpu->guest_debug = 0;
1045 kvm_s390_clear_bp_data(vcpu);
1046
2de3bfc2 1047 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21
DH		 1048 return -EINVAL;
1049
1050 if (dbg->control & KVM_GUESTDBG_ENABLE) {
1051 vcpu->guest_debug = dbg->control;
1052 /* enforce guest PER */
1053 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1054
1055 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1056 rc = kvm_s390_import_bp_data(vcpu, dbg);
1057 } else {
1058 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1059 vcpu->arch.guestdbg.last_bp = 0;
1060 }
1061
1062 if (rc) {
1063 vcpu->guest_debug = 0;
1064 kvm_s390_clear_bp_data(vcpu);
1065 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1066 }
1067
1068 return rc;
b0c632db
HC		 1069}
1070
62d9f0db
MT		 1071int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1072 struct kvm_mp_state *mp_state)
1073{
6352e4d2
DH		 1074 /* CHECK_STOP and LOAD are not supported yet */
1075 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1076 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 1077}
1078
1079int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1080 struct kvm_mp_state *mp_state)
1081{
6352e4d2
DH		 1082 int rc = 0;
1083
1084 /* user space knows about this interface - let it control the state */
1085 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1086
1087 switch (mp_state->mp_state) {
1088 case KVM_MP_STATE_STOPPED:
1089 kvm_s390_vcpu_stop(vcpu);
1090 break;
1091 case KVM_MP_STATE_OPERATING:
1092 kvm_s390_vcpu_start(vcpu);
1093 break;
1094 case KVM_MP_STATE_LOAD:
1095 case KVM_MP_STATE_CHECK_STOP:
1096 /* fall through - CHECK_STOP and LOAD are not supported yet */
1097 default:
1098 rc = -ENXIO;
1099 }
1100
1101 return rc;
62d9f0db
MT		 1102}
1103
b31605c1
DD		 1104bool kvm_s390_cmma_enabled(struct kvm *kvm)
1105{
1106 if (!MACHINE_IS_LPAR)
1107 return false;
1108 /* only enable for z10 and later */
1109 if (!MACHINE_HAS_EDAT1)
1110 return false;
1111 if (!kvm->arch.use_cmma)
1112 return false;
1113 return true;
1114}
1115
8ad35755
DH		 1116static bool ibs_enabled(struct kvm_vcpu *vcpu)
1117{
1118 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1119}
1120
2c70fe44
CB		 1121static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1122{
8ad35755
DH		 1123retry:
1124 s390_vcpu_unblock(vcpu);
2c70fe44
CB		 1125 /*
1126 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1127 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1128 * This ensures that the ipte instruction for this request has
1129 * already finished. We might race against a second unmapper that
1130 * wants to set the blocking bit. Lets just retry the request loop.
1131 */
8ad35755 1132 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44
CB		 1133 int rc;
1134 rc = gmap_ipte_notify(vcpu->arch.gmap,
fda902cb 1135 kvm_s390_get_prefix(vcpu),
2c70fe44
CB		 1136 PAGE_SIZE * 2);
1137 if (rc)
1138 return rc;
8ad35755 1139 goto retry;
2c70fe44 1140 }
8ad35755 1141
d3d692c8
DH		 1142 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1143 vcpu->arch.sie_block->ihcpu = 0xffff;
1144 goto retry;
1145 }
1146
8ad35755
DH		 1147 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1148 if (!ibs_enabled(vcpu)) {
1149 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1150 atomic_set_mask(CPUSTAT_IBS,
1151 &vcpu->arch.sie_block->cpuflags);
1152 }
1153 goto retry;
2c70fe44 1154 }
8ad35755
DH		 1155
1156 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1157 if (ibs_enabled(vcpu)) {
1158 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1159 atomic_clear_mask(CPUSTAT_IBS,
1160 &vcpu->arch.sie_block->cpuflags);
1161 }
1162 goto retry;
1163 }
1164
0759d068
DH		 1165 /* nothing to do, just clear the request */
1166 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1167
2c70fe44
CB		 1168 return 0;
1169}
1170
fa576c58
TH		 1171/**
1172 * kvm_arch_fault_in_page - fault-in guest page if necessary
1173 * @vcpu: The corresponding virtual cpu
1174 * @gpa: Guest physical address
1175 * @writable: Whether the page should be writable or not
1176 *
1177 * Make sure that a guest page has been faulted-in on the host.
1178 *
1179 * Return: Zero on success, negative error code otherwise.
1180 */
1181long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 1182{
527e30b4
MS		 1183 return gmap_fault(vcpu->arch.gmap, gpa,
1184 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 1185}
1186
3c038e6b
DD		 1187static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1188 unsigned long token)
1189{
1190 struct kvm_s390_interrupt inti;
383d0b05 1191 struct kvm_s390_irq irq;
3c038e6b
DD		 1192
1193 if (start_token) {
383d0b05
JF		 1194 irq.u.ext.ext_params2 = token;
1195 irq.type = KVM_S390_INT_PFAULT_INIT;
1196 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 1197 } else {
1198 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 1199 inti.parm64 = token;
3c038e6b
DD		 1200 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1201 }
1202}
1203
1204void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1205 struct kvm_async_pf *work)
1206{
1207 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1208 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1209}
1210
1211void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1212 struct kvm_async_pf *work)
1213{
1214 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1215 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1216}
1217
1218void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1219 struct kvm_async_pf *work)
1220{
1221 /* s390 will always inject the page directly */
1222}
1223
1224bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1225{
1226 /*
1227 * s390 will always inject the page directly,
1228 * but we still want check_async_completion to cleanup
1229 */
1230 return true;
1231}
1232
1233static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1234{
1235 hva_t hva;
1236 struct kvm_arch_async_pf arch;
1237 int rc;
1238
1239 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1240 return 0;
1241 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1242 vcpu->arch.pfault_compare)
1243 return 0;
1244 if (psw_extint_disabled(vcpu))
1245 return 0;
9a022067 1246 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 1247 return 0;
1248 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1249 return 0;
1250 if (!vcpu->arch.gmap->pfault_enabled)
1251 return 0;
1252
81480cc1
HC		 1253 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1254 hva += current->thread.gmap_addr & ~PAGE_MASK;
1255 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 1256 return 0;
1257
1258 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1259 return rc;
1260}
1261
3fb4c40f 1262static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 1263{
3fb4c40f 1264 int rc, cpuflags;
e168bf8d 1265
3c038e6b
DD		 1266 /*
1267 * On s390 notifications for arriving pages will be delivered directly
1268 * to the guest but the house keeping for completed pfaults is
1269 * handled outside the worker.
1270 */
1271 kvm_check_async_pf_completion(vcpu);
1272
5a32c1af 1273 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
b0c632db
HC		 1274
1275 if (need_resched())
1276 schedule();
1277
d3a73acb 1278 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 1279 s390_handle_mcck();
1280
79395031
JF		 1281 if (!kvm_is_ucontrol(vcpu->kvm)) {
1282 rc = kvm_s390_deliver_pending_interrupts(vcpu);
1283 if (rc)
1284 return rc;
1285 }
0ff31867 1286
2c70fe44
CB		 1287 rc = kvm_s390_handle_requests(vcpu);
1288 if (rc)
1289 return rc;
1290
27291e21
DH		 1291 if (guestdbg_enabled(vcpu)) {
1292 kvm_s390_backup_guest_per_regs(vcpu);
1293 kvm_s390_patch_guest_per_regs(vcpu);
1294 }
1295
b0c632db 1296 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 1297 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1298 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1299 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 1300
3fb4c40f
TH		 1301 return 0;
1302}
1303
1304static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1305{
24eb3a82 1306 int rc = -1;
2b29a9fd
DD		 1307
1308 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1309 vcpu->arch.sie_block->icptcode);
1310 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1311
27291e21
DH		 1312 if (guestdbg_enabled(vcpu))
1313 kvm_s390_restore_guest_per_regs(vcpu);
1314
3fb4c40f 1315 if (exit_reason >= 0) {
7c470539 1316 rc = 0;
210b1607
TH		 1317 } else if (kvm_is_ucontrol(vcpu->kvm)) {
1318 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1319 vcpu->run->s390_ucontrol.trans_exc_code =
1320 current->thread.gmap_addr;
1321 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1322 rc = -EREMOTE;
24eb3a82
DD		 1323
1324 } else if (current->thread.gmap_pfault) {
3c038e6b 1325 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 1326 current->thread.gmap_pfault = 0;
fa576c58 1327 if (kvm_arch_setup_async_pf(vcpu)) {
24eb3a82 1328 rc = 0;
fa576c58
TH		 1329 } else {
1330 gpa_t gpa = current->thread.gmap_addr;
1331 rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1332 }
24eb3a82
DD		 1333 }
1334
1335 if (rc == -1) {
699bde3b
CB		 1336 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1337 trace_kvm_s390_sie_fault(vcpu);
1338 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1f0d0f09 1339 }
b0c632db 1340
5a32c1af 1341 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
3fb4c40f 1342
a76ccff6
TH		 1343 if (rc == 0) {
1344 if (kvm_is_ucontrol(vcpu->kvm))
2955c83f
CB		 1345 /* Don't exit for host interrupts. */
1346 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
a76ccff6
TH		 1347 else
1348 rc = kvm_handle_sie_intercept(vcpu);
1349 }
1350
3fb4c40f
TH		 1351 return rc;
1352}
1353
1354static int __vcpu_run(struct kvm_vcpu *vcpu)
1355{
1356 int rc, exit_reason;
1357
800c1065
TH		 1358 /*
1359 * We try to hold kvm->srcu during most of vcpu_run (except when run-
1360 * ning the guest), so that memslots (and other stuff) are protected
1361 */
1362 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1363
a76ccff6
TH		 1364 do {
1365 rc = vcpu_pre_run(vcpu);
1366 if (rc)
1367 break;
3fb4c40f 1368
800c1065 1369 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 1370 /*
1371 * As PF_VCPU will be used in fault handler, between
1372 * guest_enter and guest_exit should be no uaccess.
1373 */
1374 preempt_disable();
1375 kvm_guest_enter();
1376 preempt_enable();
1377 exit_reason = sie64a(vcpu->arch.sie_block,
1378 vcpu->run->s.regs.gprs);
1379 kvm_guest_exit();
800c1065 1380 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 1381
1382 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 1383 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 1384
800c1065 1385 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 1386 return rc;
b0c632db
HC		 1387}
1388
b028ee3e
DH		 1389static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1390{
1391 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1392 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
1393 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
1394 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1395 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1396 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 1397 /* some control register changes require a tlb flush */
1398 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 1399 }
1400 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
1401 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
1402 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
1403 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
1404 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
1405 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
1406 }
1407 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
1408 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
1409 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
1410 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
1411 }
1412 kvm_run->kvm_dirty_regs = 0;
1413}
1414
1415static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1416{
1417 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
1418 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
1419 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
1420 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
1421 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
1422 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
1423 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
1424 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
1425 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
1426 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
1427 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
1428 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
1429}
1430
b0c632db
HC		 1431int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1432{
8f2abe6a 1433 int rc;
b0c632db
HC		 1434 sigset_t sigsaved;
1435
27291e21
DH		 1436 if (guestdbg_exit_pending(vcpu)) {
1437 kvm_s390_prepare_debug_exit(vcpu);
1438 return 0;
1439 }
1440
b0c632db
HC		 1441 if (vcpu->sigset_active)
1442 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1443
6352e4d2
DH		 1444 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
1445 kvm_s390_vcpu_start(vcpu);
1446 } else if (is_vcpu_stopped(vcpu)) {
1447 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
1448 vcpu->vcpu_id);
1449 return -EINVAL;
1450 }
b0c632db 1451
b028ee3e 1452 sync_regs(vcpu, kvm_run);
d7b0b5eb 1453
dab4079d 1454 might_fault();
a76ccff6 1455 rc = __vcpu_run(vcpu);
9ace903d 1456
b1d16c49
CE		 1457 if (signal_pending(current) && !rc) {
1458 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 1459 rc = -EINTR;
b1d16c49 1460 }
8f2abe6a 1461
27291e21
DH		 1462 if (guestdbg_exit_pending(vcpu) && !rc) {
1463 kvm_s390_prepare_debug_exit(vcpu);
1464 rc = 0;
1465 }
1466
b8e660b8 1467 if (rc == -EOPNOTSUPP) {
8f2abe6a
CB		 1468 /* intercept cannot be handled in-kernel, prepare kvm-run */
1469 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
1470 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
8f2abe6a
CB		 1471 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
1472 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
1473 rc = 0;
1474 }
1475
1476 if (rc == -EREMOTE) {
1477 /* intercept was handled, but userspace support is needed
1478 * kvm_run has been prepared by the handler */
1479 rc = 0;
1480 }
b0c632db 1481
b028ee3e 1482 store_regs(vcpu, kvm_run);
d7b0b5eb 1483
b0c632db
HC		 1484 if (vcpu->sigset_active)
1485 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1486
b0c632db 1487 vcpu->stat.exit_userspace++;
7e8e6ab4 1488 return rc;
b0c632db
HC		 1489}
1490
b0c632db
HC		 1491/*
1492 * store status at address
1493 * we use have two special cases:
1494 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1495 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1496 */
d0bce605 1497int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 1498{
092670cd 1499 unsigned char archmode = 1;
fda902cb 1500 unsigned int px;
178bd789 1501 u64 clkcomp;
d0bce605 1502 int rc;
b0c632db 1503
d0bce605
HC		 1504 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
1505 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 1506 return -EFAULT;
d0bce605
HC		 1507 gpa = SAVE_AREA_BASE;
1508 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
1509 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 1510 return -EFAULT;
d0bce605
HC		 1511 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
1512 }
1513 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
1514 vcpu->arch.guest_fpregs.fprs, 128);
1515 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
1516 vcpu->run->s.regs.gprs, 128);
1517 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
1518 &vcpu->arch.sie_block->gpsw, 16);
fda902cb 1519 px = kvm_s390_get_prefix(vcpu);
d0bce605 1520 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
fda902cb 1521 &px, 4);
d0bce605
HC		 1522 rc |= write_guest_abs(vcpu,
1523 gpa + offsetof(struct save_area, fp_ctrl_reg),
1524 &vcpu->arch.guest_fpregs.fpc, 4);
1525 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
1526 &vcpu->arch.sie_block->todpr, 4);
1527 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
1528 &vcpu->arch.sie_block->cputm, 8);
178bd789 1529 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d0bce605
HC		 1530 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
1531 &clkcomp, 8);
1532 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
1533 &vcpu->run->s.regs.acrs, 64);
1534 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
1535 &vcpu->arch.sie_block->gcr, 128);
1536 return rc ? -EFAULT : 0;
b0c632db
HC		 1537}
1538
e879892c
TH		 1539int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1540{
1541 /*
1542 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1543 * copying in vcpu load/put. Lets update our copies before we save
1544 * it into the save area
1545 */
1546 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1547 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1548 save_access_regs(vcpu->run->s.regs.acrs);
1549
1550 return kvm_s390_store_status_unloaded(vcpu, addr);
1551}
1552
8ad35755
DH		 1553static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1554{
1555 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
1556 kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu);
1557 exit_sie_sync(vcpu);
1558}
1559
1560static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
1561{
1562 unsigned int i;
1563 struct kvm_vcpu *vcpu;
1564
1565 kvm_for_each_vcpu(i, vcpu, kvm) {
1566 __disable_ibs_on_vcpu(vcpu);
1567 }
1568}
1569
1570static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1571{
1572 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
1573 kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu);
1574 exit_sie_sync(vcpu);
1575}
1576
6852d7b6
DH		 1577void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
1578{
8ad35755
DH		 1579 int i, online_vcpus, started_vcpus = 0;
1580
1581 if (!is_vcpu_stopped(vcpu))
1582 return;
1583
6852d7b6 1584 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 1585 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 1586 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 1587 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1588
1589 for (i = 0; i < online_vcpus; i++) {
1590 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
1591 started_vcpus++;
1592 }
1593
1594 if (started_vcpus == 0) {
1595 /* we're the only active VCPU -> speed it up */
1596 __enable_ibs_on_vcpu(vcpu);
1597 } else if (started_vcpus == 1) {
1598 /*
1599 * As we are starting a second VCPU, we have to disable
1600 * the IBS facility on all VCPUs to remove potentially
1601 * oustanding ENABLE requests.
1602 */
1603 __disable_ibs_on_all_vcpus(vcpu->kvm);
1604 }
1605
6852d7b6 1606 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 1607 /*
1608 * Another VCPU might have used IBS while we were offline.
1609 * Let's play safe and flush the VCPU at startup.
1610 */
d3d692c8 1611 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 1612 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 1613 return;
6852d7b6
DH		 1614}
1615
1616void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
1617{
8ad35755
DH		 1618 int i, online_vcpus, started_vcpus = 0;
1619 struct kvm_vcpu *started_vcpu = NULL;
1620
1621 if (is_vcpu_stopped(vcpu))
1622 return;
1623
6852d7b6 1624 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 1625 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 1626 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 1627 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1628
32f5ff63 1629 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 1630 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 1631
6cddd432 1632 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 1633 __disable_ibs_on_vcpu(vcpu);
1634
1635 for (i = 0; i < online_vcpus; i++) {
1636 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
1637 started_vcpus++;
1638 started_vcpu = vcpu->kvm->vcpus[i];
1639 }
1640 }
1641
1642 if (started_vcpus == 1) {
1643 /*
1644 * As we only have one VCPU left, we want to enable the
1645 * IBS facility for that VCPU to speed it up.
1646 */
1647 __enable_ibs_on_vcpu(started_vcpu);
1648 }
1649
433b9ee4 1650 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 1651 return;
6852d7b6
DH		 1652}
1653
d6712df9
CH		 1654static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1655 struct kvm_enable_cap *cap)
1656{
1657 int r;
1658
1659 if (cap->flags)
1660 return -EINVAL;
1661
1662 switch (cap->cap) {
fa6b7fe9
CH		 1663 case KVM_CAP_S390_CSS_SUPPORT:
1664 if (!vcpu->kvm->arch.css_support) {
1665 vcpu->kvm->arch.css_support = 1;
1666 trace_kvm_s390_enable_css(vcpu->kvm);
1667 }
1668 r = 0;
1669 break;
d6712df9
CH		 1670 default:
1671 r = -EINVAL;
1672 break;
1673 }
1674 return r;
1675}
1676
b0c632db
HC		 1677long kvm_arch_vcpu_ioctl(struct file *filp,
1678 unsigned int ioctl, unsigned long arg)
1679{
1680 struct kvm_vcpu *vcpu = filp->private_data;
1681 void __user *argp = (void __user *)arg;
800c1065 1682 int idx;
bc923cc9 1683 long r;
b0c632db 1684
93736624
AK		 1685 switch (ioctl) {
1686 case KVM_S390_INTERRUPT: {
ba5c1e9b 1687 struct kvm_s390_interrupt s390int;
383d0b05 1688 struct kvm_s390_irq s390irq;
ba5c1e9b 1689
93736624 1690 r = -EFAULT;
ba5c1e9b 1691 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624 1692 break;
383d0b05
JF		 1693 if (s390int_to_s390irq(&s390int, &s390irq))
1694 return -EINVAL;
1695 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
93736624 1696 break;
ba5c1e9b 1697 }
b0c632db 1698 case KVM_S390_STORE_STATUS:
800c1065 1699 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 1700 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 1701 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 1702 break;
b0c632db
HC		 1703 case KVM_S390_SET_INITIAL_PSW: {
1704 psw_t psw;
1705
bc923cc9 1706 r = -EFAULT;
b0c632db 1707 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 1708 break;
1709 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
1710 break;
b0c632db
HC		 1711 }
1712 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 1713 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
1714 break;
14eebd91
CO		 1715 case KVM_SET_ONE_REG:
1716 case KVM_GET_ONE_REG: {
1717 struct kvm_one_reg reg;
1718 r = -EFAULT;
1719 if (copy_from_user(&reg, argp, sizeof(reg)))
1720 break;
1721 if (ioctl == KVM_SET_ONE_REG)
1722 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
1723 else
1724 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
1725 break;
1726 }
27e0393f
CO		 1727#ifdef CONFIG_KVM_S390_UCONTROL
1728 case KVM_S390_UCAS_MAP: {
1729 struct kvm_s390_ucas_mapping ucasmap;
1730
1731 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1732 r = -EFAULT;
1733 break;
1734 }
1735
1736 if (!kvm_is_ucontrol(vcpu->kvm)) {
1737 r = -EINVAL;
1738 break;
1739 }
1740
1741 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
1742 ucasmap.vcpu_addr, ucasmap.length);
1743 break;
1744 }
1745 case KVM_S390_UCAS_UNMAP: {
1746 struct kvm_s390_ucas_mapping ucasmap;
1747
1748 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1749 r = -EFAULT;
1750 break;
1751 }
1752
1753 if (!kvm_is_ucontrol(vcpu->kvm)) {
1754 r = -EINVAL;
1755 break;
1756 }
1757
1758 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
1759 ucasmap.length);
1760 break;
1761 }
1762#endif
ccc7910f 1763 case KVM_S390_VCPU_FAULT: {
527e30b4 1764 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 1765 break;
1766 }
d6712df9
CH		 1767 case KVM_ENABLE_CAP:
1768 {
1769 struct kvm_enable_cap cap;
1770 r = -EFAULT;
1771 if (copy_from_user(&cap, argp, sizeof(cap)))
1772 break;
1773 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1774 break;
1775 }
b0c632db 1776 default:
3e6afcf1 1777 r = -ENOTTY;
b0c632db 1778 }
bc923cc9 1779 return r;
b0c632db
HC		 1780}
1781
5b1c1493
CO		 1782int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
1783{
1784#ifdef CONFIG_KVM_S390_UCONTROL
1785 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
1786 && (kvm_is_ucontrol(vcpu->kvm))) {
1787 vmf->page = virt_to_page(vcpu->arch.sie_block);
1788 get_page(vmf->page);
1789 return 0;
1790 }
1791#endif
1792 return VM_FAULT_SIGBUS;
1793}
1794
5587027c
AK		 1795int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
1796 unsigned long npages)
db3fe4eb
TY		 1797{
1798 return 0;
1799}
1800
b0c632db 1801/* Section: memory related */
f7784b8e
MT		 1802int kvm_arch_prepare_memory_region(struct kvm *kvm,
1803 struct kvm_memory_slot *memslot,
7b6195a9
TY		 1804 struct kvm_userspace_memory_region *mem,
1805 enum kvm_mr_change change)
b0c632db 1806{
dd2887e7
NW		 1807 /* A few sanity checks. We can have memory slots which have to be
1808 located/ended at a segment boundary (1MB). The memory in userland is
1809 ok to be fragmented into various different vmas. It is okay to mmap()
1810 and munmap() stuff in this slot after doing this call at any time */
b0c632db 1811
598841ca 1812 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 1813 return -EINVAL;
1814
598841ca 1815 if (mem->memory_size & 0xffffful)
b0c632db
HC		 1816 return -EINVAL;
1817
f7784b8e
MT		 1818 return 0;
1819}
1820
1821void kvm_arch_commit_memory_region(struct kvm *kvm,
1822 struct kvm_userspace_memory_region *mem,
8482644a
TY		 1823 const struct kvm_memory_slot *old,
1824 enum kvm_mr_change change)
f7784b8e 1825{
f7850c92 1826 int rc;
f7784b8e 1827
2cef4deb
CB		 1828 /* If the basics of the memslot do not change, we do not want
1829 * to update the gmap. Every update causes several unnecessary
1830 * segment translation exceptions. This is usually handled just
1831 * fine by the normal fault handler + gmap, but it will also
1832 * cause faults on the prefix page of running guest CPUs.
1833 */
1834 if (old->userspace_addr == mem->userspace_addr &&
1835 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
1836 old->npages * PAGE_SIZE == mem->memory_size)
1837 return;
598841ca
CO		 1838
1839 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
1840 mem->guest_phys_addr, mem->memory_size);
1841 if (rc)
f7850c92 1842 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
598841ca 1843 return;
b0c632db
HC		 1844}
1845
b0c632db
HC		 1846static int __init kvm_s390_init(void)
1847{
ef50f7ac 1848 int ret;
0ee75bea 1849 ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
ef50f7ac
CB		 1850 if (ret)
1851 return ret;
1852
1853 /*
1854 * guests can ask for up to 255+1 double words, we need a full page
25985edc 1855 * to hold the maximum amount of facilities. On the other hand, we
ef50f7ac
CB		 1856 * only set facilities that are known to work in KVM.
1857 */
78c4b59f
MM		 1858 vfacilities = (unsigned long *) get_zeroed_page(GFP_KERNEL|GFP_DMA);
1859 if (!vfacilities) {
ef50f7ac
CB		 1860 kvm_exit();
1861 return -ENOMEM;
1862 }
78c4b59f 1863 memcpy(vfacilities, S390_lowcore.stfle_fac_list, 16);
7be81a46 1864 vfacilities[0] &= 0xff82fffbf47c2000UL;
7feb6bb8 1865 vfacilities[1] &= 0x005c000000000000UL;
ef50f7ac 1866 return 0;
b0c632db
HC		 1867}
1868
1869static void __exit kvm_s390_exit(void)
1870{
78c4b59f 1871 free_page((unsigned long) vfacilities);
b0c632db
HC		 1872 kvm_exit();
1873}
1874
1875module_init(kvm_s390_init);
1876module_exit(kvm_s390_exit);
566af940
CH		 1877
1878/*
1879 * Enable autoloading of the kvm module.
1880 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
1881 * since x86 takes a different approach.
1882 */
1883#include <linux/miscdevice.h>
1884MODULE_ALIAS_MISCDEV(KVM_MINOR);
1885MODULE_ALIAS("devname:kvm");
Ubuntu Hirsute Linux kernel mirror