]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: s390: Machine Check
[mirror_ubuntu-jammy-kernel.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
24#include <linux/module.h>
a374e892 25#include <linux/random.h>
b0c632db 26#include <linux/slab.h>
ba5c1e9b 27#include <linux/timer.h>
cbb870c8 28#include <asm/asm-offsets.h>
b0c632db
HC		 29#include <asm/lowcore.h>
30#include <asm/pgtable.h>
f5daba1d 31#include <asm/nmi.h>
a0616cde 32#include <asm/switch_to.h>
1526bf9c 33#include <asm/sclp.h>
8f2abe6a 34#include "kvm-s390.h"
b0c632db
HC		 35#include "gaccess.h"
36
5786fffa
CH		 37#define CREATE_TRACE_POINTS
38#include "trace.h"
ade38c31 39#include "trace-s390.h"
5786fffa 40
b0c632db
HC		 41#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
42
43struct kvm_stats_debugfs_item debugfs_entries[] = {
44 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 45 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 46 { "exit_validity", VCPU_STAT(exit_validity) },
47 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
48 { "exit_external_request", VCPU_STAT(exit_external_request) },
49 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b
CO		 50 { "exit_instruction", VCPU_STAT(exit_instruction) },
51 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
52 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
f7819512 53 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
ce2e4f0b 54 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 55 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 56 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 57 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
58 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 59 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 60 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 61 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
62 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
63 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
64 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
65 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
66 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
67 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 68 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 69 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
70 { "instruction_spx", VCPU_STAT(instruction_spx) },
71 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
72 { "instruction_stap", VCPU_STAT(instruction_stap) },
73 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 74 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 75 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
76 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 77 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 78 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
79 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 80 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
5288fbf0 81 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 82 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 83 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 84 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 85 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
86 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 87 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 88 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
89 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 90 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 91 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
92 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
93 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 94 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
95 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
96 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 97 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 98 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 99 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
b0c632db
HC		 100 { NULL }
101};
102
9d8d5786
MM		 103/* upper facilities limit for kvm */
104unsigned long kvm_s390_fac_list_mask[] = {
105 0xff82fffbf4fc2000UL,
106 0x005c000000000000UL,
107};
b0c632db 108
9d8d5786 109unsigned long kvm_s390_fac_list_mask_size(void)
78c4b59f 110{
9d8d5786
MM		 111 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
112 return ARRAY_SIZE(kvm_s390_fac_list_mask);
78c4b59f
MM		 113}
114
9d8d5786
MM		 115static struct gmap_notifier gmap_notifier;
116
b0c632db 117/* Section: not file related */
13a34e06 118int kvm_arch_hardware_enable(void)
b0c632db
HC		 119{
120 /* every s390 is virtualization enabled ;-) */
10474ae8 121 return 0;
b0c632db
HC		 122}
123
2c70fe44
CB		 124static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
125
b0c632db
HC		 126int kvm_arch_hardware_setup(void)
127{
2c70fe44
CB		 128 gmap_notifier.notifier_call = kvm_gmap_notifier;
129 gmap_register_ipte_notifier(&gmap_notifier);
b0c632db
HC		 130 return 0;
131}
132
133void kvm_arch_hardware_unsetup(void)
134{
2c70fe44 135 gmap_unregister_ipte_notifier(&gmap_notifier);
b0c632db
HC		 136}
137
b0c632db
HC		 138int kvm_arch_init(void *opaque)
139{
84877d93
CH		 140 /* Register floating interrupt controller interface. */
141 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 142}
143
b0c632db
HC		 144/* Section: device related */
145long kvm_arch_dev_ioctl(struct file *filp,
146 unsigned int ioctl, unsigned long arg)
147{
148 if (ioctl == KVM_S390_ENABLE_SIE)
149 return s390_enable_sie();
150 return -EINVAL;
151}
152
784aa3d7 153int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 154{
d7b0b5eb
CO		 155 int r;
156
2bd0ac4e 157 switch (ext) {
d7b0b5eb 158 case KVM_CAP_S390_PSW:
b6cf8788 159 case KVM_CAP_S390_GMAP:
52e16b18 160 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 161#ifdef CONFIG_KVM_S390_UCONTROL
162 case KVM_CAP_S390_UCONTROL:
163#endif
3c038e6b 164 case KVM_CAP_ASYNC_PF:
60b413c9 165 case KVM_CAP_SYNC_REGS:
14eebd91 166 case KVM_CAP_ONE_REG:
d6712df9 167 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 168 case KVM_CAP_S390_CSS_SUPPORT:
ebc32262 169 case KVM_CAP_IRQFD:
10ccaa1e 170 case KVM_CAP_IOEVENTFD:
c05c4186 171 case KVM_CAP_DEVICE_CTRL:
d938dc55 172 case KVM_CAP_ENABLE_CAP_VM:
78599d90 173 case KVM_CAP_S390_IRQCHIP:
f2061656 174 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 175 case KVM_CAP_MP_STATE:
2444b352 176 case KVM_CAP_S390_USER_SIGP:
d7b0b5eb
CO		 177 r = 1;
178 break;
e726b1bd
CB		 179 case KVM_CAP_NR_VCPUS:
180 case KVM_CAP_MAX_VCPUS:
181 r = KVM_MAX_VCPUS;
182 break;
e1e2e605
NW		 183 case KVM_CAP_NR_MEMSLOTS:
184 r = KVM_USER_MEM_SLOTS;
185 break;
1526bf9c 186 case KVM_CAP_S390_COW:
abf09bed 187 r = MACHINE_HAS_ESOP;
1526bf9c 188 break;
68c55750
EF		 189 case KVM_CAP_S390_VECTOR_REGISTERS:
190 r = MACHINE_HAS_VX;
191 break;
2bd0ac4e 192 default:
d7b0b5eb 193 r = 0;
2bd0ac4e 194 }
d7b0b5eb 195 return r;
b0c632db
HC		 196}
197
15f36ebd
JH		 198static void kvm_s390_sync_dirty_log(struct kvm *kvm,
199 struct kvm_memory_slot *memslot)
200{
201 gfn_t cur_gfn, last_gfn;
202 unsigned long address;
203 struct gmap *gmap = kvm->arch.gmap;
204
205 down_read(&gmap->mm->mmap_sem);
206 /* Loop over all guest pages */
207 last_gfn = memslot->base_gfn + memslot->npages;
208 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
209 address = gfn_to_hva_memslot(memslot, cur_gfn);
210
211 if (gmap_test_and_clear_dirty(address, gmap))
212 mark_page_dirty(kvm, cur_gfn);
213 }
214 up_read(&gmap->mm->mmap_sem);
215}
216
b0c632db
HC		 217/* Section: vm related */
218/*
219 * Get (and clear) the dirty memory log for a memory slot.
220 */
221int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
222 struct kvm_dirty_log *log)
223{
15f36ebd
JH		 224 int r;
225 unsigned long n;
226 struct kvm_memory_slot *memslot;
227 int is_dirty = 0;
228
229 mutex_lock(&kvm->slots_lock);
230
231 r = -EINVAL;
232 if (log->slot >= KVM_USER_MEM_SLOTS)
233 goto out;
234
235 memslot = id_to_memslot(kvm->memslots, log->slot);
236 r = -ENOENT;
237 if (!memslot->dirty_bitmap)
238 goto out;
239
240 kvm_s390_sync_dirty_log(kvm, memslot);
241 r = kvm_get_dirty_log(kvm, log, &is_dirty);
242 if (r)
243 goto out;
244
245 /* Clear the dirty log */
246 if (is_dirty) {
247 n = kvm_dirty_bitmap_bytes(memslot);
248 memset(memslot->dirty_bitmap, 0, n);
249 }
250 r = 0;
251out:
252 mutex_unlock(&kvm->slots_lock);
253 return r;
b0c632db
HC		 254}
255
d938dc55
CH		 256static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
257{
258 int r;
259
260 if (cap->flags)
261 return -EINVAL;
262
263 switch (cap->cap) {
84223598
CH		 264 case KVM_CAP_S390_IRQCHIP:
265 kvm->arch.use_irqchip = 1;
266 r = 0;
267 break;
2444b352
DH		 268 case KVM_CAP_S390_USER_SIGP:
269 kvm->arch.user_sigp = 1;
270 r = 0;
271 break;
68c55750
EF		 272 case KVM_CAP_S390_VECTOR_REGISTERS:
273 kvm->arch.use_vectors = MACHINE_HAS_VX;
274 r = MACHINE_HAS_VX ? 0 : -EINVAL;
275 break;
d938dc55
CH		 276 default:
277 r = -EINVAL;
278 break;
279 }
280 return r;
281}
282
8c0a7ce6
DD		 283static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
284{
285 int ret;
286
287 switch (attr->attr) {
288 case KVM_S390_VM_MEM_LIMIT_SIZE:
289 ret = 0;
290 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
291 ret = -EFAULT;
292 break;
293 default:
294 ret = -ENXIO;
295 break;
296 }
297 return ret;
298}
299
300static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 301{
302 int ret;
303 unsigned int idx;
304 switch (attr->attr) {
305 case KVM_S390_VM_MEM_ENABLE_CMMA:
306 ret = -EBUSY;
307 mutex_lock(&kvm->lock);
308 if (atomic_read(&kvm->online_vcpus) == 0) {
309 kvm->arch.use_cmma = 1;
310 ret = 0;
311 }
312 mutex_unlock(&kvm->lock);
313 break;
314 case KVM_S390_VM_MEM_CLR_CMMA:
315 mutex_lock(&kvm->lock);
316 idx = srcu_read_lock(&kvm->srcu);
a13cff31 317 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 318 srcu_read_unlock(&kvm->srcu, idx);
319 mutex_unlock(&kvm->lock);
320 ret = 0;
321 break;
8c0a7ce6
DD		 322 case KVM_S390_VM_MEM_LIMIT_SIZE: {
323 unsigned long new_limit;
324
325 if (kvm_is_ucontrol(kvm))
326 return -EINVAL;
327
328 if (get_user(new_limit, (u64 __user *)attr->addr))
329 return -EFAULT;
330
331 if (new_limit > kvm->arch.gmap->asce_end)
332 return -E2BIG;
333
334 ret = -EBUSY;
335 mutex_lock(&kvm->lock);
336 if (atomic_read(&kvm->online_vcpus) == 0) {
337 /* gmap_alloc will round the limit up */
338 struct gmap *new = gmap_alloc(current->mm, new_limit);
339
340 if (!new) {
341 ret = -ENOMEM;
342 } else {
343 gmap_free(kvm->arch.gmap);
344 new->private = kvm;
345 kvm->arch.gmap = new;
346 ret = 0;
347 }
348 }
349 mutex_unlock(&kvm->lock);
350 break;
351 }
4f718eab
DD		 352 default:
353 ret = -ENXIO;
354 break;
355 }
356 return ret;
357}
358
a374e892
TK		 359static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
360
361static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
362{
363 struct kvm_vcpu *vcpu;
364 int i;
365
9d8d5786 366 if (!test_kvm_facility(kvm, 76))
a374e892
TK		 367 return -EINVAL;
368
369 mutex_lock(&kvm->lock);
370 switch (attr->attr) {
371 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
372 get_random_bytes(
373 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
374 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
375 kvm->arch.crypto.aes_kw = 1;
376 break;
377 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
378 get_random_bytes(
379 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
380 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
381 kvm->arch.crypto.dea_kw = 1;
382 break;
383 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
384 kvm->arch.crypto.aes_kw = 0;
385 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
386 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
387 break;
388 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
389 kvm->arch.crypto.dea_kw = 0;
390 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
391 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
392 break;
393 default:
394 mutex_unlock(&kvm->lock);
395 return -ENXIO;
396 }
397
398 kvm_for_each_vcpu(i, vcpu, kvm) {
399 kvm_s390_vcpu_crypto_setup(vcpu);
400 exit_sie(vcpu);
401 }
402 mutex_unlock(&kvm->lock);
403 return 0;
404}
405
72f25020
JH		 406static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
407{
408 u8 gtod_high;
409
410 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
411 sizeof(gtod_high)))
412 return -EFAULT;
413
414 if (gtod_high != 0)
415 return -EINVAL;
416
417 return 0;
418}
419
420static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
421{
422 struct kvm_vcpu *cur_vcpu;
423 unsigned int vcpu_idx;
424 u64 host_tod, gtod;
425 int r;
426
427 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
428 return -EFAULT;
429
430 r = store_tod_clock(&host_tod);
431 if (r)
432 return r;
433
434 mutex_lock(&kvm->lock);
435 kvm->arch.epoch = gtod - host_tod;
436 kvm_for_each_vcpu(vcpu_idx, cur_vcpu, kvm) {
437 cur_vcpu->arch.sie_block->epoch = kvm->arch.epoch;
438 exit_sie(cur_vcpu);
439 }
440 mutex_unlock(&kvm->lock);
441 return 0;
442}
443
444static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
445{
446 int ret;
447
448 if (attr->flags)
449 return -EINVAL;
450
451 switch (attr->attr) {
452 case KVM_S390_VM_TOD_HIGH:
453 ret = kvm_s390_set_tod_high(kvm, attr);
454 break;
455 case KVM_S390_VM_TOD_LOW:
456 ret = kvm_s390_set_tod_low(kvm, attr);
457 break;
458 default:
459 ret = -ENXIO;
460 break;
461 }
462 return ret;
463}
464
465static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
466{
467 u8 gtod_high = 0;
468
469 if (copy_to_user((void __user *)attr->addr, &gtod_high,
470 sizeof(gtod_high)))
471 return -EFAULT;
472
473 return 0;
474}
475
476static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
477{
478 u64 host_tod, gtod;
479 int r;
480
481 r = store_tod_clock(&host_tod);
482 if (r)
483 return r;
484
485 gtod = host_tod + kvm->arch.epoch;
486 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
487 return -EFAULT;
488
489 return 0;
490}
491
492static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
493{
494 int ret;
495
496 if (attr->flags)
497 return -EINVAL;
498
499 switch (attr->attr) {
500 case KVM_S390_VM_TOD_HIGH:
501 ret = kvm_s390_get_tod_high(kvm, attr);
502 break;
503 case KVM_S390_VM_TOD_LOW:
504 ret = kvm_s390_get_tod_low(kvm, attr);
505 break;
506 default:
507 ret = -ENXIO;
508 break;
509 }
510 return ret;
511}
512
658b6eda
MM		 513static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
514{
515 struct kvm_s390_vm_cpu_processor *proc;
516 int ret = 0;
517
518 mutex_lock(&kvm->lock);
519 if (atomic_read(&kvm->online_vcpus)) {
520 ret = -EBUSY;
521 goto out;
522 }
523 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
524 if (!proc) {
525 ret = -ENOMEM;
526 goto out;
527 }
528 if (!copy_from_user(proc, (void __user *)attr->addr,
529 sizeof(*proc))) {
530 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
531 sizeof(struct cpuid));
532 kvm->arch.model.ibc = proc->ibc;
981467c9 533 memcpy(kvm->arch.model.fac->list, proc->fac_list,
658b6eda
MM		 534 S390_ARCH_FAC_LIST_SIZE_BYTE);
535 } else
536 ret = -EFAULT;
537 kfree(proc);
538out:
539 mutex_unlock(&kvm->lock);
540 return ret;
541}
542
543static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
544{
545 int ret = -ENXIO;
546
547 switch (attr->attr) {
548 case KVM_S390_VM_CPU_PROCESSOR:
549 ret = kvm_s390_set_processor(kvm, attr);
550 break;
551 }
552 return ret;
553}
554
555static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
556{
557 struct kvm_s390_vm_cpu_processor *proc;
558 int ret = 0;
559
560 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
561 if (!proc) {
562 ret = -ENOMEM;
563 goto out;
564 }
565 memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
566 proc->ibc = kvm->arch.model.ibc;
981467c9 567 memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 568 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
569 ret = -EFAULT;
570 kfree(proc);
571out:
572 return ret;
573}
574
575static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
576{
577 struct kvm_s390_vm_cpu_machine *mach;
578 int ret = 0;
579
580 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
581 if (!mach) {
582 ret = -ENOMEM;
583 goto out;
584 }
585 get_cpu_id((struct cpuid *) &mach->cpuid);
586 mach->ibc = sclp_get_ibc();
981467c9
MM		 587 memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
588 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 589 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
94422ee8 590 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 591 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
592 ret = -EFAULT;
593 kfree(mach);
594out:
595 return ret;
596}
597
598static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
599{
600 int ret = -ENXIO;
601
602 switch (attr->attr) {
603 case KVM_S390_VM_CPU_PROCESSOR:
604 ret = kvm_s390_get_processor(kvm, attr);
605 break;
606 case KVM_S390_VM_CPU_MACHINE:
607 ret = kvm_s390_get_machine(kvm, attr);
608 break;
609 }
610 return ret;
611}
612
f2061656
DD		 613static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
614{
615 int ret;
616
617 switch (attr->group) {
4f718eab 618 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 619 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 620 break;
72f25020
JH		 621 case KVM_S390_VM_TOD:
622 ret = kvm_s390_set_tod(kvm, attr);
623 break;
658b6eda
MM		 624 case KVM_S390_VM_CPU_MODEL:
625 ret = kvm_s390_set_cpu_model(kvm, attr);
626 break;
a374e892
TK		 627 case KVM_S390_VM_CRYPTO:
628 ret = kvm_s390_vm_set_crypto(kvm, attr);
629 break;
f2061656
DD		 630 default:
631 ret = -ENXIO;
632 break;
633 }
634
635 return ret;
636}
637
638static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
639{
8c0a7ce6
DD		 640 int ret;
641
642 switch (attr->group) {
643 case KVM_S390_VM_MEM_CTRL:
644 ret = kvm_s390_get_mem_control(kvm, attr);
645 break;
72f25020
JH		 646 case KVM_S390_VM_TOD:
647 ret = kvm_s390_get_tod(kvm, attr);
648 break;
658b6eda
MM		 649 case KVM_S390_VM_CPU_MODEL:
650 ret = kvm_s390_get_cpu_model(kvm, attr);
651 break;
8c0a7ce6
DD		 652 default:
653 ret = -ENXIO;
654 break;
655 }
656
657 return ret;
f2061656
DD		 658}
659
660static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
661{
662 int ret;
663
664 switch (attr->group) {
4f718eab
DD		 665 case KVM_S390_VM_MEM_CTRL:
666 switch (attr->attr) {
667 case KVM_S390_VM_MEM_ENABLE_CMMA:
668 case KVM_S390_VM_MEM_CLR_CMMA:
8c0a7ce6 669 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 670 ret = 0;
671 break;
672 default:
673 ret = -ENXIO;
674 break;
675 }
676 break;
72f25020
JH		 677 case KVM_S390_VM_TOD:
678 switch (attr->attr) {
679 case KVM_S390_VM_TOD_LOW:
680 case KVM_S390_VM_TOD_HIGH:
681 ret = 0;
682 break;
683 default:
684 ret = -ENXIO;
685 break;
686 }
687 break;
658b6eda
MM		 688 case KVM_S390_VM_CPU_MODEL:
689 switch (attr->attr) {
690 case KVM_S390_VM_CPU_PROCESSOR:
691 case KVM_S390_VM_CPU_MACHINE:
692 ret = 0;
693 break;
694 default:
695 ret = -ENXIO;
696 break;
697 }
698 break;
a374e892
TK		 699 case KVM_S390_VM_CRYPTO:
700 switch (attr->attr) {
701 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
702 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
703 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
704 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
705 ret = 0;
706 break;
707 default:
708 ret = -ENXIO;
709 break;
710 }
711 break;
f2061656
DD		 712 default:
713 ret = -ENXIO;
714 break;
715 }
716
717 return ret;
718}
719
b0c632db
HC		 720long kvm_arch_vm_ioctl(struct file *filp,
721 unsigned int ioctl, unsigned long arg)
722{
723 struct kvm *kvm = filp->private_data;
724 void __user *argp = (void __user *)arg;
f2061656 725 struct kvm_device_attr attr;
b0c632db
HC		 726 int r;
727
728 switch (ioctl) {
ba5c1e9b
CO		 729 case KVM_S390_INTERRUPT: {
730 struct kvm_s390_interrupt s390int;
731
732 r = -EFAULT;
733 if (copy_from_user(&s390int, argp, sizeof(s390int)))
734 break;
735 r = kvm_s390_inject_vm(kvm, &s390int);
736 break;
737 }
d938dc55
CH		 738 case KVM_ENABLE_CAP: {
739 struct kvm_enable_cap cap;
740 r = -EFAULT;
741 if (copy_from_user(&cap, argp, sizeof(cap)))
742 break;
743 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
744 break;
745 }
84223598
CH		 746 case KVM_CREATE_IRQCHIP: {
747 struct kvm_irq_routing_entry routing;
748
749 r = -EINVAL;
750 if (kvm->arch.use_irqchip) {
751 /* Set up dummy routing. */
752 memset(&routing, 0, sizeof(routing));
753 kvm_set_irq_routing(kvm, &routing, 0, 0);
754 r = 0;
755 }
756 break;
757 }
f2061656
DD		 758 case KVM_SET_DEVICE_ATTR: {
759 r = -EFAULT;
760 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
761 break;
762 r = kvm_s390_vm_set_attr(kvm, &attr);
763 break;
764 }
765 case KVM_GET_DEVICE_ATTR: {
766 r = -EFAULT;
767 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
768 break;
769 r = kvm_s390_vm_get_attr(kvm, &attr);
770 break;
771 }
772 case KVM_HAS_DEVICE_ATTR: {
773 r = -EFAULT;
774 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
775 break;
776 r = kvm_s390_vm_has_attr(kvm, &attr);
777 break;
778 }
b0c632db 779 default:
367e1319 780 r = -ENOTTY;
b0c632db
HC		 781 }
782
783 return r;
784}
785
45c9b47c
TK		 786static int kvm_s390_query_ap_config(u8 *config)
787{
788 u32 fcn_code = 0x04000000UL;
86044c8c 789 u32 cc = 0;
45c9b47c 790
86044c8c 791 memset(config, 0, 128);
45c9b47c
TK		 792 asm volatile(
793 "lgr 0,%1\n"
794 "lgr 2,%2\n"
795 ".long 0xb2af0000\n" /* PQAP(QCI) */
86044c8c 796 "0: ipm %0\n"
45c9b47c 797 "srl %0,28\n"
86044c8c
CB		 798 "1:\n"
799 EX_TABLE(0b, 1b)
800 : "+r" (cc)
45c9b47c
TK		 801 : "r" (fcn_code), "r" (config)
802 : "cc", "0", "2", "memory"
803 );
804
805 return cc;
806}
807
808static int kvm_s390_apxa_installed(void)
809{
810 u8 config[128];
811 int cc;
812
813 if (test_facility(2) && test_facility(12)) {
814 cc = kvm_s390_query_ap_config(config);
815
816 if (cc)
817 pr_err("PQAP(QCI) failed with cc=%d", cc);
818 else
819 return config[0] & 0x40;
820 }
821
822 return 0;
823}
824
825static void kvm_s390_set_crycb_format(struct kvm *kvm)
826{
827 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
828
829 if (kvm_s390_apxa_installed())
830 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
831 else
832 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
833}
834
9d8d5786
MM		 835static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
836{
837 get_cpu_id(cpu_id);
838 cpu_id->version = 0xff;
839}
840
5102ee87
TK		 841static int kvm_s390_crypto_init(struct kvm *kvm)
842{
9d8d5786 843 if (!test_kvm_facility(kvm, 76))
5102ee87
TK		 844 return 0;
845
846 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
847 GFP_KERNEL | GFP_DMA);
848 if (!kvm->arch.crypto.crycb)
849 return -ENOMEM;
850
45c9b47c 851 kvm_s390_set_crycb_format(kvm);
5102ee87 852
ed6f76b4
TK		 853 /* Enable AES/DEA protected key functions by default */
854 kvm->arch.crypto.aes_kw = 1;
855 kvm->arch.crypto.dea_kw = 1;
856 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
857 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
858 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
859 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
a374e892 860
5102ee87
TK		 861 return 0;
862}
863
e08b9637 864int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 865{
9d8d5786 866 int i, rc;
b0c632db 867 char debug_name[16];
f6c137ff 868 static unsigned long sca_offset;
b0c632db 869
e08b9637
CO		 870 rc = -EINVAL;
871#ifdef CONFIG_KVM_S390_UCONTROL
872 if (type & ~KVM_VM_S390_UCONTROL)
873 goto out_err;
874 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
875 goto out_err;
876#else
877 if (type)
878 goto out_err;
879#endif
880
b0c632db
HC		 881 rc = s390_enable_sie();
882 if (rc)
d89f5eff 883 goto out_err;
b0c632db 884
b290411a
CO		 885 rc = -ENOMEM;
886
b0c632db
HC		 887 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
888 if (!kvm->arch.sca)
d89f5eff 889 goto out_err;
f6c137ff
CB		 890 spin_lock(&kvm_lock);
891 sca_offset = (sca_offset + 16) & 0x7f0;
892 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
893 spin_unlock(&kvm_lock);
b0c632db
HC		 894
895 sprintf(debug_name, "kvm-%u", current->pid);
896
897 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
898 if (!kvm->arch.dbf)
899 goto out_nodbf;
900
9d8d5786
MM		 901 /*
902 * The architectural maximum amount of facilities is 16 kbit. To store
903 * this amount, 2 kbyte of memory is required. Thus we need a full
981467c9
MM		 904 * page to hold the guest facility list (arch.model.fac->list) and the
905 * facility mask (arch.model.fac->mask). Its address size has to be
9d8d5786
MM		 906 * 31 bits and word aligned.
907 */
908 kvm->arch.model.fac =
981467c9 909 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
9d8d5786
MM		 910 if (!kvm->arch.model.fac)
911 goto out_nofac;
912
fb5bf93f 913 /* Populate the facility mask initially. */
981467c9 914 memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
94422ee8 915 S390_ARCH_FAC_LIST_SIZE_BYTE);
9d8d5786
MM		 916 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
917 if (i < kvm_s390_fac_list_mask_size())
981467c9 918 kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
9d8d5786 919 else
981467c9 920 kvm->arch.model.fac->mask[i] = 0UL;
9d8d5786
MM		 921 }
922
981467c9
MM		 923 /* Populate the facility list initially. */
924 memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
925 S390_ARCH_FAC_LIST_SIZE_BYTE);
926
9d8d5786 927 kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
658b6eda 928 kvm->arch.model.ibc = sclp_get_ibc() & 0x0fff;
9d8d5786 929
5102ee87
TK		 930 if (kvm_s390_crypto_init(kvm) < 0)
931 goto out_crypto;
932
ba5c1e9b
CO		 933 spin_lock_init(&kvm->arch.float_int.lock);
934 INIT_LIST_HEAD(&kvm->arch.float_int.list);
8a242234 935 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 936 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 937
b0c632db
HC		 938 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
939 VM_EVENT(kvm, 3, "%s", "vm created");
940
e08b9637
CO		 941 if (type & KVM_VM_S390_UCONTROL) {
942 kvm->arch.gmap = NULL;
943 } else {
0349985a 944 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
e08b9637
CO		 945 if (!kvm->arch.gmap)
946 goto out_nogmap;
2c70fe44 947 kvm->arch.gmap->private = kvm;
24eb3a82 948 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 949 }
fa6b7fe9
CH		 950
951 kvm->arch.css_support = 0;
84223598 952 kvm->arch.use_irqchip = 0;
68c55750 953 kvm->arch.use_vectors = 0;
72f25020 954 kvm->arch.epoch = 0;
fa6b7fe9 955
8ad35755
DH		 956 spin_lock_init(&kvm->arch.start_stop_lock);
957
d89f5eff 958 return 0;
598841ca 959out_nogmap:
5102ee87
TK		 960 kfree(kvm->arch.crypto.crycb);
961out_crypto:
9d8d5786
MM		 962 free_page((unsigned long)kvm->arch.model.fac);
963out_nofac:
598841ca 964 debug_unregister(kvm->arch.dbf);
b0c632db
HC		 965out_nodbf:
966 free_page((unsigned long)(kvm->arch.sca));
d89f5eff
JK		 967out_err:
968 return rc;
b0c632db
HC		 969}
970
d329c035
CB		 971void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
972{
973 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 974 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 975 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 976 kvm_clear_async_pf_completion_queue(vcpu);
58f9460b
CO		 977 if (!kvm_is_ucontrol(vcpu->kvm)) {
978 clear_bit(63 - vcpu->vcpu_id,
979 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
980 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
981 (__u64) vcpu->arch.sie_block)
982 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
983 }
abf4a71e 984 smp_mb();
27e0393f
CO		 985
986 if (kvm_is_ucontrol(vcpu->kvm))
987 gmap_free(vcpu->arch.gmap);
988
b31605c1
DD		 989 if (kvm_s390_cmma_enabled(vcpu->kvm))
990 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 991 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 992
6692cef3 993 kvm_vcpu_uninit(vcpu);
b110feaf 994 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 995}
996
997static void kvm_free_vcpus(struct kvm *kvm)
998{
999 unsigned int i;
988a2cae 1000 struct kvm_vcpu *vcpu;
d329c035 1001
988a2cae
GN		 1002 kvm_for_each_vcpu(i, vcpu, kvm)
1003 kvm_arch_vcpu_destroy(vcpu);
1004
1005 mutex_lock(&kvm->lock);
1006 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1007 kvm->vcpus[i] = NULL;
1008
1009 atomic_set(&kvm->online_vcpus, 0);
1010 mutex_unlock(&kvm->lock);
d329c035
CB		 1011}
1012
b0c632db
HC		 1013void kvm_arch_destroy_vm(struct kvm *kvm)
1014{
d329c035 1015 kvm_free_vcpus(kvm);
9d8d5786 1016 free_page((unsigned long)kvm->arch.model.fac);
b0c632db 1017 free_page((unsigned long)(kvm->arch.sca));
d329c035 1018 debug_unregister(kvm->arch.dbf);
5102ee87 1019 kfree(kvm->arch.crypto.crycb);
27e0393f
CO		 1020 if (!kvm_is_ucontrol(kvm))
1021 gmap_free(kvm->arch.gmap);
841b91c5 1022 kvm_s390_destroy_adapters(kvm);
67335e63 1023 kvm_s390_clear_float_irqs(kvm);
b0c632db
HC		 1024}
1025
1026/* Section: vcpu related */
dafd032a
DD		 1027static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1028{
1029 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1030 if (!vcpu->arch.gmap)
1031 return -ENOMEM;
1032 vcpu->arch.gmap->private = vcpu->kvm;
1033
1034 return 0;
1035}
1036
b0c632db
HC		 1037int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1038{
3c038e6b
DD		 1039 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1040 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 1041 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1042 KVM_SYNC_GPRS |
9eed0735 1043 KVM_SYNC_ACRS |
b028ee3e
DH		 1044 KVM_SYNC_CRS |
1045 KVM_SYNC_ARCH0 |
1046 KVM_SYNC_PFAULT;
68c55750
EF		 1047 if (test_kvm_facility(vcpu->kvm, 129))
1048 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
dafd032a
DD		 1049
1050 if (kvm_is_ucontrol(vcpu->kvm))
1051 return __kvm_ucontrol_vcpu_init(vcpu);
1052
b0c632db
HC		 1053 return 0;
1054}
1055
b0c632db
HC		 1056void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1057{
4725c860 1058 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
68c55750
EF		 1059 if (vcpu->kvm->arch.use_vectors)
1060 save_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
1061 else
1062 save_fp_regs(vcpu->arch.host_fpregs.fprs);
b0c632db 1063 save_access_regs(vcpu->arch.host_acrs);
68c55750
EF		 1064 if (vcpu->kvm->arch.use_vectors) {
1065 restore_fp_ctl(&vcpu->run->s.regs.fpc);
1066 restore_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
1067 } else {
1068 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1069 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1070 }
59674c1a 1071 restore_access_regs(vcpu->run->s.regs.acrs);
480e5926 1072 gmap_enable(vcpu->arch.gmap);
9e6dabef 1073 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
b0c632db
HC		 1074}
1075
1076void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1077{
9e6dabef 1078 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
480e5926 1079 gmap_disable(vcpu->arch.gmap);
68c55750
EF		 1080 if (vcpu->kvm->arch.use_vectors) {
1081 save_fp_ctl(&vcpu->run->s.regs.fpc);
1082 save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
1083 } else {
1084 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1085 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1086 }
59674c1a 1087 save_access_regs(vcpu->run->s.regs.acrs);
4725c860 1088 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
68c55750
EF		 1089 if (vcpu->kvm->arch.use_vectors)
1090 restore_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
1091 else
1092 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
b0c632db
HC		 1093 restore_access_regs(vcpu->arch.host_acrs);
1094}
1095
1096static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1097{
1098 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1099 vcpu->arch.sie_block->gpsw.mask = 0UL;
1100 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 1101 kvm_s390_set_prefix(vcpu, 0);
b0c632db
HC		 1102 vcpu->arch.sie_block->cputm = 0UL;
1103 vcpu->arch.sie_block->ckc = 0UL;
1104 vcpu->arch.sie_block->todpr = 0;
1105 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1106 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1107 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1108 vcpu->arch.guest_fpregs.fpc = 0;
1109 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
1110 vcpu->arch.sie_block->gbea = 1;
672550fb 1111 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 1112 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1113 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 1114 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1115 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 1116 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 1117}
1118
31928aa5 1119void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 1120{
72f25020
JH		 1121 mutex_lock(&vcpu->kvm->lock);
1122 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1123 mutex_unlock(&vcpu->kvm->lock);
dafd032a
DD		 1124 if (!kvm_is_ucontrol(vcpu->kvm))
1125 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
42897d86
MT		 1126}
1127
5102ee87
TK		 1128static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1129{
9d8d5786 1130 if (!test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 1131 return;
1132
a374e892
TK		 1133 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1134
1135 if (vcpu->kvm->arch.crypto.aes_kw)
1136 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1137 if (vcpu->kvm->arch.crypto.dea_kw)
1138 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1139
5102ee87
TK		 1140 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1141}
1142
b31605c1
DD		 1143void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1144{
1145 free_page(vcpu->arch.sie_block->cbrlo);
1146 vcpu->arch.sie_block->cbrlo = 0;
1147}
1148
1149int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1150{
1151 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1152 if (!vcpu->arch.sie_block->cbrlo)
1153 return -ENOMEM;
1154
1155 vcpu->arch.sie_block->ecb2 |= 0x80;
1156 vcpu->arch.sie_block->ecb2 &= ~0x08;
1157 return 0;
1158}
1159
91520f1a
MM		 1160static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1161{
1162 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1163
1164 vcpu->arch.cpu_id = model->cpu_id;
1165 vcpu->arch.sie_block->ibc = model->ibc;
1166 vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1167}
1168
b0c632db
HC		 1169int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1170{
b31605c1 1171 int rc = 0;
b31288fa 1172
9e6dabef
CH		 1173 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1174 CPUSTAT_SM |
69d0d3a3
CB		 1175 CPUSTAT_STOPPED |
1176 CPUSTAT_GED);
91520f1a
MM		 1177 kvm_s390_vcpu_setup_model(vcpu);
1178
fc34531d 1179 vcpu->arch.sie_block->ecb = 6;
9d8d5786 1180 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
7feb6bb8
MM		 1181 vcpu->arch.sie_block->ecb |= 0x10;
1182
69d0d3a3 1183 vcpu->arch.sie_block->ecb2 = 8;
ea5f4969 1184 vcpu->arch.sie_block->eca = 0xC1002000U;
217a4406
HC		 1185 if (sclp_has_siif())
1186 vcpu->arch.sie_block->eca |= 1;
ea5f4969
DH		 1187 if (sclp_has_sigpif())
1188 vcpu->arch.sie_block->eca |= 0x10000000U;
492d8642 1189 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
5a5e6536 1190
b31605c1
DD		 1191 if (kvm_s390_cmma_enabled(vcpu->kvm)) {
1192 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1193 if (rc)
1194 return rc;
b31288fa 1195 }
0ac96caf 1196 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 1197 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 1198
5102ee87
TK		 1199 kvm_s390_vcpu_crypto_setup(vcpu);
1200
b31605c1 1201 return rc;
b0c632db
HC		 1202}
1203
1204struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1205 unsigned int id)
1206{
4d47555a 1207 struct kvm_vcpu *vcpu;
7feb6bb8 1208 struct sie_page *sie_page;
4d47555a
CO		 1209 int rc = -EINVAL;
1210
1211 if (id >= KVM_MAX_VCPUS)
1212 goto out;
1213
1214 rc = -ENOMEM;
b0c632db 1215
b110feaf 1216 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 1217 if (!vcpu)
4d47555a 1218 goto out;
b0c632db 1219
7feb6bb8
MM		 1220 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1221 if (!sie_page)
b0c632db
HC		 1222 goto out_free_cpu;
1223
7feb6bb8
MM		 1224 vcpu->arch.sie_block = &sie_page->sie_block;
1225 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
68c55750 1226 vcpu->arch.host_vregs = &sie_page->vregs;
7feb6bb8 1227
b0c632db 1228 vcpu->arch.sie_block->icpua = id;
58f9460b
CO		 1229 if (!kvm_is_ucontrol(kvm)) {
1230 if (!kvm->arch.sca) {
1231 WARN_ON_ONCE(1);
1232 goto out_free_cpu;
1233 }
1234 if (!kvm->arch.sca->cpu[id].sda)
1235 kvm->arch.sca->cpu[id].sda =
1236 (__u64) vcpu->arch.sie_block;
1237 vcpu->arch.sie_block->scaoh =
1238 (__u32)(((__u64)kvm->arch.sca) >> 32);
1239 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
1240 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
1241 }
b0c632db 1242
ba5c1e9b 1243 spin_lock_init(&vcpu->arch.local_int.lock);
ba5c1e9b 1244 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 1245 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 1246 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
ba5c1e9b 1247
b0c632db
HC		 1248 rc = kvm_vcpu_init(vcpu, kvm, id);
1249 if (rc)
7b06bf2f 1250 goto out_free_sie_block;
b0c632db
HC		 1251 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
1252 vcpu->arch.sie_block);
ade38c31 1253 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 1254
b0c632db 1255 return vcpu;
7b06bf2f
WY		 1256out_free_sie_block:
1257 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 1258out_free_cpu:
b110feaf 1259 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 1260out:
b0c632db
HC		 1261 return ERR_PTR(rc);
1262}
1263
b0c632db
HC		 1264int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1265{
9a022067 1266 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 1267}
1268
49b99e1e
CB		 1269void s390_vcpu_block(struct kvm_vcpu *vcpu)
1270{
1271 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1272}
1273
1274void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1275{
1276 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1277}
1278
1279/*
1280 * Kick a guest cpu out of SIE and wait until SIE is not running.
1281 * If the CPU is not running (e.g. waiting as idle) the function will
1282 * return immediately. */
1283void exit_sie(struct kvm_vcpu *vcpu)
1284{
1285 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1286 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1287 cpu_relax();
1288}
1289
1290/* Kick a guest cpu out of SIE and prevent SIE-reentry */
1291void exit_sie_sync(struct kvm_vcpu *vcpu)
1292{
1293 s390_vcpu_block(vcpu);
1294 exit_sie(vcpu);
1295}
1296
2c70fe44
CB		 1297static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1298{
1299 int i;
1300 struct kvm *kvm = gmap->private;
1301 struct kvm_vcpu *vcpu;
1302
1303 kvm_for_each_vcpu(i, vcpu, kvm) {
1304 /* match against both prefix pages */
fda902cb 1305 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
2c70fe44
CB		 1306 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1307 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
1308 exit_sie_sync(vcpu);
1309 }
1310 }
1311}
1312
b6d33834
CD		 1313int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1314{
1315 /* kvm common code refers to this, but never calls it */
1316 BUG();
1317 return 0;
1318}
1319
14eebd91
CO		 1320static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1321 struct kvm_one_reg *reg)
1322{
1323 int r = -EINVAL;
1324
1325 switch (reg->id) {
29b7c71b
CO		 1326 case KVM_REG_S390_TODPR:
1327 r = put_user(vcpu->arch.sie_block->todpr,
1328 (u32 __user *)reg->addr);
1329 break;
1330 case KVM_REG_S390_EPOCHDIFF:
1331 r = put_user(vcpu->arch.sie_block->epoch,
1332 (u64 __user *)reg->addr);
1333 break;
46a6dd1c
J		 1334 case KVM_REG_S390_CPU_TIMER:
1335 r = put_user(vcpu->arch.sie_block->cputm,
1336 (u64 __user *)reg->addr);
1337 break;
1338 case KVM_REG_S390_CLOCK_COMP:
1339 r = put_user(vcpu->arch.sie_block->ckc,
1340 (u64 __user *)reg->addr);
1341 break;
536336c2
DD		 1342 case KVM_REG_S390_PFTOKEN:
1343 r = put_user(vcpu->arch.pfault_token,
1344 (u64 __user *)reg->addr);
1345 break;
1346 case KVM_REG_S390_PFCOMPARE:
1347 r = put_user(vcpu->arch.pfault_compare,
1348 (u64 __user *)reg->addr);
1349 break;
1350 case KVM_REG_S390_PFSELECT:
1351 r = put_user(vcpu->arch.pfault_select,
1352 (u64 __user *)reg->addr);
1353 break;
672550fb
CB		 1354 case KVM_REG_S390_PP:
1355 r = put_user(vcpu->arch.sie_block->pp,
1356 (u64 __user *)reg->addr);
1357 break;
afa45ff5
CB		 1358 case KVM_REG_S390_GBEA:
1359 r = put_user(vcpu->arch.sie_block->gbea,
1360 (u64 __user *)reg->addr);
1361 break;
14eebd91
CO		 1362 default:
1363 break;
1364 }
1365
1366 return r;
1367}
1368
1369static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1370 struct kvm_one_reg *reg)
1371{
1372 int r = -EINVAL;
1373
1374 switch (reg->id) {
29b7c71b
CO		 1375 case KVM_REG_S390_TODPR:
1376 r = get_user(vcpu->arch.sie_block->todpr,
1377 (u32 __user *)reg->addr);
1378 break;
1379 case KVM_REG_S390_EPOCHDIFF:
1380 r = get_user(vcpu->arch.sie_block->epoch,
1381 (u64 __user *)reg->addr);
1382 break;
46a6dd1c
J		 1383 case KVM_REG_S390_CPU_TIMER:
1384 r = get_user(vcpu->arch.sie_block->cputm,
1385 (u64 __user *)reg->addr);
1386 break;
1387 case KVM_REG_S390_CLOCK_COMP:
1388 r = get_user(vcpu->arch.sie_block->ckc,
1389 (u64 __user *)reg->addr);
1390 break;
536336c2
DD		 1391 case KVM_REG_S390_PFTOKEN:
1392 r = get_user(vcpu->arch.pfault_token,
1393 (u64 __user *)reg->addr);
9fbd8082
DH		 1394 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1395 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 1396 break;
1397 case KVM_REG_S390_PFCOMPARE:
1398 r = get_user(vcpu->arch.pfault_compare,
1399 (u64 __user *)reg->addr);
1400 break;
1401 case KVM_REG_S390_PFSELECT:
1402 r = get_user(vcpu->arch.pfault_select,
1403 (u64 __user *)reg->addr);
1404 break;
672550fb
CB		 1405 case KVM_REG_S390_PP:
1406 r = get_user(vcpu->arch.sie_block->pp,
1407 (u64 __user *)reg->addr);
1408 break;
afa45ff5
CB		 1409 case KVM_REG_S390_GBEA:
1410 r = get_user(vcpu->arch.sie_block->gbea,
1411 (u64 __user *)reg->addr);
1412 break;
14eebd91
CO		 1413 default:
1414 break;
1415 }
1416
1417 return r;
1418}
b6d33834 1419
b0c632db
HC		 1420static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1421{
b0c632db 1422 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 1423 return 0;
1424}
1425
1426int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1427{
5a32c1af 1428 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 1429 return 0;
1430}
1431
1432int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1433{
5a32c1af 1434 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 1435 return 0;
1436}
1437
1438int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1439 struct kvm_sregs *sregs)
1440{
59674c1a 1441 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 1442 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
59674c1a 1443 restore_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 1444 return 0;
1445}
1446
1447int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1448 struct kvm_sregs *sregs)
1449{
59674c1a 1450 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 1451 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 1452 return 0;
1453}
1454
1455int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1456{
4725c860
MS		 1457 if (test_fp_ctl(fpu->fpc))
1458 return -EINVAL;
b0c632db 1459 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
4725c860
MS		 1460 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1461 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1462 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
b0c632db
HC		 1463 return 0;
1464}
1465
1466int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1467{
b0c632db
HC		 1468 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1469 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
b0c632db
HC		 1470 return 0;
1471}
1472
1473static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1474{
1475 int rc = 0;
1476
7a42fdc2 1477 if (!is_vcpu_stopped(vcpu))
b0c632db 1478 rc = -EBUSY;
d7b0b5eb
CO		 1479 else {
1480 vcpu->run->psw_mask = psw.mask;
1481 vcpu->run->psw_addr = psw.addr;
1482 }
b0c632db
HC		 1483 return rc;
1484}
1485
1486int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1487 struct kvm_translation *tr)
1488{
1489 return -EINVAL; /* not implemented yet */
1490}
1491
27291e21
DH		 1492#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1493 KVM_GUESTDBG_USE_HW_BP | \
1494 KVM_GUESTDBG_ENABLE)
1495
d0bfb940
JK		 1496int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1497 struct kvm_guest_debug *dbg)
b0c632db 1498{
27291e21
DH		 1499 int rc = 0;
1500
1501 vcpu->guest_debug = 0;
1502 kvm_s390_clear_bp_data(vcpu);
1503
2de3bfc2 1504 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21
DH		 1505 return -EINVAL;
1506
1507 if (dbg->control & KVM_GUESTDBG_ENABLE) {
1508 vcpu->guest_debug = dbg->control;
1509 /* enforce guest PER */
1510 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1511
1512 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1513 rc = kvm_s390_import_bp_data(vcpu, dbg);
1514 } else {
1515 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1516 vcpu->arch.guestdbg.last_bp = 0;
1517 }
1518
1519 if (rc) {
1520 vcpu->guest_debug = 0;
1521 kvm_s390_clear_bp_data(vcpu);
1522 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1523 }
1524
1525 return rc;
b0c632db
HC		 1526}
1527
62d9f0db
MT		 1528int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1529 struct kvm_mp_state *mp_state)
1530{
6352e4d2
DH		 1531 /* CHECK_STOP and LOAD are not supported yet */
1532 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1533 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 1534}
1535
1536int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1537 struct kvm_mp_state *mp_state)
1538{
6352e4d2
DH		 1539 int rc = 0;
1540
1541 /* user space knows about this interface - let it control the state */
1542 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1543
1544 switch (mp_state->mp_state) {
1545 case KVM_MP_STATE_STOPPED:
1546 kvm_s390_vcpu_stop(vcpu);
1547 break;
1548 case KVM_MP_STATE_OPERATING:
1549 kvm_s390_vcpu_start(vcpu);
1550 break;
1551 case KVM_MP_STATE_LOAD:
1552 case KVM_MP_STATE_CHECK_STOP:
1553 /* fall through - CHECK_STOP and LOAD are not supported yet */
1554 default:
1555 rc = -ENXIO;
1556 }
1557
1558 return rc;
62d9f0db
MT		 1559}
1560
b31605c1
DD		 1561bool kvm_s390_cmma_enabled(struct kvm *kvm)
1562{
1563 if (!MACHINE_IS_LPAR)
1564 return false;
1565 /* only enable for z10 and later */
1566 if (!MACHINE_HAS_EDAT1)
1567 return false;
1568 if (!kvm->arch.use_cmma)
1569 return false;
1570 return true;
1571}
1572
8ad35755
DH		 1573static bool ibs_enabled(struct kvm_vcpu *vcpu)
1574{
1575 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1576}
1577
2c70fe44
CB		 1578static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1579{
8ad35755
DH		 1580retry:
1581 s390_vcpu_unblock(vcpu);
2c70fe44
CB		 1582 /*
1583 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1584 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1585 * This ensures that the ipte instruction for this request has
1586 * already finished. We might race against a second unmapper that
1587 * wants to set the blocking bit. Lets just retry the request loop.
1588 */
8ad35755 1589 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44
CB		 1590 int rc;
1591 rc = gmap_ipte_notify(vcpu->arch.gmap,
fda902cb 1592 kvm_s390_get_prefix(vcpu),
2c70fe44
CB		 1593 PAGE_SIZE * 2);
1594 if (rc)
1595 return rc;
8ad35755 1596 goto retry;
2c70fe44 1597 }
8ad35755 1598
d3d692c8
DH		 1599 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1600 vcpu->arch.sie_block->ihcpu = 0xffff;
1601 goto retry;
1602 }
1603
8ad35755
DH		 1604 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1605 if (!ibs_enabled(vcpu)) {
1606 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1607 atomic_set_mask(CPUSTAT_IBS,
1608 &vcpu->arch.sie_block->cpuflags);
1609 }
1610 goto retry;
2c70fe44 1611 }
8ad35755
DH		 1612
1613 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1614 if (ibs_enabled(vcpu)) {
1615 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1616 atomic_clear_mask(CPUSTAT_IBS,
1617 &vcpu->arch.sie_block->cpuflags);
1618 }
1619 goto retry;
1620 }
1621
0759d068
DH		 1622 /* nothing to do, just clear the request */
1623 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1624
2c70fe44
CB		 1625 return 0;
1626}
1627
fa576c58
TH		 1628/**
1629 * kvm_arch_fault_in_page - fault-in guest page if necessary
1630 * @vcpu: The corresponding virtual cpu
1631 * @gpa: Guest physical address
1632 * @writable: Whether the page should be writable or not
1633 *
1634 * Make sure that a guest page has been faulted-in on the host.
1635 *
1636 * Return: Zero on success, negative error code otherwise.
1637 */
1638long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 1639{
527e30b4
MS		 1640 return gmap_fault(vcpu->arch.gmap, gpa,
1641 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 1642}
1643
3c038e6b
DD		 1644static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1645 unsigned long token)
1646{
1647 struct kvm_s390_interrupt inti;
383d0b05 1648 struct kvm_s390_irq irq;
3c038e6b
DD		 1649
1650 if (start_token) {
383d0b05
JF		 1651 irq.u.ext.ext_params2 = token;
1652 irq.type = KVM_S390_INT_PFAULT_INIT;
1653 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 1654 } else {
1655 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 1656 inti.parm64 = token;
3c038e6b
DD		 1657 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1658 }
1659}
1660
1661void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1662 struct kvm_async_pf *work)
1663{
1664 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1665 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1666}
1667
1668void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1669 struct kvm_async_pf *work)
1670{
1671 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1672 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1673}
1674
1675void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1676 struct kvm_async_pf *work)
1677{
1678 /* s390 will always inject the page directly */
1679}
1680
1681bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1682{
1683 /*
1684 * s390 will always inject the page directly,
1685 * but we still want check_async_completion to cleanup
1686 */
1687 return true;
1688}
1689
1690static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1691{
1692 hva_t hva;
1693 struct kvm_arch_async_pf arch;
1694 int rc;
1695
1696 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1697 return 0;
1698 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1699 vcpu->arch.pfault_compare)
1700 return 0;
1701 if (psw_extint_disabled(vcpu))
1702 return 0;
9a022067 1703 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 1704 return 0;
1705 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1706 return 0;
1707 if (!vcpu->arch.gmap->pfault_enabled)
1708 return 0;
1709
81480cc1
HC		 1710 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1711 hva += current->thread.gmap_addr & ~PAGE_MASK;
1712 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 1713 return 0;
1714
1715 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1716 return rc;
1717}
1718
3fb4c40f 1719static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 1720{
3fb4c40f 1721 int rc, cpuflags;
e168bf8d 1722
3c038e6b
DD		 1723 /*
1724 * On s390 notifications for arriving pages will be delivered directly
1725 * to the guest but the house keeping for completed pfaults is
1726 * handled outside the worker.
1727 */
1728 kvm_check_async_pf_completion(vcpu);
1729
5a32c1af 1730 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
b0c632db
HC		 1731
1732 if (need_resched())
1733 schedule();
1734
d3a73acb 1735 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 1736 s390_handle_mcck();
1737
79395031
JF		 1738 if (!kvm_is_ucontrol(vcpu->kvm)) {
1739 rc = kvm_s390_deliver_pending_interrupts(vcpu);
1740 if (rc)
1741 return rc;
1742 }
0ff31867 1743
2c70fe44
CB		 1744 rc = kvm_s390_handle_requests(vcpu);
1745 if (rc)
1746 return rc;
1747
27291e21
DH		 1748 if (guestdbg_enabled(vcpu)) {
1749 kvm_s390_backup_guest_per_regs(vcpu);
1750 kvm_s390_patch_guest_per_regs(vcpu);
1751 }
1752
b0c632db 1753 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 1754 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1755 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1756 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 1757
3fb4c40f
TH		 1758 return 0;
1759}
1760
492d8642
TH		 1761static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
1762{
1763 psw_t *psw = &vcpu->arch.sie_block->gpsw;
1764 u8 opcode;
1765 int rc;
1766
1767 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1768 trace_kvm_s390_sie_fault(vcpu);
1769
1770 /*
1771 * We want to inject an addressing exception, which is defined as a
1772 * suppressing or terminating exception. However, since we came here
1773 * by a DAT access exception, the PSW still points to the faulting
1774 * instruction since DAT exceptions are nullifying. So we've got
1775 * to look up the current opcode to get the length of the instruction
1776 * to be able to forward the PSW.
1777 */
1778 rc = read_guest(vcpu, psw->addr, &opcode, 1);
1779 if (rc)
1780 return kvm_s390_inject_prog_cond(vcpu, rc);
1781 psw->addr = __rewind_psw(*psw, -insn_length(opcode));
1782
1783 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1784}
1785
3fb4c40f
TH		 1786static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1787{
24eb3a82 1788 int rc = -1;
2b29a9fd
DD		 1789
1790 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1791 vcpu->arch.sie_block->icptcode);
1792 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1793
27291e21
DH		 1794 if (guestdbg_enabled(vcpu))
1795 kvm_s390_restore_guest_per_regs(vcpu);
1796
3fb4c40f 1797 if (exit_reason >= 0) {
7c470539 1798 rc = 0;
210b1607
TH		 1799 } else if (kvm_is_ucontrol(vcpu->kvm)) {
1800 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1801 vcpu->run->s390_ucontrol.trans_exc_code =
1802 current->thread.gmap_addr;
1803 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1804 rc = -EREMOTE;
24eb3a82
DD		 1805
1806 } else if (current->thread.gmap_pfault) {
3c038e6b 1807 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 1808 current->thread.gmap_pfault = 0;
fa576c58 1809 if (kvm_arch_setup_async_pf(vcpu)) {
24eb3a82 1810 rc = 0;
fa576c58
TH		 1811 } else {
1812 gpa_t gpa = current->thread.gmap_addr;
1813 rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1814 }
24eb3a82
DD		 1815 }
1816
492d8642
TH		 1817 if (rc == -1)
1818 rc = vcpu_post_run_fault_in_sie(vcpu);
b0c632db 1819
5a32c1af 1820 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
3fb4c40f 1821
a76ccff6
TH		 1822 if (rc == 0) {
1823 if (kvm_is_ucontrol(vcpu->kvm))
2955c83f
CB		 1824 /* Don't exit for host interrupts. */
1825 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
a76ccff6
TH		 1826 else
1827 rc = kvm_handle_sie_intercept(vcpu);
1828 }
1829
3fb4c40f
TH		 1830 return rc;
1831}
1832
1833static int __vcpu_run(struct kvm_vcpu *vcpu)
1834{
1835 int rc, exit_reason;
1836
800c1065
TH		 1837 /*
1838 * We try to hold kvm->srcu during most of vcpu_run (except when run-
1839 * ning the guest), so that memslots (and other stuff) are protected
1840 */
1841 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1842
a76ccff6
TH		 1843 do {
1844 rc = vcpu_pre_run(vcpu);
1845 if (rc)
1846 break;
3fb4c40f 1847
800c1065 1848 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 1849 /*
1850 * As PF_VCPU will be used in fault handler, between
1851 * guest_enter and guest_exit should be no uaccess.
1852 */
1853 preempt_disable();
1854 kvm_guest_enter();
1855 preempt_enable();
1856 exit_reason = sie64a(vcpu->arch.sie_block,
1857 vcpu->run->s.regs.gprs);
1858 kvm_guest_exit();
800c1065 1859 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 1860
1861 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 1862 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 1863
800c1065 1864 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 1865 return rc;
b0c632db
HC		 1866}
1867
b028ee3e
DH		 1868static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1869{
1870 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1871 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
1872 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
1873 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1874 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1875 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 1876 /* some control register changes require a tlb flush */
1877 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 1878 }
1879 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
1880 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
1881 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
1882 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
1883 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
1884 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
1885 }
1886 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
1887 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
1888 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
1889 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 1890 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1891 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e
DH		 1892 }
1893 kvm_run->kvm_dirty_regs = 0;
1894}
1895
1896static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1897{
1898 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
1899 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
1900 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
1901 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
1902 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
1903 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
1904 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
1905 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
1906 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
1907 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
1908 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
1909 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
1910}
1911
b0c632db
HC		 1912int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1913{
8f2abe6a 1914 int rc;
b0c632db
HC		 1915 sigset_t sigsaved;
1916
27291e21
DH		 1917 if (guestdbg_exit_pending(vcpu)) {
1918 kvm_s390_prepare_debug_exit(vcpu);
1919 return 0;
1920 }
1921
b0c632db
HC		 1922 if (vcpu->sigset_active)
1923 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1924
6352e4d2
DH		 1925 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
1926 kvm_s390_vcpu_start(vcpu);
1927 } else if (is_vcpu_stopped(vcpu)) {
1928 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
1929 vcpu->vcpu_id);
1930 return -EINVAL;
1931 }
b0c632db 1932
b028ee3e 1933 sync_regs(vcpu, kvm_run);
d7b0b5eb 1934
dab4079d 1935 might_fault();
a76ccff6 1936 rc = __vcpu_run(vcpu);
9ace903d 1937
b1d16c49
CE		 1938 if (signal_pending(current) && !rc) {
1939 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 1940 rc = -EINTR;
b1d16c49 1941 }
8f2abe6a 1942
27291e21
DH		 1943 if (guestdbg_exit_pending(vcpu) && !rc) {
1944 kvm_s390_prepare_debug_exit(vcpu);
1945 rc = 0;
1946 }
1947
b8e660b8 1948 if (rc == -EOPNOTSUPP) {
8f2abe6a
CB		 1949 /* intercept cannot be handled in-kernel, prepare kvm-run */
1950 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
1951 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
8f2abe6a
CB		 1952 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
1953 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
1954 rc = 0;
1955 }
1956
1957 if (rc == -EREMOTE) {
1958 /* intercept was handled, but userspace support is needed
1959 * kvm_run has been prepared by the handler */
1960 rc = 0;
1961 }
b0c632db 1962
b028ee3e 1963 store_regs(vcpu, kvm_run);
d7b0b5eb 1964
b0c632db
HC		 1965 if (vcpu->sigset_active)
1966 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1967
b0c632db 1968 vcpu->stat.exit_userspace++;
7e8e6ab4 1969 return rc;
b0c632db
HC		 1970}
1971
b0c632db
HC		 1972/*
1973 * store status at address
1974 * we use have two special cases:
1975 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1976 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1977 */
d0bce605 1978int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 1979{
092670cd 1980 unsigned char archmode = 1;
fda902cb 1981 unsigned int px;
178bd789 1982 u64 clkcomp;
d0bce605 1983 int rc;
b0c632db 1984
d0bce605
HC		 1985 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
1986 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 1987 return -EFAULT;
d0bce605
HC		 1988 gpa = SAVE_AREA_BASE;
1989 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
1990 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 1991 return -EFAULT;
d0bce605
HC		 1992 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
1993 }
1994 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
1995 vcpu->arch.guest_fpregs.fprs, 128);
1996 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
1997 vcpu->run->s.regs.gprs, 128);
1998 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
1999 &vcpu->arch.sie_block->gpsw, 16);
fda902cb 2000 px = kvm_s390_get_prefix(vcpu);
d0bce605 2001 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
fda902cb 2002 &px, 4);
d0bce605
HC		 2003 rc |= write_guest_abs(vcpu,
2004 gpa + offsetof(struct save_area, fp_ctrl_reg),
2005 &vcpu->arch.guest_fpregs.fpc, 4);
2006 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
2007 &vcpu->arch.sie_block->todpr, 4);
2008 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
2009 &vcpu->arch.sie_block->cputm, 8);
178bd789 2010 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d0bce605
HC		 2011 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
2012 &clkcomp, 8);
2013 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
2014 &vcpu->run->s.regs.acrs, 64);
2015 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
2016 &vcpu->arch.sie_block->gcr, 128);
2017 return rc ? -EFAULT : 0;
b0c632db
HC		 2018}
2019
e879892c
TH		 2020int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2021{
2022 /*
2023 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2024 * copying in vcpu load/put. Lets update our copies before we save
2025 * it into the save area
2026 */
2027 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
2028 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
2029 save_access_regs(vcpu->run->s.regs.acrs);
2030
2031 return kvm_s390_store_status_unloaded(vcpu, addr);
2032}
2033
bc17de7c
EF		 2034/*
2035 * store additional status at address
2036 */
2037int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2038 unsigned long gpa)
2039{
2040 /* Only bits 0-53 are used for address formation */
2041 if (!(gpa & ~0x3ff))
2042 return 0;
2043
2044 return write_guest_abs(vcpu, gpa & ~0x3ff,
2045 (void *)&vcpu->run->s.regs.vrs, 512);
2046}
2047
2048int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2049{
2050 if (!test_kvm_facility(vcpu->kvm, 129))
2051 return 0;
2052
2053 /*
2054 * The guest VXRS are in the host VXRs due to the lazy
2055 * copying in vcpu load/put. Let's update our copies before we save
2056 * it into the save area.
2057 */
2058 save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
2059
2060 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2061}
2062
8ad35755
DH		 2063static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2064{
2065 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2066 kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu);
2067 exit_sie_sync(vcpu);
2068}
2069
2070static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2071{
2072 unsigned int i;
2073 struct kvm_vcpu *vcpu;
2074
2075 kvm_for_each_vcpu(i, vcpu, kvm) {
2076 __disable_ibs_on_vcpu(vcpu);
2077 }
2078}
2079
2080static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2081{
2082 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2083 kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu);
2084 exit_sie_sync(vcpu);
2085}
2086
6852d7b6
DH		 2087void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2088{
8ad35755
DH		 2089 int i, online_vcpus, started_vcpus = 0;
2090
2091 if (!is_vcpu_stopped(vcpu))
2092 return;
2093
6852d7b6 2094 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 2095 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2096 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2097 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2098
2099 for (i = 0; i < online_vcpus; i++) {
2100 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2101 started_vcpus++;
2102 }
2103
2104 if (started_vcpus == 0) {
2105 /* we're the only active VCPU -> speed it up */
2106 __enable_ibs_on_vcpu(vcpu);
2107 } else if (started_vcpus == 1) {
2108 /*
2109 * As we are starting a second VCPU, we have to disable
2110 * the IBS facility on all VCPUs to remove potentially
2111 * oustanding ENABLE requests.
2112 */
2113 __disable_ibs_on_all_vcpus(vcpu->kvm);
2114 }
2115
6852d7b6 2116 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2117 /*
2118 * Another VCPU might have used IBS while we were offline.
2119 * Let's play safe and flush the VCPU at startup.
2120 */
d3d692c8 2121 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 2122 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2123 return;
6852d7b6
DH		 2124}
2125
2126void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2127{
8ad35755
DH		 2128 int i, online_vcpus, started_vcpus = 0;
2129 struct kvm_vcpu *started_vcpu = NULL;
2130
2131 if (is_vcpu_stopped(vcpu))
2132 return;
2133
6852d7b6 2134 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 2135 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2136 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2137 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2138
32f5ff63 2139 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 2140 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 2141
6cddd432 2142 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2143 __disable_ibs_on_vcpu(vcpu);
2144
2145 for (i = 0; i < online_vcpus; i++) {
2146 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2147 started_vcpus++;
2148 started_vcpu = vcpu->kvm->vcpus[i];
2149 }
2150 }
2151
2152 if (started_vcpus == 1) {
2153 /*
2154 * As we only have one VCPU left, we want to enable the
2155 * IBS facility for that VCPU to speed it up.
2156 */
2157 __enable_ibs_on_vcpu(started_vcpu);
2158 }
2159
433b9ee4 2160 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2161 return;
6852d7b6
DH		 2162}
2163
d6712df9
CH		 2164static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2165 struct kvm_enable_cap *cap)
2166{
2167 int r;
2168
2169 if (cap->flags)
2170 return -EINVAL;
2171
2172 switch (cap->cap) {
fa6b7fe9
CH		 2173 case KVM_CAP_S390_CSS_SUPPORT:
2174 if (!vcpu->kvm->arch.css_support) {
2175 vcpu->kvm->arch.css_support = 1;
2176 trace_kvm_s390_enable_css(vcpu->kvm);
2177 }
2178 r = 0;
2179 break;
d6712df9
CH		 2180 default:
2181 r = -EINVAL;
2182 break;
2183 }
2184 return r;
2185}
2186
b0c632db
HC		 2187long kvm_arch_vcpu_ioctl(struct file *filp,
2188 unsigned int ioctl, unsigned long arg)
2189{
2190 struct kvm_vcpu *vcpu = filp->private_data;
2191 void __user *argp = (void __user *)arg;
800c1065 2192 int idx;
bc923cc9 2193 long r;
b0c632db 2194
93736624
AK		 2195 switch (ioctl) {
2196 case KVM_S390_INTERRUPT: {
ba5c1e9b 2197 struct kvm_s390_interrupt s390int;
383d0b05 2198 struct kvm_s390_irq s390irq;
ba5c1e9b 2199
93736624 2200 r = -EFAULT;
ba5c1e9b 2201 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624 2202 break;
383d0b05
JF		 2203 if (s390int_to_s390irq(&s390int, &s390irq))
2204 return -EINVAL;
2205 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
93736624 2206 break;
ba5c1e9b 2207 }
b0c632db 2208 case KVM_S390_STORE_STATUS:
800c1065 2209 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 2210 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 2211 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 2212 break;
b0c632db
HC		 2213 case KVM_S390_SET_INITIAL_PSW: {
2214 psw_t psw;
2215
bc923cc9 2216 r = -EFAULT;
b0c632db 2217 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 2218 break;
2219 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2220 break;
b0c632db
HC		 2221 }
2222 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 2223 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2224 break;
14eebd91
CO		 2225 case KVM_SET_ONE_REG:
2226 case KVM_GET_ONE_REG: {
2227 struct kvm_one_reg reg;
2228 r = -EFAULT;
2229 if (copy_from_user(&reg, argp, sizeof(reg)))
2230 break;
2231 if (ioctl == KVM_SET_ONE_REG)
2232 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
2233 else
2234 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
2235 break;
2236 }
27e0393f
CO		 2237#ifdef CONFIG_KVM_S390_UCONTROL
2238 case KVM_S390_UCAS_MAP: {
2239 struct kvm_s390_ucas_mapping ucasmap;
2240
2241 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2242 r = -EFAULT;
2243 break;
2244 }
2245
2246 if (!kvm_is_ucontrol(vcpu->kvm)) {
2247 r = -EINVAL;
2248 break;
2249 }
2250
2251 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2252 ucasmap.vcpu_addr, ucasmap.length);
2253 break;
2254 }
2255 case KVM_S390_UCAS_UNMAP: {
2256 struct kvm_s390_ucas_mapping ucasmap;
2257
2258 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2259 r = -EFAULT;
2260 break;
2261 }
2262
2263 if (!kvm_is_ucontrol(vcpu->kvm)) {
2264 r = -EINVAL;
2265 break;
2266 }
2267
2268 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2269 ucasmap.length);
2270 break;
2271 }
2272#endif
ccc7910f 2273 case KVM_S390_VCPU_FAULT: {
527e30b4 2274 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 2275 break;
2276 }
d6712df9
CH		 2277 case KVM_ENABLE_CAP:
2278 {
2279 struct kvm_enable_cap cap;
2280 r = -EFAULT;
2281 if (copy_from_user(&cap, argp, sizeof(cap)))
2282 break;
2283 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2284 break;
2285 }
b0c632db 2286 default:
3e6afcf1 2287 r = -ENOTTY;
b0c632db 2288 }
bc923cc9 2289 return r;
b0c632db
HC		 2290}
2291
5b1c1493
CO		 2292int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2293{
2294#ifdef CONFIG_KVM_S390_UCONTROL
2295 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2296 && (kvm_is_ucontrol(vcpu->kvm))) {
2297 vmf->page = virt_to_page(vcpu->arch.sie_block);
2298 get_page(vmf->page);
2299 return 0;
2300 }
2301#endif
2302 return VM_FAULT_SIGBUS;
2303}
2304
5587027c
AK		 2305int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2306 unsigned long npages)
db3fe4eb
TY		 2307{
2308 return 0;
2309}
2310
b0c632db 2311/* Section: memory related */
f7784b8e
MT		 2312int kvm_arch_prepare_memory_region(struct kvm *kvm,
2313 struct kvm_memory_slot *memslot,
7b6195a9
TY		 2314 struct kvm_userspace_memory_region *mem,
2315 enum kvm_mr_change change)
b0c632db 2316{
dd2887e7
NW		 2317 /* A few sanity checks. We can have memory slots which have to be
2318 located/ended at a segment boundary (1MB). The memory in userland is
2319 ok to be fragmented into various different vmas. It is okay to mmap()
2320 and munmap() stuff in this slot after doing this call at any time */
b0c632db 2321
598841ca 2322 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 2323 return -EINVAL;
2324
598841ca 2325 if (mem->memory_size & 0xffffful)
b0c632db
HC		 2326 return -EINVAL;
2327
f7784b8e
MT		 2328 return 0;
2329}
2330
2331void kvm_arch_commit_memory_region(struct kvm *kvm,
2332 struct kvm_userspace_memory_region *mem,
8482644a
TY		 2333 const struct kvm_memory_slot *old,
2334 enum kvm_mr_change change)
f7784b8e 2335{
f7850c92 2336 int rc;
f7784b8e 2337
2cef4deb
CB		 2338 /* If the basics of the memslot do not change, we do not want
2339 * to update the gmap. Every update causes several unnecessary
2340 * segment translation exceptions. This is usually handled just
2341 * fine by the normal fault handler + gmap, but it will also
2342 * cause faults on the prefix page of running guest CPUs.
2343 */
2344 if (old->userspace_addr == mem->userspace_addr &&
2345 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2346 old->npages * PAGE_SIZE == mem->memory_size)
2347 return;
598841ca
CO		 2348
2349 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2350 mem->guest_phys_addr, mem->memory_size);
2351 if (rc)
f7850c92 2352 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
598841ca 2353 return;
b0c632db
HC		 2354}
2355
b0c632db
HC		 2356static int __init kvm_s390_init(void)
2357{
9d8d5786 2358 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 2359}
2360
2361static void __exit kvm_s390_exit(void)
2362{
2363 kvm_exit();
2364}
2365
2366module_init(kvm_s390_init);
2367module_exit(kvm_s390_exit);
566af940
CH		 2368
2369/*
2370 * Enable autoloading of the kvm module.
2371 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2372 * since x86 takes a different approach.
2373 */
2374#include <linux/miscdevice.h>
2375MODULE_ALIAS_MISCDEV(KVM_MINOR);
2376MODULE_ALIAS("devname:kvm");
Ubuntu Jammy Linux kernel mirror