]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
s390/sclp: Add hmfai field
[mirror_ubuntu-hirsute-kernel.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
24#include <linux/module.h>
a374e892 25#include <linux/random.h>
b0c632db 26#include <linux/slab.h>
ba5c1e9b 27#include <linux/timer.h>
41408c28 28#include <linux/vmalloc.h>
cbb870c8 29#include <asm/asm-offsets.h>
b0c632db 30#include <asm/lowcore.h>
fdf03650 31#include <asm/etr.h>
b0c632db 32#include <asm/pgtable.h>
1e133ab2 33#include <asm/gmap.h>
f5daba1d 34#include <asm/nmi.h>
a0616cde 35#include <asm/switch_to.h>
6d3da241 36#include <asm/isc.h>
1526bf9c 37#include <asm/sclp.h>
8f2abe6a 38#include "kvm-s390.h"
b0c632db
HC		 39#include "gaccess.h"
40
ea2cdd27
DH		 41#define KMSG_COMPONENT "kvm-s390"
42#undef pr_fmt
43#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
44
5786fffa
CH		 45#define CREATE_TRACE_POINTS
46#include "trace.h"
ade38c31 47#include "trace-s390.h"
5786fffa 48
41408c28 49#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
816c7667
JF		 50#define LOCAL_IRQS 32
51#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
52 (KVM_MAX_VCPUS + LOCAL_IRQS))
41408c28 53
b0c632db
HC		 54#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
55
56struct kvm_stats_debugfs_item debugfs_entries[] = {
57 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 58 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 59 { "exit_validity", VCPU_STAT(exit_validity) },
60 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
61 { "exit_external_request", VCPU_STAT(exit_external_request) },
62 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b
CO		 63 { "exit_instruction", VCPU_STAT(exit_instruction) },
64 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
65 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
f7819512 66 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
62bea5bf 67 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
ce2e4f0b 68 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 69 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 70 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 71 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
72 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 73 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 74 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 75 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
76 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
77 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
78 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
79 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
80 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
81 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 82 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 83 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
84 { "instruction_spx", VCPU_STAT(instruction_spx) },
85 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
86 { "instruction_stap", VCPU_STAT(instruction_stap) },
87 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 88 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 89 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
90 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 91 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 92 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
93 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 94 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
5288fbf0 95 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 96 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 97 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 98 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 99 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
100 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 101 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 102 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
103 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 104 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 105 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
106 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
107 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 108 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
109 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
110 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 111 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 112 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 113 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
175a5c9e
CB		 114 { "diagnose_258", VCPU_STAT(diagnose_258) },
115 { "diagnose_308", VCPU_STAT(diagnose_308) },
116 { "diagnose_500", VCPU_STAT(diagnose_500) },
b0c632db
HC		 117 { NULL }
118};
119
9d8d5786
MM		 120/* upper facilities limit for kvm */
121unsigned long kvm_s390_fac_list_mask[] = {
ed8dda0b
AY		 122 0xffe6ffffffffffffUL,
123 0x005effffffffffffUL,
9d8d5786 124};
b0c632db 125
9d8d5786 126unsigned long kvm_s390_fac_list_mask_size(void)
78c4b59f 127{
9d8d5786
MM		 128 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
129 return ARRAY_SIZE(kvm_s390_fac_list_mask);
78c4b59f
MM		 130}
131
9d8d5786 132static struct gmap_notifier gmap_notifier;
78f26131 133debug_info_t *kvm_s390_dbf;
9d8d5786 134
b0c632db 135/* Section: not file related */
13a34e06 136int kvm_arch_hardware_enable(void)
b0c632db
HC		 137{
138 /* every s390 is virtualization enabled ;-) */
10474ae8 139 return 0;
b0c632db
HC		 140}
141
2c70fe44
CB		 142static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
143
fdf03650
FZ		 144/*
145 * This callback is executed during stop_machine(). All CPUs are therefore
146 * temporarily stopped. In order not to change guest behavior, we have to
147 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
148 * so a CPU won't be stopped while calculating with the epoch.
149 */
150static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
151 void *v)
152{
153 struct kvm *kvm;
154 struct kvm_vcpu *vcpu;
155 int i;
156 unsigned long long *delta = v;
157
158 list_for_each_entry(kvm, &vm_list, vm_list) {
159 kvm->arch.epoch -= *delta;
160 kvm_for_each_vcpu(i, vcpu, kvm) {
161 vcpu->arch.sie_block->epoch -= *delta;
db0758b2
DH		 162 if (vcpu->arch.cputm_enabled)
163 vcpu->arch.cputm_start += *delta;
fdf03650
FZ		 164 }
165 }
166 return NOTIFY_OK;
167}
168
169static struct notifier_block kvm_clock_notifier = {
170 .notifier_call = kvm_clock_sync,
171};
172
b0c632db
HC		 173int kvm_arch_hardware_setup(void)
174{
2c70fe44
CB		 175 gmap_notifier.notifier_call = kvm_gmap_notifier;
176 gmap_register_ipte_notifier(&gmap_notifier);
fdf03650
FZ		 177 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
178 &kvm_clock_notifier);
b0c632db
HC		 179 return 0;
180}
181
182void kvm_arch_hardware_unsetup(void)
183{
2c70fe44 184 gmap_unregister_ipte_notifier(&gmap_notifier);
fdf03650
FZ		 185 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
186 &kvm_clock_notifier);
b0c632db
HC		 187}
188
b0c632db
HC		 189int kvm_arch_init(void *opaque)
190{
78f26131
CB		 191 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
192 if (!kvm_s390_dbf)
193 return -ENOMEM;
194
195 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
196 debug_unregister(kvm_s390_dbf);
197 return -ENOMEM;
198 }
199
84877d93
CH		 200 /* Register floating interrupt controller interface. */
201 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 202}
203
78f26131
CB		 204void kvm_arch_exit(void)
205{
206 debug_unregister(kvm_s390_dbf);
207}
208
b0c632db
HC		 209/* Section: device related */
210long kvm_arch_dev_ioctl(struct file *filp,
211 unsigned int ioctl, unsigned long arg)
212{
213 if (ioctl == KVM_S390_ENABLE_SIE)
214 return s390_enable_sie();
215 return -EINVAL;
216}
217
784aa3d7 218int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 219{
d7b0b5eb
CO		 220 int r;
221
2bd0ac4e 222 switch (ext) {
d7b0b5eb 223 case KVM_CAP_S390_PSW:
b6cf8788 224 case KVM_CAP_S390_GMAP:
52e16b18 225 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 226#ifdef CONFIG_KVM_S390_UCONTROL
227 case KVM_CAP_S390_UCONTROL:
228#endif
3c038e6b 229 case KVM_CAP_ASYNC_PF:
60b413c9 230 case KVM_CAP_SYNC_REGS:
14eebd91 231 case KVM_CAP_ONE_REG:
d6712df9 232 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 233 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 234 case KVM_CAP_IOEVENTFD:
c05c4186 235 case KVM_CAP_DEVICE_CTRL:
d938dc55 236 case KVM_CAP_ENABLE_CAP_VM:
78599d90 237 case KVM_CAP_S390_IRQCHIP:
f2061656 238 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 239 case KVM_CAP_MP_STATE:
47b43c52 240 case KVM_CAP_S390_INJECT_IRQ:
2444b352 241 case KVM_CAP_S390_USER_SIGP:
e44fc8c9 242 case KVM_CAP_S390_USER_STSI:
30ee2a98 243 case KVM_CAP_S390_SKEYS:
816c7667 244 case KVM_CAP_S390_IRQ_STATE:
d7b0b5eb
CO		 245 r = 1;
246 break;
41408c28
TH		 247 case KVM_CAP_S390_MEM_OP:
248 r = MEM_OP_MAX_SIZE;
249 break;
e726b1bd
CB		 250 case KVM_CAP_NR_VCPUS:
251 case KVM_CAP_MAX_VCPUS:
fe0edcb7
ED		 252 r = sclp.has_esca ? KVM_S390_ESCA_CPU_SLOTS
253 : KVM_S390_BSCA_CPU_SLOTS;
e726b1bd 254 break;
e1e2e605
NW		 255 case KVM_CAP_NR_MEMSLOTS:
256 r = KVM_USER_MEM_SLOTS;
257 break;
1526bf9c 258 case KVM_CAP_S390_COW:
abf09bed 259 r = MACHINE_HAS_ESOP;
1526bf9c 260 break;
68c55750
EF		 261 case KVM_CAP_S390_VECTOR_REGISTERS:
262 r = MACHINE_HAS_VX;
263 break;
c6e5f166
FZ		 264 case KVM_CAP_S390_RI:
265 r = test_facility(64);
266 break;
2bd0ac4e 267 default:
d7b0b5eb 268 r = 0;
2bd0ac4e 269 }
d7b0b5eb 270 return r;
b0c632db
HC		 271}
272
15f36ebd
JH		 273static void kvm_s390_sync_dirty_log(struct kvm *kvm,
274 struct kvm_memory_slot *memslot)
275{
276 gfn_t cur_gfn, last_gfn;
277 unsigned long address;
278 struct gmap *gmap = kvm->arch.gmap;
279
15f36ebd
JH		 280 /* Loop over all guest pages */
281 last_gfn = memslot->base_gfn + memslot->npages;
282 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
283 address = gfn_to_hva_memslot(memslot, cur_gfn);
284
1e133ab2 285 if (test_and_clear_guest_dirty(gmap->mm, address))
15f36ebd 286 mark_page_dirty(kvm, cur_gfn);
1763f8d0
CB		 287 if (fatal_signal_pending(current))
288 return;
70c88a00 289 cond_resched();
15f36ebd 290 }
15f36ebd
JH		 291}
292
b0c632db 293/* Section: vm related */
a6e2f683
ED		 294static void sca_del_vcpu(struct kvm_vcpu *vcpu);
295
b0c632db
HC		 296/*
297 * Get (and clear) the dirty memory log for a memory slot.
298 */
299int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
300 struct kvm_dirty_log *log)
301{
15f36ebd
JH		 302 int r;
303 unsigned long n;
9f6b8029 304 struct kvm_memslots *slots;
15f36ebd
JH		 305 struct kvm_memory_slot *memslot;
306 int is_dirty = 0;
307
308 mutex_lock(&kvm->slots_lock);
309
310 r = -EINVAL;
311 if (log->slot >= KVM_USER_MEM_SLOTS)
312 goto out;
313
9f6b8029
PB		 314 slots = kvm_memslots(kvm);
315 memslot = id_to_memslot(slots, log->slot);
15f36ebd
JH		 316 r = -ENOENT;
317 if (!memslot->dirty_bitmap)
318 goto out;
319
320 kvm_s390_sync_dirty_log(kvm, memslot);
321 r = kvm_get_dirty_log(kvm, log, &is_dirty);
322 if (r)
323 goto out;
324
325 /* Clear the dirty log */
326 if (is_dirty) {
327 n = kvm_dirty_bitmap_bytes(memslot);
328 memset(memslot->dirty_bitmap, 0, n);
329 }
330 r = 0;
331out:
332 mutex_unlock(&kvm->slots_lock);
333 return r;
b0c632db
HC		 334}
335
d938dc55
CH		 336static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
337{
338 int r;
339
340 if (cap->flags)
341 return -EINVAL;
342
343 switch (cap->cap) {
84223598 344 case KVM_CAP_S390_IRQCHIP:
c92ea7b9 345 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
84223598
CH		 346 kvm->arch.use_irqchip = 1;
347 r = 0;
348 break;
2444b352 349 case KVM_CAP_S390_USER_SIGP:
c92ea7b9 350 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
2444b352
DH		 351 kvm->arch.user_sigp = 1;
352 r = 0;
353 break;
68c55750 354 case KVM_CAP_S390_VECTOR_REGISTERS:
5967c17b
DH		 355 mutex_lock(&kvm->lock);
356 if (atomic_read(&kvm->online_vcpus)) {
357 r = -EBUSY;
358 } else if (MACHINE_HAS_VX) {
c54f0d6a
DH		 359 set_kvm_facility(kvm->arch.model.fac_mask, 129);
360 set_kvm_facility(kvm->arch.model.fac_list, 129);
18280d8b
MM		 361 r = 0;
362 } else
363 r = -EINVAL;
5967c17b 364 mutex_unlock(&kvm->lock);
c92ea7b9
CB		 365 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
366 r ? "(not available)" : "(success)");
68c55750 367 break;
c6e5f166
FZ		 368 case KVM_CAP_S390_RI:
369 r = -EINVAL;
370 mutex_lock(&kvm->lock);
371 if (atomic_read(&kvm->online_vcpus)) {
372 r = -EBUSY;
373 } else if (test_facility(64)) {
c54f0d6a
DH		 374 set_kvm_facility(kvm->arch.model.fac_mask, 64);
375 set_kvm_facility(kvm->arch.model.fac_list, 64);
c6e5f166
FZ		 376 r = 0;
377 }
378 mutex_unlock(&kvm->lock);
379 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
380 r ? "(not available)" : "(success)");
381 break;
e44fc8c9 382 case KVM_CAP_S390_USER_STSI:
c92ea7b9 383 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
e44fc8c9
ET		 384 kvm->arch.user_stsi = 1;
385 r = 0;
386 break;
d938dc55
CH		 387 default:
388 r = -EINVAL;
389 break;
390 }
391 return r;
392}
393
8c0a7ce6
DD		 394static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
395{
396 int ret;
397
398 switch (attr->attr) {
399 case KVM_S390_VM_MEM_LIMIT_SIZE:
400 ret = 0;
c92ea7b9 401 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
a3a92c31
DD		 402 kvm->arch.mem_limit);
403 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
8c0a7ce6
DD		 404 ret = -EFAULT;
405 break;
406 default:
407 ret = -ENXIO;
408 break;
409 }
410 return ret;
411}
412
413static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 414{
415 int ret;
416 unsigned int idx;
417 switch (attr->attr) {
418 case KVM_S390_VM_MEM_ENABLE_CMMA:
e6db1d61
DD		 419 /* enable CMMA only for z10 and later (EDAT_1) */
420 ret = -EINVAL;
421 if (!MACHINE_IS_LPAR || !MACHINE_HAS_EDAT1)
422 break;
423
4f718eab 424 ret = -EBUSY;
c92ea7b9 425 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
4f718eab
DD		 426 mutex_lock(&kvm->lock);
427 if (atomic_read(&kvm->online_vcpus) == 0) {
428 kvm->arch.use_cmma = 1;
429 ret = 0;
430 }
431 mutex_unlock(&kvm->lock);
432 break;
433 case KVM_S390_VM_MEM_CLR_CMMA:
c3489155
DD		 434 ret = -EINVAL;
435 if (!kvm->arch.use_cmma)
436 break;
437
c92ea7b9 438 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
4f718eab
DD		 439 mutex_lock(&kvm->lock);
440 idx = srcu_read_lock(&kvm->srcu);
a13cff31 441 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 442 srcu_read_unlock(&kvm->srcu, idx);
443 mutex_unlock(&kvm->lock);
444 ret = 0;
445 break;
8c0a7ce6
DD		 446 case KVM_S390_VM_MEM_LIMIT_SIZE: {
447 unsigned long new_limit;
448
449 if (kvm_is_ucontrol(kvm))
450 return -EINVAL;
451
452 if (get_user(new_limit, (u64 __user *)attr->addr))
453 return -EFAULT;
454
a3a92c31
DD		 455 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
456 new_limit > kvm->arch.mem_limit)
8c0a7ce6
DD		 457 return -E2BIG;
458
a3a92c31
DD		 459 if (!new_limit)
460 return -EINVAL;
461
462 /* gmap_alloc takes last usable address */
463 if (new_limit != KVM_S390_NO_MEM_LIMIT)
464 new_limit -= 1;
465
8c0a7ce6
DD		 466 ret = -EBUSY;
467 mutex_lock(&kvm->lock);
468 if (atomic_read(&kvm->online_vcpus) == 0) {
469 /* gmap_alloc will round the limit up */
470 struct gmap *new = gmap_alloc(current->mm, new_limit);
471
472 if (!new) {
473 ret = -ENOMEM;
474 } else {
475 gmap_free(kvm->arch.gmap);
476 new->private = kvm;
477 kvm->arch.gmap = new;
478 ret = 0;
479 }
480 }
481 mutex_unlock(&kvm->lock);
a3a92c31
DD		 482 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
483 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
484 (void *) kvm->arch.gmap->asce);
8c0a7ce6
DD		 485 break;
486 }
4f718eab
DD		 487 default:
488 ret = -ENXIO;
489 break;
490 }
491 return ret;
492}
493
a374e892
TK		 494static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
495
496static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
497{
498 struct kvm_vcpu *vcpu;
499 int i;
500
9d8d5786 501 if (!test_kvm_facility(kvm, 76))
a374e892
TK		 502 return -EINVAL;
503
504 mutex_lock(&kvm->lock);
505 switch (attr->attr) {
506 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
507 get_random_bytes(
508 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
509 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
510 kvm->arch.crypto.aes_kw = 1;
c92ea7b9 511 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
a374e892
TK		 512 break;
513 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
514 get_random_bytes(
515 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
516 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
517 kvm->arch.crypto.dea_kw = 1;
c92ea7b9 518 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
a374e892
TK		 519 break;
520 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
521 kvm->arch.crypto.aes_kw = 0;
522 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
523 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
c92ea7b9 524 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
a374e892
TK		 525 break;
526 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
527 kvm->arch.crypto.dea_kw = 0;
528 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
529 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
c92ea7b9 530 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
a374e892
TK		 531 break;
532 default:
533 mutex_unlock(&kvm->lock);
534 return -ENXIO;
535 }
536
537 kvm_for_each_vcpu(i, vcpu, kvm) {
538 kvm_s390_vcpu_crypto_setup(vcpu);
539 exit_sie(vcpu);
540 }
541 mutex_unlock(&kvm->lock);
542 return 0;
543}
544
72f25020
JH		 545static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
546{
547 u8 gtod_high;
548
549 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
550 sizeof(gtod_high)))
551 return -EFAULT;
552
553 if (gtod_high != 0)
554 return -EINVAL;
58c383c6 555 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
72f25020
JH		 556
557 return 0;
558}
559
560static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
561{
5a3d883a 562 u64 gtod;
72f25020
JH		 563
564 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
565 return -EFAULT;
566
25ed1675 567 kvm_s390_set_tod_clock(kvm, gtod);
58c383c6 568 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
72f25020
JH		 569 return 0;
570}
571
572static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
573{
574 int ret;
575
576 if (attr->flags)
577 return -EINVAL;
578
579 switch (attr->attr) {
580 case KVM_S390_VM_TOD_HIGH:
581 ret = kvm_s390_set_tod_high(kvm, attr);
582 break;
583 case KVM_S390_VM_TOD_LOW:
584 ret = kvm_s390_set_tod_low(kvm, attr);
585 break;
586 default:
587 ret = -ENXIO;
588 break;
589 }
590 return ret;
591}
592
593static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
594{
595 u8 gtod_high = 0;
596
597 if (copy_to_user((void __user *)attr->addr, &gtod_high,
598 sizeof(gtod_high)))
599 return -EFAULT;
58c383c6 600 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
72f25020
JH		 601
602 return 0;
603}
604
605static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
606{
5a3d883a 607 u64 gtod;
72f25020 608
60417fcc 609 gtod = kvm_s390_get_tod_clock_fast(kvm);
72f25020
JH		 610 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
611 return -EFAULT;
58c383c6 612 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
72f25020
JH		 613
614 return 0;
615}
616
617static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
618{
619 int ret;
620
621 if (attr->flags)
622 return -EINVAL;
623
624 switch (attr->attr) {
625 case KVM_S390_VM_TOD_HIGH:
626 ret = kvm_s390_get_tod_high(kvm, attr);
627 break;
628 case KVM_S390_VM_TOD_LOW:
629 ret = kvm_s390_get_tod_low(kvm, attr);
630 break;
631 default:
632 ret = -ENXIO;
633 break;
634 }
635 return ret;
636}
637
658b6eda
MM		 638static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
639{
640 struct kvm_s390_vm_cpu_processor *proc;
053dd230 641 u16 lowest_ibc, unblocked_ibc;
658b6eda
MM		 642 int ret = 0;
643
644 mutex_lock(&kvm->lock);
645 if (atomic_read(&kvm->online_vcpus)) {
646 ret = -EBUSY;
647 goto out;
648 }
649 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
650 if (!proc) {
651 ret = -ENOMEM;
652 goto out;
653 }
654 if (!copy_from_user(proc, (void __user *)attr->addr,
655 sizeof(*proc))) {
9bb0ec09 656 kvm->arch.model.cpuid = proc->cpuid;
053dd230
DH		 657 lowest_ibc = sclp.ibc >> 16 & 0xfff;
658 unblocked_ibc = sclp.ibc & 0xfff;
659 if (lowest_ibc) {
660 if (proc->ibc > unblocked_ibc)
661 kvm->arch.model.ibc = unblocked_ibc;
662 else if (proc->ibc < lowest_ibc)
663 kvm->arch.model.ibc = lowest_ibc;
664 else
665 kvm->arch.model.ibc = proc->ibc;
666 }
c54f0d6a 667 memcpy(kvm->arch.model.fac_list, proc->fac_list,
658b6eda
MM		 668 S390_ARCH_FAC_LIST_SIZE_BYTE);
669 } else
670 ret = -EFAULT;
671 kfree(proc);
672out:
673 mutex_unlock(&kvm->lock);
674 return ret;
675}
676
677static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
678{
679 int ret = -ENXIO;
680
681 switch (attr->attr) {
682 case KVM_S390_VM_CPU_PROCESSOR:
683 ret = kvm_s390_set_processor(kvm, attr);
684 break;
685 }
686 return ret;
687}
688
689static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
690{
691 struct kvm_s390_vm_cpu_processor *proc;
692 int ret = 0;
693
694 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
695 if (!proc) {
696 ret = -ENOMEM;
697 goto out;
698 }
9bb0ec09 699 proc->cpuid = kvm->arch.model.cpuid;
658b6eda 700 proc->ibc = kvm->arch.model.ibc;
c54f0d6a
DH		 701 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
702 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 703 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
704 ret = -EFAULT;
705 kfree(proc);
706out:
707 return ret;
708}
709
710static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
711{
712 struct kvm_s390_vm_cpu_machine *mach;
713 int ret = 0;
714
715 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
716 if (!mach) {
717 ret = -ENOMEM;
718 goto out;
719 }
720 get_cpu_id((struct cpuid *) &mach->cpuid);
37c5f6c8 721 mach->ibc = sclp.ibc;
c54f0d6a 722 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
981467c9 723 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 724 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
94422ee8 725 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 726 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
727 ret = -EFAULT;
728 kfree(mach);
729out:
730 return ret;
731}
732
733static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
734{
735 int ret = -ENXIO;
736
737 switch (attr->attr) {
738 case KVM_S390_VM_CPU_PROCESSOR:
739 ret = kvm_s390_get_processor(kvm, attr);
740 break;
741 case KVM_S390_VM_CPU_MACHINE:
742 ret = kvm_s390_get_machine(kvm, attr);
743 break;
744 }
745 return ret;
746}
747
f2061656
DD		 748static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
749{
750 int ret;
751
752 switch (attr->group) {
4f718eab 753 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 754 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 755 break;
72f25020
JH		 756 case KVM_S390_VM_TOD:
757 ret = kvm_s390_set_tod(kvm, attr);
758 break;
658b6eda
MM		 759 case KVM_S390_VM_CPU_MODEL:
760 ret = kvm_s390_set_cpu_model(kvm, attr);
761 break;
a374e892
TK		 762 case KVM_S390_VM_CRYPTO:
763 ret = kvm_s390_vm_set_crypto(kvm, attr);
764 break;
f2061656
DD		 765 default:
766 ret = -ENXIO;
767 break;
768 }
769
770 return ret;
771}
772
773static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
774{
8c0a7ce6
DD		 775 int ret;
776
777 switch (attr->group) {
778 case KVM_S390_VM_MEM_CTRL:
779 ret = kvm_s390_get_mem_control(kvm, attr);
780 break;
72f25020
JH		 781 case KVM_S390_VM_TOD:
782 ret = kvm_s390_get_tod(kvm, attr);
783 break;
658b6eda
MM		 784 case KVM_S390_VM_CPU_MODEL:
785 ret = kvm_s390_get_cpu_model(kvm, attr);
786 break;
8c0a7ce6
DD		 787 default:
788 ret = -ENXIO;
789 break;
790 }
791
792 return ret;
f2061656
DD		 793}
794
795static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
796{
797 int ret;
798
799 switch (attr->group) {
4f718eab
DD		 800 case KVM_S390_VM_MEM_CTRL:
801 switch (attr->attr) {
802 case KVM_S390_VM_MEM_ENABLE_CMMA:
803 case KVM_S390_VM_MEM_CLR_CMMA:
8c0a7ce6 804 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 805 ret = 0;
806 break;
807 default:
808 ret = -ENXIO;
809 break;
810 }
811 break;
72f25020
JH		 812 case KVM_S390_VM_TOD:
813 switch (attr->attr) {
814 case KVM_S390_VM_TOD_LOW:
815 case KVM_S390_VM_TOD_HIGH:
816 ret = 0;
817 break;
818 default:
819 ret = -ENXIO;
820 break;
821 }
822 break;
658b6eda
MM		 823 case KVM_S390_VM_CPU_MODEL:
824 switch (attr->attr) {
825 case KVM_S390_VM_CPU_PROCESSOR:
826 case KVM_S390_VM_CPU_MACHINE:
827 ret = 0;
828 break;
829 default:
830 ret = -ENXIO;
831 break;
832 }
833 break;
a374e892
TK		 834 case KVM_S390_VM_CRYPTO:
835 switch (attr->attr) {
836 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
837 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
838 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
839 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
840 ret = 0;
841 break;
842 default:
843 ret = -ENXIO;
844 break;
845 }
846 break;
f2061656
DD		 847 default:
848 ret = -ENXIO;
849 break;
850 }
851
852 return ret;
853}
854
30ee2a98
JH		 855static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
856{
857 uint8_t *keys;
858 uint64_t hva;
859 unsigned long curkey;
860 int i, r = 0;
861
862 if (args->flags != 0)
863 return -EINVAL;
864
865 /* Is this guest using storage keys? */
866 if (!mm_use_skey(current->mm))
867 return KVM_S390_GET_SKEYS_NONE;
868
869 /* Enforce sane limit on memory allocation */
870 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
871 return -EINVAL;
872
873 keys = kmalloc_array(args->count, sizeof(uint8_t),
874 GFP_KERNEL | __GFP_NOWARN);
875 if (!keys)
876 keys = vmalloc(sizeof(uint8_t) * args->count);
877 if (!keys)
878 return -ENOMEM;
879
880 for (i = 0; i < args->count; i++) {
881 hva = gfn_to_hva(kvm, args->start_gfn + i);
882 if (kvm_is_error_hva(hva)) {
883 r = -EFAULT;
884 goto out;
885 }
886
887 curkey = get_guest_storage_key(current->mm, hva);
888 if (IS_ERR_VALUE(curkey)) {
889 r = curkey;
890 goto out;
891 }
892 keys[i] = curkey;
893 }
894
895 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
896 sizeof(uint8_t) * args->count);
897 if (r)
898 r = -EFAULT;
899out:
900 kvfree(keys);
901 return r;
902}
903
904static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
905{
906 uint8_t *keys;
907 uint64_t hva;
908 int i, r = 0;
909
910 if (args->flags != 0)
911 return -EINVAL;
912
913 /* Enforce sane limit on memory allocation */
914 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
915 return -EINVAL;
916
917 keys = kmalloc_array(args->count, sizeof(uint8_t),
918 GFP_KERNEL | __GFP_NOWARN);
919 if (!keys)
920 keys = vmalloc(sizeof(uint8_t) * args->count);
921 if (!keys)
922 return -ENOMEM;
923
924 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
925 sizeof(uint8_t) * args->count);
926 if (r) {
927 r = -EFAULT;
928 goto out;
929 }
930
931 /* Enable storage key handling for the guest */
14d4a425
DD		 932 r = s390_enable_skey();
933 if (r)
934 goto out;
30ee2a98
JH		 935
936 for (i = 0; i < args->count; i++) {
937 hva = gfn_to_hva(kvm, args->start_gfn + i);
938 if (kvm_is_error_hva(hva)) {
939 r = -EFAULT;
940 goto out;
941 }
942
943 /* Lowest order bit is reserved */
944 if (keys[i] & 0x01) {
945 r = -EINVAL;
946 goto out;
947 }
948
949 r = set_guest_storage_key(current->mm, hva,
950 (unsigned long)keys[i], 0);
951 if (r)
952 goto out;
953 }
954out:
955 kvfree(keys);
956 return r;
957}
958
b0c632db
HC		 959long kvm_arch_vm_ioctl(struct file *filp,
960 unsigned int ioctl, unsigned long arg)
961{
962 struct kvm *kvm = filp->private_data;
963 void __user *argp = (void __user *)arg;
f2061656 964 struct kvm_device_attr attr;
b0c632db
HC		 965 int r;
966
967 switch (ioctl) {
ba5c1e9b
CO		 968 case KVM_S390_INTERRUPT: {
969 struct kvm_s390_interrupt s390int;
970
971 r = -EFAULT;
972 if (copy_from_user(&s390int, argp, sizeof(s390int)))
973 break;
974 r = kvm_s390_inject_vm(kvm, &s390int);
975 break;
976 }
d938dc55
CH		 977 case KVM_ENABLE_CAP: {
978 struct kvm_enable_cap cap;
979 r = -EFAULT;
980 if (copy_from_user(&cap, argp, sizeof(cap)))
981 break;
982 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
983 break;
984 }
84223598
CH		 985 case KVM_CREATE_IRQCHIP: {
986 struct kvm_irq_routing_entry routing;
987
988 r = -EINVAL;
989 if (kvm->arch.use_irqchip) {
990 /* Set up dummy routing. */
991 memset(&routing, 0, sizeof(routing));
152b2839 992 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
84223598
CH		 993 }
994 break;
995 }
f2061656
DD		 996 case KVM_SET_DEVICE_ATTR: {
997 r = -EFAULT;
998 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
999 break;
1000 r = kvm_s390_vm_set_attr(kvm, &attr);
1001 break;
1002 }
1003 case KVM_GET_DEVICE_ATTR: {
1004 r = -EFAULT;
1005 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1006 break;
1007 r = kvm_s390_vm_get_attr(kvm, &attr);
1008 break;
1009 }
1010 case KVM_HAS_DEVICE_ATTR: {
1011 r = -EFAULT;
1012 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1013 break;
1014 r = kvm_s390_vm_has_attr(kvm, &attr);
1015 break;
1016 }
30ee2a98
JH		 1017 case KVM_S390_GET_SKEYS: {
1018 struct kvm_s390_skeys args;
1019
1020 r = -EFAULT;
1021 if (copy_from_user(&args, argp,
1022 sizeof(struct kvm_s390_skeys)))
1023 break;
1024 r = kvm_s390_get_skeys(kvm, &args);
1025 break;
1026 }
1027 case KVM_S390_SET_SKEYS: {
1028 struct kvm_s390_skeys args;
1029
1030 r = -EFAULT;
1031 if (copy_from_user(&args, argp,
1032 sizeof(struct kvm_s390_skeys)))
1033 break;
1034 r = kvm_s390_set_skeys(kvm, &args);
1035 break;
1036 }
b0c632db 1037 default:
367e1319 1038 r = -ENOTTY;
b0c632db
HC		 1039 }
1040
1041 return r;
1042}
1043
45c9b47c
TK		 1044static int kvm_s390_query_ap_config(u8 *config)
1045{
1046 u32 fcn_code = 0x04000000UL;
86044c8c 1047 u32 cc = 0;
45c9b47c 1048
86044c8c 1049 memset(config, 0, 128);
45c9b47c
TK		 1050 asm volatile(
1051 "lgr 0,%1\n"
1052 "lgr 2,%2\n"
1053 ".long 0xb2af0000\n" /* PQAP(QCI) */
86044c8c 1054 "0: ipm %0\n"
45c9b47c 1055 "srl %0,28\n"
86044c8c
CB		 1056 "1:\n"
1057 EX_TABLE(0b, 1b)
1058 : "+r" (cc)
45c9b47c
TK		 1059 : "r" (fcn_code), "r" (config)
1060 : "cc", "0", "2", "memory"
1061 );
1062
1063 return cc;
1064}
1065
1066static int kvm_s390_apxa_installed(void)
1067{
1068 u8 config[128];
1069 int cc;
1070
a6aacc3f 1071 if (test_facility(12)) {
45c9b47c
TK		 1072 cc = kvm_s390_query_ap_config(config);
1073
1074 if (cc)
1075 pr_err("PQAP(QCI) failed with cc=%d", cc);
1076 else
1077 return config[0] & 0x40;
1078 }
1079
1080 return 0;
1081}
1082
1083static void kvm_s390_set_crycb_format(struct kvm *kvm)
1084{
1085 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1086
1087 if (kvm_s390_apxa_installed())
1088 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1089 else
1090 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1091}
1092
9bb0ec09 1093static u64 kvm_s390_get_initial_cpuid(void)
9d8d5786 1094{
9bb0ec09
DH		 1095 struct cpuid cpuid;
1096
1097 get_cpu_id(&cpuid);
1098 cpuid.version = 0xff;
1099 return *((u64 *) &cpuid);
9d8d5786
MM		 1100}
1101
c54f0d6a 1102static void kvm_s390_crypto_init(struct kvm *kvm)
5102ee87 1103{
9d8d5786 1104 if (!test_kvm_facility(kvm, 76))
c54f0d6a 1105 return;
5102ee87 1106
c54f0d6a 1107 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
45c9b47c 1108 kvm_s390_set_crycb_format(kvm);
5102ee87 1109
ed6f76b4
TK		 1110 /* Enable AES/DEA protected key functions by default */
1111 kvm->arch.crypto.aes_kw = 1;
1112 kvm->arch.crypto.dea_kw = 1;
1113 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1114 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1115 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1116 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
5102ee87
TK		 1117}
1118
7d43bafc
ED		 1119static void sca_dispose(struct kvm *kvm)
1120{
1121 if (kvm->arch.use_esca)
5e044315 1122 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
7d43bafc
ED		 1123 else
1124 free_page((unsigned long)(kvm->arch.sca));
1125 kvm->arch.sca = NULL;
1126}
1127
e08b9637 1128int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 1129{
9d8d5786 1130 int i, rc;
b0c632db 1131 char debug_name[16];
f6c137ff 1132 static unsigned long sca_offset;
b0c632db 1133
e08b9637
CO		 1134 rc = -EINVAL;
1135#ifdef CONFIG_KVM_S390_UCONTROL
1136 if (type & ~KVM_VM_S390_UCONTROL)
1137 goto out_err;
1138 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1139 goto out_err;
1140#else
1141 if (type)
1142 goto out_err;
1143#endif
1144
b0c632db
HC		 1145 rc = s390_enable_sie();
1146 if (rc)
d89f5eff 1147 goto out_err;
b0c632db 1148
b290411a
CO		 1149 rc = -ENOMEM;
1150
7d43bafc 1151 kvm->arch.use_esca = 0; /* start with basic SCA */
5e044315 1152 rwlock_init(&kvm->arch.sca_lock);
bc784cce 1153 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(GFP_KERNEL);
b0c632db 1154 if (!kvm->arch.sca)
d89f5eff 1155 goto out_err;
f6c137ff 1156 spin_lock(&kvm_lock);
c5c2c393 1157 sca_offset += 16;
bc784cce 1158 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
c5c2c393 1159 sca_offset = 0;
bc784cce
ED		 1160 kvm->arch.sca = (struct bsca_block *)
1161 ((char *) kvm->arch.sca + sca_offset);
f6c137ff 1162 spin_unlock(&kvm_lock);
b0c632db
HC		 1163
1164 sprintf(debug_name, "kvm-%u", current->pid);
1165
1cb9cf72 1166 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
b0c632db 1167 if (!kvm->arch.dbf)
40f5b735 1168 goto out_err;
b0c632db 1169
c54f0d6a
DH		 1170 kvm->arch.sie_page2 =
1171 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1172 if (!kvm->arch.sie_page2)
40f5b735 1173 goto out_err;
9d8d5786 1174
fb5bf93f 1175 /* Populate the facility mask initially. */
c54f0d6a 1176 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
94422ee8 1177 S390_ARCH_FAC_LIST_SIZE_BYTE);
9d8d5786
MM		 1178 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1179 if (i < kvm_s390_fac_list_mask_size())
c54f0d6a 1180 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
9d8d5786 1181 else
c54f0d6a 1182 kvm->arch.model.fac_mask[i] = 0UL;
9d8d5786
MM		 1183 }
1184
981467c9 1185 /* Populate the facility list initially. */
c54f0d6a
DH		 1186 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1187 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
981467c9
MM		 1188 S390_ARCH_FAC_LIST_SIZE_BYTE);
1189
9bb0ec09 1190 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
37c5f6c8 1191 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
9d8d5786 1192
c54f0d6a 1193 kvm_s390_crypto_init(kvm);
5102ee87 1194
ba5c1e9b 1195 spin_lock_init(&kvm->arch.float_int.lock);
6d3da241
JF		 1196 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1197 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
8a242234 1198 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 1199 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 1200
b0c632db 1201 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
78f26131 1202 VM_EVENT(kvm, 3, "vm created with type %lu", type);
b0c632db 1203
e08b9637
CO		 1204 if (type & KVM_VM_S390_UCONTROL) {
1205 kvm->arch.gmap = NULL;
a3a92c31 1206 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
e08b9637 1207 } else {
32e6b236
GH		 1208 if (sclp.hamax == U64_MAX)
1209 kvm->arch.mem_limit = TASK_MAX_SIZE;
1210 else
1211 kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
1212 sclp.hamax + 1);
a3a92c31 1213 kvm->arch.gmap = gmap_alloc(current->mm, kvm->arch.mem_limit - 1);
e08b9637 1214 if (!kvm->arch.gmap)
40f5b735 1215 goto out_err;
2c70fe44 1216 kvm->arch.gmap->private = kvm;
24eb3a82 1217 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 1218 }
fa6b7fe9
CH		 1219
1220 kvm->arch.css_support = 0;
84223598 1221 kvm->arch.use_irqchip = 0;
72f25020 1222 kvm->arch.epoch = 0;
fa6b7fe9 1223
8ad35755 1224 spin_lock_init(&kvm->arch.start_stop_lock);
8335713a 1225 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
8ad35755 1226
d89f5eff 1227 return 0;
40f5b735 1228out_err:
c54f0d6a 1229 free_page((unsigned long)kvm->arch.sie_page2);
598841ca 1230 debug_unregister(kvm->arch.dbf);
7d43bafc 1231 sca_dispose(kvm);
78f26131 1232 KVM_EVENT(3, "creation of vm failed: %d", rc);
d89f5eff 1233 return rc;
b0c632db
HC		 1234}
1235
d329c035
CB		 1236void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1237{
1238 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 1239 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 1240 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 1241 kvm_clear_async_pf_completion_queue(vcpu);
bc784cce 1242 if (!kvm_is_ucontrol(vcpu->kvm))
a6e2f683 1243 sca_del_vcpu(vcpu);
27e0393f
CO		 1244
1245 if (kvm_is_ucontrol(vcpu->kvm))
1246 gmap_free(vcpu->arch.gmap);
1247
e6db1d61 1248 if (vcpu->kvm->arch.use_cmma)
b31605c1 1249 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 1250 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 1251
6692cef3 1252 kvm_vcpu_uninit(vcpu);
b110feaf 1253 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 1254}
1255
1256static void kvm_free_vcpus(struct kvm *kvm)
1257{
1258 unsigned int i;
988a2cae 1259 struct kvm_vcpu *vcpu;
d329c035 1260
988a2cae
GN		 1261 kvm_for_each_vcpu(i, vcpu, kvm)
1262 kvm_arch_vcpu_destroy(vcpu);
1263
1264 mutex_lock(&kvm->lock);
1265 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1266 kvm->vcpus[i] = NULL;
1267
1268 atomic_set(&kvm->online_vcpus, 0);
1269 mutex_unlock(&kvm->lock);
d329c035
CB		 1270}
1271
b0c632db
HC		 1272void kvm_arch_destroy_vm(struct kvm *kvm)
1273{
d329c035 1274 kvm_free_vcpus(kvm);
7d43bafc 1275 sca_dispose(kvm);
d329c035 1276 debug_unregister(kvm->arch.dbf);
c54f0d6a 1277 free_page((unsigned long)kvm->arch.sie_page2);
27e0393f
CO		 1278 if (!kvm_is_ucontrol(kvm))
1279 gmap_free(kvm->arch.gmap);
841b91c5 1280 kvm_s390_destroy_adapters(kvm);
67335e63 1281 kvm_s390_clear_float_irqs(kvm);
8335713a 1282 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
b0c632db
HC		 1283}
1284
1285/* Section: vcpu related */
dafd032a
DD		 1286static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1287{
1288 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1289 if (!vcpu->arch.gmap)
1290 return -ENOMEM;
1291 vcpu->arch.gmap->private = vcpu->kvm;
1292
1293 return 0;
1294}
1295
a6e2f683
ED		 1296static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1297{
5e044315 1298 read_lock(&vcpu->kvm->arch.sca_lock);
7d43bafc
ED		 1299 if (vcpu->kvm->arch.use_esca) {
1300 struct esca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1301
7d43bafc 1302 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
10ce32d5 1303 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc
ED		 1304 } else {
1305 struct bsca_block *sca = vcpu->kvm->arch.sca;
1306
1307 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
10ce32d5 1308 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc 1309 }
5e044315 1310 read_unlock(&vcpu->kvm->arch.sca_lock);
a6e2f683
ED		 1311}
1312
eaa78f34 1313static void sca_add_vcpu(struct kvm_vcpu *vcpu)
a6e2f683 1314{
eaa78f34
DH		 1315 read_lock(&vcpu->kvm->arch.sca_lock);
1316 if (vcpu->kvm->arch.use_esca) {
1317 struct esca_block *sca = vcpu->kvm->arch.sca;
7d43bafc 1318
eaa78f34 1319 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1320 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1321 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
25508824 1322 vcpu->arch.sie_block->ecb2 |= 0x04U;
eaa78f34 1323 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
7d43bafc 1324 } else {
eaa78f34 1325 struct bsca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1326
eaa78f34 1327 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1328 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1329 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
eaa78f34 1330 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
7d43bafc 1331 }
eaa78f34 1332 read_unlock(&vcpu->kvm->arch.sca_lock);
5e044315
ED		 1333}
1334
1335/* Basic SCA to Extended SCA data copy routines */
1336static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1337{
1338 d->sda = s->sda;
1339 d->sigp_ctrl.c = s->sigp_ctrl.c;
1340 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1341}
1342
1343static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1344{
1345 int i;
1346
1347 d->ipte_control = s->ipte_control;
1348 d->mcn[0] = s->mcn;
1349 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1350 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1351}
1352
1353static int sca_switch_to_extended(struct kvm *kvm)
1354{
1355 struct bsca_block *old_sca = kvm->arch.sca;
1356 struct esca_block *new_sca;
1357 struct kvm_vcpu *vcpu;
1358 unsigned int vcpu_idx;
1359 u32 scaol, scaoh;
1360
1361 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1362 if (!new_sca)
1363 return -ENOMEM;
1364
1365 scaoh = (u32)((u64)(new_sca) >> 32);
1366 scaol = (u32)(u64)(new_sca) & ~0x3fU;
1367
1368 kvm_s390_vcpu_block_all(kvm);
1369 write_lock(&kvm->arch.sca_lock);
1370
1371 sca_copy_b_to_e(new_sca, old_sca);
1372
1373 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1374 vcpu->arch.sie_block->scaoh = scaoh;
1375 vcpu->arch.sie_block->scaol = scaol;
1376 vcpu->arch.sie_block->ecb2 |= 0x04U;
1377 }
1378 kvm->arch.sca = new_sca;
1379 kvm->arch.use_esca = 1;
1380
1381 write_unlock(&kvm->arch.sca_lock);
1382 kvm_s390_vcpu_unblock_all(kvm);
1383
1384 free_page((unsigned long)old_sca);
1385
8335713a
CB		 1386 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1387 old_sca, kvm->arch.sca);
5e044315 1388 return 0;
a6e2f683
ED		 1389}
1390
1391static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1392{
5e044315
ED		 1393 int rc;
1394
1395 if (id < KVM_S390_BSCA_CPU_SLOTS)
1396 return true;
1397 if (!sclp.has_esca)
1398 return false;
1399
1400 mutex_lock(&kvm->lock);
1401 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1402 mutex_unlock(&kvm->lock);
1403
1404 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
a6e2f683
ED		 1405}
1406
b0c632db
HC		 1407int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1408{
3c038e6b
DD		 1409 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1410 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 1411 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1412 KVM_SYNC_GPRS |
9eed0735 1413 KVM_SYNC_ACRS |
b028ee3e
DH		 1414 KVM_SYNC_CRS |
1415 KVM_SYNC_ARCH0 |
1416 KVM_SYNC_PFAULT;
c6e5f166
FZ		 1417 if (test_kvm_facility(vcpu->kvm, 64))
1418 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
f6aa6dc4
DH		 1419 /* fprs can be synchronized via vrs, even if the guest has no vx. With
1420 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1421 */
1422 if (MACHINE_HAS_VX)
68c55750 1423 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
6fd8e67d
DH		 1424 else
1425 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
dafd032a
DD		 1426
1427 if (kvm_is_ucontrol(vcpu->kvm))
1428 return __kvm_ucontrol_vcpu_init(vcpu);
1429
b0c632db
HC		 1430 return 0;
1431}
1432
db0758b2
DH		 1433/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1434static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1435{
1436 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
9c23a131 1437 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2 1438 vcpu->arch.cputm_start = get_tod_clock_fast();
9c23a131 1439 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1440}
1441
1442/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1443static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1444{
1445 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
9c23a131 1446 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1447 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1448 vcpu->arch.cputm_start = 0;
9c23a131 1449 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1450}
1451
1452/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1453static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1454{
1455 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
1456 vcpu->arch.cputm_enabled = true;
1457 __start_cpu_timer_accounting(vcpu);
1458}
1459
1460/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1461static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1462{
1463 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
1464 __stop_cpu_timer_accounting(vcpu);
1465 vcpu->arch.cputm_enabled = false;
1466}
1467
1468static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1469{
1470 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1471 __enable_cpu_timer_accounting(vcpu);
1472 preempt_enable();
1473}
1474
1475static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1476{
1477 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1478 __disable_cpu_timer_accounting(vcpu);
1479 preempt_enable();
1480}
1481
4287f247
DH		 1482/* set the cpu timer - may only be called from the VCPU thread itself */
1483void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
1484{
db0758b2 1485 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
9c23a131 1486 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1487 if (vcpu->arch.cputm_enabled)
1488 vcpu->arch.cputm_start = get_tod_clock_fast();
4287f247 1489 vcpu->arch.sie_block->cputm = cputm;
9c23a131 1490 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2 1491 preempt_enable();
4287f247
DH		 1492}
1493
db0758b2 1494/* update and get the cpu timer - can also be called from other VCPU threads */
4287f247
DH		 1495__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
1496{
9c23a131 1497 unsigned int seq;
db0758b2 1498 __u64 value;
db0758b2
DH		 1499
1500 if (unlikely(!vcpu->arch.cputm_enabled))
1501 return vcpu->arch.sie_block->cputm;
1502
9c23a131
DH		 1503 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1504 do {
1505 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
1506 /*
1507 * If the writer would ever execute a read in the critical
1508 * section, e.g. in irq context, we have a deadlock.
1509 */
1510 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
1511 value = vcpu->arch.sie_block->cputm;
1512 /* if cputm_start is 0, accounting is being started/stopped */
1513 if (likely(vcpu->arch.cputm_start))
1514 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1515 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
1516 preempt_enable();
db0758b2 1517 return value;
4287f247
DH		 1518}
1519
b0c632db
HC		 1520void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1521{
9977e886 1522 /* Save host register state */
d0164ee2 1523 save_fpu_regs();
9abc2a08
DH		 1524 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
1525 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
9977e886 1526
6fd8e67d
DH		 1527 if (MACHINE_HAS_VX)
1528 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
1529 else
1530 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
9abc2a08 1531 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
9977e886 1532 if (test_fp_ctl(current->thread.fpu.fpc))
96b2d7a8 1533 /* User space provided an invalid FPC, let's clear it */
9977e886
HB		 1534 current->thread.fpu.fpc = 0;
1535
1536 save_access_regs(vcpu->arch.host_acrs);
59674c1a 1537 restore_access_regs(vcpu->run->s.regs.acrs);
480e5926 1538 gmap_enable(vcpu->arch.gmap);
805de8f4 1539 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
5ebda316 1540 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1541 __start_cpu_timer_accounting(vcpu);
01a745ac 1542 vcpu->cpu = cpu;
b0c632db
HC		 1543}
1544
1545void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1546{
01a745ac 1547 vcpu->cpu = -1;
5ebda316 1548 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1549 __stop_cpu_timer_accounting(vcpu);
805de8f4 1550 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
480e5926 1551 gmap_disable(vcpu->arch.gmap);
9977e886 1552
9abc2a08 1553 /* Save guest register state */
d0164ee2 1554 save_fpu_regs();
9abc2a08 1555 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
9977e886 1556
9abc2a08
DH		 1557 /* Restore host register state */
1558 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
1559 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
9977e886
HB		 1560
1561 save_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 1562 restore_access_regs(vcpu->arch.host_acrs);
1563}
1564
1565static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1566{
1567 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1568 vcpu->arch.sie_block->gpsw.mask = 0UL;
1569 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 1570 kvm_s390_set_prefix(vcpu, 0);
4287f247 1571 kvm_s390_set_cpu_timer(vcpu, 0);
b0c632db
HC		 1572 vcpu->arch.sie_block->ckc = 0UL;
1573 vcpu->arch.sie_block->todpr = 0;
1574 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1575 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1576 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
9abc2a08
DH		 1577 /* make sure the new fpc will be lazily loaded */
1578 save_fpu_regs();
1579 current->thread.fpu.fpc = 0;
b0c632db 1580 vcpu->arch.sie_block->gbea = 1;
672550fb 1581 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 1582 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1583 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 1584 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1585 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 1586 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 1587}
1588
31928aa5 1589void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 1590{
72f25020 1591 mutex_lock(&vcpu->kvm->lock);
fdf03650 1592 preempt_disable();
72f25020 1593 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
fdf03650 1594 preempt_enable();
72f25020 1595 mutex_unlock(&vcpu->kvm->lock);
25508824 1596 if (!kvm_is_ucontrol(vcpu->kvm)) {
dafd032a 1597 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
eaa78f34 1598 sca_add_vcpu(vcpu);
25508824
DH		 1599 }
1600
42897d86
MT		 1601}
1602
5102ee87
TK		 1603static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1604{
9d8d5786 1605 if (!test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 1606 return;
1607
a374e892
TK		 1608 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1609
1610 if (vcpu->kvm->arch.crypto.aes_kw)
1611 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1612 if (vcpu->kvm->arch.crypto.dea_kw)
1613 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1614
5102ee87
TK		 1615 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1616}
1617
b31605c1
DD		 1618void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1619{
1620 free_page(vcpu->arch.sie_block->cbrlo);
1621 vcpu->arch.sie_block->cbrlo = 0;
1622}
1623
1624int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1625{
1626 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1627 if (!vcpu->arch.sie_block->cbrlo)
1628 return -ENOMEM;
1629
1630 vcpu->arch.sie_block->ecb2 |= 0x80;
1631 vcpu->arch.sie_block->ecb2 &= ~0x08;
1632 return 0;
1633}
1634
91520f1a
MM		 1635static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1636{
1637 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1638
91520f1a 1639 vcpu->arch.sie_block->ibc = model->ibc;
80bc79dc 1640 if (test_kvm_facility(vcpu->kvm, 7))
c54f0d6a 1641 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
91520f1a
MM		 1642}
1643
b0c632db
HC		 1644int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1645{
b31605c1 1646 int rc = 0;
b31288fa 1647
9e6dabef
CH		 1648 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1649 CPUSTAT_SM |
a4a4f191
GH		 1650 CPUSTAT_STOPPED);
1651
53df84f8 1652 if (test_kvm_facility(vcpu->kvm, 78))
805de8f4 1653 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
53df84f8 1654 else if (test_kvm_facility(vcpu->kvm, 8))
805de8f4 1655 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
a4a4f191 1656
91520f1a
MM		 1657 kvm_s390_vcpu_setup_model(vcpu);
1658
bd50e8ec
DH		 1659 vcpu->arch.sie_block->ecb = 0x02;
1660 if (test_kvm_facility(vcpu->kvm, 9))
1661 vcpu->arch.sie_block->ecb |= 0x04;
9d8d5786 1662 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
7feb6bb8
MM		 1663 vcpu->arch.sie_block->ecb |= 0x10;
1664
d6af0b49
DH		 1665 if (test_kvm_facility(vcpu->kvm, 8))
1666 vcpu->arch.sie_block->ecb2 |= 0x08;
ea5f4969 1667 vcpu->arch.sie_block->eca = 0xC1002000U;
37c5f6c8 1668 if (sclp.has_siif)
217a4406 1669 vcpu->arch.sie_block->eca |= 1;
37c5f6c8 1670 if (sclp.has_sigpif)
ea5f4969 1671 vcpu->arch.sie_block->eca |= 0x10000000U;
c6e5f166
FZ		 1672 if (test_kvm_facility(vcpu->kvm, 64))
1673 vcpu->arch.sie_block->ecb3 |= 0x01;
18280d8b 1674 if (test_kvm_facility(vcpu->kvm, 129)) {
13211ea7
EF		 1675 vcpu->arch.sie_block->eca |= 0x00020000;
1676 vcpu->arch.sie_block->ecd |= 0x20000000;
1677 }
c6e5f166 1678 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
492d8642 1679 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
5a5e6536 1680
e6db1d61 1681 if (vcpu->kvm->arch.use_cmma) {
b31605c1
DD		 1682 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1683 if (rc)
1684 return rc;
b31288fa 1685 }
0ac96caf 1686 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 1687 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 1688
5102ee87
TK		 1689 kvm_s390_vcpu_crypto_setup(vcpu);
1690
b31605c1 1691 return rc;
b0c632db
HC		 1692}
1693
1694struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1695 unsigned int id)
1696{
4d47555a 1697 struct kvm_vcpu *vcpu;
7feb6bb8 1698 struct sie_page *sie_page;
4d47555a
CO		 1699 int rc = -EINVAL;
1700
4215825e 1701 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
4d47555a
CO		 1702 goto out;
1703
1704 rc = -ENOMEM;
b0c632db 1705
b110feaf 1706 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 1707 if (!vcpu)
4d47555a 1708 goto out;
b0c632db 1709
7feb6bb8
MM		 1710 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1711 if (!sie_page)
b0c632db
HC		 1712 goto out_free_cpu;
1713
7feb6bb8
MM		 1714 vcpu->arch.sie_block = &sie_page->sie_block;
1715 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1716
b0c632db 1717 vcpu->arch.sie_block->icpua = id;
ba5c1e9b 1718 spin_lock_init(&vcpu->arch.local_int.lock);
ba5c1e9b 1719 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 1720 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 1721 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
9c23a131 1722 seqcount_init(&vcpu->arch.cputm_seqcount);
ba5c1e9b 1723
b0c632db
HC		 1724 rc = kvm_vcpu_init(vcpu, kvm, id);
1725 if (rc)
9abc2a08 1726 goto out_free_sie_block;
8335713a 1727 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
b0c632db 1728 vcpu->arch.sie_block);
ade38c31 1729 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 1730
b0c632db 1731 return vcpu;
7b06bf2f
WY		 1732out_free_sie_block:
1733 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 1734out_free_cpu:
b110feaf 1735 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 1736out:
b0c632db
HC		 1737 return ERR_PTR(rc);
1738}
1739
b0c632db
HC		 1740int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1741{
9a022067 1742 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 1743}
1744
27406cd5 1745void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
49b99e1e 1746{
805de8f4 1747 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
61a6df54 1748 exit_sie(vcpu);
49b99e1e
CB		 1749}
1750
27406cd5 1751void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
49b99e1e 1752{
805de8f4 1753 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
49b99e1e
CB		 1754}
1755
8e236546
CB		 1756static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1757{
805de8f4 1758 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
61a6df54 1759 exit_sie(vcpu);
8e236546
CB		 1760}
1761
1762static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1763{
9bf9fde2 1764 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
8e236546
CB		 1765}
1766
49b99e1e
CB		 1767/*
1768 * Kick a guest cpu out of SIE and wait until SIE is not running.
1769 * If the CPU is not running (e.g. waiting as idle) the function will
1770 * return immediately. */
1771void exit_sie(struct kvm_vcpu *vcpu)
1772{
805de8f4 1773 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
49b99e1e
CB		 1774 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1775 cpu_relax();
1776}
1777
8e236546
CB		 1778/* Kick a guest cpu out of SIE to process a request synchronously */
1779void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
49b99e1e 1780{
8e236546
CB		 1781 kvm_make_request(req, vcpu);
1782 kvm_s390_vcpu_request(vcpu);
49b99e1e
CB		 1783}
1784
2c70fe44
CB		 1785static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1786{
1787 int i;
1788 struct kvm *kvm = gmap->private;
1789 struct kvm_vcpu *vcpu;
1790
1791 kvm_for_each_vcpu(i, vcpu, kvm) {
1792 /* match against both prefix pages */
fda902cb 1793 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
2c70fe44 1794 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
8e236546 1795 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44
CB		 1796 }
1797 }
1798}
1799
b6d33834
CD		 1800int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1801{
1802 /* kvm common code refers to this, but never calls it */
1803 BUG();
1804 return 0;
1805}
1806
14eebd91
CO		 1807static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1808 struct kvm_one_reg *reg)
1809{
1810 int r = -EINVAL;
1811
1812 switch (reg->id) {
29b7c71b
CO		 1813 case KVM_REG_S390_TODPR:
1814 r = put_user(vcpu->arch.sie_block->todpr,
1815 (u32 __user *)reg->addr);
1816 break;
1817 case KVM_REG_S390_EPOCHDIFF:
1818 r = put_user(vcpu->arch.sie_block->epoch,
1819 (u64 __user *)reg->addr);
1820 break;
46a6dd1c 1821 case KVM_REG_S390_CPU_TIMER:
4287f247 1822 r = put_user(kvm_s390_get_cpu_timer(vcpu),
46a6dd1c
J		 1823 (u64 __user *)reg->addr);
1824 break;
1825 case KVM_REG_S390_CLOCK_COMP:
1826 r = put_user(vcpu->arch.sie_block->ckc,
1827 (u64 __user *)reg->addr);
1828 break;
536336c2
DD		 1829 case KVM_REG_S390_PFTOKEN:
1830 r = put_user(vcpu->arch.pfault_token,
1831 (u64 __user *)reg->addr);
1832 break;
1833 case KVM_REG_S390_PFCOMPARE:
1834 r = put_user(vcpu->arch.pfault_compare,
1835 (u64 __user *)reg->addr);
1836 break;
1837 case KVM_REG_S390_PFSELECT:
1838 r = put_user(vcpu->arch.pfault_select,
1839 (u64 __user *)reg->addr);
1840 break;
672550fb
CB		 1841 case KVM_REG_S390_PP:
1842 r = put_user(vcpu->arch.sie_block->pp,
1843 (u64 __user *)reg->addr);
1844 break;
afa45ff5
CB		 1845 case KVM_REG_S390_GBEA:
1846 r = put_user(vcpu->arch.sie_block->gbea,
1847 (u64 __user *)reg->addr);
1848 break;
14eebd91
CO		 1849 default:
1850 break;
1851 }
1852
1853 return r;
1854}
1855
1856static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1857 struct kvm_one_reg *reg)
1858{
1859 int r = -EINVAL;
4287f247 1860 __u64 val;
14eebd91
CO		 1861
1862 switch (reg->id) {
29b7c71b
CO		 1863 case KVM_REG_S390_TODPR:
1864 r = get_user(vcpu->arch.sie_block->todpr,
1865 (u32 __user *)reg->addr);
1866 break;
1867 case KVM_REG_S390_EPOCHDIFF:
1868 r = get_user(vcpu->arch.sie_block->epoch,
1869 (u64 __user *)reg->addr);
1870 break;
46a6dd1c 1871 case KVM_REG_S390_CPU_TIMER:
4287f247
DH		 1872 r = get_user(val, (u64 __user *)reg->addr);
1873 if (!r)
1874 kvm_s390_set_cpu_timer(vcpu, val);
46a6dd1c
J		 1875 break;
1876 case KVM_REG_S390_CLOCK_COMP:
1877 r = get_user(vcpu->arch.sie_block->ckc,
1878 (u64 __user *)reg->addr);
1879 break;
536336c2
DD		 1880 case KVM_REG_S390_PFTOKEN:
1881 r = get_user(vcpu->arch.pfault_token,
1882 (u64 __user *)reg->addr);
9fbd8082
DH		 1883 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1884 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 1885 break;
1886 case KVM_REG_S390_PFCOMPARE:
1887 r = get_user(vcpu->arch.pfault_compare,
1888 (u64 __user *)reg->addr);
1889 break;
1890 case KVM_REG_S390_PFSELECT:
1891 r = get_user(vcpu->arch.pfault_select,
1892 (u64 __user *)reg->addr);
1893 break;
672550fb
CB		 1894 case KVM_REG_S390_PP:
1895 r = get_user(vcpu->arch.sie_block->pp,
1896 (u64 __user *)reg->addr);
1897 break;
afa45ff5
CB		 1898 case KVM_REG_S390_GBEA:
1899 r = get_user(vcpu->arch.sie_block->gbea,
1900 (u64 __user *)reg->addr);
1901 break;
14eebd91
CO		 1902 default:
1903 break;
1904 }
1905
1906 return r;
1907}
b6d33834 1908
b0c632db
HC		 1909static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1910{
b0c632db 1911 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 1912 return 0;
1913}
1914
1915int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1916{
5a32c1af 1917 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 1918 return 0;
1919}
1920
1921int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1922{
5a32c1af 1923 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 1924 return 0;
1925}
1926
1927int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1928 struct kvm_sregs *sregs)
1929{
59674c1a 1930 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 1931 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
59674c1a 1932 restore_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 1933 return 0;
1934}
1935
1936int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1937 struct kvm_sregs *sregs)
1938{
59674c1a 1939 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 1940 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 1941 return 0;
1942}
1943
1944int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1945{
9abc2a08
DH		 1946 /* make sure the new values will be lazily loaded */
1947 save_fpu_regs();
4725c860
MS		 1948 if (test_fp_ctl(fpu->fpc))
1949 return -EINVAL;
9abc2a08
DH		 1950 current->thread.fpu.fpc = fpu->fpc;
1951 if (MACHINE_HAS_VX)
1952 convert_fp_to_vx(current->thread.fpu.vxrs, (freg_t *)fpu->fprs);
1953 else
1954 memcpy(current->thread.fpu.fprs, &fpu->fprs, sizeof(fpu->fprs));
b0c632db
HC		 1955 return 0;
1956}
1957
1958int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1959{
9abc2a08
DH		 1960 /* make sure we have the latest values */
1961 save_fpu_regs();
1962 if (MACHINE_HAS_VX)
1963 convert_vx_to_fp((freg_t *)fpu->fprs, current->thread.fpu.vxrs);
1964 else
1965 memcpy(fpu->fprs, current->thread.fpu.fprs, sizeof(fpu->fprs));
1966 fpu->fpc = current->thread.fpu.fpc;
b0c632db
HC		 1967 return 0;
1968}
1969
1970static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1971{
1972 int rc = 0;
1973
7a42fdc2 1974 if (!is_vcpu_stopped(vcpu))
b0c632db 1975 rc = -EBUSY;
d7b0b5eb
CO		 1976 else {
1977 vcpu->run->psw_mask = psw.mask;
1978 vcpu->run->psw_addr = psw.addr;
1979 }
b0c632db
HC		 1980 return rc;
1981}
1982
1983int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1984 struct kvm_translation *tr)
1985{
1986 return -EINVAL; /* not implemented yet */
1987}
1988
27291e21
DH		 1989#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1990 KVM_GUESTDBG_USE_HW_BP | \
1991 KVM_GUESTDBG_ENABLE)
1992
d0bfb940
JK		 1993int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1994 struct kvm_guest_debug *dbg)
b0c632db 1995{
27291e21
DH		 1996 int rc = 0;
1997
1998 vcpu->guest_debug = 0;
1999 kvm_s390_clear_bp_data(vcpu);
2000
2de3bfc2 2001 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21
DH		 2002 return -EINVAL;
2003
2004 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2005 vcpu->guest_debug = dbg->control;
2006 /* enforce guest PER */
805de8f4 2007 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2008
2009 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2010 rc = kvm_s390_import_bp_data(vcpu, dbg);
2011 } else {
805de8f4 2012 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2013 vcpu->arch.guestdbg.last_bp = 0;
2014 }
2015
2016 if (rc) {
2017 vcpu->guest_debug = 0;
2018 kvm_s390_clear_bp_data(vcpu);
805de8f4 2019 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2020 }
2021
2022 return rc;
b0c632db
HC		 2023}
2024
62d9f0db
MT		 2025int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2026 struct kvm_mp_state *mp_state)
2027{
6352e4d2
DH		 2028 /* CHECK_STOP and LOAD are not supported yet */
2029 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2030 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 2031}
2032
2033int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2034 struct kvm_mp_state *mp_state)
2035{
6352e4d2
DH		 2036 int rc = 0;
2037
2038 /* user space knows about this interface - let it control the state */
2039 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2040
2041 switch (mp_state->mp_state) {
2042 case KVM_MP_STATE_STOPPED:
2043 kvm_s390_vcpu_stop(vcpu);
2044 break;
2045 case KVM_MP_STATE_OPERATING:
2046 kvm_s390_vcpu_start(vcpu);
2047 break;
2048 case KVM_MP_STATE_LOAD:
2049 case KVM_MP_STATE_CHECK_STOP:
2050 /* fall through - CHECK_STOP and LOAD are not supported yet */
2051 default:
2052 rc = -ENXIO;
2053 }
2054
2055 return rc;
62d9f0db
MT		 2056}
2057
8ad35755
DH		 2058static bool ibs_enabled(struct kvm_vcpu *vcpu)
2059{
2060 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2061}
2062
2c70fe44
CB		 2063static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2064{
8ad35755 2065retry:
8e236546 2066 kvm_s390_vcpu_request_handled(vcpu);
586b7ccd
CB		 2067 if (!vcpu->requests)
2068 return 0;
2c70fe44
CB		 2069 /*
2070 * We use MMU_RELOAD just to re-arm the ipte notifier for the
2071 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
2072 * This ensures that the ipte instruction for this request has
2073 * already finished. We might race against a second unmapper that
2074 * wants to set the blocking bit. Lets just retry the request loop.
2075 */
8ad35755 2076 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44
CB		 2077 int rc;
2078 rc = gmap_ipte_notify(vcpu->arch.gmap,
fda902cb 2079 kvm_s390_get_prefix(vcpu),
2c70fe44
CB		 2080 PAGE_SIZE * 2);
2081 if (rc)
2082 return rc;
8ad35755 2083 goto retry;
2c70fe44 2084 }
8ad35755 2085
d3d692c8
DH		 2086 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2087 vcpu->arch.sie_block->ihcpu = 0xffff;
2088 goto retry;
2089 }
2090
8ad35755
DH		 2091 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2092 if (!ibs_enabled(vcpu)) {
2093 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
805de8f4 2094 atomic_or(CPUSTAT_IBS,
8ad35755
DH		 2095 &vcpu->arch.sie_block->cpuflags);
2096 }
2097 goto retry;
2c70fe44 2098 }
8ad35755
DH		 2099
2100 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2101 if (ibs_enabled(vcpu)) {
2102 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
805de8f4 2103 atomic_andnot(CPUSTAT_IBS,
8ad35755
DH		 2104 &vcpu->arch.sie_block->cpuflags);
2105 }
2106 goto retry;
2107 }
2108
0759d068
DH		 2109 /* nothing to do, just clear the request */
2110 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2111
2c70fe44
CB		 2112 return 0;
2113}
2114
25ed1675
DH		 2115void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2116{
2117 struct kvm_vcpu *vcpu;
2118 int i;
2119
2120 mutex_lock(&kvm->lock);
2121 preempt_disable();
2122 kvm->arch.epoch = tod - get_tod_clock();
2123 kvm_s390_vcpu_block_all(kvm);
2124 kvm_for_each_vcpu(i, vcpu, kvm)
2125 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2126 kvm_s390_vcpu_unblock_all(kvm);
2127 preempt_enable();
2128 mutex_unlock(&kvm->lock);
2129}
2130
fa576c58
TH		 2131/**
2132 * kvm_arch_fault_in_page - fault-in guest page if necessary
2133 * @vcpu: The corresponding virtual cpu
2134 * @gpa: Guest physical address
2135 * @writable: Whether the page should be writable or not
2136 *
2137 * Make sure that a guest page has been faulted-in on the host.
2138 *
2139 * Return: Zero on success, negative error code otherwise.
2140 */
2141long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 2142{
527e30b4
MS		 2143 return gmap_fault(vcpu->arch.gmap, gpa,
2144 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 2145}
2146
3c038e6b
DD		 2147static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2148 unsigned long token)
2149{
2150 struct kvm_s390_interrupt inti;
383d0b05 2151 struct kvm_s390_irq irq;
3c038e6b
DD		 2152
2153 if (start_token) {
383d0b05
JF		 2154 irq.u.ext.ext_params2 = token;
2155 irq.type = KVM_S390_INT_PFAULT_INIT;
2156 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 2157 } else {
2158 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 2159 inti.parm64 = token;
3c038e6b
DD		 2160 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2161 }
2162}
2163
2164void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2165 struct kvm_async_pf *work)
2166{
2167 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2168 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2169}
2170
2171void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2172 struct kvm_async_pf *work)
2173{
2174 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2175 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2176}
2177
2178void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2179 struct kvm_async_pf *work)
2180{
2181 /* s390 will always inject the page directly */
2182}
2183
2184bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2185{
2186 /*
2187 * s390 will always inject the page directly,
2188 * but we still want check_async_completion to cleanup
2189 */
2190 return true;
2191}
2192
2193static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2194{
2195 hva_t hva;
2196 struct kvm_arch_async_pf arch;
2197 int rc;
2198
2199 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2200 return 0;
2201 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2202 vcpu->arch.pfault_compare)
2203 return 0;
2204 if (psw_extint_disabled(vcpu))
2205 return 0;
9a022067 2206 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 2207 return 0;
2208 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2209 return 0;
2210 if (!vcpu->arch.gmap->pfault_enabled)
2211 return 0;
2212
81480cc1
HC		 2213 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2214 hva += current->thread.gmap_addr & ~PAGE_MASK;
2215 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 2216 return 0;
2217
2218 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2219 return rc;
2220}
2221
3fb4c40f 2222static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 2223{
3fb4c40f 2224 int rc, cpuflags;
e168bf8d 2225
3c038e6b
DD		 2226 /*
2227 * On s390 notifications for arriving pages will be delivered directly
2228 * to the guest but the house keeping for completed pfaults is
2229 * handled outside the worker.
2230 */
2231 kvm_check_async_pf_completion(vcpu);
2232
7ec7c8c7
CB		 2233 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2234 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
b0c632db
HC		 2235
2236 if (need_resched())
2237 schedule();
2238
d3a73acb 2239 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 2240 s390_handle_mcck();
2241
79395031
JF		 2242 if (!kvm_is_ucontrol(vcpu->kvm)) {
2243 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2244 if (rc)
2245 return rc;
2246 }
0ff31867 2247
2c70fe44
CB		 2248 rc = kvm_s390_handle_requests(vcpu);
2249 if (rc)
2250 return rc;
2251
27291e21
DH		 2252 if (guestdbg_enabled(vcpu)) {
2253 kvm_s390_backup_guest_per_regs(vcpu);
2254 kvm_s390_patch_guest_per_regs(vcpu);
2255 }
2256
b0c632db 2257 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 2258 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2259 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2260 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 2261
3fb4c40f
TH		 2262 return 0;
2263}
2264
492d8642
TH		 2265static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2266{
56317920
DH		 2267 struct kvm_s390_pgm_info pgm_info = {
2268 .code = PGM_ADDRESSING,
2269 };
2270 u8 opcode, ilen;
492d8642
TH		 2271 int rc;
2272
2273 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2274 trace_kvm_s390_sie_fault(vcpu);
2275
2276 /*
2277 * We want to inject an addressing exception, which is defined as a
2278 * suppressing or terminating exception. However, since we came here
2279 * by a DAT access exception, the PSW still points to the faulting
2280 * instruction since DAT exceptions are nullifying. So we've got
2281 * to look up the current opcode to get the length of the instruction
2282 * to be able to forward the PSW.
2283 */
65977322 2284 rc = read_guest_instr(vcpu, &opcode, 1);
56317920 2285 ilen = insn_length(opcode);
9b0d721a
DH		 2286 if (rc < 0) {
2287 return rc;
2288 } else if (rc) {
2289 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2290 * Forward by arbitrary ilc, injection will take care of
2291 * nullification if necessary.
2292 */
2293 pgm_info = vcpu->arch.pgm;
2294 ilen = 4;
2295 }
56317920
DH		 2296 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2297 kvm_s390_forward_psw(vcpu, ilen);
2298 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
492d8642
TH		 2299}
2300
3fb4c40f
TH		 2301static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2302{
2b29a9fd
DD		 2303 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2304 vcpu->arch.sie_block->icptcode);
2305 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2306
27291e21
DH		 2307 if (guestdbg_enabled(vcpu))
2308 kvm_s390_restore_guest_per_regs(vcpu);
2309
7ec7c8c7
CB		 2310 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2311 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
71f116bf
DH		 2312
2313 if (vcpu->arch.sie_block->icptcode > 0) {
2314 int rc = kvm_handle_sie_intercept(vcpu);
2315
2316 if (rc != -EOPNOTSUPP)
2317 return rc;
2318 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2319 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2320 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2321 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2322 return -EREMOTE;
2323 } else if (exit_reason != -EFAULT) {
2324 vcpu->stat.exit_null++;
2325 return 0;
210b1607
TH		 2326 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2327 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2328 vcpu->run->s390_ucontrol.trans_exc_code =
2329 current->thread.gmap_addr;
2330 vcpu->run->s390_ucontrol.pgm_code = 0x10;
71f116bf 2331 return -EREMOTE;
24eb3a82 2332 } else if (current->thread.gmap_pfault) {
3c038e6b 2333 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 2334 current->thread.gmap_pfault = 0;
71f116bf
DH		 2335 if (kvm_arch_setup_async_pf(vcpu))
2336 return 0;
2337 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
a76ccff6 2338 }
71f116bf 2339 return vcpu_post_run_fault_in_sie(vcpu);
3fb4c40f
TH		 2340}
2341
2342static int __vcpu_run(struct kvm_vcpu *vcpu)
2343{
2344 int rc, exit_reason;
2345
800c1065
TH		 2346 /*
2347 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2348 * ning the guest), so that memslots (and other stuff) are protected
2349 */
2350 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2351
a76ccff6
TH		 2352 do {
2353 rc = vcpu_pre_run(vcpu);
2354 if (rc)
2355 break;
3fb4c40f 2356
800c1065 2357 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 2358 /*
2359 * As PF_VCPU will be used in fault handler, between
2360 * guest_enter and guest_exit should be no uaccess.
2361 */
0097d12e
CB		 2362 local_irq_disable();
2363 __kvm_guest_enter();
db0758b2 2364 __disable_cpu_timer_accounting(vcpu);
0097d12e 2365 local_irq_enable();
a76ccff6
TH		 2366 exit_reason = sie64a(vcpu->arch.sie_block,
2367 vcpu->run->s.regs.gprs);
0097d12e 2368 local_irq_disable();
db0758b2 2369 __enable_cpu_timer_accounting(vcpu);
0097d12e
CB		 2370 __kvm_guest_exit();
2371 local_irq_enable();
800c1065 2372 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 2373
2374 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 2375 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 2376
800c1065 2377 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 2378 return rc;
b0c632db
HC		 2379}
2380
b028ee3e
DH		 2381static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2382{
2383 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2384 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2385 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2386 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2387 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2388 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 2389 /* some control register changes require a tlb flush */
2390 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 2391 }
2392 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4287f247 2393 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
b028ee3e
DH		 2394 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2395 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2396 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2397 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2398 }
2399 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2400 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2401 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2402 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 2403 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2404 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e
DH		 2405 }
2406 kvm_run->kvm_dirty_regs = 0;
2407}
2408
2409static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2410{
2411 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2412 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2413 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2414 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4287f247 2415 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
b028ee3e
DH		 2416 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2417 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2418 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2419 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2420 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2421 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2422 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2423}
2424
b0c632db
HC		 2425int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2426{
8f2abe6a 2427 int rc;
b0c632db
HC		 2428 sigset_t sigsaved;
2429
27291e21
DH		 2430 if (guestdbg_exit_pending(vcpu)) {
2431 kvm_s390_prepare_debug_exit(vcpu);
2432 return 0;
2433 }
2434
b0c632db
HC		 2435 if (vcpu->sigset_active)
2436 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2437
6352e4d2
DH		 2438 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2439 kvm_s390_vcpu_start(vcpu);
2440 } else if (is_vcpu_stopped(vcpu)) {
ea2cdd27 2441 pr_err_ratelimited("can't run stopped vcpu %d\n",
6352e4d2
DH		 2442 vcpu->vcpu_id);
2443 return -EINVAL;
2444 }
b0c632db 2445
b028ee3e 2446 sync_regs(vcpu, kvm_run);
db0758b2 2447 enable_cpu_timer_accounting(vcpu);
d7b0b5eb 2448
dab4079d 2449 might_fault();
a76ccff6 2450 rc = __vcpu_run(vcpu);
9ace903d 2451
b1d16c49
CE		 2452 if (signal_pending(current) && !rc) {
2453 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 2454 rc = -EINTR;
b1d16c49 2455 }
8f2abe6a 2456
27291e21
DH		 2457 if (guestdbg_exit_pending(vcpu) && !rc) {
2458 kvm_s390_prepare_debug_exit(vcpu);
2459 rc = 0;
2460 }
2461
8f2abe6a 2462 if (rc == -EREMOTE) {
71f116bf 2463 /* userspace support is needed, kvm_run has been prepared */
8f2abe6a
CB		 2464 rc = 0;
2465 }
b0c632db 2466
db0758b2 2467 disable_cpu_timer_accounting(vcpu);
b028ee3e 2468 store_regs(vcpu, kvm_run);
d7b0b5eb 2469
b0c632db
HC		 2470 if (vcpu->sigset_active)
2471 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2472
b0c632db 2473 vcpu->stat.exit_userspace++;
7e8e6ab4 2474 return rc;
b0c632db
HC		 2475}
2476
b0c632db
HC		 2477/*
2478 * store status at address
2479 * we use have two special cases:
2480 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2481 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2482 */
d0bce605 2483int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 2484{
092670cd 2485 unsigned char archmode = 1;
9abc2a08 2486 freg_t fprs[NUM_FPRS];
fda902cb 2487 unsigned int px;
4287f247 2488 u64 clkcomp, cputm;
d0bce605 2489 int rc;
b0c632db 2490
d9a3a09a 2491 px = kvm_s390_get_prefix(vcpu);
d0bce605
HC		 2492 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2493 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 2494 return -EFAULT;
d9a3a09a 2495 gpa = 0;
d0bce605
HC		 2496 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2497 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 2498 return -EFAULT;
d9a3a09a
MS		 2499 gpa = px;
2500 } else
2501 gpa -= __LC_FPREGS_SAVE_AREA;
9abc2a08
DH		 2502
2503 /* manually convert vector registers if necessary */
2504 if (MACHINE_HAS_VX) {
9522b37f 2505 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08
DH		 2506 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2507 fprs, 128);
2508 } else {
2509 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
6fd8e67d 2510 vcpu->run->s.regs.fprs, 128);
9abc2a08 2511 }
d9a3a09a 2512 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
d0bce605 2513 vcpu->run->s.regs.gprs, 128);
d9a3a09a 2514 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
d0bce605 2515 &vcpu->arch.sie_block->gpsw, 16);
d9a3a09a 2516 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
fda902cb 2517 &px, 4);
d9a3a09a 2518 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
9abc2a08 2519 &vcpu->run->s.regs.fpc, 4);
d9a3a09a 2520 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
d0bce605 2521 &vcpu->arch.sie_block->todpr, 4);
4287f247 2522 cputm = kvm_s390_get_cpu_timer(vcpu);
d9a3a09a 2523 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4287f247 2524 &cputm, 8);
178bd789 2525 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d9a3a09a 2526 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
d0bce605 2527 &clkcomp, 8);
d9a3a09a 2528 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
d0bce605 2529 &vcpu->run->s.regs.acrs, 64);
d9a3a09a 2530 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
d0bce605
HC		 2531 &vcpu->arch.sie_block->gcr, 128);
2532 return rc ? -EFAULT : 0;
b0c632db
HC		 2533}
2534
e879892c
TH		 2535int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2536{
2537 /*
2538 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2539 * copying in vcpu load/put. Lets update our copies before we save
2540 * it into the save area
2541 */
d0164ee2 2542 save_fpu_regs();
9abc2a08 2543 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
e879892c
TH		 2544 save_access_regs(vcpu->run->s.regs.acrs);
2545
2546 return kvm_s390_store_status_unloaded(vcpu, addr);
2547}
2548
bc17de7c
EF		 2549/*
2550 * store additional status at address
2551 */
2552int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2553 unsigned long gpa)
2554{
2555 /* Only bits 0-53 are used for address formation */
2556 if (!(gpa & ~0x3ff))
2557 return 0;
2558
2559 return write_guest_abs(vcpu, gpa & ~0x3ff,
2560 (void *)&vcpu->run->s.regs.vrs, 512);
2561}
2562
2563int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2564{
2565 if (!test_kvm_facility(vcpu->kvm, 129))
2566 return 0;
2567
2568 /*
2569 * The guest VXRS are in the host VXRs due to the lazy
9977e886
HB		 2570 * copying in vcpu load/put. We can simply call save_fpu_regs()
2571 * to save the current register state because we are in the
2572 * middle of a load/put cycle.
2573 *
2574 * Let's update our copies before we save it into the save area.
bc17de7c 2575 */
d0164ee2 2576 save_fpu_regs();
bc17de7c
EF		 2577
2578 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2579}
2580
8ad35755
DH		 2581static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2582{
2583 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
8e236546 2584 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
8ad35755
DH		 2585}
2586
2587static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2588{
2589 unsigned int i;
2590 struct kvm_vcpu *vcpu;
2591
2592 kvm_for_each_vcpu(i, vcpu, kvm) {
2593 __disable_ibs_on_vcpu(vcpu);
2594 }
2595}
2596
2597static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2598{
2599 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
8e236546 2600 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
8ad35755
DH		 2601}
2602
6852d7b6
DH		 2603void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2604{
8ad35755
DH		 2605 int i, online_vcpus, started_vcpus = 0;
2606
2607 if (!is_vcpu_stopped(vcpu))
2608 return;
2609
6852d7b6 2610 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 2611 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2612 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2613 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2614
2615 for (i = 0; i < online_vcpus; i++) {
2616 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2617 started_vcpus++;
2618 }
2619
2620 if (started_vcpus == 0) {
2621 /* we're the only active VCPU -> speed it up */
2622 __enable_ibs_on_vcpu(vcpu);
2623 } else if (started_vcpus == 1) {
2624 /*
2625 * As we are starting a second VCPU, we have to disable
2626 * the IBS facility on all VCPUs to remove potentially
2627 * oustanding ENABLE requests.
2628 */
2629 __disable_ibs_on_all_vcpus(vcpu->kvm);
2630 }
2631
805de8f4 2632 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2633 /*
2634 * Another VCPU might have used IBS while we were offline.
2635 * Let's play safe and flush the VCPU at startup.
2636 */
d3d692c8 2637 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 2638 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2639 return;
6852d7b6
DH		 2640}
2641
2642void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2643{
8ad35755
DH		 2644 int i, online_vcpus, started_vcpus = 0;
2645 struct kvm_vcpu *started_vcpu = NULL;
2646
2647 if (is_vcpu_stopped(vcpu))
2648 return;
2649
6852d7b6 2650 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 2651 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2652 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2653 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2654
32f5ff63 2655 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 2656 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 2657
805de8f4 2658 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2659 __disable_ibs_on_vcpu(vcpu);
2660
2661 for (i = 0; i < online_vcpus; i++) {
2662 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2663 started_vcpus++;
2664 started_vcpu = vcpu->kvm->vcpus[i];
2665 }
2666 }
2667
2668 if (started_vcpus == 1) {
2669 /*
2670 * As we only have one VCPU left, we want to enable the
2671 * IBS facility for that VCPU to speed it up.
2672 */
2673 __enable_ibs_on_vcpu(started_vcpu);
2674 }
2675
433b9ee4 2676 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2677 return;
6852d7b6
DH		 2678}
2679
d6712df9
CH		 2680static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2681 struct kvm_enable_cap *cap)
2682{
2683 int r;
2684
2685 if (cap->flags)
2686 return -EINVAL;
2687
2688 switch (cap->cap) {
fa6b7fe9
CH		 2689 case KVM_CAP_S390_CSS_SUPPORT:
2690 if (!vcpu->kvm->arch.css_support) {
2691 vcpu->kvm->arch.css_support = 1;
c92ea7b9 2692 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
fa6b7fe9
CH		 2693 trace_kvm_s390_enable_css(vcpu->kvm);
2694 }
2695 r = 0;
2696 break;
d6712df9
CH		 2697 default:
2698 r = -EINVAL;
2699 break;
2700 }
2701 return r;
2702}
2703
41408c28
TH		 2704static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2705 struct kvm_s390_mem_op *mop)
2706{
2707 void __user *uaddr = (void __user *)mop->buf;
2708 void *tmpbuf = NULL;
2709 int r, srcu_idx;
2710 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2711 | KVM_S390_MEMOP_F_CHECK_ONLY;
2712
2713 if (mop->flags & ~supported_flags)
2714 return -EINVAL;
2715
2716 if (mop->size > MEM_OP_MAX_SIZE)
2717 return -E2BIG;
2718
2719 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2720 tmpbuf = vmalloc(mop->size);
2721 if (!tmpbuf)
2722 return -ENOMEM;
2723 }
2724
2725 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2726
2727 switch (mop->op) {
2728 case KVM_S390_MEMOP_LOGICAL_READ:
2729 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 2730 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2731 mop->size, GACC_FETCH);
41408c28
TH		 2732 break;
2733 }
2734 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2735 if (r == 0) {
2736 if (copy_to_user(uaddr, tmpbuf, mop->size))
2737 r = -EFAULT;
2738 }
2739 break;
2740 case KVM_S390_MEMOP_LOGICAL_WRITE:
2741 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 2742 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2743 mop->size, GACC_STORE);
41408c28
TH		 2744 break;
2745 }
2746 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2747 r = -EFAULT;
2748 break;
2749 }
2750 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2751 break;
2752 default:
2753 r = -EINVAL;
2754 }
2755
2756 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2757
2758 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2759 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2760
2761 vfree(tmpbuf);
2762 return r;
2763}
2764
b0c632db
HC		 2765long kvm_arch_vcpu_ioctl(struct file *filp,
2766 unsigned int ioctl, unsigned long arg)
2767{
2768 struct kvm_vcpu *vcpu = filp->private_data;
2769 void __user *argp = (void __user *)arg;
800c1065 2770 int idx;
bc923cc9 2771 long r;
b0c632db 2772
93736624 2773 switch (ioctl) {
47b43c52
JF		 2774 case KVM_S390_IRQ: {
2775 struct kvm_s390_irq s390irq;
2776
2777 r = -EFAULT;
2778 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2779 break;
2780 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2781 break;
2782 }
93736624 2783 case KVM_S390_INTERRUPT: {
ba5c1e9b 2784 struct kvm_s390_interrupt s390int;
383d0b05 2785 struct kvm_s390_irq s390irq;
ba5c1e9b 2786
93736624 2787 r = -EFAULT;
ba5c1e9b 2788 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624 2789 break;
383d0b05
JF		 2790 if (s390int_to_s390irq(&s390int, &s390irq))
2791 return -EINVAL;
2792 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
93736624 2793 break;
ba5c1e9b 2794 }
b0c632db 2795 case KVM_S390_STORE_STATUS:
800c1065 2796 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 2797 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 2798 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 2799 break;
b0c632db
HC		 2800 case KVM_S390_SET_INITIAL_PSW: {
2801 psw_t psw;
2802
bc923cc9 2803 r = -EFAULT;
b0c632db 2804 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 2805 break;
2806 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2807 break;
b0c632db
HC		 2808 }
2809 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 2810 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2811 break;
14eebd91
CO		 2812 case KVM_SET_ONE_REG:
2813 case KVM_GET_ONE_REG: {
2814 struct kvm_one_reg reg;
2815 r = -EFAULT;
2816 if (copy_from_user(&reg, argp, sizeof(reg)))
2817 break;
2818 if (ioctl == KVM_SET_ONE_REG)
2819 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
2820 else
2821 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
2822 break;
2823 }
27e0393f
CO		 2824#ifdef CONFIG_KVM_S390_UCONTROL
2825 case KVM_S390_UCAS_MAP: {
2826 struct kvm_s390_ucas_mapping ucasmap;
2827
2828 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2829 r = -EFAULT;
2830 break;
2831 }
2832
2833 if (!kvm_is_ucontrol(vcpu->kvm)) {
2834 r = -EINVAL;
2835 break;
2836 }
2837
2838 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2839 ucasmap.vcpu_addr, ucasmap.length);
2840 break;
2841 }
2842 case KVM_S390_UCAS_UNMAP: {
2843 struct kvm_s390_ucas_mapping ucasmap;
2844
2845 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2846 r = -EFAULT;
2847 break;
2848 }
2849
2850 if (!kvm_is_ucontrol(vcpu->kvm)) {
2851 r = -EINVAL;
2852 break;
2853 }
2854
2855 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2856 ucasmap.length);
2857 break;
2858 }
2859#endif
ccc7910f 2860 case KVM_S390_VCPU_FAULT: {
527e30b4 2861 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 2862 break;
2863 }
d6712df9
CH		 2864 case KVM_ENABLE_CAP:
2865 {
2866 struct kvm_enable_cap cap;
2867 r = -EFAULT;
2868 if (copy_from_user(&cap, argp, sizeof(cap)))
2869 break;
2870 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2871 break;
2872 }
41408c28
TH		 2873 case KVM_S390_MEM_OP: {
2874 struct kvm_s390_mem_op mem_op;
2875
2876 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2877 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2878 else
2879 r = -EFAULT;
2880 break;
2881 }
816c7667
JF		 2882 case KVM_S390_SET_IRQ_STATE: {
2883 struct kvm_s390_irq_state irq_state;
2884
2885 r = -EFAULT;
2886 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2887 break;
2888 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
2889 irq_state.len == 0 ||
2890 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
2891 r = -EINVAL;
2892 break;
2893 }
2894 r = kvm_s390_set_irq_state(vcpu,
2895 (void __user *) irq_state.buf,
2896 irq_state.len);
2897 break;
2898 }
2899 case KVM_S390_GET_IRQ_STATE: {
2900 struct kvm_s390_irq_state irq_state;
2901
2902 r = -EFAULT;
2903 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2904 break;
2905 if (irq_state.len == 0) {
2906 r = -EINVAL;
2907 break;
2908 }
2909 r = kvm_s390_get_irq_state(vcpu,
2910 (__u8 __user *) irq_state.buf,
2911 irq_state.len);
2912 break;
2913 }
b0c632db 2914 default:
3e6afcf1 2915 r = -ENOTTY;
b0c632db 2916 }
bc923cc9 2917 return r;
b0c632db
HC		 2918}
2919
5b1c1493
CO		 2920int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2921{
2922#ifdef CONFIG_KVM_S390_UCONTROL
2923 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2924 && (kvm_is_ucontrol(vcpu->kvm))) {
2925 vmf->page = virt_to_page(vcpu->arch.sie_block);
2926 get_page(vmf->page);
2927 return 0;
2928 }
2929#endif
2930 return VM_FAULT_SIGBUS;
2931}
2932
5587027c
AK		 2933int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2934 unsigned long npages)
db3fe4eb
TY		 2935{
2936 return 0;
2937}
2938
b0c632db 2939/* Section: memory related */
f7784b8e
MT		 2940int kvm_arch_prepare_memory_region(struct kvm *kvm,
2941 struct kvm_memory_slot *memslot,
09170a49 2942 const struct kvm_userspace_memory_region *mem,
7b6195a9 2943 enum kvm_mr_change change)
b0c632db 2944{
dd2887e7
NW		 2945 /* A few sanity checks. We can have memory slots which have to be
2946 located/ended at a segment boundary (1MB). The memory in userland is
2947 ok to be fragmented into various different vmas. It is okay to mmap()
2948 and munmap() stuff in this slot after doing this call at any time */
b0c632db 2949
598841ca 2950 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 2951 return -EINVAL;
2952
598841ca 2953 if (mem->memory_size & 0xffffful)
b0c632db
HC		 2954 return -EINVAL;
2955
a3a92c31
DD		 2956 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
2957 return -EINVAL;
2958
f7784b8e
MT		 2959 return 0;
2960}
2961
2962void kvm_arch_commit_memory_region(struct kvm *kvm,
09170a49 2963 const struct kvm_userspace_memory_region *mem,
8482644a 2964 const struct kvm_memory_slot *old,
f36f3f28 2965 const struct kvm_memory_slot *new,
8482644a 2966 enum kvm_mr_change change)
f7784b8e 2967{
f7850c92 2968 int rc;
f7784b8e 2969
2cef4deb
CB		 2970 /* If the basics of the memslot do not change, we do not want
2971 * to update the gmap. Every update causes several unnecessary
2972 * segment translation exceptions. This is usually handled just
2973 * fine by the normal fault handler + gmap, but it will also
2974 * cause faults on the prefix page of running guest CPUs.
2975 */
2976 if (old->userspace_addr == mem->userspace_addr &&
2977 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2978 old->npages * PAGE_SIZE == mem->memory_size)
2979 return;
598841ca
CO		 2980
2981 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2982 mem->guest_phys_addr, mem->memory_size);
2983 if (rc)
ea2cdd27 2984 pr_warn("failed to commit memory region\n");
598841ca 2985 return;
b0c632db
HC		 2986}
2987
b0c632db
HC		 2988static int __init kvm_s390_init(void)
2989{
07197fd0
DH		 2990 if (!sclp.has_sief2) {
2991 pr_info("SIE not available\n");
2992 return -ENODEV;
2993 }
2994
9d8d5786 2995 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 2996}
2997
2998static void __exit kvm_s390_exit(void)
2999{
3000 kvm_exit();
3001}
3002
3003module_init(kvm_s390_init);
3004module_exit(kvm_s390_exit);
566af940
CH		 3005
3006/*
3007 * Enable autoloading of the kvm module.
3008 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3009 * since x86 takes a different approach.
3010 */
3011#include <linux/miscdevice.h>
3012MODULE_ALIAS_MISCDEV(KVM_MINOR);
3013MODULE_ALIAS("devname:kvm");
Ubuntu Hirsute Linux kernel mirror