]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: s390: Enable the KVM-VFIO device
[mirror_ubuntu-jammy-kernel.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
24#include <linux/module.h>
a374e892 25#include <linux/random.h>
b0c632db 26#include <linux/slab.h>
ba5c1e9b 27#include <linux/timer.h>
41408c28 28#include <linux/vmalloc.h>
cbb870c8 29#include <asm/asm-offsets.h>
b0c632db 30#include <asm/lowcore.h>
fdf03650 31#include <asm/etr.h>
b0c632db 32#include <asm/pgtable.h>
f5daba1d 33#include <asm/nmi.h>
a0616cde 34#include <asm/switch_to.h>
6d3da241 35#include <asm/isc.h>
1526bf9c 36#include <asm/sclp.h>
8f2abe6a 37#include "kvm-s390.h"
b0c632db
HC		 38#include "gaccess.h"
39
ea2cdd27
DH		 40#define KMSG_COMPONENT "kvm-s390"
41#undef pr_fmt
42#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
43
5786fffa
CH		 44#define CREATE_TRACE_POINTS
45#include "trace.h"
ade38c31 46#include "trace-s390.h"
5786fffa 47
41408c28 48#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
816c7667
JF		 49#define LOCAL_IRQS 32
50#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
51 (KVM_MAX_VCPUS + LOCAL_IRQS))
41408c28 52
b0c632db
HC		 53#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
54
55struct kvm_stats_debugfs_item debugfs_entries[] = {
56 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 57 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 58 { "exit_validity", VCPU_STAT(exit_validity) },
59 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
60 { "exit_external_request", VCPU_STAT(exit_external_request) },
61 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b
CO		 62 { "exit_instruction", VCPU_STAT(exit_instruction) },
63 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
64 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
f7819512 65 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
62bea5bf 66 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
ce2e4f0b 67 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 68 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 69 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 70 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
71 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 72 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 73 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 74 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
75 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
76 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
77 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
78 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
79 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
80 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 81 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 82 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
83 { "instruction_spx", VCPU_STAT(instruction_spx) },
84 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
85 { "instruction_stap", VCPU_STAT(instruction_stap) },
86 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 87 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 88 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
89 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 90 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 91 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
92 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 93 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
5288fbf0 94 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 95 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 96 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 97 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 98 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
99 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 100 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 101 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
102 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 103 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 104 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
105 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
106 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 107 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
108 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
109 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 110 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 111 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 112 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
175a5c9e
CB		 113 { "diagnose_258", VCPU_STAT(diagnose_258) },
114 { "diagnose_308", VCPU_STAT(diagnose_308) },
115 { "diagnose_500", VCPU_STAT(diagnose_500) },
b0c632db
HC		 116 { NULL }
117};
118
9d8d5786
MM		 119/* upper facilities limit for kvm */
120unsigned long kvm_s390_fac_list_mask[] = {
a3ed8dae 121 0xffe6fffbfcfdfc40UL,
53df84f8 122 0x005e800000000000UL,
9d8d5786 123};
b0c632db 124
9d8d5786 125unsigned long kvm_s390_fac_list_mask_size(void)
78c4b59f 126{
9d8d5786
MM		 127 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
128 return ARRAY_SIZE(kvm_s390_fac_list_mask);
78c4b59f
MM		 129}
130
9d8d5786 131static struct gmap_notifier gmap_notifier;
78f26131 132debug_info_t *kvm_s390_dbf;
9d8d5786 133
b0c632db 134/* Section: not file related */
13a34e06 135int kvm_arch_hardware_enable(void)
b0c632db
HC		 136{
137 /* every s390 is virtualization enabled ;-) */
10474ae8 138 return 0;
b0c632db
HC		 139}
140
2c70fe44
CB		 141static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
142
fdf03650
FZ		 143/*
144 * This callback is executed during stop_machine(). All CPUs are therefore
145 * temporarily stopped. In order not to change guest behavior, we have to
146 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
147 * so a CPU won't be stopped while calculating with the epoch.
148 */
149static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
150 void *v)
151{
152 struct kvm *kvm;
153 struct kvm_vcpu *vcpu;
154 int i;
155 unsigned long long *delta = v;
156
157 list_for_each_entry(kvm, &vm_list, vm_list) {
158 kvm->arch.epoch -= *delta;
159 kvm_for_each_vcpu(i, vcpu, kvm) {
160 vcpu->arch.sie_block->epoch -= *delta;
161 }
162 }
163 return NOTIFY_OK;
164}
165
166static struct notifier_block kvm_clock_notifier = {
167 .notifier_call = kvm_clock_sync,
168};
169
b0c632db
HC		 170int kvm_arch_hardware_setup(void)
171{
2c70fe44
CB		 172 gmap_notifier.notifier_call = kvm_gmap_notifier;
173 gmap_register_ipte_notifier(&gmap_notifier);
fdf03650
FZ		 174 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
175 &kvm_clock_notifier);
b0c632db
HC		 176 return 0;
177}
178
179void kvm_arch_hardware_unsetup(void)
180{
2c70fe44 181 gmap_unregister_ipte_notifier(&gmap_notifier);
fdf03650
FZ		 182 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
183 &kvm_clock_notifier);
b0c632db
HC		 184}
185
b0c632db
HC		 186int kvm_arch_init(void *opaque)
187{
78f26131
CB		 188 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
189 if (!kvm_s390_dbf)
190 return -ENOMEM;
191
192 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
193 debug_unregister(kvm_s390_dbf);
194 return -ENOMEM;
195 }
196
84877d93
CH		 197 /* Register floating interrupt controller interface. */
198 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 199}
200
78f26131
CB		 201void kvm_arch_exit(void)
202{
203 debug_unregister(kvm_s390_dbf);
204}
205
b0c632db
HC		 206/* Section: device related */
207long kvm_arch_dev_ioctl(struct file *filp,
208 unsigned int ioctl, unsigned long arg)
209{
210 if (ioctl == KVM_S390_ENABLE_SIE)
211 return s390_enable_sie();
212 return -EINVAL;
213}
214
784aa3d7 215int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 216{
d7b0b5eb
CO		 217 int r;
218
2bd0ac4e 219 switch (ext) {
d7b0b5eb 220 case KVM_CAP_S390_PSW:
b6cf8788 221 case KVM_CAP_S390_GMAP:
52e16b18 222 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 223#ifdef CONFIG_KVM_S390_UCONTROL
224 case KVM_CAP_S390_UCONTROL:
225#endif
3c038e6b 226 case KVM_CAP_ASYNC_PF:
60b413c9 227 case KVM_CAP_SYNC_REGS:
14eebd91 228 case KVM_CAP_ONE_REG:
d6712df9 229 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 230 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 231 case KVM_CAP_IOEVENTFD:
c05c4186 232 case KVM_CAP_DEVICE_CTRL:
d938dc55 233 case KVM_CAP_ENABLE_CAP_VM:
78599d90 234 case KVM_CAP_S390_IRQCHIP:
f2061656 235 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 236 case KVM_CAP_MP_STATE:
47b43c52 237 case KVM_CAP_S390_INJECT_IRQ:
2444b352 238 case KVM_CAP_S390_USER_SIGP:
e44fc8c9 239 case KVM_CAP_S390_USER_STSI:
30ee2a98 240 case KVM_CAP_S390_SKEYS:
816c7667 241 case KVM_CAP_S390_IRQ_STATE:
d7b0b5eb
CO		 242 r = 1;
243 break;
41408c28
TH		 244 case KVM_CAP_S390_MEM_OP:
245 r = MEM_OP_MAX_SIZE;
246 break;
e726b1bd
CB		 247 case KVM_CAP_NR_VCPUS:
248 case KVM_CAP_MAX_VCPUS:
fe0edcb7
ED		 249 r = sclp.has_esca ? KVM_S390_ESCA_CPU_SLOTS
250 : KVM_S390_BSCA_CPU_SLOTS;
e726b1bd 251 break;
e1e2e605
NW		 252 case KVM_CAP_NR_MEMSLOTS:
253 r = KVM_USER_MEM_SLOTS;
254 break;
1526bf9c 255 case KVM_CAP_S390_COW:
abf09bed 256 r = MACHINE_HAS_ESOP;
1526bf9c 257 break;
68c55750
EF		 258 case KVM_CAP_S390_VECTOR_REGISTERS:
259 r = MACHINE_HAS_VX;
260 break;
c6e5f166
FZ		 261 case KVM_CAP_S390_RI:
262 r = test_facility(64);
263 break;
2bd0ac4e 264 default:
d7b0b5eb 265 r = 0;
2bd0ac4e 266 }
d7b0b5eb 267 return r;
b0c632db
HC		 268}
269
15f36ebd
JH		 270static void kvm_s390_sync_dirty_log(struct kvm *kvm,
271 struct kvm_memory_slot *memslot)
272{
273 gfn_t cur_gfn, last_gfn;
274 unsigned long address;
275 struct gmap *gmap = kvm->arch.gmap;
276
277 down_read(&gmap->mm->mmap_sem);
278 /* Loop over all guest pages */
279 last_gfn = memslot->base_gfn + memslot->npages;
280 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
281 address = gfn_to_hva_memslot(memslot, cur_gfn);
282
283 if (gmap_test_and_clear_dirty(address, gmap))
284 mark_page_dirty(kvm, cur_gfn);
285 }
286 up_read(&gmap->mm->mmap_sem);
287}
288
b0c632db 289/* Section: vm related */
a6e2f683
ED		 290static void sca_del_vcpu(struct kvm_vcpu *vcpu);
291
b0c632db
HC		 292/*
293 * Get (and clear) the dirty memory log for a memory slot.
294 */
295int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
296 struct kvm_dirty_log *log)
297{
15f36ebd
JH		 298 int r;
299 unsigned long n;
9f6b8029 300 struct kvm_memslots *slots;
15f36ebd
JH		 301 struct kvm_memory_slot *memslot;
302 int is_dirty = 0;
303
304 mutex_lock(&kvm->slots_lock);
305
306 r = -EINVAL;
307 if (log->slot >= KVM_USER_MEM_SLOTS)
308 goto out;
309
9f6b8029
PB		 310 slots = kvm_memslots(kvm);
311 memslot = id_to_memslot(slots, log->slot);
15f36ebd
JH		 312 r = -ENOENT;
313 if (!memslot->dirty_bitmap)
314 goto out;
315
316 kvm_s390_sync_dirty_log(kvm, memslot);
317 r = kvm_get_dirty_log(kvm, log, &is_dirty);
318 if (r)
319 goto out;
320
321 /* Clear the dirty log */
322 if (is_dirty) {
323 n = kvm_dirty_bitmap_bytes(memslot);
324 memset(memslot->dirty_bitmap, 0, n);
325 }
326 r = 0;
327out:
328 mutex_unlock(&kvm->slots_lock);
329 return r;
b0c632db
HC		 330}
331
d938dc55
CH		 332static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
333{
334 int r;
335
336 if (cap->flags)
337 return -EINVAL;
338
339 switch (cap->cap) {
84223598 340 case KVM_CAP_S390_IRQCHIP:
c92ea7b9 341 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
84223598
CH		 342 kvm->arch.use_irqchip = 1;
343 r = 0;
344 break;
2444b352 345 case KVM_CAP_S390_USER_SIGP:
c92ea7b9 346 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
2444b352
DH		 347 kvm->arch.user_sigp = 1;
348 r = 0;
349 break;
68c55750 350 case KVM_CAP_S390_VECTOR_REGISTERS:
5967c17b
DH		 351 mutex_lock(&kvm->lock);
352 if (atomic_read(&kvm->online_vcpus)) {
353 r = -EBUSY;
354 } else if (MACHINE_HAS_VX) {
18280d8b
MM		 355 set_kvm_facility(kvm->arch.model.fac->mask, 129);
356 set_kvm_facility(kvm->arch.model.fac->list, 129);
357 r = 0;
358 } else
359 r = -EINVAL;
5967c17b 360 mutex_unlock(&kvm->lock);
c92ea7b9
CB		 361 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
362 r ? "(not available)" : "(success)");
68c55750 363 break;
c6e5f166
FZ		 364 case KVM_CAP_S390_RI:
365 r = -EINVAL;
366 mutex_lock(&kvm->lock);
367 if (atomic_read(&kvm->online_vcpus)) {
368 r = -EBUSY;
369 } else if (test_facility(64)) {
370 set_kvm_facility(kvm->arch.model.fac->mask, 64);
371 set_kvm_facility(kvm->arch.model.fac->list, 64);
372 r = 0;
373 }
374 mutex_unlock(&kvm->lock);
375 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
376 r ? "(not available)" : "(success)");
377 break;
e44fc8c9 378 case KVM_CAP_S390_USER_STSI:
c92ea7b9 379 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
e44fc8c9
ET		 380 kvm->arch.user_stsi = 1;
381 r = 0;
382 break;
d938dc55
CH		 383 default:
384 r = -EINVAL;
385 break;
386 }
387 return r;
388}
389
8c0a7ce6
DD		 390static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
391{
392 int ret;
393
394 switch (attr->attr) {
395 case KVM_S390_VM_MEM_LIMIT_SIZE:
396 ret = 0;
c92ea7b9 397 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
a3a92c31
DD		 398 kvm->arch.mem_limit);
399 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
8c0a7ce6
DD		 400 ret = -EFAULT;
401 break;
402 default:
403 ret = -ENXIO;
404 break;
405 }
406 return ret;
407}
408
409static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 410{
411 int ret;
412 unsigned int idx;
413 switch (attr->attr) {
414 case KVM_S390_VM_MEM_ENABLE_CMMA:
e6db1d61
DD		 415 /* enable CMMA only for z10 and later (EDAT_1) */
416 ret = -EINVAL;
417 if (!MACHINE_IS_LPAR || !MACHINE_HAS_EDAT1)
418 break;
419
4f718eab 420 ret = -EBUSY;
c92ea7b9 421 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
4f718eab
DD		 422 mutex_lock(&kvm->lock);
423 if (atomic_read(&kvm->online_vcpus) == 0) {
424 kvm->arch.use_cmma = 1;
425 ret = 0;
426 }
427 mutex_unlock(&kvm->lock);
428 break;
429 case KVM_S390_VM_MEM_CLR_CMMA:
c3489155
DD		 430 ret = -EINVAL;
431 if (!kvm->arch.use_cmma)
432 break;
433
c92ea7b9 434 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
4f718eab
DD		 435 mutex_lock(&kvm->lock);
436 idx = srcu_read_lock(&kvm->srcu);
a13cff31 437 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 438 srcu_read_unlock(&kvm->srcu, idx);
439 mutex_unlock(&kvm->lock);
440 ret = 0;
441 break;
8c0a7ce6
DD		 442 case KVM_S390_VM_MEM_LIMIT_SIZE: {
443 unsigned long new_limit;
444
445 if (kvm_is_ucontrol(kvm))
446 return -EINVAL;
447
448 if (get_user(new_limit, (u64 __user *)attr->addr))
449 return -EFAULT;
450
a3a92c31
DD		 451 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
452 new_limit > kvm->arch.mem_limit)
8c0a7ce6
DD		 453 return -E2BIG;
454
a3a92c31
DD		 455 if (!new_limit)
456 return -EINVAL;
457
458 /* gmap_alloc takes last usable address */
459 if (new_limit != KVM_S390_NO_MEM_LIMIT)
460 new_limit -= 1;
461
8c0a7ce6
DD		 462 ret = -EBUSY;
463 mutex_lock(&kvm->lock);
464 if (atomic_read(&kvm->online_vcpus) == 0) {
465 /* gmap_alloc will round the limit up */
466 struct gmap *new = gmap_alloc(current->mm, new_limit);
467
468 if (!new) {
469 ret = -ENOMEM;
470 } else {
471 gmap_free(kvm->arch.gmap);
472 new->private = kvm;
473 kvm->arch.gmap = new;
474 ret = 0;
475 }
476 }
477 mutex_unlock(&kvm->lock);
a3a92c31
DD		 478 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
479 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
480 (void *) kvm->arch.gmap->asce);
8c0a7ce6
DD		 481 break;
482 }
4f718eab
DD		 483 default:
484 ret = -ENXIO;
485 break;
486 }
487 return ret;
488}
489
a374e892
TK		 490static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
491
492static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
493{
494 struct kvm_vcpu *vcpu;
495 int i;
496
9d8d5786 497 if (!test_kvm_facility(kvm, 76))
a374e892
TK		 498 return -EINVAL;
499
500 mutex_lock(&kvm->lock);
501 switch (attr->attr) {
502 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
503 get_random_bytes(
504 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
505 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
506 kvm->arch.crypto.aes_kw = 1;
c92ea7b9 507 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
a374e892
TK		 508 break;
509 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
510 get_random_bytes(
511 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
512 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
513 kvm->arch.crypto.dea_kw = 1;
c92ea7b9 514 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
a374e892
TK		 515 break;
516 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
517 kvm->arch.crypto.aes_kw = 0;
518 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
519 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
c92ea7b9 520 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
a374e892
TK		 521 break;
522 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
523 kvm->arch.crypto.dea_kw = 0;
524 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
525 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
c92ea7b9 526 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
a374e892
TK		 527 break;
528 default:
529 mutex_unlock(&kvm->lock);
530 return -ENXIO;
531 }
532
533 kvm_for_each_vcpu(i, vcpu, kvm) {
534 kvm_s390_vcpu_crypto_setup(vcpu);
535 exit_sie(vcpu);
536 }
537 mutex_unlock(&kvm->lock);
538 return 0;
539}
540
72f25020
JH		 541static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
542{
543 u8 gtod_high;
544
545 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
546 sizeof(gtod_high)))
547 return -EFAULT;
548
549 if (gtod_high != 0)
550 return -EINVAL;
58c383c6 551 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
72f25020
JH		 552
553 return 0;
554}
555
556static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
557{
5a3d883a 558 u64 gtod;
72f25020
JH		 559
560 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
561 return -EFAULT;
562
25ed1675 563 kvm_s390_set_tod_clock(kvm, gtod);
58c383c6 564 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
72f25020
JH		 565 return 0;
566}
567
568static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
569{
570 int ret;
571
572 if (attr->flags)
573 return -EINVAL;
574
575 switch (attr->attr) {
576 case KVM_S390_VM_TOD_HIGH:
577 ret = kvm_s390_set_tod_high(kvm, attr);
578 break;
579 case KVM_S390_VM_TOD_LOW:
580 ret = kvm_s390_set_tod_low(kvm, attr);
581 break;
582 default:
583 ret = -ENXIO;
584 break;
585 }
586 return ret;
587}
588
589static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
590{
591 u8 gtod_high = 0;
592
593 if (copy_to_user((void __user *)attr->addr, &gtod_high,
594 sizeof(gtod_high)))
595 return -EFAULT;
58c383c6 596 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
72f25020
JH		 597
598 return 0;
599}
600
601static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
602{
5a3d883a 603 u64 gtod;
72f25020 604
60417fcc 605 gtod = kvm_s390_get_tod_clock_fast(kvm);
72f25020
JH		 606 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
607 return -EFAULT;
58c383c6 608 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
72f25020
JH		 609
610 return 0;
611}
612
613static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
614{
615 int ret;
616
617 if (attr->flags)
618 return -EINVAL;
619
620 switch (attr->attr) {
621 case KVM_S390_VM_TOD_HIGH:
622 ret = kvm_s390_get_tod_high(kvm, attr);
623 break;
624 case KVM_S390_VM_TOD_LOW:
625 ret = kvm_s390_get_tod_low(kvm, attr);
626 break;
627 default:
628 ret = -ENXIO;
629 break;
630 }
631 return ret;
632}
633
658b6eda
MM		 634static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
635{
636 struct kvm_s390_vm_cpu_processor *proc;
637 int ret = 0;
638
639 mutex_lock(&kvm->lock);
640 if (atomic_read(&kvm->online_vcpus)) {
641 ret = -EBUSY;
642 goto out;
643 }
644 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
645 if (!proc) {
646 ret = -ENOMEM;
647 goto out;
648 }
649 if (!copy_from_user(proc, (void __user *)attr->addr,
650 sizeof(*proc))) {
651 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
652 sizeof(struct cpuid));
653 kvm->arch.model.ibc = proc->ibc;
981467c9 654 memcpy(kvm->arch.model.fac->list, proc->fac_list,
658b6eda
MM		 655 S390_ARCH_FAC_LIST_SIZE_BYTE);
656 } else
657 ret = -EFAULT;
658 kfree(proc);
659out:
660 mutex_unlock(&kvm->lock);
661 return ret;
662}
663
664static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
665{
666 int ret = -ENXIO;
667
668 switch (attr->attr) {
669 case KVM_S390_VM_CPU_PROCESSOR:
670 ret = kvm_s390_set_processor(kvm, attr);
671 break;
672 }
673 return ret;
674}
675
676static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
677{
678 struct kvm_s390_vm_cpu_processor *proc;
679 int ret = 0;
680
681 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
682 if (!proc) {
683 ret = -ENOMEM;
684 goto out;
685 }
686 memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
687 proc->ibc = kvm->arch.model.ibc;
981467c9 688 memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 689 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
690 ret = -EFAULT;
691 kfree(proc);
692out:
693 return ret;
694}
695
696static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
697{
698 struct kvm_s390_vm_cpu_machine *mach;
699 int ret = 0;
700
701 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
702 if (!mach) {
703 ret = -ENOMEM;
704 goto out;
705 }
706 get_cpu_id((struct cpuid *) &mach->cpuid);
37c5f6c8 707 mach->ibc = sclp.ibc;
981467c9
MM		 708 memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
709 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 710 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
94422ee8 711 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 712 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
713 ret = -EFAULT;
714 kfree(mach);
715out:
716 return ret;
717}
718
719static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
720{
721 int ret = -ENXIO;
722
723 switch (attr->attr) {
724 case KVM_S390_VM_CPU_PROCESSOR:
725 ret = kvm_s390_get_processor(kvm, attr);
726 break;
727 case KVM_S390_VM_CPU_MACHINE:
728 ret = kvm_s390_get_machine(kvm, attr);
729 break;
730 }
731 return ret;
732}
733
f2061656
DD		 734static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
735{
736 int ret;
737
738 switch (attr->group) {
4f718eab 739 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 740 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 741 break;
72f25020
JH		 742 case KVM_S390_VM_TOD:
743 ret = kvm_s390_set_tod(kvm, attr);
744 break;
658b6eda
MM		 745 case KVM_S390_VM_CPU_MODEL:
746 ret = kvm_s390_set_cpu_model(kvm, attr);
747 break;
a374e892
TK		 748 case KVM_S390_VM_CRYPTO:
749 ret = kvm_s390_vm_set_crypto(kvm, attr);
750 break;
f2061656
DD		 751 default:
752 ret = -ENXIO;
753 break;
754 }
755
756 return ret;
757}
758
759static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
760{
8c0a7ce6
DD		 761 int ret;
762
763 switch (attr->group) {
764 case KVM_S390_VM_MEM_CTRL:
765 ret = kvm_s390_get_mem_control(kvm, attr);
766 break;
72f25020
JH		 767 case KVM_S390_VM_TOD:
768 ret = kvm_s390_get_tod(kvm, attr);
769 break;
658b6eda
MM		 770 case KVM_S390_VM_CPU_MODEL:
771 ret = kvm_s390_get_cpu_model(kvm, attr);
772 break;
8c0a7ce6
DD		 773 default:
774 ret = -ENXIO;
775 break;
776 }
777
778 return ret;
f2061656
DD		 779}
780
781static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
782{
783 int ret;
784
785 switch (attr->group) {
4f718eab
DD		 786 case KVM_S390_VM_MEM_CTRL:
787 switch (attr->attr) {
788 case KVM_S390_VM_MEM_ENABLE_CMMA:
789 case KVM_S390_VM_MEM_CLR_CMMA:
8c0a7ce6 790 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 791 ret = 0;
792 break;
793 default:
794 ret = -ENXIO;
795 break;
796 }
797 break;
72f25020
JH		 798 case KVM_S390_VM_TOD:
799 switch (attr->attr) {
800 case KVM_S390_VM_TOD_LOW:
801 case KVM_S390_VM_TOD_HIGH:
802 ret = 0;
803 break;
804 default:
805 ret = -ENXIO;
806 break;
807 }
808 break;
658b6eda
MM		 809 case KVM_S390_VM_CPU_MODEL:
810 switch (attr->attr) {
811 case KVM_S390_VM_CPU_PROCESSOR:
812 case KVM_S390_VM_CPU_MACHINE:
813 ret = 0;
814 break;
815 default:
816 ret = -ENXIO;
817 break;
818 }
819 break;
a374e892
TK		 820 case KVM_S390_VM_CRYPTO:
821 switch (attr->attr) {
822 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
823 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
824 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
825 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
826 ret = 0;
827 break;
828 default:
829 ret = -ENXIO;
830 break;
831 }
832 break;
f2061656
DD		 833 default:
834 ret = -ENXIO;
835 break;
836 }
837
838 return ret;
839}
840
30ee2a98
JH		 841static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
842{
843 uint8_t *keys;
844 uint64_t hva;
845 unsigned long curkey;
846 int i, r = 0;
847
848 if (args->flags != 0)
849 return -EINVAL;
850
851 /* Is this guest using storage keys? */
852 if (!mm_use_skey(current->mm))
853 return KVM_S390_GET_SKEYS_NONE;
854
855 /* Enforce sane limit on memory allocation */
856 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
857 return -EINVAL;
858
859 keys = kmalloc_array(args->count, sizeof(uint8_t),
860 GFP_KERNEL | __GFP_NOWARN);
861 if (!keys)
862 keys = vmalloc(sizeof(uint8_t) * args->count);
863 if (!keys)
864 return -ENOMEM;
865
866 for (i = 0; i < args->count; i++) {
867 hva = gfn_to_hva(kvm, args->start_gfn + i);
868 if (kvm_is_error_hva(hva)) {
869 r = -EFAULT;
870 goto out;
871 }
872
873 curkey = get_guest_storage_key(current->mm, hva);
874 if (IS_ERR_VALUE(curkey)) {
875 r = curkey;
876 goto out;
877 }
878 keys[i] = curkey;
879 }
880
881 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
882 sizeof(uint8_t) * args->count);
883 if (r)
884 r = -EFAULT;
885out:
886 kvfree(keys);
887 return r;
888}
889
890static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
891{
892 uint8_t *keys;
893 uint64_t hva;
894 int i, r = 0;
895
896 if (args->flags != 0)
897 return -EINVAL;
898
899 /* Enforce sane limit on memory allocation */
900 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
901 return -EINVAL;
902
903 keys = kmalloc_array(args->count, sizeof(uint8_t),
904 GFP_KERNEL | __GFP_NOWARN);
905 if (!keys)
906 keys = vmalloc(sizeof(uint8_t) * args->count);
907 if (!keys)
908 return -ENOMEM;
909
910 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
911 sizeof(uint8_t) * args->count);
912 if (r) {
913 r = -EFAULT;
914 goto out;
915 }
916
917 /* Enable storage key handling for the guest */
14d4a425
DD		 918 r = s390_enable_skey();
919 if (r)
920 goto out;
30ee2a98
JH		 921
922 for (i = 0; i < args->count; i++) {
923 hva = gfn_to_hva(kvm, args->start_gfn + i);
924 if (kvm_is_error_hva(hva)) {
925 r = -EFAULT;
926 goto out;
927 }
928
929 /* Lowest order bit is reserved */
930 if (keys[i] & 0x01) {
931 r = -EINVAL;
932 goto out;
933 }
934
935 r = set_guest_storage_key(current->mm, hva,
936 (unsigned long)keys[i], 0);
937 if (r)
938 goto out;
939 }
940out:
941 kvfree(keys);
942 return r;
943}
944
b0c632db
HC		 945long kvm_arch_vm_ioctl(struct file *filp,
946 unsigned int ioctl, unsigned long arg)
947{
948 struct kvm *kvm = filp->private_data;
949 void __user *argp = (void __user *)arg;
f2061656 950 struct kvm_device_attr attr;
b0c632db
HC		 951 int r;
952
953 switch (ioctl) {
ba5c1e9b
CO		 954 case KVM_S390_INTERRUPT: {
955 struct kvm_s390_interrupt s390int;
956
957 r = -EFAULT;
958 if (copy_from_user(&s390int, argp, sizeof(s390int)))
959 break;
960 r = kvm_s390_inject_vm(kvm, &s390int);
961 break;
962 }
d938dc55
CH		 963 case KVM_ENABLE_CAP: {
964 struct kvm_enable_cap cap;
965 r = -EFAULT;
966 if (copy_from_user(&cap, argp, sizeof(cap)))
967 break;
968 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
969 break;
970 }
84223598
CH		 971 case KVM_CREATE_IRQCHIP: {
972 struct kvm_irq_routing_entry routing;
973
974 r = -EINVAL;
975 if (kvm->arch.use_irqchip) {
976 /* Set up dummy routing. */
977 memset(&routing, 0, sizeof(routing));
152b2839 978 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
84223598
CH		 979 }
980 break;
981 }
f2061656
DD		 982 case KVM_SET_DEVICE_ATTR: {
983 r = -EFAULT;
984 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
985 break;
986 r = kvm_s390_vm_set_attr(kvm, &attr);
987 break;
988 }
989 case KVM_GET_DEVICE_ATTR: {
990 r = -EFAULT;
991 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
992 break;
993 r = kvm_s390_vm_get_attr(kvm, &attr);
994 break;
995 }
996 case KVM_HAS_DEVICE_ATTR: {
997 r = -EFAULT;
998 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
999 break;
1000 r = kvm_s390_vm_has_attr(kvm, &attr);
1001 break;
1002 }
30ee2a98
JH		 1003 case KVM_S390_GET_SKEYS: {
1004 struct kvm_s390_skeys args;
1005
1006 r = -EFAULT;
1007 if (copy_from_user(&args, argp,
1008 sizeof(struct kvm_s390_skeys)))
1009 break;
1010 r = kvm_s390_get_skeys(kvm, &args);
1011 break;
1012 }
1013 case KVM_S390_SET_SKEYS: {
1014 struct kvm_s390_skeys args;
1015
1016 r = -EFAULT;
1017 if (copy_from_user(&args, argp,
1018 sizeof(struct kvm_s390_skeys)))
1019 break;
1020 r = kvm_s390_set_skeys(kvm, &args);
1021 break;
1022 }
b0c632db 1023 default:
367e1319 1024 r = -ENOTTY;
b0c632db
HC		 1025 }
1026
1027 return r;
1028}
1029
45c9b47c
TK		 1030static int kvm_s390_query_ap_config(u8 *config)
1031{
1032 u32 fcn_code = 0x04000000UL;
86044c8c 1033 u32 cc = 0;
45c9b47c 1034
86044c8c 1035 memset(config, 0, 128);
45c9b47c
TK		 1036 asm volatile(
1037 "lgr 0,%1\n"
1038 "lgr 2,%2\n"
1039 ".long 0xb2af0000\n" /* PQAP(QCI) */
86044c8c 1040 "0: ipm %0\n"
45c9b47c 1041 "srl %0,28\n"
86044c8c
CB		 1042 "1:\n"
1043 EX_TABLE(0b, 1b)
1044 : "+r" (cc)
45c9b47c
TK		 1045 : "r" (fcn_code), "r" (config)
1046 : "cc", "0", "2", "memory"
1047 );
1048
1049 return cc;
1050}
1051
1052static int kvm_s390_apxa_installed(void)
1053{
1054 u8 config[128];
1055 int cc;
1056
a6aacc3f 1057 if (test_facility(12)) {
45c9b47c
TK		 1058 cc = kvm_s390_query_ap_config(config);
1059
1060 if (cc)
1061 pr_err("PQAP(QCI) failed with cc=%d", cc);
1062 else
1063 return config[0] & 0x40;
1064 }
1065
1066 return 0;
1067}
1068
1069static void kvm_s390_set_crycb_format(struct kvm *kvm)
1070{
1071 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1072
1073 if (kvm_s390_apxa_installed())
1074 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1075 else
1076 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1077}
1078
9d8d5786
MM		 1079static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
1080{
1081 get_cpu_id(cpu_id);
1082 cpu_id->version = 0xff;
1083}
1084
5102ee87
TK		 1085static int kvm_s390_crypto_init(struct kvm *kvm)
1086{
9d8d5786 1087 if (!test_kvm_facility(kvm, 76))
5102ee87
TK		 1088 return 0;
1089
1090 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
1091 GFP_KERNEL | GFP_DMA);
1092 if (!kvm->arch.crypto.crycb)
1093 return -ENOMEM;
1094
45c9b47c 1095 kvm_s390_set_crycb_format(kvm);
5102ee87 1096
ed6f76b4
TK		 1097 /* Enable AES/DEA protected key functions by default */
1098 kvm->arch.crypto.aes_kw = 1;
1099 kvm->arch.crypto.dea_kw = 1;
1100 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1101 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1102 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1103 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
a374e892 1104
5102ee87
TK		 1105 return 0;
1106}
1107
7d43bafc
ED		 1108static void sca_dispose(struct kvm *kvm)
1109{
1110 if (kvm->arch.use_esca)
5e044315 1111 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
7d43bafc
ED		 1112 else
1113 free_page((unsigned long)(kvm->arch.sca));
1114 kvm->arch.sca = NULL;
1115}
1116
e08b9637 1117int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 1118{
9d8d5786 1119 int i, rc;
b0c632db 1120 char debug_name[16];
f6c137ff 1121 static unsigned long sca_offset;
b0c632db 1122
e08b9637
CO		 1123 rc = -EINVAL;
1124#ifdef CONFIG_KVM_S390_UCONTROL
1125 if (type & ~KVM_VM_S390_UCONTROL)
1126 goto out_err;
1127 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1128 goto out_err;
1129#else
1130 if (type)
1131 goto out_err;
1132#endif
1133
b0c632db
HC		 1134 rc = s390_enable_sie();
1135 if (rc)
d89f5eff 1136 goto out_err;
b0c632db 1137
b290411a
CO		 1138 rc = -ENOMEM;
1139
7d43bafc 1140 kvm->arch.use_esca = 0; /* start with basic SCA */
5e044315 1141 rwlock_init(&kvm->arch.sca_lock);
bc784cce 1142 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(GFP_KERNEL);
b0c632db 1143 if (!kvm->arch.sca)
d89f5eff 1144 goto out_err;
f6c137ff 1145 spin_lock(&kvm_lock);
c5c2c393 1146 sca_offset += 16;
bc784cce 1147 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
c5c2c393 1148 sca_offset = 0;
bc784cce
ED		 1149 kvm->arch.sca = (struct bsca_block *)
1150 ((char *) kvm->arch.sca + sca_offset);
f6c137ff 1151 spin_unlock(&kvm_lock);
b0c632db
HC		 1152
1153 sprintf(debug_name, "kvm-%u", current->pid);
1154
1cb9cf72 1155 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
b0c632db 1156 if (!kvm->arch.dbf)
40f5b735 1157 goto out_err;
b0c632db 1158
9d8d5786
MM		 1159 /*
1160 * The architectural maximum amount of facilities is 16 kbit. To store
1161 * this amount, 2 kbyte of memory is required. Thus we need a full
981467c9
MM		 1162 * page to hold the guest facility list (arch.model.fac->list) and the
1163 * facility mask (arch.model.fac->mask). Its address size has to be
9d8d5786
MM		 1164 * 31 bits and word aligned.
1165 */
1166 kvm->arch.model.fac =
981467c9 1167 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
9d8d5786 1168 if (!kvm->arch.model.fac)
40f5b735 1169 goto out_err;
9d8d5786 1170
fb5bf93f 1171 /* Populate the facility mask initially. */
981467c9 1172 memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
94422ee8 1173 S390_ARCH_FAC_LIST_SIZE_BYTE);
9d8d5786
MM		 1174 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1175 if (i < kvm_s390_fac_list_mask_size())
981467c9 1176 kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
9d8d5786 1177 else
981467c9 1178 kvm->arch.model.fac->mask[i] = 0UL;
9d8d5786
MM		 1179 }
1180
981467c9
MM		 1181 /* Populate the facility list initially. */
1182 memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
1183 S390_ARCH_FAC_LIST_SIZE_BYTE);
1184
9d8d5786 1185 kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
37c5f6c8 1186 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
9d8d5786 1187
5102ee87 1188 if (kvm_s390_crypto_init(kvm) < 0)
40f5b735 1189 goto out_err;
5102ee87 1190
ba5c1e9b 1191 spin_lock_init(&kvm->arch.float_int.lock);
6d3da241
JF		 1192 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1193 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
8a242234 1194 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 1195 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 1196
b0c632db 1197 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
78f26131 1198 VM_EVENT(kvm, 3, "vm created with type %lu", type);
b0c632db 1199
e08b9637
CO		 1200 if (type & KVM_VM_S390_UCONTROL) {
1201 kvm->arch.gmap = NULL;
a3a92c31 1202 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
e08b9637 1203 } else {
32e6b236
GH		 1204 if (sclp.hamax == U64_MAX)
1205 kvm->arch.mem_limit = TASK_MAX_SIZE;
1206 else
1207 kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
1208 sclp.hamax + 1);
a3a92c31 1209 kvm->arch.gmap = gmap_alloc(current->mm, kvm->arch.mem_limit - 1);
e08b9637 1210 if (!kvm->arch.gmap)
40f5b735 1211 goto out_err;
2c70fe44 1212 kvm->arch.gmap->private = kvm;
24eb3a82 1213 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 1214 }
fa6b7fe9
CH		 1215
1216 kvm->arch.css_support = 0;
84223598 1217 kvm->arch.use_irqchip = 0;
72f25020 1218 kvm->arch.epoch = 0;
fa6b7fe9 1219
8ad35755 1220 spin_lock_init(&kvm->arch.start_stop_lock);
8335713a 1221 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
8ad35755 1222
d89f5eff 1223 return 0;
40f5b735 1224out_err:
5102ee87 1225 kfree(kvm->arch.crypto.crycb);
9d8d5786 1226 free_page((unsigned long)kvm->arch.model.fac);
598841ca 1227 debug_unregister(kvm->arch.dbf);
7d43bafc 1228 sca_dispose(kvm);
78f26131 1229 KVM_EVENT(3, "creation of vm failed: %d", rc);
d89f5eff 1230 return rc;
b0c632db
HC		 1231}
1232
d329c035
CB		 1233void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1234{
1235 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 1236 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 1237 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 1238 kvm_clear_async_pf_completion_queue(vcpu);
bc784cce 1239 if (!kvm_is_ucontrol(vcpu->kvm))
a6e2f683 1240 sca_del_vcpu(vcpu);
27e0393f
CO		 1241
1242 if (kvm_is_ucontrol(vcpu->kvm))
1243 gmap_free(vcpu->arch.gmap);
1244
e6db1d61 1245 if (vcpu->kvm->arch.use_cmma)
b31605c1 1246 kvm_s390_vcpu_unsetup_cmma(vcpu);
9c7ebb61 1247 kfree(vcpu->arch.guest_fpregs.fprs);
d329c035 1248 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 1249
6692cef3 1250 kvm_vcpu_uninit(vcpu);
b110feaf 1251 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 1252}
1253
1254static void kvm_free_vcpus(struct kvm *kvm)
1255{
1256 unsigned int i;
988a2cae 1257 struct kvm_vcpu *vcpu;
d329c035 1258
988a2cae
GN		 1259 kvm_for_each_vcpu(i, vcpu, kvm)
1260 kvm_arch_vcpu_destroy(vcpu);
1261
1262 mutex_lock(&kvm->lock);
1263 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1264 kvm->vcpus[i] = NULL;
1265
1266 atomic_set(&kvm->online_vcpus, 0);
1267 mutex_unlock(&kvm->lock);
d329c035
CB		 1268}
1269
b0c632db
HC		 1270void kvm_arch_destroy_vm(struct kvm *kvm)
1271{
d329c035 1272 kvm_free_vcpus(kvm);
9d8d5786 1273 free_page((unsigned long)kvm->arch.model.fac);
7d43bafc 1274 sca_dispose(kvm);
d329c035 1275 debug_unregister(kvm->arch.dbf);
5102ee87 1276 kfree(kvm->arch.crypto.crycb);
27e0393f
CO		 1277 if (!kvm_is_ucontrol(kvm))
1278 gmap_free(kvm->arch.gmap);
841b91c5 1279 kvm_s390_destroy_adapters(kvm);
67335e63 1280 kvm_s390_clear_float_irqs(kvm);
8335713a 1281 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
b0c632db
HC		 1282}
1283
1284/* Section: vcpu related */
dafd032a
DD		 1285static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1286{
1287 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1288 if (!vcpu->arch.gmap)
1289 return -ENOMEM;
1290 vcpu->arch.gmap->private = vcpu->kvm;
1291
1292 return 0;
1293}
1294
a6e2f683
ED		 1295static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1296{
5e044315 1297 read_lock(&vcpu->kvm->arch.sca_lock);
7d43bafc
ED		 1298 if (vcpu->kvm->arch.use_esca) {
1299 struct esca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1300
7d43bafc 1301 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
10ce32d5 1302 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc
ED		 1303 } else {
1304 struct bsca_block *sca = vcpu->kvm->arch.sca;
1305
1306 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
10ce32d5 1307 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc 1308 }
5e044315 1309 read_unlock(&vcpu->kvm->arch.sca_lock);
a6e2f683
ED		 1310}
1311
eaa78f34 1312static void sca_add_vcpu(struct kvm_vcpu *vcpu)
a6e2f683 1313{
eaa78f34
DH		 1314 read_lock(&vcpu->kvm->arch.sca_lock);
1315 if (vcpu->kvm->arch.use_esca) {
1316 struct esca_block *sca = vcpu->kvm->arch.sca;
7d43bafc 1317
eaa78f34 1318 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1319 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1320 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
25508824 1321 vcpu->arch.sie_block->ecb2 |= 0x04U;
eaa78f34 1322 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
7d43bafc 1323 } else {
eaa78f34 1324 struct bsca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1325
eaa78f34 1326 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1327 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1328 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
eaa78f34 1329 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
7d43bafc 1330 }
eaa78f34 1331 read_unlock(&vcpu->kvm->arch.sca_lock);
5e044315
ED		 1332}
1333
1334/* Basic SCA to Extended SCA data copy routines */
1335static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1336{
1337 d->sda = s->sda;
1338 d->sigp_ctrl.c = s->sigp_ctrl.c;
1339 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1340}
1341
1342static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1343{
1344 int i;
1345
1346 d->ipte_control = s->ipte_control;
1347 d->mcn[0] = s->mcn;
1348 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1349 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1350}
1351
1352static int sca_switch_to_extended(struct kvm *kvm)
1353{
1354 struct bsca_block *old_sca = kvm->arch.sca;
1355 struct esca_block *new_sca;
1356 struct kvm_vcpu *vcpu;
1357 unsigned int vcpu_idx;
1358 u32 scaol, scaoh;
1359
1360 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1361 if (!new_sca)
1362 return -ENOMEM;
1363
1364 scaoh = (u32)((u64)(new_sca) >> 32);
1365 scaol = (u32)(u64)(new_sca) & ~0x3fU;
1366
1367 kvm_s390_vcpu_block_all(kvm);
1368 write_lock(&kvm->arch.sca_lock);
1369
1370 sca_copy_b_to_e(new_sca, old_sca);
1371
1372 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1373 vcpu->arch.sie_block->scaoh = scaoh;
1374 vcpu->arch.sie_block->scaol = scaol;
1375 vcpu->arch.sie_block->ecb2 |= 0x04U;
1376 }
1377 kvm->arch.sca = new_sca;
1378 kvm->arch.use_esca = 1;
1379
1380 write_unlock(&kvm->arch.sca_lock);
1381 kvm_s390_vcpu_unblock_all(kvm);
1382
1383 free_page((unsigned long)old_sca);
1384
8335713a
CB		 1385 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1386 old_sca, kvm->arch.sca);
5e044315 1387 return 0;
a6e2f683
ED		 1388}
1389
1390static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1391{
5e044315
ED		 1392 int rc;
1393
1394 if (id < KVM_S390_BSCA_CPU_SLOTS)
1395 return true;
1396 if (!sclp.has_esca)
1397 return false;
1398
1399 mutex_lock(&kvm->lock);
1400 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1401 mutex_unlock(&kvm->lock);
1402
1403 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
a6e2f683
ED		 1404}
1405
b0c632db
HC		 1406int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1407{
3c038e6b
DD		 1408 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1409 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 1410 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1411 KVM_SYNC_GPRS |
9eed0735 1412 KVM_SYNC_ACRS |
b028ee3e
DH		 1413 KVM_SYNC_CRS |
1414 KVM_SYNC_ARCH0 |
1415 KVM_SYNC_PFAULT;
c6e5f166
FZ		 1416 if (test_kvm_facility(vcpu->kvm, 64))
1417 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
68c55750
EF		 1418 if (test_kvm_facility(vcpu->kvm, 129))
1419 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
dafd032a
DD		 1420
1421 if (kvm_is_ucontrol(vcpu->kvm))
1422 return __kvm_ucontrol_vcpu_init(vcpu);
1423
b0c632db
HC		 1424 return 0;
1425}
1426
9977e886
HB		 1427/*
1428 * Backs up the current FP/VX register save area on a particular
1429 * destination. Used to switch between different register save
1430 * areas.
1431 */
1432static inline void save_fpu_to(struct fpu *dst)
1433{
1434 dst->fpc = current->thread.fpu.fpc;
9977e886
HB		 1435 dst->regs = current->thread.fpu.regs;
1436}
1437
1438/*
1439 * Switches the FP/VX register save area from which to lazy
1440 * restore register contents.
1441 */
1442static inline void load_fpu_from(struct fpu *from)
1443{
1444 current->thread.fpu.fpc = from->fpc;
9977e886
HB		 1445 current->thread.fpu.regs = from->regs;
1446}
1447
b0c632db
HC		 1448void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1449{
9977e886 1450 /* Save host register state */
d0164ee2 1451 save_fpu_regs();
9977e886 1452 save_fpu_to(&vcpu->arch.host_fpregs);
96b2d7a8 1453
18280d8b 1454 if (test_kvm_facility(vcpu->kvm, 129)) {
9977e886 1455 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
9977e886
HB		 1456 /*
1457 * Use the register save area in the SIE-control block
1458 * for register restore and save in kvm_arch_vcpu_put()
1459 */
1460 current->thread.fpu.vxrs =
1461 (__vector128 *)&vcpu->run->s.regs.vrs;
9977e886
HB		 1462 } else
1463 load_fpu_from(&vcpu->arch.guest_fpregs);
1464
1465 if (test_fp_ctl(current->thread.fpu.fpc))
96b2d7a8 1466 /* User space provided an invalid FPC, let's clear it */
9977e886
HB		 1467 current->thread.fpu.fpc = 0;
1468
1469 save_access_regs(vcpu->arch.host_acrs);
59674c1a 1470 restore_access_regs(vcpu->run->s.regs.acrs);
480e5926 1471 gmap_enable(vcpu->arch.gmap);
805de8f4 1472 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
b0c632db
HC		 1473}
1474
1475void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1476{
805de8f4 1477 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
480e5926 1478 gmap_disable(vcpu->arch.gmap);
9977e886 1479
d0164ee2 1480 save_fpu_regs();
9977e886 1481
18280d8b 1482 if (test_kvm_facility(vcpu->kvm, 129))
9977e886
HB		 1483 /*
1484 * kvm_arch_vcpu_load() set up the register save area to
1485 * the &vcpu->run->s.regs.vrs and, thus, the vector registers
1486 * are already saved. Only the floating-point control must be
1487 * copied.
1488 */
1489 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
68c55750 1490 else
9977e886
HB		 1491 save_fpu_to(&vcpu->arch.guest_fpregs);
1492 load_fpu_from(&vcpu->arch.host_fpregs);
1493
1494 save_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 1495 restore_access_regs(vcpu->arch.host_acrs);
1496}
1497
1498static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1499{
1500 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1501 vcpu->arch.sie_block->gpsw.mask = 0UL;
1502 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 1503 kvm_s390_set_prefix(vcpu, 0);
b0c632db
HC		 1504 vcpu->arch.sie_block->cputm = 0UL;
1505 vcpu->arch.sie_block->ckc = 0UL;
1506 vcpu->arch.sie_block->todpr = 0;
1507 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1508 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1509 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1510 vcpu->arch.guest_fpregs.fpc = 0;
1511 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
1512 vcpu->arch.sie_block->gbea = 1;
672550fb 1513 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 1514 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1515 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 1516 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1517 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 1518 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 1519}
1520
31928aa5 1521void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 1522{
72f25020 1523 mutex_lock(&vcpu->kvm->lock);
fdf03650 1524 preempt_disable();
72f25020 1525 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
fdf03650 1526 preempt_enable();
72f25020 1527 mutex_unlock(&vcpu->kvm->lock);
25508824 1528 if (!kvm_is_ucontrol(vcpu->kvm)) {
dafd032a 1529 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
eaa78f34 1530 sca_add_vcpu(vcpu);
25508824
DH		 1531 }
1532
42897d86
MT		 1533}
1534
5102ee87
TK		 1535static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1536{
9d8d5786 1537 if (!test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 1538 return;
1539
a374e892
TK		 1540 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1541
1542 if (vcpu->kvm->arch.crypto.aes_kw)
1543 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1544 if (vcpu->kvm->arch.crypto.dea_kw)
1545 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1546
5102ee87
TK		 1547 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1548}
1549
b31605c1
DD		 1550void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1551{
1552 free_page(vcpu->arch.sie_block->cbrlo);
1553 vcpu->arch.sie_block->cbrlo = 0;
1554}
1555
1556int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1557{
1558 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1559 if (!vcpu->arch.sie_block->cbrlo)
1560 return -ENOMEM;
1561
1562 vcpu->arch.sie_block->ecb2 |= 0x80;
1563 vcpu->arch.sie_block->ecb2 &= ~0x08;
1564 return 0;
1565}
1566
91520f1a
MM		 1567static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1568{
1569 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1570
1571 vcpu->arch.cpu_id = model->cpu_id;
1572 vcpu->arch.sie_block->ibc = model->ibc;
1573 vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1574}
1575
b0c632db
HC		 1576int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1577{
b31605c1 1578 int rc = 0;
b31288fa 1579
9e6dabef
CH		 1580 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1581 CPUSTAT_SM |
a4a4f191
GH		 1582 CPUSTAT_STOPPED);
1583
53df84f8 1584 if (test_kvm_facility(vcpu->kvm, 78))
805de8f4 1585 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
53df84f8 1586 else if (test_kvm_facility(vcpu->kvm, 8))
805de8f4 1587 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
a4a4f191 1588
91520f1a
MM		 1589 kvm_s390_vcpu_setup_model(vcpu);
1590
fc34531d 1591 vcpu->arch.sie_block->ecb = 6;
9d8d5786 1592 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
7feb6bb8
MM		 1593 vcpu->arch.sie_block->ecb |= 0x10;
1594
69d0d3a3 1595 vcpu->arch.sie_block->ecb2 = 8;
ea5f4969 1596 vcpu->arch.sie_block->eca = 0xC1002000U;
37c5f6c8 1597 if (sclp.has_siif)
217a4406 1598 vcpu->arch.sie_block->eca |= 1;
37c5f6c8 1599 if (sclp.has_sigpif)
ea5f4969 1600 vcpu->arch.sie_block->eca |= 0x10000000U;
c6e5f166
FZ		 1601 if (test_kvm_facility(vcpu->kvm, 64))
1602 vcpu->arch.sie_block->ecb3 |= 0x01;
18280d8b 1603 if (test_kvm_facility(vcpu->kvm, 129)) {
13211ea7
EF		 1604 vcpu->arch.sie_block->eca |= 0x00020000;
1605 vcpu->arch.sie_block->ecd |= 0x20000000;
1606 }
c6e5f166 1607 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
492d8642 1608 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
5a5e6536 1609
e6db1d61 1610 if (vcpu->kvm->arch.use_cmma) {
b31605c1
DD		 1611 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1612 if (rc)
1613 return rc;
b31288fa 1614 }
0ac96caf 1615 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 1616 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 1617
5102ee87
TK		 1618 kvm_s390_vcpu_crypto_setup(vcpu);
1619
b31605c1 1620 return rc;
b0c632db
HC		 1621}
1622
1623struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1624 unsigned int id)
1625{
4d47555a 1626 struct kvm_vcpu *vcpu;
7feb6bb8 1627 struct sie_page *sie_page;
4d47555a
CO		 1628 int rc = -EINVAL;
1629
4215825e 1630 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
4d47555a
CO		 1631 goto out;
1632
1633 rc = -ENOMEM;
b0c632db 1634
b110feaf 1635 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 1636 if (!vcpu)
4d47555a 1637 goto out;
b0c632db 1638
7feb6bb8
MM		 1639 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1640 if (!sie_page)
b0c632db
HC		 1641 goto out_free_cpu;
1642
7feb6bb8
MM		 1643 vcpu->arch.sie_block = &sie_page->sie_block;
1644 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1645
b0c632db 1646 vcpu->arch.sie_block->icpua = id;
ba5c1e9b 1647 spin_lock_init(&vcpu->arch.local_int.lock);
ba5c1e9b 1648 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 1649 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 1650 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
ba5c1e9b 1651
9977e886
HB		 1652 /*
1653 * Allocate a save area for floating-point registers. If the vector
1654 * extension is available, register contents are saved in the SIE
1655 * control block. The allocated save area is still required in
1656 * particular places, for example, in kvm_s390_vcpu_store_status().
1657 */
1658 vcpu->arch.guest_fpregs.fprs = kzalloc(sizeof(freg_t) * __NUM_FPRS,
1659 GFP_KERNEL);
2f8a43d4 1660 if (!vcpu->arch.guest_fpregs.fprs)
9977e886 1661 goto out_free_sie_block;
9977e886 1662
b0c632db
HC		 1663 rc = kvm_vcpu_init(vcpu, kvm, id);
1664 if (rc)
9c7ebb61 1665 goto out_free_fprs;
8335713a 1666 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
b0c632db 1667 vcpu->arch.sie_block);
ade38c31 1668 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 1669
b0c632db 1670 return vcpu;
9c7ebb61
DH		 1671out_free_fprs:
1672 kfree(vcpu->arch.guest_fpregs.fprs);
7b06bf2f
WY		 1673out_free_sie_block:
1674 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 1675out_free_cpu:
b110feaf 1676 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 1677out:
b0c632db
HC		 1678 return ERR_PTR(rc);
1679}
1680
b0c632db
HC		 1681int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1682{
9a022067 1683 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 1684}
1685
27406cd5 1686void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
49b99e1e 1687{
805de8f4 1688 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
61a6df54 1689 exit_sie(vcpu);
49b99e1e
CB		 1690}
1691
27406cd5 1692void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
49b99e1e 1693{
805de8f4 1694 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
49b99e1e
CB		 1695}
1696
8e236546
CB		 1697static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1698{
805de8f4 1699 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
61a6df54 1700 exit_sie(vcpu);
8e236546
CB		 1701}
1702
1703static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1704{
9bf9fde2 1705 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
8e236546
CB		 1706}
1707
49b99e1e
CB		 1708/*
1709 * Kick a guest cpu out of SIE and wait until SIE is not running.
1710 * If the CPU is not running (e.g. waiting as idle) the function will
1711 * return immediately. */
1712void exit_sie(struct kvm_vcpu *vcpu)
1713{
805de8f4 1714 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
49b99e1e
CB		 1715 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1716 cpu_relax();
1717}
1718
8e236546
CB		 1719/* Kick a guest cpu out of SIE to process a request synchronously */
1720void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
49b99e1e 1721{
8e236546
CB		 1722 kvm_make_request(req, vcpu);
1723 kvm_s390_vcpu_request(vcpu);
49b99e1e
CB		 1724}
1725
2c70fe44
CB		 1726static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1727{
1728 int i;
1729 struct kvm *kvm = gmap->private;
1730 struct kvm_vcpu *vcpu;
1731
1732 kvm_for_each_vcpu(i, vcpu, kvm) {
1733 /* match against both prefix pages */
fda902cb 1734 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
2c70fe44 1735 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
8e236546 1736 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44
CB		 1737 }
1738 }
1739}
1740
b6d33834
CD		 1741int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1742{
1743 /* kvm common code refers to this, but never calls it */
1744 BUG();
1745 return 0;
1746}
1747
14eebd91
CO		 1748static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1749 struct kvm_one_reg *reg)
1750{
1751 int r = -EINVAL;
1752
1753 switch (reg->id) {
29b7c71b
CO		 1754 case KVM_REG_S390_TODPR:
1755 r = put_user(vcpu->arch.sie_block->todpr,
1756 (u32 __user *)reg->addr);
1757 break;
1758 case KVM_REG_S390_EPOCHDIFF:
1759 r = put_user(vcpu->arch.sie_block->epoch,
1760 (u64 __user *)reg->addr);
1761 break;
46a6dd1c
J		 1762 case KVM_REG_S390_CPU_TIMER:
1763 r = put_user(vcpu->arch.sie_block->cputm,
1764 (u64 __user *)reg->addr);
1765 break;
1766 case KVM_REG_S390_CLOCK_COMP:
1767 r = put_user(vcpu->arch.sie_block->ckc,
1768 (u64 __user *)reg->addr);
1769 break;
536336c2
DD		 1770 case KVM_REG_S390_PFTOKEN:
1771 r = put_user(vcpu->arch.pfault_token,
1772 (u64 __user *)reg->addr);
1773 break;
1774 case KVM_REG_S390_PFCOMPARE:
1775 r = put_user(vcpu->arch.pfault_compare,
1776 (u64 __user *)reg->addr);
1777 break;
1778 case KVM_REG_S390_PFSELECT:
1779 r = put_user(vcpu->arch.pfault_select,
1780 (u64 __user *)reg->addr);
1781 break;
672550fb
CB		 1782 case KVM_REG_S390_PP:
1783 r = put_user(vcpu->arch.sie_block->pp,
1784 (u64 __user *)reg->addr);
1785 break;
afa45ff5
CB		 1786 case KVM_REG_S390_GBEA:
1787 r = put_user(vcpu->arch.sie_block->gbea,
1788 (u64 __user *)reg->addr);
1789 break;
14eebd91
CO		 1790 default:
1791 break;
1792 }
1793
1794 return r;
1795}
1796
1797static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1798 struct kvm_one_reg *reg)
1799{
1800 int r = -EINVAL;
1801
1802 switch (reg->id) {
29b7c71b
CO		 1803 case KVM_REG_S390_TODPR:
1804 r = get_user(vcpu->arch.sie_block->todpr,
1805 (u32 __user *)reg->addr);
1806 break;
1807 case KVM_REG_S390_EPOCHDIFF:
1808 r = get_user(vcpu->arch.sie_block->epoch,
1809 (u64 __user *)reg->addr);
1810 break;
46a6dd1c
J		 1811 case KVM_REG_S390_CPU_TIMER:
1812 r = get_user(vcpu->arch.sie_block->cputm,
1813 (u64 __user *)reg->addr);
1814 break;
1815 case KVM_REG_S390_CLOCK_COMP:
1816 r = get_user(vcpu->arch.sie_block->ckc,
1817 (u64 __user *)reg->addr);
1818 break;
536336c2
DD		 1819 case KVM_REG_S390_PFTOKEN:
1820 r = get_user(vcpu->arch.pfault_token,
1821 (u64 __user *)reg->addr);
9fbd8082
DH		 1822 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1823 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 1824 break;
1825 case KVM_REG_S390_PFCOMPARE:
1826 r = get_user(vcpu->arch.pfault_compare,
1827 (u64 __user *)reg->addr);
1828 break;
1829 case KVM_REG_S390_PFSELECT:
1830 r = get_user(vcpu->arch.pfault_select,
1831 (u64 __user *)reg->addr);
1832 break;
672550fb
CB		 1833 case KVM_REG_S390_PP:
1834 r = get_user(vcpu->arch.sie_block->pp,
1835 (u64 __user *)reg->addr);
1836 break;
afa45ff5
CB		 1837 case KVM_REG_S390_GBEA:
1838 r = get_user(vcpu->arch.sie_block->gbea,
1839 (u64 __user *)reg->addr);
1840 break;
14eebd91
CO		 1841 default:
1842 break;
1843 }
1844
1845 return r;
1846}
b6d33834 1847
b0c632db
HC		 1848static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1849{
b0c632db 1850 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 1851 return 0;
1852}
1853
1854int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1855{
5a32c1af 1856 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 1857 return 0;
1858}
1859
1860int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1861{
5a32c1af 1862 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 1863 return 0;
1864}
1865
1866int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1867 struct kvm_sregs *sregs)
1868{
59674c1a 1869 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 1870 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
59674c1a 1871 restore_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 1872 return 0;
1873}
1874
1875int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1876 struct kvm_sregs *sregs)
1877{
59674c1a 1878 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 1879 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 1880 return 0;
1881}
1882
1883int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1884{
4725c860
MS		 1885 if (test_fp_ctl(fpu->fpc))
1886 return -EINVAL;
9977e886 1887 memcpy(vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
4725c860 1888 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
d0164ee2 1889 save_fpu_regs();
9977e886 1890 load_fpu_from(&vcpu->arch.guest_fpregs);
b0c632db
HC		 1891 return 0;
1892}
1893
1894int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1895{
9977e886 1896 memcpy(&fpu->fprs, vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
b0c632db 1897 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
b0c632db
HC		 1898 return 0;
1899}
1900
1901static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1902{
1903 int rc = 0;
1904
7a42fdc2 1905 if (!is_vcpu_stopped(vcpu))
b0c632db 1906 rc = -EBUSY;
d7b0b5eb
CO		 1907 else {
1908 vcpu->run->psw_mask = psw.mask;
1909 vcpu->run->psw_addr = psw.addr;
1910 }
b0c632db
HC		 1911 return rc;
1912}
1913
1914int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1915 struct kvm_translation *tr)
1916{
1917 return -EINVAL; /* not implemented yet */
1918}
1919
27291e21
DH		 1920#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1921 KVM_GUESTDBG_USE_HW_BP | \
1922 KVM_GUESTDBG_ENABLE)
1923
d0bfb940
JK		 1924int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1925 struct kvm_guest_debug *dbg)
b0c632db 1926{
27291e21
DH		 1927 int rc = 0;
1928
1929 vcpu->guest_debug = 0;
1930 kvm_s390_clear_bp_data(vcpu);
1931
2de3bfc2 1932 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21
DH		 1933 return -EINVAL;
1934
1935 if (dbg->control & KVM_GUESTDBG_ENABLE) {
1936 vcpu->guest_debug = dbg->control;
1937 /* enforce guest PER */
805de8f4 1938 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 1939
1940 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1941 rc = kvm_s390_import_bp_data(vcpu, dbg);
1942 } else {
805de8f4 1943 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 1944 vcpu->arch.guestdbg.last_bp = 0;
1945 }
1946
1947 if (rc) {
1948 vcpu->guest_debug = 0;
1949 kvm_s390_clear_bp_data(vcpu);
805de8f4 1950 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 1951 }
1952
1953 return rc;
b0c632db
HC		 1954}
1955
62d9f0db
MT		 1956int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1957 struct kvm_mp_state *mp_state)
1958{
6352e4d2
DH		 1959 /* CHECK_STOP and LOAD are not supported yet */
1960 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1961 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 1962}
1963
1964int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1965 struct kvm_mp_state *mp_state)
1966{
6352e4d2
DH		 1967 int rc = 0;
1968
1969 /* user space knows about this interface - let it control the state */
1970 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1971
1972 switch (mp_state->mp_state) {
1973 case KVM_MP_STATE_STOPPED:
1974 kvm_s390_vcpu_stop(vcpu);
1975 break;
1976 case KVM_MP_STATE_OPERATING:
1977 kvm_s390_vcpu_start(vcpu);
1978 break;
1979 case KVM_MP_STATE_LOAD:
1980 case KVM_MP_STATE_CHECK_STOP:
1981 /* fall through - CHECK_STOP and LOAD are not supported yet */
1982 default:
1983 rc = -ENXIO;
1984 }
1985
1986 return rc;
62d9f0db
MT		 1987}
1988
8ad35755
DH		 1989static bool ibs_enabled(struct kvm_vcpu *vcpu)
1990{
1991 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1992}
1993
2c70fe44
CB		 1994static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1995{
8ad35755 1996retry:
8e236546 1997 kvm_s390_vcpu_request_handled(vcpu);
586b7ccd
CB		 1998 if (!vcpu->requests)
1999 return 0;
2c70fe44
CB		 2000 /*
2001 * We use MMU_RELOAD just to re-arm the ipte notifier for the
2002 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
2003 * This ensures that the ipte instruction for this request has
2004 * already finished. We might race against a second unmapper that
2005 * wants to set the blocking bit. Lets just retry the request loop.
2006 */
8ad35755 2007 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44
CB		 2008 int rc;
2009 rc = gmap_ipte_notify(vcpu->arch.gmap,
fda902cb 2010 kvm_s390_get_prefix(vcpu),
2c70fe44
CB		 2011 PAGE_SIZE * 2);
2012 if (rc)
2013 return rc;
8ad35755 2014 goto retry;
2c70fe44 2015 }
8ad35755 2016
d3d692c8
DH		 2017 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2018 vcpu->arch.sie_block->ihcpu = 0xffff;
2019 goto retry;
2020 }
2021
8ad35755
DH		 2022 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2023 if (!ibs_enabled(vcpu)) {
2024 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
805de8f4 2025 atomic_or(CPUSTAT_IBS,
8ad35755
DH		 2026 &vcpu->arch.sie_block->cpuflags);
2027 }
2028 goto retry;
2c70fe44 2029 }
8ad35755
DH		 2030
2031 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2032 if (ibs_enabled(vcpu)) {
2033 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
805de8f4 2034 atomic_andnot(CPUSTAT_IBS,
8ad35755
DH		 2035 &vcpu->arch.sie_block->cpuflags);
2036 }
2037 goto retry;
2038 }
2039
0759d068
DH		 2040 /* nothing to do, just clear the request */
2041 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2042
2c70fe44
CB		 2043 return 0;
2044}
2045
25ed1675
DH		 2046void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2047{
2048 struct kvm_vcpu *vcpu;
2049 int i;
2050
2051 mutex_lock(&kvm->lock);
2052 preempt_disable();
2053 kvm->arch.epoch = tod - get_tod_clock();
2054 kvm_s390_vcpu_block_all(kvm);
2055 kvm_for_each_vcpu(i, vcpu, kvm)
2056 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2057 kvm_s390_vcpu_unblock_all(kvm);
2058 preempt_enable();
2059 mutex_unlock(&kvm->lock);
2060}
2061
fa576c58
TH		 2062/**
2063 * kvm_arch_fault_in_page - fault-in guest page if necessary
2064 * @vcpu: The corresponding virtual cpu
2065 * @gpa: Guest physical address
2066 * @writable: Whether the page should be writable or not
2067 *
2068 * Make sure that a guest page has been faulted-in on the host.
2069 *
2070 * Return: Zero on success, negative error code otherwise.
2071 */
2072long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 2073{
527e30b4
MS		 2074 return gmap_fault(vcpu->arch.gmap, gpa,
2075 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 2076}
2077
3c038e6b
DD		 2078static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2079 unsigned long token)
2080{
2081 struct kvm_s390_interrupt inti;
383d0b05 2082 struct kvm_s390_irq irq;
3c038e6b
DD		 2083
2084 if (start_token) {
383d0b05
JF		 2085 irq.u.ext.ext_params2 = token;
2086 irq.type = KVM_S390_INT_PFAULT_INIT;
2087 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 2088 } else {
2089 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 2090 inti.parm64 = token;
3c038e6b
DD		 2091 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2092 }
2093}
2094
2095void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2096 struct kvm_async_pf *work)
2097{
2098 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2099 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2100}
2101
2102void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2103 struct kvm_async_pf *work)
2104{
2105 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2106 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2107}
2108
2109void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2110 struct kvm_async_pf *work)
2111{
2112 /* s390 will always inject the page directly */
2113}
2114
2115bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2116{
2117 /*
2118 * s390 will always inject the page directly,
2119 * but we still want check_async_completion to cleanup
2120 */
2121 return true;
2122}
2123
2124static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2125{
2126 hva_t hva;
2127 struct kvm_arch_async_pf arch;
2128 int rc;
2129
2130 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2131 return 0;
2132 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2133 vcpu->arch.pfault_compare)
2134 return 0;
2135 if (psw_extint_disabled(vcpu))
2136 return 0;
9a022067 2137 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 2138 return 0;
2139 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2140 return 0;
2141 if (!vcpu->arch.gmap->pfault_enabled)
2142 return 0;
2143
81480cc1
HC		 2144 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2145 hva += current->thread.gmap_addr & ~PAGE_MASK;
2146 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 2147 return 0;
2148
2149 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2150 return rc;
2151}
2152
3fb4c40f 2153static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 2154{
3fb4c40f 2155 int rc, cpuflags;
e168bf8d 2156
3c038e6b
DD		 2157 /*
2158 * On s390 notifications for arriving pages will be delivered directly
2159 * to the guest but the house keeping for completed pfaults is
2160 * handled outside the worker.
2161 */
2162 kvm_check_async_pf_completion(vcpu);
2163
7ec7c8c7
CB		 2164 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2165 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
b0c632db
HC		 2166
2167 if (need_resched())
2168 schedule();
2169
d3a73acb 2170 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 2171 s390_handle_mcck();
2172
79395031
JF		 2173 if (!kvm_is_ucontrol(vcpu->kvm)) {
2174 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2175 if (rc)
2176 return rc;
2177 }
0ff31867 2178
2c70fe44
CB		 2179 rc = kvm_s390_handle_requests(vcpu);
2180 if (rc)
2181 return rc;
2182
27291e21
DH		 2183 if (guestdbg_enabled(vcpu)) {
2184 kvm_s390_backup_guest_per_regs(vcpu);
2185 kvm_s390_patch_guest_per_regs(vcpu);
2186 }
2187
b0c632db 2188 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 2189 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2190 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2191 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 2192
3fb4c40f
TH		 2193 return 0;
2194}
2195
492d8642
TH		 2196static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2197{
2198 psw_t *psw = &vcpu->arch.sie_block->gpsw;
2199 u8 opcode;
2200 int rc;
2201
2202 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2203 trace_kvm_s390_sie_fault(vcpu);
2204
2205 /*
2206 * We want to inject an addressing exception, which is defined as a
2207 * suppressing or terminating exception. However, since we came here
2208 * by a DAT access exception, the PSW still points to the faulting
2209 * instruction since DAT exceptions are nullifying. So we've got
2210 * to look up the current opcode to get the length of the instruction
2211 * to be able to forward the PSW.
2212 */
8ae04b8f 2213 rc = read_guest(vcpu, psw->addr, 0, &opcode, 1);
492d8642
TH		 2214 if (rc)
2215 return kvm_s390_inject_prog_cond(vcpu, rc);
2216 psw->addr = __rewind_psw(*psw, -insn_length(opcode));
2217
2218 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
2219}
2220
3fb4c40f
TH		 2221static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2222{
2b29a9fd
DD		 2223 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2224 vcpu->arch.sie_block->icptcode);
2225 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2226
27291e21
DH		 2227 if (guestdbg_enabled(vcpu))
2228 kvm_s390_restore_guest_per_regs(vcpu);
2229
7ec7c8c7
CB		 2230 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2231 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
71f116bf
DH		 2232
2233 if (vcpu->arch.sie_block->icptcode > 0) {
2234 int rc = kvm_handle_sie_intercept(vcpu);
2235
2236 if (rc != -EOPNOTSUPP)
2237 return rc;
2238 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2239 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2240 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2241 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2242 return -EREMOTE;
2243 } else if (exit_reason != -EFAULT) {
2244 vcpu->stat.exit_null++;
2245 return 0;
210b1607
TH		 2246 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2247 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2248 vcpu->run->s390_ucontrol.trans_exc_code =
2249 current->thread.gmap_addr;
2250 vcpu->run->s390_ucontrol.pgm_code = 0x10;
71f116bf 2251 return -EREMOTE;
24eb3a82 2252 } else if (current->thread.gmap_pfault) {
3c038e6b 2253 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 2254 current->thread.gmap_pfault = 0;
71f116bf
DH		 2255 if (kvm_arch_setup_async_pf(vcpu))
2256 return 0;
2257 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
a76ccff6 2258 }
71f116bf 2259 return vcpu_post_run_fault_in_sie(vcpu);
3fb4c40f
TH		 2260}
2261
2262static int __vcpu_run(struct kvm_vcpu *vcpu)
2263{
2264 int rc, exit_reason;
2265
800c1065
TH		 2266 /*
2267 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2268 * ning the guest), so that memslots (and other stuff) are protected
2269 */
2270 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2271
a76ccff6
TH		 2272 do {
2273 rc = vcpu_pre_run(vcpu);
2274 if (rc)
2275 break;
3fb4c40f 2276
800c1065 2277 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 2278 /*
2279 * As PF_VCPU will be used in fault handler, between
2280 * guest_enter and guest_exit should be no uaccess.
2281 */
0097d12e
CB		 2282 local_irq_disable();
2283 __kvm_guest_enter();
2284 local_irq_enable();
a76ccff6
TH		 2285 exit_reason = sie64a(vcpu->arch.sie_block,
2286 vcpu->run->s.regs.gprs);
0097d12e
CB		 2287 local_irq_disable();
2288 __kvm_guest_exit();
2289 local_irq_enable();
800c1065 2290 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 2291
2292 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 2293 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 2294
800c1065 2295 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 2296 return rc;
b0c632db
HC		 2297}
2298
b028ee3e
DH		 2299static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2300{
2301 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2302 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2303 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2304 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2305 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2306 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 2307 /* some control register changes require a tlb flush */
2308 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 2309 }
2310 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2311 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
2312 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2313 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2314 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2315 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2316 }
2317 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2318 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2319 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2320 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 2321 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2322 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e
DH		 2323 }
2324 kvm_run->kvm_dirty_regs = 0;
2325}
2326
2327static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2328{
2329 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2330 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2331 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2332 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2333 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
2334 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2335 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2336 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2337 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2338 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2339 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2340 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2341}
2342
b0c632db
HC		 2343int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2344{
8f2abe6a 2345 int rc;
b0c632db
HC		 2346 sigset_t sigsaved;
2347
27291e21
DH		 2348 if (guestdbg_exit_pending(vcpu)) {
2349 kvm_s390_prepare_debug_exit(vcpu);
2350 return 0;
2351 }
2352
b0c632db
HC		 2353 if (vcpu->sigset_active)
2354 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2355
6352e4d2
DH		 2356 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2357 kvm_s390_vcpu_start(vcpu);
2358 } else if (is_vcpu_stopped(vcpu)) {
ea2cdd27 2359 pr_err_ratelimited("can't run stopped vcpu %d\n",
6352e4d2
DH		 2360 vcpu->vcpu_id);
2361 return -EINVAL;
2362 }
b0c632db 2363
b028ee3e 2364 sync_regs(vcpu, kvm_run);
d7b0b5eb 2365
dab4079d 2366 might_fault();
a76ccff6 2367 rc = __vcpu_run(vcpu);
9ace903d 2368
b1d16c49
CE		 2369 if (signal_pending(current) && !rc) {
2370 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 2371 rc = -EINTR;
b1d16c49 2372 }
8f2abe6a 2373
27291e21
DH		 2374 if (guestdbg_exit_pending(vcpu) && !rc) {
2375 kvm_s390_prepare_debug_exit(vcpu);
2376 rc = 0;
2377 }
2378
8f2abe6a 2379 if (rc == -EREMOTE) {
71f116bf 2380 /* userspace support is needed, kvm_run has been prepared */
8f2abe6a
CB		 2381 rc = 0;
2382 }
b0c632db 2383
b028ee3e 2384 store_regs(vcpu, kvm_run);
d7b0b5eb 2385
b0c632db
HC		 2386 if (vcpu->sigset_active)
2387 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2388
b0c632db 2389 vcpu->stat.exit_userspace++;
7e8e6ab4 2390 return rc;
b0c632db
HC		 2391}
2392
b0c632db
HC		 2393/*
2394 * store status at address
2395 * we use have two special cases:
2396 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2397 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2398 */
d0bce605 2399int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 2400{
092670cd 2401 unsigned char archmode = 1;
fda902cb 2402 unsigned int px;
178bd789 2403 u64 clkcomp;
d0bce605 2404 int rc;
b0c632db 2405
d9a3a09a 2406 px = kvm_s390_get_prefix(vcpu);
d0bce605
HC		 2407 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2408 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 2409 return -EFAULT;
d9a3a09a 2410 gpa = 0;
d0bce605
HC		 2411 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2412 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 2413 return -EFAULT;
d9a3a09a
MS		 2414 gpa = px;
2415 } else
2416 gpa -= __LC_FPREGS_SAVE_AREA;
2417 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
d0bce605 2418 vcpu->arch.guest_fpregs.fprs, 128);
d9a3a09a 2419 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
d0bce605 2420 vcpu->run->s.regs.gprs, 128);
d9a3a09a 2421 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
d0bce605 2422 &vcpu->arch.sie_block->gpsw, 16);
d9a3a09a 2423 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
fda902cb 2424 &px, 4);
d9a3a09a 2425 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
d0bce605 2426 &vcpu->arch.guest_fpregs.fpc, 4);
d9a3a09a 2427 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
d0bce605 2428 &vcpu->arch.sie_block->todpr, 4);
d9a3a09a 2429 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
d0bce605 2430 &vcpu->arch.sie_block->cputm, 8);
178bd789 2431 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d9a3a09a 2432 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
d0bce605 2433 &clkcomp, 8);
d9a3a09a 2434 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
d0bce605 2435 &vcpu->run->s.regs.acrs, 64);
d9a3a09a 2436 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
d0bce605
HC		 2437 &vcpu->arch.sie_block->gcr, 128);
2438 return rc ? -EFAULT : 0;
b0c632db
HC		 2439}
2440
e879892c
TH		 2441int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2442{
2443 /*
2444 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2445 * copying in vcpu load/put. Lets update our copies before we save
2446 * it into the save area
2447 */
d0164ee2 2448 save_fpu_regs();
9977e886
HB		 2449 if (test_kvm_facility(vcpu->kvm, 129)) {
2450 /*
2451 * If the vector extension is available, the vector registers
2452 * which overlaps with floating-point registers are saved in
2453 * the SIE-control block. Hence, extract the floating-point
2454 * registers and the FPC value and store them in the
2455 * guest_fpregs structure.
2456 */
9977e886
HB		 2457 vcpu->arch.guest_fpregs.fpc = current->thread.fpu.fpc;
2458 convert_vx_to_fp(vcpu->arch.guest_fpregs.fprs,
2459 current->thread.fpu.vxrs);
2460 } else
2461 save_fpu_to(&vcpu->arch.guest_fpregs);
e879892c
TH		 2462 save_access_regs(vcpu->run->s.regs.acrs);
2463
2464 return kvm_s390_store_status_unloaded(vcpu, addr);
2465}
2466
bc17de7c
EF		 2467/*
2468 * store additional status at address
2469 */
2470int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2471 unsigned long gpa)
2472{
2473 /* Only bits 0-53 are used for address formation */
2474 if (!(gpa & ~0x3ff))
2475 return 0;
2476
2477 return write_guest_abs(vcpu, gpa & ~0x3ff,
2478 (void *)&vcpu->run->s.regs.vrs, 512);
2479}
2480
2481int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2482{
2483 if (!test_kvm_facility(vcpu->kvm, 129))
2484 return 0;
2485
2486 /*
2487 * The guest VXRS are in the host VXRs due to the lazy
9977e886
HB		 2488 * copying in vcpu load/put. We can simply call save_fpu_regs()
2489 * to save the current register state because we are in the
2490 * middle of a load/put cycle.
2491 *
2492 * Let's update our copies before we save it into the save area.
bc17de7c 2493 */
d0164ee2 2494 save_fpu_regs();
bc17de7c
EF		 2495
2496 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2497}
2498
8ad35755
DH		 2499static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2500{
2501 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
8e236546 2502 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
8ad35755
DH		 2503}
2504
2505static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2506{
2507 unsigned int i;
2508 struct kvm_vcpu *vcpu;
2509
2510 kvm_for_each_vcpu(i, vcpu, kvm) {
2511 __disable_ibs_on_vcpu(vcpu);
2512 }
2513}
2514
2515static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2516{
2517 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
8e236546 2518 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
8ad35755
DH		 2519}
2520
6852d7b6
DH		 2521void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2522{
8ad35755
DH		 2523 int i, online_vcpus, started_vcpus = 0;
2524
2525 if (!is_vcpu_stopped(vcpu))
2526 return;
2527
6852d7b6 2528 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 2529 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2530 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2531 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2532
2533 for (i = 0; i < online_vcpus; i++) {
2534 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2535 started_vcpus++;
2536 }
2537
2538 if (started_vcpus == 0) {
2539 /* we're the only active VCPU -> speed it up */
2540 __enable_ibs_on_vcpu(vcpu);
2541 } else if (started_vcpus == 1) {
2542 /*
2543 * As we are starting a second VCPU, we have to disable
2544 * the IBS facility on all VCPUs to remove potentially
2545 * oustanding ENABLE requests.
2546 */
2547 __disable_ibs_on_all_vcpus(vcpu->kvm);
2548 }
2549
805de8f4 2550 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2551 /*
2552 * Another VCPU might have used IBS while we were offline.
2553 * Let's play safe and flush the VCPU at startup.
2554 */
d3d692c8 2555 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 2556 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2557 return;
6852d7b6
DH		 2558}
2559
2560void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2561{
8ad35755
DH		 2562 int i, online_vcpus, started_vcpus = 0;
2563 struct kvm_vcpu *started_vcpu = NULL;
2564
2565 if (is_vcpu_stopped(vcpu))
2566 return;
2567
6852d7b6 2568 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 2569 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2570 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2571 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2572
32f5ff63 2573 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 2574 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 2575
805de8f4 2576 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2577 __disable_ibs_on_vcpu(vcpu);
2578
2579 for (i = 0; i < online_vcpus; i++) {
2580 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2581 started_vcpus++;
2582 started_vcpu = vcpu->kvm->vcpus[i];
2583 }
2584 }
2585
2586 if (started_vcpus == 1) {
2587 /*
2588 * As we only have one VCPU left, we want to enable the
2589 * IBS facility for that VCPU to speed it up.
2590 */
2591 __enable_ibs_on_vcpu(started_vcpu);
2592 }
2593
433b9ee4 2594 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2595 return;
6852d7b6
DH		 2596}
2597
d6712df9
CH		 2598static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2599 struct kvm_enable_cap *cap)
2600{
2601 int r;
2602
2603 if (cap->flags)
2604 return -EINVAL;
2605
2606 switch (cap->cap) {
fa6b7fe9
CH		 2607 case KVM_CAP_S390_CSS_SUPPORT:
2608 if (!vcpu->kvm->arch.css_support) {
2609 vcpu->kvm->arch.css_support = 1;
c92ea7b9 2610 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
fa6b7fe9
CH		 2611 trace_kvm_s390_enable_css(vcpu->kvm);
2612 }
2613 r = 0;
2614 break;
d6712df9
CH		 2615 default:
2616 r = -EINVAL;
2617 break;
2618 }
2619 return r;
2620}
2621
41408c28
TH		 2622static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2623 struct kvm_s390_mem_op *mop)
2624{
2625 void __user *uaddr = (void __user *)mop->buf;
2626 void *tmpbuf = NULL;
2627 int r, srcu_idx;
2628 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2629 | KVM_S390_MEMOP_F_CHECK_ONLY;
2630
2631 if (mop->flags & ~supported_flags)
2632 return -EINVAL;
2633
2634 if (mop->size > MEM_OP_MAX_SIZE)
2635 return -E2BIG;
2636
2637 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2638 tmpbuf = vmalloc(mop->size);
2639 if (!tmpbuf)
2640 return -ENOMEM;
2641 }
2642
2643 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2644
2645 switch (mop->op) {
2646 case KVM_S390_MEMOP_LOGICAL_READ:
2647 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2648 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, false);
2649 break;
2650 }
2651 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2652 if (r == 0) {
2653 if (copy_to_user(uaddr, tmpbuf, mop->size))
2654 r = -EFAULT;
2655 }
2656 break;
2657 case KVM_S390_MEMOP_LOGICAL_WRITE:
2658 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2659 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, true);
2660 break;
2661 }
2662 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2663 r = -EFAULT;
2664 break;
2665 }
2666 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2667 break;
2668 default:
2669 r = -EINVAL;
2670 }
2671
2672 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2673
2674 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2675 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2676
2677 vfree(tmpbuf);
2678 return r;
2679}
2680
b0c632db
HC		 2681long kvm_arch_vcpu_ioctl(struct file *filp,
2682 unsigned int ioctl, unsigned long arg)
2683{
2684 struct kvm_vcpu *vcpu = filp->private_data;
2685 void __user *argp = (void __user *)arg;
800c1065 2686 int idx;
bc923cc9 2687 long r;
b0c632db 2688
93736624 2689 switch (ioctl) {
47b43c52
JF		 2690 case KVM_S390_IRQ: {
2691 struct kvm_s390_irq s390irq;
2692
2693 r = -EFAULT;
2694 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2695 break;
2696 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2697 break;
2698 }
93736624 2699 case KVM_S390_INTERRUPT: {
ba5c1e9b 2700 struct kvm_s390_interrupt s390int;
383d0b05 2701 struct kvm_s390_irq s390irq;
ba5c1e9b 2702
93736624 2703 r = -EFAULT;
ba5c1e9b 2704 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624 2705 break;
383d0b05
JF		 2706 if (s390int_to_s390irq(&s390int, &s390irq))
2707 return -EINVAL;
2708 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
93736624 2709 break;
ba5c1e9b 2710 }
b0c632db 2711 case KVM_S390_STORE_STATUS:
800c1065 2712 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 2713 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 2714 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 2715 break;
b0c632db
HC		 2716 case KVM_S390_SET_INITIAL_PSW: {
2717 psw_t psw;
2718
bc923cc9 2719 r = -EFAULT;
b0c632db 2720 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 2721 break;
2722 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2723 break;
b0c632db
HC		 2724 }
2725 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 2726 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2727 break;
14eebd91
CO		 2728 case KVM_SET_ONE_REG:
2729 case KVM_GET_ONE_REG: {
2730 struct kvm_one_reg reg;
2731 r = -EFAULT;
2732 if (copy_from_user(&reg, argp, sizeof(reg)))
2733 break;
2734 if (ioctl == KVM_SET_ONE_REG)
2735 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
2736 else
2737 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
2738 break;
2739 }
27e0393f
CO		 2740#ifdef CONFIG_KVM_S390_UCONTROL
2741 case KVM_S390_UCAS_MAP: {
2742 struct kvm_s390_ucas_mapping ucasmap;
2743
2744 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2745 r = -EFAULT;
2746 break;
2747 }
2748
2749 if (!kvm_is_ucontrol(vcpu->kvm)) {
2750 r = -EINVAL;
2751 break;
2752 }
2753
2754 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2755 ucasmap.vcpu_addr, ucasmap.length);
2756 break;
2757 }
2758 case KVM_S390_UCAS_UNMAP: {
2759 struct kvm_s390_ucas_mapping ucasmap;
2760
2761 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2762 r = -EFAULT;
2763 break;
2764 }
2765
2766 if (!kvm_is_ucontrol(vcpu->kvm)) {
2767 r = -EINVAL;
2768 break;
2769 }
2770
2771 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2772 ucasmap.length);
2773 break;
2774 }
2775#endif
ccc7910f 2776 case KVM_S390_VCPU_FAULT: {
527e30b4 2777 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 2778 break;
2779 }
d6712df9
CH		 2780 case KVM_ENABLE_CAP:
2781 {
2782 struct kvm_enable_cap cap;
2783 r = -EFAULT;
2784 if (copy_from_user(&cap, argp, sizeof(cap)))
2785 break;
2786 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2787 break;
2788 }
41408c28
TH		 2789 case KVM_S390_MEM_OP: {
2790 struct kvm_s390_mem_op mem_op;
2791
2792 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2793 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2794 else
2795 r = -EFAULT;
2796 break;
2797 }
816c7667
JF		 2798 case KVM_S390_SET_IRQ_STATE: {
2799 struct kvm_s390_irq_state irq_state;
2800
2801 r = -EFAULT;
2802 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2803 break;
2804 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
2805 irq_state.len == 0 ||
2806 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
2807 r = -EINVAL;
2808 break;
2809 }
2810 r = kvm_s390_set_irq_state(vcpu,
2811 (void __user *) irq_state.buf,
2812 irq_state.len);
2813 break;
2814 }
2815 case KVM_S390_GET_IRQ_STATE: {
2816 struct kvm_s390_irq_state irq_state;
2817
2818 r = -EFAULT;
2819 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2820 break;
2821 if (irq_state.len == 0) {
2822 r = -EINVAL;
2823 break;
2824 }
2825 r = kvm_s390_get_irq_state(vcpu,
2826 (__u8 __user *) irq_state.buf,
2827 irq_state.len);
2828 break;
2829 }
b0c632db 2830 default:
3e6afcf1 2831 r = -ENOTTY;
b0c632db 2832 }
bc923cc9 2833 return r;
b0c632db
HC		 2834}
2835
5b1c1493
CO		 2836int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2837{
2838#ifdef CONFIG_KVM_S390_UCONTROL
2839 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2840 && (kvm_is_ucontrol(vcpu->kvm))) {
2841 vmf->page = virt_to_page(vcpu->arch.sie_block);
2842 get_page(vmf->page);
2843 return 0;
2844 }
2845#endif
2846 return VM_FAULT_SIGBUS;
2847}
2848
5587027c
AK		 2849int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2850 unsigned long npages)
db3fe4eb
TY		 2851{
2852 return 0;
2853}
2854
b0c632db 2855/* Section: memory related */
f7784b8e
MT		 2856int kvm_arch_prepare_memory_region(struct kvm *kvm,
2857 struct kvm_memory_slot *memslot,
09170a49 2858 const struct kvm_userspace_memory_region *mem,
7b6195a9 2859 enum kvm_mr_change change)
b0c632db 2860{
dd2887e7
NW		 2861 /* A few sanity checks. We can have memory slots which have to be
2862 located/ended at a segment boundary (1MB). The memory in userland is
2863 ok to be fragmented into various different vmas. It is okay to mmap()
2864 and munmap() stuff in this slot after doing this call at any time */
b0c632db 2865
598841ca 2866 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 2867 return -EINVAL;
2868
598841ca 2869 if (mem->memory_size & 0xffffful)
b0c632db
HC		 2870 return -EINVAL;
2871
a3a92c31
DD		 2872 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
2873 return -EINVAL;
2874
f7784b8e
MT		 2875 return 0;
2876}
2877
2878void kvm_arch_commit_memory_region(struct kvm *kvm,
09170a49 2879 const struct kvm_userspace_memory_region *mem,
8482644a 2880 const struct kvm_memory_slot *old,
f36f3f28 2881 const struct kvm_memory_slot *new,
8482644a 2882 enum kvm_mr_change change)
f7784b8e 2883{
f7850c92 2884 int rc;
f7784b8e 2885
2cef4deb
CB		 2886 /* If the basics of the memslot do not change, we do not want
2887 * to update the gmap. Every update causes several unnecessary
2888 * segment translation exceptions. This is usually handled just
2889 * fine by the normal fault handler + gmap, but it will also
2890 * cause faults on the prefix page of running guest CPUs.
2891 */
2892 if (old->userspace_addr == mem->userspace_addr &&
2893 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2894 old->npages * PAGE_SIZE == mem->memory_size)
2895 return;
598841ca
CO		 2896
2897 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2898 mem->guest_phys_addr, mem->memory_size);
2899 if (rc)
ea2cdd27 2900 pr_warn("failed to commit memory region\n");
598841ca 2901 return;
b0c632db
HC		 2902}
2903
b0c632db
HC		 2904static int __init kvm_s390_init(void)
2905{
07197fd0
DH		 2906 if (!sclp.has_sief2) {
2907 pr_info("SIE not available\n");
2908 return -ENODEV;
2909 }
2910
9d8d5786 2911 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 2912}
2913
2914static void __exit kvm_s390_exit(void)
2915{
2916 kvm_exit();
2917}
2918
2919module_init(kvm_s390_init);
2920module_exit(kvm_s390_exit);
566af940
CH		 2921
2922/*
2923 * Enable autoloading of the kvm module.
2924 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2925 * since x86 takes a different approach.
2926 */
2927#include <linux/miscdevice.h>
2928MODULE_ALIAS_MISCDEV(KVM_MINOR);
2929MODULE_ALIAS("devname:kvm");
Ubuntu Jammy Linux kernel mirror