]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: s390: vsie: add indication for future features
[mirror_ubuntu-hirsute-kernel.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
b2d73b2a 24#include <linux/mman.h>
b0c632db 25#include <linux/module.h>
a374e892 26#include <linux/random.h>
b0c632db 27#include <linux/slab.h>
ba5c1e9b 28#include <linux/timer.h>
41408c28 29#include <linux/vmalloc.h>
15c9705f 30#include <linux/bitmap.h>
cbb870c8 31#include <asm/asm-offsets.h>
b0c632db 32#include <asm/lowcore.h>
fdf03650 33#include <asm/etr.h>
b0c632db 34#include <asm/pgtable.h>
1e133ab2 35#include <asm/gmap.h>
f5daba1d 36#include <asm/nmi.h>
a0616cde 37#include <asm/switch_to.h>
6d3da241 38#include <asm/isc.h>
1526bf9c 39#include <asm/sclp.h>
0a763c78
DH		 40#include <asm/cpacf.h>
41#include <asm/etr.h>
8f2abe6a 42#include "kvm-s390.h"
b0c632db
HC		 43#include "gaccess.h"
44
ea2cdd27
DH		 45#define KMSG_COMPONENT "kvm-s390"
46#undef pr_fmt
47#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
48
5786fffa
CH		 49#define CREATE_TRACE_POINTS
50#include "trace.h"
ade38c31 51#include "trace-s390.h"
5786fffa 52
41408c28 53#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
816c7667
JF		 54#define LOCAL_IRQS 32
55#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
56 (KVM_MAX_VCPUS + LOCAL_IRQS))
41408c28 57
b0c632db
HC		 58#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
59
60struct kvm_stats_debugfs_item debugfs_entries[] = {
61 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 62 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 63 { "exit_validity", VCPU_STAT(exit_validity) },
64 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
65 { "exit_external_request", VCPU_STAT(exit_external_request) },
66 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b
CO		 67 { "exit_instruction", VCPU_STAT(exit_instruction) },
68 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
69 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
a011eeb2 70 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
f7819512 71 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
62bea5bf 72 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
3491caf2 73 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
ce2e4f0b 74 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 75 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 76 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 77 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
78 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 79 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 80 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 81 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
82 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
83 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
84 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
85 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
86 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
87 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 88 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 89 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
90 { "instruction_spx", VCPU_STAT(instruction_spx) },
91 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
92 { "instruction_stap", VCPU_STAT(instruction_stap) },
93 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 94 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 95 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
96 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 97 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 98 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
99 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 100 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
95ca2cb5 101 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
a3508fbe 102 { "instruction_sie", VCPU_STAT(instruction_sie) },
5288fbf0 103 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 104 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 105 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 106 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 107 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
108 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 109 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 110 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
111 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 112 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 113 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
114 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
115 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 116 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
117 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
118 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 119 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 120 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 121 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
175a5c9e
CB		 122 { "diagnose_258", VCPU_STAT(diagnose_258) },
123 { "diagnose_308", VCPU_STAT(diagnose_308) },
124 { "diagnose_500", VCPU_STAT(diagnose_500) },
b0c632db
HC		 125 { NULL }
126};
127
9d8d5786 128/* upper facilities limit for kvm */
60a37709
AY		 129unsigned long kvm_s390_fac_list_mask[16] = {
130 0xffe6000000000000UL,
131 0x005e000000000000UL,
9d8d5786 132};
b0c632db 133
9d8d5786 134unsigned long kvm_s390_fac_list_mask_size(void)
78c4b59f 135{
9d8d5786
MM		 136 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
137 return ARRAY_SIZE(kvm_s390_fac_list_mask);
78c4b59f
MM		 138}
139
15c9705f
DH		 140/* available cpu features supported by kvm */
141static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
0a763c78
DH		 142/* available subfunctions indicated via query / "test bit" */
143static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
15c9705f 144
9d8d5786 145static struct gmap_notifier gmap_notifier;
a3508fbe 146static struct gmap_notifier vsie_gmap_notifier;
78f26131 147debug_info_t *kvm_s390_dbf;
9d8d5786 148
b0c632db 149/* Section: not file related */
13a34e06 150int kvm_arch_hardware_enable(void)
b0c632db
HC		 151{
152 /* every s390 is virtualization enabled ;-) */
10474ae8 153 return 0;
b0c632db
HC		 154}
155
414d3b07
MS		 156static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
157 unsigned long end);
2c70fe44 158
fdf03650
FZ		 159/*
160 * This callback is executed during stop_machine(). All CPUs are therefore
161 * temporarily stopped. In order not to change guest behavior, we have to
162 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
163 * so a CPU won't be stopped while calculating with the epoch.
164 */
165static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
166 void *v)
167{
168 struct kvm *kvm;
169 struct kvm_vcpu *vcpu;
170 int i;
171 unsigned long long *delta = v;
172
173 list_for_each_entry(kvm, &vm_list, vm_list) {
174 kvm->arch.epoch -= *delta;
175 kvm_for_each_vcpu(i, vcpu, kvm) {
176 vcpu->arch.sie_block->epoch -= *delta;
db0758b2
DH		 177 if (vcpu->arch.cputm_enabled)
178 vcpu->arch.cputm_start += *delta;
91473b48
DH		 179 if (vcpu->arch.vsie_block)
180 vcpu->arch.vsie_block->epoch -= *delta;
fdf03650
FZ		 181 }
182 }
183 return NOTIFY_OK;
184}
185
186static struct notifier_block kvm_clock_notifier = {
187 .notifier_call = kvm_clock_sync,
188};
189
b0c632db
HC		 190int kvm_arch_hardware_setup(void)
191{
2c70fe44 192 gmap_notifier.notifier_call = kvm_gmap_notifier;
b2d73b2a 193 gmap_register_pte_notifier(&gmap_notifier);
a3508fbe
DH		 194 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
195 gmap_register_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 196 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
197 &kvm_clock_notifier);
b0c632db
HC		 198 return 0;
199}
200
201void kvm_arch_hardware_unsetup(void)
202{
b2d73b2a 203 gmap_unregister_pte_notifier(&gmap_notifier);
a3508fbe 204 gmap_unregister_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 205 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
206 &kvm_clock_notifier);
b0c632db
HC		 207}
208
22be5a13
DH		 209static void allow_cpu_feat(unsigned long nr)
210{
211 set_bit_inv(nr, kvm_s390_available_cpu_feat);
212}
213
0a763c78
DH		 214static inline int plo_test_bit(unsigned char nr)
215{
216 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
217 int cc = 3; /* subfunction not available */
218
219 asm volatile(
220 /* Parameter registers are ignored for "test bit" */
221 " plo 0,0,0,0(0)\n"
222 " ipm %0\n"
223 " srl %0,28\n"
224 : "=d" (cc)
225 : "d" (r0)
226 : "cc");
227 return cc == 0;
228}
229
22be5a13
DH		 230static void kvm_s390_cpu_feat_init(void)
231{
0a763c78
DH		 232 int i;
233
234 for (i = 0; i < 256; ++i) {
235 if (plo_test_bit(i))
236 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
237 }
238
239 if (test_facility(28)) /* TOD-clock steering */
240 etr_ptff(kvm_s390_available_subfunc.ptff, ETR_PTFF_QAF);
241
242 if (test_facility(17)) { /* MSA */
243 __cpacf_query(CPACF_KMAC, kvm_s390_available_subfunc.kmac);
244 __cpacf_query(CPACF_KMC, kvm_s390_available_subfunc.kmc);
245 __cpacf_query(CPACF_KM, kvm_s390_available_subfunc.km);
246 __cpacf_query(CPACF_KIMD, kvm_s390_available_subfunc.kimd);
247 __cpacf_query(CPACF_KLMD, kvm_s390_available_subfunc.klmd);
248 }
249 if (test_facility(76)) /* MSA3 */
250 __cpacf_query(CPACF_PCKMO, kvm_s390_available_subfunc.pckmo);
251 if (test_facility(77)) { /* MSA4 */
252 __cpacf_query(CPACF_KMCTR, kvm_s390_available_subfunc.kmctr);
253 __cpacf_query(CPACF_KMF, kvm_s390_available_subfunc.kmf);
254 __cpacf_query(CPACF_KMO, kvm_s390_available_subfunc.kmo);
255 __cpacf_query(CPACF_PCC, kvm_s390_available_subfunc.pcc);
256 }
257 if (test_facility(57)) /* MSA5 */
258 __cpacf_query(CPACF_PPNO, kvm_s390_available_subfunc.ppno);
259
22be5a13
DH		 260 if (MACHINE_HAS_ESOP)
261 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
a3508fbe
DH		 262 /*
263 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
264 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
265 */
266 if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
267 !test_facility(3))
268 return;
269 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
19c439b5
DH		 270 if (sclp.has_64bscao)
271 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
0615a326
DH		 272 if (sclp.has_siif)
273 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
77d18f6d
DH		 274 if (sclp.has_gpere)
275 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
a1b7b9b2
DH		 276 if (sclp.has_gsls)
277 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
5630a8e8
DH		 278 if (sclp.has_ib)
279 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
13ee3f67
DH		 280 if (sclp.has_cei)
281 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
7fd7f39d
DH		 282 if (sclp.has_ibs)
283 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
5d3876a8
DH		 284 /*
285 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
286 * all skey handling functions read/set the skey from the PGSTE
287 * instead of the real storage key.
288 *
289 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
290 * pages being detected as preserved although they are resident.
291 *
292 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
293 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
294 *
295 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
296 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
297 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
298 *
299 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
300 * cannot easily shadow the SCA because of the ipte lock.
301 */
22be5a13
DH		 302}
303
b0c632db
HC		 304int kvm_arch_init(void *opaque)
305{
78f26131
CB		 306 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
307 if (!kvm_s390_dbf)
308 return -ENOMEM;
309
310 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
311 debug_unregister(kvm_s390_dbf);
312 return -ENOMEM;
313 }
314
22be5a13
DH		 315 kvm_s390_cpu_feat_init();
316
84877d93
CH		 317 /* Register floating interrupt controller interface. */
318 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 319}
320
78f26131
CB		 321void kvm_arch_exit(void)
322{
323 debug_unregister(kvm_s390_dbf);
324}
325
b0c632db
HC		 326/* Section: device related */
327long kvm_arch_dev_ioctl(struct file *filp,
328 unsigned int ioctl, unsigned long arg)
329{
330 if (ioctl == KVM_S390_ENABLE_SIE)
331 return s390_enable_sie();
332 return -EINVAL;
333}
334
784aa3d7 335int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 336{
d7b0b5eb
CO		 337 int r;
338
2bd0ac4e 339 switch (ext) {
d7b0b5eb 340 case KVM_CAP_S390_PSW:
b6cf8788 341 case KVM_CAP_S390_GMAP:
52e16b18 342 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 343#ifdef CONFIG_KVM_S390_UCONTROL
344 case KVM_CAP_S390_UCONTROL:
345#endif
3c038e6b 346 case KVM_CAP_ASYNC_PF:
60b413c9 347 case KVM_CAP_SYNC_REGS:
14eebd91 348 case KVM_CAP_ONE_REG:
d6712df9 349 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 350 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 351 case KVM_CAP_IOEVENTFD:
c05c4186 352 case KVM_CAP_DEVICE_CTRL:
d938dc55 353 case KVM_CAP_ENABLE_CAP_VM:
78599d90 354 case KVM_CAP_S390_IRQCHIP:
f2061656 355 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 356 case KVM_CAP_MP_STATE:
47b43c52 357 case KVM_CAP_S390_INJECT_IRQ:
2444b352 358 case KVM_CAP_S390_USER_SIGP:
e44fc8c9 359 case KVM_CAP_S390_USER_STSI:
30ee2a98 360 case KVM_CAP_S390_SKEYS:
816c7667 361 case KVM_CAP_S390_IRQ_STATE:
d7b0b5eb
CO		 362 r = 1;
363 break;
41408c28
TH		 364 case KVM_CAP_S390_MEM_OP:
365 r = MEM_OP_MAX_SIZE;
366 break;
e726b1bd
CB		 367 case KVM_CAP_NR_VCPUS:
368 case KVM_CAP_MAX_VCPUS:
76a6dd72
DH		 369 r = KVM_S390_BSCA_CPU_SLOTS;
370 if (sclp.has_esca && sclp.has_64bscao)
371 r = KVM_S390_ESCA_CPU_SLOTS;
e726b1bd 372 break;
e1e2e605
NW		 373 case KVM_CAP_NR_MEMSLOTS:
374 r = KVM_USER_MEM_SLOTS;
375 break;
1526bf9c 376 case KVM_CAP_S390_COW:
abf09bed 377 r = MACHINE_HAS_ESOP;
1526bf9c 378 break;
68c55750
EF		 379 case KVM_CAP_S390_VECTOR_REGISTERS:
380 r = MACHINE_HAS_VX;
381 break;
c6e5f166
FZ		 382 case KVM_CAP_S390_RI:
383 r = test_facility(64);
384 break;
2bd0ac4e 385 default:
d7b0b5eb 386 r = 0;
2bd0ac4e 387 }
d7b0b5eb 388 return r;
b0c632db
HC		 389}
390
15f36ebd
JH		 391static void kvm_s390_sync_dirty_log(struct kvm *kvm,
392 struct kvm_memory_slot *memslot)
393{
394 gfn_t cur_gfn, last_gfn;
395 unsigned long address;
396 struct gmap *gmap = kvm->arch.gmap;
397
15f36ebd
JH		 398 /* Loop over all guest pages */
399 last_gfn = memslot->base_gfn + memslot->npages;
400 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
401 address = gfn_to_hva_memslot(memslot, cur_gfn);
402
1e133ab2 403 if (test_and_clear_guest_dirty(gmap->mm, address))
15f36ebd 404 mark_page_dirty(kvm, cur_gfn);
1763f8d0
CB		 405 if (fatal_signal_pending(current))
406 return;
70c88a00 407 cond_resched();
15f36ebd 408 }
15f36ebd
JH		 409}
410
b0c632db 411/* Section: vm related */
a6e2f683
ED		 412static void sca_del_vcpu(struct kvm_vcpu *vcpu);
413
b0c632db
HC		 414/*
415 * Get (and clear) the dirty memory log for a memory slot.
416 */
417int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
418 struct kvm_dirty_log *log)
419{
15f36ebd
JH		 420 int r;
421 unsigned long n;
9f6b8029 422 struct kvm_memslots *slots;
15f36ebd
JH		 423 struct kvm_memory_slot *memslot;
424 int is_dirty = 0;
425
426 mutex_lock(&kvm->slots_lock);
427
428 r = -EINVAL;
429 if (log->slot >= KVM_USER_MEM_SLOTS)
430 goto out;
431
9f6b8029
PB		 432 slots = kvm_memslots(kvm);
433 memslot = id_to_memslot(slots, log->slot);
15f36ebd
JH		 434 r = -ENOENT;
435 if (!memslot->dirty_bitmap)
436 goto out;
437
438 kvm_s390_sync_dirty_log(kvm, memslot);
439 r = kvm_get_dirty_log(kvm, log, &is_dirty);
440 if (r)
441 goto out;
442
443 /* Clear the dirty log */
444 if (is_dirty) {
445 n = kvm_dirty_bitmap_bytes(memslot);
446 memset(memslot->dirty_bitmap, 0, n);
447 }
448 r = 0;
449out:
450 mutex_unlock(&kvm->slots_lock);
451 return r;
b0c632db
HC		 452}
453
d938dc55
CH		 454static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
455{
456 int r;
457
458 if (cap->flags)
459 return -EINVAL;
460
461 switch (cap->cap) {
84223598 462 case KVM_CAP_S390_IRQCHIP:
c92ea7b9 463 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
84223598
CH		 464 kvm->arch.use_irqchip = 1;
465 r = 0;
466 break;
2444b352 467 case KVM_CAP_S390_USER_SIGP:
c92ea7b9 468 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
2444b352
DH		 469 kvm->arch.user_sigp = 1;
470 r = 0;
471 break;
68c55750 472 case KVM_CAP_S390_VECTOR_REGISTERS:
5967c17b 473 mutex_lock(&kvm->lock);
a03825bb 474 if (kvm->created_vcpus) {
5967c17b
DH		 475 r = -EBUSY;
476 } else if (MACHINE_HAS_VX) {
c54f0d6a
DH		 477 set_kvm_facility(kvm->arch.model.fac_mask, 129);
478 set_kvm_facility(kvm->arch.model.fac_list, 129);
18280d8b
MM		 479 r = 0;
480 } else
481 r = -EINVAL;
5967c17b 482 mutex_unlock(&kvm->lock);
c92ea7b9
CB		 483 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
484 r ? "(not available)" : "(success)");
68c55750 485 break;
c6e5f166
FZ		 486 case KVM_CAP_S390_RI:
487 r = -EINVAL;
488 mutex_lock(&kvm->lock);
a03825bb 489 if (kvm->created_vcpus) {
c6e5f166
FZ		 490 r = -EBUSY;
491 } else if (test_facility(64)) {
c54f0d6a
DH		 492 set_kvm_facility(kvm->arch.model.fac_mask, 64);
493 set_kvm_facility(kvm->arch.model.fac_list, 64);
c6e5f166
FZ		 494 r = 0;
495 }
496 mutex_unlock(&kvm->lock);
497 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
498 r ? "(not available)" : "(success)");
499 break;
e44fc8c9 500 case KVM_CAP_S390_USER_STSI:
c92ea7b9 501 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
e44fc8c9
ET		 502 kvm->arch.user_stsi = 1;
503 r = 0;
504 break;
d938dc55
CH		 505 default:
506 r = -EINVAL;
507 break;
508 }
509 return r;
510}
511
8c0a7ce6
DD		 512static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
513{
514 int ret;
515
516 switch (attr->attr) {
517 case KVM_S390_VM_MEM_LIMIT_SIZE:
518 ret = 0;
c92ea7b9 519 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
a3a92c31
DD		 520 kvm->arch.mem_limit);
521 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
8c0a7ce6
DD		 522 ret = -EFAULT;
523 break;
524 default:
525 ret = -ENXIO;
526 break;
527 }
528 return ret;
529}
530
531static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 532{
533 int ret;
534 unsigned int idx;
535 switch (attr->attr) {
536 case KVM_S390_VM_MEM_ENABLE_CMMA:
f9cbd9b0 537 ret = -ENXIO;
c24cc9c8 538 if (!sclp.has_cmma)
e6db1d61
DD		 539 break;
540
4f718eab 541 ret = -EBUSY;
c92ea7b9 542 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
4f718eab 543 mutex_lock(&kvm->lock);
a03825bb 544 if (!kvm->created_vcpus) {
4f718eab
DD		 545 kvm->arch.use_cmma = 1;
546 ret = 0;
547 }
548 mutex_unlock(&kvm->lock);
549 break;
550 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 551 ret = -ENXIO;
552 if (!sclp.has_cmma)
553 break;
c3489155
DD		 554 ret = -EINVAL;
555 if (!kvm->arch.use_cmma)
556 break;
557
c92ea7b9 558 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
4f718eab
DD		 559 mutex_lock(&kvm->lock);
560 idx = srcu_read_lock(&kvm->srcu);
a13cff31 561 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 562 srcu_read_unlock(&kvm->srcu, idx);
563 mutex_unlock(&kvm->lock);
564 ret = 0;
565 break;
8c0a7ce6
DD		 566 case KVM_S390_VM_MEM_LIMIT_SIZE: {
567 unsigned long new_limit;
568
569 if (kvm_is_ucontrol(kvm))
570 return -EINVAL;
571
572 if (get_user(new_limit, (u64 __user *)attr->addr))
573 return -EFAULT;
574
a3a92c31
DD		 575 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
576 new_limit > kvm->arch.mem_limit)
8c0a7ce6
DD		 577 return -E2BIG;
578
a3a92c31
DD		 579 if (!new_limit)
580 return -EINVAL;
581
6ea427bb 582 /* gmap_create takes last usable address */
a3a92c31
DD		 583 if (new_limit != KVM_S390_NO_MEM_LIMIT)
584 new_limit -= 1;
585
8c0a7ce6
DD		 586 ret = -EBUSY;
587 mutex_lock(&kvm->lock);
a03825bb 588 if (!kvm->created_vcpus) {
6ea427bb
MS		 589 /* gmap_create will round the limit up */
590 struct gmap *new = gmap_create(current->mm, new_limit);
8c0a7ce6
DD		 591
592 if (!new) {
593 ret = -ENOMEM;
594 } else {
6ea427bb 595 gmap_remove(kvm->arch.gmap);
8c0a7ce6
DD		 596 new->private = kvm;
597 kvm->arch.gmap = new;
598 ret = 0;
599 }
600 }
601 mutex_unlock(&kvm->lock);
a3a92c31
DD		 602 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
603 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
604 (void *) kvm->arch.gmap->asce);
8c0a7ce6
DD		 605 break;
606 }
4f718eab
DD		 607 default:
608 ret = -ENXIO;
609 break;
610 }
611 return ret;
612}
613
a374e892
TK		 614static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
615
616static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
617{
618 struct kvm_vcpu *vcpu;
619 int i;
620
9d8d5786 621 if (!test_kvm_facility(kvm, 76))
a374e892
TK		 622 return -EINVAL;
623
624 mutex_lock(&kvm->lock);
625 switch (attr->attr) {
626 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
627 get_random_bytes(
628 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
629 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
630 kvm->arch.crypto.aes_kw = 1;
c92ea7b9 631 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
a374e892
TK		 632 break;
633 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
634 get_random_bytes(
635 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
636 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
637 kvm->arch.crypto.dea_kw = 1;
c92ea7b9 638 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
a374e892
TK		 639 break;
640 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
641 kvm->arch.crypto.aes_kw = 0;
642 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
643 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
c92ea7b9 644 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
a374e892
TK		 645 break;
646 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
647 kvm->arch.crypto.dea_kw = 0;
648 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
649 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
c92ea7b9 650 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
a374e892
TK		 651 break;
652 default:
653 mutex_unlock(&kvm->lock);
654 return -ENXIO;
655 }
656
657 kvm_for_each_vcpu(i, vcpu, kvm) {
658 kvm_s390_vcpu_crypto_setup(vcpu);
659 exit_sie(vcpu);
660 }
661 mutex_unlock(&kvm->lock);
662 return 0;
663}
664
72f25020
JH		 665static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
666{
667 u8 gtod_high;
668
669 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
670 sizeof(gtod_high)))
671 return -EFAULT;
672
673 if (gtod_high != 0)
674 return -EINVAL;
58c383c6 675 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
72f25020
JH		 676
677 return 0;
678}
679
680static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
681{
5a3d883a 682 u64 gtod;
72f25020
JH		 683
684 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
685 return -EFAULT;
686
25ed1675 687 kvm_s390_set_tod_clock(kvm, gtod);
58c383c6 688 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
72f25020
JH		 689 return 0;
690}
691
692static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
693{
694 int ret;
695
696 if (attr->flags)
697 return -EINVAL;
698
699 switch (attr->attr) {
700 case KVM_S390_VM_TOD_HIGH:
701 ret = kvm_s390_set_tod_high(kvm, attr);
702 break;
703 case KVM_S390_VM_TOD_LOW:
704 ret = kvm_s390_set_tod_low(kvm, attr);
705 break;
706 default:
707 ret = -ENXIO;
708 break;
709 }
710 return ret;
711}
712
713static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
714{
715 u8 gtod_high = 0;
716
717 if (copy_to_user((void __user *)attr->addr, &gtod_high,
718 sizeof(gtod_high)))
719 return -EFAULT;
58c383c6 720 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
72f25020
JH		 721
722 return 0;
723}
724
725static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
726{
5a3d883a 727 u64 gtod;
72f25020 728
60417fcc 729 gtod = kvm_s390_get_tod_clock_fast(kvm);
72f25020
JH		 730 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
731 return -EFAULT;
58c383c6 732 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
72f25020
JH		 733
734 return 0;
735}
736
737static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
738{
739 int ret;
740
741 if (attr->flags)
742 return -EINVAL;
743
744 switch (attr->attr) {
745 case KVM_S390_VM_TOD_HIGH:
746 ret = kvm_s390_get_tod_high(kvm, attr);
747 break;
748 case KVM_S390_VM_TOD_LOW:
749 ret = kvm_s390_get_tod_low(kvm, attr);
750 break;
751 default:
752 ret = -ENXIO;
753 break;
754 }
755 return ret;
756}
757
658b6eda
MM		 758static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
759{
760 struct kvm_s390_vm_cpu_processor *proc;
053dd230 761 u16 lowest_ibc, unblocked_ibc;
658b6eda
MM		 762 int ret = 0;
763
764 mutex_lock(&kvm->lock);
a03825bb 765 if (kvm->created_vcpus) {
658b6eda
MM		 766 ret = -EBUSY;
767 goto out;
768 }
769 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
770 if (!proc) {
771 ret = -ENOMEM;
772 goto out;
773 }
774 if (!copy_from_user(proc, (void __user *)attr->addr,
775 sizeof(*proc))) {
9bb0ec09 776 kvm->arch.model.cpuid = proc->cpuid;
053dd230
DH		 777 lowest_ibc = sclp.ibc >> 16 & 0xfff;
778 unblocked_ibc = sclp.ibc & 0xfff;
779 if (lowest_ibc) {
780 if (proc->ibc > unblocked_ibc)
781 kvm->arch.model.ibc = unblocked_ibc;
782 else if (proc->ibc < lowest_ibc)
783 kvm->arch.model.ibc = lowest_ibc;
784 else
785 kvm->arch.model.ibc = proc->ibc;
786 }
c54f0d6a 787 memcpy(kvm->arch.model.fac_list, proc->fac_list,
658b6eda
MM		 788 S390_ARCH_FAC_LIST_SIZE_BYTE);
789 } else
790 ret = -EFAULT;
791 kfree(proc);
792out:
793 mutex_unlock(&kvm->lock);
794 return ret;
795}
796
15c9705f
DH		 797static int kvm_s390_set_processor_feat(struct kvm *kvm,
798 struct kvm_device_attr *attr)
799{
800 struct kvm_s390_vm_cpu_feat data;
801 int ret = -EBUSY;
802
803 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
804 return -EFAULT;
805 if (!bitmap_subset((unsigned long *) data.feat,
806 kvm_s390_available_cpu_feat,
807 KVM_S390_VM_CPU_FEAT_NR_BITS))
808 return -EINVAL;
809
810 mutex_lock(&kvm->lock);
811 if (!atomic_read(&kvm->online_vcpus)) {
812 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
813 KVM_S390_VM_CPU_FEAT_NR_BITS);
814 ret = 0;
815 }
816 mutex_unlock(&kvm->lock);
817 return ret;
818}
819
0a763c78
DH		 820static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
821 struct kvm_device_attr *attr)
822{
823 /*
824 * Once supported by kernel + hw, we have to store the subfunctions
825 * in kvm->arch and remember that user space configured them.
826 */
827 return -ENXIO;
828}
829
658b6eda
MM		 830static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
831{
832 int ret = -ENXIO;
833
834 switch (attr->attr) {
835 case KVM_S390_VM_CPU_PROCESSOR:
836 ret = kvm_s390_set_processor(kvm, attr);
837 break;
15c9705f
DH		 838 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
839 ret = kvm_s390_set_processor_feat(kvm, attr);
840 break;
0a763c78
DH		 841 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
842 ret = kvm_s390_set_processor_subfunc(kvm, attr);
843 break;
658b6eda
MM		 844 }
845 return ret;
846}
847
848static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
849{
850 struct kvm_s390_vm_cpu_processor *proc;
851 int ret = 0;
852
853 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
854 if (!proc) {
855 ret = -ENOMEM;
856 goto out;
857 }
9bb0ec09 858 proc->cpuid = kvm->arch.model.cpuid;
658b6eda 859 proc->ibc = kvm->arch.model.ibc;
c54f0d6a
DH		 860 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
861 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 862 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
863 ret = -EFAULT;
864 kfree(proc);
865out:
866 return ret;
867}
868
869static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
870{
871 struct kvm_s390_vm_cpu_machine *mach;
872 int ret = 0;
873
874 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
875 if (!mach) {
876 ret = -ENOMEM;
877 goto out;
878 }
879 get_cpu_id((struct cpuid *) &mach->cpuid);
37c5f6c8 880 mach->ibc = sclp.ibc;
c54f0d6a 881 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
981467c9 882 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 883 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
94422ee8 884 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 885 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
886 ret = -EFAULT;
887 kfree(mach);
888out:
889 return ret;
890}
891
15c9705f
DH		 892static int kvm_s390_get_processor_feat(struct kvm *kvm,
893 struct kvm_device_attr *attr)
894{
895 struct kvm_s390_vm_cpu_feat data;
896
897 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
898 KVM_S390_VM_CPU_FEAT_NR_BITS);
899 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
900 return -EFAULT;
901 return 0;
902}
903
904static int kvm_s390_get_machine_feat(struct kvm *kvm,
905 struct kvm_device_attr *attr)
906{
907 struct kvm_s390_vm_cpu_feat data;
908
909 bitmap_copy((unsigned long *) data.feat,
910 kvm_s390_available_cpu_feat,
911 KVM_S390_VM_CPU_FEAT_NR_BITS);
912 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
913 return -EFAULT;
914 return 0;
915}
916
0a763c78
DH		 917static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
918 struct kvm_device_attr *attr)
919{
920 /*
921 * Once we can actually configure subfunctions (kernel + hw support),
922 * we have to check if they were already set by user space, if so copy
923 * them from kvm->arch.
924 */
925 return -ENXIO;
926}
927
928static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
929 struct kvm_device_attr *attr)
930{
931 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
932 sizeof(struct kvm_s390_vm_cpu_subfunc)))
933 return -EFAULT;
934 return 0;
935}
658b6eda
MM		 936static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
937{
938 int ret = -ENXIO;
939
940 switch (attr->attr) {
941 case KVM_S390_VM_CPU_PROCESSOR:
942 ret = kvm_s390_get_processor(kvm, attr);
943 break;
944 case KVM_S390_VM_CPU_MACHINE:
945 ret = kvm_s390_get_machine(kvm, attr);
946 break;
15c9705f
DH		 947 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
948 ret = kvm_s390_get_processor_feat(kvm, attr);
949 break;
950 case KVM_S390_VM_CPU_MACHINE_FEAT:
951 ret = kvm_s390_get_machine_feat(kvm, attr);
952 break;
0a763c78
DH		 953 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
954 ret = kvm_s390_get_processor_subfunc(kvm, attr);
955 break;
956 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
957 ret = kvm_s390_get_machine_subfunc(kvm, attr);
958 break;
658b6eda
MM		 959 }
960 return ret;
961}
962
f2061656
DD		 963static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
964{
965 int ret;
966
967 switch (attr->group) {
4f718eab 968 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 969 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 970 break;
72f25020
JH		 971 case KVM_S390_VM_TOD:
972 ret = kvm_s390_set_tod(kvm, attr);
973 break;
658b6eda
MM		 974 case KVM_S390_VM_CPU_MODEL:
975 ret = kvm_s390_set_cpu_model(kvm, attr);
976 break;
a374e892
TK		 977 case KVM_S390_VM_CRYPTO:
978 ret = kvm_s390_vm_set_crypto(kvm, attr);
979 break;
f2061656
DD		 980 default:
981 ret = -ENXIO;
982 break;
983 }
984
985 return ret;
986}
987
988static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
989{
8c0a7ce6
DD		 990 int ret;
991
992 switch (attr->group) {
993 case KVM_S390_VM_MEM_CTRL:
994 ret = kvm_s390_get_mem_control(kvm, attr);
995 break;
72f25020
JH		 996 case KVM_S390_VM_TOD:
997 ret = kvm_s390_get_tod(kvm, attr);
998 break;
658b6eda
MM		 999 case KVM_S390_VM_CPU_MODEL:
1000 ret = kvm_s390_get_cpu_model(kvm, attr);
1001 break;
8c0a7ce6
DD		 1002 default:
1003 ret = -ENXIO;
1004 break;
1005 }
1006
1007 return ret;
f2061656
DD		 1008}
1009
1010static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1011{
1012 int ret;
1013
1014 switch (attr->group) {
4f718eab
DD		 1015 case KVM_S390_VM_MEM_CTRL:
1016 switch (attr->attr) {
1017 case KVM_S390_VM_MEM_ENABLE_CMMA:
1018 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 1019 ret = sclp.has_cmma ? 0 : -ENXIO;
1020 break;
8c0a7ce6 1021 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 1022 ret = 0;
1023 break;
1024 default:
1025 ret = -ENXIO;
1026 break;
1027 }
1028 break;
72f25020
JH		 1029 case KVM_S390_VM_TOD:
1030 switch (attr->attr) {
1031 case KVM_S390_VM_TOD_LOW:
1032 case KVM_S390_VM_TOD_HIGH:
1033 ret = 0;
1034 break;
1035 default:
1036 ret = -ENXIO;
1037 break;
1038 }
1039 break;
658b6eda
MM		 1040 case KVM_S390_VM_CPU_MODEL:
1041 switch (attr->attr) {
1042 case KVM_S390_VM_CPU_PROCESSOR:
1043 case KVM_S390_VM_CPU_MACHINE:
15c9705f
DH		 1044 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1045 case KVM_S390_VM_CPU_MACHINE_FEAT:
0a763c78 1046 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
658b6eda
MM		 1047 ret = 0;
1048 break;
0a763c78
DH		 1049 /* configuring subfunctions is not supported yet */
1050 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
658b6eda
MM		 1051 default:
1052 ret = -ENXIO;
1053 break;
1054 }
1055 break;
a374e892
TK		 1056 case KVM_S390_VM_CRYPTO:
1057 switch (attr->attr) {
1058 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1059 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1060 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1061 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1062 ret = 0;
1063 break;
1064 default:
1065 ret = -ENXIO;
1066 break;
1067 }
1068 break;
f2061656
DD		 1069 default:
1070 ret = -ENXIO;
1071 break;
1072 }
1073
1074 return ret;
1075}
1076
30ee2a98
JH		 1077static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1078{
1079 uint8_t *keys;
1080 uint64_t hva;
30ee2a98
JH		 1081 int i, r = 0;
1082
1083 if (args->flags != 0)
1084 return -EINVAL;
1085
1086 /* Is this guest using storage keys? */
1087 if (!mm_use_skey(current->mm))
1088 return KVM_S390_GET_SKEYS_NONE;
1089
1090 /* Enforce sane limit on memory allocation */
1091 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1092 return -EINVAL;
1093
1094 keys = kmalloc_array(args->count, sizeof(uint8_t),
1095 GFP_KERNEL | __GFP_NOWARN);
1096 if (!keys)
1097 keys = vmalloc(sizeof(uint8_t) * args->count);
1098 if (!keys)
1099 return -ENOMEM;
1100
d3ed1cee 1101 down_read(&current->mm->mmap_sem);
30ee2a98
JH		 1102 for (i = 0; i < args->count; i++) {
1103 hva = gfn_to_hva(kvm, args->start_gfn + i);
1104 if (kvm_is_error_hva(hva)) {
1105 r = -EFAULT;
d3ed1cee 1106 break;
30ee2a98
JH		 1107 }
1108
154c8c19
DH		 1109 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1110 if (r)
d3ed1cee 1111 break;
30ee2a98 1112 }
d3ed1cee
MS		 1113 up_read(&current->mm->mmap_sem);
1114
1115 if (!r) {
1116 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1117 sizeof(uint8_t) * args->count);
1118 if (r)
1119 r = -EFAULT;
1120 }
30ee2a98 1121
30ee2a98
JH		 1122 kvfree(keys);
1123 return r;
1124}
1125
1126static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1127{
1128 uint8_t *keys;
1129 uint64_t hva;
1130 int i, r = 0;
1131
1132 if (args->flags != 0)
1133 return -EINVAL;
1134
1135 /* Enforce sane limit on memory allocation */
1136 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1137 return -EINVAL;
1138
1139 keys = kmalloc_array(args->count, sizeof(uint8_t),
1140 GFP_KERNEL | __GFP_NOWARN);
1141 if (!keys)
1142 keys = vmalloc(sizeof(uint8_t) * args->count);
1143 if (!keys)
1144 return -ENOMEM;
1145
1146 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1147 sizeof(uint8_t) * args->count);
1148 if (r) {
1149 r = -EFAULT;
1150 goto out;
1151 }
1152
1153 /* Enable storage key handling for the guest */
14d4a425
DD		 1154 r = s390_enable_skey();
1155 if (r)
1156 goto out;
30ee2a98 1157
d3ed1cee 1158 down_read(&current->mm->mmap_sem);
30ee2a98
JH		 1159 for (i = 0; i < args->count; i++) {
1160 hva = gfn_to_hva(kvm, args->start_gfn + i);
1161 if (kvm_is_error_hva(hva)) {
1162 r = -EFAULT;
d3ed1cee 1163 break;
30ee2a98
JH		 1164 }
1165
1166 /* Lowest order bit is reserved */
1167 if (keys[i] & 0x01) {
1168 r = -EINVAL;
d3ed1cee 1169 break;
30ee2a98
JH		 1170 }
1171
fe69eabf 1172 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
30ee2a98 1173 if (r)
d3ed1cee 1174 break;
30ee2a98 1175 }
d3ed1cee 1176 up_read(&current->mm->mmap_sem);
30ee2a98
JH		 1177out:
1178 kvfree(keys);
1179 return r;
1180}
1181
b0c632db
HC		 1182long kvm_arch_vm_ioctl(struct file *filp,
1183 unsigned int ioctl, unsigned long arg)
1184{
1185 struct kvm *kvm = filp->private_data;
1186 void __user *argp = (void __user *)arg;
f2061656 1187 struct kvm_device_attr attr;
b0c632db
HC		 1188 int r;
1189
1190 switch (ioctl) {
ba5c1e9b
CO		 1191 case KVM_S390_INTERRUPT: {
1192 struct kvm_s390_interrupt s390int;
1193
1194 r = -EFAULT;
1195 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1196 break;
1197 r = kvm_s390_inject_vm(kvm, &s390int);
1198 break;
1199 }
d938dc55
CH		 1200 case KVM_ENABLE_CAP: {
1201 struct kvm_enable_cap cap;
1202 r = -EFAULT;
1203 if (copy_from_user(&cap, argp, sizeof(cap)))
1204 break;
1205 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1206 break;
1207 }
84223598
CH		 1208 case KVM_CREATE_IRQCHIP: {
1209 struct kvm_irq_routing_entry routing;
1210
1211 r = -EINVAL;
1212 if (kvm->arch.use_irqchip) {
1213 /* Set up dummy routing. */
1214 memset(&routing, 0, sizeof(routing));
152b2839 1215 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
84223598
CH		 1216 }
1217 break;
1218 }
f2061656
DD		 1219 case KVM_SET_DEVICE_ATTR: {
1220 r = -EFAULT;
1221 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1222 break;
1223 r = kvm_s390_vm_set_attr(kvm, &attr);
1224 break;
1225 }
1226 case KVM_GET_DEVICE_ATTR: {
1227 r = -EFAULT;
1228 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1229 break;
1230 r = kvm_s390_vm_get_attr(kvm, &attr);
1231 break;
1232 }
1233 case KVM_HAS_DEVICE_ATTR: {
1234 r = -EFAULT;
1235 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1236 break;
1237 r = kvm_s390_vm_has_attr(kvm, &attr);
1238 break;
1239 }
30ee2a98
JH		 1240 case KVM_S390_GET_SKEYS: {
1241 struct kvm_s390_skeys args;
1242
1243 r = -EFAULT;
1244 if (copy_from_user(&args, argp,
1245 sizeof(struct kvm_s390_skeys)))
1246 break;
1247 r = kvm_s390_get_skeys(kvm, &args);
1248 break;
1249 }
1250 case KVM_S390_SET_SKEYS: {
1251 struct kvm_s390_skeys args;
1252
1253 r = -EFAULT;
1254 if (copy_from_user(&args, argp,
1255 sizeof(struct kvm_s390_skeys)))
1256 break;
1257 r = kvm_s390_set_skeys(kvm, &args);
1258 break;
1259 }
b0c632db 1260 default:
367e1319 1261 r = -ENOTTY;
b0c632db
HC		 1262 }
1263
1264 return r;
1265}
1266
45c9b47c
TK		 1267static int kvm_s390_query_ap_config(u8 *config)
1268{
1269 u32 fcn_code = 0x04000000UL;
86044c8c 1270 u32 cc = 0;
45c9b47c 1271
86044c8c 1272 memset(config, 0, 128);
45c9b47c
TK		 1273 asm volatile(
1274 "lgr 0,%1\n"
1275 "lgr 2,%2\n"
1276 ".long 0xb2af0000\n" /* PQAP(QCI) */
86044c8c 1277 "0: ipm %0\n"
45c9b47c 1278 "srl %0,28\n"
86044c8c
CB		 1279 "1:\n"
1280 EX_TABLE(0b, 1b)
1281 : "+r" (cc)
45c9b47c
TK		 1282 : "r" (fcn_code), "r" (config)
1283 : "cc", "0", "2", "memory"
1284 );
1285
1286 return cc;
1287}
1288
1289static int kvm_s390_apxa_installed(void)
1290{
1291 u8 config[128];
1292 int cc;
1293
a6aacc3f 1294 if (test_facility(12)) {
45c9b47c
TK		 1295 cc = kvm_s390_query_ap_config(config);
1296
1297 if (cc)
1298 pr_err("PQAP(QCI) failed with cc=%d", cc);
1299 else
1300 return config[0] & 0x40;
1301 }
1302
1303 return 0;
1304}
1305
1306static void kvm_s390_set_crycb_format(struct kvm *kvm)
1307{
1308 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1309
1310 if (kvm_s390_apxa_installed())
1311 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1312 else
1313 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1314}
1315
9bb0ec09 1316static u64 kvm_s390_get_initial_cpuid(void)
9d8d5786 1317{
9bb0ec09
DH		 1318 struct cpuid cpuid;
1319
1320 get_cpu_id(&cpuid);
1321 cpuid.version = 0xff;
1322 return *((u64 *) &cpuid);
9d8d5786
MM		 1323}
1324
c54f0d6a 1325static void kvm_s390_crypto_init(struct kvm *kvm)
5102ee87 1326{
9d8d5786 1327 if (!test_kvm_facility(kvm, 76))
c54f0d6a 1328 return;
5102ee87 1329
c54f0d6a 1330 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
45c9b47c 1331 kvm_s390_set_crycb_format(kvm);
5102ee87 1332
ed6f76b4
TK		 1333 /* Enable AES/DEA protected key functions by default */
1334 kvm->arch.crypto.aes_kw = 1;
1335 kvm->arch.crypto.dea_kw = 1;
1336 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1337 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1338 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1339 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
5102ee87
TK		 1340}
1341
7d43bafc
ED		 1342static void sca_dispose(struct kvm *kvm)
1343{
1344 if (kvm->arch.use_esca)
5e044315 1345 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
7d43bafc
ED		 1346 else
1347 free_page((unsigned long)(kvm->arch.sca));
1348 kvm->arch.sca = NULL;
1349}
1350
e08b9637 1351int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 1352{
76a6dd72 1353 gfp_t alloc_flags = GFP_KERNEL;
9d8d5786 1354 int i, rc;
b0c632db 1355 char debug_name[16];
f6c137ff 1356 static unsigned long sca_offset;
b0c632db 1357
e08b9637
CO		 1358 rc = -EINVAL;
1359#ifdef CONFIG_KVM_S390_UCONTROL
1360 if (type & ~KVM_VM_S390_UCONTROL)
1361 goto out_err;
1362 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1363 goto out_err;
1364#else
1365 if (type)
1366 goto out_err;
1367#endif
1368
b0c632db
HC		 1369 rc = s390_enable_sie();
1370 if (rc)
d89f5eff 1371 goto out_err;
b0c632db 1372
b290411a
CO		 1373 rc = -ENOMEM;
1374
7d0a5e62
JF		 1375 ratelimit_state_init(&kvm->arch.sthyi_limit, 5 * HZ, 500);
1376
7d43bafc 1377 kvm->arch.use_esca = 0; /* start with basic SCA */
76a6dd72
DH		 1378 if (!sclp.has_64bscao)
1379 alloc_flags |= GFP_DMA;
5e044315 1380 rwlock_init(&kvm->arch.sca_lock);
76a6dd72 1381 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
b0c632db 1382 if (!kvm->arch.sca)
d89f5eff 1383 goto out_err;
f6c137ff 1384 spin_lock(&kvm_lock);
c5c2c393 1385 sca_offset += 16;
bc784cce 1386 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
c5c2c393 1387 sca_offset = 0;
bc784cce
ED		 1388 kvm->arch.sca = (struct bsca_block *)
1389 ((char *) kvm->arch.sca + sca_offset);
f6c137ff 1390 spin_unlock(&kvm_lock);
b0c632db
HC		 1391
1392 sprintf(debug_name, "kvm-%u", current->pid);
1393
1cb9cf72 1394 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
b0c632db 1395 if (!kvm->arch.dbf)
40f5b735 1396 goto out_err;
b0c632db 1397
c54f0d6a
DH		 1398 kvm->arch.sie_page2 =
1399 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1400 if (!kvm->arch.sie_page2)
40f5b735 1401 goto out_err;
9d8d5786 1402
fb5bf93f 1403 /* Populate the facility mask initially. */
c54f0d6a 1404 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
94422ee8 1405 S390_ARCH_FAC_LIST_SIZE_BYTE);
9d8d5786
MM		 1406 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1407 if (i < kvm_s390_fac_list_mask_size())
c54f0d6a 1408 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
9d8d5786 1409 else
c54f0d6a 1410 kvm->arch.model.fac_mask[i] = 0UL;
9d8d5786
MM		 1411 }
1412
981467c9 1413 /* Populate the facility list initially. */
c54f0d6a
DH		 1414 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1415 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
981467c9
MM		 1416 S390_ARCH_FAC_LIST_SIZE_BYTE);
1417
95ca2cb5
JF		 1418 set_kvm_facility(kvm->arch.model.fac_mask, 74);
1419 set_kvm_facility(kvm->arch.model.fac_list, 74);
1420
9bb0ec09 1421 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
37c5f6c8 1422 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
9d8d5786 1423
c54f0d6a 1424 kvm_s390_crypto_init(kvm);
5102ee87 1425
ba5c1e9b 1426 spin_lock_init(&kvm->arch.float_int.lock);
6d3da241
JF		 1427 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1428 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
8a242234 1429 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 1430 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 1431
b0c632db 1432 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
78f26131 1433 VM_EVENT(kvm, 3, "vm created with type %lu", type);
b0c632db 1434
e08b9637
CO		 1435 if (type & KVM_VM_S390_UCONTROL) {
1436 kvm->arch.gmap = NULL;
a3a92c31 1437 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
e08b9637 1438 } else {
32e6b236
GH		 1439 if (sclp.hamax == U64_MAX)
1440 kvm->arch.mem_limit = TASK_MAX_SIZE;
1441 else
1442 kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
1443 sclp.hamax + 1);
6ea427bb 1444 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
e08b9637 1445 if (!kvm->arch.gmap)
40f5b735 1446 goto out_err;
2c70fe44 1447 kvm->arch.gmap->private = kvm;
24eb3a82 1448 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 1449 }
fa6b7fe9
CH		 1450
1451 kvm->arch.css_support = 0;
84223598 1452 kvm->arch.use_irqchip = 0;
72f25020 1453 kvm->arch.epoch = 0;
fa6b7fe9 1454
8ad35755 1455 spin_lock_init(&kvm->arch.start_stop_lock);
a3508fbe 1456 kvm_s390_vsie_init(kvm);
8335713a 1457 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
8ad35755 1458
d89f5eff 1459 return 0;
40f5b735 1460out_err:
c54f0d6a 1461 free_page((unsigned long)kvm->arch.sie_page2);
598841ca 1462 debug_unregister(kvm->arch.dbf);
7d43bafc 1463 sca_dispose(kvm);
78f26131 1464 KVM_EVENT(3, "creation of vm failed: %d", rc);
d89f5eff 1465 return rc;
b0c632db
HC		 1466}
1467
d329c035
CB		 1468void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1469{
1470 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 1471 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 1472 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 1473 kvm_clear_async_pf_completion_queue(vcpu);
bc784cce 1474 if (!kvm_is_ucontrol(vcpu->kvm))
a6e2f683 1475 sca_del_vcpu(vcpu);
27e0393f
CO		 1476
1477 if (kvm_is_ucontrol(vcpu->kvm))
6ea427bb 1478 gmap_remove(vcpu->arch.gmap);
27e0393f 1479
e6db1d61 1480 if (vcpu->kvm->arch.use_cmma)
b31605c1 1481 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 1482 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 1483
6692cef3 1484 kvm_vcpu_uninit(vcpu);
b110feaf 1485 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 1486}
1487
1488static void kvm_free_vcpus(struct kvm *kvm)
1489{
1490 unsigned int i;
988a2cae 1491 struct kvm_vcpu *vcpu;
d329c035 1492
988a2cae
GN		 1493 kvm_for_each_vcpu(i, vcpu, kvm)
1494 kvm_arch_vcpu_destroy(vcpu);
1495
1496 mutex_lock(&kvm->lock);
1497 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1498 kvm->vcpus[i] = NULL;
1499
1500 atomic_set(&kvm->online_vcpus, 0);
1501 mutex_unlock(&kvm->lock);
d329c035
CB		 1502}
1503
b0c632db
HC		 1504void kvm_arch_destroy_vm(struct kvm *kvm)
1505{
d329c035 1506 kvm_free_vcpus(kvm);
7d43bafc 1507 sca_dispose(kvm);
d329c035 1508 debug_unregister(kvm->arch.dbf);
c54f0d6a 1509 free_page((unsigned long)kvm->arch.sie_page2);
27e0393f 1510 if (!kvm_is_ucontrol(kvm))
6ea427bb 1511 gmap_remove(kvm->arch.gmap);
841b91c5 1512 kvm_s390_destroy_adapters(kvm);
67335e63 1513 kvm_s390_clear_float_irqs(kvm);
a3508fbe 1514 kvm_s390_vsie_destroy(kvm);
8335713a 1515 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
b0c632db
HC		 1516}
1517
1518/* Section: vcpu related */
dafd032a
DD		 1519static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1520{
6ea427bb 1521 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
dafd032a
DD		 1522 if (!vcpu->arch.gmap)
1523 return -ENOMEM;
1524 vcpu->arch.gmap->private = vcpu->kvm;
1525
1526 return 0;
1527}
1528
a6e2f683
ED		 1529static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1530{
5e044315 1531 read_lock(&vcpu->kvm->arch.sca_lock);
7d43bafc
ED		 1532 if (vcpu->kvm->arch.use_esca) {
1533 struct esca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1534
7d43bafc 1535 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
10ce32d5 1536 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc
ED		 1537 } else {
1538 struct bsca_block *sca = vcpu->kvm->arch.sca;
1539
1540 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
10ce32d5 1541 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc 1542 }
5e044315 1543 read_unlock(&vcpu->kvm->arch.sca_lock);
a6e2f683
ED		 1544}
1545
eaa78f34 1546static void sca_add_vcpu(struct kvm_vcpu *vcpu)
a6e2f683 1547{
eaa78f34
DH		 1548 read_lock(&vcpu->kvm->arch.sca_lock);
1549 if (vcpu->kvm->arch.use_esca) {
1550 struct esca_block *sca = vcpu->kvm->arch.sca;
7d43bafc 1551
eaa78f34 1552 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1553 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1554 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
25508824 1555 vcpu->arch.sie_block->ecb2 |= 0x04U;
eaa78f34 1556 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
7d43bafc 1557 } else {
eaa78f34 1558 struct bsca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1559
eaa78f34 1560 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1561 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1562 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
eaa78f34 1563 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
7d43bafc 1564 }
eaa78f34 1565 read_unlock(&vcpu->kvm->arch.sca_lock);
5e044315
ED		 1566}
1567
1568/* Basic SCA to Extended SCA data copy routines */
1569static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1570{
1571 d->sda = s->sda;
1572 d->sigp_ctrl.c = s->sigp_ctrl.c;
1573 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1574}
1575
1576static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1577{
1578 int i;
1579
1580 d->ipte_control = s->ipte_control;
1581 d->mcn[0] = s->mcn;
1582 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1583 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1584}
1585
1586static int sca_switch_to_extended(struct kvm *kvm)
1587{
1588 struct bsca_block *old_sca = kvm->arch.sca;
1589 struct esca_block *new_sca;
1590 struct kvm_vcpu *vcpu;
1591 unsigned int vcpu_idx;
1592 u32 scaol, scaoh;
1593
1594 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1595 if (!new_sca)
1596 return -ENOMEM;
1597
1598 scaoh = (u32)((u64)(new_sca) >> 32);
1599 scaol = (u32)(u64)(new_sca) & ~0x3fU;
1600
1601 kvm_s390_vcpu_block_all(kvm);
1602 write_lock(&kvm->arch.sca_lock);
1603
1604 sca_copy_b_to_e(new_sca, old_sca);
1605
1606 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1607 vcpu->arch.sie_block->scaoh = scaoh;
1608 vcpu->arch.sie_block->scaol = scaol;
1609 vcpu->arch.sie_block->ecb2 |= 0x04U;
1610 }
1611 kvm->arch.sca = new_sca;
1612 kvm->arch.use_esca = 1;
1613
1614 write_unlock(&kvm->arch.sca_lock);
1615 kvm_s390_vcpu_unblock_all(kvm);
1616
1617 free_page((unsigned long)old_sca);
1618
8335713a
CB		 1619 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1620 old_sca, kvm->arch.sca);
5e044315 1621 return 0;
a6e2f683
ED		 1622}
1623
1624static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1625{
5e044315
ED		 1626 int rc;
1627
1628 if (id < KVM_S390_BSCA_CPU_SLOTS)
1629 return true;
76a6dd72 1630 if (!sclp.has_esca || !sclp.has_64bscao)
5e044315
ED		 1631 return false;
1632
1633 mutex_lock(&kvm->lock);
1634 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1635 mutex_unlock(&kvm->lock);
1636
1637 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
a6e2f683
ED		 1638}
1639
b0c632db
HC		 1640int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1641{
3c038e6b
DD		 1642 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1643 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 1644 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1645 KVM_SYNC_GPRS |
9eed0735 1646 KVM_SYNC_ACRS |
b028ee3e
DH		 1647 KVM_SYNC_CRS |
1648 KVM_SYNC_ARCH0 |
1649 KVM_SYNC_PFAULT;
c6e5f166
FZ		 1650 if (test_kvm_facility(vcpu->kvm, 64))
1651 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
f6aa6dc4
DH		 1652 /* fprs can be synchronized via vrs, even if the guest has no vx. With
1653 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1654 */
1655 if (MACHINE_HAS_VX)
68c55750 1656 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
6fd8e67d
DH		 1657 else
1658 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
dafd032a
DD		 1659
1660 if (kvm_is_ucontrol(vcpu->kvm))
1661 return __kvm_ucontrol_vcpu_init(vcpu);
1662
b0c632db
HC		 1663 return 0;
1664}
1665
db0758b2
DH		 1666/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1667static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1668{
1669 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
9c23a131 1670 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2 1671 vcpu->arch.cputm_start = get_tod_clock_fast();
9c23a131 1672 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1673}
1674
1675/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1676static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1677{
1678 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
9c23a131 1679 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1680 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1681 vcpu->arch.cputm_start = 0;
9c23a131 1682 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1683}
1684
1685/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1686static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1687{
1688 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
1689 vcpu->arch.cputm_enabled = true;
1690 __start_cpu_timer_accounting(vcpu);
1691}
1692
1693/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1694static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1695{
1696 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
1697 __stop_cpu_timer_accounting(vcpu);
1698 vcpu->arch.cputm_enabled = false;
1699}
1700
1701static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1702{
1703 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1704 __enable_cpu_timer_accounting(vcpu);
1705 preempt_enable();
1706}
1707
1708static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1709{
1710 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1711 __disable_cpu_timer_accounting(vcpu);
1712 preempt_enable();
1713}
1714
4287f247
DH		 1715/* set the cpu timer - may only be called from the VCPU thread itself */
1716void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
1717{
db0758b2 1718 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
9c23a131 1719 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1720 if (vcpu->arch.cputm_enabled)
1721 vcpu->arch.cputm_start = get_tod_clock_fast();
4287f247 1722 vcpu->arch.sie_block->cputm = cputm;
9c23a131 1723 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2 1724 preempt_enable();
4287f247
DH		 1725}
1726
db0758b2 1727/* update and get the cpu timer - can also be called from other VCPU threads */
4287f247
DH		 1728__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
1729{
9c23a131 1730 unsigned int seq;
db0758b2 1731 __u64 value;
db0758b2
DH		 1732
1733 if (unlikely(!vcpu->arch.cputm_enabled))
1734 return vcpu->arch.sie_block->cputm;
1735
9c23a131
DH		 1736 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1737 do {
1738 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
1739 /*
1740 * If the writer would ever execute a read in the critical
1741 * section, e.g. in irq context, we have a deadlock.
1742 */
1743 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
1744 value = vcpu->arch.sie_block->cputm;
1745 /* if cputm_start is 0, accounting is being started/stopped */
1746 if (likely(vcpu->arch.cputm_start))
1747 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1748 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
1749 preempt_enable();
db0758b2 1750 return value;
4287f247
DH		 1751}
1752
b0c632db
HC		 1753void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1754{
9977e886 1755 /* Save host register state */
d0164ee2 1756 save_fpu_regs();
9abc2a08
DH		 1757 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
1758 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
9977e886 1759
6fd8e67d
DH		 1760 if (MACHINE_HAS_VX)
1761 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
1762 else
1763 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
9abc2a08 1764 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
9977e886 1765 if (test_fp_ctl(current->thread.fpu.fpc))
96b2d7a8 1766 /* User space provided an invalid FPC, let's clear it */
9977e886
HB		 1767 current->thread.fpu.fpc = 0;
1768
1769 save_access_regs(vcpu->arch.host_acrs);
59674c1a 1770 restore_access_regs(vcpu->run->s.regs.acrs);
37d9df98 1771 gmap_enable(vcpu->arch.enabled_gmap);
805de8f4 1772 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
5ebda316 1773 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1774 __start_cpu_timer_accounting(vcpu);
01a745ac 1775 vcpu->cpu = cpu;
b0c632db
HC		 1776}
1777
1778void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1779{
01a745ac 1780 vcpu->cpu = -1;
5ebda316 1781 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1782 __stop_cpu_timer_accounting(vcpu);
805de8f4 1783 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
37d9df98
DH		 1784 vcpu->arch.enabled_gmap = gmap_get_enabled();
1785 gmap_disable(vcpu->arch.enabled_gmap);
9977e886 1786
9abc2a08 1787 /* Save guest register state */
d0164ee2 1788 save_fpu_regs();
9abc2a08 1789 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
9977e886 1790
9abc2a08
DH		 1791 /* Restore host register state */
1792 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
1793 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
9977e886
HB		 1794
1795 save_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 1796 restore_access_regs(vcpu->arch.host_acrs);
1797}
1798
1799static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1800{
1801 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1802 vcpu->arch.sie_block->gpsw.mask = 0UL;
1803 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 1804 kvm_s390_set_prefix(vcpu, 0);
4287f247 1805 kvm_s390_set_cpu_timer(vcpu, 0);
b0c632db
HC		 1806 vcpu->arch.sie_block->ckc = 0UL;
1807 vcpu->arch.sie_block->todpr = 0;
1808 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1809 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1810 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
9abc2a08
DH		 1811 /* make sure the new fpc will be lazily loaded */
1812 save_fpu_regs();
1813 current->thread.fpu.fpc = 0;
b0c632db 1814 vcpu->arch.sie_block->gbea = 1;
672550fb 1815 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 1816 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1817 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 1818 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1819 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 1820 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 1821}
1822
31928aa5 1823void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 1824{
72f25020 1825 mutex_lock(&vcpu->kvm->lock);
fdf03650 1826 preempt_disable();
72f25020 1827 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
fdf03650 1828 preempt_enable();
72f25020 1829 mutex_unlock(&vcpu->kvm->lock);
25508824 1830 if (!kvm_is_ucontrol(vcpu->kvm)) {
dafd032a 1831 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
eaa78f34 1832 sca_add_vcpu(vcpu);
25508824 1833 }
37d9df98
DH		 1834 /* make vcpu_load load the right gmap on the first trigger */
1835 vcpu->arch.enabled_gmap = vcpu->arch.gmap;
42897d86
MT		 1836}
1837
5102ee87
TK		 1838static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1839{
9d8d5786 1840 if (!test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 1841 return;
1842
a374e892
TK		 1843 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1844
1845 if (vcpu->kvm->arch.crypto.aes_kw)
1846 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1847 if (vcpu->kvm->arch.crypto.dea_kw)
1848 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1849
5102ee87
TK		 1850 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1851}
1852
b31605c1
DD		 1853void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1854{
1855 free_page(vcpu->arch.sie_block->cbrlo);
1856 vcpu->arch.sie_block->cbrlo = 0;
1857}
1858
1859int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1860{
1861 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1862 if (!vcpu->arch.sie_block->cbrlo)
1863 return -ENOMEM;
1864
1865 vcpu->arch.sie_block->ecb2 |= 0x80;
1866 vcpu->arch.sie_block->ecb2 &= ~0x08;
1867 return 0;
1868}
1869
91520f1a
MM		 1870static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1871{
1872 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1873
91520f1a 1874 vcpu->arch.sie_block->ibc = model->ibc;
80bc79dc 1875 if (test_kvm_facility(vcpu->kvm, 7))
c54f0d6a 1876 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
91520f1a
MM		 1877}
1878
b0c632db
HC		 1879int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1880{
b31605c1 1881 int rc = 0;
b31288fa 1882
9e6dabef
CH		 1883 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1884 CPUSTAT_SM |
a4a4f191
GH		 1885 CPUSTAT_STOPPED);
1886
53df84f8 1887 if (test_kvm_facility(vcpu->kvm, 78))
805de8f4 1888 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
53df84f8 1889 else if (test_kvm_facility(vcpu->kvm, 8))
805de8f4 1890 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
a4a4f191 1891
91520f1a
MM		 1892 kvm_s390_vcpu_setup_model(vcpu);
1893
bdab09f3
DH		 1894 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
1895 if (MACHINE_HAS_ESOP)
1896 vcpu->arch.sie_block->ecb |= 0x02;
bd50e8ec
DH		 1897 if (test_kvm_facility(vcpu->kvm, 9))
1898 vcpu->arch.sie_block->ecb |= 0x04;
f597d24e 1899 if (test_kvm_facility(vcpu->kvm, 73))
7feb6bb8
MM		 1900 vcpu->arch.sie_block->ecb |= 0x10;
1901
873b425e 1902 if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
d6af0b49 1903 vcpu->arch.sie_block->ecb2 |= 0x08;
48ee7d3a
DH		 1904 vcpu->arch.sie_block->eca = 0x1002000U;
1905 if (sclp.has_cei)
1906 vcpu->arch.sie_block->eca |= 0x80000000U;
11ad65b7
DH		 1907 if (sclp.has_ib)
1908 vcpu->arch.sie_block->eca |= 0x40000000U;
37c5f6c8 1909 if (sclp.has_siif)
217a4406 1910 vcpu->arch.sie_block->eca |= 1;
37c5f6c8 1911 if (sclp.has_sigpif)
ea5f4969 1912 vcpu->arch.sie_block->eca |= 0x10000000U;
c6e5f166
FZ		 1913 if (test_kvm_facility(vcpu->kvm, 64))
1914 vcpu->arch.sie_block->ecb3 |= 0x01;
18280d8b 1915 if (test_kvm_facility(vcpu->kvm, 129)) {
13211ea7
EF		 1916 vcpu->arch.sie_block->eca |= 0x00020000;
1917 vcpu->arch.sie_block->ecd |= 0x20000000;
1918 }
c6e5f166 1919 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
492d8642 1920 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
95ca2cb5
JF		 1921 if (test_kvm_facility(vcpu->kvm, 74))
1922 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
5a5e6536 1923
e6db1d61 1924 if (vcpu->kvm->arch.use_cmma) {
b31605c1
DD		 1925 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1926 if (rc)
1927 return rc;
b31288fa 1928 }
0ac96caf 1929 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 1930 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 1931
5102ee87
TK		 1932 kvm_s390_vcpu_crypto_setup(vcpu);
1933
b31605c1 1934 return rc;
b0c632db
HC		 1935}
1936
1937struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1938 unsigned int id)
1939{
4d47555a 1940 struct kvm_vcpu *vcpu;
7feb6bb8 1941 struct sie_page *sie_page;
4d47555a
CO		 1942 int rc = -EINVAL;
1943
4215825e 1944 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
4d47555a
CO		 1945 goto out;
1946
1947 rc = -ENOMEM;
b0c632db 1948
b110feaf 1949 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 1950 if (!vcpu)
4d47555a 1951 goto out;
b0c632db 1952
7feb6bb8
MM		 1953 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1954 if (!sie_page)
b0c632db
HC		 1955 goto out_free_cpu;
1956
7feb6bb8
MM		 1957 vcpu->arch.sie_block = &sie_page->sie_block;
1958 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1959
efed1104
DH		 1960 /* the real guest size will always be smaller than msl */
1961 vcpu->arch.sie_block->mso = 0;
1962 vcpu->arch.sie_block->msl = sclp.hamax;
1963
b0c632db 1964 vcpu->arch.sie_block->icpua = id;
ba5c1e9b 1965 spin_lock_init(&vcpu->arch.local_int.lock);
ba5c1e9b 1966 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 1967 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 1968 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
9c23a131 1969 seqcount_init(&vcpu->arch.cputm_seqcount);
ba5c1e9b 1970
b0c632db
HC		 1971 rc = kvm_vcpu_init(vcpu, kvm, id);
1972 if (rc)
9abc2a08 1973 goto out_free_sie_block;
8335713a 1974 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
b0c632db 1975 vcpu->arch.sie_block);
ade38c31 1976 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 1977
b0c632db 1978 return vcpu;
7b06bf2f
WY		 1979out_free_sie_block:
1980 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 1981out_free_cpu:
b110feaf 1982 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 1983out:
b0c632db
HC		 1984 return ERR_PTR(rc);
1985}
1986
b0c632db
HC		 1987int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1988{
9a022067 1989 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 1990}
1991
27406cd5 1992void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
49b99e1e 1993{
805de8f4 1994 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
61a6df54 1995 exit_sie(vcpu);
49b99e1e
CB		 1996}
1997
27406cd5 1998void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
49b99e1e 1999{
805de8f4 2000 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
49b99e1e
CB		 2001}
2002
8e236546
CB		 2003static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
2004{
805de8f4 2005 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
61a6df54 2006 exit_sie(vcpu);
8e236546
CB		 2007}
2008
2009static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2010{
9bf9fde2 2011 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
8e236546
CB		 2012}
2013
49b99e1e
CB		 2014/*
2015 * Kick a guest cpu out of SIE and wait until SIE is not running.
2016 * If the CPU is not running (e.g. waiting as idle) the function will
2017 * return immediately. */
2018void exit_sie(struct kvm_vcpu *vcpu)
2019{
805de8f4 2020 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
49b99e1e
CB		 2021 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2022 cpu_relax();
2023}
2024
8e236546
CB		 2025/* Kick a guest cpu out of SIE to process a request synchronously */
2026void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
49b99e1e 2027{
8e236546
CB		 2028 kvm_make_request(req, vcpu);
2029 kvm_s390_vcpu_request(vcpu);
49b99e1e
CB		 2030}
2031
414d3b07
MS		 2032static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
2033 unsigned long end)
2c70fe44 2034{
2c70fe44
CB		 2035 struct kvm *kvm = gmap->private;
2036 struct kvm_vcpu *vcpu;
414d3b07
MS		 2037 unsigned long prefix;
2038 int i;
2c70fe44 2039
65d0b0d4
DH		 2040 if (gmap_is_shadow(gmap))
2041 return;
414d3b07
MS		 2042 if (start >= 1UL << 31)
2043 /* We are only interested in prefix pages */
2044 return;
2c70fe44
CB		 2045 kvm_for_each_vcpu(i, vcpu, kvm) {
2046 /* match against both prefix pages */
414d3b07
MS		 2047 prefix = kvm_s390_get_prefix(vcpu);
2048 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
2049 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
2050 start, end);
8e236546 2051 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44
CB		 2052 }
2053 }
2054}
2055
b6d33834
CD		 2056int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2057{
2058 /* kvm common code refers to this, but never calls it */
2059 BUG();
2060 return 0;
2061}
2062
14eebd91
CO		 2063static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2064 struct kvm_one_reg *reg)
2065{
2066 int r = -EINVAL;
2067
2068 switch (reg->id) {
29b7c71b
CO		 2069 case KVM_REG_S390_TODPR:
2070 r = put_user(vcpu->arch.sie_block->todpr,
2071 (u32 __user *)reg->addr);
2072 break;
2073 case KVM_REG_S390_EPOCHDIFF:
2074 r = put_user(vcpu->arch.sie_block->epoch,
2075 (u64 __user *)reg->addr);
2076 break;
46a6dd1c 2077 case KVM_REG_S390_CPU_TIMER:
4287f247 2078 r = put_user(kvm_s390_get_cpu_timer(vcpu),
46a6dd1c
J		 2079 (u64 __user *)reg->addr);
2080 break;
2081 case KVM_REG_S390_CLOCK_COMP:
2082 r = put_user(vcpu->arch.sie_block->ckc,
2083 (u64 __user *)reg->addr);
2084 break;
536336c2
DD		 2085 case KVM_REG_S390_PFTOKEN:
2086 r = put_user(vcpu->arch.pfault_token,
2087 (u64 __user *)reg->addr);
2088 break;
2089 case KVM_REG_S390_PFCOMPARE:
2090 r = put_user(vcpu->arch.pfault_compare,
2091 (u64 __user *)reg->addr);
2092 break;
2093 case KVM_REG_S390_PFSELECT:
2094 r = put_user(vcpu->arch.pfault_select,
2095 (u64 __user *)reg->addr);
2096 break;
672550fb
CB		 2097 case KVM_REG_S390_PP:
2098 r = put_user(vcpu->arch.sie_block->pp,
2099 (u64 __user *)reg->addr);
2100 break;
afa45ff5
CB		 2101 case KVM_REG_S390_GBEA:
2102 r = put_user(vcpu->arch.sie_block->gbea,
2103 (u64 __user *)reg->addr);
2104 break;
14eebd91
CO		 2105 default:
2106 break;
2107 }
2108
2109 return r;
2110}
2111
2112static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2113 struct kvm_one_reg *reg)
2114{
2115 int r = -EINVAL;
4287f247 2116 __u64 val;
14eebd91
CO		 2117
2118 switch (reg->id) {
29b7c71b
CO		 2119 case KVM_REG_S390_TODPR:
2120 r = get_user(vcpu->arch.sie_block->todpr,
2121 (u32 __user *)reg->addr);
2122 break;
2123 case KVM_REG_S390_EPOCHDIFF:
2124 r = get_user(vcpu->arch.sie_block->epoch,
2125 (u64 __user *)reg->addr);
2126 break;
46a6dd1c 2127 case KVM_REG_S390_CPU_TIMER:
4287f247
DH		 2128 r = get_user(val, (u64 __user *)reg->addr);
2129 if (!r)
2130 kvm_s390_set_cpu_timer(vcpu, val);
46a6dd1c
J		 2131 break;
2132 case KVM_REG_S390_CLOCK_COMP:
2133 r = get_user(vcpu->arch.sie_block->ckc,
2134 (u64 __user *)reg->addr);
2135 break;
536336c2
DD		 2136 case KVM_REG_S390_PFTOKEN:
2137 r = get_user(vcpu->arch.pfault_token,
2138 (u64 __user *)reg->addr);
9fbd8082
DH		 2139 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2140 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 2141 break;
2142 case KVM_REG_S390_PFCOMPARE:
2143 r = get_user(vcpu->arch.pfault_compare,
2144 (u64 __user *)reg->addr);
2145 break;
2146 case KVM_REG_S390_PFSELECT:
2147 r = get_user(vcpu->arch.pfault_select,
2148 (u64 __user *)reg->addr);
2149 break;
672550fb
CB		 2150 case KVM_REG_S390_PP:
2151 r = get_user(vcpu->arch.sie_block->pp,
2152 (u64 __user *)reg->addr);
2153 break;
afa45ff5
CB		 2154 case KVM_REG_S390_GBEA:
2155 r = get_user(vcpu->arch.sie_block->gbea,
2156 (u64 __user *)reg->addr);
2157 break;
14eebd91
CO		 2158 default:
2159 break;
2160 }
2161
2162 return r;
2163}
b6d33834 2164
b0c632db
HC		 2165static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2166{
b0c632db 2167 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 2168 return 0;
2169}
2170
2171int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2172{
5a32c1af 2173 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 2174 return 0;
2175}
2176
2177int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2178{
5a32c1af 2179 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 2180 return 0;
2181}
2182
2183int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2184 struct kvm_sregs *sregs)
2185{
59674c1a 2186 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 2187 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
59674c1a 2188 restore_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 2189 return 0;
2190}
2191
2192int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2193 struct kvm_sregs *sregs)
2194{
59674c1a 2195 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 2196 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 2197 return 0;
2198}
2199
2200int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2201{
9abc2a08
DH		 2202 /* make sure the new values will be lazily loaded */
2203 save_fpu_regs();
4725c860
MS		 2204 if (test_fp_ctl(fpu->fpc))
2205 return -EINVAL;
9abc2a08
DH		 2206 current->thread.fpu.fpc = fpu->fpc;
2207 if (MACHINE_HAS_VX)
2208 convert_fp_to_vx(current->thread.fpu.vxrs, (freg_t *)fpu->fprs);
2209 else
2210 memcpy(current->thread.fpu.fprs, &fpu->fprs, sizeof(fpu->fprs));
b0c632db
HC		 2211 return 0;
2212}
2213
2214int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2215{
9abc2a08
DH		 2216 /* make sure we have the latest values */
2217 save_fpu_regs();
2218 if (MACHINE_HAS_VX)
2219 convert_vx_to_fp((freg_t *)fpu->fprs, current->thread.fpu.vxrs);
2220 else
2221 memcpy(fpu->fprs, current->thread.fpu.fprs, sizeof(fpu->fprs));
2222 fpu->fpc = current->thread.fpu.fpc;
b0c632db
HC		 2223 return 0;
2224}
2225
2226static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2227{
2228 int rc = 0;
2229
7a42fdc2 2230 if (!is_vcpu_stopped(vcpu))
b0c632db 2231 rc = -EBUSY;
d7b0b5eb
CO		 2232 else {
2233 vcpu->run->psw_mask = psw.mask;
2234 vcpu->run->psw_addr = psw.addr;
2235 }
b0c632db
HC		 2236 return rc;
2237}
2238
2239int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2240 struct kvm_translation *tr)
2241{
2242 return -EINVAL; /* not implemented yet */
2243}
2244
27291e21
DH		 2245#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2246 KVM_GUESTDBG_USE_HW_BP | \
2247 KVM_GUESTDBG_ENABLE)
2248
d0bfb940
JK		 2249int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2250 struct kvm_guest_debug *dbg)
b0c632db 2251{
27291e21
DH		 2252 int rc = 0;
2253
2254 vcpu->guest_debug = 0;
2255 kvm_s390_clear_bp_data(vcpu);
2256
2de3bfc2 2257 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21 2258 return -EINVAL;
89b5b4de
DH		 2259 if (!sclp.has_gpere)
2260 return -EINVAL;
27291e21
DH		 2261
2262 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2263 vcpu->guest_debug = dbg->control;
2264 /* enforce guest PER */
805de8f4 2265 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2266
2267 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2268 rc = kvm_s390_import_bp_data(vcpu, dbg);
2269 } else {
805de8f4 2270 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2271 vcpu->arch.guestdbg.last_bp = 0;
2272 }
2273
2274 if (rc) {
2275 vcpu->guest_debug = 0;
2276 kvm_s390_clear_bp_data(vcpu);
805de8f4 2277 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2278 }
2279
2280 return rc;
b0c632db
HC		 2281}
2282
62d9f0db
MT		 2283int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2284 struct kvm_mp_state *mp_state)
2285{
6352e4d2
DH		 2286 /* CHECK_STOP and LOAD are not supported yet */
2287 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2288 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 2289}
2290
2291int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2292 struct kvm_mp_state *mp_state)
2293{
6352e4d2
DH		 2294 int rc = 0;
2295
2296 /* user space knows about this interface - let it control the state */
2297 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2298
2299 switch (mp_state->mp_state) {
2300 case KVM_MP_STATE_STOPPED:
2301 kvm_s390_vcpu_stop(vcpu);
2302 break;
2303 case KVM_MP_STATE_OPERATING:
2304 kvm_s390_vcpu_start(vcpu);
2305 break;
2306 case KVM_MP_STATE_LOAD:
2307 case KVM_MP_STATE_CHECK_STOP:
2308 /* fall through - CHECK_STOP and LOAD are not supported yet */
2309 default:
2310 rc = -ENXIO;
2311 }
2312
2313 return rc;
62d9f0db
MT		 2314}
2315
8ad35755
DH		 2316static bool ibs_enabled(struct kvm_vcpu *vcpu)
2317{
2318 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2319}
2320
2c70fe44
CB		 2321static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2322{
8ad35755 2323retry:
8e236546 2324 kvm_s390_vcpu_request_handled(vcpu);
586b7ccd
CB		 2325 if (!vcpu->requests)
2326 return 0;
2c70fe44
CB		 2327 /*
2328 * We use MMU_RELOAD just to re-arm the ipte notifier for the
b2d73b2a 2329 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2c70fe44
CB		 2330 * This ensures that the ipte instruction for this request has
2331 * already finished. We might race against a second unmapper that
2332 * wants to set the blocking bit. Lets just retry the request loop.
2333 */
8ad35755 2334 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44 2335 int rc;
b2d73b2a
MS		 2336 rc = gmap_mprotect_notify(vcpu->arch.gmap,
2337 kvm_s390_get_prefix(vcpu),
2338 PAGE_SIZE * 2, PROT_WRITE);
2c70fe44
CB		 2339 if (rc)
2340 return rc;
8ad35755 2341 goto retry;
2c70fe44 2342 }
8ad35755 2343
d3d692c8
DH		 2344 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2345 vcpu->arch.sie_block->ihcpu = 0xffff;
2346 goto retry;
2347 }
2348
8ad35755
DH		 2349 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2350 if (!ibs_enabled(vcpu)) {
2351 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
805de8f4 2352 atomic_or(CPUSTAT_IBS,
8ad35755
DH		 2353 &vcpu->arch.sie_block->cpuflags);
2354 }
2355 goto retry;
2c70fe44 2356 }
8ad35755
DH		 2357
2358 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2359 if (ibs_enabled(vcpu)) {
2360 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
805de8f4 2361 atomic_andnot(CPUSTAT_IBS,
8ad35755
DH		 2362 &vcpu->arch.sie_block->cpuflags);
2363 }
2364 goto retry;
2365 }
2366
0759d068
DH		 2367 /* nothing to do, just clear the request */
2368 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2369
2c70fe44
CB		 2370 return 0;
2371}
2372
25ed1675
DH		 2373void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2374{
2375 struct kvm_vcpu *vcpu;
2376 int i;
2377
2378 mutex_lock(&kvm->lock);
2379 preempt_disable();
2380 kvm->arch.epoch = tod - get_tod_clock();
2381 kvm_s390_vcpu_block_all(kvm);
2382 kvm_for_each_vcpu(i, vcpu, kvm)
2383 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2384 kvm_s390_vcpu_unblock_all(kvm);
2385 preempt_enable();
2386 mutex_unlock(&kvm->lock);
2387}
2388
fa576c58
TH		 2389/**
2390 * kvm_arch_fault_in_page - fault-in guest page if necessary
2391 * @vcpu: The corresponding virtual cpu
2392 * @gpa: Guest physical address
2393 * @writable: Whether the page should be writable or not
2394 *
2395 * Make sure that a guest page has been faulted-in on the host.
2396 *
2397 * Return: Zero on success, negative error code otherwise.
2398 */
2399long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 2400{
527e30b4
MS		 2401 return gmap_fault(vcpu->arch.gmap, gpa,
2402 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 2403}
2404
3c038e6b
DD		 2405static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2406 unsigned long token)
2407{
2408 struct kvm_s390_interrupt inti;
383d0b05 2409 struct kvm_s390_irq irq;
3c038e6b
DD		 2410
2411 if (start_token) {
383d0b05
JF		 2412 irq.u.ext.ext_params2 = token;
2413 irq.type = KVM_S390_INT_PFAULT_INIT;
2414 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 2415 } else {
2416 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 2417 inti.parm64 = token;
3c038e6b
DD		 2418 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2419 }
2420}
2421
2422void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2423 struct kvm_async_pf *work)
2424{
2425 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2426 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2427}
2428
2429void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2430 struct kvm_async_pf *work)
2431{
2432 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2433 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2434}
2435
2436void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2437 struct kvm_async_pf *work)
2438{
2439 /* s390 will always inject the page directly */
2440}
2441
2442bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2443{
2444 /*
2445 * s390 will always inject the page directly,
2446 * but we still want check_async_completion to cleanup
2447 */
2448 return true;
2449}
2450
2451static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2452{
2453 hva_t hva;
2454 struct kvm_arch_async_pf arch;
2455 int rc;
2456
2457 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2458 return 0;
2459 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2460 vcpu->arch.pfault_compare)
2461 return 0;
2462 if (psw_extint_disabled(vcpu))
2463 return 0;
9a022067 2464 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 2465 return 0;
2466 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2467 return 0;
2468 if (!vcpu->arch.gmap->pfault_enabled)
2469 return 0;
2470
81480cc1
HC		 2471 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2472 hva += current->thread.gmap_addr & ~PAGE_MASK;
2473 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 2474 return 0;
2475
2476 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2477 return rc;
2478}
2479
3fb4c40f 2480static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 2481{
3fb4c40f 2482 int rc, cpuflags;
e168bf8d 2483
3c038e6b
DD		 2484 /*
2485 * On s390 notifications for arriving pages will be delivered directly
2486 * to the guest but the house keeping for completed pfaults is
2487 * handled outside the worker.
2488 */
2489 kvm_check_async_pf_completion(vcpu);
2490
7ec7c8c7
CB		 2491 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2492 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
b0c632db
HC		 2493
2494 if (need_resched())
2495 schedule();
2496
d3a73acb 2497 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 2498 s390_handle_mcck();
2499
79395031
JF		 2500 if (!kvm_is_ucontrol(vcpu->kvm)) {
2501 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2502 if (rc)
2503 return rc;
2504 }
0ff31867 2505
2c70fe44
CB		 2506 rc = kvm_s390_handle_requests(vcpu);
2507 if (rc)
2508 return rc;
2509
27291e21
DH		 2510 if (guestdbg_enabled(vcpu)) {
2511 kvm_s390_backup_guest_per_regs(vcpu);
2512 kvm_s390_patch_guest_per_regs(vcpu);
2513 }
2514
b0c632db 2515 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 2516 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2517 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2518 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 2519
3fb4c40f
TH		 2520 return 0;
2521}
2522
492d8642
TH		 2523static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2524{
56317920
DH		 2525 struct kvm_s390_pgm_info pgm_info = {
2526 .code = PGM_ADDRESSING,
2527 };
2528 u8 opcode, ilen;
492d8642
TH		 2529 int rc;
2530
2531 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2532 trace_kvm_s390_sie_fault(vcpu);
2533
2534 /*
2535 * We want to inject an addressing exception, which is defined as a
2536 * suppressing or terminating exception. However, since we came here
2537 * by a DAT access exception, the PSW still points to the faulting
2538 * instruction since DAT exceptions are nullifying. So we've got
2539 * to look up the current opcode to get the length of the instruction
2540 * to be able to forward the PSW.
2541 */
65977322 2542 rc = read_guest_instr(vcpu, &opcode, 1);
56317920 2543 ilen = insn_length(opcode);
9b0d721a
DH		 2544 if (rc < 0) {
2545 return rc;
2546 } else if (rc) {
2547 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2548 * Forward by arbitrary ilc, injection will take care of
2549 * nullification if necessary.
2550 */
2551 pgm_info = vcpu->arch.pgm;
2552 ilen = 4;
2553 }
56317920
DH		 2554 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2555 kvm_s390_forward_psw(vcpu, ilen);
2556 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
492d8642
TH		 2557}
2558
3fb4c40f
TH		 2559static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2560{
2b29a9fd
DD		 2561 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2562 vcpu->arch.sie_block->icptcode);
2563 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2564
27291e21
DH		 2565 if (guestdbg_enabled(vcpu))
2566 kvm_s390_restore_guest_per_regs(vcpu);
2567
7ec7c8c7
CB		 2568 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2569 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
71f116bf
DH		 2570
2571 if (vcpu->arch.sie_block->icptcode > 0) {
2572 int rc = kvm_handle_sie_intercept(vcpu);
2573
2574 if (rc != -EOPNOTSUPP)
2575 return rc;
2576 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2577 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2578 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2579 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2580 return -EREMOTE;
2581 } else if (exit_reason != -EFAULT) {
2582 vcpu->stat.exit_null++;
2583 return 0;
210b1607
TH		 2584 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2585 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2586 vcpu->run->s390_ucontrol.trans_exc_code =
2587 current->thread.gmap_addr;
2588 vcpu->run->s390_ucontrol.pgm_code = 0x10;
71f116bf 2589 return -EREMOTE;
24eb3a82 2590 } else if (current->thread.gmap_pfault) {
3c038e6b 2591 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 2592 current->thread.gmap_pfault = 0;
71f116bf
DH		 2593 if (kvm_arch_setup_async_pf(vcpu))
2594 return 0;
2595 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
a76ccff6 2596 }
71f116bf 2597 return vcpu_post_run_fault_in_sie(vcpu);
3fb4c40f
TH		 2598}
2599
2600static int __vcpu_run(struct kvm_vcpu *vcpu)
2601{
2602 int rc, exit_reason;
2603
800c1065
TH		 2604 /*
2605 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2606 * ning the guest), so that memslots (and other stuff) are protected
2607 */
2608 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2609
a76ccff6
TH		 2610 do {
2611 rc = vcpu_pre_run(vcpu);
2612 if (rc)
2613 break;
3fb4c40f 2614
800c1065 2615 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 2616 /*
2617 * As PF_VCPU will be used in fault handler, between
2618 * guest_enter and guest_exit should be no uaccess.
2619 */
0097d12e
CB		 2620 local_irq_disable();
2621 __kvm_guest_enter();
db0758b2 2622 __disable_cpu_timer_accounting(vcpu);
0097d12e 2623 local_irq_enable();
a76ccff6
TH		 2624 exit_reason = sie64a(vcpu->arch.sie_block,
2625 vcpu->run->s.regs.gprs);
0097d12e 2626 local_irq_disable();
db0758b2 2627 __enable_cpu_timer_accounting(vcpu);
0097d12e
CB		 2628 __kvm_guest_exit();
2629 local_irq_enable();
800c1065 2630 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 2631
2632 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 2633 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 2634
800c1065 2635 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 2636 return rc;
b0c632db
HC		 2637}
2638
b028ee3e
DH		 2639static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2640{
2641 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2642 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2643 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2644 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2645 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2646 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 2647 /* some control register changes require a tlb flush */
2648 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 2649 }
2650 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4287f247 2651 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
b028ee3e
DH		 2652 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2653 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2654 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2655 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2656 }
2657 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2658 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2659 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2660 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 2661 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2662 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e
DH		 2663 }
2664 kvm_run->kvm_dirty_regs = 0;
2665}
2666
2667static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2668{
2669 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2670 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2671 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2672 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4287f247 2673 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
b028ee3e
DH		 2674 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2675 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2676 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2677 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2678 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2679 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2680 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2681}
2682
b0c632db
HC		 2683int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2684{
8f2abe6a 2685 int rc;
b0c632db
HC		 2686 sigset_t sigsaved;
2687
27291e21
DH		 2688 if (guestdbg_exit_pending(vcpu)) {
2689 kvm_s390_prepare_debug_exit(vcpu);
2690 return 0;
2691 }
2692
b0c632db
HC		 2693 if (vcpu->sigset_active)
2694 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2695
6352e4d2
DH		 2696 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2697 kvm_s390_vcpu_start(vcpu);
2698 } else if (is_vcpu_stopped(vcpu)) {
ea2cdd27 2699 pr_err_ratelimited("can't run stopped vcpu %d\n",
6352e4d2
DH		 2700 vcpu->vcpu_id);
2701 return -EINVAL;
2702 }
b0c632db 2703
b028ee3e 2704 sync_regs(vcpu, kvm_run);
db0758b2 2705 enable_cpu_timer_accounting(vcpu);
d7b0b5eb 2706
dab4079d 2707 might_fault();
a76ccff6 2708 rc = __vcpu_run(vcpu);
9ace903d 2709
b1d16c49
CE		 2710 if (signal_pending(current) && !rc) {
2711 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 2712 rc = -EINTR;
b1d16c49 2713 }
8f2abe6a 2714
27291e21
DH		 2715 if (guestdbg_exit_pending(vcpu) && !rc) {
2716 kvm_s390_prepare_debug_exit(vcpu);
2717 rc = 0;
2718 }
2719
8f2abe6a 2720 if (rc == -EREMOTE) {
71f116bf 2721 /* userspace support is needed, kvm_run has been prepared */
8f2abe6a
CB		 2722 rc = 0;
2723 }
b0c632db 2724
db0758b2 2725 disable_cpu_timer_accounting(vcpu);
b028ee3e 2726 store_regs(vcpu, kvm_run);
d7b0b5eb 2727
b0c632db
HC		 2728 if (vcpu->sigset_active)
2729 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2730
b0c632db 2731 vcpu->stat.exit_userspace++;
7e8e6ab4 2732 return rc;
b0c632db
HC		 2733}
2734
b0c632db
HC		 2735/*
2736 * store status at address
2737 * we use have two special cases:
2738 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2739 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2740 */
d0bce605 2741int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 2742{
092670cd 2743 unsigned char archmode = 1;
9abc2a08 2744 freg_t fprs[NUM_FPRS];
fda902cb 2745 unsigned int px;
4287f247 2746 u64 clkcomp, cputm;
d0bce605 2747 int rc;
b0c632db 2748
d9a3a09a 2749 px = kvm_s390_get_prefix(vcpu);
d0bce605
HC		 2750 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2751 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 2752 return -EFAULT;
d9a3a09a 2753 gpa = 0;
d0bce605
HC		 2754 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2755 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 2756 return -EFAULT;
d9a3a09a
MS		 2757 gpa = px;
2758 } else
2759 gpa -= __LC_FPREGS_SAVE_AREA;
9abc2a08
DH		 2760
2761 /* manually convert vector registers if necessary */
2762 if (MACHINE_HAS_VX) {
9522b37f 2763 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08
DH		 2764 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2765 fprs, 128);
2766 } else {
2767 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
6fd8e67d 2768 vcpu->run->s.regs.fprs, 128);
9abc2a08 2769 }
d9a3a09a 2770 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
d0bce605 2771 vcpu->run->s.regs.gprs, 128);
d9a3a09a 2772 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
d0bce605 2773 &vcpu->arch.sie_block->gpsw, 16);
d9a3a09a 2774 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
fda902cb 2775 &px, 4);
d9a3a09a 2776 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
9abc2a08 2777 &vcpu->run->s.regs.fpc, 4);
d9a3a09a 2778 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
d0bce605 2779 &vcpu->arch.sie_block->todpr, 4);
4287f247 2780 cputm = kvm_s390_get_cpu_timer(vcpu);
d9a3a09a 2781 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4287f247 2782 &cputm, 8);
178bd789 2783 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d9a3a09a 2784 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
d0bce605 2785 &clkcomp, 8);
d9a3a09a 2786 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
d0bce605 2787 &vcpu->run->s.regs.acrs, 64);
d9a3a09a 2788 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
d0bce605
HC		 2789 &vcpu->arch.sie_block->gcr, 128);
2790 return rc ? -EFAULT : 0;
b0c632db
HC		 2791}
2792
e879892c
TH		 2793int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2794{
2795 /*
2796 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2797 * copying in vcpu load/put. Lets update our copies before we save
2798 * it into the save area
2799 */
d0164ee2 2800 save_fpu_regs();
9abc2a08 2801 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
e879892c
TH		 2802 save_access_regs(vcpu->run->s.regs.acrs);
2803
2804 return kvm_s390_store_status_unloaded(vcpu, addr);
2805}
2806
bc17de7c
EF		 2807/*
2808 * store additional status at address
2809 */
2810int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2811 unsigned long gpa)
2812{
2813 /* Only bits 0-53 are used for address formation */
2814 if (!(gpa & ~0x3ff))
2815 return 0;
2816
2817 return write_guest_abs(vcpu, gpa & ~0x3ff,
2818 (void *)&vcpu->run->s.regs.vrs, 512);
2819}
2820
2821int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2822{
2823 if (!test_kvm_facility(vcpu->kvm, 129))
2824 return 0;
2825
2826 /*
2827 * The guest VXRS are in the host VXRs due to the lazy
9977e886
HB		 2828 * copying in vcpu load/put. We can simply call save_fpu_regs()
2829 * to save the current register state because we are in the
2830 * middle of a load/put cycle.
2831 *
2832 * Let's update our copies before we save it into the save area.
bc17de7c 2833 */
d0164ee2 2834 save_fpu_regs();
bc17de7c
EF		 2835
2836 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2837}
2838
8ad35755
DH		 2839static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2840{
2841 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
8e236546 2842 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
8ad35755
DH		 2843}
2844
2845static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2846{
2847 unsigned int i;
2848 struct kvm_vcpu *vcpu;
2849
2850 kvm_for_each_vcpu(i, vcpu, kvm) {
2851 __disable_ibs_on_vcpu(vcpu);
2852 }
2853}
2854
2855static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2856{
09a400e7
DH		 2857 if (!sclp.has_ibs)
2858 return;
8ad35755 2859 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
8e236546 2860 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
8ad35755
DH		 2861}
2862
6852d7b6
DH		 2863void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2864{
8ad35755
DH		 2865 int i, online_vcpus, started_vcpus = 0;
2866
2867 if (!is_vcpu_stopped(vcpu))
2868 return;
2869
6852d7b6 2870 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 2871 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2872 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2873 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2874
2875 for (i = 0; i < online_vcpus; i++) {
2876 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2877 started_vcpus++;
2878 }
2879
2880 if (started_vcpus == 0) {
2881 /* we're the only active VCPU -> speed it up */
2882 __enable_ibs_on_vcpu(vcpu);
2883 } else if (started_vcpus == 1) {
2884 /*
2885 * As we are starting a second VCPU, we have to disable
2886 * the IBS facility on all VCPUs to remove potentially
2887 * oustanding ENABLE requests.
2888 */
2889 __disable_ibs_on_all_vcpus(vcpu->kvm);
2890 }
2891
805de8f4 2892 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2893 /*
2894 * Another VCPU might have used IBS while we were offline.
2895 * Let's play safe and flush the VCPU at startup.
2896 */
d3d692c8 2897 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 2898 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2899 return;
6852d7b6
DH		 2900}
2901
2902void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2903{
8ad35755
DH		 2904 int i, online_vcpus, started_vcpus = 0;
2905 struct kvm_vcpu *started_vcpu = NULL;
2906
2907 if (is_vcpu_stopped(vcpu))
2908 return;
2909
6852d7b6 2910 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 2911 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2912 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2913 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2914
32f5ff63 2915 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 2916 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 2917
805de8f4 2918 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2919 __disable_ibs_on_vcpu(vcpu);
2920
2921 for (i = 0; i < online_vcpus; i++) {
2922 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2923 started_vcpus++;
2924 started_vcpu = vcpu->kvm->vcpus[i];
2925 }
2926 }
2927
2928 if (started_vcpus == 1) {
2929 /*
2930 * As we only have one VCPU left, we want to enable the
2931 * IBS facility for that VCPU to speed it up.
2932 */
2933 __enable_ibs_on_vcpu(started_vcpu);
2934 }
2935
433b9ee4 2936 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2937 return;
6852d7b6
DH		 2938}
2939
d6712df9
CH		 2940static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2941 struct kvm_enable_cap *cap)
2942{
2943 int r;
2944
2945 if (cap->flags)
2946 return -EINVAL;
2947
2948 switch (cap->cap) {
fa6b7fe9
CH		 2949 case KVM_CAP_S390_CSS_SUPPORT:
2950 if (!vcpu->kvm->arch.css_support) {
2951 vcpu->kvm->arch.css_support = 1;
c92ea7b9 2952 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
fa6b7fe9
CH		 2953 trace_kvm_s390_enable_css(vcpu->kvm);
2954 }
2955 r = 0;
2956 break;
d6712df9
CH		 2957 default:
2958 r = -EINVAL;
2959 break;
2960 }
2961 return r;
2962}
2963
41408c28
TH		 2964static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2965 struct kvm_s390_mem_op *mop)
2966{
2967 void __user *uaddr = (void __user *)mop->buf;
2968 void *tmpbuf = NULL;
2969 int r, srcu_idx;
2970 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2971 | KVM_S390_MEMOP_F_CHECK_ONLY;
2972
2973 if (mop->flags & ~supported_flags)
2974 return -EINVAL;
2975
2976 if (mop->size > MEM_OP_MAX_SIZE)
2977 return -E2BIG;
2978
2979 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2980 tmpbuf = vmalloc(mop->size);
2981 if (!tmpbuf)
2982 return -ENOMEM;
2983 }
2984
2985 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2986
2987 switch (mop->op) {
2988 case KVM_S390_MEMOP_LOGICAL_READ:
2989 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 2990 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2991 mop->size, GACC_FETCH);
41408c28
TH		 2992 break;
2993 }
2994 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2995 if (r == 0) {
2996 if (copy_to_user(uaddr, tmpbuf, mop->size))
2997 r = -EFAULT;
2998 }
2999 break;
3000 case KVM_S390_MEMOP_LOGICAL_WRITE:
3001 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 3002 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3003 mop->size, GACC_STORE);
41408c28
TH		 3004 break;
3005 }
3006 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
3007 r = -EFAULT;
3008 break;
3009 }
3010 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3011 break;
3012 default:
3013 r = -EINVAL;
3014 }
3015
3016 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
3017
3018 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
3019 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
3020
3021 vfree(tmpbuf);
3022 return r;
3023}
3024
b0c632db
HC		 3025long kvm_arch_vcpu_ioctl(struct file *filp,
3026 unsigned int ioctl, unsigned long arg)
3027{
3028 struct kvm_vcpu *vcpu = filp->private_data;
3029 void __user *argp = (void __user *)arg;
800c1065 3030 int idx;
bc923cc9 3031 long r;
b0c632db 3032
93736624 3033 switch (ioctl) {
47b43c52
JF		 3034 case KVM_S390_IRQ: {
3035 struct kvm_s390_irq s390irq;
3036
3037 r = -EFAULT;
3038 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
3039 break;
3040 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3041 break;
3042 }
93736624 3043 case KVM_S390_INTERRUPT: {
ba5c1e9b 3044 struct kvm_s390_interrupt s390int;
383d0b05 3045 struct kvm_s390_irq s390irq;
ba5c1e9b 3046
93736624 3047 r = -EFAULT;
ba5c1e9b 3048 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624 3049 break;
383d0b05
JF		 3050 if (s390int_to_s390irq(&s390int, &s390irq))
3051 return -EINVAL;
3052 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
93736624 3053 break;
ba5c1e9b 3054 }
b0c632db 3055 case KVM_S390_STORE_STATUS:
800c1065 3056 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 3057 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 3058 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 3059 break;
b0c632db
HC		 3060 case KVM_S390_SET_INITIAL_PSW: {
3061 psw_t psw;
3062
bc923cc9 3063 r = -EFAULT;
b0c632db 3064 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 3065 break;
3066 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3067 break;
b0c632db
HC		 3068 }
3069 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 3070 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3071 break;
14eebd91
CO		 3072 case KVM_SET_ONE_REG:
3073 case KVM_GET_ONE_REG: {
3074 struct kvm_one_reg reg;
3075 r = -EFAULT;
3076 if (copy_from_user(&reg, argp, sizeof(reg)))
3077 break;
3078 if (ioctl == KVM_SET_ONE_REG)
3079 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3080 else
3081 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3082 break;
3083 }
27e0393f
CO		 3084#ifdef CONFIG_KVM_S390_UCONTROL
3085 case KVM_S390_UCAS_MAP: {
3086 struct kvm_s390_ucas_mapping ucasmap;
3087
3088 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3089 r = -EFAULT;
3090 break;
3091 }
3092
3093 if (!kvm_is_ucontrol(vcpu->kvm)) {
3094 r = -EINVAL;
3095 break;
3096 }
3097
3098 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3099 ucasmap.vcpu_addr, ucasmap.length);
3100 break;
3101 }
3102 case KVM_S390_UCAS_UNMAP: {
3103 struct kvm_s390_ucas_mapping ucasmap;
3104
3105 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3106 r = -EFAULT;
3107 break;
3108 }
3109
3110 if (!kvm_is_ucontrol(vcpu->kvm)) {
3111 r = -EINVAL;
3112 break;
3113 }
3114
3115 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3116 ucasmap.length);
3117 break;
3118 }
3119#endif
ccc7910f 3120 case KVM_S390_VCPU_FAULT: {
527e30b4 3121 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 3122 break;
3123 }
d6712df9
CH		 3124 case KVM_ENABLE_CAP:
3125 {
3126 struct kvm_enable_cap cap;
3127 r = -EFAULT;
3128 if (copy_from_user(&cap, argp, sizeof(cap)))
3129 break;
3130 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3131 break;
3132 }
41408c28
TH		 3133 case KVM_S390_MEM_OP: {
3134 struct kvm_s390_mem_op mem_op;
3135
3136 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3137 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3138 else
3139 r = -EFAULT;
3140 break;
3141 }
816c7667
JF		 3142 case KVM_S390_SET_IRQ_STATE: {
3143 struct kvm_s390_irq_state irq_state;
3144
3145 r = -EFAULT;
3146 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3147 break;
3148 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3149 irq_state.len == 0 ||
3150 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3151 r = -EINVAL;
3152 break;
3153 }
3154 r = kvm_s390_set_irq_state(vcpu,
3155 (void __user *) irq_state.buf,
3156 irq_state.len);
3157 break;
3158 }
3159 case KVM_S390_GET_IRQ_STATE: {
3160 struct kvm_s390_irq_state irq_state;
3161
3162 r = -EFAULT;
3163 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3164 break;
3165 if (irq_state.len == 0) {
3166 r = -EINVAL;
3167 break;
3168 }
3169 r = kvm_s390_get_irq_state(vcpu,
3170 (__u8 __user *) irq_state.buf,
3171 irq_state.len);
3172 break;
3173 }
b0c632db 3174 default:
3e6afcf1 3175 r = -ENOTTY;
b0c632db 3176 }
bc923cc9 3177 return r;
b0c632db
HC		 3178}
3179
5b1c1493
CO		 3180int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3181{
3182#ifdef CONFIG_KVM_S390_UCONTROL
3183 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3184 && (kvm_is_ucontrol(vcpu->kvm))) {
3185 vmf->page = virt_to_page(vcpu->arch.sie_block);
3186 get_page(vmf->page);
3187 return 0;
3188 }
3189#endif
3190 return VM_FAULT_SIGBUS;
3191}
3192
5587027c
AK		 3193int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3194 unsigned long npages)
db3fe4eb
TY		 3195{
3196 return 0;
3197}
3198
b0c632db 3199/* Section: memory related */
f7784b8e
MT		 3200int kvm_arch_prepare_memory_region(struct kvm *kvm,
3201 struct kvm_memory_slot *memslot,
09170a49 3202 const struct kvm_userspace_memory_region *mem,
7b6195a9 3203 enum kvm_mr_change change)
b0c632db 3204{
dd2887e7
NW		 3205 /* A few sanity checks. We can have memory slots which have to be
3206 located/ended at a segment boundary (1MB). The memory in userland is
3207 ok to be fragmented into various different vmas. It is okay to mmap()
3208 and munmap() stuff in this slot after doing this call at any time */
b0c632db 3209
598841ca 3210 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 3211 return -EINVAL;
3212
598841ca 3213 if (mem->memory_size & 0xffffful)
b0c632db
HC		 3214 return -EINVAL;
3215
a3a92c31
DD		 3216 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3217 return -EINVAL;
3218
f7784b8e
MT		 3219 return 0;
3220}
3221
3222void kvm_arch_commit_memory_region(struct kvm *kvm,
09170a49 3223 const struct kvm_userspace_memory_region *mem,
8482644a 3224 const struct kvm_memory_slot *old,
f36f3f28 3225 const struct kvm_memory_slot *new,
8482644a 3226 enum kvm_mr_change change)
f7784b8e 3227{
f7850c92 3228 int rc;
f7784b8e 3229
2cef4deb
CB		 3230 /* If the basics of the memslot do not change, we do not want
3231 * to update the gmap. Every update causes several unnecessary
3232 * segment translation exceptions. This is usually handled just
3233 * fine by the normal fault handler + gmap, but it will also
3234 * cause faults on the prefix page of running guest CPUs.
3235 */
3236 if (old->userspace_addr == mem->userspace_addr &&
3237 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3238 old->npages * PAGE_SIZE == mem->memory_size)
3239 return;
598841ca
CO		 3240
3241 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3242 mem->guest_phys_addr, mem->memory_size);
3243 if (rc)
ea2cdd27 3244 pr_warn("failed to commit memory region\n");
598841ca 3245 return;
b0c632db
HC		 3246}
3247
60a37709
AY		 3248static inline unsigned long nonhyp_mask(int i)
3249{
3250 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3251
3252 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3253}
3254
3491caf2
CB		 3255void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3256{
3257 vcpu->valid_wakeup = false;
3258}
3259
b0c632db
HC		 3260static int __init kvm_s390_init(void)
3261{
60a37709
AY		 3262 int i;
3263
07197fd0
DH		 3264 if (!sclp.has_sief2) {
3265 pr_info("SIE not available\n");
3266 return -ENODEV;
3267 }
3268
60a37709
AY		 3269 for (i = 0; i < 16; i++)
3270 kvm_s390_fac_list_mask[i] |=
3271 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3272
9d8d5786 3273 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 3274}
3275
3276static void __exit kvm_s390_exit(void)
3277{
3278 kvm_exit();
3279}
3280
3281module_init(kvm_s390_init);
3282module_exit(kvm_s390_exit);
566af940
CH		 3283
3284/*
3285 * Enable autoloading of the kvm module.
3286 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3287 * since x86 takes a different approach.
3288 */
3289#include <linux/miscdevice.h>
3290MODULE_ALIAS_MISCDEV(KVM_MINOR);
3291MODULE_ALIAS("devname:kvm");
Ubuntu Hirsute Linux kernel mirror