]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'mmc-v4.15-2' of git://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc
[mirror_ubuntu-bionic-kernel.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
b2d73b2a 24#include <linux/mman.h>
b0c632db 25#include <linux/module.h>
d3217967 26#include <linux/moduleparam.h>
a374e892 27#include <linux/random.h>
b0c632db 28#include <linux/slab.h>
ba5c1e9b 29#include <linux/timer.h>
41408c28 30#include <linux/vmalloc.h>
15c9705f 31#include <linux/bitmap.h>
174cd4b1 32#include <linux/sched/signal.h>
190df4a2 33#include <linux/string.h>
174cd4b1 34
cbb870c8 35#include <asm/asm-offsets.h>
b0c632db 36#include <asm/lowcore.h>
fd5ada04 37#include <asm/stp.h>
b0c632db 38#include <asm/pgtable.h>
1e133ab2 39#include <asm/gmap.h>
f5daba1d 40#include <asm/nmi.h>
a0616cde 41#include <asm/switch_to.h>
6d3da241 42#include <asm/isc.h>
1526bf9c 43#include <asm/sclp.h>
0a763c78 44#include <asm/cpacf.h>
221bb8a4 45#include <asm/timex.h>
8f2abe6a 46#include "kvm-s390.h"
b0c632db
HC		 47#include "gaccess.h"
48
ea2cdd27
DH		 49#define KMSG_COMPONENT "kvm-s390"
50#undef pr_fmt
51#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
52
5786fffa
CH		 53#define CREATE_TRACE_POINTS
54#include "trace.h"
ade38c31 55#include "trace-s390.h"
5786fffa 56
41408c28 57#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
816c7667
JF		 58#define LOCAL_IRQS 32
59#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
60 (KVM_MAX_VCPUS + LOCAL_IRQS))
41408c28 61
b0c632db
HC		 62#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
63
64struct kvm_stats_debugfs_item debugfs_entries[] = {
65 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 66 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 67 { "exit_validity", VCPU_STAT(exit_validity) },
68 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
69 { "exit_external_request", VCPU_STAT(exit_external_request) },
70 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b 71 { "exit_instruction", VCPU_STAT(exit_instruction) },
9ec6de19 72 { "exit_pei", VCPU_STAT(exit_pei) },
ba5c1e9b
CO		 73 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
74 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
a011eeb2 75 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
f7819512 76 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
62bea5bf 77 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
3491caf2 78 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
ce2e4f0b 79 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 80 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 81 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 82 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
83 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 84 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 85 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 86 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
87 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
88 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
89 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
90 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
91 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
92 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 93 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 94 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
95 { "instruction_spx", VCPU_STAT(instruction_spx) },
96 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
97 { "instruction_stap", VCPU_STAT(instruction_stap) },
98 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 99 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 100 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
101 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 102 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 103 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
104 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 105 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
95ca2cb5 106 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
a3508fbe 107 { "instruction_sie", VCPU_STAT(instruction_sie) },
5288fbf0 108 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 109 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 110 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 111 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 112 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
113 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 114 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 115 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
116 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 117 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 118 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
119 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
120 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 121 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
122 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
123 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 124 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 125 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 126 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
175a5c9e
CB		 127 { "diagnose_258", VCPU_STAT(diagnose_258) },
128 { "diagnose_308", VCPU_STAT(diagnose_308) },
129 { "diagnose_500", VCPU_STAT(diagnose_500) },
b0c632db
HC		 130 { NULL }
131};
132
8fa1696e
CW		 133struct kvm_s390_tod_clock_ext {
134 __u8 epoch_idx;
135 __u64 tod;
136 __u8 reserved[7];
137} __packed;
138
a411edf1
DH		 139/* allow nested virtualization in KVM (if enabled by user space) */
140static int nested;
141module_param(nested, int, S_IRUGO);
142MODULE_PARM_DESC(nested, "Nested virtualization support");
143
9d8d5786 144/* upper facilities limit for kvm */
f6c1d359 145unsigned long kvm_s390_fac_list_mask[16] = { FACILITIES_KVM };
b0c632db 146
9d8d5786 147unsigned long kvm_s390_fac_list_mask_size(void)
78c4b59f 148{
9d8d5786
MM		 149 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
150 return ARRAY_SIZE(kvm_s390_fac_list_mask);
78c4b59f
MM		 151}
152
15c9705f
DH		 153/* available cpu features supported by kvm */
154static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
0a763c78
DH		 155/* available subfunctions indicated via query / "test bit" */
156static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
15c9705f 157
9d8d5786 158static struct gmap_notifier gmap_notifier;
a3508fbe 159static struct gmap_notifier vsie_gmap_notifier;
78f26131 160debug_info_t *kvm_s390_dbf;
9d8d5786 161
b0c632db 162/* Section: not file related */
13a34e06 163int kvm_arch_hardware_enable(void)
b0c632db
HC		 164{
165 /* every s390 is virtualization enabled ;-) */
10474ae8 166 return 0;
b0c632db
HC		 167}
168
414d3b07
MS		 169static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
170 unsigned long end);
2c70fe44 171
fdf03650
FZ		 172/*
173 * This callback is executed during stop_machine(). All CPUs are therefore
174 * temporarily stopped. In order not to change guest behavior, we have to
175 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
176 * so a CPU won't be stopped while calculating with the epoch.
177 */
178static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
179 void *v)
180{
181 struct kvm *kvm;
182 struct kvm_vcpu *vcpu;
183 int i;
184 unsigned long long *delta = v;
185
186 list_for_each_entry(kvm, &vm_list, vm_list) {
187 kvm->arch.epoch -= *delta;
188 kvm_for_each_vcpu(i, vcpu, kvm) {
189 vcpu->arch.sie_block->epoch -= *delta;
db0758b2
DH		 190 if (vcpu->arch.cputm_enabled)
191 vcpu->arch.cputm_start += *delta;
91473b48
DH		 192 if (vcpu->arch.vsie_block)
193 vcpu->arch.vsie_block->epoch -= *delta;
fdf03650
FZ		 194 }
195 }
196 return NOTIFY_OK;
197}
198
199static struct notifier_block kvm_clock_notifier = {
200 .notifier_call = kvm_clock_sync,
201};
202
b0c632db
HC		 203int kvm_arch_hardware_setup(void)
204{
2c70fe44 205 gmap_notifier.notifier_call = kvm_gmap_notifier;
b2d73b2a 206 gmap_register_pte_notifier(&gmap_notifier);
a3508fbe
DH		 207 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
208 gmap_register_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 209 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
210 &kvm_clock_notifier);
b0c632db
HC		 211 return 0;
212}
213
214void kvm_arch_hardware_unsetup(void)
215{
b2d73b2a 216 gmap_unregister_pte_notifier(&gmap_notifier);
a3508fbe 217 gmap_unregister_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 218 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
219 &kvm_clock_notifier);
b0c632db
HC		 220}
221
22be5a13
DH		 222static void allow_cpu_feat(unsigned long nr)
223{
224 set_bit_inv(nr, kvm_s390_available_cpu_feat);
225}
226
0a763c78
DH		 227static inline int plo_test_bit(unsigned char nr)
228{
229 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
d051ae53 230 int cc;
0a763c78
DH		 231
232 asm volatile(
233 /* Parameter registers are ignored for "test bit" */
234 " plo 0,0,0,0(0)\n"
235 " ipm %0\n"
236 " srl %0,28\n"
237 : "=d" (cc)
238 : "d" (r0)
239 : "cc");
240 return cc == 0;
241}
242
22be5a13
DH		 243static void kvm_s390_cpu_feat_init(void)
244{
0a763c78
DH		 245 int i;
246
247 for (i = 0; i < 256; ++i) {
248 if (plo_test_bit(i))
249 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
250 }
251
252 if (test_facility(28)) /* TOD-clock steering */
221bb8a4
LT		 253 ptff(kvm_s390_available_subfunc.ptff,
254 sizeof(kvm_s390_available_subfunc.ptff),
255 PTFF_QAF);
0a763c78
DH		 256
257 if (test_facility(17)) { /* MSA */
69c0e360
MS		 258 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
259 kvm_s390_available_subfunc.kmac);
260 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
261 kvm_s390_available_subfunc.kmc);
262 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
263 kvm_s390_available_subfunc.km);
264 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
265 kvm_s390_available_subfunc.kimd);
266 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
267 kvm_s390_available_subfunc.klmd);
0a763c78
DH		 268 }
269 if (test_facility(76)) /* MSA3 */
69c0e360
MS		 270 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
271 kvm_s390_available_subfunc.pckmo);
0a763c78 272 if (test_facility(77)) { /* MSA4 */
69c0e360
MS		 273 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
274 kvm_s390_available_subfunc.kmctr);
275 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
276 kvm_s390_available_subfunc.kmf);
277 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
278 kvm_s390_available_subfunc.kmo);
279 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
280 kvm_s390_available_subfunc.pcc);
0a763c78
DH		 281 }
282 if (test_facility(57)) /* MSA5 */
985a9d20 283 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
69c0e360 284 kvm_s390_available_subfunc.ppno);
0a763c78 285
e000b8e0
JH		 286 if (test_facility(146)) /* MSA8 */
287 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
288 kvm_s390_available_subfunc.kma);
289
22be5a13
DH		 290 if (MACHINE_HAS_ESOP)
291 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
a3508fbe
DH		 292 /*
293 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
294 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
295 */
296 if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
a411edf1 297 !test_facility(3) || !nested)
a3508fbe
DH		 298 return;
299 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
19c439b5
DH		 300 if (sclp.has_64bscao)
301 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
0615a326
DH		 302 if (sclp.has_siif)
303 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
77d18f6d
DH		 304 if (sclp.has_gpere)
305 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
a1b7b9b2
DH		 306 if (sclp.has_gsls)
307 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
5630a8e8
DH		 308 if (sclp.has_ib)
309 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
13ee3f67
DH		 310 if (sclp.has_cei)
311 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
7fd7f39d
DH		 312 if (sclp.has_ibs)
313 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
730cd632
FA		 314 if (sclp.has_kss)
315 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
5d3876a8
DH		 316 /*
317 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
318 * all skey handling functions read/set the skey from the PGSTE
319 * instead of the real storage key.
320 *
321 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
322 * pages being detected as preserved although they are resident.
323 *
324 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
325 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
326 *
327 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
328 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
329 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
330 *
331 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
332 * cannot easily shadow the SCA because of the ipte lock.
333 */
22be5a13
DH		 334}
335
b0c632db
HC		 336int kvm_arch_init(void *opaque)
337{
78f26131
CB		 338 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
339 if (!kvm_s390_dbf)
340 return -ENOMEM;
341
342 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
343 debug_unregister(kvm_s390_dbf);
344 return -ENOMEM;
345 }
346
22be5a13
DH		 347 kvm_s390_cpu_feat_init();
348
84877d93
CH		 349 /* Register floating interrupt controller interface. */
350 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 351}
352
78f26131
CB		 353void kvm_arch_exit(void)
354{
355 debug_unregister(kvm_s390_dbf);
356}
357
b0c632db
HC		 358/* Section: device related */
359long kvm_arch_dev_ioctl(struct file *filp,
360 unsigned int ioctl, unsigned long arg)
361{
362 if (ioctl == KVM_S390_ENABLE_SIE)
363 return s390_enable_sie();
364 return -EINVAL;
365}
366
784aa3d7 367int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 368{
d7b0b5eb
CO		 369 int r;
370
2bd0ac4e 371 switch (ext) {
d7b0b5eb 372 case KVM_CAP_S390_PSW:
b6cf8788 373 case KVM_CAP_S390_GMAP:
52e16b18 374 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 375#ifdef CONFIG_KVM_S390_UCONTROL
376 case KVM_CAP_S390_UCONTROL:
377#endif
3c038e6b 378 case KVM_CAP_ASYNC_PF:
60b413c9 379 case KVM_CAP_SYNC_REGS:
14eebd91 380 case KVM_CAP_ONE_REG:
d6712df9 381 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 382 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 383 case KVM_CAP_IOEVENTFD:
c05c4186 384 case KVM_CAP_DEVICE_CTRL:
d938dc55 385 case KVM_CAP_ENABLE_CAP_VM:
78599d90 386 case KVM_CAP_S390_IRQCHIP:
f2061656 387 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 388 case KVM_CAP_MP_STATE:
460df4c1 389 case KVM_CAP_IMMEDIATE_EXIT:
47b43c52 390 case KVM_CAP_S390_INJECT_IRQ:
2444b352 391 case KVM_CAP_S390_USER_SIGP:
e44fc8c9 392 case KVM_CAP_S390_USER_STSI:
30ee2a98 393 case KVM_CAP_S390_SKEYS:
816c7667 394 case KVM_CAP_S390_IRQ_STATE:
6502a34c 395 case KVM_CAP_S390_USER_INSTR0:
4036e387 396 case KVM_CAP_S390_CMMA_MIGRATION:
47a4693e 397 case KVM_CAP_S390_AIS:
da9a1446 398 case KVM_CAP_S390_AIS_MIGRATION:
d7b0b5eb
CO		 399 r = 1;
400 break;
41408c28
TH		 401 case KVM_CAP_S390_MEM_OP:
402 r = MEM_OP_MAX_SIZE;
403 break;
e726b1bd
CB		 404 case KVM_CAP_NR_VCPUS:
405 case KVM_CAP_MAX_VCPUS:
76a6dd72 406 r = KVM_S390_BSCA_CPU_SLOTS;
a6940674
DH		 407 if (!kvm_s390_use_sca_entries())
408 r = KVM_MAX_VCPUS;
409 else if (sclp.has_esca && sclp.has_64bscao)
76a6dd72 410 r = KVM_S390_ESCA_CPU_SLOTS;
e726b1bd 411 break;
e1e2e605
NW		 412 case KVM_CAP_NR_MEMSLOTS:
413 r = KVM_USER_MEM_SLOTS;
414 break;
1526bf9c 415 case KVM_CAP_S390_COW:
abf09bed 416 r = MACHINE_HAS_ESOP;
1526bf9c 417 break;
68c55750
EF		 418 case KVM_CAP_S390_VECTOR_REGISTERS:
419 r = MACHINE_HAS_VX;
420 break;
c6e5f166
FZ		 421 case KVM_CAP_S390_RI:
422 r = test_facility(64);
423 break;
4e0b1ab7
FZ		 424 case KVM_CAP_S390_GS:
425 r = test_facility(133);
426 break;
2bd0ac4e 427 default:
d7b0b5eb 428 r = 0;
2bd0ac4e 429 }
d7b0b5eb 430 return r;
b0c632db
HC		 431}
432
15f36ebd
JH		 433static void kvm_s390_sync_dirty_log(struct kvm *kvm,
434 struct kvm_memory_slot *memslot)
435{
436 gfn_t cur_gfn, last_gfn;
437 unsigned long address;
438 struct gmap *gmap = kvm->arch.gmap;
439
15f36ebd
JH		 440 /* Loop over all guest pages */
441 last_gfn = memslot->base_gfn + memslot->npages;
442 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
443 address = gfn_to_hva_memslot(memslot, cur_gfn);
444
1e133ab2 445 if (test_and_clear_guest_dirty(gmap->mm, address))
15f36ebd 446 mark_page_dirty(kvm, cur_gfn);
1763f8d0
CB		 447 if (fatal_signal_pending(current))
448 return;
70c88a00 449 cond_resched();
15f36ebd 450 }
15f36ebd
JH		 451}
452
b0c632db 453/* Section: vm related */
a6e2f683
ED		 454static void sca_del_vcpu(struct kvm_vcpu *vcpu);
455
b0c632db
HC		 456/*
457 * Get (and clear) the dirty memory log for a memory slot.
458 */
459int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
460 struct kvm_dirty_log *log)
461{
15f36ebd
JH		 462 int r;
463 unsigned long n;
9f6b8029 464 struct kvm_memslots *slots;
15f36ebd
JH		 465 struct kvm_memory_slot *memslot;
466 int is_dirty = 0;
467
e1e8a962
JF		 468 if (kvm_is_ucontrol(kvm))
469 return -EINVAL;
470
15f36ebd
JH		 471 mutex_lock(&kvm->slots_lock);
472
473 r = -EINVAL;
474 if (log->slot >= KVM_USER_MEM_SLOTS)
475 goto out;
476
9f6b8029
PB		 477 slots = kvm_memslots(kvm);
478 memslot = id_to_memslot(slots, log->slot);
15f36ebd
JH		 479 r = -ENOENT;
480 if (!memslot->dirty_bitmap)
481 goto out;
482
483 kvm_s390_sync_dirty_log(kvm, memslot);
484 r = kvm_get_dirty_log(kvm, log, &is_dirty);
485 if (r)
486 goto out;
487
488 /* Clear the dirty log */
489 if (is_dirty) {
490 n = kvm_dirty_bitmap_bytes(memslot);
491 memset(memslot->dirty_bitmap, 0, n);
492 }
493 r = 0;
494out:
495 mutex_unlock(&kvm->slots_lock);
496 return r;
b0c632db
HC		 497}
498
6502a34c
DH		 499static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
500{
501 unsigned int i;
502 struct kvm_vcpu *vcpu;
503
504 kvm_for_each_vcpu(i, vcpu, kvm) {
505 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
506 }
507}
508
d938dc55
CH		 509static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
510{
511 int r;
512
513 if (cap->flags)
514 return -EINVAL;
515
516 switch (cap->cap) {
84223598 517 case KVM_CAP_S390_IRQCHIP:
c92ea7b9 518 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
84223598
CH		 519 kvm->arch.use_irqchip = 1;
520 r = 0;
521 break;
2444b352 522 case KVM_CAP_S390_USER_SIGP:
c92ea7b9 523 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
2444b352
DH		 524 kvm->arch.user_sigp = 1;
525 r = 0;
526 break;
68c55750 527 case KVM_CAP_S390_VECTOR_REGISTERS:
5967c17b 528 mutex_lock(&kvm->lock);
a03825bb 529 if (kvm->created_vcpus) {
5967c17b
DH		 530 r = -EBUSY;
531 } else if (MACHINE_HAS_VX) {
c54f0d6a
DH		 532 set_kvm_facility(kvm->arch.model.fac_mask, 129);
533 set_kvm_facility(kvm->arch.model.fac_list, 129);
2f87d942
GH		 534 if (test_facility(134)) {
535 set_kvm_facility(kvm->arch.model.fac_mask, 134);
536 set_kvm_facility(kvm->arch.model.fac_list, 134);
537 }
53743aa7
MS		 538 if (test_facility(135)) {
539 set_kvm_facility(kvm->arch.model.fac_mask, 135);
540 set_kvm_facility(kvm->arch.model.fac_list, 135);
541 }
18280d8b
MM		 542 r = 0;
543 } else
544 r = -EINVAL;
5967c17b 545 mutex_unlock(&kvm->lock);
c92ea7b9
CB		 546 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
547 r ? "(not available)" : "(success)");
68c55750 548 break;
c6e5f166
FZ		 549 case KVM_CAP_S390_RI:
550 r = -EINVAL;
551 mutex_lock(&kvm->lock);
a03825bb 552 if (kvm->created_vcpus) {
c6e5f166
FZ		 553 r = -EBUSY;
554 } else if (test_facility(64)) {
c54f0d6a
DH		 555 set_kvm_facility(kvm->arch.model.fac_mask, 64);
556 set_kvm_facility(kvm->arch.model.fac_list, 64);
c6e5f166
FZ		 557 r = 0;
558 }
559 mutex_unlock(&kvm->lock);
560 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
561 r ? "(not available)" : "(success)");
562 break;
47a4693e
YMZ		 563 case KVM_CAP_S390_AIS:
564 mutex_lock(&kvm->lock);
565 if (kvm->created_vcpus) {
566 r = -EBUSY;
567 } else {
568 set_kvm_facility(kvm->arch.model.fac_mask, 72);
569 set_kvm_facility(kvm->arch.model.fac_list, 72);
47a4693e
YMZ		 570 r = 0;
571 }
572 mutex_unlock(&kvm->lock);
573 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
574 r ? "(not available)" : "(success)");
575 break;
4e0b1ab7
FZ		 576 case KVM_CAP_S390_GS:
577 r = -EINVAL;
578 mutex_lock(&kvm->lock);
579 if (atomic_read(&kvm->online_vcpus)) {
580 r = -EBUSY;
581 } else if (test_facility(133)) {
582 set_kvm_facility(kvm->arch.model.fac_mask, 133);
583 set_kvm_facility(kvm->arch.model.fac_list, 133);
584 r = 0;
585 }
586 mutex_unlock(&kvm->lock);
587 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
588 r ? "(not available)" : "(success)");
589 break;
e44fc8c9 590 case KVM_CAP_S390_USER_STSI:
c92ea7b9 591 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
e44fc8c9
ET		 592 kvm->arch.user_stsi = 1;
593 r = 0;
594 break;
6502a34c
DH		 595 case KVM_CAP_S390_USER_INSTR0:
596 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
597 kvm->arch.user_instr0 = 1;
598 icpt_operexc_on_all_vcpus(kvm);
599 r = 0;
600 break;
d938dc55
CH		 601 default:
602 r = -EINVAL;
603 break;
604 }
605 return r;
606}
607
8c0a7ce6
DD		 608static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
609{
610 int ret;
611
612 switch (attr->attr) {
613 case KVM_S390_VM_MEM_LIMIT_SIZE:
614 ret = 0;
c92ea7b9 615 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
a3a92c31
DD		 616 kvm->arch.mem_limit);
617 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
8c0a7ce6
DD		 618 ret = -EFAULT;
619 break;
620 default:
621 ret = -ENXIO;
622 break;
623 }
624 return ret;
625}
626
627static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 628{
629 int ret;
630 unsigned int idx;
631 switch (attr->attr) {
632 case KVM_S390_VM_MEM_ENABLE_CMMA:
f9cbd9b0 633 ret = -ENXIO;
c24cc9c8 634 if (!sclp.has_cmma)
e6db1d61
DD		 635 break;
636
4f718eab 637 ret = -EBUSY;
c92ea7b9 638 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
4f718eab 639 mutex_lock(&kvm->lock);
a03825bb 640 if (!kvm->created_vcpus) {
4f718eab
DD		 641 kvm->arch.use_cmma = 1;
642 ret = 0;
643 }
644 mutex_unlock(&kvm->lock);
645 break;
646 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 647 ret = -ENXIO;
648 if (!sclp.has_cmma)
649 break;
c3489155
DD		 650 ret = -EINVAL;
651 if (!kvm->arch.use_cmma)
652 break;
653
c92ea7b9 654 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
4f718eab
DD		 655 mutex_lock(&kvm->lock);
656 idx = srcu_read_lock(&kvm->srcu);
a13cff31 657 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 658 srcu_read_unlock(&kvm->srcu, idx);
659 mutex_unlock(&kvm->lock);
660 ret = 0;
661 break;
8c0a7ce6
DD		 662 case KVM_S390_VM_MEM_LIMIT_SIZE: {
663 unsigned long new_limit;
664
665 if (kvm_is_ucontrol(kvm))
666 return -EINVAL;
667
668 if (get_user(new_limit, (u64 __user *)attr->addr))
669 return -EFAULT;
670
a3a92c31
DD		 671 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
672 new_limit > kvm->arch.mem_limit)
8c0a7ce6
DD		 673 return -E2BIG;
674
a3a92c31
DD		 675 if (!new_limit)
676 return -EINVAL;
677
6ea427bb 678 /* gmap_create takes last usable address */
a3a92c31
DD		 679 if (new_limit != KVM_S390_NO_MEM_LIMIT)
680 new_limit -= 1;
681
8c0a7ce6
DD		 682 ret = -EBUSY;
683 mutex_lock(&kvm->lock);
a03825bb 684 if (!kvm->created_vcpus) {
6ea427bb
MS		 685 /* gmap_create will round the limit up */
686 struct gmap *new = gmap_create(current->mm, new_limit);
8c0a7ce6
DD		 687
688 if (!new) {
689 ret = -ENOMEM;
690 } else {
6ea427bb 691 gmap_remove(kvm->arch.gmap);
8c0a7ce6
DD		 692 new->private = kvm;
693 kvm->arch.gmap = new;
694 ret = 0;
695 }
696 }
697 mutex_unlock(&kvm->lock);
a3a92c31
DD		 698 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
699 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
700 (void *) kvm->arch.gmap->asce);
8c0a7ce6
DD		 701 break;
702 }
4f718eab
DD		 703 default:
704 ret = -ENXIO;
705 break;
706 }
707 return ret;
708}
709
a374e892
TK		 710static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
711
712static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
713{
714 struct kvm_vcpu *vcpu;
715 int i;
716
9d8d5786 717 if (!test_kvm_facility(kvm, 76))
a374e892
TK		 718 return -EINVAL;
719
720 mutex_lock(&kvm->lock);
721 switch (attr->attr) {
722 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
723 get_random_bytes(
724 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
725 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
726 kvm->arch.crypto.aes_kw = 1;
c92ea7b9 727 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
a374e892
TK		 728 break;
729 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
730 get_random_bytes(
731 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
732 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
733 kvm->arch.crypto.dea_kw = 1;
c92ea7b9 734 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
a374e892
TK		 735 break;
736 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
737 kvm->arch.crypto.aes_kw = 0;
738 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
739 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
c92ea7b9 740 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
a374e892
TK		 741 break;
742 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
743 kvm->arch.crypto.dea_kw = 0;
744 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
745 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
c92ea7b9 746 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
a374e892
TK		 747 break;
748 default:
749 mutex_unlock(&kvm->lock);
750 return -ENXIO;
751 }
752
753 kvm_for_each_vcpu(i, vcpu, kvm) {
754 kvm_s390_vcpu_crypto_setup(vcpu);
755 exit_sie(vcpu);
756 }
757 mutex_unlock(&kvm->lock);
758 return 0;
759}
760
190df4a2
CI		 761static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
762{
763 int cx;
764 struct kvm_vcpu *vcpu;
765
766 kvm_for_each_vcpu(cx, vcpu, kvm)
767 kvm_s390_sync_request(req, vcpu);
768}
769
770/*
771 * Must be called with kvm->srcu held to avoid races on memslots, and with
772 * kvm->lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
773 */
774static int kvm_s390_vm_start_migration(struct kvm *kvm)
775{
776 struct kvm_s390_migration_state *mgs;
777 struct kvm_memory_slot *ms;
778 /* should be the only one */
779 struct kvm_memslots *slots;
780 unsigned long ram_pages;
781 int slotnr;
782
783 /* migration mode already enabled */
784 if (kvm->arch.migration_state)
785 return 0;
786
787 slots = kvm_memslots(kvm);
788 if (!slots || !slots->used_slots)
789 return -EINVAL;
790
791 mgs = kzalloc(sizeof(*mgs), GFP_KERNEL);
792 if (!mgs)
793 return -ENOMEM;
794 kvm->arch.migration_state = mgs;
795
796 if (kvm->arch.use_cmma) {
797 /*
798 * Get the last slot. They should be sorted by base_gfn, so the
799 * last slot is also the one at the end of the address space.
800 * We have verified above that at least one slot is present.
801 */
802 ms = slots->memslots + slots->used_slots - 1;
803 /* round up so we only use full longs */
804 ram_pages = roundup(ms->base_gfn + ms->npages, BITS_PER_LONG);
805 /* allocate enough bytes to store all the bits */
806 mgs->pgste_bitmap = vmalloc(ram_pages / 8);
807 if (!mgs->pgste_bitmap) {
808 kfree(mgs);
809 kvm->arch.migration_state = NULL;
810 return -ENOMEM;
811 }
812
813 mgs->bitmap_size = ram_pages;
814 atomic64_set(&mgs->dirty_pages, ram_pages);
815 /* mark all the pages in active slots as dirty */
816 for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
817 ms = slots->memslots + slotnr;
818 bitmap_set(mgs->pgste_bitmap, ms->base_gfn, ms->npages);
819 }
820
821 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
822 }
823 return 0;
824}
825
826/*
827 * Must be called with kvm->lock to avoid races with ourselves and
828 * kvm_s390_vm_start_migration.
829 */
830static int kvm_s390_vm_stop_migration(struct kvm *kvm)
831{
832 struct kvm_s390_migration_state *mgs;
833
834 /* migration mode already disabled */
835 if (!kvm->arch.migration_state)
836 return 0;
837 mgs = kvm->arch.migration_state;
838 kvm->arch.migration_state = NULL;
839
840 if (kvm->arch.use_cmma) {
841 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
842 vfree(mgs->pgste_bitmap);
843 }
844 kfree(mgs);
845 return 0;
846}
847
848static int kvm_s390_vm_set_migration(struct kvm *kvm,
849 struct kvm_device_attr *attr)
850{
851 int idx, res = -ENXIO;
852
853 mutex_lock(&kvm->lock);
854 switch (attr->attr) {
855 case KVM_S390_VM_MIGRATION_START:
856 idx = srcu_read_lock(&kvm->srcu);
857 res = kvm_s390_vm_start_migration(kvm);
858 srcu_read_unlock(&kvm->srcu, idx);
859 break;
860 case KVM_S390_VM_MIGRATION_STOP:
861 res = kvm_s390_vm_stop_migration(kvm);
862 break;
863 default:
864 break;
865 }
866 mutex_unlock(&kvm->lock);
867
868 return res;
869}
870
871static int kvm_s390_vm_get_migration(struct kvm *kvm,
872 struct kvm_device_attr *attr)
873{
874 u64 mig = (kvm->arch.migration_state != NULL);
875
876 if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
877 return -ENXIO;
878
879 if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
880 return -EFAULT;
881 return 0;
882}
883
8fa1696e
CW		 884static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
885{
886 struct kvm_s390_vm_tod_clock gtod;
887
888 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
889 return -EFAULT;
890
891 if (test_kvm_facility(kvm, 139))
892 kvm_s390_set_tod_clock_ext(kvm, &gtod);
893 else if (gtod.epoch_idx == 0)
894 kvm_s390_set_tod_clock(kvm, gtod.tod);
895 else
896 return -EINVAL;
897
898 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
899 gtod.epoch_idx, gtod.tod);
900
901 return 0;
902}
903
72f25020
JH		 904static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
905{
906 u8 gtod_high;
907
908 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
909 sizeof(gtod_high)))
910 return -EFAULT;
911
912 if (gtod_high != 0)
913 return -EINVAL;
58c383c6 914 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
72f25020
JH		 915
916 return 0;
917}
918
919static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
920{
5a3d883a 921 u64 gtod;
72f25020
JH		 922
923 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
924 return -EFAULT;
925
25ed1675 926 kvm_s390_set_tod_clock(kvm, gtod);
58c383c6 927 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
72f25020
JH		 928 return 0;
929}
930
931static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
932{
933 int ret;
934
935 if (attr->flags)
936 return -EINVAL;
937
938 switch (attr->attr) {
8fa1696e
CW		 939 case KVM_S390_VM_TOD_EXT:
940 ret = kvm_s390_set_tod_ext(kvm, attr);
941 break;
72f25020
JH		 942 case KVM_S390_VM_TOD_HIGH:
943 ret = kvm_s390_set_tod_high(kvm, attr);
944 break;
945 case KVM_S390_VM_TOD_LOW:
946 ret = kvm_s390_set_tod_low(kvm, attr);
947 break;
948 default:
949 ret = -ENXIO;
950 break;
951 }
952 return ret;
953}
954
8fa1696e
CW		 955static void kvm_s390_get_tod_clock_ext(struct kvm *kvm,
956 struct kvm_s390_vm_tod_clock *gtod)
957{
958 struct kvm_s390_tod_clock_ext htod;
959
960 preempt_disable();
961
962 get_tod_clock_ext((char *)&htod);
963
964 gtod->tod = htod.tod + kvm->arch.epoch;
965 gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
966
967 if (gtod->tod < htod.tod)
968 gtod->epoch_idx += 1;
969
970 preempt_enable();
971}
972
973static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
974{
975 struct kvm_s390_vm_tod_clock gtod;
976
977 memset(&gtod, 0, sizeof(gtod));
978
979 if (test_kvm_facility(kvm, 139))
980 kvm_s390_get_tod_clock_ext(kvm, &gtod);
981 else
982 gtod.tod = kvm_s390_get_tod_clock_fast(kvm);
983
984 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
985 return -EFAULT;
986
987 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
988 gtod.epoch_idx, gtod.tod);
989 return 0;
990}
991
72f25020
JH		 992static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
993{
994 u8 gtod_high = 0;
995
996 if (copy_to_user((void __user *)attr->addr, &gtod_high,
997 sizeof(gtod_high)))
998 return -EFAULT;
58c383c6 999 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
72f25020
JH		 1000
1001 return 0;
1002}
1003
1004static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1005{
5a3d883a 1006 u64 gtod;
72f25020 1007
60417fcc 1008 gtod = kvm_s390_get_tod_clock_fast(kvm);
72f25020
JH		 1009 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1010 return -EFAULT;
58c383c6 1011 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
72f25020
JH		 1012
1013 return 0;
1014}
1015
1016static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1017{
1018 int ret;
1019
1020 if (attr->flags)
1021 return -EINVAL;
1022
1023 switch (attr->attr) {
8fa1696e
CW		 1024 case KVM_S390_VM_TOD_EXT:
1025 ret = kvm_s390_get_tod_ext(kvm, attr);
1026 break;
72f25020
JH		 1027 case KVM_S390_VM_TOD_HIGH:
1028 ret = kvm_s390_get_tod_high(kvm, attr);
1029 break;
1030 case KVM_S390_VM_TOD_LOW:
1031 ret = kvm_s390_get_tod_low(kvm, attr);
1032 break;
1033 default:
1034 ret = -ENXIO;
1035 break;
1036 }
1037 return ret;
1038}
1039
658b6eda
MM		 1040static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1041{
1042 struct kvm_s390_vm_cpu_processor *proc;
053dd230 1043 u16 lowest_ibc, unblocked_ibc;
658b6eda
MM		 1044 int ret = 0;
1045
1046 mutex_lock(&kvm->lock);
a03825bb 1047 if (kvm->created_vcpus) {
658b6eda
MM		 1048 ret = -EBUSY;
1049 goto out;
1050 }
1051 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1052 if (!proc) {
1053 ret = -ENOMEM;
1054 goto out;
1055 }
1056 if (!copy_from_user(proc, (void __user *)attr->addr,
1057 sizeof(*proc))) {
9bb0ec09 1058 kvm->arch.model.cpuid = proc->cpuid;
053dd230
DH		 1059 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1060 unblocked_ibc = sclp.ibc & 0xfff;
0487c44d 1061 if (lowest_ibc && proc->ibc) {
053dd230
DH		 1062 if (proc->ibc > unblocked_ibc)
1063 kvm->arch.model.ibc = unblocked_ibc;
1064 else if (proc->ibc < lowest_ibc)
1065 kvm->arch.model.ibc = lowest_ibc;
1066 else
1067 kvm->arch.model.ibc = proc->ibc;
1068 }
c54f0d6a 1069 memcpy(kvm->arch.model.fac_list, proc->fac_list,
658b6eda 1070 S390_ARCH_FAC_LIST_SIZE_BYTE);
a8c39dd7
CB		 1071 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1072 kvm->arch.model.ibc,
1073 kvm->arch.model.cpuid);
1074 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1075 kvm->arch.model.fac_list[0],
1076 kvm->arch.model.fac_list[1],
1077 kvm->arch.model.fac_list[2]);
658b6eda
MM		 1078 } else
1079 ret = -EFAULT;
1080 kfree(proc);
1081out:
1082 mutex_unlock(&kvm->lock);
1083 return ret;
1084}
1085
15c9705f
DH		 1086static int kvm_s390_set_processor_feat(struct kvm *kvm,
1087 struct kvm_device_attr *attr)
1088{
1089 struct kvm_s390_vm_cpu_feat data;
1090 int ret = -EBUSY;
1091
1092 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1093 return -EFAULT;
1094 if (!bitmap_subset((unsigned long *) data.feat,
1095 kvm_s390_available_cpu_feat,
1096 KVM_S390_VM_CPU_FEAT_NR_BITS))
1097 return -EINVAL;
1098
1099 mutex_lock(&kvm->lock);
1100 if (!atomic_read(&kvm->online_vcpus)) {
1101 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
1102 KVM_S390_VM_CPU_FEAT_NR_BITS);
1103 ret = 0;
1104 }
1105 mutex_unlock(&kvm->lock);
1106 return ret;
1107}
1108
0a763c78
DH		 1109static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1110 struct kvm_device_attr *attr)
1111{
1112 /*
1113 * Once supported by kernel + hw, we have to store the subfunctions
1114 * in kvm->arch and remember that user space configured them.
1115 */
1116 return -ENXIO;
1117}
1118
658b6eda
MM		 1119static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1120{
1121 int ret = -ENXIO;
1122
1123 switch (attr->attr) {
1124 case KVM_S390_VM_CPU_PROCESSOR:
1125 ret = kvm_s390_set_processor(kvm, attr);
1126 break;
15c9705f
DH		 1127 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1128 ret = kvm_s390_set_processor_feat(kvm, attr);
1129 break;
0a763c78
DH		 1130 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1131 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1132 break;
658b6eda
MM		 1133 }
1134 return ret;
1135}
1136
1137static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1138{
1139 struct kvm_s390_vm_cpu_processor *proc;
1140 int ret = 0;
1141
1142 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1143 if (!proc) {
1144 ret = -ENOMEM;
1145 goto out;
1146 }
9bb0ec09 1147 proc->cpuid = kvm->arch.model.cpuid;
658b6eda 1148 proc->ibc = kvm->arch.model.ibc;
c54f0d6a
DH		 1149 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1150 S390_ARCH_FAC_LIST_SIZE_BYTE);
a8c39dd7
CB		 1151 VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1152 kvm->arch.model.ibc,
1153 kvm->arch.model.cpuid);
1154 VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1155 kvm->arch.model.fac_list[0],
1156 kvm->arch.model.fac_list[1],
1157 kvm->arch.model.fac_list[2]);
658b6eda
MM		 1158 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1159 ret = -EFAULT;
1160 kfree(proc);
1161out:
1162 return ret;
1163}
1164
1165static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1166{
1167 struct kvm_s390_vm_cpu_machine *mach;
1168 int ret = 0;
1169
1170 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
1171 if (!mach) {
1172 ret = -ENOMEM;
1173 goto out;
1174 }
1175 get_cpu_id((struct cpuid *) &mach->cpuid);
37c5f6c8 1176 mach->ibc = sclp.ibc;
c54f0d6a 1177 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
981467c9 1178 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 1179 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
04478197 1180 sizeof(S390_lowcore.stfle_fac_list));
a8c39dd7
CB		 1181 VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx",
1182 kvm->arch.model.ibc,
1183 kvm->arch.model.cpuid);
1184 VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx",
1185 mach->fac_mask[0],
1186 mach->fac_mask[1],
1187 mach->fac_mask[2]);
1188 VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1189 mach->fac_list[0],
1190 mach->fac_list[1],
1191 mach->fac_list[2]);
658b6eda
MM		 1192 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1193 ret = -EFAULT;
1194 kfree(mach);
1195out:
1196 return ret;
1197}
1198
15c9705f
DH		 1199static int kvm_s390_get_processor_feat(struct kvm *kvm,
1200 struct kvm_device_attr *attr)
1201{
1202 struct kvm_s390_vm_cpu_feat data;
1203
1204 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1205 KVM_S390_VM_CPU_FEAT_NR_BITS);
1206 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1207 return -EFAULT;
1208 return 0;
1209}
1210
1211static int kvm_s390_get_machine_feat(struct kvm *kvm,
1212 struct kvm_device_attr *attr)
1213{
1214 struct kvm_s390_vm_cpu_feat data;
1215
1216 bitmap_copy((unsigned long *) data.feat,
1217 kvm_s390_available_cpu_feat,
1218 KVM_S390_VM_CPU_FEAT_NR_BITS);
1219 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1220 return -EFAULT;
1221 return 0;
1222}
1223
0a763c78
DH		 1224static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1225 struct kvm_device_attr *attr)
1226{
1227 /*
1228 * Once we can actually configure subfunctions (kernel + hw support),
1229 * we have to check if they were already set by user space, if so copy
1230 * them from kvm->arch.
1231 */
1232 return -ENXIO;
1233}
1234
1235static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1236 struct kvm_device_attr *attr)
1237{
1238 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1239 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1240 return -EFAULT;
1241 return 0;
1242}
658b6eda
MM		 1243static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1244{
1245 int ret = -ENXIO;
1246
1247 switch (attr->attr) {
1248 case KVM_S390_VM_CPU_PROCESSOR:
1249 ret = kvm_s390_get_processor(kvm, attr);
1250 break;
1251 case KVM_S390_VM_CPU_MACHINE:
1252 ret = kvm_s390_get_machine(kvm, attr);
1253 break;
15c9705f
DH		 1254 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1255 ret = kvm_s390_get_processor_feat(kvm, attr);
1256 break;
1257 case KVM_S390_VM_CPU_MACHINE_FEAT:
1258 ret = kvm_s390_get_machine_feat(kvm, attr);
1259 break;
0a763c78
DH		 1260 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1261 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1262 break;
1263 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1264 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1265 break;
658b6eda
MM		 1266 }
1267 return ret;
1268}
1269
f2061656
DD		 1270static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1271{
1272 int ret;
1273
1274 switch (attr->group) {
4f718eab 1275 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 1276 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 1277 break;
72f25020
JH		 1278 case KVM_S390_VM_TOD:
1279 ret = kvm_s390_set_tod(kvm, attr);
1280 break;
658b6eda
MM		 1281 case KVM_S390_VM_CPU_MODEL:
1282 ret = kvm_s390_set_cpu_model(kvm, attr);
1283 break;
a374e892
TK		 1284 case KVM_S390_VM_CRYPTO:
1285 ret = kvm_s390_vm_set_crypto(kvm, attr);
1286 break;
190df4a2
CI		 1287 case KVM_S390_VM_MIGRATION:
1288 ret = kvm_s390_vm_set_migration(kvm, attr);
1289 break;
f2061656
DD		 1290 default:
1291 ret = -ENXIO;
1292 break;
1293 }
1294
1295 return ret;
1296}
1297
1298static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1299{
8c0a7ce6
DD		 1300 int ret;
1301
1302 switch (attr->group) {
1303 case KVM_S390_VM_MEM_CTRL:
1304 ret = kvm_s390_get_mem_control(kvm, attr);
1305 break;
72f25020
JH		 1306 case KVM_S390_VM_TOD:
1307 ret = kvm_s390_get_tod(kvm, attr);
1308 break;
658b6eda
MM		 1309 case KVM_S390_VM_CPU_MODEL:
1310 ret = kvm_s390_get_cpu_model(kvm, attr);
1311 break;
190df4a2
CI		 1312 case KVM_S390_VM_MIGRATION:
1313 ret = kvm_s390_vm_get_migration(kvm, attr);
1314 break;
8c0a7ce6
DD		 1315 default:
1316 ret = -ENXIO;
1317 break;
1318 }
1319
1320 return ret;
f2061656
DD		 1321}
1322
1323static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1324{
1325 int ret;
1326
1327 switch (attr->group) {
4f718eab
DD		 1328 case KVM_S390_VM_MEM_CTRL:
1329 switch (attr->attr) {
1330 case KVM_S390_VM_MEM_ENABLE_CMMA:
1331 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 1332 ret = sclp.has_cmma ? 0 : -ENXIO;
1333 break;
8c0a7ce6 1334 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 1335 ret = 0;
1336 break;
1337 default:
1338 ret = -ENXIO;
1339 break;
1340 }
1341 break;
72f25020
JH		 1342 case KVM_S390_VM_TOD:
1343 switch (attr->attr) {
1344 case KVM_S390_VM_TOD_LOW:
1345 case KVM_S390_VM_TOD_HIGH:
1346 ret = 0;
1347 break;
1348 default:
1349 ret = -ENXIO;
1350 break;
1351 }
1352 break;
658b6eda
MM		 1353 case KVM_S390_VM_CPU_MODEL:
1354 switch (attr->attr) {
1355 case KVM_S390_VM_CPU_PROCESSOR:
1356 case KVM_S390_VM_CPU_MACHINE:
15c9705f
DH		 1357 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1358 case KVM_S390_VM_CPU_MACHINE_FEAT:
0a763c78 1359 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
658b6eda
MM		 1360 ret = 0;
1361 break;
0a763c78
DH		 1362 /* configuring subfunctions is not supported yet */
1363 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
658b6eda
MM		 1364 default:
1365 ret = -ENXIO;
1366 break;
1367 }
1368 break;
a374e892
TK		 1369 case KVM_S390_VM_CRYPTO:
1370 switch (attr->attr) {
1371 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1372 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1373 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1374 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1375 ret = 0;
1376 break;
1377 default:
1378 ret = -ENXIO;
1379 break;
1380 }
1381 break;
190df4a2
CI		 1382 case KVM_S390_VM_MIGRATION:
1383 ret = 0;
1384 break;
f2061656
DD		 1385 default:
1386 ret = -ENXIO;
1387 break;
1388 }
1389
1390 return ret;
1391}
1392
30ee2a98
JH		 1393static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1394{
1395 uint8_t *keys;
1396 uint64_t hva;
4f899147 1397 int srcu_idx, i, r = 0;
30ee2a98
JH		 1398
1399 if (args->flags != 0)
1400 return -EINVAL;
1401
1402 /* Is this guest using storage keys? */
1403 if (!mm_use_skey(current->mm))
1404 return KVM_S390_GET_SKEYS_NONE;
1405
1406 /* Enforce sane limit on memory allocation */
1407 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1408 return -EINVAL;
1409
752ade68 1410 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
30ee2a98
JH		 1411 if (!keys)
1412 return -ENOMEM;
1413
d3ed1cee 1414 down_read(&current->mm->mmap_sem);
4f899147 1415 srcu_idx = srcu_read_lock(&kvm->srcu);
30ee2a98
JH		 1416 for (i = 0; i < args->count; i++) {
1417 hva = gfn_to_hva(kvm, args->start_gfn + i);
1418 if (kvm_is_error_hva(hva)) {
1419 r = -EFAULT;
d3ed1cee 1420 break;
30ee2a98
JH		 1421 }
1422
154c8c19
DH		 1423 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1424 if (r)
d3ed1cee 1425 break;
30ee2a98 1426 }
4f899147 1427 srcu_read_unlock(&kvm->srcu, srcu_idx);
d3ed1cee
MS		 1428 up_read(&current->mm->mmap_sem);
1429
1430 if (!r) {
1431 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1432 sizeof(uint8_t) * args->count);
1433 if (r)
1434 r = -EFAULT;
30ee2a98
JH		 1435 }
1436
30ee2a98
JH		 1437 kvfree(keys);
1438 return r;
1439}
1440
1441static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1442{
1443 uint8_t *keys;
1444 uint64_t hva;
4f899147 1445 int srcu_idx, i, r = 0;
30ee2a98
JH		 1446
1447 if (args->flags != 0)
1448 return -EINVAL;
1449
1450 /* Enforce sane limit on memory allocation */
1451 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1452 return -EINVAL;
1453
752ade68 1454 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
30ee2a98
JH		 1455 if (!keys)
1456 return -ENOMEM;
1457
1458 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1459 sizeof(uint8_t) * args->count);
1460 if (r) {
1461 r = -EFAULT;
1462 goto out;
1463 }
1464
1465 /* Enable storage key handling for the guest */
14d4a425
DD		 1466 r = s390_enable_skey();
1467 if (r)
1468 goto out;
30ee2a98 1469
d3ed1cee 1470 down_read(&current->mm->mmap_sem);
4f899147 1471 srcu_idx = srcu_read_lock(&kvm->srcu);
30ee2a98
JH		 1472 for (i = 0; i < args->count; i++) {
1473 hva = gfn_to_hva(kvm, args->start_gfn + i);
1474 if (kvm_is_error_hva(hva)) {
1475 r = -EFAULT;
d3ed1cee 1476 break;
30ee2a98
JH		 1477 }
1478
1479 /* Lowest order bit is reserved */
1480 if (keys[i] & 0x01) {
1481 r = -EINVAL;
d3ed1cee 1482 break;
30ee2a98
JH		 1483 }
1484
fe69eabf 1485 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
30ee2a98 1486 if (r)
d3ed1cee 1487 break;
30ee2a98 1488 }
4f899147 1489 srcu_read_unlock(&kvm->srcu, srcu_idx);
d3ed1cee 1490 up_read(&current->mm->mmap_sem);
30ee2a98
JH		 1491out:
1492 kvfree(keys);
1493 return r;
1494}
1495
4036e387
CI		 1496/*
1497 * Base address and length must be sent at the start of each block, therefore
1498 * it's cheaper to send some clean data, as long as it's less than the size of
1499 * two longs.
1500 */
1501#define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
1502/* for consistency */
1503#define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1504
1505/*
1506 * This function searches for the next page with dirty CMMA attributes, and
1507 * saves the attributes in the buffer up to either the end of the buffer or
1508 * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
1509 * no trailing clean bytes are saved.
1510 * In case no dirty bits were found, or if CMMA was not enabled or used, the
1511 * output buffer will indicate 0 as length.
1512 */
1513static int kvm_s390_get_cmma_bits(struct kvm *kvm,
1514 struct kvm_s390_cmma_log *args)
1515{
1516 struct kvm_s390_migration_state *s = kvm->arch.migration_state;
1517 unsigned long bufsize, hva, pgstev, i, next, cur;
1518 int srcu_idx, peek, r = 0, rr;
1519 u8 *res;
1520
1521 cur = args->start_gfn;
1522 i = next = pgstev = 0;
1523
1524 if (unlikely(!kvm->arch.use_cmma))
1525 return -ENXIO;
1526 /* Invalid/unsupported flags were specified */
1527 if (args->flags & ~KVM_S390_CMMA_PEEK)
1528 return -EINVAL;
1529 /* Migration mode query, and we are not doing a migration */
1530 peek = !!(args->flags & KVM_S390_CMMA_PEEK);
1531 if (!peek && !s)
1532 return -EINVAL;
1533 /* CMMA is disabled or was not used, or the buffer has length zero */
1534 bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
1535 if (!bufsize || !kvm->mm->context.use_cmma) {
1536 memset(args, 0, sizeof(*args));
1537 return 0;
1538 }
1539
1540 if (!peek) {
1541 /* We are not peeking, and there are no dirty pages */
1542 if (!atomic64_read(&s->dirty_pages)) {
1543 memset(args, 0, sizeof(*args));
1544 return 0;
1545 }
1546 cur = find_next_bit(s->pgste_bitmap, s->bitmap_size,
1547 args->start_gfn);
1548 if (cur >= s->bitmap_size) /* nothing found, loop back */
1549 cur = find_next_bit(s->pgste_bitmap, s->bitmap_size, 0);
1550 if (cur >= s->bitmap_size) { /* again! (very unlikely) */
1551 memset(args, 0, sizeof(*args));
1552 return 0;
1553 }
1554 next = find_next_bit(s->pgste_bitmap, s->bitmap_size, cur + 1);
1555 }
1556
1557 res = vmalloc(bufsize);
1558 if (!res)
1559 return -ENOMEM;
1560
1561 args->start_gfn = cur;
1562
1563 down_read(&kvm->mm->mmap_sem);
1564 srcu_idx = srcu_read_lock(&kvm->srcu);
1565 while (i < bufsize) {
1566 hva = gfn_to_hva(kvm, cur);
1567 if (kvm_is_error_hva(hva)) {
1568 r = -EFAULT;
1569 break;
1570 }
1571 /* decrement only if we actually flipped the bit to 0 */
1572 if (!peek && test_and_clear_bit(cur, s->pgste_bitmap))
1573 atomic64_dec(&s->dirty_pages);
1574 r = get_pgste(kvm->mm, hva, &pgstev);
1575 if (r < 0)
1576 pgstev = 0;
1577 /* save the value */
1bab1c02 1578 res[i++] = (pgstev >> 24) & 0x43;
4036e387
CI		 1579 /*
1580 * if the next bit is too far away, stop.
1581 * if we reached the previous "next", find the next one
1582 */
1583 if (!peek) {
1584 if (next > cur + KVM_S390_MAX_BIT_DISTANCE)
1585 break;
1586 if (cur == next)
1587 next = find_next_bit(s->pgste_bitmap,
1588 s->bitmap_size, cur + 1);
1589 /* reached the end of the bitmap or of the buffer, stop */
1590 if ((next >= s->bitmap_size) ||
1591 (next >= args->start_gfn + bufsize))
1592 break;
1593 }
1594 cur++;
1595 }
1596 srcu_read_unlock(&kvm->srcu, srcu_idx);
1597 up_read(&kvm->mm->mmap_sem);
1598 args->count = i;
1599 args->remaining = s ? atomic64_read(&s->dirty_pages) : 0;
1600
1601 rr = copy_to_user((void __user *)args->values, res, args->count);
1602 if (rr)
1603 r = -EFAULT;
1604
1605 vfree(res);
1606 return r;
1607}
1608
1609/*
1610 * This function sets the CMMA attributes for the given pages. If the input
1611 * buffer has zero length, no action is taken, otherwise the attributes are
1612 * set and the mm->context.use_cmma flag is set.
1613 */
1614static int kvm_s390_set_cmma_bits(struct kvm *kvm,
1615 const struct kvm_s390_cmma_log *args)
1616{
1617 unsigned long hva, mask, pgstev, i;
1618 uint8_t *bits;
1619 int srcu_idx, r = 0;
1620
1621 mask = args->mask;
1622
1623 if (!kvm->arch.use_cmma)
1624 return -ENXIO;
1625 /* invalid/unsupported flags */
1626 if (args->flags != 0)
1627 return -EINVAL;
1628 /* Enforce sane limit on memory allocation */
1629 if (args->count > KVM_S390_CMMA_SIZE_MAX)
1630 return -EINVAL;
1631 /* Nothing to do */
1632 if (args->count == 0)
1633 return 0;
1634
1635 bits = vmalloc(sizeof(*bits) * args->count);
1636 if (!bits)
1637 return -ENOMEM;
1638
1639 r = copy_from_user(bits, (void __user *)args->values, args->count);
1640 if (r) {
1641 r = -EFAULT;
1642 goto out;
1643 }
1644
1645 down_read(&kvm->mm->mmap_sem);
1646 srcu_idx = srcu_read_lock(&kvm->srcu);
1647 for (i = 0; i < args->count; i++) {
1648 hva = gfn_to_hva(kvm, args->start_gfn + i);
1649 if (kvm_is_error_hva(hva)) {
1650 r = -EFAULT;
1651 break;
1652 }
1653
1654 pgstev = bits[i];
1655 pgstev = pgstev << 24;
1bab1c02 1656 mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
4036e387
CI		 1657 set_pgste_bits(kvm->mm, hva, mask, pgstev);
1658 }
1659 srcu_read_unlock(&kvm->srcu, srcu_idx);
1660 up_read(&kvm->mm->mmap_sem);
1661
1662 if (!kvm->mm->context.use_cmma) {
1663 down_write(&kvm->mm->mmap_sem);
1664 kvm->mm->context.use_cmma = 1;
1665 up_write(&kvm->mm->mmap_sem);
1666 }
1667out:
1668 vfree(bits);
1669 return r;
1670}
1671
b0c632db
HC		 1672long kvm_arch_vm_ioctl(struct file *filp,
1673 unsigned int ioctl, unsigned long arg)
1674{
1675 struct kvm *kvm = filp->private_data;
1676 void __user *argp = (void __user *)arg;
f2061656 1677 struct kvm_device_attr attr;
b0c632db
HC		 1678 int r;
1679
1680 switch (ioctl) {
ba5c1e9b
CO		 1681 case KVM_S390_INTERRUPT: {
1682 struct kvm_s390_interrupt s390int;
1683
1684 r = -EFAULT;
1685 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1686 break;
1687 r = kvm_s390_inject_vm(kvm, &s390int);
1688 break;
1689 }
d938dc55
CH		 1690 case KVM_ENABLE_CAP: {
1691 struct kvm_enable_cap cap;
1692 r = -EFAULT;
1693 if (copy_from_user(&cap, argp, sizeof(cap)))
1694 break;
1695 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1696 break;
1697 }
84223598
CH		 1698 case KVM_CREATE_IRQCHIP: {
1699 struct kvm_irq_routing_entry routing;
1700
1701 r = -EINVAL;
1702 if (kvm->arch.use_irqchip) {
1703 /* Set up dummy routing. */
1704 memset(&routing, 0, sizeof(routing));
152b2839 1705 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
84223598
CH		 1706 }
1707 break;
1708 }
f2061656
DD		 1709 case KVM_SET_DEVICE_ATTR: {
1710 r = -EFAULT;
1711 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1712 break;
1713 r = kvm_s390_vm_set_attr(kvm, &attr);
1714 break;
1715 }
1716 case KVM_GET_DEVICE_ATTR: {
1717 r = -EFAULT;
1718 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1719 break;
1720 r = kvm_s390_vm_get_attr(kvm, &attr);
1721 break;
1722 }
1723 case KVM_HAS_DEVICE_ATTR: {
1724 r = -EFAULT;
1725 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1726 break;
1727 r = kvm_s390_vm_has_attr(kvm, &attr);
1728 break;
1729 }
30ee2a98
JH		 1730 case KVM_S390_GET_SKEYS: {
1731 struct kvm_s390_skeys args;
1732
1733 r = -EFAULT;
1734 if (copy_from_user(&args, argp,
1735 sizeof(struct kvm_s390_skeys)))
1736 break;
1737 r = kvm_s390_get_skeys(kvm, &args);
1738 break;
1739 }
1740 case KVM_S390_SET_SKEYS: {
1741 struct kvm_s390_skeys args;
1742
1743 r = -EFAULT;
1744 if (copy_from_user(&args, argp,
1745 sizeof(struct kvm_s390_skeys)))
1746 break;
1747 r = kvm_s390_set_skeys(kvm, &args);
1748 break;
1749 }
4036e387
CI		 1750 case KVM_S390_GET_CMMA_BITS: {
1751 struct kvm_s390_cmma_log args;
1752
1753 r = -EFAULT;
1754 if (copy_from_user(&args, argp, sizeof(args)))
1755 break;
1756 r = kvm_s390_get_cmma_bits(kvm, &args);
1757 if (!r) {
1758 r = copy_to_user(argp, &args, sizeof(args));
1759 if (r)
1760 r = -EFAULT;
1761 }
1762 break;
1763 }
1764 case KVM_S390_SET_CMMA_BITS: {
1765 struct kvm_s390_cmma_log args;
1766
1767 r = -EFAULT;
1768 if (copy_from_user(&args, argp, sizeof(args)))
1769 break;
1770 r = kvm_s390_set_cmma_bits(kvm, &args);
1771 break;
1772 }
b0c632db 1773 default:
367e1319 1774 r = -ENOTTY;
b0c632db
HC		 1775 }
1776
1777 return r;
1778}
1779
45c9b47c
TK		 1780static int kvm_s390_query_ap_config(u8 *config)
1781{
1782 u32 fcn_code = 0x04000000UL;
86044c8c 1783 u32 cc = 0;
45c9b47c 1784
86044c8c 1785 memset(config, 0, 128);
45c9b47c
TK		 1786 asm volatile(
1787 "lgr 0,%1\n"
1788 "lgr 2,%2\n"
1789 ".long 0xb2af0000\n" /* PQAP(QCI) */
86044c8c 1790 "0: ipm %0\n"
45c9b47c 1791 "srl %0,28\n"
86044c8c
CB		 1792 "1:\n"
1793 EX_TABLE(0b, 1b)
1794 : "+r" (cc)
45c9b47c
TK		 1795 : "r" (fcn_code), "r" (config)
1796 : "cc", "0", "2", "memory"
1797 );
1798
1799 return cc;
1800}
1801
1802static int kvm_s390_apxa_installed(void)
1803{
1804 u8 config[128];
1805 int cc;
1806
a6aacc3f 1807 if (test_facility(12)) {
45c9b47c
TK		 1808 cc = kvm_s390_query_ap_config(config);
1809
1810 if (cc)
1811 pr_err("PQAP(QCI) failed with cc=%d", cc);
1812 else
1813 return config[0] & 0x40;
1814 }
1815
1816 return 0;
1817}
1818
1819static void kvm_s390_set_crycb_format(struct kvm *kvm)
1820{
1821 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1822
1823 if (kvm_s390_apxa_installed())
1824 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1825 else
1826 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1827}
1828
9bb0ec09 1829static u64 kvm_s390_get_initial_cpuid(void)
9d8d5786 1830{
9bb0ec09
DH		 1831 struct cpuid cpuid;
1832
1833 get_cpu_id(&cpuid);
1834 cpuid.version = 0xff;
1835 return *((u64 *) &cpuid);
9d8d5786
MM		 1836}
1837
c54f0d6a 1838static void kvm_s390_crypto_init(struct kvm *kvm)
5102ee87 1839{
9d8d5786 1840 if (!test_kvm_facility(kvm, 76))
c54f0d6a 1841 return;
5102ee87 1842
c54f0d6a 1843 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
45c9b47c 1844 kvm_s390_set_crycb_format(kvm);
5102ee87 1845
ed6f76b4
TK		 1846 /* Enable AES/DEA protected key functions by default */
1847 kvm->arch.crypto.aes_kw = 1;
1848 kvm->arch.crypto.dea_kw = 1;
1849 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1850 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1851 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1852 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
5102ee87
TK		 1853}
1854
7d43bafc
ED		 1855static void sca_dispose(struct kvm *kvm)
1856{
1857 if (kvm->arch.use_esca)
5e044315 1858 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
7d43bafc
ED		 1859 else
1860 free_page((unsigned long)(kvm->arch.sca));
1861 kvm->arch.sca = NULL;
1862}
1863
e08b9637 1864int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 1865{
76a6dd72 1866 gfp_t alloc_flags = GFP_KERNEL;
9d8d5786 1867 int i, rc;
b0c632db 1868 char debug_name[16];
f6c137ff 1869 static unsigned long sca_offset;
b0c632db 1870
e08b9637
CO		 1871 rc = -EINVAL;
1872#ifdef CONFIG_KVM_S390_UCONTROL
1873 if (type & ~KVM_VM_S390_UCONTROL)
1874 goto out_err;
1875 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1876 goto out_err;
1877#else
1878 if (type)
1879 goto out_err;
1880#endif
1881
b0c632db
HC		 1882 rc = s390_enable_sie();
1883 if (rc)
d89f5eff 1884 goto out_err;
b0c632db 1885
b290411a
CO		 1886 rc = -ENOMEM;
1887
7d43bafc 1888 kvm->arch.use_esca = 0; /* start with basic SCA */
76a6dd72
DH		 1889 if (!sclp.has_64bscao)
1890 alloc_flags |= GFP_DMA;
5e044315 1891 rwlock_init(&kvm->arch.sca_lock);
76a6dd72 1892 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
b0c632db 1893 if (!kvm->arch.sca)
d89f5eff 1894 goto out_err;
f6c137ff 1895 spin_lock(&kvm_lock);
c5c2c393 1896 sca_offset += 16;
bc784cce 1897 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
c5c2c393 1898 sca_offset = 0;
bc784cce
ED		 1899 kvm->arch.sca = (struct bsca_block *)
1900 ((char *) kvm->arch.sca + sca_offset);
f6c137ff 1901 spin_unlock(&kvm_lock);
b0c632db
HC		 1902
1903 sprintf(debug_name, "kvm-%u", current->pid);
1904
1cb9cf72 1905 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
b0c632db 1906 if (!kvm->arch.dbf)
40f5b735 1907 goto out_err;
b0c632db 1908
c54f0d6a
DH		 1909 kvm->arch.sie_page2 =
1910 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1911 if (!kvm->arch.sie_page2)
40f5b735 1912 goto out_err;
9d8d5786 1913
fb5bf93f 1914 /* Populate the facility mask initially. */
c54f0d6a 1915 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
04478197 1916 sizeof(S390_lowcore.stfle_fac_list));
9d8d5786
MM		 1917 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1918 if (i < kvm_s390_fac_list_mask_size())
c54f0d6a 1919 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
9d8d5786 1920 else
c54f0d6a 1921 kvm->arch.model.fac_mask[i] = 0UL;
9d8d5786
MM		 1922 }
1923
981467c9 1924 /* Populate the facility list initially. */
c54f0d6a
DH		 1925 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1926 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
981467c9
MM		 1927 S390_ARCH_FAC_LIST_SIZE_BYTE);
1928
1935222d
DH		 1929 /* we are always in czam mode - even on pre z14 machines */
1930 set_kvm_facility(kvm->arch.model.fac_mask, 138);
1931 set_kvm_facility(kvm->arch.model.fac_list, 138);
1932 /* we emulate STHYI in kvm */
95ca2cb5
JF		 1933 set_kvm_facility(kvm->arch.model.fac_mask, 74);
1934 set_kvm_facility(kvm->arch.model.fac_list, 74);
1bab1c02
CI		 1935 if (MACHINE_HAS_TLB_GUEST) {
1936 set_kvm_facility(kvm->arch.model.fac_mask, 147);
1937 set_kvm_facility(kvm->arch.model.fac_list, 147);
1938 }
95ca2cb5 1939
9bb0ec09 1940 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
37c5f6c8 1941 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
9d8d5786 1942
c54f0d6a 1943 kvm_s390_crypto_init(kvm);
5102ee87 1944
51978393
FL		 1945 mutex_init(&kvm->arch.float_int.ais_lock);
1946 kvm->arch.float_int.simm = 0;
1947 kvm->arch.float_int.nimm = 0;
ba5c1e9b 1948 spin_lock_init(&kvm->arch.float_int.lock);
6d3da241
JF		 1949 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1950 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
8a242234 1951 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 1952 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 1953
b0c632db 1954 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
78f26131 1955 VM_EVENT(kvm, 3, "vm created with type %lu", type);
b0c632db 1956
e08b9637
CO		 1957 if (type & KVM_VM_S390_UCONTROL) {
1958 kvm->arch.gmap = NULL;
a3a92c31 1959 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
e08b9637 1960 } else {
32e6b236 1961 if (sclp.hamax == U64_MAX)
ee71d16d 1962 kvm->arch.mem_limit = TASK_SIZE_MAX;
32e6b236 1963 else
ee71d16d 1964 kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
32e6b236 1965 sclp.hamax + 1);
6ea427bb 1966 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
e08b9637 1967 if (!kvm->arch.gmap)
40f5b735 1968 goto out_err;
2c70fe44 1969 kvm->arch.gmap->private = kvm;
24eb3a82 1970 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 1971 }
fa6b7fe9
CH		 1972
1973 kvm->arch.css_support = 0;
84223598 1974 kvm->arch.use_irqchip = 0;
72f25020 1975 kvm->arch.epoch = 0;
fa6b7fe9 1976
8ad35755 1977 spin_lock_init(&kvm->arch.start_stop_lock);
a3508fbe 1978 kvm_s390_vsie_init(kvm);
8335713a 1979 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
8ad35755 1980
d89f5eff 1981 return 0;
40f5b735 1982out_err:
c54f0d6a 1983 free_page((unsigned long)kvm->arch.sie_page2);
598841ca 1984 debug_unregister(kvm->arch.dbf);
7d43bafc 1985 sca_dispose(kvm);
78f26131 1986 KVM_EVENT(3, "creation of vm failed: %d", rc);
d89f5eff 1987 return rc;
b0c632db
HC		 1988}
1989
235539b4
LC		 1990bool kvm_arch_has_vcpu_debugfs(void)
1991{
1992 return false;
1993}
1994
1995int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
1996{
1997 return 0;
1998}
1999
d329c035
CB		 2000void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
2001{
2002 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 2003 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 2004 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 2005 kvm_clear_async_pf_completion_queue(vcpu);
bc784cce 2006 if (!kvm_is_ucontrol(vcpu->kvm))
a6e2f683 2007 sca_del_vcpu(vcpu);
27e0393f
CO		 2008
2009 if (kvm_is_ucontrol(vcpu->kvm))
6ea427bb 2010 gmap_remove(vcpu->arch.gmap);
27e0393f 2011
e6db1d61 2012 if (vcpu->kvm->arch.use_cmma)
b31605c1 2013 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 2014 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 2015
6692cef3 2016 kvm_vcpu_uninit(vcpu);
b110feaf 2017 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 2018}
2019
2020static void kvm_free_vcpus(struct kvm *kvm)
2021{
2022 unsigned int i;
988a2cae 2023 struct kvm_vcpu *vcpu;
d329c035 2024
988a2cae
GN		 2025 kvm_for_each_vcpu(i, vcpu, kvm)
2026 kvm_arch_vcpu_destroy(vcpu);
2027
2028 mutex_lock(&kvm->lock);
2029 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
2030 kvm->vcpus[i] = NULL;
2031
2032 atomic_set(&kvm->online_vcpus, 0);
2033 mutex_unlock(&kvm->lock);
d329c035
CB		 2034}
2035
b0c632db
HC		 2036void kvm_arch_destroy_vm(struct kvm *kvm)
2037{
d329c035 2038 kvm_free_vcpus(kvm);
7d43bafc 2039 sca_dispose(kvm);
d329c035 2040 debug_unregister(kvm->arch.dbf);
c54f0d6a 2041 free_page((unsigned long)kvm->arch.sie_page2);
27e0393f 2042 if (!kvm_is_ucontrol(kvm))
6ea427bb 2043 gmap_remove(kvm->arch.gmap);
841b91c5 2044 kvm_s390_destroy_adapters(kvm);
67335e63 2045 kvm_s390_clear_float_irqs(kvm);
a3508fbe 2046 kvm_s390_vsie_destroy(kvm);
190df4a2
CI		 2047 if (kvm->arch.migration_state) {
2048 vfree(kvm->arch.migration_state->pgste_bitmap);
2049 kfree(kvm->arch.migration_state);
2050 }
8335713a 2051 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
b0c632db
HC		 2052}
2053
2054/* Section: vcpu related */
dafd032a
DD		 2055static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
2056{
6ea427bb 2057 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
dafd032a
DD		 2058 if (!vcpu->arch.gmap)
2059 return -ENOMEM;
2060 vcpu->arch.gmap->private = vcpu->kvm;
2061
2062 return 0;
2063}
2064
a6e2f683
ED		 2065static void sca_del_vcpu(struct kvm_vcpu *vcpu)
2066{
a6940674
DH		 2067 if (!kvm_s390_use_sca_entries())
2068 return;
5e044315 2069 read_lock(&vcpu->kvm->arch.sca_lock);
7d43bafc
ED		 2070 if (vcpu->kvm->arch.use_esca) {
2071 struct esca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 2072
7d43bafc 2073 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
10ce32d5 2074 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc
ED		 2075 } else {
2076 struct bsca_block *sca = vcpu->kvm->arch.sca;
2077
2078 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
10ce32d5 2079 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc 2080 }
5e044315 2081 read_unlock(&vcpu->kvm->arch.sca_lock);
a6e2f683
ED		 2082}
2083
eaa78f34 2084static void sca_add_vcpu(struct kvm_vcpu *vcpu)
a6e2f683 2085{
a6940674
DH		 2086 if (!kvm_s390_use_sca_entries()) {
2087 struct bsca_block *sca = vcpu->kvm->arch.sca;
2088
2089 /* we still need the basic sca for the ipte control */
2090 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2091 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2092 }
eaa78f34
DH		 2093 read_lock(&vcpu->kvm->arch.sca_lock);
2094 if (vcpu->kvm->arch.use_esca) {
2095 struct esca_block *sca = vcpu->kvm->arch.sca;
7d43bafc 2096
eaa78f34 2097 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 2098 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2099 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
0c9d8683 2100 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
eaa78f34 2101 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
7d43bafc 2102 } else {
eaa78f34 2103 struct bsca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 2104
eaa78f34 2105 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 2106 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2107 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
eaa78f34 2108 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
7d43bafc 2109 }
eaa78f34 2110 read_unlock(&vcpu->kvm->arch.sca_lock);
5e044315
ED		 2111}
2112
2113/* Basic SCA to Extended SCA data copy routines */
2114static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
2115{
2116 d->sda = s->sda;
2117 d->sigp_ctrl.c = s->sigp_ctrl.c;
2118 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
2119}
2120
2121static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
2122{
2123 int i;
2124
2125 d->ipte_control = s->ipte_control;
2126 d->mcn[0] = s->mcn;
2127 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
2128 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
2129}
2130
2131static int sca_switch_to_extended(struct kvm *kvm)
2132{
2133 struct bsca_block *old_sca = kvm->arch.sca;
2134 struct esca_block *new_sca;
2135 struct kvm_vcpu *vcpu;
2136 unsigned int vcpu_idx;
2137 u32 scaol, scaoh;
2138
2139 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
2140 if (!new_sca)
2141 return -ENOMEM;
2142
2143 scaoh = (u32)((u64)(new_sca) >> 32);
2144 scaol = (u32)(u64)(new_sca) & ~0x3fU;
2145
2146 kvm_s390_vcpu_block_all(kvm);
2147 write_lock(&kvm->arch.sca_lock);
2148
2149 sca_copy_b_to_e(new_sca, old_sca);
2150
2151 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
2152 vcpu->arch.sie_block->scaoh = scaoh;
2153 vcpu->arch.sie_block->scaol = scaol;
0c9d8683 2154 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
5e044315
ED		 2155 }
2156 kvm->arch.sca = new_sca;
2157 kvm->arch.use_esca = 1;
2158
2159 write_unlock(&kvm->arch.sca_lock);
2160 kvm_s390_vcpu_unblock_all(kvm);
2161
2162 free_page((unsigned long)old_sca);
2163
8335713a
CB		 2164 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
2165 old_sca, kvm->arch.sca);
5e044315 2166 return 0;
a6e2f683
ED		 2167}
2168
2169static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
2170{
5e044315
ED		 2171 int rc;
2172
a6940674
DH		 2173 if (!kvm_s390_use_sca_entries()) {
2174 if (id < KVM_MAX_VCPUS)
2175 return true;
2176 return false;
2177 }
5e044315
ED		 2178 if (id < KVM_S390_BSCA_CPU_SLOTS)
2179 return true;
76a6dd72 2180 if (!sclp.has_esca || !sclp.has_64bscao)
5e044315
ED		 2181 return false;
2182
2183 mutex_lock(&kvm->lock);
2184 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
2185 mutex_unlock(&kvm->lock);
2186
2187 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
a6e2f683
ED		 2188}
2189
b0c632db
HC		 2190int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
2191{
3c038e6b
DD		 2192 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2193 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 2194 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
2195 KVM_SYNC_GPRS |
9eed0735 2196 KVM_SYNC_ACRS |
b028ee3e
DH		 2197 KVM_SYNC_CRS |
2198 KVM_SYNC_ARCH0 |
2199 KVM_SYNC_PFAULT;
75a4615c 2200 kvm_s390_set_prefix(vcpu, 0);
c6e5f166
FZ		 2201 if (test_kvm_facility(vcpu->kvm, 64))
2202 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
4e0b1ab7
FZ		 2203 if (test_kvm_facility(vcpu->kvm, 133))
2204 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
f6aa6dc4
DH		 2205 /* fprs can be synchronized via vrs, even if the guest has no vx. With
2206 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
2207 */
2208 if (MACHINE_HAS_VX)
68c55750 2209 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
6fd8e67d
DH		 2210 else
2211 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
dafd032a
DD		 2212
2213 if (kvm_is_ucontrol(vcpu->kvm))
2214 return __kvm_ucontrol_vcpu_init(vcpu);
2215
b0c632db
HC		 2216 return 0;
2217}
2218
db0758b2
DH		 2219/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2220static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2221{
2222 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
9c23a131 2223 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2 2224 vcpu->arch.cputm_start = get_tod_clock_fast();
9c23a131 2225 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 2226}
2227
2228/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2229static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2230{
2231 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
9c23a131 2232 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 2233 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2234 vcpu->arch.cputm_start = 0;
9c23a131 2235 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 2236}
2237
2238/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2239static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2240{
2241 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
2242 vcpu->arch.cputm_enabled = true;
2243 __start_cpu_timer_accounting(vcpu);
2244}
2245
2246/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2247static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2248{
2249 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
2250 __stop_cpu_timer_accounting(vcpu);
2251 vcpu->arch.cputm_enabled = false;
2252}
2253
2254static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2255{
2256 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2257 __enable_cpu_timer_accounting(vcpu);
2258 preempt_enable();
2259}
2260
2261static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2262{
2263 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2264 __disable_cpu_timer_accounting(vcpu);
2265 preempt_enable();
2266}
2267
4287f247
DH		 2268/* set the cpu timer - may only be called from the VCPU thread itself */
2269void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
2270{
db0758b2 2271 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
9c23a131 2272 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 2273 if (vcpu->arch.cputm_enabled)
2274 vcpu->arch.cputm_start = get_tod_clock_fast();
4287f247 2275 vcpu->arch.sie_block->cputm = cputm;
9c23a131 2276 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2 2277 preempt_enable();
4287f247
DH		 2278}
2279
db0758b2 2280/* update and get the cpu timer - can also be called from other VCPU threads */
4287f247
DH		 2281__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
2282{
9c23a131 2283 unsigned int seq;
db0758b2 2284 __u64 value;
db0758b2
DH		 2285
2286 if (unlikely(!vcpu->arch.cputm_enabled))
2287 return vcpu->arch.sie_block->cputm;
2288
9c23a131
DH		 2289 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2290 do {
2291 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
2292 /*
2293 * If the writer would ever execute a read in the critical
2294 * section, e.g. in irq context, we have a deadlock.
2295 */
2296 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
2297 value = vcpu->arch.sie_block->cputm;
2298 /* if cputm_start is 0, accounting is being started/stopped */
2299 if (likely(vcpu->arch.cputm_start))
2300 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2301 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
2302 preempt_enable();
db0758b2 2303 return value;
4287f247
DH		 2304}
2305
b0c632db
HC		 2306void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2307{
9977e886 2308
37d9df98 2309 gmap_enable(vcpu->arch.enabled_gmap);
805de8f4 2310 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
5ebda316 2311 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 2312 __start_cpu_timer_accounting(vcpu);
01a745ac 2313 vcpu->cpu = cpu;
b0c632db
HC		 2314}
2315
2316void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
2317{
01a745ac 2318 vcpu->cpu = -1;
5ebda316 2319 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 2320 __stop_cpu_timer_accounting(vcpu);
805de8f4 2321 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
37d9df98
DH		 2322 vcpu->arch.enabled_gmap = gmap_get_enabled();
2323 gmap_disable(vcpu->arch.enabled_gmap);
9977e886 2324
b0c632db
HC		 2325}
2326
2327static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
2328{
2329 /* this equals initial cpu reset in pop, but we don't switch to ESA */
2330 vcpu->arch.sie_block->gpsw.mask = 0UL;
2331 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 2332 kvm_s390_set_prefix(vcpu, 0);
4287f247 2333 kvm_s390_set_cpu_timer(vcpu, 0);
b0c632db
HC		 2334 vcpu->arch.sie_block->ckc = 0UL;
2335 vcpu->arch.sie_block->todpr = 0;
2336 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
2337 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
2338 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
9abc2a08
DH		 2339 /* make sure the new fpc will be lazily loaded */
2340 save_fpu_regs();
2341 current->thread.fpu.fpc = 0;
b0c632db 2342 vcpu->arch.sie_block->gbea = 1;
672550fb 2343 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 2344 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2345 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 2346 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
2347 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 2348 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 2349}
2350
31928aa5 2351void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 2352{
72f25020 2353 mutex_lock(&vcpu->kvm->lock);
fdf03650 2354 preempt_disable();
72f25020 2355 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
fdf03650 2356 preempt_enable();
72f25020 2357 mutex_unlock(&vcpu->kvm->lock);
25508824 2358 if (!kvm_is_ucontrol(vcpu->kvm)) {
dafd032a 2359 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
eaa78f34 2360 sca_add_vcpu(vcpu);
25508824 2361 }
6502a34c
DH		 2362 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
2363 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
37d9df98
DH		 2364 /* make vcpu_load load the right gmap on the first trigger */
2365 vcpu->arch.enabled_gmap = vcpu->arch.gmap;
42897d86
MT		 2366}
2367
5102ee87
TK		 2368static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
2369{
9d8d5786 2370 if (!test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 2371 return;
2372
a374e892
TK		 2373 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
2374
2375 if (vcpu->kvm->arch.crypto.aes_kw)
2376 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
2377 if (vcpu->kvm->arch.crypto.dea_kw)
2378 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
2379
5102ee87
TK		 2380 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
2381}
2382
b31605c1
DD		 2383void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
2384{
2385 free_page(vcpu->arch.sie_block->cbrlo);
2386 vcpu->arch.sie_block->cbrlo = 0;
2387}
2388
2389int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
2390{
2391 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
2392 if (!vcpu->arch.sie_block->cbrlo)
2393 return -ENOMEM;
2394
0c9d8683 2395 vcpu->arch.sie_block->ecb2 &= ~ECB2_PFMFI;
b31605c1
DD		 2396 return 0;
2397}
2398
91520f1a
MM		 2399static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
2400{
2401 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
2402
91520f1a 2403 vcpu->arch.sie_block->ibc = model->ibc;
80bc79dc 2404 if (test_kvm_facility(vcpu->kvm, 7))
c54f0d6a 2405 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
91520f1a
MM		 2406}
2407
b0c632db
HC		 2408int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
2409{
b31605c1 2410 int rc = 0;
b31288fa 2411
9e6dabef
CH		 2412 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
2413 CPUSTAT_SM |
a4a4f191
GH		 2414 CPUSTAT_STOPPED);
2415
53df84f8 2416 if (test_kvm_facility(vcpu->kvm, 78))
805de8f4 2417 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
53df84f8 2418 else if (test_kvm_facility(vcpu->kvm, 8))
805de8f4 2419 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
a4a4f191 2420
91520f1a
MM		 2421 kvm_s390_vcpu_setup_model(vcpu);
2422
bdab09f3
DH		 2423 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
2424 if (MACHINE_HAS_ESOP)
0c9d8683 2425 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
bd50e8ec 2426 if (test_kvm_facility(vcpu->kvm, 9))
0c9d8683 2427 vcpu->arch.sie_block->ecb |= ECB_SRSI;
f597d24e 2428 if (test_kvm_facility(vcpu->kvm, 73))
0c9d8683 2429 vcpu->arch.sie_block->ecb |= ECB_TE;
7feb6bb8 2430
873b425e 2431 if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
0c9d8683 2432 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
cd1836f5 2433 if (test_kvm_facility(vcpu->kvm, 130))
0c9d8683
DH		 2434 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
2435 vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
48ee7d3a 2436 if (sclp.has_cei)
0c9d8683 2437 vcpu->arch.sie_block->eca |= ECA_CEI;
11ad65b7 2438 if (sclp.has_ib)
0c9d8683 2439 vcpu->arch.sie_block->eca |= ECA_IB;
37c5f6c8 2440 if (sclp.has_siif)
0c9d8683 2441 vcpu->arch.sie_block->eca |= ECA_SII;
37c5f6c8 2442 if (sclp.has_sigpif)
0c9d8683 2443 vcpu->arch.sie_block->eca |= ECA_SIGPI;
18280d8b 2444 if (test_kvm_facility(vcpu->kvm, 129)) {
0c9d8683
DH		 2445 vcpu->arch.sie_block->eca |= ECA_VX;
2446 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
13211ea7 2447 }
8fa1696e
CW		 2448 if (test_kvm_facility(vcpu->kvm, 139))
2449 vcpu->arch.sie_block->ecd |= ECD_MEF;
2450
4e0b1ab7
FZ		 2451 vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
2452 | SDNXC;
c6e5f166 2453 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
730cd632
FA		 2454
2455 if (sclp.has_kss)
2456 atomic_or(CPUSTAT_KSS, &vcpu->arch.sie_block->cpuflags);
2457 else
2458 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
5a5e6536 2459
e6db1d61 2460 if (vcpu->kvm->arch.use_cmma) {
b31605c1
DD		 2461 rc = kvm_s390_vcpu_setup_cmma(vcpu);
2462 if (rc)
2463 return rc;
b31288fa 2464 }
0ac96caf 2465 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 2466 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 2467
5102ee87
TK		 2468 kvm_s390_vcpu_crypto_setup(vcpu);
2469
b31605c1 2470 return rc;
b0c632db
HC		 2471}
2472
2473struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
2474 unsigned int id)
2475{
4d47555a 2476 struct kvm_vcpu *vcpu;
7feb6bb8 2477 struct sie_page *sie_page;
4d47555a
CO		 2478 int rc = -EINVAL;
2479
4215825e 2480 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
4d47555a
CO		 2481 goto out;
2482
2483 rc = -ENOMEM;
b0c632db 2484
b110feaf 2485 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 2486 if (!vcpu)
4d47555a 2487 goto out;
b0c632db 2488
da72ca4d 2489 BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
7feb6bb8
MM		 2490 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
2491 if (!sie_page)
b0c632db
HC		 2492 goto out_free_cpu;
2493
7feb6bb8
MM		 2494 vcpu->arch.sie_block = &sie_page->sie_block;
2495 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
2496
efed1104
DH		 2497 /* the real guest size will always be smaller than msl */
2498 vcpu->arch.sie_block->mso = 0;
2499 vcpu->arch.sie_block->msl = sclp.hamax;
2500
b0c632db 2501 vcpu->arch.sie_block->icpua = id;
ba5c1e9b 2502 spin_lock_init(&vcpu->arch.local_int.lock);
ba5c1e9b 2503 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 2504 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 2505 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
9c23a131 2506 seqcount_init(&vcpu->arch.cputm_seqcount);
ba5c1e9b 2507
b0c632db
HC		 2508 rc = kvm_vcpu_init(vcpu, kvm, id);
2509 if (rc)
9abc2a08 2510 goto out_free_sie_block;
8335713a 2511 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
b0c632db 2512 vcpu->arch.sie_block);
ade38c31 2513 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 2514
b0c632db 2515 return vcpu;
7b06bf2f
WY		 2516out_free_sie_block:
2517 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 2518out_free_cpu:
b110feaf 2519 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 2520out:
b0c632db
HC		 2521 return ERR_PTR(rc);
2522}
2523
b0c632db
HC		 2524int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
2525{
9a022067 2526 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 2527}
2528
199b5763
LM		 2529bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
2530{
0546c63d 2531 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
199b5763
LM		 2532}
2533
27406cd5 2534void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
49b99e1e 2535{
805de8f4 2536 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
61a6df54 2537 exit_sie(vcpu);
49b99e1e
CB		 2538}
2539
27406cd5 2540void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
49b99e1e 2541{
805de8f4 2542 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
49b99e1e
CB		 2543}
2544
8e236546
CB		 2545static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
2546{
805de8f4 2547 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
61a6df54 2548 exit_sie(vcpu);
8e236546
CB		 2549}
2550
2551static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2552{
9bf9fde2 2553 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
8e236546
CB		 2554}
2555
49b99e1e
CB		 2556/*
2557 * Kick a guest cpu out of SIE and wait until SIE is not running.
2558 * If the CPU is not running (e.g. waiting as idle) the function will
2559 * return immediately. */
2560void exit_sie(struct kvm_vcpu *vcpu)
2561{
805de8f4 2562 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
49b99e1e
CB		 2563 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2564 cpu_relax();
2565}
2566
8e236546
CB		 2567/* Kick a guest cpu out of SIE to process a request synchronously */
2568void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
49b99e1e 2569{
8e236546
CB		 2570 kvm_make_request(req, vcpu);
2571 kvm_s390_vcpu_request(vcpu);
49b99e1e
CB		 2572}
2573
414d3b07
MS		 2574static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
2575 unsigned long end)
2c70fe44 2576{
2c70fe44
CB		 2577 struct kvm *kvm = gmap->private;
2578 struct kvm_vcpu *vcpu;
414d3b07
MS		 2579 unsigned long prefix;
2580 int i;
2c70fe44 2581
65d0b0d4
DH		 2582 if (gmap_is_shadow(gmap))
2583 return;
414d3b07
MS		 2584 if (start >= 1UL << 31)
2585 /* We are only interested in prefix pages */
2586 return;
2c70fe44
CB		 2587 kvm_for_each_vcpu(i, vcpu, kvm) {
2588 /* match against both prefix pages */
414d3b07
MS		 2589 prefix = kvm_s390_get_prefix(vcpu);
2590 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
2591 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
2592 start, end);
8e236546 2593 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44
CB		 2594 }
2595 }
2596}
2597
b6d33834
CD		 2598int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2599{
2600 /* kvm common code refers to this, but never calls it */
2601 BUG();
2602 return 0;
2603}
2604
14eebd91
CO		 2605static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2606 struct kvm_one_reg *reg)
2607{
2608 int r = -EINVAL;
2609
2610 switch (reg->id) {
29b7c71b
CO		 2611 case KVM_REG_S390_TODPR:
2612 r = put_user(vcpu->arch.sie_block->todpr,
2613 (u32 __user *)reg->addr);
2614 break;
2615 case KVM_REG_S390_EPOCHDIFF:
2616 r = put_user(vcpu->arch.sie_block->epoch,
2617 (u64 __user *)reg->addr);
2618 break;
46a6dd1c 2619 case KVM_REG_S390_CPU_TIMER:
4287f247 2620 r = put_user(kvm_s390_get_cpu_timer(vcpu),
46a6dd1c
J		 2621 (u64 __user *)reg->addr);
2622 break;
2623 case KVM_REG_S390_CLOCK_COMP:
2624 r = put_user(vcpu->arch.sie_block->ckc,
2625 (u64 __user *)reg->addr);
2626 break;
536336c2
DD		 2627 case KVM_REG_S390_PFTOKEN:
2628 r = put_user(vcpu->arch.pfault_token,
2629 (u64 __user *)reg->addr);
2630 break;
2631 case KVM_REG_S390_PFCOMPARE:
2632 r = put_user(vcpu->arch.pfault_compare,
2633 (u64 __user *)reg->addr);
2634 break;
2635 case KVM_REG_S390_PFSELECT:
2636 r = put_user(vcpu->arch.pfault_select,
2637 (u64 __user *)reg->addr);
2638 break;
672550fb
CB		 2639 case KVM_REG_S390_PP:
2640 r = put_user(vcpu->arch.sie_block->pp,
2641 (u64 __user *)reg->addr);
2642 break;
afa45ff5
CB		 2643 case KVM_REG_S390_GBEA:
2644 r = put_user(vcpu->arch.sie_block->gbea,
2645 (u64 __user *)reg->addr);
2646 break;
14eebd91
CO		 2647 default:
2648 break;
2649 }
2650
2651 return r;
2652}
2653
2654static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2655 struct kvm_one_reg *reg)
2656{
2657 int r = -EINVAL;
4287f247 2658 __u64 val;
14eebd91
CO		 2659
2660 switch (reg->id) {
29b7c71b
CO		 2661 case KVM_REG_S390_TODPR:
2662 r = get_user(vcpu->arch.sie_block->todpr,
2663 (u32 __user *)reg->addr);
2664 break;
2665 case KVM_REG_S390_EPOCHDIFF:
2666 r = get_user(vcpu->arch.sie_block->epoch,
2667 (u64 __user *)reg->addr);
2668 break;
46a6dd1c 2669 case KVM_REG_S390_CPU_TIMER:
4287f247
DH		 2670 r = get_user(val, (u64 __user *)reg->addr);
2671 if (!r)
2672 kvm_s390_set_cpu_timer(vcpu, val);
46a6dd1c
J		 2673 break;
2674 case KVM_REG_S390_CLOCK_COMP:
2675 r = get_user(vcpu->arch.sie_block->ckc,
2676 (u64 __user *)reg->addr);
2677 break;
536336c2
DD		 2678 case KVM_REG_S390_PFTOKEN:
2679 r = get_user(vcpu->arch.pfault_token,
2680 (u64 __user *)reg->addr);
9fbd8082
DH		 2681 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2682 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 2683 break;
2684 case KVM_REG_S390_PFCOMPARE:
2685 r = get_user(vcpu->arch.pfault_compare,
2686 (u64 __user *)reg->addr);
2687 break;
2688 case KVM_REG_S390_PFSELECT:
2689 r = get_user(vcpu->arch.pfault_select,
2690 (u64 __user *)reg->addr);
2691 break;
672550fb
CB		 2692 case KVM_REG_S390_PP:
2693 r = get_user(vcpu->arch.sie_block->pp,
2694 (u64 __user *)reg->addr);
2695 break;
afa45ff5
CB		 2696 case KVM_REG_S390_GBEA:
2697 r = get_user(vcpu->arch.sie_block->gbea,
2698 (u64 __user *)reg->addr);
2699 break;
14eebd91
CO		 2700 default:
2701 break;
2702 }
2703
2704 return r;
2705}
b6d33834 2706
b0c632db
HC		 2707static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2708{
b0c632db 2709 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 2710 return 0;
2711}
2712
2713int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2714{
5a32c1af 2715 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 2716 return 0;
2717}
2718
2719int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2720{
5a32c1af 2721 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 2722 return 0;
2723}
2724
2725int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2726 struct kvm_sregs *sregs)
2727{
59674c1a 2728 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 2729 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
b0c632db
HC		 2730 return 0;
2731}
2732
2733int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2734 struct kvm_sregs *sregs)
2735{
59674c1a 2736 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 2737 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 2738 return 0;
2739}
2740
2741int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2742{
4725c860
MS		 2743 if (test_fp_ctl(fpu->fpc))
2744 return -EINVAL;
e1788bb9 2745 vcpu->run->s.regs.fpc = fpu->fpc;
9abc2a08 2746 if (MACHINE_HAS_VX)
a7d4b8f2
DH		 2747 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
2748 (freg_t *) fpu->fprs);
9abc2a08 2749 else
a7d4b8f2 2750 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
b0c632db
HC		 2751 return 0;
2752}
2753
2754int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2755{
9abc2a08
DH		 2756 /* make sure we have the latest values */
2757 save_fpu_regs();
2758 if (MACHINE_HAS_VX)
a7d4b8f2
DH		 2759 convert_vx_to_fp((freg_t *) fpu->fprs,
2760 (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08 2761 else
a7d4b8f2 2762 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
e1788bb9 2763 fpu->fpc = vcpu->run->s.regs.fpc;
b0c632db
HC		 2764 return 0;
2765}
2766
2767static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2768{
2769 int rc = 0;
2770
7a42fdc2 2771 if (!is_vcpu_stopped(vcpu))
b0c632db 2772 rc = -EBUSY;
d7b0b5eb
CO		 2773 else {
2774 vcpu->run->psw_mask = psw.mask;
2775 vcpu->run->psw_addr = psw.addr;
2776 }
b0c632db
HC		 2777 return rc;
2778}
2779
2780int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2781 struct kvm_translation *tr)
2782{
2783 return -EINVAL; /* not implemented yet */
2784}
2785
27291e21
DH		 2786#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2787 KVM_GUESTDBG_USE_HW_BP | \
2788 KVM_GUESTDBG_ENABLE)
2789
d0bfb940
JK		 2790int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2791 struct kvm_guest_debug *dbg)
b0c632db 2792{
27291e21
DH		 2793 int rc = 0;
2794
2795 vcpu->guest_debug = 0;
2796 kvm_s390_clear_bp_data(vcpu);
2797
2de3bfc2 2798 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21 2799 return -EINVAL;
89b5b4de
DH		 2800 if (!sclp.has_gpere)
2801 return -EINVAL;
27291e21
DH		 2802
2803 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2804 vcpu->guest_debug = dbg->control;
2805 /* enforce guest PER */
805de8f4 2806 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2807
2808 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2809 rc = kvm_s390_import_bp_data(vcpu, dbg);
2810 } else {
805de8f4 2811 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2812 vcpu->arch.guestdbg.last_bp = 0;
2813 }
2814
2815 if (rc) {
2816 vcpu->guest_debug = 0;
2817 kvm_s390_clear_bp_data(vcpu);
805de8f4 2818 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2819 }
2820
2821 return rc;
b0c632db
HC		 2822}
2823
62d9f0db
MT		 2824int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2825 struct kvm_mp_state *mp_state)
2826{
6352e4d2
DH		 2827 /* CHECK_STOP and LOAD are not supported yet */
2828 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2829 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 2830}
2831
2832int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2833 struct kvm_mp_state *mp_state)
2834{
6352e4d2
DH		 2835 int rc = 0;
2836
2837 /* user space knows about this interface - let it control the state */
2838 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2839
2840 switch (mp_state->mp_state) {
2841 case KVM_MP_STATE_STOPPED:
2842 kvm_s390_vcpu_stop(vcpu);
2843 break;
2844 case KVM_MP_STATE_OPERATING:
2845 kvm_s390_vcpu_start(vcpu);
2846 break;
2847 case KVM_MP_STATE_LOAD:
2848 case KVM_MP_STATE_CHECK_STOP:
2849 /* fall through - CHECK_STOP and LOAD are not supported yet */
2850 default:
2851 rc = -ENXIO;
2852 }
2853
2854 return rc;
62d9f0db
MT		 2855}
2856
8ad35755
DH		 2857static bool ibs_enabled(struct kvm_vcpu *vcpu)
2858{
2859 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2860}
2861
2c70fe44
CB		 2862static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2863{
8ad35755 2864retry:
8e236546 2865 kvm_s390_vcpu_request_handled(vcpu);
2fa6e1e1 2866 if (!kvm_request_pending(vcpu))
586b7ccd 2867 return 0;
2c70fe44
CB		 2868 /*
2869 * We use MMU_RELOAD just to re-arm the ipte notifier for the
b2d73b2a 2870 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2c70fe44
CB		 2871 * This ensures that the ipte instruction for this request has
2872 * already finished. We might race against a second unmapper that
2873 * wants to set the blocking bit. Lets just retry the request loop.
2874 */
8ad35755 2875 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44 2876 int rc;
b2d73b2a
MS		 2877 rc = gmap_mprotect_notify(vcpu->arch.gmap,
2878 kvm_s390_get_prefix(vcpu),
2879 PAGE_SIZE * 2, PROT_WRITE);
aca411a4
JN		 2880 if (rc) {
2881 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44 2882 return rc;
aca411a4 2883 }
8ad35755 2884 goto retry;
2c70fe44 2885 }
8ad35755 2886
d3d692c8
DH		 2887 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2888 vcpu->arch.sie_block->ihcpu = 0xffff;
2889 goto retry;
2890 }
2891
8ad35755
DH		 2892 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2893 if (!ibs_enabled(vcpu)) {
2894 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
805de8f4 2895 atomic_or(CPUSTAT_IBS,
8ad35755
DH		 2896 &vcpu->arch.sie_block->cpuflags);
2897 }
2898 goto retry;
2c70fe44 2899 }
8ad35755
DH		 2900
2901 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2902 if (ibs_enabled(vcpu)) {
2903 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
805de8f4 2904 atomic_andnot(CPUSTAT_IBS,
8ad35755
DH		 2905 &vcpu->arch.sie_block->cpuflags);
2906 }
2907 goto retry;
2908 }
2909
6502a34c
DH		 2910 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
2911 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2912 goto retry;
2913 }
2914
190df4a2
CI		 2915 if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
2916 /*
2917 * Disable CMMA virtualization; we will emulate the ESSA
2918 * instruction manually, in order to provide additional
2919 * functionalities needed for live migration.
2920 */
2921 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
2922 goto retry;
2923 }
2924
2925 if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
2926 /*
2927 * Re-enable CMMA virtualization if CMMA is available and
2928 * was used.
2929 */
2930 if ((vcpu->kvm->arch.use_cmma) &&
2931 (vcpu->kvm->mm->context.use_cmma))
2932 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
2933 goto retry;
2934 }
2935
0759d068 2936 /* nothing to do, just clear the request */
72875d8a 2937 kvm_clear_request(KVM_REQ_UNHALT, vcpu);
0759d068 2938
2c70fe44
CB		 2939 return 0;
2940}
2941
8fa1696e
CW		 2942void kvm_s390_set_tod_clock_ext(struct kvm *kvm,
2943 const struct kvm_s390_vm_tod_clock *gtod)
2944{
2945 struct kvm_vcpu *vcpu;
2946 struct kvm_s390_tod_clock_ext htod;
2947 int i;
2948
2949 mutex_lock(&kvm->lock);
2950 preempt_disable();
2951
2952 get_tod_clock_ext((char *)&htod);
2953
2954 kvm->arch.epoch = gtod->tod - htod.tod;
2955 kvm->arch.epdx = gtod->epoch_idx - htod.epoch_idx;
2956
2957 if (kvm->arch.epoch > gtod->tod)
2958 kvm->arch.epdx -= 1;
2959
2960 kvm_s390_vcpu_block_all(kvm);
2961 kvm_for_each_vcpu(i, vcpu, kvm) {
2962 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2963 vcpu->arch.sie_block->epdx = kvm->arch.epdx;
2964 }
2965
2966 kvm_s390_vcpu_unblock_all(kvm);
2967 preempt_enable();
2968 mutex_unlock(&kvm->lock);
2969}
2970
25ed1675
DH		 2971void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2972{
2973 struct kvm_vcpu *vcpu;
2974 int i;
2975
2976 mutex_lock(&kvm->lock);
2977 preempt_disable();
2978 kvm->arch.epoch = tod - get_tod_clock();
2979 kvm_s390_vcpu_block_all(kvm);
2980 kvm_for_each_vcpu(i, vcpu, kvm)
2981 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2982 kvm_s390_vcpu_unblock_all(kvm);
2983 preempt_enable();
2984 mutex_unlock(&kvm->lock);
2985}
2986
fa576c58
TH		 2987/**
2988 * kvm_arch_fault_in_page - fault-in guest page if necessary
2989 * @vcpu: The corresponding virtual cpu
2990 * @gpa: Guest physical address
2991 * @writable: Whether the page should be writable or not
2992 *
2993 * Make sure that a guest page has been faulted-in on the host.
2994 *
2995 * Return: Zero on success, negative error code otherwise.
2996 */
2997long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 2998{
527e30b4
MS		 2999 return gmap_fault(vcpu->arch.gmap, gpa,
3000 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 3001}
3002
3c038e6b
DD		 3003static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
3004 unsigned long token)
3005{
3006 struct kvm_s390_interrupt inti;
383d0b05 3007 struct kvm_s390_irq irq;
3c038e6b
DD		 3008
3009 if (start_token) {
383d0b05
JF		 3010 irq.u.ext.ext_params2 = token;
3011 irq.type = KVM_S390_INT_PFAULT_INIT;
3012 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 3013 } else {
3014 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 3015 inti.parm64 = token;
3c038e6b
DD		 3016 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
3017 }
3018}
3019
3020void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
3021 struct kvm_async_pf *work)
3022{
3023 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
3024 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
3025}
3026
3027void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
3028 struct kvm_async_pf *work)
3029{
3030 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
3031 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
3032}
3033
3034void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
3035 struct kvm_async_pf *work)
3036{
3037 /* s390 will always inject the page directly */
3038}
3039
3040bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
3041{
3042 /*
3043 * s390 will always inject the page directly,
3044 * but we still want check_async_completion to cleanup
3045 */
3046 return true;
3047}
3048
3049static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
3050{
3051 hva_t hva;
3052 struct kvm_arch_async_pf arch;
3053 int rc;
3054
3055 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3056 return 0;
3057 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
3058 vcpu->arch.pfault_compare)
3059 return 0;
3060 if (psw_extint_disabled(vcpu))
3061 return 0;
9a022067 3062 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 3063 return 0;
3064 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
3065 return 0;
3066 if (!vcpu->arch.gmap->pfault_enabled)
3067 return 0;
3068
81480cc1
HC		 3069 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
3070 hva += current->thread.gmap_addr & ~PAGE_MASK;
3071 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 3072 return 0;
3073
3074 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
3075 return rc;
3076}
3077
3fb4c40f 3078static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 3079{
3fb4c40f 3080 int rc, cpuflags;
e168bf8d 3081
3c038e6b
DD		 3082 /*
3083 * On s390 notifications for arriving pages will be delivered directly
3084 * to the guest but the house keeping for completed pfaults is
3085 * handled outside the worker.
3086 */
3087 kvm_check_async_pf_completion(vcpu);
3088
7ec7c8c7
CB		 3089 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
3090 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
b0c632db
HC		 3091
3092 if (need_resched())
3093 schedule();
3094
d3a73acb 3095 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 3096 s390_handle_mcck();
3097
79395031
JF		 3098 if (!kvm_is_ucontrol(vcpu->kvm)) {
3099 rc = kvm_s390_deliver_pending_interrupts(vcpu);
3100 if (rc)
3101 return rc;
3102 }
0ff31867 3103
2c70fe44
CB		 3104 rc = kvm_s390_handle_requests(vcpu);
3105 if (rc)
3106 return rc;
3107
27291e21
DH		 3108 if (guestdbg_enabled(vcpu)) {
3109 kvm_s390_backup_guest_per_regs(vcpu);
3110 kvm_s390_patch_guest_per_regs(vcpu);
3111 }
3112
b0c632db 3113 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 3114 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
3115 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
3116 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 3117
3fb4c40f
TH		 3118 return 0;
3119}
3120
492d8642
TH		 3121static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
3122{
56317920
DH		 3123 struct kvm_s390_pgm_info pgm_info = {
3124 .code = PGM_ADDRESSING,
3125 };
3126 u8 opcode, ilen;
492d8642
TH		 3127 int rc;
3128
3129 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
3130 trace_kvm_s390_sie_fault(vcpu);
3131
3132 /*
3133 * We want to inject an addressing exception, which is defined as a
3134 * suppressing or terminating exception. However, since we came here
3135 * by a DAT access exception, the PSW still points to the faulting
3136 * instruction since DAT exceptions are nullifying. So we've got
3137 * to look up the current opcode to get the length of the instruction
3138 * to be able to forward the PSW.
3139 */
3fa8cad7 3140 rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
56317920 3141 ilen = insn_length(opcode);
9b0d721a
DH		 3142 if (rc < 0) {
3143 return rc;
3144 } else if (rc) {
3145 /* Instruction-Fetching Exceptions - we can't detect the ilen.
3146 * Forward by arbitrary ilc, injection will take care of
3147 * nullification if necessary.
3148 */
3149 pgm_info = vcpu->arch.pgm;
3150 ilen = 4;
3151 }
56317920
DH		 3152 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
3153 kvm_s390_forward_psw(vcpu, ilen);
3154 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
492d8642
TH		 3155}
3156
3fb4c40f
TH		 3157static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
3158{
4d62fcc0
QH		 3159 struct mcck_volatile_info *mcck_info;
3160 struct sie_page *sie_page;
3161
2b29a9fd
DD		 3162 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
3163 vcpu->arch.sie_block->icptcode);
3164 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
3165
27291e21
DH		 3166 if (guestdbg_enabled(vcpu))
3167 kvm_s390_restore_guest_per_regs(vcpu);
3168
7ec7c8c7
CB		 3169 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
3170 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
71f116bf 3171
4d62fcc0
QH		 3172 if (exit_reason == -EINTR) {
3173 VCPU_EVENT(vcpu, 3, "%s", "machine check");
3174 sie_page = container_of(vcpu->arch.sie_block,
3175 struct sie_page, sie_block);
3176 mcck_info = &sie_page->mcck_info;
3177 kvm_s390_reinject_machine_check(vcpu, mcck_info);
3178 return 0;
3179 }
3180
71f116bf
DH		 3181 if (vcpu->arch.sie_block->icptcode > 0) {
3182 int rc = kvm_handle_sie_intercept(vcpu);
3183
3184 if (rc != -EOPNOTSUPP)
3185 return rc;
3186 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
3187 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
3188 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
3189 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
3190 return -EREMOTE;
3191 } else if (exit_reason != -EFAULT) {
3192 vcpu->stat.exit_null++;
3193 return 0;
210b1607
TH		 3194 } else if (kvm_is_ucontrol(vcpu->kvm)) {
3195 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
3196 vcpu->run->s390_ucontrol.trans_exc_code =
3197 current->thread.gmap_addr;
3198 vcpu->run->s390_ucontrol.pgm_code = 0x10;
71f116bf 3199 return -EREMOTE;
24eb3a82 3200 } else if (current->thread.gmap_pfault) {
3c038e6b 3201 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 3202 current->thread.gmap_pfault = 0;
71f116bf
DH		 3203 if (kvm_arch_setup_async_pf(vcpu))
3204 return 0;
3205 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
a76ccff6 3206 }
71f116bf 3207 return vcpu_post_run_fault_in_sie(vcpu);
3fb4c40f
TH		 3208}
3209
3210static int __vcpu_run(struct kvm_vcpu *vcpu)
3211{
3212 int rc, exit_reason;
3213
800c1065
TH		 3214 /*
3215 * We try to hold kvm->srcu during most of vcpu_run (except when run-
3216 * ning the guest), so that memslots (and other stuff) are protected
3217 */
3218 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3219
a76ccff6
TH		 3220 do {
3221 rc = vcpu_pre_run(vcpu);
3222 if (rc)
3223 break;
3fb4c40f 3224
800c1065 3225 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 3226 /*
3227 * As PF_VCPU will be used in fault handler, between
3228 * guest_enter and guest_exit should be no uaccess.
3229 */
0097d12e 3230 local_irq_disable();
6edaa530 3231 guest_enter_irqoff();
db0758b2 3232 __disable_cpu_timer_accounting(vcpu);
0097d12e 3233 local_irq_enable();
a76ccff6
TH		 3234 exit_reason = sie64a(vcpu->arch.sie_block,
3235 vcpu->run->s.regs.gprs);
0097d12e 3236 local_irq_disable();
db0758b2 3237 __enable_cpu_timer_accounting(vcpu);
6edaa530 3238 guest_exit_irqoff();
0097d12e 3239 local_irq_enable();
800c1065 3240 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 3241
3242 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 3243 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 3244
800c1065 3245 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 3246 return rc;
b0c632db
HC		 3247}
3248
b028ee3e
DH		 3249static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3250{
4d5f2c04 3251 struct runtime_instr_cb *riccb;
4e0b1ab7 3252 struct gs_cb *gscb;
4d5f2c04
CB		 3253
3254 riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
4e0b1ab7 3255 gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
b028ee3e
DH		 3256 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
3257 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
3258 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
3259 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
3260 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
3261 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 3262 /* some control register changes require a tlb flush */
3263 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 3264 }
3265 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4287f247 3266 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
b028ee3e
DH		 3267 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
3268 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
3269 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
3270 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
3271 }
3272 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
3273 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
3274 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
3275 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 3276 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3277 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e 3278 }
80cd8763
FZ		 3279 /*
3280 * If userspace sets the riccb (e.g. after migration) to a valid state,
3281 * we should enable RI here instead of doing the lazy enablement.
3282 */
3283 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
4d5f2c04 3284 test_kvm_facility(vcpu->kvm, 64) &&
bb59c2da 3285 riccb->v &&
0c9d8683 3286 !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
4d5f2c04 3287 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
0c9d8683 3288 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
80cd8763 3289 }
4e0b1ab7
FZ		 3290 /*
3291 * If userspace sets the gscb (e.g. after migration) to non-zero,
3292 * we should enable GS here instead of doing the lazy enablement.
3293 */
3294 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
3295 test_kvm_facility(vcpu->kvm, 133) &&
3296 gscb->gssm &&
3297 !vcpu->arch.gs_enabled) {
3298 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
3299 vcpu->arch.sie_block->ecb |= ECB_GS;
3300 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3301 vcpu->arch.gs_enabled = 1;
80cd8763 3302 }
31d8b8d4
CB		 3303 save_access_regs(vcpu->arch.host_acrs);
3304 restore_access_regs(vcpu->run->s.regs.acrs);
e1788bb9
CB		 3305 /* save host (userspace) fprs/vrs */
3306 save_fpu_regs();
3307 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
3308 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
3309 if (MACHINE_HAS_VX)
3310 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
3311 else
3312 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
3313 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
3314 if (test_fp_ctl(current->thread.fpu.fpc))
3315 /* User space provided an invalid FPC, let's clear it */
3316 current->thread.fpu.fpc = 0;
4e0b1ab7
FZ		 3317 if (MACHINE_HAS_GS) {
3318 preempt_disable();
3319 __ctl_set_bit(2, 4);
3320 if (current->thread.gs_cb) {
3321 vcpu->arch.host_gscb = current->thread.gs_cb;
3322 save_gs_cb(vcpu->arch.host_gscb);
3323 }
3324 if (vcpu->arch.gs_enabled) {
3325 current->thread.gs_cb = (struct gs_cb *)
3326 &vcpu->run->s.regs.gscb;
3327 restore_gs_cb(current->thread.gs_cb);
3328 }
3329 preempt_enable();
3330 }
80cd8763 3331
b028ee3e
DH		 3332 kvm_run->kvm_dirty_regs = 0;
3333}
3334
3335static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3336{
3337 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
3338 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
3339 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
3340 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4287f247 3341 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
b028ee3e
DH		 3342 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
3343 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
3344 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
3345 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
3346 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
3347 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
3348 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
31d8b8d4
CB		 3349 save_access_regs(vcpu->run->s.regs.acrs);
3350 restore_access_regs(vcpu->arch.host_acrs);
e1788bb9
CB		 3351 /* Save guest register state */
3352 save_fpu_regs();
3353 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3354 /* Restore will be done lazily at return */
3355 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
3356 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
4e0b1ab7
FZ		 3357 if (MACHINE_HAS_GS) {
3358 __ctl_set_bit(2, 4);
3359 if (vcpu->arch.gs_enabled)
3360 save_gs_cb(current->thread.gs_cb);
3361 preempt_disable();
3362 current->thread.gs_cb = vcpu->arch.host_gscb;
3363 restore_gs_cb(vcpu->arch.host_gscb);
3364 preempt_enable();
3365 if (!vcpu->arch.host_gscb)
3366 __ctl_clear_bit(2, 4);
3367 vcpu->arch.host_gscb = NULL;
3368 }
e1788bb9 3369
b028ee3e
DH		 3370}
3371
b0c632db
HC		 3372int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3373{
8f2abe6a 3374 int rc;
b0c632db 3375
460df4c1
PB		 3376 if (kvm_run->immediate_exit)
3377 return -EINTR;
3378
27291e21
DH		 3379 if (guestdbg_exit_pending(vcpu)) {
3380 kvm_s390_prepare_debug_exit(vcpu);
3381 return 0;
3382 }
3383
20b7035c 3384 kvm_sigset_activate(vcpu);
b0c632db 3385
6352e4d2
DH		 3386 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
3387 kvm_s390_vcpu_start(vcpu);
3388 } else if (is_vcpu_stopped(vcpu)) {
ea2cdd27 3389 pr_err_ratelimited("can't run stopped vcpu %d\n",
6352e4d2
DH		 3390 vcpu->vcpu_id);
3391 return -EINVAL;
3392 }
b0c632db 3393
b028ee3e 3394 sync_regs(vcpu, kvm_run);
db0758b2 3395 enable_cpu_timer_accounting(vcpu);
d7b0b5eb 3396
dab4079d 3397 might_fault();
a76ccff6 3398 rc = __vcpu_run(vcpu);
9ace903d 3399
b1d16c49
CE		 3400 if (signal_pending(current) && !rc) {
3401 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 3402 rc = -EINTR;
b1d16c49 3403 }
8f2abe6a 3404
27291e21
DH		 3405 if (guestdbg_exit_pending(vcpu) && !rc) {
3406 kvm_s390_prepare_debug_exit(vcpu);
3407 rc = 0;
3408 }
3409
8f2abe6a 3410 if (rc == -EREMOTE) {
71f116bf 3411 /* userspace support is needed, kvm_run has been prepared */
8f2abe6a
CB		 3412 rc = 0;
3413 }
b0c632db 3414
db0758b2 3415 disable_cpu_timer_accounting(vcpu);
b028ee3e 3416 store_regs(vcpu, kvm_run);
d7b0b5eb 3417
20b7035c 3418 kvm_sigset_deactivate(vcpu);
b0c632db 3419
b0c632db 3420 vcpu->stat.exit_userspace++;
7e8e6ab4 3421 return rc;
b0c632db
HC		 3422}
3423
b0c632db
HC		 3424/*
3425 * store status at address
3426 * we use have two special cases:
3427 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
3428 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
3429 */
d0bce605 3430int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 3431{
092670cd 3432 unsigned char archmode = 1;
9abc2a08 3433 freg_t fprs[NUM_FPRS];
fda902cb 3434 unsigned int px;
4287f247 3435 u64 clkcomp, cputm;
d0bce605 3436 int rc;
b0c632db 3437
d9a3a09a 3438 px = kvm_s390_get_prefix(vcpu);
d0bce605
HC		 3439 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
3440 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 3441 return -EFAULT;
d9a3a09a 3442 gpa = 0;
d0bce605
HC		 3443 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
3444 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 3445 return -EFAULT;
d9a3a09a
MS		 3446 gpa = px;
3447 } else
3448 gpa -= __LC_FPREGS_SAVE_AREA;
9abc2a08
DH		 3449
3450 /* manually convert vector registers if necessary */
3451 if (MACHINE_HAS_VX) {
9522b37f 3452 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08
DH		 3453 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
3454 fprs, 128);
3455 } else {
3456 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
6fd8e67d 3457 vcpu->run->s.regs.fprs, 128);
9abc2a08 3458 }
d9a3a09a 3459 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
d0bce605 3460 vcpu->run->s.regs.gprs, 128);
d9a3a09a 3461 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
d0bce605 3462 &vcpu->arch.sie_block->gpsw, 16);
d9a3a09a 3463 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
fda902cb 3464 &px, 4);
d9a3a09a 3465 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
9abc2a08 3466 &vcpu->run->s.regs.fpc, 4);
d9a3a09a 3467 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
d0bce605 3468 &vcpu->arch.sie_block->todpr, 4);
4287f247 3469 cputm = kvm_s390_get_cpu_timer(vcpu);
d9a3a09a 3470 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4287f247 3471 &cputm, 8);
178bd789 3472 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d9a3a09a 3473 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
d0bce605 3474 &clkcomp, 8);
d9a3a09a 3475 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
d0bce605 3476 &vcpu->run->s.regs.acrs, 64);
d9a3a09a 3477 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
d0bce605
HC		 3478 &vcpu->arch.sie_block->gcr, 128);
3479 return rc ? -EFAULT : 0;
b0c632db
HC		 3480}
3481
e879892c
TH		 3482int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
3483{
3484 /*
3485 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
31d8b8d4 3486 * switch in the run ioctl. Let's update our copies before we save
e879892c
TH		 3487 * it into the save area
3488 */
d0164ee2 3489 save_fpu_regs();
9abc2a08 3490 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
e879892c
TH		 3491 save_access_regs(vcpu->run->s.regs.acrs);
3492
3493 return kvm_s390_store_status_unloaded(vcpu, addr);
3494}
3495
8ad35755
DH		 3496static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3497{
3498 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
8e236546 3499 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
8ad35755
DH		 3500}
3501
3502static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
3503{
3504 unsigned int i;
3505 struct kvm_vcpu *vcpu;
3506
3507 kvm_for_each_vcpu(i, vcpu, kvm) {
3508 __disable_ibs_on_vcpu(vcpu);
3509 }
3510}
3511
3512static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3513{
09a400e7
DH		 3514 if (!sclp.has_ibs)
3515 return;
8ad35755 3516 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
8e236546 3517 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
8ad35755
DH		 3518}
3519
6852d7b6
DH		 3520void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
3521{
8ad35755
DH		 3522 int i, online_vcpus, started_vcpus = 0;
3523
3524 if (!is_vcpu_stopped(vcpu))
3525 return;
3526
6852d7b6 3527 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 3528 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 3529 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 3530 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3531
3532 for (i = 0; i < online_vcpus; i++) {
3533 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
3534 started_vcpus++;
3535 }
3536
3537 if (started_vcpus == 0) {
3538 /* we're the only active VCPU -> speed it up */
3539 __enable_ibs_on_vcpu(vcpu);
3540 } else if (started_vcpus == 1) {
3541 /*
3542 * As we are starting a second VCPU, we have to disable
3543 * the IBS facility on all VCPUs to remove potentially
3544 * oustanding ENABLE requests.
3545 */
3546 __disable_ibs_on_all_vcpus(vcpu->kvm);
3547 }
3548
805de8f4 3549 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 3550 /*
3551 * Another VCPU might have used IBS while we were offline.
3552 * Let's play safe and flush the VCPU at startup.
3553 */
d3d692c8 3554 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 3555 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 3556 return;
6852d7b6
DH		 3557}
3558
3559void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
3560{
8ad35755
DH		 3561 int i, online_vcpus, started_vcpus = 0;
3562 struct kvm_vcpu *started_vcpu = NULL;
3563
3564 if (is_vcpu_stopped(vcpu))
3565 return;
3566
6852d7b6 3567 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 3568 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 3569 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 3570 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3571
32f5ff63 3572 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 3573 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 3574
805de8f4 3575 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 3576 __disable_ibs_on_vcpu(vcpu);
3577
3578 for (i = 0; i < online_vcpus; i++) {
3579 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
3580 started_vcpus++;
3581 started_vcpu = vcpu->kvm->vcpus[i];
3582 }
3583 }
3584
3585 if (started_vcpus == 1) {
3586 /*
3587 * As we only have one VCPU left, we want to enable the
3588 * IBS facility for that VCPU to speed it up.
3589 */
3590 __enable_ibs_on_vcpu(started_vcpu);
3591 }
3592
433b9ee4 3593 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 3594 return;
6852d7b6
DH		 3595}
3596
d6712df9
CH		 3597static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
3598 struct kvm_enable_cap *cap)
3599{
3600 int r;
3601
3602 if (cap->flags)
3603 return -EINVAL;
3604
3605 switch (cap->cap) {
fa6b7fe9
CH		 3606 case KVM_CAP_S390_CSS_SUPPORT:
3607 if (!vcpu->kvm->arch.css_support) {
3608 vcpu->kvm->arch.css_support = 1;
c92ea7b9 3609 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
fa6b7fe9
CH		 3610 trace_kvm_s390_enable_css(vcpu->kvm);
3611 }
3612 r = 0;
3613 break;
d6712df9
CH		 3614 default:
3615 r = -EINVAL;
3616 break;
3617 }
3618 return r;
3619}
3620
41408c28
TH		 3621static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
3622 struct kvm_s390_mem_op *mop)
3623{
3624 void __user *uaddr = (void __user *)mop->buf;
3625 void *tmpbuf = NULL;
3626 int r, srcu_idx;
3627 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
3628 | KVM_S390_MEMOP_F_CHECK_ONLY;
3629
3630 if (mop->flags & ~supported_flags)
3631 return -EINVAL;
3632
3633 if (mop->size > MEM_OP_MAX_SIZE)
3634 return -E2BIG;
3635
3636 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
3637 tmpbuf = vmalloc(mop->size);
3638 if (!tmpbuf)
3639 return -ENOMEM;
3640 }
3641
3642 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3643
3644 switch (mop->op) {
3645 case KVM_S390_MEMOP_LOGICAL_READ:
3646 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 3647 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3648 mop->size, GACC_FETCH);
41408c28
TH		 3649 break;
3650 }
3651 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3652 if (r == 0) {
3653 if (copy_to_user(uaddr, tmpbuf, mop->size))
3654 r = -EFAULT;
3655 }
3656 break;
3657 case KVM_S390_MEMOP_LOGICAL_WRITE:
3658 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 3659 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3660 mop->size, GACC_STORE);
41408c28
TH		 3661 break;
3662 }
3663 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
3664 r = -EFAULT;
3665 break;
3666 }
3667 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3668 break;
3669 default:
3670 r = -EINVAL;
3671 }
3672
3673 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
3674
3675 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
3676 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
3677
3678 vfree(tmpbuf);
3679 return r;
3680}
3681
b0c632db
HC		 3682long kvm_arch_vcpu_ioctl(struct file *filp,
3683 unsigned int ioctl, unsigned long arg)
3684{
3685 struct kvm_vcpu *vcpu = filp->private_data;
3686 void __user *argp = (void __user *)arg;
800c1065 3687 int idx;
bc923cc9 3688 long r;
b0c632db 3689
93736624 3690 switch (ioctl) {
47b43c52
JF		 3691 case KVM_S390_IRQ: {
3692 struct kvm_s390_irq s390irq;
3693
3694 r = -EFAULT;
3695 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
3696 break;
3697 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3698 break;
3699 }
93736624 3700 case KVM_S390_INTERRUPT: {
ba5c1e9b 3701 struct kvm_s390_interrupt s390int;
383d0b05 3702 struct kvm_s390_irq s390irq;
ba5c1e9b 3703
93736624 3704 r = -EFAULT;
ba5c1e9b 3705 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624 3706 break;
383d0b05
JF		 3707 if (s390int_to_s390irq(&s390int, &s390irq))
3708 return -EINVAL;
3709 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
93736624 3710 break;
ba5c1e9b 3711 }
b0c632db 3712 case KVM_S390_STORE_STATUS:
800c1065 3713 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 3714 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 3715 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 3716 break;
b0c632db
HC		 3717 case KVM_S390_SET_INITIAL_PSW: {
3718 psw_t psw;
3719
bc923cc9 3720 r = -EFAULT;
b0c632db 3721 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 3722 break;
3723 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3724 break;
b0c632db
HC		 3725 }
3726 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 3727 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3728 break;
14eebd91
CO		 3729 case KVM_SET_ONE_REG:
3730 case KVM_GET_ONE_REG: {
3731 struct kvm_one_reg reg;
3732 r = -EFAULT;
3733 if (copy_from_user(&reg, argp, sizeof(reg)))
3734 break;
3735 if (ioctl == KVM_SET_ONE_REG)
3736 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3737 else
3738 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3739 break;
3740 }
27e0393f
CO		 3741#ifdef CONFIG_KVM_S390_UCONTROL
3742 case KVM_S390_UCAS_MAP: {
3743 struct kvm_s390_ucas_mapping ucasmap;
3744
3745 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3746 r = -EFAULT;
3747 break;
3748 }
3749
3750 if (!kvm_is_ucontrol(vcpu->kvm)) {
3751 r = -EINVAL;
3752 break;
3753 }
3754
3755 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3756 ucasmap.vcpu_addr, ucasmap.length);
3757 break;
3758 }
3759 case KVM_S390_UCAS_UNMAP: {
3760 struct kvm_s390_ucas_mapping ucasmap;
3761
3762 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3763 r = -EFAULT;
3764 break;
3765 }
3766
3767 if (!kvm_is_ucontrol(vcpu->kvm)) {
3768 r = -EINVAL;
3769 break;
3770 }
3771
3772 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3773 ucasmap.length);
3774 break;
3775 }
3776#endif
ccc7910f 3777 case KVM_S390_VCPU_FAULT: {
527e30b4 3778 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 3779 break;
3780 }
d6712df9
CH		 3781 case KVM_ENABLE_CAP:
3782 {
3783 struct kvm_enable_cap cap;
3784 r = -EFAULT;
3785 if (copy_from_user(&cap, argp, sizeof(cap)))
3786 break;
3787 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3788 break;
3789 }
41408c28
TH		 3790 case KVM_S390_MEM_OP: {
3791 struct kvm_s390_mem_op mem_op;
3792
3793 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3794 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3795 else
3796 r = -EFAULT;
3797 break;
3798 }
816c7667
JF		 3799 case KVM_S390_SET_IRQ_STATE: {
3800 struct kvm_s390_irq_state irq_state;
3801
3802 r = -EFAULT;
3803 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3804 break;
3805 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3806 irq_state.len == 0 ||
3807 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3808 r = -EINVAL;
3809 break;
3810 }
3811 r = kvm_s390_set_irq_state(vcpu,
3812 (void __user *) irq_state.buf,
3813 irq_state.len);
3814 break;
3815 }
3816 case KVM_S390_GET_IRQ_STATE: {
3817 struct kvm_s390_irq_state irq_state;
3818
3819 r = -EFAULT;
3820 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3821 break;
3822 if (irq_state.len == 0) {
3823 r = -EINVAL;
3824 break;
3825 }
3826 r = kvm_s390_get_irq_state(vcpu,
3827 (__u8 __user *) irq_state.buf,
3828 irq_state.len);
3829 break;
3830 }
b0c632db 3831 default:
3e6afcf1 3832 r = -ENOTTY;
b0c632db 3833 }
bc923cc9 3834 return r;
b0c632db
HC		 3835}
3836
5b1c1493
CO		 3837int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3838{
3839#ifdef CONFIG_KVM_S390_UCONTROL
3840 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3841 && (kvm_is_ucontrol(vcpu->kvm))) {
3842 vmf->page = virt_to_page(vcpu->arch.sie_block);
3843 get_page(vmf->page);
3844 return 0;
3845 }
3846#endif
3847 return VM_FAULT_SIGBUS;
3848}
3849
5587027c
AK		 3850int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3851 unsigned long npages)
db3fe4eb
TY		 3852{
3853 return 0;
3854}
3855
b0c632db 3856/* Section: memory related */
f7784b8e
MT		 3857int kvm_arch_prepare_memory_region(struct kvm *kvm,
3858 struct kvm_memory_slot *memslot,
09170a49 3859 const struct kvm_userspace_memory_region *mem,
7b6195a9 3860 enum kvm_mr_change change)
b0c632db 3861{
dd2887e7
NW		 3862 /* A few sanity checks. We can have memory slots which have to be
3863 located/ended at a segment boundary (1MB). The memory in userland is
3864 ok to be fragmented into various different vmas. It is okay to mmap()
3865 and munmap() stuff in this slot after doing this call at any time */
b0c632db 3866
598841ca 3867 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 3868 return -EINVAL;
3869
598841ca 3870 if (mem->memory_size & 0xffffful)
b0c632db
HC		 3871 return -EINVAL;
3872
a3a92c31
DD		 3873 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3874 return -EINVAL;
3875
f7784b8e
MT		 3876 return 0;
3877}
3878
3879void kvm_arch_commit_memory_region(struct kvm *kvm,
09170a49 3880 const struct kvm_userspace_memory_region *mem,
8482644a 3881 const struct kvm_memory_slot *old,
f36f3f28 3882 const struct kvm_memory_slot *new,
8482644a 3883 enum kvm_mr_change change)
f7784b8e 3884{
f7850c92 3885 int rc;
f7784b8e 3886
2cef4deb
CB		 3887 /* If the basics of the memslot do not change, we do not want
3888 * to update the gmap. Every update causes several unnecessary
3889 * segment translation exceptions. This is usually handled just
3890 * fine by the normal fault handler + gmap, but it will also
3891 * cause faults on the prefix page of running guest CPUs.
3892 */
3893 if (old->userspace_addr == mem->userspace_addr &&
3894 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3895 old->npages * PAGE_SIZE == mem->memory_size)
3896 return;
598841ca
CO		 3897
3898 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3899 mem->guest_phys_addr, mem->memory_size);
3900 if (rc)
ea2cdd27 3901 pr_warn("failed to commit memory region\n");
598841ca 3902 return;
b0c632db
HC		 3903}
3904
60a37709
AY		 3905static inline unsigned long nonhyp_mask(int i)
3906{
3907 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3908
3909 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3910}
3911
3491caf2
CB		 3912void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3913{
3914 vcpu->valid_wakeup = false;
3915}
3916
b0c632db
HC		 3917static int __init kvm_s390_init(void)
3918{
60a37709
AY		 3919 int i;
3920
07197fd0
DH		 3921 if (!sclp.has_sief2) {
3922 pr_info("SIE not available\n");
3923 return -ENODEV;
3924 }
3925
60a37709
AY		 3926 for (i = 0; i < 16; i++)
3927 kvm_s390_fac_list_mask[i] |=
3928 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3929
9d8d5786 3930 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 3931}
3932
3933static void __exit kvm_s390_exit(void)
3934{
3935 kvm_exit();
3936}
3937
3938module_init(kvm_s390_init);
3939module_exit(kvm_s390_exit);
566af940
CH		 3940
3941/*
3942 * Enable autoloading of the kvm module.
3943 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3944 * since x86 takes a different approach.
3945 */
3946#include <linux/miscdevice.h>
3947MODULE_ALIAS_MISCDEV(KVM_MINOR);
3948MODULE_ALIAS("devname:kvm");
ubuntu-bionic-kernel mirror