]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: s390: mark irq_state.flags as non-usable
[mirror_ubuntu-jammy-kernel.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
d809aa23 1// SPDX-License-Identifier: GPL-2.0
b0c632db 2/*
bb64da9a 3 * hosting IBM Z kernel virtual machines (s390x)
b0c632db 4 *
bb64da9a 5 * Copyright IBM Corp. 2008, 2017
b0c632db 6 *
b0c632db
HC		 7 * Author(s): Carsten Otte <cotte@de.ibm.com>
8 * Christian Borntraeger <borntraeger@de.ibm.com>
9 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 10 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 11 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 12 */
13
14#include <linux/compiler.h>
15#include <linux/err.h>
16#include <linux/fs.h>
ca872302 17#include <linux/hrtimer.h>
b0c632db
HC		 18#include <linux/init.h>
19#include <linux/kvm.h>
20#include <linux/kvm_host.h>
b2d73b2a 21#include <linux/mman.h>
b0c632db 22#include <linux/module.h>
d3217967 23#include <linux/moduleparam.h>
a374e892 24#include <linux/random.h>
b0c632db 25#include <linux/slab.h>
ba5c1e9b 26#include <linux/timer.h>
41408c28 27#include <linux/vmalloc.h>
15c9705f 28#include <linux/bitmap.h>
174cd4b1 29#include <linux/sched/signal.h>
190df4a2 30#include <linux/string.h>
174cd4b1 31
cbb870c8 32#include <asm/asm-offsets.h>
b0c632db 33#include <asm/lowcore.h>
fd5ada04 34#include <asm/stp.h>
b0c632db 35#include <asm/pgtable.h>
1e133ab2 36#include <asm/gmap.h>
f5daba1d 37#include <asm/nmi.h>
a0616cde 38#include <asm/switch_to.h>
6d3da241 39#include <asm/isc.h>
1526bf9c 40#include <asm/sclp.h>
0a763c78 41#include <asm/cpacf.h>
221bb8a4 42#include <asm/timex.h>
8f2abe6a 43#include "kvm-s390.h"
b0c632db
HC		 44#include "gaccess.h"
45
ea2cdd27
DH		 46#define KMSG_COMPONENT "kvm-s390"
47#undef pr_fmt
48#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
49
5786fffa
CH		 50#define CREATE_TRACE_POINTS
51#include "trace.h"
ade38c31 52#include "trace-s390.h"
5786fffa 53
41408c28 54#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
816c7667
JF		 55#define LOCAL_IRQS 32
56#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
57 (KVM_MAX_VCPUS + LOCAL_IRQS))
41408c28 58
b0c632db
HC		 59#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
60
61struct kvm_stats_debugfs_item debugfs_entries[] = {
62 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 63 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 64 { "exit_validity", VCPU_STAT(exit_validity) },
65 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
66 { "exit_external_request", VCPU_STAT(exit_external_request) },
67 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b 68 { "exit_instruction", VCPU_STAT(exit_instruction) },
9ec6de19 69 { "exit_pei", VCPU_STAT(exit_pei) },
ba5c1e9b
CO		 70 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
71 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
a011eeb2 72 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
f7819512 73 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
62bea5bf 74 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
3491caf2 75 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
ce2e4f0b 76 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 77 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 78 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 79 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
80 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 81 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 82 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 83 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
84 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
85 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
86 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
87 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
88 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
89 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 90 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 91 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
92 { "instruction_spx", VCPU_STAT(instruction_spx) },
93 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
94 { "instruction_stap", VCPU_STAT(instruction_stap) },
95 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 96 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 97 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
98 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 99 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 100 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
101 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 102 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
95ca2cb5 103 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
a3508fbe 104 { "instruction_sie", VCPU_STAT(instruction_sie) },
5288fbf0 105 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 106 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 107 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 108 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 109 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
110 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 111 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 112 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
113 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 114 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 115 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
116 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
117 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 118 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
119 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
120 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 121 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 122 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 123 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
175a5c9e
CB		 124 { "diagnose_258", VCPU_STAT(diagnose_258) },
125 { "diagnose_308", VCPU_STAT(diagnose_308) },
126 { "diagnose_500", VCPU_STAT(diagnose_500) },
b0c632db
HC		 127 { NULL }
128};
129
8fa1696e
CW		 130struct kvm_s390_tod_clock_ext {
131 __u8 epoch_idx;
132 __u64 tod;
133 __u8 reserved[7];
134} __packed;
135
a411edf1
DH		 136/* allow nested virtualization in KVM (if enabled by user space) */
137static int nested;
138module_param(nested, int, S_IRUGO);
139MODULE_PARM_DESC(nested, "Nested virtualization support");
140
9d8d5786 141/* upper facilities limit for kvm */
f6c1d359 142unsigned long kvm_s390_fac_list_mask[16] = { FACILITIES_KVM };
b0c632db 143
9d8d5786 144unsigned long kvm_s390_fac_list_mask_size(void)
78c4b59f 145{
9d8d5786
MM		 146 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
147 return ARRAY_SIZE(kvm_s390_fac_list_mask);
78c4b59f
MM		 148}
149
15c9705f
DH		 150/* available cpu features supported by kvm */
151static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
0a763c78
DH		 152/* available subfunctions indicated via query / "test bit" */
153static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
15c9705f 154
9d8d5786 155static struct gmap_notifier gmap_notifier;
a3508fbe 156static struct gmap_notifier vsie_gmap_notifier;
78f26131 157debug_info_t *kvm_s390_dbf;
9d8d5786 158
b0c632db 159/* Section: not file related */
13a34e06 160int kvm_arch_hardware_enable(void)
b0c632db
HC		 161{
162 /* every s390 is virtualization enabled ;-) */
10474ae8 163 return 0;
b0c632db
HC		 164}
165
414d3b07
MS		 166static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
167 unsigned long end);
2c70fe44 168
fdf03650
FZ		 169/*
170 * This callback is executed during stop_machine(). All CPUs are therefore
171 * temporarily stopped. In order not to change guest behavior, we have to
172 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
173 * so a CPU won't be stopped while calculating with the epoch.
174 */
175static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
176 void *v)
177{
178 struct kvm *kvm;
179 struct kvm_vcpu *vcpu;
180 int i;
181 unsigned long long *delta = v;
182
183 list_for_each_entry(kvm, &vm_list, vm_list) {
184 kvm->arch.epoch -= *delta;
185 kvm_for_each_vcpu(i, vcpu, kvm) {
186 vcpu->arch.sie_block->epoch -= *delta;
db0758b2
DH		 187 if (vcpu->arch.cputm_enabled)
188 vcpu->arch.cputm_start += *delta;
91473b48
DH		 189 if (vcpu->arch.vsie_block)
190 vcpu->arch.vsie_block->epoch -= *delta;
fdf03650
FZ		 191 }
192 }
193 return NOTIFY_OK;
194}
195
196static struct notifier_block kvm_clock_notifier = {
197 .notifier_call = kvm_clock_sync,
198};
199
b0c632db
HC		 200int kvm_arch_hardware_setup(void)
201{
2c70fe44 202 gmap_notifier.notifier_call = kvm_gmap_notifier;
b2d73b2a 203 gmap_register_pte_notifier(&gmap_notifier);
a3508fbe
DH		 204 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
205 gmap_register_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 206 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
207 &kvm_clock_notifier);
b0c632db
HC		 208 return 0;
209}
210
211void kvm_arch_hardware_unsetup(void)
212{
b2d73b2a 213 gmap_unregister_pte_notifier(&gmap_notifier);
a3508fbe 214 gmap_unregister_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 215 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
216 &kvm_clock_notifier);
b0c632db
HC		 217}
218
22be5a13
DH		 219static void allow_cpu_feat(unsigned long nr)
220{
221 set_bit_inv(nr, kvm_s390_available_cpu_feat);
222}
223
0a763c78
DH		 224static inline int plo_test_bit(unsigned char nr)
225{
226 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
d051ae53 227 int cc;
0a763c78
DH		 228
229 asm volatile(
230 /* Parameter registers are ignored for "test bit" */
231 " plo 0,0,0,0(0)\n"
232 " ipm %0\n"
233 " srl %0,28\n"
234 : "=d" (cc)
235 : "d" (r0)
236 : "cc");
237 return cc == 0;
238}
239
22be5a13
DH		 240static void kvm_s390_cpu_feat_init(void)
241{
0a763c78
DH		 242 int i;
243
244 for (i = 0; i < 256; ++i) {
245 if (plo_test_bit(i))
246 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
247 }
248
249 if (test_facility(28)) /* TOD-clock steering */
221bb8a4
LT		 250 ptff(kvm_s390_available_subfunc.ptff,
251 sizeof(kvm_s390_available_subfunc.ptff),
252 PTFF_QAF);
0a763c78
DH		 253
254 if (test_facility(17)) { /* MSA */
69c0e360
MS		 255 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
256 kvm_s390_available_subfunc.kmac);
257 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
258 kvm_s390_available_subfunc.kmc);
259 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
260 kvm_s390_available_subfunc.km);
261 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
262 kvm_s390_available_subfunc.kimd);
263 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
264 kvm_s390_available_subfunc.klmd);
0a763c78
DH		 265 }
266 if (test_facility(76)) /* MSA3 */
69c0e360
MS		 267 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
268 kvm_s390_available_subfunc.pckmo);
0a763c78 269 if (test_facility(77)) { /* MSA4 */
69c0e360
MS		 270 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
271 kvm_s390_available_subfunc.kmctr);
272 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
273 kvm_s390_available_subfunc.kmf);
274 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
275 kvm_s390_available_subfunc.kmo);
276 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
277 kvm_s390_available_subfunc.pcc);
0a763c78
DH		 278 }
279 if (test_facility(57)) /* MSA5 */
985a9d20 280 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
69c0e360 281 kvm_s390_available_subfunc.ppno);
0a763c78 282
e000b8e0
JH		 283 if (test_facility(146)) /* MSA8 */
284 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
285 kvm_s390_available_subfunc.kma);
286
22be5a13
DH		 287 if (MACHINE_HAS_ESOP)
288 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
a3508fbe
DH		 289 /*
290 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
291 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
292 */
293 if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
a411edf1 294 !test_facility(3) || !nested)
a3508fbe
DH		 295 return;
296 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
19c439b5
DH		 297 if (sclp.has_64bscao)
298 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
0615a326
DH		 299 if (sclp.has_siif)
300 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
77d18f6d
DH		 301 if (sclp.has_gpere)
302 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
a1b7b9b2
DH		 303 if (sclp.has_gsls)
304 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
5630a8e8
DH		 305 if (sclp.has_ib)
306 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
13ee3f67
DH		 307 if (sclp.has_cei)
308 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
7fd7f39d
DH		 309 if (sclp.has_ibs)
310 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
730cd632
FA		 311 if (sclp.has_kss)
312 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
5d3876a8
DH		 313 /*
314 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
315 * all skey handling functions read/set the skey from the PGSTE
316 * instead of the real storage key.
317 *
318 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
319 * pages being detected as preserved although they are resident.
320 *
321 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
322 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
323 *
324 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
325 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
326 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
327 *
328 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
329 * cannot easily shadow the SCA because of the ipte lock.
330 */
22be5a13
DH		 331}
332
b0c632db
HC		 333int kvm_arch_init(void *opaque)
334{
78f26131
CB		 335 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
336 if (!kvm_s390_dbf)
337 return -ENOMEM;
338
339 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
340 debug_unregister(kvm_s390_dbf);
341 return -ENOMEM;
342 }
343
22be5a13
DH		 344 kvm_s390_cpu_feat_init();
345
84877d93
CH		 346 /* Register floating interrupt controller interface. */
347 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 348}
349
78f26131
CB		 350void kvm_arch_exit(void)
351{
352 debug_unregister(kvm_s390_dbf);
353}
354
b0c632db
HC		 355/* Section: device related */
356long kvm_arch_dev_ioctl(struct file *filp,
357 unsigned int ioctl, unsigned long arg)
358{
359 if (ioctl == KVM_S390_ENABLE_SIE)
360 return s390_enable_sie();
361 return -EINVAL;
362}
363
784aa3d7 364int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 365{
d7b0b5eb
CO		 366 int r;
367
2bd0ac4e 368 switch (ext) {
d7b0b5eb 369 case KVM_CAP_S390_PSW:
b6cf8788 370 case KVM_CAP_S390_GMAP:
52e16b18 371 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 372#ifdef CONFIG_KVM_S390_UCONTROL
373 case KVM_CAP_S390_UCONTROL:
374#endif
3c038e6b 375 case KVM_CAP_ASYNC_PF:
60b413c9 376 case KVM_CAP_SYNC_REGS:
14eebd91 377 case KVM_CAP_ONE_REG:
d6712df9 378 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 379 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 380 case KVM_CAP_IOEVENTFD:
c05c4186 381 case KVM_CAP_DEVICE_CTRL:
d938dc55 382 case KVM_CAP_ENABLE_CAP_VM:
78599d90 383 case KVM_CAP_S390_IRQCHIP:
f2061656 384 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 385 case KVM_CAP_MP_STATE:
460df4c1 386 case KVM_CAP_IMMEDIATE_EXIT:
47b43c52 387 case KVM_CAP_S390_INJECT_IRQ:
2444b352 388 case KVM_CAP_S390_USER_SIGP:
e44fc8c9 389 case KVM_CAP_S390_USER_STSI:
30ee2a98 390 case KVM_CAP_S390_SKEYS:
816c7667 391 case KVM_CAP_S390_IRQ_STATE:
6502a34c 392 case KVM_CAP_S390_USER_INSTR0:
4036e387 393 case KVM_CAP_S390_CMMA_MIGRATION:
47a4693e 394 case KVM_CAP_S390_AIS:
da9a1446 395 case KVM_CAP_S390_AIS_MIGRATION:
d7b0b5eb
CO		 396 r = 1;
397 break;
41408c28
TH		 398 case KVM_CAP_S390_MEM_OP:
399 r = MEM_OP_MAX_SIZE;
400 break;
e726b1bd
CB		 401 case KVM_CAP_NR_VCPUS:
402 case KVM_CAP_MAX_VCPUS:
76a6dd72 403 r = KVM_S390_BSCA_CPU_SLOTS;
a6940674
DH		 404 if (!kvm_s390_use_sca_entries())
405 r = KVM_MAX_VCPUS;
406 else if (sclp.has_esca && sclp.has_64bscao)
76a6dd72 407 r = KVM_S390_ESCA_CPU_SLOTS;
e726b1bd 408 break;
e1e2e605
NW		 409 case KVM_CAP_NR_MEMSLOTS:
410 r = KVM_USER_MEM_SLOTS;
411 break;
1526bf9c 412 case KVM_CAP_S390_COW:
abf09bed 413 r = MACHINE_HAS_ESOP;
1526bf9c 414 break;
68c55750
EF		 415 case KVM_CAP_S390_VECTOR_REGISTERS:
416 r = MACHINE_HAS_VX;
417 break;
c6e5f166
FZ		 418 case KVM_CAP_S390_RI:
419 r = test_facility(64);
420 break;
4e0b1ab7
FZ		 421 case KVM_CAP_S390_GS:
422 r = test_facility(133);
423 break;
2bd0ac4e 424 default:
d7b0b5eb 425 r = 0;
2bd0ac4e 426 }
d7b0b5eb 427 return r;
b0c632db
HC		 428}
429
15f36ebd
JH		 430static void kvm_s390_sync_dirty_log(struct kvm *kvm,
431 struct kvm_memory_slot *memslot)
432{
433 gfn_t cur_gfn, last_gfn;
434 unsigned long address;
435 struct gmap *gmap = kvm->arch.gmap;
436
15f36ebd
JH		 437 /* Loop over all guest pages */
438 last_gfn = memslot->base_gfn + memslot->npages;
439 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
440 address = gfn_to_hva_memslot(memslot, cur_gfn);
441
1e133ab2 442 if (test_and_clear_guest_dirty(gmap->mm, address))
15f36ebd 443 mark_page_dirty(kvm, cur_gfn);
1763f8d0
CB		 444 if (fatal_signal_pending(current))
445 return;
70c88a00 446 cond_resched();
15f36ebd 447 }
15f36ebd
JH		 448}
449
b0c632db 450/* Section: vm related */
a6e2f683
ED		 451static void sca_del_vcpu(struct kvm_vcpu *vcpu);
452
b0c632db
HC		 453/*
454 * Get (and clear) the dirty memory log for a memory slot.
455 */
456int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
457 struct kvm_dirty_log *log)
458{
15f36ebd
JH		 459 int r;
460 unsigned long n;
9f6b8029 461 struct kvm_memslots *slots;
15f36ebd
JH		 462 struct kvm_memory_slot *memslot;
463 int is_dirty = 0;
464
e1e8a962
JF		 465 if (kvm_is_ucontrol(kvm))
466 return -EINVAL;
467
15f36ebd
JH		 468 mutex_lock(&kvm->slots_lock);
469
470 r = -EINVAL;
471 if (log->slot >= KVM_USER_MEM_SLOTS)
472 goto out;
473
9f6b8029
PB		 474 slots = kvm_memslots(kvm);
475 memslot = id_to_memslot(slots, log->slot);
15f36ebd
JH		 476 r = -ENOENT;
477 if (!memslot->dirty_bitmap)
478 goto out;
479
480 kvm_s390_sync_dirty_log(kvm, memslot);
481 r = kvm_get_dirty_log(kvm, log, &is_dirty);
482 if (r)
483 goto out;
484
485 /* Clear the dirty log */
486 if (is_dirty) {
487 n = kvm_dirty_bitmap_bytes(memslot);
488 memset(memslot->dirty_bitmap, 0, n);
489 }
490 r = 0;
491out:
492 mutex_unlock(&kvm->slots_lock);
493 return r;
b0c632db
HC		 494}
495
6502a34c
DH		 496static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
497{
498 unsigned int i;
499 struct kvm_vcpu *vcpu;
500
501 kvm_for_each_vcpu(i, vcpu, kvm) {
502 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
503 }
504}
505
d938dc55
CH		 506static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
507{
508 int r;
509
510 if (cap->flags)
511 return -EINVAL;
512
513 switch (cap->cap) {
84223598 514 case KVM_CAP_S390_IRQCHIP:
c92ea7b9 515 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
84223598
CH		 516 kvm->arch.use_irqchip = 1;
517 r = 0;
518 break;
2444b352 519 case KVM_CAP_S390_USER_SIGP:
c92ea7b9 520 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
2444b352
DH		 521 kvm->arch.user_sigp = 1;
522 r = 0;
523 break;
68c55750 524 case KVM_CAP_S390_VECTOR_REGISTERS:
5967c17b 525 mutex_lock(&kvm->lock);
a03825bb 526 if (kvm->created_vcpus) {
5967c17b
DH		 527 r = -EBUSY;
528 } else if (MACHINE_HAS_VX) {
c54f0d6a
DH		 529 set_kvm_facility(kvm->arch.model.fac_mask, 129);
530 set_kvm_facility(kvm->arch.model.fac_list, 129);
2f87d942
GH		 531 if (test_facility(134)) {
532 set_kvm_facility(kvm->arch.model.fac_mask, 134);
533 set_kvm_facility(kvm->arch.model.fac_list, 134);
534 }
53743aa7
MS		 535 if (test_facility(135)) {
536 set_kvm_facility(kvm->arch.model.fac_mask, 135);
537 set_kvm_facility(kvm->arch.model.fac_list, 135);
538 }
18280d8b
MM		 539 r = 0;
540 } else
541 r = -EINVAL;
5967c17b 542 mutex_unlock(&kvm->lock);
c92ea7b9
CB		 543 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
544 r ? "(not available)" : "(success)");
68c55750 545 break;
c6e5f166
FZ		 546 case KVM_CAP_S390_RI:
547 r = -EINVAL;
548 mutex_lock(&kvm->lock);
a03825bb 549 if (kvm->created_vcpus) {
c6e5f166
FZ		 550 r = -EBUSY;
551 } else if (test_facility(64)) {
c54f0d6a
DH		 552 set_kvm_facility(kvm->arch.model.fac_mask, 64);
553 set_kvm_facility(kvm->arch.model.fac_list, 64);
c6e5f166
FZ		 554 r = 0;
555 }
556 mutex_unlock(&kvm->lock);
557 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
558 r ? "(not available)" : "(success)");
559 break;
47a4693e
YMZ		 560 case KVM_CAP_S390_AIS:
561 mutex_lock(&kvm->lock);
562 if (kvm->created_vcpus) {
563 r = -EBUSY;
564 } else {
565 set_kvm_facility(kvm->arch.model.fac_mask, 72);
566 set_kvm_facility(kvm->arch.model.fac_list, 72);
47a4693e
YMZ		 567 r = 0;
568 }
569 mutex_unlock(&kvm->lock);
570 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
571 r ? "(not available)" : "(success)");
572 break;
4e0b1ab7
FZ		 573 case KVM_CAP_S390_GS:
574 r = -EINVAL;
575 mutex_lock(&kvm->lock);
576 if (atomic_read(&kvm->online_vcpus)) {
577 r = -EBUSY;
578 } else if (test_facility(133)) {
579 set_kvm_facility(kvm->arch.model.fac_mask, 133);
580 set_kvm_facility(kvm->arch.model.fac_list, 133);
581 r = 0;
582 }
583 mutex_unlock(&kvm->lock);
584 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
585 r ? "(not available)" : "(success)");
586 break;
e44fc8c9 587 case KVM_CAP_S390_USER_STSI:
c92ea7b9 588 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
e44fc8c9
ET		 589 kvm->arch.user_stsi = 1;
590 r = 0;
591 break;
6502a34c
DH		 592 case KVM_CAP_S390_USER_INSTR0:
593 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
594 kvm->arch.user_instr0 = 1;
595 icpt_operexc_on_all_vcpus(kvm);
596 r = 0;
597 break;
d938dc55
CH		 598 default:
599 r = -EINVAL;
600 break;
601 }
602 return r;
603}
604
8c0a7ce6
DD		 605static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
606{
607 int ret;
608
609 switch (attr->attr) {
610 case KVM_S390_VM_MEM_LIMIT_SIZE:
611 ret = 0;
c92ea7b9 612 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
a3a92c31
DD		 613 kvm->arch.mem_limit);
614 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
8c0a7ce6
DD		 615 ret = -EFAULT;
616 break;
617 default:
618 ret = -ENXIO;
619 break;
620 }
621 return ret;
622}
623
624static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 625{
626 int ret;
627 unsigned int idx;
628 switch (attr->attr) {
629 case KVM_S390_VM_MEM_ENABLE_CMMA:
f9cbd9b0 630 ret = -ENXIO;
c24cc9c8 631 if (!sclp.has_cmma)
e6db1d61
DD		 632 break;
633
4f718eab 634 ret = -EBUSY;
c92ea7b9 635 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
4f718eab 636 mutex_lock(&kvm->lock);
a03825bb 637 if (!kvm->created_vcpus) {
4f718eab
DD		 638 kvm->arch.use_cmma = 1;
639 ret = 0;
640 }
641 mutex_unlock(&kvm->lock);
642 break;
643 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 644 ret = -ENXIO;
645 if (!sclp.has_cmma)
646 break;
c3489155
DD		 647 ret = -EINVAL;
648 if (!kvm->arch.use_cmma)
649 break;
650
c92ea7b9 651 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
4f718eab
DD		 652 mutex_lock(&kvm->lock);
653 idx = srcu_read_lock(&kvm->srcu);
a13cff31 654 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 655 srcu_read_unlock(&kvm->srcu, idx);
656 mutex_unlock(&kvm->lock);
657 ret = 0;
658 break;
8c0a7ce6
DD		 659 case KVM_S390_VM_MEM_LIMIT_SIZE: {
660 unsigned long new_limit;
661
662 if (kvm_is_ucontrol(kvm))
663 return -EINVAL;
664
665 if (get_user(new_limit, (u64 __user *)attr->addr))
666 return -EFAULT;
667
a3a92c31
DD		 668 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
669 new_limit > kvm->arch.mem_limit)
8c0a7ce6
DD		 670 return -E2BIG;
671
a3a92c31
DD		 672 if (!new_limit)
673 return -EINVAL;
674
6ea427bb 675 /* gmap_create takes last usable address */
a3a92c31
DD		 676 if (new_limit != KVM_S390_NO_MEM_LIMIT)
677 new_limit -= 1;
678
8c0a7ce6
DD		 679 ret = -EBUSY;
680 mutex_lock(&kvm->lock);
a03825bb 681 if (!kvm->created_vcpus) {
6ea427bb
MS		 682 /* gmap_create will round the limit up */
683 struct gmap *new = gmap_create(current->mm, new_limit);
8c0a7ce6
DD		 684
685 if (!new) {
686 ret = -ENOMEM;
687 } else {
6ea427bb 688 gmap_remove(kvm->arch.gmap);
8c0a7ce6
DD		 689 new->private = kvm;
690 kvm->arch.gmap = new;
691 ret = 0;
692 }
693 }
694 mutex_unlock(&kvm->lock);
a3a92c31
DD		 695 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
696 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
697 (void *) kvm->arch.gmap->asce);
8c0a7ce6
DD		 698 break;
699 }
4f718eab
DD		 700 default:
701 ret = -ENXIO;
702 break;
703 }
704 return ret;
705}
706
a374e892
TK		 707static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
708
709static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
710{
711 struct kvm_vcpu *vcpu;
712 int i;
713
9d8d5786 714 if (!test_kvm_facility(kvm, 76))
a374e892
TK		 715 return -EINVAL;
716
717 mutex_lock(&kvm->lock);
718 switch (attr->attr) {
719 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
720 get_random_bytes(
721 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
722 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
723 kvm->arch.crypto.aes_kw = 1;
c92ea7b9 724 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
a374e892
TK		 725 break;
726 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
727 get_random_bytes(
728 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
729 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
730 kvm->arch.crypto.dea_kw = 1;
c92ea7b9 731 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
a374e892
TK		 732 break;
733 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
734 kvm->arch.crypto.aes_kw = 0;
735 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
736 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
c92ea7b9 737 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
a374e892
TK		 738 break;
739 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
740 kvm->arch.crypto.dea_kw = 0;
741 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
742 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
c92ea7b9 743 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
a374e892
TK		 744 break;
745 default:
746 mutex_unlock(&kvm->lock);
747 return -ENXIO;
748 }
749
750 kvm_for_each_vcpu(i, vcpu, kvm) {
751 kvm_s390_vcpu_crypto_setup(vcpu);
752 exit_sie(vcpu);
753 }
754 mutex_unlock(&kvm->lock);
755 return 0;
756}
757
190df4a2
CI		 758static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
759{
760 int cx;
761 struct kvm_vcpu *vcpu;
762
763 kvm_for_each_vcpu(cx, vcpu, kvm)
764 kvm_s390_sync_request(req, vcpu);
765}
766
767/*
768 * Must be called with kvm->srcu held to avoid races on memslots, and with
769 * kvm->lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
770 */
771static int kvm_s390_vm_start_migration(struct kvm *kvm)
772{
773 struct kvm_s390_migration_state *mgs;
774 struct kvm_memory_slot *ms;
775 /* should be the only one */
776 struct kvm_memslots *slots;
777 unsigned long ram_pages;
778 int slotnr;
779
780 /* migration mode already enabled */
781 if (kvm->arch.migration_state)
782 return 0;
783
784 slots = kvm_memslots(kvm);
785 if (!slots || !slots->used_slots)
786 return -EINVAL;
787
788 mgs = kzalloc(sizeof(*mgs), GFP_KERNEL);
789 if (!mgs)
790 return -ENOMEM;
791 kvm->arch.migration_state = mgs;
792
793 if (kvm->arch.use_cmma) {
794 /*
795 * Get the last slot. They should be sorted by base_gfn, so the
796 * last slot is also the one at the end of the address space.
797 * We have verified above that at least one slot is present.
798 */
799 ms = slots->memslots + slots->used_slots - 1;
800 /* round up so we only use full longs */
801 ram_pages = roundup(ms->base_gfn + ms->npages, BITS_PER_LONG);
802 /* allocate enough bytes to store all the bits */
803 mgs->pgste_bitmap = vmalloc(ram_pages / 8);
804 if (!mgs->pgste_bitmap) {
805 kfree(mgs);
806 kvm->arch.migration_state = NULL;
807 return -ENOMEM;
808 }
809
810 mgs->bitmap_size = ram_pages;
811 atomic64_set(&mgs->dirty_pages, ram_pages);
812 /* mark all the pages in active slots as dirty */
813 for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
814 ms = slots->memslots + slotnr;
815 bitmap_set(mgs->pgste_bitmap, ms->base_gfn, ms->npages);
816 }
817
818 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
819 }
820 return 0;
821}
822
823/*
824 * Must be called with kvm->lock to avoid races with ourselves and
825 * kvm_s390_vm_start_migration.
826 */
827static int kvm_s390_vm_stop_migration(struct kvm *kvm)
828{
829 struct kvm_s390_migration_state *mgs;
830
831 /* migration mode already disabled */
832 if (!kvm->arch.migration_state)
833 return 0;
834 mgs = kvm->arch.migration_state;
835 kvm->arch.migration_state = NULL;
836
837 if (kvm->arch.use_cmma) {
838 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
839 vfree(mgs->pgste_bitmap);
840 }
841 kfree(mgs);
842 return 0;
843}
844
845static int kvm_s390_vm_set_migration(struct kvm *kvm,
846 struct kvm_device_attr *attr)
847{
848 int idx, res = -ENXIO;
849
850 mutex_lock(&kvm->lock);
851 switch (attr->attr) {
852 case KVM_S390_VM_MIGRATION_START:
853 idx = srcu_read_lock(&kvm->srcu);
854 res = kvm_s390_vm_start_migration(kvm);
855 srcu_read_unlock(&kvm->srcu, idx);
856 break;
857 case KVM_S390_VM_MIGRATION_STOP:
858 res = kvm_s390_vm_stop_migration(kvm);
859 break;
860 default:
861 break;
862 }
863 mutex_unlock(&kvm->lock);
864
865 return res;
866}
867
868static int kvm_s390_vm_get_migration(struct kvm *kvm,
869 struct kvm_device_attr *attr)
870{
871 u64 mig = (kvm->arch.migration_state != NULL);
872
873 if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
874 return -ENXIO;
875
876 if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
877 return -EFAULT;
878 return 0;
879}
880
8fa1696e
CW		 881static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
882{
883 struct kvm_s390_vm_tod_clock gtod;
884
885 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
886 return -EFAULT;
887
888 if (test_kvm_facility(kvm, 139))
889 kvm_s390_set_tod_clock_ext(kvm, &gtod);
890 else if (gtod.epoch_idx == 0)
891 kvm_s390_set_tod_clock(kvm, gtod.tod);
892 else
893 return -EINVAL;
894
895 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
896 gtod.epoch_idx, gtod.tod);
897
898 return 0;
899}
900
72f25020
JH		 901static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
902{
903 u8 gtod_high;
904
905 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
906 sizeof(gtod_high)))
907 return -EFAULT;
908
909 if (gtod_high != 0)
910 return -EINVAL;
58c383c6 911 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
72f25020
JH		 912
913 return 0;
914}
915
916static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
917{
5a3d883a 918 u64 gtod;
72f25020
JH		 919
920 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
921 return -EFAULT;
922
25ed1675 923 kvm_s390_set_tod_clock(kvm, gtod);
58c383c6 924 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
72f25020
JH		 925 return 0;
926}
927
928static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
929{
930 int ret;
931
932 if (attr->flags)
933 return -EINVAL;
934
935 switch (attr->attr) {
8fa1696e
CW		 936 case KVM_S390_VM_TOD_EXT:
937 ret = kvm_s390_set_tod_ext(kvm, attr);
938 break;
72f25020
JH		 939 case KVM_S390_VM_TOD_HIGH:
940 ret = kvm_s390_set_tod_high(kvm, attr);
941 break;
942 case KVM_S390_VM_TOD_LOW:
943 ret = kvm_s390_set_tod_low(kvm, attr);
944 break;
945 default:
946 ret = -ENXIO;
947 break;
948 }
949 return ret;
950}
951
8fa1696e
CW		 952static void kvm_s390_get_tod_clock_ext(struct kvm *kvm,
953 struct kvm_s390_vm_tod_clock *gtod)
954{
955 struct kvm_s390_tod_clock_ext htod;
956
957 preempt_disable();
958
959 get_tod_clock_ext((char *)&htod);
960
961 gtod->tod = htod.tod + kvm->arch.epoch;
962 gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
963
964 if (gtod->tod < htod.tod)
965 gtod->epoch_idx += 1;
966
967 preempt_enable();
968}
969
970static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
971{
972 struct kvm_s390_vm_tod_clock gtod;
973
974 memset(&gtod, 0, sizeof(gtod));
975
976 if (test_kvm_facility(kvm, 139))
977 kvm_s390_get_tod_clock_ext(kvm, &gtod);
978 else
979 gtod.tod = kvm_s390_get_tod_clock_fast(kvm);
980
981 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
982 return -EFAULT;
983
984 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
985 gtod.epoch_idx, gtod.tod);
986 return 0;
987}
988
72f25020
JH		 989static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
990{
991 u8 gtod_high = 0;
992
993 if (copy_to_user((void __user *)attr->addr, &gtod_high,
994 sizeof(gtod_high)))
995 return -EFAULT;
58c383c6 996 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
72f25020
JH		 997
998 return 0;
999}
1000
1001static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1002{
5a3d883a 1003 u64 gtod;
72f25020 1004
60417fcc 1005 gtod = kvm_s390_get_tod_clock_fast(kvm);
72f25020
JH		 1006 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1007 return -EFAULT;
58c383c6 1008 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
72f25020
JH		 1009
1010 return 0;
1011}
1012
1013static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1014{
1015 int ret;
1016
1017 if (attr->flags)
1018 return -EINVAL;
1019
1020 switch (attr->attr) {
8fa1696e
CW		 1021 case KVM_S390_VM_TOD_EXT:
1022 ret = kvm_s390_get_tod_ext(kvm, attr);
1023 break;
72f25020
JH		 1024 case KVM_S390_VM_TOD_HIGH:
1025 ret = kvm_s390_get_tod_high(kvm, attr);
1026 break;
1027 case KVM_S390_VM_TOD_LOW:
1028 ret = kvm_s390_get_tod_low(kvm, attr);
1029 break;
1030 default:
1031 ret = -ENXIO;
1032 break;
1033 }
1034 return ret;
1035}
1036
658b6eda
MM		 1037static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1038{
1039 struct kvm_s390_vm_cpu_processor *proc;
053dd230 1040 u16 lowest_ibc, unblocked_ibc;
658b6eda
MM		 1041 int ret = 0;
1042
1043 mutex_lock(&kvm->lock);
a03825bb 1044 if (kvm->created_vcpus) {
658b6eda
MM		 1045 ret = -EBUSY;
1046 goto out;
1047 }
1048 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1049 if (!proc) {
1050 ret = -ENOMEM;
1051 goto out;
1052 }
1053 if (!copy_from_user(proc, (void __user *)attr->addr,
1054 sizeof(*proc))) {
9bb0ec09 1055 kvm->arch.model.cpuid = proc->cpuid;
053dd230
DH		 1056 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1057 unblocked_ibc = sclp.ibc & 0xfff;
0487c44d 1058 if (lowest_ibc && proc->ibc) {
053dd230
DH		 1059 if (proc->ibc > unblocked_ibc)
1060 kvm->arch.model.ibc = unblocked_ibc;
1061 else if (proc->ibc < lowest_ibc)
1062 kvm->arch.model.ibc = lowest_ibc;
1063 else
1064 kvm->arch.model.ibc = proc->ibc;
1065 }
c54f0d6a 1066 memcpy(kvm->arch.model.fac_list, proc->fac_list,
658b6eda 1067 S390_ARCH_FAC_LIST_SIZE_BYTE);
a8c39dd7
CB		 1068 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1069 kvm->arch.model.ibc,
1070 kvm->arch.model.cpuid);
1071 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1072 kvm->arch.model.fac_list[0],
1073 kvm->arch.model.fac_list[1],
1074 kvm->arch.model.fac_list[2]);
658b6eda
MM		 1075 } else
1076 ret = -EFAULT;
1077 kfree(proc);
1078out:
1079 mutex_unlock(&kvm->lock);
1080 return ret;
1081}
1082
15c9705f
DH		 1083static int kvm_s390_set_processor_feat(struct kvm *kvm,
1084 struct kvm_device_attr *attr)
1085{
1086 struct kvm_s390_vm_cpu_feat data;
1087 int ret = -EBUSY;
1088
1089 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1090 return -EFAULT;
1091 if (!bitmap_subset((unsigned long *) data.feat,
1092 kvm_s390_available_cpu_feat,
1093 KVM_S390_VM_CPU_FEAT_NR_BITS))
1094 return -EINVAL;
1095
1096 mutex_lock(&kvm->lock);
1097 if (!atomic_read(&kvm->online_vcpus)) {
1098 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
1099 KVM_S390_VM_CPU_FEAT_NR_BITS);
1100 ret = 0;
1101 }
1102 mutex_unlock(&kvm->lock);
1103 return ret;
1104}
1105
0a763c78
DH		 1106static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1107 struct kvm_device_attr *attr)
1108{
1109 /*
1110 * Once supported by kernel + hw, we have to store the subfunctions
1111 * in kvm->arch and remember that user space configured them.
1112 */
1113 return -ENXIO;
1114}
1115
658b6eda
MM		 1116static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1117{
1118 int ret = -ENXIO;
1119
1120 switch (attr->attr) {
1121 case KVM_S390_VM_CPU_PROCESSOR:
1122 ret = kvm_s390_set_processor(kvm, attr);
1123 break;
15c9705f
DH		 1124 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1125 ret = kvm_s390_set_processor_feat(kvm, attr);
1126 break;
0a763c78
DH		 1127 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1128 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1129 break;
658b6eda
MM		 1130 }
1131 return ret;
1132}
1133
1134static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1135{
1136 struct kvm_s390_vm_cpu_processor *proc;
1137 int ret = 0;
1138
1139 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1140 if (!proc) {
1141 ret = -ENOMEM;
1142 goto out;
1143 }
9bb0ec09 1144 proc->cpuid = kvm->arch.model.cpuid;
658b6eda 1145 proc->ibc = kvm->arch.model.ibc;
c54f0d6a
DH		 1146 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1147 S390_ARCH_FAC_LIST_SIZE_BYTE);
a8c39dd7
CB		 1148 VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1149 kvm->arch.model.ibc,
1150 kvm->arch.model.cpuid);
1151 VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1152 kvm->arch.model.fac_list[0],
1153 kvm->arch.model.fac_list[1],
1154 kvm->arch.model.fac_list[2]);
658b6eda
MM		 1155 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1156 ret = -EFAULT;
1157 kfree(proc);
1158out:
1159 return ret;
1160}
1161
1162static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1163{
1164 struct kvm_s390_vm_cpu_machine *mach;
1165 int ret = 0;
1166
1167 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
1168 if (!mach) {
1169 ret = -ENOMEM;
1170 goto out;
1171 }
1172 get_cpu_id((struct cpuid *) &mach->cpuid);
37c5f6c8 1173 mach->ibc = sclp.ibc;
c54f0d6a 1174 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
981467c9 1175 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 1176 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
04478197 1177 sizeof(S390_lowcore.stfle_fac_list));
a8c39dd7
CB		 1178 VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx",
1179 kvm->arch.model.ibc,
1180 kvm->arch.model.cpuid);
1181 VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx",
1182 mach->fac_mask[0],
1183 mach->fac_mask[1],
1184 mach->fac_mask[2]);
1185 VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1186 mach->fac_list[0],
1187 mach->fac_list[1],
1188 mach->fac_list[2]);
658b6eda
MM		 1189 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1190 ret = -EFAULT;
1191 kfree(mach);
1192out:
1193 return ret;
1194}
1195
15c9705f
DH		 1196static int kvm_s390_get_processor_feat(struct kvm *kvm,
1197 struct kvm_device_attr *attr)
1198{
1199 struct kvm_s390_vm_cpu_feat data;
1200
1201 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1202 KVM_S390_VM_CPU_FEAT_NR_BITS);
1203 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1204 return -EFAULT;
1205 return 0;
1206}
1207
1208static int kvm_s390_get_machine_feat(struct kvm *kvm,
1209 struct kvm_device_attr *attr)
1210{
1211 struct kvm_s390_vm_cpu_feat data;
1212
1213 bitmap_copy((unsigned long *) data.feat,
1214 kvm_s390_available_cpu_feat,
1215 KVM_S390_VM_CPU_FEAT_NR_BITS);
1216 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1217 return -EFAULT;
1218 return 0;
1219}
1220
0a763c78
DH		 1221static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1222 struct kvm_device_attr *attr)
1223{
1224 /*
1225 * Once we can actually configure subfunctions (kernel + hw support),
1226 * we have to check if they were already set by user space, if so copy
1227 * them from kvm->arch.
1228 */
1229 return -ENXIO;
1230}
1231
1232static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1233 struct kvm_device_attr *attr)
1234{
1235 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1236 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1237 return -EFAULT;
1238 return 0;
1239}
658b6eda
MM		 1240static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1241{
1242 int ret = -ENXIO;
1243
1244 switch (attr->attr) {
1245 case KVM_S390_VM_CPU_PROCESSOR:
1246 ret = kvm_s390_get_processor(kvm, attr);
1247 break;
1248 case KVM_S390_VM_CPU_MACHINE:
1249 ret = kvm_s390_get_machine(kvm, attr);
1250 break;
15c9705f
DH		 1251 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1252 ret = kvm_s390_get_processor_feat(kvm, attr);
1253 break;
1254 case KVM_S390_VM_CPU_MACHINE_FEAT:
1255 ret = kvm_s390_get_machine_feat(kvm, attr);
1256 break;
0a763c78
DH		 1257 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1258 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1259 break;
1260 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1261 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1262 break;
658b6eda
MM		 1263 }
1264 return ret;
1265}
1266
f2061656
DD		 1267static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1268{
1269 int ret;
1270
1271 switch (attr->group) {
4f718eab 1272 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 1273 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 1274 break;
72f25020
JH		 1275 case KVM_S390_VM_TOD:
1276 ret = kvm_s390_set_tod(kvm, attr);
1277 break;
658b6eda
MM		 1278 case KVM_S390_VM_CPU_MODEL:
1279 ret = kvm_s390_set_cpu_model(kvm, attr);
1280 break;
a374e892
TK		 1281 case KVM_S390_VM_CRYPTO:
1282 ret = kvm_s390_vm_set_crypto(kvm, attr);
1283 break;
190df4a2
CI		 1284 case KVM_S390_VM_MIGRATION:
1285 ret = kvm_s390_vm_set_migration(kvm, attr);
1286 break;
f2061656
DD		 1287 default:
1288 ret = -ENXIO;
1289 break;
1290 }
1291
1292 return ret;
1293}
1294
1295static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1296{
8c0a7ce6
DD		 1297 int ret;
1298
1299 switch (attr->group) {
1300 case KVM_S390_VM_MEM_CTRL:
1301 ret = kvm_s390_get_mem_control(kvm, attr);
1302 break;
72f25020
JH		 1303 case KVM_S390_VM_TOD:
1304 ret = kvm_s390_get_tod(kvm, attr);
1305 break;
658b6eda
MM		 1306 case KVM_S390_VM_CPU_MODEL:
1307 ret = kvm_s390_get_cpu_model(kvm, attr);
1308 break;
190df4a2
CI		 1309 case KVM_S390_VM_MIGRATION:
1310 ret = kvm_s390_vm_get_migration(kvm, attr);
1311 break;
8c0a7ce6
DD		 1312 default:
1313 ret = -ENXIO;
1314 break;
1315 }
1316
1317 return ret;
f2061656
DD		 1318}
1319
1320static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1321{
1322 int ret;
1323
1324 switch (attr->group) {
4f718eab
DD		 1325 case KVM_S390_VM_MEM_CTRL:
1326 switch (attr->attr) {
1327 case KVM_S390_VM_MEM_ENABLE_CMMA:
1328 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 1329 ret = sclp.has_cmma ? 0 : -ENXIO;
1330 break;
8c0a7ce6 1331 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 1332 ret = 0;
1333 break;
1334 default:
1335 ret = -ENXIO;
1336 break;
1337 }
1338 break;
72f25020
JH		 1339 case KVM_S390_VM_TOD:
1340 switch (attr->attr) {
1341 case KVM_S390_VM_TOD_LOW:
1342 case KVM_S390_VM_TOD_HIGH:
1343 ret = 0;
1344 break;
1345 default:
1346 ret = -ENXIO;
1347 break;
1348 }
1349 break;
658b6eda
MM		 1350 case KVM_S390_VM_CPU_MODEL:
1351 switch (attr->attr) {
1352 case KVM_S390_VM_CPU_PROCESSOR:
1353 case KVM_S390_VM_CPU_MACHINE:
15c9705f
DH		 1354 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1355 case KVM_S390_VM_CPU_MACHINE_FEAT:
0a763c78 1356 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
658b6eda
MM		 1357 ret = 0;
1358 break;
0a763c78
DH		 1359 /* configuring subfunctions is not supported yet */
1360 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
658b6eda
MM		 1361 default:
1362 ret = -ENXIO;
1363 break;
1364 }
1365 break;
a374e892
TK		 1366 case KVM_S390_VM_CRYPTO:
1367 switch (attr->attr) {
1368 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1369 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1370 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1371 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1372 ret = 0;
1373 break;
1374 default:
1375 ret = -ENXIO;
1376 break;
1377 }
1378 break;
190df4a2
CI		 1379 case KVM_S390_VM_MIGRATION:
1380 ret = 0;
1381 break;
f2061656
DD		 1382 default:
1383 ret = -ENXIO;
1384 break;
1385 }
1386
1387 return ret;
1388}
1389
30ee2a98
JH		 1390static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1391{
1392 uint8_t *keys;
1393 uint64_t hva;
4f899147 1394 int srcu_idx, i, r = 0;
30ee2a98
JH		 1395
1396 if (args->flags != 0)
1397 return -EINVAL;
1398
1399 /* Is this guest using storage keys? */
1400 if (!mm_use_skey(current->mm))
1401 return KVM_S390_GET_SKEYS_NONE;
1402
1403 /* Enforce sane limit on memory allocation */
1404 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1405 return -EINVAL;
1406
752ade68 1407 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
30ee2a98
JH		 1408 if (!keys)
1409 return -ENOMEM;
1410
d3ed1cee 1411 down_read(&current->mm->mmap_sem);
4f899147 1412 srcu_idx = srcu_read_lock(&kvm->srcu);
30ee2a98
JH		 1413 for (i = 0; i < args->count; i++) {
1414 hva = gfn_to_hva(kvm, args->start_gfn + i);
1415 if (kvm_is_error_hva(hva)) {
1416 r = -EFAULT;
d3ed1cee 1417 break;
30ee2a98
JH		 1418 }
1419
154c8c19
DH		 1420 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1421 if (r)
d3ed1cee 1422 break;
30ee2a98 1423 }
4f899147 1424 srcu_read_unlock(&kvm->srcu, srcu_idx);
d3ed1cee
MS		 1425 up_read(&current->mm->mmap_sem);
1426
1427 if (!r) {
1428 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1429 sizeof(uint8_t) * args->count);
1430 if (r)
1431 r = -EFAULT;
30ee2a98
JH		 1432 }
1433
30ee2a98
JH		 1434 kvfree(keys);
1435 return r;
1436}
1437
1438static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1439{
1440 uint8_t *keys;
1441 uint64_t hva;
4f899147 1442 int srcu_idx, i, r = 0;
30ee2a98
JH		 1443
1444 if (args->flags != 0)
1445 return -EINVAL;
1446
1447 /* Enforce sane limit on memory allocation */
1448 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1449 return -EINVAL;
1450
752ade68 1451 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
30ee2a98
JH		 1452 if (!keys)
1453 return -ENOMEM;
1454
1455 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1456 sizeof(uint8_t) * args->count);
1457 if (r) {
1458 r = -EFAULT;
1459 goto out;
1460 }
1461
1462 /* Enable storage key handling for the guest */
14d4a425
DD		 1463 r = s390_enable_skey();
1464 if (r)
1465 goto out;
30ee2a98 1466
d3ed1cee 1467 down_read(&current->mm->mmap_sem);
4f899147 1468 srcu_idx = srcu_read_lock(&kvm->srcu);
30ee2a98
JH		 1469 for (i = 0; i < args->count; i++) {
1470 hva = gfn_to_hva(kvm, args->start_gfn + i);
1471 if (kvm_is_error_hva(hva)) {
1472 r = -EFAULT;
d3ed1cee 1473 break;
30ee2a98
JH		 1474 }
1475
1476 /* Lowest order bit is reserved */
1477 if (keys[i] & 0x01) {
1478 r = -EINVAL;
d3ed1cee 1479 break;
30ee2a98
JH		 1480 }
1481
fe69eabf 1482 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
30ee2a98 1483 if (r)
d3ed1cee 1484 break;
30ee2a98 1485 }
4f899147 1486 srcu_read_unlock(&kvm->srcu, srcu_idx);
d3ed1cee 1487 up_read(&current->mm->mmap_sem);
30ee2a98
JH		 1488out:
1489 kvfree(keys);
1490 return r;
1491}
1492
4036e387
CI		 1493/*
1494 * Base address and length must be sent at the start of each block, therefore
1495 * it's cheaper to send some clean data, as long as it's less than the size of
1496 * two longs.
1497 */
1498#define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
1499/* for consistency */
1500#define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1501
1502/*
1503 * This function searches for the next page with dirty CMMA attributes, and
1504 * saves the attributes in the buffer up to either the end of the buffer or
1505 * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
1506 * no trailing clean bytes are saved.
1507 * In case no dirty bits were found, or if CMMA was not enabled or used, the
1508 * output buffer will indicate 0 as length.
1509 */
1510static int kvm_s390_get_cmma_bits(struct kvm *kvm,
1511 struct kvm_s390_cmma_log *args)
1512{
1513 struct kvm_s390_migration_state *s = kvm->arch.migration_state;
1514 unsigned long bufsize, hva, pgstev, i, next, cur;
1515 int srcu_idx, peek, r = 0, rr;
1516 u8 *res;
1517
1518 cur = args->start_gfn;
1519 i = next = pgstev = 0;
1520
1521 if (unlikely(!kvm->arch.use_cmma))
1522 return -ENXIO;
1523 /* Invalid/unsupported flags were specified */
1524 if (args->flags & ~KVM_S390_CMMA_PEEK)
1525 return -EINVAL;
1526 /* Migration mode query, and we are not doing a migration */
1527 peek = !!(args->flags & KVM_S390_CMMA_PEEK);
1528 if (!peek && !s)
1529 return -EINVAL;
1530 /* CMMA is disabled or was not used, or the buffer has length zero */
1531 bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
1532 if (!bufsize || !kvm->mm->context.use_cmma) {
1533 memset(args, 0, sizeof(*args));
1534 return 0;
1535 }
1536
1537 if (!peek) {
1538 /* We are not peeking, and there are no dirty pages */
1539 if (!atomic64_read(&s->dirty_pages)) {
1540 memset(args, 0, sizeof(*args));
1541 return 0;
1542 }
1543 cur = find_next_bit(s->pgste_bitmap, s->bitmap_size,
1544 args->start_gfn);
1545 if (cur >= s->bitmap_size) /* nothing found, loop back */
1546 cur = find_next_bit(s->pgste_bitmap, s->bitmap_size, 0);
1547 if (cur >= s->bitmap_size) { /* again! (very unlikely) */
1548 memset(args, 0, sizeof(*args));
1549 return 0;
1550 }
1551 next = find_next_bit(s->pgste_bitmap, s->bitmap_size, cur + 1);
1552 }
1553
1554 res = vmalloc(bufsize);
1555 if (!res)
1556 return -ENOMEM;
1557
1558 args->start_gfn = cur;
1559
1560 down_read(&kvm->mm->mmap_sem);
1561 srcu_idx = srcu_read_lock(&kvm->srcu);
1562 while (i < bufsize) {
1563 hva = gfn_to_hva(kvm, cur);
1564 if (kvm_is_error_hva(hva)) {
1565 r = -EFAULT;
1566 break;
1567 }
1568 /* decrement only if we actually flipped the bit to 0 */
1569 if (!peek && test_and_clear_bit(cur, s->pgste_bitmap))
1570 atomic64_dec(&s->dirty_pages);
1571 r = get_pgste(kvm->mm, hva, &pgstev);
1572 if (r < 0)
1573 pgstev = 0;
1574 /* save the value */
1bab1c02 1575 res[i++] = (pgstev >> 24) & 0x43;
4036e387
CI		 1576 /*
1577 * if the next bit is too far away, stop.
1578 * if we reached the previous "next", find the next one
1579 */
1580 if (!peek) {
1581 if (next > cur + KVM_S390_MAX_BIT_DISTANCE)
1582 break;
1583 if (cur == next)
1584 next = find_next_bit(s->pgste_bitmap,
1585 s->bitmap_size, cur + 1);
1586 /* reached the end of the bitmap or of the buffer, stop */
1587 if ((next >= s->bitmap_size) ||
1588 (next >= args->start_gfn + bufsize))
1589 break;
1590 }
1591 cur++;
1592 }
1593 srcu_read_unlock(&kvm->srcu, srcu_idx);
1594 up_read(&kvm->mm->mmap_sem);
1595 args->count = i;
1596 args->remaining = s ? atomic64_read(&s->dirty_pages) : 0;
1597
1598 rr = copy_to_user((void __user *)args->values, res, args->count);
1599 if (rr)
1600 r = -EFAULT;
1601
1602 vfree(res);
1603 return r;
1604}
1605
1606/*
1607 * This function sets the CMMA attributes for the given pages. If the input
1608 * buffer has zero length, no action is taken, otherwise the attributes are
1609 * set and the mm->context.use_cmma flag is set.
1610 */
1611static int kvm_s390_set_cmma_bits(struct kvm *kvm,
1612 const struct kvm_s390_cmma_log *args)
1613{
1614 unsigned long hva, mask, pgstev, i;
1615 uint8_t *bits;
1616 int srcu_idx, r = 0;
1617
1618 mask = args->mask;
1619
1620 if (!kvm->arch.use_cmma)
1621 return -ENXIO;
1622 /* invalid/unsupported flags */
1623 if (args->flags != 0)
1624 return -EINVAL;
1625 /* Enforce sane limit on memory allocation */
1626 if (args->count > KVM_S390_CMMA_SIZE_MAX)
1627 return -EINVAL;
1628 /* Nothing to do */
1629 if (args->count == 0)
1630 return 0;
1631
1632 bits = vmalloc(sizeof(*bits) * args->count);
1633 if (!bits)
1634 return -ENOMEM;
1635
1636 r = copy_from_user(bits, (void __user *)args->values, args->count);
1637 if (r) {
1638 r = -EFAULT;
1639 goto out;
1640 }
1641
1642 down_read(&kvm->mm->mmap_sem);
1643 srcu_idx = srcu_read_lock(&kvm->srcu);
1644 for (i = 0; i < args->count; i++) {
1645 hva = gfn_to_hva(kvm, args->start_gfn + i);
1646 if (kvm_is_error_hva(hva)) {
1647 r = -EFAULT;
1648 break;
1649 }
1650
1651 pgstev = bits[i];
1652 pgstev = pgstev << 24;
1bab1c02 1653 mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
4036e387
CI		 1654 set_pgste_bits(kvm->mm, hva, mask, pgstev);
1655 }
1656 srcu_read_unlock(&kvm->srcu, srcu_idx);
1657 up_read(&kvm->mm->mmap_sem);
1658
1659 if (!kvm->mm->context.use_cmma) {
1660 down_write(&kvm->mm->mmap_sem);
1661 kvm->mm->context.use_cmma = 1;
1662 up_write(&kvm->mm->mmap_sem);
1663 }
1664out:
1665 vfree(bits);
1666 return r;
1667}
1668
b0c632db
HC		 1669long kvm_arch_vm_ioctl(struct file *filp,
1670 unsigned int ioctl, unsigned long arg)
1671{
1672 struct kvm *kvm = filp->private_data;
1673 void __user *argp = (void __user *)arg;
f2061656 1674 struct kvm_device_attr attr;
b0c632db
HC		 1675 int r;
1676
1677 switch (ioctl) {
ba5c1e9b
CO		 1678 case KVM_S390_INTERRUPT: {
1679 struct kvm_s390_interrupt s390int;
1680
1681 r = -EFAULT;
1682 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1683 break;
1684 r = kvm_s390_inject_vm(kvm, &s390int);
1685 break;
1686 }
d938dc55
CH		 1687 case KVM_ENABLE_CAP: {
1688 struct kvm_enable_cap cap;
1689 r = -EFAULT;
1690 if (copy_from_user(&cap, argp, sizeof(cap)))
1691 break;
1692 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1693 break;
1694 }
84223598
CH		 1695 case KVM_CREATE_IRQCHIP: {
1696 struct kvm_irq_routing_entry routing;
1697
1698 r = -EINVAL;
1699 if (kvm->arch.use_irqchip) {
1700 /* Set up dummy routing. */
1701 memset(&routing, 0, sizeof(routing));
152b2839 1702 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
84223598
CH		 1703 }
1704 break;
1705 }
f2061656
DD		 1706 case KVM_SET_DEVICE_ATTR: {
1707 r = -EFAULT;
1708 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1709 break;
1710 r = kvm_s390_vm_set_attr(kvm, &attr);
1711 break;
1712 }
1713 case KVM_GET_DEVICE_ATTR: {
1714 r = -EFAULT;
1715 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1716 break;
1717 r = kvm_s390_vm_get_attr(kvm, &attr);
1718 break;
1719 }
1720 case KVM_HAS_DEVICE_ATTR: {
1721 r = -EFAULT;
1722 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1723 break;
1724 r = kvm_s390_vm_has_attr(kvm, &attr);
1725 break;
1726 }
30ee2a98
JH		 1727 case KVM_S390_GET_SKEYS: {
1728 struct kvm_s390_skeys args;
1729
1730 r = -EFAULT;
1731 if (copy_from_user(&args, argp,
1732 sizeof(struct kvm_s390_skeys)))
1733 break;
1734 r = kvm_s390_get_skeys(kvm, &args);
1735 break;
1736 }
1737 case KVM_S390_SET_SKEYS: {
1738 struct kvm_s390_skeys args;
1739
1740 r = -EFAULT;
1741 if (copy_from_user(&args, argp,
1742 sizeof(struct kvm_s390_skeys)))
1743 break;
1744 r = kvm_s390_set_skeys(kvm, &args);
1745 break;
1746 }
4036e387
CI		 1747 case KVM_S390_GET_CMMA_BITS: {
1748 struct kvm_s390_cmma_log args;
1749
1750 r = -EFAULT;
1751 if (copy_from_user(&args, argp, sizeof(args)))
1752 break;
1753 r = kvm_s390_get_cmma_bits(kvm, &args);
1754 if (!r) {
1755 r = copy_to_user(argp, &args, sizeof(args));
1756 if (r)
1757 r = -EFAULT;
1758 }
1759 break;
1760 }
1761 case KVM_S390_SET_CMMA_BITS: {
1762 struct kvm_s390_cmma_log args;
1763
1764 r = -EFAULT;
1765 if (copy_from_user(&args, argp, sizeof(args)))
1766 break;
1767 r = kvm_s390_set_cmma_bits(kvm, &args);
1768 break;
1769 }
b0c632db 1770 default:
367e1319 1771 r = -ENOTTY;
b0c632db
HC		 1772 }
1773
1774 return r;
1775}
1776
45c9b47c
TK		 1777static int kvm_s390_query_ap_config(u8 *config)
1778{
1779 u32 fcn_code = 0x04000000UL;
86044c8c 1780 u32 cc = 0;
45c9b47c 1781
86044c8c 1782 memset(config, 0, 128);
45c9b47c
TK		 1783 asm volatile(
1784 "lgr 0,%1\n"
1785 "lgr 2,%2\n"
1786 ".long 0xb2af0000\n" /* PQAP(QCI) */
86044c8c 1787 "0: ipm %0\n"
45c9b47c 1788 "srl %0,28\n"
86044c8c
CB		 1789 "1:\n"
1790 EX_TABLE(0b, 1b)
1791 : "+r" (cc)
45c9b47c
TK		 1792 : "r" (fcn_code), "r" (config)
1793 : "cc", "0", "2", "memory"
1794 );
1795
1796 return cc;
1797}
1798
1799static int kvm_s390_apxa_installed(void)
1800{
1801 u8 config[128];
1802 int cc;
1803
a6aacc3f 1804 if (test_facility(12)) {
45c9b47c
TK		 1805 cc = kvm_s390_query_ap_config(config);
1806
1807 if (cc)
1808 pr_err("PQAP(QCI) failed with cc=%d", cc);
1809 else
1810 return config[0] & 0x40;
1811 }
1812
1813 return 0;
1814}
1815
1816static void kvm_s390_set_crycb_format(struct kvm *kvm)
1817{
1818 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1819
1820 if (kvm_s390_apxa_installed())
1821 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1822 else
1823 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1824}
1825
9bb0ec09 1826static u64 kvm_s390_get_initial_cpuid(void)
9d8d5786 1827{
9bb0ec09
DH		 1828 struct cpuid cpuid;
1829
1830 get_cpu_id(&cpuid);
1831 cpuid.version = 0xff;
1832 return *((u64 *) &cpuid);
9d8d5786
MM		 1833}
1834
c54f0d6a 1835static void kvm_s390_crypto_init(struct kvm *kvm)
5102ee87 1836{
9d8d5786 1837 if (!test_kvm_facility(kvm, 76))
c54f0d6a 1838 return;
5102ee87 1839
c54f0d6a 1840 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
45c9b47c 1841 kvm_s390_set_crycb_format(kvm);
5102ee87 1842
ed6f76b4
TK		 1843 /* Enable AES/DEA protected key functions by default */
1844 kvm->arch.crypto.aes_kw = 1;
1845 kvm->arch.crypto.dea_kw = 1;
1846 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1847 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1848 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1849 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
5102ee87
TK		 1850}
1851
7d43bafc
ED		 1852static void sca_dispose(struct kvm *kvm)
1853{
1854 if (kvm->arch.use_esca)
5e044315 1855 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
7d43bafc
ED		 1856 else
1857 free_page((unsigned long)(kvm->arch.sca));
1858 kvm->arch.sca = NULL;
1859}
1860
e08b9637 1861int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 1862{
76a6dd72 1863 gfp_t alloc_flags = GFP_KERNEL;
9d8d5786 1864 int i, rc;
b0c632db 1865 char debug_name[16];
f6c137ff 1866 static unsigned long sca_offset;
b0c632db 1867
e08b9637
CO		 1868 rc = -EINVAL;
1869#ifdef CONFIG_KVM_S390_UCONTROL
1870 if (type & ~KVM_VM_S390_UCONTROL)
1871 goto out_err;
1872 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1873 goto out_err;
1874#else
1875 if (type)
1876 goto out_err;
1877#endif
1878
b0c632db
HC		 1879 rc = s390_enable_sie();
1880 if (rc)
d89f5eff 1881 goto out_err;
b0c632db 1882
b290411a
CO		 1883 rc = -ENOMEM;
1884
7d43bafc 1885 kvm->arch.use_esca = 0; /* start with basic SCA */
76a6dd72
DH		 1886 if (!sclp.has_64bscao)
1887 alloc_flags |= GFP_DMA;
5e044315 1888 rwlock_init(&kvm->arch.sca_lock);
76a6dd72 1889 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
b0c632db 1890 if (!kvm->arch.sca)
d89f5eff 1891 goto out_err;
f6c137ff 1892 spin_lock(&kvm_lock);
c5c2c393 1893 sca_offset += 16;
bc784cce 1894 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
c5c2c393 1895 sca_offset = 0;
bc784cce
ED		 1896 kvm->arch.sca = (struct bsca_block *)
1897 ((char *) kvm->arch.sca + sca_offset);
f6c137ff 1898 spin_unlock(&kvm_lock);
b0c632db
HC		 1899
1900 sprintf(debug_name, "kvm-%u", current->pid);
1901
1cb9cf72 1902 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
b0c632db 1903 if (!kvm->arch.dbf)
40f5b735 1904 goto out_err;
b0c632db 1905
c54f0d6a
DH		 1906 kvm->arch.sie_page2 =
1907 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1908 if (!kvm->arch.sie_page2)
40f5b735 1909 goto out_err;
9d8d5786 1910
fb5bf93f 1911 /* Populate the facility mask initially. */
c54f0d6a 1912 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
04478197 1913 sizeof(S390_lowcore.stfle_fac_list));
9d8d5786
MM		 1914 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1915 if (i < kvm_s390_fac_list_mask_size())
c54f0d6a 1916 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
9d8d5786 1917 else
c54f0d6a 1918 kvm->arch.model.fac_mask[i] = 0UL;
9d8d5786
MM		 1919 }
1920
981467c9 1921 /* Populate the facility list initially. */
c54f0d6a
DH		 1922 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1923 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
981467c9
MM		 1924 S390_ARCH_FAC_LIST_SIZE_BYTE);
1925
1935222d
DH		 1926 /* we are always in czam mode - even on pre z14 machines */
1927 set_kvm_facility(kvm->arch.model.fac_mask, 138);
1928 set_kvm_facility(kvm->arch.model.fac_list, 138);
1929 /* we emulate STHYI in kvm */
95ca2cb5
JF		 1930 set_kvm_facility(kvm->arch.model.fac_mask, 74);
1931 set_kvm_facility(kvm->arch.model.fac_list, 74);
1bab1c02
CI		 1932 if (MACHINE_HAS_TLB_GUEST) {
1933 set_kvm_facility(kvm->arch.model.fac_mask, 147);
1934 set_kvm_facility(kvm->arch.model.fac_list, 147);
1935 }
95ca2cb5 1936
9bb0ec09 1937 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
37c5f6c8 1938 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
9d8d5786 1939
c54f0d6a 1940 kvm_s390_crypto_init(kvm);
5102ee87 1941
51978393
FL		 1942 mutex_init(&kvm->arch.float_int.ais_lock);
1943 kvm->arch.float_int.simm = 0;
1944 kvm->arch.float_int.nimm = 0;
ba5c1e9b 1945 spin_lock_init(&kvm->arch.float_int.lock);
6d3da241
JF		 1946 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1947 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
8a242234 1948 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 1949 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 1950
b0c632db 1951 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
78f26131 1952 VM_EVENT(kvm, 3, "vm created with type %lu", type);
b0c632db 1953
e08b9637
CO		 1954 if (type & KVM_VM_S390_UCONTROL) {
1955 kvm->arch.gmap = NULL;
a3a92c31 1956 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
e08b9637 1957 } else {
32e6b236 1958 if (sclp.hamax == U64_MAX)
ee71d16d 1959 kvm->arch.mem_limit = TASK_SIZE_MAX;
32e6b236 1960 else
ee71d16d 1961 kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
32e6b236 1962 sclp.hamax + 1);
6ea427bb 1963 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
e08b9637 1964 if (!kvm->arch.gmap)
40f5b735 1965 goto out_err;
2c70fe44 1966 kvm->arch.gmap->private = kvm;
24eb3a82 1967 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 1968 }
fa6b7fe9
CH		 1969
1970 kvm->arch.css_support = 0;
84223598 1971 kvm->arch.use_irqchip = 0;
72f25020 1972 kvm->arch.epoch = 0;
fa6b7fe9 1973
8ad35755 1974 spin_lock_init(&kvm->arch.start_stop_lock);
a3508fbe 1975 kvm_s390_vsie_init(kvm);
8335713a 1976 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
8ad35755 1977
d89f5eff 1978 return 0;
40f5b735 1979out_err:
c54f0d6a 1980 free_page((unsigned long)kvm->arch.sie_page2);
598841ca 1981 debug_unregister(kvm->arch.dbf);
7d43bafc 1982 sca_dispose(kvm);
78f26131 1983 KVM_EVENT(3, "creation of vm failed: %d", rc);
d89f5eff 1984 return rc;
b0c632db
HC		 1985}
1986
235539b4
LC		 1987bool kvm_arch_has_vcpu_debugfs(void)
1988{
1989 return false;
1990}
1991
1992int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
1993{
1994 return 0;
1995}
1996
d329c035
CB		 1997void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1998{
1999 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 2000 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 2001 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 2002 kvm_clear_async_pf_completion_queue(vcpu);
bc784cce 2003 if (!kvm_is_ucontrol(vcpu->kvm))
a6e2f683 2004 sca_del_vcpu(vcpu);
27e0393f
CO		 2005
2006 if (kvm_is_ucontrol(vcpu->kvm))
6ea427bb 2007 gmap_remove(vcpu->arch.gmap);
27e0393f 2008
e6db1d61 2009 if (vcpu->kvm->arch.use_cmma)
b31605c1 2010 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 2011 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 2012
6692cef3 2013 kvm_vcpu_uninit(vcpu);
b110feaf 2014 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 2015}
2016
2017static void kvm_free_vcpus(struct kvm *kvm)
2018{
2019 unsigned int i;
988a2cae 2020 struct kvm_vcpu *vcpu;
d329c035 2021
988a2cae
GN		 2022 kvm_for_each_vcpu(i, vcpu, kvm)
2023 kvm_arch_vcpu_destroy(vcpu);
2024
2025 mutex_lock(&kvm->lock);
2026 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
2027 kvm->vcpus[i] = NULL;
2028
2029 atomic_set(&kvm->online_vcpus, 0);
2030 mutex_unlock(&kvm->lock);
d329c035
CB		 2031}
2032
b0c632db
HC		 2033void kvm_arch_destroy_vm(struct kvm *kvm)
2034{
d329c035 2035 kvm_free_vcpus(kvm);
7d43bafc 2036 sca_dispose(kvm);
d329c035 2037 debug_unregister(kvm->arch.dbf);
c54f0d6a 2038 free_page((unsigned long)kvm->arch.sie_page2);
27e0393f 2039 if (!kvm_is_ucontrol(kvm))
6ea427bb 2040 gmap_remove(kvm->arch.gmap);
841b91c5 2041 kvm_s390_destroy_adapters(kvm);
67335e63 2042 kvm_s390_clear_float_irqs(kvm);
a3508fbe 2043 kvm_s390_vsie_destroy(kvm);
190df4a2
CI		 2044 if (kvm->arch.migration_state) {
2045 vfree(kvm->arch.migration_state->pgste_bitmap);
2046 kfree(kvm->arch.migration_state);
2047 }
8335713a 2048 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
b0c632db
HC		 2049}
2050
2051/* Section: vcpu related */
dafd032a
DD		 2052static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
2053{
6ea427bb 2054 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
dafd032a
DD		 2055 if (!vcpu->arch.gmap)
2056 return -ENOMEM;
2057 vcpu->arch.gmap->private = vcpu->kvm;
2058
2059 return 0;
2060}
2061
a6e2f683
ED		 2062static void sca_del_vcpu(struct kvm_vcpu *vcpu)
2063{
a6940674
DH		 2064 if (!kvm_s390_use_sca_entries())
2065 return;
5e044315 2066 read_lock(&vcpu->kvm->arch.sca_lock);
7d43bafc
ED		 2067 if (vcpu->kvm->arch.use_esca) {
2068 struct esca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 2069
7d43bafc 2070 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
10ce32d5 2071 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc
ED		 2072 } else {
2073 struct bsca_block *sca = vcpu->kvm->arch.sca;
2074
2075 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
10ce32d5 2076 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc 2077 }
5e044315 2078 read_unlock(&vcpu->kvm->arch.sca_lock);
a6e2f683
ED		 2079}
2080
eaa78f34 2081static void sca_add_vcpu(struct kvm_vcpu *vcpu)
a6e2f683 2082{
a6940674
DH		 2083 if (!kvm_s390_use_sca_entries()) {
2084 struct bsca_block *sca = vcpu->kvm->arch.sca;
2085
2086 /* we still need the basic sca for the ipte control */
2087 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2088 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2089 }
eaa78f34
DH		 2090 read_lock(&vcpu->kvm->arch.sca_lock);
2091 if (vcpu->kvm->arch.use_esca) {
2092 struct esca_block *sca = vcpu->kvm->arch.sca;
7d43bafc 2093
eaa78f34 2094 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 2095 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2096 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
0c9d8683 2097 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
eaa78f34 2098 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
7d43bafc 2099 } else {
eaa78f34 2100 struct bsca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 2101
eaa78f34 2102 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 2103 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2104 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
eaa78f34 2105 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
7d43bafc 2106 }
eaa78f34 2107 read_unlock(&vcpu->kvm->arch.sca_lock);
5e044315
ED		 2108}
2109
2110/* Basic SCA to Extended SCA data copy routines */
2111static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
2112{
2113 d->sda = s->sda;
2114 d->sigp_ctrl.c = s->sigp_ctrl.c;
2115 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
2116}
2117
2118static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
2119{
2120 int i;
2121
2122 d->ipte_control = s->ipte_control;
2123 d->mcn[0] = s->mcn;
2124 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
2125 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
2126}
2127
2128static int sca_switch_to_extended(struct kvm *kvm)
2129{
2130 struct bsca_block *old_sca = kvm->arch.sca;
2131 struct esca_block *new_sca;
2132 struct kvm_vcpu *vcpu;
2133 unsigned int vcpu_idx;
2134 u32 scaol, scaoh;
2135
2136 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
2137 if (!new_sca)
2138 return -ENOMEM;
2139
2140 scaoh = (u32)((u64)(new_sca) >> 32);
2141 scaol = (u32)(u64)(new_sca) & ~0x3fU;
2142
2143 kvm_s390_vcpu_block_all(kvm);
2144 write_lock(&kvm->arch.sca_lock);
2145
2146 sca_copy_b_to_e(new_sca, old_sca);
2147
2148 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
2149 vcpu->arch.sie_block->scaoh = scaoh;
2150 vcpu->arch.sie_block->scaol = scaol;
0c9d8683 2151 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
5e044315
ED		 2152 }
2153 kvm->arch.sca = new_sca;
2154 kvm->arch.use_esca = 1;
2155
2156 write_unlock(&kvm->arch.sca_lock);
2157 kvm_s390_vcpu_unblock_all(kvm);
2158
2159 free_page((unsigned long)old_sca);
2160
8335713a
CB		 2161 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
2162 old_sca, kvm->arch.sca);
5e044315 2163 return 0;
a6e2f683
ED		 2164}
2165
2166static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
2167{
5e044315
ED		 2168 int rc;
2169
a6940674
DH		 2170 if (!kvm_s390_use_sca_entries()) {
2171 if (id < KVM_MAX_VCPUS)
2172 return true;
2173 return false;
2174 }
5e044315
ED		 2175 if (id < KVM_S390_BSCA_CPU_SLOTS)
2176 return true;
76a6dd72 2177 if (!sclp.has_esca || !sclp.has_64bscao)
5e044315
ED		 2178 return false;
2179
2180 mutex_lock(&kvm->lock);
2181 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
2182 mutex_unlock(&kvm->lock);
2183
2184 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
a6e2f683
ED		 2185}
2186
b0c632db
HC		 2187int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
2188{
3c038e6b
DD		 2189 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2190 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 2191 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
2192 KVM_SYNC_GPRS |
9eed0735 2193 KVM_SYNC_ACRS |
b028ee3e
DH		 2194 KVM_SYNC_CRS |
2195 KVM_SYNC_ARCH0 |
2196 KVM_SYNC_PFAULT;
75a4615c 2197 kvm_s390_set_prefix(vcpu, 0);
c6e5f166
FZ		 2198 if (test_kvm_facility(vcpu->kvm, 64))
2199 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
4e0b1ab7
FZ		 2200 if (test_kvm_facility(vcpu->kvm, 133))
2201 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
f6aa6dc4
DH		 2202 /* fprs can be synchronized via vrs, even if the guest has no vx. With
2203 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
2204 */
2205 if (MACHINE_HAS_VX)
68c55750 2206 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
6fd8e67d
DH		 2207 else
2208 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
dafd032a
DD		 2209
2210 if (kvm_is_ucontrol(vcpu->kvm))
2211 return __kvm_ucontrol_vcpu_init(vcpu);
2212
b0c632db
HC		 2213 return 0;
2214}
2215
db0758b2
DH		 2216/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2217static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2218{
2219 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
9c23a131 2220 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2 2221 vcpu->arch.cputm_start = get_tod_clock_fast();
9c23a131 2222 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 2223}
2224
2225/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2226static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2227{
2228 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
9c23a131 2229 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 2230 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2231 vcpu->arch.cputm_start = 0;
9c23a131 2232 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 2233}
2234
2235/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2236static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2237{
2238 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
2239 vcpu->arch.cputm_enabled = true;
2240 __start_cpu_timer_accounting(vcpu);
2241}
2242
2243/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2244static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2245{
2246 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
2247 __stop_cpu_timer_accounting(vcpu);
2248 vcpu->arch.cputm_enabled = false;
2249}
2250
2251static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2252{
2253 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2254 __enable_cpu_timer_accounting(vcpu);
2255 preempt_enable();
2256}
2257
2258static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2259{
2260 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2261 __disable_cpu_timer_accounting(vcpu);
2262 preempt_enable();
2263}
2264
4287f247
DH		 2265/* set the cpu timer - may only be called from the VCPU thread itself */
2266void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
2267{
db0758b2 2268 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
9c23a131 2269 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 2270 if (vcpu->arch.cputm_enabled)
2271 vcpu->arch.cputm_start = get_tod_clock_fast();
4287f247 2272 vcpu->arch.sie_block->cputm = cputm;
9c23a131 2273 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2 2274 preempt_enable();
4287f247
DH		 2275}
2276
db0758b2 2277/* update and get the cpu timer - can also be called from other VCPU threads */
4287f247
DH		 2278__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
2279{
9c23a131 2280 unsigned int seq;
db0758b2 2281 __u64 value;
db0758b2
DH		 2282
2283 if (unlikely(!vcpu->arch.cputm_enabled))
2284 return vcpu->arch.sie_block->cputm;
2285
9c23a131
DH		 2286 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2287 do {
2288 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
2289 /*
2290 * If the writer would ever execute a read in the critical
2291 * section, e.g. in irq context, we have a deadlock.
2292 */
2293 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
2294 value = vcpu->arch.sie_block->cputm;
2295 /* if cputm_start is 0, accounting is being started/stopped */
2296 if (likely(vcpu->arch.cputm_start))
2297 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2298 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
2299 preempt_enable();
db0758b2 2300 return value;
4287f247
DH		 2301}
2302
b0c632db
HC		 2303void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2304{
9977e886 2305
37d9df98 2306 gmap_enable(vcpu->arch.enabled_gmap);
805de8f4 2307 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
5ebda316 2308 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 2309 __start_cpu_timer_accounting(vcpu);
01a745ac 2310 vcpu->cpu = cpu;
b0c632db
HC		 2311}
2312
2313void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
2314{
01a745ac 2315 vcpu->cpu = -1;
5ebda316 2316 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 2317 __stop_cpu_timer_accounting(vcpu);
805de8f4 2318 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
37d9df98
DH		 2319 vcpu->arch.enabled_gmap = gmap_get_enabled();
2320 gmap_disable(vcpu->arch.enabled_gmap);
9977e886 2321
b0c632db
HC		 2322}
2323
2324static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
2325{
2326 /* this equals initial cpu reset in pop, but we don't switch to ESA */
2327 vcpu->arch.sie_block->gpsw.mask = 0UL;
2328 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 2329 kvm_s390_set_prefix(vcpu, 0);
4287f247 2330 kvm_s390_set_cpu_timer(vcpu, 0);
b0c632db
HC		 2331 vcpu->arch.sie_block->ckc = 0UL;
2332 vcpu->arch.sie_block->todpr = 0;
2333 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
2334 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
2335 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
9abc2a08
DH		 2336 /* make sure the new fpc will be lazily loaded */
2337 save_fpu_regs();
2338 current->thread.fpu.fpc = 0;
b0c632db 2339 vcpu->arch.sie_block->gbea = 1;
672550fb 2340 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 2341 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2342 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 2343 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
2344 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 2345 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 2346}
2347
31928aa5 2348void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 2349{
72f25020 2350 mutex_lock(&vcpu->kvm->lock);
fdf03650 2351 preempt_disable();
72f25020 2352 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
fdf03650 2353 preempt_enable();
72f25020 2354 mutex_unlock(&vcpu->kvm->lock);
25508824 2355 if (!kvm_is_ucontrol(vcpu->kvm)) {
dafd032a 2356 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
eaa78f34 2357 sca_add_vcpu(vcpu);
25508824 2358 }
6502a34c
DH		 2359 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
2360 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
37d9df98
DH		 2361 /* make vcpu_load load the right gmap on the first trigger */
2362 vcpu->arch.enabled_gmap = vcpu->arch.gmap;
42897d86
MT		 2363}
2364
5102ee87
TK		 2365static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
2366{
9d8d5786 2367 if (!test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 2368 return;
2369
a374e892
TK		 2370 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
2371
2372 if (vcpu->kvm->arch.crypto.aes_kw)
2373 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
2374 if (vcpu->kvm->arch.crypto.dea_kw)
2375 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
2376
5102ee87
TK		 2377 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
2378}
2379
b31605c1
DD		 2380void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
2381{
2382 free_page(vcpu->arch.sie_block->cbrlo);
2383 vcpu->arch.sie_block->cbrlo = 0;
2384}
2385
2386int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
2387{
2388 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
2389 if (!vcpu->arch.sie_block->cbrlo)
2390 return -ENOMEM;
2391
0c9d8683 2392 vcpu->arch.sie_block->ecb2 &= ~ECB2_PFMFI;
b31605c1
DD		 2393 return 0;
2394}
2395
91520f1a
MM		 2396static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
2397{
2398 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
2399
91520f1a 2400 vcpu->arch.sie_block->ibc = model->ibc;
80bc79dc 2401 if (test_kvm_facility(vcpu->kvm, 7))
c54f0d6a 2402 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
91520f1a
MM		 2403}
2404
b0c632db
HC		 2405int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
2406{
b31605c1 2407 int rc = 0;
b31288fa 2408
9e6dabef
CH		 2409 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
2410 CPUSTAT_SM |
a4a4f191
GH		 2411 CPUSTAT_STOPPED);
2412
53df84f8 2413 if (test_kvm_facility(vcpu->kvm, 78))
805de8f4 2414 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
53df84f8 2415 else if (test_kvm_facility(vcpu->kvm, 8))
805de8f4 2416 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
a4a4f191 2417
91520f1a
MM		 2418 kvm_s390_vcpu_setup_model(vcpu);
2419
bdab09f3
DH		 2420 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
2421 if (MACHINE_HAS_ESOP)
0c9d8683 2422 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
bd50e8ec 2423 if (test_kvm_facility(vcpu->kvm, 9))
0c9d8683 2424 vcpu->arch.sie_block->ecb |= ECB_SRSI;
f597d24e 2425 if (test_kvm_facility(vcpu->kvm, 73))
0c9d8683 2426 vcpu->arch.sie_block->ecb |= ECB_TE;
7feb6bb8 2427
873b425e 2428 if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
0c9d8683 2429 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
cd1836f5 2430 if (test_kvm_facility(vcpu->kvm, 130))
0c9d8683
DH		 2431 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
2432 vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
48ee7d3a 2433 if (sclp.has_cei)
0c9d8683 2434 vcpu->arch.sie_block->eca |= ECA_CEI;
11ad65b7 2435 if (sclp.has_ib)
0c9d8683 2436 vcpu->arch.sie_block->eca |= ECA_IB;
37c5f6c8 2437 if (sclp.has_siif)
0c9d8683 2438 vcpu->arch.sie_block->eca |= ECA_SII;
37c5f6c8 2439 if (sclp.has_sigpif)
0c9d8683 2440 vcpu->arch.sie_block->eca |= ECA_SIGPI;
18280d8b 2441 if (test_kvm_facility(vcpu->kvm, 129)) {
0c9d8683
DH		 2442 vcpu->arch.sie_block->eca |= ECA_VX;
2443 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
13211ea7 2444 }
8fa1696e
CW		 2445 if (test_kvm_facility(vcpu->kvm, 139))
2446 vcpu->arch.sie_block->ecd |= ECD_MEF;
2447
4e0b1ab7
FZ		 2448 vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
2449 | SDNXC;
c6e5f166 2450 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
730cd632
FA		 2451
2452 if (sclp.has_kss)
2453 atomic_or(CPUSTAT_KSS, &vcpu->arch.sie_block->cpuflags);
2454 else
2455 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
5a5e6536 2456
e6db1d61 2457 if (vcpu->kvm->arch.use_cmma) {
b31605c1
DD		 2458 rc = kvm_s390_vcpu_setup_cmma(vcpu);
2459 if (rc)
2460 return rc;
b31288fa 2461 }
0ac96caf 2462 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 2463 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 2464
5102ee87
TK		 2465 kvm_s390_vcpu_crypto_setup(vcpu);
2466
b31605c1 2467 return rc;
b0c632db
HC		 2468}
2469
2470struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
2471 unsigned int id)
2472{
4d47555a 2473 struct kvm_vcpu *vcpu;
7feb6bb8 2474 struct sie_page *sie_page;
4d47555a
CO		 2475 int rc = -EINVAL;
2476
4215825e 2477 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
4d47555a
CO		 2478 goto out;
2479
2480 rc = -ENOMEM;
b0c632db 2481
b110feaf 2482 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 2483 if (!vcpu)
4d47555a 2484 goto out;
b0c632db 2485
da72ca4d 2486 BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
7feb6bb8
MM		 2487 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
2488 if (!sie_page)
b0c632db
HC		 2489 goto out_free_cpu;
2490
7feb6bb8
MM		 2491 vcpu->arch.sie_block = &sie_page->sie_block;
2492 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
2493
efed1104
DH		 2494 /* the real guest size will always be smaller than msl */
2495 vcpu->arch.sie_block->mso = 0;
2496 vcpu->arch.sie_block->msl = sclp.hamax;
2497
b0c632db 2498 vcpu->arch.sie_block->icpua = id;
ba5c1e9b 2499 spin_lock_init(&vcpu->arch.local_int.lock);
ba5c1e9b 2500 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 2501 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 2502 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
9c23a131 2503 seqcount_init(&vcpu->arch.cputm_seqcount);
ba5c1e9b 2504
b0c632db
HC		 2505 rc = kvm_vcpu_init(vcpu, kvm, id);
2506 if (rc)
9abc2a08 2507 goto out_free_sie_block;
8335713a 2508 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
b0c632db 2509 vcpu->arch.sie_block);
ade38c31 2510 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 2511
b0c632db 2512 return vcpu;
7b06bf2f
WY		 2513out_free_sie_block:
2514 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 2515out_free_cpu:
b110feaf 2516 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 2517out:
b0c632db
HC		 2518 return ERR_PTR(rc);
2519}
2520
b0c632db
HC		 2521int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
2522{
9a022067 2523 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 2524}
2525
199b5763
LM		 2526bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
2527{
0546c63d 2528 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
199b5763
LM		 2529}
2530
27406cd5 2531void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
49b99e1e 2532{
805de8f4 2533 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
61a6df54 2534 exit_sie(vcpu);
49b99e1e
CB		 2535}
2536
27406cd5 2537void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
49b99e1e 2538{
805de8f4 2539 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
49b99e1e
CB		 2540}
2541
8e236546
CB		 2542static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
2543{
805de8f4 2544 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
61a6df54 2545 exit_sie(vcpu);
8e236546
CB		 2546}
2547
2548static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2549{
9bf9fde2 2550 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
8e236546
CB		 2551}
2552
49b99e1e
CB		 2553/*
2554 * Kick a guest cpu out of SIE and wait until SIE is not running.
2555 * If the CPU is not running (e.g. waiting as idle) the function will
2556 * return immediately. */
2557void exit_sie(struct kvm_vcpu *vcpu)
2558{
805de8f4 2559 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
49b99e1e
CB		 2560 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2561 cpu_relax();
2562}
2563
8e236546
CB		 2564/* Kick a guest cpu out of SIE to process a request synchronously */
2565void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
49b99e1e 2566{
8e236546
CB		 2567 kvm_make_request(req, vcpu);
2568 kvm_s390_vcpu_request(vcpu);
49b99e1e
CB		 2569}
2570
414d3b07
MS		 2571static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
2572 unsigned long end)
2c70fe44 2573{
2c70fe44
CB		 2574 struct kvm *kvm = gmap->private;
2575 struct kvm_vcpu *vcpu;
414d3b07
MS		 2576 unsigned long prefix;
2577 int i;
2c70fe44 2578
65d0b0d4
DH		 2579 if (gmap_is_shadow(gmap))
2580 return;
414d3b07
MS		 2581 if (start >= 1UL << 31)
2582 /* We are only interested in prefix pages */
2583 return;
2c70fe44
CB		 2584 kvm_for_each_vcpu(i, vcpu, kvm) {
2585 /* match against both prefix pages */
414d3b07
MS		 2586 prefix = kvm_s390_get_prefix(vcpu);
2587 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
2588 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
2589 start, end);
8e236546 2590 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44
CB		 2591 }
2592 }
2593}
2594
b6d33834
CD		 2595int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2596{
2597 /* kvm common code refers to this, but never calls it */
2598 BUG();
2599 return 0;
2600}
2601
14eebd91
CO		 2602static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2603 struct kvm_one_reg *reg)
2604{
2605 int r = -EINVAL;
2606
2607 switch (reg->id) {
29b7c71b
CO		 2608 case KVM_REG_S390_TODPR:
2609 r = put_user(vcpu->arch.sie_block->todpr,
2610 (u32 __user *)reg->addr);
2611 break;
2612 case KVM_REG_S390_EPOCHDIFF:
2613 r = put_user(vcpu->arch.sie_block->epoch,
2614 (u64 __user *)reg->addr);
2615 break;
46a6dd1c 2616 case KVM_REG_S390_CPU_TIMER:
4287f247 2617 r = put_user(kvm_s390_get_cpu_timer(vcpu),
46a6dd1c
J		 2618 (u64 __user *)reg->addr);
2619 break;
2620 case KVM_REG_S390_CLOCK_COMP:
2621 r = put_user(vcpu->arch.sie_block->ckc,
2622 (u64 __user *)reg->addr);
2623 break;
536336c2
DD		 2624 case KVM_REG_S390_PFTOKEN:
2625 r = put_user(vcpu->arch.pfault_token,
2626 (u64 __user *)reg->addr);
2627 break;
2628 case KVM_REG_S390_PFCOMPARE:
2629 r = put_user(vcpu->arch.pfault_compare,
2630 (u64 __user *)reg->addr);
2631 break;
2632 case KVM_REG_S390_PFSELECT:
2633 r = put_user(vcpu->arch.pfault_select,
2634 (u64 __user *)reg->addr);
2635 break;
672550fb
CB		 2636 case KVM_REG_S390_PP:
2637 r = put_user(vcpu->arch.sie_block->pp,
2638 (u64 __user *)reg->addr);
2639 break;
afa45ff5
CB		 2640 case KVM_REG_S390_GBEA:
2641 r = put_user(vcpu->arch.sie_block->gbea,
2642 (u64 __user *)reg->addr);
2643 break;
14eebd91
CO		 2644 default:
2645 break;
2646 }
2647
2648 return r;
2649}
2650
2651static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2652 struct kvm_one_reg *reg)
2653{
2654 int r = -EINVAL;
4287f247 2655 __u64 val;
14eebd91
CO		 2656
2657 switch (reg->id) {
29b7c71b
CO		 2658 case KVM_REG_S390_TODPR:
2659 r = get_user(vcpu->arch.sie_block->todpr,
2660 (u32 __user *)reg->addr);
2661 break;
2662 case KVM_REG_S390_EPOCHDIFF:
2663 r = get_user(vcpu->arch.sie_block->epoch,
2664 (u64 __user *)reg->addr);
2665 break;
46a6dd1c 2666 case KVM_REG_S390_CPU_TIMER:
4287f247
DH		 2667 r = get_user(val, (u64 __user *)reg->addr);
2668 if (!r)
2669 kvm_s390_set_cpu_timer(vcpu, val);
46a6dd1c
J		 2670 break;
2671 case KVM_REG_S390_CLOCK_COMP:
2672 r = get_user(vcpu->arch.sie_block->ckc,
2673 (u64 __user *)reg->addr);
2674 break;
536336c2
DD		 2675 case KVM_REG_S390_PFTOKEN:
2676 r = get_user(vcpu->arch.pfault_token,
2677 (u64 __user *)reg->addr);
9fbd8082
DH		 2678 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2679 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 2680 break;
2681 case KVM_REG_S390_PFCOMPARE:
2682 r = get_user(vcpu->arch.pfault_compare,
2683 (u64 __user *)reg->addr);
2684 break;
2685 case KVM_REG_S390_PFSELECT:
2686 r = get_user(vcpu->arch.pfault_select,
2687 (u64 __user *)reg->addr);
2688 break;
672550fb
CB		 2689 case KVM_REG_S390_PP:
2690 r = get_user(vcpu->arch.sie_block->pp,
2691 (u64 __user *)reg->addr);
2692 break;
afa45ff5
CB		 2693 case KVM_REG_S390_GBEA:
2694 r = get_user(vcpu->arch.sie_block->gbea,
2695 (u64 __user *)reg->addr);
2696 break;
14eebd91
CO		 2697 default:
2698 break;
2699 }
2700
2701 return r;
2702}
b6d33834 2703
b0c632db
HC		 2704static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2705{
b0c632db 2706 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 2707 return 0;
2708}
2709
2710int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2711{
5a32c1af 2712 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 2713 return 0;
2714}
2715
2716int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2717{
5a32c1af 2718 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 2719 return 0;
2720}
2721
2722int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2723 struct kvm_sregs *sregs)
2724{
59674c1a 2725 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 2726 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
b0c632db
HC		 2727 return 0;
2728}
2729
2730int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2731 struct kvm_sregs *sregs)
2732{
59674c1a 2733 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 2734 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 2735 return 0;
2736}
2737
2738int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2739{
4725c860
MS		 2740 if (test_fp_ctl(fpu->fpc))
2741 return -EINVAL;
e1788bb9 2742 vcpu->run->s.regs.fpc = fpu->fpc;
9abc2a08 2743 if (MACHINE_HAS_VX)
a7d4b8f2
DH		 2744 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
2745 (freg_t *) fpu->fprs);
9abc2a08 2746 else
a7d4b8f2 2747 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
b0c632db
HC		 2748 return 0;
2749}
2750
2751int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2752{
9abc2a08
DH		 2753 /* make sure we have the latest values */
2754 save_fpu_regs();
2755 if (MACHINE_HAS_VX)
a7d4b8f2
DH		 2756 convert_vx_to_fp((freg_t *) fpu->fprs,
2757 (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08 2758 else
a7d4b8f2 2759 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
e1788bb9 2760 fpu->fpc = vcpu->run->s.regs.fpc;
b0c632db
HC		 2761 return 0;
2762}
2763
2764static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2765{
2766 int rc = 0;
2767
7a42fdc2 2768 if (!is_vcpu_stopped(vcpu))
b0c632db 2769 rc = -EBUSY;
d7b0b5eb
CO		 2770 else {
2771 vcpu->run->psw_mask = psw.mask;
2772 vcpu->run->psw_addr = psw.addr;
2773 }
b0c632db
HC		 2774 return rc;
2775}
2776
2777int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2778 struct kvm_translation *tr)
2779{
2780 return -EINVAL; /* not implemented yet */
2781}
2782
27291e21
DH		 2783#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2784 KVM_GUESTDBG_USE_HW_BP | \
2785 KVM_GUESTDBG_ENABLE)
2786
d0bfb940
JK		 2787int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2788 struct kvm_guest_debug *dbg)
b0c632db 2789{
27291e21
DH		 2790 int rc = 0;
2791
2792 vcpu->guest_debug = 0;
2793 kvm_s390_clear_bp_data(vcpu);
2794
2de3bfc2 2795 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21 2796 return -EINVAL;
89b5b4de
DH		 2797 if (!sclp.has_gpere)
2798 return -EINVAL;
27291e21
DH		 2799
2800 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2801 vcpu->guest_debug = dbg->control;
2802 /* enforce guest PER */
805de8f4 2803 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2804
2805 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2806 rc = kvm_s390_import_bp_data(vcpu, dbg);
2807 } else {
805de8f4 2808 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2809 vcpu->arch.guestdbg.last_bp = 0;
2810 }
2811
2812 if (rc) {
2813 vcpu->guest_debug = 0;
2814 kvm_s390_clear_bp_data(vcpu);
805de8f4 2815 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2816 }
2817
2818 return rc;
b0c632db
HC		 2819}
2820
62d9f0db
MT		 2821int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2822 struct kvm_mp_state *mp_state)
2823{
6352e4d2
DH		 2824 /* CHECK_STOP and LOAD are not supported yet */
2825 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2826 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 2827}
2828
2829int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2830 struct kvm_mp_state *mp_state)
2831{
6352e4d2
DH		 2832 int rc = 0;
2833
2834 /* user space knows about this interface - let it control the state */
2835 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2836
2837 switch (mp_state->mp_state) {
2838 case KVM_MP_STATE_STOPPED:
2839 kvm_s390_vcpu_stop(vcpu);
2840 break;
2841 case KVM_MP_STATE_OPERATING:
2842 kvm_s390_vcpu_start(vcpu);
2843 break;
2844 case KVM_MP_STATE_LOAD:
2845 case KVM_MP_STATE_CHECK_STOP:
2846 /* fall through - CHECK_STOP and LOAD are not supported yet */
2847 default:
2848 rc = -ENXIO;
2849 }
2850
2851 return rc;
62d9f0db
MT		 2852}
2853
8ad35755
DH		 2854static bool ibs_enabled(struct kvm_vcpu *vcpu)
2855{
2856 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2857}
2858
2c70fe44
CB		 2859static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2860{
8ad35755 2861retry:
8e236546 2862 kvm_s390_vcpu_request_handled(vcpu);
2fa6e1e1 2863 if (!kvm_request_pending(vcpu))
586b7ccd 2864 return 0;
2c70fe44
CB		 2865 /*
2866 * We use MMU_RELOAD just to re-arm the ipte notifier for the
b2d73b2a 2867 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2c70fe44
CB		 2868 * This ensures that the ipte instruction for this request has
2869 * already finished. We might race against a second unmapper that
2870 * wants to set the blocking bit. Lets just retry the request loop.
2871 */
8ad35755 2872 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44 2873 int rc;
b2d73b2a
MS		 2874 rc = gmap_mprotect_notify(vcpu->arch.gmap,
2875 kvm_s390_get_prefix(vcpu),
2876 PAGE_SIZE * 2, PROT_WRITE);
aca411a4
JN		 2877 if (rc) {
2878 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44 2879 return rc;
aca411a4 2880 }
8ad35755 2881 goto retry;
2c70fe44 2882 }
8ad35755 2883
d3d692c8
DH		 2884 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2885 vcpu->arch.sie_block->ihcpu = 0xffff;
2886 goto retry;
2887 }
2888
8ad35755
DH		 2889 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2890 if (!ibs_enabled(vcpu)) {
2891 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
805de8f4 2892 atomic_or(CPUSTAT_IBS,
8ad35755
DH		 2893 &vcpu->arch.sie_block->cpuflags);
2894 }
2895 goto retry;
2c70fe44 2896 }
8ad35755
DH		 2897
2898 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2899 if (ibs_enabled(vcpu)) {
2900 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
805de8f4 2901 atomic_andnot(CPUSTAT_IBS,
8ad35755
DH		 2902 &vcpu->arch.sie_block->cpuflags);
2903 }
2904 goto retry;
2905 }
2906
6502a34c
DH		 2907 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
2908 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2909 goto retry;
2910 }
2911
190df4a2
CI		 2912 if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
2913 /*
2914 * Disable CMMA virtualization; we will emulate the ESSA
2915 * instruction manually, in order to provide additional
2916 * functionalities needed for live migration.
2917 */
2918 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
2919 goto retry;
2920 }
2921
2922 if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
2923 /*
2924 * Re-enable CMMA virtualization if CMMA is available and
2925 * was used.
2926 */
2927 if ((vcpu->kvm->arch.use_cmma) &&
2928 (vcpu->kvm->mm->context.use_cmma))
2929 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
2930 goto retry;
2931 }
2932
0759d068 2933 /* nothing to do, just clear the request */
72875d8a 2934 kvm_clear_request(KVM_REQ_UNHALT, vcpu);
0759d068 2935
2c70fe44
CB		 2936 return 0;
2937}
2938
8fa1696e
CW		 2939void kvm_s390_set_tod_clock_ext(struct kvm *kvm,
2940 const struct kvm_s390_vm_tod_clock *gtod)
2941{
2942 struct kvm_vcpu *vcpu;
2943 struct kvm_s390_tod_clock_ext htod;
2944 int i;
2945
2946 mutex_lock(&kvm->lock);
2947 preempt_disable();
2948
2949 get_tod_clock_ext((char *)&htod);
2950
2951 kvm->arch.epoch = gtod->tod - htod.tod;
2952 kvm->arch.epdx = gtod->epoch_idx - htod.epoch_idx;
2953
2954 if (kvm->arch.epoch > gtod->tod)
2955 kvm->arch.epdx -= 1;
2956
2957 kvm_s390_vcpu_block_all(kvm);
2958 kvm_for_each_vcpu(i, vcpu, kvm) {
2959 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2960 vcpu->arch.sie_block->epdx = kvm->arch.epdx;
2961 }
2962
2963 kvm_s390_vcpu_unblock_all(kvm);
2964 preempt_enable();
2965 mutex_unlock(&kvm->lock);
2966}
2967
25ed1675
DH		 2968void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2969{
2970 struct kvm_vcpu *vcpu;
2971 int i;
2972
2973 mutex_lock(&kvm->lock);
2974 preempt_disable();
2975 kvm->arch.epoch = tod - get_tod_clock();
2976 kvm_s390_vcpu_block_all(kvm);
2977 kvm_for_each_vcpu(i, vcpu, kvm)
2978 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2979 kvm_s390_vcpu_unblock_all(kvm);
2980 preempt_enable();
2981 mutex_unlock(&kvm->lock);
2982}
2983
fa576c58
TH		 2984/**
2985 * kvm_arch_fault_in_page - fault-in guest page if necessary
2986 * @vcpu: The corresponding virtual cpu
2987 * @gpa: Guest physical address
2988 * @writable: Whether the page should be writable or not
2989 *
2990 * Make sure that a guest page has been faulted-in on the host.
2991 *
2992 * Return: Zero on success, negative error code otherwise.
2993 */
2994long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 2995{
527e30b4
MS		 2996 return gmap_fault(vcpu->arch.gmap, gpa,
2997 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 2998}
2999
3c038e6b
DD		 3000static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
3001 unsigned long token)
3002{
3003 struct kvm_s390_interrupt inti;
383d0b05 3004 struct kvm_s390_irq irq;
3c038e6b
DD		 3005
3006 if (start_token) {
383d0b05
JF		 3007 irq.u.ext.ext_params2 = token;
3008 irq.type = KVM_S390_INT_PFAULT_INIT;
3009 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 3010 } else {
3011 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 3012 inti.parm64 = token;
3c038e6b
DD		 3013 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
3014 }
3015}
3016
3017void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
3018 struct kvm_async_pf *work)
3019{
3020 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
3021 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
3022}
3023
3024void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
3025 struct kvm_async_pf *work)
3026{
3027 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
3028 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
3029}
3030
3031void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
3032 struct kvm_async_pf *work)
3033{
3034 /* s390 will always inject the page directly */
3035}
3036
3037bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
3038{
3039 /*
3040 * s390 will always inject the page directly,
3041 * but we still want check_async_completion to cleanup
3042 */
3043 return true;
3044}
3045
3046static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
3047{
3048 hva_t hva;
3049 struct kvm_arch_async_pf arch;
3050 int rc;
3051
3052 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3053 return 0;
3054 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
3055 vcpu->arch.pfault_compare)
3056 return 0;
3057 if (psw_extint_disabled(vcpu))
3058 return 0;
9a022067 3059 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 3060 return 0;
3061 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
3062 return 0;
3063 if (!vcpu->arch.gmap->pfault_enabled)
3064 return 0;
3065
81480cc1
HC		 3066 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
3067 hva += current->thread.gmap_addr & ~PAGE_MASK;
3068 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 3069 return 0;
3070
3071 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
3072 return rc;
3073}
3074
3fb4c40f 3075static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 3076{
3fb4c40f 3077 int rc, cpuflags;
e168bf8d 3078
3c038e6b
DD		 3079 /*
3080 * On s390 notifications for arriving pages will be delivered directly
3081 * to the guest but the house keeping for completed pfaults is
3082 * handled outside the worker.
3083 */
3084 kvm_check_async_pf_completion(vcpu);
3085
7ec7c8c7
CB		 3086 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
3087 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
b0c632db
HC		 3088
3089 if (need_resched())
3090 schedule();
3091
d3a73acb 3092 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 3093 s390_handle_mcck();
3094
79395031
JF		 3095 if (!kvm_is_ucontrol(vcpu->kvm)) {
3096 rc = kvm_s390_deliver_pending_interrupts(vcpu);
3097 if (rc)
3098 return rc;
3099 }
0ff31867 3100
2c70fe44
CB		 3101 rc = kvm_s390_handle_requests(vcpu);
3102 if (rc)
3103 return rc;
3104
27291e21
DH		 3105 if (guestdbg_enabled(vcpu)) {
3106 kvm_s390_backup_guest_per_regs(vcpu);
3107 kvm_s390_patch_guest_per_regs(vcpu);
3108 }
3109
b0c632db 3110 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 3111 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
3112 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
3113 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 3114
3fb4c40f
TH		 3115 return 0;
3116}
3117
492d8642
TH		 3118static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
3119{
56317920
DH		 3120 struct kvm_s390_pgm_info pgm_info = {
3121 .code = PGM_ADDRESSING,
3122 };
3123 u8 opcode, ilen;
492d8642
TH		 3124 int rc;
3125
3126 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
3127 trace_kvm_s390_sie_fault(vcpu);
3128
3129 /*
3130 * We want to inject an addressing exception, which is defined as a
3131 * suppressing or terminating exception. However, since we came here
3132 * by a DAT access exception, the PSW still points to the faulting
3133 * instruction since DAT exceptions are nullifying. So we've got
3134 * to look up the current opcode to get the length of the instruction
3135 * to be able to forward the PSW.
3136 */
3fa8cad7 3137 rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
56317920 3138 ilen = insn_length(opcode);
9b0d721a
DH		 3139 if (rc < 0) {
3140 return rc;
3141 } else if (rc) {
3142 /* Instruction-Fetching Exceptions - we can't detect the ilen.
3143 * Forward by arbitrary ilc, injection will take care of
3144 * nullification if necessary.
3145 */
3146 pgm_info = vcpu->arch.pgm;
3147 ilen = 4;
3148 }
56317920
DH		 3149 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
3150 kvm_s390_forward_psw(vcpu, ilen);
3151 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
492d8642
TH		 3152}
3153
3fb4c40f
TH		 3154static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
3155{
4d62fcc0
QH		 3156 struct mcck_volatile_info *mcck_info;
3157 struct sie_page *sie_page;
3158
2b29a9fd
DD		 3159 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
3160 vcpu->arch.sie_block->icptcode);
3161 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
3162
27291e21
DH		 3163 if (guestdbg_enabled(vcpu))
3164 kvm_s390_restore_guest_per_regs(vcpu);
3165
7ec7c8c7
CB		 3166 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
3167 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
71f116bf 3168
4d62fcc0
QH		 3169 if (exit_reason == -EINTR) {
3170 VCPU_EVENT(vcpu, 3, "%s", "machine check");
3171 sie_page = container_of(vcpu->arch.sie_block,
3172 struct sie_page, sie_block);
3173 mcck_info = &sie_page->mcck_info;
3174 kvm_s390_reinject_machine_check(vcpu, mcck_info);
3175 return 0;
3176 }
3177
71f116bf
DH		 3178 if (vcpu->arch.sie_block->icptcode > 0) {
3179 int rc = kvm_handle_sie_intercept(vcpu);
3180
3181 if (rc != -EOPNOTSUPP)
3182 return rc;
3183 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
3184 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
3185 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
3186 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
3187 return -EREMOTE;
3188 } else if (exit_reason != -EFAULT) {
3189 vcpu->stat.exit_null++;
3190 return 0;
210b1607
TH		 3191 } else if (kvm_is_ucontrol(vcpu->kvm)) {
3192 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
3193 vcpu->run->s390_ucontrol.trans_exc_code =
3194 current->thread.gmap_addr;
3195 vcpu->run->s390_ucontrol.pgm_code = 0x10;
71f116bf 3196 return -EREMOTE;
24eb3a82 3197 } else if (current->thread.gmap_pfault) {
3c038e6b 3198 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 3199 current->thread.gmap_pfault = 0;
71f116bf
DH		 3200 if (kvm_arch_setup_async_pf(vcpu))
3201 return 0;
3202 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
a76ccff6 3203 }
71f116bf 3204 return vcpu_post_run_fault_in_sie(vcpu);
3fb4c40f
TH		 3205}
3206
3207static int __vcpu_run(struct kvm_vcpu *vcpu)
3208{
3209 int rc, exit_reason;
3210
800c1065
TH		 3211 /*
3212 * We try to hold kvm->srcu during most of vcpu_run (except when run-
3213 * ning the guest), so that memslots (and other stuff) are protected
3214 */
3215 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3216
a76ccff6
TH		 3217 do {
3218 rc = vcpu_pre_run(vcpu);
3219 if (rc)
3220 break;
3fb4c40f 3221
800c1065 3222 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 3223 /*
3224 * As PF_VCPU will be used in fault handler, between
3225 * guest_enter and guest_exit should be no uaccess.
3226 */
0097d12e 3227 local_irq_disable();
6edaa530 3228 guest_enter_irqoff();
db0758b2 3229 __disable_cpu_timer_accounting(vcpu);
0097d12e 3230 local_irq_enable();
a76ccff6
TH		 3231 exit_reason = sie64a(vcpu->arch.sie_block,
3232 vcpu->run->s.regs.gprs);
0097d12e 3233 local_irq_disable();
db0758b2 3234 __enable_cpu_timer_accounting(vcpu);
6edaa530 3235 guest_exit_irqoff();
0097d12e 3236 local_irq_enable();
800c1065 3237 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 3238
3239 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 3240 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 3241
800c1065 3242 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 3243 return rc;
b0c632db
HC		 3244}
3245
b028ee3e
DH		 3246static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3247{
4d5f2c04 3248 struct runtime_instr_cb *riccb;
4e0b1ab7 3249 struct gs_cb *gscb;
4d5f2c04
CB		 3250
3251 riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
4e0b1ab7 3252 gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
b028ee3e
DH		 3253 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
3254 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
3255 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
3256 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
3257 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
3258 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 3259 /* some control register changes require a tlb flush */
3260 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 3261 }
3262 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4287f247 3263 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
b028ee3e
DH		 3264 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
3265 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
3266 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
3267 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
3268 }
3269 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
3270 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
3271 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
3272 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 3273 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3274 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e 3275 }
80cd8763
FZ		 3276 /*
3277 * If userspace sets the riccb (e.g. after migration) to a valid state,
3278 * we should enable RI here instead of doing the lazy enablement.
3279 */
3280 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
4d5f2c04 3281 test_kvm_facility(vcpu->kvm, 64) &&
bb59c2da 3282 riccb->v &&
0c9d8683 3283 !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
4d5f2c04 3284 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
0c9d8683 3285 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
80cd8763 3286 }
4e0b1ab7
FZ		 3287 /*
3288 * If userspace sets the gscb (e.g. after migration) to non-zero,
3289 * we should enable GS here instead of doing the lazy enablement.
3290 */
3291 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
3292 test_kvm_facility(vcpu->kvm, 133) &&
3293 gscb->gssm &&
3294 !vcpu->arch.gs_enabled) {
3295 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
3296 vcpu->arch.sie_block->ecb |= ECB_GS;
3297 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3298 vcpu->arch.gs_enabled = 1;
80cd8763 3299 }
31d8b8d4
CB		 3300 save_access_regs(vcpu->arch.host_acrs);
3301 restore_access_regs(vcpu->run->s.regs.acrs);
e1788bb9
CB		 3302 /* save host (userspace) fprs/vrs */
3303 save_fpu_regs();
3304 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
3305 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
3306 if (MACHINE_HAS_VX)
3307 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
3308 else
3309 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
3310 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
3311 if (test_fp_ctl(current->thread.fpu.fpc))
3312 /* User space provided an invalid FPC, let's clear it */
3313 current->thread.fpu.fpc = 0;
4e0b1ab7
FZ		 3314 if (MACHINE_HAS_GS) {
3315 preempt_disable();
3316 __ctl_set_bit(2, 4);
3317 if (current->thread.gs_cb) {
3318 vcpu->arch.host_gscb = current->thread.gs_cb;
3319 save_gs_cb(vcpu->arch.host_gscb);
3320 }
3321 if (vcpu->arch.gs_enabled) {
3322 current->thread.gs_cb = (struct gs_cb *)
3323 &vcpu->run->s.regs.gscb;
3324 restore_gs_cb(current->thread.gs_cb);
3325 }
3326 preempt_enable();
3327 }
80cd8763 3328
b028ee3e
DH		 3329 kvm_run->kvm_dirty_regs = 0;
3330}
3331
3332static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3333{
3334 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
3335 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
3336 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
3337 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4287f247 3338 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
b028ee3e
DH		 3339 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
3340 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
3341 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
3342 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
3343 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
3344 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
3345 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
31d8b8d4
CB		 3346 save_access_regs(vcpu->run->s.regs.acrs);
3347 restore_access_regs(vcpu->arch.host_acrs);
e1788bb9
CB		 3348 /* Save guest register state */
3349 save_fpu_regs();
3350 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3351 /* Restore will be done lazily at return */
3352 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
3353 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
4e0b1ab7
FZ		 3354 if (MACHINE_HAS_GS) {
3355 __ctl_set_bit(2, 4);
3356 if (vcpu->arch.gs_enabled)
3357 save_gs_cb(current->thread.gs_cb);
3358 preempt_disable();
3359 current->thread.gs_cb = vcpu->arch.host_gscb;
3360 restore_gs_cb(vcpu->arch.host_gscb);
3361 preempt_enable();
3362 if (!vcpu->arch.host_gscb)
3363 __ctl_clear_bit(2, 4);
3364 vcpu->arch.host_gscb = NULL;
3365 }
e1788bb9 3366
b028ee3e
DH		 3367}
3368
b0c632db
HC		 3369int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3370{
8f2abe6a 3371 int rc;
b0c632db
HC		 3372 sigset_t sigsaved;
3373
460df4c1
PB		 3374 if (kvm_run->immediate_exit)
3375 return -EINTR;
3376
27291e21
DH		 3377 if (guestdbg_exit_pending(vcpu)) {
3378 kvm_s390_prepare_debug_exit(vcpu);
3379 return 0;
3380 }
3381
b0c632db
HC		 3382 if (vcpu->sigset_active)
3383 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
3384
6352e4d2
DH		 3385 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
3386 kvm_s390_vcpu_start(vcpu);
3387 } else if (is_vcpu_stopped(vcpu)) {
ea2cdd27 3388 pr_err_ratelimited("can't run stopped vcpu %d\n",
6352e4d2
DH		 3389 vcpu->vcpu_id);
3390 return -EINVAL;
3391 }
b0c632db 3392
b028ee3e 3393 sync_regs(vcpu, kvm_run);
db0758b2 3394 enable_cpu_timer_accounting(vcpu);
d7b0b5eb 3395
dab4079d 3396 might_fault();
a76ccff6 3397 rc = __vcpu_run(vcpu);
9ace903d 3398
b1d16c49
CE		 3399 if (signal_pending(current) && !rc) {
3400 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 3401 rc = -EINTR;
b1d16c49 3402 }
8f2abe6a 3403
27291e21
DH		 3404 if (guestdbg_exit_pending(vcpu) && !rc) {
3405 kvm_s390_prepare_debug_exit(vcpu);
3406 rc = 0;
3407 }
3408
8f2abe6a 3409 if (rc == -EREMOTE) {
71f116bf 3410 /* userspace support is needed, kvm_run has been prepared */
8f2abe6a
CB		 3411 rc = 0;
3412 }
b0c632db 3413
db0758b2 3414 disable_cpu_timer_accounting(vcpu);
b028ee3e 3415 store_regs(vcpu, kvm_run);
d7b0b5eb 3416
b0c632db
HC		 3417 if (vcpu->sigset_active)
3418 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
3419
b0c632db 3420 vcpu->stat.exit_userspace++;
7e8e6ab4 3421 return rc;
b0c632db
HC		 3422}
3423
b0c632db
HC		 3424/*
3425 * store status at address
3426 * we use have two special cases:
3427 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
3428 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
3429 */
d0bce605 3430int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 3431{
092670cd 3432 unsigned char archmode = 1;
9abc2a08 3433 freg_t fprs[NUM_FPRS];
fda902cb 3434 unsigned int px;
4287f247 3435 u64 clkcomp, cputm;
d0bce605 3436 int rc;
b0c632db 3437
d9a3a09a 3438 px = kvm_s390_get_prefix(vcpu);
d0bce605
HC		 3439 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
3440 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 3441 return -EFAULT;
d9a3a09a 3442 gpa = 0;
d0bce605
HC		 3443 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
3444 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 3445 return -EFAULT;
d9a3a09a
MS		 3446 gpa = px;
3447 } else
3448 gpa -= __LC_FPREGS_SAVE_AREA;
9abc2a08
DH		 3449
3450 /* manually convert vector registers if necessary */
3451 if (MACHINE_HAS_VX) {
9522b37f 3452 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08
DH		 3453 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
3454 fprs, 128);
3455 } else {
3456 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
6fd8e67d 3457 vcpu->run->s.regs.fprs, 128);
9abc2a08 3458 }
d9a3a09a 3459 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
d0bce605 3460 vcpu->run->s.regs.gprs, 128);
d9a3a09a 3461 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
d0bce605 3462 &vcpu->arch.sie_block->gpsw, 16);
d9a3a09a 3463 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
fda902cb 3464 &px, 4);
d9a3a09a 3465 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
9abc2a08 3466 &vcpu->run->s.regs.fpc, 4);
d9a3a09a 3467 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
d0bce605 3468 &vcpu->arch.sie_block->todpr, 4);
4287f247 3469 cputm = kvm_s390_get_cpu_timer(vcpu);
d9a3a09a 3470 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4287f247 3471 &cputm, 8);
178bd789 3472 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d9a3a09a 3473 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
d0bce605 3474 &clkcomp, 8);
d9a3a09a 3475 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
d0bce605 3476 &vcpu->run->s.regs.acrs, 64);
d9a3a09a 3477 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
d0bce605
HC		 3478 &vcpu->arch.sie_block->gcr, 128);
3479 return rc ? -EFAULT : 0;
b0c632db
HC		 3480}
3481
e879892c
TH		 3482int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
3483{
3484 /*
3485 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
31d8b8d4 3486 * switch in the run ioctl. Let's update our copies before we save
e879892c
TH		 3487 * it into the save area
3488 */
d0164ee2 3489 save_fpu_regs();
9abc2a08 3490 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
e879892c
TH		 3491 save_access_regs(vcpu->run->s.regs.acrs);
3492
3493 return kvm_s390_store_status_unloaded(vcpu, addr);
3494}
3495
8ad35755
DH		 3496static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3497{
3498 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
8e236546 3499 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
8ad35755
DH		 3500}
3501
3502static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
3503{
3504 unsigned int i;
3505 struct kvm_vcpu *vcpu;
3506
3507 kvm_for_each_vcpu(i, vcpu, kvm) {
3508 __disable_ibs_on_vcpu(vcpu);
3509 }
3510}
3511
3512static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3513{
09a400e7
DH		 3514 if (!sclp.has_ibs)
3515 return;
8ad35755 3516 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
8e236546 3517 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
8ad35755
DH		 3518}
3519
6852d7b6
DH		 3520void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
3521{
8ad35755
DH		 3522 int i, online_vcpus, started_vcpus = 0;
3523
3524 if (!is_vcpu_stopped(vcpu))
3525 return;
3526
6852d7b6 3527 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 3528 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 3529 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 3530 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3531
3532 for (i = 0; i < online_vcpus; i++) {
3533 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
3534 started_vcpus++;
3535 }
3536
3537 if (started_vcpus == 0) {
3538 /* we're the only active VCPU -> speed it up */
3539 __enable_ibs_on_vcpu(vcpu);
3540 } else if (started_vcpus == 1) {
3541 /*
3542 * As we are starting a second VCPU, we have to disable
3543 * the IBS facility on all VCPUs to remove potentially
3544 * oustanding ENABLE requests.
3545 */
3546 __disable_ibs_on_all_vcpus(vcpu->kvm);
3547 }
3548
805de8f4 3549 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 3550 /*
3551 * Another VCPU might have used IBS while we were offline.
3552 * Let's play safe and flush the VCPU at startup.
3553 */
d3d692c8 3554 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 3555 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 3556 return;
6852d7b6
DH		 3557}
3558
3559void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
3560{
8ad35755
DH		 3561 int i, online_vcpus, started_vcpus = 0;
3562 struct kvm_vcpu *started_vcpu = NULL;
3563
3564 if (is_vcpu_stopped(vcpu))
3565 return;
3566
6852d7b6 3567 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 3568 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 3569 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 3570 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3571
32f5ff63 3572 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 3573 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 3574
805de8f4 3575 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 3576 __disable_ibs_on_vcpu(vcpu);
3577
3578 for (i = 0; i < online_vcpus; i++) {
3579 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
3580 started_vcpus++;
3581 started_vcpu = vcpu->kvm->vcpus[i];
3582 }
3583 }
3584
3585 if (started_vcpus == 1) {
3586 /*
3587 * As we only have one VCPU left, we want to enable the
3588 * IBS facility for that VCPU to speed it up.
3589 */
3590 __enable_ibs_on_vcpu(started_vcpu);
3591 }
3592
433b9ee4 3593 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 3594 return;
6852d7b6
DH		 3595}
3596
d6712df9
CH		 3597static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
3598 struct kvm_enable_cap *cap)
3599{
3600 int r;
3601
3602 if (cap->flags)
3603 return -EINVAL;
3604
3605 switch (cap->cap) {
fa6b7fe9
CH		 3606 case KVM_CAP_S390_CSS_SUPPORT:
3607 if (!vcpu->kvm->arch.css_support) {
3608 vcpu->kvm->arch.css_support = 1;
c92ea7b9 3609 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
fa6b7fe9
CH		 3610 trace_kvm_s390_enable_css(vcpu->kvm);
3611 }
3612 r = 0;
3613 break;
d6712df9
CH		 3614 default:
3615 r = -EINVAL;
3616 break;
3617 }
3618 return r;
3619}
3620
41408c28
TH		 3621static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
3622 struct kvm_s390_mem_op *mop)
3623{
3624 void __user *uaddr = (void __user *)mop->buf;
3625 void *tmpbuf = NULL;
3626 int r, srcu_idx;
3627 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
3628 | KVM_S390_MEMOP_F_CHECK_ONLY;
3629
3630 if (mop->flags & ~supported_flags)
3631 return -EINVAL;
3632
3633 if (mop->size > MEM_OP_MAX_SIZE)
3634 return -E2BIG;
3635
3636 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
3637 tmpbuf = vmalloc(mop->size);
3638 if (!tmpbuf)
3639 return -ENOMEM;
3640 }
3641
3642 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3643
3644 switch (mop->op) {
3645 case KVM_S390_MEMOP_LOGICAL_READ:
3646 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 3647 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3648 mop->size, GACC_FETCH);
41408c28
TH		 3649 break;
3650 }
3651 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3652 if (r == 0) {
3653 if (copy_to_user(uaddr, tmpbuf, mop->size))
3654 r = -EFAULT;
3655 }
3656 break;
3657 case KVM_S390_MEMOP_LOGICAL_WRITE:
3658 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 3659 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3660 mop->size, GACC_STORE);
41408c28
TH		 3661 break;
3662 }
3663 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
3664 r = -EFAULT;
3665 break;
3666 }
3667 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3668 break;
3669 default:
3670 r = -EINVAL;
3671 }
3672
3673 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
3674
3675 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
3676 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
3677
3678 vfree(tmpbuf);
3679 return r;
3680}
3681
b0c632db
HC		 3682long kvm_arch_vcpu_ioctl(struct file *filp,
3683 unsigned int ioctl, unsigned long arg)
3684{
3685 struct kvm_vcpu *vcpu = filp->private_data;
3686 void __user *argp = (void __user *)arg;
800c1065 3687 int idx;
bc923cc9 3688 long r;
b0c632db 3689
93736624 3690 switch (ioctl) {
47b43c52
JF		 3691 case KVM_S390_IRQ: {
3692 struct kvm_s390_irq s390irq;
3693
3694 r = -EFAULT;
3695 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
3696 break;
3697 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3698 break;
3699 }
93736624 3700 case KVM_S390_INTERRUPT: {
ba5c1e9b 3701 struct kvm_s390_interrupt s390int;
383d0b05 3702 struct kvm_s390_irq s390irq;
ba5c1e9b 3703
93736624 3704 r = -EFAULT;
ba5c1e9b 3705 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624 3706 break;
383d0b05
JF		 3707 if (s390int_to_s390irq(&s390int, &s390irq))
3708 return -EINVAL;
3709 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
93736624 3710 break;
ba5c1e9b 3711 }
b0c632db 3712 case KVM_S390_STORE_STATUS:
800c1065 3713 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 3714 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 3715 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 3716 break;
b0c632db
HC		 3717 case KVM_S390_SET_INITIAL_PSW: {
3718 psw_t psw;
3719
bc923cc9 3720 r = -EFAULT;
b0c632db 3721 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 3722 break;
3723 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3724 break;
b0c632db
HC		 3725 }
3726 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 3727 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3728 break;
14eebd91
CO		 3729 case KVM_SET_ONE_REG:
3730 case KVM_GET_ONE_REG: {
3731 struct kvm_one_reg reg;
3732 r = -EFAULT;
3733 if (copy_from_user(&reg, argp, sizeof(reg)))
3734 break;
3735 if (ioctl == KVM_SET_ONE_REG)
3736 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3737 else
3738 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3739 break;
3740 }
27e0393f
CO		 3741#ifdef CONFIG_KVM_S390_UCONTROL
3742 case KVM_S390_UCAS_MAP: {
3743 struct kvm_s390_ucas_mapping ucasmap;
3744
3745 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3746 r = -EFAULT;
3747 break;
3748 }
3749
3750 if (!kvm_is_ucontrol(vcpu->kvm)) {
3751 r = -EINVAL;
3752 break;
3753 }
3754
3755 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3756 ucasmap.vcpu_addr, ucasmap.length);
3757 break;
3758 }
3759 case KVM_S390_UCAS_UNMAP: {
3760 struct kvm_s390_ucas_mapping ucasmap;
3761
3762 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3763 r = -EFAULT;
3764 break;
3765 }
3766
3767 if (!kvm_is_ucontrol(vcpu->kvm)) {
3768 r = -EINVAL;
3769 break;
3770 }
3771
3772 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3773 ucasmap.length);
3774 break;
3775 }
3776#endif
ccc7910f 3777 case KVM_S390_VCPU_FAULT: {
527e30b4 3778 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 3779 break;
3780 }
d6712df9
CH		 3781 case KVM_ENABLE_CAP:
3782 {
3783 struct kvm_enable_cap cap;
3784 r = -EFAULT;
3785 if (copy_from_user(&cap, argp, sizeof(cap)))
3786 break;
3787 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3788 break;
3789 }
41408c28
TH		 3790 case KVM_S390_MEM_OP: {
3791 struct kvm_s390_mem_op mem_op;
3792
3793 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3794 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3795 else
3796 r = -EFAULT;
3797 break;
3798 }
816c7667
JF		 3799 case KVM_S390_SET_IRQ_STATE: {
3800 struct kvm_s390_irq_state irq_state;
3801
3802 r = -EFAULT;
3803 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3804 break;
3805 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3806 irq_state.len == 0 ||
3807 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3808 r = -EINVAL;
3809 break;
3810 }
bb64da9a 3811 /* do not use irq_state.flags, it will break old QEMUs */
816c7667
JF		 3812 r = kvm_s390_set_irq_state(vcpu,
3813 (void __user *) irq_state.buf,
3814 irq_state.len);
3815 break;
3816 }
3817 case KVM_S390_GET_IRQ_STATE: {
3818 struct kvm_s390_irq_state irq_state;
3819
3820 r = -EFAULT;
3821 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3822 break;
3823 if (irq_state.len == 0) {
3824 r = -EINVAL;
3825 break;
3826 }
bb64da9a 3827 /* do not use irq_state.flags, it will break old QEMUs */
816c7667
JF		 3828 r = kvm_s390_get_irq_state(vcpu,
3829 (__u8 __user *) irq_state.buf,
3830 irq_state.len);
3831 break;
3832 }
b0c632db 3833 default:
3e6afcf1 3834 r = -ENOTTY;
b0c632db 3835 }
bc923cc9 3836 return r;
b0c632db
HC		 3837}
3838
5b1c1493
CO		 3839int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3840{
3841#ifdef CONFIG_KVM_S390_UCONTROL
3842 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3843 && (kvm_is_ucontrol(vcpu->kvm))) {
3844 vmf->page = virt_to_page(vcpu->arch.sie_block);
3845 get_page(vmf->page);
3846 return 0;
3847 }
3848#endif
3849 return VM_FAULT_SIGBUS;
3850}
3851
5587027c
AK		 3852int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3853 unsigned long npages)
db3fe4eb
TY		 3854{
3855 return 0;
3856}
3857
b0c632db 3858/* Section: memory related */
f7784b8e
MT		 3859int kvm_arch_prepare_memory_region(struct kvm *kvm,
3860 struct kvm_memory_slot *memslot,
09170a49 3861 const struct kvm_userspace_memory_region *mem,
7b6195a9 3862 enum kvm_mr_change change)
b0c632db 3863{
dd2887e7
NW		 3864 /* A few sanity checks. We can have memory slots which have to be
3865 located/ended at a segment boundary (1MB). The memory in userland is
3866 ok to be fragmented into various different vmas. It is okay to mmap()
3867 and munmap() stuff in this slot after doing this call at any time */
b0c632db 3868
598841ca 3869 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 3870 return -EINVAL;
3871
598841ca 3872 if (mem->memory_size & 0xffffful)
b0c632db
HC		 3873 return -EINVAL;
3874
a3a92c31
DD		 3875 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3876 return -EINVAL;
3877
f7784b8e
MT		 3878 return 0;
3879}
3880
3881void kvm_arch_commit_memory_region(struct kvm *kvm,
09170a49 3882 const struct kvm_userspace_memory_region *mem,
8482644a 3883 const struct kvm_memory_slot *old,
f36f3f28 3884 const struct kvm_memory_slot *new,
8482644a 3885 enum kvm_mr_change change)
f7784b8e 3886{
f7850c92 3887 int rc;
f7784b8e 3888
2cef4deb
CB		 3889 /* If the basics of the memslot do not change, we do not want
3890 * to update the gmap. Every update causes several unnecessary
3891 * segment translation exceptions. This is usually handled just
3892 * fine by the normal fault handler + gmap, but it will also
3893 * cause faults on the prefix page of running guest CPUs.
3894 */
3895 if (old->userspace_addr == mem->userspace_addr &&
3896 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3897 old->npages * PAGE_SIZE == mem->memory_size)
3898 return;
598841ca
CO		 3899
3900 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3901 mem->guest_phys_addr, mem->memory_size);
3902 if (rc)
ea2cdd27 3903 pr_warn("failed to commit memory region\n");
598841ca 3904 return;
b0c632db
HC		 3905}
3906
60a37709
AY		 3907static inline unsigned long nonhyp_mask(int i)
3908{
3909 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3910
3911 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3912}
3913
3491caf2
CB		 3914void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3915{
3916 vcpu->valid_wakeup = false;
3917}
3918
b0c632db
HC		 3919static int __init kvm_s390_init(void)
3920{
60a37709
AY		 3921 int i;
3922
07197fd0
DH		 3923 if (!sclp.has_sief2) {
3924 pr_info("SIE not available\n");
3925 return -ENODEV;
3926 }
3927
60a37709
AY		 3928 for (i = 0; i < 16; i++)
3929 kvm_s390_fac_list_mask[i] |=
3930 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3931
9d8d5786 3932 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 3933}
3934
3935static void __exit kvm_s390_exit(void)
3936{
3937 kvm_exit();
3938}
3939
3940module_init(kvm_s390_init);
3941module_exit(kvm_s390_exit);
566af940
CH		 3942
3943/*
3944 * Enable autoloading of the kvm module.
3945 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3946 * since x86 takes a different approach.
3947 */
3948#include <linux/miscdevice.h>
3949MODULE_ALIAS_MISCDEV(KVM_MINOR);
3950MODULE_ALIAS("devname:kvm");
Ubuntu Jammy Linux kernel mirror