]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'kvm-s390-master-4.10-1' of git://git.kernel.org/pub/scm/linux/kernel/git...
[mirror_ubuntu-jammy-kernel.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
b2d73b2a 24#include <linux/mman.h>
b0c632db 25#include <linux/module.h>
a374e892 26#include <linux/random.h>
b0c632db 27#include <linux/slab.h>
ba5c1e9b 28#include <linux/timer.h>
41408c28 29#include <linux/vmalloc.h>
15c9705f 30#include <linux/bitmap.h>
cbb870c8 31#include <asm/asm-offsets.h>
b0c632db 32#include <asm/lowcore.h>
fd5ada04 33#include <asm/stp.h>
b0c632db 34#include <asm/pgtable.h>
1e133ab2 35#include <asm/gmap.h>
f5daba1d 36#include <asm/nmi.h>
a0616cde 37#include <asm/switch_to.h>
6d3da241 38#include <asm/isc.h>
1526bf9c 39#include <asm/sclp.h>
0a763c78 40#include <asm/cpacf.h>
221bb8a4 41#include <asm/timex.h>
8f2abe6a 42#include "kvm-s390.h"
b0c632db
HC		 43#include "gaccess.h"
44
ea2cdd27
DH		 45#define KMSG_COMPONENT "kvm-s390"
46#undef pr_fmt
47#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
48
5786fffa
CH		 49#define CREATE_TRACE_POINTS
50#include "trace.h"
ade38c31 51#include "trace-s390.h"
5786fffa 52
41408c28 53#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
816c7667
JF		 54#define LOCAL_IRQS 32
55#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
56 (KVM_MAX_VCPUS + LOCAL_IRQS))
41408c28 57
b0c632db
HC		 58#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
59
60struct kvm_stats_debugfs_item debugfs_entries[] = {
61 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 62 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 63 { "exit_validity", VCPU_STAT(exit_validity) },
64 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
65 { "exit_external_request", VCPU_STAT(exit_external_request) },
66 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b 67 { "exit_instruction", VCPU_STAT(exit_instruction) },
9ec6de19 68 { "exit_pei", VCPU_STAT(exit_pei) },
ba5c1e9b
CO		 69 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
70 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
a011eeb2 71 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
f7819512 72 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
62bea5bf 73 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
3491caf2 74 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
ce2e4f0b 75 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 76 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 77 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 78 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
79 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 80 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 81 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 82 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
83 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
84 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
85 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
86 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
87 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
88 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 89 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 90 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
91 { "instruction_spx", VCPU_STAT(instruction_spx) },
92 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
93 { "instruction_stap", VCPU_STAT(instruction_stap) },
94 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 95 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 96 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
97 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 98 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 99 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
100 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 101 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
95ca2cb5 102 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
a3508fbe 103 { "instruction_sie", VCPU_STAT(instruction_sie) },
5288fbf0 104 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 105 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 106 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 107 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 108 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
109 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 110 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 111 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
112 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 113 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 114 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
115 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
116 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 117 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
118 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
119 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 120 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 121 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 122 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
175a5c9e
CB		 123 { "diagnose_258", VCPU_STAT(diagnose_258) },
124 { "diagnose_308", VCPU_STAT(diagnose_308) },
125 { "diagnose_500", VCPU_STAT(diagnose_500) },
b0c632db
HC		 126 { NULL }
127};
128
a411edf1
DH		 129/* allow nested virtualization in KVM (if enabled by user space) */
130static int nested;
131module_param(nested, int, S_IRUGO);
132MODULE_PARM_DESC(nested, "Nested virtualization support");
133
9d8d5786 134/* upper facilities limit for kvm */
f6c1d359 135unsigned long kvm_s390_fac_list_mask[16] = { FACILITIES_KVM };
b0c632db 136
9d8d5786 137unsigned long kvm_s390_fac_list_mask_size(void)
78c4b59f 138{
9d8d5786
MM		 139 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
140 return ARRAY_SIZE(kvm_s390_fac_list_mask);
78c4b59f
MM		 141}
142
15c9705f
DH		 143/* available cpu features supported by kvm */
144static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
0a763c78
DH		 145/* available subfunctions indicated via query / "test bit" */
146static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
15c9705f 147
9d8d5786 148static struct gmap_notifier gmap_notifier;
a3508fbe 149static struct gmap_notifier vsie_gmap_notifier;
78f26131 150debug_info_t *kvm_s390_dbf;
9d8d5786 151
b0c632db 152/* Section: not file related */
13a34e06 153int kvm_arch_hardware_enable(void)
b0c632db
HC		 154{
155 /* every s390 is virtualization enabled ;-) */
10474ae8 156 return 0;
b0c632db
HC		 157}
158
414d3b07
MS		 159static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
160 unsigned long end);
2c70fe44 161
fdf03650
FZ		 162/*
163 * This callback is executed during stop_machine(). All CPUs are therefore
164 * temporarily stopped. In order not to change guest behavior, we have to
165 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
166 * so a CPU won't be stopped while calculating with the epoch.
167 */
168static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
169 void *v)
170{
171 struct kvm *kvm;
172 struct kvm_vcpu *vcpu;
173 int i;
174 unsigned long long *delta = v;
175
176 list_for_each_entry(kvm, &vm_list, vm_list) {
177 kvm->arch.epoch -= *delta;
178 kvm_for_each_vcpu(i, vcpu, kvm) {
179 vcpu->arch.sie_block->epoch -= *delta;
db0758b2
DH		 180 if (vcpu->arch.cputm_enabled)
181 vcpu->arch.cputm_start += *delta;
91473b48
DH		 182 if (vcpu->arch.vsie_block)
183 vcpu->arch.vsie_block->epoch -= *delta;
fdf03650
FZ		 184 }
185 }
186 return NOTIFY_OK;
187}
188
189static struct notifier_block kvm_clock_notifier = {
190 .notifier_call = kvm_clock_sync,
191};
192
b0c632db
HC		 193int kvm_arch_hardware_setup(void)
194{
2c70fe44 195 gmap_notifier.notifier_call = kvm_gmap_notifier;
b2d73b2a 196 gmap_register_pte_notifier(&gmap_notifier);
a3508fbe
DH		 197 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
198 gmap_register_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 199 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
200 &kvm_clock_notifier);
b0c632db
HC		 201 return 0;
202}
203
204void kvm_arch_hardware_unsetup(void)
205{
b2d73b2a 206 gmap_unregister_pte_notifier(&gmap_notifier);
a3508fbe 207 gmap_unregister_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 208 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
209 &kvm_clock_notifier);
b0c632db
HC		 210}
211
22be5a13
DH		 212static void allow_cpu_feat(unsigned long nr)
213{
214 set_bit_inv(nr, kvm_s390_available_cpu_feat);
215}
216
0a763c78
DH		 217static inline int plo_test_bit(unsigned char nr)
218{
219 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
d051ae53 220 int cc;
0a763c78
DH		 221
222 asm volatile(
223 /* Parameter registers are ignored for "test bit" */
224 " plo 0,0,0,0(0)\n"
225 " ipm %0\n"
226 " srl %0,28\n"
227 : "=d" (cc)
228 : "d" (r0)
229 : "cc");
230 return cc == 0;
231}
232
22be5a13
DH		 233static void kvm_s390_cpu_feat_init(void)
234{
0a763c78
DH		 235 int i;
236
237 for (i = 0; i < 256; ++i) {
238 if (plo_test_bit(i))
239 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
240 }
241
242 if (test_facility(28)) /* TOD-clock steering */
221bb8a4
LT		 243 ptff(kvm_s390_available_subfunc.ptff,
244 sizeof(kvm_s390_available_subfunc.ptff),
245 PTFF_QAF);
0a763c78
DH		 246
247 if (test_facility(17)) { /* MSA */
69c0e360
MS		 248 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
249 kvm_s390_available_subfunc.kmac);
250 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
251 kvm_s390_available_subfunc.kmc);
252 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
253 kvm_s390_available_subfunc.km);
254 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
255 kvm_s390_available_subfunc.kimd);
256 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
257 kvm_s390_available_subfunc.klmd);
0a763c78
DH		 258 }
259 if (test_facility(76)) /* MSA3 */
69c0e360
MS		 260 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
261 kvm_s390_available_subfunc.pckmo);
0a763c78 262 if (test_facility(77)) { /* MSA4 */
69c0e360
MS		 263 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
264 kvm_s390_available_subfunc.kmctr);
265 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
266 kvm_s390_available_subfunc.kmf);
267 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
268 kvm_s390_available_subfunc.kmo);
269 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
270 kvm_s390_available_subfunc.pcc);
0a763c78
DH		 271 }
272 if (test_facility(57)) /* MSA5 */
69c0e360
MS		 273 __cpacf_query(CPACF_PPNO, (cpacf_mask_t *)
274 kvm_s390_available_subfunc.ppno);
0a763c78 275
22be5a13
DH		 276 if (MACHINE_HAS_ESOP)
277 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
a3508fbe
DH		 278 /*
279 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
280 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
281 */
282 if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
a411edf1 283 !test_facility(3) || !nested)
a3508fbe
DH		 284 return;
285 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
19c439b5
DH		 286 if (sclp.has_64bscao)
287 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
0615a326
DH		 288 if (sclp.has_siif)
289 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
77d18f6d
DH		 290 if (sclp.has_gpere)
291 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
a1b7b9b2
DH		 292 if (sclp.has_gsls)
293 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
5630a8e8
DH		 294 if (sclp.has_ib)
295 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
13ee3f67
DH		 296 if (sclp.has_cei)
297 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
7fd7f39d
DH		 298 if (sclp.has_ibs)
299 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
5d3876a8
DH		 300 /*
301 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
302 * all skey handling functions read/set the skey from the PGSTE
303 * instead of the real storage key.
304 *
305 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
306 * pages being detected as preserved although they are resident.
307 *
308 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
309 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
310 *
311 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
312 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
313 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
314 *
315 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
316 * cannot easily shadow the SCA because of the ipte lock.
317 */
22be5a13
DH		 318}
319
b0c632db
HC		 320int kvm_arch_init(void *opaque)
321{
78f26131
CB		 322 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
323 if (!kvm_s390_dbf)
324 return -ENOMEM;
325
326 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
327 debug_unregister(kvm_s390_dbf);
328 return -ENOMEM;
329 }
330
22be5a13
DH		 331 kvm_s390_cpu_feat_init();
332
84877d93
CH		 333 /* Register floating interrupt controller interface. */
334 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 335}
336
78f26131
CB		 337void kvm_arch_exit(void)
338{
339 debug_unregister(kvm_s390_dbf);
340}
341
b0c632db
HC		 342/* Section: device related */
343long kvm_arch_dev_ioctl(struct file *filp,
344 unsigned int ioctl, unsigned long arg)
345{
346 if (ioctl == KVM_S390_ENABLE_SIE)
347 return s390_enable_sie();
348 return -EINVAL;
349}
350
784aa3d7 351int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 352{
d7b0b5eb
CO		 353 int r;
354
2bd0ac4e 355 switch (ext) {
d7b0b5eb 356 case KVM_CAP_S390_PSW:
b6cf8788 357 case KVM_CAP_S390_GMAP:
52e16b18 358 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 359#ifdef CONFIG_KVM_S390_UCONTROL
360 case KVM_CAP_S390_UCONTROL:
361#endif
3c038e6b 362 case KVM_CAP_ASYNC_PF:
60b413c9 363 case KVM_CAP_SYNC_REGS:
14eebd91 364 case KVM_CAP_ONE_REG:
d6712df9 365 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 366 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 367 case KVM_CAP_IOEVENTFD:
c05c4186 368 case KVM_CAP_DEVICE_CTRL:
d938dc55 369 case KVM_CAP_ENABLE_CAP_VM:
78599d90 370 case KVM_CAP_S390_IRQCHIP:
f2061656 371 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 372 case KVM_CAP_MP_STATE:
47b43c52 373 case KVM_CAP_S390_INJECT_IRQ:
2444b352 374 case KVM_CAP_S390_USER_SIGP:
e44fc8c9 375 case KVM_CAP_S390_USER_STSI:
30ee2a98 376 case KVM_CAP_S390_SKEYS:
816c7667 377 case KVM_CAP_S390_IRQ_STATE:
6502a34c 378 case KVM_CAP_S390_USER_INSTR0:
d7b0b5eb
CO		 379 r = 1;
380 break;
41408c28
TH		 381 case KVM_CAP_S390_MEM_OP:
382 r = MEM_OP_MAX_SIZE;
383 break;
e726b1bd
CB		 384 case KVM_CAP_NR_VCPUS:
385 case KVM_CAP_MAX_VCPUS:
76a6dd72 386 r = KVM_S390_BSCA_CPU_SLOTS;
a6940674
DH		 387 if (!kvm_s390_use_sca_entries())
388 r = KVM_MAX_VCPUS;
389 else if (sclp.has_esca && sclp.has_64bscao)
76a6dd72 390 r = KVM_S390_ESCA_CPU_SLOTS;
e726b1bd 391 break;
e1e2e605
NW		 392 case KVM_CAP_NR_MEMSLOTS:
393 r = KVM_USER_MEM_SLOTS;
394 break;
1526bf9c 395 case KVM_CAP_S390_COW:
abf09bed 396 r = MACHINE_HAS_ESOP;
1526bf9c 397 break;
68c55750
EF		 398 case KVM_CAP_S390_VECTOR_REGISTERS:
399 r = MACHINE_HAS_VX;
400 break;
c6e5f166
FZ		 401 case KVM_CAP_S390_RI:
402 r = test_facility(64);
403 break;
2bd0ac4e 404 default:
d7b0b5eb 405 r = 0;
2bd0ac4e 406 }
d7b0b5eb 407 return r;
b0c632db
HC		 408}
409
15f36ebd
JH		 410static void kvm_s390_sync_dirty_log(struct kvm *kvm,
411 struct kvm_memory_slot *memslot)
412{
413 gfn_t cur_gfn, last_gfn;
414 unsigned long address;
415 struct gmap *gmap = kvm->arch.gmap;
416
15f36ebd
JH		 417 /* Loop over all guest pages */
418 last_gfn = memslot->base_gfn + memslot->npages;
419 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
420 address = gfn_to_hva_memslot(memslot, cur_gfn);
421
1e133ab2 422 if (test_and_clear_guest_dirty(gmap->mm, address))
15f36ebd 423 mark_page_dirty(kvm, cur_gfn);
1763f8d0
CB		 424 if (fatal_signal_pending(current))
425 return;
70c88a00 426 cond_resched();
15f36ebd 427 }
15f36ebd
JH		 428}
429
b0c632db 430/* Section: vm related */
a6e2f683
ED		 431static void sca_del_vcpu(struct kvm_vcpu *vcpu);
432
b0c632db
HC		 433/*
434 * Get (and clear) the dirty memory log for a memory slot.
435 */
436int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
437 struct kvm_dirty_log *log)
438{
15f36ebd
JH		 439 int r;
440 unsigned long n;
9f6b8029 441 struct kvm_memslots *slots;
15f36ebd
JH		 442 struct kvm_memory_slot *memslot;
443 int is_dirty = 0;
444
445 mutex_lock(&kvm->slots_lock);
446
447 r = -EINVAL;
448 if (log->slot >= KVM_USER_MEM_SLOTS)
449 goto out;
450
9f6b8029
PB		 451 slots = kvm_memslots(kvm);
452 memslot = id_to_memslot(slots, log->slot);
15f36ebd
JH		 453 r = -ENOENT;
454 if (!memslot->dirty_bitmap)
455 goto out;
456
457 kvm_s390_sync_dirty_log(kvm, memslot);
458 r = kvm_get_dirty_log(kvm, log, &is_dirty);
459 if (r)
460 goto out;
461
462 /* Clear the dirty log */
463 if (is_dirty) {
464 n = kvm_dirty_bitmap_bytes(memslot);
465 memset(memslot->dirty_bitmap, 0, n);
466 }
467 r = 0;
468out:
469 mutex_unlock(&kvm->slots_lock);
470 return r;
b0c632db
HC		 471}
472
6502a34c
DH		 473static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
474{
475 unsigned int i;
476 struct kvm_vcpu *vcpu;
477
478 kvm_for_each_vcpu(i, vcpu, kvm) {
479 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
480 }
481}
482
d938dc55
CH		 483static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
484{
485 int r;
486
487 if (cap->flags)
488 return -EINVAL;
489
490 switch (cap->cap) {
84223598 491 case KVM_CAP_S390_IRQCHIP:
c92ea7b9 492 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
84223598
CH		 493 kvm->arch.use_irqchip = 1;
494 r = 0;
495 break;
2444b352 496 case KVM_CAP_S390_USER_SIGP:
c92ea7b9 497 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
2444b352
DH		 498 kvm->arch.user_sigp = 1;
499 r = 0;
500 break;
68c55750 501 case KVM_CAP_S390_VECTOR_REGISTERS:
5967c17b 502 mutex_lock(&kvm->lock);
a03825bb 503 if (kvm->created_vcpus) {
5967c17b
DH		 504 r = -EBUSY;
505 } else if (MACHINE_HAS_VX) {
c54f0d6a
DH		 506 set_kvm_facility(kvm->arch.model.fac_mask, 129);
507 set_kvm_facility(kvm->arch.model.fac_list, 129);
2f87d942
GH		 508 if (test_facility(134)) {
509 set_kvm_facility(kvm->arch.model.fac_mask, 134);
510 set_kvm_facility(kvm->arch.model.fac_list, 134);
511 }
53743aa7
MS		 512 if (test_facility(135)) {
513 set_kvm_facility(kvm->arch.model.fac_mask, 135);
514 set_kvm_facility(kvm->arch.model.fac_list, 135);
515 }
18280d8b
MM		 516 r = 0;
517 } else
518 r = -EINVAL;
5967c17b 519 mutex_unlock(&kvm->lock);
c92ea7b9
CB		 520 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
521 r ? "(not available)" : "(success)");
68c55750 522 break;
c6e5f166
FZ		 523 case KVM_CAP_S390_RI:
524 r = -EINVAL;
525 mutex_lock(&kvm->lock);
a03825bb 526 if (kvm->created_vcpus) {
c6e5f166
FZ		 527 r = -EBUSY;
528 } else if (test_facility(64)) {
c54f0d6a
DH		 529 set_kvm_facility(kvm->arch.model.fac_mask, 64);
530 set_kvm_facility(kvm->arch.model.fac_list, 64);
c6e5f166
FZ		 531 r = 0;
532 }
533 mutex_unlock(&kvm->lock);
534 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
535 r ? "(not available)" : "(success)");
536 break;
e44fc8c9 537 case KVM_CAP_S390_USER_STSI:
c92ea7b9 538 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
e44fc8c9
ET		 539 kvm->arch.user_stsi = 1;
540 r = 0;
541 break;
6502a34c
DH		 542 case KVM_CAP_S390_USER_INSTR0:
543 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
544 kvm->arch.user_instr0 = 1;
545 icpt_operexc_on_all_vcpus(kvm);
546 r = 0;
547 break;
d938dc55
CH		 548 default:
549 r = -EINVAL;
550 break;
551 }
552 return r;
553}
554
8c0a7ce6
DD		 555static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
556{
557 int ret;
558
559 switch (attr->attr) {
560 case KVM_S390_VM_MEM_LIMIT_SIZE:
561 ret = 0;
c92ea7b9 562 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
a3a92c31
DD		 563 kvm->arch.mem_limit);
564 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
8c0a7ce6
DD		 565 ret = -EFAULT;
566 break;
567 default:
568 ret = -ENXIO;
569 break;
570 }
571 return ret;
572}
573
574static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 575{
576 int ret;
577 unsigned int idx;
578 switch (attr->attr) {
579 case KVM_S390_VM_MEM_ENABLE_CMMA:
f9cbd9b0 580 ret = -ENXIO;
c24cc9c8 581 if (!sclp.has_cmma)
e6db1d61
DD		 582 break;
583
4f718eab 584 ret = -EBUSY;
c92ea7b9 585 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
4f718eab 586 mutex_lock(&kvm->lock);
a03825bb 587 if (!kvm->created_vcpus) {
4f718eab
DD		 588 kvm->arch.use_cmma = 1;
589 ret = 0;
590 }
591 mutex_unlock(&kvm->lock);
592 break;
593 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 594 ret = -ENXIO;
595 if (!sclp.has_cmma)
596 break;
c3489155
DD		 597 ret = -EINVAL;
598 if (!kvm->arch.use_cmma)
599 break;
600
c92ea7b9 601 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
4f718eab
DD		 602 mutex_lock(&kvm->lock);
603 idx = srcu_read_lock(&kvm->srcu);
a13cff31 604 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 605 srcu_read_unlock(&kvm->srcu, idx);
606 mutex_unlock(&kvm->lock);
607 ret = 0;
608 break;
8c0a7ce6
DD		 609 case KVM_S390_VM_MEM_LIMIT_SIZE: {
610 unsigned long new_limit;
611
612 if (kvm_is_ucontrol(kvm))
613 return -EINVAL;
614
615 if (get_user(new_limit, (u64 __user *)attr->addr))
616 return -EFAULT;
617
a3a92c31
DD		 618 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
619 new_limit > kvm->arch.mem_limit)
8c0a7ce6
DD		 620 return -E2BIG;
621
a3a92c31
DD		 622 if (!new_limit)
623 return -EINVAL;
624
6ea427bb 625 /* gmap_create takes last usable address */
a3a92c31
DD		 626 if (new_limit != KVM_S390_NO_MEM_LIMIT)
627 new_limit -= 1;
628
8c0a7ce6
DD		 629 ret = -EBUSY;
630 mutex_lock(&kvm->lock);
a03825bb 631 if (!kvm->created_vcpus) {
6ea427bb
MS		 632 /* gmap_create will round the limit up */
633 struct gmap *new = gmap_create(current->mm, new_limit);
8c0a7ce6
DD		 634
635 if (!new) {
636 ret = -ENOMEM;
637 } else {
6ea427bb 638 gmap_remove(kvm->arch.gmap);
8c0a7ce6
DD		 639 new->private = kvm;
640 kvm->arch.gmap = new;
641 ret = 0;
642 }
643 }
644 mutex_unlock(&kvm->lock);
a3a92c31
DD		 645 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
646 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
647 (void *) kvm->arch.gmap->asce);
8c0a7ce6
DD		 648 break;
649 }
4f718eab
DD		 650 default:
651 ret = -ENXIO;
652 break;
653 }
654 return ret;
655}
656
a374e892
TK		 657static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
658
659static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
660{
661 struct kvm_vcpu *vcpu;
662 int i;
663
9d8d5786 664 if (!test_kvm_facility(kvm, 76))
a374e892
TK		 665 return -EINVAL;
666
667 mutex_lock(&kvm->lock);
668 switch (attr->attr) {
669 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
670 get_random_bytes(
671 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
672 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
673 kvm->arch.crypto.aes_kw = 1;
c92ea7b9 674 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
a374e892
TK		 675 break;
676 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
677 get_random_bytes(
678 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
679 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
680 kvm->arch.crypto.dea_kw = 1;
c92ea7b9 681 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
a374e892
TK		 682 break;
683 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
684 kvm->arch.crypto.aes_kw = 0;
685 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
686 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
c92ea7b9 687 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
a374e892
TK		 688 break;
689 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
690 kvm->arch.crypto.dea_kw = 0;
691 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
692 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
c92ea7b9 693 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
a374e892
TK		 694 break;
695 default:
696 mutex_unlock(&kvm->lock);
697 return -ENXIO;
698 }
699
700 kvm_for_each_vcpu(i, vcpu, kvm) {
701 kvm_s390_vcpu_crypto_setup(vcpu);
702 exit_sie(vcpu);
703 }
704 mutex_unlock(&kvm->lock);
705 return 0;
706}
707
72f25020
JH		 708static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
709{
710 u8 gtod_high;
711
712 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
713 sizeof(gtod_high)))
714 return -EFAULT;
715
716 if (gtod_high != 0)
717 return -EINVAL;
58c383c6 718 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
72f25020
JH		 719
720 return 0;
721}
722
723static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
724{
5a3d883a 725 u64 gtod;
72f25020
JH		 726
727 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
728 return -EFAULT;
729
25ed1675 730 kvm_s390_set_tod_clock(kvm, gtod);
58c383c6 731 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
72f25020
JH		 732 return 0;
733}
734
735static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
736{
737 int ret;
738
739 if (attr->flags)
740 return -EINVAL;
741
742 switch (attr->attr) {
743 case KVM_S390_VM_TOD_HIGH:
744 ret = kvm_s390_set_tod_high(kvm, attr);
745 break;
746 case KVM_S390_VM_TOD_LOW:
747 ret = kvm_s390_set_tod_low(kvm, attr);
748 break;
749 default:
750 ret = -ENXIO;
751 break;
752 }
753 return ret;
754}
755
756static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
757{
758 u8 gtod_high = 0;
759
760 if (copy_to_user((void __user *)attr->addr, &gtod_high,
761 sizeof(gtod_high)))
762 return -EFAULT;
58c383c6 763 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
72f25020
JH		 764
765 return 0;
766}
767
768static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
769{
5a3d883a 770 u64 gtod;
72f25020 771
60417fcc 772 gtod = kvm_s390_get_tod_clock_fast(kvm);
72f25020
JH		 773 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
774 return -EFAULT;
58c383c6 775 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
72f25020
JH		 776
777 return 0;
778}
779
780static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
781{
782 int ret;
783
784 if (attr->flags)
785 return -EINVAL;
786
787 switch (attr->attr) {
788 case KVM_S390_VM_TOD_HIGH:
789 ret = kvm_s390_get_tod_high(kvm, attr);
790 break;
791 case KVM_S390_VM_TOD_LOW:
792 ret = kvm_s390_get_tod_low(kvm, attr);
793 break;
794 default:
795 ret = -ENXIO;
796 break;
797 }
798 return ret;
799}
800
658b6eda
MM		 801static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
802{
803 struct kvm_s390_vm_cpu_processor *proc;
053dd230 804 u16 lowest_ibc, unblocked_ibc;
658b6eda
MM		 805 int ret = 0;
806
807 mutex_lock(&kvm->lock);
a03825bb 808 if (kvm->created_vcpus) {
658b6eda
MM		 809 ret = -EBUSY;
810 goto out;
811 }
812 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
813 if (!proc) {
814 ret = -ENOMEM;
815 goto out;
816 }
817 if (!copy_from_user(proc, (void __user *)attr->addr,
818 sizeof(*proc))) {
9bb0ec09 819 kvm->arch.model.cpuid = proc->cpuid;
053dd230
DH		 820 lowest_ibc = sclp.ibc >> 16 & 0xfff;
821 unblocked_ibc = sclp.ibc & 0xfff;
0487c44d 822 if (lowest_ibc && proc->ibc) {
053dd230
DH		 823 if (proc->ibc > unblocked_ibc)
824 kvm->arch.model.ibc = unblocked_ibc;
825 else if (proc->ibc < lowest_ibc)
826 kvm->arch.model.ibc = lowest_ibc;
827 else
828 kvm->arch.model.ibc = proc->ibc;
829 }
c54f0d6a 830 memcpy(kvm->arch.model.fac_list, proc->fac_list,
658b6eda
MM		 831 S390_ARCH_FAC_LIST_SIZE_BYTE);
832 } else
833 ret = -EFAULT;
834 kfree(proc);
835out:
836 mutex_unlock(&kvm->lock);
837 return ret;
838}
839
15c9705f
DH		 840static int kvm_s390_set_processor_feat(struct kvm *kvm,
841 struct kvm_device_attr *attr)
842{
843 struct kvm_s390_vm_cpu_feat data;
844 int ret = -EBUSY;
845
846 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
847 return -EFAULT;
848 if (!bitmap_subset((unsigned long *) data.feat,
849 kvm_s390_available_cpu_feat,
850 KVM_S390_VM_CPU_FEAT_NR_BITS))
851 return -EINVAL;
852
853 mutex_lock(&kvm->lock);
854 if (!atomic_read(&kvm->online_vcpus)) {
855 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
856 KVM_S390_VM_CPU_FEAT_NR_BITS);
857 ret = 0;
858 }
859 mutex_unlock(&kvm->lock);
860 return ret;
861}
862
0a763c78
DH		 863static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
864 struct kvm_device_attr *attr)
865{
866 /*
867 * Once supported by kernel + hw, we have to store the subfunctions
868 * in kvm->arch and remember that user space configured them.
869 */
870 return -ENXIO;
871}
872
658b6eda
MM		 873static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
874{
875 int ret = -ENXIO;
876
877 switch (attr->attr) {
878 case KVM_S390_VM_CPU_PROCESSOR:
879 ret = kvm_s390_set_processor(kvm, attr);
880 break;
15c9705f
DH		 881 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
882 ret = kvm_s390_set_processor_feat(kvm, attr);
883 break;
0a763c78
DH		 884 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
885 ret = kvm_s390_set_processor_subfunc(kvm, attr);
886 break;
658b6eda
MM		 887 }
888 return ret;
889}
890
891static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
892{
893 struct kvm_s390_vm_cpu_processor *proc;
894 int ret = 0;
895
896 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
897 if (!proc) {
898 ret = -ENOMEM;
899 goto out;
900 }
9bb0ec09 901 proc->cpuid = kvm->arch.model.cpuid;
658b6eda 902 proc->ibc = kvm->arch.model.ibc;
c54f0d6a
DH		 903 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
904 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 905 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
906 ret = -EFAULT;
907 kfree(proc);
908out:
909 return ret;
910}
911
912static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
913{
914 struct kvm_s390_vm_cpu_machine *mach;
915 int ret = 0;
916
917 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
918 if (!mach) {
919 ret = -ENOMEM;
920 goto out;
921 }
922 get_cpu_id((struct cpuid *) &mach->cpuid);
37c5f6c8 923 mach->ibc = sclp.ibc;
c54f0d6a 924 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
981467c9 925 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 926 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
04478197 927 sizeof(S390_lowcore.stfle_fac_list));
658b6eda
MM		 928 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
929 ret = -EFAULT;
930 kfree(mach);
931out:
932 return ret;
933}
934
15c9705f
DH		 935static int kvm_s390_get_processor_feat(struct kvm *kvm,
936 struct kvm_device_attr *attr)
937{
938 struct kvm_s390_vm_cpu_feat data;
939
940 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
941 KVM_S390_VM_CPU_FEAT_NR_BITS);
942 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
943 return -EFAULT;
944 return 0;
945}
946
947static int kvm_s390_get_machine_feat(struct kvm *kvm,
948 struct kvm_device_attr *attr)
949{
950 struct kvm_s390_vm_cpu_feat data;
951
952 bitmap_copy((unsigned long *) data.feat,
953 kvm_s390_available_cpu_feat,
954 KVM_S390_VM_CPU_FEAT_NR_BITS);
955 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
956 return -EFAULT;
957 return 0;
958}
959
0a763c78
DH		 960static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
961 struct kvm_device_attr *attr)
962{
963 /*
964 * Once we can actually configure subfunctions (kernel + hw support),
965 * we have to check if they were already set by user space, if so copy
966 * them from kvm->arch.
967 */
968 return -ENXIO;
969}
970
971static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
972 struct kvm_device_attr *attr)
973{
974 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
975 sizeof(struct kvm_s390_vm_cpu_subfunc)))
976 return -EFAULT;
977 return 0;
978}
658b6eda
MM		 979static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
980{
981 int ret = -ENXIO;
982
983 switch (attr->attr) {
984 case KVM_S390_VM_CPU_PROCESSOR:
985 ret = kvm_s390_get_processor(kvm, attr);
986 break;
987 case KVM_S390_VM_CPU_MACHINE:
988 ret = kvm_s390_get_machine(kvm, attr);
989 break;
15c9705f
DH		 990 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
991 ret = kvm_s390_get_processor_feat(kvm, attr);
992 break;
993 case KVM_S390_VM_CPU_MACHINE_FEAT:
994 ret = kvm_s390_get_machine_feat(kvm, attr);
995 break;
0a763c78
DH		 996 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
997 ret = kvm_s390_get_processor_subfunc(kvm, attr);
998 break;
999 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1000 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1001 break;
658b6eda
MM		 1002 }
1003 return ret;
1004}
1005
f2061656
DD		 1006static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1007{
1008 int ret;
1009
1010 switch (attr->group) {
4f718eab 1011 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 1012 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 1013 break;
72f25020
JH		 1014 case KVM_S390_VM_TOD:
1015 ret = kvm_s390_set_tod(kvm, attr);
1016 break;
658b6eda
MM		 1017 case KVM_S390_VM_CPU_MODEL:
1018 ret = kvm_s390_set_cpu_model(kvm, attr);
1019 break;
a374e892
TK		 1020 case KVM_S390_VM_CRYPTO:
1021 ret = kvm_s390_vm_set_crypto(kvm, attr);
1022 break;
f2061656
DD		 1023 default:
1024 ret = -ENXIO;
1025 break;
1026 }
1027
1028 return ret;
1029}
1030
1031static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1032{
8c0a7ce6
DD		 1033 int ret;
1034
1035 switch (attr->group) {
1036 case KVM_S390_VM_MEM_CTRL:
1037 ret = kvm_s390_get_mem_control(kvm, attr);
1038 break;
72f25020
JH		 1039 case KVM_S390_VM_TOD:
1040 ret = kvm_s390_get_tod(kvm, attr);
1041 break;
658b6eda
MM		 1042 case KVM_S390_VM_CPU_MODEL:
1043 ret = kvm_s390_get_cpu_model(kvm, attr);
1044 break;
8c0a7ce6
DD		 1045 default:
1046 ret = -ENXIO;
1047 break;
1048 }
1049
1050 return ret;
f2061656
DD		 1051}
1052
1053static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1054{
1055 int ret;
1056
1057 switch (attr->group) {
4f718eab
DD		 1058 case KVM_S390_VM_MEM_CTRL:
1059 switch (attr->attr) {
1060 case KVM_S390_VM_MEM_ENABLE_CMMA:
1061 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 1062 ret = sclp.has_cmma ? 0 : -ENXIO;
1063 break;
8c0a7ce6 1064 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 1065 ret = 0;
1066 break;
1067 default:
1068 ret = -ENXIO;
1069 break;
1070 }
1071 break;
72f25020
JH		 1072 case KVM_S390_VM_TOD:
1073 switch (attr->attr) {
1074 case KVM_S390_VM_TOD_LOW:
1075 case KVM_S390_VM_TOD_HIGH:
1076 ret = 0;
1077 break;
1078 default:
1079 ret = -ENXIO;
1080 break;
1081 }
1082 break;
658b6eda
MM		 1083 case KVM_S390_VM_CPU_MODEL:
1084 switch (attr->attr) {
1085 case KVM_S390_VM_CPU_PROCESSOR:
1086 case KVM_S390_VM_CPU_MACHINE:
15c9705f
DH		 1087 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1088 case KVM_S390_VM_CPU_MACHINE_FEAT:
0a763c78 1089 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
658b6eda
MM		 1090 ret = 0;
1091 break;
0a763c78
DH		 1092 /* configuring subfunctions is not supported yet */
1093 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
658b6eda
MM		 1094 default:
1095 ret = -ENXIO;
1096 break;
1097 }
1098 break;
a374e892
TK		 1099 case KVM_S390_VM_CRYPTO:
1100 switch (attr->attr) {
1101 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1102 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1103 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1104 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1105 ret = 0;
1106 break;
1107 default:
1108 ret = -ENXIO;
1109 break;
1110 }
1111 break;
f2061656
DD		 1112 default:
1113 ret = -ENXIO;
1114 break;
1115 }
1116
1117 return ret;
1118}
1119
30ee2a98
JH		 1120static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1121{
1122 uint8_t *keys;
1123 uint64_t hva;
30ee2a98
JH		 1124 int i, r = 0;
1125
1126 if (args->flags != 0)
1127 return -EINVAL;
1128
1129 /* Is this guest using storage keys? */
1130 if (!mm_use_skey(current->mm))
1131 return KVM_S390_GET_SKEYS_NONE;
1132
1133 /* Enforce sane limit on memory allocation */
1134 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1135 return -EINVAL;
1136
1137 keys = kmalloc_array(args->count, sizeof(uint8_t),
1138 GFP_KERNEL | __GFP_NOWARN);
1139 if (!keys)
1140 keys = vmalloc(sizeof(uint8_t) * args->count);
1141 if (!keys)
1142 return -ENOMEM;
1143
d3ed1cee 1144 down_read(&current->mm->mmap_sem);
30ee2a98
JH		 1145 for (i = 0; i < args->count; i++) {
1146 hva = gfn_to_hva(kvm, args->start_gfn + i);
1147 if (kvm_is_error_hva(hva)) {
1148 r = -EFAULT;
d3ed1cee 1149 break;
30ee2a98
JH		 1150 }
1151
154c8c19
DH		 1152 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1153 if (r)
d3ed1cee 1154 break;
30ee2a98 1155 }
d3ed1cee
MS		 1156 up_read(&current->mm->mmap_sem);
1157
1158 if (!r) {
1159 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1160 sizeof(uint8_t) * args->count);
1161 if (r)
1162 r = -EFAULT;
30ee2a98
JH		 1163 }
1164
30ee2a98
JH		 1165 kvfree(keys);
1166 return r;
1167}
1168
1169static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1170{
1171 uint8_t *keys;
1172 uint64_t hva;
1173 int i, r = 0;
1174
1175 if (args->flags != 0)
1176 return -EINVAL;
1177
1178 /* Enforce sane limit on memory allocation */
1179 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1180 return -EINVAL;
1181
1182 keys = kmalloc_array(args->count, sizeof(uint8_t),
1183 GFP_KERNEL | __GFP_NOWARN);
1184 if (!keys)
1185 keys = vmalloc(sizeof(uint8_t) * args->count);
1186 if (!keys)
1187 return -ENOMEM;
1188
1189 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1190 sizeof(uint8_t) * args->count);
1191 if (r) {
1192 r = -EFAULT;
1193 goto out;
1194 }
1195
1196 /* Enable storage key handling for the guest */
14d4a425
DD		 1197 r = s390_enable_skey();
1198 if (r)
1199 goto out;
30ee2a98 1200
d3ed1cee 1201 down_read(&current->mm->mmap_sem);
30ee2a98
JH		 1202 for (i = 0; i < args->count; i++) {
1203 hva = gfn_to_hva(kvm, args->start_gfn + i);
1204 if (kvm_is_error_hva(hva)) {
1205 r = -EFAULT;
d3ed1cee 1206 break;
30ee2a98
JH		 1207 }
1208
1209 /* Lowest order bit is reserved */
1210 if (keys[i] & 0x01) {
1211 r = -EINVAL;
d3ed1cee 1212 break;
30ee2a98
JH		 1213 }
1214
fe69eabf 1215 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
30ee2a98 1216 if (r)
d3ed1cee 1217 break;
30ee2a98 1218 }
d3ed1cee 1219 up_read(&current->mm->mmap_sem);
30ee2a98
JH		 1220out:
1221 kvfree(keys);
1222 return r;
1223}
1224
b0c632db
HC		 1225long kvm_arch_vm_ioctl(struct file *filp,
1226 unsigned int ioctl, unsigned long arg)
1227{
1228 struct kvm *kvm = filp->private_data;
1229 void __user *argp = (void __user *)arg;
f2061656 1230 struct kvm_device_attr attr;
b0c632db
HC		 1231 int r;
1232
1233 switch (ioctl) {
ba5c1e9b
CO		 1234 case KVM_S390_INTERRUPT: {
1235 struct kvm_s390_interrupt s390int;
1236
1237 r = -EFAULT;
1238 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1239 break;
1240 r = kvm_s390_inject_vm(kvm, &s390int);
1241 break;
1242 }
d938dc55
CH		 1243 case KVM_ENABLE_CAP: {
1244 struct kvm_enable_cap cap;
1245 r = -EFAULT;
1246 if (copy_from_user(&cap, argp, sizeof(cap)))
1247 break;
1248 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1249 break;
1250 }
84223598
CH		 1251 case KVM_CREATE_IRQCHIP: {
1252 struct kvm_irq_routing_entry routing;
1253
1254 r = -EINVAL;
1255 if (kvm->arch.use_irqchip) {
1256 /* Set up dummy routing. */
1257 memset(&routing, 0, sizeof(routing));
152b2839 1258 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
84223598
CH		 1259 }
1260 break;
1261 }
f2061656
DD		 1262 case KVM_SET_DEVICE_ATTR: {
1263 r = -EFAULT;
1264 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1265 break;
1266 r = kvm_s390_vm_set_attr(kvm, &attr);
1267 break;
1268 }
1269 case KVM_GET_DEVICE_ATTR: {
1270 r = -EFAULT;
1271 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1272 break;
1273 r = kvm_s390_vm_get_attr(kvm, &attr);
1274 break;
1275 }
1276 case KVM_HAS_DEVICE_ATTR: {
1277 r = -EFAULT;
1278 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1279 break;
1280 r = kvm_s390_vm_has_attr(kvm, &attr);
1281 break;
1282 }
30ee2a98
JH		 1283 case KVM_S390_GET_SKEYS: {
1284 struct kvm_s390_skeys args;
1285
1286 r = -EFAULT;
1287 if (copy_from_user(&args, argp,
1288 sizeof(struct kvm_s390_skeys)))
1289 break;
1290 r = kvm_s390_get_skeys(kvm, &args);
1291 break;
1292 }
1293 case KVM_S390_SET_SKEYS: {
1294 struct kvm_s390_skeys args;
1295
1296 r = -EFAULT;
1297 if (copy_from_user(&args, argp,
1298 sizeof(struct kvm_s390_skeys)))
1299 break;
1300 r = kvm_s390_set_skeys(kvm, &args);
1301 break;
1302 }
b0c632db 1303 default:
367e1319 1304 r = -ENOTTY;
b0c632db
HC		 1305 }
1306
1307 return r;
1308}
1309
45c9b47c
TK		 1310static int kvm_s390_query_ap_config(u8 *config)
1311{
1312 u32 fcn_code = 0x04000000UL;
86044c8c 1313 u32 cc = 0;
45c9b47c 1314
86044c8c 1315 memset(config, 0, 128);
45c9b47c
TK		 1316 asm volatile(
1317 "lgr 0,%1\n"
1318 "lgr 2,%2\n"
1319 ".long 0xb2af0000\n" /* PQAP(QCI) */
86044c8c 1320 "0: ipm %0\n"
45c9b47c 1321 "srl %0,28\n"
86044c8c
CB		 1322 "1:\n"
1323 EX_TABLE(0b, 1b)
1324 : "+r" (cc)
45c9b47c
TK		 1325 : "r" (fcn_code), "r" (config)
1326 : "cc", "0", "2", "memory"
1327 );
1328
1329 return cc;
1330}
1331
1332static int kvm_s390_apxa_installed(void)
1333{
1334 u8 config[128];
1335 int cc;
1336
a6aacc3f 1337 if (test_facility(12)) {
45c9b47c
TK		 1338 cc = kvm_s390_query_ap_config(config);
1339
1340 if (cc)
1341 pr_err("PQAP(QCI) failed with cc=%d", cc);
1342 else
1343 return config[0] & 0x40;
1344 }
1345
1346 return 0;
1347}
1348
1349static void kvm_s390_set_crycb_format(struct kvm *kvm)
1350{
1351 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1352
1353 if (kvm_s390_apxa_installed())
1354 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1355 else
1356 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1357}
1358
9bb0ec09 1359static u64 kvm_s390_get_initial_cpuid(void)
9d8d5786 1360{
9bb0ec09
DH		 1361 struct cpuid cpuid;
1362
1363 get_cpu_id(&cpuid);
1364 cpuid.version = 0xff;
1365 return *((u64 *) &cpuid);
9d8d5786
MM		 1366}
1367
c54f0d6a 1368static void kvm_s390_crypto_init(struct kvm *kvm)
5102ee87 1369{
9d8d5786 1370 if (!test_kvm_facility(kvm, 76))
c54f0d6a 1371 return;
5102ee87 1372
c54f0d6a 1373 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
45c9b47c 1374 kvm_s390_set_crycb_format(kvm);
5102ee87 1375
ed6f76b4
TK		 1376 /* Enable AES/DEA protected key functions by default */
1377 kvm->arch.crypto.aes_kw = 1;
1378 kvm->arch.crypto.dea_kw = 1;
1379 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1380 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1381 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1382 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
5102ee87
TK		 1383}
1384
7d43bafc
ED		 1385static void sca_dispose(struct kvm *kvm)
1386{
1387 if (kvm->arch.use_esca)
5e044315 1388 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
7d43bafc
ED		 1389 else
1390 free_page((unsigned long)(kvm->arch.sca));
1391 kvm->arch.sca = NULL;
1392}
1393
e08b9637 1394int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 1395{
76a6dd72 1396 gfp_t alloc_flags = GFP_KERNEL;
9d8d5786 1397 int i, rc;
b0c632db 1398 char debug_name[16];
f6c137ff 1399 static unsigned long sca_offset;
b0c632db 1400
e08b9637
CO		 1401 rc = -EINVAL;
1402#ifdef CONFIG_KVM_S390_UCONTROL
1403 if (type & ~KVM_VM_S390_UCONTROL)
1404 goto out_err;
1405 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1406 goto out_err;
1407#else
1408 if (type)
1409 goto out_err;
1410#endif
1411
b0c632db
HC		 1412 rc = s390_enable_sie();
1413 if (rc)
d89f5eff 1414 goto out_err;
b0c632db 1415
b290411a
CO		 1416 rc = -ENOMEM;
1417
7d0a5e62
JF		 1418 ratelimit_state_init(&kvm->arch.sthyi_limit, 5 * HZ, 500);
1419
7d43bafc 1420 kvm->arch.use_esca = 0; /* start with basic SCA */
76a6dd72
DH		 1421 if (!sclp.has_64bscao)
1422 alloc_flags |= GFP_DMA;
5e044315 1423 rwlock_init(&kvm->arch.sca_lock);
76a6dd72 1424 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
b0c632db 1425 if (!kvm->arch.sca)
d89f5eff 1426 goto out_err;
f6c137ff 1427 spin_lock(&kvm_lock);
c5c2c393 1428 sca_offset += 16;
bc784cce 1429 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
c5c2c393 1430 sca_offset = 0;
bc784cce
ED		 1431 kvm->arch.sca = (struct bsca_block *)
1432 ((char *) kvm->arch.sca + sca_offset);
f6c137ff 1433 spin_unlock(&kvm_lock);
b0c632db
HC		 1434
1435 sprintf(debug_name, "kvm-%u", current->pid);
1436
1cb9cf72 1437 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
b0c632db 1438 if (!kvm->arch.dbf)
40f5b735 1439 goto out_err;
b0c632db 1440
c54f0d6a
DH		 1441 kvm->arch.sie_page2 =
1442 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1443 if (!kvm->arch.sie_page2)
40f5b735 1444 goto out_err;
9d8d5786 1445
fb5bf93f 1446 /* Populate the facility mask initially. */
c54f0d6a 1447 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
04478197 1448 sizeof(S390_lowcore.stfle_fac_list));
9d8d5786
MM		 1449 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1450 if (i < kvm_s390_fac_list_mask_size())
c54f0d6a 1451 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
9d8d5786 1452 else
c54f0d6a 1453 kvm->arch.model.fac_mask[i] = 0UL;
9d8d5786
MM		 1454 }
1455
981467c9 1456 /* Populate the facility list initially. */
c54f0d6a
DH		 1457 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1458 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
981467c9
MM		 1459 S390_ARCH_FAC_LIST_SIZE_BYTE);
1460
95ca2cb5
JF		 1461 set_kvm_facility(kvm->arch.model.fac_mask, 74);
1462 set_kvm_facility(kvm->arch.model.fac_list, 74);
1463
9bb0ec09 1464 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
37c5f6c8 1465 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
9d8d5786 1466
c54f0d6a 1467 kvm_s390_crypto_init(kvm);
5102ee87 1468
ba5c1e9b 1469 spin_lock_init(&kvm->arch.float_int.lock);
6d3da241
JF		 1470 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1471 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
8a242234 1472 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 1473 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 1474
b0c632db 1475 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
78f26131 1476 VM_EVENT(kvm, 3, "vm created with type %lu", type);
b0c632db 1477
e08b9637
CO		 1478 if (type & KVM_VM_S390_UCONTROL) {
1479 kvm->arch.gmap = NULL;
a3a92c31 1480 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
e08b9637 1481 } else {
32e6b236
GH		 1482 if (sclp.hamax == U64_MAX)
1483 kvm->arch.mem_limit = TASK_MAX_SIZE;
1484 else
1485 kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
1486 sclp.hamax + 1);
6ea427bb 1487 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
e08b9637 1488 if (!kvm->arch.gmap)
40f5b735 1489 goto out_err;
2c70fe44 1490 kvm->arch.gmap->private = kvm;
24eb3a82 1491 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 1492 }
fa6b7fe9
CH		 1493
1494 kvm->arch.css_support = 0;
84223598 1495 kvm->arch.use_irqchip = 0;
72f25020 1496 kvm->arch.epoch = 0;
fa6b7fe9 1497
8ad35755 1498 spin_lock_init(&kvm->arch.start_stop_lock);
a3508fbe 1499 kvm_s390_vsie_init(kvm);
8335713a 1500 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
8ad35755 1501
d89f5eff 1502 return 0;
40f5b735 1503out_err:
c54f0d6a 1504 free_page((unsigned long)kvm->arch.sie_page2);
598841ca 1505 debug_unregister(kvm->arch.dbf);
7d43bafc 1506 sca_dispose(kvm);
78f26131 1507 KVM_EVENT(3, "creation of vm failed: %d", rc);
d89f5eff 1508 return rc;
b0c632db
HC		 1509}
1510
235539b4
LC		 1511bool kvm_arch_has_vcpu_debugfs(void)
1512{
1513 return false;
1514}
1515
1516int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
1517{
1518 return 0;
1519}
1520
d329c035
CB		 1521void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1522{
1523 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 1524 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 1525 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 1526 kvm_clear_async_pf_completion_queue(vcpu);
bc784cce 1527 if (!kvm_is_ucontrol(vcpu->kvm))
a6e2f683 1528 sca_del_vcpu(vcpu);
27e0393f
CO		 1529
1530 if (kvm_is_ucontrol(vcpu->kvm))
6ea427bb 1531 gmap_remove(vcpu->arch.gmap);
27e0393f 1532
e6db1d61 1533 if (vcpu->kvm->arch.use_cmma)
b31605c1 1534 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 1535 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 1536
6692cef3 1537 kvm_vcpu_uninit(vcpu);
b110feaf 1538 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 1539}
1540
1541static void kvm_free_vcpus(struct kvm *kvm)
1542{
1543 unsigned int i;
988a2cae 1544 struct kvm_vcpu *vcpu;
d329c035 1545
988a2cae
GN		 1546 kvm_for_each_vcpu(i, vcpu, kvm)
1547 kvm_arch_vcpu_destroy(vcpu);
1548
1549 mutex_lock(&kvm->lock);
1550 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1551 kvm->vcpus[i] = NULL;
1552
1553 atomic_set(&kvm->online_vcpus, 0);
1554 mutex_unlock(&kvm->lock);
d329c035
CB		 1555}
1556
b0c632db
HC		 1557void kvm_arch_destroy_vm(struct kvm *kvm)
1558{
d329c035 1559 kvm_free_vcpus(kvm);
7d43bafc 1560 sca_dispose(kvm);
d329c035 1561 debug_unregister(kvm->arch.dbf);
c54f0d6a 1562 free_page((unsigned long)kvm->arch.sie_page2);
27e0393f 1563 if (!kvm_is_ucontrol(kvm))
6ea427bb 1564 gmap_remove(kvm->arch.gmap);
841b91c5 1565 kvm_s390_destroy_adapters(kvm);
67335e63 1566 kvm_s390_clear_float_irqs(kvm);
a3508fbe 1567 kvm_s390_vsie_destroy(kvm);
8335713a 1568 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
b0c632db
HC		 1569}
1570
1571/* Section: vcpu related */
dafd032a
DD		 1572static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1573{
6ea427bb 1574 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
dafd032a
DD		 1575 if (!vcpu->arch.gmap)
1576 return -ENOMEM;
1577 vcpu->arch.gmap->private = vcpu->kvm;
1578
1579 return 0;
1580}
1581
a6e2f683
ED		 1582static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1583{
a6940674
DH		 1584 if (!kvm_s390_use_sca_entries())
1585 return;
5e044315 1586 read_lock(&vcpu->kvm->arch.sca_lock);
7d43bafc
ED		 1587 if (vcpu->kvm->arch.use_esca) {
1588 struct esca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1589
7d43bafc 1590 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
10ce32d5 1591 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc
ED		 1592 } else {
1593 struct bsca_block *sca = vcpu->kvm->arch.sca;
1594
1595 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
10ce32d5 1596 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc 1597 }
5e044315 1598 read_unlock(&vcpu->kvm->arch.sca_lock);
a6e2f683
ED		 1599}
1600
eaa78f34 1601static void sca_add_vcpu(struct kvm_vcpu *vcpu)
a6e2f683 1602{
a6940674
DH		 1603 if (!kvm_s390_use_sca_entries()) {
1604 struct bsca_block *sca = vcpu->kvm->arch.sca;
1605
1606 /* we still need the basic sca for the ipte control */
1607 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1608 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
1609 }
eaa78f34
DH		 1610 read_lock(&vcpu->kvm->arch.sca_lock);
1611 if (vcpu->kvm->arch.use_esca) {
1612 struct esca_block *sca = vcpu->kvm->arch.sca;
7d43bafc 1613
eaa78f34 1614 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1615 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1616 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
25508824 1617 vcpu->arch.sie_block->ecb2 |= 0x04U;
eaa78f34 1618 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
7d43bafc 1619 } else {
eaa78f34 1620 struct bsca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1621
eaa78f34 1622 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1623 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1624 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
eaa78f34 1625 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
7d43bafc 1626 }
eaa78f34 1627 read_unlock(&vcpu->kvm->arch.sca_lock);
5e044315
ED		 1628}
1629
1630/* Basic SCA to Extended SCA data copy routines */
1631static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1632{
1633 d->sda = s->sda;
1634 d->sigp_ctrl.c = s->sigp_ctrl.c;
1635 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1636}
1637
1638static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1639{
1640 int i;
1641
1642 d->ipte_control = s->ipte_control;
1643 d->mcn[0] = s->mcn;
1644 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1645 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1646}
1647
1648static int sca_switch_to_extended(struct kvm *kvm)
1649{
1650 struct bsca_block *old_sca = kvm->arch.sca;
1651 struct esca_block *new_sca;
1652 struct kvm_vcpu *vcpu;
1653 unsigned int vcpu_idx;
1654 u32 scaol, scaoh;
1655
1656 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1657 if (!new_sca)
1658 return -ENOMEM;
1659
1660 scaoh = (u32)((u64)(new_sca) >> 32);
1661 scaol = (u32)(u64)(new_sca) & ~0x3fU;
1662
1663 kvm_s390_vcpu_block_all(kvm);
1664 write_lock(&kvm->arch.sca_lock);
1665
1666 sca_copy_b_to_e(new_sca, old_sca);
1667
1668 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1669 vcpu->arch.sie_block->scaoh = scaoh;
1670 vcpu->arch.sie_block->scaol = scaol;
1671 vcpu->arch.sie_block->ecb2 |= 0x04U;
1672 }
1673 kvm->arch.sca = new_sca;
1674 kvm->arch.use_esca = 1;
1675
1676 write_unlock(&kvm->arch.sca_lock);
1677 kvm_s390_vcpu_unblock_all(kvm);
1678
1679 free_page((unsigned long)old_sca);
1680
8335713a
CB		 1681 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1682 old_sca, kvm->arch.sca);
5e044315 1683 return 0;
a6e2f683
ED		 1684}
1685
1686static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1687{
5e044315
ED		 1688 int rc;
1689
a6940674
DH		 1690 if (!kvm_s390_use_sca_entries()) {
1691 if (id < KVM_MAX_VCPUS)
1692 return true;
1693 return false;
1694 }
5e044315
ED		 1695 if (id < KVM_S390_BSCA_CPU_SLOTS)
1696 return true;
76a6dd72 1697 if (!sclp.has_esca || !sclp.has_64bscao)
5e044315
ED		 1698 return false;
1699
1700 mutex_lock(&kvm->lock);
1701 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1702 mutex_unlock(&kvm->lock);
1703
1704 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
a6e2f683
ED		 1705}
1706
b0c632db
HC		 1707int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1708{
3c038e6b
DD		 1709 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1710 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 1711 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1712 KVM_SYNC_GPRS |
9eed0735 1713 KVM_SYNC_ACRS |
b028ee3e
DH		 1714 KVM_SYNC_CRS |
1715 KVM_SYNC_ARCH0 |
1716 KVM_SYNC_PFAULT;
75a4615c 1717 kvm_s390_set_prefix(vcpu, 0);
c6e5f166
FZ		 1718 if (test_kvm_facility(vcpu->kvm, 64))
1719 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
f6aa6dc4
DH		 1720 /* fprs can be synchronized via vrs, even if the guest has no vx. With
1721 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1722 */
1723 if (MACHINE_HAS_VX)
68c55750 1724 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
6fd8e67d
DH		 1725 else
1726 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
dafd032a
DD		 1727
1728 if (kvm_is_ucontrol(vcpu->kvm))
1729 return __kvm_ucontrol_vcpu_init(vcpu);
1730
b0c632db
HC		 1731 return 0;
1732}
1733
db0758b2
DH		 1734/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1735static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1736{
1737 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
9c23a131 1738 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2 1739 vcpu->arch.cputm_start = get_tod_clock_fast();
9c23a131 1740 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1741}
1742
1743/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1744static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1745{
1746 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
9c23a131 1747 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1748 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1749 vcpu->arch.cputm_start = 0;
9c23a131 1750 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1751}
1752
1753/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1754static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1755{
1756 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
1757 vcpu->arch.cputm_enabled = true;
1758 __start_cpu_timer_accounting(vcpu);
1759}
1760
1761/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1762static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1763{
1764 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
1765 __stop_cpu_timer_accounting(vcpu);
1766 vcpu->arch.cputm_enabled = false;
1767}
1768
1769static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1770{
1771 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1772 __enable_cpu_timer_accounting(vcpu);
1773 preempt_enable();
1774}
1775
1776static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1777{
1778 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1779 __disable_cpu_timer_accounting(vcpu);
1780 preempt_enable();
1781}
1782
4287f247
DH		 1783/* set the cpu timer - may only be called from the VCPU thread itself */
1784void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
1785{
db0758b2 1786 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
9c23a131 1787 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1788 if (vcpu->arch.cputm_enabled)
1789 vcpu->arch.cputm_start = get_tod_clock_fast();
4287f247 1790 vcpu->arch.sie_block->cputm = cputm;
9c23a131 1791 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2 1792 preempt_enable();
4287f247
DH		 1793}
1794
db0758b2 1795/* update and get the cpu timer - can also be called from other VCPU threads */
4287f247
DH		 1796__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
1797{
9c23a131 1798 unsigned int seq;
db0758b2 1799 __u64 value;
db0758b2
DH		 1800
1801 if (unlikely(!vcpu->arch.cputm_enabled))
1802 return vcpu->arch.sie_block->cputm;
1803
9c23a131
DH		 1804 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1805 do {
1806 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
1807 /*
1808 * If the writer would ever execute a read in the critical
1809 * section, e.g. in irq context, we have a deadlock.
1810 */
1811 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
1812 value = vcpu->arch.sie_block->cputm;
1813 /* if cputm_start is 0, accounting is being started/stopped */
1814 if (likely(vcpu->arch.cputm_start))
1815 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1816 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
1817 preempt_enable();
db0758b2 1818 return value;
4287f247
DH		 1819}
1820
b0c632db
HC		 1821void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1822{
9977e886 1823
37d9df98 1824 gmap_enable(vcpu->arch.enabled_gmap);
805de8f4 1825 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
5ebda316 1826 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1827 __start_cpu_timer_accounting(vcpu);
01a745ac 1828 vcpu->cpu = cpu;
b0c632db
HC		 1829}
1830
1831void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1832{
01a745ac 1833 vcpu->cpu = -1;
5ebda316 1834 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1835 __stop_cpu_timer_accounting(vcpu);
805de8f4 1836 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
37d9df98
DH		 1837 vcpu->arch.enabled_gmap = gmap_get_enabled();
1838 gmap_disable(vcpu->arch.enabled_gmap);
9977e886 1839
b0c632db
HC		 1840}
1841
1842static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1843{
1844 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1845 vcpu->arch.sie_block->gpsw.mask = 0UL;
1846 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 1847 kvm_s390_set_prefix(vcpu, 0);
4287f247 1848 kvm_s390_set_cpu_timer(vcpu, 0);
b0c632db
HC		 1849 vcpu->arch.sie_block->ckc = 0UL;
1850 vcpu->arch.sie_block->todpr = 0;
1851 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1852 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1853 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
9abc2a08
DH		 1854 /* make sure the new fpc will be lazily loaded */
1855 save_fpu_regs();
1856 current->thread.fpu.fpc = 0;
b0c632db 1857 vcpu->arch.sie_block->gbea = 1;
672550fb 1858 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 1859 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1860 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 1861 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1862 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 1863 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 1864}
1865
31928aa5 1866void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 1867{
72f25020 1868 mutex_lock(&vcpu->kvm->lock);
fdf03650 1869 preempt_disable();
72f25020 1870 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
fdf03650 1871 preempt_enable();
72f25020 1872 mutex_unlock(&vcpu->kvm->lock);
25508824 1873 if (!kvm_is_ucontrol(vcpu->kvm)) {
dafd032a 1874 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
eaa78f34 1875 sca_add_vcpu(vcpu);
25508824 1876 }
6502a34c
DH		 1877 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
1878 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
37d9df98
DH		 1879 /* make vcpu_load load the right gmap on the first trigger */
1880 vcpu->arch.enabled_gmap = vcpu->arch.gmap;
42897d86
MT		 1881}
1882
5102ee87
TK		 1883static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1884{
9d8d5786 1885 if (!test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 1886 return;
1887
a374e892
TK		 1888 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1889
1890 if (vcpu->kvm->arch.crypto.aes_kw)
1891 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1892 if (vcpu->kvm->arch.crypto.dea_kw)
1893 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1894
5102ee87
TK		 1895 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1896}
1897
b31605c1
DD		 1898void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1899{
1900 free_page(vcpu->arch.sie_block->cbrlo);
1901 vcpu->arch.sie_block->cbrlo = 0;
1902}
1903
1904int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1905{
1906 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1907 if (!vcpu->arch.sie_block->cbrlo)
1908 return -ENOMEM;
1909
1910 vcpu->arch.sie_block->ecb2 |= 0x80;
1911 vcpu->arch.sie_block->ecb2 &= ~0x08;
1912 return 0;
1913}
1914
91520f1a
MM		 1915static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1916{
1917 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1918
91520f1a 1919 vcpu->arch.sie_block->ibc = model->ibc;
80bc79dc 1920 if (test_kvm_facility(vcpu->kvm, 7))
c54f0d6a 1921 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
91520f1a
MM		 1922}
1923
b0c632db
HC		 1924int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1925{
b31605c1 1926 int rc = 0;
b31288fa 1927
9e6dabef
CH		 1928 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1929 CPUSTAT_SM |
a4a4f191
GH		 1930 CPUSTAT_STOPPED);
1931
53df84f8 1932 if (test_kvm_facility(vcpu->kvm, 78))
805de8f4 1933 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
53df84f8 1934 else if (test_kvm_facility(vcpu->kvm, 8))
805de8f4 1935 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
a4a4f191 1936
91520f1a
MM		 1937 kvm_s390_vcpu_setup_model(vcpu);
1938
bdab09f3
DH		 1939 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
1940 if (MACHINE_HAS_ESOP)
1941 vcpu->arch.sie_block->ecb |= 0x02;
bd50e8ec
DH		 1942 if (test_kvm_facility(vcpu->kvm, 9))
1943 vcpu->arch.sie_block->ecb |= 0x04;
f597d24e 1944 if (test_kvm_facility(vcpu->kvm, 73))
7feb6bb8
MM		 1945 vcpu->arch.sie_block->ecb |= 0x10;
1946
873b425e 1947 if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
d6af0b49 1948 vcpu->arch.sie_block->ecb2 |= 0x08;
cd1836f5
JF		 1949 if (test_kvm_facility(vcpu->kvm, 130))
1950 vcpu->arch.sie_block->ecb2 |= 0x20;
48ee7d3a
DH		 1951 vcpu->arch.sie_block->eca = 0x1002000U;
1952 if (sclp.has_cei)
1953 vcpu->arch.sie_block->eca |= 0x80000000U;
11ad65b7
DH		 1954 if (sclp.has_ib)
1955 vcpu->arch.sie_block->eca |= 0x40000000U;
37c5f6c8 1956 if (sclp.has_siif)
217a4406 1957 vcpu->arch.sie_block->eca |= 1;
37c5f6c8 1958 if (sclp.has_sigpif)
ea5f4969 1959 vcpu->arch.sie_block->eca |= 0x10000000U;
18280d8b 1960 if (test_kvm_facility(vcpu->kvm, 129)) {
13211ea7
EF		 1961 vcpu->arch.sie_block->eca |= 0x00020000;
1962 vcpu->arch.sie_block->ecd |= 0x20000000;
1963 }
c6e5f166 1964 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
492d8642 1965 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
5a5e6536 1966
e6db1d61 1967 if (vcpu->kvm->arch.use_cmma) {
b31605c1
DD		 1968 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1969 if (rc)
1970 return rc;
b31288fa 1971 }
0ac96caf 1972 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 1973 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 1974
5102ee87
TK		 1975 kvm_s390_vcpu_crypto_setup(vcpu);
1976
b31605c1 1977 return rc;
b0c632db
HC		 1978}
1979
1980struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1981 unsigned int id)
1982{
4d47555a 1983 struct kvm_vcpu *vcpu;
7feb6bb8 1984 struct sie_page *sie_page;
4d47555a
CO		 1985 int rc = -EINVAL;
1986
4215825e 1987 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
4d47555a
CO		 1988 goto out;
1989
1990 rc = -ENOMEM;
b0c632db 1991
b110feaf 1992 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 1993 if (!vcpu)
4d47555a 1994 goto out;
b0c632db 1995
7feb6bb8
MM		 1996 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1997 if (!sie_page)
b0c632db
HC		 1998 goto out_free_cpu;
1999
7feb6bb8
MM		 2000 vcpu->arch.sie_block = &sie_page->sie_block;
2001 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
2002
efed1104
DH		 2003 /* the real guest size will always be smaller than msl */
2004 vcpu->arch.sie_block->mso = 0;
2005 vcpu->arch.sie_block->msl = sclp.hamax;
2006
b0c632db 2007 vcpu->arch.sie_block->icpua = id;
ba5c1e9b 2008 spin_lock_init(&vcpu->arch.local_int.lock);
ba5c1e9b 2009 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 2010 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 2011 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
9c23a131 2012 seqcount_init(&vcpu->arch.cputm_seqcount);
ba5c1e9b 2013
b0c632db
HC		 2014 rc = kvm_vcpu_init(vcpu, kvm, id);
2015 if (rc)
9abc2a08 2016 goto out_free_sie_block;
8335713a 2017 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
b0c632db 2018 vcpu->arch.sie_block);
ade38c31 2019 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 2020
b0c632db 2021 return vcpu;
7b06bf2f
WY		 2022out_free_sie_block:
2023 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 2024out_free_cpu:
b110feaf 2025 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 2026out:
b0c632db
HC		 2027 return ERR_PTR(rc);
2028}
2029
b0c632db
HC		 2030int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
2031{
9a022067 2032 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 2033}
2034
27406cd5 2035void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
49b99e1e 2036{
805de8f4 2037 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
61a6df54 2038 exit_sie(vcpu);
49b99e1e
CB		 2039}
2040
27406cd5 2041void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
49b99e1e 2042{
805de8f4 2043 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
49b99e1e
CB		 2044}
2045
8e236546
CB		 2046static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
2047{
805de8f4 2048 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
61a6df54 2049 exit_sie(vcpu);
8e236546
CB		 2050}
2051
2052static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2053{
9bf9fde2 2054 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
8e236546
CB		 2055}
2056
49b99e1e
CB		 2057/*
2058 * Kick a guest cpu out of SIE and wait until SIE is not running.
2059 * If the CPU is not running (e.g. waiting as idle) the function will
2060 * return immediately. */
2061void exit_sie(struct kvm_vcpu *vcpu)
2062{
805de8f4 2063 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
49b99e1e
CB		 2064 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2065 cpu_relax();
2066}
2067
8e236546
CB		 2068/* Kick a guest cpu out of SIE to process a request synchronously */
2069void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
49b99e1e 2070{
8e236546
CB		 2071 kvm_make_request(req, vcpu);
2072 kvm_s390_vcpu_request(vcpu);
49b99e1e
CB		 2073}
2074
414d3b07
MS		 2075static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
2076 unsigned long end)
2c70fe44 2077{
2c70fe44
CB		 2078 struct kvm *kvm = gmap->private;
2079 struct kvm_vcpu *vcpu;
414d3b07
MS		 2080 unsigned long prefix;
2081 int i;
2c70fe44 2082
65d0b0d4
DH		 2083 if (gmap_is_shadow(gmap))
2084 return;
414d3b07
MS		 2085 if (start >= 1UL << 31)
2086 /* We are only interested in prefix pages */
2087 return;
2c70fe44
CB		 2088 kvm_for_each_vcpu(i, vcpu, kvm) {
2089 /* match against both prefix pages */
414d3b07
MS		 2090 prefix = kvm_s390_get_prefix(vcpu);
2091 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
2092 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
2093 start, end);
8e236546 2094 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44
CB		 2095 }
2096 }
2097}
2098
b6d33834
CD		 2099int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2100{
2101 /* kvm common code refers to this, but never calls it */
2102 BUG();
2103 return 0;
2104}
2105
14eebd91
CO		 2106static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2107 struct kvm_one_reg *reg)
2108{
2109 int r = -EINVAL;
2110
2111 switch (reg->id) {
29b7c71b
CO		 2112 case KVM_REG_S390_TODPR:
2113 r = put_user(vcpu->arch.sie_block->todpr,
2114 (u32 __user *)reg->addr);
2115 break;
2116 case KVM_REG_S390_EPOCHDIFF:
2117 r = put_user(vcpu->arch.sie_block->epoch,
2118 (u64 __user *)reg->addr);
2119 break;
46a6dd1c 2120 case KVM_REG_S390_CPU_TIMER:
4287f247 2121 r = put_user(kvm_s390_get_cpu_timer(vcpu),
46a6dd1c
J		 2122 (u64 __user *)reg->addr);
2123 break;
2124 case KVM_REG_S390_CLOCK_COMP:
2125 r = put_user(vcpu->arch.sie_block->ckc,
2126 (u64 __user *)reg->addr);
2127 break;
536336c2
DD		 2128 case KVM_REG_S390_PFTOKEN:
2129 r = put_user(vcpu->arch.pfault_token,
2130 (u64 __user *)reg->addr);
2131 break;
2132 case KVM_REG_S390_PFCOMPARE:
2133 r = put_user(vcpu->arch.pfault_compare,
2134 (u64 __user *)reg->addr);
2135 break;
2136 case KVM_REG_S390_PFSELECT:
2137 r = put_user(vcpu->arch.pfault_select,
2138 (u64 __user *)reg->addr);
2139 break;
672550fb
CB		 2140 case KVM_REG_S390_PP:
2141 r = put_user(vcpu->arch.sie_block->pp,
2142 (u64 __user *)reg->addr);
2143 break;
afa45ff5
CB		 2144 case KVM_REG_S390_GBEA:
2145 r = put_user(vcpu->arch.sie_block->gbea,
2146 (u64 __user *)reg->addr);
2147 break;
14eebd91
CO		 2148 default:
2149 break;
2150 }
2151
2152 return r;
2153}
2154
2155static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2156 struct kvm_one_reg *reg)
2157{
2158 int r = -EINVAL;
4287f247 2159 __u64 val;
14eebd91
CO		 2160
2161 switch (reg->id) {
29b7c71b
CO		 2162 case KVM_REG_S390_TODPR:
2163 r = get_user(vcpu->arch.sie_block->todpr,
2164 (u32 __user *)reg->addr);
2165 break;
2166 case KVM_REG_S390_EPOCHDIFF:
2167 r = get_user(vcpu->arch.sie_block->epoch,
2168 (u64 __user *)reg->addr);
2169 break;
46a6dd1c 2170 case KVM_REG_S390_CPU_TIMER:
4287f247
DH		 2171 r = get_user(val, (u64 __user *)reg->addr);
2172 if (!r)
2173 kvm_s390_set_cpu_timer(vcpu, val);
46a6dd1c
J		 2174 break;
2175 case KVM_REG_S390_CLOCK_COMP:
2176 r = get_user(vcpu->arch.sie_block->ckc,
2177 (u64 __user *)reg->addr);
2178 break;
536336c2
DD		 2179 case KVM_REG_S390_PFTOKEN:
2180 r = get_user(vcpu->arch.pfault_token,
2181 (u64 __user *)reg->addr);
9fbd8082
DH		 2182 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2183 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 2184 break;
2185 case KVM_REG_S390_PFCOMPARE:
2186 r = get_user(vcpu->arch.pfault_compare,
2187 (u64 __user *)reg->addr);
2188 break;
2189 case KVM_REG_S390_PFSELECT:
2190 r = get_user(vcpu->arch.pfault_select,
2191 (u64 __user *)reg->addr);
2192 break;
672550fb
CB		 2193 case KVM_REG_S390_PP:
2194 r = get_user(vcpu->arch.sie_block->pp,
2195 (u64 __user *)reg->addr);
2196 break;
afa45ff5
CB		 2197 case KVM_REG_S390_GBEA:
2198 r = get_user(vcpu->arch.sie_block->gbea,
2199 (u64 __user *)reg->addr);
2200 break;
14eebd91
CO		 2201 default:
2202 break;
2203 }
2204
2205 return r;
2206}
b6d33834 2207
b0c632db
HC		 2208static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2209{
b0c632db 2210 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 2211 return 0;
2212}
2213
2214int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2215{
5a32c1af 2216 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 2217 return 0;
2218}
2219
2220int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2221{
5a32c1af 2222 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 2223 return 0;
2224}
2225
2226int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2227 struct kvm_sregs *sregs)
2228{
59674c1a 2229 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 2230 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
b0c632db
HC		 2231 return 0;
2232}
2233
2234int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2235 struct kvm_sregs *sregs)
2236{
59674c1a 2237 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 2238 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 2239 return 0;
2240}
2241
2242int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2243{
4725c860
MS		 2244 if (test_fp_ctl(fpu->fpc))
2245 return -EINVAL;
e1788bb9 2246 vcpu->run->s.regs.fpc = fpu->fpc;
9abc2a08 2247 if (MACHINE_HAS_VX)
a7d4b8f2
DH		 2248 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
2249 (freg_t *) fpu->fprs);
9abc2a08 2250 else
a7d4b8f2 2251 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
b0c632db
HC		 2252 return 0;
2253}
2254
2255int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2256{
9abc2a08
DH		 2257 /* make sure we have the latest values */
2258 save_fpu_regs();
2259 if (MACHINE_HAS_VX)
a7d4b8f2
DH		 2260 convert_vx_to_fp((freg_t *) fpu->fprs,
2261 (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08 2262 else
a7d4b8f2 2263 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
e1788bb9 2264 fpu->fpc = vcpu->run->s.regs.fpc;
b0c632db
HC		 2265 return 0;
2266}
2267
2268static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2269{
2270 int rc = 0;
2271
7a42fdc2 2272 if (!is_vcpu_stopped(vcpu))
b0c632db 2273 rc = -EBUSY;
d7b0b5eb
CO		 2274 else {
2275 vcpu->run->psw_mask = psw.mask;
2276 vcpu->run->psw_addr = psw.addr;
2277 }
b0c632db
HC		 2278 return rc;
2279}
2280
2281int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2282 struct kvm_translation *tr)
2283{
2284 return -EINVAL; /* not implemented yet */
2285}
2286
27291e21
DH		 2287#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2288 KVM_GUESTDBG_USE_HW_BP | \
2289 KVM_GUESTDBG_ENABLE)
2290
d0bfb940
JK		 2291int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2292 struct kvm_guest_debug *dbg)
b0c632db 2293{
27291e21
DH		 2294 int rc = 0;
2295
2296 vcpu->guest_debug = 0;
2297 kvm_s390_clear_bp_data(vcpu);
2298
2de3bfc2 2299 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21 2300 return -EINVAL;
89b5b4de
DH		 2301 if (!sclp.has_gpere)
2302 return -EINVAL;
27291e21
DH		 2303
2304 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2305 vcpu->guest_debug = dbg->control;
2306 /* enforce guest PER */
805de8f4 2307 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2308
2309 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2310 rc = kvm_s390_import_bp_data(vcpu, dbg);
2311 } else {
805de8f4 2312 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2313 vcpu->arch.guestdbg.last_bp = 0;
2314 }
2315
2316 if (rc) {
2317 vcpu->guest_debug = 0;
2318 kvm_s390_clear_bp_data(vcpu);
805de8f4 2319 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2320 }
2321
2322 return rc;
b0c632db
HC		 2323}
2324
62d9f0db
MT		 2325int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2326 struct kvm_mp_state *mp_state)
2327{
6352e4d2
DH		 2328 /* CHECK_STOP and LOAD are not supported yet */
2329 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2330 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 2331}
2332
2333int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2334 struct kvm_mp_state *mp_state)
2335{
6352e4d2
DH		 2336 int rc = 0;
2337
2338 /* user space knows about this interface - let it control the state */
2339 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2340
2341 switch (mp_state->mp_state) {
2342 case KVM_MP_STATE_STOPPED:
2343 kvm_s390_vcpu_stop(vcpu);
2344 break;
2345 case KVM_MP_STATE_OPERATING:
2346 kvm_s390_vcpu_start(vcpu);
2347 break;
2348 case KVM_MP_STATE_LOAD:
2349 case KVM_MP_STATE_CHECK_STOP:
2350 /* fall through - CHECK_STOP and LOAD are not supported yet */
2351 default:
2352 rc = -ENXIO;
2353 }
2354
2355 return rc;
62d9f0db
MT		 2356}
2357
8ad35755
DH		 2358static bool ibs_enabled(struct kvm_vcpu *vcpu)
2359{
2360 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2361}
2362
2c70fe44
CB		 2363static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2364{
8ad35755 2365retry:
8e236546 2366 kvm_s390_vcpu_request_handled(vcpu);
586b7ccd
CB		 2367 if (!vcpu->requests)
2368 return 0;
2c70fe44
CB		 2369 /*
2370 * We use MMU_RELOAD just to re-arm the ipte notifier for the
b2d73b2a 2371 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2c70fe44
CB		 2372 * This ensures that the ipte instruction for this request has
2373 * already finished. We might race against a second unmapper that
2374 * wants to set the blocking bit. Lets just retry the request loop.
2375 */
8ad35755 2376 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44 2377 int rc;
b2d73b2a
MS		 2378 rc = gmap_mprotect_notify(vcpu->arch.gmap,
2379 kvm_s390_get_prefix(vcpu),
2380 PAGE_SIZE * 2, PROT_WRITE);
aca411a4
JN		 2381 if (rc) {
2382 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44 2383 return rc;
aca411a4 2384 }
8ad35755 2385 goto retry;
2c70fe44 2386 }
8ad35755 2387
d3d692c8
DH		 2388 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2389 vcpu->arch.sie_block->ihcpu = 0xffff;
2390 goto retry;
2391 }
2392
8ad35755
DH		 2393 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2394 if (!ibs_enabled(vcpu)) {
2395 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
805de8f4 2396 atomic_or(CPUSTAT_IBS,
8ad35755
DH		 2397 &vcpu->arch.sie_block->cpuflags);
2398 }
2399 goto retry;
2c70fe44 2400 }
8ad35755
DH		 2401
2402 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2403 if (ibs_enabled(vcpu)) {
2404 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
805de8f4 2405 atomic_andnot(CPUSTAT_IBS,
8ad35755
DH		 2406 &vcpu->arch.sie_block->cpuflags);
2407 }
2408 goto retry;
2409 }
2410
6502a34c
DH		 2411 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
2412 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2413 goto retry;
2414 }
2415
0759d068
DH		 2416 /* nothing to do, just clear the request */
2417 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2418
2c70fe44
CB		 2419 return 0;
2420}
2421
25ed1675
DH		 2422void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2423{
2424 struct kvm_vcpu *vcpu;
2425 int i;
2426
2427 mutex_lock(&kvm->lock);
2428 preempt_disable();
2429 kvm->arch.epoch = tod - get_tod_clock();
2430 kvm_s390_vcpu_block_all(kvm);
2431 kvm_for_each_vcpu(i, vcpu, kvm)
2432 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2433 kvm_s390_vcpu_unblock_all(kvm);
2434 preempt_enable();
2435 mutex_unlock(&kvm->lock);
2436}
2437
fa576c58
TH		 2438/**
2439 * kvm_arch_fault_in_page - fault-in guest page if necessary
2440 * @vcpu: The corresponding virtual cpu
2441 * @gpa: Guest physical address
2442 * @writable: Whether the page should be writable or not
2443 *
2444 * Make sure that a guest page has been faulted-in on the host.
2445 *
2446 * Return: Zero on success, negative error code otherwise.
2447 */
2448long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 2449{
527e30b4
MS		 2450 return gmap_fault(vcpu->arch.gmap, gpa,
2451 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 2452}
2453
3c038e6b
DD		 2454static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2455 unsigned long token)
2456{
2457 struct kvm_s390_interrupt inti;
383d0b05 2458 struct kvm_s390_irq irq;
3c038e6b
DD		 2459
2460 if (start_token) {
383d0b05
JF		 2461 irq.u.ext.ext_params2 = token;
2462 irq.type = KVM_S390_INT_PFAULT_INIT;
2463 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 2464 } else {
2465 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 2466 inti.parm64 = token;
3c038e6b
DD		 2467 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2468 }
2469}
2470
2471void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2472 struct kvm_async_pf *work)
2473{
2474 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2475 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2476}
2477
2478void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2479 struct kvm_async_pf *work)
2480{
2481 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2482 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2483}
2484
2485void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2486 struct kvm_async_pf *work)
2487{
2488 /* s390 will always inject the page directly */
2489}
2490
2491bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2492{
2493 /*
2494 * s390 will always inject the page directly,
2495 * but we still want check_async_completion to cleanup
2496 */
2497 return true;
2498}
2499
2500static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2501{
2502 hva_t hva;
2503 struct kvm_arch_async_pf arch;
2504 int rc;
2505
2506 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2507 return 0;
2508 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2509 vcpu->arch.pfault_compare)
2510 return 0;
2511 if (psw_extint_disabled(vcpu))
2512 return 0;
9a022067 2513 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 2514 return 0;
2515 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2516 return 0;
2517 if (!vcpu->arch.gmap->pfault_enabled)
2518 return 0;
2519
81480cc1
HC		 2520 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2521 hva += current->thread.gmap_addr & ~PAGE_MASK;
2522 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 2523 return 0;
2524
2525 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2526 return rc;
2527}
2528
3fb4c40f 2529static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 2530{
3fb4c40f 2531 int rc, cpuflags;
e168bf8d 2532
3c038e6b
DD		 2533 /*
2534 * On s390 notifications for arriving pages will be delivered directly
2535 * to the guest but the house keeping for completed pfaults is
2536 * handled outside the worker.
2537 */
2538 kvm_check_async_pf_completion(vcpu);
2539
7ec7c8c7
CB		 2540 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2541 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
b0c632db
HC		 2542
2543 if (need_resched())
2544 schedule();
2545
d3a73acb 2546 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 2547 s390_handle_mcck();
2548
79395031
JF		 2549 if (!kvm_is_ucontrol(vcpu->kvm)) {
2550 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2551 if (rc)
2552 return rc;
2553 }
0ff31867 2554
2c70fe44
CB		 2555 rc = kvm_s390_handle_requests(vcpu);
2556 if (rc)
2557 return rc;
2558
27291e21
DH		 2559 if (guestdbg_enabled(vcpu)) {
2560 kvm_s390_backup_guest_per_regs(vcpu);
2561 kvm_s390_patch_guest_per_regs(vcpu);
2562 }
2563
b0c632db 2564 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 2565 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2566 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2567 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 2568
3fb4c40f
TH		 2569 return 0;
2570}
2571
492d8642
TH		 2572static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2573{
56317920
DH		 2574 struct kvm_s390_pgm_info pgm_info = {
2575 .code = PGM_ADDRESSING,
2576 };
2577 u8 opcode, ilen;
492d8642
TH		 2578 int rc;
2579
2580 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2581 trace_kvm_s390_sie_fault(vcpu);
2582
2583 /*
2584 * We want to inject an addressing exception, which is defined as a
2585 * suppressing or terminating exception. However, since we came here
2586 * by a DAT access exception, the PSW still points to the faulting
2587 * instruction since DAT exceptions are nullifying. So we've got
2588 * to look up the current opcode to get the length of the instruction
2589 * to be able to forward the PSW.
2590 */
3fa8cad7 2591 rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
56317920 2592 ilen = insn_length(opcode);
9b0d721a
DH		 2593 if (rc < 0) {
2594 return rc;
2595 } else if (rc) {
2596 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2597 * Forward by arbitrary ilc, injection will take care of
2598 * nullification if necessary.
2599 */
2600 pgm_info = vcpu->arch.pgm;
2601 ilen = 4;
2602 }
56317920
DH		 2603 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2604 kvm_s390_forward_psw(vcpu, ilen);
2605 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
492d8642
TH		 2606}
2607
3fb4c40f
TH		 2608static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2609{
2b29a9fd
DD		 2610 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2611 vcpu->arch.sie_block->icptcode);
2612 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2613
27291e21
DH		 2614 if (guestdbg_enabled(vcpu))
2615 kvm_s390_restore_guest_per_regs(vcpu);
2616
7ec7c8c7
CB		 2617 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2618 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
71f116bf
DH		 2619
2620 if (vcpu->arch.sie_block->icptcode > 0) {
2621 int rc = kvm_handle_sie_intercept(vcpu);
2622
2623 if (rc != -EOPNOTSUPP)
2624 return rc;
2625 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2626 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2627 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2628 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2629 return -EREMOTE;
2630 } else if (exit_reason != -EFAULT) {
2631 vcpu->stat.exit_null++;
2632 return 0;
210b1607
TH		 2633 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2634 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2635 vcpu->run->s390_ucontrol.trans_exc_code =
2636 current->thread.gmap_addr;
2637 vcpu->run->s390_ucontrol.pgm_code = 0x10;
71f116bf 2638 return -EREMOTE;
24eb3a82 2639 } else if (current->thread.gmap_pfault) {
3c038e6b 2640 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 2641 current->thread.gmap_pfault = 0;
71f116bf
DH		 2642 if (kvm_arch_setup_async_pf(vcpu))
2643 return 0;
2644 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
a76ccff6 2645 }
71f116bf 2646 return vcpu_post_run_fault_in_sie(vcpu);
3fb4c40f
TH		 2647}
2648
2649static int __vcpu_run(struct kvm_vcpu *vcpu)
2650{
2651 int rc, exit_reason;
2652
800c1065
TH		 2653 /*
2654 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2655 * ning the guest), so that memslots (and other stuff) are protected
2656 */
2657 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2658
a76ccff6
TH		 2659 do {
2660 rc = vcpu_pre_run(vcpu);
2661 if (rc)
2662 break;
3fb4c40f 2663
800c1065 2664 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 2665 /*
2666 * As PF_VCPU will be used in fault handler, between
2667 * guest_enter and guest_exit should be no uaccess.
2668 */
0097d12e 2669 local_irq_disable();
6edaa530 2670 guest_enter_irqoff();
db0758b2 2671 __disable_cpu_timer_accounting(vcpu);
0097d12e 2672 local_irq_enable();
a76ccff6
TH		 2673 exit_reason = sie64a(vcpu->arch.sie_block,
2674 vcpu->run->s.regs.gprs);
0097d12e 2675 local_irq_disable();
db0758b2 2676 __enable_cpu_timer_accounting(vcpu);
6edaa530 2677 guest_exit_irqoff();
0097d12e 2678 local_irq_enable();
800c1065 2679 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 2680
2681 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 2682 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 2683
800c1065 2684 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 2685 return rc;
b0c632db
HC		 2686}
2687
b028ee3e
DH		 2688static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2689{
2690 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2691 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2692 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2693 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2694 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2695 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 2696 /* some control register changes require a tlb flush */
2697 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 2698 }
2699 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4287f247 2700 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
b028ee3e
DH		 2701 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2702 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2703 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2704 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2705 }
2706 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2707 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2708 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2709 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 2710 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2711 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e 2712 }
80cd8763
FZ		 2713 /*
2714 * If userspace sets the riccb (e.g. after migration) to a valid state,
2715 * we should enable RI here instead of doing the lazy enablement.
2716 */
2717 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
2718 test_kvm_facility(vcpu->kvm, 64)) {
2719 struct runtime_instr_cb *riccb =
2720 (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
2721
2722 if (riccb->valid)
2723 vcpu->arch.sie_block->ecb3 |= 0x01;
2724 }
31d8b8d4
CB		 2725 save_access_regs(vcpu->arch.host_acrs);
2726 restore_access_regs(vcpu->run->s.regs.acrs);
e1788bb9
CB		 2727 /* save host (userspace) fprs/vrs */
2728 save_fpu_regs();
2729 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
2730 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
2731 if (MACHINE_HAS_VX)
2732 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
2733 else
2734 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
2735 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
2736 if (test_fp_ctl(current->thread.fpu.fpc))
2737 /* User space provided an invalid FPC, let's clear it */
2738 current->thread.fpu.fpc = 0;
80cd8763 2739
b028ee3e
DH		 2740 kvm_run->kvm_dirty_regs = 0;
2741}
2742
2743static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2744{
2745 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2746 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2747 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2748 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4287f247 2749 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
b028ee3e
DH		 2750 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2751 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2752 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2753 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2754 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2755 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2756 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
31d8b8d4
CB		 2757 save_access_regs(vcpu->run->s.regs.acrs);
2758 restore_access_regs(vcpu->arch.host_acrs);
e1788bb9
CB		 2759 /* Save guest register state */
2760 save_fpu_regs();
2761 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
2762 /* Restore will be done lazily at return */
2763 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
2764 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
2765
b028ee3e
DH		 2766}
2767
b0c632db
HC		 2768int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2769{
8f2abe6a 2770 int rc;
b0c632db
HC		 2771 sigset_t sigsaved;
2772
27291e21
DH		 2773 if (guestdbg_exit_pending(vcpu)) {
2774 kvm_s390_prepare_debug_exit(vcpu);
2775 return 0;
2776 }
2777
b0c632db
HC		 2778 if (vcpu->sigset_active)
2779 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2780
6352e4d2
DH		 2781 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2782 kvm_s390_vcpu_start(vcpu);
2783 } else if (is_vcpu_stopped(vcpu)) {
ea2cdd27 2784 pr_err_ratelimited("can't run stopped vcpu %d\n",
6352e4d2
DH		 2785 vcpu->vcpu_id);
2786 return -EINVAL;
2787 }
b0c632db 2788
b028ee3e 2789 sync_regs(vcpu, kvm_run);
db0758b2 2790 enable_cpu_timer_accounting(vcpu);
d7b0b5eb 2791
dab4079d 2792 might_fault();
a76ccff6 2793 rc = __vcpu_run(vcpu);
9ace903d 2794
b1d16c49
CE		 2795 if (signal_pending(current) && !rc) {
2796 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 2797 rc = -EINTR;
b1d16c49 2798 }
8f2abe6a 2799
27291e21
DH		 2800 if (guestdbg_exit_pending(vcpu) && !rc) {
2801 kvm_s390_prepare_debug_exit(vcpu);
2802 rc = 0;
2803 }
2804
8f2abe6a 2805 if (rc == -EREMOTE) {
71f116bf 2806 /* userspace support is needed, kvm_run has been prepared */
8f2abe6a
CB		 2807 rc = 0;
2808 }
b0c632db 2809
db0758b2 2810 disable_cpu_timer_accounting(vcpu);
b028ee3e 2811 store_regs(vcpu, kvm_run);
d7b0b5eb 2812
b0c632db
HC		 2813 if (vcpu->sigset_active)
2814 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2815
b0c632db 2816 vcpu->stat.exit_userspace++;
7e8e6ab4 2817 return rc;
b0c632db
HC		 2818}
2819
b0c632db
HC		 2820/*
2821 * store status at address
2822 * we use have two special cases:
2823 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2824 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2825 */
d0bce605 2826int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 2827{
092670cd 2828 unsigned char archmode = 1;
9abc2a08 2829 freg_t fprs[NUM_FPRS];
fda902cb 2830 unsigned int px;
4287f247 2831 u64 clkcomp, cputm;
d0bce605 2832 int rc;
b0c632db 2833
d9a3a09a 2834 px = kvm_s390_get_prefix(vcpu);
d0bce605
HC		 2835 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2836 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 2837 return -EFAULT;
d9a3a09a 2838 gpa = 0;
d0bce605
HC		 2839 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2840 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 2841 return -EFAULT;
d9a3a09a
MS		 2842 gpa = px;
2843 } else
2844 gpa -= __LC_FPREGS_SAVE_AREA;
9abc2a08
DH		 2845
2846 /* manually convert vector registers if necessary */
2847 if (MACHINE_HAS_VX) {
9522b37f 2848 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08
DH		 2849 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2850 fprs, 128);
2851 } else {
2852 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
6fd8e67d 2853 vcpu->run->s.regs.fprs, 128);
9abc2a08 2854 }
d9a3a09a 2855 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
d0bce605 2856 vcpu->run->s.regs.gprs, 128);
d9a3a09a 2857 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
d0bce605 2858 &vcpu->arch.sie_block->gpsw, 16);
d9a3a09a 2859 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
fda902cb 2860 &px, 4);
d9a3a09a 2861 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
9abc2a08 2862 &vcpu->run->s.regs.fpc, 4);
d9a3a09a 2863 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
d0bce605 2864 &vcpu->arch.sie_block->todpr, 4);
4287f247 2865 cputm = kvm_s390_get_cpu_timer(vcpu);
d9a3a09a 2866 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4287f247 2867 &cputm, 8);
178bd789 2868 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d9a3a09a 2869 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
d0bce605 2870 &clkcomp, 8);
d9a3a09a 2871 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
d0bce605 2872 &vcpu->run->s.regs.acrs, 64);
d9a3a09a 2873 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
d0bce605
HC		 2874 &vcpu->arch.sie_block->gcr, 128);
2875 return rc ? -EFAULT : 0;
b0c632db
HC		 2876}
2877
e879892c
TH		 2878int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2879{
2880 /*
2881 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
31d8b8d4 2882 * switch in the run ioctl. Let's update our copies before we save
e879892c
TH		 2883 * it into the save area
2884 */
d0164ee2 2885 save_fpu_regs();
9abc2a08 2886 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
e879892c
TH		 2887 save_access_regs(vcpu->run->s.regs.acrs);
2888
2889 return kvm_s390_store_status_unloaded(vcpu, addr);
2890}
2891
8ad35755
DH		 2892static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2893{
2894 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
8e236546 2895 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
8ad35755
DH		 2896}
2897
2898static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2899{
2900 unsigned int i;
2901 struct kvm_vcpu *vcpu;
2902
2903 kvm_for_each_vcpu(i, vcpu, kvm) {
2904 __disable_ibs_on_vcpu(vcpu);
2905 }
2906}
2907
2908static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2909{
09a400e7
DH		 2910 if (!sclp.has_ibs)
2911 return;
8ad35755 2912 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
8e236546 2913 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
8ad35755
DH		 2914}
2915
6852d7b6
DH		 2916void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2917{
8ad35755
DH		 2918 int i, online_vcpus, started_vcpus = 0;
2919
2920 if (!is_vcpu_stopped(vcpu))
2921 return;
2922
6852d7b6 2923 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 2924 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2925 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2926 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2927
2928 for (i = 0; i < online_vcpus; i++) {
2929 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2930 started_vcpus++;
2931 }
2932
2933 if (started_vcpus == 0) {
2934 /* we're the only active VCPU -> speed it up */
2935 __enable_ibs_on_vcpu(vcpu);
2936 } else if (started_vcpus == 1) {
2937 /*
2938 * As we are starting a second VCPU, we have to disable
2939 * the IBS facility on all VCPUs to remove potentially
2940 * oustanding ENABLE requests.
2941 */
2942 __disable_ibs_on_all_vcpus(vcpu->kvm);
2943 }
2944
805de8f4 2945 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2946 /*
2947 * Another VCPU might have used IBS while we were offline.
2948 * Let's play safe and flush the VCPU at startup.
2949 */
d3d692c8 2950 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 2951 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2952 return;
6852d7b6
DH		 2953}
2954
2955void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2956{
8ad35755
DH		 2957 int i, online_vcpus, started_vcpus = 0;
2958 struct kvm_vcpu *started_vcpu = NULL;
2959
2960 if (is_vcpu_stopped(vcpu))
2961 return;
2962
6852d7b6 2963 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 2964 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2965 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2966 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2967
32f5ff63 2968 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 2969 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 2970
805de8f4 2971 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2972 __disable_ibs_on_vcpu(vcpu);
2973
2974 for (i = 0; i < online_vcpus; i++) {
2975 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2976 started_vcpus++;
2977 started_vcpu = vcpu->kvm->vcpus[i];
2978 }
2979 }
2980
2981 if (started_vcpus == 1) {
2982 /*
2983 * As we only have one VCPU left, we want to enable the
2984 * IBS facility for that VCPU to speed it up.
2985 */
2986 __enable_ibs_on_vcpu(started_vcpu);
2987 }
2988
433b9ee4 2989 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2990 return;
6852d7b6
DH		 2991}
2992
d6712df9
CH		 2993static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2994 struct kvm_enable_cap *cap)
2995{
2996 int r;
2997
2998 if (cap->flags)
2999 return -EINVAL;
3000
3001 switch (cap->cap) {
fa6b7fe9
CH		 3002 case KVM_CAP_S390_CSS_SUPPORT:
3003 if (!vcpu->kvm->arch.css_support) {
3004 vcpu->kvm->arch.css_support = 1;
c92ea7b9 3005 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
fa6b7fe9
CH		 3006 trace_kvm_s390_enable_css(vcpu->kvm);
3007 }
3008 r = 0;
3009 break;
d6712df9
CH		 3010 default:
3011 r = -EINVAL;
3012 break;
3013 }
3014 return r;
3015}
3016
41408c28
TH		 3017static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
3018 struct kvm_s390_mem_op *mop)
3019{
3020 void __user *uaddr = (void __user *)mop->buf;
3021 void *tmpbuf = NULL;
3022 int r, srcu_idx;
3023 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
3024 | KVM_S390_MEMOP_F_CHECK_ONLY;
3025
3026 if (mop->flags & ~supported_flags)
3027 return -EINVAL;
3028
3029 if (mop->size > MEM_OP_MAX_SIZE)
3030 return -E2BIG;
3031
3032 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
3033 tmpbuf = vmalloc(mop->size);
3034 if (!tmpbuf)
3035 return -ENOMEM;
3036 }
3037
3038 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3039
3040 switch (mop->op) {
3041 case KVM_S390_MEMOP_LOGICAL_READ:
3042 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 3043 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3044 mop->size, GACC_FETCH);
41408c28
TH		 3045 break;
3046 }
3047 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3048 if (r == 0) {
3049 if (copy_to_user(uaddr, tmpbuf, mop->size))
3050 r = -EFAULT;
3051 }
3052 break;
3053 case KVM_S390_MEMOP_LOGICAL_WRITE:
3054 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 3055 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3056 mop->size, GACC_STORE);
41408c28
TH		 3057 break;
3058 }
3059 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
3060 r = -EFAULT;
3061 break;
3062 }
3063 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3064 break;
3065 default:
3066 r = -EINVAL;
3067 }
3068
3069 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
3070
3071 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
3072 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
3073
3074 vfree(tmpbuf);
3075 return r;
3076}
3077
b0c632db
HC		 3078long kvm_arch_vcpu_ioctl(struct file *filp,
3079 unsigned int ioctl, unsigned long arg)
3080{
3081 struct kvm_vcpu *vcpu = filp->private_data;
3082 void __user *argp = (void __user *)arg;
800c1065 3083 int idx;
bc923cc9 3084 long r;
b0c632db 3085
93736624 3086 switch (ioctl) {
47b43c52
JF		 3087 case KVM_S390_IRQ: {
3088 struct kvm_s390_irq s390irq;
3089
3090 r = -EFAULT;
3091 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
3092 break;
3093 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3094 break;
3095 }
93736624 3096 case KVM_S390_INTERRUPT: {
ba5c1e9b 3097 struct kvm_s390_interrupt s390int;
383d0b05 3098 struct kvm_s390_irq s390irq;
ba5c1e9b 3099
93736624 3100 r = -EFAULT;
ba5c1e9b 3101 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624 3102 break;
383d0b05
JF		 3103 if (s390int_to_s390irq(&s390int, &s390irq))
3104 return -EINVAL;
3105 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
93736624 3106 break;
ba5c1e9b 3107 }
b0c632db 3108 case KVM_S390_STORE_STATUS:
800c1065 3109 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 3110 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 3111 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 3112 break;
b0c632db
HC		 3113 case KVM_S390_SET_INITIAL_PSW: {
3114 psw_t psw;
3115
bc923cc9 3116 r = -EFAULT;
b0c632db 3117 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 3118 break;
3119 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3120 break;
b0c632db
HC		 3121 }
3122 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 3123 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3124 break;
14eebd91
CO		 3125 case KVM_SET_ONE_REG:
3126 case KVM_GET_ONE_REG: {
3127 struct kvm_one_reg reg;
3128 r = -EFAULT;
3129 if (copy_from_user(&reg, argp, sizeof(reg)))
3130 break;
3131 if (ioctl == KVM_SET_ONE_REG)
3132 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3133 else
3134 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3135 break;
3136 }
27e0393f
CO		 3137#ifdef CONFIG_KVM_S390_UCONTROL
3138 case KVM_S390_UCAS_MAP: {
3139 struct kvm_s390_ucas_mapping ucasmap;
3140
3141 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3142 r = -EFAULT;
3143 break;
3144 }
3145
3146 if (!kvm_is_ucontrol(vcpu->kvm)) {
3147 r = -EINVAL;
3148 break;
3149 }
3150
3151 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3152 ucasmap.vcpu_addr, ucasmap.length);
3153 break;
3154 }
3155 case KVM_S390_UCAS_UNMAP: {
3156 struct kvm_s390_ucas_mapping ucasmap;
3157
3158 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3159 r = -EFAULT;
3160 break;
3161 }
3162
3163 if (!kvm_is_ucontrol(vcpu->kvm)) {
3164 r = -EINVAL;
3165 break;
3166 }
3167
3168 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3169 ucasmap.length);
3170 break;
3171 }
3172#endif
ccc7910f 3173 case KVM_S390_VCPU_FAULT: {
527e30b4 3174 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 3175 break;
3176 }
d6712df9
CH		 3177 case KVM_ENABLE_CAP:
3178 {
3179 struct kvm_enable_cap cap;
3180 r = -EFAULT;
3181 if (copy_from_user(&cap, argp, sizeof(cap)))
3182 break;
3183 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3184 break;
3185 }
41408c28
TH		 3186 case KVM_S390_MEM_OP: {
3187 struct kvm_s390_mem_op mem_op;
3188
3189 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3190 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3191 else
3192 r = -EFAULT;
3193 break;
3194 }
816c7667
JF		 3195 case KVM_S390_SET_IRQ_STATE: {
3196 struct kvm_s390_irq_state irq_state;
3197
3198 r = -EFAULT;
3199 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3200 break;
3201 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3202 irq_state.len == 0 ||
3203 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3204 r = -EINVAL;
3205 break;
3206 }
3207 r = kvm_s390_set_irq_state(vcpu,
3208 (void __user *) irq_state.buf,
3209 irq_state.len);
3210 break;
3211 }
3212 case KVM_S390_GET_IRQ_STATE: {
3213 struct kvm_s390_irq_state irq_state;
3214
3215 r = -EFAULT;
3216 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3217 break;
3218 if (irq_state.len == 0) {
3219 r = -EINVAL;
3220 break;
3221 }
3222 r = kvm_s390_get_irq_state(vcpu,
3223 (__u8 __user *) irq_state.buf,
3224 irq_state.len);
3225 break;
3226 }
b0c632db 3227 default:
3e6afcf1 3228 r = -ENOTTY;
b0c632db 3229 }
bc923cc9 3230 return r;
b0c632db
HC		 3231}
3232
5b1c1493
CO		 3233int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3234{
3235#ifdef CONFIG_KVM_S390_UCONTROL
3236 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3237 && (kvm_is_ucontrol(vcpu->kvm))) {
3238 vmf->page = virt_to_page(vcpu->arch.sie_block);
3239 get_page(vmf->page);
3240 return 0;
3241 }
3242#endif
3243 return VM_FAULT_SIGBUS;
3244}
3245
5587027c
AK		 3246int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3247 unsigned long npages)
db3fe4eb
TY		 3248{
3249 return 0;
3250}
3251
b0c632db 3252/* Section: memory related */
f7784b8e
MT		 3253int kvm_arch_prepare_memory_region(struct kvm *kvm,
3254 struct kvm_memory_slot *memslot,
09170a49 3255 const struct kvm_userspace_memory_region *mem,
7b6195a9 3256 enum kvm_mr_change change)
b0c632db 3257{
dd2887e7
NW		 3258 /* A few sanity checks. We can have memory slots which have to be
3259 located/ended at a segment boundary (1MB). The memory in userland is
3260 ok to be fragmented into various different vmas. It is okay to mmap()
3261 and munmap() stuff in this slot after doing this call at any time */
b0c632db 3262
598841ca 3263 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 3264 return -EINVAL;
3265
598841ca 3266 if (mem->memory_size & 0xffffful)
b0c632db
HC		 3267 return -EINVAL;
3268
a3a92c31
DD		 3269 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3270 return -EINVAL;
3271
f7784b8e
MT		 3272 return 0;
3273}
3274
3275void kvm_arch_commit_memory_region(struct kvm *kvm,
09170a49 3276 const struct kvm_userspace_memory_region *mem,
8482644a 3277 const struct kvm_memory_slot *old,
f36f3f28 3278 const struct kvm_memory_slot *new,
8482644a 3279 enum kvm_mr_change change)
f7784b8e 3280{
f7850c92 3281 int rc;
f7784b8e 3282
2cef4deb
CB		 3283 /* If the basics of the memslot do not change, we do not want
3284 * to update the gmap. Every update causes several unnecessary
3285 * segment translation exceptions. This is usually handled just
3286 * fine by the normal fault handler + gmap, but it will also
3287 * cause faults on the prefix page of running guest CPUs.
3288 */
3289 if (old->userspace_addr == mem->userspace_addr &&
3290 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3291 old->npages * PAGE_SIZE == mem->memory_size)
3292 return;
598841ca
CO		 3293
3294 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3295 mem->guest_phys_addr, mem->memory_size);
3296 if (rc)
ea2cdd27 3297 pr_warn("failed to commit memory region\n");
598841ca 3298 return;
b0c632db
HC		 3299}
3300
60a37709
AY		 3301static inline unsigned long nonhyp_mask(int i)
3302{
3303 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3304
3305 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3306}
3307
3491caf2
CB		 3308void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3309{
3310 vcpu->valid_wakeup = false;
3311}
3312
b0c632db
HC		 3313static int __init kvm_s390_init(void)
3314{
60a37709
AY		 3315 int i;
3316
07197fd0
DH		 3317 if (!sclp.has_sief2) {
3318 pr_info("SIE not available\n");
3319 return -ENODEV;
3320 }
3321
60a37709
AY		 3322 for (i = 0; i < 16; i++)
3323 kvm_s390_fac_list_mask[i] |=
3324 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3325
9d8d5786 3326 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 3327}
3328
3329static void __exit kvm_s390_exit(void)
3330{
3331 kvm_exit();
3332}
3333
3334module_init(kvm_s390_init);
3335module_exit(kvm_s390_exit);
566af940
CH		 3336
3337/*
3338 * Enable autoloading of the kvm module.
3339 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3340 * since x86 takes a different approach.
3341 */
3342#include <linux/miscdevice.h>
3343MODULE_ALIAS_MISCDEV(KVM_MINOR);
3344MODULE_ALIAS("devname:kvm");
Ubuntu Jammy Linux kernel mirror