]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge remote-tracking branch 'regulator/fix/max77802' into regulator-linus
[mirror_ubuntu-artful-kernel.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
b2d73b2a 24#include <linux/mman.h>
b0c632db 25#include <linux/module.h>
d3217967 26#include <linux/moduleparam.h>
a374e892 27#include <linux/random.h>
b0c632db 28#include <linux/slab.h>
ba5c1e9b 29#include <linux/timer.h>
41408c28 30#include <linux/vmalloc.h>
15c9705f 31#include <linux/bitmap.h>
174cd4b1
IM		 32#include <linux/sched/signal.h>
33
cbb870c8 34#include <asm/asm-offsets.h>
b0c632db 35#include <asm/lowcore.h>
fd5ada04 36#include <asm/stp.h>
b0c632db 37#include <asm/pgtable.h>
1e133ab2 38#include <asm/gmap.h>
f5daba1d 39#include <asm/nmi.h>
a0616cde 40#include <asm/switch_to.h>
6d3da241 41#include <asm/isc.h>
1526bf9c 42#include <asm/sclp.h>
0a763c78 43#include <asm/cpacf.h>
221bb8a4 44#include <asm/timex.h>
8f2abe6a 45#include "kvm-s390.h"
b0c632db
HC		 46#include "gaccess.h"
47
ea2cdd27
DH		 48#define KMSG_COMPONENT "kvm-s390"
49#undef pr_fmt
50#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
51
5786fffa
CH		 52#define CREATE_TRACE_POINTS
53#include "trace.h"
ade38c31 54#include "trace-s390.h"
5786fffa 55
41408c28 56#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
816c7667
JF		 57#define LOCAL_IRQS 32
58#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
59 (KVM_MAX_VCPUS + LOCAL_IRQS))
41408c28 60
b0c632db
HC		 61#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
62
63struct kvm_stats_debugfs_item debugfs_entries[] = {
64 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 65 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 66 { "exit_validity", VCPU_STAT(exit_validity) },
67 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
68 { "exit_external_request", VCPU_STAT(exit_external_request) },
69 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b 70 { "exit_instruction", VCPU_STAT(exit_instruction) },
9ec6de19 71 { "exit_pei", VCPU_STAT(exit_pei) },
ba5c1e9b
CO		 72 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
73 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
a011eeb2 74 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
f7819512 75 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
62bea5bf 76 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
3491caf2 77 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
ce2e4f0b 78 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 79 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 80 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 81 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
82 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 83 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 84 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 85 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
86 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
87 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
88 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
89 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
90 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
91 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 92 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 93 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
94 { "instruction_spx", VCPU_STAT(instruction_spx) },
95 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
96 { "instruction_stap", VCPU_STAT(instruction_stap) },
97 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 98 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 99 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
100 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 101 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 102 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
103 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 104 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
95ca2cb5 105 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
a3508fbe 106 { "instruction_sie", VCPU_STAT(instruction_sie) },
5288fbf0 107 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 108 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 109 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 110 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 111 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
112 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 113 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 114 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
115 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 116 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 117 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
118 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
119 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 120 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
121 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
122 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 123 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 124 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 125 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
175a5c9e
CB		 126 { "diagnose_258", VCPU_STAT(diagnose_258) },
127 { "diagnose_308", VCPU_STAT(diagnose_308) },
128 { "diagnose_500", VCPU_STAT(diagnose_500) },
b0c632db
HC		 129 { NULL }
130};
131
a411edf1
DH		 132/* allow nested virtualization in KVM (if enabled by user space) */
133static int nested;
134module_param(nested, int, S_IRUGO);
135MODULE_PARM_DESC(nested, "Nested virtualization support");
136
9d8d5786 137/* upper facilities limit for kvm */
f6c1d359 138unsigned long kvm_s390_fac_list_mask[16] = { FACILITIES_KVM };
b0c632db 139
9d8d5786 140unsigned long kvm_s390_fac_list_mask_size(void)
78c4b59f 141{
9d8d5786
MM		 142 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
143 return ARRAY_SIZE(kvm_s390_fac_list_mask);
78c4b59f
MM		 144}
145
15c9705f
DH		 146/* available cpu features supported by kvm */
147static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
0a763c78
DH		 148/* available subfunctions indicated via query / "test bit" */
149static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
15c9705f 150
9d8d5786 151static struct gmap_notifier gmap_notifier;
a3508fbe 152static struct gmap_notifier vsie_gmap_notifier;
78f26131 153debug_info_t *kvm_s390_dbf;
9d8d5786 154
b0c632db 155/* Section: not file related */
13a34e06 156int kvm_arch_hardware_enable(void)
b0c632db
HC		 157{
158 /* every s390 is virtualization enabled ;-) */
10474ae8 159 return 0;
b0c632db
HC		 160}
161
414d3b07
MS		 162static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
163 unsigned long end);
2c70fe44 164
fdf03650
FZ		 165/*
166 * This callback is executed during stop_machine(). All CPUs are therefore
167 * temporarily stopped. In order not to change guest behavior, we have to
168 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
169 * so a CPU won't be stopped while calculating with the epoch.
170 */
171static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
172 void *v)
173{
174 struct kvm *kvm;
175 struct kvm_vcpu *vcpu;
176 int i;
177 unsigned long long *delta = v;
178
179 list_for_each_entry(kvm, &vm_list, vm_list) {
180 kvm->arch.epoch -= *delta;
181 kvm_for_each_vcpu(i, vcpu, kvm) {
182 vcpu->arch.sie_block->epoch -= *delta;
db0758b2
DH		 183 if (vcpu->arch.cputm_enabled)
184 vcpu->arch.cputm_start += *delta;
91473b48
DH		 185 if (vcpu->arch.vsie_block)
186 vcpu->arch.vsie_block->epoch -= *delta;
fdf03650
FZ		 187 }
188 }
189 return NOTIFY_OK;
190}
191
192static struct notifier_block kvm_clock_notifier = {
193 .notifier_call = kvm_clock_sync,
194};
195
b0c632db
HC		 196int kvm_arch_hardware_setup(void)
197{
2c70fe44 198 gmap_notifier.notifier_call = kvm_gmap_notifier;
b2d73b2a 199 gmap_register_pte_notifier(&gmap_notifier);
a3508fbe
DH		 200 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
201 gmap_register_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 202 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
203 &kvm_clock_notifier);
b0c632db
HC		 204 return 0;
205}
206
207void kvm_arch_hardware_unsetup(void)
208{
b2d73b2a 209 gmap_unregister_pte_notifier(&gmap_notifier);
a3508fbe 210 gmap_unregister_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 211 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
212 &kvm_clock_notifier);
b0c632db
HC		 213}
214
22be5a13
DH		 215static void allow_cpu_feat(unsigned long nr)
216{
217 set_bit_inv(nr, kvm_s390_available_cpu_feat);
218}
219
0a763c78
DH		 220static inline int plo_test_bit(unsigned char nr)
221{
222 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
d051ae53 223 int cc;
0a763c78
DH		 224
225 asm volatile(
226 /* Parameter registers are ignored for "test bit" */
227 " plo 0,0,0,0(0)\n"
228 " ipm %0\n"
229 " srl %0,28\n"
230 : "=d" (cc)
231 : "d" (r0)
232 : "cc");
233 return cc == 0;
234}
235
22be5a13
DH		 236static void kvm_s390_cpu_feat_init(void)
237{
0a763c78
DH		 238 int i;
239
240 for (i = 0; i < 256; ++i) {
241 if (plo_test_bit(i))
242 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
243 }
244
245 if (test_facility(28)) /* TOD-clock steering */
221bb8a4
LT		 246 ptff(kvm_s390_available_subfunc.ptff,
247 sizeof(kvm_s390_available_subfunc.ptff),
248 PTFF_QAF);
0a763c78
DH		 249
250 if (test_facility(17)) { /* MSA */
69c0e360
MS		 251 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
252 kvm_s390_available_subfunc.kmac);
253 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
254 kvm_s390_available_subfunc.kmc);
255 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
256 kvm_s390_available_subfunc.km);
257 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
258 kvm_s390_available_subfunc.kimd);
259 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
260 kvm_s390_available_subfunc.klmd);
0a763c78
DH		 261 }
262 if (test_facility(76)) /* MSA3 */
69c0e360
MS		 263 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
264 kvm_s390_available_subfunc.pckmo);
0a763c78 265 if (test_facility(77)) { /* MSA4 */
69c0e360
MS		 266 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
267 kvm_s390_available_subfunc.kmctr);
268 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
269 kvm_s390_available_subfunc.kmf);
270 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
271 kvm_s390_available_subfunc.kmo);
272 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
273 kvm_s390_available_subfunc.pcc);
0a763c78
DH		 274 }
275 if (test_facility(57)) /* MSA5 */
985a9d20 276 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
69c0e360 277 kvm_s390_available_subfunc.ppno);
0a763c78 278
e000b8e0
JH		 279 if (test_facility(146)) /* MSA8 */
280 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
281 kvm_s390_available_subfunc.kma);
282
22be5a13
DH		 283 if (MACHINE_HAS_ESOP)
284 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
a3508fbe
DH		 285 /*
286 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
287 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
288 */
289 if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
a411edf1 290 !test_facility(3) || !nested)
a3508fbe
DH		 291 return;
292 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
19c439b5
DH		 293 if (sclp.has_64bscao)
294 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
0615a326
DH		 295 if (sclp.has_siif)
296 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
77d18f6d
DH		 297 if (sclp.has_gpere)
298 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
a1b7b9b2
DH		 299 if (sclp.has_gsls)
300 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
5630a8e8
DH		 301 if (sclp.has_ib)
302 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
13ee3f67
DH		 303 if (sclp.has_cei)
304 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
7fd7f39d
DH		 305 if (sclp.has_ibs)
306 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
730cd632
FA		 307 if (sclp.has_kss)
308 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
5d3876a8
DH		 309 /*
310 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
311 * all skey handling functions read/set the skey from the PGSTE
312 * instead of the real storage key.
313 *
314 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
315 * pages being detected as preserved although they are resident.
316 *
317 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
318 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
319 *
320 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
321 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
322 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
323 *
324 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
325 * cannot easily shadow the SCA because of the ipte lock.
326 */
22be5a13
DH		 327}
328
b0c632db
HC		 329int kvm_arch_init(void *opaque)
330{
78f26131
CB		 331 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
332 if (!kvm_s390_dbf)
333 return -ENOMEM;
334
335 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
336 debug_unregister(kvm_s390_dbf);
337 return -ENOMEM;
338 }
339
22be5a13
DH		 340 kvm_s390_cpu_feat_init();
341
84877d93
CH		 342 /* Register floating interrupt controller interface. */
343 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 344}
345
78f26131
CB		 346void kvm_arch_exit(void)
347{
348 debug_unregister(kvm_s390_dbf);
349}
350
b0c632db
HC		 351/* Section: device related */
352long kvm_arch_dev_ioctl(struct file *filp,
353 unsigned int ioctl, unsigned long arg)
354{
355 if (ioctl == KVM_S390_ENABLE_SIE)
356 return s390_enable_sie();
357 return -EINVAL;
358}
359
784aa3d7 360int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 361{
d7b0b5eb
CO		 362 int r;
363
2bd0ac4e 364 switch (ext) {
d7b0b5eb 365 case KVM_CAP_S390_PSW:
b6cf8788 366 case KVM_CAP_S390_GMAP:
52e16b18 367 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 368#ifdef CONFIG_KVM_S390_UCONTROL
369 case KVM_CAP_S390_UCONTROL:
370#endif
3c038e6b 371 case KVM_CAP_ASYNC_PF:
60b413c9 372 case KVM_CAP_SYNC_REGS:
14eebd91 373 case KVM_CAP_ONE_REG:
d6712df9 374 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 375 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 376 case KVM_CAP_IOEVENTFD:
c05c4186 377 case KVM_CAP_DEVICE_CTRL:
d938dc55 378 case KVM_CAP_ENABLE_CAP_VM:
78599d90 379 case KVM_CAP_S390_IRQCHIP:
f2061656 380 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 381 case KVM_CAP_MP_STATE:
460df4c1 382 case KVM_CAP_IMMEDIATE_EXIT:
47b43c52 383 case KVM_CAP_S390_INJECT_IRQ:
2444b352 384 case KVM_CAP_S390_USER_SIGP:
e44fc8c9 385 case KVM_CAP_S390_USER_STSI:
30ee2a98 386 case KVM_CAP_S390_SKEYS:
816c7667 387 case KVM_CAP_S390_IRQ_STATE:
6502a34c 388 case KVM_CAP_S390_USER_INSTR0:
47a4693e 389 case KVM_CAP_S390_AIS:
d7b0b5eb
CO		 390 r = 1;
391 break;
41408c28
TH		 392 case KVM_CAP_S390_MEM_OP:
393 r = MEM_OP_MAX_SIZE;
394 break;
e726b1bd
CB		 395 case KVM_CAP_NR_VCPUS:
396 case KVM_CAP_MAX_VCPUS:
76a6dd72 397 r = KVM_S390_BSCA_CPU_SLOTS;
a6940674
DH		 398 if (!kvm_s390_use_sca_entries())
399 r = KVM_MAX_VCPUS;
400 else if (sclp.has_esca && sclp.has_64bscao)
76a6dd72 401 r = KVM_S390_ESCA_CPU_SLOTS;
e726b1bd 402 break;
e1e2e605
NW		 403 case KVM_CAP_NR_MEMSLOTS:
404 r = KVM_USER_MEM_SLOTS;
405 break;
1526bf9c 406 case KVM_CAP_S390_COW:
abf09bed 407 r = MACHINE_HAS_ESOP;
1526bf9c 408 break;
68c55750
EF		 409 case KVM_CAP_S390_VECTOR_REGISTERS:
410 r = MACHINE_HAS_VX;
411 break;
c6e5f166
FZ		 412 case KVM_CAP_S390_RI:
413 r = test_facility(64);
414 break;
4e0b1ab7
FZ		 415 case KVM_CAP_S390_GS:
416 r = test_facility(133);
417 break;
2bd0ac4e 418 default:
d7b0b5eb 419 r = 0;
2bd0ac4e 420 }
d7b0b5eb 421 return r;
b0c632db
HC		 422}
423
15f36ebd
JH		 424static void kvm_s390_sync_dirty_log(struct kvm *kvm,
425 struct kvm_memory_slot *memslot)
426{
427 gfn_t cur_gfn, last_gfn;
428 unsigned long address;
429 struct gmap *gmap = kvm->arch.gmap;
430
15f36ebd
JH		 431 /* Loop over all guest pages */
432 last_gfn = memslot->base_gfn + memslot->npages;
433 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
434 address = gfn_to_hva_memslot(memslot, cur_gfn);
435
1e133ab2 436 if (test_and_clear_guest_dirty(gmap->mm, address))
15f36ebd 437 mark_page_dirty(kvm, cur_gfn);
1763f8d0
CB		 438 if (fatal_signal_pending(current))
439 return;
70c88a00 440 cond_resched();
15f36ebd 441 }
15f36ebd
JH		 442}
443
b0c632db 444/* Section: vm related */
a6e2f683
ED		 445static void sca_del_vcpu(struct kvm_vcpu *vcpu);
446
b0c632db
HC		 447/*
448 * Get (and clear) the dirty memory log for a memory slot.
449 */
450int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
451 struct kvm_dirty_log *log)
452{
15f36ebd
JH		 453 int r;
454 unsigned long n;
9f6b8029 455 struct kvm_memslots *slots;
15f36ebd
JH		 456 struct kvm_memory_slot *memslot;
457 int is_dirty = 0;
458
e1e8a962
JF		 459 if (kvm_is_ucontrol(kvm))
460 return -EINVAL;
461
15f36ebd
JH		 462 mutex_lock(&kvm->slots_lock);
463
464 r = -EINVAL;
465 if (log->slot >= KVM_USER_MEM_SLOTS)
466 goto out;
467
9f6b8029
PB		 468 slots = kvm_memslots(kvm);
469 memslot = id_to_memslot(slots, log->slot);
15f36ebd
JH		 470 r = -ENOENT;
471 if (!memslot->dirty_bitmap)
472 goto out;
473
474 kvm_s390_sync_dirty_log(kvm, memslot);
475 r = kvm_get_dirty_log(kvm, log, &is_dirty);
476 if (r)
477 goto out;
478
479 /* Clear the dirty log */
480 if (is_dirty) {
481 n = kvm_dirty_bitmap_bytes(memslot);
482 memset(memslot->dirty_bitmap, 0, n);
483 }
484 r = 0;
485out:
486 mutex_unlock(&kvm->slots_lock);
487 return r;
b0c632db
HC		 488}
489
6502a34c
DH		 490static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
491{
492 unsigned int i;
493 struct kvm_vcpu *vcpu;
494
495 kvm_for_each_vcpu(i, vcpu, kvm) {
496 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
497 }
498}
499
d938dc55
CH		 500static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
501{
502 int r;
503
504 if (cap->flags)
505 return -EINVAL;
506
507 switch (cap->cap) {
84223598 508 case KVM_CAP_S390_IRQCHIP:
c92ea7b9 509 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
84223598
CH		 510 kvm->arch.use_irqchip = 1;
511 r = 0;
512 break;
2444b352 513 case KVM_CAP_S390_USER_SIGP:
c92ea7b9 514 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
2444b352
DH		 515 kvm->arch.user_sigp = 1;
516 r = 0;
517 break;
68c55750 518 case KVM_CAP_S390_VECTOR_REGISTERS:
5967c17b 519 mutex_lock(&kvm->lock);
a03825bb 520 if (kvm->created_vcpus) {
5967c17b
DH		 521 r = -EBUSY;
522 } else if (MACHINE_HAS_VX) {
c54f0d6a
DH		 523 set_kvm_facility(kvm->arch.model.fac_mask, 129);
524 set_kvm_facility(kvm->arch.model.fac_list, 129);
2f87d942
GH		 525 if (test_facility(134)) {
526 set_kvm_facility(kvm->arch.model.fac_mask, 134);
527 set_kvm_facility(kvm->arch.model.fac_list, 134);
528 }
53743aa7
MS		 529 if (test_facility(135)) {
530 set_kvm_facility(kvm->arch.model.fac_mask, 135);
531 set_kvm_facility(kvm->arch.model.fac_list, 135);
532 }
18280d8b
MM		 533 r = 0;
534 } else
535 r = -EINVAL;
5967c17b 536 mutex_unlock(&kvm->lock);
c92ea7b9
CB		 537 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
538 r ? "(not available)" : "(success)");
68c55750 539 break;
c6e5f166
FZ		 540 case KVM_CAP_S390_RI:
541 r = -EINVAL;
542 mutex_lock(&kvm->lock);
a03825bb 543 if (kvm->created_vcpus) {
c6e5f166
FZ		 544 r = -EBUSY;
545 } else if (test_facility(64)) {
c54f0d6a
DH		 546 set_kvm_facility(kvm->arch.model.fac_mask, 64);
547 set_kvm_facility(kvm->arch.model.fac_list, 64);
c6e5f166
FZ		 548 r = 0;
549 }
550 mutex_unlock(&kvm->lock);
551 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
552 r ? "(not available)" : "(success)");
553 break;
47a4693e
YMZ		 554 case KVM_CAP_S390_AIS:
555 mutex_lock(&kvm->lock);
556 if (kvm->created_vcpus) {
557 r = -EBUSY;
558 } else {
559 set_kvm_facility(kvm->arch.model.fac_mask, 72);
560 set_kvm_facility(kvm->arch.model.fac_list, 72);
47a4693e
YMZ		 561 r = 0;
562 }
563 mutex_unlock(&kvm->lock);
564 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
565 r ? "(not available)" : "(success)");
566 break;
4e0b1ab7
FZ		 567 case KVM_CAP_S390_GS:
568 r = -EINVAL;
569 mutex_lock(&kvm->lock);
570 if (atomic_read(&kvm->online_vcpus)) {
571 r = -EBUSY;
572 } else if (test_facility(133)) {
573 set_kvm_facility(kvm->arch.model.fac_mask, 133);
574 set_kvm_facility(kvm->arch.model.fac_list, 133);
575 r = 0;
576 }
577 mutex_unlock(&kvm->lock);
578 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
579 r ? "(not available)" : "(success)");
580 break;
e44fc8c9 581 case KVM_CAP_S390_USER_STSI:
c92ea7b9 582 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
e44fc8c9
ET		 583 kvm->arch.user_stsi = 1;
584 r = 0;
585 break;
6502a34c
DH		 586 case KVM_CAP_S390_USER_INSTR0:
587 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
588 kvm->arch.user_instr0 = 1;
589 icpt_operexc_on_all_vcpus(kvm);
590 r = 0;
591 break;
d938dc55
CH		 592 default:
593 r = -EINVAL;
594 break;
595 }
596 return r;
597}
598
8c0a7ce6
DD		 599static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
600{
601 int ret;
602
603 switch (attr->attr) {
604 case KVM_S390_VM_MEM_LIMIT_SIZE:
605 ret = 0;
c92ea7b9 606 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
a3a92c31
DD		 607 kvm->arch.mem_limit);
608 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
8c0a7ce6
DD		 609 ret = -EFAULT;
610 break;
611 default:
612 ret = -ENXIO;
613 break;
614 }
615 return ret;
616}
617
618static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 619{
620 int ret;
621 unsigned int idx;
622 switch (attr->attr) {
623 case KVM_S390_VM_MEM_ENABLE_CMMA:
f9cbd9b0 624 ret = -ENXIO;
c24cc9c8 625 if (!sclp.has_cmma)
e6db1d61
DD		 626 break;
627
4f718eab 628 ret = -EBUSY;
c92ea7b9 629 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
4f718eab 630 mutex_lock(&kvm->lock);
a03825bb 631 if (!kvm->created_vcpus) {
4f718eab
DD		 632 kvm->arch.use_cmma = 1;
633 ret = 0;
634 }
635 mutex_unlock(&kvm->lock);
636 break;
637 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 638 ret = -ENXIO;
639 if (!sclp.has_cmma)
640 break;
c3489155
DD		 641 ret = -EINVAL;
642 if (!kvm->arch.use_cmma)
643 break;
644
c92ea7b9 645 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
4f718eab
DD		 646 mutex_lock(&kvm->lock);
647 idx = srcu_read_lock(&kvm->srcu);
a13cff31 648 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 649 srcu_read_unlock(&kvm->srcu, idx);
650 mutex_unlock(&kvm->lock);
651 ret = 0;
652 break;
8c0a7ce6
DD		 653 case KVM_S390_VM_MEM_LIMIT_SIZE: {
654 unsigned long new_limit;
655
656 if (kvm_is_ucontrol(kvm))
657 return -EINVAL;
658
659 if (get_user(new_limit, (u64 __user *)attr->addr))
660 return -EFAULT;
661
a3a92c31
DD		 662 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
663 new_limit > kvm->arch.mem_limit)
8c0a7ce6
DD		 664 return -E2BIG;
665
a3a92c31
DD		 666 if (!new_limit)
667 return -EINVAL;
668
6ea427bb 669 /* gmap_create takes last usable address */
a3a92c31
DD		 670 if (new_limit != KVM_S390_NO_MEM_LIMIT)
671 new_limit -= 1;
672
8c0a7ce6
DD		 673 ret = -EBUSY;
674 mutex_lock(&kvm->lock);
a03825bb 675 if (!kvm->created_vcpus) {
6ea427bb
MS		 676 /* gmap_create will round the limit up */
677 struct gmap *new = gmap_create(current->mm, new_limit);
8c0a7ce6
DD		 678
679 if (!new) {
680 ret = -ENOMEM;
681 } else {
6ea427bb 682 gmap_remove(kvm->arch.gmap);
8c0a7ce6
DD		 683 new->private = kvm;
684 kvm->arch.gmap = new;
685 ret = 0;
686 }
687 }
688 mutex_unlock(&kvm->lock);
a3a92c31
DD		 689 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
690 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
691 (void *) kvm->arch.gmap->asce);
8c0a7ce6
DD		 692 break;
693 }
4f718eab
DD		 694 default:
695 ret = -ENXIO;
696 break;
697 }
698 return ret;
699}
700
a374e892
TK		 701static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
702
703static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
704{
705 struct kvm_vcpu *vcpu;
706 int i;
707
9d8d5786 708 if (!test_kvm_facility(kvm, 76))
a374e892
TK		 709 return -EINVAL;
710
711 mutex_lock(&kvm->lock);
712 switch (attr->attr) {
713 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
714 get_random_bytes(
715 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
716 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
717 kvm->arch.crypto.aes_kw = 1;
c92ea7b9 718 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
a374e892
TK		 719 break;
720 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
721 get_random_bytes(
722 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
723 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
724 kvm->arch.crypto.dea_kw = 1;
c92ea7b9 725 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
a374e892
TK		 726 break;
727 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
728 kvm->arch.crypto.aes_kw = 0;
729 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
730 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
c92ea7b9 731 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
a374e892
TK		 732 break;
733 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
734 kvm->arch.crypto.dea_kw = 0;
735 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
736 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
c92ea7b9 737 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
a374e892
TK		 738 break;
739 default:
740 mutex_unlock(&kvm->lock);
741 return -ENXIO;
742 }
743
744 kvm_for_each_vcpu(i, vcpu, kvm) {
745 kvm_s390_vcpu_crypto_setup(vcpu);
746 exit_sie(vcpu);
747 }
748 mutex_unlock(&kvm->lock);
749 return 0;
750}
751
72f25020
JH		 752static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
753{
754 u8 gtod_high;
755
756 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
757 sizeof(gtod_high)))
758 return -EFAULT;
759
760 if (gtod_high != 0)
761 return -EINVAL;
58c383c6 762 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
72f25020
JH		 763
764 return 0;
765}
766
767static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
768{
5a3d883a 769 u64 gtod;
72f25020
JH		 770
771 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
772 return -EFAULT;
773
25ed1675 774 kvm_s390_set_tod_clock(kvm, gtod);
58c383c6 775 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
72f25020
JH		 776 return 0;
777}
778
779static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
780{
781 int ret;
782
783 if (attr->flags)
784 return -EINVAL;
785
786 switch (attr->attr) {
787 case KVM_S390_VM_TOD_HIGH:
788 ret = kvm_s390_set_tod_high(kvm, attr);
789 break;
790 case KVM_S390_VM_TOD_LOW:
791 ret = kvm_s390_set_tod_low(kvm, attr);
792 break;
793 default:
794 ret = -ENXIO;
795 break;
796 }
797 return ret;
798}
799
800static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
801{
802 u8 gtod_high = 0;
803
804 if (copy_to_user((void __user *)attr->addr, &gtod_high,
805 sizeof(gtod_high)))
806 return -EFAULT;
58c383c6 807 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
72f25020
JH		 808
809 return 0;
810}
811
812static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
813{
5a3d883a 814 u64 gtod;
72f25020 815
60417fcc 816 gtod = kvm_s390_get_tod_clock_fast(kvm);
72f25020
JH		 817 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
818 return -EFAULT;
58c383c6 819 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
72f25020
JH		 820
821 return 0;
822}
823
824static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
825{
826 int ret;
827
828 if (attr->flags)
829 return -EINVAL;
830
831 switch (attr->attr) {
832 case KVM_S390_VM_TOD_HIGH:
833 ret = kvm_s390_get_tod_high(kvm, attr);
834 break;
835 case KVM_S390_VM_TOD_LOW:
836 ret = kvm_s390_get_tod_low(kvm, attr);
837 break;
838 default:
839 ret = -ENXIO;
840 break;
841 }
842 return ret;
843}
844
658b6eda
MM		 845static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
846{
847 struct kvm_s390_vm_cpu_processor *proc;
053dd230 848 u16 lowest_ibc, unblocked_ibc;
658b6eda
MM		 849 int ret = 0;
850
851 mutex_lock(&kvm->lock);
a03825bb 852 if (kvm->created_vcpus) {
658b6eda
MM		 853 ret = -EBUSY;
854 goto out;
855 }
856 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
857 if (!proc) {
858 ret = -ENOMEM;
859 goto out;
860 }
861 if (!copy_from_user(proc, (void __user *)attr->addr,
862 sizeof(*proc))) {
9bb0ec09 863 kvm->arch.model.cpuid = proc->cpuid;
053dd230
DH		 864 lowest_ibc = sclp.ibc >> 16 & 0xfff;
865 unblocked_ibc = sclp.ibc & 0xfff;
0487c44d 866 if (lowest_ibc && proc->ibc) {
053dd230
DH		 867 if (proc->ibc > unblocked_ibc)
868 kvm->arch.model.ibc = unblocked_ibc;
869 else if (proc->ibc < lowest_ibc)
870 kvm->arch.model.ibc = lowest_ibc;
871 else
872 kvm->arch.model.ibc = proc->ibc;
873 }
c54f0d6a 874 memcpy(kvm->arch.model.fac_list, proc->fac_list,
658b6eda 875 S390_ARCH_FAC_LIST_SIZE_BYTE);
a8c39dd7
CB		 876 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
877 kvm->arch.model.ibc,
878 kvm->arch.model.cpuid);
879 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
880 kvm->arch.model.fac_list[0],
881 kvm->arch.model.fac_list[1],
882 kvm->arch.model.fac_list[2]);
658b6eda
MM		 883 } else
884 ret = -EFAULT;
885 kfree(proc);
886out:
887 mutex_unlock(&kvm->lock);
888 return ret;
889}
890
15c9705f
DH		 891static int kvm_s390_set_processor_feat(struct kvm *kvm,
892 struct kvm_device_attr *attr)
893{
894 struct kvm_s390_vm_cpu_feat data;
895 int ret = -EBUSY;
896
897 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
898 return -EFAULT;
899 if (!bitmap_subset((unsigned long *) data.feat,
900 kvm_s390_available_cpu_feat,
901 KVM_S390_VM_CPU_FEAT_NR_BITS))
902 return -EINVAL;
903
904 mutex_lock(&kvm->lock);
905 if (!atomic_read(&kvm->online_vcpus)) {
906 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
907 KVM_S390_VM_CPU_FEAT_NR_BITS);
908 ret = 0;
909 }
910 mutex_unlock(&kvm->lock);
911 return ret;
912}
913
0a763c78
DH		 914static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
915 struct kvm_device_attr *attr)
916{
917 /*
918 * Once supported by kernel + hw, we have to store the subfunctions
919 * in kvm->arch and remember that user space configured them.
920 */
921 return -ENXIO;
922}
923
658b6eda
MM		 924static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
925{
926 int ret = -ENXIO;
927
928 switch (attr->attr) {
929 case KVM_S390_VM_CPU_PROCESSOR:
930 ret = kvm_s390_set_processor(kvm, attr);
931 break;
15c9705f
DH		 932 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
933 ret = kvm_s390_set_processor_feat(kvm, attr);
934 break;
0a763c78
DH		 935 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
936 ret = kvm_s390_set_processor_subfunc(kvm, attr);
937 break;
658b6eda
MM		 938 }
939 return ret;
940}
941
942static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
943{
944 struct kvm_s390_vm_cpu_processor *proc;
945 int ret = 0;
946
947 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
948 if (!proc) {
949 ret = -ENOMEM;
950 goto out;
951 }
9bb0ec09 952 proc->cpuid = kvm->arch.model.cpuid;
658b6eda 953 proc->ibc = kvm->arch.model.ibc;
c54f0d6a
DH		 954 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
955 S390_ARCH_FAC_LIST_SIZE_BYTE);
a8c39dd7
CB		 956 VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
957 kvm->arch.model.ibc,
958 kvm->arch.model.cpuid);
959 VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
960 kvm->arch.model.fac_list[0],
961 kvm->arch.model.fac_list[1],
962 kvm->arch.model.fac_list[2]);
658b6eda
MM		 963 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
964 ret = -EFAULT;
965 kfree(proc);
966out:
967 return ret;
968}
969
970static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
971{
972 struct kvm_s390_vm_cpu_machine *mach;
973 int ret = 0;
974
975 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
976 if (!mach) {
977 ret = -ENOMEM;
978 goto out;
979 }
980 get_cpu_id((struct cpuid *) &mach->cpuid);
37c5f6c8 981 mach->ibc = sclp.ibc;
c54f0d6a 982 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
981467c9 983 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 984 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
04478197 985 sizeof(S390_lowcore.stfle_fac_list));
a8c39dd7
CB		 986 VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx",
987 kvm->arch.model.ibc,
988 kvm->arch.model.cpuid);
989 VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx",
990 mach->fac_mask[0],
991 mach->fac_mask[1],
992 mach->fac_mask[2]);
993 VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx",
994 mach->fac_list[0],
995 mach->fac_list[1],
996 mach->fac_list[2]);
658b6eda
MM		 997 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
998 ret = -EFAULT;
999 kfree(mach);
1000out:
1001 return ret;
1002}
1003
15c9705f
DH		 1004static int kvm_s390_get_processor_feat(struct kvm *kvm,
1005 struct kvm_device_attr *attr)
1006{
1007 struct kvm_s390_vm_cpu_feat data;
1008
1009 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1010 KVM_S390_VM_CPU_FEAT_NR_BITS);
1011 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1012 return -EFAULT;
1013 return 0;
1014}
1015
1016static int kvm_s390_get_machine_feat(struct kvm *kvm,
1017 struct kvm_device_attr *attr)
1018{
1019 struct kvm_s390_vm_cpu_feat data;
1020
1021 bitmap_copy((unsigned long *) data.feat,
1022 kvm_s390_available_cpu_feat,
1023 KVM_S390_VM_CPU_FEAT_NR_BITS);
1024 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1025 return -EFAULT;
1026 return 0;
1027}
1028
0a763c78
DH		 1029static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1030 struct kvm_device_attr *attr)
1031{
1032 /*
1033 * Once we can actually configure subfunctions (kernel + hw support),
1034 * we have to check if they were already set by user space, if so copy
1035 * them from kvm->arch.
1036 */
1037 return -ENXIO;
1038}
1039
1040static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1041 struct kvm_device_attr *attr)
1042{
1043 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1044 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1045 return -EFAULT;
1046 return 0;
1047}
658b6eda
MM		 1048static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1049{
1050 int ret = -ENXIO;
1051
1052 switch (attr->attr) {
1053 case KVM_S390_VM_CPU_PROCESSOR:
1054 ret = kvm_s390_get_processor(kvm, attr);
1055 break;
1056 case KVM_S390_VM_CPU_MACHINE:
1057 ret = kvm_s390_get_machine(kvm, attr);
1058 break;
15c9705f
DH		 1059 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1060 ret = kvm_s390_get_processor_feat(kvm, attr);
1061 break;
1062 case KVM_S390_VM_CPU_MACHINE_FEAT:
1063 ret = kvm_s390_get_machine_feat(kvm, attr);
1064 break;
0a763c78
DH		 1065 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1066 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1067 break;
1068 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1069 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1070 break;
658b6eda
MM		 1071 }
1072 return ret;
1073}
1074
f2061656
DD		 1075static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1076{
1077 int ret;
1078
1079 switch (attr->group) {
4f718eab 1080 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 1081 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 1082 break;
72f25020
JH		 1083 case KVM_S390_VM_TOD:
1084 ret = kvm_s390_set_tod(kvm, attr);
1085 break;
658b6eda
MM		 1086 case KVM_S390_VM_CPU_MODEL:
1087 ret = kvm_s390_set_cpu_model(kvm, attr);
1088 break;
a374e892
TK		 1089 case KVM_S390_VM_CRYPTO:
1090 ret = kvm_s390_vm_set_crypto(kvm, attr);
1091 break;
f2061656
DD		 1092 default:
1093 ret = -ENXIO;
1094 break;
1095 }
1096
1097 return ret;
1098}
1099
1100static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1101{
8c0a7ce6
DD		 1102 int ret;
1103
1104 switch (attr->group) {
1105 case KVM_S390_VM_MEM_CTRL:
1106 ret = kvm_s390_get_mem_control(kvm, attr);
1107 break;
72f25020
JH		 1108 case KVM_S390_VM_TOD:
1109 ret = kvm_s390_get_tod(kvm, attr);
1110 break;
658b6eda
MM		 1111 case KVM_S390_VM_CPU_MODEL:
1112 ret = kvm_s390_get_cpu_model(kvm, attr);
1113 break;
8c0a7ce6
DD		 1114 default:
1115 ret = -ENXIO;
1116 break;
1117 }
1118
1119 return ret;
f2061656
DD		 1120}
1121
1122static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1123{
1124 int ret;
1125
1126 switch (attr->group) {
4f718eab
DD		 1127 case KVM_S390_VM_MEM_CTRL:
1128 switch (attr->attr) {
1129 case KVM_S390_VM_MEM_ENABLE_CMMA:
1130 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 1131 ret = sclp.has_cmma ? 0 : -ENXIO;
1132 break;
8c0a7ce6 1133 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 1134 ret = 0;
1135 break;
1136 default:
1137 ret = -ENXIO;
1138 break;
1139 }
1140 break;
72f25020
JH		 1141 case KVM_S390_VM_TOD:
1142 switch (attr->attr) {
1143 case KVM_S390_VM_TOD_LOW:
1144 case KVM_S390_VM_TOD_HIGH:
1145 ret = 0;
1146 break;
1147 default:
1148 ret = -ENXIO;
1149 break;
1150 }
1151 break;
658b6eda
MM		 1152 case KVM_S390_VM_CPU_MODEL:
1153 switch (attr->attr) {
1154 case KVM_S390_VM_CPU_PROCESSOR:
1155 case KVM_S390_VM_CPU_MACHINE:
15c9705f
DH		 1156 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1157 case KVM_S390_VM_CPU_MACHINE_FEAT:
0a763c78 1158 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
658b6eda
MM		 1159 ret = 0;
1160 break;
0a763c78
DH		 1161 /* configuring subfunctions is not supported yet */
1162 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
658b6eda
MM		 1163 default:
1164 ret = -ENXIO;
1165 break;
1166 }
1167 break;
a374e892
TK		 1168 case KVM_S390_VM_CRYPTO:
1169 switch (attr->attr) {
1170 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1171 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1172 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1173 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1174 ret = 0;
1175 break;
1176 default:
1177 ret = -ENXIO;
1178 break;
1179 }
1180 break;
f2061656
DD		 1181 default:
1182 ret = -ENXIO;
1183 break;
1184 }
1185
1186 return ret;
1187}
1188
30ee2a98
JH		 1189static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1190{
1191 uint8_t *keys;
1192 uint64_t hva;
30ee2a98
JH		 1193 int i, r = 0;
1194
1195 if (args->flags != 0)
1196 return -EINVAL;
1197
1198 /* Is this guest using storage keys? */
1199 if (!mm_use_skey(current->mm))
1200 return KVM_S390_GET_SKEYS_NONE;
1201
1202 /* Enforce sane limit on memory allocation */
1203 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1204 return -EINVAL;
1205
752ade68 1206 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
30ee2a98
JH		 1207 if (!keys)
1208 return -ENOMEM;
1209
d3ed1cee 1210 down_read(&current->mm->mmap_sem);
30ee2a98
JH		 1211 for (i = 0; i < args->count; i++) {
1212 hva = gfn_to_hva(kvm, args->start_gfn + i);
1213 if (kvm_is_error_hva(hva)) {
1214 r = -EFAULT;
d3ed1cee 1215 break;
30ee2a98
JH		 1216 }
1217
154c8c19
DH		 1218 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1219 if (r)
d3ed1cee 1220 break;
30ee2a98 1221 }
d3ed1cee
MS		 1222 up_read(&current->mm->mmap_sem);
1223
1224 if (!r) {
1225 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1226 sizeof(uint8_t) * args->count);
1227 if (r)
1228 r = -EFAULT;
30ee2a98
JH		 1229 }
1230
30ee2a98
JH		 1231 kvfree(keys);
1232 return r;
1233}
1234
1235static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1236{
1237 uint8_t *keys;
1238 uint64_t hva;
1239 int i, r = 0;
1240
1241 if (args->flags != 0)
1242 return -EINVAL;
1243
1244 /* Enforce sane limit on memory allocation */
1245 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1246 return -EINVAL;
1247
752ade68 1248 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
30ee2a98
JH		 1249 if (!keys)
1250 return -ENOMEM;
1251
1252 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1253 sizeof(uint8_t) * args->count);
1254 if (r) {
1255 r = -EFAULT;
1256 goto out;
1257 }
1258
1259 /* Enable storage key handling for the guest */
14d4a425
DD		 1260 r = s390_enable_skey();
1261 if (r)
1262 goto out;
30ee2a98 1263
d3ed1cee 1264 down_read(&current->mm->mmap_sem);
30ee2a98
JH		 1265 for (i = 0; i < args->count; i++) {
1266 hva = gfn_to_hva(kvm, args->start_gfn + i);
1267 if (kvm_is_error_hva(hva)) {
1268 r = -EFAULT;
d3ed1cee 1269 break;
30ee2a98
JH		 1270 }
1271
1272 /* Lowest order bit is reserved */
1273 if (keys[i] & 0x01) {
1274 r = -EINVAL;
d3ed1cee 1275 break;
30ee2a98
JH		 1276 }
1277
fe69eabf 1278 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
30ee2a98 1279 if (r)
d3ed1cee 1280 break;
30ee2a98 1281 }
d3ed1cee 1282 up_read(&current->mm->mmap_sem);
30ee2a98
JH		 1283out:
1284 kvfree(keys);
1285 return r;
1286}
1287
b0c632db
HC		 1288long kvm_arch_vm_ioctl(struct file *filp,
1289 unsigned int ioctl, unsigned long arg)
1290{
1291 struct kvm *kvm = filp->private_data;
1292 void __user *argp = (void __user *)arg;
f2061656 1293 struct kvm_device_attr attr;
b0c632db
HC		 1294 int r;
1295
1296 switch (ioctl) {
ba5c1e9b
CO		 1297 case KVM_S390_INTERRUPT: {
1298 struct kvm_s390_interrupt s390int;
1299
1300 r = -EFAULT;
1301 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1302 break;
1303 r = kvm_s390_inject_vm(kvm, &s390int);
1304 break;
1305 }
d938dc55
CH		 1306 case KVM_ENABLE_CAP: {
1307 struct kvm_enable_cap cap;
1308 r = -EFAULT;
1309 if (copy_from_user(&cap, argp, sizeof(cap)))
1310 break;
1311 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1312 break;
1313 }
84223598
CH		 1314 case KVM_CREATE_IRQCHIP: {
1315 struct kvm_irq_routing_entry routing;
1316
1317 r = -EINVAL;
1318 if (kvm->arch.use_irqchip) {
1319 /* Set up dummy routing. */
1320 memset(&routing, 0, sizeof(routing));
152b2839 1321 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
84223598
CH		 1322 }
1323 break;
1324 }
f2061656
DD		 1325 case KVM_SET_DEVICE_ATTR: {
1326 r = -EFAULT;
1327 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1328 break;
1329 r = kvm_s390_vm_set_attr(kvm, &attr);
1330 break;
1331 }
1332 case KVM_GET_DEVICE_ATTR: {
1333 r = -EFAULT;
1334 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1335 break;
1336 r = kvm_s390_vm_get_attr(kvm, &attr);
1337 break;
1338 }
1339 case KVM_HAS_DEVICE_ATTR: {
1340 r = -EFAULT;
1341 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1342 break;
1343 r = kvm_s390_vm_has_attr(kvm, &attr);
1344 break;
1345 }
30ee2a98
JH		 1346 case KVM_S390_GET_SKEYS: {
1347 struct kvm_s390_skeys args;
1348
1349 r = -EFAULT;
1350 if (copy_from_user(&args, argp,
1351 sizeof(struct kvm_s390_skeys)))
1352 break;
1353 r = kvm_s390_get_skeys(kvm, &args);
1354 break;
1355 }
1356 case KVM_S390_SET_SKEYS: {
1357 struct kvm_s390_skeys args;
1358
1359 r = -EFAULT;
1360 if (copy_from_user(&args, argp,
1361 sizeof(struct kvm_s390_skeys)))
1362 break;
1363 r = kvm_s390_set_skeys(kvm, &args);
1364 break;
1365 }
b0c632db 1366 default:
367e1319 1367 r = -ENOTTY;
b0c632db
HC		 1368 }
1369
1370 return r;
1371}
1372
45c9b47c
TK		 1373static int kvm_s390_query_ap_config(u8 *config)
1374{
1375 u32 fcn_code = 0x04000000UL;
86044c8c 1376 u32 cc = 0;
45c9b47c 1377
86044c8c 1378 memset(config, 0, 128);
45c9b47c
TK		 1379 asm volatile(
1380 "lgr 0,%1\n"
1381 "lgr 2,%2\n"
1382 ".long 0xb2af0000\n" /* PQAP(QCI) */
86044c8c 1383 "0: ipm %0\n"
45c9b47c 1384 "srl %0,28\n"
86044c8c
CB		 1385 "1:\n"
1386 EX_TABLE(0b, 1b)
1387 : "+r" (cc)
45c9b47c
TK		 1388 : "r" (fcn_code), "r" (config)
1389 : "cc", "0", "2", "memory"
1390 );
1391
1392 return cc;
1393}
1394
1395static int kvm_s390_apxa_installed(void)
1396{
1397 u8 config[128];
1398 int cc;
1399
a6aacc3f 1400 if (test_facility(12)) {
45c9b47c
TK		 1401 cc = kvm_s390_query_ap_config(config);
1402
1403 if (cc)
1404 pr_err("PQAP(QCI) failed with cc=%d", cc);
1405 else
1406 return config[0] & 0x40;
1407 }
1408
1409 return 0;
1410}
1411
1412static void kvm_s390_set_crycb_format(struct kvm *kvm)
1413{
1414 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1415
1416 if (kvm_s390_apxa_installed())
1417 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1418 else
1419 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1420}
1421
9bb0ec09 1422static u64 kvm_s390_get_initial_cpuid(void)
9d8d5786 1423{
9bb0ec09
DH		 1424 struct cpuid cpuid;
1425
1426 get_cpu_id(&cpuid);
1427 cpuid.version = 0xff;
1428 return *((u64 *) &cpuid);
9d8d5786
MM		 1429}
1430
c54f0d6a 1431static void kvm_s390_crypto_init(struct kvm *kvm)
5102ee87 1432{
9d8d5786 1433 if (!test_kvm_facility(kvm, 76))
c54f0d6a 1434 return;
5102ee87 1435
c54f0d6a 1436 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
45c9b47c 1437 kvm_s390_set_crycb_format(kvm);
5102ee87 1438
ed6f76b4
TK		 1439 /* Enable AES/DEA protected key functions by default */
1440 kvm->arch.crypto.aes_kw = 1;
1441 kvm->arch.crypto.dea_kw = 1;
1442 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1443 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1444 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1445 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
5102ee87
TK		 1446}
1447
7d43bafc
ED		 1448static void sca_dispose(struct kvm *kvm)
1449{
1450 if (kvm->arch.use_esca)
5e044315 1451 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
7d43bafc
ED		 1452 else
1453 free_page((unsigned long)(kvm->arch.sca));
1454 kvm->arch.sca = NULL;
1455}
1456
e08b9637 1457int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 1458{
76a6dd72 1459 gfp_t alloc_flags = GFP_KERNEL;
9d8d5786 1460 int i, rc;
b0c632db 1461 char debug_name[16];
f6c137ff 1462 static unsigned long sca_offset;
b0c632db 1463
e08b9637
CO		 1464 rc = -EINVAL;
1465#ifdef CONFIG_KVM_S390_UCONTROL
1466 if (type & ~KVM_VM_S390_UCONTROL)
1467 goto out_err;
1468 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1469 goto out_err;
1470#else
1471 if (type)
1472 goto out_err;
1473#endif
1474
b0c632db
HC		 1475 rc = s390_enable_sie();
1476 if (rc)
d89f5eff 1477 goto out_err;
b0c632db 1478
b290411a
CO		 1479 rc = -ENOMEM;
1480
7d0a5e62
JF		 1481 ratelimit_state_init(&kvm->arch.sthyi_limit, 5 * HZ, 500);
1482
7d43bafc 1483 kvm->arch.use_esca = 0; /* start with basic SCA */
76a6dd72
DH		 1484 if (!sclp.has_64bscao)
1485 alloc_flags |= GFP_DMA;
5e044315 1486 rwlock_init(&kvm->arch.sca_lock);
76a6dd72 1487 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
b0c632db 1488 if (!kvm->arch.sca)
d89f5eff 1489 goto out_err;
f6c137ff 1490 spin_lock(&kvm_lock);
c5c2c393 1491 sca_offset += 16;
bc784cce 1492 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
c5c2c393 1493 sca_offset = 0;
bc784cce
ED		 1494 kvm->arch.sca = (struct bsca_block *)
1495 ((char *) kvm->arch.sca + sca_offset);
f6c137ff 1496 spin_unlock(&kvm_lock);
b0c632db
HC		 1497
1498 sprintf(debug_name, "kvm-%u", current->pid);
1499
1cb9cf72 1500 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
b0c632db 1501 if (!kvm->arch.dbf)
40f5b735 1502 goto out_err;
b0c632db 1503
c54f0d6a
DH		 1504 kvm->arch.sie_page2 =
1505 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1506 if (!kvm->arch.sie_page2)
40f5b735 1507 goto out_err;
9d8d5786 1508
fb5bf93f 1509 /* Populate the facility mask initially. */
c54f0d6a 1510 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
04478197 1511 sizeof(S390_lowcore.stfle_fac_list));
9d8d5786
MM		 1512 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1513 if (i < kvm_s390_fac_list_mask_size())
c54f0d6a 1514 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
9d8d5786 1515 else
c54f0d6a 1516 kvm->arch.model.fac_mask[i] = 0UL;
9d8d5786
MM		 1517 }
1518
981467c9 1519 /* Populate the facility list initially. */
c54f0d6a
DH		 1520 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1521 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
981467c9
MM		 1522 S390_ARCH_FAC_LIST_SIZE_BYTE);
1523
95ca2cb5
JF		 1524 set_kvm_facility(kvm->arch.model.fac_mask, 74);
1525 set_kvm_facility(kvm->arch.model.fac_list, 74);
1526
9bb0ec09 1527 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
37c5f6c8 1528 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
9d8d5786 1529
c54f0d6a 1530 kvm_s390_crypto_init(kvm);
5102ee87 1531
51978393
FL		 1532 mutex_init(&kvm->arch.float_int.ais_lock);
1533 kvm->arch.float_int.simm = 0;
1534 kvm->arch.float_int.nimm = 0;
ba5c1e9b 1535 spin_lock_init(&kvm->arch.float_int.lock);
6d3da241
JF		 1536 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1537 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
8a242234 1538 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 1539 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 1540
b0c632db 1541 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
78f26131 1542 VM_EVENT(kvm, 3, "vm created with type %lu", type);
b0c632db 1543
e08b9637
CO		 1544 if (type & KVM_VM_S390_UCONTROL) {
1545 kvm->arch.gmap = NULL;
a3a92c31 1546 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
e08b9637 1547 } else {
32e6b236 1548 if (sclp.hamax == U64_MAX)
ee71d16d 1549 kvm->arch.mem_limit = TASK_SIZE_MAX;
32e6b236 1550 else
ee71d16d 1551 kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
32e6b236 1552 sclp.hamax + 1);
6ea427bb 1553 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
e08b9637 1554 if (!kvm->arch.gmap)
40f5b735 1555 goto out_err;
2c70fe44 1556 kvm->arch.gmap->private = kvm;
24eb3a82 1557 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 1558 }
fa6b7fe9
CH		 1559
1560 kvm->arch.css_support = 0;
84223598 1561 kvm->arch.use_irqchip = 0;
72f25020 1562 kvm->arch.epoch = 0;
fa6b7fe9 1563
8ad35755 1564 spin_lock_init(&kvm->arch.start_stop_lock);
a3508fbe 1565 kvm_s390_vsie_init(kvm);
8335713a 1566 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
8ad35755 1567
d89f5eff 1568 return 0;
40f5b735 1569out_err:
c54f0d6a 1570 free_page((unsigned long)kvm->arch.sie_page2);
598841ca 1571 debug_unregister(kvm->arch.dbf);
7d43bafc 1572 sca_dispose(kvm);
78f26131 1573 KVM_EVENT(3, "creation of vm failed: %d", rc);
d89f5eff 1574 return rc;
b0c632db
HC		 1575}
1576
235539b4
LC		 1577bool kvm_arch_has_vcpu_debugfs(void)
1578{
1579 return false;
1580}
1581
1582int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
1583{
1584 return 0;
1585}
1586
d329c035
CB		 1587void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1588{
1589 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 1590 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 1591 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 1592 kvm_clear_async_pf_completion_queue(vcpu);
bc784cce 1593 if (!kvm_is_ucontrol(vcpu->kvm))
a6e2f683 1594 sca_del_vcpu(vcpu);
27e0393f
CO		 1595
1596 if (kvm_is_ucontrol(vcpu->kvm))
6ea427bb 1597 gmap_remove(vcpu->arch.gmap);
27e0393f 1598
e6db1d61 1599 if (vcpu->kvm->arch.use_cmma)
b31605c1 1600 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 1601 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 1602
6692cef3 1603 kvm_vcpu_uninit(vcpu);
b110feaf 1604 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 1605}
1606
1607static void kvm_free_vcpus(struct kvm *kvm)
1608{
1609 unsigned int i;
988a2cae 1610 struct kvm_vcpu *vcpu;
d329c035 1611
988a2cae
GN		 1612 kvm_for_each_vcpu(i, vcpu, kvm)
1613 kvm_arch_vcpu_destroy(vcpu);
1614
1615 mutex_lock(&kvm->lock);
1616 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1617 kvm->vcpus[i] = NULL;
1618
1619 atomic_set(&kvm->online_vcpus, 0);
1620 mutex_unlock(&kvm->lock);
d329c035
CB		 1621}
1622
b0c632db
HC		 1623void kvm_arch_destroy_vm(struct kvm *kvm)
1624{
d329c035 1625 kvm_free_vcpus(kvm);
7d43bafc 1626 sca_dispose(kvm);
d329c035 1627 debug_unregister(kvm->arch.dbf);
c54f0d6a 1628 free_page((unsigned long)kvm->arch.sie_page2);
27e0393f 1629 if (!kvm_is_ucontrol(kvm))
6ea427bb 1630 gmap_remove(kvm->arch.gmap);
841b91c5 1631 kvm_s390_destroy_adapters(kvm);
67335e63 1632 kvm_s390_clear_float_irqs(kvm);
a3508fbe 1633 kvm_s390_vsie_destroy(kvm);
8335713a 1634 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
b0c632db
HC		 1635}
1636
1637/* Section: vcpu related */
dafd032a
DD		 1638static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1639{
6ea427bb 1640 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
dafd032a
DD		 1641 if (!vcpu->arch.gmap)
1642 return -ENOMEM;
1643 vcpu->arch.gmap->private = vcpu->kvm;
1644
1645 return 0;
1646}
1647
a6e2f683
ED		 1648static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1649{
a6940674
DH		 1650 if (!kvm_s390_use_sca_entries())
1651 return;
5e044315 1652 read_lock(&vcpu->kvm->arch.sca_lock);
7d43bafc
ED		 1653 if (vcpu->kvm->arch.use_esca) {
1654 struct esca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1655
7d43bafc 1656 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
10ce32d5 1657 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc
ED		 1658 } else {
1659 struct bsca_block *sca = vcpu->kvm->arch.sca;
1660
1661 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
10ce32d5 1662 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc 1663 }
5e044315 1664 read_unlock(&vcpu->kvm->arch.sca_lock);
a6e2f683
ED		 1665}
1666
eaa78f34 1667static void sca_add_vcpu(struct kvm_vcpu *vcpu)
a6e2f683 1668{
a6940674
DH		 1669 if (!kvm_s390_use_sca_entries()) {
1670 struct bsca_block *sca = vcpu->kvm->arch.sca;
1671
1672 /* we still need the basic sca for the ipte control */
1673 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1674 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
1675 }
eaa78f34
DH		 1676 read_lock(&vcpu->kvm->arch.sca_lock);
1677 if (vcpu->kvm->arch.use_esca) {
1678 struct esca_block *sca = vcpu->kvm->arch.sca;
7d43bafc 1679
eaa78f34 1680 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1681 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1682 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
0c9d8683 1683 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
eaa78f34 1684 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
7d43bafc 1685 } else {
eaa78f34 1686 struct bsca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1687
eaa78f34 1688 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1689 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1690 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
eaa78f34 1691 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
7d43bafc 1692 }
eaa78f34 1693 read_unlock(&vcpu->kvm->arch.sca_lock);
5e044315
ED		 1694}
1695
1696/* Basic SCA to Extended SCA data copy routines */
1697static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1698{
1699 d->sda = s->sda;
1700 d->sigp_ctrl.c = s->sigp_ctrl.c;
1701 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1702}
1703
1704static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1705{
1706 int i;
1707
1708 d->ipte_control = s->ipte_control;
1709 d->mcn[0] = s->mcn;
1710 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1711 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1712}
1713
1714static int sca_switch_to_extended(struct kvm *kvm)
1715{
1716 struct bsca_block *old_sca = kvm->arch.sca;
1717 struct esca_block *new_sca;
1718 struct kvm_vcpu *vcpu;
1719 unsigned int vcpu_idx;
1720 u32 scaol, scaoh;
1721
1722 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1723 if (!new_sca)
1724 return -ENOMEM;
1725
1726 scaoh = (u32)((u64)(new_sca) >> 32);
1727 scaol = (u32)(u64)(new_sca) & ~0x3fU;
1728
1729 kvm_s390_vcpu_block_all(kvm);
1730 write_lock(&kvm->arch.sca_lock);
1731
1732 sca_copy_b_to_e(new_sca, old_sca);
1733
1734 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1735 vcpu->arch.sie_block->scaoh = scaoh;
1736 vcpu->arch.sie_block->scaol = scaol;
0c9d8683 1737 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
5e044315
ED		 1738 }
1739 kvm->arch.sca = new_sca;
1740 kvm->arch.use_esca = 1;
1741
1742 write_unlock(&kvm->arch.sca_lock);
1743 kvm_s390_vcpu_unblock_all(kvm);
1744
1745 free_page((unsigned long)old_sca);
1746
8335713a
CB		 1747 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1748 old_sca, kvm->arch.sca);
5e044315 1749 return 0;
a6e2f683
ED		 1750}
1751
1752static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1753{
5e044315
ED		 1754 int rc;
1755
a6940674
DH		 1756 if (!kvm_s390_use_sca_entries()) {
1757 if (id < KVM_MAX_VCPUS)
1758 return true;
1759 return false;
1760 }
5e044315
ED		 1761 if (id < KVM_S390_BSCA_CPU_SLOTS)
1762 return true;
76a6dd72 1763 if (!sclp.has_esca || !sclp.has_64bscao)
5e044315
ED		 1764 return false;
1765
1766 mutex_lock(&kvm->lock);
1767 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1768 mutex_unlock(&kvm->lock);
1769
1770 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
a6e2f683
ED		 1771}
1772
b0c632db
HC		 1773int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1774{
3c038e6b
DD		 1775 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1776 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 1777 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1778 KVM_SYNC_GPRS |
9eed0735 1779 KVM_SYNC_ACRS |
b028ee3e
DH		 1780 KVM_SYNC_CRS |
1781 KVM_SYNC_ARCH0 |
1782 KVM_SYNC_PFAULT;
75a4615c 1783 kvm_s390_set_prefix(vcpu, 0);
c6e5f166
FZ		 1784 if (test_kvm_facility(vcpu->kvm, 64))
1785 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
4e0b1ab7
FZ		 1786 if (test_kvm_facility(vcpu->kvm, 133))
1787 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
f6aa6dc4
DH		 1788 /* fprs can be synchronized via vrs, even if the guest has no vx. With
1789 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1790 */
1791 if (MACHINE_HAS_VX)
68c55750 1792 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
6fd8e67d
DH		 1793 else
1794 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
dafd032a
DD		 1795
1796 if (kvm_is_ucontrol(vcpu->kvm))
1797 return __kvm_ucontrol_vcpu_init(vcpu);
1798
b0c632db
HC		 1799 return 0;
1800}
1801
db0758b2
DH		 1802/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1803static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1804{
1805 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
9c23a131 1806 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2 1807 vcpu->arch.cputm_start = get_tod_clock_fast();
9c23a131 1808 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1809}
1810
1811/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1812static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1813{
1814 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
9c23a131 1815 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1816 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1817 vcpu->arch.cputm_start = 0;
9c23a131 1818 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1819}
1820
1821/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1822static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1823{
1824 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
1825 vcpu->arch.cputm_enabled = true;
1826 __start_cpu_timer_accounting(vcpu);
1827}
1828
1829/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1830static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1831{
1832 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
1833 __stop_cpu_timer_accounting(vcpu);
1834 vcpu->arch.cputm_enabled = false;
1835}
1836
1837static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1838{
1839 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1840 __enable_cpu_timer_accounting(vcpu);
1841 preempt_enable();
1842}
1843
1844static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1845{
1846 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1847 __disable_cpu_timer_accounting(vcpu);
1848 preempt_enable();
1849}
1850
4287f247
DH		 1851/* set the cpu timer - may only be called from the VCPU thread itself */
1852void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
1853{
db0758b2 1854 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
9c23a131 1855 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1856 if (vcpu->arch.cputm_enabled)
1857 vcpu->arch.cputm_start = get_tod_clock_fast();
4287f247 1858 vcpu->arch.sie_block->cputm = cputm;
9c23a131 1859 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2 1860 preempt_enable();
4287f247
DH		 1861}
1862
db0758b2 1863/* update and get the cpu timer - can also be called from other VCPU threads */
4287f247
DH		 1864__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
1865{
9c23a131 1866 unsigned int seq;
db0758b2 1867 __u64 value;
db0758b2
DH		 1868
1869 if (unlikely(!vcpu->arch.cputm_enabled))
1870 return vcpu->arch.sie_block->cputm;
1871
9c23a131
DH		 1872 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1873 do {
1874 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
1875 /*
1876 * If the writer would ever execute a read in the critical
1877 * section, e.g. in irq context, we have a deadlock.
1878 */
1879 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
1880 value = vcpu->arch.sie_block->cputm;
1881 /* if cputm_start is 0, accounting is being started/stopped */
1882 if (likely(vcpu->arch.cputm_start))
1883 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1884 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
1885 preempt_enable();
db0758b2 1886 return value;
4287f247
DH		 1887}
1888
b0c632db
HC		 1889void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1890{
9977e886 1891
37d9df98 1892 gmap_enable(vcpu->arch.enabled_gmap);
805de8f4 1893 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
5ebda316 1894 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1895 __start_cpu_timer_accounting(vcpu);
01a745ac 1896 vcpu->cpu = cpu;
b0c632db
HC		 1897}
1898
1899void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1900{
01a745ac 1901 vcpu->cpu = -1;
5ebda316 1902 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1903 __stop_cpu_timer_accounting(vcpu);
805de8f4 1904 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
37d9df98
DH		 1905 vcpu->arch.enabled_gmap = gmap_get_enabled();
1906 gmap_disable(vcpu->arch.enabled_gmap);
9977e886 1907
b0c632db
HC		 1908}
1909
1910static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1911{
1912 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1913 vcpu->arch.sie_block->gpsw.mask = 0UL;
1914 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 1915 kvm_s390_set_prefix(vcpu, 0);
4287f247 1916 kvm_s390_set_cpu_timer(vcpu, 0);
b0c632db
HC		 1917 vcpu->arch.sie_block->ckc = 0UL;
1918 vcpu->arch.sie_block->todpr = 0;
1919 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1920 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1921 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
9abc2a08
DH		 1922 /* make sure the new fpc will be lazily loaded */
1923 save_fpu_regs();
1924 current->thread.fpu.fpc = 0;
b0c632db 1925 vcpu->arch.sie_block->gbea = 1;
672550fb 1926 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 1927 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1928 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 1929 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1930 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 1931 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 1932}
1933
31928aa5 1934void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 1935{
72f25020 1936 mutex_lock(&vcpu->kvm->lock);
fdf03650 1937 preempt_disable();
72f25020 1938 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
fdf03650 1939 preempt_enable();
72f25020 1940 mutex_unlock(&vcpu->kvm->lock);
25508824 1941 if (!kvm_is_ucontrol(vcpu->kvm)) {
dafd032a 1942 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
eaa78f34 1943 sca_add_vcpu(vcpu);
25508824 1944 }
6502a34c
DH		 1945 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
1946 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
37d9df98
DH		 1947 /* make vcpu_load load the right gmap on the first trigger */
1948 vcpu->arch.enabled_gmap = vcpu->arch.gmap;
42897d86
MT		 1949}
1950
5102ee87
TK		 1951static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1952{
9d8d5786 1953 if (!test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 1954 return;
1955
a374e892
TK		 1956 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1957
1958 if (vcpu->kvm->arch.crypto.aes_kw)
1959 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1960 if (vcpu->kvm->arch.crypto.dea_kw)
1961 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1962
5102ee87
TK		 1963 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1964}
1965
b31605c1
DD		 1966void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1967{
1968 free_page(vcpu->arch.sie_block->cbrlo);
1969 vcpu->arch.sie_block->cbrlo = 0;
1970}
1971
1972int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1973{
1974 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1975 if (!vcpu->arch.sie_block->cbrlo)
1976 return -ENOMEM;
1977
0c9d8683
DH		 1978 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
1979 vcpu->arch.sie_block->ecb2 &= ~ECB2_PFMFI;
b31605c1
DD		 1980 return 0;
1981}
1982
91520f1a
MM		 1983static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1984{
1985 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1986
91520f1a 1987 vcpu->arch.sie_block->ibc = model->ibc;
80bc79dc 1988 if (test_kvm_facility(vcpu->kvm, 7))
c54f0d6a 1989 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
91520f1a
MM		 1990}
1991
b0c632db
HC		 1992int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1993{
b31605c1 1994 int rc = 0;
b31288fa 1995
9e6dabef
CH		 1996 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1997 CPUSTAT_SM |
a4a4f191
GH		 1998 CPUSTAT_STOPPED);
1999
53df84f8 2000 if (test_kvm_facility(vcpu->kvm, 78))
805de8f4 2001 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
53df84f8 2002 else if (test_kvm_facility(vcpu->kvm, 8))
805de8f4 2003 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
a4a4f191 2004
91520f1a
MM		 2005 kvm_s390_vcpu_setup_model(vcpu);
2006
bdab09f3
DH		 2007 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
2008 if (MACHINE_HAS_ESOP)
0c9d8683 2009 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
bd50e8ec 2010 if (test_kvm_facility(vcpu->kvm, 9))
0c9d8683 2011 vcpu->arch.sie_block->ecb |= ECB_SRSI;
f597d24e 2012 if (test_kvm_facility(vcpu->kvm, 73))
0c9d8683 2013 vcpu->arch.sie_block->ecb |= ECB_TE;
7feb6bb8 2014
873b425e 2015 if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
0c9d8683 2016 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
cd1836f5 2017 if (test_kvm_facility(vcpu->kvm, 130))
0c9d8683
DH		 2018 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
2019 vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
48ee7d3a 2020 if (sclp.has_cei)
0c9d8683 2021 vcpu->arch.sie_block->eca |= ECA_CEI;
11ad65b7 2022 if (sclp.has_ib)
0c9d8683 2023 vcpu->arch.sie_block->eca |= ECA_IB;
37c5f6c8 2024 if (sclp.has_siif)
0c9d8683 2025 vcpu->arch.sie_block->eca |= ECA_SII;
37c5f6c8 2026 if (sclp.has_sigpif)
0c9d8683 2027 vcpu->arch.sie_block->eca |= ECA_SIGPI;
18280d8b 2028 if (test_kvm_facility(vcpu->kvm, 129)) {
0c9d8683
DH		 2029 vcpu->arch.sie_block->eca |= ECA_VX;
2030 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
13211ea7 2031 }
4e0b1ab7
FZ		 2032 vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
2033 | SDNXC;
c6e5f166 2034 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
730cd632
FA		 2035
2036 if (sclp.has_kss)
2037 atomic_or(CPUSTAT_KSS, &vcpu->arch.sie_block->cpuflags);
2038 else
2039 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
5a5e6536 2040
e6db1d61 2041 if (vcpu->kvm->arch.use_cmma) {
b31605c1
DD		 2042 rc = kvm_s390_vcpu_setup_cmma(vcpu);
2043 if (rc)
2044 return rc;
b31288fa 2045 }
0ac96caf 2046 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 2047 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 2048
5102ee87
TK		 2049 kvm_s390_vcpu_crypto_setup(vcpu);
2050
b31605c1 2051 return rc;
b0c632db
HC		 2052}
2053
2054struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
2055 unsigned int id)
2056{
4d47555a 2057 struct kvm_vcpu *vcpu;
7feb6bb8 2058 struct sie_page *sie_page;
4d47555a
CO		 2059 int rc = -EINVAL;
2060
4215825e 2061 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
4d47555a
CO		 2062 goto out;
2063
2064 rc = -ENOMEM;
b0c632db 2065
b110feaf 2066 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 2067 if (!vcpu)
4d47555a 2068 goto out;
b0c632db 2069
7feb6bb8
MM		 2070 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
2071 if (!sie_page)
b0c632db
HC		 2072 goto out_free_cpu;
2073
7feb6bb8
MM		 2074 vcpu->arch.sie_block = &sie_page->sie_block;
2075 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
2076
efed1104
DH		 2077 /* the real guest size will always be smaller than msl */
2078 vcpu->arch.sie_block->mso = 0;
2079 vcpu->arch.sie_block->msl = sclp.hamax;
2080
b0c632db 2081 vcpu->arch.sie_block->icpua = id;
ba5c1e9b 2082 spin_lock_init(&vcpu->arch.local_int.lock);
ba5c1e9b 2083 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 2084 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 2085 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
9c23a131 2086 seqcount_init(&vcpu->arch.cputm_seqcount);
ba5c1e9b 2087
b0c632db
HC		 2088 rc = kvm_vcpu_init(vcpu, kvm, id);
2089 if (rc)
9abc2a08 2090 goto out_free_sie_block;
8335713a 2091 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
b0c632db 2092 vcpu->arch.sie_block);
ade38c31 2093 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 2094
b0c632db 2095 return vcpu;
7b06bf2f
WY		 2096out_free_sie_block:
2097 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 2098out_free_cpu:
b110feaf 2099 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 2100out:
b0c632db
HC		 2101 return ERR_PTR(rc);
2102}
2103
b0c632db
HC		 2104int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
2105{
9a022067 2106 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 2107}
2108
27406cd5 2109void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
49b99e1e 2110{
805de8f4 2111 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
61a6df54 2112 exit_sie(vcpu);
49b99e1e
CB		 2113}
2114
27406cd5 2115void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
49b99e1e 2116{
805de8f4 2117 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
49b99e1e
CB		 2118}
2119
8e236546
CB		 2120static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
2121{
805de8f4 2122 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
61a6df54 2123 exit_sie(vcpu);
8e236546
CB		 2124}
2125
2126static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2127{
9bf9fde2 2128 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
8e236546
CB		 2129}
2130
49b99e1e
CB		 2131/*
2132 * Kick a guest cpu out of SIE and wait until SIE is not running.
2133 * If the CPU is not running (e.g. waiting as idle) the function will
2134 * return immediately. */
2135void exit_sie(struct kvm_vcpu *vcpu)
2136{
805de8f4 2137 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
49b99e1e
CB		 2138 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2139 cpu_relax();
2140}
2141
8e236546
CB		 2142/* Kick a guest cpu out of SIE to process a request synchronously */
2143void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
49b99e1e 2144{
8e236546
CB		 2145 kvm_make_request(req, vcpu);
2146 kvm_s390_vcpu_request(vcpu);
49b99e1e
CB		 2147}
2148
414d3b07
MS		 2149static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
2150 unsigned long end)
2c70fe44 2151{
2c70fe44
CB		 2152 struct kvm *kvm = gmap->private;
2153 struct kvm_vcpu *vcpu;
414d3b07
MS		 2154 unsigned long prefix;
2155 int i;
2c70fe44 2156
65d0b0d4
DH		 2157 if (gmap_is_shadow(gmap))
2158 return;
414d3b07
MS		 2159 if (start >= 1UL << 31)
2160 /* We are only interested in prefix pages */
2161 return;
2c70fe44
CB		 2162 kvm_for_each_vcpu(i, vcpu, kvm) {
2163 /* match against both prefix pages */
414d3b07
MS		 2164 prefix = kvm_s390_get_prefix(vcpu);
2165 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
2166 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
2167 start, end);
8e236546 2168 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44
CB		 2169 }
2170 }
2171}
2172
b6d33834
CD		 2173int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2174{
2175 /* kvm common code refers to this, but never calls it */
2176 BUG();
2177 return 0;
2178}
2179
14eebd91
CO		 2180static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2181 struct kvm_one_reg *reg)
2182{
2183 int r = -EINVAL;
2184
2185 switch (reg->id) {
29b7c71b
CO		 2186 case KVM_REG_S390_TODPR:
2187 r = put_user(vcpu->arch.sie_block->todpr,
2188 (u32 __user *)reg->addr);
2189 break;
2190 case KVM_REG_S390_EPOCHDIFF:
2191 r = put_user(vcpu->arch.sie_block->epoch,
2192 (u64 __user *)reg->addr);
2193 break;
46a6dd1c 2194 case KVM_REG_S390_CPU_TIMER:
4287f247 2195 r = put_user(kvm_s390_get_cpu_timer(vcpu),
46a6dd1c
J		 2196 (u64 __user *)reg->addr);
2197 break;
2198 case KVM_REG_S390_CLOCK_COMP:
2199 r = put_user(vcpu->arch.sie_block->ckc,
2200 (u64 __user *)reg->addr);
2201 break;
536336c2
DD		 2202 case KVM_REG_S390_PFTOKEN:
2203 r = put_user(vcpu->arch.pfault_token,
2204 (u64 __user *)reg->addr);
2205 break;
2206 case KVM_REG_S390_PFCOMPARE:
2207 r = put_user(vcpu->arch.pfault_compare,
2208 (u64 __user *)reg->addr);
2209 break;
2210 case KVM_REG_S390_PFSELECT:
2211 r = put_user(vcpu->arch.pfault_select,
2212 (u64 __user *)reg->addr);
2213 break;
672550fb
CB		 2214 case KVM_REG_S390_PP:
2215 r = put_user(vcpu->arch.sie_block->pp,
2216 (u64 __user *)reg->addr);
2217 break;
afa45ff5
CB		 2218 case KVM_REG_S390_GBEA:
2219 r = put_user(vcpu->arch.sie_block->gbea,
2220 (u64 __user *)reg->addr);
2221 break;
14eebd91
CO		 2222 default:
2223 break;
2224 }
2225
2226 return r;
2227}
2228
2229static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2230 struct kvm_one_reg *reg)
2231{
2232 int r = -EINVAL;
4287f247 2233 __u64 val;
14eebd91
CO		 2234
2235 switch (reg->id) {
29b7c71b
CO		 2236 case KVM_REG_S390_TODPR:
2237 r = get_user(vcpu->arch.sie_block->todpr,
2238 (u32 __user *)reg->addr);
2239 break;
2240 case KVM_REG_S390_EPOCHDIFF:
2241 r = get_user(vcpu->arch.sie_block->epoch,
2242 (u64 __user *)reg->addr);
2243 break;
46a6dd1c 2244 case KVM_REG_S390_CPU_TIMER:
4287f247
DH		 2245 r = get_user(val, (u64 __user *)reg->addr);
2246 if (!r)
2247 kvm_s390_set_cpu_timer(vcpu, val);
46a6dd1c
J		 2248 break;
2249 case KVM_REG_S390_CLOCK_COMP:
2250 r = get_user(vcpu->arch.sie_block->ckc,
2251 (u64 __user *)reg->addr);
2252 break;
536336c2
DD		 2253 case KVM_REG_S390_PFTOKEN:
2254 r = get_user(vcpu->arch.pfault_token,
2255 (u64 __user *)reg->addr);
9fbd8082
DH		 2256 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2257 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 2258 break;
2259 case KVM_REG_S390_PFCOMPARE:
2260 r = get_user(vcpu->arch.pfault_compare,
2261 (u64 __user *)reg->addr);
2262 break;
2263 case KVM_REG_S390_PFSELECT:
2264 r = get_user(vcpu->arch.pfault_select,
2265 (u64 __user *)reg->addr);
2266 break;
672550fb
CB		 2267 case KVM_REG_S390_PP:
2268 r = get_user(vcpu->arch.sie_block->pp,
2269 (u64 __user *)reg->addr);
2270 break;
afa45ff5
CB		 2271 case KVM_REG_S390_GBEA:
2272 r = get_user(vcpu->arch.sie_block->gbea,
2273 (u64 __user *)reg->addr);
2274 break;
14eebd91
CO		 2275 default:
2276 break;
2277 }
2278
2279 return r;
2280}
b6d33834 2281
b0c632db
HC		 2282static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2283{
b0c632db 2284 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 2285 return 0;
2286}
2287
2288int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2289{
5a32c1af 2290 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 2291 return 0;
2292}
2293
2294int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2295{
5a32c1af 2296 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 2297 return 0;
2298}
2299
2300int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2301 struct kvm_sregs *sregs)
2302{
59674c1a 2303 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 2304 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
b0c632db
HC		 2305 return 0;
2306}
2307
2308int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2309 struct kvm_sregs *sregs)
2310{
59674c1a 2311 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 2312 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 2313 return 0;
2314}
2315
2316int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2317{
4725c860
MS		 2318 if (test_fp_ctl(fpu->fpc))
2319 return -EINVAL;
e1788bb9 2320 vcpu->run->s.regs.fpc = fpu->fpc;
9abc2a08 2321 if (MACHINE_HAS_VX)
a7d4b8f2
DH		 2322 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
2323 (freg_t *) fpu->fprs);
9abc2a08 2324 else
a7d4b8f2 2325 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
b0c632db
HC		 2326 return 0;
2327}
2328
2329int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2330{
9abc2a08
DH		 2331 /* make sure we have the latest values */
2332 save_fpu_regs();
2333 if (MACHINE_HAS_VX)
a7d4b8f2
DH		 2334 convert_vx_to_fp((freg_t *) fpu->fprs,
2335 (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08 2336 else
a7d4b8f2 2337 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
e1788bb9 2338 fpu->fpc = vcpu->run->s.regs.fpc;
b0c632db
HC		 2339 return 0;
2340}
2341
2342static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2343{
2344 int rc = 0;
2345
7a42fdc2 2346 if (!is_vcpu_stopped(vcpu))
b0c632db 2347 rc = -EBUSY;
d7b0b5eb
CO		 2348 else {
2349 vcpu->run->psw_mask = psw.mask;
2350 vcpu->run->psw_addr = psw.addr;
2351 }
b0c632db
HC		 2352 return rc;
2353}
2354
2355int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2356 struct kvm_translation *tr)
2357{
2358 return -EINVAL; /* not implemented yet */
2359}
2360
27291e21
DH		 2361#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2362 KVM_GUESTDBG_USE_HW_BP | \
2363 KVM_GUESTDBG_ENABLE)
2364
d0bfb940
JK		 2365int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2366 struct kvm_guest_debug *dbg)
b0c632db 2367{
27291e21
DH		 2368 int rc = 0;
2369
2370 vcpu->guest_debug = 0;
2371 kvm_s390_clear_bp_data(vcpu);
2372
2de3bfc2 2373 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21 2374 return -EINVAL;
89b5b4de
DH		 2375 if (!sclp.has_gpere)
2376 return -EINVAL;
27291e21
DH		 2377
2378 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2379 vcpu->guest_debug = dbg->control;
2380 /* enforce guest PER */
805de8f4 2381 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2382
2383 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2384 rc = kvm_s390_import_bp_data(vcpu, dbg);
2385 } else {
805de8f4 2386 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2387 vcpu->arch.guestdbg.last_bp = 0;
2388 }
2389
2390 if (rc) {
2391 vcpu->guest_debug = 0;
2392 kvm_s390_clear_bp_data(vcpu);
805de8f4 2393 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2394 }
2395
2396 return rc;
b0c632db
HC		 2397}
2398
62d9f0db
MT		 2399int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2400 struct kvm_mp_state *mp_state)
2401{
6352e4d2
DH		 2402 /* CHECK_STOP and LOAD are not supported yet */
2403 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2404 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 2405}
2406
2407int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2408 struct kvm_mp_state *mp_state)
2409{
6352e4d2
DH		 2410 int rc = 0;
2411
2412 /* user space knows about this interface - let it control the state */
2413 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2414
2415 switch (mp_state->mp_state) {
2416 case KVM_MP_STATE_STOPPED:
2417 kvm_s390_vcpu_stop(vcpu);
2418 break;
2419 case KVM_MP_STATE_OPERATING:
2420 kvm_s390_vcpu_start(vcpu);
2421 break;
2422 case KVM_MP_STATE_LOAD:
2423 case KVM_MP_STATE_CHECK_STOP:
2424 /* fall through - CHECK_STOP and LOAD are not supported yet */
2425 default:
2426 rc = -ENXIO;
2427 }
2428
2429 return rc;
62d9f0db
MT		 2430}
2431
8ad35755
DH		 2432static bool ibs_enabled(struct kvm_vcpu *vcpu)
2433{
2434 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2435}
2436
2c70fe44
CB		 2437static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2438{
8ad35755 2439retry:
8e236546 2440 kvm_s390_vcpu_request_handled(vcpu);
586b7ccd
CB		 2441 if (!vcpu->requests)
2442 return 0;
2c70fe44
CB		 2443 /*
2444 * We use MMU_RELOAD just to re-arm the ipte notifier for the
b2d73b2a 2445 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2c70fe44
CB		 2446 * This ensures that the ipte instruction for this request has
2447 * already finished. We might race against a second unmapper that
2448 * wants to set the blocking bit. Lets just retry the request loop.
2449 */
8ad35755 2450 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44 2451 int rc;
b2d73b2a
MS		 2452 rc = gmap_mprotect_notify(vcpu->arch.gmap,
2453 kvm_s390_get_prefix(vcpu),
2454 PAGE_SIZE * 2, PROT_WRITE);
aca411a4
JN		 2455 if (rc) {
2456 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44 2457 return rc;
aca411a4 2458 }
8ad35755 2459 goto retry;
2c70fe44 2460 }
8ad35755 2461
d3d692c8
DH		 2462 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2463 vcpu->arch.sie_block->ihcpu = 0xffff;
2464 goto retry;
2465 }
2466
8ad35755
DH		 2467 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2468 if (!ibs_enabled(vcpu)) {
2469 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
805de8f4 2470 atomic_or(CPUSTAT_IBS,
8ad35755
DH		 2471 &vcpu->arch.sie_block->cpuflags);
2472 }
2473 goto retry;
2c70fe44 2474 }
8ad35755
DH		 2475
2476 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2477 if (ibs_enabled(vcpu)) {
2478 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
805de8f4 2479 atomic_andnot(CPUSTAT_IBS,
8ad35755
DH		 2480 &vcpu->arch.sie_block->cpuflags);
2481 }
2482 goto retry;
2483 }
2484
6502a34c
DH		 2485 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
2486 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2487 goto retry;
2488 }
2489
0759d068 2490 /* nothing to do, just clear the request */
72875d8a 2491 kvm_clear_request(KVM_REQ_UNHALT, vcpu);
0759d068 2492
2c70fe44
CB		 2493 return 0;
2494}
2495
25ed1675
DH		 2496void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2497{
2498 struct kvm_vcpu *vcpu;
2499 int i;
2500
2501 mutex_lock(&kvm->lock);
2502 preempt_disable();
2503 kvm->arch.epoch = tod - get_tod_clock();
2504 kvm_s390_vcpu_block_all(kvm);
2505 kvm_for_each_vcpu(i, vcpu, kvm)
2506 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2507 kvm_s390_vcpu_unblock_all(kvm);
2508 preempt_enable();
2509 mutex_unlock(&kvm->lock);
2510}
2511
fa576c58
TH		 2512/**
2513 * kvm_arch_fault_in_page - fault-in guest page if necessary
2514 * @vcpu: The corresponding virtual cpu
2515 * @gpa: Guest physical address
2516 * @writable: Whether the page should be writable or not
2517 *
2518 * Make sure that a guest page has been faulted-in on the host.
2519 *
2520 * Return: Zero on success, negative error code otherwise.
2521 */
2522long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 2523{
527e30b4
MS		 2524 return gmap_fault(vcpu->arch.gmap, gpa,
2525 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 2526}
2527
3c038e6b
DD		 2528static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2529 unsigned long token)
2530{
2531 struct kvm_s390_interrupt inti;
383d0b05 2532 struct kvm_s390_irq irq;
3c038e6b
DD		 2533
2534 if (start_token) {
383d0b05
JF		 2535 irq.u.ext.ext_params2 = token;
2536 irq.type = KVM_S390_INT_PFAULT_INIT;
2537 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 2538 } else {
2539 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 2540 inti.parm64 = token;
3c038e6b
DD		 2541 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2542 }
2543}
2544
2545void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2546 struct kvm_async_pf *work)
2547{
2548 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2549 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2550}
2551
2552void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2553 struct kvm_async_pf *work)
2554{
2555 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2556 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2557}
2558
2559void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2560 struct kvm_async_pf *work)
2561{
2562 /* s390 will always inject the page directly */
2563}
2564
2565bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2566{
2567 /*
2568 * s390 will always inject the page directly,
2569 * but we still want check_async_completion to cleanup
2570 */
2571 return true;
2572}
2573
2574static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2575{
2576 hva_t hva;
2577 struct kvm_arch_async_pf arch;
2578 int rc;
2579
2580 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2581 return 0;
2582 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2583 vcpu->arch.pfault_compare)
2584 return 0;
2585 if (psw_extint_disabled(vcpu))
2586 return 0;
9a022067 2587 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 2588 return 0;
2589 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2590 return 0;
2591 if (!vcpu->arch.gmap->pfault_enabled)
2592 return 0;
2593
81480cc1
HC		 2594 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2595 hva += current->thread.gmap_addr & ~PAGE_MASK;
2596 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 2597 return 0;
2598
2599 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2600 return rc;
2601}
2602
3fb4c40f 2603static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 2604{
3fb4c40f 2605 int rc, cpuflags;
e168bf8d 2606
3c038e6b
DD		 2607 /*
2608 * On s390 notifications for arriving pages will be delivered directly
2609 * to the guest but the house keeping for completed pfaults is
2610 * handled outside the worker.
2611 */
2612 kvm_check_async_pf_completion(vcpu);
2613
7ec7c8c7
CB		 2614 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2615 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
b0c632db
HC		 2616
2617 if (need_resched())
2618 schedule();
2619
d3a73acb 2620 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 2621 s390_handle_mcck();
2622
79395031
JF		 2623 if (!kvm_is_ucontrol(vcpu->kvm)) {
2624 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2625 if (rc)
2626 return rc;
2627 }
0ff31867 2628
2c70fe44
CB		 2629 rc = kvm_s390_handle_requests(vcpu);
2630 if (rc)
2631 return rc;
2632
27291e21
DH		 2633 if (guestdbg_enabled(vcpu)) {
2634 kvm_s390_backup_guest_per_regs(vcpu);
2635 kvm_s390_patch_guest_per_regs(vcpu);
2636 }
2637
b0c632db 2638 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 2639 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2640 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2641 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 2642
3fb4c40f
TH		 2643 return 0;
2644}
2645
492d8642
TH		 2646static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2647{
56317920
DH		 2648 struct kvm_s390_pgm_info pgm_info = {
2649 .code = PGM_ADDRESSING,
2650 };
2651 u8 opcode, ilen;
492d8642
TH		 2652 int rc;
2653
2654 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2655 trace_kvm_s390_sie_fault(vcpu);
2656
2657 /*
2658 * We want to inject an addressing exception, which is defined as a
2659 * suppressing or terminating exception. However, since we came here
2660 * by a DAT access exception, the PSW still points to the faulting
2661 * instruction since DAT exceptions are nullifying. So we've got
2662 * to look up the current opcode to get the length of the instruction
2663 * to be able to forward the PSW.
2664 */
3fa8cad7 2665 rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
56317920 2666 ilen = insn_length(opcode);
9b0d721a
DH		 2667 if (rc < 0) {
2668 return rc;
2669 } else if (rc) {
2670 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2671 * Forward by arbitrary ilc, injection will take care of
2672 * nullification if necessary.
2673 */
2674 pgm_info = vcpu->arch.pgm;
2675 ilen = 4;
2676 }
56317920
DH		 2677 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2678 kvm_s390_forward_psw(vcpu, ilen);
2679 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
492d8642
TH		 2680}
2681
3fb4c40f
TH		 2682static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2683{
2b29a9fd
DD		 2684 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2685 vcpu->arch.sie_block->icptcode);
2686 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2687
27291e21
DH		 2688 if (guestdbg_enabled(vcpu))
2689 kvm_s390_restore_guest_per_regs(vcpu);
2690
7ec7c8c7
CB		 2691 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2692 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
71f116bf
DH		 2693
2694 if (vcpu->arch.sie_block->icptcode > 0) {
2695 int rc = kvm_handle_sie_intercept(vcpu);
2696
2697 if (rc != -EOPNOTSUPP)
2698 return rc;
2699 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2700 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2701 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2702 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2703 return -EREMOTE;
2704 } else if (exit_reason != -EFAULT) {
2705 vcpu->stat.exit_null++;
2706 return 0;
210b1607
TH		 2707 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2708 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2709 vcpu->run->s390_ucontrol.trans_exc_code =
2710 current->thread.gmap_addr;
2711 vcpu->run->s390_ucontrol.pgm_code = 0x10;
71f116bf 2712 return -EREMOTE;
24eb3a82 2713 } else if (current->thread.gmap_pfault) {
3c038e6b 2714 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 2715 current->thread.gmap_pfault = 0;
71f116bf
DH		 2716 if (kvm_arch_setup_async_pf(vcpu))
2717 return 0;
2718 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
a76ccff6 2719 }
71f116bf 2720 return vcpu_post_run_fault_in_sie(vcpu);
3fb4c40f
TH		 2721}
2722
2723static int __vcpu_run(struct kvm_vcpu *vcpu)
2724{
2725 int rc, exit_reason;
2726
800c1065
TH		 2727 /*
2728 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2729 * ning the guest), so that memslots (and other stuff) are protected
2730 */
2731 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2732
a76ccff6
TH		 2733 do {
2734 rc = vcpu_pre_run(vcpu);
2735 if (rc)
2736 break;
3fb4c40f 2737
800c1065 2738 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 2739 /*
2740 * As PF_VCPU will be used in fault handler, between
2741 * guest_enter and guest_exit should be no uaccess.
2742 */
0097d12e 2743 local_irq_disable();
6edaa530 2744 guest_enter_irqoff();
db0758b2 2745 __disable_cpu_timer_accounting(vcpu);
0097d12e 2746 local_irq_enable();
a76ccff6
TH		 2747 exit_reason = sie64a(vcpu->arch.sie_block,
2748 vcpu->run->s.regs.gprs);
0097d12e 2749 local_irq_disable();
db0758b2 2750 __enable_cpu_timer_accounting(vcpu);
6edaa530 2751 guest_exit_irqoff();
0097d12e 2752 local_irq_enable();
800c1065 2753 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 2754
2755 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 2756 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 2757
800c1065 2758 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 2759 return rc;
b0c632db
HC		 2760}
2761
b028ee3e
DH		 2762static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2763{
4d5f2c04 2764 struct runtime_instr_cb *riccb;
4e0b1ab7 2765 struct gs_cb *gscb;
4d5f2c04
CB		 2766
2767 riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
4e0b1ab7 2768 gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
b028ee3e
DH		 2769 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2770 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2771 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2772 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2773 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2774 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 2775 /* some control register changes require a tlb flush */
2776 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 2777 }
2778 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4287f247 2779 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
b028ee3e
DH		 2780 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2781 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2782 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2783 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2784 }
2785 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2786 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2787 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2788 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 2789 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2790 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e 2791 }
80cd8763
FZ		 2792 /*
2793 * If userspace sets the riccb (e.g. after migration) to a valid state,
2794 * we should enable RI here instead of doing the lazy enablement.
2795 */
2796 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
4d5f2c04
CB		 2797 test_kvm_facility(vcpu->kvm, 64) &&
2798 riccb->valid &&
0c9d8683 2799 !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
4d5f2c04 2800 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
0c9d8683 2801 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
80cd8763 2802 }
4e0b1ab7
FZ		 2803 /*
2804 * If userspace sets the gscb (e.g. after migration) to non-zero,
2805 * we should enable GS here instead of doing the lazy enablement.
2806 */
2807 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
2808 test_kvm_facility(vcpu->kvm, 133) &&
2809 gscb->gssm &&
2810 !vcpu->arch.gs_enabled) {
2811 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
2812 vcpu->arch.sie_block->ecb |= ECB_GS;
2813 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
2814 vcpu->arch.gs_enabled = 1;
80cd8763 2815 }
31d8b8d4
CB		 2816 save_access_regs(vcpu->arch.host_acrs);
2817 restore_access_regs(vcpu->run->s.regs.acrs);
e1788bb9
CB		 2818 /* save host (userspace) fprs/vrs */
2819 save_fpu_regs();
2820 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
2821 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
2822 if (MACHINE_HAS_VX)
2823 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
2824 else
2825 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
2826 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
2827 if (test_fp_ctl(current->thread.fpu.fpc))
2828 /* User space provided an invalid FPC, let's clear it */
2829 current->thread.fpu.fpc = 0;
4e0b1ab7
FZ		 2830 if (MACHINE_HAS_GS) {
2831 preempt_disable();
2832 __ctl_set_bit(2, 4);
2833 if (current->thread.gs_cb) {
2834 vcpu->arch.host_gscb = current->thread.gs_cb;
2835 save_gs_cb(vcpu->arch.host_gscb);
2836 }
2837 if (vcpu->arch.gs_enabled) {
2838 current->thread.gs_cb = (struct gs_cb *)
2839 &vcpu->run->s.regs.gscb;
2840 restore_gs_cb(current->thread.gs_cb);
2841 }
2842 preempt_enable();
2843 }
80cd8763 2844
b028ee3e
DH		 2845 kvm_run->kvm_dirty_regs = 0;
2846}
2847
2848static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2849{
2850 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2851 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2852 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2853 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4287f247 2854 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
b028ee3e
DH		 2855 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2856 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2857 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2858 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2859 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2860 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2861 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
31d8b8d4
CB		 2862 save_access_regs(vcpu->run->s.regs.acrs);
2863 restore_access_regs(vcpu->arch.host_acrs);
e1788bb9
CB		 2864 /* Save guest register state */
2865 save_fpu_regs();
2866 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
2867 /* Restore will be done lazily at return */
2868 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
2869 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
4e0b1ab7
FZ		 2870 if (MACHINE_HAS_GS) {
2871 __ctl_set_bit(2, 4);
2872 if (vcpu->arch.gs_enabled)
2873 save_gs_cb(current->thread.gs_cb);
2874 preempt_disable();
2875 current->thread.gs_cb = vcpu->arch.host_gscb;
2876 restore_gs_cb(vcpu->arch.host_gscb);
2877 preempt_enable();
2878 if (!vcpu->arch.host_gscb)
2879 __ctl_clear_bit(2, 4);
2880 vcpu->arch.host_gscb = NULL;
2881 }
e1788bb9 2882
b028ee3e
DH		 2883}
2884
b0c632db
HC		 2885int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2886{
8f2abe6a 2887 int rc;
b0c632db
HC		 2888 sigset_t sigsaved;
2889
460df4c1
PB		 2890 if (kvm_run->immediate_exit)
2891 return -EINTR;
2892
27291e21
DH		 2893 if (guestdbg_exit_pending(vcpu)) {
2894 kvm_s390_prepare_debug_exit(vcpu);
2895 return 0;
2896 }
2897
b0c632db
HC		 2898 if (vcpu->sigset_active)
2899 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2900
6352e4d2
DH		 2901 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2902 kvm_s390_vcpu_start(vcpu);
2903 } else if (is_vcpu_stopped(vcpu)) {
ea2cdd27 2904 pr_err_ratelimited("can't run stopped vcpu %d\n",
6352e4d2
DH		 2905 vcpu->vcpu_id);
2906 return -EINVAL;
2907 }
b0c632db 2908
b028ee3e 2909 sync_regs(vcpu, kvm_run);
db0758b2 2910 enable_cpu_timer_accounting(vcpu);
d7b0b5eb 2911
dab4079d 2912 might_fault();
a76ccff6 2913 rc = __vcpu_run(vcpu);
9ace903d 2914
b1d16c49
CE		 2915 if (signal_pending(current) && !rc) {
2916 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 2917 rc = -EINTR;
b1d16c49 2918 }
8f2abe6a 2919
27291e21
DH		 2920 if (guestdbg_exit_pending(vcpu) && !rc) {
2921 kvm_s390_prepare_debug_exit(vcpu);
2922 rc = 0;
2923 }
2924
8f2abe6a 2925 if (rc == -EREMOTE) {
71f116bf 2926 /* userspace support is needed, kvm_run has been prepared */
8f2abe6a
CB		 2927 rc = 0;
2928 }
b0c632db 2929
db0758b2 2930 disable_cpu_timer_accounting(vcpu);
b028ee3e 2931 store_regs(vcpu, kvm_run);
d7b0b5eb 2932
b0c632db
HC		 2933 if (vcpu->sigset_active)
2934 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2935
b0c632db 2936 vcpu->stat.exit_userspace++;
7e8e6ab4 2937 return rc;
b0c632db
HC		 2938}
2939
b0c632db
HC		 2940/*
2941 * store status at address
2942 * we use have two special cases:
2943 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2944 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2945 */
d0bce605 2946int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 2947{
092670cd 2948 unsigned char archmode = 1;
9abc2a08 2949 freg_t fprs[NUM_FPRS];
fda902cb 2950 unsigned int px;
4287f247 2951 u64 clkcomp, cputm;
d0bce605 2952 int rc;
b0c632db 2953
d9a3a09a 2954 px = kvm_s390_get_prefix(vcpu);
d0bce605
HC		 2955 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2956 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 2957 return -EFAULT;
d9a3a09a 2958 gpa = 0;
d0bce605
HC		 2959 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2960 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 2961 return -EFAULT;
d9a3a09a
MS		 2962 gpa = px;
2963 } else
2964 gpa -= __LC_FPREGS_SAVE_AREA;
9abc2a08
DH		 2965
2966 /* manually convert vector registers if necessary */
2967 if (MACHINE_HAS_VX) {
9522b37f 2968 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08
DH		 2969 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2970 fprs, 128);
2971 } else {
2972 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
6fd8e67d 2973 vcpu->run->s.regs.fprs, 128);
9abc2a08 2974 }
d9a3a09a 2975 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
d0bce605 2976 vcpu->run->s.regs.gprs, 128);
d9a3a09a 2977 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
d0bce605 2978 &vcpu->arch.sie_block->gpsw, 16);
d9a3a09a 2979 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
fda902cb 2980 &px, 4);
d9a3a09a 2981 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
9abc2a08 2982 &vcpu->run->s.regs.fpc, 4);
d9a3a09a 2983 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
d0bce605 2984 &vcpu->arch.sie_block->todpr, 4);
4287f247 2985 cputm = kvm_s390_get_cpu_timer(vcpu);
d9a3a09a 2986 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4287f247 2987 &cputm, 8);
178bd789 2988 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d9a3a09a 2989 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
d0bce605 2990 &clkcomp, 8);
d9a3a09a 2991 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
d0bce605 2992 &vcpu->run->s.regs.acrs, 64);
d9a3a09a 2993 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
d0bce605
HC		 2994 &vcpu->arch.sie_block->gcr, 128);
2995 return rc ? -EFAULT : 0;
b0c632db
HC		 2996}
2997
e879892c
TH		 2998int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2999{
3000 /*
3001 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
31d8b8d4 3002 * switch in the run ioctl. Let's update our copies before we save
e879892c
TH		 3003 * it into the save area
3004 */
d0164ee2 3005 save_fpu_regs();
9abc2a08 3006 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
e879892c
TH		 3007 save_access_regs(vcpu->run->s.regs.acrs);
3008
3009 return kvm_s390_store_status_unloaded(vcpu, addr);
3010}
3011
8ad35755
DH		 3012static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3013{
3014 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
8e236546 3015 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
8ad35755
DH		 3016}
3017
3018static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
3019{
3020 unsigned int i;
3021 struct kvm_vcpu *vcpu;
3022
3023 kvm_for_each_vcpu(i, vcpu, kvm) {
3024 __disable_ibs_on_vcpu(vcpu);
3025 }
3026}
3027
3028static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3029{
09a400e7
DH		 3030 if (!sclp.has_ibs)
3031 return;
8ad35755 3032 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
8e236546 3033 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
8ad35755
DH		 3034}
3035
6852d7b6
DH		 3036void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
3037{
8ad35755
DH		 3038 int i, online_vcpus, started_vcpus = 0;
3039
3040 if (!is_vcpu_stopped(vcpu))
3041 return;
3042
6852d7b6 3043 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 3044 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 3045 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 3046 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3047
3048 for (i = 0; i < online_vcpus; i++) {
3049 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
3050 started_vcpus++;
3051 }
3052
3053 if (started_vcpus == 0) {
3054 /* we're the only active VCPU -> speed it up */
3055 __enable_ibs_on_vcpu(vcpu);
3056 } else if (started_vcpus == 1) {
3057 /*
3058 * As we are starting a second VCPU, we have to disable
3059 * the IBS facility on all VCPUs to remove potentially
3060 * oustanding ENABLE requests.
3061 */
3062 __disable_ibs_on_all_vcpus(vcpu->kvm);
3063 }
3064
805de8f4 3065 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 3066 /*
3067 * Another VCPU might have used IBS while we were offline.
3068 * Let's play safe and flush the VCPU at startup.
3069 */
d3d692c8 3070 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 3071 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 3072 return;
6852d7b6
DH		 3073}
3074
3075void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
3076{
8ad35755
DH		 3077 int i, online_vcpus, started_vcpus = 0;
3078 struct kvm_vcpu *started_vcpu = NULL;
3079
3080 if (is_vcpu_stopped(vcpu))
3081 return;
3082
6852d7b6 3083 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 3084 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 3085 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 3086 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3087
32f5ff63 3088 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 3089 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 3090
805de8f4 3091 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 3092 __disable_ibs_on_vcpu(vcpu);
3093
3094 for (i = 0; i < online_vcpus; i++) {
3095 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
3096 started_vcpus++;
3097 started_vcpu = vcpu->kvm->vcpus[i];
3098 }
3099 }
3100
3101 if (started_vcpus == 1) {
3102 /*
3103 * As we only have one VCPU left, we want to enable the
3104 * IBS facility for that VCPU to speed it up.
3105 */
3106 __enable_ibs_on_vcpu(started_vcpu);
3107 }
3108
433b9ee4 3109 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 3110 return;
6852d7b6
DH		 3111}
3112
d6712df9
CH		 3113static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
3114 struct kvm_enable_cap *cap)
3115{
3116 int r;
3117
3118 if (cap->flags)
3119 return -EINVAL;
3120
3121 switch (cap->cap) {
fa6b7fe9
CH		 3122 case KVM_CAP_S390_CSS_SUPPORT:
3123 if (!vcpu->kvm->arch.css_support) {
3124 vcpu->kvm->arch.css_support = 1;
c92ea7b9 3125 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
fa6b7fe9
CH		 3126 trace_kvm_s390_enable_css(vcpu->kvm);
3127 }
3128 r = 0;
3129 break;
d6712df9
CH		 3130 default:
3131 r = -EINVAL;
3132 break;
3133 }
3134 return r;
3135}
3136
41408c28
TH		 3137static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
3138 struct kvm_s390_mem_op *mop)
3139{
3140 void __user *uaddr = (void __user *)mop->buf;
3141 void *tmpbuf = NULL;
3142 int r, srcu_idx;
3143 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
3144 | KVM_S390_MEMOP_F_CHECK_ONLY;
3145
3146 if (mop->flags & ~supported_flags)
3147 return -EINVAL;
3148
3149 if (mop->size > MEM_OP_MAX_SIZE)
3150 return -E2BIG;
3151
3152 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
3153 tmpbuf = vmalloc(mop->size);
3154 if (!tmpbuf)
3155 return -ENOMEM;
3156 }
3157
3158 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3159
3160 switch (mop->op) {
3161 case KVM_S390_MEMOP_LOGICAL_READ:
3162 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 3163 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3164 mop->size, GACC_FETCH);
41408c28
TH		 3165 break;
3166 }
3167 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3168 if (r == 0) {
3169 if (copy_to_user(uaddr, tmpbuf, mop->size))
3170 r = -EFAULT;
3171 }
3172 break;
3173 case KVM_S390_MEMOP_LOGICAL_WRITE:
3174 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 3175 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3176 mop->size, GACC_STORE);
41408c28
TH		 3177 break;
3178 }
3179 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
3180 r = -EFAULT;
3181 break;
3182 }
3183 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3184 break;
3185 default:
3186 r = -EINVAL;
3187 }
3188
3189 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
3190
3191 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
3192 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
3193
3194 vfree(tmpbuf);
3195 return r;
3196}
3197
b0c632db
HC		 3198long kvm_arch_vcpu_ioctl(struct file *filp,
3199 unsigned int ioctl, unsigned long arg)
3200{
3201 struct kvm_vcpu *vcpu = filp->private_data;
3202 void __user *argp = (void __user *)arg;
800c1065 3203 int idx;
bc923cc9 3204 long r;
b0c632db 3205
93736624 3206 switch (ioctl) {
47b43c52
JF		 3207 case KVM_S390_IRQ: {
3208 struct kvm_s390_irq s390irq;
3209
3210 r = -EFAULT;
3211 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
3212 break;
3213 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3214 break;
3215 }
93736624 3216 case KVM_S390_INTERRUPT: {
ba5c1e9b 3217 struct kvm_s390_interrupt s390int;
383d0b05 3218 struct kvm_s390_irq s390irq;
ba5c1e9b 3219
93736624 3220 r = -EFAULT;
ba5c1e9b 3221 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624 3222 break;
383d0b05
JF		 3223 if (s390int_to_s390irq(&s390int, &s390irq))
3224 return -EINVAL;
3225 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
93736624 3226 break;
ba5c1e9b 3227 }
b0c632db 3228 case KVM_S390_STORE_STATUS:
800c1065 3229 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 3230 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 3231 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 3232 break;
b0c632db
HC		 3233 case KVM_S390_SET_INITIAL_PSW: {
3234 psw_t psw;
3235
bc923cc9 3236 r = -EFAULT;
b0c632db 3237 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 3238 break;
3239 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3240 break;
b0c632db
HC		 3241 }
3242 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 3243 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3244 break;
14eebd91
CO		 3245 case KVM_SET_ONE_REG:
3246 case KVM_GET_ONE_REG: {
3247 struct kvm_one_reg reg;
3248 r = -EFAULT;
3249 if (copy_from_user(&reg, argp, sizeof(reg)))
3250 break;
3251 if (ioctl == KVM_SET_ONE_REG)
3252 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3253 else
3254 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3255 break;
3256 }
27e0393f
CO		 3257#ifdef CONFIG_KVM_S390_UCONTROL
3258 case KVM_S390_UCAS_MAP: {
3259 struct kvm_s390_ucas_mapping ucasmap;
3260
3261 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3262 r = -EFAULT;
3263 break;
3264 }
3265
3266 if (!kvm_is_ucontrol(vcpu->kvm)) {
3267 r = -EINVAL;
3268 break;
3269 }
3270
3271 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3272 ucasmap.vcpu_addr, ucasmap.length);
3273 break;
3274 }
3275 case KVM_S390_UCAS_UNMAP: {
3276 struct kvm_s390_ucas_mapping ucasmap;
3277
3278 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3279 r = -EFAULT;
3280 break;
3281 }
3282
3283 if (!kvm_is_ucontrol(vcpu->kvm)) {
3284 r = -EINVAL;
3285 break;
3286 }
3287
3288 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3289 ucasmap.length);
3290 break;
3291 }
3292#endif
ccc7910f 3293 case KVM_S390_VCPU_FAULT: {
527e30b4 3294 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 3295 break;
3296 }
d6712df9
CH		 3297 case KVM_ENABLE_CAP:
3298 {
3299 struct kvm_enable_cap cap;
3300 r = -EFAULT;
3301 if (copy_from_user(&cap, argp, sizeof(cap)))
3302 break;
3303 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3304 break;
3305 }
41408c28
TH		 3306 case KVM_S390_MEM_OP: {
3307 struct kvm_s390_mem_op mem_op;
3308
3309 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3310 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3311 else
3312 r = -EFAULT;
3313 break;
3314 }
816c7667
JF		 3315 case KVM_S390_SET_IRQ_STATE: {
3316 struct kvm_s390_irq_state irq_state;
3317
3318 r = -EFAULT;
3319 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3320 break;
3321 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3322 irq_state.len == 0 ||
3323 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3324 r = -EINVAL;
3325 break;
3326 }
3327 r = kvm_s390_set_irq_state(vcpu,
3328 (void __user *) irq_state.buf,
3329 irq_state.len);
3330 break;
3331 }
3332 case KVM_S390_GET_IRQ_STATE: {
3333 struct kvm_s390_irq_state irq_state;
3334
3335 r = -EFAULT;
3336 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3337 break;
3338 if (irq_state.len == 0) {
3339 r = -EINVAL;
3340 break;
3341 }
3342 r = kvm_s390_get_irq_state(vcpu,
3343 (__u8 __user *) irq_state.buf,
3344 irq_state.len);
3345 break;
3346 }
b0c632db 3347 default:
3e6afcf1 3348 r = -ENOTTY;
b0c632db 3349 }
bc923cc9 3350 return r;
b0c632db
HC		 3351}
3352
5b1c1493
CO		 3353int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3354{
3355#ifdef CONFIG_KVM_S390_UCONTROL
3356 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3357 && (kvm_is_ucontrol(vcpu->kvm))) {
3358 vmf->page = virt_to_page(vcpu->arch.sie_block);
3359 get_page(vmf->page);
3360 return 0;
3361 }
3362#endif
3363 return VM_FAULT_SIGBUS;
3364}
3365
5587027c
AK		 3366int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3367 unsigned long npages)
db3fe4eb
TY		 3368{
3369 return 0;
3370}
3371
b0c632db 3372/* Section: memory related */
f7784b8e
MT		 3373int kvm_arch_prepare_memory_region(struct kvm *kvm,
3374 struct kvm_memory_slot *memslot,
09170a49 3375 const struct kvm_userspace_memory_region *mem,
7b6195a9 3376 enum kvm_mr_change change)
b0c632db 3377{
dd2887e7
NW		 3378 /* A few sanity checks. We can have memory slots which have to be
3379 located/ended at a segment boundary (1MB). The memory in userland is
3380 ok to be fragmented into various different vmas. It is okay to mmap()
3381 and munmap() stuff in this slot after doing this call at any time */
b0c632db 3382
598841ca 3383 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 3384 return -EINVAL;
3385
598841ca 3386 if (mem->memory_size & 0xffffful)
b0c632db
HC		 3387 return -EINVAL;
3388
a3a92c31
DD		 3389 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3390 return -EINVAL;
3391
f7784b8e
MT		 3392 return 0;
3393}
3394
3395void kvm_arch_commit_memory_region(struct kvm *kvm,
09170a49 3396 const struct kvm_userspace_memory_region *mem,
8482644a 3397 const struct kvm_memory_slot *old,
f36f3f28 3398 const struct kvm_memory_slot *new,
8482644a 3399 enum kvm_mr_change change)
f7784b8e 3400{
f7850c92 3401 int rc;
f7784b8e 3402
2cef4deb
CB		 3403 /* If the basics of the memslot do not change, we do not want
3404 * to update the gmap. Every update causes several unnecessary
3405 * segment translation exceptions. This is usually handled just
3406 * fine by the normal fault handler + gmap, but it will also
3407 * cause faults on the prefix page of running guest CPUs.
3408 */
3409 if (old->userspace_addr == mem->userspace_addr &&
3410 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3411 old->npages * PAGE_SIZE == mem->memory_size)
3412 return;
598841ca
CO		 3413
3414 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3415 mem->guest_phys_addr, mem->memory_size);
3416 if (rc)
ea2cdd27 3417 pr_warn("failed to commit memory region\n");
598841ca 3418 return;
b0c632db
HC		 3419}
3420
60a37709
AY		 3421static inline unsigned long nonhyp_mask(int i)
3422{
3423 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3424
3425 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3426}
3427
3491caf2
CB		 3428void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3429{
3430 vcpu->valid_wakeup = false;
3431}
3432
b0c632db
HC		 3433static int __init kvm_s390_init(void)
3434{
60a37709
AY		 3435 int i;
3436
07197fd0
DH		 3437 if (!sclp.has_sief2) {
3438 pr_info("SIE not available\n");
3439 return -ENODEV;
3440 }
3441
60a37709
AY		 3442 for (i = 0; i < 16; i++)
3443 kvm_s390_fac_list_mask[i] |=
3444 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3445
9d8d5786 3446 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 3447}
3448
3449static void __exit kvm_s390_exit(void)
3450{
3451 kvm_exit();
3452}
3453
3454module_init(kvm_s390_init);
3455module_exit(kvm_s390_exit);
566af940
CH		 3456
3457/*
3458 * Enable autoloading of the kvm module.
3459 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3460 * since x86 takes a different approach.
3461 */
3462#include <linux/miscdevice.h>
3463MODULE_ALIAS_MISCDEV(KVM_MINOR);
3464MODULE_ALIAS("devname:kvm");
Ubuntu Artful kernel mirror