]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
treewide: use kv[mz]alloc* rather than opencoded variants
[mirror_ubuntu-artful-kernel.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
b2d73b2a 24#include <linux/mman.h>
b0c632db 25#include <linux/module.h>
d3217967 26#include <linux/moduleparam.h>
a374e892 27#include <linux/random.h>
b0c632db 28#include <linux/slab.h>
ba5c1e9b 29#include <linux/timer.h>
41408c28 30#include <linux/vmalloc.h>
15c9705f 31#include <linux/bitmap.h>
174cd4b1
IM		 32#include <linux/sched/signal.h>
33
cbb870c8 34#include <asm/asm-offsets.h>
b0c632db 35#include <asm/lowcore.h>
fd5ada04 36#include <asm/stp.h>
b0c632db 37#include <asm/pgtable.h>
1e133ab2 38#include <asm/gmap.h>
f5daba1d 39#include <asm/nmi.h>
a0616cde 40#include <asm/switch_to.h>
6d3da241 41#include <asm/isc.h>
1526bf9c 42#include <asm/sclp.h>
0a763c78 43#include <asm/cpacf.h>
221bb8a4 44#include <asm/timex.h>
8f2abe6a 45#include "kvm-s390.h"
b0c632db
HC		 46#include "gaccess.h"
47
ea2cdd27
DH		 48#define KMSG_COMPONENT "kvm-s390"
49#undef pr_fmt
50#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
51
5786fffa
CH		 52#define CREATE_TRACE_POINTS
53#include "trace.h"
ade38c31 54#include "trace-s390.h"
5786fffa 55
41408c28 56#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
816c7667
JF		 57#define LOCAL_IRQS 32
58#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
59 (KVM_MAX_VCPUS + LOCAL_IRQS))
41408c28 60
b0c632db
HC		 61#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
62
63struct kvm_stats_debugfs_item debugfs_entries[] = {
64 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 65 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 66 { "exit_validity", VCPU_STAT(exit_validity) },
67 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
68 { "exit_external_request", VCPU_STAT(exit_external_request) },
69 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b 70 { "exit_instruction", VCPU_STAT(exit_instruction) },
9ec6de19 71 { "exit_pei", VCPU_STAT(exit_pei) },
ba5c1e9b
CO		 72 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
73 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
a011eeb2 74 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
f7819512 75 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
62bea5bf 76 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
3491caf2 77 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
ce2e4f0b 78 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 79 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 80 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 81 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
82 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 83 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 84 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 85 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
86 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
87 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
88 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
89 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
90 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
91 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 92 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 93 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
94 { "instruction_spx", VCPU_STAT(instruction_spx) },
95 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
96 { "instruction_stap", VCPU_STAT(instruction_stap) },
97 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 98 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 99 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
100 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 101 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 102 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
103 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 104 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
95ca2cb5 105 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
a3508fbe 106 { "instruction_sie", VCPU_STAT(instruction_sie) },
5288fbf0 107 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 108 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 109 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 110 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 111 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
112 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 113 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 114 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
115 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 116 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 117 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
118 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
119 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 120 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
121 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
122 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 123 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 124 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 125 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
175a5c9e
CB		 126 { "diagnose_258", VCPU_STAT(diagnose_258) },
127 { "diagnose_308", VCPU_STAT(diagnose_308) },
128 { "diagnose_500", VCPU_STAT(diagnose_500) },
b0c632db
HC		 129 { NULL }
130};
131
a411edf1
DH		 132/* allow nested virtualization in KVM (if enabled by user space) */
133static int nested;
134module_param(nested, int, S_IRUGO);
135MODULE_PARM_DESC(nested, "Nested virtualization support");
136
9d8d5786 137/* upper facilities limit for kvm */
f6c1d359 138unsigned long kvm_s390_fac_list_mask[16] = { FACILITIES_KVM };
b0c632db 139
9d8d5786 140unsigned long kvm_s390_fac_list_mask_size(void)
78c4b59f 141{
9d8d5786
MM		 142 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
143 return ARRAY_SIZE(kvm_s390_fac_list_mask);
78c4b59f
MM		 144}
145
15c9705f
DH		 146/* available cpu features supported by kvm */
147static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
0a763c78
DH		 148/* available subfunctions indicated via query / "test bit" */
149static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
15c9705f 150
9d8d5786 151static struct gmap_notifier gmap_notifier;
a3508fbe 152static struct gmap_notifier vsie_gmap_notifier;
78f26131 153debug_info_t *kvm_s390_dbf;
9d8d5786 154
b0c632db 155/* Section: not file related */
13a34e06 156int kvm_arch_hardware_enable(void)
b0c632db
HC		 157{
158 /* every s390 is virtualization enabled ;-) */
10474ae8 159 return 0;
b0c632db
HC		 160}
161
414d3b07
MS		 162static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
163 unsigned long end);
2c70fe44 164
fdf03650
FZ		 165/*
166 * This callback is executed during stop_machine(). All CPUs are therefore
167 * temporarily stopped. In order not to change guest behavior, we have to
168 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
169 * so a CPU won't be stopped while calculating with the epoch.
170 */
171static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
172 void *v)
173{
174 struct kvm *kvm;
175 struct kvm_vcpu *vcpu;
176 int i;
177 unsigned long long *delta = v;
178
179 list_for_each_entry(kvm, &vm_list, vm_list) {
180 kvm->arch.epoch -= *delta;
181 kvm_for_each_vcpu(i, vcpu, kvm) {
182 vcpu->arch.sie_block->epoch -= *delta;
db0758b2
DH		 183 if (vcpu->arch.cputm_enabled)
184 vcpu->arch.cputm_start += *delta;
91473b48
DH		 185 if (vcpu->arch.vsie_block)
186 vcpu->arch.vsie_block->epoch -= *delta;
fdf03650
FZ		 187 }
188 }
189 return NOTIFY_OK;
190}
191
192static struct notifier_block kvm_clock_notifier = {
193 .notifier_call = kvm_clock_sync,
194};
195
b0c632db
HC		 196int kvm_arch_hardware_setup(void)
197{
2c70fe44 198 gmap_notifier.notifier_call = kvm_gmap_notifier;
b2d73b2a 199 gmap_register_pte_notifier(&gmap_notifier);
a3508fbe
DH		 200 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
201 gmap_register_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 202 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
203 &kvm_clock_notifier);
b0c632db
HC		 204 return 0;
205}
206
207void kvm_arch_hardware_unsetup(void)
208{
b2d73b2a 209 gmap_unregister_pte_notifier(&gmap_notifier);
a3508fbe 210 gmap_unregister_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 211 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
212 &kvm_clock_notifier);
b0c632db
HC		 213}
214
22be5a13
DH		 215static void allow_cpu_feat(unsigned long nr)
216{
217 set_bit_inv(nr, kvm_s390_available_cpu_feat);
218}
219
0a763c78
DH		 220static inline int plo_test_bit(unsigned char nr)
221{
222 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
d051ae53 223 int cc;
0a763c78
DH		 224
225 asm volatile(
226 /* Parameter registers are ignored for "test bit" */
227 " plo 0,0,0,0(0)\n"
228 " ipm %0\n"
229 " srl %0,28\n"
230 : "=d" (cc)
231 : "d" (r0)
232 : "cc");
233 return cc == 0;
234}
235
22be5a13
DH		 236static void kvm_s390_cpu_feat_init(void)
237{
0a763c78
DH		 238 int i;
239
240 for (i = 0; i < 256; ++i) {
241 if (plo_test_bit(i))
242 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
243 }
244
245 if (test_facility(28)) /* TOD-clock steering */
221bb8a4
LT		 246 ptff(kvm_s390_available_subfunc.ptff,
247 sizeof(kvm_s390_available_subfunc.ptff),
248 PTFF_QAF);
0a763c78
DH		 249
250 if (test_facility(17)) { /* MSA */
69c0e360
MS		 251 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
252 kvm_s390_available_subfunc.kmac);
253 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
254 kvm_s390_available_subfunc.kmc);
255 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
256 kvm_s390_available_subfunc.km);
257 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
258 kvm_s390_available_subfunc.kimd);
259 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
260 kvm_s390_available_subfunc.klmd);
0a763c78
DH		 261 }
262 if (test_facility(76)) /* MSA3 */
69c0e360
MS		 263 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
264 kvm_s390_available_subfunc.pckmo);
0a763c78 265 if (test_facility(77)) { /* MSA4 */
69c0e360
MS		 266 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
267 kvm_s390_available_subfunc.kmctr);
268 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
269 kvm_s390_available_subfunc.kmf);
270 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
271 kvm_s390_available_subfunc.kmo);
272 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
273 kvm_s390_available_subfunc.pcc);
0a763c78
DH		 274 }
275 if (test_facility(57)) /* MSA5 */
985a9d20 276 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
69c0e360 277 kvm_s390_available_subfunc.ppno);
0a763c78 278
22be5a13
DH		 279 if (MACHINE_HAS_ESOP)
280 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
a3508fbe
DH		 281 /*
282 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
283 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
284 */
285 if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
a411edf1 286 !test_facility(3) || !nested)
a3508fbe
DH		 287 return;
288 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
19c439b5
DH		 289 if (sclp.has_64bscao)
290 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
0615a326
DH		 291 if (sclp.has_siif)
292 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
77d18f6d
DH		 293 if (sclp.has_gpere)
294 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
a1b7b9b2
DH		 295 if (sclp.has_gsls)
296 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
5630a8e8
DH		 297 if (sclp.has_ib)
298 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
13ee3f67
DH		 299 if (sclp.has_cei)
300 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
7fd7f39d
DH		 301 if (sclp.has_ibs)
302 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
5d3876a8
DH		 303 /*
304 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
305 * all skey handling functions read/set the skey from the PGSTE
306 * instead of the real storage key.
307 *
308 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
309 * pages being detected as preserved although they are resident.
310 *
311 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
312 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
313 *
314 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
315 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
316 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
317 *
318 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
319 * cannot easily shadow the SCA because of the ipte lock.
320 */
22be5a13
DH		 321}
322
b0c632db
HC		 323int kvm_arch_init(void *opaque)
324{
78f26131
CB		 325 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
326 if (!kvm_s390_dbf)
327 return -ENOMEM;
328
329 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
330 debug_unregister(kvm_s390_dbf);
331 return -ENOMEM;
332 }
333
22be5a13
DH		 334 kvm_s390_cpu_feat_init();
335
84877d93
CH		 336 /* Register floating interrupt controller interface. */
337 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 338}
339
78f26131
CB		 340void kvm_arch_exit(void)
341{
342 debug_unregister(kvm_s390_dbf);
343}
344
b0c632db
HC		 345/* Section: device related */
346long kvm_arch_dev_ioctl(struct file *filp,
347 unsigned int ioctl, unsigned long arg)
348{
349 if (ioctl == KVM_S390_ENABLE_SIE)
350 return s390_enable_sie();
351 return -EINVAL;
352}
353
784aa3d7 354int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 355{
d7b0b5eb
CO		 356 int r;
357
2bd0ac4e 358 switch (ext) {
d7b0b5eb 359 case KVM_CAP_S390_PSW:
b6cf8788 360 case KVM_CAP_S390_GMAP:
52e16b18 361 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 362#ifdef CONFIG_KVM_S390_UCONTROL
363 case KVM_CAP_S390_UCONTROL:
364#endif
3c038e6b 365 case KVM_CAP_ASYNC_PF:
60b413c9 366 case KVM_CAP_SYNC_REGS:
14eebd91 367 case KVM_CAP_ONE_REG:
d6712df9 368 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 369 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 370 case KVM_CAP_IOEVENTFD:
c05c4186 371 case KVM_CAP_DEVICE_CTRL:
d938dc55 372 case KVM_CAP_ENABLE_CAP_VM:
78599d90 373 case KVM_CAP_S390_IRQCHIP:
f2061656 374 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 375 case KVM_CAP_MP_STATE:
460df4c1 376 case KVM_CAP_IMMEDIATE_EXIT:
47b43c52 377 case KVM_CAP_S390_INJECT_IRQ:
2444b352 378 case KVM_CAP_S390_USER_SIGP:
e44fc8c9 379 case KVM_CAP_S390_USER_STSI:
30ee2a98 380 case KVM_CAP_S390_SKEYS:
816c7667 381 case KVM_CAP_S390_IRQ_STATE:
6502a34c 382 case KVM_CAP_S390_USER_INSTR0:
d7b0b5eb
CO		 383 r = 1;
384 break;
41408c28
TH		 385 case KVM_CAP_S390_MEM_OP:
386 r = MEM_OP_MAX_SIZE;
387 break;
e726b1bd
CB		 388 case KVM_CAP_NR_VCPUS:
389 case KVM_CAP_MAX_VCPUS:
76a6dd72 390 r = KVM_S390_BSCA_CPU_SLOTS;
a6940674
DH		 391 if (!kvm_s390_use_sca_entries())
392 r = KVM_MAX_VCPUS;
393 else if (sclp.has_esca && sclp.has_64bscao)
76a6dd72 394 r = KVM_S390_ESCA_CPU_SLOTS;
e726b1bd 395 break;
e1e2e605
NW		 396 case KVM_CAP_NR_MEMSLOTS:
397 r = KVM_USER_MEM_SLOTS;
398 break;
1526bf9c 399 case KVM_CAP_S390_COW:
abf09bed 400 r = MACHINE_HAS_ESOP;
1526bf9c 401 break;
68c55750
EF		 402 case KVM_CAP_S390_VECTOR_REGISTERS:
403 r = MACHINE_HAS_VX;
404 break;
c6e5f166
FZ		 405 case KVM_CAP_S390_RI:
406 r = test_facility(64);
407 break;
2bd0ac4e 408 default:
d7b0b5eb 409 r = 0;
2bd0ac4e 410 }
d7b0b5eb 411 return r;
b0c632db
HC		 412}
413
15f36ebd
JH		 414static void kvm_s390_sync_dirty_log(struct kvm *kvm,
415 struct kvm_memory_slot *memslot)
416{
417 gfn_t cur_gfn, last_gfn;
418 unsigned long address;
419 struct gmap *gmap = kvm->arch.gmap;
420
15f36ebd
JH		 421 /* Loop over all guest pages */
422 last_gfn = memslot->base_gfn + memslot->npages;
423 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
424 address = gfn_to_hva_memslot(memslot, cur_gfn);
425
1e133ab2 426 if (test_and_clear_guest_dirty(gmap->mm, address))
15f36ebd 427 mark_page_dirty(kvm, cur_gfn);
1763f8d0
CB		 428 if (fatal_signal_pending(current))
429 return;
70c88a00 430 cond_resched();
15f36ebd 431 }
15f36ebd
JH		 432}
433
b0c632db 434/* Section: vm related */
a6e2f683
ED		 435static void sca_del_vcpu(struct kvm_vcpu *vcpu);
436
b0c632db
HC		 437/*
438 * Get (and clear) the dirty memory log for a memory slot.
439 */
440int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
441 struct kvm_dirty_log *log)
442{
15f36ebd
JH		 443 int r;
444 unsigned long n;
9f6b8029 445 struct kvm_memslots *slots;
15f36ebd
JH		 446 struct kvm_memory_slot *memslot;
447 int is_dirty = 0;
448
e1e8a962
JF		 449 if (kvm_is_ucontrol(kvm))
450 return -EINVAL;
451
15f36ebd
JH		 452 mutex_lock(&kvm->slots_lock);
453
454 r = -EINVAL;
455 if (log->slot >= KVM_USER_MEM_SLOTS)
456 goto out;
457
9f6b8029
PB		 458 slots = kvm_memslots(kvm);
459 memslot = id_to_memslot(slots, log->slot);
15f36ebd
JH		 460 r = -ENOENT;
461 if (!memslot->dirty_bitmap)
462 goto out;
463
464 kvm_s390_sync_dirty_log(kvm, memslot);
465 r = kvm_get_dirty_log(kvm, log, &is_dirty);
466 if (r)
467 goto out;
468
469 /* Clear the dirty log */
470 if (is_dirty) {
471 n = kvm_dirty_bitmap_bytes(memslot);
472 memset(memslot->dirty_bitmap, 0, n);
473 }
474 r = 0;
475out:
476 mutex_unlock(&kvm->slots_lock);
477 return r;
b0c632db
HC		 478}
479
6502a34c
DH		 480static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
481{
482 unsigned int i;
483 struct kvm_vcpu *vcpu;
484
485 kvm_for_each_vcpu(i, vcpu, kvm) {
486 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
487 }
488}
489
d938dc55
CH		 490static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
491{
492 int r;
493
494 if (cap->flags)
495 return -EINVAL;
496
497 switch (cap->cap) {
84223598 498 case KVM_CAP_S390_IRQCHIP:
c92ea7b9 499 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
84223598
CH		 500 kvm->arch.use_irqchip = 1;
501 r = 0;
502 break;
2444b352 503 case KVM_CAP_S390_USER_SIGP:
c92ea7b9 504 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
2444b352
DH		 505 kvm->arch.user_sigp = 1;
506 r = 0;
507 break;
68c55750 508 case KVM_CAP_S390_VECTOR_REGISTERS:
5967c17b 509 mutex_lock(&kvm->lock);
a03825bb 510 if (kvm->created_vcpus) {
5967c17b
DH		 511 r = -EBUSY;
512 } else if (MACHINE_HAS_VX) {
c54f0d6a
DH		 513 set_kvm_facility(kvm->arch.model.fac_mask, 129);
514 set_kvm_facility(kvm->arch.model.fac_list, 129);
2f87d942
GH		 515 if (test_facility(134)) {
516 set_kvm_facility(kvm->arch.model.fac_mask, 134);
517 set_kvm_facility(kvm->arch.model.fac_list, 134);
518 }
53743aa7
MS		 519 if (test_facility(135)) {
520 set_kvm_facility(kvm->arch.model.fac_mask, 135);
521 set_kvm_facility(kvm->arch.model.fac_list, 135);
522 }
18280d8b
MM		 523 r = 0;
524 } else
525 r = -EINVAL;
5967c17b 526 mutex_unlock(&kvm->lock);
c92ea7b9
CB		 527 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
528 r ? "(not available)" : "(success)");
68c55750 529 break;
c6e5f166
FZ		 530 case KVM_CAP_S390_RI:
531 r = -EINVAL;
532 mutex_lock(&kvm->lock);
a03825bb 533 if (kvm->created_vcpus) {
c6e5f166
FZ		 534 r = -EBUSY;
535 } else if (test_facility(64)) {
c54f0d6a
DH		 536 set_kvm_facility(kvm->arch.model.fac_mask, 64);
537 set_kvm_facility(kvm->arch.model.fac_list, 64);
c6e5f166
FZ		 538 r = 0;
539 }
540 mutex_unlock(&kvm->lock);
541 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
542 r ? "(not available)" : "(success)");
543 break;
e44fc8c9 544 case KVM_CAP_S390_USER_STSI:
c92ea7b9 545 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
e44fc8c9
ET		 546 kvm->arch.user_stsi = 1;
547 r = 0;
548 break;
6502a34c
DH		 549 case KVM_CAP_S390_USER_INSTR0:
550 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
551 kvm->arch.user_instr0 = 1;
552 icpt_operexc_on_all_vcpus(kvm);
553 r = 0;
554 break;
d938dc55
CH		 555 default:
556 r = -EINVAL;
557 break;
558 }
559 return r;
560}
561
8c0a7ce6
DD		 562static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
563{
564 int ret;
565
566 switch (attr->attr) {
567 case KVM_S390_VM_MEM_LIMIT_SIZE:
568 ret = 0;
c92ea7b9 569 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
a3a92c31
DD		 570 kvm->arch.mem_limit);
571 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
8c0a7ce6
DD		 572 ret = -EFAULT;
573 break;
574 default:
575 ret = -ENXIO;
576 break;
577 }
578 return ret;
579}
580
581static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 582{
583 int ret;
584 unsigned int idx;
585 switch (attr->attr) {
586 case KVM_S390_VM_MEM_ENABLE_CMMA:
f9cbd9b0 587 ret = -ENXIO;
c24cc9c8 588 if (!sclp.has_cmma)
e6db1d61
DD		 589 break;
590
4f718eab 591 ret = -EBUSY;
c92ea7b9 592 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
4f718eab 593 mutex_lock(&kvm->lock);
a03825bb 594 if (!kvm->created_vcpus) {
4f718eab
DD		 595 kvm->arch.use_cmma = 1;
596 ret = 0;
597 }
598 mutex_unlock(&kvm->lock);
599 break;
600 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 601 ret = -ENXIO;
602 if (!sclp.has_cmma)
603 break;
c3489155
DD		 604 ret = -EINVAL;
605 if (!kvm->arch.use_cmma)
606 break;
607
c92ea7b9 608 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
4f718eab
DD		 609 mutex_lock(&kvm->lock);
610 idx = srcu_read_lock(&kvm->srcu);
a13cff31 611 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 612 srcu_read_unlock(&kvm->srcu, idx);
613 mutex_unlock(&kvm->lock);
614 ret = 0;
615 break;
8c0a7ce6
DD		 616 case KVM_S390_VM_MEM_LIMIT_SIZE: {
617 unsigned long new_limit;
618
619 if (kvm_is_ucontrol(kvm))
620 return -EINVAL;
621
622 if (get_user(new_limit, (u64 __user *)attr->addr))
623 return -EFAULT;
624
a3a92c31
DD		 625 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
626 new_limit > kvm->arch.mem_limit)
8c0a7ce6
DD		 627 return -E2BIG;
628
a3a92c31
DD		 629 if (!new_limit)
630 return -EINVAL;
631
6ea427bb 632 /* gmap_create takes last usable address */
a3a92c31
DD		 633 if (new_limit != KVM_S390_NO_MEM_LIMIT)
634 new_limit -= 1;
635
8c0a7ce6
DD		 636 ret = -EBUSY;
637 mutex_lock(&kvm->lock);
a03825bb 638 if (!kvm->created_vcpus) {
6ea427bb
MS		 639 /* gmap_create will round the limit up */
640 struct gmap *new = gmap_create(current->mm, new_limit);
8c0a7ce6
DD		 641
642 if (!new) {
643 ret = -ENOMEM;
644 } else {
6ea427bb 645 gmap_remove(kvm->arch.gmap);
8c0a7ce6
DD		 646 new->private = kvm;
647 kvm->arch.gmap = new;
648 ret = 0;
649 }
650 }
651 mutex_unlock(&kvm->lock);
a3a92c31
DD		 652 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
653 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
654 (void *) kvm->arch.gmap->asce);
8c0a7ce6
DD		 655 break;
656 }
4f718eab
DD		 657 default:
658 ret = -ENXIO;
659 break;
660 }
661 return ret;
662}
663
a374e892
TK		 664static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
665
666static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
667{
668 struct kvm_vcpu *vcpu;
669 int i;
670
9d8d5786 671 if (!test_kvm_facility(kvm, 76))
a374e892
TK		 672 return -EINVAL;
673
674 mutex_lock(&kvm->lock);
675 switch (attr->attr) {
676 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
677 get_random_bytes(
678 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
679 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
680 kvm->arch.crypto.aes_kw = 1;
c92ea7b9 681 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
a374e892
TK		 682 break;
683 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
684 get_random_bytes(
685 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
686 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
687 kvm->arch.crypto.dea_kw = 1;
c92ea7b9 688 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
a374e892
TK		 689 break;
690 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
691 kvm->arch.crypto.aes_kw = 0;
692 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
693 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
c92ea7b9 694 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
a374e892
TK		 695 break;
696 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
697 kvm->arch.crypto.dea_kw = 0;
698 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
699 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
c92ea7b9 700 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
a374e892
TK		 701 break;
702 default:
703 mutex_unlock(&kvm->lock);
704 return -ENXIO;
705 }
706
707 kvm_for_each_vcpu(i, vcpu, kvm) {
708 kvm_s390_vcpu_crypto_setup(vcpu);
709 exit_sie(vcpu);
710 }
711 mutex_unlock(&kvm->lock);
712 return 0;
713}
714
72f25020
JH		 715static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
716{
717 u8 gtod_high;
718
719 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
720 sizeof(gtod_high)))
721 return -EFAULT;
722
723 if (gtod_high != 0)
724 return -EINVAL;
58c383c6 725 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
72f25020
JH		 726
727 return 0;
728}
729
730static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
731{
5a3d883a 732 u64 gtod;
72f25020
JH		 733
734 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
735 return -EFAULT;
736
25ed1675 737 kvm_s390_set_tod_clock(kvm, gtod);
58c383c6 738 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
72f25020
JH		 739 return 0;
740}
741
742static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
743{
744 int ret;
745
746 if (attr->flags)
747 return -EINVAL;
748
749 switch (attr->attr) {
750 case KVM_S390_VM_TOD_HIGH:
751 ret = kvm_s390_set_tod_high(kvm, attr);
752 break;
753 case KVM_S390_VM_TOD_LOW:
754 ret = kvm_s390_set_tod_low(kvm, attr);
755 break;
756 default:
757 ret = -ENXIO;
758 break;
759 }
760 return ret;
761}
762
763static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
764{
765 u8 gtod_high = 0;
766
767 if (copy_to_user((void __user *)attr->addr, &gtod_high,
768 sizeof(gtod_high)))
769 return -EFAULT;
58c383c6 770 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
72f25020
JH		 771
772 return 0;
773}
774
775static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
776{
5a3d883a 777 u64 gtod;
72f25020 778
60417fcc 779 gtod = kvm_s390_get_tod_clock_fast(kvm);
72f25020
JH		 780 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
781 return -EFAULT;
58c383c6 782 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
72f25020
JH		 783
784 return 0;
785}
786
787static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
788{
789 int ret;
790
791 if (attr->flags)
792 return -EINVAL;
793
794 switch (attr->attr) {
795 case KVM_S390_VM_TOD_HIGH:
796 ret = kvm_s390_get_tod_high(kvm, attr);
797 break;
798 case KVM_S390_VM_TOD_LOW:
799 ret = kvm_s390_get_tod_low(kvm, attr);
800 break;
801 default:
802 ret = -ENXIO;
803 break;
804 }
805 return ret;
806}
807
658b6eda
MM		 808static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
809{
810 struct kvm_s390_vm_cpu_processor *proc;
053dd230 811 u16 lowest_ibc, unblocked_ibc;
658b6eda
MM		 812 int ret = 0;
813
814 mutex_lock(&kvm->lock);
a03825bb 815 if (kvm->created_vcpus) {
658b6eda
MM		 816 ret = -EBUSY;
817 goto out;
818 }
819 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
820 if (!proc) {
821 ret = -ENOMEM;
822 goto out;
823 }
824 if (!copy_from_user(proc, (void __user *)attr->addr,
825 sizeof(*proc))) {
9bb0ec09 826 kvm->arch.model.cpuid = proc->cpuid;
053dd230
DH		 827 lowest_ibc = sclp.ibc >> 16 & 0xfff;
828 unblocked_ibc = sclp.ibc & 0xfff;
0487c44d 829 if (lowest_ibc && proc->ibc) {
053dd230
DH		 830 if (proc->ibc > unblocked_ibc)
831 kvm->arch.model.ibc = unblocked_ibc;
832 else if (proc->ibc < lowest_ibc)
833 kvm->arch.model.ibc = lowest_ibc;
834 else
835 kvm->arch.model.ibc = proc->ibc;
836 }
c54f0d6a 837 memcpy(kvm->arch.model.fac_list, proc->fac_list,
658b6eda 838 S390_ARCH_FAC_LIST_SIZE_BYTE);
a8c39dd7
CB		 839 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
840 kvm->arch.model.ibc,
841 kvm->arch.model.cpuid);
842 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
843 kvm->arch.model.fac_list[0],
844 kvm->arch.model.fac_list[1],
845 kvm->arch.model.fac_list[2]);
658b6eda
MM		 846 } else
847 ret = -EFAULT;
848 kfree(proc);
849out:
850 mutex_unlock(&kvm->lock);
851 return ret;
852}
853
15c9705f
DH		 854static int kvm_s390_set_processor_feat(struct kvm *kvm,
855 struct kvm_device_attr *attr)
856{
857 struct kvm_s390_vm_cpu_feat data;
858 int ret = -EBUSY;
859
860 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
861 return -EFAULT;
862 if (!bitmap_subset((unsigned long *) data.feat,
863 kvm_s390_available_cpu_feat,
864 KVM_S390_VM_CPU_FEAT_NR_BITS))
865 return -EINVAL;
866
867 mutex_lock(&kvm->lock);
868 if (!atomic_read(&kvm->online_vcpus)) {
869 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
870 KVM_S390_VM_CPU_FEAT_NR_BITS);
871 ret = 0;
872 }
873 mutex_unlock(&kvm->lock);
874 return ret;
875}
876
0a763c78
DH		 877static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
878 struct kvm_device_attr *attr)
879{
880 /*
881 * Once supported by kernel + hw, we have to store the subfunctions
882 * in kvm->arch and remember that user space configured them.
883 */
884 return -ENXIO;
885}
886
658b6eda
MM		 887static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
888{
889 int ret = -ENXIO;
890
891 switch (attr->attr) {
892 case KVM_S390_VM_CPU_PROCESSOR:
893 ret = kvm_s390_set_processor(kvm, attr);
894 break;
15c9705f
DH		 895 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
896 ret = kvm_s390_set_processor_feat(kvm, attr);
897 break;
0a763c78
DH		 898 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
899 ret = kvm_s390_set_processor_subfunc(kvm, attr);
900 break;
658b6eda
MM		 901 }
902 return ret;
903}
904
905static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
906{
907 struct kvm_s390_vm_cpu_processor *proc;
908 int ret = 0;
909
910 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
911 if (!proc) {
912 ret = -ENOMEM;
913 goto out;
914 }
9bb0ec09 915 proc->cpuid = kvm->arch.model.cpuid;
658b6eda 916 proc->ibc = kvm->arch.model.ibc;
c54f0d6a
DH		 917 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
918 S390_ARCH_FAC_LIST_SIZE_BYTE);
a8c39dd7
CB		 919 VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
920 kvm->arch.model.ibc,
921 kvm->arch.model.cpuid);
922 VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
923 kvm->arch.model.fac_list[0],
924 kvm->arch.model.fac_list[1],
925 kvm->arch.model.fac_list[2]);
658b6eda
MM		 926 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
927 ret = -EFAULT;
928 kfree(proc);
929out:
930 return ret;
931}
932
933static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
934{
935 struct kvm_s390_vm_cpu_machine *mach;
936 int ret = 0;
937
938 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
939 if (!mach) {
940 ret = -ENOMEM;
941 goto out;
942 }
943 get_cpu_id((struct cpuid *) &mach->cpuid);
37c5f6c8 944 mach->ibc = sclp.ibc;
c54f0d6a 945 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
981467c9 946 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 947 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
04478197 948 sizeof(S390_lowcore.stfle_fac_list));
a8c39dd7
CB		 949 VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx",
950 kvm->arch.model.ibc,
951 kvm->arch.model.cpuid);
952 VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx",
953 mach->fac_mask[0],
954 mach->fac_mask[1],
955 mach->fac_mask[2]);
956 VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx",
957 mach->fac_list[0],
958 mach->fac_list[1],
959 mach->fac_list[2]);
658b6eda
MM		 960 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
961 ret = -EFAULT;
962 kfree(mach);
963out:
964 return ret;
965}
966
15c9705f
DH		 967static int kvm_s390_get_processor_feat(struct kvm *kvm,
968 struct kvm_device_attr *attr)
969{
970 struct kvm_s390_vm_cpu_feat data;
971
972 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
973 KVM_S390_VM_CPU_FEAT_NR_BITS);
974 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
975 return -EFAULT;
976 return 0;
977}
978
979static int kvm_s390_get_machine_feat(struct kvm *kvm,
980 struct kvm_device_attr *attr)
981{
982 struct kvm_s390_vm_cpu_feat data;
983
984 bitmap_copy((unsigned long *) data.feat,
985 kvm_s390_available_cpu_feat,
986 KVM_S390_VM_CPU_FEAT_NR_BITS);
987 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
988 return -EFAULT;
989 return 0;
990}
991
0a763c78
DH		 992static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
993 struct kvm_device_attr *attr)
994{
995 /*
996 * Once we can actually configure subfunctions (kernel + hw support),
997 * we have to check if they were already set by user space, if so copy
998 * them from kvm->arch.
999 */
1000 return -ENXIO;
1001}
1002
1003static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1004 struct kvm_device_attr *attr)
1005{
1006 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1007 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1008 return -EFAULT;
1009 return 0;
1010}
658b6eda
MM		 1011static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1012{
1013 int ret = -ENXIO;
1014
1015 switch (attr->attr) {
1016 case KVM_S390_VM_CPU_PROCESSOR:
1017 ret = kvm_s390_get_processor(kvm, attr);
1018 break;
1019 case KVM_S390_VM_CPU_MACHINE:
1020 ret = kvm_s390_get_machine(kvm, attr);
1021 break;
15c9705f
DH		 1022 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1023 ret = kvm_s390_get_processor_feat(kvm, attr);
1024 break;
1025 case KVM_S390_VM_CPU_MACHINE_FEAT:
1026 ret = kvm_s390_get_machine_feat(kvm, attr);
1027 break;
0a763c78
DH		 1028 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1029 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1030 break;
1031 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1032 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1033 break;
658b6eda
MM		 1034 }
1035 return ret;
1036}
1037
f2061656
DD		 1038static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1039{
1040 int ret;
1041
1042 switch (attr->group) {
4f718eab 1043 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 1044 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 1045 break;
72f25020
JH		 1046 case KVM_S390_VM_TOD:
1047 ret = kvm_s390_set_tod(kvm, attr);
1048 break;
658b6eda
MM		 1049 case KVM_S390_VM_CPU_MODEL:
1050 ret = kvm_s390_set_cpu_model(kvm, attr);
1051 break;
a374e892
TK		 1052 case KVM_S390_VM_CRYPTO:
1053 ret = kvm_s390_vm_set_crypto(kvm, attr);
1054 break;
f2061656
DD		 1055 default:
1056 ret = -ENXIO;
1057 break;
1058 }
1059
1060 return ret;
1061}
1062
1063static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1064{
8c0a7ce6
DD		 1065 int ret;
1066
1067 switch (attr->group) {
1068 case KVM_S390_VM_MEM_CTRL:
1069 ret = kvm_s390_get_mem_control(kvm, attr);
1070 break;
72f25020
JH		 1071 case KVM_S390_VM_TOD:
1072 ret = kvm_s390_get_tod(kvm, attr);
1073 break;
658b6eda
MM		 1074 case KVM_S390_VM_CPU_MODEL:
1075 ret = kvm_s390_get_cpu_model(kvm, attr);
1076 break;
8c0a7ce6
DD		 1077 default:
1078 ret = -ENXIO;
1079 break;
1080 }
1081
1082 return ret;
f2061656
DD		 1083}
1084
1085static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1086{
1087 int ret;
1088
1089 switch (attr->group) {
4f718eab
DD		 1090 case KVM_S390_VM_MEM_CTRL:
1091 switch (attr->attr) {
1092 case KVM_S390_VM_MEM_ENABLE_CMMA:
1093 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 1094 ret = sclp.has_cmma ? 0 : -ENXIO;
1095 break;
8c0a7ce6 1096 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 1097 ret = 0;
1098 break;
1099 default:
1100 ret = -ENXIO;
1101 break;
1102 }
1103 break;
72f25020
JH		 1104 case KVM_S390_VM_TOD:
1105 switch (attr->attr) {
1106 case KVM_S390_VM_TOD_LOW:
1107 case KVM_S390_VM_TOD_HIGH:
1108 ret = 0;
1109 break;
1110 default:
1111 ret = -ENXIO;
1112 break;
1113 }
1114 break;
658b6eda
MM		 1115 case KVM_S390_VM_CPU_MODEL:
1116 switch (attr->attr) {
1117 case KVM_S390_VM_CPU_PROCESSOR:
1118 case KVM_S390_VM_CPU_MACHINE:
15c9705f
DH		 1119 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1120 case KVM_S390_VM_CPU_MACHINE_FEAT:
0a763c78 1121 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
658b6eda
MM		 1122 ret = 0;
1123 break;
0a763c78
DH		 1124 /* configuring subfunctions is not supported yet */
1125 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
658b6eda
MM		 1126 default:
1127 ret = -ENXIO;
1128 break;
1129 }
1130 break;
a374e892
TK		 1131 case KVM_S390_VM_CRYPTO:
1132 switch (attr->attr) {
1133 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1134 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1135 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1136 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1137 ret = 0;
1138 break;
1139 default:
1140 ret = -ENXIO;
1141 break;
1142 }
1143 break;
f2061656
DD		 1144 default:
1145 ret = -ENXIO;
1146 break;
1147 }
1148
1149 return ret;
1150}
1151
30ee2a98
JH		 1152static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1153{
1154 uint8_t *keys;
1155 uint64_t hva;
30ee2a98
JH		 1156 int i, r = 0;
1157
1158 if (args->flags != 0)
1159 return -EINVAL;
1160
1161 /* Is this guest using storage keys? */
1162 if (!mm_use_skey(current->mm))
1163 return KVM_S390_GET_SKEYS_NONE;
1164
1165 /* Enforce sane limit on memory allocation */
1166 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1167 return -EINVAL;
1168
752ade68 1169 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
30ee2a98
JH		 1170 if (!keys)
1171 return -ENOMEM;
1172
d3ed1cee 1173 down_read(&current->mm->mmap_sem);
30ee2a98
JH		 1174 for (i = 0; i < args->count; i++) {
1175 hva = gfn_to_hva(kvm, args->start_gfn + i);
1176 if (kvm_is_error_hva(hva)) {
1177 r = -EFAULT;
d3ed1cee 1178 break;
30ee2a98
JH		 1179 }
1180
154c8c19
DH		 1181 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1182 if (r)
d3ed1cee 1183 break;
30ee2a98 1184 }
d3ed1cee
MS		 1185 up_read(&current->mm->mmap_sem);
1186
1187 if (!r) {
1188 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1189 sizeof(uint8_t) * args->count);
1190 if (r)
1191 r = -EFAULT;
30ee2a98
JH		 1192 }
1193
30ee2a98
JH		 1194 kvfree(keys);
1195 return r;
1196}
1197
1198static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1199{
1200 uint8_t *keys;
1201 uint64_t hva;
1202 int i, r = 0;
1203
1204 if (args->flags != 0)
1205 return -EINVAL;
1206
1207 /* Enforce sane limit on memory allocation */
1208 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1209 return -EINVAL;
1210
752ade68 1211 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
30ee2a98
JH		 1212 if (!keys)
1213 return -ENOMEM;
1214
1215 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1216 sizeof(uint8_t) * args->count);
1217 if (r) {
1218 r = -EFAULT;
1219 goto out;
1220 }
1221
1222 /* Enable storage key handling for the guest */
14d4a425
DD		 1223 r = s390_enable_skey();
1224 if (r)
1225 goto out;
30ee2a98 1226
d3ed1cee 1227 down_read(&current->mm->mmap_sem);
30ee2a98
JH		 1228 for (i = 0; i < args->count; i++) {
1229 hva = gfn_to_hva(kvm, args->start_gfn + i);
1230 if (kvm_is_error_hva(hva)) {
1231 r = -EFAULT;
d3ed1cee 1232 break;
30ee2a98
JH		 1233 }
1234
1235 /* Lowest order bit is reserved */
1236 if (keys[i] & 0x01) {
1237 r = -EINVAL;
d3ed1cee 1238 break;
30ee2a98
JH		 1239 }
1240
fe69eabf 1241 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
30ee2a98 1242 if (r)
d3ed1cee 1243 break;
30ee2a98 1244 }
d3ed1cee 1245 up_read(&current->mm->mmap_sem);
30ee2a98
JH		 1246out:
1247 kvfree(keys);
1248 return r;
1249}
1250
b0c632db
HC		 1251long kvm_arch_vm_ioctl(struct file *filp,
1252 unsigned int ioctl, unsigned long arg)
1253{
1254 struct kvm *kvm = filp->private_data;
1255 void __user *argp = (void __user *)arg;
f2061656 1256 struct kvm_device_attr attr;
b0c632db
HC		 1257 int r;
1258
1259 switch (ioctl) {
ba5c1e9b
CO		 1260 case KVM_S390_INTERRUPT: {
1261 struct kvm_s390_interrupt s390int;
1262
1263 r = -EFAULT;
1264 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1265 break;
1266 r = kvm_s390_inject_vm(kvm, &s390int);
1267 break;
1268 }
d938dc55
CH		 1269 case KVM_ENABLE_CAP: {
1270 struct kvm_enable_cap cap;
1271 r = -EFAULT;
1272 if (copy_from_user(&cap, argp, sizeof(cap)))
1273 break;
1274 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1275 break;
1276 }
84223598
CH		 1277 case KVM_CREATE_IRQCHIP: {
1278 struct kvm_irq_routing_entry routing;
1279
1280 r = -EINVAL;
1281 if (kvm->arch.use_irqchip) {
1282 /* Set up dummy routing. */
1283 memset(&routing, 0, sizeof(routing));
152b2839 1284 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
84223598
CH		 1285 }
1286 break;
1287 }
f2061656
DD		 1288 case KVM_SET_DEVICE_ATTR: {
1289 r = -EFAULT;
1290 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1291 break;
1292 r = kvm_s390_vm_set_attr(kvm, &attr);
1293 break;
1294 }
1295 case KVM_GET_DEVICE_ATTR: {
1296 r = -EFAULT;
1297 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1298 break;
1299 r = kvm_s390_vm_get_attr(kvm, &attr);
1300 break;
1301 }
1302 case KVM_HAS_DEVICE_ATTR: {
1303 r = -EFAULT;
1304 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1305 break;
1306 r = kvm_s390_vm_has_attr(kvm, &attr);
1307 break;
1308 }
30ee2a98
JH		 1309 case KVM_S390_GET_SKEYS: {
1310 struct kvm_s390_skeys args;
1311
1312 r = -EFAULT;
1313 if (copy_from_user(&args, argp,
1314 sizeof(struct kvm_s390_skeys)))
1315 break;
1316 r = kvm_s390_get_skeys(kvm, &args);
1317 break;
1318 }
1319 case KVM_S390_SET_SKEYS: {
1320 struct kvm_s390_skeys args;
1321
1322 r = -EFAULT;
1323 if (copy_from_user(&args, argp,
1324 sizeof(struct kvm_s390_skeys)))
1325 break;
1326 r = kvm_s390_set_skeys(kvm, &args);
1327 break;
1328 }
b0c632db 1329 default:
367e1319 1330 r = -ENOTTY;
b0c632db
HC		 1331 }
1332
1333 return r;
1334}
1335
45c9b47c
TK		 1336static int kvm_s390_query_ap_config(u8 *config)
1337{
1338 u32 fcn_code = 0x04000000UL;
86044c8c 1339 u32 cc = 0;
45c9b47c 1340
86044c8c 1341 memset(config, 0, 128);
45c9b47c
TK		 1342 asm volatile(
1343 "lgr 0,%1\n"
1344 "lgr 2,%2\n"
1345 ".long 0xb2af0000\n" /* PQAP(QCI) */
86044c8c 1346 "0: ipm %0\n"
45c9b47c 1347 "srl %0,28\n"
86044c8c
CB		 1348 "1:\n"
1349 EX_TABLE(0b, 1b)
1350 : "+r" (cc)
45c9b47c
TK		 1351 : "r" (fcn_code), "r" (config)
1352 : "cc", "0", "2", "memory"
1353 );
1354
1355 return cc;
1356}
1357
1358static int kvm_s390_apxa_installed(void)
1359{
1360 u8 config[128];
1361 int cc;
1362
a6aacc3f 1363 if (test_facility(12)) {
45c9b47c
TK		 1364 cc = kvm_s390_query_ap_config(config);
1365
1366 if (cc)
1367 pr_err("PQAP(QCI) failed with cc=%d", cc);
1368 else
1369 return config[0] & 0x40;
1370 }
1371
1372 return 0;
1373}
1374
1375static void kvm_s390_set_crycb_format(struct kvm *kvm)
1376{
1377 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1378
1379 if (kvm_s390_apxa_installed())
1380 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1381 else
1382 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1383}
1384
9bb0ec09 1385static u64 kvm_s390_get_initial_cpuid(void)
9d8d5786 1386{
9bb0ec09
DH		 1387 struct cpuid cpuid;
1388
1389 get_cpu_id(&cpuid);
1390 cpuid.version = 0xff;
1391 return *((u64 *) &cpuid);
9d8d5786
MM		 1392}
1393
c54f0d6a 1394static void kvm_s390_crypto_init(struct kvm *kvm)
5102ee87 1395{
9d8d5786 1396 if (!test_kvm_facility(kvm, 76))
c54f0d6a 1397 return;
5102ee87 1398
c54f0d6a 1399 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
45c9b47c 1400 kvm_s390_set_crycb_format(kvm);
5102ee87 1401
ed6f76b4
TK		 1402 /* Enable AES/DEA protected key functions by default */
1403 kvm->arch.crypto.aes_kw = 1;
1404 kvm->arch.crypto.dea_kw = 1;
1405 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1406 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1407 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1408 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
5102ee87
TK		 1409}
1410
7d43bafc
ED		 1411static void sca_dispose(struct kvm *kvm)
1412{
1413 if (kvm->arch.use_esca)
5e044315 1414 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
7d43bafc
ED		 1415 else
1416 free_page((unsigned long)(kvm->arch.sca));
1417 kvm->arch.sca = NULL;
1418}
1419
e08b9637 1420int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 1421{
76a6dd72 1422 gfp_t alloc_flags = GFP_KERNEL;
9d8d5786 1423 int i, rc;
b0c632db 1424 char debug_name[16];
f6c137ff 1425 static unsigned long sca_offset;
b0c632db 1426
e08b9637
CO		 1427 rc = -EINVAL;
1428#ifdef CONFIG_KVM_S390_UCONTROL
1429 if (type & ~KVM_VM_S390_UCONTROL)
1430 goto out_err;
1431 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1432 goto out_err;
1433#else
1434 if (type)
1435 goto out_err;
1436#endif
1437
b0c632db
HC		 1438 rc = s390_enable_sie();
1439 if (rc)
d89f5eff 1440 goto out_err;
b0c632db 1441
b290411a
CO		 1442 rc = -ENOMEM;
1443
7d0a5e62
JF		 1444 ratelimit_state_init(&kvm->arch.sthyi_limit, 5 * HZ, 500);
1445
7d43bafc 1446 kvm->arch.use_esca = 0; /* start with basic SCA */
76a6dd72
DH		 1447 if (!sclp.has_64bscao)
1448 alloc_flags |= GFP_DMA;
5e044315 1449 rwlock_init(&kvm->arch.sca_lock);
76a6dd72 1450 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
b0c632db 1451 if (!kvm->arch.sca)
d89f5eff 1452 goto out_err;
f6c137ff 1453 spin_lock(&kvm_lock);
c5c2c393 1454 sca_offset += 16;
bc784cce 1455 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
c5c2c393 1456 sca_offset = 0;
bc784cce
ED		 1457 kvm->arch.sca = (struct bsca_block *)
1458 ((char *) kvm->arch.sca + sca_offset);
f6c137ff 1459 spin_unlock(&kvm_lock);
b0c632db
HC		 1460
1461 sprintf(debug_name, "kvm-%u", current->pid);
1462
1cb9cf72 1463 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
b0c632db 1464 if (!kvm->arch.dbf)
40f5b735 1465 goto out_err;
b0c632db 1466
c54f0d6a
DH		 1467 kvm->arch.sie_page2 =
1468 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1469 if (!kvm->arch.sie_page2)
40f5b735 1470 goto out_err;
9d8d5786 1471
fb5bf93f 1472 /* Populate the facility mask initially. */
c54f0d6a 1473 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
04478197 1474 sizeof(S390_lowcore.stfle_fac_list));
9d8d5786
MM		 1475 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1476 if (i < kvm_s390_fac_list_mask_size())
c54f0d6a 1477 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
9d8d5786 1478 else
c54f0d6a 1479 kvm->arch.model.fac_mask[i] = 0UL;
9d8d5786
MM		 1480 }
1481
981467c9 1482 /* Populate the facility list initially. */
c54f0d6a
DH		 1483 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1484 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
981467c9
MM		 1485 S390_ARCH_FAC_LIST_SIZE_BYTE);
1486
95ca2cb5
JF		 1487 set_kvm_facility(kvm->arch.model.fac_mask, 74);
1488 set_kvm_facility(kvm->arch.model.fac_list, 74);
1489
9bb0ec09 1490 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
37c5f6c8 1491 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
9d8d5786 1492
c54f0d6a 1493 kvm_s390_crypto_init(kvm);
5102ee87 1494
ba5c1e9b 1495 spin_lock_init(&kvm->arch.float_int.lock);
6d3da241
JF		 1496 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1497 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
8a242234 1498 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 1499 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 1500
b0c632db 1501 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
78f26131 1502 VM_EVENT(kvm, 3, "vm created with type %lu", type);
b0c632db 1503
e08b9637
CO		 1504 if (type & KVM_VM_S390_UCONTROL) {
1505 kvm->arch.gmap = NULL;
a3a92c31 1506 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
e08b9637 1507 } else {
32e6b236 1508 if (sclp.hamax == U64_MAX)
ee71d16d 1509 kvm->arch.mem_limit = TASK_SIZE_MAX;
32e6b236 1510 else
ee71d16d 1511 kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
32e6b236 1512 sclp.hamax + 1);
6ea427bb 1513 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
e08b9637 1514 if (!kvm->arch.gmap)
40f5b735 1515 goto out_err;
2c70fe44 1516 kvm->arch.gmap->private = kvm;
24eb3a82 1517 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 1518 }
fa6b7fe9
CH		 1519
1520 kvm->arch.css_support = 0;
84223598 1521 kvm->arch.use_irqchip = 0;
72f25020 1522 kvm->arch.epoch = 0;
fa6b7fe9 1523
8ad35755 1524 spin_lock_init(&kvm->arch.start_stop_lock);
a3508fbe 1525 kvm_s390_vsie_init(kvm);
8335713a 1526 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
8ad35755 1527
d89f5eff 1528 return 0;
40f5b735 1529out_err:
c54f0d6a 1530 free_page((unsigned long)kvm->arch.sie_page2);
598841ca 1531 debug_unregister(kvm->arch.dbf);
7d43bafc 1532 sca_dispose(kvm);
78f26131 1533 KVM_EVENT(3, "creation of vm failed: %d", rc);
d89f5eff 1534 return rc;
b0c632db
HC		 1535}
1536
235539b4
LC		 1537bool kvm_arch_has_vcpu_debugfs(void)
1538{
1539 return false;
1540}
1541
1542int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
1543{
1544 return 0;
1545}
1546
d329c035
CB		 1547void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1548{
1549 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 1550 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 1551 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 1552 kvm_clear_async_pf_completion_queue(vcpu);
bc784cce 1553 if (!kvm_is_ucontrol(vcpu->kvm))
a6e2f683 1554 sca_del_vcpu(vcpu);
27e0393f
CO		 1555
1556 if (kvm_is_ucontrol(vcpu->kvm))
6ea427bb 1557 gmap_remove(vcpu->arch.gmap);
27e0393f 1558
e6db1d61 1559 if (vcpu->kvm->arch.use_cmma)
b31605c1 1560 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 1561 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 1562
6692cef3 1563 kvm_vcpu_uninit(vcpu);
b110feaf 1564 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 1565}
1566
1567static void kvm_free_vcpus(struct kvm *kvm)
1568{
1569 unsigned int i;
988a2cae 1570 struct kvm_vcpu *vcpu;
d329c035 1571
988a2cae
GN		 1572 kvm_for_each_vcpu(i, vcpu, kvm)
1573 kvm_arch_vcpu_destroy(vcpu);
1574
1575 mutex_lock(&kvm->lock);
1576 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1577 kvm->vcpus[i] = NULL;
1578
1579 atomic_set(&kvm->online_vcpus, 0);
1580 mutex_unlock(&kvm->lock);
d329c035
CB		 1581}
1582
b0c632db
HC		 1583void kvm_arch_destroy_vm(struct kvm *kvm)
1584{
d329c035 1585 kvm_free_vcpus(kvm);
7d43bafc 1586 sca_dispose(kvm);
d329c035 1587 debug_unregister(kvm->arch.dbf);
c54f0d6a 1588 free_page((unsigned long)kvm->arch.sie_page2);
27e0393f 1589 if (!kvm_is_ucontrol(kvm))
6ea427bb 1590 gmap_remove(kvm->arch.gmap);
841b91c5 1591 kvm_s390_destroy_adapters(kvm);
67335e63 1592 kvm_s390_clear_float_irqs(kvm);
a3508fbe 1593 kvm_s390_vsie_destroy(kvm);
8335713a 1594 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
b0c632db
HC		 1595}
1596
1597/* Section: vcpu related */
dafd032a
DD		 1598static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1599{
6ea427bb 1600 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
dafd032a
DD		 1601 if (!vcpu->arch.gmap)
1602 return -ENOMEM;
1603 vcpu->arch.gmap->private = vcpu->kvm;
1604
1605 return 0;
1606}
1607
a6e2f683
ED		 1608static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1609{
a6940674
DH		 1610 if (!kvm_s390_use_sca_entries())
1611 return;
5e044315 1612 read_lock(&vcpu->kvm->arch.sca_lock);
7d43bafc
ED		 1613 if (vcpu->kvm->arch.use_esca) {
1614 struct esca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1615
7d43bafc 1616 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
10ce32d5 1617 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc
ED		 1618 } else {
1619 struct bsca_block *sca = vcpu->kvm->arch.sca;
1620
1621 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
10ce32d5 1622 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc 1623 }
5e044315 1624 read_unlock(&vcpu->kvm->arch.sca_lock);
a6e2f683
ED		 1625}
1626
eaa78f34 1627static void sca_add_vcpu(struct kvm_vcpu *vcpu)
a6e2f683 1628{
a6940674
DH		 1629 if (!kvm_s390_use_sca_entries()) {
1630 struct bsca_block *sca = vcpu->kvm->arch.sca;
1631
1632 /* we still need the basic sca for the ipte control */
1633 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1634 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
1635 }
eaa78f34
DH		 1636 read_lock(&vcpu->kvm->arch.sca_lock);
1637 if (vcpu->kvm->arch.use_esca) {
1638 struct esca_block *sca = vcpu->kvm->arch.sca;
7d43bafc 1639
eaa78f34 1640 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1641 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1642 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
25508824 1643 vcpu->arch.sie_block->ecb2 |= 0x04U;
eaa78f34 1644 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
7d43bafc 1645 } else {
eaa78f34 1646 struct bsca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1647
eaa78f34 1648 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1649 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1650 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
eaa78f34 1651 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
7d43bafc 1652 }
eaa78f34 1653 read_unlock(&vcpu->kvm->arch.sca_lock);
5e044315
ED		 1654}
1655
1656/* Basic SCA to Extended SCA data copy routines */
1657static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1658{
1659 d->sda = s->sda;
1660 d->sigp_ctrl.c = s->sigp_ctrl.c;
1661 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1662}
1663
1664static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1665{
1666 int i;
1667
1668 d->ipte_control = s->ipte_control;
1669 d->mcn[0] = s->mcn;
1670 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1671 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1672}
1673
1674static int sca_switch_to_extended(struct kvm *kvm)
1675{
1676 struct bsca_block *old_sca = kvm->arch.sca;
1677 struct esca_block *new_sca;
1678 struct kvm_vcpu *vcpu;
1679 unsigned int vcpu_idx;
1680 u32 scaol, scaoh;
1681
1682 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1683 if (!new_sca)
1684 return -ENOMEM;
1685
1686 scaoh = (u32)((u64)(new_sca) >> 32);
1687 scaol = (u32)(u64)(new_sca) & ~0x3fU;
1688
1689 kvm_s390_vcpu_block_all(kvm);
1690 write_lock(&kvm->arch.sca_lock);
1691
1692 sca_copy_b_to_e(new_sca, old_sca);
1693
1694 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1695 vcpu->arch.sie_block->scaoh = scaoh;
1696 vcpu->arch.sie_block->scaol = scaol;
1697 vcpu->arch.sie_block->ecb2 |= 0x04U;
1698 }
1699 kvm->arch.sca = new_sca;
1700 kvm->arch.use_esca = 1;
1701
1702 write_unlock(&kvm->arch.sca_lock);
1703 kvm_s390_vcpu_unblock_all(kvm);
1704
1705 free_page((unsigned long)old_sca);
1706
8335713a
CB		 1707 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1708 old_sca, kvm->arch.sca);
5e044315 1709 return 0;
a6e2f683
ED		 1710}
1711
1712static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1713{
5e044315
ED		 1714 int rc;
1715
a6940674
DH		 1716 if (!kvm_s390_use_sca_entries()) {
1717 if (id < KVM_MAX_VCPUS)
1718 return true;
1719 return false;
1720 }
5e044315
ED		 1721 if (id < KVM_S390_BSCA_CPU_SLOTS)
1722 return true;
76a6dd72 1723 if (!sclp.has_esca || !sclp.has_64bscao)
5e044315
ED		 1724 return false;
1725
1726 mutex_lock(&kvm->lock);
1727 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1728 mutex_unlock(&kvm->lock);
1729
1730 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
a6e2f683
ED		 1731}
1732
b0c632db
HC		 1733int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1734{
3c038e6b
DD		 1735 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1736 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 1737 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1738 KVM_SYNC_GPRS |
9eed0735 1739 KVM_SYNC_ACRS |
b028ee3e
DH		 1740 KVM_SYNC_CRS |
1741 KVM_SYNC_ARCH0 |
1742 KVM_SYNC_PFAULT;
75a4615c 1743 kvm_s390_set_prefix(vcpu, 0);
c6e5f166
FZ		 1744 if (test_kvm_facility(vcpu->kvm, 64))
1745 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
f6aa6dc4
DH		 1746 /* fprs can be synchronized via vrs, even if the guest has no vx. With
1747 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1748 */
1749 if (MACHINE_HAS_VX)
68c55750 1750 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
6fd8e67d
DH		 1751 else
1752 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
dafd032a
DD		 1753
1754 if (kvm_is_ucontrol(vcpu->kvm))
1755 return __kvm_ucontrol_vcpu_init(vcpu);
1756
b0c632db
HC		 1757 return 0;
1758}
1759
db0758b2
DH		 1760/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1761static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1762{
1763 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
9c23a131 1764 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2 1765 vcpu->arch.cputm_start = get_tod_clock_fast();
9c23a131 1766 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1767}
1768
1769/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1770static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1771{
1772 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
9c23a131 1773 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1774 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1775 vcpu->arch.cputm_start = 0;
9c23a131 1776 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1777}
1778
1779/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1780static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1781{
1782 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
1783 vcpu->arch.cputm_enabled = true;
1784 __start_cpu_timer_accounting(vcpu);
1785}
1786
1787/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1788static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1789{
1790 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
1791 __stop_cpu_timer_accounting(vcpu);
1792 vcpu->arch.cputm_enabled = false;
1793}
1794
1795static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1796{
1797 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1798 __enable_cpu_timer_accounting(vcpu);
1799 preempt_enable();
1800}
1801
1802static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1803{
1804 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1805 __disable_cpu_timer_accounting(vcpu);
1806 preempt_enable();
1807}
1808
4287f247
DH		 1809/* set the cpu timer - may only be called from the VCPU thread itself */
1810void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
1811{
db0758b2 1812 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
9c23a131 1813 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1814 if (vcpu->arch.cputm_enabled)
1815 vcpu->arch.cputm_start = get_tod_clock_fast();
4287f247 1816 vcpu->arch.sie_block->cputm = cputm;
9c23a131 1817 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2 1818 preempt_enable();
4287f247
DH		 1819}
1820
db0758b2 1821/* update and get the cpu timer - can also be called from other VCPU threads */
4287f247
DH		 1822__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
1823{
9c23a131 1824 unsigned int seq;
db0758b2 1825 __u64 value;
db0758b2
DH		 1826
1827 if (unlikely(!vcpu->arch.cputm_enabled))
1828 return vcpu->arch.sie_block->cputm;
1829
9c23a131
DH		 1830 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1831 do {
1832 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
1833 /*
1834 * If the writer would ever execute a read in the critical
1835 * section, e.g. in irq context, we have a deadlock.
1836 */
1837 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
1838 value = vcpu->arch.sie_block->cputm;
1839 /* if cputm_start is 0, accounting is being started/stopped */
1840 if (likely(vcpu->arch.cputm_start))
1841 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1842 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
1843 preempt_enable();
db0758b2 1844 return value;
4287f247
DH		 1845}
1846
b0c632db
HC		 1847void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1848{
9977e886 1849
37d9df98 1850 gmap_enable(vcpu->arch.enabled_gmap);
805de8f4 1851 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
5ebda316 1852 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1853 __start_cpu_timer_accounting(vcpu);
01a745ac 1854 vcpu->cpu = cpu;
b0c632db
HC		 1855}
1856
1857void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1858{
01a745ac 1859 vcpu->cpu = -1;
5ebda316 1860 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1861 __stop_cpu_timer_accounting(vcpu);
805de8f4 1862 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
37d9df98
DH		 1863 vcpu->arch.enabled_gmap = gmap_get_enabled();
1864 gmap_disable(vcpu->arch.enabled_gmap);
9977e886 1865
b0c632db
HC		 1866}
1867
1868static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1869{
1870 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1871 vcpu->arch.sie_block->gpsw.mask = 0UL;
1872 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 1873 kvm_s390_set_prefix(vcpu, 0);
4287f247 1874 kvm_s390_set_cpu_timer(vcpu, 0);
b0c632db
HC		 1875 vcpu->arch.sie_block->ckc = 0UL;
1876 vcpu->arch.sie_block->todpr = 0;
1877 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1878 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1879 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
9abc2a08
DH		 1880 /* make sure the new fpc will be lazily loaded */
1881 save_fpu_regs();
1882 current->thread.fpu.fpc = 0;
b0c632db 1883 vcpu->arch.sie_block->gbea = 1;
672550fb 1884 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 1885 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1886 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 1887 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1888 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 1889 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 1890}
1891
31928aa5 1892void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 1893{
72f25020 1894 mutex_lock(&vcpu->kvm->lock);
fdf03650 1895 preempt_disable();
72f25020 1896 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
fdf03650 1897 preempt_enable();
72f25020 1898 mutex_unlock(&vcpu->kvm->lock);
25508824 1899 if (!kvm_is_ucontrol(vcpu->kvm)) {
dafd032a 1900 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
eaa78f34 1901 sca_add_vcpu(vcpu);
25508824 1902 }
6502a34c
DH		 1903 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
1904 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
37d9df98
DH		 1905 /* make vcpu_load load the right gmap on the first trigger */
1906 vcpu->arch.enabled_gmap = vcpu->arch.gmap;
42897d86
MT		 1907}
1908
5102ee87
TK		 1909static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1910{
9d8d5786 1911 if (!test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 1912 return;
1913
a374e892
TK		 1914 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1915
1916 if (vcpu->kvm->arch.crypto.aes_kw)
1917 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1918 if (vcpu->kvm->arch.crypto.dea_kw)
1919 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1920
5102ee87
TK		 1921 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1922}
1923
b31605c1
DD		 1924void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1925{
1926 free_page(vcpu->arch.sie_block->cbrlo);
1927 vcpu->arch.sie_block->cbrlo = 0;
1928}
1929
1930int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1931{
1932 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1933 if (!vcpu->arch.sie_block->cbrlo)
1934 return -ENOMEM;
1935
1936 vcpu->arch.sie_block->ecb2 |= 0x80;
1937 vcpu->arch.sie_block->ecb2 &= ~0x08;
1938 return 0;
1939}
1940
91520f1a
MM		 1941static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1942{
1943 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1944
91520f1a 1945 vcpu->arch.sie_block->ibc = model->ibc;
80bc79dc 1946 if (test_kvm_facility(vcpu->kvm, 7))
c54f0d6a 1947 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
91520f1a
MM		 1948}
1949
b0c632db
HC		 1950int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1951{
b31605c1 1952 int rc = 0;
b31288fa 1953
9e6dabef
CH		 1954 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1955 CPUSTAT_SM |
a4a4f191
GH		 1956 CPUSTAT_STOPPED);
1957
53df84f8 1958 if (test_kvm_facility(vcpu->kvm, 78))
805de8f4 1959 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
53df84f8 1960 else if (test_kvm_facility(vcpu->kvm, 8))
805de8f4 1961 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
a4a4f191 1962
91520f1a
MM		 1963 kvm_s390_vcpu_setup_model(vcpu);
1964
bdab09f3
DH		 1965 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
1966 if (MACHINE_HAS_ESOP)
1967 vcpu->arch.sie_block->ecb |= 0x02;
bd50e8ec
DH		 1968 if (test_kvm_facility(vcpu->kvm, 9))
1969 vcpu->arch.sie_block->ecb |= 0x04;
f597d24e 1970 if (test_kvm_facility(vcpu->kvm, 73))
7feb6bb8
MM		 1971 vcpu->arch.sie_block->ecb |= 0x10;
1972
873b425e 1973 if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
d6af0b49 1974 vcpu->arch.sie_block->ecb2 |= 0x08;
cd1836f5
JF		 1975 if (test_kvm_facility(vcpu->kvm, 130))
1976 vcpu->arch.sie_block->ecb2 |= 0x20;
48ee7d3a
DH		 1977 vcpu->arch.sie_block->eca = 0x1002000U;
1978 if (sclp.has_cei)
1979 vcpu->arch.sie_block->eca |= 0x80000000U;
11ad65b7
DH		 1980 if (sclp.has_ib)
1981 vcpu->arch.sie_block->eca |= 0x40000000U;
37c5f6c8 1982 if (sclp.has_siif)
217a4406 1983 vcpu->arch.sie_block->eca |= 1;
37c5f6c8 1984 if (sclp.has_sigpif)
ea5f4969 1985 vcpu->arch.sie_block->eca |= 0x10000000U;
18280d8b 1986 if (test_kvm_facility(vcpu->kvm, 129)) {
13211ea7
EF		 1987 vcpu->arch.sie_block->eca |= 0x00020000;
1988 vcpu->arch.sie_block->ecd |= 0x20000000;
1989 }
c6e5f166 1990 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
492d8642 1991 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
5a5e6536 1992
e6db1d61 1993 if (vcpu->kvm->arch.use_cmma) {
b31605c1
DD		 1994 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1995 if (rc)
1996 return rc;
b31288fa 1997 }
0ac96caf 1998 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 1999 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 2000
5102ee87
TK		 2001 kvm_s390_vcpu_crypto_setup(vcpu);
2002
b31605c1 2003 return rc;
b0c632db
HC		 2004}
2005
2006struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
2007 unsigned int id)
2008{
4d47555a 2009 struct kvm_vcpu *vcpu;
7feb6bb8 2010 struct sie_page *sie_page;
4d47555a
CO		 2011 int rc = -EINVAL;
2012
4215825e 2013 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
4d47555a
CO		 2014 goto out;
2015
2016 rc = -ENOMEM;
b0c632db 2017
b110feaf 2018 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 2019 if (!vcpu)
4d47555a 2020 goto out;
b0c632db 2021
7feb6bb8
MM		 2022 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
2023 if (!sie_page)
b0c632db
HC		 2024 goto out_free_cpu;
2025
7feb6bb8
MM		 2026 vcpu->arch.sie_block = &sie_page->sie_block;
2027 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
2028
efed1104
DH		 2029 /* the real guest size will always be smaller than msl */
2030 vcpu->arch.sie_block->mso = 0;
2031 vcpu->arch.sie_block->msl = sclp.hamax;
2032
b0c632db 2033 vcpu->arch.sie_block->icpua = id;
ba5c1e9b 2034 spin_lock_init(&vcpu->arch.local_int.lock);
ba5c1e9b 2035 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 2036 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 2037 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
9c23a131 2038 seqcount_init(&vcpu->arch.cputm_seqcount);
ba5c1e9b 2039
b0c632db
HC		 2040 rc = kvm_vcpu_init(vcpu, kvm, id);
2041 if (rc)
9abc2a08 2042 goto out_free_sie_block;
8335713a 2043 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
b0c632db 2044 vcpu->arch.sie_block);
ade38c31 2045 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 2046
b0c632db 2047 return vcpu;
7b06bf2f
WY		 2048out_free_sie_block:
2049 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 2050out_free_cpu:
b110feaf 2051 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 2052out:
b0c632db
HC		 2053 return ERR_PTR(rc);
2054}
2055
b0c632db
HC		 2056int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
2057{
9a022067 2058 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 2059}
2060
27406cd5 2061void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
49b99e1e 2062{
805de8f4 2063 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
61a6df54 2064 exit_sie(vcpu);
49b99e1e
CB		 2065}
2066
27406cd5 2067void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
49b99e1e 2068{
805de8f4 2069 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
49b99e1e
CB		 2070}
2071
8e236546
CB		 2072static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
2073{
805de8f4 2074 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
61a6df54 2075 exit_sie(vcpu);
8e236546
CB		 2076}
2077
2078static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2079{
9bf9fde2 2080 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
8e236546
CB		 2081}
2082
49b99e1e
CB		 2083/*
2084 * Kick a guest cpu out of SIE and wait until SIE is not running.
2085 * If the CPU is not running (e.g. waiting as idle) the function will
2086 * return immediately. */
2087void exit_sie(struct kvm_vcpu *vcpu)
2088{
805de8f4 2089 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
49b99e1e
CB		 2090 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2091 cpu_relax();
2092}
2093
8e236546
CB		 2094/* Kick a guest cpu out of SIE to process a request synchronously */
2095void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
49b99e1e 2096{
8e236546
CB		 2097 kvm_make_request(req, vcpu);
2098 kvm_s390_vcpu_request(vcpu);
49b99e1e
CB		 2099}
2100
414d3b07
MS		 2101static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
2102 unsigned long end)
2c70fe44 2103{
2c70fe44
CB		 2104 struct kvm *kvm = gmap->private;
2105 struct kvm_vcpu *vcpu;
414d3b07
MS		 2106 unsigned long prefix;
2107 int i;
2c70fe44 2108
65d0b0d4
DH		 2109 if (gmap_is_shadow(gmap))
2110 return;
414d3b07
MS		 2111 if (start >= 1UL << 31)
2112 /* We are only interested in prefix pages */
2113 return;
2c70fe44
CB		 2114 kvm_for_each_vcpu(i, vcpu, kvm) {
2115 /* match against both prefix pages */
414d3b07
MS		 2116 prefix = kvm_s390_get_prefix(vcpu);
2117 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
2118 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
2119 start, end);
8e236546 2120 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44
CB		 2121 }
2122 }
2123}
2124
b6d33834
CD		 2125int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2126{
2127 /* kvm common code refers to this, but never calls it */
2128 BUG();
2129 return 0;
2130}
2131
14eebd91
CO		 2132static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2133 struct kvm_one_reg *reg)
2134{
2135 int r = -EINVAL;
2136
2137 switch (reg->id) {
29b7c71b
CO		 2138 case KVM_REG_S390_TODPR:
2139 r = put_user(vcpu->arch.sie_block->todpr,
2140 (u32 __user *)reg->addr);
2141 break;
2142 case KVM_REG_S390_EPOCHDIFF:
2143 r = put_user(vcpu->arch.sie_block->epoch,
2144 (u64 __user *)reg->addr);
2145 break;
46a6dd1c 2146 case KVM_REG_S390_CPU_TIMER:
4287f247 2147 r = put_user(kvm_s390_get_cpu_timer(vcpu),
46a6dd1c
J		 2148 (u64 __user *)reg->addr);
2149 break;
2150 case KVM_REG_S390_CLOCK_COMP:
2151 r = put_user(vcpu->arch.sie_block->ckc,
2152 (u64 __user *)reg->addr);
2153 break;
536336c2
DD		 2154 case KVM_REG_S390_PFTOKEN:
2155 r = put_user(vcpu->arch.pfault_token,
2156 (u64 __user *)reg->addr);
2157 break;
2158 case KVM_REG_S390_PFCOMPARE:
2159 r = put_user(vcpu->arch.pfault_compare,
2160 (u64 __user *)reg->addr);
2161 break;
2162 case KVM_REG_S390_PFSELECT:
2163 r = put_user(vcpu->arch.pfault_select,
2164 (u64 __user *)reg->addr);
2165 break;
672550fb
CB		 2166 case KVM_REG_S390_PP:
2167 r = put_user(vcpu->arch.sie_block->pp,
2168 (u64 __user *)reg->addr);
2169 break;
afa45ff5
CB		 2170 case KVM_REG_S390_GBEA:
2171 r = put_user(vcpu->arch.sie_block->gbea,
2172 (u64 __user *)reg->addr);
2173 break;
14eebd91
CO		 2174 default:
2175 break;
2176 }
2177
2178 return r;
2179}
2180
2181static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2182 struct kvm_one_reg *reg)
2183{
2184 int r = -EINVAL;
4287f247 2185 __u64 val;
14eebd91
CO		 2186
2187 switch (reg->id) {
29b7c71b
CO		 2188 case KVM_REG_S390_TODPR:
2189 r = get_user(vcpu->arch.sie_block->todpr,
2190 (u32 __user *)reg->addr);
2191 break;
2192 case KVM_REG_S390_EPOCHDIFF:
2193 r = get_user(vcpu->arch.sie_block->epoch,
2194 (u64 __user *)reg->addr);
2195 break;
46a6dd1c 2196 case KVM_REG_S390_CPU_TIMER:
4287f247
DH		 2197 r = get_user(val, (u64 __user *)reg->addr);
2198 if (!r)
2199 kvm_s390_set_cpu_timer(vcpu, val);
46a6dd1c
J		 2200 break;
2201 case KVM_REG_S390_CLOCK_COMP:
2202 r = get_user(vcpu->arch.sie_block->ckc,
2203 (u64 __user *)reg->addr);
2204 break;
536336c2
DD		 2205 case KVM_REG_S390_PFTOKEN:
2206 r = get_user(vcpu->arch.pfault_token,
2207 (u64 __user *)reg->addr);
9fbd8082
DH		 2208 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2209 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 2210 break;
2211 case KVM_REG_S390_PFCOMPARE:
2212 r = get_user(vcpu->arch.pfault_compare,
2213 (u64 __user *)reg->addr);
2214 break;
2215 case KVM_REG_S390_PFSELECT:
2216 r = get_user(vcpu->arch.pfault_select,
2217 (u64 __user *)reg->addr);
2218 break;
672550fb
CB		 2219 case KVM_REG_S390_PP:
2220 r = get_user(vcpu->arch.sie_block->pp,
2221 (u64 __user *)reg->addr);
2222 break;
afa45ff5
CB		 2223 case KVM_REG_S390_GBEA:
2224 r = get_user(vcpu->arch.sie_block->gbea,
2225 (u64 __user *)reg->addr);
2226 break;
14eebd91
CO		 2227 default:
2228 break;
2229 }
2230
2231 return r;
2232}
b6d33834 2233
b0c632db
HC		 2234static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2235{
b0c632db 2236 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 2237 return 0;
2238}
2239
2240int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2241{
5a32c1af 2242 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 2243 return 0;
2244}
2245
2246int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2247{
5a32c1af 2248 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 2249 return 0;
2250}
2251
2252int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2253 struct kvm_sregs *sregs)
2254{
59674c1a 2255 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 2256 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
b0c632db
HC		 2257 return 0;
2258}
2259
2260int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2261 struct kvm_sregs *sregs)
2262{
59674c1a 2263 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 2264 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 2265 return 0;
2266}
2267
2268int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2269{
4725c860
MS		 2270 if (test_fp_ctl(fpu->fpc))
2271 return -EINVAL;
e1788bb9 2272 vcpu->run->s.regs.fpc = fpu->fpc;
9abc2a08 2273 if (MACHINE_HAS_VX)
a7d4b8f2
DH		 2274 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
2275 (freg_t *) fpu->fprs);
9abc2a08 2276 else
a7d4b8f2 2277 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
b0c632db
HC		 2278 return 0;
2279}
2280
2281int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2282{
9abc2a08
DH		 2283 /* make sure we have the latest values */
2284 save_fpu_regs();
2285 if (MACHINE_HAS_VX)
a7d4b8f2
DH		 2286 convert_vx_to_fp((freg_t *) fpu->fprs,
2287 (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08 2288 else
a7d4b8f2 2289 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
e1788bb9 2290 fpu->fpc = vcpu->run->s.regs.fpc;
b0c632db
HC		 2291 return 0;
2292}
2293
2294static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2295{
2296 int rc = 0;
2297
7a42fdc2 2298 if (!is_vcpu_stopped(vcpu))
b0c632db 2299 rc = -EBUSY;
d7b0b5eb
CO		 2300 else {
2301 vcpu->run->psw_mask = psw.mask;
2302 vcpu->run->psw_addr = psw.addr;
2303 }
b0c632db
HC		 2304 return rc;
2305}
2306
2307int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2308 struct kvm_translation *tr)
2309{
2310 return -EINVAL; /* not implemented yet */
2311}
2312
27291e21
DH		 2313#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2314 KVM_GUESTDBG_USE_HW_BP | \
2315 KVM_GUESTDBG_ENABLE)
2316
d0bfb940
JK		 2317int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2318 struct kvm_guest_debug *dbg)
b0c632db 2319{
27291e21
DH		 2320 int rc = 0;
2321
2322 vcpu->guest_debug = 0;
2323 kvm_s390_clear_bp_data(vcpu);
2324
2de3bfc2 2325 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21 2326 return -EINVAL;
89b5b4de
DH		 2327 if (!sclp.has_gpere)
2328 return -EINVAL;
27291e21
DH		 2329
2330 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2331 vcpu->guest_debug = dbg->control;
2332 /* enforce guest PER */
805de8f4 2333 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2334
2335 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2336 rc = kvm_s390_import_bp_data(vcpu, dbg);
2337 } else {
805de8f4 2338 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2339 vcpu->arch.guestdbg.last_bp = 0;
2340 }
2341
2342 if (rc) {
2343 vcpu->guest_debug = 0;
2344 kvm_s390_clear_bp_data(vcpu);
805de8f4 2345 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2346 }
2347
2348 return rc;
b0c632db
HC		 2349}
2350
62d9f0db
MT		 2351int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2352 struct kvm_mp_state *mp_state)
2353{
6352e4d2
DH		 2354 /* CHECK_STOP and LOAD are not supported yet */
2355 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2356 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 2357}
2358
2359int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2360 struct kvm_mp_state *mp_state)
2361{
6352e4d2
DH		 2362 int rc = 0;
2363
2364 /* user space knows about this interface - let it control the state */
2365 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2366
2367 switch (mp_state->mp_state) {
2368 case KVM_MP_STATE_STOPPED:
2369 kvm_s390_vcpu_stop(vcpu);
2370 break;
2371 case KVM_MP_STATE_OPERATING:
2372 kvm_s390_vcpu_start(vcpu);
2373 break;
2374 case KVM_MP_STATE_LOAD:
2375 case KVM_MP_STATE_CHECK_STOP:
2376 /* fall through - CHECK_STOP and LOAD are not supported yet */
2377 default:
2378 rc = -ENXIO;
2379 }
2380
2381 return rc;
62d9f0db
MT		 2382}
2383
8ad35755
DH		 2384static bool ibs_enabled(struct kvm_vcpu *vcpu)
2385{
2386 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2387}
2388
2c70fe44
CB		 2389static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2390{
8ad35755 2391retry:
8e236546 2392 kvm_s390_vcpu_request_handled(vcpu);
586b7ccd
CB		 2393 if (!vcpu->requests)
2394 return 0;
2c70fe44
CB		 2395 /*
2396 * We use MMU_RELOAD just to re-arm the ipte notifier for the
b2d73b2a 2397 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2c70fe44
CB		 2398 * This ensures that the ipte instruction for this request has
2399 * already finished. We might race against a second unmapper that
2400 * wants to set the blocking bit. Lets just retry the request loop.
2401 */
8ad35755 2402 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44 2403 int rc;
b2d73b2a
MS		 2404 rc = gmap_mprotect_notify(vcpu->arch.gmap,
2405 kvm_s390_get_prefix(vcpu),
2406 PAGE_SIZE * 2, PROT_WRITE);
aca411a4
JN		 2407 if (rc) {
2408 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44 2409 return rc;
aca411a4 2410 }
8ad35755 2411 goto retry;
2c70fe44 2412 }
8ad35755 2413
d3d692c8
DH		 2414 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2415 vcpu->arch.sie_block->ihcpu = 0xffff;
2416 goto retry;
2417 }
2418
8ad35755
DH		 2419 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2420 if (!ibs_enabled(vcpu)) {
2421 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
805de8f4 2422 atomic_or(CPUSTAT_IBS,
8ad35755
DH		 2423 &vcpu->arch.sie_block->cpuflags);
2424 }
2425 goto retry;
2c70fe44 2426 }
8ad35755
DH		 2427
2428 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2429 if (ibs_enabled(vcpu)) {
2430 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
805de8f4 2431 atomic_andnot(CPUSTAT_IBS,
8ad35755
DH		 2432 &vcpu->arch.sie_block->cpuflags);
2433 }
2434 goto retry;
2435 }
2436
6502a34c
DH		 2437 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
2438 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2439 goto retry;
2440 }
2441
0759d068
DH		 2442 /* nothing to do, just clear the request */
2443 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2444
2c70fe44
CB		 2445 return 0;
2446}
2447
25ed1675
DH		 2448void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2449{
2450 struct kvm_vcpu *vcpu;
2451 int i;
2452
2453 mutex_lock(&kvm->lock);
2454 preempt_disable();
2455 kvm->arch.epoch = tod - get_tod_clock();
2456 kvm_s390_vcpu_block_all(kvm);
2457 kvm_for_each_vcpu(i, vcpu, kvm)
2458 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2459 kvm_s390_vcpu_unblock_all(kvm);
2460 preempt_enable();
2461 mutex_unlock(&kvm->lock);
2462}
2463
fa576c58
TH		 2464/**
2465 * kvm_arch_fault_in_page - fault-in guest page if necessary
2466 * @vcpu: The corresponding virtual cpu
2467 * @gpa: Guest physical address
2468 * @writable: Whether the page should be writable or not
2469 *
2470 * Make sure that a guest page has been faulted-in on the host.
2471 *
2472 * Return: Zero on success, negative error code otherwise.
2473 */
2474long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 2475{
527e30b4
MS		 2476 return gmap_fault(vcpu->arch.gmap, gpa,
2477 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 2478}
2479
3c038e6b
DD		 2480static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2481 unsigned long token)
2482{
2483 struct kvm_s390_interrupt inti;
383d0b05 2484 struct kvm_s390_irq irq;
3c038e6b
DD		 2485
2486 if (start_token) {
383d0b05
JF		 2487 irq.u.ext.ext_params2 = token;
2488 irq.type = KVM_S390_INT_PFAULT_INIT;
2489 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 2490 } else {
2491 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 2492 inti.parm64 = token;
3c038e6b
DD		 2493 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2494 }
2495}
2496
2497void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2498 struct kvm_async_pf *work)
2499{
2500 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2501 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2502}
2503
2504void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2505 struct kvm_async_pf *work)
2506{
2507 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2508 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2509}
2510
2511void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2512 struct kvm_async_pf *work)
2513{
2514 /* s390 will always inject the page directly */
2515}
2516
2517bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2518{
2519 /*
2520 * s390 will always inject the page directly,
2521 * but we still want check_async_completion to cleanup
2522 */
2523 return true;
2524}
2525
2526static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2527{
2528 hva_t hva;
2529 struct kvm_arch_async_pf arch;
2530 int rc;
2531
2532 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2533 return 0;
2534 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2535 vcpu->arch.pfault_compare)
2536 return 0;
2537 if (psw_extint_disabled(vcpu))
2538 return 0;
9a022067 2539 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 2540 return 0;
2541 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2542 return 0;
2543 if (!vcpu->arch.gmap->pfault_enabled)
2544 return 0;
2545
81480cc1
HC		 2546 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2547 hva += current->thread.gmap_addr & ~PAGE_MASK;
2548 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 2549 return 0;
2550
2551 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2552 return rc;
2553}
2554
3fb4c40f 2555static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 2556{
3fb4c40f 2557 int rc, cpuflags;
e168bf8d 2558
3c038e6b
DD		 2559 /*
2560 * On s390 notifications for arriving pages will be delivered directly
2561 * to the guest but the house keeping for completed pfaults is
2562 * handled outside the worker.
2563 */
2564 kvm_check_async_pf_completion(vcpu);
2565
7ec7c8c7
CB		 2566 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2567 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
b0c632db
HC		 2568
2569 if (need_resched())
2570 schedule();
2571
d3a73acb 2572 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 2573 s390_handle_mcck();
2574
79395031
JF		 2575 if (!kvm_is_ucontrol(vcpu->kvm)) {
2576 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2577 if (rc)
2578 return rc;
2579 }
0ff31867 2580
2c70fe44
CB		 2581 rc = kvm_s390_handle_requests(vcpu);
2582 if (rc)
2583 return rc;
2584
27291e21
DH		 2585 if (guestdbg_enabled(vcpu)) {
2586 kvm_s390_backup_guest_per_regs(vcpu);
2587 kvm_s390_patch_guest_per_regs(vcpu);
2588 }
2589
b0c632db 2590 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 2591 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2592 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2593 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 2594
3fb4c40f
TH		 2595 return 0;
2596}
2597
492d8642
TH		 2598static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2599{
56317920
DH		 2600 struct kvm_s390_pgm_info pgm_info = {
2601 .code = PGM_ADDRESSING,
2602 };
2603 u8 opcode, ilen;
492d8642
TH		 2604 int rc;
2605
2606 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2607 trace_kvm_s390_sie_fault(vcpu);
2608
2609 /*
2610 * We want to inject an addressing exception, which is defined as a
2611 * suppressing or terminating exception. However, since we came here
2612 * by a DAT access exception, the PSW still points to the faulting
2613 * instruction since DAT exceptions are nullifying. So we've got
2614 * to look up the current opcode to get the length of the instruction
2615 * to be able to forward the PSW.
2616 */
3fa8cad7 2617 rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
56317920 2618 ilen = insn_length(opcode);
9b0d721a
DH		 2619 if (rc < 0) {
2620 return rc;
2621 } else if (rc) {
2622 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2623 * Forward by arbitrary ilc, injection will take care of
2624 * nullification if necessary.
2625 */
2626 pgm_info = vcpu->arch.pgm;
2627 ilen = 4;
2628 }
56317920
DH		 2629 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2630 kvm_s390_forward_psw(vcpu, ilen);
2631 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
492d8642
TH		 2632}
2633
3fb4c40f
TH		 2634static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2635{
2b29a9fd
DD		 2636 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2637 vcpu->arch.sie_block->icptcode);
2638 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2639
27291e21
DH		 2640 if (guestdbg_enabled(vcpu))
2641 kvm_s390_restore_guest_per_regs(vcpu);
2642
7ec7c8c7
CB		 2643 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2644 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
71f116bf
DH		 2645
2646 if (vcpu->arch.sie_block->icptcode > 0) {
2647 int rc = kvm_handle_sie_intercept(vcpu);
2648
2649 if (rc != -EOPNOTSUPP)
2650 return rc;
2651 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2652 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2653 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2654 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2655 return -EREMOTE;
2656 } else if (exit_reason != -EFAULT) {
2657 vcpu->stat.exit_null++;
2658 return 0;
210b1607
TH		 2659 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2660 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2661 vcpu->run->s390_ucontrol.trans_exc_code =
2662 current->thread.gmap_addr;
2663 vcpu->run->s390_ucontrol.pgm_code = 0x10;
71f116bf 2664 return -EREMOTE;
24eb3a82 2665 } else if (current->thread.gmap_pfault) {
3c038e6b 2666 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 2667 current->thread.gmap_pfault = 0;
71f116bf
DH		 2668 if (kvm_arch_setup_async_pf(vcpu))
2669 return 0;
2670 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
a76ccff6 2671 }
71f116bf 2672 return vcpu_post_run_fault_in_sie(vcpu);
3fb4c40f
TH		 2673}
2674
2675static int __vcpu_run(struct kvm_vcpu *vcpu)
2676{
2677 int rc, exit_reason;
2678
800c1065
TH		 2679 /*
2680 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2681 * ning the guest), so that memslots (and other stuff) are protected
2682 */
2683 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2684
a76ccff6
TH		 2685 do {
2686 rc = vcpu_pre_run(vcpu);
2687 if (rc)
2688 break;
3fb4c40f 2689
800c1065 2690 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 2691 /*
2692 * As PF_VCPU will be used in fault handler, between
2693 * guest_enter and guest_exit should be no uaccess.
2694 */
0097d12e 2695 local_irq_disable();
6edaa530 2696 guest_enter_irqoff();
db0758b2 2697 __disable_cpu_timer_accounting(vcpu);
0097d12e 2698 local_irq_enable();
a76ccff6
TH		 2699 exit_reason = sie64a(vcpu->arch.sie_block,
2700 vcpu->run->s.regs.gprs);
0097d12e 2701 local_irq_disable();
db0758b2 2702 __enable_cpu_timer_accounting(vcpu);
6edaa530 2703 guest_exit_irqoff();
0097d12e 2704 local_irq_enable();
800c1065 2705 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 2706
2707 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 2708 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 2709
800c1065 2710 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 2711 return rc;
b0c632db
HC		 2712}
2713
b028ee3e
DH		 2714static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2715{
2716 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2717 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2718 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2719 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2720 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2721 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 2722 /* some control register changes require a tlb flush */
2723 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 2724 }
2725 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4287f247 2726 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
b028ee3e
DH		 2727 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2728 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2729 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2730 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2731 }
2732 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2733 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2734 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2735 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 2736 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2737 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e 2738 }
80cd8763
FZ		 2739 /*
2740 * If userspace sets the riccb (e.g. after migration) to a valid state,
2741 * we should enable RI here instead of doing the lazy enablement.
2742 */
2743 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
2744 test_kvm_facility(vcpu->kvm, 64)) {
2745 struct runtime_instr_cb *riccb =
2746 (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
2747
2748 if (riccb->valid)
2749 vcpu->arch.sie_block->ecb3 |= 0x01;
2750 }
31d8b8d4
CB		 2751 save_access_regs(vcpu->arch.host_acrs);
2752 restore_access_regs(vcpu->run->s.regs.acrs);
e1788bb9
CB		 2753 /* save host (userspace) fprs/vrs */
2754 save_fpu_regs();
2755 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
2756 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
2757 if (MACHINE_HAS_VX)
2758 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
2759 else
2760 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
2761 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
2762 if (test_fp_ctl(current->thread.fpu.fpc))
2763 /* User space provided an invalid FPC, let's clear it */
2764 current->thread.fpu.fpc = 0;
80cd8763 2765
b028ee3e
DH		 2766 kvm_run->kvm_dirty_regs = 0;
2767}
2768
2769static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2770{
2771 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2772 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2773 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2774 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4287f247 2775 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
b028ee3e
DH		 2776 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2777 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2778 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2779 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2780 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2781 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2782 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
31d8b8d4
CB		 2783 save_access_regs(vcpu->run->s.regs.acrs);
2784 restore_access_regs(vcpu->arch.host_acrs);
e1788bb9
CB		 2785 /* Save guest register state */
2786 save_fpu_regs();
2787 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
2788 /* Restore will be done lazily at return */
2789 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
2790 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
2791
b028ee3e
DH		 2792}
2793
b0c632db
HC		 2794int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2795{
8f2abe6a 2796 int rc;
b0c632db
HC		 2797 sigset_t sigsaved;
2798
460df4c1
PB		 2799 if (kvm_run->immediate_exit)
2800 return -EINTR;
2801
27291e21
DH		 2802 if (guestdbg_exit_pending(vcpu)) {
2803 kvm_s390_prepare_debug_exit(vcpu);
2804 return 0;
2805 }
2806
b0c632db
HC		 2807 if (vcpu->sigset_active)
2808 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2809
6352e4d2
DH		 2810 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2811 kvm_s390_vcpu_start(vcpu);
2812 } else if (is_vcpu_stopped(vcpu)) {
ea2cdd27 2813 pr_err_ratelimited("can't run stopped vcpu %d\n",
6352e4d2
DH		 2814 vcpu->vcpu_id);
2815 return -EINVAL;
2816 }
b0c632db 2817
b028ee3e 2818 sync_regs(vcpu, kvm_run);
db0758b2 2819 enable_cpu_timer_accounting(vcpu);
d7b0b5eb 2820
dab4079d 2821 might_fault();
a76ccff6 2822 rc = __vcpu_run(vcpu);
9ace903d 2823
b1d16c49
CE		 2824 if (signal_pending(current) && !rc) {
2825 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 2826 rc = -EINTR;
b1d16c49 2827 }
8f2abe6a 2828
27291e21
DH		 2829 if (guestdbg_exit_pending(vcpu) && !rc) {
2830 kvm_s390_prepare_debug_exit(vcpu);
2831 rc = 0;
2832 }
2833
8f2abe6a 2834 if (rc == -EREMOTE) {
71f116bf 2835 /* userspace support is needed, kvm_run has been prepared */
8f2abe6a
CB		 2836 rc = 0;
2837 }
b0c632db 2838
db0758b2 2839 disable_cpu_timer_accounting(vcpu);
b028ee3e 2840 store_regs(vcpu, kvm_run);
d7b0b5eb 2841
b0c632db
HC		 2842 if (vcpu->sigset_active)
2843 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2844
b0c632db 2845 vcpu->stat.exit_userspace++;
7e8e6ab4 2846 return rc;
b0c632db
HC		 2847}
2848
b0c632db
HC		 2849/*
2850 * store status at address
2851 * we use have two special cases:
2852 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2853 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2854 */
d0bce605 2855int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 2856{
092670cd 2857 unsigned char archmode = 1;
9abc2a08 2858 freg_t fprs[NUM_FPRS];
fda902cb 2859 unsigned int px;
4287f247 2860 u64 clkcomp, cputm;
d0bce605 2861 int rc;
b0c632db 2862
d9a3a09a 2863 px = kvm_s390_get_prefix(vcpu);
d0bce605
HC		 2864 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2865 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 2866 return -EFAULT;
d9a3a09a 2867 gpa = 0;
d0bce605
HC		 2868 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2869 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 2870 return -EFAULT;
d9a3a09a
MS		 2871 gpa = px;
2872 } else
2873 gpa -= __LC_FPREGS_SAVE_AREA;
9abc2a08
DH		 2874
2875 /* manually convert vector registers if necessary */
2876 if (MACHINE_HAS_VX) {
9522b37f 2877 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08
DH		 2878 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2879 fprs, 128);
2880 } else {
2881 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
6fd8e67d 2882 vcpu->run->s.regs.fprs, 128);
9abc2a08 2883 }
d9a3a09a 2884 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
d0bce605 2885 vcpu->run->s.regs.gprs, 128);
d9a3a09a 2886 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
d0bce605 2887 &vcpu->arch.sie_block->gpsw, 16);
d9a3a09a 2888 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
fda902cb 2889 &px, 4);
d9a3a09a 2890 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
9abc2a08 2891 &vcpu->run->s.regs.fpc, 4);
d9a3a09a 2892 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
d0bce605 2893 &vcpu->arch.sie_block->todpr, 4);
4287f247 2894 cputm = kvm_s390_get_cpu_timer(vcpu);
d9a3a09a 2895 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4287f247 2896 &cputm, 8);
178bd789 2897 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d9a3a09a 2898 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
d0bce605 2899 &clkcomp, 8);
d9a3a09a 2900 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
d0bce605 2901 &vcpu->run->s.regs.acrs, 64);
d9a3a09a 2902 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
d0bce605
HC		 2903 &vcpu->arch.sie_block->gcr, 128);
2904 return rc ? -EFAULT : 0;
b0c632db
HC		 2905}
2906
e879892c
TH		 2907int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2908{
2909 /*
2910 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
31d8b8d4 2911 * switch in the run ioctl. Let's update our copies before we save
e879892c
TH		 2912 * it into the save area
2913 */
d0164ee2 2914 save_fpu_regs();
9abc2a08 2915 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
e879892c
TH		 2916 save_access_regs(vcpu->run->s.regs.acrs);
2917
2918 return kvm_s390_store_status_unloaded(vcpu, addr);
2919}
2920
8ad35755
DH		 2921static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2922{
2923 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
8e236546 2924 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
8ad35755
DH		 2925}
2926
2927static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2928{
2929 unsigned int i;
2930 struct kvm_vcpu *vcpu;
2931
2932 kvm_for_each_vcpu(i, vcpu, kvm) {
2933 __disable_ibs_on_vcpu(vcpu);
2934 }
2935}
2936
2937static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2938{
09a400e7
DH		 2939 if (!sclp.has_ibs)
2940 return;
8ad35755 2941 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
8e236546 2942 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
8ad35755
DH		 2943}
2944
6852d7b6
DH		 2945void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2946{
8ad35755
DH		 2947 int i, online_vcpus, started_vcpus = 0;
2948
2949 if (!is_vcpu_stopped(vcpu))
2950 return;
2951
6852d7b6 2952 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 2953 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2954 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2955 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2956
2957 for (i = 0; i < online_vcpus; i++) {
2958 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2959 started_vcpus++;
2960 }
2961
2962 if (started_vcpus == 0) {
2963 /* we're the only active VCPU -> speed it up */
2964 __enable_ibs_on_vcpu(vcpu);
2965 } else if (started_vcpus == 1) {
2966 /*
2967 * As we are starting a second VCPU, we have to disable
2968 * the IBS facility on all VCPUs to remove potentially
2969 * oustanding ENABLE requests.
2970 */
2971 __disable_ibs_on_all_vcpus(vcpu->kvm);
2972 }
2973
805de8f4 2974 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2975 /*
2976 * Another VCPU might have used IBS while we were offline.
2977 * Let's play safe and flush the VCPU at startup.
2978 */
d3d692c8 2979 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 2980 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2981 return;
6852d7b6
DH		 2982}
2983
2984void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2985{
8ad35755
DH		 2986 int i, online_vcpus, started_vcpus = 0;
2987 struct kvm_vcpu *started_vcpu = NULL;
2988
2989 if (is_vcpu_stopped(vcpu))
2990 return;
2991
6852d7b6 2992 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 2993 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2994 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2995 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2996
32f5ff63 2997 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 2998 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 2999
805de8f4 3000 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 3001 __disable_ibs_on_vcpu(vcpu);
3002
3003 for (i = 0; i < online_vcpus; i++) {
3004 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
3005 started_vcpus++;
3006 started_vcpu = vcpu->kvm->vcpus[i];
3007 }
3008 }
3009
3010 if (started_vcpus == 1) {
3011 /*
3012 * As we only have one VCPU left, we want to enable the
3013 * IBS facility for that VCPU to speed it up.
3014 */
3015 __enable_ibs_on_vcpu(started_vcpu);
3016 }
3017
433b9ee4 3018 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 3019 return;
6852d7b6
DH		 3020}
3021
d6712df9
CH		 3022static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
3023 struct kvm_enable_cap *cap)
3024{
3025 int r;
3026
3027 if (cap->flags)
3028 return -EINVAL;
3029
3030 switch (cap->cap) {
fa6b7fe9
CH		 3031 case KVM_CAP_S390_CSS_SUPPORT:
3032 if (!vcpu->kvm->arch.css_support) {
3033 vcpu->kvm->arch.css_support = 1;
c92ea7b9 3034 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
fa6b7fe9
CH		 3035 trace_kvm_s390_enable_css(vcpu->kvm);
3036 }
3037 r = 0;
3038 break;
d6712df9
CH		 3039 default:
3040 r = -EINVAL;
3041 break;
3042 }
3043 return r;
3044}
3045
41408c28
TH		 3046static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
3047 struct kvm_s390_mem_op *mop)
3048{
3049 void __user *uaddr = (void __user *)mop->buf;
3050 void *tmpbuf = NULL;
3051 int r, srcu_idx;
3052 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
3053 | KVM_S390_MEMOP_F_CHECK_ONLY;
3054
3055 if (mop->flags & ~supported_flags)
3056 return -EINVAL;
3057
3058 if (mop->size > MEM_OP_MAX_SIZE)
3059 return -E2BIG;
3060
3061 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
3062 tmpbuf = vmalloc(mop->size);
3063 if (!tmpbuf)
3064 return -ENOMEM;
3065 }
3066
3067 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3068
3069 switch (mop->op) {
3070 case KVM_S390_MEMOP_LOGICAL_READ:
3071 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 3072 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3073 mop->size, GACC_FETCH);
41408c28
TH		 3074 break;
3075 }
3076 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3077 if (r == 0) {
3078 if (copy_to_user(uaddr, tmpbuf, mop->size))
3079 r = -EFAULT;
3080 }
3081 break;
3082 case KVM_S390_MEMOP_LOGICAL_WRITE:
3083 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 3084 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3085 mop->size, GACC_STORE);
41408c28
TH		 3086 break;
3087 }
3088 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
3089 r = -EFAULT;
3090 break;
3091 }
3092 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3093 break;
3094 default:
3095 r = -EINVAL;
3096 }
3097
3098 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
3099
3100 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
3101 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
3102
3103 vfree(tmpbuf);
3104 return r;
3105}
3106
b0c632db
HC		 3107long kvm_arch_vcpu_ioctl(struct file *filp,
3108 unsigned int ioctl, unsigned long arg)
3109{
3110 struct kvm_vcpu *vcpu = filp->private_data;
3111 void __user *argp = (void __user *)arg;
800c1065 3112 int idx;
bc923cc9 3113 long r;
b0c632db 3114
93736624 3115 switch (ioctl) {
47b43c52
JF		 3116 case KVM_S390_IRQ: {
3117 struct kvm_s390_irq s390irq;
3118
3119 r = -EFAULT;
3120 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
3121 break;
3122 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3123 break;
3124 }
93736624 3125 case KVM_S390_INTERRUPT: {
ba5c1e9b 3126 struct kvm_s390_interrupt s390int;
383d0b05 3127 struct kvm_s390_irq s390irq;
ba5c1e9b 3128
93736624 3129 r = -EFAULT;
ba5c1e9b 3130 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624 3131 break;
383d0b05
JF		 3132 if (s390int_to_s390irq(&s390int, &s390irq))
3133 return -EINVAL;
3134 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
93736624 3135 break;
ba5c1e9b 3136 }
b0c632db 3137 case KVM_S390_STORE_STATUS:
800c1065 3138 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 3139 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 3140 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 3141 break;
b0c632db
HC		 3142 case KVM_S390_SET_INITIAL_PSW: {
3143 psw_t psw;
3144
bc923cc9 3145 r = -EFAULT;
b0c632db 3146 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 3147 break;
3148 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3149 break;
b0c632db
HC		 3150 }
3151 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 3152 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3153 break;
14eebd91
CO		 3154 case KVM_SET_ONE_REG:
3155 case KVM_GET_ONE_REG: {
3156 struct kvm_one_reg reg;
3157 r = -EFAULT;
3158 if (copy_from_user(&reg, argp, sizeof(reg)))
3159 break;
3160 if (ioctl == KVM_SET_ONE_REG)
3161 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3162 else
3163 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3164 break;
3165 }
27e0393f
CO		 3166#ifdef CONFIG_KVM_S390_UCONTROL
3167 case KVM_S390_UCAS_MAP: {
3168 struct kvm_s390_ucas_mapping ucasmap;
3169
3170 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3171 r = -EFAULT;
3172 break;
3173 }
3174
3175 if (!kvm_is_ucontrol(vcpu->kvm)) {
3176 r = -EINVAL;
3177 break;
3178 }
3179
3180 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3181 ucasmap.vcpu_addr, ucasmap.length);
3182 break;
3183 }
3184 case KVM_S390_UCAS_UNMAP: {
3185 struct kvm_s390_ucas_mapping ucasmap;
3186
3187 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3188 r = -EFAULT;
3189 break;
3190 }
3191
3192 if (!kvm_is_ucontrol(vcpu->kvm)) {
3193 r = -EINVAL;
3194 break;
3195 }
3196
3197 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3198 ucasmap.length);
3199 break;
3200 }
3201#endif
ccc7910f 3202 case KVM_S390_VCPU_FAULT: {
527e30b4 3203 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 3204 break;
3205 }
d6712df9
CH		 3206 case KVM_ENABLE_CAP:
3207 {
3208 struct kvm_enable_cap cap;
3209 r = -EFAULT;
3210 if (copy_from_user(&cap, argp, sizeof(cap)))
3211 break;
3212 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3213 break;
3214 }
41408c28
TH		 3215 case KVM_S390_MEM_OP: {
3216 struct kvm_s390_mem_op mem_op;
3217
3218 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3219 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3220 else
3221 r = -EFAULT;
3222 break;
3223 }
816c7667
JF		 3224 case KVM_S390_SET_IRQ_STATE: {
3225 struct kvm_s390_irq_state irq_state;
3226
3227 r = -EFAULT;
3228 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3229 break;
3230 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3231 irq_state.len == 0 ||
3232 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3233 r = -EINVAL;
3234 break;
3235 }
3236 r = kvm_s390_set_irq_state(vcpu,
3237 (void __user *) irq_state.buf,
3238 irq_state.len);
3239 break;
3240 }
3241 case KVM_S390_GET_IRQ_STATE: {
3242 struct kvm_s390_irq_state irq_state;
3243
3244 r = -EFAULT;
3245 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3246 break;
3247 if (irq_state.len == 0) {
3248 r = -EINVAL;
3249 break;
3250 }
3251 r = kvm_s390_get_irq_state(vcpu,
3252 (__u8 __user *) irq_state.buf,
3253 irq_state.len);
3254 break;
3255 }
b0c632db 3256 default:
3e6afcf1 3257 r = -ENOTTY;
b0c632db 3258 }
bc923cc9 3259 return r;
b0c632db
HC		 3260}
3261
5b1c1493
CO		 3262int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3263{
3264#ifdef CONFIG_KVM_S390_UCONTROL
3265 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3266 && (kvm_is_ucontrol(vcpu->kvm))) {
3267 vmf->page = virt_to_page(vcpu->arch.sie_block);
3268 get_page(vmf->page);
3269 return 0;
3270 }
3271#endif
3272 return VM_FAULT_SIGBUS;
3273}
3274
5587027c
AK		 3275int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3276 unsigned long npages)
db3fe4eb
TY		 3277{
3278 return 0;
3279}
3280
b0c632db 3281/* Section: memory related */
f7784b8e
MT		 3282int kvm_arch_prepare_memory_region(struct kvm *kvm,
3283 struct kvm_memory_slot *memslot,
09170a49 3284 const struct kvm_userspace_memory_region *mem,
7b6195a9 3285 enum kvm_mr_change change)
b0c632db 3286{
dd2887e7
NW		 3287 /* A few sanity checks. We can have memory slots which have to be
3288 located/ended at a segment boundary (1MB). The memory in userland is
3289 ok to be fragmented into various different vmas. It is okay to mmap()
3290 and munmap() stuff in this slot after doing this call at any time */
b0c632db 3291
598841ca 3292 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 3293 return -EINVAL;
3294
598841ca 3295 if (mem->memory_size & 0xffffful)
b0c632db
HC		 3296 return -EINVAL;
3297
a3a92c31
DD		 3298 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3299 return -EINVAL;
3300
f7784b8e
MT		 3301 return 0;
3302}
3303
3304void kvm_arch_commit_memory_region(struct kvm *kvm,
09170a49 3305 const struct kvm_userspace_memory_region *mem,
8482644a 3306 const struct kvm_memory_slot *old,
f36f3f28 3307 const struct kvm_memory_slot *new,
8482644a 3308 enum kvm_mr_change change)
f7784b8e 3309{
f7850c92 3310 int rc;
f7784b8e 3311
2cef4deb
CB		 3312 /* If the basics of the memslot do not change, we do not want
3313 * to update the gmap. Every update causes several unnecessary
3314 * segment translation exceptions. This is usually handled just
3315 * fine by the normal fault handler + gmap, but it will also
3316 * cause faults on the prefix page of running guest CPUs.
3317 */
3318 if (old->userspace_addr == mem->userspace_addr &&
3319 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3320 old->npages * PAGE_SIZE == mem->memory_size)
3321 return;
598841ca
CO		 3322
3323 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3324 mem->guest_phys_addr, mem->memory_size);
3325 if (rc)
ea2cdd27 3326 pr_warn("failed to commit memory region\n");
598841ca 3327 return;
b0c632db
HC		 3328}
3329
60a37709
AY		 3330static inline unsigned long nonhyp_mask(int i)
3331{
3332 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3333
3334 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3335}
3336
3491caf2
CB		 3337void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3338{
3339 vcpu->valid_wakeup = false;
3340}
3341
b0c632db
HC		 3342static int __init kvm_s390_init(void)
3343{
60a37709
AY		 3344 int i;
3345
07197fd0
DH		 3346 if (!sclp.has_sief2) {
3347 pr_info("SIE not available\n");
3348 return -ENODEV;
3349 }
3350
60a37709
AY		 3351 for (i = 0; i < 16; i++)
3352 kvm_s390_fac_list_mask[i] |=
3353 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3354
9d8d5786 3355 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 3356}
3357
3358static void __exit kvm_s390_exit(void)
3359{
3360 kvm_exit();
3361}
3362
3363module_init(kvm_s390_init);
3364module_exit(kvm_s390_exit);
566af940
CH		 3365
3366/*
3367 * Enable autoloading of the kvm module.
3368 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3369 * since x86 takes a different approach.
3370 */
3371#include <linux/miscdevice.h>
3372MODULE_ALIAS_MISCDEV(KVM_MINOR);
3373MODULE_ALIAS("devname:kvm");
Ubuntu Artful kernel mirror