]> git.proxmox.com Git - mirror_ubuntu-focal-kernel.git/blame - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-focal-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
UBUNTU: Ubuntu-5.4.0-117.132
[mirror_ubuntu-focal-kernel.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
d809aa23 1// SPDX-License-Identifier: GPL-2.0
b0c632db 2/*
bb64da9a 3 * hosting IBM Z kernel virtual machines (s390x)
b0c632db 4 *
da7fba15 5 * Copyright IBM Corp. 2008, 2020
b0c632db
HC		 6 *
7 * Author(s): Carsten Otte <cotte@de.ibm.com>
8 * Christian Borntraeger <borntraeger@de.ibm.com>
9 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 10 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 11 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 12 */
13
7aedd9d4
MM		 14#define KMSG_COMPONENT "kvm-s390"
15#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
16
b0c632db
HC		 17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
b2d73b2a 24#include <linux/mman.h>
b0c632db 25#include <linux/module.h>
d3217967 26#include <linux/moduleparam.h>
a374e892 27#include <linux/random.h>
b0c632db 28#include <linux/slab.h>
ba5c1e9b 29#include <linux/timer.h>
41408c28 30#include <linux/vmalloc.h>
15c9705f 31#include <linux/bitmap.h>
174cd4b1 32#include <linux/sched/signal.h>
190df4a2 33#include <linux/string.h>
174cd4b1 34
cbb870c8 35#include <asm/asm-offsets.h>
b0c632db 36#include <asm/lowcore.h>
fd5ada04 37#include <asm/stp.h>
b0c632db 38#include <asm/pgtable.h>
1e133ab2 39#include <asm/gmap.h>
f5daba1d 40#include <asm/nmi.h>
a0616cde 41#include <asm/switch_to.h>
6d3da241 42#include <asm/isc.h>
1526bf9c 43#include <asm/sclp.h>
0a763c78 44#include <asm/cpacf.h>
221bb8a4 45#include <asm/timex.h>
e585b24a 46#include <asm/ap.h>
e2ed2e2d 47#include <asm/uv.h>
8f2abe6a 48#include "kvm-s390.h"
b0c632db
HC		 49#include "gaccess.h"
50
5786fffa
CH		 51#define CREATE_TRACE_POINTS
52#include "trace.h"
ade38c31 53#include "trace-s390.h"
5786fffa 54
41408c28 55#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
816c7667
JF		 56#define LOCAL_IRQS 32
57#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
58 (KVM_MAX_VCPUS + LOCAL_IRQS))
41408c28 59
b0c632db 60#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
ccc40c53 61#define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
b0c632db
HC		 62
63struct kvm_stats_debugfs_item debugfs_entries[] = {
64 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 65 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 66 { "exit_validity", VCPU_STAT(exit_validity) },
67 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
68 { "exit_external_request", VCPU_STAT(exit_external_request) },
a5e0acea 69 { "exit_io_request", VCPU_STAT(exit_io_request) },
8f2abe6a 70 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b 71 { "exit_instruction", VCPU_STAT(exit_instruction) },
9ec6de19 72 { "exit_pei", VCPU_STAT(exit_pei) },
ba5c1e9b
CO		 73 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
74 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
a011eeb2 75 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
f7819512 76 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
62bea5bf 77 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
3491caf2 78 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
8b905d28 79 { "halt_no_poll_steal", VCPU_STAT(halt_no_poll_steal) },
ce2e4f0b 80 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 81 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 82 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 83 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
84 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ccc40c53
CB		 85 { "deliver_ckc", VCPU_STAT(deliver_ckc) },
86 { "deliver_cputm", VCPU_STAT(deliver_cputm) },
ba5c1e9b 87 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 88 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b 89 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
ccc40c53 90 { "deliver_virtio", VCPU_STAT(deliver_virtio) },
ba5c1e9b
CO		 91 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
92 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
93 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
ccc40c53
CB		 94 { "deliver_program", VCPU_STAT(deliver_program) },
95 { "deliver_io", VCPU_STAT(deliver_io) },
32de0749 96 { "deliver_machine_check", VCPU_STAT(deliver_machine_check) },
ba5c1e9b 97 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
ccc40c53
CB		 98 { "inject_ckc", VCPU_STAT(inject_ckc) },
99 { "inject_cputm", VCPU_STAT(inject_cputm) },
100 { "inject_external_call", VCPU_STAT(inject_external_call) },
101 { "inject_float_mchk", VM_STAT(inject_float_mchk) },
102 { "inject_emergency_signal", VCPU_STAT(inject_emergency_signal) },
103 { "inject_io", VM_STAT(inject_io) },
104 { "inject_mchk", VCPU_STAT(inject_mchk) },
105 { "inject_pfault_done", VM_STAT(inject_pfault_done) },
106 { "inject_program", VCPU_STAT(inject_program) },
107 { "inject_restart", VCPU_STAT(inject_restart) },
108 { "inject_service_signal", VM_STAT(inject_service_signal) },
109 { "inject_set_prefix", VCPU_STAT(inject_set_prefix) },
110 { "inject_stop_signal", VCPU_STAT(inject_stop_signal) },
111 { "inject_pfault_init", VCPU_STAT(inject_pfault_init) },
112 { "inject_virtio", VM_STAT(inject_virtio) },
a37cb07a
CB		 113 { "instruction_epsw", VCPU_STAT(instruction_epsw) },
114 { "instruction_gs", VCPU_STAT(instruction_gs) },
115 { "instruction_io_other", VCPU_STAT(instruction_io_other) },
116 { "instruction_lpsw", VCPU_STAT(instruction_lpsw) },
117 { "instruction_lpswe", VCPU_STAT(instruction_lpswe) },
69d0d3a3 118 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
a37cb07a 119 { "instruction_ptff", VCPU_STAT(instruction_ptff) },
453423dc 120 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
a37cb07a
CB		 121 { "instruction_sck", VCPU_STAT(instruction_sck) },
122 { "instruction_sckpf", VCPU_STAT(instruction_sckpf) },
453423dc
CB		 123 { "instruction_spx", VCPU_STAT(instruction_spx) },
124 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
125 { "instruction_stap", VCPU_STAT(instruction_stap) },
a37cb07a
CB		 126 { "instruction_iske", VCPU_STAT(instruction_iske) },
127 { "instruction_ri", VCPU_STAT(instruction_ri) },
128 { "instruction_rrbe", VCPU_STAT(instruction_rrbe) },
129 { "instruction_sske", VCPU_STAT(instruction_sske) },
8a242234 130 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
b31288fa 131 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 132 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
133 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
a37cb07a
CB		 134 { "instruction_tb", VCPU_STAT(instruction_tb) },
135 { "instruction_tpi", VCPU_STAT(instruction_tpi) },
bb25b9ba 136 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
a37cb07a 137 { "instruction_tsch", VCPU_STAT(instruction_tsch) },
95ca2cb5 138 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
a3508fbe 139 { "instruction_sie", VCPU_STAT(instruction_sie) },
5288fbf0 140 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 141 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 142 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 143 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 144 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
145 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 146 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 147 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
148 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 149 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 150 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
151 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
152 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 153 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
154 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
155 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
866c138c
CB		 156 { "instruction_diag_10", VCPU_STAT(diagnose_10) },
157 { "instruction_diag_44", VCPU_STAT(diagnose_44) },
158 { "instruction_diag_9c", VCPU_STAT(diagnose_9c) },
159 { "instruction_diag_258", VCPU_STAT(diagnose_258) },
160 { "instruction_diag_308", VCPU_STAT(diagnose_308) },
161 { "instruction_diag_500", VCPU_STAT(diagnose_500) },
a37cb07a 162 { "instruction_diag_other", VCPU_STAT(diagnose_other) },
b0c632db
HC		 163 { NULL }
164};
165
8fa1696e
CW		 166struct kvm_s390_tod_clock_ext {
167 __u8 epoch_idx;
168 __u64 tod;
169 __u8 reserved[7];
170} __packed;
171
a411edf1
DH		 172/* allow nested virtualization in KVM (if enabled by user space) */
173static int nested;
174module_param(nested, int, S_IRUGO);
175MODULE_PARM_DESC(nested, "Nested virtualization support");
176
a4499382
JF		 177/* allow 1m huge page guest backing, if !nested */
178static int hpage;
179module_param(hpage, int, 0444);
180MODULE_PARM_DESC(hpage, "1m huge page backing support");
b0c632db 181
8b905d28
CB		 182/* maximum percentage of steal time for polling. >100 is treated like 100 */
183static u8 halt_poll_max_steal = 10;
184module_param(halt_poll_max_steal, byte, 0644);
b41fb528 185MODULE_PARM_DESC(halt_poll_max_steal, "Maximum percentage of steal time to allow polling");
8b905d28 186
c3b9e3e1
CB		 187/*
188 * For now we handle at most 16 double words as this is what the s390 base
189 * kernel handles and stores in the prefix page. If we ever need to go beyond
190 * this, this requires changes to code, but the external uapi can stay.
191 */
192#define SIZE_INTERNAL 16
193
194/*
195 * Base feature mask that defines default mask for facilities. Consists of the
196 * defines in FACILITIES_KVM and the non-hypervisor managed bits.
197 */
198static unsigned long kvm_s390_fac_base[SIZE_INTERNAL] = { FACILITIES_KVM };
199/*
200 * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL
201 * and defines the facilities that can be enabled via a cpu model.
202 */
203static unsigned long kvm_s390_fac_ext[SIZE_INTERNAL] = { FACILITIES_KVM_CPUMODEL };
204
205static unsigned long kvm_s390_fac_size(void)
78c4b59f 206{
c3b9e3e1
CB		 207 BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_MASK_SIZE_U64);
208 BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_LIST_SIZE_U64);
209 BUILD_BUG_ON(SIZE_INTERNAL * sizeof(unsigned long) >
210 sizeof(S390_lowcore.stfle_fac_list));
211
212 return SIZE_INTERNAL;
78c4b59f
MM		 213}
214
15c9705f
DH		 215/* available cpu features supported by kvm */
216static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
0a763c78
DH		 217/* available subfunctions indicated via query / "test bit" */
218static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
15c9705f 219
9d8d5786 220static struct gmap_notifier gmap_notifier;
a3508fbe 221static struct gmap_notifier vsie_gmap_notifier;
78f26131 222debug_info_t *kvm_s390_dbf;
da7fba15 223debug_info_t *kvm_s390_dbf_uv;
9d8d5786 224
b0c632db 225/* Section: not file related */
13a34e06 226int kvm_arch_hardware_enable(void)
b0c632db
HC		 227{
228 /* every s390 is virtualization enabled ;-) */
10474ae8 229 return 0;
b0c632db
HC		 230}
231
f257d6dc
SC		 232int kvm_arch_check_processor_compat(void)
233{
234 return 0;
235}
236
e2ed2e2d 237/* forward declarations */
414d3b07
MS		 238static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
239 unsigned long end);
e2ed2e2d 240static int sca_switch_to_extended(struct kvm *kvm);
2c70fe44 241
1575767e
DH		 242static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
243{
244 u8 delta_idx = 0;
245
246 /*
247 * The TOD jumps by delta, we have to compensate this by adding
248 * -delta to the epoch.
249 */
250 delta = -delta;
251
252 /* sign-extension - we're adding to signed values below */
253 if ((s64)delta < 0)
254 delta_idx = -1;
255
256 scb->epoch += delta;
257 if (scb->ecd & ECD_MEF) {
258 scb->epdx += delta_idx;
259 if (scb->epoch < delta)
260 scb->epdx += 1;
261 }
262}
263
fdf03650
FZ		 264/*
265 * This callback is executed during stop_machine(). All CPUs are therefore
266 * temporarily stopped. In order not to change guest behavior, we have to
267 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
268 * so a CPU won't be stopped while calculating with the epoch.
269 */
270static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
271 void *v)
272{
273 struct kvm *kvm;
274 struct kvm_vcpu *vcpu;
275 int i;
276 unsigned long long *delta = v;
277
278 list_for_each_entry(kvm, &vm_list, vm_list) {
fdf03650 279 kvm_for_each_vcpu(i, vcpu, kvm) {
1575767e
DH		 280 kvm_clock_sync_scb(vcpu->arch.sie_block, *delta);
281 if (i == 0) {
282 kvm->arch.epoch = vcpu->arch.sie_block->epoch;
283 kvm->arch.epdx = vcpu->arch.sie_block->epdx;
284 }
db0758b2
DH		 285 if (vcpu->arch.cputm_enabled)
286 vcpu->arch.cputm_start += *delta;
91473b48 287 if (vcpu->arch.vsie_block)
1575767e
DH		 288 kvm_clock_sync_scb(vcpu->arch.vsie_block,
289 *delta);
fdf03650
FZ		 290 }
291 }
292 return NOTIFY_OK;
293}
294
295static struct notifier_block kvm_clock_notifier = {
296 .notifier_call = kvm_clock_sync,
297};
298
b0c632db
HC		 299int kvm_arch_hardware_setup(void)
300{
2c70fe44 301 gmap_notifier.notifier_call = kvm_gmap_notifier;
b2d73b2a 302 gmap_register_pte_notifier(&gmap_notifier);
a3508fbe
DH		 303 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
304 gmap_register_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 305 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
306 &kvm_clock_notifier);
b0c632db
HC		 307 return 0;
308}
309
310void kvm_arch_hardware_unsetup(void)
311{
b2d73b2a 312 gmap_unregister_pte_notifier(&gmap_notifier);
a3508fbe 313 gmap_unregister_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 314 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
315 &kvm_clock_notifier);
b0c632db
HC		 316}
317
22be5a13
DH		 318static void allow_cpu_feat(unsigned long nr)
319{
320 set_bit_inv(nr, kvm_s390_available_cpu_feat);
321}
322
0a763c78
DH		 323static inline int plo_test_bit(unsigned char nr)
324{
78cc563a 325 unsigned long function = (unsigned long)nr | 0x100;
d051ae53 326 int cc;
0a763c78
DH		 327
328 asm volatile(
78cc563a 329 " lgr 0,%[function]\n"
0a763c78
DH		 330 /* Parameter registers are ignored for "test bit" */
331 " plo 0,0,0,0(0)\n"
332 " ipm %0\n"
333 " srl %0,28\n"
334 : "=d" (cc)
78cc563a
HC		 335 : [function] "d" (function)
336 : "cc", "0");
0a763c78
DH		 337 return cc == 0;
338}
339
d0dea733 340static __always_inline void __insn32_query(unsigned int opcode, u8 *query)
d6681397 341{
d6681397 342 asm volatile(
78cc563a
HC		 343 " lghi 0,0\n"
344 " lgr 1,%[query]\n"
345 /* Parameter registers are ignored */
d6681397 346 " .insn rrf,%[opc] << 16,2,4,6,0\n"
b1c41ac3 347 :
78cc563a
HC		 348 : [query] "d" ((unsigned long)query), [opc] "i" (opcode)
349 : "cc", "memory", "0", "1");
d6681397
CB		 350}
351
173aec2d 352#define INSN_SORTL 0xb938
4f45b90e 353#define INSN_DFLTCC 0xb939
173aec2d 354
22be5a13
DH		 355static void kvm_s390_cpu_feat_init(void)
356{
0a763c78
DH		 357 int i;
358
359 for (i = 0; i < 256; ++i) {
360 if (plo_test_bit(i))
361 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
362 }
363
364 if (test_facility(28)) /* TOD-clock steering */
221bb8a4
LT		 365 ptff(kvm_s390_available_subfunc.ptff,
366 sizeof(kvm_s390_available_subfunc.ptff),
367 PTFF_QAF);
0a763c78
DH		 368
369 if (test_facility(17)) { /* MSA */
69c0e360
MS		 370 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
371 kvm_s390_available_subfunc.kmac);
372 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
373 kvm_s390_available_subfunc.kmc);
374 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
375 kvm_s390_available_subfunc.km);
376 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
377 kvm_s390_available_subfunc.kimd);
378 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
379 kvm_s390_available_subfunc.klmd);
0a763c78
DH		 380 }
381 if (test_facility(76)) /* MSA3 */
69c0e360
MS		 382 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
383 kvm_s390_available_subfunc.pckmo);
0a763c78 384 if (test_facility(77)) { /* MSA4 */
69c0e360
MS		 385 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
386 kvm_s390_available_subfunc.kmctr);
387 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
388 kvm_s390_available_subfunc.kmf);
389 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
390 kvm_s390_available_subfunc.kmo);
391 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
392 kvm_s390_available_subfunc.pcc);
0a763c78
DH		 393 }
394 if (test_facility(57)) /* MSA5 */
985a9d20 395 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
69c0e360 396 kvm_s390_available_subfunc.ppno);
0a763c78 397
e000b8e0
JH		 398 if (test_facility(146)) /* MSA8 */
399 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
400 kvm_s390_available_subfunc.kma);
401
13209ad0
CB		 402 if (test_facility(155)) /* MSA9 */
403 __cpacf_query(CPACF_KDSA, (cpacf_mask_t *)
404 kvm_s390_available_subfunc.kdsa);
405
173aec2d
CB		 406 if (test_facility(150)) /* SORTL */
407 __insn32_query(INSN_SORTL, kvm_s390_available_subfunc.sortl);
408
4f45b90e
CB		 409 if (test_facility(151)) /* DFLTCC */
410 __insn32_query(INSN_DFLTCC, kvm_s390_available_subfunc.dfltcc);
411
22be5a13
DH		 412 if (MACHINE_HAS_ESOP)
413 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
a3508fbe
DH		 414 /*
415 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
416 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
417 */
418 if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
a411edf1 419 !test_facility(3) || !nested)
a3508fbe
DH		 420 return;
421 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
19c439b5
DH		 422 if (sclp.has_64bscao)
423 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
0615a326
DH		 424 if (sclp.has_siif)
425 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
77d18f6d
DH		 426 if (sclp.has_gpere)
427 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
a1b7b9b2
DH		 428 if (sclp.has_gsls)
429 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
5630a8e8
DH		 430 if (sclp.has_ib)
431 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
13ee3f67
DH		 432 if (sclp.has_cei)
433 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
7fd7f39d
DH		 434 if (sclp.has_ibs)
435 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
730cd632
FA		 436 if (sclp.has_kss)
437 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
5d3876a8
DH		 438 /*
439 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
440 * all skey handling functions read/set the skey from the PGSTE
441 * instead of the real storage key.
442 *
443 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
444 * pages being detected as preserved although they are resident.
445 *
446 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
447 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
448 *
449 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
450 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
451 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
452 *
453 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
454 * cannot easily shadow the SCA because of the ipte lock.
455 */
22be5a13
DH		 456}
457
b0c632db
HC		 458int kvm_arch_init(void *opaque)
459{
5045428d 460 int rc = -ENOMEM;
308c3e66 461
78f26131
CB		 462 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
463 if (!kvm_s390_dbf)
464 return -ENOMEM;
465
da7fba15
JF		 466 kvm_s390_dbf_uv = debug_register("kvm-uv", 32, 1, 7 * sizeof(long));
467 if (!kvm_s390_dbf_uv)
468 goto out;
469
470 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view) ||
471 debug_register_view(kvm_s390_dbf_uv, &debug_sprintf_view))
5045428d 472 goto out;
78f26131 473
22be5a13
DH		 474 kvm_s390_cpu_feat_init();
475
84877d93 476 /* Register floating interrupt controller interface. */
308c3e66
MM		 477 rc = kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
478 if (rc) {
8d43d570 479 pr_err("A FLIC registration call failed with rc=%d\n", rc);
5045428d 480 goto out;
308c3e66 481 }
b1d1e76e
MM		 482
483 rc = kvm_s390_gib_init(GAL_ISC);
484 if (rc)
5045428d 485 goto out;
b1d1e76e 486
308c3e66
MM		 487 return 0;
488
5045428d
JF		 489out:
490 kvm_arch_exit();
308c3e66 491 return rc;
b0c632db
HC		 492}
493
78f26131
CB		 494void kvm_arch_exit(void)
495{
1282c21e 496 kvm_s390_gib_destroy();
78f26131 497 debug_unregister(kvm_s390_dbf);
da7fba15 498 debug_unregister(kvm_s390_dbf_uv);
78f26131
CB		 499}
500
b0c632db
HC		 501/* Section: device related */
502long kvm_arch_dev_ioctl(struct file *filp,
503 unsigned int ioctl, unsigned long arg)
504{
505 if (ioctl == KVM_S390_ENABLE_SIE)
506 return s390_enable_sie();
507 return -EINVAL;
508}
509
784aa3d7 510int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 511{
d7b0b5eb
CO		 512 int r;
513
2bd0ac4e 514 switch (ext) {
d7b0b5eb 515 case KVM_CAP_S390_PSW:
b6cf8788 516 case KVM_CAP_S390_GMAP:
52e16b18 517 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 518#ifdef CONFIG_KVM_S390_UCONTROL
519 case KVM_CAP_S390_UCONTROL:
520#endif
3c038e6b 521 case KVM_CAP_ASYNC_PF:
60b413c9 522 case KVM_CAP_SYNC_REGS:
14eebd91 523 case KVM_CAP_ONE_REG:
d6712df9 524 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 525 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 526 case KVM_CAP_IOEVENTFD:
c05c4186 527 case KVM_CAP_DEVICE_CTRL:
78599d90 528 case KVM_CAP_S390_IRQCHIP:
f2061656 529 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 530 case KVM_CAP_MP_STATE:
460df4c1 531 case KVM_CAP_IMMEDIATE_EXIT:
47b43c52 532 case KVM_CAP_S390_INJECT_IRQ:
2444b352 533 case KVM_CAP_S390_USER_SIGP:
e44fc8c9 534 case KVM_CAP_S390_USER_STSI:
30ee2a98 535 case KVM_CAP_S390_SKEYS:
816c7667 536 case KVM_CAP_S390_IRQ_STATE:
6502a34c 537 case KVM_CAP_S390_USER_INSTR0:
4036e387 538 case KVM_CAP_S390_CMMA_MIGRATION:
47a4693e 539 case KVM_CAP_S390_AIS:
da9a1446 540 case KVM_CAP_S390_AIS_MIGRATION:
68369234 541 case KVM_CAP_S390_VCPU_RESETS:
dad38237 542 case KVM_CAP_S390_DIAG318:
05b2b2e9 543 case KVM_CAP_S390_MEM_OP_EXTENSION:
d7b0b5eb
CO		 544 r = 1;
545 break;
a4499382
JF		 546 case KVM_CAP_S390_HPAGE_1M:
547 r = 0;
40ebdb8e 548 if (hpage && !kvm_is_ucontrol(kvm))
a4499382
JF		 549 r = 1;
550 break;
41408c28
TH		 551 case KVM_CAP_S390_MEM_OP:
552 r = MEM_OP_MAX_SIZE;
553 break;
e726b1bd
CB		 554 case KVM_CAP_NR_VCPUS:
555 case KVM_CAP_MAX_VCPUS:
a86cb413 556 case KVM_CAP_MAX_VCPU_ID:
76a6dd72 557 r = KVM_S390_BSCA_CPU_SLOTS;
a6940674
DH		 558 if (!kvm_s390_use_sca_entries())
559 r = KVM_MAX_VCPUS;
560 else if (sclp.has_esca && sclp.has_64bscao)
76a6dd72 561 r = KVM_S390_ESCA_CPU_SLOTS;
e726b1bd 562 break;
1526bf9c 563 case KVM_CAP_S390_COW:
abf09bed 564 r = MACHINE_HAS_ESOP;
1526bf9c 565 break;
68c55750
EF		 566 case KVM_CAP_S390_VECTOR_REGISTERS:
567 r = MACHINE_HAS_VX;
568 break;
c6e5f166
FZ		 569 case KVM_CAP_S390_RI:
570 r = test_facility(64);
571 break;
4e0b1ab7
FZ		 572 case KVM_CAP_S390_GS:
573 r = test_facility(133);
574 break;
35b3fde6
CB		 575 case KVM_CAP_S390_BPB:
576 r = test_facility(82);
577 break;
06abdc74
CB		 578 case KVM_CAP_S390_PROTECTED:
579 r = is_prot_virt_host();
580 break;
2bd0ac4e 581 default:
d7b0b5eb 582 r = 0;
2bd0ac4e 583 }
d7b0b5eb 584 return r;
b0c632db
HC		 585}
586
15f36ebd 587static void kvm_s390_sync_dirty_log(struct kvm *kvm,
0959e168 588 struct kvm_memory_slot *memslot)
15f36ebd 589{
0959e168 590 int i;
15f36ebd 591 gfn_t cur_gfn, last_gfn;
0959e168 592 unsigned long gaddr, vmaddr;
15f36ebd 593 struct gmap *gmap = kvm->arch.gmap;
0959e168 594 DECLARE_BITMAP(bitmap, _PAGE_ENTRIES);
15f36ebd 595
0959e168
JF		 596 /* Loop over all guest segments */
597 cur_gfn = memslot->base_gfn;
15f36ebd 598 last_gfn = memslot->base_gfn + memslot->npages;
0959e168
JF		 599 for (; cur_gfn <= last_gfn; cur_gfn += _PAGE_ENTRIES) {
600 gaddr = gfn_to_gpa(cur_gfn);
601 vmaddr = gfn_to_hva_memslot(memslot, cur_gfn);
602 if (kvm_is_error_hva(vmaddr))
603 continue;
604
605 bitmap_zero(bitmap, _PAGE_ENTRIES);
606 gmap_sync_dirty_log_pmd(gmap, bitmap, gaddr, vmaddr);
607 for (i = 0; i < _PAGE_ENTRIES; i++) {
608 if (test_bit(i, bitmap))
609 mark_page_dirty(kvm, cur_gfn + i);
610 }
15f36ebd 611
1763f8d0
CB		 612 if (fatal_signal_pending(current))
613 return;
70c88a00 614 cond_resched();
15f36ebd 615 }
15f36ebd
JH		 616}
617
b0c632db 618/* Section: vm related */
a6e2f683
ED		 619static void sca_del_vcpu(struct kvm_vcpu *vcpu);
620
b0c632db
HC		 621/*
622 * Get (and clear) the dirty memory log for a memory slot.
623 */
624int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
625 struct kvm_dirty_log *log)
626{
15f36ebd
JH		 627 int r;
628 unsigned long n;
9f6b8029 629 struct kvm_memslots *slots;
15f36ebd
JH		 630 struct kvm_memory_slot *memslot;
631 int is_dirty = 0;
632
e1e8a962
JF		 633 if (kvm_is_ucontrol(kvm))
634 return -EINVAL;
635
15f36ebd
JH		 636 mutex_lock(&kvm->slots_lock);
637
638 r = -EINVAL;
639 if (log->slot >= KVM_USER_MEM_SLOTS)
640 goto out;
641
9f6b8029
PB		 642 slots = kvm_memslots(kvm);
643 memslot = id_to_memslot(slots, log->slot);
15f36ebd
JH		 644 r = -ENOENT;
645 if (!memslot->dirty_bitmap)
646 goto out;
647
648 kvm_s390_sync_dirty_log(kvm, memslot);
649 r = kvm_get_dirty_log(kvm, log, &is_dirty);
650 if (r)
651 goto out;
652
653 /* Clear the dirty log */
654 if (is_dirty) {
655 n = kvm_dirty_bitmap_bytes(memslot);
656 memset(memslot->dirty_bitmap, 0, n);
657 }
658 r = 0;
659out:
660 mutex_unlock(&kvm->slots_lock);
661 return r;
b0c632db
HC		 662}
663
6502a34c
DH		 664static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
665{
666 unsigned int i;
667 struct kvm_vcpu *vcpu;
668
669 kvm_for_each_vcpu(i, vcpu, kvm) {
670 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
671 }
672}
673
e5d83c74 674int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
d938dc55
CH		 675{
676 int r;
677
678 if (cap->flags)
679 return -EINVAL;
680
681 switch (cap->cap) {
84223598 682 case KVM_CAP_S390_IRQCHIP:
c92ea7b9 683 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
84223598
CH		 684 kvm->arch.use_irqchip = 1;
685 r = 0;
686 break;
2444b352 687 case KVM_CAP_S390_USER_SIGP:
c92ea7b9 688 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
2444b352
DH		 689 kvm->arch.user_sigp = 1;
690 r = 0;
691 break;
68c55750 692 case KVM_CAP_S390_VECTOR_REGISTERS:
5967c17b 693 mutex_lock(&kvm->lock);
a03825bb 694 if (kvm->created_vcpus) {
5967c17b
DH		 695 r = -EBUSY;
696 } else if (MACHINE_HAS_VX) {
c54f0d6a
DH		 697 set_kvm_facility(kvm->arch.model.fac_mask, 129);
698 set_kvm_facility(kvm->arch.model.fac_list, 129);
2f87d942
GH		 699 if (test_facility(134)) {
700 set_kvm_facility(kvm->arch.model.fac_mask, 134);
701 set_kvm_facility(kvm->arch.model.fac_list, 134);
702 }
53743aa7
MS		 703 if (test_facility(135)) {
704 set_kvm_facility(kvm->arch.model.fac_mask, 135);
705 set_kvm_facility(kvm->arch.model.fac_list, 135);
706 }
7832e91c
CB		 707 if (test_facility(148)) {
708 set_kvm_facility(kvm->arch.model.fac_mask, 148);
709 set_kvm_facility(kvm->arch.model.fac_list, 148);
710 }
d5cb6ab1
CB		 711 if (test_facility(152)) {
712 set_kvm_facility(kvm->arch.model.fac_mask, 152);
713 set_kvm_facility(kvm->arch.model.fac_list, 152);
714 }
18280d8b
MM		 715 r = 0;
716 } else
717 r = -EINVAL;
5967c17b 718 mutex_unlock(&kvm->lock);
c92ea7b9
CB		 719 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
720 r ? "(not available)" : "(success)");
68c55750 721 break;
c6e5f166
FZ		 722 case KVM_CAP_S390_RI:
723 r = -EINVAL;
724 mutex_lock(&kvm->lock);
a03825bb 725 if (kvm->created_vcpus) {
c6e5f166
FZ		 726 r = -EBUSY;
727 } else if (test_facility(64)) {
c54f0d6a
DH		 728 set_kvm_facility(kvm->arch.model.fac_mask, 64);
729 set_kvm_facility(kvm->arch.model.fac_list, 64);
c6e5f166
FZ		 730 r = 0;
731 }
732 mutex_unlock(&kvm->lock);
733 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
734 r ? "(not available)" : "(success)");
735 break;
47a4693e
YMZ		 736 case KVM_CAP_S390_AIS:
737 mutex_lock(&kvm->lock);
738 if (kvm->created_vcpus) {
739 r = -EBUSY;
740 } else {
741 set_kvm_facility(kvm->arch.model.fac_mask, 72);
742 set_kvm_facility(kvm->arch.model.fac_list, 72);
47a4693e
YMZ		 743 r = 0;
744 }
745 mutex_unlock(&kvm->lock);
746 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
747 r ? "(not available)" : "(success)");
748 break;
4e0b1ab7
FZ		 749 case KVM_CAP_S390_GS:
750 r = -EINVAL;
751 mutex_lock(&kvm->lock);
241e3ec0 752 if (kvm->created_vcpus) {
4e0b1ab7
FZ		 753 r = -EBUSY;
754 } else if (test_facility(133)) {
755 set_kvm_facility(kvm->arch.model.fac_mask, 133);
756 set_kvm_facility(kvm->arch.model.fac_list, 133);
757 r = 0;
758 }
759 mutex_unlock(&kvm->lock);
760 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
761 r ? "(not available)" : "(success)");
762 break;
a4499382
JF		 763 case KVM_CAP_S390_HPAGE_1M:
764 mutex_lock(&kvm->lock);
765 if (kvm->created_vcpus)
766 r = -EBUSY;
40ebdb8e 767 else if (!hpage || kvm->arch.use_cmma || kvm_is_ucontrol(kvm))
a4499382
JF		 768 r = -EINVAL;
769 else {
770 r = 0;
df88f318 771 down_write(&kvm->mm->mmap_sem);
a4499382 772 kvm->mm->context.allow_gmap_hpage_1m = 1;
df88f318 773 up_write(&kvm->mm->mmap_sem);
a4499382
JF		 774 /*
775 * We might have to create fake 4k page
776 * tables. To avoid that the hardware works on
777 * stale PGSTEs, we emulate these instructions.
778 */
779 kvm->arch.use_skf = 0;
780 kvm->arch.use_pfmfi = 0;
781 }
782 mutex_unlock(&kvm->lock);
783 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s",
784 r ? "(not available)" : "(success)");
785 break;
e44fc8c9 786 case KVM_CAP_S390_USER_STSI:
c92ea7b9 787 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
e44fc8c9
ET		 788 kvm->arch.user_stsi = 1;
789 r = 0;
790 break;
6502a34c
DH		 791 case KVM_CAP_S390_USER_INSTR0:
792 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
793 kvm->arch.user_instr0 = 1;
794 icpt_operexc_on_all_vcpus(kvm);
795 r = 0;
796 break;
d938dc55
CH		 797 default:
798 r = -EINVAL;
799 break;
800 }
801 return r;
802}
803
8c0a7ce6
DD		 804static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
805{
806 int ret;
807
808 switch (attr->attr) {
809 case KVM_S390_VM_MEM_LIMIT_SIZE:
810 ret = 0;
c92ea7b9 811 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
a3a92c31
DD		 812 kvm->arch.mem_limit);
813 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
8c0a7ce6
DD		 814 ret = -EFAULT;
815 break;
816 default:
817 ret = -ENXIO;
818 break;
819 }
820 return ret;
821}
822
823static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 824{
825 int ret;
826 unsigned int idx;
827 switch (attr->attr) {
828 case KVM_S390_VM_MEM_ENABLE_CMMA:
f9cbd9b0 829 ret = -ENXIO;
c24cc9c8 830 if (!sclp.has_cmma)
e6db1d61
DD		 831 break;
832
c92ea7b9 833 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
4f718eab 834 mutex_lock(&kvm->lock);
a4499382
JF		 835 if (kvm->created_vcpus)
836 ret = -EBUSY;
837 else if (kvm->mm->context.allow_gmap_hpage_1m)
838 ret = -EINVAL;
839 else {
4f718eab 840 kvm->arch.use_cmma = 1;
c9f0a2b8
JF		 841 /* Not compatible with cmma. */
842 kvm->arch.use_pfmfi = 0;
4f718eab
DD		 843 ret = 0;
844 }
845 mutex_unlock(&kvm->lock);
846 break;
847 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 848 ret = -ENXIO;
849 if (!sclp.has_cmma)
850 break;
c3489155
DD		 851 ret = -EINVAL;
852 if (!kvm->arch.use_cmma)
853 break;
854
c92ea7b9 855 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
4f718eab
DD		 856 mutex_lock(&kvm->lock);
857 idx = srcu_read_lock(&kvm->srcu);
a13cff31 858 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 859 srcu_read_unlock(&kvm->srcu, idx);
860 mutex_unlock(&kvm->lock);
861 ret = 0;
862 break;
8c0a7ce6
DD		 863 case KVM_S390_VM_MEM_LIMIT_SIZE: {
864 unsigned long new_limit;
865
866 if (kvm_is_ucontrol(kvm))
867 return -EINVAL;
868
869 if (get_user(new_limit, (u64 __user *)attr->addr))
870 return -EFAULT;
871
a3a92c31
DD		 872 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
873 new_limit > kvm->arch.mem_limit)
8c0a7ce6
DD		 874 return -E2BIG;
875
a3a92c31
DD		 876 if (!new_limit)
877 return -EINVAL;
878
6ea427bb 879 /* gmap_create takes last usable address */
a3a92c31
DD		 880 if (new_limit != KVM_S390_NO_MEM_LIMIT)
881 new_limit -= 1;
882
8c0a7ce6
DD		 883 ret = -EBUSY;
884 mutex_lock(&kvm->lock);
a03825bb 885 if (!kvm->created_vcpus) {
6ea427bb
MS		 886 /* gmap_create will round the limit up */
887 struct gmap *new = gmap_create(current->mm, new_limit);
8c0a7ce6
DD		 888
889 if (!new) {
890 ret = -ENOMEM;
891 } else {
6ea427bb 892 gmap_remove(kvm->arch.gmap);
8c0a7ce6
DD		 893 new->private = kvm;
894 kvm->arch.gmap = new;
895 ret = 0;
896 }
897 }
898 mutex_unlock(&kvm->lock);
a3a92c31
DD		 899 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
900 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
901 (void *) kvm->arch.gmap->asce);
8c0a7ce6
DD		 902 break;
903 }
4f718eab
DD		 904 default:
905 ret = -ENXIO;
906 break;
907 }
908 return ret;
909}
910
a374e892
TK		 911static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
912
20c922f0 913void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
a374e892
TK		 914{
915 struct kvm_vcpu *vcpu;
916 int i;
917
20c922f0
TK		 918 kvm_s390_vcpu_block_all(kvm);
919
3194cdb7 920 kvm_for_each_vcpu(i, vcpu, kvm) {
20c922f0 921 kvm_s390_vcpu_crypto_setup(vcpu);
3194cdb7
DH		 922 /* recreate the shadow crycb by leaving the VSIE handler */
923 kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
924 }
20c922f0
TK		 925
926 kvm_s390_vcpu_unblock_all(kvm);
927}
928
929static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
930{
a374e892
TK		 931 mutex_lock(&kvm->lock);
932 switch (attr->attr) {
933 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
8e41bd54
CB		 934 if (!test_kvm_facility(kvm, 76)) {
935 mutex_unlock(&kvm->lock);
37940fb0 936 return -EINVAL;
8e41bd54 937 }
a374e892
TK		 938 get_random_bytes(
939 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
940 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
941 kvm->arch.crypto.aes_kw = 1;
c92ea7b9 942 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
a374e892
TK		 943 break;
944 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
8e41bd54
CB		 945 if (!test_kvm_facility(kvm, 76)) {
946 mutex_unlock(&kvm->lock);
37940fb0 947 return -EINVAL;
8e41bd54 948 }
a374e892
TK		 949 get_random_bytes(
950 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
951 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
952 kvm->arch.crypto.dea_kw = 1;
c92ea7b9 953 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
a374e892
TK		 954 break;
955 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
8e41bd54
CB		 956 if (!test_kvm_facility(kvm, 76)) {
957 mutex_unlock(&kvm->lock);
37940fb0 958 return -EINVAL;
8e41bd54 959 }
a374e892
TK		 960 kvm->arch.crypto.aes_kw = 0;
961 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
962 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
c92ea7b9 963 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
a374e892
TK		 964 break;
965 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
8e41bd54
CB		 966 if (!test_kvm_facility(kvm, 76)) {
967 mutex_unlock(&kvm->lock);
37940fb0 968 return -EINVAL;
8e41bd54 969 }
a374e892
TK		 970 kvm->arch.crypto.dea_kw = 0;
971 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
972 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
c92ea7b9 973 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
a374e892 974 break;
37940fb0
TK		 975 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
976 if (!ap_instructions_available()) {
977 mutex_unlock(&kvm->lock);
978 return -EOPNOTSUPP;
979 }
980 kvm->arch.crypto.apie = 1;
981 break;
982 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
983 if (!ap_instructions_available()) {
984 mutex_unlock(&kvm->lock);
985 return -EOPNOTSUPP;
986 }
987 kvm->arch.crypto.apie = 0;
988 break;
a374e892
TK		 989 default:
990 mutex_unlock(&kvm->lock);
991 return -ENXIO;
992 }
993
20c922f0 994 kvm_s390_vcpu_crypto_reset_all(kvm);
a374e892
TK		 995 mutex_unlock(&kvm->lock);
996 return 0;
997}
998
190df4a2
CI		 999static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
1000{
1001 int cx;
1002 struct kvm_vcpu *vcpu;
1003
1004 kvm_for_each_vcpu(cx, vcpu, kvm)
1005 kvm_s390_sync_request(req, vcpu);
1006}
1007
1008/*
1009 * Must be called with kvm->srcu held to avoid races on memslots, and with
1de1ea7e 1010 * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
190df4a2
CI		 1011 */
1012static int kvm_s390_vm_start_migration(struct kvm *kvm)
1013{
190df4a2 1014 struct kvm_memory_slot *ms;
190df4a2 1015 struct kvm_memslots *slots;
afdad616 1016 unsigned long ram_pages = 0;
190df4a2
CI		 1017 int slotnr;
1018
1019 /* migration mode already enabled */
afdad616 1020 if (kvm->arch.migration_mode)
190df4a2 1021 return 0;
190df4a2
CI		 1022 slots = kvm_memslots(kvm);
1023 if (!slots || !slots->used_slots)
1024 return -EINVAL;
1025
afdad616
CI		 1026 if (!kvm->arch.use_cmma) {
1027 kvm->arch.migration_mode = 1;
1028 return 0;
1029 }
1030 /* mark all the pages in active slots as dirty */
1031 for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
1032 ms = slots->memslots + slotnr;
13a17cc0
IM		 1033 if (!ms->dirty_bitmap)
1034 return -EINVAL;
190df4a2 1035 /*
afdad616
CI		 1036 * The second half of the bitmap is only used on x86,
1037 * and would be wasted otherwise, so we put it to good
1038 * use here to keep track of the state of the storage
1039 * attributes.
190df4a2 1040 */
afdad616
CI		 1041 memset(kvm_second_dirty_bitmap(ms), 0xff, kvm_dirty_bitmap_bytes(ms));
1042 ram_pages += ms->npages;
190df4a2 1043 }
afdad616
CI		 1044 atomic64_set(&kvm->arch.cmma_dirty_pages, ram_pages);
1045 kvm->arch.migration_mode = 1;
1046 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
190df4a2
CI		 1047 return 0;
1048}
1049
1050/*
1de1ea7e 1051 * Must be called with kvm->slots_lock to avoid races with ourselves and
190df4a2
CI		 1052 * kvm_s390_vm_start_migration.
1053 */
1054static int kvm_s390_vm_stop_migration(struct kvm *kvm)
1055{
190df4a2 1056 /* migration mode already disabled */
afdad616 1057 if (!kvm->arch.migration_mode)
190df4a2 1058 return 0;
afdad616
CI		 1059 kvm->arch.migration_mode = 0;
1060 if (kvm->arch.use_cmma)
190df4a2 1061 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
190df4a2
CI		 1062 return 0;
1063}
1064
1065static int kvm_s390_vm_set_migration(struct kvm *kvm,
1066 struct kvm_device_attr *attr)
1067{
1de1ea7e 1068 int res = -ENXIO;
190df4a2 1069
1de1ea7e 1070 mutex_lock(&kvm->slots_lock);
190df4a2
CI		 1071 switch (attr->attr) {
1072 case KVM_S390_VM_MIGRATION_START:
190df4a2 1073 res = kvm_s390_vm_start_migration(kvm);
190df4a2
CI		 1074 break;
1075 case KVM_S390_VM_MIGRATION_STOP:
1076 res = kvm_s390_vm_stop_migration(kvm);
1077 break;
1078 default:
1079 break;
1080 }
1de1ea7e 1081 mutex_unlock(&kvm->slots_lock);
190df4a2
CI		 1082
1083 return res;
1084}
1085
1086static int kvm_s390_vm_get_migration(struct kvm *kvm,
1087 struct kvm_device_attr *attr)
1088{
afdad616 1089 u64 mig = kvm->arch.migration_mode;
190df4a2
CI		 1090
1091 if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
1092 return -ENXIO;
1093
1094 if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
1095 return -EFAULT;
1096 return 0;
1097}
1098
8fa1696e
CW		 1099static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1100{
1101 struct kvm_s390_vm_tod_clock gtod;
1102
1103 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
1104 return -EFAULT;
1105
0e7def5f 1106 if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
8fa1696e 1107 return -EINVAL;
0e7def5f 1108 kvm_s390_set_tod_clock(kvm, &gtod);
8fa1696e
CW		 1109
1110 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
1111 gtod.epoch_idx, gtod.tod);
1112
1113 return 0;
1114}
1115
72f25020
JH		 1116static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1117{
1118 u8 gtod_high;
1119
1120 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
1121 sizeof(gtod_high)))
1122 return -EFAULT;
1123
1124 if (gtod_high != 0)
1125 return -EINVAL;
58c383c6 1126 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
72f25020
JH		 1127
1128 return 0;
1129}
1130
1131static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1132{
0e7def5f 1133 struct kvm_s390_vm_tod_clock gtod = { 0 };
72f25020 1134
0e7def5f
DH		 1135 if (copy_from_user(&gtod.tod, (void __user *)attr->addr,
1136 sizeof(gtod.tod)))
72f25020
JH		 1137 return -EFAULT;
1138
0e7def5f
DH		 1139 kvm_s390_set_tod_clock(kvm, &gtod);
1140 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
72f25020
JH		 1141 return 0;
1142}
1143
1144static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1145{
1146 int ret;
1147
1148 if (attr->flags)
1149 return -EINVAL;
1150
1151 switch (attr->attr) {
8fa1696e
CW		 1152 case KVM_S390_VM_TOD_EXT:
1153 ret = kvm_s390_set_tod_ext(kvm, attr);
1154 break;
72f25020
JH		 1155 case KVM_S390_VM_TOD_HIGH:
1156 ret = kvm_s390_set_tod_high(kvm, attr);
1157 break;
1158 case KVM_S390_VM_TOD_LOW:
1159 ret = kvm_s390_set_tod_low(kvm, attr);
1160 break;
1161 default:
1162 ret = -ENXIO;
1163 break;
1164 }
1165 return ret;
1166}
1167
33d1b272
DH		 1168static void kvm_s390_get_tod_clock(struct kvm *kvm,
1169 struct kvm_s390_vm_tod_clock *gtod)
8fa1696e
CW		 1170{
1171 struct kvm_s390_tod_clock_ext htod;
1172
1173 preempt_disable();
1174
1175 get_tod_clock_ext((char *)&htod);
1176
1177 gtod->tod = htod.tod + kvm->arch.epoch;
33d1b272
DH		 1178 gtod->epoch_idx = 0;
1179 if (test_kvm_facility(kvm, 139)) {
1180 gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
1181 if (gtod->tod < htod.tod)
1182 gtod->epoch_idx += 1;
1183 }
8fa1696e
CW		 1184
1185 preempt_enable();
1186}
1187
1188static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1189{
1190 struct kvm_s390_vm_tod_clock gtod;
1191
1192 memset(&gtod, 0, sizeof(gtod));
33d1b272 1193 kvm_s390_get_tod_clock(kvm, &gtod);
8fa1696e
CW		 1194 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1195 return -EFAULT;
1196
1197 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
1198 gtod.epoch_idx, gtod.tod);
1199 return 0;
1200}
1201
72f25020
JH		 1202static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1203{
1204 u8 gtod_high = 0;
1205
1206 if (copy_to_user((void __user *)attr->addr, &gtod_high,
1207 sizeof(gtod_high)))
1208 return -EFAULT;
58c383c6 1209 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
72f25020
JH		 1210
1211 return 0;
1212}
1213
1214static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1215{
5a3d883a 1216 u64 gtod;
72f25020 1217
60417fcc 1218 gtod = kvm_s390_get_tod_clock_fast(kvm);
72f25020
JH		 1219 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1220 return -EFAULT;
58c383c6 1221 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
72f25020
JH		 1222
1223 return 0;
1224}
1225
1226static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1227{
1228 int ret;
1229
1230 if (attr->flags)
1231 return -EINVAL;
1232
1233 switch (attr->attr) {
8fa1696e
CW		 1234 case KVM_S390_VM_TOD_EXT:
1235 ret = kvm_s390_get_tod_ext(kvm, attr);
1236 break;
72f25020
JH		 1237 case KVM_S390_VM_TOD_HIGH:
1238 ret = kvm_s390_get_tod_high(kvm, attr);
1239 break;
1240 case KVM_S390_VM_TOD_LOW:
1241 ret = kvm_s390_get_tod_low(kvm, attr);
1242 break;
1243 default:
1244 ret = -ENXIO;
1245 break;
1246 }
1247 return ret;
1248}
1249
658b6eda
MM		 1250static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1251{
1252 struct kvm_s390_vm_cpu_processor *proc;
053dd230 1253 u16 lowest_ibc, unblocked_ibc;
658b6eda
MM		 1254 int ret = 0;
1255
1256 mutex_lock(&kvm->lock);
a03825bb 1257 if (kvm->created_vcpus) {
658b6eda
MM		 1258 ret = -EBUSY;
1259 goto out;
1260 }
1261 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1262 if (!proc) {
1263 ret = -ENOMEM;
1264 goto out;
1265 }
1266 if (!copy_from_user(proc, (void __user *)attr->addr,
1267 sizeof(*proc))) {
9bb0ec09 1268 kvm->arch.model.cpuid = proc->cpuid;
053dd230
DH		 1269 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1270 unblocked_ibc = sclp.ibc & 0xfff;
0487c44d 1271 if (lowest_ibc && proc->ibc) {
053dd230
DH		 1272 if (proc->ibc > unblocked_ibc)
1273 kvm->arch.model.ibc = unblocked_ibc;
1274 else if (proc->ibc < lowest_ibc)
1275 kvm->arch.model.ibc = lowest_ibc;
1276 else
1277 kvm->arch.model.ibc = proc->ibc;
1278 }
c54f0d6a 1279 memcpy(kvm->arch.model.fac_list, proc->fac_list,
658b6eda 1280 S390_ARCH_FAC_LIST_SIZE_BYTE);
a8c39dd7
CB		 1281 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1282 kvm->arch.model.ibc,
1283 kvm->arch.model.cpuid);
1284 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1285 kvm->arch.model.fac_list[0],
1286 kvm->arch.model.fac_list[1],
1287 kvm->arch.model.fac_list[2]);
658b6eda
MM		 1288 } else
1289 ret = -EFAULT;
1290 kfree(proc);
1291out:
1292 mutex_unlock(&kvm->lock);
1293 return ret;
1294}
1295
15c9705f
DH		 1296static int kvm_s390_set_processor_feat(struct kvm *kvm,
1297 struct kvm_device_attr *attr)
1298{
1299 struct kvm_s390_vm_cpu_feat data;
15c9705f
DH		 1300
1301 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1302 return -EFAULT;
1303 if (!bitmap_subset((unsigned long *) data.feat,
1304 kvm_s390_available_cpu_feat,
1305 KVM_S390_VM_CPU_FEAT_NR_BITS))
1306 return -EINVAL;
1307
1308 mutex_lock(&kvm->lock);
2f8311c9
CB		 1309 if (kvm->created_vcpus) {
1310 mutex_unlock(&kvm->lock);
1311 return -EBUSY;
15c9705f 1312 }
2f8311c9
CB		 1313 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
1314 KVM_S390_VM_CPU_FEAT_NR_BITS);
15c9705f 1315 mutex_unlock(&kvm->lock);
2f8311c9
CB		 1316 VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1317 data.feat[0],
1318 data.feat[1],
1319 data.feat[2]);
1320 return 0;
15c9705f
DH		 1321}
1322
0a763c78
DH		 1323static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1324 struct kvm_device_attr *attr)
1325{
346fa2f8
CB		 1326 mutex_lock(&kvm->lock);
1327 if (kvm->created_vcpus) {
1328 mutex_unlock(&kvm->lock);
1329 return -EBUSY;
1330 }
1331
1332 if (copy_from_user(&kvm->arch.model.subfuncs, (void __user *)attr->addr,
1333 sizeof(struct kvm_s390_vm_cpu_subfunc))) {
1334 mutex_unlock(&kvm->lock);
1335 return -EFAULT;
1336 }
1337 mutex_unlock(&kvm->lock);
1338
11ba5961
CB		 1339 VM_EVENT(kvm, 3, "SET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1340 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1341 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1342 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1343 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1344 VM_EVENT(kvm, 3, "SET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1345 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1346 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1347 VM_EVENT(kvm, 3, "SET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1348 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1349 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1350 VM_EVENT(kvm, 3, "SET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1351 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1352 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1353 VM_EVENT(kvm, 3, "SET: guest KM subfunc 0x%16.16lx.%16.16lx",
1354 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1355 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1356 VM_EVENT(kvm, 3, "SET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1357 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1358 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1359 VM_EVENT(kvm, 3, "SET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1360 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1361 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1362 VM_EVENT(kvm, 3, "SET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1363 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1364 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1365 VM_EVENT(kvm, 3, "SET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1366 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1367 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1368 VM_EVENT(kvm, 3, "SET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1369 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1370 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1371 VM_EVENT(kvm, 3, "SET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1372 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1373 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1374 VM_EVENT(kvm, 3, "SET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1375 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1376 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1377 VM_EVENT(kvm, 3, "SET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1378 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1379 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1380 VM_EVENT(kvm, 3, "SET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1381 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1382 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
13209ad0
CB		 1383 VM_EVENT(kvm, 3, "SET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1384 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1385 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
173aec2d
CB		 1386 VM_EVENT(kvm, 3, "SET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1387 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1388 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1389 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1390 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
4f45b90e
CB		 1391 VM_EVENT(kvm, 3, "SET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1392 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1393 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1394 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1395 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
11ba5961 1396
346fa2f8 1397 return 0;
0a763c78
DH		 1398}
1399
658b6eda
MM		 1400static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1401{
1402 int ret = -ENXIO;
1403
1404 switch (attr->attr) {
1405 case KVM_S390_VM_CPU_PROCESSOR:
1406 ret = kvm_s390_set_processor(kvm, attr);
1407 break;
15c9705f
DH		 1408 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1409 ret = kvm_s390_set_processor_feat(kvm, attr);
1410 break;
0a763c78
DH		 1411 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1412 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1413 break;
658b6eda
MM		 1414 }
1415 return ret;
1416}
1417
1418static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1419{
1420 struct kvm_s390_vm_cpu_processor *proc;
1421 int ret = 0;
1422
1423 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1424 if (!proc) {
1425 ret = -ENOMEM;
1426 goto out;
1427 }
9bb0ec09 1428 proc->cpuid = kvm->arch.model.cpuid;
658b6eda 1429 proc->ibc = kvm->arch.model.ibc;
c54f0d6a
DH		 1430 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1431 S390_ARCH_FAC_LIST_SIZE_BYTE);
a8c39dd7
CB		 1432 VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1433 kvm->arch.model.ibc,
1434 kvm->arch.model.cpuid);
1435 VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1436 kvm->arch.model.fac_list[0],
1437 kvm->arch.model.fac_list[1],
1438 kvm->arch.model.fac_list[2]);
658b6eda
MM		 1439 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1440 ret = -EFAULT;
1441 kfree(proc);
1442out:
1443 return ret;
1444}
1445
1446static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1447{
1448 struct kvm_s390_vm_cpu_machine *mach;
1449 int ret = 0;
1450
1451 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
1452 if (!mach) {
1453 ret = -ENOMEM;
1454 goto out;
1455 }
1456 get_cpu_id((struct cpuid *) &mach->cpuid);
37c5f6c8 1457 mach->ibc = sclp.ibc;
c54f0d6a 1458 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
981467c9 1459 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 1460 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
04478197 1461 sizeof(S390_lowcore.stfle_fac_list));
a8c39dd7
CB		 1462 VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx",
1463 kvm->arch.model.ibc,
1464 kvm->arch.model.cpuid);
1465 VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx",
1466 mach->fac_mask[0],
1467 mach->fac_mask[1],
1468 mach->fac_mask[2]);
1469 VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1470 mach->fac_list[0],
1471 mach->fac_list[1],
1472 mach->fac_list[2]);
658b6eda
MM		 1473 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1474 ret = -EFAULT;
1475 kfree(mach);
1476out:
1477 return ret;
1478}
1479
15c9705f
DH		 1480static int kvm_s390_get_processor_feat(struct kvm *kvm,
1481 struct kvm_device_attr *attr)
1482{
1483 struct kvm_s390_vm_cpu_feat data;
1484
1485 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1486 KVM_S390_VM_CPU_FEAT_NR_BITS);
1487 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1488 return -EFAULT;
2f8311c9
CB		 1489 VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1490 data.feat[0],
1491 data.feat[1],
1492 data.feat[2]);
15c9705f
DH		 1493 return 0;
1494}
1495
1496static int kvm_s390_get_machine_feat(struct kvm *kvm,
1497 struct kvm_device_attr *attr)
1498{
1499 struct kvm_s390_vm_cpu_feat data;
1500
1501 bitmap_copy((unsigned long *) data.feat,
1502 kvm_s390_available_cpu_feat,
1503 KVM_S390_VM_CPU_FEAT_NR_BITS);
1504 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1505 return -EFAULT;
2f8311c9
CB		 1506 VM_EVENT(kvm, 3, "GET: host feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1507 data.feat[0],
1508 data.feat[1],
1509 data.feat[2]);
15c9705f
DH		 1510 return 0;
1511}
1512
0a763c78
DH		 1513static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1514 struct kvm_device_attr *attr)
1515{
346fa2f8
CB		 1516 if (copy_to_user((void __user *)attr->addr, &kvm->arch.model.subfuncs,
1517 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1518 return -EFAULT;
1519
11ba5961
CB		 1520 VM_EVENT(kvm, 3, "GET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1521 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1522 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1523 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1524 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1525 VM_EVENT(kvm, 3, "GET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1526 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1527 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1528 VM_EVENT(kvm, 3, "GET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1529 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1530 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1531 VM_EVENT(kvm, 3, "GET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1532 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1533 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1534 VM_EVENT(kvm, 3, "GET: guest KM subfunc 0x%16.16lx.%16.16lx",
1535 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1536 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1537 VM_EVENT(kvm, 3, "GET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1538 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1539 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1540 VM_EVENT(kvm, 3, "GET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1541 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1542 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1543 VM_EVENT(kvm, 3, "GET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1544 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1545 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1546 VM_EVENT(kvm, 3, "GET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1547 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1548 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1549 VM_EVENT(kvm, 3, "GET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1550 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1551 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1552 VM_EVENT(kvm, 3, "GET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1553 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1554 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1555 VM_EVENT(kvm, 3, "GET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1556 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1557 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1558 VM_EVENT(kvm, 3, "GET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1559 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1560 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1561 VM_EVENT(kvm, 3, "GET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1562 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1563 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
13209ad0
CB		 1564 VM_EVENT(kvm, 3, "GET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1565 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1566 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
173aec2d
CB		 1567 VM_EVENT(kvm, 3, "GET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1568 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1569 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1570 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1571 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
4f45b90e
CB		 1572 VM_EVENT(kvm, 3, "GET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1573 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1574 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1575 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1576 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
11ba5961 1577
346fa2f8 1578 return 0;
0a763c78
DH		 1579}
1580
1581static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1582 struct kvm_device_attr *attr)
1583{
1584 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1585 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1586 return -EFAULT;
11ba5961
CB		 1587
1588 VM_EVENT(kvm, 3, "GET: host PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1589 ((unsigned long *) &kvm_s390_available_subfunc.plo)[0],
1590 ((unsigned long *) &kvm_s390_available_subfunc.plo)[1],
1591 ((unsigned long *) &kvm_s390_available_subfunc.plo)[2],
1592 ((unsigned long *) &kvm_s390_available_subfunc.plo)[3]);
1593 VM_EVENT(kvm, 3, "GET: host PTFF subfunc 0x%16.16lx.%16.16lx",
1594 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[0],
1595 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[1]);
1596 VM_EVENT(kvm, 3, "GET: host KMAC subfunc 0x%16.16lx.%16.16lx",
1597 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[0],
1598 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[1]);
1599 VM_EVENT(kvm, 3, "GET: host KMC subfunc 0x%16.16lx.%16.16lx",
1600 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[0],
1601 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[1]);
1602 VM_EVENT(kvm, 3, "GET: host KM subfunc 0x%16.16lx.%16.16lx",
1603 ((unsigned long *) &kvm_s390_available_subfunc.km)[0],
1604 ((unsigned long *) &kvm_s390_available_subfunc.km)[1]);
1605 VM_EVENT(kvm, 3, "GET: host KIMD subfunc 0x%16.16lx.%16.16lx",
1606 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[0],
1607 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[1]);
1608 VM_EVENT(kvm, 3, "GET: host KLMD subfunc 0x%16.16lx.%16.16lx",
1609 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[0],
1610 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[1]);
1611 VM_EVENT(kvm, 3, "GET: host PCKMO subfunc 0x%16.16lx.%16.16lx",
1612 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[0],
1613 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[1]);
1614 VM_EVENT(kvm, 3, "GET: host KMCTR subfunc 0x%16.16lx.%16.16lx",
1615 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[0],
1616 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[1]);
1617 VM_EVENT(kvm, 3, "GET: host KMF subfunc 0x%16.16lx.%16.16lx",
1618 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[0],
1619 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[1]);
1620 VM_EVENT(kvm, 3, "GET: host KMO subfunc 0x%16.16lx.%16.16lx",
1621 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[0],
1622 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[1]);
1623 VM_EVENT(kvm, 3, "GET: host PCC subfunc 0x%16.16lx.%16.16lx",
1624 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[0],
1625 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[1]);
1626 VM_EVENT(kvm, 3, "GET: host PPNO subfunc 0x%16.16lx.%16.16lx",
1627 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[0],
1628 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[1]);
1629 VM_EVENT(kvm, 3, "GET: host KMA subfunc 0x%16.16lx.%16.16lx",
1630 ((unsigned long *) &kvm_s390_available_subfunc.kma)[0],
1631 ((unsigned long *) &kvm_s390_available_subfunc.kma)[1]);
13209ad0
CB		 1632 VM_EVENT(kvm, 3, "GET: host KDSA subfunc 0x%16.16lx.%16.16lx",
1633 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[0],
1634 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[1]);
173aec2d
CB		 1635 VM_EVENT(kvm, 3, "GET: host SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1636 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[0],
1637 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[1],
1638 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[2],
1639 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[3]);
4f45b90e
CB		 1640 VM_EVENT(kvm, 3, "GET: host DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1641 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[0],
1642 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[1],
1643 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[2],
1644 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[3]);
11ba5961 1645
0a763c78
DH		 1646 return 0;
1647}
346fa2f8 1648
658b6eda
MM		 1649static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1650{
1651 int ret = -ENXIO;
1652
1653 switch (attr->attr) {
1654 case KVM_S390_VM_CPU_PROCESSOR:
1655 ret = kvm_s390_get_processor(kvm, attr);
1656 break;
1657 case KVM_S390_VM_CPU_MACHINE:
1658 ret = kvm_s390_get_machine(kvm, attr);
1659 break;
15c9705f
DH		 1660 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1661 ret = kvm_s390_get_processor_feat(kvm, attr);
1662 break;
1663 case KVM_S390_VM_CPU_MACHINE_FEAT:
1664 ret = kvm_s390_get_machine_feat(kvm, attr);
1665 break;
0a763c78
DH		 1666 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1667 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1668 break;
1669 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1670 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1671 break;
658b6eda
MM		 1672 }
1673 return ret;
1674}
1675
f2061656
DD		 1676static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1677{
1678 int ret;
1679
1680 switch (attr->group) {
4f718eab 1681 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 1682 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 1683 break;
72f25020
JH		 1684 case KVM_S390_VM_TOD:
1685 ret = kvm_s390_set_tod(kvm, attr);
1686 break;
658b6eda
MM		 1687 case KVM_S390_VM_CPU_MODEL:
1688 ret = kvm_s390_set_cpu_model(kvm, attr);
1689 break;
a374e892
TK		 1690 case KVM_S390_VM_CRYPTO:
1691 ret = kvm_s390_vm_set_crypto(kvm, attr);
1692 break;
190df4a2
CI		 1693 case KVM_S390_VM_MIGRATION:
1694 ret = kvm_s390_vm_set_migration(kvm, attr);
1695 break;
f2061656
DD		 1696 default:
1697 ret = -ENXIO;
1698 break;
1699 }
1700
1701 return ret;
1702}
1703
1704static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1705{
8c0a7ce6
DD		 1706 int ret;
1707
1708 switch (attr->group) {
1709 case KVM_S390_VM_MEM_CTRL:
1710 ret = kvm_s390_get_mem_control(kvm, attr);
1711 break;
72f25020
JH		 1712 case KVM_S390_VM_TOD:
1713 ret = kvm_s390_get_tod(kvm, attr);
1714 break;
658b6eda
MM		 1715 case KVM_S390_VM_CPU_MODEL:
1716 ret = kvm_s390_get_cpu_model(kvm, attr);
1717 break;
190df4a2
CI		 1718 case KVM_S390_VM_MIGRATION:
1719 ret = kvm_s390_vm_get_migration(kvm, attr);
1720 break;
8c0a7ce6
DD		 1721 default:
1722 ret = -ENXIO;
1723 break;
1724 }
1725
1726 return ret;
f2061656
DD		 1727}
1728
1729static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1730{
1731 int ret;
1732
1733 switch (attr->group) {
4f718eab
DD		 1734 case KVM_S390_VM_MEM_CTRL:
1735 switch (attr->attr) {
1736 case KVM_S390_VM_MEM_ENABLE_CMMA:
1737 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 1738 ret = sclp.has_cmma ? 0 : -ENXIO;
1739 break;
8c0a7ce6 1740 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 1741 ret = 0;
1742 break;
1743 default:
1744 ret = -ENXIO;
1745 break;
1746 }
1747 break;
72f25020
JH		 1748 case KVM_S390_VM_TOD:
1749 switch (attr->attr) {
1750 case KVM_S390_VM_TOD_LOW:
1751 case KVM_S390_VM_TOD_HIGH:
1752 ret = 0;
1753 break;
1754 default:
1755 ret = -ENXIO;
1756 break;
1757 }
1758 break;
658b6eda
MM		 1759 case KVM_S390_VM_CPU_MODEL:
1760 switch (attr->attr) {
1761 case KVM_S390_VM_CPU_PROCESSOR:
1762 case KVM_S390_VM_CPU_MACHINE:
15c9705f
DH		 1763 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1764 case KVM_S390_VM_CPU_MACHINE_FEAT:
0a763c78 1765 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
346fa2f8 1766 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
658b6eda
MM		 1767 ret = 0;
1768 break;
1769 default:
1770 ret = -ENXIO;
1771 break;
1772 }
1773 break;
a374e892
TK		 1774 case KVM_S390_VM_CRYPTO:
1775 switch (attr->attr) {
1776 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1777 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1778 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1779 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1780 ret = 0;
1781 break;
37940fb0
TK		 1782 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
1783 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
1784 ret = ap_instructions_available() ? 0 : -ENXIO;
1785 break;
a374e892
TK		 1786 default:
1787 ret = -ENXIO;
1788 break;
1789 }
1790 break;
190df4a2
CI		 1791 case KVM_S390_VM_MIGRATION:
1792 ret = 0;
1793 break;
f2061656
DD		 1794 default:
1795 ret = -ENXIO;
1796 break;
1797 }
1798
1799 return ret;
1800}
1801
30ee2a98
JH		 1802static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1803{
1804 uint8_t *keys;
1805 uint64_t hva;
4f899147 1806 int srcu_idx, i, r = 0;
30ee2a98
JH		 1807
1808 if (args->flags != 0)
1809 return -EINVAL;
1810
1811 /* Is this guest using storage keys? */
55531b74 1812 if (!mm_uses_skeys(current->mm))
30ee2a98
JH		 1813 return KVM_S390_GET_SKEYS_NONE;
1814
1815 /* Enforce sane limit on memory allocation */
1816 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1817 return -EINVAL;
1818
752ade68 1819 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
30ee2a98
JH		 1820 if (!keys)
1821 return -ENOMEM;
1822
d3ed1cee 1823 down_read(&current->mm->mmap_sem);
4f899147 1824 srcu_idx = srcu_read_lock(&kvm->srcu);
30ee2a98
JH		 1825 for (i = 0; i < args->count; i++) {
1826 hva = gfn_to_hva(kvm, args->start_gfn + i);
1827 if (kvm_is_error_hva(hva)) {
1828 r = -EFAULT;
d3ed1cee 1829 break;
30ee2a98
JH		 1830 }
1831
154c8c19
DH		 1832 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1833 if (r)
d3ed1cee 1834 break;
30ee2a98 1835 }
4f899147 1836 srcu_read_unlock(&kvm->srcu, srcu_idx);
d3ed1cee
MS		 1837 up_read(&current->mm->mmap_sem);
1838
1839 if (!r) {
1840 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1841 sizeof(uint8_t) * args->count);
1842 if (r)
1843 r = -EFAULT;
30ee2a98
JH		 1844 }
1845
30ee2a98
JH		 1846 kvfree(keys);
1847 return r;
1848}
1849
1850static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1851{
1852 uint8_t *keys;
1853 uint64_t hva;
4f899147 1854 int srcu_idx, i, r = 0;
bd096f64 1855 bool unlocked;
30ee2a98
JH		 1856
1857 if (args->flags != 0)
1858 return -EINVAL;
1859
1860 /* Enforce sane limit on memory allocation */
1861 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1862 return -EINVAL;
1863
752ade68 1864 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
30ee2a98
JH		 1865 if (!keys)
1866 return -ENOMEM;
1867
1868 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1869 sizeof(uint8_t) * args->count);
1870 if (r) {
1871 r = -EFAULT;
1872 goto out;
1873 }
1874
1875 /* Enable storage key handling for the guest */
14d4a425
DD		 1876 r = s390_enable_skey();
1877 if (r)
1878 goto out;
30ee2a98 1879
bd096f64 1880 i = 0;
d3ed1cee 1881 down_read(&current->mm->mmap_sem);
4f899147 1882 srcu_idx = srcu_read_lock(&kvm->srcu);
bd096f64
JF		 1883 while (i < args->count) {
1884 unlocked = false;
30ee2a98
JH		 1885 hva = gfn_to_hva(kvm, args->start_gfn + i);
1886 if (kvm_is_error_hva(hva)) {
1887 r = -EFAULT;
d3ed1cee 1888 break;
30ee2a98
JH		 1889 }
1890
1891 /* Lowest order bit is reserved */
1892 if (keys[i] & 0x01) {
1893 r = -EINVAL;
d3ed1cee 1894 break;
30ee2a98
JH		 1895 }
1896
fe69eabf 1897 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
bd096f64
JF		 1898 if (r) {
1899 r = fixup_user_fault(current, current->mm, hva,
1900 FAULT_FLAG_WRITE, &unlocked);
1901 if (r)
1902 break;
1903 }
1904 if (!r)
1905 i++;
30ee2a98 1906 }
4f899147 1907 srcu_read_unlock(&kvm->srcu, srcu_idx);
d3ed1cee 1908 up_read(&current->mm->mmap_sem);
30ee2a98
JH		 1909out:
1910 kvfree(keys);
1911 return r;
1912}
1913
4036e387
CI		 1914/*
1915 * Base address and length must be sent at the start of each block, therefore
1916 * it's cheaper to send some clean data, as long as it's less than the size of
1917 * two longs.
1918 */
1919#define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
1920/* for consistency */
1921#define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1922
afdad616
CI		 1923/*
1924 * Similar to gfn_to_memslot, but returns the index of a memslot also when the
1925 * address falls in a hole. In that case the index of one of the memslots
1926 * bordering the hole is returned.
1927 */
1928static int gfn_to_memslot_approx(struct kvm_memslots *slots, gfn_t gfn)
1929{
1930 int start = 0, end = slots->used_slots;
1931 int slot = atomic_read(&slots->lru_slot);
1932 struct kvm_memory_slot *memslots = slots->memslots;
1933
1934 if (gfn >= memslots[slot].base_gfn &&
1935 gfn < memslots[slot].base_gfn + memslots[slot].npages)
1936 return slot;
1937
1938 while (start < end) {
1939 slot = start + (end - start) / 2;
1940
1941 if (gfn >= memslots[slot].base_gfn)
1942 end = slot;
1943 else
1944 start = slot + 1;
1945 }
1946
d3afc4c1
SC		 1947 if (start >= slots->used_slots)
1948 return slots->used_slots - 1;
1949
afdad616
CI		 1950 if (gfn >= memslots[start].base_gfn &&
1951 gfn < memslots[start].base_gfn + memslots[start].npages) {
1952 atomic_set(&slots->lru_slot, start);
1953 }
1954
1955 return start;
1956}
1957
1958static int kvm_s390_peek_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
1959 u8 *res, unsigned long bufsize)
1960{
1961 unsigned long pgstev, hva, cur_gfn = args->start_gfn;
1962
1963 args->count = 0;
1964 while (args->count < bufsize) {
1965 hva = gfn_to_hva(kvm, cur_gfn);
1966 /*
1967 * We return an error if the first value was invalid, but we
1968 * return successfully if at least one value was copied.
1969 */
1970 if (kvm_is_error_hva(hva))
1971 return args->count ? 0 : -EFAULT;
1972 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
1973 pgstev = 0;
1974 res[args->count++] = (pgstev >> 24) & 0x43;
1975 cur_gfn++;
1976 }
1977
1978 return 0;
1979}
1980
1981static unsigned long kvm_s390_next_dirty_cmma(struct kvm_memslots *slots,
1982 unsigned long cur_gfn)
1983{
1984 int slotidx = gfn_to_memslot_approx(slots, cur_gfn);
1985 struct kvm_memory_slot *ms = slots->memslots + slotidx;
1986 unsigned long ofs = cur_gfn - ms->base_gfn;
1987
1988 if (ms->base_gfn + ms->npages <= cur_gfn) {
1989 slotidx--;
1990 /* If we are above the highest slot, wrap around */
1991 if (slotidx < 0)
1992 slotidx = slots->used_slots - 1;
1993
1994 ms = slots->memslots + slotidx;
1995 ofs = 0;
1996 }
1997 ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, ofs);
1998 while ((slotidx > 0) && (ofs >= ms->npages)) {
1999 slotidx--;
2000 ms = slots->memslots + slotidx;
2001 ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, 0);
2002 }
2003 return ms->base_gfn + ofs;
2004}
2005
2006static int kvm_s390_get_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
2007 u8 *res, unsigned long bufsize)
2008{
2009 unsigned long mem_end, cur_gfn, next_gfn, hva, pgstev;
2010 struct kvm_memslots *slots = kvm_memslots(kvm);
2011 struct kvm_memory_slot *ms;
2012
2013 cur_gfn = kvm_s390_next_dirty_cmma(slots, args->start_gfn);
2014 ms = gfn_to_memslot(kvm, cur_gfn);
2015 args->count = 0;
2016 args->start_gfn = cur_gfn;
2017 if (!ms)
2018 return 0;
2019 next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2020 mem_end = slots->memslots[0].base_gfn + slots->memslots[0].npages;
2021
2022 while (args->count < bufsize) {
2023 hva = gfn_to_hva(kvm, cur_gfn);
2024 if (kvm_is_error_hva(hva))
2025 return 0;
2026 /* Decrement only if we actually flipped the bit to 0 */
2027 if (test_and_clear_bit(cur_gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms)))
2028 atomic64_dec(&kvm->arch.cmma_dirty_pages);
2029 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
2030 pgstev = 0;
2031 /* Save the value */
2032 res[args->count++] = (pgstev >> 24) & 0x43;
2033 /* If the next bit is too far away, stop. */
2034 if (next_gfn > cur_gfn + KVM_S390_MAX_BIT_DISTANCE)
2035 return 0;
2036 /* If we reached the previous "next", find the next one */
2037 if (cur_gfn == next_gfn)
2038 next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2039 /* Reached the end of memory or of the buffer, stop */
2040 if ((next_gfn >= mem_end) ||
2041 (next_gfn - args->start_gfn >= bufsize))
2042 return 0;
2043 cur_gfn++;
2044 /* Reached the end of the current memslot, take the next one. */
2045 if (cur_gfn - ms->base_gfn >= ms->npages) {
2046 ms = gfn_to_memslot(kvm, cur_gfn);
2047 if (!ms)
2048 return 0;
2049 }
2050 }
2051 return 0;
2052}
2053
4036e387
CI		 2054/*
2055 * This function searches for the next page with dirty CMMA attributes, and
2056 * saves the attributes in the buffer up to either the end of the buffer or
2057 * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
2058 * no trailing clean bytes are saved.
2059 * In case no dirty bits were found, or if CMMA was not enabled or used, the
2060 * output buffer will indicate 0 as length.
2061 */
2062static int kvm_s390_get_cmma_bits(struct kvm *kvm,
2063 struct kvm_s390_cmma_log *args)
2064{
afdad616
CI		 2065 unsigned long bufsize;
2066 int srcu_idx, peek, ret;
2067 u8 *values;
4036e387 2068
afdad616 2069 if (!kvm->arch.use_cmma)
4036e387
CI		 2070 return -ENXIO;
2071 /* Invalid/unsupported flags were specified */
2072 if (args->flags & ~KVM_S390_CMMA_PEEK)
2073 return -EINVAL;
2074 /* Migration mode query, and we are not doing a migration */
2075 peek = !!(args->flags & KVM_S390_CMMA_PEEK);
afdad616 2076 if (!peek && !kvm->arch.migration_mode)
4036e387
CI		 2077 return -EINVAL;
2078 /* CMMA is disabled or was not used, or the buffer has length zero */
2079 bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
c9f0a2b8 2080 if (!bufsize || !kvm->mm->context.uses_cmm) {
4036e387
CI		 2081 memset(args, 0, sizeof(*args));
2082 return 0;
2083 }
afdad616
CI		 2084 /* We are not peeking, and there are no dirty pages */
2085 if (!peek && !atomic64_read(&kvm->arch.cmma_dirty_pages)) {
2086 memset(args, 0, sizeof(*args));
2087 return 0;
4036e387
CI		 2088 }
2089
afdad616
CI		 2090 values = vmalloc(bufsize);
2091 if (!values)
4036e387
CI		 2092 return -ENOMEM;
2093
4036e387
CI		 2094 down_read(&kvm->mm->mmap_sem);
2095 srcu_idx = srcu_read_lock(&kvm->srcu);
afdad616
CI		 2096 if (peek)
2097 ret = kvm_s390_peek_cmma(kvm, args, values, bufsize);
2098 else
2099 ret = kvm_s390_get_cmma(kvm, args, values, bufsize);
4036e387
CI		 2100 srcu_read_unlock(&kvm->srcu, srcu_idx);
2101 up_read(&kvm->mm->mmap_sem);
4036e387 2102
afdad616
CI		 2103 if (kvm->arch.migration_mode)
2104 args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages);
2105 else
2106 args->remaining = 0;
4036e387 2107
afdad616
CI		 2108 if (copy_to_user((void __user *)args->values, values, args->count))
2109 ret = -EFAULT;
2110
2111 vfree(values);
2112 return ret;
4036e387
CI		 2113}
2114
2115/*
2116 * This function sets the CMMA attributes for the given pages. If the input
2117 * buffer has zero length, no action is taken, otherwise the attributes are
c9f0a2b8 2118 * set and the mm->context.uses_cmm flag is set.
4036e387
CI		 2119 */
2120static int kvm_s390_set_cmma_bits(struct kvm *kvm,
2121 const struct kvm_s390_cmma_log *args)
2122{
2123 unsigned long hva, mask, pgstev, i;
2124 uint8_t *bits;
2125 int srcu_idx, r = 0;
2126
2127 mask = args->mask;
2128
2129 if (!kvm->arch.use_cmma)
2130 return -ENXIO;
2131 /* invalid/unsupported flags */
2132 if (args->flags != 0)
2133 return -EINVAL;
2134 /* Enforce sane limit on memory allocation */
2135 if (args->count > KVM_S390_CMMA_SIZE_MAX)
2136 return -EINVAL;
2137 /* Nothing to do */
2138 if (args->count == 0)
2139 return 0;
2140
42bc47b3 2141 bits = vmalloc(array_size(sizeof(*bits), args->count));
4036e387
CI		 2142 if (!bits)
2143 return -ENOMEM;
2144
2145 r = copy_from_user(bits, (void __user *)args->values, args->count);
2146 if (r) {
2147 r = -EFAULT;
2148 goto out;
2149 }
2150
2151 down_read(&kvm->mm->mmap_sem);
2152 srcu_idx = srcu_read_lock(&kvm->srcu);
2153 for (i = 0; i < args->count; i++) {
2154 hva = gfn_to_hva(kvm, args->start_gfn + i);
2155 if (kvm_is_error_hva(hva)) {
2156 r = -EFAULT;
2157 break;
2158 }
2159
2160 pgstev = bits[i];
2161 pgstev = pgstev << 24;
1bab1c02 2162 mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
4036e387
CI		 2163 set_pgste_bits(kvm->mm, hva, mask, pgstev);
2164 }
2165 srcu_read_unlock(&kvm->srcu, srcu_idx);
2166 up_read(&kvm->mm->mmap_sem);
2167
c9f0a2b8 2168 if (!kvm->mm->context.uses_cmm) {
4036e387 2169 down_write(&kvm->mm->mmap_sem);
c9f0a2b8 2170 kvm->mm->context.uses_cmm = 1;
4036e387
CI		 2171 up_write(&kvm->mm->mmap_sem);
2172 }
2173out:
2174 vfree(bits);
2175 return r;
2176}
2177
e2ed2e2d
JF		 2178static int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rcp, u16 *rrcp)
2179{
2180 struct kvm_vcpu *vcpu;
2181 u16 rc, rrc;
2182 int ret = 0;
2183 int i;
2184
2185 /*
2186 * We ignore failures and try to destroy as many CPUs as possible.
2187 * At the same time we must not free the assigned resources when
2188 * this fails, as the ultravisor has still access to that memory.
2189 * So kvm_s390_pv_destroy_cpu can leave a "wanted" memory leak
2190 * behind.
2191 * We want to return the first failure rc and rrc, though.
2192 */
2193 kvm_for_each_vcpu(i, vcpu, kvm) {
2194 mutex_lock(&vcpu->mutex);
2195 if (kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc) && !ret) {
2196 *rcp = rc;
2197 *rrcp = rrc;
2198 ret = -EIO;
2199 }
2200 mutex_unlock(&vcpu->mutex);
2201 }
2202 return ret;
2203}
2204
2205static int kvm_s390_cpus_to_pv(struct kvm *kvm, u16 *rc, u16 *rrc)
2206{
2207 int i, r = 0;
2208 u16 dummy;
2209
2210 struct kvm_vcpu *vcpu;
2211
2212 kvm_for_each_vcpu(i, vcpu, kvm) {
2213 mutex_lock(&vcpu->mutex);
2214 r = kvm_s390_pv_create_cpu(vcpu, rc, rrc);
2215 mutex_unlock(&vcpu->mutex);
2216 if (r)
2217 break;
2218 }
2219 if (r)
2220 kvm_s390_cpus_from_pv(kvm, &dummy, &dummy);
2221 return r;
2222}
2223
2224static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd)
2225{
2226 int r = 0;
2227 u16 dummy;
2228 void __user *argp = (void __user *)cmd->data;
2229
2230 switch (cmd->cmd) {
2231 case KVM_PV_ENABLE: {
2232 r = -EINVAL;
2233 if (kvm_s390_pv_is_protected(kvm))
2234 break;
2235
2236 /*
2237 * FMT 4 SIE needs esca. As we never switch back to bsca from
2238 * esca, we need no cleanup in the error cases below
2239 */
2240 r = sca_switch_to_extended(kvm);
2241 if (r)
2242 break;
2243
88d4cc6c
JF		 2244 down_write(&current->mm->mmap_sem);
2245 r = gmap_mark_unmergeable();
2246 up_write(&current->mm->mmap_sem);
2247 if (r)
2248 break;
2249
e2ed2e2d
JF		 2250 r = kvm_s390_pv_init_vm(kvm, &cmd->rc, &cmd->rrc);
2251 if (r)
2252 break;
2253
2254 r = kvm_s390_cpus_to_pv(kvm, &cmd->rc, &cmd->rrc);
2255 if (r)
2256 kvm_s390_pv_deinit_vm(kvm, &dummy, &dummy);
be5c8f43
CB		 2257
2258 /* we need to block service interrupts from now on */
2259 set_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
e2ed2e2d
JF		 2260 break;
2261 }
2262 case KVM_PV_DISABLE: {
2263 r = -EINVAL;
2264 if (!kvm_s390_pv_is_protected(kvm))
2265 break;
2266
2267 r = kvm_s390_cpus_from_pv(kvm, &cmd->rc, &cmd->rrc);
2268 /*
2269 * If a CPU could not be destroyed, destroy VM will also fail.
2270 * There is no point in trying to destroy it. Instead return
2271 * the rc and rrc from the first CPU that failed destroying.
2272 */
2273 if (r)
2274 break;
2275 r = kvm_s390_pv_deinit_vm(kvm, &cmd->rc, &cmd->rrc);
be5c8f43
CB		 2276
2277 /* no need to block service interrupts any more */
2278 clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
e2ed2e2d
JF		 2279 break;
2280 }
2281 case KVM_PV_SET_SEC_PARMS: {
2282 struct kvm_s390_pv_sec_parm parms = {};
2283 void *hdr;
2284
2285 r = -EINVAL;
2286 if (!kvm_s390_pv_is_protected(kvm))
2287 break;
2288
2289 r = -EFAULT;
2290 if (copy_from_user(&parms, argp, sizeof(parms)))
2291 break;
2292
2293 /* Currently restricted to 8KB */
2294 r = -EINVAL;
2295 if (parms.length > PAGE_SIZE * 2)
2296 break;
2297
2298 r = -ENOMEM;
2299 hdr = vmalloc(parms.length);
2300 if (!hdr)
2301 break;
2302
2303 r = -EFAULT;
2304 if (!copy_from_user(hdr, (void __user *)parms.origin,
2305 parms.length))
2306 r = kvm_s390_pv_set_sec_parms(kvm, hdr, parms.length,
2307 &cmd->rc, &cmd->rrc);
2308
2309 vfree(hdr);
2310 break;
2311 }
2312 case KVM_PV_UNPACK: {
2313 struct kvm_s390_pv_unp unp = {};
2314
2315 r = -EINVAL;
2316 if (!kvm_s390_pv_is_protected(kvm))
2317 break;
2318
2319 r = -EFAULT;
2320 if (copy_from_user(&unp, argp, sizeof(unp)))
2321 break;
2322
2323 r = kvm_s390_pv_unpack(kvm, unp.addr, unp.size, unp.tweak,
2324 &cmd->rc, &cmd->rrc);
2325 break;
2326 }
2327 case KVM_PV_VERIFY: {
2328 r = -EINVAL;
2329 if (!kvm_s390_pv_is_protected(kvm))
2330 break;
2331
2332 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2333 UVC_CMD_VERIFY_IMG, &cmd->rc, &cmd->rrc);
2334 KVM_UV_EVENT(kvm, 3, "PROTVIRT VERIFY: rc %x rrc %x", cmd->rc,
2335 cmd->rrc);
2336 break;
2337 }
7dc51f22
JF		 2338 case KVM_PV_PREP_RESET: {
2339 r = -EINVAL;
2340 if (!kvm_s390_pv_is_protected(kvm))
2341 break;
2342
2343 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2344 UVC_CMD_PREPARE_RESET, &cmd->rc, &cmd->rrc);
2345 KVM_UV_EVENT(kvm, 3, "PROTVIRT PREP RESET: rc %x rrc %x",
2346 cmd->rc, cmd->rrc);
2347 break;
2348 }
2349 case KVM_PV_UNSHARE_ALL: {
2350 r = -EINVAL;
2351 if (!kvm_s390_pv_is_protected(kvm))
2352 break;
2353
2354 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2355 UVC_CMD_SET_UNSHARE_ALL, &cmd->rc, &cmd->rrc);
2356 KVM_UV_EVENT(kvm, 3, "PROTVIRT UNSHARE: rc %x rrc %x",
2357 cmd->rc, cmd->rrc);
2358 break;
2359 }
e2ed2e2d
JF		 2360 default:
2361 r = -ENOTTY;
2362 }
2363 return r;
2364}
2365
0b8b9918
JSG		 2366static bool access_key_invalid(u8 access_key)
2367{
2368 return access_key > 0xf;
2369}
2370
8016411b
JSG		 2371static int kvm_s390_vm_mem_op(struct kvm *kvm, struct kvm_s390_mem_op *mop)
2372{
2373 void __user *uaddr = (void __user *)mop->buf;
2374 u64 supported_flags;
2375 void *tmpbuf = NULL;
2376 int r, srcu_idx;
2377
2378 supported_flags = KVM_S390_MEMOP_F_SKEY_PROTECTION
2379 | KVM_S390_MEMOP_F_CHECK_ONLY;
61e9c42c 2380 if (mop->flags & ~supported_flags || !mop->size)
8016411b
JSG		 2381 return -EINVAL;
2382 if (mop->size > MEM_OP_MAX_SIZE)
2383 return -E2BIG;
2384 if (kvm_s390_pv_is_protected(kvm))
2385 return -EINVAL;
2386 if (mop->flags & KVM_S390_MEMOP_F_SKEY_PROTECTION) {
2387 if (access_key_invalid(mop->key))
2388 return -EINVAL;
2389 } else {
2390 mop->key = 0;
2391 }
2392 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2393 tmpbuf = vmalloc(mop->size);
2394 if (!tmpbuf)
2395 return -ENOMEM;
2396 }
2397
2398 srcu_idx = srcu_read_lock(&kvm->srcu);
2399
2400 if (kvm_is_error_gpa(kvm, mop->gaddr)) {
2401 r = PGM_ADDRESSING;
2402 goto out_unlock;
2403 }
2404
2405 switch (mop->op) {
2406 case KVM_S390_MEMOP_ABSOLUTE_READ: {
2407 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2408 r = check_gpa_range(kvm, mop->gaddr, mop->size, GACC_FETCH, mop->key);
2409 } else {
2410 r = access_guest_abs_with_key(kvm, mop->gaddr, tmpbuf,
2411 mop->size, GACC_FETCH, mop->key);
2412 if (r == 0) {
2413 if (copy_to_user(uaddr, tmpbuf, mop->size))
2414 r = -EFAULT;
2415 }
2416 }
2417 break;
2418 }
2419 case KVM_S390_MEMOP_ABSOLUTE_WRITE: {
2420 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2421 r = check_gpa_range(kvm, mop->gaddr, mop->size, GACC_STORE, mop->key);
2422 } else {
2423 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2424 r = -EFAULT;
2425 break;
2426 }
2427 r = access_guest_abs_with_key(kvm, mop->gaddr, tmpbuf,
2428 mop->size, GACC_STORE, mop->key);
2429 }
2430 break;
2431 }
2432 default:
2433 r = -EINVAL;
2434 }
2435
2436out_unlock:
2437 srcu_read_unlock(&kvm->srcu, srcu_idx);
2438
2439 vfree(tmpbuf);
2440 return r;
2441}
2442
b0c632db
HC		 2443long kvm_arch_vm_ioctl(struct file *filp,
2444 unsigned int ioctl, unsigned long arg)
2445{
2446 struct kvm *kvm = filp->private_data;
2447 void __user *argp = (void __user *)arg;
f2061656 2448 struct kvm_device_attr attr;
b0c632db
HC		 2449 int r;
2450
2451 switch (ioctl) {
ba5c1e9b
CO		 2452 case KVM_S390_INTERRUPT: {
2453 struct kvm_s390_interrupt s390int;
2454
2455 r = -EFAULT;
2456 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2457 break;
2458 r = kvm_s390_inject_vm(kvm, &s390int);
2459 break;
2460 }
84223598
CH		 2461 case KVM_CREATE_IRQCHIP: {
2462 struct kvm_irq_routing_entry routing;
2463
2464 r = -EINVAL;
2465 if (kvm->arch.use_irqchip) {
2466 /* Set up dummy routing. */
2467 memset(&routing, 0, sizeof(routing));
152b2839 2468 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
84223598
CH		 2469 }
2470 break;
2471 }
f2061656
DD		 2472 case KVM_SET_DEVICE_ATTR: {
2473 r = -EFAULT;
2474 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2475 break;
2476 r = kvm_s390_vm_set_attr(kvm, &attr);
2477 break;
2478 }
2479 case KVM_GET_DEVICE_ATTR: {
2480 r = -EFAULT;
2481 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2482 break;
2483 r = kvm_s390_vm_get_attr(kvm, &attr);
2484 break;
2485 }
2486 case KVM_HAS_DEVICE_ATTR: {
2487 r = -EFAULT;
2488 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2489 break;
2490 r = kvm_s390_vm_has_attr(kvm, &attr);
2491 break;
2492 }
30ee2a98
JH		 2493 case KVM_S390_GET_SKEYS: {
2494 struct kvm_s390_skeys args;
2495
2496 r = -EFAULT;
2497 if (copy_from_user(&args, argp,
2498 sizeof(struct kvm_s390_skeys)))
2499 break;
2500 r = kvm_s390_get_skeys(kvm, &args);
2501 break;
2502 }
2503 case KVM_S390_SET_SKEYS: {
2504 struct kvm_s390_skeys args;
2505
2506 r = -EFAULT;
2507 if (copy_from_user(&args, argp,
2508 sizeof(struct kvm_s390_skeys)))
2509 break;
2510 r = kvm_s390_set_skeys(kvm, &args);
2511 break;
2512 }
4036e387
CI		 2513 case KVM_S390_GET_CMMA_BITS: {
2514 struct kvm_s390_cmma_log args;
2515
2516 r = -EFAULT;
2517 if (copy_from_user(&args, argp, sizeof(args)))
2518 break;
1de1ea7e 2519 mutex_lock(&kvm->slots_lock);
4036e387 2520 r = kvm_s390_get_cmma_bits(kvm, &args);
1de1ea7e 2521 mutex_unlock(&kvm->slots_lock);
4036e387
CI		 2522 if (!r) {
2523 r = copy_to_user(argp, &args, sizeof(args));
2524 if (r)
2525 r = -EFAULT;
2526 }
2527 break;
2528 }
2529 case KVM_S390_SET_CMMA_BITS: {
2530 struct kvm_s390_cmma_log args;
2531
2532 r = -EFAULT;
2533 if (copy_from_user(&args, argp, sizeof(args)))
2534 break;
1de1ea7e 2535 mutex_lock(&kvm->slots_lock);
4036e387 2536 r = kvm_s390_set_cmma_bits(kvm, &args);
1de1ea7e 2537 mutex_unlock(&kvm->slots_lock);
4036e387
CI		 2538 break;
2539 }
e2ed2e2d
JF		 2540 case KVM_S390_PV_COMMAND: {
2541 struct kvm_pv_cmd args;
2542
efba558a
EF		 2543 /* protvirt means user cpu state */
2544 kvm_s390_set_user_cpu_state_ctrl(kvm);
e2ed2e2d
JF		 2545 r = 0;
2546 if (!is_prot_virt_host()) {
2547 r = -EINVAL;
2548 break;
2549 }
2550 if (copy_from_user(&args, argp, sizeof(args))) {
2551 r = -EFAULT;
2552 break;
2553 }
2554 if (args.flags) {
2555 r = -EINVAL;
2556 break;
2557 }
2558 mutex_lock(&kvm->lock);
2559 r = kvm_s390_handle_pv(kvm, &args);
2560 mutex_unlock(&kvm->lock);
2561 if (copy_to_user(argp, &args, sizeof(args))) {
2562 r = -EFAULT;
2563 break;
2564 }
2565 break;
2566 }
8016411b
JSG		 2567 case KVM_S390_MEM_OP: {
2568 struct kvm_s390_mem_op mem_op;
2569
2570 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2571 r = kvm_s390_vm_mem_op(kvm, &mem_op);
2572 else
2573 r = -EFAULT;
2574 break;
2575 }
b0c632db 2576 default:
367e1319 2577 r = -ENOTTY;
b0c632db
HC		 2578 }
2579
2580 return r;
2581}
2582
45c9b47c
TK		 2583static int kvm_s390_apxa_installed(void)
2584{
e585b24a 2585 struct ap_config_info info;
45c9b47c 2586
e585b24a
TK		 2587 if (ap_instructions_available()) {
2588 if (ap_qci(&info) == 0)
2589 return info.apxa;
45c9b47c
TK		 2590 }
2591
2592 return 0;
2593}
2594
e585b24a
TK		 2595/*
2596 * The format of the crypto control block (CRYCB) is specified in the 3 low
2597 * order bits of the CRYCB designation (CRYCBD) field as follows:
2598 * Format 0: Neither the message security assist extension 3 (MSAX3) nor the
2599 * AP extended addressing (APXA) facility are installed.
2600 * Format 1: The APXA facility is not installed but the MSAX3 facility is.
2601 * Format 2: Both the APXA and MSAX3 facilities are installed
2602 */
45c9b47c
TK		 2603static void kvm_s390_set_crycb_format(struct kvm *kvm)
2604{
2605 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
2606
e585b24a
TK		 2607 /* Clear the CRYCB format bits - i.e., set format 0 by default */
2608 kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
2609
2610 /* Check whether MSAX3 is installed */
2611 if (!test_kvm_facility(kvm, 76))
2612 return;
2613
45c9b47c
TK		 2614 if (kvm_s390_apxa_installed())
2615 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
2616 else
2617 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
2618}
2619
0e237e44
PM		 2620void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
2621 unsigned long *aqm, unsigned long *adm)
2622{
2623 struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
2624
2625 mutex_lock(&kvm->lock);
2626 kvm_s390_vcpu_block_all(kvm);
2627
2628 switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
2629 case CRYCB_FORMAT2: /* APCB1 use 256 bits */
2630 memcpy(crycb->apcb1.apm, apm, 32);
2631 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx %016lx %016lx %016lx",
2632 apm[0], apm[1], apm[2], apm[3]);
2633 memcpy(crycb->apcb1.aqm, aqm, 32);
2634 VM_EVENT(kvm, 3, "SET CRYCB: aqm %016lx %016lx %016lx %016lx",
2635 aqm[0], aqm[1], aqm[2], aqm[3]);
2636 memcpy(crycb->apcb1.adm, adm, 32);
2637 VM_EVENT(kvm, 3, "SET CRYCB: adm %016lx %016lx %016lx %016lx",
2638 adm[0], adm[1], adm[2], adm[3]);
2639 break;
2640 case CRYCB_FORMAT1:
2641 case CRYCB_FORMAT0: /* Fall through both use APCB0 */
2642 memcpy(crycb->apcb0.apm, apm, 8);
2643 memcpy(crycb->apcb0.aqm, aqm, 2);
2644 memcpy(crycb->apcb0.adm, adm, 2);
2645 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx aqm %04x adm %04x",
2646 apm[0], *((unsigned short *)aqm),
2647 *((unsigned short *)adm));
2648 break;
2649 default: /* Can not happen */
2650 break;
2651 }
2652
2653 /* recreate the shadow crycb for each vcpu */
2654 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2655 kvm_s390_vcpu_unblock_all(kvm);
2656 mutex_unlock(&kvm->lock);
2657}
2658EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks);
2659
42104598
TK		 2660void kvm_arch_crypto_clear_masks(struct kvm *kvm)
2661{
2662 mutex_lock(&kvm->lock);
2663 kvm_s390_vcpu_block_all(kvm);
2664
2665 memset(&kvm->arch.crypto.crycb->apcb0, 0,
2666 sizeof(kvm->arch.crypto.crycb->apcb0));
2667 memset(&kvm->arch.crypto.crycb->apcb1, 0,
2668 sizeof(kvm->arch.crypto.crycb->apcb1));
2669
0e237e44 2670 VM_EVENT(kvm, 3, "%s", "CLR CRYCB:");
6cc571b1
PM		 2671 /* recreate the shadow crycb for each vcpu */
2672 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
42104598
TK		 2673 kvm_s390_vcpu_unblock_all(kvm);
2674 mutex_unlock(&kvm->lock);
2675}
2676EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks);
2677
9bb0ec09 2678static u64 kvm_s390_get_initial_cpuid(void)
9d8d5786 2679{
9bb0ec09
DH		 2680 struct cpuid cpuid;
2681
2682 get_cpu_id(&cpuid);
2683 cpuid.version = 0xff;
2684 return *((u64 *) &cpuid);
9d8d5786
MM		 2685}
2686
c54f0d6a 2687static void kvm_s390_crypto_init(struct kvm *kvm)
5102ee87 2688{
c54f0d6a 2689 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
45c9b47c 2690 kvm_s390_set_crycb_format(kvm);
5102ee87 2691
e585b24a
TK		 2692 if (!test_kvm_facility(kvm, 76))
2693 return;
2694
ed6f76b4
TK		 2695 /* Enable AES/DEA protected key functions by default */
2696 kvm->arch.crypto.aes_kw = 1;
2697 kvm->arch.crypto.dea_kw = 1;
2698 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
2699 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
2700 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
2701 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
5102ee87
TK		 2702}
2703
7d43bafc
ED		 2704static void sca_dispose(struct kvm *kvm)
2705{
2706 if (kvm->arch.use_esca)
5e044315 2707 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
7d43bafc
ED		 2708 else
2709 free_page((unsigned long)(kvm->arch.sca));
2710 kvm->arch.sca = NULL;
2711}
2712
e08b9637 2713int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 2714{
76a6dd72 2715 gfp_t alloc_flags = GFP_KERNEL;
9d8d5786 2716 int i, rc;
b0c632db 2717 char debug_name[16];
f6c137ff 2718 static unsigned long sca_offset;
b0c632db 2719
e08b9637
CO		 2720 rc = -EINVAL;
2721#ifdef CONFIG_KVM_S390_UCONTROL
2722 if (type & ~KVM_VM_S390_UCONTROL)
2723 goto out_err;
2724 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
2725 goto out_err;
2726#else
2727 if (type)
2728 goto out_err;
2729#endif
2730
b0c632db
HC		 2731 rc = s390_enable_sie();
2732 if (rc)
d89f5eff 2733 goto out_err;
b0c632db 2734
b290411a
CO		 2735 rc = -ENOMEM;
2736
76a6dd72
DH		 2737 if (!sclp.has_64bscao)
2738 alloc_flags |= GFP_DMA;
5e044315 2739 rwlock_init(&kvm->arch.sca_lock);
9ac96d75 2740 /* start with basic SCA */
76a6dd72 2741 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
b0c632db 2742 if (!kvm->arch.sca)
d89f5eff 2743 goto out_err;
0d9ce162 2744 mutex_lock(&kvm_lock);
c5c2c393 2745 sca_offset += 16;
bc784cce 2746 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
c5c2c393 2747 sca_offset = 0;
bc784cce
ED		 2748 kvm->arch.sca = (struct bsca_block *)
2749 ((char *) kvm->arch.sca + sca_offset);
0d9ce162 2750 mutex_unlock(&kvm_lock);
b0c632db
HC		 2751
2752 sprintf(debug_name, "kvm-%u", current->pid);
2753
1cb9cf72 2754 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
b0c632db 2755 if (!kvm->arch.dbf)
40f5b735 2756 goto out_err;
b0c632db 2757
19114beb 2758 BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
c54f0d6a
DH		 2759 kvm->arch.sie_page2 =
2760 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
2761 if (!kvm->arch.sie_page2)
40f5b735 2762 goto out_err;
9d8d5786 2763
25c84dba 2764 kvm->arch.sie_page2->kvm = kvm;
c54f0d6a 2765 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
c3b9e3e1
CB		 2766
2767 for (i = 0; i < kvm_s390_fac_size(); i++) {
2768 kvm->arch.model.fac_mask[i] = S390_lowcore.stfle_fac_list[i] &
2769 (kvm_s390_fac_base[i] |
2770 kvm_s390_fac_ext[i]);
2771 kvm->arch.model.fac_list[i] = S390_lowcore.stfle_fac_list[i] &
2772 kvm_s390_fac_base[i];
2773 }
346fa2f8 2774 kvm->arch.model.subfuncs = kvm_s390_available_subfunc;
981467c9 2775
1935222d
DH		 2776 /* we are always in czam mode - even on pre z14 machines */
2777 set_kvm_facility(kvm->arch.model.fac_mask, 138);
2778 set_kvm_facility(kvm->arch.model.fac_list, 138);
2779 /* we emulate STHYI in kvm */
95ca2cb5
JF		 2780 set_kvm_facility(kvm->arch.model.fac_mask, 74);
2781 set_kvm_facility(kvm->arch.model.fac_list, 74);
1bab1c02
CI		 2782 if (MACHINE_HAS_TLB_GUEST) {
2783 set_kvm_facility(kvm->arch.model.fac_mask, 147);
2784 set_kvm_facility(kvm->arch.model.fac_list, 147);
2785 }
95ca2cb5 2786
05f31e3b
PM		 2787 if (css_general_characteristics.aiv && test_facility(65))
2788 set_kvm_facility(kvm->arch.model.fac_mask, 65);
2789
9bb0ec09 2790 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
37c5f6c8 2791 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
9d8d5786 2792
c54f0d6a 2793 kvm_s390_crypto_init(kvm);
5102ee87 2794
51978393 2795 mutex_init(&kvm->arch.float_int.ais_lock);
ba5c1e9b 2796 spin_lock_init(&kvm->arch.float_int.lock);
6d3da241
JF		 2797 for (i = 0; i < FIRQ_LIST_COUNT; i++)
2798 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
8a242234 2799 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 2800 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 2801
b0c632db 2802 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
78f26131 2803 VM_EVENT(kvm, 3, "vm created with type %lu", type);
b0c632db 2804
e08b9637
CO		 2805 if (type & KVM_VM_S390_UCONTROL) {
2806 kvm->arch.gmap = NULL;
a3a92c31 2807 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
e08b9637 2808 } else {
32e6b236 2809 if (sclp.hamax == U64_MAX)
ee71d16d 2810 kvm->arch.mem_limit = TASK_SIZE_MAX;
32e6b236 2811 else
ee71d16d 2812 kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
32e6b236 2813 sclp.hamax + 1);
6ea427bb 2814 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
e08b9637 2815 if (!kvm->arch.gmap)
40f5b735 2816 goto out_err;
2c70fe44 2817 kvm->arch.gmap->private = kvm;
24eb3a82 2818 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 2819 }
fa6b7fe9 2820
c9f0a2b8 2821 kvm->arch.use_pfmfi = sclp.has_pfmfi;
55531b74 2822 kvm->arch.use_skf = sclp.has_skey;
8ad35755 2823 spin_lock_init(&kvm->arch.start_stop_lock);
a3508fbe 2824 kvm_s390_vsie_init(kvm);
d7c5cb01 2825 kvm_s390_gisa_init(kvm);
8335713a 2826 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
8ad35755 2827
d89f5eff 2828 return 0;
40f5b735 2829out_err:
c54f0d6a 2830 free_page((unsigned long)kvm->arch.sie_page2);
598841ca 2831 debug_unregister(kvm->arch.dbf);
7d43bafc 2832 sca_dispose(kvm);
78f26131 2833 KVM_EVENT(3, "creation of vm failed: %d", rc);
d89f5eff 2834 return rc;
b0c632db
HC		 2835}
2836
d329c035
CB		 2837void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
2838{
e2ed2e2d
JF		 2839 u16 rc, rrc;
2840
d329c035 2841 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 2842 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 2843 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 2844 kvm_clear_async_pf_completion_queue(vcpu);
bc784cce 2845 if (!kvm_is_ucontrol(vcpu->kvm))
a6e2f683 2846 sca_del_vcpu(vcpu);
27e0393f
CO		 2847
2848 if (kvm_is_ucontrol(vcpu->kvm))
6ea427bb 2849 gmap_remove(vcpu->arch.gmap);
27e0393f 2850
e6db1d61 2851 if (vcpu->kvm->arch.use_cmma)
b31605c1 2852 kvm_s390_vcpu_unsetup_cmma(vcpu);
e2ed2e2d
JF		 2853 /* We can not hold the vcpu mutex here, we are already dying */
2854 if (kvm_s390_pv_cpu_get_handle(vcpu))
2855 kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc);
d329c035 2856 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 2857
6692cef3 2858 kvm_vcpu_uninit(vcpu);
b110feaf 2859 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 2860}
2861
2862static void kvm_free_vcpus(struct kvm *kvm)
2863{
2864 unsigned int i;
988a2cae 2865 struct kvm_vcpu *vcpu;
d329c035 2866
988a2cae
GN		 2867 kvm_for_each_vcpu(i, vcpu, kvm)
2868 kvm_arch_vcpu_destroy(vcpu);
2869
2870 mutex_lock(&kvm->lock);
2871 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
2872 kvm->vcpus[i] = NULL;
2873
2874 atomic_set(&kvm->online_vcpus, 0);
2875 mutex_unlock(&kvm->lock);
d329c035
CB		 2876}
2877
b0c632db
HC		 2878void kvm_arch_destroy_vm(struct kvm *kvm)
2879{
e2ed2e2d
JF		 2880 u16 rc, rrc;
2881
d329c035 2882 kvm_free_vcpus(kvm);
7d43bafc 2883 sca_dispose(kvm);
d7c5cb01 2884 kvm_s390_gisa_destroy(kvm);
e2ed2e2d
JF		 2885 /*
2886 * We are already at the end of life and kvm->lock is not taken.
2887 * This is ok as the file descriptor is closed by now and nobody
2888 * can mess with the pv state. To avoid lockdep_assert_held from
2889 * complaining we do not use kvm_s390_pv_is_protected.
2890 */
2891 if (kvm_s390_pv_get_handle(kvm))
2892 kvm_s390_pv_deinit_vm(kvm, &rc, &rrc);
2893 debug_unregister(kvm->arch.dbf);
c54f0d6a 2894 free_page((unsigned long)kvm->arch.sie_page2);
27e0393f 2895 if (!kvm_is_ucontrol(kvm))
6ea427bb 2896 gmap_remove(kvm->arch.gmap);
841b91c5 2897 kvm_s390_destroy_adapters(kvm);
67335e63 2898 kvm_s390_clear_float_irqs(kvm);
a3508fbe 2899 kvm_s390_vsie_destroy(kvm);
8335713a 2900 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
b0c632db
HC		 2901}
2902
2903/* Section: vcpu related */
dafd032a
DD		 2904static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
2905{
6ea427bb 2906 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
dafd032a
DD		 2907 if (!vcpu->arch.gmap)
2908 return -ENOMEM;
2909 vcpu->arch.gmap->private = vcpu->kvm;
2910
2911 return 0;
2912}
2913
a6e2f683
ED		 2914static void sca_del_vcpu(struct kvm_vcpu *vcpu)
2915{
a6940674
DH		 2916 if (!kvm_s390_use_sca_entries())
2917 return;
5e044315 2918 read_lock(&vcpu->kvm->arch.sca_lock);
7d43bafc
ED		 2919 if (vcpu->kvm->arch.use_esca) {
2920 struct esca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 2921
7d43bafc 2922 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
10ce32d5 2923 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc
ED		 2924 } else {
2925 struct bsca_block *sca = vcpu->kvm->arch.sca;
2926
2927 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
10ce32d5 2928 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc 2929 }
5e044315 2930 read_unlock(&vcpu->kvm->arch.sca_lock);
a6e2f683
ED		 2931}
2932
eaa78f34 2933static void sca_add_vcpu(struct kvm_vcpu *vcpu)
a6e2f683 2934{
a6940674
DH		 2935 if (!kvm_s390_use_sca_entries()) {
2936 struct bsca_block *sca = vcpu->kvm->arch.sca;
2937
2938 /* we still need the basic sca for the ipte control */
2939 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2940 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
f07afa04 2941 return;
a6940674 2942 }
eaa78f34
DH		 2943 read_lock(&vcpu->kvm->arch.sca_lock);
2944 if (vcpu->kvm->arch.use_esca) {
2945 struct esca_block *sca = vcpu->kvm->arch.sca;
7d43bafc 2946
eaa78f34 2947 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 2948 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2949 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
0c9d8683 2950 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
eaa78f34 2951 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
7d43bafc 2952 } else {
eaa78f34 2953 struct bsca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 2954
eaa78f34 2955 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 2956 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2957 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
eaa78f34 2958 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
7d43bafc 2959 }
eaa78f34 2960 read_unlock(&vcpu->kvm->arch.sca_lock);
5e044315
ED		 2961}
2962
2963/* Basic SCA to Extended SCA data copy routines */
2964static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
2965{
2966 d->sda = s->sda;
2967 d->sigp_ctrl.c = s->sigp_ctrl.c;
2968 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
2969}
2970
2971static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
2972{
2973 int i;
2974
2975 d->ipte_control = s->ipte_control;
2976 d->mcn[0] = s->mcn;
2977 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
2978 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
2979}
2980
2981static int sca_switch_to_extended(struct kvm *kvm)
2982{
2983 struct bsca_block *old_sca = kvm->arch.sca;
2984 struct esca_block *new_sca;
2985 struct kvm_vcpu *vcpu;
2986 unsigned int vcpu_idx;
2987 u32 scaol, scaoh;
2988
e2ed2e2d
JF		 2989 if (kvm->arch.use_esca)
2990 return 0;
2991
5e044315
ED		 2992 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
2993 if (!new_sca)
2994 return -ENOMEM;
2995
2996 scaoh = (u32)((u64)(new_sca) >> 32);
2997 scaol = (u32)(u64)(new_sca) & ~0x3fU;
2998
2999 kvm_s390_vcpu_block_all(kvm);
3000 write_lock(&kvm->arch.sca_lock);
3001
3002 sca_copy_b_to_e(new_sca, old_sca);
3003
3004 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
3005 vcpu->arch.sie_block->scaoh = scaoh;
3006 vcpu->arch.sie_block->scaol = scaol;
0c9d8683 3007 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
5e044315
ED		 3008 }
3009 kvm->arch.sca = new_sca;
3010 kvm->arch.use_esca = 1;
3011
3012 write_unlock(&kvm->arch.sca_lock);
3013 kvm_s390_vcpu_unblock_all(kvm);
3014
3015 free_page((unsigned long)old_sca);
3016
8335713a
CB		 3017 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
3018 old_sca, kvm->arch.sca);
5e044315 3019 return 0;
a6e2f683
ED		 3020}
3021
3022static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
3023{
5e044315
ED		 3024 int rc;
3025
a6940674
DH		 3026 if (!kvm_s390_use_sca_entries()) {
3027 if (id < KVM_MAX_VCPUS)
3028 return true;
3029 return false;
3030 }
5e044315
ED		 3031 if (id < KVM_S390_BSCA_CPU_SLOTS)
3032 return true;
76a6dd72 3033 if (!sclp.has_esca || !sclp.has_64bscao)
5e044315
ED		 3034 return false;
3035
3036 mutex_lock(&kvm->lock);
3037 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
3038 mutex_unlock(&kvm->lock);
3039
3040 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
a6e2f683
ED		 3041}
3042
b0c632db
HC		 3043int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
3044{
3c038e6b
DD		 3045 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
3046 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 3047 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
3048 KVM_SYNC_GPRS |
9eed0735 3049 KVM_SYNC_ACRS |
b028ee3e
DH		 3050 KVM_SYNC_CRS |
3051 KVM_SYNC_ARCH0 |
dad38237
CW		 3052 KVM_SYNC_PFAULT |
3053 KVM_SYNC_DIAG318;
75a4615c 3054 kvm_s390_set_prefix(vcpu, 0);
c6e5f166
FZ		 3055 if (test_kvm_facility(vcpu->kvm, 64))
3056 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
35b3fde6
CB		 3057 if (test_kvm_facility(vcpu->kvm, 82))
3058 vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
4e0b1ab7
FZ		 3059 if (test_kvm_facility(vcpu->kvm, 133))
3060 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
a3da7b4a
CB		 3061 if (test_kvm_facility(vcpu->kvm, 156))
3062 vcpu->run->kvm_valid_regs |= KVM_SYNC_ETOKEN;
f6aa6dc4
DH		 3063 /* fprs can be synchronized via vrs, even if the guest has no vx. With
3064 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
3065 */
3066 if (MACHINE_HAS_VX)
68c55750 3067 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
6fd8e67d
DH		 3068 else
3069 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
dafd032a
DD		 3070
3071 if (kvm_is_ucontrol(vcpu->kvm))
3072 return __kvm_ucontrol_vcpu_init(vcpu);
3073
b0c632db
HC		 3074 return 0;
3075}
3076
db0758b2
DH		 3077/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
3078static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3079{
3080 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
9c23a131 3081 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2 3082 vcpu->arch.cputm_start = get_tod_clock_fast();
9c23a131 3083 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 3084}
3085
3086/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
3087static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3088{
3089 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
9c23a131 3090 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 3091 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
3092 vcpu->arch.cputm_start = 0;
9c23a131 3093 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 3094}
3095
3096/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
3097static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3098{
3099 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
3100 vcpu->arch.cputm_enabled = true;
3101 __start_cpu_timer_accounting(vcpu);
3102}
3103
3104/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
3105static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3106{
3107 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
3108 __stop_cpu_timer_accounting(vcpu);
3109 vcpu->arch.cputm_enabled = false;
3110}
3111
3112static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3113{
3114 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3115 __enable_cpu_timer_accounting(vcpu);
3116 preempt_enable();
3117}
3118
3119static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3120{
3121 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3122 __disable_cpu_timer_accounting(vcpu);
3123 preempt_enable();
3124}
3125
4287f247
DH		 3126/* set the cpu timer - may only be called from the VCPU thread itself */
3127void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
3128{
db0758b2 3129 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
9c23a131 3130 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 3131 if (vcpu->arch.cputm_enabled)
3132 vcpu->arch.cputm_start = get_tod_clock_fast();
4287f247 3133 vcpu->arch.sie_block->cputm = cputm;
9c23a131 3134 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2 3135 preempt_enable();
4287f247
DH		 3136}
3137
db0758b2 3138/* update and get the cpu timer - can also be called from other VCPU threads */
4287f247
DH		 3139__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
3140{
9c23a131 3141 unsigned int seq;
db0758b2 3142 __u64 value;
db0758b2
DH		 3143
3144 if (unlikely(!vcpu->arch.cputm_enabled))
3145 return vcpu->arch.sie_block->cputm;
3146
9c23a131
DH		 3147 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3148 do {
3149 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
3150 /*
3151 * If the writer would ever execute a read in the critical
3152 * section, e.g. in irq context, we have a deadlock.
3153 */
3154 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
3155 value = vcpu->arch.sie_block->cputm;
3156 /* if cputm_start is 0, accounting is being started/stopped */
3157 if (likely(vcpu->arch.cputm_start))
3158 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
3159 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
3160 preempt_enable();
db0758b2 3161 return value;
4287f247
DH		 3162}
3163
b0c632db
HC		 3164void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
3165{
9977e886 3166
37d9df98 3167 gmap_enable(vcpu->arch.enabled_gmap);
ef8f4f49 3168 kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
5ebda316 3169 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 3170 __start_cpu_timer_accounting(vcpu);
01a745ac 3171 vcpu->cpu = cpu;
b0c632db
HC		 3172}
3173
3174void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
3175{
01a745ac 3176 vcpu->cpu = -1;
5ebda316 3177 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 3178 __stop_cpu_timer_accounting(vcpu);
9daecfc6 3179 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
37d9df98
DH		 3180 vcpu->arch.enabled_gmap = gmap_get_enabled();
3181 gmap_disable(vcpu->arch.enabled_gmap);
9977e886 3182
b0c632db
HC		 3183}
3184
31928aa5 3185void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 3186{
72f25020 3187 mutex_lock(&vcpu->kvm->lock);
fdf03650 3188 preempt_disable();
72f25020 3189 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
d16b52cb 3190 vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
fdf03650 3191 preempt_enable();
72f25020 3192 mutex_unlock(&vcpu->kvm->lock);
25508824 3193 if (!kvm_is_ucontrol(vcpu->kvm)) {
dafd032a 3194 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
eaa78f34 3195 sca_add_vcpu(vcpu);
25508824 3196 }
6502a34c
DH		 3197 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
3198 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
37d9df98
DH		 3199 /* make vcpu_load load the right gmap on the first trigger */
3200 vcpu->arch.enabled_gmap = vcpu->arch.gmap;
42897d86
MT		 3201}
3202
8ec2fa52
CB		 3203static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr)
3204{
3205 if (test_bit_inv(nr, (unsigned long *)&kvm->arch.model.subfuncs.pckmo) &&
3206 test_bit_inv(nr, (unsigned long *)&kvm_s390_available_subfunc.pckmo))
3207 return true;
3208 return false;
3209}
3210
3211static bool kvm_has_pckmo_ecc(struct kvm *kvm)
3212{
3213 /* At least one ECC subfunction must be present */
3214 return kvm_has_pckmo_subfunc(kvm, 32) ||
3215 kvm_has_pckmo_subfunc(kvm, 33) ||
3216 kvm_has_pckmo_subfunc(kvm, 34) ||
3217 kvm_has_pckmo_subfunc(kvm, 40) ||
3218 kvm_has_pckmo_subfunc(kvm, 41);
3219
3220}
3221
5102ee87
TK		 3222static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
3223{
e585b24a
TK		 3224 /*
3225 * If the AP instructions are not being interpreted and the MSAX3
3226 * facility is not configured for the guest, there is nothing to set up.
3227 */
3228 if (!vcpu->kvm->arch.crypto.apie && !test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 3229 return;
3230
e585b24a 3231 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
a374e892 3232 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
37940fb0 3233 vcpu->arch.sie_block->eca &= ~ECA_APIE;
8ec2fa52 3234 vcpu->arch.sie_block->ecd &= ~ECD_ECC;
a374e892 3235
e585b24a
TK		 3236 if (vcpu->kvm->arch.crypto.apie)
3237 vcpu->arch.sie_block->eca |= ECA_APIE;
a374e892 3238
e585b24a 3239 /* Set up protected key support */
8ec2fa52 3240 if (vcpu->kvm->arch.crypto.aes_kw) {
a374e892 3241 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
8ec2fa52
CB		 3242 /* ecc is also wrapped with AES key */
3243 if (kvm_has_pckmo_ecc(vcpu->kvm))
3244 vcpu->arch.sie_block->ecd |= ECD_ECC;
3245 }
3246
a374e892
TK		 3247 if (vcpu->kvm->arch.crypto.dea_kw)
3248 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
5102ee87
TK		 3249}
3250
b31605c1
DD		 3251void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
3252{
3253 free_page(vcpu->arch.sie_block->cbrlo);
3254 vcpu->arch.sie_block->cbrlo = 0;
3255}
3256
3257int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
3258{
3259 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
3260 if (!vcpu->arch.sie_block->cbrlo)
3261 return -ENOMEM;
b31605c1
DD		 3262 return 0;
3263}
3264
91520f1a
MM		 3265static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
3266{
3267 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
3268
91520f1a 3269 vcpu->arch.sie_block->ibc = model->ibc;
80bc79dc 3270 if (test_kvm_facility(vcpu->kvm, 7))
c54f0d6a 3271 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
91520f1a
MM		 3272}
3273
b0c632db
HC		 3274int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
3275{
b31605c1 3276 int rc = 0;
e2ed2e2d 3277 u16 uvrc, uvrrc;
b31288fa 3278
9e6dabef
CH		 3279 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
3280 CPUSTAT_SM |
a4a4f191
GH		 3281 CPUSTAT_STOPPED);
3282
53df84f8 3283 if (test_kvm_facility(vcpu->kvm, 78))
ef8f4f49 3284 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
53df84f8 3285 else if (test_kvm_facility(vcpu->kvm, 8))
ef8f4f49 3286 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
a4a4f191 3287
91520f1a
MM		 3288 kvm_s390_vcpu_setup_model(vcpu);
3289
bdab09f3
DH		 3290 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
3291 if (MACHINE_HAS_ESOP)
0c9d8683 3292 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
bd50e8ec 3293 if (test_kvm_facility(vcpu->kvm, 9))
0c9d8683 3294 vcpu->arch.sie_block->ecb |= ECB_SRSI;
f597d24e 3295 if (test_kvm_facility(vcpu->kvm, 73))
0c9d8683 3296 vcpu->arch.sie_block->ecb |= ECB_TE;
7feb6bb8 3297
c9f0a2b8 3298 if (test_kvm_facility(vcpu->kvm, 8) && vcpu->kvm->arch.use_pfmfi)
0c9d8683 3299 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
cd1836f5 3300 if (test_kvm_facility(vcpu->kvm, 130))
0c9d8683
DH		 3301 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
3302 vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
48ee7d3a 3303 if (sclp.has_cei)
0c9d8683 3304 vcpu->arch.sie_block->eca |= ECA_CEI;
11ad65b7 3305 if (sclp.has_ib)
0c9d8683 3306 vcpu->arch.sie_block->eca |= ECA_IB;
37c5f6c8 3307 if (sclp.has_siif)
0c9d8683 3308 vcpu->arch.sie_block->eca |= ECA_SII;
37c5f6c8 3309 if (sclp.has_sigpif)
0c9d8683 3310 vcpu->arch.sie_block->eca |= ECA_SIGPI;
18280d8b 3311 if (test_kvm_facility(vcpu->kvm, 129)) {
0c9d8683
DH		 3312 vcpu->arch.sie_block->eca |= ECA_VX;
3313 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
13211ea7 3314 }
8fa1696e
CW		 3315 if (test_kvm_facility(vcpu->kvm, 139))
3316 vcpu->arch.sie_block->ecd |= ECD_MEF;
a3da7b4a
CB		 3317 if (test_kvm_facility(vcpu->kvm, 156))
3318 vcpu->arch.sie_block->ecd |= ECD_ETOKENF;
d7c5cb01
MM		 3319 if (vcpu->arch.sie_block->gd) {
3320 vcpu->arch.sie_block->eca |= ECA_AIV;
3321 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
3322 vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
3323 }
4e0b1ab7
FZ		 3324 vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
3325 | SDNXC;
c6e5f166 3326 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
730cd632
FA		 3327
3328 if (sclp.has_kss)
ef8f4f49 3329 kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
730cd632
FA		 3330 else
3331 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
5a5e6536 3332
e6db1d61 3333 if (vcpu->kvm->arch.use_cmma) {
b31605c1
DD		 3334 rc = kvm_s390_vcpu_setup_cmma(vcpu);
3335 if (rc)
3336 return rc;
b31288fa 3337 }
0ac96caf 3338 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 3339 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 3340
67d49d52
CW		 3341 vcpu->arch.sie_block->hpid = HPID_KVM;
3342
5102ee87
TK		 3343 kvm_s390_vcpu_crypto_setup(vcpu);
3344
e2ed2e2d
JF		 3345 mutex_lock(&vcpu->kvm->lock);
3346 if (kvm_s390_pv_is_protected(vcpu->kvm)) {
3347 rc = kvm_s390_pv_create_cpu(vcpu, &uvrc, &uvrrc);
3348 if (rc)
3349 kvm_s390_vcpu_unsetup_cmma(vcpu);
3350 }
3351 mutex_unlock(&vcpu->kvm->lock);
3352
b31605c1 3353 return rc;
b0c632db
HC		 3354}
3355
3356struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
3357 unsigned int id)
3358{
4d47555a 3359 struct kvm_vcpu *vcpu;
7feb6bb8 3360 struct sie_page *sie_page;
4d47555a
CO		 3361 int rc = -EINVAL;
3362
4215825e 3363 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
4d47555a
CO		 3364 goto out;
3365
3366 rc = -ENOMEM;
b0c632db 3367
b110feaf 3368 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 3369 if (!vcpu)
4d47555a 3370 goto out;
b0c632db 3371
da72ca4d 3372 BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
7feb6bb8
MM		 3373 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
3374 if (!sie_page)
b0c632db
HC		 3375 goto out_free_cpu;
3376
7feb6bb8
MM		 3377 vcpu->arch.sie_block = &sie_page->sie_block;
3378 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
3379
efed1104
DH		 3380 /* the real guest size will always be smaller than msl */
3381 vcpu->arch.sie_block->mso = 0;
3382 vcpu->arch.sie_block->msl = sclp.hamax;
3383
b0c632db 3384 vcpu->arch.sie_block->icpua = id;
ba5c1e9b 3385 spin_lock_init(&vcpu->arch.local_int.lock);
982cff42 3386 vcpu->arch.sie_block->gd = (u32)(u64)kvm->arch.gisa_int.origin;
4b9f9525
MM		 3387 if (vcpu->arch.sie_block->gd && sclp.has_gisaf)
3388 vcpu->arch.sie_block->gd |= GISA_FORMAT1;
9c23a131 3389 seqcount_init(&vcpu->arch.cputm_seqcount);
ba5c1e9b 3390
b0c632db
HC		 3391 rc = kvm_vcpu_init(vcpu, kvm, id);
3392 if (rc)
9abc2a08 3393 goto out_free_sie_block;
8335713a 3394 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
b0c632db 3395 vcpu->arch.sie_block);
ade38c31 3396 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 3397
b0c632db 3398 return vcpu;
7b06bf2f
WY		 3399out_free_sie_block:
3400 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 3401out_free_cpu:
b110feaf 3402 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 3403out:
b0c632db
HC		 3404 return ERR_PTR(rc);
3405}
3406
b0c632db
HC		 3407int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
3408{
727abd33 3409 clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.gisa_int.kicked_mask);
9a022067 3410 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 3411}
3412
199b5763
LM		 3413bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
3414{
0546c63d 3415 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
199b5763
LM		 3416}
3417
27406cd5 3418void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
49b99e1e 3419{
805de8f4 3420 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
61a6df54 3421 exit_sie(vcpu);
49b99e1e
CB		 3422}
3423
27406cd5 3424void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
49b99e1e 3425{
805de8f4 3426 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
49b99e1e
CB		 3427}
3428
8e236546
CB		 3429static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
3430{
805de8f4 3431 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
61a6df54 3432 exit_sie(vcpu);
8e236546
CB		 3433}
3434
9ea59728
DH		 3435bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu)
3436{
3437 return atomic_read(&vcpu->arch.sie_block->prog20) &
3438 (PROG_BLOCK_SIE | PROG_REQUEST);
3439}
3440
8e236546
CB		 3441static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
3442{
9bf9fde2 3443 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
8e236546
CB		 3444}
3445
49b99e1e 3446/*
9ea59728 3447 * Kick a guest cpu out of (v)SIE and wait until (v)SIE is not running.
49b99e1e
CB		 3448 * If the CPU is not running (e.g. waiting as idle) the function will
3449 * return immediately. */
3450void exit_sie(struct kvm_vcpu *vcpu)
3451{
ef8f4f49 3452 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
9ea59728 3453 kvm_s390_vsie_kick(vcpu);
49b99e1e
CB		 3454 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
3455 cpu_relax();
3456}
3457
8e236546
CB		 3458/* Kick a guest cpu out of SIE to process a request synchronously */
3459void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
49b99e1e 3460{
8e236546
CB		 3461 kvm_make_request(req, vcpu);
3462 kvm_s390_vcpu_request(vcpu);
49b99e1e
CB		 3463}
3464
414d3b07
MS		 3465static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
3466 unsigned long end)
2c70fe44 3467{
2c70fe44
CB		 3468 struct kvm *kvm = gmap->private;
3469 struct kvm_vcpu *vcpu;
414d3b07
MS		 3470 unsigned long prefix;
3471 int i;
2c70fe44 3472
65d0b0d4
DH		 3473 if (gmap_is_shadow(gmap))
3474 return;
414d3b07
MS		 3475 if (start >= 1UL << 31)
3476 /* We are only interested in prefix pages */
3477 return;
2c70fe44
CB		 3478 kvm_for_each_vcpu(i, vcpu, kvm) {
3479 /* match against both prefix pages */
414d3b07
MS		 3480 prefix = kvm_s390_get_prefix(vcpu);
3481 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
3482 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
3483 start, end);
8e236546 3484 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44
CB		 3485 }
3486 }
3487}
3488
8b905d28
CB		 3489bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
3490{
3491 /* do not poll with more than halt_poll_max_steal percent of steal time */
3492 if (S390_lowcore.avg_steal_timer * 100 / (TICK_USEC << 12) >=
3493 halt_poll_max_steal) {
3494 vcpu->stat.halt_no_poll_steal++;
3495 return true;
3496 }
3497 return false;
3498}
3499
b6d33834
CD		 3500int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
3501{
3502 /* kvm common code refers to this, but never calls it */
3503 BUG();
3504 return 0;
3505}
3506
14eebd91
CO		 3507static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
3508 struct kvm_one_reg *reg)
3509{
3510 int r = -EINVAL;
3511
3512 switch (reg->id) {
29b7c71b
CO		 3513 case KVM_REG_S390_TODPR:
3514 r = put_user(vcpu->arch.sie_block->todpr,
3515 (u32 __user *)reg->addr);
3516 break;
3517 case KVM_REG_S390_EPOCHDIFF:
3518 r = put_user(vcpu->arch.sie_block->epoch,
3519 (u64 __user *)reg->addr);
3520 break;
46a6dd1c 3521 case KVM_REG_S390_CPU_TIMER:
4287f247 3522 r = put_user(kvm_s390_get_cpu_timer(vcpu),
46a6dd1c
J		 3523 (u64 __user *)reg->addr);
3524 break;
3525 case KVM_REG_S390_CLOCK_COMP:
3526 r = put_user(vcpu->arch.sie_block->ckc,
3527 (u64 __user *)reg->addr);
3528 break;
536336c2
DD		 3529 case KVM_REG_S390_PFTOKEN:
3530 r = put_user(vcpu->arch.pfault_token,
3531 (u64 __user *)reg->addr);
3532 break;
3533 case KVM_REG_S390_PFCOMPARE:
3534 r = put_user(vcpu->arch.pfault_compare,
3535 (u64 __user *)reg->addr);
3536 break;
3537 case KVM_REG_S390_PFSELECT:
3538 r = put_user(vcpu->arch.pfault_select,
3539 (u64 __user *)reg->addr);
3540 break;
672550fb
CB		 3541 case KVM_REG_S390_PP:
3542 r = put_user(vcpu->arch.sie_block->pp,
3543 (u64 __user *)reg->addr);
3544 break;
afa45ff5
CB		 3545 case KVM_REG_S390_GBEA:
3546 r = put_user(vcpu->arch.sie_block->gbea,
3547 (u64 __user *)reg->addr);
3548 break;
14eebd91
CO		 3549 default:
3550 break;
3551 }
3552
3553 return r;
3554}
3555
3556static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
3557 struct kvm_one_reg *reg)
3558{
3559 int r = -EINVAL;
4287f247 3560 __u64 val;
14eebd91
CO		 3561
3562 switch (reg->id) {
29b7c71b
CO		 3563 case KVM_REG_S390_TODPR:
3564 r = get_user(vcpu->arch.sie_block->todpr,
3565 (u32 __user *)reg->addr);
3566 break;
3567 case KVM_REG_S390_EPOCHDIFF:
3568 r = get_user(vcpu->arch.sie_block->epoch,
3569 (u64 __user *)reg->addr);
3570 break;
46a6dd1c 3571 case KVM_REG_S390_CPU_TIMER:
4287f247
DH		 3572 r = get_user(val, (u64 __user *)reg->addr);
3573 if (!r)
3574 kvm_s390_set_cpu_timer(vcpu, val);
46a6dd1c
J		 3575 break;
3576 case KVM_REG_S390_CLOCK_COMP:
3577 r = get_user(vcpu->arch.sie_block->ckc,
3578 (u64 __user *)reg->addr);
3579 break;
536336c2
DD		 3580 case KVM_REG_S390_PFTOKEN:
3581 r = get_user(vcpu->arch.pfault_token,
3582 (u64 __user *)reg->addr);
9fbd8082
DH		 3583 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3584 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 3585 break;
3586 case KVM_REG_S390_PFCOMPARE:
3587 r = get_user(vcpu->arch.pfault_compare,
3588 (u64 __user *)reg->addr);
3589 break;
3590 case KVM_REG_S390_PFSELECT:
3591 r = get_user(vcpu->arch.pfault_select,
3592 (u64 __user *)reg->addr);
3593 break;
672550fb
CB		 3594 case KVM_REG_S390_PP:
3595 r = get_user(vcpu->arch.sie_block->pp,
3596 (u64 __user *)reg->addr);
3597 break;
afa45ff5
CB		 3598 case KVM_REG_S390_GBEA:
3599 r = get_user(vcpu->arch.sie_block->gbea,
3600 (u64 __user *)reg->addr);
3601 break;
14eebd91
CO		 3602 default:
3603 break;
3604 }
3605
3606 return r;
3607}
b6d33834 3608
68369234 3609static void kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
b0c632db 3610{
68369234
JF		 3611 vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_RI;
3612 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
3613 memset(vcpu->run->s.regs.riccb, 0, sizeof(vcpu->run->s.regs.riccb));
3614
3615 kvm_clear_async_pf_completion_queue(vcpu);
3616 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
3617 kvm_s390_vcpu_stop(vcpu);
3618 kvm_s390_clear_local_irqs(vcpu);
3619}
3620
3621static void kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
3622{
3623 /* Initial reset is a superset of the normal reset */
3624 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
3625
3626 /* this equals initial cpu reset in pop, but we don't switch to ESA */
3627 vcpu->arch.sie_block->gpsw.mask = 0;
3628 vcpu->arch.sie_block->gpsw.addr = 0;
3629 kvm_s390_set_prefix(vcpu, 0);
3630 kvm_s390_set_cpu_timer(vcpu, 0);
3631 vcpu->arch.sie_block->ckc = 0;
68369234
JF		 3632 memset(vcpu->arch.sie_block->gcr, 0, sizeof(vcpu->arch.sie_block->gcr));
3633 vcpu->arch.sie_block->gcr[0] = CR0_INITIAL_MASK;
3634 vcpu->arch.sie_block->gcr[14] = CR14_INITIAL_MASK;
3635 vcpu->run->s.regs.fpc = 0;
6ad5db22
JF		 3636 /*
3637 * Do not reset these registers in the protected case, as some of
3638 * them are overlayed and they are not accessible in this case
3639 * anyway.
3640 */
3641 if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
3642 vcpu->arch.sie_block->gbea = 1;
3643 vcpu->arch.sie_block->pp = 0;
3644 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
3645 vcpu->arch.sie_block->todpr = 0;
3646 }
68369234
JF		 3647}
3648
3649static void kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
3650{
3651 struct kvm_sync_regs *regs = &vcpu->run->s.regs;
3652
3653 /* Clear reset is a superset of the initial reset */
3654 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3655
3656 memset(&regs->gprs, 0, sizeof(regs->gprs));
3657 memset(&regs->vrs, 0, sizeof(regs->vrs));
3658 memset(&regs->acrs, 0, sizeof(regs->acrs));
3659 memset(&regs->gscb, 0, sizeof(regs->gscb));
3660
3661 regs->etoken = 0;
3662 regs->etoken_extension = 0;
b0c632db
HC		 3663}
3664
3665int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3666{
875656fe 3667 vcpu_load(vcpu);
5a32c1af 3668 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
875656fe 3669 vcpu_put(vcpu);
b0c632db
HC		 3670 return 0;
3671}
3672
3673int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3674{
1fc9b76b 3675 vcpu_load(vcpu);
5a32c1af 3676 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1fc9b76b 3677 vcpu_put(vcpu);
b0c632db
HC		 3678 return 0;
3679}
3680
3681int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
3682 struct kvm_sregs *sregs)
3683{
b4ef9d4e
CD		 3684 vcpu_load(vcpu);
3685
59674c1a 3686 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 3687 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
b4ef9d4e
CD		 3688
3689 vcpu_put(vcpu);
b0c632db
HC		 3690 return 0;
3691}
3692
3693int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
3694 struct kvm_sregs *sregs)
3695{
bcdec41c
CD		 3696 vcpu_load(vcpu);
3697
59674c1a 3698 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 3699 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
bcdec41c
CD		 3700
3701 vcpu_put(vcpu);
b0c632db
HC		 3702 return 0;
3703}
3704
3705int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3706{
6a96bc7f
CD		 3707 int ret = 0;
3708
3709 vcpu_load(vcpu);
3710
3711 if (test_fp_ctl(fpu->fpc)) {
3712 ret = -EINVAL;
3713 goto out;
3714 }
e1788bb9 3715 vcpu->run->s.regs.fpc = fpu->fpc;
9abc2a08 3716 if (MACHINE_HAS_VX)
a7d4b8f2
DH		 3717 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
3718 (freg_t *) fpu->fprs);
9abc2a08 3719 else
a7d4b8f2 3720 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
6a96bc7f
CD		 3721
3722out:
3723 vcpu_put(vcpu);
3724 return ret;
b0c632db
HC		 3725}
3726
3727int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3728{
1393123e
CD		 3729 vcpu_load(vcpu);
3730
9abc2a08
DH		 3731 /* make sure we have the latest values */
3732 save_fpu_regs();
3733 if (MACHINE_HAS_VX)
a7d4b8f2
DH		 3734 convert_vx_to_fp((freg_t *) fpu->fprs,
3735 (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08 3736 else
a7d4b8f2 3737 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
e1788bb9 3738 fpu->fpc = vcpu->run->s.regs.fpc;
1393123e
CD		 3739
3740 vcpu_put(vcpu);
b0c632db
HC		 3741 return 0;
3742}
3743
3744static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
3745{
3746 int rc = 0;
3747
7a42fdc2 3748 if (!is_vcpu_stopped(vcpu))
b0c632db 3749 rc = -EBUSY;
d7b0b5eb
CO		 3750 else {
3751 vcpu->run->psw_mask = psw.mask;
3752 vcpu->run->psw_addr = psw.addr;
3753 }
b0c632db
HC		 3754 return rc;
3755}
3756
3757int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
3758 struct kvm_translation *tr)
3759{
3760 return -EINVAL; /* not implemented yet */
3761}
3762
27291e21
DH		 3763#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
3764 KVM_GUESTDBG_USE_HW_BP | \
3765 KVM_GUESTDBG_ENABLE)
3766
d0bfb940
JK		 3767int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
3768 struct kvm_guest_debug *dbg)
b0c632db 3769{
27291e21
DH		 3770 int rc = 0;
3771
66b56562
CD		 3772 vcpu_load(vcpu);
3773
27291e21
DH		 3774 vcpu->guest_debug = 0;
3775 kvm_s390_clear_bp_data(vcpu);
3776
66b56562
CD		 3777 if (dbg->control & ~VALID_GUESTDBG_FLAGS) {
3778 rc = -EINVAL;
3779 goto out;
3780 }
3781 if (!sclp.has_gpere) {
3782 rc = -EINVAL;
3783 goto out;
3784 }
27291e21
DH		 3785
3786 if (dbg->control & KVM_GUESTDBG_ENABLE) {
3787 vcpu->guest_debug = dbg->control;
3788 /* enforce guest PER */
ef8f4f49 3789 kvm_s390_set_cpuflags(vcpu, CPUSTAT_P);
27291e21
DH		 3790
3791 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
3792 rc = kvm_s390_import_bp_data(vcpu, dbg);
3793 } else {
9daecfc6 3794 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
27291e21
DH		 3795 vcpu->arch.guestdbg.last_bp = 0;
3796 }
3797
3798 if (rc) {
3799 vcpu->guest_debug = 0;
3800 kvm_s390_clear_bp_data(vcpu);
9daecfc6 3801 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
27291e21
DH		 3802 }
3803
66b56562
CD		 3804out:
3805 vcpu_put(vcpu);
27291e21 3806 return rc;
b0c632db
HC		 3807}
3808
62d9f0db
MT		 3809int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
3810 struct kvm_mp_state *mp_state)
3811{
fd232561
CD		 3812 int ret;
3813
3814 vcpu_load(vcpu);
3815
6352e4d2 3816 /* CHECK_STOP and LOAD are not supported yet */
fd232561
CD		 3817 ret = is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
3818 KVM_MP_STATE_OPERATING;
3819
3820 vcpu_put(vcpu);
3821 return ret;
62d9f0db
MT		 3822}
3823
3824int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
3825 struct kvm_mp_state *mp_state)
3826{
6352e4d2
DH		 3827 int rc = 0;
3828
e83dff5e
CD		 3829 vcpu_load(vcpu);
3830
6352e4d2 3831 /* user space knows about this interface - let it control the state */
efba558a 3832 kvm_s390_set_user_cpu_state_ctrl(vcpu->kvm);
6352e4d2
DH		 3833
3834 switch (mp_state->mp_state) {
3835 case KVM_MP_STATE_STOPPED:
1a9fe91e 3836 rc = kvm_s390_vcpu_stop(vcpu);
6352e4d2
DH		 3837 break;
3838 case KVM_MP_STATE_OPERATING:
1a9fe91e 3839 rc = kvm_s390_vcpu_start(vcpu);
6352e4d2
DH		 3840 break;
3841 case KVM_MP_STATE_LOAD:
e1e83305
JF		 3842 if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
3843 rc = -ENXIO;
3844 break;
3845 }
3846 rc = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR_LOAD);
3847 break;
6352e4d2
DH		 3848 case KVM_MP_STATE_CHECK_STOP:
3849 /* fall through - CHECK_STOP and LOAD are not supported yet */
3850 default:
3851 rc = -ENXIO;
3852 }
3853
e83dff5e 3854 vcpu_put(vcpu);
6352e4d2 3855 return rc;
62d9f0db
MT		 3856}
3857
8ad35755
DH		 3858static bool ibs_enabled(struct kvm_vcpu *vcpu)
3859{
8d5fb0dc 3860 return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS);
8ad35755
DH		 3861}
3862
2c70fe44
CB		 3863static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
3864{
8ad35755 3865retry:
8e236546 3866 kvm_s390_vcpu_request_handled(vcpu);
2fa6e1e1 3867 if (!kvm_request_pending(vcpu))
586b7ccd 3868 return 0;
2c70fe44
CB		 3869 /*
3870 * We use MMU_RELOAD just to re-arm the ipte notifier for the
b2d73b2a 3871 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2c70fe44
CB		 3872 * This ensures that the ipte instruction for this request has
3873 * already finished. We might race against a second unmapper that
3874 * wants to set the blocking bit. Lets just retry the request loop.
3875 */
8ad35755 3876 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44 3877 int rc;
b2d73b2a
MS		 3878 rc = gmap_mprotect_notify(vcpu->arch.gmap,
3879 kvm_s390_get_prefix(vcpu),
3880 PAGE_SIZE * 2, PROT_WRITE);
aca411a4
JN		 3881 if (rc) {
3882 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44 3883 return rc;
aca411a4 3884 }
8ad35755 3885 goto retry;
2c70fe44 3886 }
8ad35755 3887
d3d692c8
DH		 3888 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
3889 vcpu->arch.sie_block->ihcpu = 0xffff;
3890 goto retry;
3891 }
3892
8ad35755
DH		 3893 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
3894 if (!ibs_enabled(vcpu)) {
3895 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
ef8f4f49 3896 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS);
8ad35755
DH		 3897 }
3898 goto retry;
2c70fe44 3899 }
8ad35755
DH		 3900
3901 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
3902 if (ibs_enabled(vcpu)) {
3903 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
9daecfc6 3904 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS);
8ad35755
DH		 3905 }
3906 goto retry;
3907 }
3908
6502a34c
DH		 3909 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
3910 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
3911 goto retry;
3912 }
3913
190df4a2
CI		 3914 if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
3915 /*
c9f0a2b8 3916 * Disable CMM virtualization; we will emulate the ESSA
190df4a2
CI		 3917 * instruction manually, in order to provide additional
3918 * functionalities needed for live migration.
3919 */
3920 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
3921 goto retry;
3922 }
3923
3924 if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
3925 /*
c9f0a2b8
JF		 3926 * Re-enable CMM virtualization if CMMA is available and
3927 * CMM has been used.
190df4a2
CI		 3928 */
3929 if ((vcpu->kvm->arch.use_cmma) &&
c9f0a2b8 3930 (vcpu->kvm->mm->context.uses_cmm))
190df4a2
CI		 3931 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
3932 goto retry;
3933 }
3934
0759d068 3935 /* nothing to do, just clear the request */
72875d8a 3936 kvm_clear_request(KVM_REQ_UNHALT, vcpu);
3194cdb7
DH		 3937 /* we left the vsie handler, nothing to do, just clear the request */
3938 kvm_clear_request(KVM_REQ_VSIE_RESTART, vcpu);
0759d068 3939
2c70fe44
CB		 3940 return 0;
3941}
3942
0e7def5f
DH		 3943void kvm_s390_set_tod_clock(struct kvm *kvm,
3944 const struct kvm_s390_vm_tod_clock *gtod)
8fa1696e
CW		 3945{
3946 struct kvm_vcpu *vcpu;
3947 struct kvm_s390_tod_clock_ext htod;
3948 int i;
3949
3950 mutex_lock(&kvm->lock);
3951 preempt_disable();
3952
3953 get_tod_clock_ext((char *)&htod);
3954
3955 kvm->arch.epoch = gtod->tod - htod.tod;
0e7def5f
DH		 3956 kvm->arch.epdx = 0;
3957 if (test_kvm_facility(kvm, 139)) {
3958 kvm->arch.epdx = gtod->epoch_idx - htod.epoch_idx;
3959 if (kvm->arch.epoch > gtod->tod)
3960 kvm->arch.epdx -= 1;
3961 }
8fa1696e
CW		 3962
3963 kvm_s390_vcpu_block_all(kvm);
3964 kvm_for_each_vcpu(i, vcpu, kvm) {
3965 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
3966 vcpu->arch.sie_block->epdx = kvm->arch.epdx;
3967 }
3968
3969 kvm_s390_vcpu_unblock_all(kvm);
3970 preempt_enable();
3971 mutex_unlock(&kvm->lock);
3972}
3973
fa576c58
TH		 3974/**
3975 * kvm_arch_fault_in_page - fault-in guest page if necessary
3976 * @vcpu: The corresponding virtual cpu
3977 * @gpa: Guest physical address
3978 * @writable: Whether the page should be writable or not
3979 *
3980 * Make sure that a guest page has been faulted-in on the host.
3981 *
3982 * Return: Zero on success, negative error code otherwise.
3983 */
3984long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 3985{
527e30b4
MS		 3986 return gmap_fault(vcpu->arch.gmap, gpa,
3987 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 3988}
3989
3c038e6b
DD		 3990static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
3991 unsigned long token)
3992{
3993 struct kvm_s390_interrupt inti;
383d0b05 3994 struct kvm_s390_irq irq;
3c038e6b
DD		 3995
3996 if (start_token) {
383d0b05
JF		 3997 irq.u.ext.ext_params2 = token;
3998 irq.type = KVM_S390_INT_PFAULT_INIT;
3999 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 4000 } else {
4001 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 4002 inti.parm64 = token;
3c038e6b
DD		 4003 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
4004 }
4005}
4006
4007void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
4008 struct kvm_async_pf *work)
4009{
4010 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
4011 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
4012}
4013
4014void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
4015 struct kvm_async_pf *work)
4016{
4017 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
4018 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
4019}
4020
4021void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
4022 struct kvm_async_pf *work)
4023{
4024 /* s390 will always inject the page directly */
4025}
4026
4027bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
4028{
4029 /*
4030 * s390 will always inject the page directly,
4031 * but we still want check_async_completion to cleanup
4032 */
4033 return true;
4034}
4035
4036static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
4037{
4038 hva_t hva;
4039 struct kvm_arch_async_pf arch;
4040 int rc;
4041
4042 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
4043 return 0;
4044 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
4045 vcpu->arch.pfault_compare)
4046 return 0;
4047 if (psw_extint_disabled(vcpu))
4048 return 0;
9a022067 4049 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b 4050 return 0;
b9224cd7 4051 if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
3c038e6b
DD		 4052 return 0;
4053 if (!vcpu->arch.gmap->pfault_enabled)
4054 return 0;
4055
81480cc1
HC		 4056 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
4057 hva += current->thread.gmap_addr & ~PAGE_MASK;
4058 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 4059 return 0;
4060
4061 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
4062 return rc;
4063}
4064
3fb4c40f 4065static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 4066{
3fb4c40f 4067 int rc, cpuflags;
e168bf8d 4068
3c038e6b
DD		 4069 /*
4070 * On s390 notifications for arriving pages will be delivered directly
4071 * to the guest but the house keeping for completed pfaults is
4072 * handled outside the worker.
4073 */
4074 kvm_check_async_pf_completion(vcpu);
4075
7ec7c8c7
CB		 4076 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
4077 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
b0c632db
HC		 4078
4079 if (need_resched())
4080 schedule();
4081
d3a73acb 4082 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 4083 s390_handle_mcck();
4084
79395031
JF		 4085 if (!kvm_is_ucontrol(vcpu->kvm)) {
4086 rc = kvm_s390_deliver_pending_interrupts(vcpu);
4087 if (rc)
4088 return rc;
4089 }
0ff31867 4090
2c70fe44
CB		 4091 rc = kvm_s390_handle_requests(vcpu);
4092 if (rc)
4093 return rc;
4094
27291e21
DH		 4095 if (guestdbg_enabled(vcpu)) {
4096 kvm_s390_backup_guest_per_regs(vcpu);
4097 kvm_s390_patch_guest_per_regs(vcpu);
4098 }
4099
b8265c71 4100 clear_bit(kvm_vcpu_get_idx(vcpu), vcpu->kvm->arch.gisa_int.kicked_mask);
9f30f621 4101
b0c632db 4102 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 4103 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
4104 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
4105 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 4106
3fb4c40f
TH		 4107 return 0;
4108}
4109
492d8642
TH		 4110static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
4111{
56317920
DH		 4112 struct kvm_s390_pgm_info pgm_info = {
4113 .code = PGM_ADDRESSING,
4114 };
4115 u8 opcode, ilen;
492d8642
TH		 4116 int rc;
4117
4118 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
4119 trace_kvm_s390_sie_fault(vcpu);
4120
4121 /*
4122 * We want to inject an addressing exception, which is defined as a
4123 * suppressing or terminating exception. However, since we came here
4124 * by a DAT access exception, the PSW still points to the faulting
4125 * instruction since DAT exceptions are nullifying. So we've got
4126 * to look up the current opcode to get the length of the instruction
4127 * to be able to forward the PSW.
4128 */
3fa8cad7 4129 rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
56317920 4130 ilen = insn_length(opcode);
9b0d721a
DH		 4131 if (rc < 0) {
4132 return rc;
4133 } else if (rc) {
4134 /* Instruction-Fetching Exceptions - we can't detect the ilen.
4135 * Forward by arbitrary ilc, injection will take care of
4136 * nullification if necessary.
4137 */
4138 pgm_info = vcpu->arch.pgm;
4139 ilen = 4;
4140 }
56317920
DH		 4141 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
4142 kvm_s390_forward_psw(vcpu, ilen);
4143 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
492d8642
TH		 4144}
4145
3fb4c40f
TH		 4146static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
4147{
4d62fcc0
QH		 4148 struct mcck_volatile_info *mcck_info;
4149 struct sie_page *sie_page;
4150
2b29a9fd
DD		 4151 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
4152 vcpu->arch.sie_block->icptcode);
4153 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
4154
27291e21
DH		 4155 if (guestdbg_enabled(vcpu))
4156 kvm_s390_restore_guest_per_regs(vcpu);
4157
7ec7c8c7
CB		 4158 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
4159 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
71f116bf 4160
4d62fcc0
QH		 4161 if (exit_reason == -EINTR) {
4162 VCPU_EVENT(vcpu, 3, "%s", "machine check");
4163 sie_page = container_of(vcpu->arch.sie_block,
4164 struct sie_page, sie_block);
4165 mcck_info = &sie_page->mcck_info;
4166 kvm_s390_reinject_machine_check(vcpu, mcck_info);
4167 return 0;
4168 }
4169
71f116bf
DH		 4170 if (vcpu->arch.sie_block->icptcode > 0) {
4171 int rc = kvm_handle_sie_intercept(vcpu);
4172
4173 if (rc != -EOPNOTSUPP)
4174 return rc;
4175 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
4176 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
4177 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
4178 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
4179 return -EREMOTE;
4180 } else if (exit_reason != -EFAULT) {
4181 vcpu->stat.exit_null++;
4182 return 0;
210b1607
TH		 4183 } else if (kvm_is_ucontrol(vcpu->kvm)) {
4184 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
4185 vcpu->run->s390_ucontrol.trans_exc_code =
4186 current->thread.gmap_addr;
4187 vcpu->run->s390_ucontrol.pgm_code = 0x10;
71f116bf 4188 return -EREMOTE;
24eb3a82 4189 } else if (current->thread.gmap_pfault) {
3c038e6b 4190 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 4191 current->thread.gmap_pfault = 0;
71f116bf
DH		 4192 if (kvm_arch_setup_async_pf(vcpu))
4193 return 0;
4194 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
a76ccff6 4195 }
71f116bf 4196 return vcpu_post_run_fault_in_sie(vcpu);
3fb4c40f
TH		 4197}
4198
1775d704 4199#define PSW_INT_MASK (PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_MCHECK)
3fb4c40f
TH		 4200static int __vcpu_run(struct kvm_vcpu *vcpu)
4201{
4202 int rc, exit_reason;
dcb6bd52 4203 struct sie_page *sie_page = (struct sie_page *)vcpu->arch.sie_block;
3fb4c40f 4204
800c1065
TH		 4205 /*
4206 * We try to hold kvm->srcu during most of vcpu_run (except when run-
4207 * ning the guest), so that memslots (and other stuff) are protected
4208 */
4209 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
4210
a76ccff6
TH		 4211 do {
4212 rc = vcpu_pre_run(vcpu);
4213 if (rc)
4214 break;
3fb4c40f 4215
800c1065 4216 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 4217 /*
4218 * As PF_VCPU will be used in fault handler, between
4219 * guest_enter and guest_exit should be no uaccess.
4220 */
0097d12e 4221 local_irq_disable();
6edaa530 4222 guest_enter_irqoff();
db0758b2 4223 __disable_cpu_timer_accounting(vcpu);
0097d12e 4224 local_irq_enable();
dcb6bd52
JF		 4225 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4226 memcpy(sie_page->pv_grregs,
4227 vcpu->run->s.regs.gprs,
4228 sizeof(sie_page->pv_grregs));
4229 }
a76ccff6
TH		 4230 exit_reason = sie64a(vcpu->arch.sie_block,
4231 vcpu->run->s.regs.gprs);
dcb6bd52
JF		 4232 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4233 memcpy(vcpu->run->s.regs.gprs,
4234 sie_page->pv_grregs,
4235 sizeof(sie_page->pv_grregs));
1775d704
JF		 4236 /*
4237 * We're not allowed to inject interrupts on intercepts
4238 * that leave the guest state in an "in-between" state
4239 * where the next SIE entry will do a continuation.
4240 * Fence interrupts in our "internal" PSW.
4241 */
4242 if (vcpu->arch.sie_block->icptcode == ICPT_PV_INSTR ||
4243 vcpu->arch.sie_block->icptcode == ICPT_PV_PREF) {
4244 vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
4245 }
dcb6bd52 4246 }
0097d12e 4247 local_irq_disable();
db0758b2 4248 __enable_cpu_timer_accounting(vcpu);
6edaa530 4249 guest_exit_irqoff();
0097d12e 4250 local_irq_enable();
800c1065 4251 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 4252
4253 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 4254 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 4255
800c1065 4256 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 4257 return rc;
b0c632db
HC		 4258}
4259
3653a999 4260static void sync_regs_fmt2(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
b028ee3e 4261{
4d5f2c04 4262 struct runtime_instr_cb *riccb;
4e0b1ab7 4263 struct gs_cb *gscb;
4d5f2c04
CB		 4264
4265 riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
4e0b1ab7 4266 gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
b028ee3e
DH		 4267 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
4268 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
b028ee3e 4269 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
b028ee3e
DH		 4270 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
4271 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
4272 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
4273 }
4274 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
4275 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
4276 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
4277 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 4278 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
4279 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e 4280 }
dad38237
CW		 4281 if (kvm_run->kvm_dirty_regs & KVM_SYNC_DIAG318) {
4282 vcpu->arch.diag318_info.val = kvm_run->s.regs.diag318;
4283 vcpu->arch.sie_block->cpnc = vcpu->arch.diag318_info.cpnc;
6a3448fd 4284 VCPU_EVENT(vcpu, 3, "setting cpnc to %d", vcpu->arch.diag318_info.cpnc);
dad38237 4285 }
80cd8763
FZ		 4286 /*
4287 * If userspace sets the riccb (e.g. after migration) to a valid state,
4288 * we should enable RI here instead of doing the lazy enablement.
4289 */
4290 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
4d5f2c04 4291 test_kvm_facility(vcpu->kvm, 64) &&
bb59c2da 4292 riccb->v &&
0c9d8683 4293 !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
4d5f2c04 4294 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
0c9d8683 4295 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
80cd8763 4296 }
4e0b1ab7
FZ		 4297 /*
4298 * If userspace sets the gscb (e.g. after migration) to non-zero,
4299 * we should enable GS here instead of doing the lazy enablement.
4300 */
4301 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
4302 test_kvm_facility(vcpu->kvm, 133) &&
4303 gscb->gssm &&
4304 !vcpu->arch.gs_enabled) {
4305 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
4306 vcpu->arch.sie_block->ecb |= ECB_GS;
4307 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
4308 vcpu->arch.gs_enabled = 1;
80cd8763 4309 }
35b3fde6
CB		 4310 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) &&
4311 test_kvm_facility(vcpu->kvm, 82)) {
4312 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
4313 vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
4314 }
3653a999
JF		 4315 if (MACHINE_HAS_GS) {
4316 preempt_disable();
4317 __ctl_set_bit(2, 4);
4318 if (current->thread.gs_cb) {
4319 vcpu->arch.host_gscb = current->thread.gs_cb;
4320 save_gs_cb(vcpu->arch.host_gscb);
4321 }
4322 if (vcpu->arch.gs_enabled) {
4323 current->thread.gs_cb = (struct gs_cb *)
4324 &vcpu->run->s.regs.gscb;
4325 restore_gs_cb(current->thread.gs_cb);
4326 }
4327 preempt_enable();
4328 }
4329 /* SIE will load etoken directly from SDNX and therefore kvm_run */
4330}
4331
4332static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
4333{
4334 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
4335 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
4336 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
4337 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
4338 /* some control register changes require a tlb flush */
4339 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
4340 }
4341 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4342 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
4343 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
4344 }
31d8b8d4
CB		 4345 save_access_regs(vcpu->arch.host_acrs);
4346 restore_access_regs(vcpu->run->s.regs.acrs);
e1788bb9
CB		 4347 /* save host (userspace) fprs/vrs */
4348 save_fpu_regs();
4349 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
4350 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
4351 if (MACHINE_HAS_VX)
4352 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
4353 else
4354 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
4355 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
4356 if (test_fp_ctl(current->thread.fpu.fpc))
4357 /* User space provided an invalid FPC, let's clear it */
4358 current->thread.fpu.fpc = 0;
3653a999
JF		 4359
4360 /* Sync fmt2 only data */
4361 if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) {
4362 sync_regs_fmt2(vcpu, kvm_run);
4363 } else {
4364 /*
4365 * In several places we have to modify our internal view to
4366 * not do things that are disallowed by the ultravisor. For
4367 * example we must not inject interrupts after specific exits
4368 * (e.g. 112 prefix page not secure). We do this by turning
4369 * off the machine check, external and I/O interrupt bits
4370 * of our PSW copy. To avoid getting validity intercepts, we
4371 * do only accept the condition code from userspace.
4372 */
4373 vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_CC;
4374 vcpu->arch.sie_block->gpsw.mask |= kvm_run->psw_mask &
4375 PSW_MASK_CC;
4376 }
4377
4378 kvm_run->kvm_dirty_regs = 0;
4379}
4380
4381static void store_regs_fmt2(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
4382{
4383 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
4384 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
4385 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
4386 kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
dad38237 4387 kvm_run->s.regs.diag318 = vcpu->arch.diag318_info.val;
4e0b1ab7 4388 if (MACHINE_HAS_GS) {
dde06655 4389 preempt_disable();
4e0b1ab7 4390 __ctl_set_bit(2, 4);
3653a999
JF		 4391 if (vcpu->arch.gs_enabled)
4392 save_gs_cb(current->thread.gs_cb);
3653a999
JF		 4393 current->thread.gs_cb = vcpu->arch.host_gscb;
4394 restore_gs_cb(vcpu->arch.host_gscb);
3653a999
JF		 4395 if (!vcpu->arch.host_gscb)
4396 __ctl_clear_bit(2, 4);
4397 vcpu->arch.host_gscb = NULL;
dde06655 4398 preempt_enable();
4e0b1ab7 4399 }
3653a999 4400 /* SIE will save etoken directly into SDNX and therefore kvm_run */
b028ee3e
DH		 4401}
4402
4403static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
4404{
4405 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
4406 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
4407 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
4408 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4287f247 4409 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
b028ee3e 4410 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
b028ee3e
DH		 4411 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
4412 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
4413 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
31d8b8d4
CB		 4414 save_access_regs(vcpu->run->s.regs.acrs);
4415 restore_access_regs(vcpu->arch.host_acrs);
e1788bb9
CB		 4416 /* Save guest register state */
4417 save_fpu_regs();
4418 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
4419 /* Restore will be done lazily at return */
4420 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
4421 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
3653a999
JF		 4422 if (likely(!kvm_s390_pv_cpu_is_protected(vcpu)))
4423 store_regs_fmt2(vcpu, kvm_run);
b028ee3e
DH		 4424}
4425
b0c632db
HC		 4426int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
4427{
8f2abe6a 4428 int rc;
b0c632db 4429
460df4c1
PB		 4430 if (kvm_run->immediate_exit)
4431 return -EINTR;
4432
200824f5
TH		 4433 if (kvm_run->kvm_valid_regs & ~KVM_SYNC_S390_VALID_FIELDS ||
4434 kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS)
4435 return -EINVAL;
4436
accb757d
CD		 4437 vcpu_load(vcpu);
4438
27291e21
DH		 4439 if (guestdbg_exit_pending(vcpu)) {
4440 kvm_s390_prepare_debug_exit(vcpu);
accb757d
CD		 4441 rc = 0;
4442 goto out;
27291e21
DH		 4443 }
4444
20b7035c 4445 kvm_sigset_activate(vcpu);
b0c632db 4446
1a9fe91e
JF		 4447 /*
4448 * no need to check the return value of vcpu_start as it can only have
4449 * an error for protvirt, but protvirt means user cpu state
4450 */
6352e4d2
DH		 4451 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
4452 kvm_s390_vcpu_start(vcpu);
4453 } else if (is_vcpu_stopped(vcpu)) {
ea2cdd27 4454 pr_err_ratelimited("can't run stopped vcpu %d\n",
6352e4d2 4455 vcpu->vcpu_id);
accb757d
CD		 4456 rc = -EINVAL;
4457 goto out;
6352e4d2 4458 }
b0c632db 4459
b028ee3e 4460 sync_regs(vcpu, kvm_run);
db0758b2 4461 enable_cpu_timer_accounting(vcpu);
d7b0b5eb 4462
dab4079d 4463 might_fault();
a76ccff6 4464 rc = __vcpu_run(vcpu);
9ace903d 4465
b1d16c49
CE		 4466 if (signal_pending(current) && !rc) {
4467 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 4468 rc = -EINTR;
b1d16c49 4469 }
8f2abe6a 4470
27291e21
DH		 4471 if (guestdbg_exit_pending(vcpu) && !rc) {
4472 kvm_s390_prepare_debug_exit(vcpu);
4473 rc = 0;
4474 }
4475
8f2abe6a 4476 if (rc == -EREMOTE) {
71f116bf 4477 /* userspace support is needed, kvm_run has been prepared */
8f2abe6a
CB		 4478 rc = 0;
4479 }
b0c632db 4480
db0758b2 4481 disable_cpu_timer_accounting(vcpu);
b028ee3e 4482 store_regs(vcpu, kvm_run);
d7b0b5eb 4483
20b7035c 4484 kvm_sigset_deactivate(vcpu);
b0c632db 4485
b0c632db 4486 vcpu->stat.exit_userspace++;
accb757d
CD		 4487out:
4488 vcpu_put(vcpu);
7e8e6ab4 4489 return rc;
b0c632db
HC		 4490}
4491
b0c632db
HC		 4492/*
4493 * store status at address
4494 * we use have two special cases:
4495 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
4496 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
4497 */
d0bce605 4498int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 4499{
092670cd 4500 unsigned char archmode = 1;
9abc2a08 4501 freg_t fprs[NUM_FPRS];
fda902cb 4502 unsigned int px;
4287f247 4503 u64 clkcomp, cputm;
d0bce605 4504 int rc;
b0c632db 4505
d9a3a09a 4506 px = kvm_s390_get_prefix(vcpu);
d0bce605
HC		 4507 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
4508 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 4509 return -EFAULT;
d9a3a09a 4510 gpa = 0;
d0bce605
HC		 4511 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
4512 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 4513 return -EFAULT;
d9a3a09a
MS		 4514 gpa = px;
4515 } else
4516 gpa -= __LC_FPREGS_SAVE_AREA;
9abc2a08
DH		 4517
4518 /* manually convert vector registers if necessary */
4519 if (MACHINE_HAS_VX) {
9522b37f 4520 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08
DH		 4521 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
4522 fprs, 128);
4523 } else {
4524 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
6fd8e67d 4525 vcpu->run->s.regs.fprs, 128);
9abc2a08 4526 }
d9a3a09a 4527 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
d0bce605 4528 vcpu->run->s.regs.gprs, 128);
d9a3a09a 4529 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
d0bce605 4530 &vcpu->arch.sie_block->gpsw, 16);
d9a3a09a 4531 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
fda902cb 4532 &px, 4);
d9a3a09a 4533 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
9abc2a08 4534 &vcpu->run->s.regs.fpc, 4);
d9a3a09a 4535 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
d0bce605 4536 &vcpu->arch.sie_block->todpr, 4);
4287f247 4537 cputm = kvm_s390_get_cpu_timer(vcpu);
d9a3a09a 4538 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4287f247 4539 &cputm, 8);
178bd789 4540 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d9a3a09a 4541 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
d0bce605 4542 &clkcomp, 8);
d9a3a09a 4543 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
d0bce605 4544 &vcpu->run->s.regs.acrs, 64);
d9a3a09a 4545 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
d0bce605
HC		 4546 &vcpu->arch.sie_block->gcr, 128);
4547 return rc ? -EFAULT : 0;
b0c632db
HC		 4548}
4549
e879892c
TH		 4550int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
4551{
4552 /*
4553 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
31d8b8d4 4554 * switch in the run ioctl. Let's update our copies before we save
e879892c
TH		 4555 * it into the save area
4556 */
d0164ee2 4557 save_fpu_regs();
9abc2a08 4558 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
e879892c
TH		 4559 save_access_regs(vcpu->run->s.regs.acrs);
4560
4561 return kvm_s390_store_status_unloaded(vcpu, addr);
4562}
4563
8ad35755
DH		 4564static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
4565{
4566 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
8e236546 4567 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
8ad35755
DH		 4568}
4569
4570static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
4571{
4572 unsigned int i;
4573 struct kvm_vcpu *vcpu;
4574
4575 kvm_for_each_vcpu(i, vcpu, kvm) {
4576 __disable_ibs_on_vcpu(vcpu);
4577 }
4578}
4579
4580static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
4581{
09a400e7
DH		 4582 if (!sclp.has_ibs)
4583 return;
8ad35755 4584 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
8e236546 4585 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
8ad35755
DH		 4586}
4587
1a9fe91e 4588int kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
6852d7b6 4589{
1a9fe91e 4590 int i, online_vcpus, r = 0, started_vcpus = 0;
8ad35755
DH		 4591
4592 if (!is_vcpu_stopped(vcpu))
1a9fe91e 4593 return 0;
8ad35755 4594
6852d7b6 4595 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 4596 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 4597 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 4598 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
4599
1a9fe91e
JF		 4600 /* Let's tell the UV that we want to change into the operating state */
4601 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4602 r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR);
4603 if (r) {
4604 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4605 return r;
4606 }
4607 }
4608
8ad35755
DH		 4609 for (i = 0; i < online_vcpus; i++) {
4610 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
4611 started_vcpus++;
4612 }
4613
4614 if (started_vcpus == 0) {
4615 /* we're the only active VCPU -> speed it up */
4616 __enable_ibs_on_vcpu(vcpu);
4617 } else if (started_vcpus == 1) {
4618 /*
4619 * As we are starting a second VCPU, we have to disable
4620 * the IBS facility on all VCPUs to remove potentially
4621 * oustanding ENABLE requests.
4622 */
4623 __disable_ibs_on_all_vcpus(vcpu->kvm);
4624 }
4625
9daecfc6 4626 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
550a5848
CB		 4627 /*
4628 * The real PSW might have changed due to a RESTART interpreted by the
4629 * ultravisor. We block all interrupts and let the next sie exit
4630 * refresh our view.
4631 */
4632 if (kvm_s390_pv_cpu_is_protected(vcpu))
4633 vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
8ad35755
DH		 4634 /*
4635 * Another VCPU might have used IBS while we were offline.
4636 * Let's play safe and flush the VCPU at startup.
4637 */
d3d692c8 4638 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 4639 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
1a9fe91e 4640 return 0;
6852d7b6
DH		 4641}
4642
1a9fe91e 4643int kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
6852d7b6 4644{
1a9fe91e 4645 int i, online_vcpus, r = 0, started_vcpus = 0;
8ad35755
DH		 4646 struct kvm_vcpu *started_vcpu = NULL;
4647
4648 if (is_vcpu_stopped(vcpu))
1a9fe91e 4649 return 0;
8ad35755 4650
6852d7b6 4651 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 4652 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 4653 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 4654 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
4655
1a9fe91e
JF		 4656 /* Let's tell the UV that we want to change into the stopped state */
4657 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4658 r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_STP);
4659 if (r) {
4660 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4661 return r;
4662 }
4663 }
4664
4c226bbf
EF		 4665 /*
4666 * Set the VCPU to STOPPED and THEN clear the interrupt flag,
4667 * now that the SIGP STOP and SIGP STOP AND STORE STATUS orders
4668 * have been fully processed. This will ensure that the VCPU
4669 * is kept BUSY if another VCPU is inquiring with SIGP SENSE.
4670 */
4671 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
6cddd432 4672 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 4673
8ad35755
DH		 4674 __disable_ibs_on_vcpu(vcpu);
4675
4676 for (i = 0; i < online_vcpus; i++) {
4677 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
4678 started_vcpus++;
4679 started_vcpu = vcpu->kvm->vcpus[i];
4680 }
4681 }
4682
4683 if (started_vcpus == 1) {
4684 /*
4685 * As we only have one VCPU left, we want to enable the
4686 * IBS facility for that VCPU to speed it up.
4687 */
4688 __enable_ibs_on_vcpu(started_vcpu);
4689 }
4690
433b9ee4 4691 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
1a9fe91e 4692 return 0;
6852d7b6
DH		 4693}
4694
d6712df9
CH		 4695static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
4696 struct kvm_enable_cap *cap)
4697{
4698 int r;
4699
4700 if (cap->flags)
4701 return -EINVAL;
4702
4703 switch (cap->cap) {
fa6b7fe9
CH		 4704 case KVM_CAP_S390_CSS_SUPPORT:
4705 if (!vcpu->kvm->arch.css_support) {
4706 vcpu->kvm->arch.css_support = 1;
c92ea7b9 4707 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
fa6b7fe9
CH		 4708 trace_kvm_s390_enable_css(vcpu->kvm);
4709 }
4710 r = 0;
4711 break;
d6712df9
CH		 4712 default:
4713 r = -EINVAL;
4714 break;
4715 }
4716 return r;
4717}
4718
c1548876
JSG		 4719static long kvm_s390_vcpu_sida_op(struct kvm_vcpu *vcpu,
4720 struct kvm_s390_mem_op *mop)
3bb61e16
JF		 4721{
4722 void __user *uaddr = (void __user *)mop->buf;
4723 int r = 0;
4724
4725 if (mop->flags || !mop->size)
4726 return -EINVAL;
4727 if (mop->size + mop->sida_offset < mop->size)
4728 return -EINVAL;
4729 if (mop->size + mop->sida_offset > sida_size(vcpu->arch.sie_block))
4730 return -E2BIG;
3a5e3dcc
JSG		 4731 if (!kvm_s390_pv_cpu_is_protected(vcpu))
4732 return -EINVAL;
3bb61e16
JF		 4733
4734 switch (mop->op) {
4735 case KVM_S390_MEMOP_SIDA_READ:
4736 if (copy_to_user(uaddr, (void *)(sida_origin(vcpu->arch.sie_block) +
4737 mop->sida_offset), mop->size))
4738 r = -EFAULT;
4739
4740 break;
4741 case KVM_S390_MEMOP_SIDA_WRITE:
4742 if (copy_from_user((void *)(sida_origin(vcpu->arch.sie_block) +
4743 mop->sida_offset), uaddr, mop->size))
4744 r = -EFAULT;
4745 break;
4746 }
4747 return r;
4748}
c1548876
JSG		 4749
4750static long kvm_s390_vcpu_mem_op(struct kvm_vcpu *vcpu,
4751 struct kvm_s390_mem_op *mop)
41408c28
TH		 4752{
4753 void __user *uaddr = (void __user *)mop->buf;
4754 void *tmpbuf = NULL;
3bb61e16 4755 int r = 0;
41408c28 4756 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
0b8b9918
JSG		 4757 | KVM_S390_MEMOP_F_CHECK_ONLY
4758 | KVM_S390_MEMOP_F_SKEY_PROTECTION;
41408c28 4759
a13b03bb 4760 if (mop->flags & ~supported_flags || mop->ar >= NUM_ACRS || !mop->size)
41408c28 4761 return -EINVAL;
41408c28
TH		 4762 if (mop->size > MEM_OP_MAX_SIZE)
4763 return -E2BIG;
3bb61e16
JF		 4764 if (kvm_s390_pv_cpu_is_protected(vcpu))
4765 return -EINVAL;
0b8b9918
JSG		 4766 if (mop->flags & KVM_S390_MEMOP_F_SKEY_PROTECTION) {
4767 if (access_key_invalid(mop->key))
4768 return -EINVAL;
4769 } else {
4770 mop->key = 0;
4771 }
41408c28
TH		 4772 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
4773 tmpbuf = vmalloc(mop->size);
4774 if (!tmpbuf)
4775 return -ENOMEM;
4776 }
4777
41408c28
TH		 4778 switch (mop->op) {
4779 case KVM_S390_MEMOP_LOGICAL_READ:
4780 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
0b8b9918
JSG		 4781 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size,
4782 GACC_FETCH, mop->key);
41408c28
TH		 4783 break;
4784 }
0b8b9918
JSG		 4785 r = read_guest_with_key(vcpu, mop->gaddr, mop->ar, tmpbuf,
4786 mop->size, mop->key);
41408c28
TH		 4787 if (r == 0) {
4788 if (copy_to_user(uaddr, tmpbuf, mop->size))
4789 r = -EFAULT;
4790 }
4791 break;
4792 case KVM_S390_MEMOP_LOGICAL_WRITE:
4793 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
0b8b9918
JSG		 4794 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size,
4795 GACC_STORE, mop->key);
41408c28
TH		 4796 break;
4797 }
4798 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
4799 r = -EFAULT;
4800 break;
4801 }
0b8b9918
JSG		 4802 r = write_guest_with_key(vcpu, mop->gaddr, mop->ar, tmpbuf,
4803 mop->size, mop->key);
41408c28 4804 break;
41408c28
TH		 4805 }
4806
41408c28
TH		 4807 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
4808 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
4809
4810 vfree(tmpbuf);
4811 return r;
4812}
4813
c1548876
JSG		 4814static long kvm_s390_vcpu_memsida_op(struct kvm_vcpu *vcpu,
4815 struct kvm_s390_mem_op *mop)
3bb61e16
JF		 4816{
4817 int r, srcu_idx;
4818
4819 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
4820
4821 switch (mop->op) {
4822 case KVM_S390_MEMOP_LOGICAL_READ:
4823 case KVM_S390_MEMOP_LOGICAL_WRITE:
c1548876 4824 r = kvm_s390_vcpu_mem_op(vcpu, mop);
3bb61e16
JF		 4825 break;
4826 case KVM_S390_MEMOP_SIDA_READ:
4827 case KVM_S390_MEMOP_SIDA_WRITE:
4828 /* we are locked against sida going away by the vcpu->mutex */
c1548876 4829 r = kvm_s390_vcpu_sida_op(vcpu, mop);
3bb61e16
JF		 4830 break;
4831 default:
4832 r = -EINVAL;
4833 }
4834
4835 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
4836 return r;
4837}
4838
5cb0944c
PB		 4839long kvm_arch_vcpu_async_ioctl(struct file *filp,
4840 unsigned int ioctl, unsigned long arg)
b0c632db
HC		 4841{
4842 struct kvm_vcpu *vcpu = filp->private_data;
4843 void __user *argp = (void __user *)arg;
4844
93736624 4845 switch (ioctl) {
47b43c52
JF		 4846 case KVM_S390_IRQ: {
4847 struct kvm_s390_irq s390irq;
4848
47b43c52 4849 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
9b062471
CD		 4850 return -EFAULT;
4851 return kvm_s390_inject_vcpu(vcpu, &s390irq);
47b43c52 4852 }
93736624 4853 case KVM_S390_INTERRUPT: {
ba5c1e9b 4854 struct kvm_s390_interrupt s390int;
53936b5b 4855 struct kvm_s390_irq s390irq = {};
ba5c1e9b
CO		 4856
4857 if (copy_from_user(&s390int, argp, sizeof(s390int)))
9b062471 4858 return -EFAULT;
383d0b05
JF		 4859 if (s390int_to_s390irq(&s390int, &s390irq))
4860 return -EINVAL;
9b062471 4861 return kvm_s390_inject_vcpu(vcpu, &s390irq);
ba5c1e9b 4862 }
9b062471 4863 }
5cb0944c
PB		 4864 return -ENOIOCTLCMD;
4865}
4866
4867long kvm_arch_vcpu_ioctl(struct file *filp,
4868 unsigned int ioctl, unsigned long arg)
4869{
4870 struct kvm_vcpu *vcpu = filp->private_data;
4871 void __user *argp = (void __user *)arg;
4872 int idx;
4873 long r;
4742a31b 4874 u16 rc, rrc;
9b062471
CD		 4875
4876 vcpu_load(vcpu);
4877
4878 switch (ioctl) {
b0c632db 4879 case KVM_S390_STORE_STATUS:
800c1065 4880 idx = srcu_read_lock(&vcpu->kvm->srcu);
f00ad9a8 4881 r = kvm_s390_store_status_unloaded(vcpu, arg);
800c1065 4882 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 4883 break;
b0c632db
HC		 4884 case KVM_S390_SET_INITIAL_PSW: {
4885 psw_t psw;
4886
bc923cc9 4887 r = -EFAULT;
b0c632db 4888 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 4889 break;
4890 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
4891 break;
b0c632db 4892 }
68369234
JF		 4893 case KVM_S390_CLEAR_RESET:
4894 r = 0;
4895 kvm_arch_vcpu_ioctl_clear_reset(vcpu);
4742a31b
JF		 4896 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4897 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
4898 UVC_CMD_CPU_RESET_CLEAR, &rc, &rrc);
4899 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET CLEAR VCPU: rc %x rrc %x",
4900 rc, rrc);
4901 }
68369234 4902 break;
b0c632db 4903 case KVM_S390_INITIAL_RESET:
68369234
JF		 4904 r = 0;
4905 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
4742a31b
JF		 4906 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4907 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
4908 UVC_CMD_CPU_RESET_INITIAL,
4909 &rc, &rrc);
4910 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET INITIAL VCPU: rc %x rrc %x",
4911 rc, rrc);
4912 }
68369234
JF		 4913 break;
4914 case KVM_S390_NORMAL_RESET:
4915 r = 0;
4916 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
4742a31b
JF		 4917 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4918 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
4919 UVC_CMD_CPU_RESET, &rc, &rrc);
4920 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET NORMAL VCPU: rc %x rrc %x",
4921 rc, rrc);
4922 }
bc923cc9 4923 break;
14eebd91
CO		 4924 case KVM_SET_ONE_REG:
4925 case KVM_GET_ONE_REG: {
4926 struct kvm_one_reg reg;
f84ac97c
JF		 4927 r = -EINVAL;
4928 if (kvm_s390_pv_cpu_is_protected(vcpu))
4929 break;
14eebd91
CO		 4930 r = -EFAULT;
4931 if (copy_from_user(&reg, argp, sizeof(reg)))
4932 break;
4933 if (ioctl == KVM_SET_ONE_REG)
4934 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
4935 else
4936 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
4937 break;
4938 }
27e0393f
CO		 4939#ifdef CONFIG_KVM_S390_UCONTROL
4940 case KVM_S390_UCAS_MAP: {
4941 struct kvm_s390_ucas_mapping ucasmap;
4942
4943 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4944 r = -EFAULT;
4945 break;
4946 }
4947
4948 if (!kvm_is_ucontrol(vcpu->kvm)) {
4949 r = -EINVAL;
4950 break;
4951 }
4952
4953 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
4954 ucasmap.vcpu_addr, ucasmap.length);
4955 break;
4956 }
4957 case KVM_S390_UCAS_UNMAP: {
4958 struct kvm_s390_ucas_mapping ucasmap;
4959
4960 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4961 r = -EFAULT;
4962 break;
4963 }
4964
4965 if (!kvm_is_ucontrol(vcpu->kvm)) {
4966 r = -EINVAL;
4967 break;
4968 }
4969
4970 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
4971 ucasmap.length);
4972 break;
4973 }
4974#endif
ccc7910f 4975 case KVM_S390_VCPU_FAULT: {
527e30b4 4976 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 4977 break;
4978 }
d6712df9
CH		 4979 case KVM_ENABLE_CAP:
4980 {
4981 struct kvm_enable_cap cap;
4982 r = -EFAULT;
4983 if (copy_from_user(&cap, argp, sizeof(cap)))
4984 break;
4985 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
4986 break;
4987 }
41408c28
TH		 4988 case KVM_S390_MEM_OP: {
4989 struct kvm_s390_mem_op mem_op;
4990
4991 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
c1548876 4992 r = kvm_s390_vcpu_memsida_op(vcpu, &mem_op);
41408c28
TH		 4993 else
4994 r = -EFAULT;
4995 break;
4996 }
816c7667
JF		 4997 case KVM_S390_SET_IRQ_STATE: {
4998 struct kvm_s390_irq_state irq_state;
4999
5000 r = -EFAULT;
5001 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
5002 break;
5003 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
5004 irq_state.len == 0 ||
5005 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
5006 r = -EINVAL;
5007 break;
5008 }
bb64da9a 5009 /* do not use irq_state.flags, it will break old QEMUs */
816c7667
JF		 5010 r = kvm_s390_set_irq_state(vcpu,
5011 (void __user *) irq_state.buf,
5012 irq_state.len);
5013 break;
5014 }
5015 case KVM_S390_GET_IRQ_STATE: {
5016 struct kvm_s390_irq_state irq_state;
5017
5018 r = -EFAULT;
5019 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
5020 break;
5021 if (irq_state.len == 0) {
5022 r = -EINVAL;
5023 break;
5024 }
bb64da9a 5025 /* do not use irq_state.flags, it will break old QEMUs */
816c7667
JF		 5026 r = kvm_s390_get_irq_state(vcpu,
5027 (__u8 __user *) irq_state.buf,
5028 irq_state.len);
5029 break;
5030 }
b0c632db 5031 default:
3e6afcf1 5032 r = -ENOTTY;
b0c632db 5033 }
9b062471
CD		 5034
5035 vcpu_put(vcpu);
bc923cc9 5036 return r;
b0c632db
HC		 5037}
5038
1499fa80 5039vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
5b1c1493
CO		 5040{
5041#ifdef CONFIG_KVM_S390_UCONTROL
5042 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
5043 && (kvm_is_ucontrol(vcpu->kvm))) {
5044 vmf->page = virt_to_page(vcpu->arch.sie_block);
5045 get_page(vmf->page);
5046 return 0;
5047 }
5048#endif
5049 return VM_FAULT_SIGBUS;
5050}
5051
5587027c
AK		 5052int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
5053 unsigned long npages)
db3fe4eb
TY		 5054{
5055 return 0;
5056}
5057
b0c632db 5058/* Section: memory related */
f7784b8e
MT		 5059int kvm_arch_prepare_memory_region(struct kvm *kvm,
5060 struct kvm_memory_slot *memslot,
09170a49 5061 const struct kvm_userspace_memory_region *mem,
7b6195a9 5062 enum kvm_mr_change change)
b0c632db 5063{
dd2887e7
NW		 5064 /* A few sanity checks. We can have memory slots which have to be
5065 located/ended at a segment boundary (1MB). The memory in userland is
5066 ok to be fragmented into various different vmas. It is okay to mmap()
5067 and munmap() stuff in this slot after doing this call at any time */
b0c632db 5068
598841ca 5069 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 5070 return -EINVAL;
5071
598841ca 5072 if (mem->memory_size & 0xffffful)
b0c632db
HC		 5073 return -EINVAL;
5074
a3a92c31
DD		 5075 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
5076 return -EINVAL;
5077
e2ed2e2d
JF		 5078 /* When we are protected, we should not change the memory slots */
5079 if (kvm_s390_pv_get_handle(kvm))
5080 return -EINVAL;
f7784b8e
MT		 5081 return 0;
5082}
5083
5084void kvm_arch_commit_memory_region(struct kvm *kvm,
09170a49 5085 const struct kvm_userspace_memory_region *mem,
8482644a 5086 const struct kvm_memory_slot *old,
f36f3f28 5087 const struct kvm_memory_slot *new,
8482644a 5088 enum kvm_mr_change change)
f7784b8e 5089{
19ec166c 5090 int rc = 0;
598841ca 5091
19ec166c
CB		 5092 switch (change) {
5093 case KVM_MR_DELETE:
5094 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
5095 old->npages * PAGE_SIZE);
5096 break;
5097 case KVM_MR_MOVE:
5098 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
5099 old->npages * PAGE_SIZE);
5100 if (rc)
5101 break;
5102 /* FALLTHROUGH */
5103 case KVM_MR_CREATE:
5104 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
5105 mem->guest_phys_addr, mem->memory_size);
5106 break;
5107 case KVM_MR_FLAGS_ONLY:
5108 break;
5109 default:
5110 WARN(1, "Unknown KVM MR CHANGE: %d\n", change);
5111 }
598841ca 5112 if (rc)
ea2cdd27 5113 pr_warn("failed to commit memory region\n");
598841ca 5114 return;
b0c632db
HC		 5115}
5116
60a37709
AY		 5117static inline unsigned long nonhyp_mask(int i)
5118{
5119 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
5120
5121 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
5122}
5123
3491caf2
CB		 5124void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
5125{
5126 vcpu->valid_wakeup = false;
5127}
5128
b0c632db
HC		 5129static int __init kvm_s390_init(void)
5130{
60a37709
AY		 5131 int i;
5132
07197fd0 5133 if (!sclp.has_sief2) {
8d43d570 5134 pr_info("SIE is not available\n");
07197fd0
DH		 5135 return -ENODEV;
5136 }
5137
a4499382 5138 if (nested && hpage) {
8d43d570 5139 pr_info("A KVM host that supports nesting cannot back its KVM guests with huge pages\n");
a4499382
JF		 5140 return -EINVAL;
5141 }
5142
60a37709 5143 for (i = 0; i < 16; i++)
c3b9e3e1 5144 kvm_s390_fac_base[i] |=
60a37709
AY		 5145 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
5146
9d8d5786 5147 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 5148}
5149
5150static void __exit kvm_s390_exit(void)
5151{
5152 kvm_exit();
5153}
5154
5155module_init(kvm_s390_init);
5156module_exit(kvm_s390_exit);
566af940
CH		 5157
5158/*
5159 * Enable autoloading of the kvm module.
5160 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
5161 * since x86 takes a different approach.
5162 */
5163#include <linux/miscdevice.h>
5164MODULE_ALIAS_MISCDEV(KVM_MINOR);
5165MODULE_ALIAS("devname:kvm");
Ubuntu Focal Linux kernel mirror