]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
s390/sclp: detect interlock-and-broadcast-suppression facility
[mirror_ubuntu-jammy-kernel.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
24#include <linux/module.h>
a374e892 25#include <linux/random.h>
b0c632db 26#include <linux/slab.h>
ba5c1e9b 27#include <linux/timer.h>
41408c28 28#include <linux/vmalloc.h>
15c9705f 29#include <linux/bitmap.h>
cbb870c8 30#include <asm/asm-offsets.h>
b0c632db 31#include <asm/lowcore.h>
fdf03650 32#include <asm/etr.h>
b0c632db 33#include <asm/pgtable.h>
1e133ab2 34#include <asm/gmap.h>
f5daba1d 35#include <asm/nmi.h>
a0616cde 36#include <asm/switch_to.h>
6d3da241 37#include <asm/isc.h>
1526bf9c 38#include <asm/sclp.h>
0a763c78
DH		 39#include <asm/cpacf.h>
40#include <asm/etr.h>
8f2abe6a 41#include "kvm-s390.h"
b0c632db
HC		 42#include "gaccess.h"
43
ea2cdd27
DH		 44#define KMSG_COMPONENT "kvm-s390"
45#undef pr_fmt
46#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
47
5786fffa
CH		 48#define CREATE_TRACE_POINTS
49#include "trace.h"
ade38c31 50#include "trace-s390.h"
5786fffa 51
41408c28 52#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
816c7667
JF		 53#define LOCAL_IRQS 32
54#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
55 (KVM_MAX_VCPUS + LOCAL_IRQS))
41408c28 56
b0c632db
HC		 57#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
58
59struct kvm_stats_debugfs_item debugfs_entries[] = {
60 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 61 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 62 { "exit_validity", VCPU_STAT(exit_validity) },
63 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
64 { "exit_external_request", VCPU_STAT(exit_external_request) },
65 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b
CO		 66 { "exit_instruction", VCPU_STAT(exit_instruction) },
67 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
68 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
a011eeb2 69 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
f7819512 70 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
62bea5bf 71 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
3491caf2 72 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
ce2e4f0b 73 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 74 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 75 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 76 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
77 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 78 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 79 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 80 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
81 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
82 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
83 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
84 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
85 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
86 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 87 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 88 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
89 { "instruction_spx", VCPU_STAT(instruction_spx) },
90 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
91 { "instruction_stap", VCPU_STAT(instruction_stap) },
92 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 93 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 94 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
95 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 96 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 97 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
98 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 99 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
95ca2cb5 100 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
5288fbf0 101 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 102 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 103 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 104 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 105 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
106 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 107 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 108 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
109 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 110 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 111 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
112 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
113 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 114 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
115 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
116 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 117 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 118 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 119 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
175a5c9e
CB		 120 { "diagnose_258", VCPU_STAT(diagnose_258) },
121 { "diagnose_308", VCPU_STAT(diagnose_308) },
122 { "diagnose_500", VCPU_STAT(diagnose_500) },
b0c632db
HC		 123 { NULL }
124};
125
9d8d5786 126/* upper facilities limit for kvm */
60a37709
AY		 127unsigned long kvm_s390_fac_list_mask[16] = {
128 0xffe6000000000000UL,
129 0x005e000000000000UL,
9d8d5786 130};
b0c632db 131
9d8d5786 132unsigned long kvm_s390_fac_list_mask_size(void)
78c4b59f 133{
9d8d5786
MM		 134 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
135 return ARRAY_SIZE(kvm_s390_fac_list_mask);
78c4b59f
MM		 136}
137
15c9705f
DH		 138/* available cpu features supported by kvm */
139static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
0a763c78
DH		 140/* available subfunctions indicated via query / "test bit" */
141static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
15c9705f 142
9d8d5786 143static struct gmap_notifier gmap_notifier;
78f26131 144debug_info_t *kvm_s390_dbf;
9d8d5786 145
b0c632db 146/* Section: not file related */
13a34e06 147int kvm_arch_hardware_enable(void)
b0c632db
HC		 148{
149 /* every s390 is virtualization enabled ;-) */
10474ae8 150 return 0;
b0c632db
HC		 151}
152
2c70fe44
CB		 153static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
154
fdf03650
FZ		 155/*
156 * This callback is executed during stop_machine(). All CPUs are therefore
157 * temporarily stopped. In order not to change guest behavior, we have to
158 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
159 * so a CPU won't be stopped while calculating with the epoch.
160 */
161static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
162 void *v)
163{
164 struct kvm *kvm;
165 struct kvm_vcpu *vcpu;
166 int i;
167 unsigned long long *delta = v;
168
169 list_for_each_entry(kvm, &vm_list, vm_list) {
170 kvm->arch.epoch -= *delta;
171 kvm_for_each_vcpu(i, vcpu, kvm) {
172 vcpu->arch.sie_block->epoch -= *delta;
db0758b2
DH		 173 if (vcpu->arch.cputm_enabled)
174 vcpu->arch.cputm_start += *delta;
fdf03650
FZ		 175 }
176 }
177 return NOTIFY_OK;
178}
179
180static struct notifier_block kvm_clock_notifier = {
181 .notifier_call = kvm_clock_sync,
182};
183
b0c632db
HC		 184int kvm_arch_hardware_setup(void)
185{
2c70fe44
CB		 186 gmap_notifier.notifier_call = kvm_gmap_notifier;
187 gmap_register_ipte_notifier(&gmap_notifier);
fdf03650
FZ		 188 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
189 &kvm_clock_notifier);
b0c632db
HC		 190 return 0;
191}
192
193void kvm_arch_hardware_unsetup(void)
194{
2c70fe44 195 gmap_unregister_ipte_notifier(&gmap_notifier);
fdf03650
FZ		 196 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
197 &kvm_clock_notifier);
b0c632db
HC		 198}
199
22be5a13
DH		 200static void allow_cpu_feat(unsigned long nr)
201{
202 set_bit_inv(nr, kvm_s390_available_cpu_feat);
203}
204
0a763c78
DH		 205static inline int plo_test_bit(unsigned char nr)
206{
207 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
208 int cc = 3; /* subfunction not available */
209
210 asm volatile(
211 /* Parameter registers are ignored for "test bit" */
212 " plo 0,0,0,0(0)\n"
213 " ipm %0\n"
214 " srl %0,28\n"
215 : "=d" (cc)
216 : "d" (r0)
217 : "cc");
218 return cc == 0;
219}
220
22be5a13
DH		 221static void kvm_s390_cpu_feat_init(void)
222{
0a763c78
DH		 223 int i;
224
225 for (i = 0; i < 256; ++i) {
226 if (plo_test_bit(i))
227 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
228 }
229
230 if (test_facility(28)) /* TOD-clock steering */
231 etr_ptff(kvm_s390_available_subfunc.ptff, ETR_PTFF_QAF);
232
233 if (test_facility(17)) { /* MSA */
234 __cpacf_query(CPACF_KMAC, kvm_s390_available_subfunc.kmac);
235 __cpacf_query(CPACF_KMC, kvm_s390_available_subfunc.kmc);
236 __cpacf_query(CPACF_KM, kvm_s390_available_subfunc.km);
237 __cpacf_query(CPACF_KIMD, kvm_s390_available_subfunc.kimd);
238 __cpacf_query(CPACF_KLMD, kvm_s390_available_subfunc.klmd);
239 }
240 if (test_facility(76)) /* MSA3 */
241 __cpacf_query(CPACF_PCKMO, kvm_s390_available_subfunc.pckmo);
242 if (test_facility(77)) { /* MSA4 */
243 __cpacf_query(CPACF_KMCTR, kvm_s390_available_subfunc.kmctr);
244 __cpacf_query(CPACF_KMF, kvm_s390_available_subfunc.kmf);
245 __cpacf_query(CPACF_KMO, kvm_s390_available_subfunc.kmo);
246 __cpacf_query(CPACF_PCC, kvm_s390_available_subfunc.pcc);
247 }
248 if (test_facility(57)) /* MSA5 */
249 __cpacf_query(CPACF_PPNO, kvm_s390_available_subfunc.ppno);
250
22be5a13
DH		 251 if (MACHINE_HAS_ESOP)
252 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
253}
254
b0c632db
HC		 255int kvm_arch_init(void *opaque)
256{
78f26131
CB		 257 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
258 if (!kvm_s390_dbf)
259 return -ENOMEM;
260
261 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
262 debug_unregister(kvm_s390_dbf);
263 return -ENOMEM;
264 }
265
22be5a13
DH		 266 kvm_s390_cpu_feat_init();
267
84877d93
CH		 268 /* Register floating interrupt controller interface. */
269 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 270}
271
78f26131
CB		 272void kvm_arch_exit(void)
273{
274 debug_unregister(kvm_s390_dbf);
275}
276
b0c632db
HC		 277/* Section: device related */
278long kvm_arch_dev_ioctl(struct file *filp,
279 unsigned int ioctl, unsigned long arg)
280{
281 if (ioctl == KVM_S390_ENABLE_SIE)
282 return s390_enable_sie();
283 return -EINVAL;
284}
285
784aa3d7 286int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 287{
d7b0b5eb
CO		 288 int r;
289
2bd0ac4e 290 switch (ext) {
d7b0b5eb 291 case KVM_CAP_S390_PSW:
b6cf8788 292 case KVM_CAP_S390_GMAP:
52e16b18 293 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 294#ifdef CONFIG_KVM_S390_UCONTROL
295 case KVM_CAP_S390_UCONTROL:
296#endif
3c038e6b 297 case KVM_CAP_ASYNC_PF:
60b413c9 298 case KVM_CAP_SYNC_REGS:
14eebd91 299 case KVM_CAP_ONE_REG:
d6712df9 300 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 301 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 302 case KVM_CAP_IOEVENTFD:
c05c4186 303 case KVM_CAP_DEVICE_CTRL:
d938dc55 304 case KVM_CAP_ENABLE_CAP_VM:
78599d90 305 case KVM_CAP_S390_IRQCHIP:
f2061656 306 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 307 case KVM_CAP_MP_STATE:
47b43c52 308 case KVM_CAP_S390_INJECT_IRQ:
2444b352 309 case KVM_CAP_S390_USER_SIGP:
e44fc8c9 310 case KVM_CAP_S390_USER_STSI:
30ee2a98 311 case KVM_CAP_S390_SKEYS:
816c7667 312 case KVM_CAP_S390_IRQ_STATE:
d7b0b5eb
CO		 313 r = 1;
314 break;
41408c28
TH		 315 case KVM_CAP_S390_MEM_OP:
316 r = MEM_OP_MAX_SIZE;
317 break;
e726b1bd
CB		 318 case KVM_CAP_NR_VCPUS:
319 case KVM_CAP_MAX_VCPUS:
76a6dd72
DH		 320 r = KVM_S390_BSCA_CPU_SLOTS;
321 if (sclp.has_esca && sclp.has_64bscao)
322 r = KVM_S390_ESCA_CPU_SLOTS;
e726b1bd 323 break;
e1e2e605
NW		 324 case KVM_CAP_NR_MEMSLOTS:
325 r = KVM_USER_MEM_SLOTS;
326 break;
1526bf9c 327 case KVM_CAP_S390_COW:
abf09bed 328 r = MACHINE_HAS_ESOP;
1526bf9c 329 break;
68c55750
EF		 330 case KVM_CAP_S390_VECTOR_REGISTERS:
331 r = MACHINE_HAS_VX;
332 break;
c6e5f166
FZ		 333 case KVM_CAP_S390_RI:
334 r = test_facility(64);
335 break;
2bd0ac4e 336 default:
d7b0b5eb 337 r = 0;
2bd0ac4e 338 }
d7b0b5eb 339 return r;
b0c632db
HC		 340}
341
15f36ebd
JH		 342static void kvm_s390_sync_dirty_log(struct kvm *kvm,
343 struct kvm_memory_slot *memslot)
344{
345 gfn_t cur_gfn, last_gfn;
346 unsigned long address;
347 struct gmap *gmap = kvm->arch.gmap;
348
15f36ebd
JH		 349 /* Loop over all guest pages */
350 last_gfn = memslot->base_gfn + memslot->npages;
351 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
352 address = gfn_to_hva_memslot(memslot, cur_gfn);
353
1e133ab2 354 if (test_and_clear_guest_dirty(gmap->mm, address))
15f36ebd 355 mark_page_dirty(kvm, cur_gfn);
1763f8d0
CB		 356 if (fatal_signal_pending(current))
357 return;
70c88a00 358 cond_resched();
15f36ebd 359 }
15f36ebd
JH		 360}
361
b0c632db 362/* Section: vm related */
a6e2f683
ED		 363static void sca_del_vcpu(struct kvm_vcpu *vcpu);
364
b0c632db
HC		 365/*
366 * Get (and clear) the dirty memory log for a memory slot.
367 */
368int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
369 struct kvm_dirty_log *log)
370{
15f36ebd
JH		 371 int r;
372 unsigned long n;
9f6b8029 373 struct kvm_memslots *slots;
15f36ebd
JH		 374 struct kvm_memory_slot *memslot;
375 int is_dirty = 0;
376
377 mutex_lock(&kvm->slots_lock);
378
379 r = -EINVAL;
380 if (log->slot >= KVM_USER_MEM_SLOTS)
381 goto out;
382
9f6b8029
PB		 383 slots = kvm_memslots(kvm);
384 memslot = id_to_memslot(slots, log->slot);
15f36ebd
JH		 385 r = -ENOENT;
386 if (!memslot->dirty_bitmap)
387 goto out;
388
389 kvm_s390_sync_dirty_log(kvm, memslot);
390 r = kvm_get_dirty_log(kvm, log, &is_dirty);
391 if (r)
392 goto out;
393
394 /* Clear the dirty log */
395 if (is_dirty) {
396 n = kvm_dirty_bitmap_bytes(memslot);
397 memset(memslot->dirty_bitmap, 0, n);
398 }
399 r = 0;
400out:
401 mutex_unlock(&kvm->slots_lock);
402 return r;
b0c632db
HC		 403}
404
d938dc55
CH		 405static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
406{
407 int r;
408
409 if (cap->flags)
410 return -EINVAL;
411
412 switch (cap->cap) {
84223598 413 case KVM_CAP_S390_IRQCHIP:
c92ea7b9 414 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
84223598
CH		 415 kvm->arch.use_irqchip = 1;
416 r = 0;
417 break;
2444b352 418 case KVM_CAP_S390_USER_SIGP:
c92ea7b9 419 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
2444b352
DH		 420 kvm->arch.user_sigp = 1;
421 r = 0;
422 break;
68c55750 423 case KVM_CAP_S390_VECTOR_REGISTERS:
5967c17b
DH		 424 mutex_lock(&kvm->lock);
425 if (atomic_read(&kvm->online_vcpus)) {
426 r = -EBUSY;
427 } else if (MACHINE_HAS_VX) {
c54f0d6a
DH		 428 set_kvm_facility(kvm->arch.model.fac_mask, 129);
429 set_kvm_facility(kvm->arch.model.fac_list, 129);
18280d8b
MM		 430 r = 0;
431 } else
432 r = -EINVAL;
5967c17b 433 mutex_unlock(&kvm->lock);
c92ea7b9
CB		 434 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
435 r ? "(not available)" : "(success)");
68c55750 436 break;
c6e5f166
FZ		 437 case KVM_CAP_S390_RI:
438 r = -EINVAL;
439 mutex_lock(&kvm->lock);
440 if (atomic_read(&kvm->online_vcpus)) {
441 r = -EBUSY;
442 } else if (test_facility(64)) {
c54f0d6a
DH		 443 set_kvm_facility(kvm->arch.model.fac_mask, 64);
444 set_kvm_facility(kvm->arch.model.fac_list, 64);
c6e5f166
FZ		 445 r = 0;
446 }
447 mutex_unlock(&kvm->lock);
448 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
449 r ? "(not available)" : "(success)");
450 break;
e44fc8c9 451 case KVM_CAP_S390_USER_STSI:
c92ea7b9 452 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
e44fc8c9
ET		 453 kvm->arch.user_stsi = 1;
454 r = 0;
455 break;
d938dc55
CH		 456 default:
457 r = -EINVAL;
458 break;
459 }
460 return r;
461}
462
8c0a7ce6
DD		 463static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
464{
465 int ret;
466
467 switch (attr->attr) {
468 case KVM_S390_VM_MEM_LIMIT_SIZE:
469 ret = 0;
c92ea7b9 470 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
a3a92c31
DD		 471 kvm->arch.mem_limit);
472 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
8c0a7ce6
DD		 473 ret = -EFAULT;
474 break;
475 default:
476 ret = -ENXIO;
477 break;
478 }
479 return ret;
480}
481
482static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 483{
484 int ret;
485 unsigned int idx;
486 switch (attr->attr) {
487 case KVM_S390_VM_MEM_ENABLE_CMMA:
f9cbd9b0 488 ret = -ENXIO;
c24cc9c8 489 if (!sclp.has_cmma)
e6db1d61
DD		 490 break;
491
4f718eab 492 ret = -EBUSY;
c92ea7b9 493 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
4f718eab
DD		 494 mutex_lock(&kvm->lock);
495 if (atomic_read(&kvm->online_vcpus) == 0) {
496 kvm->arch.use_cmma = 1;
497 ret = 0;
498 }
499 mutex_unlock(&kvm->lock);
500 break;
501 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 502 ret = -ENXIO;
503 if (!sclp.has_cmma)
504 break;
c3489155
DD		 505 ret = -EINVAL;
506 if (!kvm->arch.use_cmma)
507 break;
508
c92ea7b9 509 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
4f718eab
DD		 510 mutex_lock(&kvm->lock);
511 idx = srcu_read_lock(&kvm->srcu);
a13cff31 512 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 513 srcu_read_unlock(&kvm->srcu, idx);
514 mutex_unlock(&kvm->lock);
515 ret = 0;
516 break;
8c0a7ce6
DD		 517 case KVM_S390_VM_MEM_LIMIT_SIZE: {
518 unsigned long new_limit;
519
520 if (kvm_is_ucontrol(kvm))
521 return -EINVAL;
522
523 if (get_user(new_limit, (u64 __user *)attr->addr))
524 return -EFAULT;
525
a3a92c31
DD		 526 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
527 new_limit > kvm->arch.mem_limit)
8c0a7ce6
DD		 528 return -E2BIG;
529
a3a92c31
DD		 530 if (!new_limit)
531 return -EINVAL;
532
533 /* gmap_alloc takes last usable address */
534 if (new_limit != KVM_S390_NO_MEM_LIMIT)
535 new_limit -= 1;
536
8c0a7ce6
DD		 537 ret = -EBUSY;
538 mutex_lock(&kvm->lock);
539 if (atomic_read(&kvm->online_vcpus) == 0) {
540 /* gmap_alloc will round the limit up */
541 struct gmap *new = gmap_alloc(current->mm, new_limit);
542
543 if (!new) {
544 ret = -ENOMEM;
545 } else {
546 gmap_free(kvm->arch.gmap);
547 new->private = kvm;
548 kvm->arch.gmap = new;
549 ret = 0;
550 }
551 }
552 mutex_unlock(&kvm->lock);
a3a92c31
DD		 553 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
554 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
555 (void *) kvm->arch.gmap->asce);
8c0a7ce6
DD		 556 break;
557 }
4f718eab
DD		 558 default:
559 ret = -ENXIO;
560 break;
561 }
562 return ret;
563}
564
a374e892
TK		 565static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
566
567static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
568{
569 struct kvm_vcpu *vcpu;
570 int i;
571
9d8d5786 572 if (!test_kvm_facility(kvm, 76))
a374e892
TK		 573 return -EINVAL;
574
575 mutex_lock(&kvm->lock);
576 switch (attr->attr) {
577 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
578 get_random_bytes(
579 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
580 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
581 kvm->arch.crypto.aes_kw = 1;
c92ea7b9 582 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
a374e892
TK		 583 break;
584 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
585 get_random_bytes(
586 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
587 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
588 kvm->arch.crypto.dea_kw = 1;
c92ea7b9 589 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
a374e892
TK		 590 break;
591 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
592 kvm->arch.crypto.aes_kw = 0;
593 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
594 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
c92ea7b9 595 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
a374e892
TK		 596 break;
597 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
598 kvm->arch.crypto.dea_kw = 0;
599 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
600 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
c92ea7b9 601 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
a374e892
TK		 602 break;
603 default:
604 mutex_unlock(&kvm->lock);
605 return -ENXIO;
606 }
607
608 kvm_for_each_vcpu(i, vcpu, kvm) {
609 kvm_s390_vcpu_crypto_setup(vcpu);
610 exit_sie(vcpu);
611 }
612 mutex_unlock(&kvm->lock);
613 return 0;
614}
615
72f25020
JH		 616static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
617{
618 u8 gtod_high;
619
620 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
621 sizeof(gtod_high)))
622 return -EFAULT;
623
624 if (gtod_high != 0)
625 return -EINVAL;
58c383c6 626 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
72f25020
JH		 627
628 return 0;
629}
630
631static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
632{
5a3d883a 633 u64 gtod;
72f25020
JH		 634
635 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
636 return -EFAULT;
637
25ed1675 638 kvm_s390_set_tod_clock(kvm, gtod);
58c383c6 639 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
72f25020
JH		 640 return 0;
641}
642
643static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
644{
645 int ret;
646
647 if (attr->flags)
648 return -EINVAL;
649
650 switch (attr->attr) {
651 case KVM_S390_VM_TOD_HIGH:
652 ret = kvm_s390_set_tod_high(kvm, attr);
653 break;
654 case KVM_S390_VM_TOD_LOW:
655 ret = kvm_s390_set_tod_low(kvm, attr);
656 break;
657 default:
658 ret = -ENXIO;
659 break;
660 }
661 return ret;
662}
663
664static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
665{
666 u8 gtod_high = 0;
667
668 if (copy_to_user((void __user *)attr->addr, &gtod_high,
669 sizeof(gtod_high)))
670 return -EFAULT;
58c383c6 671 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
72f25020
JH		 672
673 return 0;
674}
675
676static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
677{
5a3d883a 678 u64 gtod;
72f25020 679
60417fcc 680 gtod = kvm_s390_get_tod_clock_fast(kvm);
72f25020
JH		 681 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
682 return -EFAULT;
58c383c6 683 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
72f25020
JH		 684
685 return 0;
686}
687
688static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
689{
690 int ret;
691
692 if (attr->flags)
693 return -EINVAL;
694
695 switch (attr->attr) {
696 case KVM_S390_VM_TOD_HIGH:
697 ret = kvm_s390_get_tod_high(kvm, attr);
698 break;
699 case KVM_S390_VM_TOD_LOW:
700 ret = kvm_s390_get_tod_low(kvm, attr);
701 break;
702 default:
703 ret = -ENXIO;
704 break;
705 }
706 return ret;
707}
708
658b6eda
MM		 709static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
710{
711 struct kvm_s390_vm_cpu_processor *proc;
053dd230 712 u16 lowest_ibc, unblocked_ibc;
658b6eda
MM		 713 int ret = 0;
714
715 mutex_lock(&kvm->lock);
716 if (atomic_read(&kvm->online_vcpus)) {
717 ret = -EBUSY;
718 goto out;
719 }
720 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
721 if (!proc) {
722 ret = -ENOMEM;
723 goto out;
724 }
725 if (!copy_from_user(proc, (void __user *)attr->addr,
726 sizeof(*proc))) {
9bb0ec09 727 kvm->arch.model.cpuid = proc->cpuid;
053dd230
DH		 728 lowest_ibc = sclp.ibc >> 16 & 0xfff;
729 unblocked_ibc = sclp.ibc & 0xfff;
730 if (lowest_ibc) {
731 if (proc->ibc > unblocked_ibc)
732 kvm->arch.model.ibc = unblocked_ibc;
733 else if (proc->ibc < lowest_ibc)
734 kvm->arch.model.ibc = lowest_ibc;
735 else
736 kvm->arch.model.ibc = proc->ibc;
737 }
c54f0d6a 738 memcpy(kvm->arch.model.fac_list, proc->fac_list,
658b6eda
MM		 739 S390_ARCH_FAC_LIST_SIZE_BYTE);
740 } else
741 ret = -EFAULT;
742 kfree(proc);
743out:
744 mutex_unlock(&kvm->lock);
745 return ret;
746}
747
15c9705f
DH		 748static int kvm_s390_set_processor_feat(struct kvm *kvm,
749 struct kvm_device_attr *attr)
750{
751 struct kvm_s390_vm_cpu_feat data;
752 int ret = -EBUSY;
753
754 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
755 return -EFAULT;
756 if (!bitmap_subset((unsigned long *) data.feat,
757 kvm_s390_available_cpu_feat,
758 KVM_S390_VM_CPU_FEAT_NR_BITS))
759 return -EINVAL;
760
761 mutex_lock(&kvm->lock);
762 if (!atomic_read(&kvm->online_vcpus)) {
763 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
764 KVM_S390_VM_CPU_FEAT_NR_BITS);
765 ret = 0;
766 }
767 mutex_unlock(&kvm->lock);
768 return ret;
769}
770
0a763c78
DH		 771static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
772 struct kvm_device_attr *attr)
773{
774 /*
775 * Once supported by kernel + hw, we have to store the subfunctions
776 * in kvm->arch and remember that user space configured them.
777 */
778 return -ENXIO;
779}
780
658b6eda
MM		 781static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
782{
783 int ret = -ENXIO;
784
785 switch (attr->attr) {
786 case KVM_S390_VM_CPU_PROCESSOR:
787 ret = kvm_s390_set_processor(kvm, attr);
788 break;
15c9705f
DH		 789 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
790 ret = kvm_s390_set_processor_feat(kvm, attr);
791 break;
0a763c78
DH		 792 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
793 ret = kvm_s390_set_processor_subfunc(kvm, attr);
794 break;
658b6eda
MM		 795 }
796 return ret;
797}
798
799static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
800{
801 struct kvm_s390_vm_cpu_processor *proc;
802 int ret = 0;
803
804 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
805 if (!proc) {
806 ret = -ENOMEM;
807 goto out;
808 }
9bb0ec09 809 proc->cpuid = kvm->arch.model.cpuid;
658b6eda 810 proc->ibc = kvm->arch.model.ibc;
c54f0d6a
DH		 811 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
812 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 813 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
814 ret = -EFAULT;
815 kfree(proc);
816out:
817 return ret;
818}
819
820static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
821{
822 struct kvm_s390_vm_cpu_machine *mach;
823 int ret = 0;
824
825 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
826 if (!mach) {
827 ret = -ENOMEM;
828 goto out;
829 }
830 get_cpu_id((struct cpuid *) &mach->cpuid);
37c5f6c8 831 mach->ibc = sclp.ibc;
c54f0d6a 832 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
981467c9 833 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 834 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
94422ee8 835 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 836 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
837 ret = -EFAULT;
838 kfree(mach);
839out:
840 return ret;
841}
842
15c9705f
DH		 843static int kvm_s390_get_processor_feat(struct kvm *kvm,
844 struct kvm_device_attr *attr)
845{
846 struct kvm_s390_vm_cpu_feat data;
847
848 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
849 KVM_S390_VM_CPU_FEAT_NR_BITS);
850 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
851 return -EFAULT;
852 return 0;
853}
854
855static int kvm_s390_get_machine_feat(struct kvm *kvm,
856 struct kvm_device_attr *attr)
857{
858 struct kvm_s390_vm_cpu_feat data;
859
860 bitmap_copy((unsigned long *) data.feat,
861 kvm_s390_available_cpu_feat,
862 KVM_S390_VM_CPU_FEAT_NR_BITS);
863 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
864 return -EFAULT;
865 return 0;
866}
867
0a763c78
DH		 868static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
869 struct kvm_device_attr *attr)
870{
871 /*
872 * Once we can actually configure subfunctions (kernel + hw support),
873 * we have to check if they were already set by user space, if so copy
874 * them from kvm->arch.
875 */
876 return -ENXIO;
877}
878
879static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
880 struct kvm_device_attr *attr)
881{
882 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
883 sizeof(struct kvm_s390_vm_cpu_subfunc)))
884 return -EFAULT;
885 return 0;
886}
658b6eda
MM		 887static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
888{
889 int ret = -ENXIO;
890
891 switch (attr->attr) {
892 case KVM_S390_VM_CPU_PROCESSOR:
893 ret = kvm_s390_get_processor(kvm, attr);
894 break;
895 case KVM_S390_VM_CPU_MACHINE:
896 ret = kvm_s390_get_machine(kvm, attr);
897 break;
15c9705f
DH		 898 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
899 ret = kvm_s390_get_processor_feat(kvm, attr);
900 break;
901 case KVM_S390_VM_CPU_MACHINE_FEAT:
902 ret = kvm_s390_get_machine_feat(kvm, attr);
903 break;
0a763c78
DH		 904 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
905 ret = kvm_s390_get_processor_subfunc(kvm, attr);
906 break;
907 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
908 ret = kvm_s390_get_machine_subfunc(kvm, attr);
909 break;
658b6eda
MM		 910 }
911 return ret;
912}
913
f2061656
DD		 914static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
915{
916 int ret;
917
918 switch (attr->group) {
4f718eab 919 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 920 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 921 break;
72f25020
JH		 922 case KVM_S390_VM_TOD:
923 ret = kvm_s390_set_tod(kvm, attr);
924 break;
658b6eda
MM		 925 case KVM_S390_VM_CPU_MODEL:
926 ret = kvm_s390_set_cpu_model(kvm, attr);
927 break;
a374e892
TK		 928 case KVM_S390_VM_CRYPTO:
929 ret = kvm_s390_vm_set_crypto(kvm, attr);
930 break;
f2061656
DD		 931 default:
932 ret = -ENXIO;
933 break;
934 }
935
936 return ret;
937}
938
939static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
940{
8c0a7ce6
DD		 941 int ret;
942
943 switch (attr->group) {
944 case KVM_S390_VM_MEM_CTRL:
945 ret = kvm_s390_get_mem_control(kvm, attr);
946 break;
72f25020
JH		 947 case KVM_S390_VM_TOD:
948 ret = kvm_s390_get_tod(kvm, attr);
949 break;
658b6eda
MM		 950 case KVM_S390_VM_CPU_MODEL:
951 ret = kvm_s390_get_cpu_model(kvm, attr);
952 break;
8c0a7ce6
DD		 953 default:
954 ret = -ENXIO;
955 break;
956 }
957
958 return ret;
f2061656
DD		 959}
960
961static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
962{
963 int ret;
964
965 switch (attr->group) {
4f718eab
DD		 966 case KVM_S390_VM_MEM_CTRL:
967 switch (attr->attr) {
968 case KVM_S390_VM_MEM_ENABLE_CMMA:
969 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 970 ret = sclp.has_cmma ? 0 : -ENXIO;
971 break;
8c0a7ce6 972 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 973 ret = 0;
974 break;
975 default:
976 ret = -ENXIO;
977 break;
978 }
979 break;
72f25020
JH		 980 case KVM_S390_VM_TOD:
981 switch (attr->attr) {
982 case KVM_S390_VM_TOD_LOW:
983 case KVM_S390_VM_TOD_HIGH:
984 ret = 0;
985 break;
986 default:
987 ret = -ENXIO;
988 break;
989 }
990 break;
658b6eda
MM		 991 case KVM_S390_VM_CPU_MODEL:
992 switch (attr->attr) {
993 case KVM_S390_VM_CPU_PROCESSOR:
994 case KVM_S390_VM_CPU_MACHINE:
15c9705f
DH		 995 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
996 case KVM_S390_VM_CPU_MACHINE_FEAT:
0a763c78 997 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
658b6eda
MM		 998 ret = 0;
999 break;
0a763c78
DH		 1000 /* configuring subfunctions is not supported yet */
1001 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
658b6eda
MM		 1002 default:
1003 ret = -ENXIO;
1004 break;
1005 }
1006 break;
a374e892
TK		 1007 case KVM_S390_VM_CRYPTO:
1008 switch (attr->attr) {
1009 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1010 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1011 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1012 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1013 ret = 0;
1014 break;
1015 default:
1016 ret = -ENXIO;
1017 break;
1018 }
1019 break;
f2061656
DD		 1020 default:
1021 ret = -ENXIO;
1022 break;
1023 }
1024
1025 return ret;
1026}
1027
30ee2a98
JH		 1028static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1029{
1030 uint8_t *keys;
1031 uint64_t hva;
1032 unsigned long curkey;
1033 int i, r = 0;
1034
1035 if (args->flags != 0)
1036 return -EINVAL;
1037
1038 /* Is this guest using storage keys? */
1039 if (!mm_use_skey(current->mm))
1040 return KVM_S390_GET_SKEYS_NONE;
1041
1042 /* Enforce sane limit on memory allocation */
1043 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1044 return -EINVAL;
1045
1046 keys = kmalloc_array(args->count, sizeof(uint8_t),
1047 GFP_KERNEL | __GFP_NOWARN);
1048 if (!keys)
1049 keys = vmalloc(sizeof(uint8_t) * args->count);
1050 if (!keys)
1051 return -ENOMEM;
1052
1053 for (i = 0; i < args->count; i++) {
1054 hva = gfn_to_hva(kvm, args->start_gfn + i);
1055 if (kvm_is_error_hva(hva)) {
1056 r = -EFAULT;
1057 goto out;
1058 }
1059
1060 curkey = get_guest_storage_key(current->mm, hva);
1061 if (IS_ERR_VALUE(curkey)) {
1062 r = curkey;
1063 goto out;
1064 }
1065 keys[i] = curkey;
1066 }
1067
1068 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1069 sizeof(uint8_t) * args->count);
1070 if (r)
1071 r = -EFAULT;
1072out:
1073 kvfree(keys);
1074 return r;
1075}
1076
1077static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1078{
1079 uint8_t *keys;
1080 uint64_t hva;
1081 int i, r = 0;
1082
1083 if (args->flags != 0)
1084 return -EINVAL;
1085
1086 /* Enforce sane limit on memory allocation */
1087 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1088 return -EINVAL;
1089
1090 keys = kmalloc_array(args->count, sizeof(uint8_t),
1091 GFP_KERNEL | __GFP_NOWARN);
1092 if (!keys)
1093 keys = vmalloc(sizeof(uint8_t) * args->count);
1094 if (!keys)
1095 return -ENOMEM;
1096
1097 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1098 sizeof(uint8_t) * args->count);
1099 if (r) {
1100 r = -EFAULT;
1101 goto out;
1102 }
1103
1104 /* Enable storage key handling for the guest */
14d4a425
DD		 1105 r = s390_enable_skey();
1106 if (r)
1107 goto out;
30ee2a98
JH		 1108
1109 for (i = 0; i < args->count; i++) {
1110 hva = gfn_to_hva(kvm, args->start_gfn + i);
1111 if (kvm_is_error_hva(hva)) {
1112 r = -EFAULT;
1113 goto out;
1114 }
1115
1116 /* Lowest order bit is reserved */
1117 if (keys[i] & 0x01) {
1118 r = -EINVAL;
1119 goto out;
1120 }
1121
1122 r = set_guest_storage_key(current->mm, hva,
1123 (unsigned long)keys[i], 0);
1124 if (r)
1125 goto out;
1126 }
1127out:
1128 kvfree(keys);
1129 return r;
1130}
1131
b0c632db
HC		 1132long kvm_arch_vm_ioctl(struct file *filp,
1133 unsigned int ioctl, unsigned long arg)
1134{
1135 struct kvm *kvm = filp->private_data;
1136 void __user *argp = (void __user *)arg;
f2061656 1137 struct kvm_device_attr attr;
b0c632db
HC		 1138 int r;
1139
1140 switch (ioctl) {
ba5c1e9b
CO		 1141 case KVM_S390_INTERRUPT: {
1142 struct kvm_s390_interrupt s390int;
1143
1144 r = -EFAULT;
1145 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1146 break;
1147 r = kvm_s390_inject_vm(kvm, &s390int);
1148 break;
1149 }
d938dc55
CH		 1150 case KVM_ENABLE_CAP: {
1151 struct kvm_enable_cap cap;
1152 r = -EFAULT;
1153 if (copy_from_user(&cap, argp, sizeof(cap)))
1154 break;
1155 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1156 break;
1157 }
84223598
CH		 1158 case KVM_CREATE_IRQCHIP: {
1159 struct kvm_irq_routing_entry routing;
1160
1161 r = -EINVAL;
1162 if (kvm->arch.use_irqchip) {
1163 /* Set up dummy routing. */
1164 memset(&routing, 0, sizeof(routing));
152b2839 1165 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
84223598
CH		 1166 }
1167 break;
1168 }
f2061656
DD		 1169 case KVM_SET_DEVICE_ATTR: {
1170 r = -EFAULT;
1171 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1172 break;
1173 r = kvm_s390_vm_set_attr(kvm, &attr);
1174 break;
1175 }
1176 case KVM_GET_DEVICE_ATTR: {
1177 r = -EFAULT;
1178 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1179 break;
1180 r = kvm_s390_vm_get_attr(kvm, &attr);
1181 break;
1182 }
1183 case KVM_HAS_DEVICE_ATTR: {
1184 r = -EFAULT;
1185 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1186 break;
1187 r = kvm_s390_vm_has_attr(kvm, &attr);
1188 break;
1189 }
30ee2a98
JH		 1190 case KVM_S390_GET_SKEYS: {
1191 struct kvm_s390_skeys args;
1192
1193 r = -EFAULT;
1194 if (copy_from_user(&args, argp,
1195 sizeof(struct kvm_s390_skeys)))
1196 break;
1197 r = kvm_s390_get_skeys(kvm, &args);
1198 break;
1199 }
1200 case KVM_S390_SET_SKEYS: {
1201 struct kvm_s390_skeys args;
1202
1203 r = -EFAULT;
1204 if (copy_from_user(&args, argp,
1205 sizeof(struct kvm_s390_skeys)))
1206 break;
1207 r = kvm_s390_set_skeys(kvm, &args);
1208 break;
1209 }
b0c632db 1210 default:
367e1319 1211 r = -ENOTTY;
b0c632db
HC		 1212 }
1213
1214 return r;
1215}
1216
45c9b47c
TK		 1217static int kvm_s390_query_ap_config(u8 *config)
1218{
1219 u32 fcn_code = 0x04000000UL;
86044c8c 1220 u32 cc = 0;
45c9b47c 1221
86044c8c 1222 memset(config, 0, 128);
45c9b47c
TK		 1223 asm volatile(
1224 "lgr 0,%1\n"
1225 "lgr 2,%2\n"
1226 ".long 0xb2af0000\n" /* PQAP(QCI) */
86044c8c 1227 "0: ipm %0\n"
45c9b47c 1228 "srl %0,28\n"
86044c8c
CB		 1229 "1:\n"
1230 EX_TABLE(0b, 1b)
1231 : "+r" (cc)
45c9b47c
TK		 1232 : "r" (fcn_code), "r" (config)
1233 : "cc", "0", "2", "memory"
1234 );
1235
1236 return cc;
1237}
1238
1239static int kvm_s390_apxa_installed(void)
1240{
1241 u8 config[128];
1242 int cc;
1243
a6aacc3f 1244 if (test_facility(12)) {
45c9b47c
TK		 1245 cc = kvm_s390_query_ap_config(config);
1246
1247 if (cc)
1248 pr_err("PQAP(QCI) failed with cc=%d", cc);
1249 else
1250 return config[0] & 0x40;
1251 }
1252
1253 return 0;
1254}
1255
1256static void kvm_s390_set_crycb_format(struct kvm *kvm)
1257{
1258 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1259
1260 if (kvm_s390_apxa_installed())
1261 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1262 else
1263 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1264}
1265
9bb0ec09 1266static u64 kvm_s390_get_initial_cpuid(void)
9d8d5786 1267{
9bb0ec09
DH		 1268 struct cpuid cpuid;
1269
1270 get_cpu_id(&cpuid);
1271 cpuid.version = 0xff;
1272 return *((u64 *) &cpuid);
9d8d5786
MM		 1273}
1274
c54f0d6a 1275static void kvm_s390_crypto_init(struct kvm *kvm)
5102ee87 1276{
9d8d5786 1277 if (!test_kvm_facility(kvm, 76))
c54f0d6a 1278 return;
5102ee87 1279
c54f0d6a 1280 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
45c9b47c 1281 kvm_s390_set_crycb_format(kvm);
5102ee87 1282
ed6f76b4
TK		 1283 /* Enable AES/DEA protected key functions by default */
1284 kvm->arch.crypto.aes_kw = 1;
1285 kvm->arch.crypto.dea_kw = 1;
1286 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1287 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1288 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1289 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
5102ee87
TK		 1290}
1291
7d43bafc
ED		 1292static void sca_dispose(struct kvm *kvm)
1293{
1294 if (kvm->arch.use_esca)
5e044315 1295 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
7d43bafc
ED		 1296 else
1297 free_page((unsigned long)(kvm->arch.sca));
1298 kvm->arch.sca = NULL;
1299}
1300
e08b9637 1301int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 1302{
76a6dd72 1303 gfp_t alloc_flags = GFP_KERNEL;
9d8d5786 1304 int i, rc;
b0c632db 1305 char debug_name[16];
f6c137ff 1306 static unsigned long sca_offset;
b0c632db 1307
e08b9637
CO		 1308 rc = -EINVAL;
1309#ifdef CONFIG_KVM_S390_UCONTROL
1310 if (type & ~KVM_VM_S390_UCONTROL)
1311 goto out_err;
1312 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1313 goto out_err;
1314#else
1315 if (type)
1316 goto out_err;
1317#endif
1318
b0c632db
HC		 1319 rc = s390_enable_sie();
1320 if (rc)
d89f5eff 1321 goto out_err;
b0c632db 1322
b290411a
CO		 1323 rc = -ENOMEM;
1324
7d0a5e62
JF		 1325 ratelimit_state_init(&kvm->arch.sthyi_limit, 5 * HZ, 500);
1326
7d43bafc 1327 kvm->arch.use_esca = 0; /* start with basic SCA */
76a6dd72
DH		 1328 if (!sclp.has_64bscao)
1329 alloc_flags |= GFP_DMA;
5e044315 1330 rwlock_init(&kvm->arch.sca_lock);
76a6dd72 1331 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
b0c632db 1332 if (!kvm->arch.sca)
d89f5eff 1333 goto out_err;
f6c137ff 1334 spin_lock(&kvm_lock);
c5c2c393 1335 sca_offset += 16;
bc784cce 1336 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
c5c2c393 1337 sca_offset = 0;
bc784cce
ED		 1338 kvm->arch.sca = (struct bsca_block *)
1339 ((char *) kvm->arch.sca + sca_offset);
f6c137ff 1340 spin_unlock(&kvm_lock);
b0c632db
HC		 1341
1342 sprintf(debug_name, "kvm-%u", current->pid);
1343
1cb9cf72 1344 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
b0c632db 1345 if (!kvm->arch.dbf)
40f5b735 1346 goto out_err;
b0c632db 1347
c54f0d6a
DH		 1348 kvm->arch.sie_page2 =
1349 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1350 if (!kvm->arch.sie_page2)
40f5b735 1351 goto out_err;
9d8d5786 1352
fb5bf93f 1353 /* Populate the facility mask initially. */
c54f0d6a 1354 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
94422ee8 1355 S390_ARCH_FAC_LIST_SIZE_BYTE);
9d8d5786
MM		 1356 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1357 if (i < kvm_s390_fac_list_mask_size())
c54f0d6a 1358 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
9d8d5786 1359 else
c54f0d6a 1360 kvm->arch.model.fac_mask[i] = 0UL;
9d8d5786
MM		 1361 }
1362
981467c9 1363 /* Populate the facility list initially. */
c54f0d6a
DH		 1364 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1365 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
981467c9
MM		 1366 S390_ARCH_FAC_LIST_SIZE_BYTE);
1367
95ca2cb5
JF		 1368 set_kvm_facility(kvm->arch.model.fac_mask, 74);
1369 set_kvm_facility(kvm->arch.model.fac_list, 74);
1370
9bb0ec09 1371 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
37c5f6c8 1372 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
9d8d5786 1373
c54f0d6a 1374 kvm_s390_crypto_init(kvm);
5102ee87 1375
ba5c1e9b 1376 spin_lock_init(&kvm->arch.float_int.lock);
6d3da241
JF		 1377 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1378 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
8a242234 1379 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 1380 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 1381
b0c632db 1382 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
78f26131 1383 VM_EVENT(kvm, 3, "vm created with type %lu", type);
b0c632db 1384
e08b9637
CO		 1385 if (type & KVM_VM_S390_UCONTROL) {
1386 kvm->arch.gmap = NULL;
a3a92c31 1387 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
e08b9637 1388 } else {
32e6b236
GH		 1389 if (sclp.hamax == U64_MAX)
1390 kvm->arch.mem_limit = TASK_MAX_SIZE;
1391 else
1392 kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
1393 sclp.hamax + 1);
a3a92c31 1394 kvm->arch.gmap = gmap_alloc(current->mm, kvm->arch.mem_limit - 1);
e08b9637 1395 if (!kvm->arch.gmap)
40f5b735 1396 goto out_err;
2c70fe44 1397 kvm->arch.gmap->private = kvm;
24eb3a82 1398 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 1399 }
fa6b7fe9
CH		 1400
1401 kvm->arch.css_support = 0;
84223598 1402 kvm->arch.use_irqchip = 0;
72f25020 1403 kvm->arch.epoch = 0;
fa6b7fe9 1404
8ad35755 1405 spin_lock_init(&kvm->arch.start_stop_lock);
8335713a 1406 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
8ad35755 1407
d89f5eff 1408 return 0;
40f5b735 1409out_err:
c54f0d6a 1410 free_page((unsigned long)kvm->arch.sie_page2);
598841ca 1411 debug_unregister(kvm->arch.dbf);
7d43bafc 1412 sca_dispose(kvm);
78f26131 1413 KVM_EVENT(3, "creation of vm failed: %d", rc);
d89f5eff 1414 return rc;
b0c632db
HC		 1415}
1416
d329c035
CB		 1417void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1418{
1419 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 1420 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 1421 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 1422 kvm_clear_async_pf_completion_queue(vcpu);
bc784cce 1423 if (!kvm_is_ucontrol(vcpu->kvm))
a6e2f683 1424 sca_del_vcpu(vcpu);
27e0393f
CO		 1425
1426 if (kvm_is_ucontrol(vcpu->kvm))
1427 gmap_free(vcpu->arch.gmap);
1428
e6db1d61 1429 if (vcpu->kvm->arch.use_cmma)
b31605c1 1430 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 1431 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 1432
6692cef3 1433 kvm_vcpu_uninit(vcpu);
b110feaf 1434 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 1435}
1436
1437static void kvm_free_vcpus(struct kvm *kvm)
1438{
1439 unsigned int i;
988a2cae 1440 struct kvm_vcpu *vcpu;
d329c035 1441
988a2cae
GN		 1442 kvm_for_each_vcpu(i, vcpu, kvm)
1443 kvm_arch_vcpu_destroy(vcpu);
1444
1445 mutex_lock(&kvm->lock);
1446 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1447 kvm->vcpus[i] = NULL;
1448
1449 atomic_set(&kvm->online_vcpus, 0);
1450 mutex_unlock(&kvm->lock);
d329c035
CB		 1451}
1452
b0c632db
HC		 1453void kvm_arch_destroy_vm(struct kvm *kvm)
1454{
d329c035 1455 kvm_free_vcpus(kvm);
7d43bafc 1456 sca_dispose(kvm);
d329c035 1457 debug_unregister(kvm->arch.dbf);
c54f0d6a 1458 free_page((unsigned long)kvm->arch.sie_page2);
27e0393f
CO		 1459 if (!kvm_is_ucontrol(kvm))
1460 gmap_free(kvm->arch.gmap);
841b91c5 1461 kvm_s390_destroy_adapters(kvm);
67335e63 1462 kvm_s390_clear_float_irqs(kvm);
8335713a 1463 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
b0c632db
HC		 1464}
1465
1466/* Section: vcpu related */
dafd032a
DD		 1467static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1468{
1469 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1470 if (!vcpu->arch.gmap)
1471 return -ENOMEM;
1472 vcpu->arch.gmap->private = vcpu->kvm;
1473
1474 return 0;
1475}
1476
a6e2f683
ED		 1477static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1478{
5e044315 1479 read_lock(&vcpu->kvm->arch.sca_lock);
7d43bafc
ED		 1480 if (vcpu->kvm->arch.use_esca) {
1481 struct esca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1482
7d43bafc 1483 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
10ce32d5 1484 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc
ED		 1485 } else {
1486 struct bsca_block *sca = vcpu->kvm->arch.sca;
1487
1488 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
10ce32d5 1489 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc 1490 }
5e044315 1491 read_unlock(&vcpu->kvm->arch.sca_lock);
a6e2f683
ED		 1492}
1493
eaa78f34 1494static void sca_add_vcpu(struct kvm_vcpu *vcpu)
a6e2f683 1495{
eaa78f34
DH		 1496 read_lock(&vcpu->kvm->arch.sca_lock);
1497 if (vcpu->kvm->arch.use_esca) {
1498 struct esca_block *sca = vcpu->kvm->arch.sca;
7d43bafc 1499
eaa78f34 1500 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1501 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1502 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
25508824 1503 vcpu->arch.sie_block->ecb2 |= 0x04U;
eaa78f34 1504 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
7d43bafc 1505 } else {
eaa78f34 1506 struct bsca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1507
eaa78f34 1508 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1509 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1510 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
eaa78f34 1511 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
7d43bafc 1512 }
eaa78f34 1513 read_unlock(&vcpu->kvm->arch.sca_lock);
5e044315
ED		 1514}
1515
1516/* Basic SCA to Extended SCA data copy routines */
1517static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1518{
1519 d->sda = s->sda;
1520 d->sigp_ctrl.c = s->sigp_ctrl.c;
1521 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1522}
1523
1524static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1525{
1526 int i;
1527
1528 d->ipte_control = s->ipte_control;
1529 d->mcn[0] = s->mcn;
1530 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1531 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1532}
1533
1534static int sca_switch_to_extended(struct kvm *kvm)
1535{
1536 struct bsca_block *old_sca = kvm->arch.sca;
1537 struct esca_block *new_sca;
1538 struct kvm_vcpu *vcpu;
1539 unsigned int vcpu_idx;
1540 u32 scaol, scaoh;
1541
1542 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1543 if (!new_sca)
1544 return -ENOMEM;
1545
1546 scaoh = (u32)((u64)(new_sca) >> 32);
1547 scaol = (u32)(u64)(new_sca) & ~0x3fU;
1548
1549 kvm_s390_vcpu_block_all(kvm);
1550 write_lock(&kvm->arch.sca_lock);
1551
1552 sca_copy_b_to_e(new_sca, old_sca);
1553
1554 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1555 vcpu->arch.sie_block->scaoh = scaoh;
1556 vcpu->arch.sie_block->scaol = scaol;
1557 vcpu->arch.sie_block->ecb2 |= 0x04U;
1558 }
1559 kvm->arch.sca = new_sca;
1560 kvm->arch.use_esca = 1;
1561
1562 write_unlock(&kvm->arch.sca_lock);
1563 kvm_s390_vcpu_unblock_all(kvm);
1564
1565 free_page((unsigned long)old_sca);
1566
8335713a
CB		 1567 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1568 old_sca, kvm->arch.sca);
5e044315 1569 return 0;
a6e2f683
ED		 1570}
1571
1572static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1573{
5e044315
ED		 1574 int rc;
1575
1576 if (id < KVM_S390_BSCA_CPU_SLOTS)
1577 return true;
76a6dd72 1578 if (!sclp.has_esca || !sclp.has_64bscao)
5e044315
ED		 1579 return false;
1580
1581 mutex_lock(&kvm->lock);
1582 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1583 mutex_unlock(&kvm->lock);
1584
1585 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
a6e2f683
ED		 1586}
1587
b0c632db
HC		 1588int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1589{
3c038e6b
DD		 1590 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1591 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 1592 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1593 KVM_SYNC_GPRS |
9eed0735 1594 KVM_SYNC_ACRS |
b028ee3e
DH		 1595 KVM_SYNC_CRS |
1596 KVM_SYNC_ARCH0 |
1597 KVM_SYNC_PFAULT;
c6e5f166
FZ		 1598 if (test_kvm_facility(vcpu->kvm, 64))
1599 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
f6aa6dc4
DH		 1600 /* fprs can be synchronized via vrs, even if the guest has no vx. With
1601 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1602 */
1603 if (MACHINE_HAS_VX)
68c55750 1604 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
6fd8e67d
DH		 1605 else
1606 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
dafd032a
DD		 1607
1608 if (kvm_is_ucontrol(vcpu->kvm))
1609 return __kvm_ucontrol_vcpu_init(vcpu);
1610
b0c632db
HC		 1611 return 0;
1612}
1613
db0758b2
DH		 1614/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1615static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1616{
1617 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
9c23a131 1618 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2 1619 vcpu->arch.cputm_start = get_tod_clock_fast();
9c23a131 1620 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1621}
1622
1623/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1624static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1625{
1626 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
9c23a131 1627 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1628 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1629 vcpu->arch.cputm_start = 0;
9c23a131 1630 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1631}
1632
1633/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1634static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1635{
1636 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
1637 vcpu->arch.cputm_enabled = true;
1638 __start_cpu_timer_accounting(vcpu);
1639}
1640
1641/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1642static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1643{
1644 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
1645 __stop_cpu_timer_accounting(vcpu);
1646 vcpu->arch.cputm_enabled = false;
1647}
1648
1649static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1650{
1651 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1652 __enable_cpu_timer_accounting(vcpu);
1653 preempt_enable();
1654}
1655
1656static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1657{
1658 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1659 __disable_cpu_timer_accounting(vcpu);
1660 preempt_enable();
1661}
1662
4287f247
DH		 1663/* set the cpu timer - may only be called from the VCPU thread itself */
1664void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
1665{
db0758b2 1666 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
9c23a131 1667 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1668 if (vcpu->arch.cputm_enabled)
1669 vcpu->arch.cputm_start = get_tod_clock_fast();
4287f247 1670 vcpu->arch.sie_block->cputm = cputm;
9c23a131 1671 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2 1672 preempt_enable();
4287f247
DH		 1673}
1674
db0758b2 1675/* update and get the cpu timer - can also be called from other VCPU threads */
4287f247
DH		 1676__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
1677{
9c23a131 1678 unsigned int seq;
db0758b2 1679 __u64 value;
db0758b2
DH		 1680
1681 if (unlikely(!vcpu->arch.cputm_enabled))
1682 return vcpu->arch.sie_block->cputm;
1683
9c23a131
DH		 1684 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1685 do {
1686 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
1687 /*
1688 * If the writer would ever execute a read in the critical
1689 * section, e.g. in irq context, we have a deadlock.
1690 */
1691 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
1692 value = vcpu->arch.sie_block->cputm;
1693 /* if cputm_start is 0, accounting is being started/stopped */
1694 if (likely(vcpu->arch.cputm_start))
1695 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1696 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
1697 preempt_enable();
db0758b2 1698 return value;
4287f247
DH		 1699}
1700
b0c632db
HC		 1701void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1702{
9977e886 1703 /* Save host register state */
d0164ee2 1704 save_fpu_regs();
9abc2a08
DH		 1705 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
1706 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
9977e886 1707
6fd8e67d
DH		 1708 if (MACHINE_HAS_VX)
1709 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
1710 else
1711 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
9abc2a08 1712 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
9977e886 1713 if (test_fp_ctl(current->thread.fpu.fpc))
96b2d7a8 1714 /* User space provided an invalid FPC, let's clear it */
9977e886
HB		 1715 current->thread.fpu.fpc = 0;
1716
1717 save_access_regs(vcpu->arch.host_acrs);
59674c1a 1718 restore_access_regs(vcpu->run->s.regs.acrs);
480e5926 1719 gmap_enable(vcpu->arch.gmap);
805de8f4 1720 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
5ebda316 1721 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1722 __start_cpu_timer_accounting(vcpu);
01a745ac 1723 vcpu->cpu = cpu;
b0c632db
HC		 1724}
1725
1726void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1727{
01a745ac 1728 vcpu->cpu = -1;
5ebda316 1729 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1730 __stop_cpu_timer_accounting(vcpu);
805de8f4 1731 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
480e5926 1732 gmap_disable(vcpu->arch.gmap);
9977e886 1733
9abc2a08 1734 /* Save guest register state */
d0164ee2 1735 save_fpu_regs();
9abc2a08 1736 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
9977e886 1737
9abc2a08
DH		 1738 /* Restore host register state */
1739 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
1740 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
9977e886
HB		 1741
1742 save_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 1743 restore_access_regs(vcpu->arch.host_acrs);
1744}
1745
1746static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1747{
1748 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1749 vcpu->arch.sie_block->gpsw.mask = 0UL;
1750 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 1751 kvm_s390_set_prefix(vcpu, 0);
4287f247 1752 kvm_s390_set_cpu_timer(vcpu, 0);
b0c632db
HC		 1753 vcpu->arch.sie_block->ckc = 0UL;
1754 vcpu->arch.sie_block->todpr = 0;
1755 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1756 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1757 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
9abc2a08
DH		 1758 /* make sure the new fpc will be lazily loaded */
1759 save_fpu_regs();
1760 current->thread.fpu.fpc = 0;
b0c632db 1761 vcpu->arch.sie_block->gbea = 1;
672550fb 1762 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 1763 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1764 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 1765 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1766 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 1767 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 1768}
1769
31928aa5 1770void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 1771{
72f25020 1772 mutex_lock(&vcpu->kvm->lock);
fdf03650 1773 preempt_disable();
72f25020 1774 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
fdf03650 1775 preempt_enable();
72f25020 1776 mutex_unlock(&vcpu->kvm->lock);
25508824 1777 if (!kvm_is_ucontrol(vcpu->kvm)) {
dafd032a 1778 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
eaa78f34 1779 sca_add_vcpu(vcpu);
25508824
DH		 1780 }
1781
42897d86
MT		 1782}
1783
5102ee87
TK		 1784static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1785{
9d8d5786 1786 if (!test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 1787 return;
1788
a374e892
TK		 1789 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1790
1791 if (vcpu->kvm->arch.crypto.aes_kw)
1792 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1793 if (vcpu->kvm->arch.crypto.dea_kw)
1794 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1795
5102ee87
TK		 1796 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1797}
1798
b31605c1
DD		 1799void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1800{
1801 free_page(vcpu->arch.sie_block->cbrlo);
1802 vcpu->arch.sie_block->cbrlo = 0;
1803}
1804
1805int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1806{
1807 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1808 if (!vcpu->arch.sie_block->cbrlo)
1809 return -ENOMEM;
1810
1811 vcpu->arch.sie_block->ecb2 |= 0x80;
1812 vcpu->arch.sie_block->ecb2 &= ~0x08;
1813 return 0;
1814}
1815
91520f1a
MM		 1816static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1817{
1818 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1819
91520f1a 1820 vcpu->arch.sie_block->ibc = model->ibc;
80bc79dc 1821 if (test_kvm_facility(vcpu->kvm, 7))
c54f0d6a 1822 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
91520f1a
MM		 1823}
1824
b0c632db
HC		 1825int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1826{
b31605c1 1827 int rc = 0;
b31288fa 1828
9e6dabef
CH		 1829 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1830 CPUSTAT_SM |
a4a4f191
GH		 1831 CPUSTAT_STOPPED);
1832
53df84f8 1833 if (test_kvm_facility(vcpu->kvm, 78))
805de8f4 1834 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
53df84f8 1835 else if (test_kvm_facility(vcpu->kvm, 8))
805de8f4 1836 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
a4a4f191 1837
91520f1a
MM		 1838 kvm_s390_vcpu_setup_model(vcpu);
1839
bd50e8ec
DH		 1840 vcpu->arch.sie_block->ecb = 0x02;
1841 if (test_kvm_facility(vcpu->kvm, 9))
1842 vcpu->arch.sie_block->ecb |= 0x04;
9d8d5786 1843 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
7feb6bb8
MM		 1844 vcpu->arch.sie_block->ecb |= 0x10;
1845
873b425e 1846 if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
d6af0b49 1847 vcpu->arch.sie_block->ecb2 |= 0x08;
48ee7d3a
DH		 1848 vcpu->arch.sie_block->eca = 0x1002000U;
1849 if (sclp.has_cei)
1850 vcpu->arch.sie_block->eca |= 0x80000000U;
11ad65b7
DH		 1851 if (sclp.has_ib)
1852 vcpu->arch.sie_block->eca |= 0x40000000U;
37c5f6c8 1853 if (sclp.has_siif)
217a4406 1854 vcpu->arch.sie_block->eca |= 1;
37c5f6c8 1855 if (sclp.has_sigpif)
ea5f4969 1856 vcpu->arch.sie_block->eca |= 0x10000000U;
c6e5f166
FZ		 1857 if (test_kvm_facility(vcpu->kvm, 64))
1858 vcpu->arch.sie_block->ecb3 |= 0x01;
18280d8b 1859 if (test_kvm_facility(vcpu->kvm, 129)) {
13211ea7
EF		 1860 vcpu->arch.sie_block->eca |= 0x00020000;
1861 vcpu->arch.sie_block->ecd |= 0x20000000;
1862 }
c6e5f166 1863 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
492d8642 1864 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
95ca2cb5
JF		 1865 if (test_kvm_facility(vcpu->kvm, 74))
1866 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
5a5e6536 1867
e6db1d61 1868 if (vcpu->kvm->arch.use_cmma) {
b31605c1
DD		 1869 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1870 if (rc)
1871 return rc;
b31288fa 1872 }
0ac96caf 1873 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 1874 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 1875
5102ee87
TK		 1876 kvm_s390_vcpu_crypto_setup(vcpu);
1877
b31605c1 1878 return rc;
b0c632db
HC		 1879}
1880
1881struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1882 unsigned int id)
1883{
4d47555a 1884 struct kvm_vcpu *vcpu;
7feb6bb8 1885 struct sie_page *sie_page;
4d47555a
CO		 1886 int rc = -EINVAL;
1887
4215825e 1888 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
4d47555a
CO		 1889 goto out;
1890
1891 rc = -ENOMEM;
b0c632db 1892
b110feaf 1893 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 1894 if (!vcpu)
4d47555a 1895 goto out;
b0c632db 1896
7feb6bb8
MM		 1897 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1898 if (!sie_page)
b0c632db
HC		 1899 goto out_free_cpu;
1900
7feb6bb8
MM		 1901 vcpu->arch.sie_block = &sie_page->sie_block;
1902 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1903
efed1104
DH		 1904 /* the real guest size will always be smaller than msl */
1905 vcpu->arch.sie_block->mso = 0;
1906 vcpu->arch.sie_block->msl = sclp.hamax;
1907
b0c632db 1908 vcpu->arch.sie_block->icpua = id;
ba5c1e9b 1909 spin_lock_init(&vcpu->arch.local_int.lock);
ba5c1e9b 1910 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 1911 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 1912 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
9c23a131 1913 seqcount_init(&vcpu->arch.cputm_seqcount);
ba5c1e9b 1914
b0c632db
HC		 1915 rc = kvm_vcpu_init(vcpu, kvm, id);
1916 if (rc)
9abc2a08 1917 goto out_free_sie_block;
8335713a 1918 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
b0c632db 1919 vcpu->arch.sie_block);
ade38c31 1920 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 1921
b0c632db 1922 return vcpu;
7b06bf2f
WY		 1923out_free_sie_block:
1924 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 1925out_free_cpu:
b110feaf 1926 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 1927out:
b0c632db
HC		 1928 return ERR_PTR(rc);
1929}
1930
b0c632db
HC		 1931int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1932{
9a022067 1933 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 1934}
1935
27406cd5 1936void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
49b99e1e 1937{
805de8f4 1938 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
61a6df54 1939 exit_sie(vcpu);
49b99e1e
CB		 1940}
1941
27406cd5 1942void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
49b99e1e 1943{
805de8f4 1944 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
49b99e1e
CB		 1945}
1946
8e236546
CB		 1947static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1948{
805de8f4 1949 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
61a6df54 1950 exit_sie(vcpu);
8e236546
CB		 1951}
1952
1953static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1954{
9bf9fde2 1955 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
8e236546
CB		 1956}
1957
49b99e1e
CB		 1958/*
1959 * Kick a guest cpu out of SIE and wait until SIE is not running.
1960 * If the CPU is not running (e.g. waiting as idle) the function will
1961 * return immediately. */
1962void exit_sie(struct kvm_vcpu *vcpu)
1963{
805de8f4 1964 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
49b99e1e
CB		 1965 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1966 cpu_relax();
1967}
1968
8e236546
CB		 1969/* Kick a guest cpu out of SIE to process a request synchronously */
1970void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
49b99e1e 1971{
8e236546
CB		 1972 kvm_make_request(req, vcpu);
1973 kvm_s390_vcpu_request(vcpu);
49b99e1e
CB		 1974}
1975
2c70fe44
CB		 1976static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1977{
1978 int i;
1979 struct kvm *kvm = gmap->private;
1980 struct kvm_vcpu *vcpu;
1981
1982 kvm_for_each_vcpu(i, vcpu, kvm) {
1983 /* match against both prefix pages */
fda902cb 1984 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
2c70fe44 1985 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
8e236546 1986 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44
CB		 1987 }
1988 }
1989}
1990
b6d33834
CD		 1991int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1992{
1993 /* kvm common code refers to this, but never calls it */
1994 BUG();
1995 return 0;
1996}
1997
14eebd91
CO		 1998static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1999 struct kvm_one_reg *reg)
2000{
2001 int r = -EINVAL;
2002
2003 switch (reg->id) {
29b7c71b
CO		 2004 case KVM_REG_S390_TODPR:
2005 r = put_user(vcpu->arch.sie_block->todpr,
2006 (u32 __user *)reg->addr);
2007 break;
2008 case KVM_REG_S390_EPOCHDIFF:
2009 r = put_user(vcpu->arch.sie_block->epoch,
2010 (u64 __user *)reg->addr);
2011 break;
46a6dd1c 2012 case KVM_REG_S390_CPU_TIMER:
4287f247 2013 r = put_user(kvm_s390_get_cpu_timer(vcpu),
46a6dd1c
J		 2014 (u64 __user *)reg->addr);
2015 break;
2016 case KVM_REG_S390_CLOCK_COMP:
2017 r = put_user(vcpu->arch.sie_block->ckc,
2018 (u64 __user *)reg->addr);
2019 break;
536336c2
DD		 2020 case KVM_REG_S390_PFTOKEN:
2021 r = put_user(vcpu->arch.pfault_token,
2022 (u64 __user *)reg->addr);
2023 break;
2024 case KVM_REG_S390_PFCOMPARE:
2025 r = put_user(vcpu->arch.pfault_compare,
2026 (u64 __user *)reg->addr);
2027 break;
2028 case KVM_REG_S390_PFSELECT:
2029 r = put_user(vcpu->arch.pfault_select,
2030 (u64 __user *)reg->addr);
2031 break;
672550fb
CB		 2032 case KVM_REG_S390_PP:
2033 r = put_user(vcpu->arch.sie_block->pp,
2034 (u64 __user *)reg->addr);
2035 break;
afa45ff5
CB		 2036 case KVM_REG_S390_GBEA:
2037 r = put_user(vcpu->arch.sie_block->gbea,
2038 (u64 __user *)reg->addr);
2039 break;
14eebd91
CO		 2040 default:
2041 break;
2042 }
2043
2044 return r;
2045}
2046
2047static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2048 struct kvm_one_reg *reg)
2049{
2050 int r = -EINVAL;
4287f247 2051 __u64 val;
14eebd91
CO		 2052
2053 switch (reg->id) {
29b7c71b
CO		 2054 case KVM_REG_S390_TODPR:
2055 r = get_user(vcpu->arch.sie_block->todpr,
2056 (u32 __user *)reg->addr);
2057 break;
2058 case KVM_REG_S390_EPOCHDIFF:
2059 r = get_user(vcpu->arch.sie_block->epoch,
2060 (u64 __user *)reg->addr);
2061 break;
46a6dd1c 2062 case KVM_REG_S390_CPU_TIMER:
4287f247
DH		 2063 r = get_user(val, (u64 __user *)reg->addr);
2064 if (!r)
2065 kvm_s390_set_cpu_timer(vcpu, val);
46a6dd1c
J		 2066 break;
2067 case KVM_REG_S390_CLOCK_COMP:
2068 r = get_user(vcpu->arch.sie_block->ckc,
2069 (u64 __user *)reg->addr);
2070 break;
536336c2
DD		 2071 case KVM_REG_S390_PFTOKEN:
2072 r = get_user(vcpu->arch.pfault_token,
2073 (u64 __user *)reg->addr);
9fbd8082
DH		 2074 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2075 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 2076 break;
2077 case KVM_REG_S390_PFCOMPARE:
2078 r = get_user(vcpu->arch.pfault_compare,
2079 (u64 __user *)reg->addr);
2080 break;
2081 case KVM_REG_S390_PFSELECT:
2082 r = get_user(vcpu->arch.pfault_select,
2083 (u64 __user *)reg->addr);
2084 break;
672550fb
CB		 2085 case KVM_REG_S390_PP:
2086 r = get_user(vcpu->arch.sie_block->pp,
2087 (u64 __user *)reg->addr);
2088 break;
afa45ff5
CB		 2089 case KVM_REG_S390_GBEA:
2090 r = get_user(vcpu->arch.sie_block->gbea,
2091 (u64 __user *)reg->addr);
2092 break;
14eebd91
CO		 2093 default:
2094 break;
2095 }
2096
2097 return r;
2098}
b6d33834 2099
b0c632db
HC		 2100static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2101{
b0c632db 2102 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 2103 return 0;
2104}
2105
2106int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2107{
5a32c1af 2108 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 2109 return 0;
2110}
2111
2112int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2113{
5a32c1af 2114 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 2115 return 0;
2116}
2117
2118int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2119 struct kvm_sregs *sregs)
2120{
59674c1a 2121 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 2122 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
59674c1a 2123 restore_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 2124 return 0;
2125}
2126
2127int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2128 struct kvm_sregs *sregs)
2129{
59674c1a 2130 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 2131 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 2132 return 0;
2133}
2134
2135int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2136{
9abc2a08
DH		 2137 /* make sure the new values will be lazily loaded */
2138 save_fpu_regs();
4725c860
MS		 2139 if (test_fp_ctl(fpu->fpc))
2140 return -EINVAL;
9abc2a08
DH		 2141 current->thread.fpu.fpc = fpu->fpc;
2142 if (MACHINE_HAS_VX)
2143 convert_fp_to_vx(current->thread.fpu.vxrs, (freg_t *)fpu->fprs);
2144 else
2145 memcpy(current->thread.fpu.fprs, &fpu->fprs, sizeof(fpu->fprs));
b0c632db
HC		 2146 return 0;
2147}
2148
2149int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2150{
9abc2a08
DH		 2151 /* make sure we have the latest values */
2152 save_fpu_regs();
2153 if (MACHINE_HAS_VX)
2154 convert_vx_to_fp((freg_t *)fpu->fprs, current->thread.fpu.vxrs);
2155 else
2156 memcpy(fpu->fprs, current->thread.fpu.fprs, sizeof(fpu->fprs));
2157 fpu->fpc = current->thread.fpu.fpc;
b0c632db
HC		 2158 return 0;
2159}
2160
2161static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2162{
2163 int rc = 0;
2164
7a42fdc2 2165 if (!is_vcpu_stopped(vcpu))
b0c632db 2166 rc = -EBUSY;
d7b0b5eb
CO		 2167 else {
2168 vcpu->run->psw_mask = psw.mask;
2169 vcpu->run->psw_addr = psw.addr;
2170 }
b0c632db
HC		 2171 return rc;
2172}
2173
2174int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2175 struct kvm_translation *tr)
2176{
2177 return -EINVAL; /* not implemented yet */
2178}
2179
27291e21
DH		 2180#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2181 KVM_GUESTDBG_USE_HW_BP | \
2182 KVM_GUESTDBG_ENABLE)
2183
d0bfb940
JK		 2184int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2185 struct kvm_guest_debug *dbg)
b0c632db 2186{
27291e21
DH		 2187 int rc = 0;
2188
2189 vcpu->guest_debug = 0;
2190 kvm_s390_clear_bp_data(vcpu);
2191
2de3bfc2 2192 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21 2193 return -EINVAL;
89b5b4de
DH		 2194 if (!sclp.has_gpere)
2195 return -EINVAL;
27291e21
DH		 2196
2197 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2198 vcpu->guest_debug = dbg->control;
2199 /* enforce guest PER */
805de8f4 2200 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2201
2202 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2203 rc = kvm_s390_import_bp_data(vcpu, dbg);
2204 } else {
805de8f4 2205 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2206 vcpu->arch.guestdbg.last_bp = 0;
2207 }
2208
2209 if (rc) {
2210 vcpu->guest_debug = 0;
2211 kvm_s390_clear_bp_data(vcpu);
805de8f4 2212 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2213 }
2214
2215 return rc;
b0c632db
HC		 2216}
2217
62d9f0db
MT		 2218int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2219 struct kvm_mp_state *mp_state)
2220{
6352e4d2
DH		 2221 /* CHECK_STOP and LOAD are not supported yet */
2222 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2223 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 2224}
2225
2226int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2227 struct kvm_mp_state *mp_state)
2228{
6352e4d2
DH		 2229 int rc = 0;
2230
2231 /* user space knows about this interface - let it control the state */
2232 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2233
2234 switch (mp_state->mp_state) {
2235 case KVM_MP_STATE_STOPPED:
2236 kvm_s390_vcpu_stop(vcpu);
2237 break;
2238 case KVM_MP_STATE_OPERATING:
2239 kvm_s390_vcpu_start(vcpu);
2240 break;
2241 case KVM_MP_STATE_LOAD:
2242 case KVM_MP_STATE_CHECK_STOP:
2243 /* fall through - CHECK_STOP and LOAD are not supported yet */
2244 default:
2245 rc = -ENXIO;
2246 }
2247
2248 return rc;
62d9f0db
MT		 2249}
2250
8ad35755
DH		 2251static bool ibs_enabled(struct kvm_vcpu *vcpu)
2252{
2253 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2254}
2255
2c70fe44
CB		 2256static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2257{
8ad35755 2258retry:
8e236546 2259 kvm_s390_vcpu_request_handled(vcpu);
586b7ccd
CB		 2260 if (!vcpu->requests)
2261 return 0;
2c70fe44
CB		 2262 /*
2263 * We use MMU_RELOAD just to re-arm the ipte notifier for the
2264 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
2265 * This ensures that the ipte instruction for this request has
2266 * already finished. We might race against a second unmapper that
2267 * wants to set the blocking bit. Lets just retry the request loop.
2268 */
8ad35755 2269 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44
CB		 2270 int rc;
2271 rc = gmap_ipte_notify(vcpu->arch.gmap,
fda902cb 2272 kvm_s390_get_prefix(vcpu),
2c70fe44
CB		 2273 PAGE_SIZE * 2);
2274 if (rc)
2275 return rc;
8ad35755 2276 goto retry;
2c70fe44 2277 }
8ad35755 2278
d3d692c8
DH		 2279 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2280 vcpu->arch.sie_block->ihcpu = 0xffff;
2281 goto retry;
2282 }
2283
8ad35755
DH		 2284 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2285 if (!ibs_enabled(vcpu)) {
2286 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
805de8f4 2287 atomic_or(CPUSTAT_IBS,
8ad35755
DH		 2288 &vcpu->arch.sie_block->cpuflags);
2289 }
2290 goto retry;
2c70fe44 2291 }
8ad35755
DH		 2292
2293 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2294 if (ibs_enabled(vcpu)) {
2295 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
805de8f4 2296 atomic_andnot(CPUSTAT_IBS,
8ad35755
DH		 2297 &vcpu->arch.sie_block->cpuflags);
2298 }
2299 goto retry;
2300 }
2301
0759d068
DH		 2302 /* nothing to do, just clear the request */
2303 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2304
2c70fe44
CB		 2305 return 0;
2306}
2307
25ed1675
DH		 2308void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2309{
2310 struct kvm_vcpu *vcpu;
2311 int i;
2312
2313 mutex_lock(&kvm->lock);
2314 preempt_disable();
2315 kvm->arch.epoch = tod - get_tod_clock();
2316 kvm_s390_vcpu_block_all(kvm);
2317 kvm_for_each_vcpu(i, vcpu, kvm)
2318 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2319 kvm_s390_vcpu_unblock_all(kvm);
2320 preempt_enable();
2321 mutex_unlock(&kvm->lock);
2322}
2323
fa576c58
TH		 2324/**
2325 * kvm_arch_fault_in_page - fault-in guest page if necessary
2326 * @vcpu: The corresponding virtual cpu
2327 * @gpa: Guest physical address
2328 * @writable: Whether the page should be writable or not
2329 *
2330 * Make sure that a guest page has been faulted-in on the host.
2331 *
2332 * Return: Zero on success, negative error code otherwise.
2333 */
2334long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 2335{
527e30b4
MS		 2336 return gmap_fault(vcpu->arch.gmap, gpa,
2337 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 2338}
2339
3c038e6b
DD		 2340static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2341 unsigned long token)
2342{
2343 struct kvm_s390_interrupt inti;
383d0b05 2344 struct kvm_s390_irq irq;
3c038e6b
DD		 2345
2346 if (start_token) {
383d0b05
JF		 2347 irq.u.ext.ext_params2 = token;
2348 irq.type = KVM_S390_INT_PFAULT_INIT;
2349 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 2350 } else {
2351 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 2352 inti.parm64 = token;
3c038e6b
DD		 2353 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2354 }
2355}
2356
2357void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2358 struct kvm_async_pf *work)
2359{
2360 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2361 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2362}
2363
2364void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2365 struct kvm_async_pf *work)
2366{
2367 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2368 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2369}
2370
2371void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2372 struct kvm_async_pf *work)
2373{
2374 /* s390 will always inject the page directly */
2375}
2376
2377bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2378{
2379 /*
2380 * s390 will always inject the page directly,
2381 * but we still want check_async_completion to cleanup
2382 */
2383 return true;
2384}
2385
2386static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2387{
2388 hva_t hva;
2389 struct kvm_arch_async_pf arch;
2390 int rc;
2391
2392 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2393 return 0;
2394 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2395 vcpu->arch.pfault_compare)
2396 return 0;
2397 if (psw_extint_disabled(vcpu))
2398 return 0;
9a022067 2399 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 2400 return 0;
2401 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2402 return 0;
2403 if (!vcpu->arch.gmap->pfault_enabled)
2404 return 0;
2405
81480cc1
HC		 2406 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2407 hva += current->thread.gmap_addr & ~PAGE_MASK;
2408 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 2409 return 0;
2410
2411 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2412 return rc;
2413}
2414
3fb4c40f 2415static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 2416{
3fb4c40f 2417 int rc, cpuflags;
e168bf8d 2418
3c038e6b
DD		 2419 /*
2420 * On s390 notifications for arriving pages will be delivered directly
2421 * to the guest but the house keeping for completed pfaults is
2422 * handled outside the worker.
2423 */
2424 kvm_check_async_pf_completion(vcpu);
2425
7ec7c8c7
CB		 2426 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2427 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
b0c632db
HC		 2428
2429 if (need_resched())
2430 schedule();
2431
d3a73acb 2432 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 2433 s390_handle_mcck();
2434
79395031
JF		 2435 if (!kvm_is_ucontrol(vcpu->kvm)) {
2436 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2437 if (rc)
2438 return rc;
2439 }
0ff31867 2440
2c70fe44
CB		 2441 rc = kvm_s390_handle_requests(vcpu);
2442 if (rc)
2443 return rc;
2444
27291e21
DH		 2445 if (guestdbg_enabled(vcpu)) {
2446 kvm_s390_backup_guest_per_regs(vcpu);
2447 kvm_s390_patch_guest_per_regs(vcpu);
2448 }
2449
b0c632db 2450 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 2451 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2452 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2453 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 2454
3fb4c40f
TH		 2455 return 0;
2456}
2457
492d8642
TH		 2458static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2459{
56317920
DH		 2460 struct kvm_s390_pgm_info pgm_info = {
2461 .code = PGM_ADDRESSING,
2462 };
2463 u8 opcode, ilen;
492d8642
TH		 2464 int rc;
2465
2466 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2467 trace_kvm_s390_sie_fault(vcpu);
2468
2469 /*
2470 * We want to inject an addressing exception, which is defined as a
2471 * suppressing or terminating exception. However, since we came here
2472 * by a DAT access exception, the PSW still points to the faulting
2473 * instruction since DAT exceptions are nullifying. So we've got
2474 * to look up the current opcode to get the length of the instruction
2475 * to be able to forward the PSW.
2476 */
65977322 2477 rc = read_guest_instr(vcpu, &opcode, 1);
56317920 2478 ilen = insn_length(opcode);
9b0d721a
DH		 2479 if (rc < 0) {
2480 return rc;
2481 } else if (rc) {
2482 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2483 * Forward by arbitrary ilc, injection will take care of
2484 * nullification if necessary.
2485 */
2486 pgm_info = vcpu->arch.pgm;
2487 ilen = 4;
2488 }
56317920
DH		 2489 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2490 kvm_s390_forward_psw(vcpu, ilen);
2491 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
492d8642
TH		 2492}
2493
3fb4c40f
TH		 2494static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2495{
2b29a9fd
DD		 2496 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2497 vcpu->arch.sie_block->icptcode);
2498 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2499
27291e21
DH		 2500 if (guestdbg_enabled(vcpu))
2501 kvm_s390_restore_guest_per_regs(vcpu);
2502
7ec7c8c7
CB		 2503 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2504 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
71f116bf
DH		 2505
2506 if (vcpu->arch.sie_block->icptcode > 0) {
2507 int rc = kvm_handle_sie_intercept(vcpu);
2508
2509 if (rc != -EOPNOTSUPP)
2510 return rc;
2511 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2512 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2513 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2514 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2515 return -EREMOTE;
2516 } else if (exit_reason != -EFAULT) {
2517 vcpu->stat.exit_null++;
2518 return 0;
210b1607
TH		 2519 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2520 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2521 vcpu->run->s390_ucontrol.trans_exc_code =
2522 current->thread.gmap_addr;
2523 vcpu->run->s390_ucontrol.pgm_code = 0x10;
71f116bf 2524 return -EREMOTE;
24eb3a82 2525 } else if (current->thread.gmap_pfault) {
3c038e6b 2526 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 2527 current->thread.gmap_pfault = 0;
71f116bf
DH		 2528 if (kvm_arch_setup_async_pf(vcpu))
2529 return 0;
2530 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
a76ccff6 2531 }
71f116bf 2532 return vcpu_post_run_fault_in_sie(vcpu);
3fb4c40f
TH		 2533}
2534
2535static int __vcpu_run(struct kvm_vcpu *vcpu)
2536{
2537 int rc, exit_reason;
2538
800c1065
TH		 2539 /*
2540 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2541 * ning the guest), so that memslots (and other stuff) are protected
2542 */
2543 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2544
a76ccff6
TH		 2545 do {
2546 rc = vcpu_pre_run(vcpu);
2547 if (rc)
2548 break;
3fb4c40f 2549
800c1065 2550 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 2551 /*
2552 * As PF_VCPU will be used in fault handler, between
2553 * guest_enter and guest_exit should be no uaccess.
2554 */
0097d12e
CB		 2555 local_irq_disable();
2556 __kvm_guest_enter();
db0758b2 2557 __disable_cpu_timer_accounting(vcpu);
0097d12e 2558 local_irq_enable();
a76ccff6
TH		 2559 exit_reason = sie64a(vcpu->arch.sie_block,
2560 vcpu->run->s.regs.gprs);
0097d12e 2561 local_irq_disable();
db0758b2 2562 __enable_cpu_timer_accounting(vcpu);
0097d12e
CB		 2563 __kvm_guest_exit();
2564 local_irq_enable();
800c1065 2565 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 2566
2567 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 2568 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 2569
800c1065 2570 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 2571 return rc;
b0c632db
HC		 2572}
2573
b028ee3e
DH		 2574static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2575{
2576 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2577 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2578 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2579 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2580 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2581 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 2582 /* some control register changes require a tlb flush */
2583 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 2584 }
2585 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4287f247 2586 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
b028ee3e
DH		 2587 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2588 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2589 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2590 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2591 }
2592 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2593 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2594 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2595 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 2596 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2597 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e
DH		 2598 }
2599 kvm_run->kvm_dirty_regs = 0;
2600}
2601
2602static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2603{
2604 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2605 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2606 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2607 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4287f247 2608 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
b028ee3e
DH		 2609 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2610 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2611 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2612 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2613 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2614 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2615 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2616}
2617
b0c632db
HC		 2618int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2619{
8f2abe6a 2620 int rc;
b0c632db
HC		 2621 sigset_t sigsaved;
2622
27291e21
DH		 2623 if (guestdbg_exit_pending(vcpu)) {
2624 kvm_s390_prepare_debug_exit(vcpu);
2625 return 0;
2626 }
2627
b0c632db
HC		 2628 if (vcpu->sigset_active)
2629 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2630
6352e4d2
DH		 2631 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2632 kvm_s390_vcpu_start(vcpu);
2633 } else if (is_vcpu_stopped(vcpu)) {
ea2cdd27 2634 pr_err_ratelimited("can't run stopped vcpu %d\n",
6352e4d2
DH		 2635 vcpu->vcpu_id);
2636 return -EINVAL;
2637 }
b0c632db 2638
b028ee3e 2639 sync_regs(vcpu, kvm_run);
db0758b2 2640 enable_cpu_timer_accounting(vcpu);
d7b0b5eb 2641
dab4079d 2642 might_fault();
a76ccff6 2643 rc = __vcpu_run(vcpu);
9ace903d 2644
b1d16c49
CE		 2645 if (signal_pending(current) && !rc) {
2646 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 2647 rc = -EINTR;
b1d16c49 2648 }
8f2abe6a 2649
27291e21
DH		 2650 if (guestdbg_exit_pending(vcpu) && !rc) {
2651 kvm_s390_prepare_debug_exit(vcpu);
2652 rc = 0;
2653 }
2654
8f2abe6a 2655 if (rc == -EREMOTE) {
71f116bf 2656 /* userspace support is needed, kvm_run has been prepared */
8f2abe6a
CB		 2657 rc = 0;
2658 }
b0c632db 2659
db0758b2 2660 disable_cpu_timer_accounting(vcpu);
b028ee3e 2661 store_regs(vcpu, kvm_run);
d7b0b5eb 2662
b0c632db
HC		 2663 if (vcpu->sigset_active)
2664 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2665
b0c632db 2666 vcpu->stat.exit_userspace++;
7e8e6ab4 2667 return rc;
b0c632db
HC		 2668}
2669
b0c632db
HC		 2670/*
2671 * store status at address
2672 * we use have two special cases:
2673 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2674 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2675 */
d0bce605 2676int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 2677{
092670cd 2678 unsigned char archmode = 1;
9abc2a08 2679 freg_t fprs[NUM_FPRS];
fda902cb 2680 unsigned int px;
4287f247 2681 u64 clkcomp, cputm;
d0bce605 2682 int rc;
b0c632db 2683
d9a3a09a 2684 px = kvm_s390_get_prefix(vcpu);
d0bce605
HC		 2685 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2686 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 2687 return -EFAULT;
d9a3a09a 2688 gpa = 0;
d0bce605
HC		 2689 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2690 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 2691 return -EFAULT;
d9a3a09a
MS		 2692 gpa = px;
2693 } else
2694 gpa -= __LC_FPREGS_SAVE_AREA;
9abc2a08
DH		 2695
2696 /* manually convert vector registers if necessary */
2697 if (MACHINE_HAS_VX) {
9522b37f 2698 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08
DH		 2699 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2700 fprs, 128);
2701 } else {
2702 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
6fd8e67d 2703 vcpu->run->s.regs.fprs, 128);
9abc2a08 2704 }
d9a3a09a 2705 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
d0bce605 2706 vcpu->run->s.regs.gprs, 128);
d9a3a09a 2707 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
d0bce605 2708 &vcpu->arch.sie_block->gpsw, 16);
d9a3a09a 2709 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
fda902cb 2710 &px, 4);
d9a3a09a 2711 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
9abc2a08 2712 &vcpu->run->s.regs.fpc, 4);
d9a3a09a 2713 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
d0bce605 2714 &vcpu->arch.sie_block->todpr, 4);
4287f247 2715 cputm = kvm_s390_get_cpu_timer(vcpu);
d9a3a09a 2716 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4287f247 2717 &cputm, 8);
178bd789 2718 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d9a3a09a 2719 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
d0bce605 2720 &clkcomp, 8);
d9a3a09a 2721 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
d0bce605 2722 &vcpu->run->s.regs.acrs, 64);
d9a3a09a 2723 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
d0bce605
HC		 2724 &vcpu->arch.sie_block->gcr, 128);
2725 return rc ? -EFAULT : 0;
b0c632db
HC		 2726}
2727
e879892c
TH		 2728int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2729{
2730 /*
2731 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2732 * copying in vcpu load/put. Lets update our copies before we save
2733 * it into the save area
2734 */
d0164ee2 2735 save_fpu_regs();
9abc2a08 2736 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
e879892c
TH		 2737 save_access_regs(vcpu->run->s.regs.acrs);
2738
2739 return kvm_s390_store_status_unloaded(vcpu, addr);
2740}
2741
bc17de7c
EF		 2742/*
2743 * store additional status at address
2744 */
2745int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2746 unsigned long gpa)
2747{
2748 /* Only bits 0-53 are used for address formation */
2749 if (!(gpa & ~0x3ff))
2750 return 0;
2751
2752 return write_guest_abs(vcpu, gpa & ~0x3ff,
2753 (void *)&vcpu->run->s.regs.vrs, 512);
2754}
2755
2756int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2757{
2758 if (!test_kvm_facility(vcpu->kvm, 129))
2759 return 0;
2760
2761 /*
2762 * The guest VXRS are in the host VXRs due to the lazy
9977e886
HB		 2763 * copying in vcpu load/put. We can simply call save_fpu_regs()
2764 * to save the current register state because we are in the
2765 * middle of a load/put cycle.
2766 *
2767 * Let's update our copies before we save it into the save area.
bc17de7c 2768 */
d0164ee2 2769 save_fpu_regs();
bc17de7c
EF		 2770
2771 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2772}
2773
8ad35755
DH		 2774static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2775{
2776 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
8e236546 2777 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
8ad35755
DH		 2778}
2779
2780static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2781{
2782 unsigned int i;
2783 struct kvm_vcpu *vcpu;
2784
2785 kvm_for_each_vcpu(i, vcpu, kvm) {
2786 __disable_ibs_on_vcpu(vcpu);
2787 }
2788}
2789
2790static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2791{
2792 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
8e236546 2793 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
8ad35755
DH		 2794}
2795
6852d7b6
DH		 2796void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2797{
8ad35755
DH		 2798 int i, online_vcpus, started_vcpus = 0;
2799
2800 if (!is_vcpu_stopped(vcpu))
2801 return;
2802
6852d7b6 2803 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 2804 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2805 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2806 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2807
2808 for (i = 0; i < online_vcpus; i++) {
2809 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2810 started_vcpus++;
2811 }
2812
2813 if (started_vcpus == 0) {
2814 /* we're the only active VCPU -> speed it up */
2815 __enable_ibs_on_vcpu(vcpu);
2816 } else if (started_vcpus == 1) {
2817 /*
2818 * As we are starting a second VCPU, we have to disable
2819 * the IBS facility on all VCPUs to remove potentially
2820 * oustanding ENABLE requests.
2821 */
2822 __disable_ibs_on_all_vcpus(vcpu->kvm);
2823 }
2824
805de8f4 2825 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2826 /*
2827 * Another VCPU might have used IBS while we were offline.
2828 * Let's play safe and flush the VCPU at startup.
2829 */
d3d692c8 2830 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 2831 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2832 return;
6852d7b6
DH		 2833}
2834
2835void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2836{
8ad35755
DH		 2837 int i, online_vcpus, started_vcpus = 0;
2838 struct kvm_vcpu *started_vcpu = NULL;
2839
2840 if (is_vcpu_stopped(vcpu))
2841 return;
2842
6852d7b6 2843 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 2844 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2845 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2846 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2847
32f5ff63 2848 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 2849 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 2850
805de8f4 2851 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2852 __disable_ibs_on_vcpu(vcpu);
2853
2854 for (i = 0; i < online_vcpus; i++) {
2855 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2856 started_vcpus++;
2857 started_vcpu = vcpu->kvm->vcpus[i];
2858 }
2859 }
2860
2861 if (started_vcpus == 1) {
2862 /*
2863 * As we only have one VCPU left, we want to enable the
2864 * IBS facility for that VCPU to speed it up.
2865 */
2866 __enable_ibs_on_vcpu(started_vcpu);
2867 }
2868
433b9ee4 2869 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2870 return;
6852d7b6
DH		 2871}
2872
d6712df9
CH		 2873static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2874 struct kvm_enable_cap *cap)
2875{
2876 int r;
2877
2878 if (cap->flags)
2879 return -EINVAL;
2880
2881 switch (cap->cap) {
fa6b7fe9
CH		 2882 case KVM_CAP_S390_CSS_SUPPORT:
2883 if (!vcpu->kvm->arch.css_support) {
2884 vcpu->kvm->arch.css_support = 1;
c92ea7b9 2885 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
fa6b7fe9
CH		 2886 trace_kvm_s390_enable_css(vcpu->kvm);
2887 }
2888 r = 0;
2889 break;
d6712df9
CH		 2890 default:
2891 r = -EINVAL;
2892 break;
2893 }
2894 return r;
2895}
2896
41408c28
TH		 2897static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2898 struct kvm_s390_mem_op *mop)
2899{
2900 void __user *uaddr = (void __user *)mop->buf;
2901 void *tmpbuf = NULL;
2902 int r, srcu_idx;
2903 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2904 | KVM_S390_MEMOP_F_CHECK_ONLY;
2905
2906 if (mop->flags & ~supported_flags)
2907 return -EINVAL;
2908
2909 if (mop->size > MEM_OP_MAX_SIZE)
2910 return -E2BIG;
2911
2912 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2913 tmpbuf = vmalloc(mop->size);
2914 if (!tmpbuf)
2915 return -ENOMEM;
2916 }
2917
2918 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2919
2920 switch (mop->op) {
2921 case KVM_S390_MEMOP_LOGICAL_READ:
2922 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 2923 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2924 mop->size, GACC_FETCH);
41408c28
TH		 2925 break;
2926 }
2927 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2928 if (r == 0) {
2929 if (copy_to_user(uaddr, tmpbuf, mop->size))
2930 r = -EFAULT;
2931 }
2932 break;
2933 case KVM_S390_MEMOP_LOGICAL_WRITE:
2934 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 2935 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2936 mop->size, GACC_STORE);
41408c28
TH		 2937 break;
2938 }
2939 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2940 r = -EFAULT;
2941 break;
2942 }
2943 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2944 break;
2945 default:
2946 r = -EINVAL;
2947 }
2948
2949 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2950
2951 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2952 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2953
2954 vfree(tmpbuf);
2955 return r;
2956}
2957
b0c632db
HC		 2958long kvm_arch_vcpu_ioctl(struct file *filp,
2959 unsigned int ioctl, unsigned long arg)
2960{
2961 struct kvm_vcpu *vcpu = filp->private_data;
2962 void __user *argp = (void __user *)arg;
800c1065 2963 int idx;
bc923cc9 2964 long r;
b0c632db 2965
93736624 2966 switch (ioctl) {
47b43c52
JF		 2967 case KVM_S390_IRQ: {
2968 struct kvm_s390_irq s390irq;
2969
2970 r = -EFAULT;
2971 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2972 break;
2973 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2974 break;
2975 }
93736624 2976 case KVM_S390_INTERRUPT: {
ba5c1e9b 2977 struct kvm_s390_interrupt s390int;
383d0b05 2978 struct kvm_s390_irq s390irq;
ba5c1e9b 2979
93736624 2980 r = -EFAULT;
ba5c1e9b 2981 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624 2982 break;
383d0b05
JF		 2983 if (s390int_to_s390irq(&s390int, &s390irq))
2984 return -EINVAL;
2985 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
93736624 2986 break;
ba5c1e9b 2987 }
b0c632db 2988 case KVM_S390_STORE_STATUS:
800c1065 2989 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 2990 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 2991 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 2992 break;
b0c632db
HC		 2993 case KVM_S390_SET_INITIAL_PSW: {
2994 psw_t psw;
2995
bc923cc9 2996 r = -EFAULT;
b0c632db 2997 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 2998 break;
2999 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3000 break;
b0c632db
HC		 3001 }
3002 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 3003 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3004 break;
14eebd91
CO		 3005 case KVM_SET_ONE_REG:
3006 case KVM_GET_ONE_REG: {
3007 struct kvm_one_reg reg;
3008 r = -EFAULT;
3009 if (copy_from_user(&reg, argp, sizeof(reg)))
3010 break;
3011 if (ioctl == KVM_SET_ONE_REG)
3012 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3013 else
3014 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3015 break;
3016 }
27e0393f
CO		 3017#ifdef CONFIG_KVM_S390_UCONTROL
3018 case KVM_S390_UCAS_MAP: {
3019 struct kvm_s390_ucas_mapping ucasmap;
3020
3021 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3022 r = -EFAULT;
3023 break;
3024 }
3025
3026 if (!kvm_is_ucontrol(vcpu->kvm)) {
3027 r = -EINVAL;
3028 break;
3029 }
3030
3031 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3032 ucasmap.vcpu_addr, ucasmap.length);
3033 break;
3034 }
3035 case KVM_S390_UCAS_UNMAP: {
3036 struct kvm_s390_ucas_mapping ucasmap;
3037
3038 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3039 r = -EFAULT;
3040 break;
3041 }
3042
3043 if (!kvm_is_ucontrol(vcpu->kvm)) {
3044 r = -EINVAL;
3045 break;
3046 }
3047
3048 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3049 ucasmap.length);
3050 break;
3051 }
3052#endif
ccc7910f 3053 case KVM_S390_VCPU_FAULT: {
527e30b4 3054 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 3055 break;
3056 }
d6712df9
CH		 3057 case KVM_ENABLE_CAP:
3058 {
3059 struct kvm_enable_cap cap;
3060 r = -EFAULT;
3061 if (copy_from_user(&cap, argp, sizeof(cap)))
3062 break;
3063 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3064 break;
3065 }
41408c28
TH		 3066 case KVM_S390_MEM_OP: {
3067 struct kvm_s390_mem_op mem_op;
3068
3069 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3070 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3071 else
3072 r = -EFAULT;
3073 break;
3074 }
816c7667
JF		 3075 case KVM_S390_SET_IRQ_STATE: {
3076 struct kvm_s390_irq_state irq_state;
3077
3078 r = -EFAULT;
3079 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3080 break;
3081 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3082 irq_state.len == 0 ||
3083 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3084 r = -EINVAL;
3085 break;
3086 }
3087 r = kvm_s390_set_irq_state(vcpu,
3088 (void __user *) irq_state.buf,
3089 irq_state.len);
3090 break;
3091 }
3092 case KVM_S390_GET_IRQ_STATE: {
3093 struct kvm_s390_irq_state irq_state;
3094
3095 r = -EFAULT;
3096 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3097 break;
3098 if (irq_state.len == 0) {
3099 r = -EINVAL;
3100 break;
3101 }
3102 r = kvm_s390_get_irq_state(vcpu,
3103 (__u8 __user *) irq_state.buf,
3104 irq_state.len);
3105 break;
3106 }
b0c632db 3107 default:
3e6afcf1 3108 r = -ENOTTY;
b0c632db 3109 }
bc923cc9 3110 return r;
b0c632db
HC		 3111}
3112
5b1c1493
CO		 3113int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3114{
3115#ifdef CONFIG_KVM_S390_UCONTROL
3116 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3117 && (kvm_is_ucontrol(vcpu->kvm))) {
3118 vmf->page = virt_to_page(vcpu->arch.sie_block);
3119 get_page(vmf->page);
3120 return 0;
3121 }
3122#endif
3123 return VM_FAULT_SIGBUS;
3124}
3125
5587027c
AK		 3126int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3127 unsigned long npages)
db3fe4eb
TY		 3128{
3129 return 0;
3130}
3131
b0c632db 3132/* Section: memory related */
f7784b8e
MT		 3133int kvm_arch_prepare_memory_region(struct kvm *kvm,
3134 struct kvm_memory_slot *memslot,
09170a49 3135 const struct kvm_userspace_memory_region *mem,
7b6195a9 3136 enum kvm_mr_change change)
b0c632db 3137{
dd2887e7
NW		 3138 /* A few sanity checks. We can have memory slots which have to be
3139 located/ended at a segment boundary (1MB). The memory in userland is
3140 ok to be fragmented into various different vmas. It is okay to mmap()
3141 and munmap() stuff in this slot after doing this call at any time */
b0c632db 3142
598841ca 3143 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 3144 return -EINVAL;
3145
598841ca 3146 if (mem->memory_size & 0xffffful)
b0c632db
HC		 3147 return -EINVAL;
3148
a3a92c31
DD		 3149 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3150 return -EINVAL;
3151
f7784b8e
MT		 3152 return 0;
3153}
3154
3155void kvm_arch_commit_memory_region(struct kvm *kvm,
09170a49 3156 const struct kvm_userspace_memory_region *mem,
8482644a 3157 const struct kvm_memory_slot *old,
f36f3f28 3158 const struct kvm_memory_slot *new,
8482644a 3159 enum kvm_mr_change change)
f7784b8e 3160{
f7850c92 3161 int rc;
f7784b8e 3162
2cef4deb
CB		 3163 /* If the basics of the memslot do not change, we do not want
3164 * to update the gmap. Every update causes several unnecessary
3165 * segment translation exceptions. This is usually handled just
3166 * fine by the normal fault handler + gmap, but it will also
3167 * cause faults on the prefix page of running guest CPUs.
3168 */
3169 if (old->userspace_addr == mem->userspace_addr &&
3170 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3171 old->npages * PAGE_SIZE == mem->memory_size)
3172 return;
598841ca
CO		 3173
3174 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3175 mem->guest_phys_addr, mem->memory_size);
3176 if (rc)
ea2cdd27 3177 pr_warn("failed to commit memory region\n");
598841ca 3178 return;
b0c632db
HC		 3179}
3180
60a37709
AY		 3181static inline unsigned long nonhyp_mask(int i)
3182{
3183 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3184
3185 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3186}
3187
3491caf2
CB		 3188void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3189{
3190 vcpu->valid_wakeup = false;
3191}
3192
b0c632db
HC		 3193static int __init kvm_s390_init(void)
3194{
60a37709
AY		 3195 int i;
3196
07197fd0
DH		 3197 if (!sclp.has_sief2) {
3198 pr_info("SIE not available\n");
3199 return -ENODEV;
3200 }
3201
60a37709
AY		 3202 for (i = 0; i < 16; i++)
3203 kvm_s390_fac_list_mask[i] |=
3204 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3205
9d8d5786 3206 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 3207}
3208
3209static void __exit kvm_s390_exit(void)
3210{
3211 kvm_exit();
3212}
3213
3214module_init(kvm_s390_init);
3215module_exit(kvm_s390_exit);
566af940
CH		 3216
3217/*
3218 * Enable autoloading of the kvm module.
3219 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3220 * since x86 takes a different approach.
3221 */
3222#include <linux/miscdevice.h>
3223MODULE_ALIAS_MISCDEV(KVM_MINOR);
3224MODULE_ALIAS("devname:kvm");
Ubuntu Jammy Linux kernel mirror