]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
s390/mm: use RCU for gmap notifier list and the per-mm gmap list
[mirror_ubuntu-hirsute-kernel.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
24#include <linux/module.h>
a374e892 25#include <linux/random.h>
b0c632db 26#include <linux/slab.h>
ba5c1e9b 27#include <linux/timer.h>
41408c28 28#include <linux/vmalloc.h>
15c9705f 29#include <linux/bitmap.h>
cbb870c8 30#include <asm/asm-offsets.h>
b0c632db 31#include <asm/lowcore.h>
fdf03650 32#include <asm/etr.h>
b0c632db 33#include <asm/pgtable.h>
1e133ab2 34#include <asm/gmap.h>
f5daba1d 35#include <asm/nmi.h>
a0616cde 36#include <asm/switch_to.h>
6d3da241 37#include <asm/isc.h>
1526bf9c 38#include <asm/sclp.h>
0a763c78
DH		 39#include <asm/cpacf.h>
40#include <asm/etr.h>
8f2abe6a 41#include "kvm-s390.h"
b0c632db
HC		 42#include "gaccess.h"
43
ea2cdd27
DH		 44#define KMSG_COMPONENT "kvm-s390"
45#undef pr_fmt
46#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
47
5786fffa
CH		 48#define CREATE_TRACE_POINTS
49#include "trace.h"
ade38c31 50#include "trace-s390.h"
5786fffa 51
41408c28 52#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
816c7667
JF		 53#define LOCAL_IRQS 32
54#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
55 (KVM_MAX_VCPUS + LOCAL_IRQS))
41408c28 56
b0c632db
HC		 57#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
58
59struct kvm_stats_debugfs_item debugfs_entries[] = {
60 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 61 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 62 { "exit_validity", VCPU_STAT(exit_validity) },
63 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
64 { "exit_external_request", VCPU_STAT(exit_external_request) },
65 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b
CO		 66 { "exit_instruction", VCPU_STAT(exit_instruction) },
67 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
68 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
a011eeb2 69 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
f7819512 70 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
62bea5bf 71 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
3491caf2 72 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
ce2e4f0b 73 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 74 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 75 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 76 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
77 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 78 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 79 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 80 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
81 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
82 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
83 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
84 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
85 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
86 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 87 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 88 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
89 { "instruction_spx", VCPU_STAT(instruction_spx) },
90 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
91 { "instruction_stap", VCPU_STAT(instruction_stap) },
92 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 93 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 94 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
95 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 96 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 97 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
98 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 99 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
95ca2cb5 100 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
5288fbf0 101 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 102 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 103 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 104 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 105 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
106 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 107 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 108 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
109 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 110 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 111 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
112 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
113 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 114 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
115 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
116 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 117 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 118 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 119 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
175a5c9e
CB		 120 { "diagnose_258", VCPU_STAT(diagnose_258) },
121 { "diagnose_308", VCPU_STAT(diagnose_308) },
122 { "diagnose_500", VCPU_STAT(diagnose_500) },
b0c632db
HC		 123 { NULL }
124};
125
9d8d5786 126/* upper facilities limit for kvm */
60a37709
AY		 127unsigned long kvm_s390_fac_list_mask[16] = {
128 0xffe6000000000000UL,
129 0x005e000000000000UL,
9d8d5786 130};
b0c632db 131
9d8d5786 132unsigned long kvm_s390_fac_list_mask_size(void)
78c4b59f 133{
9d8d5786
MM		 134 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
135 return ARRAY_SIZE(kvm_s390_fac_list_mask);
78c4b59f
MM		 136}
137
15c9705f
DH		 138/* available cpu features supported by kvm */
139static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
0a763c78
DH		 140/* available subfunctions indicated via query / "test bit" */
141static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
15c9705f 142
9d8d5786 143static struct gmap_notifier gmap_notifier;
78f26131 144debug_info_t *kvm_s390_dbf;
9d8d5786 145
b0c632db 146/* Section: not file related */
13a34e06 147int kvm_arch_hardware_enable(void)
b0c632db
HC		 148{
149 /* every s390 is virtualization enabled ;-) */
10474ae8 150 return 0;
b0c632db
HC		 151}
152
414d3b07
MS		 153static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
154 unsigned long end);
2c70fe44 155
fdf03650
FZ		 156/*
157 * This callback is executed during stop_machine(). All CPUs are therefore
158 * temporarily stopped. In order not to change guest behavior, we have to
159 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
160 * so a CPU won't be stopped while calculating with the epoch.
161 */
162static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
163 void *v)
164{
165 struct kvm *kvm;
166 struct kvm_vcpu *vcpu;
167 int i;
168 unsigned long long *delta = v;
169
170 list_for_each_entry(kvm, &vm_list, vm_list) {
171 kvm->arch.epoch -= *delta;
172 kvm_for_each_vcpu(i, vcpu, kvm) {
173 vcpu->arch.sie_block->epoch -= *delta;
db0758b2
DH		 174 if (vcpu->arch.cputm_enabled)
175 vcpu->arch.cputm_start += *delta;
fdf03650
FZ		 176 }
177 }
178 return NOTIFY_OK;
179}
180
181static struct notifier_block kvm_clock_notifier = {
182 .notifier_call = kvm_clock_sync,
183};
184
b0c632db
HC		 185int kvm_arch_hardware_setup(void)
186{
2c70fe44
CB		 187 gmap_notifier.notifier_call = kvm_gmap_notifier;
188 gmap_register_ipte_notifier(&gmap_notifier);
fdf03650
FZ		 189 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
190 &kvm_clock_notifier);
b0c632db
HC		 191 return 0;
192}
193
194void kvm_arch_hardware_unsetup(void)
195{
2c70fe44 196 gmap_unregister_ipte_notifier(&gmap_notifier);
fdf03650
FZ		 197 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
198 &kvm_clock_notifier);
b0c632db
HC		 199}
200
22be5a13
DH		 201static void allow_cpu_feat(unsigned long nr)
202{
203 set_bit_inv(nr, kvm_s390_available_cpu_feat);
204}
205
0a763c78
DH		 206static inline int plo_test_bit(unsigned char nr)
207{
208 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
209 int cc = 3; /* subfunction not available */
210
211 asm volatile(
212 /* Parameter registers are ignored for "test bit" */
213 " plo 0,0,0,0(0)\n"
214 " ipm %0\n"
215 " srl %0,28\n"
216 : "=d" (cc)
217 : "d" (r0)
218 : "cc");
219 return cc == 0;
220}
221
22be5a13
DH		 222static void kvm_s390_cpu_feat_init(void)
223{
0a763c78
DH		 224 int i;
225
226 for (i = 0; i < 256; ++i) {
227 if (plo_test_bit(i))
228 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
229 }
230
231 if (test_facility(28)) /* TOD-clock steering */
232 etr_ptff(kvm_s390_available_subfunc.ptff, ETR_PTFF_QAF);
233
234 if (test_facility(17)) { /* MSA */
235 __cpacf_query(CPACF_KMAC, kvm_s390_available_subfunc.kmac);
236 __cpacf_query(CPACF_KMC, kvm_s390_available_subfunc.kmc);
237 __cpacf_query(CPACF_KM, kvm_s390_available_subfunc.km);
238 __cpacf_query(CPACF_KIMD, kvm_s390_available_subfunc.kimd);
239 __cpacf_query(CPACF_KLMD, kvm_s390_available_subfunc.klmd);
240 }
241 if (test_facility(76)) /* MSA3 */
242 __cpacf_query(CPACF_PCKMO, kvm_s390_available_subfunc.pckmo);
243 if (test_facility(77)) { /* MSA4 */
244 __cpacf_query(CPACF_KMCTR, kvm_s390_available_subfunc.kmctr);
245 __cpacf_query(CPACF_KMF, kvm_s390_available_subfunc.kmf);
246 __cpacf_query(CPACF_KMO, kvm_s390_available_subfunc.kmo);
247 __cpacf_query(CPACF_PCC, kvm_s390_available_subfunc.pcc);
248 }
249 if (test_facility(57)) /* MSA5 */
250 __cpacf_query(CPACF_PPNO, kvm_s390_available_subfunc.ppno);
251
22be5a13
DH		 252 if (MACHINE_HAS_ESOP)
253 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
254}
255
b0c632db
HC		 256int kvm_arch_init(void *opaque)
257{
78f26131
CB		 258 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
259 if (!kvm_s390_dbf)
260 return -ENOMEM;
261
262 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
263 debug_unregister(kvm_s390_dbf);
264 return -ENOMEM;
265 }
266
22be5a13
DH		 267 kvm_s390_cpu_feat_init();
268
84877d93
CH		 269 /* Register floating interrupt controller interface. */
270 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 271}
272
78f26131
CB		 273void kvm_arch_exit(void)
274{
275 debug_unregister(kvm_s390_dbf);
276}
277
b0c632db
HC		 278/* Section: device related */
279long kvm_arch_dev_ioctl(struct file *filp,
280 unsigned int ioctl, unsigned long arg)
281{
282 if (ioctl == KVM_S390_ENABLE_SIE)
283 return s390_enable_sie();
284 return -EINVAL;
285}
286
784aa3d7 287int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 288{
d7b0b5eb
CO		 289 int r;
290
2bd0ac4e 291 switch (ext) {
d7b0b5eb 292 case KVM_CAP_S390_PSW:
b6cf8788 293 case KVM_CAP_S390_GMAP:
52e16b18 294 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 295#ifdef CONFIG_KVM_S390_UCONTROL
296 case KVM_CAP_S390_UCONTROL:
297#endif
3c038e6b 298 case KVM_CAP_ASYNC_PF:
60b413c9 299 case KVM_CAP_SYNC_REGS:
14eebd91 300 case KVM_CAP_ONE_REG:
d6712df9 301 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 302 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 303 case KVM_CAP_IOEVENTFD:
c05c4186 304 case KVM_CAP_DEVICE_CTRL:
d938dc55 305 case KVM_CAP_ENABLE_CAP_VM:
78599d90 306 case KVM_CAP_S390_IRQCHIP:
f2061656 307 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 308 case KVM_CAP_MP_STATE:
47b43c52 309 case KVM_CAP_S390_INJECT_IRQ:
2444b352 310 case KVM_CAP_S390_USER_SIGP:
e44fc8c9 311 case KVM_CAP_S390_USER_STSI:
30ee2a98 312 case KVM_CAP_S390_SKEYS:
816c7667 313 case KVM_CAP_S390_IRQ_STATE:
d7b0b5eb
CO		 314 r = 1;
315 break;
41408c28
TH		 316 case KVM_CAP_S390_MEM_OP:
317 r = MEM_OP_MAX_SIZE;
318 break;
e726b1bd
CB		 319 case KVM_CAP_NR_VCPUS:
320 case KVM_CAP_MAX_VCPUS:
76a6dd72
DH		 321 r = KVM_S390_BSCA_CPU_SLOTS;
322 if (sclp.has_esca && sclp.has_64bscao)
323 r = KVM_S390_ESCA_CPU_SLOTS;
e726b1bd 324 break;
e1e2e605
NW		 325 case KVM_CAP_NR_MEMSLOTS:
326 r = KVM_USER_MEM_SLOTS;
327 break;
1526bf9c 328 case KVM_CAP_S390_COW:
abf09bed 329 r = MACHINE_HAS_ESOP;
1526bf9c 330 break;
68c55750
EF		 331 case KVM_CAP_S390_VECTOR_REGISTERS:
332 r = MACHINE_HAS_VX;
333 break;
c6e5f166
FZ		 334 case KVM_CAP_S390_RI:
335 r = test_facility(64);
336 break;
2bd0ac4e 337 default:
d7b0b5eb 338 r = 0;
2bd0ac4e 339 }
d7b0b5eb 340 return r;
b0c632db
HC		 341}
342
15f36ebd
JH		 343static void kvm_s390_sync_dirty_log(struct kvm *kvm,
344 struct kvm_memory_slot *memslot)
345{
346 gfn_t cur_gfn, last_gfn;
347 unsigned long address;
348 struct gmap *gmap = kvm->arch.gmap;
349
15f36ebd
JH		 350 /* Loop over all guest pages */
351 last_gfn = memslot->base_gfn + memslot->npages;
352 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
353 address = gfn_to_hva_memslot(memslot, cur_gfn);
354
1e133ab2 355 if (test_and_clear_guest_dirty(gmap->mm, address))
15f36ebd 356 mark_page_dirty(kvm, cur_gfn);
1763f8d0
CB		 357 if (fatal_signal_pending(current))
358 return;
70c88a00 359 cond_resched();
15f36ebd 360 }
15f36ebd
JH		 361}
362
b0c632db 363/* Section: vm related */
a6e2f683
ED		 364static void sca_del_vcpu(struct kvm_vcpu *vcpu);
365
b0c632db
HC		 366/*
367 * Get (and clear) the dirty memory log for a memory slot.
368 */
369int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
370 struct kvm_dirty_log *log)
371{
15f36ebd
JH		 372 int r;
373 unsigned long n;
9f6b8029 374 struct kvm_memslots *slots;
15f36ebd
JH		 375 struct kvm_memory_slot *memslot;
376 int is_dirty = 0;
377
378 mutex_lock(&kvm->slots_lock);
379
380 r = -EINVAL;
381 if (log->slot >= KVM_USER_MEM_SLOTS)
382 goto out;
383
9f6b8029
PB		 384 slots = kvm_memslots(kvm);
385 memslot = id_to_memslot(slots, log->slot);
15f36ebd
JH		 386 r = -ENOENT;
387 if (!memslot->dirty_bitmap)
388 goto out;
389
390 kvm_s390_sync_dirty_log(kvm, memslot);
391 r = kvm_get_dirty_log(kvm, log, &is_dirty);
392 if (r)
393 goto out;
394
395 /* Clear the dirty log */
396 if (is_dirty) {
397 n = kvm_dirty_bitmap_bytes(memslot);
398 memset(memslot->dirty_bitmap, 0, n);
399 }
400 r = 0;
401out:
402 mutex_unlock(&kvm->slots_lock);
403 return r;
b0c632db
HC		 404}
405
d938dc55
CH		 406static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
407{
408 int r;
409
410 if (cap->flags)
411 return -EINVAL;
412
413 switch (cap->cap) {
84223598 414 case KVM_CAP_S390_IRQCHIP:
c92ea7b9 415 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
84223598
CH		 416 kvm->arch.use_irqchip = 1;
417 r = 0;
418 break;
2444b352 419 case KVM_CAP_S390_USER_SIGP:
c92ea7b9 420 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
2444b352
DH		 421 kvm->arch.user_sigp = 1;
422 r = 0;
423 break;
68c55750 424 case KVM_CAP_S390_VECTOR_REGISTERS:
5967c17b 425 mutex_lock(&kvm->lock);
a03825bb 426 if (kvm->created_vcpus) {
5967c17b
DH		 427 r = -EBUSY;
428 } else if (MACHINE_HAS_VX) {
c54f0d6a
DH		 429 set_kvm_facility(kvm->arch.model.fac_mask, 129);
430 set_kvm_facility(kvm->arch.model.fac_list, 129);
18280d8b
MM		 431 r = 0;
432 } else
433 r = -EINVAL;
5967c17b 434 mutex_unlock(&kvm->lock);
c92ea7b9
CB		 435 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
436 r ? "(not available)" : "(success)");
68c55750 437 break;
c6e5f166
FZ		 438 case KVM_CAP_S390_RI:
439 r = -EINVAL;
440 mutex_lock(&kvm->lock);
a03825bb 441 if (kvm->created_vcpus) {
c6e5f166
FZ		 442 r = -EBUSY;
443 } else if (test_facility(64)) {
c54f0d6a
DH		 444 set_kvm_facility(kvm->arch.model.fac_mask, 64);
445 set_kvm_facility(kvm->arch.model.fac_list, 64);
c6e5f166
FZ		 446 r = 0;
447 }
448 mutex_unlock(&kvm->lock);
449 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
450 r ? "(not available)" : "(success)");
451 break;
e44fc8c9 452 case KVM_CAP_S390_USER_STSI:
c92ea7b9 453 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
e44fc8c9
ET		 454 kvm->arch.user_stsi = 1;
455 r = 0;
456 break;
d938dc55
CH		 457 default:
458 r = -EINVAL;
459 break;
460 }
461 return r;
462}
463
8c0a7ce6
DD		 464static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
465{
466 int ret;
467
468 switch (attr->attr) {
469 case KVM_S390_VM_MEM_LIMIT_SIZE:
470 ret = 0;
c92ea7b9 471 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
a3a92c31
DD		 472 kvm->arch.mem_limit);
473 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
8c0a7ce6
DD		 474 ret = -EFAULT;
475 break;
476 default:
477 ret = -ENXIO;
478 break;
479 }
480 return ret;
481}
482
483static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 484{
485 int ret;
486 unsigned int idx;
487 switch (attr->attr) {
488 case KVM_S390_VM_MEM_ENABLE_CMMA:
f9cbd9b0 489 ret = -ENXIO;
c24cc9c8 490 if (!sclp.has_cmma)
e6db1d61
DD		 491 break;
492
4f718eab 493 ret = -EBUSY;
c92ea7b9 494 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
4f718eab 495 mutex_lock(&kvm->lock);
a03825bb 496 if (!kvm->created_vcpus) {
4f718eab
DD		 497 kvm->arch.use_cmma = 1;
498 ret = 0;
499 }
500 mutex_unlock(&kvm->lock);
501 break;
502 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 503 ret = -ENXIO;
504 if (!sclp.has_cmma)
505 break;
c3489155
DD		 506 ret = -EINVAL;
507 if (!kvm->arch.use_cmma)
508 break;
509
c92ea7b9 510 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
4f718eab
DD		 511 mutex_lock(&kvm->lock);
512 idx = srcu_read_lock(&kvm->srcu);
a13cff31 513 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 514 srcu_read_unlock(&kvm->srcu, idx);
515 mutex_unlock(&kvm->lock);
516 ret = 0;
517 break;
8c0a7ce6
DD		 518 case KVM_S390_VM_MEM_LIMIT_SIZE: {
519 unsigned long new_limit;
520
521 if (kvm_is_ucontrol(kvm))
522 return -EINVAL;
523
524 if (get_user(new_limit, (u64 __user *)attr->addr))
525 return -EFAULT;
526
a3a92c31
DD		 527 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
528 new_limit > kvm->arch.mem_limit)
8c0a7ce6
DD		 529 return -E2BIG;
530
a3a92c31
DD		 531 if (!new_limit)
532 return -EINVAL;
533
534 /* gmap_alloc takes last usable address */
535 if (new_limit != KVM_S390_NO_MEM_LIMIT)
536 new_limit -= 1;
537
8c0a7ce6
DD		 538 ret = -EBUSY;
539 mutex_lock(&kvm->lock);
a03825bb 540 if (!kvm->created_vcpus) {
8c0a7ce6
DD		 541 /* gmap_alloc will round the limit up */
542 struct gmap *new = gmap_alloc(current->mm, new_limit);
543
544 if (!new) {
545 ret = -ENOMEM;
546 } else {
547 gmap_free(kvm->arch.gmap);
548 new->private = kvm;
549 kvm->arch.gmap = new;
550 ret = 0;
551 }
552 }
553 mutex_unlock(&kvm->lock);
a3a92c31
DD		 554 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
555 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
556 (void *) kvm->arch.gmap->asce);
8c0a7ce6
DD		 557 break;
558 }
4f718eab
DD		 559 default:
560 ret = -ENXIO;
561 break;
562 }
563 return ret;
564}
565
a374e892
TK		 566static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
567
568static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
569{
570 struct kvm_vcpu *vcpu;
571 int i;
572
9d8d5786 573 if (!test_kvm_facility(kvm, 76))
a374e892
TK		 574 return -EINVAL;
575
576 mutex_lock(&kvm->lock);
577 switch (attr->attr) {
578 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
579 get_random_bytes(
580 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
581 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
582 kvm->arch.crypto.aes_kw = 1;
c92ea7b9 583 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
a374e892
TK		 584 break;
585 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
586 get_random_bytes(
587 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
588 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
589 kvm->arch.crypto.dea_kw = 1;
c92ea7b9 590 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
a374e892
TK		 591 break;
592 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
593 kvm->arch.crypto.aes_kw = 0;
594 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
595 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
c92ea7b9 596 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
a374e892
TK		 597 break;
598 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
599 kvm->arch.crypto.dea_kw = 0;
600 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
601 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
c92ea7b9 602 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
a374e892
TK		 603 break;
604 default:
605 mutex_unlock(&kvm->lock);
606 return -ENXIO;
607 }
608
609 kvm_for_each_vcpu(i, vcpu, kvm) {
610 kvm_s390_vcpu_crypto_setup(vcpu);
611 exit_sie(vcpu);
612 }
613 mutex_unlock(&kvm->lock);
614 return 0;
615}
616
72f25020
JH		 617static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
618{
619 u8 gtod_high;
620
621 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
622 sizeof(gtod_high)))
623 return -EFAULT;
624
625 if (gtod_high != 0)
626 return -EINVAL;
58c383c6 627 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
72f25020
JH		 628
629 return 0;
630}
631
632static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
633{
5a3d883a 634 u64 gtod;
72f25020
JH		 635
636 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
637 return -EFAULT;
638
25ed1675 639 kvm_s390_set_tod_clock(kvm, gtod);
58c383c6 640 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
72f25020
JH		 641 return 0;
642}
643
644static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
645{
646 int ret;
647
648 if (attr->flags)
649 return -EINVAL;
650
651 switch (attr->attr) {
652 case KVM_S390_VM_TOD_HIGH:
653 ret = kvm_s390_set_tod_high(kvm, attr);
654 break;
655 case KVM_S390_VM_TOD_LOW:
656 ret = kvm_s390_set_tod_low(kvm, attr);
657 break;
658 default:
659 ret = -ENXIO;
660 break;
661 }
662 return ret;
663}
664
665static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
666{
667 u8 gtod_high = 0;
668
669 if (copy_to_user((void __user *)attr->addr, &gtod_high,
670 sizeof(gtod_high)))
671 return -EFAULT;
58c383c6 672 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
72f25020
JH		 673
674 return 0;
675}
676
677static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
678{
5a3d883a 679 u64 gtod;
72f25020 680
60417fcc 681 gtod = kvm_s390_get_tod_clock_fast(kvm);
72f25020
JH		 682 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
683 return -EFAULT;
58c383c6 684 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
72f25020
JH		 685
686 return 0;
687}
688
689static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
690{
691 int ret;
692
693 if (attr->flags)
694 return -EINVAL;
695
696 switch (attr->attr) {
697 case KVM_S390_VM_TOD_HIGH:
698 ret = kvm_s390_get_tod_high(kvm, attr);
699 break;
700 case KVM_S390_VM_TOD_LOW:
701 ret = kvm_s390_get_tod_low(kvm, attr);
702 break;
703 default:
704 ret = -ENXIO;
705 break;
706 }
707 return ret;
708}
709
658b6eda
MM		 710static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
711{
712 struct kvm_s390_vm_cpu_processor *proc;
053dd230 713 u16 lowest_ibc, unblocked_ibc;
658b6eda
MM		 714 int ret = 0;
715
716 mutex_lock(&kvm->lock);
a03825bb 717 if (kvm->created_vcpus) {
658b6eda
MM		 718 ret = -EBUSY;
719 goto out;
720 }
721 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
722 if (!proc) {
723 ret = -ENOMEM;
724 goto out;
725 }
726 if (!copy_from_user(proc, (void __user *)attr->addr,
727 sizeof(*proc))) {
9bb0ec09 728 kvm->arch.model.cpuid = proc->cpuid;
053dd230
DH		 729 lowest_ibc = sclp.ibc >> 16 & 0xfff;
730 unblocked_ibc = sclp.ibc & 0xfff;
731 if (lowest_ibc) {
732 if (proc->ibc > unblocked_ibc)
733 kvm->arch.model.ibc = unblocked_ibc;
734 else if (proc->ibc < lowest_ibc)
735 kvm->arch.model.ibc = lowest_ibc;
736 else
737 kvm->arch.model.ibc = proc->ibc;
738 }
c54f0d6a 739 memcpy(kvm->arch.model.fac_list, proc->fac_list,
658b6eda
MM		 740 S390_ARCH_FAC_LIST_SIZE_BYTE);
741 } else
742 ret = -EFAULT;
743 kfree(proc);
744out:
745 mutex_unlock(&kvm->lock);
746 return ret;
747}
748
15c9705f
DH		 749static int kvm_s390_set_processor_feat(struct kvm *kvm,
750 struct kvm_device_attr *attr)
751{
752 struct kvm_s390_vm_cpu_feat data;
753 int ret = -EBUSY;
754
755 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
756 return -EFAULT;
757 if (!bitmap_subset((unsigned long *) data.feat,
758 kvm_s390_available_cpu_feat,
759 KVM_S390_VM_CPU_FEAT_NR_BITS))
760 return -EINVAL;
761
762 mutex_lock(&kvm->lock);
763 if (!atomic_read(&kvm->online_vcpus)) {
764 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
765 KVM_S390_VM_CPU_FEAT_NR_BITS);
766 ret = 0;
767 }
768 mutex_unlock(&kvm->lock);
769 return ret;
770}
771
0a763c78
DH		 772static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
773 struct kvm_device_attr *attr)
774{
775 /*
776 * Once supported by kernel + hw, we have to store the subfunctions
777 * in kvm->arch and remember that user space configured them.
778 */
779 return -ENXIO;
780}
781
658b6eda
MM		 782static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
783{
784 int ret = -ENXIO;
785
786 switch (attr->attr) {
787 case KVM_S390_VM_CPU_PROCESSOR:
788 ret = kvm_s390_set_processor(kvm, attr);
789 break;
15c9705f
DH		 790 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
791 ret = kvm_s390_set_processor_feat(kvm, attr);
792 break;
0a763c78
DH		 793 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
794 ret = kvm_s390_set_processor_subfunc(kvm, attr);
795 break;
658b6eda
MM		 796 }
797 return ret;
798}
799
800static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
801{
802 struct kvm_s390_vm_cpu_processor *proc;
803 int ret = 0;
804
805 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
806 if (!proc) {
807 ret = -ENOMEM;
808 goto out;
809 }
9bb0ec09 810 proc->cpuid = kvm->arch.model.cpuid;
658b6eda 811 proc->ibc = kvm->arch.model.ibc;
c54f0d6a
DH		 812 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
813 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 814 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
815 ret = -EFAULT;
816 kfree(proc);
817out:
818 return ret;
819}
820
821static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
822{
823 struct kvm_s390_vm_cpu_machine *mach;
824 int ret = 0;
825
826 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
827 if (!mach) {
828 ret = -ENOMEM;
829 goto out;
830 }
831 get_cpu_id((struct cpuid *) &mach->cpuid);
37c5f6c8 832 mach->ibc = sclp.ibc;
c54f0d6a 833 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
981467c9 834 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 835 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
94422ee8 836 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 837 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
838 ret = -EFAULT;
839 kfree(mach);
840out:
841 return ret;
842}
843
15c9705f
DH		 844static int kvm_s390_get_processor_feat(struct kvm *kvm,
845 struct kvm_device_attr *attr)
846{
847 struct kvm_s390_vm_cpu_feat data;
848
849 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
850 KVM_S390_VM_CPU_FEAT_NR_BITS);
851 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
852 return -EFAULT;
853 return 0;
854}
855
856static int kvm_s390_get_machine_feat(struct kvm *kvm,
857 struct kvm_device_attr *attr)
858{
859 struct kvm_s390_vm_cpu_feat data;
860
861 bitmap_copy((unsigned long *) data.feat,
862 kvm_s390_available_cpu_feat,
863 KVM_S390_VM_CPU_FEAT_NR_BITS);
864 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
865 return -EFAULT;
866 return 0;
867}
868
0a763c78
DH		 869static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
870 struct kvm_device_attr *attr)
871{
872 /*
873 * Once we can actually configure subfunctions (kernel + hw support),
874 * we have to check if they were already set by user space, if so copy
875 * them from kvm->arch.
876 */
877 return -ENXIO;
878}
879
880static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
881 struct kvm_device_attr *attr)
882{
883 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
884 sizeof(struct kvm_s390_vm_cpu_subfunc)))
885 return -EFAULT;
886 return 0;
887}
658b6eda
MM		 888static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
889{
890 int ret = -ENXIO;
891
892 switch (attr->attr) {
893 case KVM_S390_VM_CPU_PROCESSOR:
894 ret = kvm_s390_get_processor(kvm, attr);
895 break;
896 case KVM_S390_VM_CPU_MACHINE:
897 ret = kvm_s390_get_machine(kvm, attr);
898 break;
15c9705f
DH		 899 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
900 ret = kvm_s390_get_processor_feat(kvm, attr);
901 break;
902 case KVM_S390_VM_CPU_MACHINE_FEAT:
903 ret = kvm_s390_get_machine_feat(kvm, attr);
904 break;
0a763c78
DH		 905 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
906 ret = kvm_s390_get_processor_subfunc(kvm, attr);
907 break;
908 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
909 ret = kvm_s390_get_machine_subfunc(kvm, attr);
910 break;
658b6eda
MM		 911 }
912 return ret;
913}
914
f2061656
DD		 915static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
916{
917 int ret;
918
919 switch (attr->group) {
4f718eab 920 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 921 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 922 break;
72f25020
JH		 923 case KVM_S390_VM_TOD:
924 ret = kvm_s390_set_tod(kvm, attr);
925 break;
658b6eda
MM		 926 case KVM_S390_VM_CPU_MODEL:
927 ret = kvm_s390_set_cpu_model(kvm, attr);
928 break;
a374e892
TK		 929 case KVM_S390_VM_CRYPTO:
930 ret = kvm_s390_vm_set_crypto(kvm, attr);
931 break;
f2061656
DD		 932 default:
933 ret = -ENXIO;
934 break;
935 }
936
937 return ret;
938}
939
940static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
941{
8c0a7ce6
DD		 942 int ret;
943
944 switch (attr->group) {
945 case KVM_S390_VM_MEM_CTRL:
946 ret = kvm_s390_get_mem_control(kvm, attr);
947 break;
72f25020
JH		 948 case KVM_S390_VM_TOD:
949 ret = kvm_s390_get_tod(kvm, attr);
950 break;
658b6eda
MM		 951 case KVM_S390_VM_CPU_MODEL:
952 ret = kvm_s390_get_cpu_model(kvm, attr);
953 break;
8c0a7ce6
DD		 954 default:
955 ret = -ENXIO;
956 break;
957 }
958
959 return ret;
f2061656
DD		 960}
961
962static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
963{
964 int ret;
965
966 switch (attr->group) {
4f718eab
DD		 967 case KVM_S390_VM_MEM_CTRL:
968 switch (attr->attr) {
969 case KVM_S390_VM_MEM_ENABLE_CMMA:
970 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 971 ret = sclp.has_cmma ? 0 : -ENXIO;
972 break;
8c0a7ce6 973 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 974 ret = 0;
975 break;
976 default:
977 ret = -ENXIO;
978 break;
979 }
980 break;
72f25020
JH		 981 case KVM_S390_VM_TOD:
982 switch (attr->attr) {
983 case KVM_S390_VM_TOD_LOW:
984 case KVM_S390_VM_TOD_HIGH:
985 ret = 0;
986 break;
987 default:
988 ret = -ENXIO;
989 break;
990 }
991 break;
658b6eda
MM		 992 case KVM_S390_VM_CPU_MODEL:
993 switch (attr->attr) {
994 case KVM_S390_VM_CPU_PROCESSOR:
995 case KVM_S390_VM_CPU_MACHINE:
15c9705f
DH		 996 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
997 case KVM_S390_VM_CPU_MACHINE_FEAT:
0a763c78 998 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
658b6eda
MM		 999 ret = 0;
1000 break;
0a763c78
DH		 1001 /* configuring subfunctions is not supported yet */
1002 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
658b6eda
MM		 1003 default:
1004 ret = -ENXIO;
1005 break;
1006 }
1007 break;
a374e892
TK		 1008 case KVM_S390_VM_CRYPTO:
1009 switch (attr->attr) {
1010 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1011 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1012 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1013 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1014 ret = 0;
1015 break;
1016 default:
1017 ret = -ENXIO;
1018 break;
1019 }
1020 break;
f2061656
DD		 1021 default:
1022 ret = -ENXIO;
1023 break;
1024 }
1025
1026 return ret;
1027}
1028
30ee2a98
JH		 1029static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1030{
1031 uint8_t *keys;
1032 uint64_t hva;
30ee2a98
JH		 1033 int i, r = 0;
1034
1035 if (args->flags != 0)
1036 return -EINVAL;
1037
1038 /* Is this guest using storage keys? */
1039 if (!mm_use_skey(current->mm))
1040 return KVM_S390_GET_SKEYS_NONE;
1041
1042 /* Enforce sane limit on memory allocation */
1043 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1044 return -EINVAL;
1045
1046 keys = kmalloc_array(args->count, sizeof(uint8_t),
1047 GFP_KERNEL | __GFP_NOWARN);
1048 if (!keys)
1049 keys = vmalloc(sizeof(uint8_t) * args->count);
1050 if (!keys)
1051 return -ENOMEM;
1052
d3ed1cee 1053 down_read(&current->mm->mmap_sem);
30ee2a98
JH		 1054 for (i = 0; i < args->count; i++) {
1055 hva = gfn_to_hva(kvm, args->start_gfn + i);
1056 if (kvm_is_error_hva(hva)) {
1057 r = -EFAULT;
d3ed1cee 1058 break;
30ee2a98
JH		 1059 }
1060
154c8c19
DH		 1061 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1062 if (r)
d3ed1cee 1063 break;
30ee2a98 1064 }
d3ed1cee
MS		 1065 up_read(&current->mm->mmap_sem);
1066
1067 if (!r) {
1068 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1069 sizeof(uint8_t) * args->count);
1070 if (r)
1071 r = -EFAULT;
1072 }
30ee2a98 1073
30ee2a98
JH		 1074 kvfree(keys);
1075 return r;
1076}
1077
1078static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1079{
1080 uint8_t *keys;
1081 uint64_t hva;
1082 int i, r = 0;
1083
1084 if (args->flags != 0)
1085 return -EINVAL;
1086
1087 /* Enforce sane limit on memory allocation */
1088 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1089 return -EINVAL;
1090
1091 keys = kmalloc_array(args->count, sizeof(uint8_t),
1092 GFP_KERNEL | __GFP_NOWARN);
1093 if (!keys)
1094 keys = vmalloc(sizeof(uint8_t) * args->count);
1095 if (!keys)
1096 return -ENOMEM;
1097
1098 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1099 sizeof(uint8_t) * args->count);
1100 if (r) {
1101 r = -EFAULT;
1102 goto out;
1103 }
1104
1105 /* Enable storage key handling for the guest */
14d4a425
DD		 1106 r = s390_enable_skey();
1107 if (r)
1108 goto out;
30ee2a98 1109
d3ed1cee 1110 down_read(&current->mm->mmap_sem);
30ee2a98
JH		 1111 for (i = 0; i < args->count; i++) {
1112 hva = gfn_to_hva(kvm, args->start_gfn + i);
1113 if (kvm_is_error_hva(hva)) {
1114 r = -EFAULT;
d3ed1cee 1115 break;
30ee2a98
JH		 1116 }
1117
1118 /* Lowest order bit is reserved */
1119 if (keys[i] & 0x01) {
1120 r = -EINVAL;
d3ed1cee 1121 break;
30ee2a98
JH		 1122 }
1123
fe69eabf 1124 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
30ee2a98 1125 if (r)
d3ed1cee 1126 break;
30ee2a98 1127 }
d3ed1cee 1128 up_read(&current->mm->mmap_sem);
30ee2a98
JH		 1129out:
1130 kvfree(keys);
1131 return r;
1132}
1133
b0c632db
HC		 1134long kvm_arch_vm_ioctl(struct file *filp,
1135 unsigned int ioctl, unsigned long arg)
1136{
1137 struct kvm *kvm = filp->private_data;
1138 void __user *argp = (void __user *)arg;
f2061656 1139 struct kvm_device_attr attr;
b0c632db
HC		 1140 int r;
1141
1142 switch (ioctl) {
ba5c1e9b
CO		 1143 case KVM_S390_INTERRUPT: {
1144 struct kvm_s390_interrupt s390int;
1145
1146 r = -EFAULT;
1147 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1148 break;
1149 r = kvm_s390_inject_vm(kvm, &s390int);
1150 break;
1151 }
d938dc55
CH		 1152 case KVM_ENABLE_CAP: {
1153 struct kvm_enable_cap cap;
1154 r = -EFAULT;
1155 if (copy_from_user(&cap, argp, sizeof(cap)))
1156 break;
1157 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1158 break;
1159 }
84223598
CH		 1160 case KVM_CREATE_IRQCHIP: {
1161 struct kvm_irq_routing_entry routing;
1162
1163 r = -EINVAL;
1164 if (kvm->arch.use_irqchip) {
1165 /* Set up dummy routing. */
1166 memset(&routing, 0, sizeof(routing));
152b2839 1167 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
84223598
CH		 1168 }
1169 break;
1170 }
f2061656
DD		 1171 case KVM_SET_DEVICE_ATTR: {
1172 r = -EFAULT;
1173 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1174 break;
1175 r = kvm_s390_vm_set_attr(kvm, &attr);
1176 break;
1177 }
1178 case KVM_GET_DEVICE_ATTR: {
1179 r = -EFAULT;
1180 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1181 break;
1182 r = kvm_s390_vm_get_attr(kvm, &attr);
1183 break;
1184 }
1185 case KVM_HAS_DEVICE_ATTR: {
1186 r = -EFAULT;
1187 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1188 break;
1189 r = kvm_s390_vm_has_attr(kvm, &attr);
1190 break;
1191 }
30ee2a98
JH		 1192 case KVM_S390_GET_SKEYS: {
1193 struct kvm_s390_skeys args;
1194
1195 r = -EFAULT;
1196 if (copy_from_user(&args, argp,
1197 sizeof(struct kvm_s390_skeys)))
1198 break;
1199 r = kvm_s390_get_skeys(kvm, &args);
1200 break;
1201 }
1202 case KVM_S390_SET_SKEYS: {
1203 struct kvm_s390_skeys args;
1204
1205 r = -EFAULT;
1206 if (copy_from_user(&args, argp,
1207 sizeof(struct kvm_s390_skeys)))
1208 break;
1209 r = kvm_s390_set_skeys(kvm, &args);
1210 break;
1211 }
b0c632db 1212 default:
367e1319 1213 r = -ENOTTY;
b0c632db
HC		 1214 }
1215
1216 return r;
1217}
1218
45c9b47c
TK		 1219static int kvm_s390_query_ap_config(u8 *config)
1220{
1221 u32 fcn_code = 0x04000000UL;
86044c8c 1222 u32 cc = 0;
45c9b47c 1223
86044c8c 1224 memset(config, 0, 128);
45c9b47c
TK		 1225 asm volatile(
1226 "lgr 0,%1\n"
1227 "lgr 2,%2\n"
1228 ".long 0xb2af0000\n" /* PQAP(QCI) */
86044c8c 1229 "0: ipm %0\n"
45c9b47c 1230 "srl %0,28\n"
86044c8c
CB		 1231 "1:\n"
1232 EX_TABLE(0b, 1b)
1233 : "+r" (cc)
45c9b47c
TK		 1234 : "r" (fcn_code), "r" (config)
1235 : "cc", "0", "2", "memory"
1236 );
1237
1238 return cc;
1239}
1240
1241static int kvm_s390_apxa_installed(void)
1242{
1243 u8 config[128];
1244 int cc;
1245
a6aacc3f 1246 if (test_facility(12)) {
45c9b47c
TK		 1247 cc = kvm_s390_query_ap_config(config);
1248
1249 if (cc)
1250 pr_err("PQAP(QCI) failed with cc=%d", cc);
1251 else
1252 return config[0] & 0x40;
1253 }
1254
1255 return 0;
1256}
1257
1258static void kvm_s390_set_crycb_format(struct kvm *kvm)
1259{
1260 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1261
1262 if (kvm_s390_apxa_installed())
1263 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1264 else
1265 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1266}
1267
9bb0ec09 1268static u64 kvm_s390_get_initial_cpuid(void)
9d8d5786 1269{
9bb0ec09
DH		 1270 struct cpuid cpuid;
1271
1272 get_cpu_id(&cpuid);
1273 cpuid.version = 0xff;
1274 return *((u64 *) &cpuid);
9d8d5786
MM		 1275}
1276
c54f0d6a 1277static void kvm_s390_crypto_init(struct kvm *kvm)
5102ee87 1278{
9d8d5786 1279 if (!test_kvm_facility(kvm, 76))
c54f0d6a 1280 return;
5102ee87 1281
c54f0d6a 1282 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
45c9b47c 1283 kvm_s390_set_crycb_format(kvm);
5102ee87 1284
ed6f76b4
TK		 1285 /* Enable AES/DEA protected key functions by default */
1286 kvm->arch.crypto.aes_kw = 1;
1287 kvm->arch.crypto.dea_kw = 1;
1288 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1289 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1290 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1291 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
5102ee87
TK		 1292}
1293
7d43bafc
ED		 1294static void sca_dispose(struct kvm *kvm)
1295{
1296 if (kvm->arch.use_esca)
5e044315 1297 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
7d43bafc
ED		 1298 else
1299 free_page((unsigned long)(kvm->arch.sca));
1300 kvm->arch.sca = NULL;
1301}
1302
e08b9637 1303int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 1304{
76a6dd72 1305 gfp_t alloc_flags = GFP_KERNEL;
9d8d5786 1306 int i, rc;
b0c632db 1307 char debug_name[16];
f6c137ff 1308 static unsigned long sca_offset;
b0c632db 1309
e08b9637
CO		 1310 rc = -EINVAL;
1311#ifdef CONFIG_KVM_S390_UCONTROL
1312 if (type & ~KVM_VM_S390_UCONTROL)
1313 goto out_err;
1314 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1315 goto out_err;
1316#else
1317 if (type)
1318 goto out_err;
1319#endif
1320
b0c632db
HC		 1321 rc = s390_enable_sie();
1322 if (rc)
d89f5eff 1323 goto out_err;
b0c632db 1324
b290411a
CO		 1325 rc = -ENOMEM;
1326
7d0a5e62
JF		 1327 ratelimit_state_init(&kvm->arch.sthyi_limit, 5 * HZ, 500);
1328
7d43bafc 1329 kvm->arch.use_esca = 0; /* start with basic SCA */
76a6dd72
DH		 1330 if (!sclp.has_64bscao)
1331 alloc_flags |= GFP_DMA;
5e044315 1332 rwlock_init(&kvm->arch.sca_lock);
76a6dd72 1333 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
b0c632db 1334 if (!kvm->arch.sca)
d89f5eff 1335 goto out_err;
f6c137ff 1336 spin_lock(&kvm_lock);
c5c2c393 1337 sca_offset += 16;
bc784cce 1338 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
c5c2c393 1339 sca_offset = 0;
bc784cce
ED		 1340 kvm->arch.sca = (struct bsca_block *)
1341 ((char *) kvm->arch.sca + sca_offset);
f6c137ff 1342 spin_unlock(&kvm_lock);
b0c632db
HC		 1343
1344 sprintf(debug_name, "kvm-%u", current->pid);
1345
1cb9cf72 1346 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
b0c632db 1347 if (!kvm->arch.dbf)
40f5b735 1348 goto out_err;
b0c632db 1349
c54f0d6a
DH		 1350 kvm->arch.sie_page2 =
1351 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1352 if (!kvm->arch.sie_page2)
40f5b735 1353 goto out_err;
9d8d5786 1354
fb5bf93f 1355 /* Populate the facility mask initially. */
c54f0d6a 1356 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
94422ee8 1357 S390_ARCH_FAC_LIST_SIZE_BYTE);
9d8d5786
MM		 1358 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1359 if (i < kvm_s390_fac_list_mask_size())
c54f0d6a 1360 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
9d8d5786 1361 else
c54f0d6a 1362 kvm->arch.model.fac_mask[i] = 0UL;
9d8d5786
MM		 1363 }
1364
981467c9 1365 /* Populate the facility list initially. */
c54f0d6a
DH		 1366 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1367 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
981467c9
MM		 1368 S390_ARCH_FAC_LIST_SIZE_BYTE);
1369
95ca2cb5
JF		 1370 set_kvm_facility(kvm->arch.model.fac_mask, 74);
1371 set_kvm_facility(kvm->arch.model.fac_list, 74);
1372
9bb0ec09 1373 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
37c5f6c8 1374 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
9d8d5786 1375
c54f0d6a 1376 kvm_s390_crypto_init(kvm);
5102ee87 1377
ba5c1e9b 1378 spin_lock_init(&kvm->arch.float_int.lock);
6d3da241
JF		 1379 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1380 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
8a242234 1381 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 1382 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 1383
b0c632db 1384 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
78f26131 1385 VM_EVENT(kvm, 3, "vm created with type %lu", type);
b0c632db 1386
e08b9637
CO		 1387 if (type & KVM_VM_S390_UCONTROL) {
1388 kvm->arch.gmap = NULL;
a3a92c31 1389 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
e08b9637 1390 } else {
32e6b236
GH		 1391 if (sclp.hamax == U64_MAX)
1392 kvm->arch.mem_limit = TASK_MAX_SIZE;
1393 else
1394 kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
1395 sclp.hamax + 1);
a3a92c31 1396 kvm->arch.gmap = gmap_alloc(current->mm, kvm->arch.mem_limit - 1);
e08b9637 1397 if (!kvm->arch.gmap)
40f5b735 1398 goto out_err;
2c70fe44 1399 kvm->arch.gmap->private = kvm;
24eb3a82 1400 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 1401 }
fa6b7fe9
CH		 1402
1403 kvm->arch.css_support = 0;
84223598 1404 kvm->arch.use_irqchip = 0;
72f25020 1405 kvm->arch.epoch = 0;
fa6b7fe9 1406
8ad35755 1407 spin_lock_init(&kvm->arch.start_stop_lock);
8335713a 1408 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
8ad35755 1409
d89f5eff 1410 return 0;
40f5b735 1411out_err:
c54f0d6a 1412 free_page((unsigned long)kvm->arch.sie_page2);
598841ca 1413 debug_unregister(kvm->arch.dbf);
7d43bafc 1414 sca_dispose(kvm);
78f26131 1415 KVM_EVENT(3, "creation of vm failed: %d", rc);
d89f5eff 1416 return rc;
b0c632db
HC		 1417}
1418
d329c035
CB		 1419void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1420{
1421 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 1422 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 1423 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 1424 kvm_clear_async_pf_completion_queue(vcpu);
bc784cce 1425 if (!kvm_is_ucontrol(vcpu->kvm))
a6e2f683 1426 sca_del_vcpu(vcpu);
27e0393f
CO		 1427
1428 if (kvm_is_ucontrol(vcpu->kvm))
1429 gmap_free(vcpu->arch.gmap);
1430
e6db1d61 1431 if (vcpu->kvm->arch.use_cmma)
b31605c1 1432 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 1433 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 1434
6692cef3 1435 kvm_vcpu_uninit(vcpu);
b110feaf 1436 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 1437}
1438
1439static void kvm_free_vcpus(struct kvm *kvm)
1440{
1441 unsigned int i;
988a2cae 1442 struct kvm_vcpu *vcpu;
d329c035 1443
988a2cae
GN		 1444 kvm_for_each_vcpu(i, vcpu, kvm)
1445 kvm_arch_vcpu_destroy(vcpu);
1446
1447 mutex_lock(&kvm->lock);
1448 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1449 kvm->vcpus[i] = NULL;
1450
1451 atomic_set(&kvm->online_vcpus, 0);
1452 mutex_unlock(&kvm->lock);
d329c035
CB		 1453}
1454
b0c632db
HC		 1455void kvm_arch_destroy_vm(struct kvm *kvm)
1456{
d329c035 1457 kvm_free_vcpus(kvm);
7d43bafc 1458 sca_dispose(kvm);
d329c035 1459 debug_unregister(kvm->arch.dbf);
c54f0d6a 1460 free_page((unsigned long)kvm->arch.sie_page2);
27e0393f
CO		 1461 if (!kvm_is_ucontrol(kvm))
1462 gmap_free(kvm->arch.gmap);
841b91c5 1463 kvm_s390_destroy_adapters(kvm);
67335e63 1464 kvm_s390_clear_float_irqs(kvm);
8335713a 1465 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
b0c632db
HC		 1466}
1467
1468/* Section: vcpu related */
dafd032a
DD		 1469static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1470{
1471 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1472 if (!vcpu->arch.gmap)
1473 return -ENOMEM;
1474 vcpu->arch.gmap->private = vcpu->kvm;
1475
1476 return 0;
1477}
1478
a6e2f683
ED		 1479static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1480{
5e044315 1481 read_lock(&vcpu->kvm->arch.sca_lock);
7d43bafc
ED		 1482 if (vcpu->kvm->arch.use_esca) {
1483 struct esca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1484
7d43bafc 1485 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
10ce32d5 1486 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc
ED		 1487 } else {
1488 struct bsca_block *sca = vcpu->kvm->arch.sca;
1489
1490 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
10ce32d5 1491 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc 1492 }
5e044315 1493 read_unlock(&vcpu->kvm->arch.sca_lock);
a6e2f683
ED		 1494}
1495
eaa78f34 1496static void sca_add_vcpu(struct kvm_vcpu *vcpu)
a6e2f683 1497{
eaa78f34
DH		 1498 read_lock(&vcpu->kvm->arch.sca_lock);
1499 if (vcpu->kvm->arch.use_esca) {
1500 struct esca_block *sca = vcpu->kvm->arch.sca;
7d43bafc 1501
eaa78f34 1502 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1503 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1504 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
25508824 1505 vcpu->arch.sie_block->ecb2 |= 0x04U;
eaa78f34 1506 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
7d43bafc 1507 } else {
eaa78f34 1508 struct bsca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1509
eaa78f34 1510 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1511 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1512 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
eaa78f34 1513 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
7d43bafc 1514 }
eaa78f34 1515 read_unlock(&vcpu->kvm->arch.sca_lock);
5e044315
ED		 1516}
1517
1518/* Basic SCA to Extended SCA data copy routines */
1519static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1520{
1521 d->sda = s->sda;
1522 d->sigp_ctrl.c = s->sigp_ctrl.c;
1523 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1524}
1525
1526static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1527{
1528 int i;
1529
1530 d->ipte_control = s->ipte_control;
1531 d->mcn[0] = s->mcn;
1532 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1533 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1534}
1535
1536static int sca_switch_to_extended(struct kvm *kvm)
1537{
1538 struct bsca_block *old_sca = kvm->arch.sca;
1539 struct esca_block *new_sca;
1540 struct kvm_vcpu *vcpu;
1541 unsigned int vcpu_idx;
1542 u32 scaol, scaoh;
1543
1544 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1545 if (!new_sca)
1546 return -ENOMEM;
1547
1548 scaoh = (u32)((u64)(new_sca) >> 32);
1549 scaol = (u32)(u64)(new_sca) & ~0x3fU;
1550
1551 kvm_s390_vcpu_block_all(kvm);
1552 write_lock(&kvm->arch.sca_lock);
1553
1554 sca_copy_b_to_e(new_sca, old_sca);
1555
1556 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1557 vcpu->arch.sie_block->scaoh = scaoh;
1558 vcpu->arch.sie_block->scaol = scaol;
1559 vcpu->arch.sie_block->ecb2 |= 0x04U;
1560 }
1561 kvm->arch.sca = new_sca;
1562 kvm->arch.use_esca = 1;
1563
1564 write_unlock(&kvm->arch.sca_lock);
1565 kvm_s390_vcpu_unblock_all(kvm);
1566
1567 free_page((unsigned long)old_sca);
1568
8335713a
CB		 1569 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1570 old_sca, kvm->arch.sca);
5e044315 1571 return 0;
a6e2f683
ED		 1572}
1573
1574static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1575{
5e044315
ED		 1576 int rc;
1577
1578 if (id < KVM_S390_BSCA_CPU_SLOTS)
1579 return true;
76a6dd72 1580 if (!sclp.has_esca || !sclp.has_64bscao)
5e044315
ED		 1581 return false;
1582
1583 mutex_lock(&kvm->lock);
1584 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1585 mutex_unlock(&kvm->lock);
1586
1587 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
a6e2f683
ED		 1588}
1589
b0c632db
HC		 1590int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1591{
3c038e6b
DD		 1592 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1593 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 1594 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1595 KVM_SYNC_GPRS |
9eed0735 1596 KVM_SYNC_ACRS |
b028ee3e
DH		 1597 KVM_SYNC_CRS |
1598 KVM_SYNC_ARCH0 |
1599 KVM_SYNC_PFAULT;
c6e5f166
FZ		 1600 if (test_kvm_facility(vcpu->kvm, 64))
1601 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
f6aa6dc4
DH		 1602 /* fprs can be synchronized via vrs, even if the guest has no vx. With
1603 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1604 */
1605 if (MACHINE_HAS_VX)
68c55750 1606 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
6fd8e67d
DH		 1607 else
1608 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
dafd032a
DD		 1609
1610 if (kvm_is_ucontrol(vcpu->kvm))
1611 return __kvm_ucontrol_vcpu_init(vcpu);
1612
b0c632db
HC		 1613 return 0;
1614}
1615
db0758b2
DH		 1616/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1617static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1618{
1619 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
9c23a131 1620 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2 1621 vcpu->arch.cputm_start = get_tod_clock_fast();
9c23a131 1622 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1623}
1624
1625/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1626static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1627{
1628 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
9c23a131 1629 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1630 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1631 vcpu->arch.cputm_start = 0;
9c23a131 1632 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1633}
1634
1635/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1636static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1637{
1638 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
1639 vcpu->arch.cputm_enabled = true;
1640 __start_cpu_timer_accounting(vcpu);
1641}
1642
1643/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1644static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1645{
1646 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
1647 __stop_cpu_timer_accounting(vcpu);
1648 vcpu->arch.cputm_enabled = false;
1649}
1650
1651static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1652{
1653 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1654 __enable_cpu_timer_accounting(vcpu);
1655 preempt_enable();
1656}
1657
1658static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1659{
1660 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1661 __disable_cpu_timer_accounting(vcpu);
1662 preempt_enable();
1663}
1664
4287f247
DH		 1665/* set the cpu timer - may only be called from the VCPU thread itself */
1666void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
1667{
db0758b2 1668 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
9c23a131 1669 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1670 if (vcpu->arch.cputm_enabled)
1671 vcpu->arch.cputm_start = get_tod_clock_fast();
4287f247 1672 vcpu->arch.sie_block->cputm = cputm;
9c23a131 1673 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2 1674 preempt_enable();
4287f247
DH		 1675}
1676
db0758b2 1677/* update and get the cpu timer - can also be called from other VCPU threads */
4287f247
DH		 1678__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
1679{
9c23a131 1680 unsigned int seq;
db0758b2 1681 __u64 value;
db0758b2
DH		 1682
1683 if (unlikely(!vcpu->arch.cputm_enabled))
1684 return vcpu->arch.sie_block->cputm;
1685
9c23a131
DH		 1686 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1687 do {
1688 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
1689 /*
1690 * If the writer would ever execute a read in the critical
1691 * section, e.g. in irq context, we have a deadlock.
1692 */
1693 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
1694 value = vcpu->arch.sie_block->cputm;
1695 /* if cputm_start is 0, accounting is being started/stopped */
1696 if (likely(vcpu->arch.cputm_start))
1697 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1698 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
1699 preempt_enable();
db0758b2 1700 return value;
4287f247
DH		 1701}
1702
b0c632db
HC		 1703void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1704{
9977e886 1705 /* Save host register state */
d0164ee2 1706 save_fpu_regs();
9abc2a08
DH		 1707 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
1708 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
9977e886 1709
6fd8e67d
DH		 1710 if (MACHINE_HAS_VX)
1711 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
1712 else
1713 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
9abc2a08 1714 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
9977e886 1715 if (test_fp_ctl(current->thread.fpu.fpc))
96b2d7a8 1716 /* User space provided an invalid FPC, let's clear it */
9977e886
HB		 1717 current->thread.fpu.fpc = 0;
1718
1719 save_access_regs(vcpu->arch.host_acrs);
59674c1a 1720 restore_access_regs(vcpu->run->s.regs.acrs);
480e5926 1721 gmap_enable(vcpu->arch.gmap);
805de8f4 1722 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
5ebda316 1723 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1724 __start_cpu_timer_accounting(vcpu);
01a745ac 1725 vcpu->cpu = cpu;
b0c632db
HC		 1726}
1727
1728void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1729{
01a745ac 1730 vcpu->cpu = -1;
5ebda316 1731 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1732 __stop_cpu_timer_accounting(vcpu);
805de8f4 1733 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
480e5926 1734 gmap_disable(vcpu->arch.gmap);
9977e886 1735
9abc2a08 1736 /* Save guest register state */
d0164ee2 1737 save_fpu_regs();
9abc2a08 1738 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
9977e886 1739
9abc2a08
DH		 1740 /* Restore host register state */
1741 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
1742 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
9977e886
HB		 1743
1744 save_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 1745 restore_access_regs(vcpu->arch.host_acrs);
1746}
1747
1748static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1749{
1750 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1751 vcpu->arch.sie_block->gpsw.mask = 0UL;
1752 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 1753 kvm_s390_set_prefix(vcpu, 0);
4287f247 1754 kvm_s390_set_cpu_timer(vcpu, 0);
b0c632db
HC		 1755 vcpu->arch.sie_block->ckc = 0UL;
1756 vcpu->arch.sie_block->todpr = 0;
1757 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1758 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1759 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
9abc2a08
DH		 1760 /* make sure the new fpc will be lazily loaded */
1761 save_fpu_regs();
1762 current->thread.fpu.fpc = 0;
b0c632db 1763 vcpu->arch.sie_block->gbea = 1;
672550fb 1764 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 1765 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1766 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 1767 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1768 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 1769 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 1770}
1771
31928aa5 1772void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 1773{
72f25020 1774 mutex_lock(&vcpu->kvm->lock);
fdf03650 1775 preempt_disable();
72f25020 1776 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
fdf03650 1777 preempt_enable();
72f25020 1778 mutex_unlock(&vcpu->kvm->lock);
25508824 1779 if (!kvm_is_ucontrol(vcpu->kvm)) {
dafd032a 1780 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
eaa78f34 1781 sca_add_vcpu(vcpu);
25508824
DH		 1782 }
1783
42897d86
MT		 1784}
1785
5102ee87
TK		 1786static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1787{
9d8d5786 1788 if (!test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 1789 return;
1790
a374e892
TK		 1791 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1792
1793 if (vcpu->kvm->arch.crypto.aes_kw)
1794 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1795 if (vcpu->kvm->arch.crypto.dea_kw)
1796 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1797
5102ee87
TK		 1798 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1799}
1800
b31605c1
DD		 1801void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1802{
1803 free_page(vcpu->arch.sie_block->cbrlo);
1804 vcpu->arch.sie_block->cbrlo = 0;
1805}
1806
1807int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1808{
1809 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1810 if (!vcpu->arch.sie_block->cbrlo)
1811 return -ENOMEM;
1812
1813 vcpu->arch.sie_block->ecb2 |= 0x80;
1814 vcpu->arch.sie_block->ecb2 &= ~0x08;
1815 return 0;
1816}
1817
91520f1a
MM		 1818static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1819{
1820 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1821
91520f1a 1822 vcpu->arch.sie_block->ibc = model->ibc;
80bc79dc 1823 if (test_kvm_facility(vcpu->kvm, 7))
c54f0d6a 1824 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
91520f1a
MM		 1825}
1826
b0c632db
HC		 1827int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1828{
b31605c1 1829 int rc = 0;
b31288fa 1830
9e6dabef
CH		 1831 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1832 CPUSTAT_SM |
a4a4f191
GH		 1833 CPUSTAT_STOPPED);
1834
53df84f8 1835 if (test_kvm_facility(vcpu->kvm, 78))
805de8f4 1836 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
53df84f8 1837 else if (test_kvm_facility(vcpu->kvm, 8))
805de8f4 1838 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
a4a4f191 1839
91520f1a
MM		 1840 kvm_s390_vcpu_setup_model(vcpu);
1841
bdab09f3
DH		 1842 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
1843 if (MACHINE_HAS_ESOP)
1844 vcpu->arch.sie_block->ecb |= 0x02;
bd50e8ec
DH		 1845 if (test_kvm_facility(vcpu->kvm, 9))
1846 vcpu->arch.sie_block->ecb |= 0x04;
f597d24e 1847 if (test_kvm_facility(vcpu->kvm, 73))
7feb6bb8
MM		 1848 vcpu->arch.sie_block->ecb |= 0x10;
1849
873b425e 1850 if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
d6af0b49 1851 vcpu->arch.sie_block->ecb2 |= 0x08;
48ee7d3a
DH		 1852 vcpu->arch.sie_block->eca = 0x1002000U;
1853 if (sclp.has_cei)
1854 vcpu->arch.sie_block->eca |= 0x80000000U;
11ad65b7
DH		 1855 if (sclp.has_ib)
1856 vcpu->arch.sie_block->eca |= 0x40000000U;
37c5f6c8 1857 if (sclp.has_siif)
217a4406 1858 vcpu->arch.sie_block->eca |= 1;
37c5f6c8 1859 if (sclp.has_sigpif)
ea5f4969 1860 vcpu->arch.sie_block->eca |= 0x10000000U;
c6e5f166
FZ		 1861 if (test_kvm_facility(vcpu->kvm, 64))
1862 vcpu->arch.sie_block->ecb3 |= 0x01;
18280d8b 1863 if (test_kvm_facility(vcpu->kvm, 129)) {
13211ea7
EF		 1864 vcpu->arch.sie_block->eca |= 0x00020000;
1865 vcpu->arch.sie_block->ecd |= 0x20000000;
1866 }
c6e5f166 1867 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
492d8642 1868 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
95ca2cb5
JF		 1869 if (test_kvm_facility(vcpu->kvm, 74))
1870 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
5a5e6536 1871
e6db1d61 1872 if (vcpu->kvm->arch.use_cmma) {
b31605c1
DD		 1873 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1874 if (rc)
1875 return rc;
b31288fa 1876 }
0ac96caf 1877 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 1878 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 1879
5102ee87
TK		 1880 kvm_s390_vcpu_crypto_setup(vcpu);
1881
b31605c1 1882 return rc;
b0c632db
HC		 1883}
1884
1885struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1886 unsigned int id)
1887{
4d47555a 1888 struct kvm_vcpu *vcpu;
7feb6bb8 1889 struct sie_page *sie_page;
4d47555a
CO		 1890 int rc = -EINVAL;
1891
4215825e 1892 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
4d47555a
CO		 1893 goto out;
1894
1895 rc = -ENOMEM;
b0c632db 1896
b110feaf 1897 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 1898 if (!vcpu)
4d47555a 1899 goto out;
b0c632db 1900
7feb6bb8
MM		 1901 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1902 if (!sie_page)
b0c632db
HC		 1903 goto out_free_cpu;
1904
7feb6bb8
MM		 1905 vcpu->arch.sie_block = &sie_page->sie_block;
1906 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1907
efed1104
DH		 1908 /* the real guest size will always be smaller than msl */
1909 vcpu->arch.sie_block->mso = 0;
1910 vcpu->arch.sie_block->msl = sclp.hamax;
1911
b0c632db 1912 vcpu->arch.sie_block->icpua = id;
ba5c1e9b 1913 spin_lock_init(&vcpu->arch.local_int.lock);
ba5c1e9b 1914 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 1915 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 1916 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
9c23a131 1917 seqcount_init(&vcpu->arch.cputm_seqcount);
ba5c1e9b 1918
b0c632db
HC		 1919 rc = kvm_vcpu_init(vcpu, kvm, id);
1920 if (rc)
9abc2a08 1921 goto out_free_sie_block;
8335713a 1922 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
b0c632db 1923 vcpu->arch.sie_block);
ade38c31 1924 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 1925
b0c632db 1926 return vcpu;
7b06bf2f
WY		 1927out_free_sie_block:
1928 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 1929out_free_cpu:
b110feaf 1930 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 1931out:
b0c632db
HC		 1932 return ERR_PTR(rc);
1933}
1934
b0c632db
HC		 1935int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1936{
9a022067 1937 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 1938}
1939
27406cd5 1940void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
49b99e1e 1941{
805de8f4 1942 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
61a6df54 1943 exit_sie(vcpu);
49b99e1e
CB		 1944}
1945
27406cd5 1946void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
49b99e1e 1947{
805de8f4 1948 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
49b99e1e
CB		 1949}
1950
8e236546
CB		 1951static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1952{
805de8f4 1953 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
61a6df54 1954 exit_sie(vcpu);
8e236546
CB		 1955}
1956
1957static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1958{
9bf9fde2 1959 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
8e236546
CB		 1960}
1961
49b99e1e
CB		 1962/*
1963 * Kick a guest cpu out of SIE and wait until SIE is not running.
1964 * If the CPU is not running (e.g. waiting as idle) the function will
1965 * return immediately. */
1966void exit_sie(struct kvm_vcpu *vcpu)
1967{
805de8f4 1968 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
49b99e1e
CB		 1969 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1970 cpu_relax();
1971}
1972
8e236546
CB		 1973/* Kick a guest cpu out of SIE to process a request synchronously */
1974void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
49b99e1e 1975{
8e236546
CB		 1976 kvm_make_request(req, vcpu);
1977 kvm_s390_vcpu_request(vcpu);
49b99e1e
CB		 1978}
1979
414d3b07
MS		 1980static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
1981 unsigned long end)
2c70fe44 1982{
2c70fe44
CB		 1983 struct kvm *kvm = gmap->private;
1984 struct kvm_vcpu *vcpu;
414d3b07
MS		 1985 unsigned long prefix;
1986 int i;
2c70fe44 1987
414d3b07
MS		 1988 if (start >= 1UL << 31)
1989 /* We are only interested in prefix pages */
1990 return;
2c70fe44
CB		 1991 kvm_for_each_vcpu(i, vcpu, kvm) {
1992 /* match against both prefix pages */
414d3b07
MS		 1993 prefix = kvm_s390_get_prefix(vcpu);
1994 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
1995 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
1996 start, end);
8e236546 1997 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44
CB		 1998 }
1999 }
2000}
2001
b6d33834
CD		 2002int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2003{
2004 /* kvm common code refers to this, but never calls it */
2005 BUG();
2006 return 0;
2007}
2008
14eebd91
CO		 2009static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2010 struct kvm_one_reg *reg)
2011{
2012 int r = -EINVAL;
2013
2014 switch (reg->id) {
29b7c71b
CO		 2015 case KVM_REG_S390_TODPR:
2016 r = put_user(vcpu->arch.sie_block->todpr,
2017 (u32 __user *)reg->addr);
2018 break;
2019 case KVM_REG_S390_EPOCHDIFF:
2020 r = put_user(vcpu->arch.sie_block->epoch,
2021 (u64 __user *)reg->addr);
2022 break;
46a6dd1c 2023 case KVM_REG_S390_CPU_TIMER:
4287f247 2024 r = put_user(kvm_s390_get_cpu_timer(vcpu),
46a6dd1c
J		 2025 (u64 __user *)reg->addr);
2026 break;
2027 case KVM_REG_S390_CLOCK_COMP:
2028 r = put_user(vcpu->arch.sie_block->ckc,
2029 (u64 __user *)reg->addr);
2030 break;
536336c2
DD		 2031 case KVM_REG_S390_PFTOKEN:
2032 r = put_user(vcpu->arch.pfault_token,
2033 (u64 __user *)reg->addr);
2034 break;
2035 case KVM_REG_S390_PFCOMPARE:
2036 r = put_user(vcpu->arch.pfault_compare,
2037 (u64 __user *)reg->addr);
2038 break;
2039 case KVM_REG_S390_PFSELECT:
2040 r = put_user(vcpu->arch.pfault_select,
2041 (u64 __user *)reg->addr);
2042 break;
672550fb
CB		 2043 case KVM_REG_S390_PP:
2044 r = put_user(vcpu->arch.sie_block->pp,
2045 (u64 __user *)reg->addr);
2046 break;
afa45ff5
CB		 2047 case KVM_REG_S390_GBEA:
2048 r = put_user(vcpu->arch.sie_block->gbea,
2049 (u64 __user *)reg->addr);
2050 break;
14eebd91
CO		 2051 default:
2052 break;
2053 }
2054
2055 return r;
2056}
2057
2058static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2059 struct kvm_one_reg *reg)
2060{
2061 int r = -EINVAL;
4287f247 2062 __u64 val;
14eebd91
CO		 2063
2064 switch (reg->id) {
29b7c71b
CO		 2065 case KVM_REG_S390_TODPR:
2066 r = get_user(vcpu->arch.sie_block->todpr,
2067 (u32 __user *)reg->addr);
2068 break;
2069 case KVM_REG_S390_EPOCHDIFF:
2070 r = get_user(vcpu->arch.sie_block->epoch,
2071 (u64 __user *)reg->addr);
2072 break;
46a6dd1c 2073 case KVM_REG_S390_CPU_TIMER:
4287f247
DH		 2074 r = get_user(val, (u64 __user *)reg->addr);
2075 if (!r)
2076 kvm_s390_set_cpu_timer(vcpu, val);
46a6dd1c
J		 2077 break;
2078 case KVM_REG_S390_CLOCK_COMP:
2079 r = get_user(vcpu->arch.sie_block->ckc,
2080 (u64 __user *)reg->addr);
2081 break;
536336c2
DD		 2082 case KVM_REG_S390_PFTOKEN:
2083 r = get_user(vcpu->arch.pfault_token,
2084 (u64 __user *)reg->addr);
9fbd8082
DH		 2085 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2086 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 2087 break;
2088 case KVM_REG_S390_PFCOMPARE:
2089 r = get_user(vcpu->arch.pfault_compare,
2090 (u64 __user *)reg->addr);
2091 break;
2092 case KVM_REG_S390_PFSELECT:
2093 r = get_user(vcpu->arch.pfault_select,
2094 (u64 __user *)reg->addr);
2095 break;
672550fb
CB		 2096 case KVM_REG_S390_PP:
2097 r = get_user(vcpu->arch.sie_block->pp,
2098 (u64 __user *)reg->addr);
2099 break;
afa45ff5
CB		 2100 case KVM_REG_S390_GBEA:
2101 r = get_user(vcpu->arch.sie_block->gbea,
2102 (u64 __user *)reg->addr);
2103 break;
14eebd91
CO		 2104 default:
2105 break;
2106 }
2107
2108 return r;
2109}
b6d33834 2110
b0c632db
HC		 2111static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2112{
b0c632db 2113 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 2114 return 0;
2115}
2116
2117int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2118{
5a32c1af 2119 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 2120 return 0;
2121}
2122
2123int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2124{
5a32c1af 2125 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 2126 return 0;
2127}
2128
2129int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2130 struct kvm_sregs *sregs)
2131{
59674c1a 2132 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 2133 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
59674c1a 2134 restore_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 2135 return 0;
2136}
2137
2138int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2139 struct kvm_sregs *sregs)
2140{
59674c1a 2141 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 2142 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 2143 return 0;
2144}
2145
2146int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2147{
9abc2a08
DH		 2148 /* make sure the new values will be lazily loaded */
2149 save_fpu_regs();
4725c860
MS		 2150 if (test_fp_ctl(fpu->fpc))
2151 return -EINVAL;
9abc2a08
DH		 2152 current->thread.fpu.fpc = fpu->fpc;
2153 if (MACHINE_HAS_VX)
2154 convert_fp_to_vx(current->thread.fpu.vxrs, (freg_t *)fpu->fprs);
2155 else
2156 memcpy(current->thread.fpu.fprs, &fpu->fprs, sizeof(fpu->fprs));
b0c632db
HC		 2157 return 0;
2158}
2159
2160int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2161{
9abc2a08
DH		 2162 /* make sure we have the latest values */
2163 save_fpu_regs();
2164 if (MACHINE_HAS_VX)
2165 convert_vx_to_fp((freg_t *)fpu->fprs, current->thread.fpu.vxrs);
2166 else
2167 memcpy(fpu->fprs, current->thread.fpu.fprs, sizeof(fpu->fprs));
2168 fpu->fpc = current->thread.fpu.fpc;
b0c632db
HC		 2169 return 0;
2170}
2171
2172static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2173{
2174 int rc = 0;
2175
7a42fdc2 2176 if (!is_vcpu_stopped(vcpu))
b0c632db 2177 rc = -EBUSY;
d7b0b5eb
CO		 2178 else {
2179 vcpu->run->psw_mask = psw.mask;
2180 vcpu->run->psw_addr = psw.addr;
2181 }
b0c632db
HC		 2182 return rc;
2183}
2184
2185int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2186 struct kvm_translation *tr)
2187{
2188 return -EINVAL; /* not implemented yet */
2189}
2190
27291e21
DH		 2191#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2192 KVM_GUESTDBG_USE_HW_BP | \
2193 KVM_GUESTDBG_ENABLE)
2194
d0bfb940
JK		 2195int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2196 struct kvm_guest_debug *dbg)
b0c632db 2197{
27291e21
DH		 2198 int rc = 0;
2199
2200 vcpu->guest_debug = 0;
2201 kvm_s390_clear_bp_data(vcpu);
2202
2de3bfc2 2203 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21 2204 return -EINVAL;
89b5b4de
DH		 2205 if (!sclp.has_gpere)
2206 return -EINVAL;
27291e21
DH		 2207
2208 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2209 vcpu->guest_debug = dbg->control;
2210 /* enforce guest PER */
805de8f4 2211 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2212
2213 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2214 rc = kvm_s390_import_bp_data(vcpu, dbg);
2215 } else {
805de8f4 2216 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2217 vcpu->arch.guestdbg.last_bp = 0;
2218 }
2219
2220 if (rc) {
2221 vcpu->guest_debug = 0;
2222 kvm_s390_clear_bp_data(vcpu);
805de8f4 2223 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2224 }
2225
2226 return rc;
b0c632db
HC		 2227}
2228
62d9f0db
MT		 2229int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2230 struct kvm_mp_state *mp_state)
2231{
6352e4d2
DH		 2232 /* CHECK_STOP and LOAD are not supported yet */
2233 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2234 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 2235}
2236
2237int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2238 struct kvm_mp_state *mp_state)
2239{
6352e4d2
DH		 2240 int rc = 0;
2241
2242 /* user space knows about this interface - let it control the state */
2243 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2244
2245 switch (mp_state->mp_state) {
2246 case KVM_MP_STATE_STOPPED:
2247 kvm_s390_vcpu_stop(vcpu);
2248 break;
2249 case KVM_MP_STATE_OPERATING:
2250 kvm_s390_vcpu_start(vcpu);
2251 break;
2252 case KVM_MP_STATE_LOAD:
2253 case KVM_MP_STATE_CHECK_STOP:
2254 /* fall through - CHECK_STOP and LOAD are not supported yet */
2255 default:
2256 rc = -ENXIO;
2257 }
2258
2259 return rc;
62d9f0db
MT		 2260}
2261
8ad35755
DH		 2262static bool ibs_enabled(struct kvm_vcpu *vcpu)
2263{
2264 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2265}
2266
2c70fe44
CB		 2267static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2268{
8ad35755 2269retry:
8e236546 2270 kvm_s390_vcpu_request_handled(vcpu);
586b7ccd
CB		 2271 if (!vcpu->requests)
2272 return 0;
2c70fe44
CB		 2273 /*
2274 * We use MMU_RELOAD just to re-arm the ipte notifier for the
2275 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
2276 * This ensures that the ipte instruction for this request has
2277 * already finished. We might race against a second unmapper that
2278 * wants to set the blocking bit. Lets just retry the request loop.
2279 */
8ad35755 2280 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44
CB		 2281 int rc;
2282 rc = gmap_ipte_notify(vcpu->arch.gmap,
fda902cb 2283 kvm_s390_get_prefix(vcpu),
2c70fe44
CB		 2284 PAGE_SIZE * 2);
2285 if (rc)
2286 return rc;
8ad35755 2287 goto retry;
2c70fe44 2288 }
8ad35755 2289
d3d692c8
DH		 2290 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2291 vcpu->arch.sie_block->ihcpu = 0xffff;
2292 goto retry;
2293 }
2294
8ad35755
DH		 2295 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2296 if (!ibs_enabled(vcpu)) {
2297 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
805de8f4 2298 atomic_or(CPUSTAT_IBS,
8ad35755
DH		 2299 &vcpu->arch.sie_block->cpuflags);
2300 }
2301 goto retry;
2c70fe44 2302 }
8ad35755
DH		 2303
2304 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2305 if (ibs_enabled(vcpu)) {
2306 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
805de8f4 2307 atomic_andnot(CPUSTAT_IBS,
8ad35755
DH		 2308 &vcpu->arch.sie_block->cpuflags);
2309 }
2310 goto retry;
2311 }
2312
0759d068
DH		 2313 /* nothing to do, just clear the request */
2314 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2315
2c70fe44
CB		 2316 return 0;
2317}
2318
25ed1675
DH		 2319void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2320{
2321 struct kvm_vcpu *vcpu;
2322 int i;
2323
2324 mutex_lock(&kvm->lock);
2325 preempt_disable();
2326 kvm->arch.epoch = tod - get_tod_clock();
2327 kvm_s390_vcpu_block_all(kvm);
2328 kvm_for_each_vcpu(i, vcpu, kvm)
2329 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2330 kvm_s390_vcpu_unblock_all(kvm);
2331 preempt_enable();
2332 mutex_unlock(&kvm->lock);
2333}
2334
fa576c58
TH		 2335/**
2336 * kvm_arch_fault_in_page - fault-in guest page if necessary
2337 * @vcpu: The corresponding virtual cpu
2338 * @gpa: Guest physical address
2339 * @writable: Whether the page should be writable or not
2340 *
2341 * Make sure that a guest page has been faulted-in on the host.
2342 *
2343 * Return: Zero on success, negative error code otherwise.
2344 */
2345long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 2346{
527e30b4
MS		 2347 return gmap_fault(vcpu->arch.gmap, gpa,
2348 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 2349}
2350
3c038e6b
DD		 2351static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2352 unsigned long token)
2353{
2354 struct kvm_s390_interrupt inti;
383d0b05 2355 struct kvm_s390_irq irq;
3c038e6b
DD		 2356
2357 if (start_token) {
383d0b05
JF		 2358 irq.u.ext.ext_params2 = token;
2359 irq.type = KVM_S390_INT_PFAULT_INIT;
2360 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 2361 } else {
2362 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 2363 inti.parm64 = token;
3c038e6b
DD		 2364 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2365 }
2366}
2367
2368void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2369 struct kvm_async_pf *work)
2370{
2371 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2372 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2373}
2374
2375void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2376 struct kvm_async_pf *work)
2377{
2378 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2379 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2380}
2381
2382void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2383 struct kvm_async_pf *work)
2384{
2385 /* s390 will always inject the page directly */
2386}
2387
2388bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2389{
2390 /*
2391 * s390 will always inject the page directly,
2392 * but we still want check_async_completion to cleanup
2393 */
2394 return true;
2395}
2396
2397static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2398{
2399 hva_t hva;
2400 struct kvm_arch_async_pf arch;
2401 int rc;
2402
2403 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2404 return 0;
2405 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2406 vcpu->arch.pfault_compare)
2407 return 0;
2408 if (psw_extint_disabled(vcpu))
2409 return 0;
9a022067 2410 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 2411 return 0;
2412 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2413 return 0;
2414 if (!vcpu->arch.gmap->pfault_enabled)
2415 return 0;
2416
81480cc1
HC		 2417 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2418 hva += current->thread.gmap_addr & ~PAGE_MASK;
2419 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 2420 return 0;
2421
2422 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2423 return rc;
2424}
2425
3fb4c40f 2426static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 2427{
3fb4c40f 2428 int rc, cpuflags;
e168bf8d 2429
3c038e6b
DD		 2430 /*
2431 * On s390 notifications for arriving pages will be delivered directly
2432 * to the guest but the house keeping for completed pfaults is
2433 * handled outside the worker.
2434 */
2435 kvm_check_async_pf_completion(vcpu);
2436
7ec7c8c7
CB		 2437 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2438 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
b0c632db
HC		 2439
2440 if (need_resched())
2441 schedule();
2442
d3a73acb 2443 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 2444 s390_handle_mcck();
2445
79395031
JF		 2446 if (!kvm_is_ucontrol(vcpu->kvm)) {
2447 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2448 if (rc)
2449 return rc;
2450 }
0ff31867 2451
2c70fe44
CB		 2452 rc = kvm_s390_handle_requests(vcpu);
2453 if (rc)
2454 return rc;
2455
27291e21
DH		 2456 if (guestdbg_enabled(vcpu)) {
2457 kvm_s390_backup_guest_per_regs(vcpu);
2458 kvm_s390_patch_guest_per_regs(vcpu);
2459 }
2460
b0c632db 2461 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 2462 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2463 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2464 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 2465
3fb4c40f
TH		 2466 return 0;
2467}
2468
492d8642
TH		 2469static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2470{
56317920
DH		 2471 struct kvm_s390_pgm_info pgm_info = {
2472 .code = PGM_ADDRESSING,
2473 };
2474 u8 opcode, ilen;
492d8642
TH		 2475 int rc;
2476
2477 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2478 trace_kvm_s390_sie_fault(vcpu);
2479
2480 /*
2481 * We want to inject an addressing exception, which is defined as a
2482 * suppressing or terminating exception. However, since we came here
2483 * by a DAT access exception, the PSW still points to the faulting
2484 * instruction since DAT exceptions are nullifying. So we've got
2485 * to look up the current opcode to get the length of the instruction
2486 * to be able to forward the PSW.
2487 */
65977322 2488 rc = read_guest_instr(vcpu, &opcode, 1);
56317920 2489 ilen = insn_length(opcode);
9b0d721a
DH		 2490 if (rc < 0) {
2491 return rc;
2492 } else if (rc) {
2493 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2494 * Forward by arbitrary ilc, injection will take care of
2495 * nullification if necessary.
2496 */
2497 pgm_info = vcpu->arch.pgm;
2498 ilen = 4;
2499 }
56317920
DH		 2500 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2501 kvm_s390_forward_psw(vcpu, ilen);
2502 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
492d8642
TH		 2503}
2504
3fb4c40f
TH		 2505static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2506{
2b29a9fd
DD		 2507 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2508 vcpu->arch.sie_block->icptcode);
2509 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2510
27291e21
DH		 2511 if (guestdbg_enabled(vcpu))
2512 kvm_s390_restore_guest_per_regs(vcpu);
2513
7ec7c8c7
CB		 2514 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2515 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
71f116bf
DH		 2516
2517 if (vcpu->arch.sie_block->icptcode > 0) {
2518 int rc = kvm_handle_sie_intercept(vcpu);
2519
2520 if (rc != -EOPNOTSUPP)
2521 return rc;
2522 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2523 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2524 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2525 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2526 return -EREMOTE;
2527 } else if (exit_reason != -EFAULT) {
2528 vcpu->stat.exit_null++;
2529 return 0;
210b1607
TH		 2530 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2531 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2532 vcpu->run->s390_ucontrol.trans_exc_code =
2533 current->thread.gmap_addr;
2534 vcpu->run->s390_ucontrol.pgm_code = 0x10;
71f116bf 2535 return -EREMOTE;
24eb3a82 2536 } else if (current->thread.gmap_pfault) {
3c038e6b 2537 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 2538 current->thread.gmap_pfault = 0;
71f116bf
DH		 2539 if (kvm_arch_setup_async_pf(vcpu))
2540 return 0;
2541 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
a76ccff6 2542 }
71f116bf 2543 return vcpu_post_run_fault_in_sie(vcpu);
3fb4c40f
TH		 2544}
2545
2546static int __vcpu_run(struct kvm_vcpu *vcpu)
2547{
2548 int rc, exit_reason;
2549
800c1065
TH		 2550 /*
2551 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2552 * ning the guest), so that memslots (and other stuff) are protected
2553 */
2554 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2555
a76ccff6
TH		 2556 do {
2557 rc = vcpu_pre_run(vcpu);
2558 if (rc)
2559 break;
3fb4c40f 2560
800c1065 2561 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 2562 /*
2563 * As PF_VCPU will be used in fault handler, between
2564 * guest_enter and guest_exit should be no uaccess.
2565 */
0097d12e
CB		 2566 local_irq_disable();
2567 __kvm_guest_enter();
db0758b2 2568 __disable_cpu_timer_accounting(vcpu);
0097d12e 2569 local_irq_enable();
a76ccff6
TH		 2570 exit_reason = sie64a(vcpu->arch.sie_block,
2571 vcpu->run->s.regs.gprs);
0097d12e 2572 local_irq_disable();
db0758b2 2573 __enable_cpu_timer_accounting(vcpu);
0097d12e
CB		 2574 __kvm_guest_exit();
2575 local_irq_enable();
800c1065 2576 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 2577
2578 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 2579 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 2580
800c1065 2581 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 2582 return rc;
b0c632db
HC		 2583}
2584
b028ee3e
DH		 2585static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2586{
2587 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2588 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2589 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2590 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2591 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2592 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 2593 /* some control register changes require a tlb flush */
2594 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 2595 }
2596 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4287f247 2597 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
b028ee3e
DH		 2598 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2599 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2600 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2601 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2602 }
2603 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2604 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2605 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2606 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 2607 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2608 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e
DH		 2609 }
2610 kvm_run->kvm_dirty_regs = 0;
2611}
2612
2613static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2614{
2615 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2616 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2617 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2618 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4287f247 2619 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
b028ee3e
DH		 2620 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2621 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2622 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2623 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2624 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2625 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2626 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2627}
2628
b0c632db
HC		 2629int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2630{
8f2abe6a 2631 int rc;
b0c632db
HC		 2632 sigset_t sigsaved;
2633
27291e21
DH		 2634 if (guestdbg_exit_pending(vcpu)) {
2635 kvm_s390_prepare_debug_exit(vcpu);
2636 return 0;
2637 }
2638
b0c632db
HC		 2639 if (vcpu->sigset_active)
2640 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2641
6352e4d2
DH		 2642 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2643 kvm_s390_vcpu_start(vcpu);
2644 } else if (is_vcpu_stopped(vcpu)) {
ea2cdd27 2645 pr_err_ratelimited("can't run stopped vcpu %d\n",
6352e4d2
DH		 2646 vcpu->vcpu_id);
2647 return -EINVAL;
2648 }
b0c632db 2649
b028ee3e 2650 sync_regs(vcpu, kvm_run);
db0758b2 2651 enable_cpu_timer_accounting(vcpu);
d7b0b5eb 2652
dab4079d 2653 might_fault();
a76ccff6 2654 rc = __vcpu_run(vcpu);
9ace903d 2655
b1d16c49
CE		 2656 if (signal_pending(current) && !rc) {
2657 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 2658 rc = -EINTR;
b1d16c49 2659 }
8f2abe6a 2660
27291e21
DH		 2661 if (guestdbg_exit_pending(vcpu) && !rc) {
2662 kvm_s390_prepare_debug_exit(vcpu);
2663 rc = 0;
2664 }
2665
8f2abe6a 2666 if (rc == -EREMOTE) {
71f116bf 2667 /* userspace support is needed, kvm_run has been prepared */
8f2abe6a
CB		 2668 rc = 0;
2669 }
b0c632db 2670
db0758b2 2671 disable_cpu_timer_accounting(vcpu);
b028ee3e 2672 store_regs(vcpu, kvm_run);
d7b0b5eb 2673
b0c632db
HC		 2674 if (vcpu->sigset_active)
2675 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2676
b0c632db 2677 vcpu->stat.exit_userspace++;
7e8e6ab4 2678 return rc;
b0c632db
HC		 2679}
2680
b0c632db
HC		 2681/*
2682 * store status at address
2683 * we use have two special cases:
2684 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2685 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2686 */
d0bce605 2687int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 2688{
092670cd 2689 unsigned char archmode = 1;
9abc2a08 2690 freg_t fprs[NUM_FPRS];
fda902cb 2691 unsigned int px;
4287f247 2692 u64 clkcomp, cputm;
d0bce605 2693 int rc;
b0c632db 2694
d9a3a09a 2695 px = kvm_s390_get_prefix(vcpu);
d0bce605
HC		 2696 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2697 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 2698 return -EFAULT;
d9a3a09a 2699 gpa = 0;
d0bce605
HC		 2700 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2701 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 2702 return -EFAULT;
d9a3a09a
MS		 2703 gpa = px;
2704 } else
2705 gpa -= __LC_FPREGS_SAVE_AREA;
9abc2a08
DH		 2706
2707 /* manually convert vector registers if necessary */
2708 if (MACHINE_HAS_VX) {
9522b37f 2709 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08
DH		 2710 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2711 fprs, 128);
2712 } else {
2713 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
6fd8e67d 2714 vcpu->run->s.regs.fprs, 128);
9abc2a08 2715 }
d9a3a09a 2716 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
d0bce605 2717 vcpu->run->s.regs.gprs, 128);
d9a3a09a 2718 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
d0bce605 2719 &vcpu->arch.sie_block->gpsw, 16);
d9a3a09a 2720 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
fda902cb 2721 &px, 4);
d9a3a09a 2722 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
9abc2a08 2723 &vcpu->run->s.regs.fpc, 4);
d9a3a09a 2724 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
d0bce605 2725 &vcpu->arch.sie_block->todpr, 4);
4287f247 2726 cputm = kvm_s390_get_cpu_timer(vcpu);
d9a3a09a 2727 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4287f247 2728 &cputm, 8);
178bd789 2729 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d9a3a09a 2730 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
d0bce605 2731 &clkcomp, 8);
d9a3a09a 2732 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
d0bce605 2733 &vcpu->run->s.regs.acrs, 64);
d9a3a09a 2734 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
d0bce605
HC		 2735 &vcpu->arch.sie_block->gcr, 128);
2736 return rc ? -EFAULT : 0;
b0c632db
HC		 2737}
2738
e879892c
TH		 2739int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2740{
2741 /*
2742 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2743 * copying in vcpu load/put. Lets update our copies before we save
2744 * it into the save area
2745 */
d0164ee2 2746 save_fpu_regs();
9abc2a08 2747 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
e879892c
TH		 2748 save_access_regs(vcpu->run->s.regs.acrs);
2749
2750 return kvm_s390_store_status_unloaded(vcpu, addr);
2751}
2752
bc17de7c
EF		 2753/*
2754 * store additional status at address
2755 */
2756int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2757 unsigned long gpa)
2758{
2759 /* Only bits 0-53 are used for address formation */
2760 if (!(gpa & ~0x3ff))
2761 return 0;
2762
2763 return write_guest_abs(vcpu, gpa & ~0x3ff,
2764 (void *)&vcpu->run->s.regs.vrs, 512);
2765}
2766
2767int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2768{
2769 if (!test_kvm_facility(vcpu->kvm, 129))
2770 return 0;
2771
2772 /*
2773 * The guest VXRS are in the host VXRs due to the lazy
9977e886
HB		 2774 * copying in vcpu load/put. We can simply call save_fpu_regs()
2775 * to save the current register state because we are in the
2776 * middle of a load/put cycle.
2777 *
2778 * Let's update our copies before we save it into the save area.
bc17de7c 2779 */
d0164ee2 2780 save_fpu_regs();
bc17de7c
EF		 2781
2782 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2783}
2784
8ad35755
DH		 2785static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2786{
2787 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
8e236546 2788 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
8ad35755
DH		 2789}
2790
2791static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2792{
2793 unsigned int i;
2794 struct kvm_vcpu *vcpu;
2795
2796 kvm_for_each_vcpu(i, vcpu, kvm) {
2797 __disable_ibs_on_vcpu(vcpu);
2798 }
2799}
2800
2801static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2802{
09a400e7
DH		 2803 if (!sclp.has_ibs)
2804 return;
8ad35755 2805 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
8e236546 2806 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
8ad35755
DH		 2807}
2808
6852d7b6
DH		 2809void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2810{
8ad35755
DH		 2811 int i, online_vcpus, started_vcpus = 0;
2812
2813 if (!is_vcpu_stopped(vcpu))
2814 return;
2815
6852d7b6 2816 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 2817 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2818 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2819 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2820
2821 for (i = 0; i < online_vcpus; i++) {
2822 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2823 started_vcpus++;
2824 }
2825
2826 if (started_vcpus == 0) {
2827 /* we're the only active VCPU -> speed it up */
2828 __enable_ibs_on_vcpu(vcpu);
2829 } else if (started_vcpus == 1) {
2830 /*
2831 * As we are starting a second VCPU, we have to disable
2832 * the IBS facility on all VCPUs to remove potentially
2833 * oustanding ENABLE requests.
2834 */
2835 __disable_ibs_on_all_vcpus(vcpu->kvm);
2836 }
2837
805de8f4 2838 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2839 /*
2840 * Another VCPU might have used IBS while we were offline.
2841 * Let's play safe and flush the VCPU at startup.
2842 */
d3d692c8 2843 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 2844 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2845 return;
6852d7b6
DH		 2846}
2847
2848void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2849{
8ad35755
DH		 2850 int i, online_vcpus, started_vcpus = 0;
2851 struct kvm_vcpu *started_vcpu = NULL;
2852
2853 if (is_vcpu_stopped(vcpu))
2854 return;
2855
6852d7b6 2856 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 2857 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2858 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2859 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2860
32f5ff63 2861 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 2862 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 2863
805de8f4 2864 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2865 __disable_ibs_on_vcpu(vcpu);
2866
2867 for (i = 0; i < online_vcpus; i++) {
2868 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2869 started_vcpus++;
2870 started_vcpu = vcpu->kvm->vcpus[i];
2871 }
2872 }
2873
2874 if (started_vcpus == 1) {
2875 /*
2876 * As we only have one VCPU left, we want to enable the
2877 * IBS facility for that VCPU to speed it up.
2878 */
2879 __enable_ibs_on_vcpu(started_vcpu);
2880 }
2881
433b9ee4 2882 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2883 return;
6852d7b6
DH		 2884}
2885
d6712df9
CH		 2886static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2887 struct kvm_enable_cap *cap)
2888{
2889 int r;
2890
2891 if (cap->flags)
2892 return -EINVAL;
2893
2894 switch (cap->cap) {
fa6b7fe9
CH		 2895 case KVM_CAP_S390_CSS_SUPPORT:
2896 if (!vcpu->kvm->arch.css_support) {
2897 vcpu->kvm->arch.css_support = 1;
c92ea7b9 2898 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
fa6b7fe9
CH		 2899 trace_kvm_s390_enable_css(vcpu->kvm);
2900 }
2901 r = 0;
2902 break;
d6712df9
CH		 2903 default:
2904 r = -EINVAL;
2905 break;
2906 }
2907 return r;
2908}
2909
41408c28
TH		 2910static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2911 struct kvm_s390_mem_op *mop)
2912{
2913 void __user *uaddr = (void __user *)mop->buf;
2914 void *tmpbuf = NULL;
2915 int r, srcu_idx;
2916 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2917 | KVM_S390_MEMOP_F_CHECK_ONLY;
2918
2919 if (mop->flags & ~supported_flags)
2920 return -EINVAL;
2921
2922 if (mop->size > MEM_OP_MAX_SIZE)
2923 return -E2BIG;
2924
2925 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2926 tmpbuf = vmalloc(mop->size);
2927 if (!tmpbuf)
2928 return -ENOMEM;
2929 }
2930
2931 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2932
2933 switch (mop->op) {
2934 case KVM_S390_MEMOP_LOGICAL_READ:
2935 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 2936 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2937 mop->size, GACC_FETCH);
41408c28
TH		 2938 break;
2939 }
2940 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2941 if (r == 0) {
2942 if (copy_to_user(uaddr, tmpbuf, mop->size))
2943 r = -EFAULT;
2944 }
2945 break;
2946 case KVM_S390_MEMOP_LOGICAL_WRITE:
2947 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 2948 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2949 mop->size, GACC_STORE);
41408c28
TH		 2950 break;
2951 }
2952 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2953 r = -EFAULT;
2954 break;
2955 }
2956 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2957 break;
2958 default:
2959 r = -EINVAL;
2960 }
2961
2962 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2963
2964 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2965 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2966
2967 vfree(tmpbuf);
2968 return r;
2969}
2970
b0c632db
HC		 2971long kvm_arch_vcpu_ioctl(struct file *filp,
2972 unsigned int ioctl, unsigned long arg)
2973{
2974 struct kvm_vcpu *vcpu = filp->private_data;
2975 void __user *argp = (void __user *)arg;
800c1065 2976 int idx;
bc923cc9 2977 long r;
b0c632db 2978
93736624 2979 switch (ioctl) {
47b43c52
JF		 2980 case KVM_S390_IRQ: {
2981 struct kvm_s390_irq s390irq;
2982
2983 r = -EFAULT;
2984 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2985 break;
2986 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2987 break;
2988 }
93736624 2989 case KVM_S390_INTERRUPT: {
ba5c1e9b 2990 struct kvm_s390_interrupt s390int;
383d0b05 2991 struct kvm_s390_irq s390irq;
ba5c1e9b 2992
93736624 2993 r = -EFAULT;
ba5c1e9b 2994 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624 2995 break;
383d0b05
JF		 2996 if (s390int_to_s390irq(&s390int, &s390irq))
2997 return -EINVAL;
2998 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
93736624 2999 break;
ba5c1e9b 3000 }
b0c632db 3001 case KVM_S390_STORE_STATUS:
800c1065 3002 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 3003 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 3004 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 3005 break;
b0c632db
HC		 3006 case KVM_S390_SET_INITIAL_PSW: {
3007 psw_t psw;
3008
bc923cc9 3009 r = -EFAULT;
b0c632db 3010 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 3011 break;
3012 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3013 break;
b0c632db
HC		 3014 }
3015 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 3016 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3017 break;
14eebd91
CO		 3018 case KVM_SET_ONE_REG:
3019 case KVM_GET_ONE_REG: {
3020 struct kvm_one_reg reg;
3021 r = -EFAULT;
3022 if (copy_from_user(&reg, argp, sizeof(reg)))
3023 break;
3024 if (ioctl == KVM_SET_ONE_REG)
3025 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3026 else
3027 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3028 break;
3029 }
27e0393f
CO		 3030#ifdef CONFIG_KVM_S390_UCONTROL
3031 case KVM_S390_UCAS_MAP: {
3032 struct kvm_s390_ucas_mapping ucasmap;
3033
3034 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3035 r = -EFAULT;
3036 break;
3037 }
3038
3039 if (!kvm_is_ucontrol(vcpu->kvm)) {
3040 r = -EINVAL;
3041 break;
3042 }
3043
3044 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3045 ucasmap.vcpu_addr, ucasmap.length);
3046 break;
3047 }
3048 case KVM_S390_UCAS_UNMAP: {
3049 struct kvm_s390_ucas_mapping ucasmap;
3050
3051 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3052 r = -EFAULT;
3053 break;
3054 }
3055
3056 if (!kvm_is_ucontrol(vcpu->kvm)) {
3057 r = -EINVAL;
3058 break;
3059 }
3060
3061 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3062 ucasmap.length);
3063 break;
3064 }
3065#endif
ccc7910f 3066 case KVM_S390_VCPU_FAULT: {
527e30b4 3067 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 3068 break;
3069 }
d6712df9
CH		 3070 case KVM_ENABLE_CAP:
3071 {
3072 struct kvm_enable_cap cap;
3073 r = -EFAULT;
3074 if (copy_from_user(&cap, argp, sizeof(cap)))
3075 break;
3076 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3077 break;
3078 }
41408c28
TH		 3079 case KVM_S390_MEM_OP: {
3080 struct kvm_s390_mem_op mem_op;
3081
3082 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3083 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3084 else
3085 r = -EFAULT;
3086 break;
3087 }
816c7667
JF		 3088 case KVM_S390_SET_IRQ_STATE: {
3089 struct kvm_s390_irq_state irq_state;
3090
3091 r = -EFAULT;
3092 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3093 break;
3094 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3095 irq_state.len == 0 ||
3096 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3097 r = -EINVAL;
3098 break;
3099 }
3100 r = kvm_s390_set_irq_state(vcpu,
3101 (void __user *) irq_state.buf,
3102 irq_state.len);
3103 break;
3104 }
3105 case KVM_S390_GET_IRQ_STATE: {
3106 struct kvm_s390_irq_state irq_state;
3107
3108 r = -EFAULT;
3109 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3110 break;
3111 if (irq_state.len == 0) {
3112 r = -EINVAL;
3113 break;
3114 }
3115 r = kvm_s390_get_irq_state(vcpu,
3116 (__u8 __user *) irq_state.buf,
3117 irq_state.len);
3118 break;
3119 }
b0c632db 3120 default:
3e6afcf1 3121 r = -ENOTTY;
b0c632db 3122 }
bc923cc9 3123 return r;
b0c632db
HC		 3124}
3125
5b1c1493
CO		 3126int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3127{
3128#ifdef CONFIG_KVM_S390_UCONTROL
3129 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3130 && (kvm_is_ucontrol(vcpu->kvm))) {
3131 vmf->page = virt_to_page(vcpu->arch.sie_block);
3132 get_page(vmf->page);
3133 return 0;
3134 }
3135#endif
3136 return VM_FAULT_SIGBUS;
3137}
3138
5587027c
AK		 3139int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3140 unsigned long npages)
db3fe4eb
TY		 3141{
3142 return 0;
3143}
3144
b0c632db 3145/* Section: memory related */
f7784b8e
MT		 3146int kvm_arch_prepare_memory_region(struct kvm *kvm,
3147 struct kvm_memory_slot *memslot,
09170a49 3148 const struct kvm_userspace_memory_region *mem,
7b6195a9 3149 enum kvm_mr_change change)
b0c632db 3150{
dd2887e7
NW		 3151 /* A few sanity checks. We can have memory slots which have to be
3152 located/ended at a segment boundary (1MB). The memory in userland is
3153 ok to be fragmented into various different vmas. It is okay to mmap()
3154 and munmap() stuff in this slot after doing this call at any time */
b0c632db 3155
598841ca 3156 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 3157 return -EINVAL;
3158
598841ca 3159 if (mem->memory_size & 0xffffful)
b0c632db
HC		 3160 return -EINVAL;
3161
a3a92c31
DD		 3162 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3163 return -EINVAL;
3164
f7784b8e
MT		 3165 return 0;
3166}
3167
3168void kvm_arch_commit_memory_region(struct kvm *kvm,
09170a49 3169 const struct kvm_userspace_memory_region *mem,
8482644a 3170 const struct kvm_memory_slot *old,
f36f3f28 3171 const struct kvm_memory_slot *new,
8482644a 3172 enum kvm_mr_change change)
f7784b8e 3173{
f7850c92 3174 int rc;
f7784b8e 3175
2cef4deb
CB		 3176 /* If the basics of the memslot do not change, we do not want
3177 * to update the gmap. Every update causes several unnecessary
3178 * segment translation exceptions. This is usually handled just
3179 * fine by the normal fault handler + gmap, but it will also
3180 * cause faults on the prefix page of running guest CPUs.
3181 */
3182 if (old->userspace_addr == mem->userspace_addr &&
3183 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3184 old->npages * PAGE_SIZE == mem->memory_size)
3185 return;
598841ca
CO		 3186
3187 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3188 mem->guest_phys_addr, mem->memory_size);
3189 if (rc)
ea2cdd27 3190 pr_warn("failed to commit memory region\n");
598841ca 3191 return;
b0c632db
HC		 3192}
3193
60a37709
AY		 3194static inline unsigned long nonhyp_mask(int i)
3195{
3196 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3197
3198 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3199}
3200
3491caf2
CB		 3201void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3202{
3203 vcpu->valid_wakeup = false;
3204}
3205
b0c632db
HC		 3206static int __init kvm_s390_init(void)
3207{
60a37709
AY		 3208 int i;
3209
07197fd0
DH		 3210 if (!sclp.has_sief2) {
3211 pr_info("SIE not available\n");
3212 return -ENODEV;
3213 }
3214
60a37709
AY		 3215 for (i = 0; i < 16; i++)
3216 kvm_s390_fac_list_mask[i] |=
3217 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3218
9d8d5786 3219 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 3220}
3221
3222static void __exit kvm_s390_exit(void)
3223{
3224 kvm_exit();
3225}
3226
3227module_init(kvm_s390_init);
3228module_exit(kvm_s390_exit);
566af940
CH		 3229
3230/*
3231 * Enable autoloading of the kvm module.
3232 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3233 * since x86 takes a different approach.
3234 */
3235#include <linux/miscdevice.h>
3236MODULE_ALIAS_MISCDEV(KVM_MINOR);
3237MODULE_ALIAS("devname:kvm");
Ubuntu Hirsute Linux kernel mirror