]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
s390/mm: don't fault everything in read-write in gmap_pte_op_fixup()
[mirror_ubuntu-jammy-kernel.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
b2d73b2a 24#include <linux/mman.h>
b0c632db 25#include <linux/module.h>
a374e892 26#include <linux/random.h>
b0c632db 27#include <linux/slab.h>
ba5c1e9b 28#include <linux/timer.h>
41408c28 29#include <linux/vmalloc.h>
15c9705f 30#include <linux/bitmap.h>
cbb870c8 31#include <asm/asm-offsets.h>
b0c632db 32#include <asm/lowcore.h>
fdf03650 33#include <asm/etr.h>
b0c632db 34#include <asm/pgtable.h>
1e133ab2 35#include <asm/gmap.h>
f5daba1d 36#include <asm/nmi.h>
a0616cde 37#include <asm/switch_to.h>
6d3da241 38#include <asm/isc.h>
1526bf9c 39#include <asm/sclp.h>
0a763c78
DH		 40#include <asm/cpacf.h>
41#include <asm/etr.h>
8f2abe6a 42#include "kvm-s390.h"
b0c632db
HC		 43#include "gaccess.h"
44
ea2cdd27
DH		 45#define KMSG_COMPONENT "kvm-s390"
46#undef pr_fmt
47#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
48
5786fffa
CH		 49#define CREATE_TRACE_POINTS
50#include "trace.h"
ade38c31 51#include "trace-s390.h"
5786fffa 52
41408c28 53#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
816c7667
JF		 54#define LOCAL_IRQS 32
55#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
56 (KVM_MAX_VCPUS + LOCAL_IRQS))
41408c28 57
b0c632db
HC		 58#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
59
60struct kvm_stats_debugfs_item debugfs_entries[] = {
61 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 62 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 63 { "exit_validity", VCPU_STAT(exit_validity) },
64 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
65 { "exit_external_request", VCPU_STAT(exit_external_request) },
66 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b
CO		 67 { "exit_instruction", VCPU_STAT(exit_instruction) },
68 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
69 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
a011eeb2 70 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
f7819512 71 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
62bea5bf 72 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
3491caf2 73 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
ce2e4f0b 74 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 75 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 76 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 77 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
78 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 79 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 80 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 81 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
82 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
83 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
84 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
85 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
86 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
87 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 88 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 89 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
90 { "instruction_spx", VCPU_STAT(instruction_spx) },
91 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
92 { "instruction_stap", VCPU_STAT(instruction_stap) },
93 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 94 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 95 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
96 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 97 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 98 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
99 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 100 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
95ca2cb5 101 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
5288fbf0 102 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 103 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 104 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 105 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 106 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
107 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 108 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 109 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
110 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 111 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 112 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
113 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
114 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 115 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
116 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
117 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 118 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 119 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 120 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
175a5c9e
CB		 121 { "diagnose_258", VCPU_STAT(diagnose_258) },
122 { "diagnose_308", VCPU_STAT(diagnose_308) },
123 { "diagnose_500", VCPU_STAT(diagnose_500) },
b0c632db
HC		 124 { NULL }
125};
126
9d8d5786 127/* upper facilities limit for kvm */
60a37709
AY		 128unsigned long kvm_s390_fac_list_mask[16] = {
129 0xffe6000000000000UL,
130 0x005e000000000000UL,
9d8d5786 131};
b0c632db 132
9d8d5786 133unsigned long kvm_s390_fac_list_mask_size(void)
78c4b59f 134{
9d8d5786
MM		 135 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
136 return ARRAY_SIZE(kvm_s390_fac_list_mask);
78c4b59f
MM		 137}
138
15c9705f
DH		 139/* available cpu features supported by kvm */
140static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
0a763c78
DH		 141/* available subfunctions indicated via query / "test bit" */
142static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
15c9705f 143
9d8d5786 144static struct gmap_notifier gmap_notifier;
78f26131 145debug_info_t *kvm_s390_dbf;
9d8d5786 146
b0c632db 147/* Section: not file related */
13a34e06 148int kvm_arch_hardware_enable(void)
b0c632db
HC		 149{
150 /* every s390 is virtualization enabled ;-) */
10474ae8 151 return 0;
b0c632db
HC		 152}
153
414d3b07
MS		 154static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
155 unsigned long end);
2c70fe44 156
fdf03650
FZ		 157/*
158 * This callback is executed during stop_machine(). All CPUs are therefore
159 * temporarily stopped. In order not to change guest behavior, we have to
160 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
161 * so a CPU won't be stopped while calculating with the epoch.
162 */
163static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
164 void *v)
165{
166 struct kvm *kvm;
167 struct kvm_vcpu *vcpu;
168 int i;
169 unsigned long long *delta = v;
170
171 list_for_each_entry(kvm, &vm_list, vm_list) {
172 kvm->arch.epoch -= *delta;
173 kvm_for_each_vcpu(i, vcpu, kvm) {
174 vcpu->arch.sie_block->epoch -= *delta;
db0758b2
DH		 175 if (vcpu->arch.cputm_enabled)
176 vcpu->arch.cputm_start += *delta;
fdf03650
FZ		 177 }
178 }
179 return NOTIFY_OK;
180}
181
182static struct notifier_block kvm_clock_notifier = {
183 .notifier_call = kvm_clock_sync,
184};
185
b0c632db
HC		 186int kvm_arch_hardware_setup(void)
187{
2c70fe44 188 gmap_notifier.notifier_call = kvm_gmap_notifier;
b2d73b2a 189 gmap_register_pte_notifier(&gmap_notifier);
fdf03650
FZ		 190 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
191 &kvm_clock_notifier);
b0c632db
HC		 192 return 0;
193}
194
195void kvm_arch_hardware_unsetup(void)
196{
b2d73b2a 197 gmap_unregister_pte_notifier(&gmap_notifier);
fdf03650
FZ		 198 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
199 &kvm_clock_notifier);
b0c632db
HC		 200}
201
22be5a13
DH		 202static void allow_cpu_feat(unsigned long nr)
203{
204 set_bit_inv(nr, kvm_s390_available_cpu_feat);
205}
206
0a763c78
DH		 207static inline int plo_test_bit(unsigned char nr)
208{
209 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
210 int cc = 3; /* subfunction not available */
211
212 asm volatile(
213 /* Parameter registers are ignored for "test bit" */
214 " plo 0,0,0,0(0)\n"
215 " ipm %0\n"
216 " srl %0,28\n"
217 : "=d" (cc)
218 : "d" (r0)
219 : "cc");
220 return cc == 0;
221}
222
22be5a13
DH		 223static void kvm_s390_cpu_feat_init(void)
224{
0a763c78
DH		 225 int i;
226
227 for (i = 0; i < 256; ++i) {
228 if (plo_test_bit(i))
229 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
230 }
231
232 if (test_facility(28)) /* TOD-clock steering */
233 etr_ptff(kvm_s390_available_subfunc.ptff, ETR_PTFF_QAF);
234
235 if (test_facility(17)) { /* MSA */
236 __cpacf_query(CPACF_KMAC, kvm_s390_available_subfunc.kmac);
237 __cpacf_query(CPACF_KMC, kvm_s390_available_subfunc.kmc);
238 __cpacf_query(CPACF_KM, kvm_s390_available_subfunc.km);
239 __cpacf_query(CPACF_KIMD, kvm_s390_available_subfunc.kimd);
240 __cpacf_query(CPACF_KLMD, kvm_s390_available_subfunc.klmd);
241 }
242 if (test_facility(76)) /* MSA3 */
243 __cpacf_query(CPACF_PCKMO, kvm_s390_available_subfunc.pckmo);
244 if (test_facility(77)) { /* MSA4 */
245 __cpacf_query(CPACF_KMCTR, kvm_s390_available_subfunc.kmctr);
246 __cpacf_query(CPACF_KMF, kvm_s390_available_subfunc.kmf);
247 __cpacf_query(CPACF_KMO, kvm_s390_available_subfunc.kmo);
248 __cpacf_query(CPACF_PCC, kvm_s390_available_subfunc.pcc);
249 }
250 if (test_facility(57)) /* MSA5 */
251 __cpacf_query(CPACF_PPNO, kvm_s390_available_subfunc.ppno);
252
22be5a13
DH		 253 if (MACHINE_HAS_ESOP)
254 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
255}
256
b0c632db
HC		 257int kvm_arch_init(void *opaque)
258{
78f26131
CB		 259 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
260 if (!kvm_s390_dbf)
261 return -ENOMEM;
262
263 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
264 debug_unregister(kvm_s390_dbf);
265 return -ENOMEM;
266 }
267
22be5a13
DH		 268 kvm_s390_cpu_feat_init();
269
84877d93
CH		 270 /* Register floating interrupt controller interface. */
271 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 272}
273
78f26131
CB		 274void kvm_arch_exit(void)
275{
276 debug_unregister(kvm_s390_dbf);
277}
278
b0c632db
HC		 279/* Section: device related */
280long kvm_arch_dev_ioctl(struct file *filp,
281 unsigned int ioctl, unsigned long arg)
282{
283 if (ioctl == KVM_S390_ENABLE_SIE)
284 return s390_enable_sie();
285 return -EINVAL;
286}
287
784aa3d7 288int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 289{
d7b0b5eb
CO		 290 int r;
291
2bd0ac4e 292 switch (ext) {
d7b0b5eb 293 case KVM_CAP_S390_PSW:
b6cf8788 294 case KVM_CAP_S390_GMAP:
52e16b18 295 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 296#ifdef CONFIG_KVM_S390_UCONTROL
297 case KVM_CAP_S390_UCONTROL:
298#endif
3c038e6b 299 case KVM_CAP_ASYNC_PF:
60b413c9 300 case KVM_CAP_SYNC_REGS:
14eebd91 301 case KVM_CAP_ONE_REG:
d6712df9 302 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 303 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 304 case KVM_CAP_IOEVENTFD:
c05c4186 305 case KVM_CAP_DEVICE_CTRL:
d938dc55 306 case KVM_CAP_ENABLE_CAP_VM:
78599d90 307 case KVM_CAP_S390_IRQCHIP:
f2061656 308 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 309 case KVM_CAP_MP_STATE:
47b43c52 310 case KVM_CAP_S390_INJECT_IRQ:
2444b352 311 case KVM_CAP_S390_USER_SIGP:
e44fc8c9 312 case KVM_CAP_S390_USER_STSI:
30ee2a98 313 case KVM_CAP_S390_SKEYS:
816c7667 314 case KVM_CAP_S390_IRQ_STATE:
d7b0b5eb
CO		 315 r = 1;
316 break;
41408c28
TH		 317 case KVM_CAP_S390_MEM_OP:
318 r = MEM_OP_MAX_SIZE;
319 break;
e726b1bd
CB		 320 case KVM_CAP_NR_VCPUS:
321 case KVM_CAP_MAX_VCPUS:
76a6dd72
DH		 322 r = KVM_S390_BSCA_CPU_SLOTS;
323 if (sclp.has_esca && sclp.has_64bscao)
324 r = KVM_S390_ESCA_CPU_SLOTS;
e726b1bd 325 break;
e1e2e605
NW		 326 case KVM_CAP_NR_MEMSLOTS:
327 r = KVM_USER_MEM_SLOTS;
328 break;
1526bf9c 329 case KVM_CAP_S390_COW:
abf09bed 330 r = MACHINE_HAS_ESOP;
1526bf9c 331 break;
68c55750
EF		 332 case KVM_CAP_S390_VECTOR_REGISTERS:
333 r = MACHINE_HAS_VX;
334 break;
c6e5f166
FZ		 335 case KVM_CAP_S390_RI:
336 r = test_facility(64);
337 break;
2bd0ac4e 338 default:
d7b0b5eb 339 r = 0;
2bd0ac4e 340 }
d7b0b5eb 341 return r;
b0c632db
HC		 342}
343
15f36ebd
JH		 344static void kvm_s390_sync_dirty_log(struct kvm *kvm,
345 struct kvm_memory_slot *memslot)
346{
347 gfn_t cur_gfn, last_gfn;
348 unsigned long address;
349 struct gmap *gmap = kvm->arch.gmap;
350
15f36ebd
JH		 351 /* Loop over all guest pages */
352 last_gfn = memslot->base_gfn + memslot->npages;
353 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
354 address = gfn_to_hva_memslot(memslot, cur_gfn);
355
1e133ab2 356 if (test_and_clear_guest_dirty(gmap->mm, address))
15f36ebd 357 mark_page_dirty(kvm, cur_gfn);
1763f8d0
CB		 358 if (fatal_signal_pending(current))
359 return;
70c88a00 360 cond_resched();
15f36ebd 361 }
15f36ebd
JH		 362}
363
b0c632db 364/* Section: vm related */
a6e2f683
ED		 365static void sca_del_vcpu(struct kvm_vcpu *vcpu);
366
b0c632db
HC		 367/*
368 * Get (and clear) the dirty memory log for a memory slot.
369 */
370int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
371 struct kvm_dirty_log *log)
372{
15f36ebd
JH		 373 int r;
374 unsigned long n;
9f6b8029 375 struct kvm_memslots *slots;
15f36ebd
JH		 376 struct kvm_memory_slot *memslot;
377 int is_dirty = 0;
378
379 mutex_lock(&kvm->slots_lock);
380
381 r = -EINVAL;
382 if (log->slot >= KVM_USER_MEM_SLOTS)
383 goto out;
384
9f6b8029
PB		 385 slots = kvm_memslots(kvm);
386 memslot = id_to_memslot(slots, log->slot);
15f36ebd
JH		 387 r = -ENOENT;
388 if (!memslot->dirty_bitmap)
389 goto out;
390
391 kvm_s390_sync_dirty_log(kvm, memslot);
392 r = kvm_get_dirty_log(kvm, log, &is_dirty);
393 if (r)
394 goto out;
395
396 /* Clear the dirty log */
397 if (is_dirty) {
398 n = kvm_dirty_bitmap_bytes(memslot);
399 memset(memslot->dirty_bitmap, 0, n);
400 }
401 r = 0;
402out:
403 mutex_unlock(&kvm->slots_lock);
404 return r;
b0c632db
HC		 405}
406
d938dc55
CH		 407static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
408{
409 int r;
410
411 if (cap->flags)
412 return -EINVAL;
413
414 switch (cap->cap) {
84223598 415 case KVM_CAP_S390_IRQCHIP:
c92ea7b9 416 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
84223598
CH		 417 kvm->arch.use_irqchip = 1;
418 r = 0;
419 break;
2444b352 420 case KVM_CAP_S390_USER_SIGP:
c92ea7b9 421 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
2444b352
DH		 422 kvm->arch.user_sigp = 1;
423 r = 0;
424 break;
68c55750 425 case KVM_CAP_S390_VECTOR_REGISTERS:
5967c17b 426 mutex_lock(&kvm->lock);
a03825bb 427 if (kvm->created_vcpus) {
5967c17b
DH		 428 r = -EBUSY;
429 } else if (MACHINE_HAS_VX) {
c54f0d6a
DH		 430 set_kvm_facility(kvm->arch.model.fac_mask, 129);
431 set_kvm_facility(kvm->arch.model.fac_list, 129);
18280d8b
MM		 432 r = 0;
433 } else
434 r = -EINVAL;
5967c17b 435 mutex_unlock(&kvm->lock);
c92ea7b9
CB		 436 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
437 r ? "(not available)" : "(success)");
68c55750 438 break;
c6e5f166
FZ		 439 case KVM_CAP_S390_RI:
440 r = -EINVAL;
441 mutex_lock(&kvm->lock);
a03825bb 442 if (kvm->created_vcpus) {
c6e5f166
FZ		 443 r = -EBUSY;
444 } else if (test_facility(64)) {
c54f0d6a
DH		 445 set_kvm_facility(kvm->arch.model.fac_mask, 64);
446 set_kvm_facility(kvm->arch.model.fac_list, 64);
c6e5f166
FZ		 447 r = 0;
448 }
449 mutex_unlock(&kvm->lock);
450 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
451 r ? "(not available)" : "(success)");
452 break;
e44fc8c9 453 case KVM_CAP_S390_USER_STSI:
c92ea7b9 454 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
e44fc8c9
ET		 455 kvm->arch.user_stsi = 1;
456 r = 0;
457 break;
d938dc55
CH		 458 default:
459 r = -EINVAL;
460 break;
461 }
462 return r;
463}
464
8c0a7ce6
DD		 465static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
466{
467 int ret;
468
469 switch (attr->attr) {
470 case KVM_S390_VM_MEM_LIMIT_SIZE:
471 ret = 0;
c92ea7b9 472 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
a3a92c31
DD		 473 kvm->arch.mem_limit);
474 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
8c0a7ce6
DD		 475 ret = -EFAULT;
476 break;
477 default:
478 ret = -ENXIO;
479 break;
480 }
481 return ret;
482}
483
484static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 485{
486 int ret;
487 unsigned int idx;
488 switch (attr->attr) {
489 case KVM_S390_VM_MEM_ENABLE_CMMA:
f9cbd9b0 490 ret = -ENXIO;
c24cc9c8 491 if (!sclp.has_cmma)
e6db1d61
DD		 492 break;
493
4f718eab 494 ret = -EBUSY;
c92ea7b9 495 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
4f718eab 496 mutex_lock(&kvm->lock);
a03825bb 497 if (!kvm->created_vcpus) {
4f718eab
DD		 498 kvm->arch.use_cmma = 1;
499 ret = 0;
500 }
501 mutex_unlock(&kvm->lock);
502 break;
503 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 504 ret = -ENXIO;
505 if (!sclp.has_cmma)
506 break;
c3489155
DD		 507 ret = -EINVAL;
508 if (!kvm->arch.use_cmma)
509 break;
510
c92ea7b9 511 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
4f718eab
DD		 512 mutex_lock(&kvm->lock);
513 idx = srcu_read_lock(&kvm->srcu);
a13cff31 514 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 515 srcu_read_unlock(&kvm->srcu, idx);
516 mutex_unlock(&kvm->lock);
517 ret = 0;
518 break;
8c0a7ce6
DD		 519 case KVM_S390_VM_MEM_LIMIT_SIZE: {
520 unsigned long new_limit;
521
522 if (kvm_is_ucontrol(kvm))
523 return -EINVAL;
524
525 if (get_user(new_limit, (u64 __user *)attr->addr))
526 return -EFAULT;
527
a3a92c31
DD		 528 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
529 new_limit > kvm->arch.mem_limit)
8c0a7ce6
DD		 530 return -E2BIG;
531
a3a92c31
DD		 532 if (!new_limit)
533 return -EINVAL;
534
6ea427bb 535 /* gmap_create takes last usable address */
a3a92c31
DD		 536 if (new_limit != KVM_S390_NO_MEM_LIMIT)
537 new_limit -= 1;
538
8c0a7ce6
DD		 539 ret = -EBUSY;
540 mutex_lock(&kvm->lock);
a03825bb 541 if (!kvm->created_vcpus) {
6ea427bb
MS		 542 /* gmap_create will round the limit up */
543 struct gmap *new = gmap_create(current->mm, new_limit);
8c0a7ce6
DD		 544
545 if (!new) {
546 ret = -ENOMEM;
547 } else {
6ea427bb 548 gmap_remove(kvm->arch.gmap);
8c0a7ce6
DD		 549 new->private = kvm;
550 kvm->arch.gmap = new;
551 ret = 0;
552 }
553 }
554 mutex_unlock(&kvm->lock);
a3a92c31
DD		 555 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
556 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
557 (void *) kvm->arch.gmap->asce);
8c0a7ce6
DD		 558 break;
559 }
4f718eab
DD		 560 default:
561 ret = -ENXIO;
562 break;
563 }
564 return ret;
565}
566
a374e892
TK		 567static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
568
569static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
570{
571 struct kvm_vcpu *vcpu;
572 int i;
573
9d8d5786 574 if (!test_kvm_facility(kvm, 76))
a374e892
TK		 575 return -EINVAL;
576
577 mutex_lock(&kvm->lock);
578 switch (attr->attr) {
579 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
580 get_random_bytes(
581 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
582 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
583 kvm->arch.crypto.aes_kw = 1;
c92ea7b9 584 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
a374e892
TK		 585 break;
586 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
587 get_random_bytes(
588 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
589 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
590 kvm->arch.crypto.dea_kw = 1;
c92ea7b9 591 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
a374e892
TK		 592 break;
593 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
594 kvm->arch.crypto.aes_kw = 0;
595 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
596 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
c92ea7b9 597 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
a374e892
TK		 598 break;
599 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
600 kvm->arch.crypto.dea_kw = 0;
601 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
602 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
c92ea7b9 603 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
a374e892
TK		 604 break;
605 default:
606 mutex_unlock(&kvm->lock);
607 return -ENXIO;
608 }
609
610 kvm_for_each_vcpu(i, vcpu, kvm) {
611 kvm_s390_vcpu_crypto_setup(vcpu);
612 exit_sie(vcpu);
613 }
614 mutex_unlock(&kvm->lock);
615 return 0;
616}
617
72f25020
JH		 618static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
619{
620 u8 gtod_high;
621
622 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
623 sizeof(gtod_high)))
624 return -EFAULT;
625
626 if (gtod_high != 0)
627 return -EINVAL;
58c383c6 628 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
72f25020
JH		 629
630 return 0;
631}
632
633static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
634{
5a3d883a 635 u64 gtod;
72f25020
JH		 636
637 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
638 return -EFAULT;
639
25ed1675 640 kvm_s390_set_tod_clock(kvm, gtod);
58c383c6 641 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
72f25020
JH		 642 return 0;
643}
644
645static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
646{
647 int ret;
648
649 if (attr->flags)
650 return -EINVAL;
651
652 switch (attr->attr) {
653 case KVM_S390_VM_TOD_HIGH:
654 ret = kvm_s390_set_tod_high(kvm, attr);
655 break;
656 case KVM_S390_VM_TOD_LOW:
657 ret = kvm_s390_set_tod_low(kvm, attr);
658 break;
659 default:
660 ret = -ENXIO;
661 break;
662 }
663 return ret;
664}
665
666static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
667{
668 u8 gtod_high = 0;
669
670 if (copy_to_user((void __user *)attr->addr, &gtod_high,
671 sizeof(gtod_high)))
672 return -EFAULT;
58c383c6 673 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
72f25020
JH		 674
675 return 0;
676}
677
678static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
679{
5a3d883a 680 u64 gtod;
72f25020 681
60417fcc 682 gtod = kvm_s390_get_tod_clock_fast(kvm);
72f25020
JH		 683 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
684 return -EFAULT;
58c383c6 685 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
72f25020
JH		 686
687 return 0;
688}
689
690static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
691{
692 int ret;
693
694 if (attr->flags)
695 return -EINVAL;
696
697 switch (attr->attr) {
698 case KVM_S390_VM_TOD_HIGH:
699 ret = kvm_s390_get_tod_high(kvm, attr);
700 break;
701 case KVM_S390_VM_TOD_LOW:
702 ret = kvm_s390_get_tod_low(kvm, attr);
703 break;
704 default:
705 ret = -ENXIO;
706 break;
707 }
708 return ret;
709}
710
658b6eda
MM		 711static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
712{
713 struct kvm_s390_vm_cpu_processor *proc;
053dd230 714 u16 lowest_ibc, unblocked_ibc;
658b6eda
MM		 715 int ret = 0;
716
717 mutex_lock(&kvm->lock);
a03825bb 718 if (kvm->created_vcpus) {
658b6eda
MM		 719 ret = -EBUSY;
720 goto out;
721 }
722 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
723 if (!proc) {
724 ret = -ENOMEM;
725 goto out;
726 }
727 if (!copy_from_user(proc, (void __user *)attr->addr,
728 sizeof(*proc))) {
9bb0ec09 729 kvm->arch.model.cpuid = proc->cpuid;
053dd230
DH		 730 lowest_ibc = sclp.ibc >> 16 & 0xfff;
731 unblocked_ibc = sclp.ibc & 0xfff;
732 if (lowest_ibc) {
733 if (proc->ibc > unblocked_ibc)
734 kvm->arch.model.ibc = unblocked_ibc;
735 else if (proc->ibc < lowest_ibc)
736 kvm->arch.model.ibc = lowest_ibc;
737 else
738 kvm->arch.model.ibc = proc->ibc;
739 }
c54f0d6a 740 memcpy(kvm->arch.model.fac_list, proc->fac_list,
658b6eda
MM		 741 S390_ARCH_FAC_LIST_SIZE_BYTE);
742 } else
743 ret = -EFAULT;
744 kfree(proc);
745out:
746 mutex_unlock(&kvm->lock);
747 return ret;
748}
749
15c9705f
DH		 750static int kvm_s390_set_processor_feat(struct kvm *kvm,
751 struct kvm_device_attr *attr)
752{
753 struct kvm_s390_vm_cpu_feat data;
754 int ret = -EBUSY;
755
756 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
757 return -EFAULT;
758 if (!bitmap_subset((unsigned long *) data.feat,
759 kvm_s390_available_cpu_feat,
760 KVM_S390_VM_CPU_FEAT_NR_BITS))
761 return -EINVAL;
762
763 mutex_lock(&kvm->lock);
764 if (!atomic_read(&kvm->online_vcpus)) {
765 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
766 KVM_S390_VM_CPU_FEAT_NR_BITS);
767 ret = 0;
768 }
769 mutex_unlock(&kvm->lock);
770 return ret;
771}
772
0a763c78
DH		 773static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
774 struct kvm_device_attr *attr)
775{
776 /*
777 * Once supported by kernel + hw, we have to store the subfunctions
778 * in kvm->arch and remember that user space configured them.
779 */
780 return -ENXIO;
781}
782
658b6eda
MM		 783static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
784{
785 int ret = -ENXIO;
786
787 switch (attr->attr) {
788 case KVM_S390_VM_CPU_PROCESSOR:
789 ret = kvm_s390_set_processor(kvm, attr);
790 break;
15c9705f
DH		 791 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
792 ret = kvm_s390_set_processor_feat(kvm, attr);
793 break;
0a763c78
DH		 794 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
795 ret = kvm_s390_set_processor_subfunc(kvm, attr);
796 break;
658b6eda
MM		 797 }
798 return ret;
799}
800
801static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
802{
803 struct kvm_s390_vm_cpu_processor *proc;
804 int ret = 0;
805
806 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
807 if (!proc) {
808 ret = -ENOMEM;
809 goto out;
810 }
9bb0ec09 811 proc->cpuid = kvm->arch.model.cpuid;
658b6eda 812 proc->ibc = kvm->arch.model.ibc;
c54f0d6a
DH		 813 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
814 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 815 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
816 ret = -EFAULT;
817 kfree(proc);
818out:
819 return ret;
820}
821
822static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
823{
824 struct kvm_s390_vm_cpu_machine *mach;
825 int ret = 0;
826
827 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
828 if (!mach) {
829 ret = -ENOMEM;
830 goto out;
831 }
832 get_cpu_id((struct cpuid *) &mach->cpuid);
37c5f6c8 833 mach->ibc = sclp.ibc;
c54f0d6a 834 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
981467c9 835 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 836 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
94422ee8 837 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 838 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
839 ret = -EFAULT;
840 kfree(mach);
841out:
842 return ret;
843}
844
15c9705f
DH		 845static int kvm_s390_get_processor_feat(struct kvm *kvm,
846 struct kvm_device_attr *attr)
847{
848 struct kvm_s390_vm_cpu_feat data;
849
850 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
851 KVM_S390_VM_CPU_FEAT_NR_BITS);
852 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
853 return -EFAULT;
854 return 0;
855}
856
857static int kvm_s390_get_machine_feat(struct kvm *kvm,
858 struct kvm_device_attr *attr)
859{
860 struct kvm_s390_vm_cpu_feat data;
861
862 bitmap_copy((unsigned long *) data.feat,
863 kvm_s390_available_cpu_feat,
864 KVM_S390_VM_CPU_FEAT_NR_BITS);
865 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
866 return -EFAULT;
867 return 0;
868}
869
0a763c78
DH		 870static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
871 struct kvm_device_attr *attr)
872{
873 /*
874 * Once we can actually configure subfunctions (kernel + hw support),
875 * we have to check if they were already set by user space, if so copy
876 * them from kvm->arch.
877 */
878 return -ENXIO;
879}
880
881static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
882 struct kvm_device_attr *attr)
883{
884 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
885 sizeof(struct kvm_s390_vm_cpu_subfunc)))
886 return -EFAULT;
887 return 0;
888}
658b6eda
MM		 889static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
890{
891 int ret = -ENXIO;
892
893 switch (attr->attr) {
894 case KVM_S390_VM_CPU_PROCESSOR:
895 ret = kvm_s390_get_processor(kvm, attr);
896 break;
897 case KVM_S390_VM_CPU_MACHINE:
898 ret = kvm_s390_get_machine(kvm, attr);
899 break;
15c9705f
DH		 900 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
901 ret = kvm_s390_get_processor_feat(kvm, attr);
902 break;
903 case KVM_S390_VM_CPU_MACHINE_FEAT:
904 ret = kvm_s390_get_machine_feat(kvm, attr);
905 break;
0a763c78
DH		 906 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
907 ret = kvm_s390_get_processor_subfunc(kvm, attr);
908 break;
909 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
910 ret = kvm_s390_get_machine_subfunc(kvm, attr);
911 break;
658b6eda
MM		 912 }
913 return ret;
914}
915
f2061656
DD		 916static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
917{
918 int ret;
919
920 switch (attr->group) {
4f718eab 921 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 922 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 923 break;
72f25020
JH		 924 case KVM_S390_VM_TOD:
925 ret = kvm_s390_set_tod(kvm, attr);
926 break;
658b6eda
MM		 927 case KVM_S390_VM_CPU_MODEL:
928 ret = kvm_s390_set_cpu_model(kvm, attr);
929 break;
a374e892
TK		 930 case KVM_S390_VM_CRYPTO:
931 ret = kvm_s390_vm_set_crypto(kvm, attr);
932 break;
f2061656
DD		 933 default:
934 ret = -ENXIO;
935 break;
936 }
937
938 return ret;
939}
940
941static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
942{
8c0a7ce6
DD		 943 int ret;
944
945 switch (attr->group) {
946 case KVM_S390_VM_MEM_CTRL:
947 ret = kvm_s390_get_mem_control(kvm, attr);
948 break;
72f25020
JH		 949 case KVM_S390_VM_TOD:
950 ret = kvm_s390_get_tod(kvm, attr);
951 break;
658b6eda
MM		 952 case KVM_S390_VM_CPU_MODEL:
953 ret = kvm_s390_get_cpu_model(kvm, attr);
954 break;
8c0a7ce6
DD		 955 default:
956 ret = -ENXIO;
957 break;
958 }
959
960 return ret;
f2061656
DD		 961}
962
963static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
964{
965 int ret;
966
967 switch (attr->group) {
4f718eab
DD		 968 case KVM_S390_VM_MEM_CTRL:
969 switch (attr->attr) {
970 case KVM_S390_VM_MEM_ENABLE_CMMA:
971 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 972 ret = sclp.has_cmma ? 0 : -ENXIO;
973 break;
8c0a7ce6 974 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 975 ret = 0;
976 break;
977 default:
978 ret = -ENXIO;
979 break;
980 }
981 break;
72f25020
JH		 982 case KVM_S390_VM_TOD:
983 switch (attr->attr) {
984 case KVM_S390_VM_TOD_LOW:
985 case KVM_S390_VM_TOD_HIGH:
986 ret = 0;
987 break;
988 default:
989 ret = -ENXIO;
990 break;
991 }
992 break;
658b6eda
MM		 993 case KVM_S390_VM_CPU_MODEL:
994 switch (attr->attr) {
995 case KVM_S390_VM_CPU_PROCESSOR:
996 case KVM_S390_VM_CPU_MACHINE:
15c9705f
DH		 997 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
998 case KVM_S390_VM_CPU_MACHINE_FEAT:
0a763c78 999 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
658b6eda
MM		 1000 ret = 0;
1001 break;
0a763c78
DH		 1002 /* configuring subfunctions is not supported yet */
1003 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
658b6eda
MM		 1004 default:
1005 ret = -ENXIO;
1006 break;
1007 }
1008 break;
a374e892
TK		 1009 case KVM_S390_VM_CRYPTO:
1010 switch (attr->attr) {
1011 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1012 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1013 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1014 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1015 ret = 0;
1016 break;
1017 default:
1018 ret = -ENXIO;
1019 break;
1020 }
1021 break;
f2061656
DD		 1022 default:
1023 ret = -ENXIO;
1024 break;
1025 }
1026
1027 return ret;
1028}
1029
30ee2a98
JH		 1030static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1031{
1032 uint8_t *keys;
1033 uint64_t hva;
30ee2a98
JH		 1034 int i, r = 0;
1035
1036 if (args->flags != 0)
1037 return -EINVAL;
1038
1039 /* Is this guest using storage keys? */
1040 if (!mm_use_skey(current->mm))
1041 return KVM_S390_GET_SKEYS_NONE;
1042
1043 /* Enforce sane limit on memory allocation */
1044 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1045 return -EINVAL;
1046
1047 keys = kmalloc_array(args->count, sizeof(uint8_t),
1048 GFP_KERNEL | __GFP_NOWARN);
1049 if (!keys)
1050 keys = vmalloc(sizeof(uint8_t) * args->count);
1051 if (!keys)
1052 return -ENOMEM;
1053
d3ed1cee 1054 down_read(&current->mm->mmap_sem);
30ee2a98
JH		 1055 for (i = 0; i < args->count; i++) {
1056 hva = gfn_to_hva(kvm, args->start_gfn + i);
1057 if (kvm_is_error_hva(hva)) {
1058 r = -EFAULT;
d3ed1cee 1059 break;
30ee2a98
JH		 1060 }
1061
154c8c19
DH		 1062 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1063 if (r)
d3ed1cee 1064 break;
30ee2a98 1065 }
d3ed1cee
MS		 1066 up_read(&current->mm->mmap_sem);
1067
1068 if (!r) {
1069 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1070 sizeof(uint8_t) * args->count);
1071 if (r)
1072 r = -EFAULT;
1073 }
30ee2a98 1074
30ee2a98
JH		 1075 kvfree(keys);
1076 return r;
1077}
1078
1079static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1080{
1081 uint8_t *keys;
1082 uint64_t hva;
1083 int i, r = 0;
1084
1085 if (args->flags != 0)
1086 return -EINVAL;
1087
1088 /* Enforce sane limit on memory allocation */
1089 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1090 return -EINVAL;
1091
1092 keys = kmalloc_array(args->count, sizeof(uint8_t),
1093 GFP_KERNEL | __GFP_NOWARN);
1094 if (!keys)
1095 keys = vmalloc(sizeof(uint8_t) * args->count);
1096 if (!keys)
1097 return -ENOMEM;
1098
1099 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1100 sizeof(uint8_t) * args->count);
1101 if (r) {
1102 r = -EFAULT;
1103 goto out;
1104 }
1105
1106 /* Enable storage key handling for the guest */
14d4a425
DD		 1107 r = s390_enable_skey();
1108 if (r)
1109 goto out;
30ee2a98 1110
d3ed1cee 1111 down_read(&current->mm->mmap_sem);
30ee2a98
JH		 1112 for (i = 0; i < args->count; i++) {
1113 hva = gfn_to_hva(kvm, args->start_gfn + i);
1114 if (kvm_is_error_hva(hva)) {
1115 r = -EFAULT;
d3ed1cee 1116 break;
30ee2a98
JH		 1117 }
1118
1119 /* Lowest order bit is reserved */
1120 if (keys[i] & 0x01) {
1121 r = -EINVAL;
d3ed1cee 1122 break;
30ee2a98
JH		 1123 }
1124
fe69eabf 1125 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
30ee2a98 1126 if (r)
d3ed1cee 1127 break;
30ee2a98 1128 }
d3ed1cee 1129 up_read(&current->mm->mmap_sem);
30ee2a98
JH		 1130out:
1131 kvfree(keys);
1132 return r;
1133}
1134
b0c632db
HC		 1135long kvm_arch_vm_ioctl(struct file *filp,
1136 unsigned int ioctl, unsigned long arg)
1137{
1138 struct kvm *kvm = filp->private_data;
1139 void __user *argp = (void __user *)arg;
f2061656 1140 struct kvm_device_attr attr;
b0c632db
HC		 1141 int r;
1142
1143 switch (ioctl) {
ba5c1e9b
CO		 1144 case KVM_S390_INTERRUPT: {
1145 struct kvm_s390_interrupt s390int;
1146
1147 r = -EFAULT;
1148 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1149 break;
1150 r = kvm_s390_inject_vm(kvm, &s390int);
1151 break;
1152 }
d938dc55
CH		 1153 case KVM_ENABLE_CAP: {
1154 struct kvm_enable_cap cap;
1155 r = -EFAULT;
1156 if (copy_from_user(&cap, argp, sizeof(cap)))
1157 break;
1158 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1159 break;
1160 }
84223598
CH		 1161 case KVM_CREATE_IRQCHIP: {
1162 struct kvm_irq_routing_entry routing;
1163
1164 r = -EINVAL;
1165 if (kvm->arch.use_irqchip) {
1166 /* Set up dummy routing. */
1167 memset(&routing, 0, sizeof(routing));
152b2839 1168 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
84223598
CH		 1169 }
1170 break;
1171 }
f2061656
DD		 1172 case KVM_SET_DEVICE_ATTR: {
1173 r = -EFAULT;
1174 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1175 break;
1176 r = kvm_s390_vm_set_attr(kvm, &attr);
1177 break;
1178 }
1179 case KVM_GET_DEVICE_ATTR: {
1180 r = -EFAULT;
1181 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1182 break;
1183 r = kvm_s390_vm_get_attr(kvm, &attr);
1184 break;
1185 }
1186 case KVM_HAS_DEVICE_ATTR: {
1187 r = -EFAULT;
1188 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1189 break;
1190 r = kvm_s390_vm_has_attr(kvm, &attr);
1191 break;
1192 }
30ee2a98
JH		 1193 case KVM_S390_GET_SKEYS: {
1194 struct kvm_s390_skeys args;
1195
1196 r = -EFAULT;
1197 if (copy_from_user(&args, argp,
1198 sizeof(struct kvm_s390_skeys)))
1199 break;
1200 r = kvm_s390_get_skeys(kvm, &args);
1201 break;
1202 }
1203 case KVM_S390_SET_SKEYS: {
1204 struct kvm_s390_skeys args;
1205
1206 r = -EFAULT;
1207 if (copy_from_user(&args, argp,
1208 sizeof(struct kvm_s390_skeys)))
1209 break;
1210 r = kvm_s390_set_skeys(kvm, &args);
1211 break;
1212 }
b0c632db 1213 default:
367e1319 1214 r = -ENOTTY;
b0c632db
HC		 1215 }
1216
1217 return r;
1218}
1219
45c9b47c
TK		 1220static int kvm_s390_query_ap_config(u8 *config)
1221{
1222 u32 fcn_code = 0x04000000UL;
86044c8c 1223 u32 cc = 0;
45c9b47c 1224
86044c8c 1225 memset(config, 0, 128);
45c9b47c
TK		 1226 asm volatile(
1227 "lgr 0,%1\n"
1228 "lgr 2,%2\n"
1229 ".long 0xb2af0000\n" /* PQAP(QCI) */
86044c8c 1230 "0: ipm %0\n"
45c9b47c 1231 "srl %0,28\n"
86044c8c
CB		 1232 "1:\n"
1233 EX_TABLE(0b, 1b)
1234 : "+r" (cc)
45c9b47c
TK		 1235 : "r" (fcn_code), "r" (config)
1236 : "cc", "0", "2", "memory"
1237 );
1238
1239 return cc;
1240}
1241
1242static int kvm_s390_apxa_installed(void)
1243{
1244 u8 config[128];
1245 int cc;
1246
a6aacc3f 1247 if (test_facility(12)) {
45c9b47c
TK		 1248 cc = kvm_s390_query_ap_config(config);
1249
1250 if (cc)
1251 pr_err("PQAP(QCI) failed with cc=%d", cc);
1252 else
1253 return config[0] & 0x40;
1254 }
1255
1256 return 0;
1257}
1258
1259static void kvm_s390_set_crycb_format(struct kvm *kvm)
1260{
1261 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1262
1263 if (kvm_s390_apxa_installed())
1264 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1265 else
1266 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1267}
1268
9bb0ec09 1269static u64 kvm_s390_get_initial_cpuid(void)
9d8d5786 1270{
9bb0ec09
DH		 1271 struct cpuid cpuid;
1272
1273 get_cpu_id(&cpuid);
1274 cpuid.version = 0xff;
1275 return *((u64 *) &cpuid);
9d8d5786
MM		 1276}
1277
c54f0d6a 1278static void kvm_s390_crypto_init(struct kvm *kvm)
5102ee87 1279{
9d8d5786 1280 if (!test_kvm_facility(kvm, 76))
c54f0d6a 1281 return;
5102ee87 1282
c54f0d6a 1283 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
45c9b47c 1284 kvm_s390_set_crycb_format(kvm);
5102ee87 1285
ed6f76b4
TK		 1286 /* Enable AES/DEA protected key functions by default */
1287 kvm->arch.crypto.aes_kw = 1;
1288 kvm->arch.crypto.dea_kw = 1;
1289 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1290 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1291 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1292 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
5102ee87
TK		 1293}
1294
7d43bafc
ED		 1295static void sca_dispose(struct kvm *kvm)
1296{
1297 if (kvm->arch.use_esca)
5e044315 1298 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
7d43bafc
ED		 1299 else
1300 free_page((unsigned long)(kvm->arch.sca));
1301 kvm->arch.sca = NULL;
1302}
1303
e08b9637 1304int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 1305{
76a6dd72 1306 gfp_t alloc_flags = GFP_KERNEL;
9d8d5786 1307 int i, rc;
b0c632db 1308 char debug_name[16];
f6c137ff 1309 static unsigned long sca_offset;
b0c632db 1310
e08b9637
CO		 1311 rc = -EINVAL;
1312#ifdef CONFIG_KVM_S390_UCONTROL
1313 if (type & ~KVM_VM_S390_UCONTROL)
1314 goto out_err;
1315 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1316 goto out_err;
1317#else
1318 if (type)
1319 goto out_err;
1320#endif
1321
b0c632db
HC		 1322 rc = s390_enable_sie();
1323 if (rc)
d89f5eff 1324 goto out_err;
b0c632db 1325
b290411a
CO		 1326 rc = -ENOMEM;
1327
7d0a5e62
JF		 1328 ratelimit_state_init(&kvm->arch.sthyi_limit, 5 * HZ, 500);
1329
7d43bafc 1330 kvm->arch.use_esca = 0; /* start with basic SCA */
76a6dd72
DH		 1331 if (!sclp.has_64bscao)
1332 alloc_flags |= GFP_DMA;
5e044315 1333 rwlock_init(&kvm->arch.sca_lock);
76a6dd72 1334 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
b0c632db 1335 if (!kvm->arch.sca)
d89f5eff 1336 goto out_err;
f6c137ff 1337 spin_lock(&kvm_lock);
c5c2c393 1338 sca_offset += 16;
bc784cce 1339 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
c5c2c393 1340 sca_offset = 0;
bc784cce
ED		 1341 kvm->arch.sca = (struct bsca_block *)
1342 ((char *) kvm->arch.sca + sca_offset);
f6c137ff 1343 spin_unlock(&kvm_lock);
b0c632db
HC		 1344
1345 sprintf(debug_name, "kvm-%u", current->pid);
1346
1cb9cf72 1347 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
b0c632db 1348 if (!kvm->arch.dbf)
40f5b735 1349 goto out_err;
b0c632db 1350
c54f0d6a
DH		 1351 kvm->arch.sie_page2 =
1352 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1353 if (!kvm->arch.sie_page2)
40f5b735 1354 goto out_err;
9d8d5786 1355
fb5bf93f 1356 /* Populate the facility mask initially. */
c54f0d6a 1357 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
94422ee8 1358 S390_ARCH_FAC_LIST_SIZE_BYTE);
9d8d5786
MM		 1359 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1360 if (i < kvm_s390_fac_list_mask_size())
c54f0d6a 1361 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
9d8d5786 1362 else
c54f0d6a 1363 kvm->arch.model.fac_mask[i] = 0UL;
9d8d5786
MM		 1364 }
1365
981467c9 1366 /* Populate the facility list initially. */
c54f0d6a
DH		 1367 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1368 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
981467c9
MM		 1369 S390_ARCH_FAC_LIST_SIZE_BYTE);
1370
95ca2cb5
JF		 1371 set_kvm_facility(kvm->arch.model.fac_mask, 74);
1372 set_kvm_facility(kvm->arch.model.fac_list, 74);
1373
9bb0ec09 1374 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
37c5f6c8 1375 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
9d8d5786 1376
c54f0d6a 1377 kvm_s390_crypto_init(kvm);
5102ee87 1378
ba5c1e9b 1379 spin_lock_init(&kvm->arch.float_int.lock);
6d3da241
JF		 1380 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1381 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
8a242234 1382 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 1383 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 1384
b0c632db 1385 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
78f26131 1386 VM_EVENT(kvm, 3, "vm created with type %lu", type);
b0c632db 1387
e08b9637
CO		 1388 if (type & KVM_VM_S390_UCONTROL) {
1389 kvm->arch.gmap = NULL;
a3a92c31 1390 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
e08b9637 1391 } else {
32e6b236
GH		 1392 if (sclp.hamax == U64_MAX)
1393 kvm->arch.mem_limit = TASK_MAX_SIZE;
1394 else
1395 kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
1396 sclp.hamax + 1);
6ea427bb 1397 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
e08b9637 1398 if (!kvm->arch.gmap)
40f5b735 1399 goto out_err;
2c70fe44 1400 kvm->arch.gmap->private = kvm;
24eb3a82 1401 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 1402 }
fa6b7fe9
CH		 1403
1404 kvm->arch.css_support = 0;
84223598 1405 kvm->arch.use_irqchip = 0;
72f25020 1406 kvm->arch.epoch = 0;
fa6b7fe9 1407
8ad35755 1408 spin_lock_init(&kvm->arch.start_stop_lock);
8335713a 1409 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
8ad35755 1410
d89f5eff 1411 return 0;
40f5b735 1412out_err:
c54f0d6a 1413 free_page((unsigned long)kvm->arch.sie_page2);
598841ca 1414 debug_unregister(kvm->arch.dbf);
7d43bafc 1415 sca_dispose(kvm);
78f26131 1416 KVM_EVENT(3, "creation of vm failed: %d", rc);
d89f5eff 1417 return rc;
b0c632db
HC		 1418}
1419
d329c035
CB		 1420void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1421{
1422 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 1423 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 1424 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 1425 kvm_clear_async_pf_completion_queue(vcpu);
bc784cce 1426 if (!kvm_is_ucontrol(vcpu->kvm))
a6e2f683 1427 sca_del_vcpu(vcpu);
27e0393f
CO		 1428
1429 if (kvm_is_ucontrol(vcpu->kvm))
6ea427bb 1430 gmap_remove(vcpu->arch.gmap);
27e0393f 1431
e6db1d61 1432 if (vcpu->kvm->arch.use_cmma)
b31605c1 1433 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 1434 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 1435
6692cef3 1436 kvm_vcpu_uninit(vcpu);
b110feaf 1437 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 1438}
1439
1440static void kvm_free_vcpus(struct kvm *kvm)
1441{
1442 unsigned int i;
988a2cae 1443 struct kvm_vcpu *vcpu;
d329c035 1444
988a2cae
GN		 1445 kvm_for_each_vcpu(i, vcpu, kvm)
1446 kvm_arch_vcpu_destroy(vcpu);
1447
1448 mutex_lock(&kvm->lock);
1449 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1450 kvm->vcpus[i] = NULL;
1451
1452 atomic_set(&kvm->online_vcpus, 0);
1453 mutex_unlock(&kvm->lock);
d329c035
CB		 1454}
1455
b0c632db
HC		 1456void kvm_arch_destroy_vm(struct kvm *kvm)
1457{
d329c035 1458 kvm_free_vcpus(kvm);
7d43bafc 1459 sca_dispose(kvm);
d329c035 1460 debug_unregister(kvm->arch.dbf);
c54f0d6a 1461 free_page((unsigned long)kvm->arch.sie_page2);
27e0393f 1462 if (!kvm_is_ucontrol(kvm))
6ea427bb 1463 gmap_remove(kvm->arch.gmap);
841b91c5 1464 kvm_s390_destroy_adapters(kvm);
67335e63 1465 kvm_s390_clear_float_irqs(kvm);
8335713a 1466 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
b0c632db
HC		 1467}
1468
1469/* Section: vcpu related */
dafd032a
DD		 1470static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1471{
6ea427bb 1472 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
dafd032a
DD		 1473 if (!vcpu->arch.gmap)
1474 return -ENOMEM;
1475 vcpu->arch.gmap->private = vcpu->kvm;
1476
1477 return 0;
1478}
1479
a6e2f683
ED		 1480static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1481{
5e044315 1482 read_lock(&vcpu->kvm->arch.sca_lock);
7d43bafc
ED		 1483 if (vcpu->kvm->arch.use_esca) {
1484 struct esca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1485
7d43bafc 1486 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
10ce32d5 1487 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc
ED		 1488 } else {
1489 struct bsca_block *sca = vcpu->kvm->arch.sca;
1490
1491 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
10ce32d5 1492 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc 1493 }
5e044315 1494 read_unlock(&vcpu->kvm->arch.sca_lock);
a6e2f683
ED		 1495}
1496
eaa78f34 1497static void sca_add_vcpu(struct kvm_vcpu *vcpu)
a6e2f683 1498{
eaa78f34
DH		 1499 read_lock(&vcpu->kvm->arch.sca_lock);
1500 if (vcpu->kvm->arch.use_esca) {
1501 struct esca_block *sca = vcpu->kvm->arch.sca;
7d43bafc 1502
eaa78f34 1503 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1504 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1505 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
25508824 1506 vcpu->arch.sie_block->ecb2 |= 0x04U;
eaa78f34 1507 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
7d43bafc 1508 } else {
eaa78f34 1509 struct bsca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1510
eaa78f34 1511 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1512 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1513 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
eaa78f34 1514 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
7d43bafc 1515 }
eaa78f34 1516 read_unlock(&vcpu->kvm->arch.sca_lock);
5e044315
ED		 1517}
1518
1519/* Basic SCA to Extended SCA data copy routines */
1520static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1521{
1522 d->sda = s->sda;
1523 d->sigp_ctrl.c = s->sigp_ctrl.c;
1524 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1525}
1526
1527static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1528{
1529 int i;
1530
1531 d->ipte_control = s->ipte_control;
1532 d->mcn[0] = s->mcn;
1533 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1534 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1535}
1536
1537static int sca_switch_to_extended(struct kvm *kvm)
1538{
1539 struct bsca_block *old_sca = kvm->arch.sca;
1540 struct esca_block *new_sca;
1541 struct kvm_vcpu *vcpu;
1542 unsigned int vcpu_idx;
1543 u32 scaol, scaoh;
1544
1545 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1546 if (!new_sca)
1547 return -ENOMEM;
1548
1549 scaoh = (u32)((u64)(new_sca) >> 32);
1550 scaol = (u32)(u64)(new_sca) & ~0x3fU;
1551
1552 kvm_s390_vcpu_block_all(kvm);
1553 write_lock(&kvm->arch.sca_lock);
1554
1555 sca_copy_b_to_e(new_sca, old_sca);
1556
1557 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1558 vcpu->arch.sie_block->scaoh = scaoh;
1559 vcpu->arch.sie_block->scaol = scaol;
1560 vcpu->arch.sie_block->ecb2 |= 0x04U;
1561 }
1562 kvm->arch.sca = new_sca;
1563 kvm->arch.use_esca = 1;
1564
1565 write_unlock(&kvm->arch.sca_lock);
1566 kvm_s390_vcpu_unblock_all(kvm);
1567
1568 free_page((unsigned long)old_sca);
1569
8335713a
CB		 1570 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1571 old_sca, kvm->arch.sca);
5e044315 1572 return 0;
a6e2f683
ED		 1573}
1574
1575static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1576{
5e044315
ED		 1577 int rc;
1578
1579 if (id < KVM_S390_BSCA_CPU_SLOTS)
1580 return true;
76a6dd72 1581 if (!sclp.has_esca || !sclp.has_64bscao)
5e044315
ED		 1582 return false;
1583
1584 mutex_lock(&kvm->lock);
1585 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1586 mutex_unlock(&kvm->lock);
1587
1588 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
a6e2f683
ED		 1589}
1590
b0c632db
HC		 1591int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1592{
3c038e6b
DD		 1593 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1594 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 1595 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1596 KVM_SYNC_GPRS |
9eed0735 1597 KVM_SYNC_ACRS |
b028ee3e
DH		 1598 KVM_SYNC_CRS |
1599 KVM_SYNC_ARCH0 |
1600 KVM_SYNC_PFAULT;
c6e5f166
FZ		 1601 if (test_kvm_facility(vcpu->kvm, 64))
1602 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
f6aa6dc4
DH		 1603 /* fprs can be synchronized via vrs, even if the guest has no vx. With
1604 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1605 */
1606 if (MACHINE_HAS_VX)
68c55750 1607 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
6fd8e67d
DH		 1608 else
1609 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
dafd032a
DD		 1610
1611 if (kvm_is_ucontrol(vcpu->kvm))
1612 return __kvm_ucontrol_vcpu_init(vcpu);
1613
b0c632db
HC		 1614 return 0;
1615}
1616
db0758b2
DH		 1617/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1618static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1619{
1620 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
9c23a131 1621 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2 1622 vcpu->arch.cputm_start = get_tod_clock_fast();
9c23a131 1623 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1624}
1625
1626/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1627static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1628{
1629 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
9c23a131 1630 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1631 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1632 vcpu->arch.cputm_start = 0;
9c23a131 1633 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1634}
1635
1636/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1637static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1638{
1639 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
1640 vcpu->arch.cputm_enabled = true;
1641 __start_cpu_timer_accounting(vcpu);
1642}
1643
1644/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1645static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1646{
1647 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
1648 __stop_cpu_timer_accounting(vcpu);
1649 vcpu->arch.cputm_enabled = false;
1650}
1651
1652static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1653{
1654 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1655 __enable_cpu_timer_accounting(vcpu);
1656 preempt_enable();
1657}
1658
1659static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1660{
1661 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1662 __disable_cpu_timer_accounting(vcpu);
1663 preempt_enable();
1664}
1665
4287f247
DH		 1666/* set the cpu timer - may only be called from the VCPU thread itself */
1667void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
1668{
db0758b2 1669 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
9c23a131 1670 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1671 if (vcpu->arch.cputm_enabled)
1672 vcpu->arch.cputm_start = get_tod_clock_fast();
4287f247 1673 vcpu->arch.sie_block->cputm = cputm;
9c23a131 1674 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2 1675 preempt_enable();
4287f247
DH		 1676}
1677
db0758b2 1678/* update and get the cpu timer - can also be called from other VCPU threads */
4287f247
DH		 1679__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
1680{
9c23a131 1681 unsigned int seq;
db0758b2 1682 __u64 value;
db0758b2
DH		 1683
1684 if (unlikely(!vcpu->arch.cputm_enabled))
1685 return vcpu->arch.sie_block->cputm;
1686
9c23a131
DH		 1687 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1688 do {
1689 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
1690 /*
1691 * If the writer would ever execute a read in the critical
1692 * section, e.g. in irq context, we have a deadlock.
1693 */
1694 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
1695 value = vcpu->arch.sie_block->cputm;
1696 /* if cputm_start is 0, accounting is being started/stopped */
1697 if (likely(vcpu->arch.cputm_start))
1698 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1699 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
1700 preempt_enable();
db0758b2 1701 return value;
4287f247
DH		 1702}
1703
b0c632db
HC		 1704void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1705{
9977e886 1706 /* Save host register state */
d0164ee2 1707 save_fpu_regs();
9abc2a08
DH		 1708 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
1709 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
9977e886 1710
6fd8e67d
DH		 1711 if (MACHINE_HAS_VX)
1712 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
1713 else
1714 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
9abc2a08 1715 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
9977e886 1716 if (test_fp_ctl(current->thread.fpu.fpc))
96b2d7a8 1717 /* User space provided an invalid FPC, let's clear it */
9977e886
HB		 1718 current->thread.fpu.fpc = 0;
1719
1720 save_access_regs(vcpu->arch.host_acrs);
59674c1a 1721 restore_access_regs(vcpu->run->s.regs.acrs);
480e5926 1722 gmap_enable(vcpu->arch.gmap);
805de8f4 1723 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
5ebda316 1724 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1725 __start_cpu_timer_accounting(vcpu);
01a745ac 1726 vcpu->cpu = cpu;
b0c632db
HC		 1727}
1728
1729void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1730{
01a745ac 1731 vcpu->cpu = -1;
5ebda316 1732 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1733 __stop_cpu_timer_accounting(vcpu);
805de8f4 1734 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
480e5926 1735 gmap_disable(vcpu->arch.gmap);
9977e886 1736
9abc2a08 1737 /* Save guest register state */
d0164ee2 1738 save_fpu_regs();
9abc2a08 1739 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
9977e886 1740
9abc2a08
DH		 1741 /* Restore host register state */
1742 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
1743 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
9977e886
HB		 1744
1745 save_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 1746 restore_access_regs(vcpu->arch.host_acrs);
1747}
1748
1749static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1750{
1751 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1752 vcpu->arch.sie_block->gpsw.mask = 0UL;
1753 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 1754 kvm_s390_set_prefix(vcpu, 0);
4287f247 1755 kvm_s390_set_cpu_timer(vcpu, 0);
b0c632db
HC		 1756 vcpu->arch.sie_block->ckc = 0UL;
1757 vcpu->arch.sie_block->todpr = 0;
1758 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1759 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1760 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
9abc2a08
DH		 1761 /* make sure the new fpc will be lazily loaded */
1762 save_fpu_regs();
1763 current->thread.fpu.fpc = 0;
b0c632db 1764 vcpu->arch.sie_block->gbea = 1;
672550fb 1765 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 1766 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1767 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 1768 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1769 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 1770 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 1771}
1772
31928aa5 1773void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 1774{
72f25020 1775 mutex_lock(&vcpu->kvm->lock);
fdf03650 1776 preempt_disable();
72f25020 1777 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
fdf03650 1778 preempt_enable();
72f25020 1779 mutex_unlock(&vcpu->kvm->lock);
25508824 1780 if (!kvm_is_ucontrol(vcpu->kvm)) {
dafd032a 1781 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
eaa78f34 1782 sca_add_vcpu(vcpu);
25508824
DH		 1783 }
1784
42897d86
MT		 1785}
1786
5102ee87
TK		 1787static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1788{
9d8d5786 1789 if (!test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 1790 return;
1791
a374e892
TK		 1792 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1793
1794 if (vcpu->kvm->arch.crypto.aes_kw)
1795 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1796 if (vcpu->kvm->arch.crypto.dea_kw)
1797 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1798
5102ee87
TK		 1799 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1800}
1801
b31605c1
DD		 1802void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1803{
1804 free_page(vcpu->arch.sie_block->cbrlo);
1805 vcpu->arch.sie_block->cbrlo = 0;
1806}
1807
1808int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1809{
1810 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1811 if (!vcpu->arch.sie_block->cbrlo)
1812 return -ENOMEM;
1813
1814 vcpu->arch.sie_block->ecb2 |= 0x80;
1815 vcpu->arch.sie_block->ecb2 &= ~0x08;
1816 return 0;
1817}
1818
91520f1a
MM		 1819static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1820{
1821 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1822
91520f1a 1823 vcpu->arch.sie_block->ibc = model->ibc;
80bc79dc 1824 if (test_kvm_facility(vcpu->kvm, 7))
c54f0d6a 1825 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
91520f1a
MM		 1826}
1827
b0c632db
HC		 1828int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1829{
b31605c1 1830 int rc = 0;
b31288fa 1831
9e6dabef
CH		 1832 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1833 CPUSTAT_SM |
a4a4f191
GH		 1834 CPUSTAT_STOPPED);
1835
53df84f8 1836 if (test_kvm_facility(vcpu->kvm, 78))
805de8f4 1837 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
53df84f8 1838 else if (test_kvm_facility(vcpu->kvm, 8))
805de8f4 1839 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
a4a4f191 1840
91520f1a
MM		 1841 kvm_s390_vcpu_setup_model(vcpu);
1842
bdab09f3
DH		 1843 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
1844 if (MACHINE_HAS_ESOP)
1845 vcpu->arch.sie_block->ecb |= 0x02;
bd50e8ec
DH		 1846 if (test_kvm_facility(vcpu->kvm, 9))
1847 vcpu->arch.sie_block->ecb |= 0x04;
f597d24e 1848 if (test_kvm_facility(vcpu->kvm, 73))
7feb6bb8
MM		 1849 vcpu->arch.sie_block->ecb |= 0x10;
1850
873b425e 1851 if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
d6af0b49 1852 vcpu->arch.sie_block->ecb2 |= 0x08;
48ee7d3a
DH		 1853 vcpu->arch.sie_block->eca = 0x1002000U;
1854 if (sclp.has_cei)
1855 vcpu->arch.sie_block->eca |= 0x80000000U;
11ad65b7
DH		 1856 if (sclp.has_ib)
1857 vcpu->arch.sie_block->eca |= 0x40000000U;
37c5f6c8 1858 if (sclp.has_siif)
217a4406 1859 vcpu->arch.sie_block->eca |= 1;
37c5f6c8 1860 if (sclp.has_sigpif)
ea5f4969 1861 vcpu->arch.sie_block->eca |= 0x10000000U;
c6e5f166
FZ		 1862 if (test_kvm_facility(vcpu->kvm, 64))
1863 vcpu->arch.sie_block->ecb3 |= 0x01;
18280d8b 1864 if (test_kvm_facility(vcpu->kvm, 129)) {
13211ea7
EF		 1865 vcpu->arch.sie_block->eca |= 0x00020000;
1866 vcpu->arch.sie_block->ecd |= 0x20000000;
1867 }
c6e5f166 1868 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
492d8642 1869 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
95ca2cb5
JF		 1870 if (test_kvm_facility(vcpu->kvm, 74))
1871 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
5a5e6536 1872
e6db1d61 1873 if (vcpu->kvm->arch.use_cmma) {
b31605c1
DD		 1874 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1875 if (rc)
1876 return rc;
b31288fa 1877 }
0ac96caf 1878 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 1879 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 1880
5102ee87
TK		 1881 kvm_s390_vcpu_crypto_setup(vcpu);
1882
b31605c1 1883 return rc;
b0c632db
HC		 1884}
1885
1886struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1887 unsigned int id)
1888{
4d47555a 1889 struct kvm_vcpu *vcpu;
7feb6bb8 1890 struct sie_page *sie_page;
4d47555a
CO		 1891 int rc = -EINVAL;
1892
4215825e 1893 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
4d47555a
CO		 1894 goto out;
1895
1896 rc = -ENOMEM;
b0c632db 1897
b110feaf 1898 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 1899 if (!vcpu)
4d47555a 1900 goto out;
b0c632db 1901
7feb6bb8
MM		 1902 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1903 if (!sie_page)
b0c632db
HC		 1904 goto out_free_cpu;
1905
7feb6bb8
MM		 1906 vcpu->arch.sie_block = &sie_page->sie_block;
1907 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1908
efed1104
DH		 1909 /* the real guest size will always be smaller than msl */
1910 vcpu->arch.sie_block->mso = 0;
1911 vcpu->arch.sie_block->msl = sclp.hamax;
1912
b0c632db 1913 vcpu->arch.sie_block->icpua = id;
ba5c1e9b 1914 spin_lock_init(&vcpu->arch.local_int.lock);
ba5c1e9b 1915 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 1916 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 1917 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
9c23a131 1918 seqcount_init(&vcpu->arch.cputm_seqcount);
ba5c1e9b 1919
b0c632db
HC		 1920 rc = kvm_vcpu_init(vcpu, kvm, id);
1921 if (rc)
9abc2a08 1922 goto out_free_sie_block;
8335713a 1923 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
b0c632db 1924 vcpu->arch.sie_block);
ade38c31 1925 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 1926
b0c632db 1927 return vcpu;
7b06bf2f
WY		 1928out_free_sie_block:
1929 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 1930out_free_cpu:
b110feaf 1931 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 1932out:
b0c632db
HC		 1933 return ERR_PTR(rc);
1934}
1935
b0c632db
HC		 1936int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1937{
9a022067 1938 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 1939}
1940
27406cd5 1941void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
49b99e1e 1942{
805de8f4 1943 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
61a6df54 1944 exit_sie(vcpu);
49b99e1e
CB		 1945}
1946
27406cd5 1947void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
49b99e1e 1948{
805de8f4 1949 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
49b99e1e
CB		 1950}
1951
8e236546
CB		 1952static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1953{
805de8f4 1954 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
61a6df54 1955 exit_sie(vcpu);
8e236546
CB		 1956}
1957
1958static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1959{
9bf9fde2 1960 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
8e236546
CB		 1961}
1962
49b99e1e
CB		 1963/*
1964 * Kick a guest cpu out of SIE and wait until SIE is not running.
1965 * If the CPU is not running (e.g. waiting as idle) the function will
1966 * return immediately. */
1967void exit_sie(struct kvm_vcpu *vcpu)
1968{
805de8f4 1969 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
49b99e1e
CB		 1970 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1971 cpu_relax();
1972}
1973
8e236546
CB		 1974/* Kick a guest cpu out of SIE to process a request synchronously */
1975void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
49b99e1e 1976{
8e236546
CB		 1977 kvm_make_request(req, vcpu);
1978 kvm_s390_vcpu_request(vcpu);
49b99e1e
CB		 1979}
1980
414d3b07
MS		 1981static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
1982 unsigned long end)
2c70fe44 1983{
2c70fe44
CB		 1984 struct kvm *kvm = gmap->private;
1985 struct kvm_vcpu *vcpu;
414d3b07
MS		 1986 unsigned long prefix;
1987 int i;
2c70fe44 1988
414d3b07
MS		 1989 if (start >= 1UL << 31)
1990 /* We are only interested in prefix pages */
1991 return;
2c70fe44
CB		 1992 kvm_for_each_vcpu(i, vcpu, kvm) {
1993 /* match against both prefix pages */
414d3b07
MS		 1994 prefix = kvm_s390_get_prefix(vcpu);
1995 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
1996 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
1997 start, end);
8e236546 1998 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44
CB		 1999 }
2000 }
2001}
2002
b6d33834
CD		 2003int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2004{
2005 /* kvm common code refers to this, but never calls it */
2006 BUG();
2007 return 0;
2008}
2009
14eebd91
CO		 2010static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2011 struct kvm_one_reg *reg)
2012{
2013 int r = -EINVAL;
2014
2015 switch (reg->id) {
29b7c71b
CO		 2016 case KVM_REG_S390_TODPR:
2017 r = put_user(vcpu->arch.sie_block->todpr,
2018 (u32 __user *)reg->addr);
2019 break;
2020 case KVM_REG_S390_EPOCHDIFF:
2021 r = put_user(vcpu->arch.sie_block->epoch,
2022 (u64 __user *)reg->addr);
2023 break;
46a6dd1c 2024 case KVM_REG_S390_CPU_TIMER:
4287f247 2025 r = put_user(kvm_s390_get_cpu_timer(vcpu),
46a6dd1c
J		 2026 (u64 __user *)reg->addr);
2027 break;
2028 case KVM_REG_S390_CLOCK_COMP:
2029 r = put_user(vcpu->arch.sie_block->ckc,
2030 (u64 __user *)reg->addr);
2031 break;
536336c2
DD		 2032 case KVM_REG_S390_PFTOKEN:
2033 r = put_user(vcpu->arch.pfault_token,
2034 (u64 __user *)reg->addr);
2035 break;
2036 case KVM_REG_S390_PFCOMPARE:
2037 r = put_user(vcpu->arch.pfault_compare,
2038 (u64 __user *)reg->addr);
2039 break;
2040 case KVM_REG_S390_PFSELECT:
2041 r = put_user(vcpu->arch.pfault_select,
2042 (u64 __user *)reg->addr);
2043 break;
672550fb
CB		 2044 case KVM_REG_S390_PP:
2045 r = put_user(vcpu->arch.sie_block->pp,
2046 (u64 __user *)reg->addr);
2047 break;
afa45ff5
CB		 2048 case KVM_REG_S390_GBEA:
2049 r = put_user(vcpu->arch.sie_block->gbea,
2050 (u64 __user *)reg->addr);
2051 break;
14eebd91
CO		 2052 default:
2053 break;
2054 }
2055
2056 return r;
2057}
2058
2059static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2060 struct kvm_one_reg *reg)
2061{
2062 int r = -EINVAL;
4287f247 2063 __u64 val;
14eebd91
CO		 2064
2065 switch (reg->id) {
29b7c71b
CO		 2066 case KVM_REG_S390_TODPR:
2067 r = get_user(vcpu->arch.sie_block->todpr,
2068 (u32 __user *)reg->addr);
2069 break;
2070 case KVM_REG_S390_EPOCHDIFF:
2071 r = get_user(vcpu->arch.sie_block->epoch,
2072 (u64 __user *)reg->addr);
2073 break;
46a6dd1c 2074 case KVM_REG_S390_CPU_TIMER:
4287f247
DH		 2075 r = get_user(val, (u64 __user *)reg->addr);
2076 if (!r)
2077 kvm_s390_set_cpu_timer(vcpu, val);
46a6dd1c
J		 2078 break;
2079 case KVM_REG_S390_CLOCK_COMP:
2080 r = get_user(vcpu->arch.sie_block->ckc,
2081 (u64 __user *)reg->addr);
2082 break;
536336c2
DD		 2083 case KVM_REG_S390_PFTOKEN:
2084 r = get_user(vcpu->arch.pfault_token,
2085 (u64 __user *)reg->addr);
9fbd8082
DH		 2086 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2087 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 2088 break;
2089 case KVM_REG_S390_PFCOMPARE:
2090 r = get_user(vcpu->arch.pfault_compare,
2091 (u64 __user *)reg->addr);
2092 break;
2093 case KVM_REG_S390_PFSELECT:
2094 r = get_user(vcpu->arch.pfault_select,
2095 (u64 __user *)reg->addr);
2096 break;
672550fb
CB		 2097 case KVM_REG_S390_PP:
2098 r = get_user(vcpu->arch.sie_block->pp,
2099 (u64 __user *)reg->addr);
2100 break;
afa45ff5
CB		 2101 case KVM_REG_S390_GBEA:
2102 r = get_user(vcpu->arch.sie_block->gbea,
2103 (u64 __user *)reg->addr);
2104 break;
14eebd91
CO		 2105 default:
2106 break;
2107 }
2108
2109 return r;
2110}
b6d33834 2111
b0c632db
HC		 2112static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2113{
b0c632db 2114 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 2115 return 0;
2116}
2117
2118int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2119{
5a32c1af 2120 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 2121 return 0;
2122}
2123
2124int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2125{
5a32c1af 2126 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 2127 return 0;
2128}
2129
2130int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2131 struct kvm_sregs *sregs)
2132{
59674c1a 2133 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 2134 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
59674c1a 2135 restore_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 2136 return 0;
2137}
2138
2139int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2140 struct kvm_sregs *sregs)
2141{
59674c1a 2142 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 2143 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 2144 return 0;
2145}
2146
2147int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2148{
9abc2a08
DH		 2149 /* make sure the new values will be lazily loaded */
2150 save_fpu_regs();
4725c860
MS		 2151 if (test_fp_ctl(fpu->fpc))
2152 return -EINVAL;
9abc2a08
DH		 2153 current->thread.fpu.fpc = fpu->fpc;
2154 if (MACHINE_HAS_VX)
2155 convert_fp_to_vx(current->thread.fpu.vxrs, (freg_t *)fpu->fprs);
2156 else
2157 memcpy(current->thread.fpu.fprs, &fpu->fprs, sizeof(fpu->fprs));
b0c632db
HC		 2158 return 0;
2159}
2160
2161int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2162{
9abc2a08
DH		 2163 /* make sure we have the latest values */
2164 save_fpu_regs();
2165 if (MACHINE_HAS_VX)
2166 convert_vx_to_fp((freg_t *)fpu->fprs, current->thread.fpu.vxrs);
2167 else
2168 memcpy(fpu->fprs, current->thread.fpu.fprs, sizeof(fpu->fprs));
2169 fpu->fpc = current->thread.fpu.fpc;
b0c632db
HC		 2170 return 0;
2171}
2172
2173static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2174{
2175 int rc = 0;
2176
7a42fdc2 2177 if (!is_vcpu_stopped(vcpu))
b0c632db 2178 rc = -EBUSY;
d7b0b5eb
CO		 2179 else {
2180 vcpu->run->psw_mask = psw.mask;
2181 vcpu->run->psw_addr = psw.addr;
2182 }
b0c632db
HC		 2183 return rc;
2184}
2185
2186int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2187 struct kvm_translation *tr)
2188{
2189 return -EINVAL; /* not implemented yet */
2190}
2191
27291e21
DH		 2192#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2193 KVM_GUESTDBG_USE_HW_BP | \
2194 KVM_GUESTDBG_ENABLE)
2195
d0bfb940
JK		 2196int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2197 struct kvm_guest_debug *dbg)
b0c632db 2198{
27291e21
DH		 2199 int rc = 0;
2200
2201 vcpu->guest_debug = 0;
2202 kvm_s390_clear_bp_data(vcpu);
2203
2de3bfc2 2204 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21 2205 return -EINVAL;
89b5b4de
DH		 2206 if (!sclp.has_gpere)
2207 return -EINVAL;
27291e21
DH		 2208
2209 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2210 vcpu->guest_debug = dbg->control;
2211 /* enforce guest PER */
805de8f4 2212 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2213
2214 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2215 rc = kvm_s390_import_bp_data(vcpu, dbg);
2216 } else {
805de8f4 2217 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2218 vcpu->arch.guestdbg.last_bp = 0;
2219 }
2220
2221 if (rc) {
2222 vcpu->guest_debug = 0;
2223 kvm_s390_clear_bp_data(vcpu);
805de8f4 2224 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2225 }
2226
2227 return rc;
b0c632db
HC		 2228}
2229
62d9f0db
MT		 2230int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2231 struct kvm_mp_state *mp_state)
2232{
6352e4d2
DH		 2233 /* CHECK_STOP and LOAD are not supported yet */
2234 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2235 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 2236}
2237
2238int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2239 struct kvm_mp_state *mp_state)
2240{
6352e4d2
DH		 2241 int rc = 0;
2242
2243 /* user space knows about this interface - let it control the state */
2244 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2245
2246 switch (mp_state->mp_state) {
2247 case KVM_MP_STATE_STOPPED:
2248 kvm_s390_vcpu_stop(vcpu);
2249 break;
2250 case KVM_MP_STATE_OPERATING:
2251 kvm_s390_vcpu_start(vcpu);
2252 break;
2253 case KVM_MP_STATE_LOAD:
2254 case KVM_MP_STATE_CHECK_STOP:
2255 /* fall through - CHECK_STOP and LOAD are not supported yet */
2256 default:
2257 rc = -ENXIO;
2258 }
2259
2260 return rc;
62d9f0db
MT		 2261}
2262
8ad35755
DH		 2263static bool ibs_enabled(struct kvm_vcpu *vcpu)
2264{
2265 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2266}
2267
2c70fe44
CB		 2268static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2269{
8ad35755 2270retry:
8e236546 2271 kvm_s390_vcpu_request_handled(vcpu);
586b7ccd
CB		 2272 if (!vcpu->requests)
2273 return 0;
2c70fe44
CB		 2274 /*
2275 * We use MMU_RELOAD just to re-arm the ipte notifier for the
b2d73b2a 2276 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2c70fe44
CB		 2277 * This ensures that the ipte instruction for this request has
2278 * already finished. We might race against a second unmapper that
2279 * wants to set the blocking bit. Lets just retry the request loop.
2280 */
8ad35755 2281 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44 2282 int rc;
b2d73b2a
MS		 2283 rc = gmap_mprotect_notify(vcpu->arch.gmap,
2284 kvm_s390_get_prefix(vcpu),
2285 PAGE_SIZE * 2, PROT_WRITE);
2c70fe44
CB		 2286 if (rc)
2287 return rc;
8ad35755 2288 goto retry;
2c70fe44 2289 }
8ad35755 2290
d3d692c8
DH		 2291 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2292 vcpu->arch.sie_block->ihcpu = 0xffff;
2293 goto retry;
2294 }
2295
8ad35755
DH		 2296 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2297 if (!ibs_enabled(vcpu)) {
2298 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
805de8f4 2299 atomic_or(CPUSTAT_IBS,
8ad35755
DH		 2300 &vcpu->arch.sie_block->cpuflags);
2301 }
2302 goto retry;
2c70fe44 2303 }
8ad35755
DH		 2304
2305 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2306 if (ibs_enabled(vcpu)) {
2307 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
805de8f4 2308 atomic_andnot(CPUSTAT_IBS,
8ad35755
DH		 2309 &vcpu->arch.sie_block->cpuflags);
2310 }
2311 goto retry;
2312 }
2313
0759d068
DH		 2314 /* nothing to do, just clear the request */
2315 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2316
2c70fe44
CB		 2317 return 0;
2318}
2319
25ed1675
DH		 2320void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2321{
2322 struct kvm_vcpu *vcpu;
2323 int i;
2324
2325 mutex_lock(&kvm->lock);
2326 preempt_disable();
2327 kvm->arch.epoch = tod - get_tod_clock();
2328 kvm_s390_vcpu_block_all(kvm);
2329 kvm_for_each_vcpu(i, vcpu, kvm)
2330 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2331 kvm_s390_vcpu_unblock_all(kvm);
2332 preempt_enable();
2333 mutex_unlock(&kvm->lock);
2334}
2335
fa576c58
TH		 2336/**
2337 * kvm_arch_fault_in_page - fault-in guest page if necessary
2338 * @vcpu: The corresponding virtual cpu
2339 * @gpa: Guest physical address
2340 * @writable: Whether the page should be writable or not
2341 *
2342 * Make sure that a guest page has been faulted-in on the host.
2343 *
2344 * Return: Zero on success, negative error code otherwise.
2345 */
2346long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 2347{
527e30b4
MS		 2348 return gmap_fault(vcpu->arch.gmap, gpa,
2349 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 2350}
2351
3c038e6b
DD		 2352static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2353 unsigned long token)
2354{
2355 struct kvm_s390_interrupt inti;
383d0b05 2356 struct kvm_s390_irq irq;
3c038e6b
DD		 2357
2358 if (start_token) {
383d0b05
JF		 2359 irq.u.ext.ext_params2 = token;
2360 irq.type = KVM_S390_INT_PFAULT_INIT;
2361 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 2362 } else {
2363 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 2364 inti.parm64 = token;
3c038e6b
DD		 2365 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2366 }
2367}
2368
2369void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2370 struct kvm_async_pf *work)
2371{
2372 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2373 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2374}
2375
2376void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2377 struct kvm_async_pf *work)
2378{
2379 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2380 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2381}
2382
2383void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2384 struct kvm_async_pf *work)
2385{
2386 /* s390 will always inject the page directly */
2387}
2388
2389bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2390{
2391 /*
2392 * s390 will always inject the page directly,
2393 * but we still want check_async_completion to cleanup
2394 */
2395 return true;
2396}
2397
2398static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2399{
2400 hva_t hva;
2401 struct kvm_arch_async_pf arch;
2402 int rc;
2403
2404 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2405 return 0;
2406 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2407 vcpu->arch.pfault_compare)
2408 return 0;
2409 if (psw_extint_disabled(vcpu))
2410 return 0;
9a022067 2411 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 2412 return 0;
2413 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2414 return 0;
2415 if (!vcpu->arch.gmap->pfault_enabled)
2416 return 0;
2417
81480cc1
HC		 2418 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2419 hva += current->thread.gmap_addr & ~PAGE_MASK;
2420 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 2421 return 0;
2422
2423 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2424 return rc;
2425}
2426
3fb4c40f 2427static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 2428{
3fb4c40f 2429 int rc, cpuflags;
e168bf8d 2430
3c038e6b
DD		 2431 /*
2432 * On s390 notifications for arriving pages will be delivered directly
2433 * to the guest but the house keeping for completed pfaults is
2434 * handled outside the worker.
2435 */
2436 kvm_check_async_pf_completion(vcpu);
2437
7ec7c8c7
CB		 2438 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2439 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
b0c632db
HC		 2440
2441 if (need_resched())
2442 schedule();
2443
d3a73acb 2444 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 2445 s390_handle_mcck();
2446
79395031
JF		 2447 if (!kvm_is_ucontrol(vcpu->kvm)) {
2448 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2449 if (rc)
2450 return rc;
2451 }
0ff31867 2452
2c70fe44
CB		 2453 rc = kvm_s390_handle_requests(vcpu);
2454 if (rc)
2455 return rc;
2456
27291e21
DH		 2457 if (guestdbg_enabled(vcpu)) {
2458 kvm_s390_backup_guest_per_regs(vcpu);
2459 kvm_s390_patch_guest_per_regs(vcpu);
2460 }
2461
b0c632db 2462 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 2463 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2464 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2465 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 2466
3fb4c40f
TH		 2467 return 0;
2468}
2469
492d8642
TH		 2470static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2471{
56317920
DH		 2472 struct kvm_s390_pgm_info pgm_info = {
2473 .code = PGM_ADDRESSING,
2474 };
2475 u8 opcode, ilen;
492d8642
TH		 2476 int rc;
2477
2478 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2479 trace_kvm_s390_sie_fault(vcpu);
2480
2481 /*
2482 * We want to inject an addressing exception, which is defined as a
2483 * suppressing or terminating exception. However, since we came here
2484 * by a DAT access exception, the PSW still points to the faulting
2485 * instruction since DAT exceptions are nullifying. So we've got
2486 * to look up the current opcode to get the length of the instruction
2487 * to be able to forward the PSW.
2488 */
65977322 2489 rc = read_guest_instr(vcpu, &opcode, 1);
56317920 2490 ilen = insn_length(opcode);
9b0d721a
DH		 2491 if (rc < 0) {
2492 return rc;
2493 } else if (rc) {
2494 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2495 * Forward by arbitrary ilc, injection will take care of
2496 * nullification if necessary.
2497 */
2498 pgm_info = vcpu->arch.pgm;
2499 ilen = 4;
2500 }
56317920
DH		 2501 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2502 kvm_s390_forward_psw(vcpu, ilen);
2503 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
492d8642
TH		 2504}
2505
3fb4c40f
TH		 2506static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2507{
2b29a9fd
DD		 2508 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2509 vcpu->arch.sie_block->icptcode);
2510 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2511
27291e21
DH		 2512 if (guestdbg_enabled(vcpu))
2513 kvm_s390_restore_guest_per_regs(vcpu);
2514
7ec7c8c7
CB		 2515 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2516 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
71f116bf
DH		 2517
2518 if (vcpu->arch.sie_block->icptcode > 0) {
2519 int rc = kvm_handle_sie_intercept(vcpu);
2520
2521 if (rc != -EOPNOTSUPP)
2522 return rc;
2523 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2524 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2525 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2526 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2527 return -EREMOTE;
2528 } else if (exit_reason != -EFAULT) {
2529 vcpu->stat.exit_null++;
2530 return 0;
210b1607
TH		 2531 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2532 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2533 vcpu->run->s390_ucontrol.trans_exc_code =
2534 current->thread.gmap_addr;
2535 vcpu->run->s390_ucontrol.pgm_code = 0x10;
71f116bf 2536 return -EREMOTE;
24eb3a82 2537 } else if (current->thread.gmap_pfault) {
3c038e6b 2538 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 2539 current->thread.gmap_pfault = 0;
71f116bf
DH		 2540 if (kvm_arch_setup_async_pf(vcpu))
2541 return 0;
2542 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
a76ccff6 2543 }
71f116bf 2544 return vcpu_post_run_fault_in_sie(vcpu);
3fb4c40f
TH		 2545}
2546
2547static int __vcpu_run(struct kvm_vcpu *vcpu)
2548{
2549 int rc, exit_reason;
2550
800c1065
TH		 2551 /*
2552 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2553 * ning the guest), so that memslots (and other stuff) are protected
2554 */
2555 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2556
a76ccff6
TH		 2557 do {
2558 rc = vcpu_pre_run(vcpu);
2559 if (rc)
2560 break;
3fb4c40f 2561
800c1065 2562 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 2563 /*
2564 * As PF_VCPU will be used in fault handler, between
2565 * guest_enter and guest_exit should be no uaccess.
2566 */
0097d12e
CB		 2567 local_irq_disable();
2568 __kvm_guest_enter();
db0758b2 2569 __disable_cpu_timer_accounting(vcpu);
0097d12e 2570 local_irq_enable();
a76ccff6
TH		 2571 exit_reason = sie64a(vcpu->arch.sie_block,
2572 vcpu->run->s.regs.gprs);
0097d12e 2573 local_irq_disable();
db0758b2 2574 __enable_cpu_timer_accounting(vcpu);
0097d12e
CB		 2575 __kvm_guest_exit();
2576 local_irq_enable();
800c1065 2577 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 2578
2579 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 2580 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 2581
800c1065 2582 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 2583 return rc;
b0c632db
HC		 2584}
2585
b028ee3e
DH		 2586static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2587{
2588 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2589 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2590 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2591 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2592 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2593 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 2594 /* some control register changes require a tlb flush */
2595 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 2596 }
2597 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4287f247 2598 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
b028ee3e
DH		 2599 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2600 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2601 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2602 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2603 }
2604 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2605 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2606 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2607 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 2608 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2609 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e
DH		 2610 }
2611 kvm_run->kvm_dirty_regs = 0;
2612}
2613
2614static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2615{
2616 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2617 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2618 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2619 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4287f247 2620 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
b028ee3e
DH		 2621 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2622 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2623 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2624 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2625 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2626 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2627 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2628}
2629
b0c632db
HC		 2630int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2631{
8f2abe6a 2632 int rc;
b0c632db
HC		 2633 sigset_t sigsaved;
2634
27291e21
DH		 2635 if (guestdbg_exit_pending(vcpu)) {
2636 kvm_s390_prepare_debug_exit(vcpu);
2637 return 0;
2638 }
2639
b0c632db
HC		 2640 if (vcpu->sigset_active)
2641 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2642
6352e4d2
DH		 2643 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2644 kvm_s390_vcpu_start(vcpu);
2645 } else if (is_vcpu_stopped(vcpu)) {
ea2cdd27 2646 pr_err_ratelimited("can't run stopped vcpu %d\n",
6352e4d2
DH		 2647 vcpu->vcpu_id);
2648 return -EINVAL;
2649 }
b0c632db 2650
b028ee3e 2651 sync_regs(vcpu, kvm_run);
db0758b2 2652 enable_cpu_timer_accounting(vcpu);
d7b0b5eb 2653
dab4079d 2654 might_fault();
a76ccff6 2655 rc = __vcpu_run(vcpu);
9ace903d 2656
b1d16c49
CE		 2657 if (signal_pending(current) && !rc) {
2658 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 2659 rc = -EINTR;
b1d16c49 2660 }
8f2abe6a 2661
27291e21
DH		 2662 if (guestdbg_exit_pending(vcpu) && !rc) {
2663 kvm_s390_prepare_debug_exit(vcpu);
2664 rc = 0;
2665 }
2666
8f2abe6a 2667 if (rc == -EREMOTE) {
71f116bf 2668 /* userspace support is needed, kvm_run has been prepared */
8f2abe6a
CB		 2669 rc = 0;
2670 }
b0c632db 2671
db0758b2 2672 disable_cpu_timer_accounting(vcpu);
b028ee3e 2673 store_regs(vcpu, kvm_run);
d7b0b5eb 2674
b0c632db
HC		 2675 if (vcpu->sigset_active)
2676 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2677
b0c632db 2678 vcpu->stat.exit_userspace++;
7e8e6ab4 2679 return rc;
b0c632db
HC		 2680}
2681
b0c632db
HC		 2682/*
2683 * store status at address
2684 * we use have two special cases:
2685 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2686 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2687 */
d0bce605 2688int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 2689{
092670cd 2690 unsigned char archmode = 1;
9abc2a08 2691 freg_t fprs[NUM_FPRS];
fda902cb 2692 unsigned int px;
4287f247 2693 u64 clkcomp, cputm;
d0bce605 2694 int rc;
b0c632db 2695
d9a3a09a 2696 px = kvm_s390_get_prefix(vcpu);
d0bce605
HC		 2697 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2698 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 2699 return -EFAULT;
d9a3a09a 2700 gpa = 0;
d0bce605
HC		 2701 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2702 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 2703 return -EFAULT;
d9a3a09a
MS		 2704 gpa = px;
2705 } else
2706 gpa -= __LC_FPREGS_SAVE_AREA;
9abc2a08
DH		 2707
2708 /* manually convert vector registers if necessary */
2709 if (MACHINE_HAS_VX) {
9522b37f 2710 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08
DH		 2711 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2712 fprs, 128);
2713 } else {
2714 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
6fd8e67d 2715 vcpu->run->s.regs.fprs, 128);
9abc2a08 2716 }
d9a3a09a 2717 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
d0bce605 2718 vcpu->run->s.regs.gprs, 128);
d9a3a09a 2719 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
d0bce605 2720 &vcpu->arch.sie_block->gpsw, 16);
d9a3a09a 2721 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
fda902cb 2722 &px, 4);
d9a3a09a 2723 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
9abc2a08 2724 &vcpu->run->s.regs.fpc, 4);
d9a3a09a 2725 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
d0bce605 2726 &vcpu->arch.sie_block->todpr, 4);
4287f247 2727 cputm = kvm_s390_get_cpu_timer(vcpu);
d9a3a09a 2728 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4287f247 2729 &cputm, 8);
178bd789 2730 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d9a3a09a 2731 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
d0bce605 2732 &clkcomp, 8);
d9a3a09a 2733 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
d0bce605 2734 &vcpu->run->s.regs.acrs, 64);
d9a3a09a 2735 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
d0bce605
HC		 2736 &vcpu->arch.sie_block->gcr, 128);
2737 return rc ? -EFAULT : 0;
b0c632db
HC		 2738}
2739
e879892c
TH		 2740int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2741{
2742 /*
2743 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2744 * copying in vcpu load/put. Lets update our copies before we save
2745 * it into the save area
2746 */
d0164ee2 2747 save_fpu_regs();
9abc2a08 2748 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
e879892c
TH		 2749 save_access_regs(vcpu->run->s.regs.acrs);
2750
2751 return kvm_s390_store_status_unloaded(vcpu, addr);
2752}
2753
bc17de7c
EF		 2754/*
2755 * store additional status at address
2756 */
2757int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2758 unsigned long gpa)
2759{
2760 /* Only bits 0-53 are used for address formation */
2761 if (!(gpa & ~0x3ff))
2762 return 0;
2763
2764 return write_guest_abs(vcpu, gpa & ~0x3ff,
2765 (void *)&vcpu->run->s.regs.vrs, 512);
2766}
2767
2768int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2769{
2770 if (!test_kvm_facility(vcpu->kvm, 129))
2771 return 0;
2772
2773 /*
2774 * The guest VXRS are in the host VXRs due to the lazy
9977e886
HB		 2775 * copying in vcpu load/put. We can simply call save_fpu_regs()
2776 * to save the current register state because we are in the
2777 * middle of a load/put cycle.
2778 *
2779 * Let's update our copies before we save it into the save area.
bc17de7c 2780 */
d0164ee2 2781 save_fpu_regs();
bc17de7c
EF		 2782
2783 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2784}
2785
8ad35755
DH		 2786static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2787{
2788 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
8e236546 2789 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
8ad35755
DH		 2790}
2791
2792static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2793{
2794 unsigned int i;
2795 struct kvm_vcpu *vcpu;
2796
2797 kvm_for_each_vcpu(i, vcpu, kvm) {
2798 __disable_ibs_on_vcpu(vcpu);
2799 }
2800}
2801
2802static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2803{
09a400e7
DH		 2804 if (!sclp.has_ibs)
2805 return;
8ad35755 2806 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
8e236546 2807 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
8ad35755
DH		 2808}
2809
6852d7b6
DH		 2810void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2811{
8ad35755
DH		 2812 int i, online_vcpus, started_vcpus = 0;
2813
2814 if (!is_vcpu_stopped(vcpu))
2815 return;
2816
6852d7b6 2817 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 2818 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2819 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2820 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2821
2822 for (i = 0; i < online_vcpus; i++) {
2823 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2824 started_vcpus++;
2825 }
2826
2827 if (started_vcpus == 0) {
2828 /* we're the only active VCPU -> speed it up */
2829 __enable_ibs_on_vcpu(vcpu);
2830 } else if (started_vcpus == 1) {
2831 /*
2832 * As we are starting a second VCPU, we have to disable
2833 * the IBS facility on all VCPUs to remove potentially
2834 * oustanding ENABLE requests.
2835 */
2836 __disable_ibs_on_all_vcpus(vcpu->kvm);
2837 }
2838
805de8f4 2839 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2840 /*
2841 * Another VCPU might have used IBS while we were offline.
2842 * Let's play safe and flush the VCPU at startup.
2843 */
d3d692c8 2844 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 2845 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2846 return;
6852d7b6
DH		 2847}
2848
2849void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2850{
8ad35755
DH		 2851 int i, online_vcpus, started_vcpus = 0;
2852 struct kvm_vcpu *started_vcpu = NULL;
2853
2854 if (is_vcpu_stopped(vcpu))
2855 return;
2856
6852d7b6 2857 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 2858 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2859 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2860 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2861
32f5ff63 2862 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 2863 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 2864
805de8f4 2865 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2866 __disable_ibs_on_vcpu(vcpu);
2867
2868 for (i = 0; i < online_vcpus; i++) {
2869 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2870 started_vcpus++;
2871 started_vcpu = vcpu->kvm->vcpus[i];
2872 }
2873 }
2874
2875 if (started_vcpus == 1) {
2876 /*
2877 * As we only have one VCPU left, we want to enable the
2878 * IBS facility for that VCPU to speed it up.
2879 */
2880 __enable_ibs_on_vcpu(started_vcpu);
2881 }
2882
433b9ee4 2883 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2884 return;
6852d7b6
DH		 2885}
2886
d6712df9
CH		 2887static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2888 struct kvm_enable_cap *cap)
2889{
2890 int r;
2891
2892 if (cap->flags)
2893 return -EINVAL;
2894
2895 switch (cap->cap) {
fa6b7fe9
CH		 2896 case KVM_CAP_S390_CSS_SUPPORT:
2897 if (!vcpu->kvm->arch.css_support) {
2898 vcpu->kvm->arch.css_support = 1;
c92ea7b9 2899 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
fa6b7fe9
CH		 2900 trace_kvm_s390_enable_css(vcpu->kvm);
2901 }
2902 r = 0;
2903 break;
d6712df9
CH		 2904 default:
2905 r = -EINVAL;
2906 break;
2907 }
2908 return r;
2909}
2910
41408c28
TH		 2911static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2912 struct kvm_s390_mem_op *mop)
2913{
2914 void __user *uaddr = (void __user *)mop->buf;
2915 void *tmpbuf = NULL;
2916 int r, srcu_idx;
2917 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2918 | KVM_S390_MEMOP_F_CHECK_ONLY;
2919
2920 if (mop->flags & ~supported_flags)
2921 return -EINVAL;
2922
2923 if (mop->size > MEM_OP_MAX_SIZE)
2924 return -E2BIG;
2925
2926 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2927 tmpbuf = vmalloc(mop->size);
2928 if (!tmpbuf)
2929 return -ENOMEM;
2930 }
2931
2932 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2933
2934 switch (mop->op) {
2935 case KVM_S390_MEMOP_LOGICAL_READ:
2936 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 2937 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2938 mop->size, GACC_FETCH);
41408c28
TH		 2939 break;
2940 }
2941 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2942 if (r == 0) {
2943 if (copy_to_user(uaddr, tmpbuf, mop->size))
2944 r = -EFAULT;
2945 }
2946 break;
2947 case KVM_S390_MEMOP_LOGICAL_WRITE:
2948 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 2949 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2950 mop->size, GACC_STORE);
41408c28
TH		 2951 break;
2952 }
2953 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2954 r = -EFAULT;
2955 break;
2956 }
2957 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2958 break;
2959 default:
2960 r = -EINVAL;
2961 }
2962
2963 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2964
2965 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2966 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2967
2968 vfree(tmpbuf);
2969 return r;
2970}
2971
b0c632db
HC		 2972long kvm_arch_vcpu_ioctl(struct file *filp,
2973 unsigned int ioctl, unsigned long arg)
2974{
2975 struct kvm_vcpu *vcpu = filp->private_data;
2976 void __user *argp = (void __user *)arg;
800c1065 2977 int idx;
bc923cc9 2978 long r;
b0c632db 2979
93736624 2980 switch (ioctl) {
47b43c52
JF		 2981 case KVM_S390_IRQ: {
2982 struct kvm_s390_irq s390irq;
2983
2984 r = -EFAULT;
2985 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2986 break;
2987 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2988 break;
2989 }
93736624 2990 case KVM_S390_INTERRUPT: {
ba5c1e9b 2991 struct kvm_s390_interrupt s390int;
383d0b05 2992 struct kvm_s390_irq s390irq;
ba5c1e9b 2993
93736624 2994 r = -EFAULT;
ba5c1e9b 2995 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624 2996 break;
383d0b05
JF		 2997 if (s390int_to_s390irq(&s390int, &s390irq))
2998 return -EINVAL;
2999 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
93736624 3000 break;
ba5c1e9b 3001 }
b0c632db 3002 case KVM_S390_STORE_STATUS:
800c1065 3003 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 3004 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 3005 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 3006 break;
b0c632db
HC		 3007 case KVM_S390_SET_INITIAL_PSW: {
3008 psw_t psw;
3009
bc923cc9 3010 r = -EFAULT;
b0c632db 3011 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 3012 break;
3013 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3014 break;
b0c632db
HC		 3015 }
3016 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 3017 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3018 break;
14eebd91
CO		 3019 case KVM_SET_ONE_REG:
3020 case KVM_GET_ONE_REG: {
3021 struct kvm_one_reg reg;
3022 r = -EFAULT;
3023 if (copy_from_user(&reg, argp, sizeof(reg)))
3024 break;
3025 if (ioctl == KVM_SET_ONE_REG)
3026 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3027 else
3028 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3029 break;
3030 }
27e0393f
CO		 3031#ifdef CONFIG_KVM_S390_UCONTROL
3032 case KVM_S390_UCAS_MAP: {
3033 struct kvm_s390_ucas_mapping ucasmap;
3034
3035 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3036 r = -EFAULT;
3037 break;
3038 }
3039
3040 if (!kvm_is_ucontrol(vcpu->kvm)) {
3041 r = -EINVAL;
3042 break;
3043 }
3044
3045 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3046 ucasmap.vcpu_addr, ucasmap.length);
3047 break;
3048 }
3049 case KVM_S390_UCAS_UNMAP: {
3050 struct kvm_s390_ucas_mapping ucasmap;
3051
3052 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3053 r = -EFAULT;
3054 break;
3055 }
3056
3057 if (!kvm_is_ucontrol(vcpu->kvm)) {
3058 r = -EINVAL;
3059 break;
3060 }
3061
3062 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3063 ucasmap.length);
3064 break;
3065 }
3066#endif
ccc7910f 3067 case KVM_S390_VCPU_FAULT: {
527e30b4 3068 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 3069 break;
3070 }
d6712df9
CH		 3071 case KVM_ENABLE_CAP:
3072 {
3073 struct kvm_enable_cap cap;
3074 r = -EFAULT;
3075 if (copy_from_user(&cap, argp, sizeof(cap)))
3076 break;
3077 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3078 break;
3079 }
41408c28
TH		 3080 case KVM_S390_MEM_OP: {
3081 struct kvm_s390_mem_op mem_op;
3082
3083 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3084 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3085 else
3086 r = -EFAULT;
3087 break;
3088 }
816c7667
JF		 3089 case KVM_S390_SET_IRQ_STATE: {
3090 struct kvm_s390_irq_state irq_state;
3091
3092 r = -EFAULT;
3093 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3094 break;
3095 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3096 irq_state.len == 0 ||
3097 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3098 r = -EINVAL;
3099 break;
3100 }
3101 r = kvm_s390_set_irq_state(vcpu,
3102 (void __user *) irq_state.buf,
3103 irq_state.len);
3104 break;
3105 }
3106 case KVM_S390_GET_IRQ_STATE: {
3107 struct kvm_s390_irq_state irq_state;
3108
3109 r = -EFAULT;
3110 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3111 break;
3112 if (irq_state.len == 0) {
3113 r = -EINVAL;
3114 break;
3115 }
3116 r = kvm_s390_get_irq_state(vcpu,
3117 (__u8 __user *) irq_state.buf,
3118 irq_state.len);
3119 break;
3120 }
b0c632db 3121 default:
3e6afcf1 3122 r = -ENOTTY;
b0c632db 3123 }
bc923cc9 3124 return r;
b0c632db
HC		 3125}
3126
5b1c1493
CO		 3127int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3128{
3129#ifdef CONFIG_KVM_S390_UCONTROL
3130 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3131 && (kvm_is_ucontrol(vcpu->kvm))) {
3132 vmf->page = virt_to_page(vcpu->arch.sie_block);
3133 get_page(vmf->page);
3134 return 0;
3135 }
3136#endif
3137 return VM_FAULT_SIGBUS;
3138}
3139
5587027c
AK		 3140int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3141 unsigned long npages)
db3fe4eb
TY		 3142{
3143 return 0;
3144}
3145
b0c632db 3146/* Section: memory related */
f7784b8e
MT		 3147int kvm_arch_prepare_memory_region(struct kvm *kvm,
3148 struct kvm_memory_slot *memslot,
09170a49 3149 const struct kvm_userspace_memory_region *mem,
7b6195a9 3150 enum kvm_mr_change change)
b0c632db 3151{
dd2887e7
NW		 3152 /* A few sanity checks. We can have memory slots which have to be
3153 located/ended at a segment boundary (1MB). The memory in userland is
3154 ok to be fragmented into various different vmas. It is okay to mmap()
3155 and munmap() stuff in this slot after doing this call at any time */
b0c632db 3156
598841ca 3157 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 3158 return -EINVAL;
3159
598841ca 3160 if (mem->memory_size & 0xffffful)
b0c632db
HC		 3161 return -EINVAL;
3162
a3a92c31
DD		 3163 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3164 return -EINVAL;
3165
f7784b8e
MT		 3166 return 0;
3167}
3168
3169void kvm_arch_commit_memory_region(struct kvm *kvm,
09170a49 3170 const struct kvm_userspace_memory_region *mem,
8482644a 3171 const struct kvm_memory_slot *old,
f36f3f28 3172 const struct kvm_memory_slot *new,
8482644a 3173 enum kvm_mr_change change)
f7784b8e 3174{
f7850c92 3175 int rc;
f7784b8e 3176
2cef4deb
CB		 3177 /* If the basics of the memslot do not change, we do not want
3178 * to update the gmap. Every update causes several unnecessary
3179 * segment translation exceptions. This is usually handled just
3180 * fine by the normal fault handler + gmap, but it will also
3181 * cause faults on the prefix page of running guest CPUs.
3182 */
3183 if (old->userspace_addr == mem->userspace_addr &&
3184 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3185 old->npages * PAGE_SIZE == mem->memory_size)
3186 return;
598841ca
CO		 3187
3188 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3189 mem->guest_phys_addr, mem->memory_size);
3190 if (rc)
ea2cdd27 3191 pr_warn("failed to commit memory region\n");
598841ca 3192 return;
b0c632db
HC		 3193}
3194
60a37709
AY		 3195static inline unsigned long nonhyp_mask(int i)
3196{
3197 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3198
3199 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3200}
3201
3491caf2
CB		 3202void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3203{
3204 vcpu->valid_wakeup = false;
3205}
3206
b0c632db
HC		 3207static int __init kvm_s390_init(void)
3208{
60a37709
AY		 3209 int i;
3210
07197fd0
DH		 3211 if (!sclp.has_sief2) {
3212 pr_info("SIE not available\n");
3213 return -ENODEV;
3214 }
3215
60a37709
AY		 3216 for (i = 0; i < 16; i++)
3217 kvm_s390_fac_list_mask[i] |=
3218 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3219
9d8d5786 3220 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 3221}
3222
3223static void __exit kvm_s390_exit(void)
3224{
3225 kvm_exit();
3226}
3227
3228module_init(kvm_s390_init);
3229module_exit(kvm_s390_exit);
566af940
CH		 3230
3231/*
3232 * Enable autoloading of the kvm module.
3233 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3234 * since x86 takes a different approach.
3235 */
3236#include <linux/miscdevice.h>
3237MODULE_ALIAS_MISCDEV(KVM_MINOR);
3238MODULE_ALIAS("devname:kvm");
Ubuntu Jammy Linux kernel mirror