]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blob - arch/s390/kvm/kvm-s390.c
projects / mirror_ubuntu-artful-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'nios2-fixes-v4.0-rc5' of git://git.rocketboards.org/linux-socfpga-next
[mirror_ubuntu-artful-kernel.git] / arch / s390 / kvm / kvm-s390.c
1 /*
2 * hosting zSeries kernel virtual machines
3 *
4 * Copyright IBM Corp. 2008, 2009
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
15 */
16
17 #include <linux/compiler.h>
18 #include <linux/err.h>
19 #include <linux/fs.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/module.h>
25 #include <linux/random.h>
26 #include <linux/slab.h>
27 #include <linux/timer.h>
28 #include <asm/asm-offsets.h>
29 #include <asm/lowcore.h>
30 #include <asm/pgtable.h>
31 #include <asm/nmi.h>
32 #include <asm/switch_to.h>
33 #include <asm/sclp.h>
34 #include "kvm-s390.h"
35 #include "gaccess.h"
36
37 #define CREATE_TRACE_POINTS
38 #include "trace.h"
39 #include "trace-s390.h"
40
41 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
42
43 struct kvm_stats_debugfs_item debugfs_entries[] = {
44 { "userspace_handled", VCPU_STAT(exit_userspace) },
45 { "exit_null", VCPU_STAT(exit_null) },
46 { "exit_validity", VCPU_STAT(exit_validity) },
47 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
48 { "exit_external_request", VCPU_STAT(exit_external_request) },
49 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
50 { "exit_instruction", VCPU_STAT(exit_instruction) },
51 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
52 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
53 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
54 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
55 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
56 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
57 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
58 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
59 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
60 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
61 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
62 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
63 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
64 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
65 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
66 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
67 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
68 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
69 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
70 { "instruction_spx", VCPU_STAT(instruction_spx) },
71 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
72 { "instruction_stap", VCPU_STAT(instruction_stap) },
73 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
74 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
75 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
76 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
77 { "instruction_essa", VCPU_STAT(instruction_essa) },
78 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
79 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
80 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
81 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
82 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
83 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
84 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
85 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
86 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
87 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
88 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
89 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
90 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
91 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
92 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
93 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
94 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
95 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
96 { "diagnose_10", VCPU_STAT(diagnose_10) },
97 { "diagnose_44", VCPU_STAT(diagnose_44) },
98 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
99 { NULL }
100 };
101
102 /* upper facilities limit for kvm */
103 unsigned long kvm_s390_fac_list_mask[] = {
104 0xff82fffbf4fc2000UL,
105 0x005c000000000000UL,
106 };
107
108 unsigned long kvm_s390_fac_list_mask_size(void)
109 {
110 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
111 return ARRAY_SIZE(kvm_s390_fac_list_mask);
112 }
113
114 static struct gmap_notifier gmap_notifier;
115
116 /* Section: not file related */
117 int kvm_arch_hardware_enable(void)
118 {
119 /* every s390 is virtualization enabled ;-) */
120 return 0;
121 }
122
123 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
124
125 int kvm_arch_hardware_setup(void)
126 {
127 gmap_notifier.notifier_call = kvm_gmap_notifier;
128 gmap_register_ipte_notifier(&gmap_notifier);
129 return 0;
130 }
131
132 void kvm_arch_hardware_unsetup(void)
133 {
134 gmap_unregister_ipte_notifier(&gmap_notifier);
135 }
136
137 int kvm_arch_init(void *opaque)
138 {
139 /* Register floating interrupt controller interface. */
140 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
141 }
142
143 /* Section: device related */
144 long kvm_arch_dev_ioctl(struct file *filp,
145 unsigned int ioctl, unsigned long arg)
146 {
147 if (ioctl == KVM_S390_ENABLE_SIE)
148 return s390_enable_sie();
149 return -EINVAL;
150 }
151
152 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
153 {
154 int r;
155
156 switch (ext) {
157 case KVM_CAP_S390_PSW:
158 case KVM_CAP_S390_GMAP:
159 case KVM_CAP_SYNC_MMU:
160 #ifdef CONFIG_KVM_S390_UCONTROL
161 case KVM_CAP_S390_UCONTROL:
162 #endif
163 case KVM_CAP_ASYNC_PF:
164 case KVM_CAP_SYNC_REGS:
165 case KVM_CAP_ONE_REG:
166 case KVM_CAP_ENABLE_CAP:
167 case KVM_CAP_S390_CSS_SUPPORT:
168 case KVM_CAP_IOEVENTFD:
169 case KVM_CAP_DEVICE_CTRL:
170 case KVM_CAP_ENABLE_CAP_VM:
171 case KVM_CAP_S390_IRQCHIP:
172 case KVM_CAP_VM_ATTRIBUTES:
173 case KVM_CAP_MP_STATE:
174 case KVM_CAP_S390_USER_SIGP:
175 r = 1;
176 break;
177 case KVM_CAP_NR_VCPUS:
178 case KVM_CAP_MAX_VCPUS:
179 r = KVM_MAX_VCPUS;
180 break;
181 case KVM_CAP_NR_MEMSLOTS:
182 r = KVM_USER_MEM_SLOTS;
183 break;
184 case KVM_CAP_S390_COW:
185 r = MACHINE_HAS_ESOP;
186 break;
187 default:
188 r = 0;
189 }
190 return r;
191 }
192
193 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
194 struct kvm_memory_slot *memslot)
195 {
196 gfn_t cur_gfn, last_gfn;
197 unsigned long address;
198 struct gmap *gmap = kvm->arch.gmap;
199
200 down_read(&gmap->mm->mmap_sem);
201 /* Loop over all guest pages */
202 last_gfn = memslot->base_gfn + memslot->npages;
203 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
204 address = gfn_to_hva_memslot(memslot, cur_gfn);
205
206 if (gmap_test_and_clear_dirty(address, gmap))
207 mark_page_dirty(kvm, cur_gfn);
208 }
209 up_read(&gmap->mm->mmap_sem);
210 }
211
212 /* Section: vm related */
213 /*
214 * Get (and clear) the dirty memory log for a memory slot.
215 */
216 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
217 struct kvm_dirty_log *log)
218 {
219 int r;
220 unsigned long n;
221 struct kvm_memory_slot *memslot;
222 int is_dirty = 0;
223
224 mutex_lock(&kvm->slots_lock);
225
226 r = -EINVAL;
227 if (log->slot >= KVM_USER_MEM_SLOTS)
228 goto out;
229
230 memslot = id_to_memslot(kvm->memslots, log->slot);
231 r = -ENOENT;
232 if (!memslot->dirty_bitmap)
233 goto out;
234
235 kvm_s390_sync_dirty_log(kvm, memslot);
236 r = kvm_get_dirty_log(kvm, log, &is_dirty);
237 if (r)
238 goto out;
239
240 /* Clear the dirty log */
241 if (is_dirty) {
242 n = kvm_dirty_bitmap_bytes(memslot);
243 memset(memslot->dirty_bitmap, 0, n);
244 }
245 r = 0;
246 out:
247 mutex_unlock(&kvm->slots_lock);
248 return r;
249 }
250
251 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
252 {
253 int r;
254
255 if (cap->flags)
256 return -EINVAL;
257
258 switch (cap->cap) {
259 case KVM_CAP_S390_IRQCHIP:
260 kvm->arch.use_irqchip = 1;
261 r = 0;
262 break;
263 case KVM_CAP_S390_USER_SIGP:
264 kvm->arch.user_sigp = 1;
265 r = 0;
266 break;
267 default:
268 r = -EINVAL;
269 break;
270 }
271 return r;
272 }
273
274 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
275 {
276 int ret;
277
278 switch (attr->attr) {
279 case KVM_S390_VM_MEM_LIMIT_SIZE:
280 ret = 0;
281 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
282 ret = -EFAULT;
283 break;
284 default:
285 ret = -ENXIO;
286 break;
287 }
288 return ret;
289 }
290
291 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
292 {
293 int ret;
294 unsigned int idx;
295 switch (attr->attr) {
296 case KVM_S390_VM_MEM_ENABLE_CMMA:
297 ret = -EBUSY;
298 mutex_lock(&kvm->lock);
299 if (atomic_read(&kvm->online_vcpus) == 0) {
300 kvm->arch.use_cmma = 1;
301 ret = 0;
302 }
303 mutex_unlock(&kvm->lock);
304 break;
305 case KVM_S390_VM_MEM_CLR_CMMA:
306 mutex_lock(&kvm->lock);
307 idx = srcu_read_lock(&kvm->srcu);
308 s390_reset_cmma(kvm->arch.gmap->mm);
309 srcu_read_unlock(&kvm->srcu, idx);
310 mutex_unlock(&kvm->lock);
311 ret = 0;
312 break;
313 case KVM_S390_VM_MEM_LIMIT_SIZE: {
314 unsigned long new_limit;
315
316 if (kvm_is_ucontrol(kvm))
317 return -EINVAL;
318
319 if (get_user(new_limit, (u64 __user *)attr->addr))
320 return -EFAULT;
321
322 if (new_limit > kvm->arch.gmap->asce_end)
323 return -E2BIG;
324
325 ret = -EBUSY;
326 mutex_lock(&kvm->lock);
327 if (atomic_read(&kvm->online_vcpus) == 0) {
328 /* gmap_alloc will round the limit up */
329 struct gmap *new = gmap_alloc(current->mm, new_limit);
330
331 if (!new) {
332 ret = -ENOMEM;
333 } else {
334 gmap_free(kvm->arch.gmap);
335 new->private = kvm;
336 kvm->arch.gmap = new;
337 ret = 0;
338 }
339 }
340 mutex_unlock(&kvm->lock);
341 break;
342 }
343 default:
344 ret = -ENXIO;
345 break;
346 }
347 return ret;
348 }
349
350 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
351
352 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
353 {
354 struct kvm_vcpu *vcpu;
355 int i;
356
357 if (!test_kvm_facility(kvm, 76))
358 return -EINVAL;
359
360 mutex_lock(&kvm->lock);
361 switch (attr->attr) {
362 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
363 get_random_bytes(
364 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
365 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
366 kvm->arch.crypto.aes_kw = 1;
367 break;
368 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
369 get_random_bytes(
370 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
371 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
372 kvm->arch.crypto.dea_kw = 1;
373 break;
374 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
375 kvm->arch.crypto.aes_kw = 0;
376 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
377 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
378 break;
379 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
380 kvm->arch.crypto.dea_kw = 0;
381 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
382 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
383 break;
384 default:
385 mutex_unlock(&kvm->lock);
386 return -ENXIO;
387 }
388
389 kvm_for_each_vcpu(i, vcpu, kvm) {
390 kvm_s390_vcpu_crypto_setup(vcpu);
391 exit_sie(vcpu);
392 }
393 mutex_unlock(&kvm->lock);
394 return 0;
395 }
396
397 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
398 {
399 u8 gtod_high;
400
401 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
402 sizeof(gtod_high)))
403 return -EFAULT;
404
405 if (gtod_high != 0)
406 return -EINVAL;
407
408 return 0;
409 }
410
411 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
412 {
413 struct kvm_vcpu *cur_vcpu;
414 unsigned int vcpu_idx;
415 u64 host_tod, gtod;
416 int r;
417
418 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
419 return -EFAULT;
420
421 r = store_tod_clock(&host_tod);
422 if (r)
423 return r;
424
425 mutex_lock(&kvm->lock);
426 kvm->arch.epoch = gtod - host_tod;
427 kvm_for_each_vcpu(vcpu_idx, cur_vcpu, kvm) {
428 cur_vcpu->arch.sie_block->epoch = kvm->arch.epoch;
429 exit_sie(cur_vcpu);
430 }
431 mutex_unlock(&kvm->lock);
432 return 0;
433 }
434
435 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
436 {
437 int ret;
438
439 if (attr->flags)
440 return -EINVAL;
441
442 switch (attr->attr) {
443 case KVM_S390_VM_TOD_HIGH:
444 ret = kvm_s390_set_tod_high(kvm, attr);
445 break;
446 case KVM_S390_VM_TOD_LOW:
447 ret = kvm_s390_set_tod_low(kvm, attr);
448 break;
449 default:
450 ret = -ENXIO;
451 break;
452 }
453 return ret;
454 }
455
456 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
457 {
458 u8 gtod_high = 0;
459
460 if (copy_to_user((void __user *)attr->addr, &gtod_high,
461 sizeof(gtod_high)))
462 return -EFAULT;
463
464 return 0;
465 }
466
467 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
468 {
469 u64 host_tod, gtod;
470 int r;
471
472 r = store_tod_clock(&host_tod);
473 if (r)
474 return r;
475
476 gtod = host_tod + kvm->arch.epoch;
477 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
478 return -EFAULT;
479
480 return 0;
481 }
482
483 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
484 {
485 int ret;
486
487 if (attr->flags)
488 return -EINVAL;
489
490 switch (attr->attr) {
491 case KVM_S390_VM_TOD_HIGH:
492 ret = kvm_s390_get_tod_high(kvm, attr);
493 break;
494 case KVM_S390_VM_TOD_LOW:
495 ret = kvm_s390_get_tod_low(kvm, attr);
496 break;
497 default:
498 ret = -ENXIO;
499 break;
500 }
501 return ret;
502 }
503
504 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
505 {
506 struct kvm_s390_vm_cpu_processor *proc;
507 int ret = 0;
508
509 mutex_lock(&kvm->lock);
510 if (atomic_read(&kvm->online_vcpus)) {
511 ret = -EBUSY;
512 goto out;
513 }
514 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
515 if (!proc) {
516 ret = -ENOMEM;
517 goto out;
518 }
519 if (!copy_from_user(proc, (void __user *)attr->addr,
520 sizeof(*proc))) {
521 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
522 sizeof(struct cpuid));
523 kvm->arch.model.ibc = proc->ibc;
524 memcpy(kvm->arch.model.fac->list, proc->fac_list,
525 S390_ARCH_FAC_LIST_SIZE_BYTE);
526 } else
527 ret = -EFAULT;
528 kfree(proc);
529 out:
530 mutex_unlock(&kvm->lock);
531 return ret;
532 }
533
534 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
535 {
536 int ret = -ENXIO;
537
538 switch (attr->attr) {
539 case KVM_S390_VM_CPU_PROCESSOR:
540 ret = kvm_s390_set_processor(kvm, attr);
541 break;
542 }
543 return ret;
544 }
545
546 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
547 {
548 struct kvm_s390_vm_cpu_processor *proc;
549 int ret = 0;
550
551 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
552 if (!proc) {
553 ret = -ENOMEM;
554 goto out;
555 }
556 memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
557 proc->ibc = kvm->arch.model.ibc;
558 memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
559 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
560 ret = -EFAULT;
561 kfree(proc);
562 out:
563 return ret;
564 }
565
566 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
567 {
568 struct kvm_s390_vm_cpu_machine *mach;
569 int ret = 0;
570
571 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
572 if (!mach) {
573 ret = -ENOMEM;
574 goto out;
575 }
576 get_cpu_id((struct cpuid *) &mach->cpuid);
577 mach->ibc = sclp_get_ibc();
578 memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
579 S390_ARCH_FAC_LIST_SIZE_BYTE);
580 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
581 S390_ARCH_FAC_LIST_SIZE_BYTE);
582 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
583 ret = -EFAULT;
584 kfree(mach);
585 out:
586 return ret;
587 }
588
589 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
590 {
591 int ret = -ENXIO;
592
593 switch (attr->attr) {
594 case KVM_S390_VM_CPU_PROCESSOR:
595 ret = kvm_s390_get_processor(kvm, attr);
596 break;
597 case KVM_S390_VM_CPU_MACHINE:
598 ret = kvm_s390_get_machine(kvm, attr);
599 break;
600 }
601 return ret;
602 }
603
604 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
605 {
606 int ret;
607
608 switch (attr->group) {
609 case KVM_S390_VM_MEM_CTRL:
610 ret = kvm_s390_set_mem_control(kvm, attr);
611 break;
612 case KVM_S390_VM_TOD:
613 ret = kvm_s390_set_tod(kvm, attr);
614 break;
615 case KVM_S390_VM_CPU_MODEL:
616 ret = kvm_s390_set_cpu_model(kvm, attr);
617 break;
618 case KVM_S390_VM_CRYPTO:
619 ret = kvm_s390_vm_set_crypto(kvm, attr);
620 break;
621 default:
622 ret = -ENXIO;
623 break;
624 }
625
626 return ret;
627 }
628
629 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
630 {
631 int ret;
632
633 switch (attr->group) {
634 case KVM_S390_VM_MEM_CTRL:
635 ret = kvm_s390_get_mem_control(kvm, attr);
636 break;
637 case KVM_S390_VM_TOD:
638 ret = kvm_s390_get_tod(kvm, attr);
639 break;
640 case KVM_S390_VM_CPU_MODEL:
641 ret = kvm_s390_get_cpu_model(kvm, attr);
642 break;
643 default:
644 ret = -ENXIO;
645 break;
646 }
647
648 return ret;
649 }
650
651 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
652 {
653 int ret;
654
655 switch (attr->group) {
656 case KVM_S390_VM_MEM_CTRL:
657 switch (attr->attr) {
658 case KVM_S390_VM_MEM_ENABLE_CMMA:
659 case KVM_S390_VM_MEM_CLR_CMMA:
660 case KVM_S390_VM_MEM_LIMIT_SIZE:
661 ret = 0;
662 break;
663 default:
664 ret = -ENXIO;
665 break;
666 }
667 break;
668 case KVM_S390_VM_TOD:
669 switch (attr->attr) {
670 case KVM_S390_VM_TOD_LOW:
671 case KVM_S390_VM_TOD_HIGH:
672 ret = 0;
673 break;
674 default:
675 ret = -ENXIO;
676 break;
677 }
678 break;
679 case KVM_S390_VM_CPU_MODEL:
680 switch (attr->attr) {
681 case KVM_S390_VM_CPU_PROCESSOR:
682 case KVM_S390_VM_CPU_MACHINE:
683 ret = 0;
684 break;
685 default:
686 ret = -ENXIO;
687 break;
688 }
689 break;
690 case KVM_S390_VM_CRYPTO:
691 switch (attr->attr) {
692 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
693 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
694 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
695 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
696 ret = 0;
697 break;
698 default:
699 ret = -ENXIO;
700 break;
701 }
702 break;
703 default:
704 ret = -ENXIO;
705 break;
706 }
707
708 return ret;
709 }
710
711 long kvm_arch_vm_ioctl(struct file *filp,
712 unsigned int ioctl, unsigned long arg)
713 {
714 struct kvm *kvm = filp->private_data;
715 void __user *argp = (void __user *)arg;
716 struct kvm_device_attr attr;
717 int r;
718
719 switch (ioctl) {
720 case KVM_S390_INTERRUPT: {
721 struct kvm_s390_interrupt s390int;
722
723 r = -EFAULT;
724 if (copy_from_user(&s390int, argp, sizeof(s390int)))
725 break;
726 r = kvm_s390_inject_vm(kvm, &s390int);
727 break;
728 }
729 case KVM_ENABLE_CAP: {
730 struct kvm_enable_cap cap;
731 r = -EFAULT;
732 if (copy_from_user(&cap, argp, sizeof(cap)))
733 break;
734 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
735 break;
736 }
737 case KVM_CREATE_IRQCHIP: {
738 struct kvm_irq_routing_entry routing;
739
740 r = -EINVAL;
741 if (kvm->arch.use_irqchip) {
742 /* Set up dummy routing. */
743 memset(&routing, 0, sizeof(routing));
744 kvm_set_irq_routing(kvm, &routing, 0, 0);
745 r = 0;
746 }
747 break;
748 }
749 case KVM_SET_DEVICE_ATTR: {
750 r = -EFAULT;
751 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
752 break;
753 r = kvm_s390_vm_set_attr(kvm, &attr);
754 break;
755 }
756 case KVM_GET_DEVICE_ATTR: {
757 r = -EFAULT;
758 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
759 break;
760 r = kvm_s390_vm_get_attr(kvm, &attr);
761 break;
762 }
763 case KVM_HAS_DEVICE_ATTR: {
764 r = -EFAULT;
765 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
766 break;
767 r = kvm_s390_vm_has_attr(kvm, &attr);
768 break;
769 }
770 default:
771 r = -ENOTTY;
772 }
773
774 return r;
775 }
776
777 static int kvm_s390_query_ap_config(u8 *config)
778 {
779 u32 fcn_code = 0x04000000UL;
780 u32 cc = 0;
781
782 memset(config, 0, 128);
783 asm volatile(
784 "lgr 0,%1\n"
785 "lgr 2,%2\n"
786 ".long 0xb2af0000\n" /* PQAP(QCI) */
787 "0: ipm %0\n"
788 "srl %0,28\n"
789 "1:\n"
790 EX_TABLE(0b, 1b)
791 : "+r" (cc)
792 : "r" (fcn_code), "r" (config)
793 : "cc", "0", "2", "memory"
794 );
795
796 return cc;
797 }
798
799 static int kvm_s390_apxa_installed(void)
800 {
801 u8 config[128];
802 int cc;
803
804 if (test_facility(2) && test_facility(12)) {
805 cc = kvm_s390_query_ap_config(config);
806
807 if (cc)
808 pr_err("PQAP(QCI) failed with cc=%d", cc);
809 else
810 return config[0] & 0x40;
811 }
812
813 return 0;
814 }
815
816 static void kvm_s390_set_crycb_format(struct kvm *kvm)
817 {
818 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
819
820 if (kvm_s390_apxa_installed())
821 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
822 else
823 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
824 }
825
826 static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
827 {
828 get_cpu_id(cpu_id);
829 cpu_id->version = 0xff;
830 }
831
832 static int kvm_s390_crypto_init(struct kvm *kvm)
833 {
834 if (!test_kvm_facility(kvm, 76))
835 return 0;
836
837 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
838 GFP_KERNEL | GFP_DMA);
839 if (!kvm->arch.crypto.crycb)
840 return -ENOMEM;
841
842 kvm_s390_set_crycb_format(kvm);
843
844 /* Enable AES/DEA protected key functions by default */
845 kvm->arch.crypto.aes_kw = 1;
846 kvm->arch.crypto.dea_kw = 1;
847 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
848 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
849 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
850 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
851
852 return 0;
853 }
854
855 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
856 {
857 int i, rc;
858 char debug_name[16];
859 static unsigned long sca_offset;
860
861 rc = -EINVAL;
862 #ifdef CONFIG_KVM_S390_UCONTROL
863 if (type & ~KVM_VM_S390_UCONTROL)
864 goto out_err;
865 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
866 goto out_err;
867 #else
868 if (type)
869 goto out_err;
870 #endif
871
872 rc = s390_enable_sie();
873 if (rc)
874 goto out_err;
875
876 rc = -ENOMEM;
877
878 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
879 if (!kvm->arch.sca)
880 goto out_err;
881 spin_lock(&kvm_lock);
882 sca_offset = (sca_offset + 16) & 0x7f0;
883 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
884 spin_unlock(&kvm_lock);
885
886 sprintf(debug_name, "kvm-%u", current->pid);
887
888 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
889 if (!kvm->arch.dbf)
890 goto out_nodbf;
891
892 /*
893 * The architectural maximum amount of facilities is 16 kbit. To store
894 * this amount, 2 kbyte of memory is required. Thus we need a full
895 * page to hold the guest facility list (arch.model.fac->list) and the
896 * facility mask (arch.model.fac->mask). Its address size has to be
897 * 31 bits and word aligned.
898 */
899 kvm->arch.model.fac =
900 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
901 if (!kvm->arch.model.fac)
902 goto out_nofac;
903
904 /* Populate the facility mask initially. */
905 memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
906 S390_ARCH_FAC_LIST_SIZE_BYTE);
907 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
908 if (i < kvm_s390_fac_list_mask_size())
909 kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
910 else
911 kvm->arch.model.fac->mask[i] = 0UL;
912 }
913
914 /* Populate the facility list initially. */
915 memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
916 S390_ARCH_FAC_LIST_SIZE_BYTE);
917
918 kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
919 kvm->arch.model.ibc = sclp_get_ibc() & 0x0fff;
920
921 if (kvm_s390_crypto_init(kvm) < 0)
922 goto out_crypto;
923
924 spin_lock_init(&kvm->arch.float_int.lock);
925 INIT_LIST_HEAD(&kvm->arch.float_int.list);
926 init_waitqueue_head(&kvm->arch.ipte_wq);
927 mutex_init(&kvm->arch.ipte_mutex);
928
929 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
930 VM_EVENT(kvm, 3, "%s", "vm created");
931
932 if (type & KVM_VM_S390_UCONTROL) {
933 kvm->arch.gmap = NULL;
934 } else {
935 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
936 if (!kvm->arch.gmap)
937 goto out_nogmap;
938 kvm->arch.gmap->private = kvm;
939 kvm->arch.gmap->pfault_enabled = 0;
940 }
941
942 kvm->arch.css_support = 0;
943 kvm->arch.use_irqchip = 0;
944 kvm->arch.epoch = 0;
945
946 spin_lock_init(&kvm->arch.start_stop_lock);
947
948 return 0;
949 out_nogmap:
950 kfree(kvm->arch.crypto.crycb);
951 out_crypto:
952 free_page((unsigned long)kvm->arch.model.fac);
953 out_nofac:
954 debug_unregister(kvm->arch.dbf);
955 out_nodbf:
956 free_page((unsigned long)(kvm->arch.sca));
957 out_err:
958 return rc;
959 }
960
961 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
962 {
963 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
964 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
965 kvm_s390_clear_local_irqs(vcpu);
966 kvm_clear_async_pf_completion_queue(vcpu);
967 if (!kvm_is_ucontrol(vcpu->kvm)) {
968 clear_bit(63 - vcpu->vcpu_id,
969 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
970 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
971 (__u64) vcpu->arch.sie_block)
972 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
973 }
974 smp_mb();
975
976 if (kvm_is_ucontrol(vcpu->kvm))
977 gmap_free(vcpu->arch.gmap);
978
979 if (kvm_s390_cmma_enabled(vcpu->kvm))
980 kvm_s390_vcpu_unsetup_cmma(vcpu);
981 free_page((unsigned long)(vcpu->arch.sie_block));
982
983 kvm_vcpu_uninit(vcpu);
984 kmem_cache_free(kvm_vcpu_cache, vcpu);
985 }
986
987 static void kvm_free_vcpus(struct kvm *kvm)
988 {
989 unsigned int i;
990 struct kvm_vcpu *vcpu;
991
992 kvm_for_each_vcpu(i, vcpu, kvm)
993 kvm_arch_vcpu_destroy(vcpu);
994
995 mutex_lock(&kvm->lock);
996 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
997 kvm->vcpus[i] = NULL;
998
999 atomic_set(&kvm->online_vcpus, 0);
1000 mutex_unlock(&kvm->lock);
1001 }
1002
1003 void kvm_arch_destroy_vm(struct kvm *kvm)
1004 {
1005 kvm_free_vcpus(kvm);
1006 free_page((unsigned long)kvm->arch.model.fac);
1007 free_page((unsigned long)(kvm->arch.sca));
1008 debug_unregister(kvm->arch.dbf);
1009 kfree(kvm->arch.crypto.crycb);
1010 if (!kvm_is_ucontrol(kvm))
1011 gmap_free(kvm->arch.gmap);
1012 kvm_s390_destroy_adapters(kvm);
1013 kvm_s390_clear_float_irqs(kvm);
1014 }
1015
1016 /* Section: vcpu related */
1017 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1018 {
1019 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1020 if (!vcpu->arch.gmap)
1021 return -ENOMEM;
1022 vcpu->arch.gmap->private = vcpu->kvm;
1023
1024 return 0;
1025 }
1026
1027 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1028 {
1029 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1030 kvm_clear_async_pf_completion_queue(vcpu);
1031 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1032 KVM_SYNC_GPRS |
1033 KVM_SYNC_ACRS |
1034 KVM_SYNC_CRS |
1035 KVM_SYNC_ARCH0 |
1036 KVM_SYNC_PFAULT;
1037
1038 if (kvm_is_ucontrol(vcpu->kvm))
1039 return __kvm_ucontrol_vcpu_init(vcpu);
1040
1041 return 0;
1042 }
1043
1044 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1045 {
1046 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1047 save_fp_regs(vcpu->arch.host_fpregs.fprs);
1048 save_access_regs(vcpu->arch.host_acrs);
1049 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1050 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1051 restore_access_regs(vcpu->run->s.regs.acrs);
1052 gmap_enable(vcpu->arch.gmap);
1053 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1054 }
1055
1056 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1057 {
1058 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1059 gmap_disable(vcpu->arch.gmap);
1060 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1061 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1062 save_access_regs(vcpu->run->s.regs.acrs);
1063 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1064 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
1065 restore_access_regs(vcpu->arch.host_acrs);
1066 }
1067
1068 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1069 {
1070 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1071 vcpu->arch.sie_block->gpsw.mask = 0UL;
1072 vcpu->arch.sie_block->gpsw.addr = 0UL;
1073 kvm_s390_set_prefix(vcpu, 0);
1074 vcpu->arch.sie_block->cputm = 0UL;
1075 vcpu->arch.sie_block->ckc = 0UL;
1076 vcpu->arch.sie_block->todpr = 0;
1077 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1078 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1079 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1080 vcpu->arch.guest_fpregs.fpc = 0;
1081 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
1082 vcpu->arch.sie_block->gbea = 1;
1083 vcpu->arch.sie_block->pp = 0;
1084 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1085 kvm_clear_async_pf_completion_queue(vcpu);
1086 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1087 kvm_s390_vcpu_stop(vcpu);
1088 kvm_s390_clear_local_irqs(vcpu);
1089 }
1090
1091 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1092 {
1093 mutex_lock(&vcpu->kvm->lock);
1094 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1095 mutex_unlock(&vcpu->kvm->lock);
1096 if (!kvm_is_ucontrol(vcpu->kvm))
1097 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1098 }
1099
1100 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1101 {
1102 if (!test_kvm_facility(vcpu->kvm, 76))
1103 return;
1104
1105 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1106
1107 if (vcpu->kvm->arch.crypto.aes_kw)
1108 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1109 if (vcpu->kvm->arch.crypto.dea_kw)
1110 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1111
1112 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1113 }
1114
1115 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1116 {
1117 free_page(vcpu->arch.sie_block->cbrlo);
1118 vcpu->arch.sie_block->cbrlo = 0;
1119 }
1120
1121 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1122 {
1123 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1124 if (!vcpu->arch.sie_block->cbrlo)
1125 return -ENOMEM;
1126
1127 vcpu->arch.sie_block->ecb2 |= 0x80;
1128 vcpu->arch.sie_block->ecb2 &= ~0x08;
1129 return 0;
1130 }
1131
1132 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1133 {
1134 int rc = 0;
1135
1136 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1137 CPUSTAT_SM |
1138 CPUSTAT_STOPPED |
1139 CPUSTAT_GED);
1140 vcpu->arch.sie_block->ecb = 6;
1141 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1142 vcpu->arch.sie_block->ecb |= 0x10;
1143
1144 vcpu->arch.sie_block->ecb2 = 8;
1145 vcpu->arch.sie_block->eca = 0xC1002000U;
1146 if (sclp_has_siif())
1147 vcpu->arch.sie_block->eca |= 1;
1148 if (sclp_has_sigpif())
1149 vcpu->arch.sie_block->eca |= 0x10000000U;
1150 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE |
1151 ICTL_TPROT;
1152
1153 if (kvm_s390_cmma_enabled(vcpu->kvm)) {
1154 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1155 if (rc)
1156 return rc;
1157 }
1158 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1159 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1160
1161 mutex_lock(&vcpu->kvm->lock);
1162 vcpu->arch.cpu_id = vcpu->kvm->arch.model.cpu_id;
1163 vcpu->arch.sie_block->ibc = vcpu->kvm->arch.model.ibc;
1164 mutex_unlock(&vcpu->kvm->lock);
1165
1166 kvm_s390_vcpu_crypto_setup(vcpu);
1167
1168 return rc;
1169 }
1170
1171 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1172 unsigned int id)
1173 {
1174 struct kvm_vcpu *vcpu;
1175 struct sie_page *sie_page;
1176 int rc = -EINVAL;
1177
1178 if (id >= KVM_MAX_VCPUS)
1179 goto out;
1180
1181 rc = -ENOMEM;
1182
1183 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1184 if (!vcpu)
1185 goto out;
1186
1187 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1188 if (!sie_page)
1189 goto out_free_cpu;
1190
1191 vcpu->arch.sie_block = &sie_page->sie_block;
1192 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1193
1194 vcpu->arch.sie_block->icpua = id;
1195 if (!kvm_is_ucontrol(kvm)) {
1196 if (!kvm->arch.sca) {
1197 WARN_ON_ONCE(1);
1198 goto out_free_cpu;
1199 }
1200 if (!kvm->arch.sca->cpu[id].sda)
1201 kvm->arch.sca->cpu[id].sda =
1202 (__u64) vcpu->arch.sie_block;
1203 vcpu->arch.sie_block->scaoh =
1204 (__u32)(((__u64)kvm->arch.sca) >> 32);
1205 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
1206 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
1207 }
1208 vcpu->arch.sie_block->fac = (int) (long) kvm->arch.model.fac->list;
1209
1210 spin_lock_init(&vcpu->arch.local_int.lock);
1211 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1212 vcpu->arch.local_int.wq = &vcpu->wq;
1213 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1214
1215 rc = kvm_vcpu_init(vcpu, kvm, id);
1216 if (rc)
1217 goto out_free_sie_block;
1218 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
1219 vcpu->arch.sie_block);
1220 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1221
1222 return vcpu;
1223 out_free_sie_block:
1224 free_page((unsigned long)(vcpu->arch.sie_block));
1225 out_free_cpu:
1226 kmem_cache_free(kvm_vcpu_cache, vcpu);
1227 out:
1228 return ERR_PTR(rc);
1229 }
1230
1231 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1232 {
1233 return kvm_s390_vcpu_has_irq(vcpu, 0);
1234 }
1235
1236 void s390_vcpu_block(struct kvm_vcpu *vcpu)
1237 {
1238 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1239 }
1240
1241 void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1242 {
1243 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1244 }
1245
1246 /*
1247 * Kick a guest cpu out of SIE and wait until SIE is not running.
1248 * If the CPU is not running (e.g. waiting as idle) the function will
1249 * return immediately. */
1250 void exit_sie(struct kvm_vcpu *vcpu)
1251 {
1252 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1253 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1254 cpu_relax();
1255 }
1256
1257 /* Kick a guest cpu out of SIE and prevent SIE-reentry */
1258 void exit_sie_sync(struct kvm_vcpu *vcpu)
1259 {
1260 s390_vcpu_block(vcpu);
1261 exit_sie(vcpu);
1262 }
1263
1264 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1265 {
1266 int i;
1267 struct kvm *kvm = gmap->private;
1268 struct kvm_vcpu *vcpu;
1269
1270 kvm_for_each_vcpu(i, vcpu, kvm) {
1271 /* match against both prefix pages */
1272 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
1273 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1274 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
1275 exit_sie_sync(vcpu);
1276 }
1277 }
1278 }
1279
1280 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1281 {
1282 /* kvm common code refers to this, but never calls it */
1283 BUG();
1284 return 0;
1285 }
1286
1287 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1288 struct kvm_one_reg *reg)
1289 {
1290 int r = -EINVAL;
1291
1292 switch (reg->id) {
1293 case KVM_REG_S390_TODPR:
1294 r = put_user(vcpu->arch.sie_block->todpr,
1295 (u32 __user *)reg->addr);
1296 break;
1297 case KVM_REG_S390_EPOCHDIFF:
1298 r = put_user(vcpu->arch.sie_block->epoch,
1299 (u64 __user *)reg->addr);
1300 break;
1301 case KVM_REG_S390_CPU_TIMER:
1302 r = put_user(vcpu->arch.sie_block->cputm,
1303 (u64 __user *)reg->addr);
1304 break;
1305 case KVM_REG_S390_CLOCK_COMP:
1306 r = put_user(vcpu->arch.sie_block->ckc,
1307 (u64 __user *)reg->addr);
1308 break;
1309 case KVM_REG_S390_PFTOKEN:
1310 r = put_user(vcpu->arch.pfault_token,
1311 (u64 __user *)reg->addr);
1312 break;
1313 case KVM_REG_S390_PFCOMPARE:
1314 r = put_user(vcpu->arch.pfault_compare,
1315 (u64 __user *)reg->addr);
1316 break;
1317 case KVM_REG_S390_PFSELECT:
1318 r = put_user(vcpu->arch.pfault_select,
1319 (u64 __user *)reg->addr);
1320 break;
1321 case KVM_REG_S390_PP:
1322 r = put_user(vcpu->arch.sie_block->pp,
1323 (u64 __user *)reg->addr);
1324 break;
1325 case KVM_REG_S390_GBEA:
1326 r = put_user(vcpu->arch.sie_block->gbea,
1327 (u64 __user *)reg->addr);
1328 break;
1329 default:
1330 break;
1331 }
1332
1333 return r;
1334 }
1335
1336 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1337 struct kvm_one_reg *reg)
1338 {
1339 int r = -EINVAL;
1340
1341 switch (reg->id) {
1342 case KVM_REG_S390_TODPR:
1343 r = get_user(vcpu->arch.sie_block->todpr,
1344 (u32 __user *)reg->addr);
1345 break;
1346 case KVM_REG_S390_EPOCHDIFF:
1347 r = get_user(vcpu->arch.sie_block->epoch,
1348 (u64 __user *)reg->addr);
1349 break;
1350 case KVM_REG_S390_CPU_TIMER:
1351 r = get_user(vcpu->arch.sie_block->cputm,
1352 (u64 __user *)reg->addr);
1353 break;
1354 case KVM_REG_S390_CLOCK_COMP:
1355 r = get_user(vcpu->arch.sie_block->ckc,
1356 (u64 __user *)reg->addr);
1357 break;
1358 case KVM_REG_S390_PFTOKEN:
1359 r = get_user(vcpu->arch.pfault_token,
1360 (u64 __user *)reg->addr);
1361 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1362 kvm_clear_async_pf_completion_queue(vcpu);
1363 break;
1364 case KVM_REG_S390_PFCOMPARE:
1365 r = get_user(vcpu->arch.pfault_compare,
1366 (u64 __user *)reg->addr);
1367 break;
1368 case KVM_REG_S390_PFSELECT:
1369 r = get_user(vcpu->arch.pfault_select,
1370 (u64 __user *)reg->addr);
1371 break;
1372 case KVM_REG_S390_PP:
1373 r = get_user(vcpu->arch.sie_block->pp,
1374 (u64 __user *)reg->addr);
1375 break;
1376 case KVM_REG_S390_GBEA:
1377 r = get_user(vcpu->arch.sie_block->gbea,
1378 (u64 __user *)reg->addr);
1379 break;
1380 default:
1381 break;
1382 }
1383
1384 return r;
1385 }
1386
1387 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1388 {
1389 kvm_s390_vcpu_initial_reset(vcpu);
1390 return 0;
1391 }
1392
1393 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1394 {
1395 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
1396 return 0;
1397 }
1398
1399 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1400 {
1401 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1402 return 0;
1403 }
1404
1405 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1406 struct kvm_sregs *sregs)
1407 {
1408 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
1409 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
1410 restore_access_regs(vcpu->run->s.regs.acrs);
1411 return 0;
1412 }
1413
1414 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1415 struct kvm_sregs *sregs)
1416 {
1417 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
1418 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
1419 return 0;
1420 }
1421
1422 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1423 {
1424 if (test_fp_ctl(fpu->fpc))
1425 return -EINVAL;
1426 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
1427 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1428 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1429 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1430 return 0;
1431 }
1432
1433 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1434 {
1435 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1436 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
1437 return 0;
1438 }
1439
1440 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1441 {
1442 int rc = 0;
1443
1444 if (!is_vcpu_stopped(vcpu))
1445 rc = -EBUSY;
1446 else {
1447 vcpu->run->psw_mask = psw.mask;
1448 vcpu->run->psw_addr = psw.addr;
1449 }
1450 return rc;
1451 }
1452
1453 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1454 struct kvm_translation *tr)
1455 {
1456 return -EINVAL; /* not implemented yet */
1457 }
1458
1459 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1460 KVM_GUESTDBG_USE_HW_BP | \
1461 KVM_GUESTDBG_ENABLE)
1462
1463 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1464 struct kvm_guest_debug *dbg)
1465 {
1466 int rc = 0;
1467
1468 vcpu->guest_debug = 0;
1469 kvm_s390_clear_bp_data(vcpu);
1470
1471 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
1472 return -EINVAL;
1473
1474 if (dbg->control & KVM_GUESTDBG_ENABLE) {
1475 vcpu->guest_debug = dbg->control;
1476 /* enforce guest PER */
1477 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1478
1479 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1480 rc = kvm_s390_import_bp_data(vcpu, dbg);
1481 } else {
1482 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1483 vcpu->arch.guestdbg.last_bp = 0;
1484 }
1485
1486 if (rc) {
1487 vcpu->guest_debug = 0;
1488 kvm_s390_clear_bp_data(vcpu);
1489 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1490 }
1491
1492 return rc;
1493 }
1494
1495 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1496 struct kvm_mp_state *mp_state)
1497 {
1498 /* CHECK_STOP and LOAD are not supported yet */
1499 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1500 KVM_MP_STATE_OPERATING;
1501 }
1502
1503 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1504 struct kvm_mp_state *mp_state)
1505 {
1506 int rc = 0;
1507
1508 /* user space knows about this interface - let it control the state */
1509 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1510
1511 switch (mp_state->mp_state) {
1512 case KVM_MP_STATE_STOPPED:
1513 kvm_s390_vcpu_stop(vcpu);
1514 break;
1515 case KVM_MP_STATE_OPERATING:
1516 kvm_s390_vcpu_start(vcpu);
1517 break;
1518 case KVM_MP_STATE_LOAD:
1519 case KVM_MP_STATE_CHECK_STOP:
1520 /* fall through - CHECK_STOP and LOAD are not supported yet */
1521 default:
1522 rc = -ENXIO;
1523 }
1524
1525 return rc;
1526 }
1527
1528 bool kvm_s390_cmma_enabled(struct kvm *kvm)
1529 {
1530 if (!MACHINE_IS_LPAR)
1531 return false;
1532 /* only enable for z10 and later */
1533 if (!MACHINE_HAS_EDAT1)
1534 return false;
1535 if (!kvm->arch.use_cmma)
1536 return false;
1537 return true;
1538 }
1539
1540 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1541 {
1542 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1543 }
1544
1545 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1546 {
1547 retry:
1548 s390_vcpu_unblock(vcpu);
1549 /*
1550 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1551 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1552 * This ensures that the ipte instruction for this request has
1553 * already finished. We might race against a second unmapper that
1554 * wants to set the blocking bit. Lets just retry the request loop.
1555 */
1556 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1557 int rc;
1558 rc = gmap_ipte_notify(vcpu->arch.gmap,
1559 kvm_s390_get_prefix(vcpu),
1560 PAGE_SIZE * 2);
1561 if (rc)
1562 return rc;
1563 goto retry;
1564 }
1565
1566 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1567 vcpu->arch.sie_block->ihcpu = 0xffff;
1568 goto retry;
1569 }
1570
1571 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1572 if (!ibs_enabled(vcpu)) {
1573 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1574 atomic_set_mask(CPUSTAT_IBS,
1575 &vcpu->arch.sie_block->cpuflags);
1576 }
1577 goto retry;
1578 }
1579
1580 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1581 if (ibs_enabled(vcpu)) {
1582 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1583 atomic_clear_mask(CPUSTAT_IBS,
1584 &vcpu->arch.sie_block->cpuflags);
1585 }
1586 goto retry;
1587 }
1588
1589 /* nothing to do, just clear the request */
1590 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1591
1592 return 0;
1593 }
1594
1595 /**
1596 * kvm_arch_fault_in_page - fault-in guest page if necessary
1597 * @vcpu: The corresponding virtual cpu
1598 * @gpa: Guest physical address
1599 * @writable: Whether the page should be writable or not
1600 *
1601 * Make sure that a guest page has been faulted-in on the host.
1602 *
1603 * Return: Zero on success, negative error code otherwise.
1604 */
1605 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
1606 {
1607 return gmap_fault(vcpu->arch.gmap, gpa,
1608 writable ? FAULT_FLAG_WRITE : 0);
1609 }
1610
1611 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1612 unsigned long token)
1613 {
1614 struct kvm_s390_interrupt inti;
1615 struct kvm_s390_irq irq;
1616
1617 if (start_token) {
1618 irq.u.ext.ext_params2 = token;
1619 irq.type = KVM_S390_INT_PFAULT_INIT;
1620 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
1621 } else {
1622 inti.type = KVM_S390_INT_PFAULT_DONE;
1623 inti.parm64 = token;
1624 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1625 }
1626 }
1627
1628 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1629 struct kvm_async_pf *work)
1630 {
1631 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1632 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1633 }
1634
1635 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1636 struct kvm_async_pf *work)
1637 {
1638 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1639 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1640 }
1641
1642 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1643 struct kvm_async_pf *work)
1644 {
1645 /* s390 will always inject the page directly */
1646 }
1647
1648 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1649 {
1650 /*
1651 * s390 will always inject the page directly,
1652 * but we still want check_async_completion to cleanup
1653 */
1654 return true;
1655 }
1656
1657 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1658 {
1659 hva_t hva;
1660 struct kvm_arch_async_pf arch;
1661 int rc;
1662
1663 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1664 return 0;
1665 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1666 vcpu->arch.pfault_compare)
1667 return 0;
1668 if (psw_extint_disabled(vcpu))
1669 return 0;
1670 if (kvm_s390_vcpu_has_irq(vcpu, 0))
1671 return 0;
1672 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1673 return 0;
1674 if (!vcpu->arch.gmap->pfault_enabled)
1675 return 0;
1676
1677 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1678 hva += current->thread.gmap_addr & ~PAGE_MASK;
1679 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
1680 return 0;
1681
1682 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1683 return rc;
1684 }
1685
1686 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
1687 {
1688 int rc, cpuflags;
1689
1690 /*
1691 * On s390 notifications for arriving pages will be delivered directly
1692 * to the guest but the house keeping for completed pfaults is
1693 * handled outside the worker.
1694 */
1695 kvm_check_async_pf_completion(vcpu);
1696
1697 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
1698
1699 if (need_resched())
1700 schedule();
1701
1702 if (test_cpu_flag(CIF_MCCK_PENDING))
1703 s390_handle_mcck();
1704
1705 if (!kvm_is_ucontrol(vcpu->kvm)) {
1706 rc = kvm_s390_deliver_pending_interrupts(vcpu);
1707 if (rc)
1708 return rc;
1709 }
1710
1711 rc = kvm_s390_handle_requests(vcpu);
1712 if (rc)
1713 return rc;
1714
1715 if (guestdbg_enabled(vcpu)) {
1716 kvm_s390_backup_guest_per_regs(vcpu);
1717 kvm_s390_patch_guest_per_regs(vcpu);
1718 }
1719
1720 vcpu->arch.sie_block->icptcode = 0;
1721 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1722 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1723 trace_kvm_s390_sie_enter(vcpu, cpuflags);
1724
1725 return 0;
1726 }
1727
1728 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1729 {
1730 int rc = -1;
1731
1732 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1733 vcpu->arch.sie_block->icptcode);
1734 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1735
1736 if (guestdbg_enabled(vcpu))
1737 kvm_s390_restore_guest_per_regs(vcpu);
1738
1739 if (exit_reason >= 0) {
1740 rc = 0;
1741 } else if (kvm_is_ucontrol(vcpu->kvm)) {
1742 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1743 vcpu->run->s390_ucontrol.trans_exc_code =
1744 current->thread.gmap_addr;
1745 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1746 rc = -EREMOTE;
1747
1748 } else if (current->thread.gmap_pfault) {
1749 trace_kvm_s390_major_guest_pfault(vcpu);
1750 current->thread.gmap_pfault = 0;
1751 if (kvm_arch_setup_async_pf(vcpu)) {
1752 rc = 0;
1753 } else {
1754 gpa_t gpa = current->thread.gmap_addr;
1755 rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1756 }
1757 }
1758
1759 if (rc == -1) {
1760 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1761 trace_kvm_s390_sie_fault(vcpu);
1762 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1763 }
1764
1765 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
1766
1767 if (rc == 0) {
1768 if (kvm_is_ucontrol(vcpu->kvm))
1769 /* Don't exit for host interrupts. */
1770 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
1771 else
1772 rc = kvm_handle_sie_intercept(vcpu);
1773 }
1774
1775 return rc;
1776 }
1777
1778 static int __vcpu_run(struct kvm_vcpu *vcpu)
1779 {
1780 int rc, exit_reason;
1781
1782 /*
1783 * We try to hold kvm->srcu during most of vcpu_run (except when run-
1784 * ning the guest), so that memslots (and other stuff) are protected
1785 */
1786 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1787
1788 do {
1789 rc = vcpu_pre_run(vcpu);
1790 if (rc)
1791 break;
1792
1793 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1794 /*
1795 * As PF_VCPU will be used in fault handler, between
1796 * guest_enter and guest_exit should be no uaccess.
1797 */
1798 preempt_disable();
1799 kvm_guest_enter();
1800 preempt_enable();
1801 exit_reason = sie64a(vcpu->arch.sie_block,
1802 vcpu->run->s.regs.gprs);
1803 kvm_guest_exit();
1804 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1805
1806 rc = vcpu_post_run(vcpu, exit_reason);
1807 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
1808
1809 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1810 return rc;
1811 }
1812
1813 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1814 {
1815 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1816 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
1817 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
1818 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1819 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1820 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
1821 /* some control register changes require a tlb flush */
1822 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1823 }
1824 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
1825 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
1826 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
1827 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
1828 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
1829 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
1830 }
1831 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
1832 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
1833 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
1834 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
1835 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1836 kvm_clear_async_pf_completion_queue(vcpu);
1837 }
1838 kvm_run->kvm_dirty_regs = 0;
1839 }
1840
1841 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1842 {
1843 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
1844 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
1845 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
1846 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
1847 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
1848 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
1849 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
1850 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
1851 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
1852 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
1853 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
1854 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
1855 }
1856
1857 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1858 {
1859 int rc;
1860 sigset_t sigsaved;
1861
1862 if (guestdbg_exit_pending(vcpu)) {
1863 kvm_s390_prepare_debug_exit(vcpu);
1864 return 0;
1865 }
1866
1867 if (vcpu->sigset_active)
1868 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1869
1870 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
1871 kvm_s390_vcpu_start(vcpu);
1872 } else if (is_vcpu_stopped(vcpu)) {
1873 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
1874 vcpu->vcpu_id);
1875 return -EINVAL;
1876 }
1877
1878 sync_regs(vcpu, kvm_run);
1879
1880 might_fault();
1881 rc = __vcpu_run(vcpu);
1882
1883 if (signal_pending(current) && !rc) {
1884 kvm_run->exit_reason = KVM_EXIT_INTR;
1885 rc = -EINTR;
1886 }
1887
1888 if (guestdbg_exit_pending(vcpu) && !rc) {
1889 kvm_s390_prepare_debug_exit(vcpu);
1890 rc = 0;
1891 }
1892
1893 if (rc == -EOPNOTSUPP) {
1894 /* intercept cannot be handled in-kernel, prepare kvm-run */
1895 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
1896 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
1897 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
1898 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
1899 rc = 0;
1900 }
1901
1902 if (rc == -EREMOTE) {
1903 /* intercept was handled, but userspace support is needed
1904 * kvm_run has been prepared by the handler */
1905 rc = 0;
1906 }
1907
1908 store_regs(vcpu, kvm_run);
1909
1910 if (vcpu->sigset_active)
1911 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1912
1913 vcpu->stat.exit_userspace++;
1914 return rc;
1915 }
1916
1917 /*
1918 * store status at address
1919 * we use have two special cases:
1920 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1921 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1922 */
1923 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
1924 {
1925 unsigned char archmode = 1;
1926 unsigned int px;
1927 u64 clkcomp;
1928 int rc;
1929
1930 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
1931 if (write_guest_abs(vcpu, 163, &archmode, 1))
1932 return -EFAULT;
1933 gpa = SAVE_AREA_BASE;
1934 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
1935 if (write_guest_real(vcpu, 163, &archmode, 1))
1936 return -EFAULT;
1937 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
1938 }
1939 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
1940 vcpu->arch.guest_fpregs.fprs, 128);
1941 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
1942 vcpu->run->s.regs.gprs, 128);
1943 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
1944 &vcpu->arch.sie_block->gpsw, 16);
1945 px = kvm_s390_get_prefix(vcpu);
1946 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
1947 &px, 4);
1948 rc |= write_guest_abs(vcpu,
1949 gpa + offsetof(struct save_area, fp_ctrl_reg),
1950 &vcpu->arch.guest_fpregs.fpc, 4);
1951 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
1952 &vcpu->arch.sie_block->todpr, 4);
1953 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
1954 &vcpu->arch.sie_block->cputm, 8);
1955 clkcomp = vcpu->arch.sie_block->ckc >> 8;
1956 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
1957 &clkcomp, 8);
1958 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
1959 &vcpu->run->s.regs.acrs, 64);
1960 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
1961 &vcpu->arch.sie_block->gcr, 128);
1962 return rc ? -EFAULT : 0;
1963 }
1964
1965 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1966 {
1967 /*
1968 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1969 * copying in vcpu load/put. Lets update our copies before we save
1970 * it into the save area
1971 */
1972 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1973 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1974 save_access_regs(vcpu->run->s.regs.acrs);
1975
1976 return kvm_s390_store_status_unloaded(vcpu, addr);
1977 }
1978
1979 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1980 {
1981 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
1982 kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu);
1983 exit_sie_sync(vcpu);
1984 }
1985
1986 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
1987 {
1988 unsigned int i;
1989 struct kvm_vcpu *vcpu;
1990
1991 kvm_for_each_vcpu(i, vcpu, kvm) {
1992 __disable_ibs_on_vcpu(vcpu);
1993 }
1994 }
1995
1996 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1997 {
1998 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
1999 kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu);
2000 exit_sie_sync(vcpu);
2001 }
2002
2003 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2004 {
2005 int i, online_vcpus, started_vcpus = 0;
2006
2007 if (!is_vcpu_stopped(vcpu))
2008 return;
2009
2010 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2011 /* Only one cpu at a time may enter/leave the STOPPED state. */
2012 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2013 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2014
2015 for (i = 0; i < online_vcpus; i++) {
2016 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2017 started_vcpus++;
2018 }
2019
2020 if (started_vcpus == 0) {
2021 /* we're the only active VCPU -> speed it up */
2022 __enable_ibs_on_vcpu(vcpu);
2023 } else if (started_vcpus == 1) {
2024 /*
2025 * As we are starting a second VCPU, we have to disable
2026 * the IBS facility on all VCPUs to remove potentially
2027 * oustanding ENABLE requests.
2028 */
2029 __disable_ibs_on_all_vcpus(vcpu->kvm);
2030 }
2031
2032 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2033 /*
2034 * Another VCPU might have used IBS while we were offline.
2035 * Let's play safe and flush the VCPU at startup.
2036 */
2037 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2038 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2039 return;
2040 }
2041
2042 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2043 {
2044 int i, online_vcpus, started_vcpus = 0;
2045 struct kvm_vcpu *started_vcpu = NULL;
2046
2047 if (is_vcpu_stopped(vcpu))
2048 return;
2049
2050 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2051 /* Only one cpu at a time may enter/leave the STOPPED state. */
2052 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2053 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2054
2055 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2056 kvm_s390_clear_stop_irq(vcpu);
2057
2058 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2059 __disable_ibs_on_vcpu(vcpu);
2060
2061 for (i = 0; i < online_vcpus; i++) {
2062 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2063 started_vcpus++;
2064 started_vcpu = vcpu->kvm->vcpus[i];
2065 }
2066 }
2067
2068 if (started_vcpus == 1) {
2069 /*
2070 * As we only have one VCPU left, we want to enable the
2071 * IBS facility for that VCPU to speed it up.
2072 */
2073 __enable_ibs_on_vcpu(started_vcpu);
2074 }
2075
2076 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2077 return;
2078 }
2079
2080 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2081 struct kvm_enable_cap *cap)
2082 {
2083 int r;
2084
2085 if (cap->flags)
2086 return -EINVAL;
2087
2088 switch (cap->cap) {
2089 case KVM_CAP_S390_CSS_SUPPORT:
2090 if (!vcpu->kvm->arch.css_support) {
2091 vcpu->kvm->arch.css_support = 1;
2092 trace_kvm_s390_enable_css(vcpu->kvm);
2093 }
2094 r = 0;
2095 break;
2096 default:
2097 r = -EINVAL;
2098 break;
2099 }
2100 return r;
2101 }
2102
2103 long kvm_arch_vcpu_ioctl(struct file *filp,
2104 unsigned int ioctl, unsigned long arg)
2105 {
2106 struct kvm_vcpu *vcpu = filp->private_data;
2107 void __user *argp = (void __user *)arg;
2108 int idx;
2109 long r;
2110
2111 switch (ioctl) {
2112 case KVM_S390_INTERRUPT: {
2113 struct kvm_s390_interrupt s390int;
2114 struct kvm_s390_irq s390irq;
2115
2116 r = -EFAULT;
2117 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2118 break;
2119 if (s390int_to_s390irq(&s390int, &s390irq))
2120 return -EINVAL;
2121 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2122 break;
2123 }
2124 case KVM_S390_STORE_STATUS:
2125 idx = srcu_read_lock(&vcpu->kvm->srcu);
2126 r = kvm_s390_vcpu_store_status(vcpu, arg);
2127 srcu_read_unlock(&vcpu->kvm->srcu, idx);
2128 break;
2129 case KVM_S390_SET_INITIAL_PSW: {
2130 psw_t psw;
2131
2132 r = -EFAULT;
2133 if (copy_from_user(&psw, argp, sizeof(psw)))
2134 break;
2135 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2136 break;
2137 }
2138 case KVM_S390_INITIAL_RESET:
2139 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2140 break;
2141 case KVM_SET_ONE_REG:
2142 case KVM_GET_ONE_REG: {
2143 struct kvm_one_reg reg;
2144 r = -EFAULT;
2145 if (copy_from_user(&reg, argp, sizeof(reg)))
2146 break;
2147 if (ioctl == KVM_SET_ONE_REG)
2148 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
2149 else
2150 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
2151 break;
2152 }
2153 #ifdef CONFIG_KVM_S390_UCONTROL
2154 case KVM_S390_UCAS_MAP: {
2155 struct kvm_s390_ucas_mapping ucasmap;
2156
2157 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2158 r = -EFAULT;
2159 break;
2160 }
2161
2162 if (!kvm_is_ucontrol(vcpu->kvm)) {
2163 r = -EINVAL;
2164 break;
2165 }
2166
2167 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2168 ucasmap.vcpu_addr, ucasmap.length);
2169 break;
2170 }
2171 case KVM_S390_UCAS_UNMAP: {
2172 struct kvm_s390_ucas_mapping ucasmap;
2173
2174 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2175 r = -EFAULT;
2176 break;
2177 }
2178
2179 if (!kvm_is_ucontrol(vcpu->kvm)) {
2180 r = -EINVAL;
2181 break;
2182 }
2183
2184 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2185 ucasmap.length);
2186 break;
2187 }
2188 #endif
2189 case KVM_S390_VCPU_FAULT: {
2190 r = gmap_fault(vcpu->arch.gmap, arg, 0);
2191 break;
2192 }
2193 case KVM_ENABLE_CAP:
2194 {
2195 struct kvm_enable_cap cap;
2196 r = -EFAULT;
2197 if (copy_from_user(&cap, argp, sizeof(cap)))
2198 break;
2199 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2200 break;
2201 }
2202 default:
2203 r = -ENOTTY;
2204 }
2205 return r;
2206 }
2207
2208 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2209 {
2210 #ifdef CONFIG_KVM_S390_UCONTROL
2211 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2212 && (kvm_is_ucontrol(vcpu->kvm))) {
2213 vmf->page = virt_to_page(vcpu->arch.sie_block);
2214 get_page(vmf->page);
2215 return 0;
2216 }
2217 #endif
2218 return VM_FAULT_SIGBUS;
2219 }
2220
2221 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2222 unsigned long npages)
2223 {
2224 return 0;
2225 }
2226
2227 /* Section: memory related */
2228 int kvm_arch_prepare_memory_region(struct kvm *kvm,
2229 struct kvm_memory_slot *memslot,
2230 struct kvm_userspace_memory_region *mem,
2231 enum kvm_mr_change change)
2232 {
2233 /* A few sanity checks. We can have memory slots which have to be
2234 located/ended at a segment boundary (1MB). The memory in userland is
2235 ok to be fragmented into various different vmas. It is okay to mmap()
2236 and munmap() stuff in this slot after doing this call at any time */
2237
2238 if (mem->userspace_addr & 0xffffful)
2239 return -EINVAL;
2240
2241 if (mem->memory_size & 0xffffful)
2242 return -EINVAL;
2243
2244 return 0;
2245 }
2246
2247 void kvm_arch_commit_memory_region(struct kvm *kvm,
2248 struct kvm_userspace_memory_region *mem,
2249 const struct kvm_memory_slot *old,
2250 enum kvm_mr_change change)
2251 {
2252 int rc;
2253
2254 /* If the basics of the memslot do not change, we do not want
2255 * to update the gmap. Every update causes several unnecessary
2256 * segment translation exceptions. This is usually handled just
2257 * fine by the normal fault handler + gmap, but it will also
2258 * cause faults on the prefix page of running guest CPUs.
2259 */
2260 if (old->userspace_addr == mem->userspace_addr &&
2261 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2262 old->npages * PAGE_SIZE == mem->memory_size)
2263 return;
2264
2265 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2266 mem->guest_phys_addr, mem->memory_size);
2267 if (rc)
2268 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
2269 return;
2270 }
2271
2272 static int __init kvm_s390_init(void)
2273 {
2274 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2275 }
2276
2277 static void __exit kvm_s390_exit(void)
2278 {
2279 kvm_exit();
2280 }
2281
2282 module_init(kvm_s390_init);
2283 module_exit(kvm_s390_exit);
2284
2285 /*
2286 * Enable autoloading of the kvm module.
2287 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2288 * since x86 takes a different approach.
2289 */
2290 #include <linux/miscdevice.h>
2291 MODULE_ALIAS_MISCDEV(KVM_MINOR);
2292 MODULE_ALIAS("devname:kvm");
Ubuntu Artful kernel mirror