2 * kvm nested virtualization support for s390x
4 * Copyright IBM Corp. 2016
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
10 * Author(s): David Hildenbrand <dahi@linux.vnet.ibm.com>
12 #include <linux/vmalloc.h>
13 #include <linux/kvm_host.h>
14 #include <linux/bug.h>
15 #include <linux/list.h>
16 #include <linux/bitmap.h>
18 #include <asm/mmu_context.h>
25 struct kvm_s390_sie_block scb_s
; /* 0x0000 */
26 /* the pinned originial scb */
27 struct kvm_s390_sie_block
*scb_o
; /* 0x0200 */
28 /* the shadow gmap in use by the vsie_page */
29 struct gmap
*gmap
; /* 0x0208 */
30 __u8 reserved
[0x1000 - 0x0210]; /* 0x0210 */
33 /* trigger a validity icpt for the given scb */
34 static int set_validity_icpt(struct kvm_s390_sie_block
*scb
,
38 scb
->ipb
= ((__u32
) reason_code
) << 16;
39 scb
->icptcode
= ICPT_VALIDITY
;
43 /* mark the prefix as unmapped, this will block the VSIE */
44 static void prefix_unmapped(struct vsie_page
*vsie_page
)
46 atomic_or(PROG_REQUEST
, &vsie_page
->scb_s
.prog20
);
49 /* mark the prefix as unmapped and wait until the VSIE has been left */
50 static void prefix_unmapped_sync(struct vsie_page
*vsie_page
)
52 prefix_unmapped(vsie_page
);
53 if (vsie_page
->scb_s
.prog0c
& PROG_IN_SIE
)
54 atomic_or(CPUSTAT_STOP_INT
, &vsie_page
->scb_s
.cpuflags
);
55 while (vsie_page
->scb_s
.prog0c
& PROG_IN_SIE
)
59 /* mark the prefix as mapped, this will allow the VSIE to run */
60 static void prefix_mapped(struct vsie_page
*vsie_page
)
62 atomic_andnot(PROG_REQUEST
, &vsie_page
->scb_s
.prog20
);
65 /* test if the prefix is mapped into the gmap shadow */
66 static int prefix_is_mapped(struct vsie_page
*vsie_page
)
68 return !(atomic_read(&vsie_page
->scb_s
.prog20
) & PROG_REQUEST
);
71 /* copy the updated intervention request bits into the shadow scb */
72 static void update_intervention_requests(struct vsie_page
*vsie_page
)
74 const int bits
= CPUSTAT_STOP_INT
| CPUSTAT_IO_INT
| CPUSTAT_EXT_INT
;
77 cpuflags
= atomic_read(&vsie_page
->scb_o
->cpuflags
);
78 atomic_andnot(bits
, &vsie_page
->scb_s
.cpuflags
);
79 atomic_or(cpuflags
& bits
, &vsie_page
->scb_s
.cpuflags
);
82 /* shadow (filter and validate) the cpuflags */
83 static int prepare_cpuflags(struct kvm_vcpu
*vcpu
, struct vsie_page
*vsie_page
)
85 struct kvm_s390_sie_block
*scb_s
= &vsie_page
->scb_s
;
86 struct kvm_s390_sie_block
*scb_o
= vsie_page
->scb_o
;
87 int newflags
, cpuflags
= atomic_read(&scb_o
->cpuflags
);
89 /* we don't allow ESA/390 guests */
90 if (!(cpuflags
& CPUSTAT_ZARCH
))
91 return set_validity_icpt(scb_s
, 0x0001U
);
93 if (cpuflags
& (CPUSTAT_RRF
| CPUSTAT_MCDS
))
94 return set_validity_icpt(scb_s
, 0x0001U
);
95 else if (cpuflags
& (CPUSTAT_SLSV
| CPUSTAT_SLSR
))
96 return set_validity_icpt(scb_s
, 0x0007U
);
98 /* intervention requests will be set later */
99 newflags
= CPUSTAT_ZARCH
;
101 atomic_set(&scb_s
->cpuflags
, newflags
);
105 /* shadow (round up/down) the ibc to avoid validity icpt */
106 static void prepare_ibc(struct kvm_vcpu
*vcpu
, struct vsie_page
*vsie_page
)
108 struct kvm_s390_sie_block
*scb_s
= &vsie_page
->scb_s
;
109 struct kvm_s390_sie_block
*scb_o
= vsie_page
->scb_o
;
110 __u64 min_ibc
= (sclp
.ibc
>> 16) & 0x0fffU
;
113 /* ibc installed in g2 and requested for g3 */
114 if (vcpu
->kvm
->arch
.model
.ibc
&& (scb_o
->ibc
& 0x0fffU
)) {
115 scb_s
->ibc
= scb_o
->ibc
& 0x0fffU
;
116 /* takte care of the minimum ibc level of the machine */
117 if (scb_s
->ibc
< min_ibc
)
118 scb_s
->ibc
= min_ibc
;
119 /* take care of the maximum ibc level set for the guest */
120 if (scb_s
->ibc
> vcpu
->kvm
->arch
.model
.ibc
)
121 scb_s
->ibc
= vcpu
->kvm
->arch
.model
.ibc
;
125 /* unshadow the scb, copying parameters back to the real scb */
126 static void unshadow_scb(struct kvm_vcpu
*vcpu
, struct vsie_page
*vsie_page
)
128 struct kvm_s390_sie_block
*scb_s
= &vsie_page
->scb_s
;
129 struct kvm_s390_sie_block
*scb_o
= vsie_page
->scb_o
;
132 scb_o
->icptcode
= scb_s
->icptcode
;
133 scb_o
->icptstatus
= scb_s
->icptstatus
;
134 scb_o
->ipa
= scb_s
->ipa
;
135 scb_o
->ipb
= scb_s
->ipb
;
136 scb_o
->gbea
= scb_s
->gbea
;
139 scb_o
->cputm
= scb_s
->cputm
;
140 scb_o
->ckc
= scb_s
->ckc
;
141 scb_o
->todpr
= scb_s
->todpr
;
144 scb_o
->gpsw
= scb_s
->gpsw
;
145 scb_o
->gg14
= scb_s
->gg14
;
146 scb_o
->gg15
= scb_s
->gg15
;
147 memcpy(scb_o
->gcr
, scb_s
->gcr
, 128);
148 scb_o
->pp
= scb_s
->pp
;
150 /* interrupt intercept */
151 switch (scb_s
->icptcode
) {
155 memcpy((void *)((u64
)scb_o
+ 0xc0),
156 (void *)((u64
)scb_s
+ 0xc0), 0xf0 - 0xc0);
160 memcpy((void *)((u64
)scb_o
+ 0xc0),
161 (void *)((u64
)scb_s
+ 0xc0), 0xd0 - 0xc0);
165 if (scb_s
->ihcpu
!= 0xffffU
)
166 scb_o
->ihcpu
= scb_s
->ihcpu
;
170 * Setup the shadow scb by copying and checking the relevant parts of the g2
173 * Returns: - 0 if the scb has been shadowed
174 * - > 0 if control has to be given to guest 2
176 static int shadow_scb(struct kvm_vcpu
*vcpu
, struct vsie_page
*vsie_page
)
178 struct kvm_s390_sie_block
*scb_o
= vsie_page
->scb_o
;
179 struct kvm_s390_sie_block
*scb_s
= &vsie_page
->scb_s
;
180 unsigned long new_mso
;
183 /* make sure we don't have any leftovers when reusing the scb */
191 rc
= prepare_cpuflags(vcpu
, vsie_page
);
196 scb_s
->cputm
= scb_o
->cputm
;
197 scb_s
->ckc
= scb_o
->ckc
;
198 scb_s
->todpr
= scb_o
->todpr
;
199 scb_s
->epoch
= scb_o
->epoch
;
202 scb_s
->gpsw
= scb_o
->gpsw
;
203 scb_s
->gg14
= scb_o
->gg14
;
204 scb_s
->gg15
= scb_o
->gg15
;
205 memcpy(scb_s
->gcr
, scb_o
->gcr
, 128);
206 scb_s
->pp
= scb_o
->pp
;
208 /* interception / execution handling */
209 scb_s
->gbea
= scb_o
->gbea
;
210 scb_s
->lctl
= scb_o
->lctl
;
211 scb_s
->svcc
= scb_o
->svcc
;
212 scb_s
->ictl
= scb_o
->ictl
;
214 * SKEY handling functions can't deal with false setting of PTE invalid
215 * bits. Therefore we cannot provide interpretation and would later
216 * have to provide own emulation handlers.
218 scb_s
->ictl
|= ICTL_ISKE
| ICTL_SSKE
| ICTL_RRBE
;
219 scb_s
->icpua
= scb_o
->icpua
;
221 new_mso
= scb_o
->mso
& 0xfffffffffff00000UL
;
222 /* if the hva of the prefix changes, we have to remap the prefix */
223 if (scb_s
->mso
!= new_mso
|| scb_s
->prefix
!= scb_o
->prefix
)
224 prefix_unmapped(vsie_page
);
225 /* SIE will do mso/msl validity and exception checks for us */
226 scb_s
->msl
= scb_o
->msl
& 0xfffffffffff00000UL
;
227 scb_s
->mso
= new_mso
;
228 scb_s
->prefix
= scb_o
->prefix
;
230 /* We have to definetly flush the tlb if this scb never ran */
231 if (scb_s
->ihcpu
!= 0xffffU
)
232 scb_s
->ihcpu
= scb_o
->ihcpu
;
234 /* MVPG and Protection Exception Interpretation are always available */
235 scb_s
->eca
|= scb_o
->eca
& 0x01002000U
;
237 prepare_ibc(vcpu
, vsie_page
);
240 unshadow_scb(vcpu
, vsie_page
);
244 void kvm_s390_vsie_gmap_notifier(struct gmap
*gmap
, unsigned long start
,
247 struct kvm
*kvm
= gmap
->private;
248 struct vsie_page
*cur
;
249 unsigned long prefix
;
253 if (!gmap_is_shadow(gmap
))
255 if (start
>= 1UL << 31)
256 /* We are only interested in prefix pages */
260 * Only new shadow blocks are added to the list during runtime,
261 * therefore we can safely reference them all the time.
263 for (i
= 0; i
< kvm
->arch
.vsie
.page_count
; i
++) {
264 page
= READ_ONCE(kvm
->arch
.vsie
.pages
[i
]);
267 cur
= page_to_virt(page
);
268 if (READ_ONCE(cur
->gmap
) != gmap
)
270 prefix
= cur
->scb_s
.prefix
<< GUEST_PREFIX_SHIFT
;
271 /* with mso/msl, the prefix lies at an offset */
272 prefix
+= cur
->scb_s
.mso
;
273 if (prefix
<= end
&& start
<= prefix
+ PAGE_SIZE
- 1)
274 prefix_unmapped_sync(cur
);
279 * Map the first prefix page.
281 * The prefix will be protected, a gmap notifier will inform about unmaps.
282 * The shadow scb must not be executed until the prefix is remapped, this is
283 * guaranteed by properly handling PROG_REQUEST.
285 * Returns: - 0 on if successfully mapped or already mapped
286 * - > 0 if control has to be given to guest 2
287 * - -EAGAIN if the caller can retry immediately
288 * - -ENOMEM if out of memory
290 static int map_prefix(struct kvm_vcpu
*vcpu
, struct vsie_page
*vsie_page
)
292 struct kvm_s390_sie_block
*scb_s
= &vsie_page
->scb_s
;
293 u64 prefix
= scb_s
->prefix
<< GUEST_PREFIX_SHIFT
;
296 if (prefix_is_mapped(vsie_page
))
299 /* mark it as mapped so we can catch any concurrent unmappers */
300 prefix_mapped(vsie_page
);
302 /* with mso/msl, the prefix lies at offset *mso* */
303 prefix
+= scb_s
->mso
;
305 rc
= kvm_s390_shadow_fault(vcpu
, vsie_page
->gmap
, prefix
);
307 * We don't have to mprotect, we will be called for all unshadows.
308 * SIE will detect if protection applies and trigger a validity.
311 prefix_unmapped(vsie_page
);
312 if (rc
> 0 || rc
== -EFAULT
)
313 rc
= set_validity_icpt(scb_s
, 0x0037U
);
318 * Pin the guest page given by gpa and set hpa to the pinned host address.
319 * Will always be pinned writable.
321 * Returns: - 0 on success
322 * - -EINVAL if the gpa is not valid guest storage
323 * - -ENOMEM if out of memory
325 static int pin_guest_page(struct kvm
*kvm
, gpa_t gpa
, hpa_t
*hpa
)
331 hva
= gfn_to_hva(kvm
, gpa_to_gfn(gpa
));
332 if (kvm_is_error_hva(hva
))
334 rc
= get_user_pages_fast(hva
, 1, 1, &page
);
339 *hpa
= (hpa_t
) page_to_virt(page
) + (gpa
& ~PAGE_MASK
);
343 /* Unpins a page previously pinned via pin_guest_page, marking it as dirty. */
344 static void unpin_guest_page(struct kvm
*kvm
, gpa_t gpa
, hpa_t hpa
)
348 page
= virt_to_page(hpa
);
349 set_page_dirty_lock(page
);
351 /* mark the page always as dirty for migration */
352 mark_page_dirty(kvm
, gpa_to_gfn(gpa
));
355 /* unpin all blocks previously pinned by pin_blocks(), marking them dirty */
356 static void unpin_blocks(struct kvm_vcpu
*vcpu
, struct vsie_page
*vsie_page
)
358 struct kvm_s390_sie_block
*scb_o
= vsie_page
->scb_o
;
359 struct kvm_s390_sie_block
*scb_s
= &vsie_page
->scb_s
;
363 hpa
= (u64
) scb_s
->scaoh
<< 32 | scb_s
->scaol
;
365 gpa
= scb_o
->scaol
& ~0xfUL
;
366 unpin_guest_page(vcpu
->kvm
, gpa
, hpa
);
373 * Instead of shadowing some blocks, we can simply forward them because the
374 * addresses in the scb are 64 bit long.
376 * This works as long as the data lies in one page. If blocks ever exceed one
377 * page, we have to fall back to shadowing.
379 * As we reuse the sca, the vcpu pointers contained in it are invalid. We must
380 * therefore not enable any facilities that access these pointers (e.g. SIGPIF).
382 * Returns: - 0 if all blocks were pinned.
383 * - > 0 if control has to be given to guest 2
384 * - -ENOMEM if out of memory
386 static int pin_blocks(struct kvm_vcpu
*vcpu
, struct vsie_page
*vsie_page
)
388 struct kvm_s390_sie_block
*scb_o
= vsie_page
->scb_o
;
389 struct kvm_s390_sie_block
*scb_s
= &vsie_page
->scb_s
;
394 gpa
= scb_o
->scaol
& ~0xfUL
;
396 if (!(gpa
& ~0x1fffUL
))
397 rc
= set_validity_icpt(scb_s
, 0x0038U
);
398 else if ((gpa
& ~0x1fffUL
) == kvm_s390_get_prefix(vcpu
))
399 rc
= set_validity_icpt(scb_s
, 0x0011U
);
400 else if ((gpa
& PAGE_MASK
) !=
401 ((gpa
+ sizeof(struct bsca_block
) - 1) & PAGE_MASK
))
402 rc
= set_validity_icpt(scb_s
, 0x003bU
);
404 rc
= pin_guest_page(vcpu
->kvm
, gpa
, &hpa
);
406 rc
= set_validity_icpt(scb_s
, 0x0034U
);
410 scb_s
->scaoh
= (u32
)((u64
)hpa
>> 32);
411 scb_s
->scaol
= (u32
)(u64
)hpa
;
415 unpin_blocks(vcpu
, vsie_page
);
419 /* unpin the scb provided by guest 2, marking it as dirty */
420 static void unpin_scb(struct kvm_vcpu
*vcpu
, struct vsie_page
*vsie_page
,
423 hpa_t hpa
= (hpa_t
) vsie_page
->scb_o
;
426 unpin_guest_page(vcpu
->kvm
, gpa
, hpa
);
427 vsie_page
->scb_o
= NULL
;
431 * Pin the scb at gpa provided by guest 2 at vsie_page->scb_o.
433 * Returns: - 0 if the scb was pinned.
434 * - > 0 if control has to be given to guest 2
435 * - -ENOMEM if out of memory
437 static int pin_scb(struct kvm_vcpu
*vcpu
, struct vsie_page
*vsie_page
,
443 rc
= pin_guest_page(vcpu
->kvm
, gpa
, &hpa
);
445 rc
= kvm_s390_inject_program_int(vcpu
, PGM_ADDRESSING
);
450 vsie_page
->scb_o
= (struct kvm_s390_sie_block
*) hpa
;
455 * Inject a fault into guest 2.
457 * Returns: - > 0 if control has to be given to guest 2
458 * < 0 if an error occurred during injection.
460 static int inject_fault(struct kvm_vcpu
*vcpu
, __u16 code
, __u64 vaddr
,
463 struct kvm_s390_pgm_info pgm
= {
466 /* 0-51: virtual address */
467 (vaddr
& 0xfffffffffffff000UL
) |
468 /* 52-53: store / fetch */
469 (((unsigned int) !write_flag
) + 1) << 10,
470 /* 62-63: asce id (alway primary == 0) */
471 .exc_access_id
= 0, /* always primary */
472 .op_access_id
= 0, /* not MVPG */
476 if (code
== PGM_PROTECTION
)
477 pgm
.trans_exc_code
|= 0x4UL
;
479 rc
= kvm_s390_inject_prog_irq(vcpu
, &pgm
);
484 * Handle a fault during vsie execution on a gmap shadow.
486 * Returns: - 0 if the fault was resolved
487 * - > 0 if control has to be given to guest 2
488 * - < 0 if an error occurred
490 static int handle_fault(struct kvm_vcpu
*vcpu
, struct vsie_page
*vsie_page
)
494 if (current
->thread
.gmap_int_code
== PGM_PROTECTION
)
495 /* we can directly forward all protection exceptions */
496 return inject_fault(vcpu
, PGM_PROTECTION
,
497 current
->thread
.gmap_addr
, 1);
499 rc
= kvm_s390_shadow_fault(vcpu
, vsie_page
->gmap
,
500 current
->thread
.gmap_addr
);
502 rc
= inject_fault(vcpu
, rc
,
503 current
->thread
.gmap_addr
,
504 current
->thread
.gmap_write_flag
);
509 static inline void clear_vsie_icpt(struct vsie_page
*vsie_page
)
511 vsie_page
->scb_s
.icptcode
= 0;
515 * Run the vsie on a shadow scb and a shadow gmap, without any further
516 * sanity checks, handling SIE faults.
518 * Returns: - 0 everything went fine
519 * - > 0 if control has to be given to guest 2
520 * - < 0 if an error occurred
522 static int do_vsie_run(struct kvm_vcpu
*vcpu
, struct vsie_page
*vsie_page
)
524 struct kvm_s390_sie_block
*scb_s
= &vsie_page
->scb_s
;
525 struct kvm_s390_sie_block
*scb_o
= vsie_page
->scb_o
;
530 if (test_cpu_flag(CIF_MCCK_PENDING
))
533 srcu_read_unlock(&vcpu
->kvm
->srcu
, vcpu
->srcu_idx
);
538 rc
= sie64a(scb_s
, vcpu
->run
->s
.regs
.gprs
);
543 vcpu
->srcu_idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
546 rc
= 0; /* we could still have an icpt */
547 else if (rc
== -EFAULT
)
548 return handle_fault(vcpu
, vsie_page
);
550 switch (scb_s
->icptcode
) {
552 /* stop not requested by g2 - must have been a kick */
553 if (!(atomic_read(&scb_o
->cpuflags
) & CPUSTAT_STOP_INT
))
554 clear_vsie_icpt(vsie_page
);
557 if ((scb_s
->ipa
& 0xf000) != 0xf000)
558 scb_s
->ipa
+= 0x1000;
564 static void release_gmap_shadow(struct vsie_page
*vsie_page
)
567 gmap_put(vsie_page
->gmap
);
568 WRITE_ONCE(vsie_page
->gmap
, NULL
);
569 prefix_unmapped(vsie_page
);
572 static int acquire_gmap_shadow(struct kvm_vcpu
*vcpu
,
573 struct vsie_page
*vsie_page
)
580 asce
= vcpu
->arch
.sie_block
->gcr
[1];
581 cr0
.val
= vcpu
->arch
.sie_block
->gcr
[0];
582 edat
= cr0
.edat
&& test_kvm_facility(vcpu
->kvm
, 8);
583 edat
+= edat
&& test_kvm_facility(vcpu
->kvm
, 78);
586 * ASCE or EDAT could have changed since last icpt, or the gmap
587 * we're holding has been unshadowed. If the gmap is still valid,
588 * we can safely reuse it.
590 if (vsie_page
->gmap
&& gmap_shadow_valid(vsie_page
->gmap
, asce
, edat
))
593 /* release the old shadow - if any, and mark the prefix as unmapped */
594 release_gmap_shadow(vsie_page
);
595 gmap
= gmap_shadow(vcpu
->arch
.gmap
, asce
, edat
);
597 return PTR_ERR(gmap
);
598 gmap
->private = vcpu
->kvm
;
599 WRITE_ONCE(vsie_page
->gmap
, gmap
);
604 * Run the vsie on a shadowed scb, managing the gmap shadow, handling
605 * prefix pages and faults.
607 * Returns: - 0 if no errors occurred
608 * - > 0 if control has to be given to guest 2
609 * - -ENOMEM if out of memory
611 static int vsie_run(struct kvm_vcpu
*vcpu
, struct vsie_page
*vsie_page
)
613 struct kvm_s390_sie_block
*scb_s
= &vsie_page
->scb_s
;
617 rc
= acquire_gmap_shadow(vcpu
, vsie_page
);
619 rc
= map_prefix(vcpu
, vsie_page
);
621 gmap_enable(vsie_page
->gmap
);
622 update_intervention_requests(vsie_page
);
623 rc
= do_vsie_run(vcpu
, vsie_page
);
624 gmap_enable(vcpu
->arch
.gmap
);
629 if (rc
|| scb_s
->icptcode
|| signal_pending(current
) ||
630 kvm_s390_vcpu_has_irq(vcpu
, 0))
636 * Addressing exceptions are always presentes as intercepts.
637 * As addressing exceptions are suppressing and our guest 3 PSW
638 * points at the responsible instruction, we have to
639 * forward the PSW and set the ilc. If we can't read guest 3
640 * instruction, we can use an arbitrary ilc. Let's always use
641 * ilen = 4 for now, so we can avoid reading in guest 3 virtual
642 * memory. (we could also fake the shadow so the hardware
645 scb_s
->icptcode
= ICPT_PROGI
;
646 scb_s
->iprcc
= PGM_ADDRESSING
;
648 scb_s
->gpsw
.addr
= __rewind_psw(scb_s
->gpsw
, 4);
654 * Get or create a vsie page for a scb address.
656 * Returns: - address of a vsie page (cached or new one)
657 * - NULL if the same scb address is already used by another VCPU
658 * - ERR_PTR(-ENOMEM) if out of memory
660 static struct vsie_page
*get_vsie_page(struct kvm
*kvm
, unsigned long addr
)
662 struct vsie_page
*vsie_page
;
667 page
= radix_tree_lookup(&kvm
->arch
.vsie
.addr_to_page
, addr
>> 9);
670 if (page_ref_inc_return(page
) == 2)
671 return page_to_virt(page
);
676 * We want at least #online_vcpus shadows, so every VCPU can execute
677 * the VSIE in parallel.
679 nr_vcpus
= atomic_read(&kvm
->online_vcpus
);
681 mutex_lock(&kvm
->arch
.vsie
.mutex
);
682 if (kvm
->arch
.vsie
.page_count
< nr_vcpus
) {
683 page
= alloc_page(GFP_KERNEL
| __GFP_ZERO
);
685 mutex_unlock(&kvm
->arch
.vsie
.mutex
);
686 return ERR_PTR(-ENOMEM
);
689 kvm
->arch
.vsie
.pages
[kvm
->arch
.vsie
.page_count
] = page
;
690 kvm
->arch
.vsie
.page_count
++;
692 /* reuse an existing entry that belongs to nobody */
694 page
= kvm
->arch
.vsie
.pages
[kvm
->arch
.vsie
.next
];
695 if (page_ref_inc_return(page
) == 2)
698 kvm
->arch
.vsie
.next
++;
699 kvm
->arch
.vsie
.next
%= nr_vcpus
;
701 radix_tree_delete(&kvm
->arch
.vsie
.addr_to_page
, page
->index
>> 9);
704 /* double use of the same address */
705 if (radix_tree_insert(&kvm
->arch
.vsie
.addr_to_page
, addr
>> 9, page
)) {
707 mutex_unlock(&kvm
->arch
.vsie
.mutex
);
710 mutex_unlock(&kvm
->arch
.vsie
.mutex
);
712 vsie_page
= page_to_virt(page
);
713 memset(&vsie_page
->scb_s
, 0, sizeof(struct kvm_s390_sie_block
));
714 release_gmap_shadow(vsie_page
);
715 vsie_page
->scb_s
.ihcpu
= 0xffffU
;
719 /* put a vsie page acquired via get_vsie_page */
720 static void put_vsie_page(struct kvm
*kvm
, struct vsie_page
*vsie_page
)
722 struct page
*page
= pfn_to_page(__pa(vsie_page
) >> PAGE_SHIFT
);
727 int kvm_s390_handle_vsie(struct kvm_vcpu
*vcpu
)
729 struct vsie_page
*vsie_page
;
730 unsigned long scb_addr
;
733 vcpu
->stat
.instruction_sie
++;
734 if (!test_kvm_cpu_feat(vcpu
->kvm
, KVM_S390_VM_CPU_FEAT_SIEF2
))
736 if (vcpu
->arch
.sie_block
->gpsw
.mask
& PSW_MASK_PSTATE
)
737 return kvm_s390_inject_program_int(vcpu
, PGM_PRIVILEGED_OP
);
739 BUILD_BUG_ON(sizeof(struct vsie_page
) != 4096);
740 scb_addr
= kvm_s390_get_base_disp_s(vcpu
, NULL
);
742 /* 512 byte alignment */
743 if (unlikely(scb_addr
& 0x1ffUL
))
744 return kvm_s390_inject_program_int(vcpu
, PGM_SPECIFICATION
);
746 if (signal_pending(current
) || kvm_s390_vcpu_has_irq(vcpu
, 0))
749 vsie_page
= get_vsie_page(vcpu
->kvm
, scb_addr
);
750 if (IS_ERR(vsie_page
))
751 return PTR_ERR(vsie_page
);
753 /* double use of sie control block - simply do nothing */
756 rc
= pin_scb(vcpu
, vsie_page
, scb_addr
);
759 rc
= shadow_scb(vcpu
, vsie_page
);
762 rc
= pin_blocks(vcpu
, vsie_page
);
765 rc
= vsie_run(vcpu
, vsie_page
);
766 unpin_blocks(vcpu
, vsie_page
);
768 unshadow_scb(vcpu
, vsie_page
);
770 unpin_scb(vcpu
, vsie_page
, scb_addr
);
772 put_vsie_page(vcpu
->kvm
, vsie_page
);
774 return rc
< 0 ? rc
: 0;
777 /* Init the vsie data structures. To be called when a vm is initialized. */
778 void kvm_s390_vsie_init(struct kvm
*kvm
)
780 mutex_init(&kvm
->arch
.vsie
.mutex
);
781 INIT_RADIX_TREE(&kvm
->arch
.vsie
.addr_to_page
, GFP_KERNEL
);
784 /* Destroy the vsie data structures. To be called when a vm is destroyed. */
785 void kvm_s390_vsie_destroy(struct kvm
*kvm
)
787 struct vsie_page
*vsie_page
;
791 mutex_lock(&kvm
->arch
.vsie
.mutex
);
792 for (i
= 0; i
< kvm
->arch
.vsie
.page_count
; i
++) {
793 page
= kvm
->arch
.vsie
.pages
[i
];
794 kvm
->arch
.vsie
.pages
[i
] = NULL
;
795 vsie_page
= page_to_virt(page
);
796 release_gmap_shadow(vsie_page
);
797 /* free the radix tree entry */
798 radix_tree_delete(&kvm
->arch
.vsie
.addr_to_page
, page
->index
>> 9);
801 kvm
->arch
.vsie
.page_count
= 0;
802 mutex_unlock(&kvm
->arch
.vsie
.mutex
);