2 * guest access functions
4 * Copyright IBM Corp. 2014
8 #include <linux/vmalloc.h>
10 #include <asm/pgtable.h>
17 unsigned long origin
: 52; /* Region- or Segment-Table Origin */
19 unsigned long g
: 1; /* Subspace Group Control */
20 unsigned long p
: 1; /* Private Space Control */
21 unsigned long s
: 1; /* Storage-Alteration-Event Control */
22 unsigned long x
: 1; /* Space-Switch-Event Control */
23 unsigned long r
: 1; /* Real-Space Control */
25 unsigned long dt
: 2; /* Designation-Type Control */
26 unsigned long tl
: 2; /* Region- or Segment-Table Length */
31 ASCE_TYPE_SEGMENT
= 0,
32 ASCE_TYPE_REGION3
= 1,
33 ASCE_TYPE_REGION2
= 2,
37 union region1_table_entry
{
40 unsigned long rto
: 52;/* Region-Table Origin */
42 unsigned long p
: 1; /* DAT-Protection Bit */
44 unsigned long tf
: 2; /* Region-Second-Table Offset */
45 unsigned long i
: 1; /* Region-Invalid Bit */
47 unsigned long tt
: 2; /* Table-Type Bits */
48 unsigned long tl
: 2; /* Region-Second-Table Length */
52 union region2_table_entry
{
55 unsigned long rto
: 52;/* Region-Table Origin */
57 unsigned long p
: 1; /* DAT-Protection Bit */
59 unsigned long tf
: 2; /* Region-Third-Table Offset */
60 unsigned long i
: 1; /* Region-Invalid Bit */
62 unsigned long tt
: 2; /* Table-Type Bits */
63 unsigned long tl
: 2; /* Region-Third-Table Length */
67 struct region3_table_entry_fc0
{
68 unsigned long sto
: 52;/* Segment-Table Origin */
70 unsigned long fc
: 1; /* Format-Control */
71 unsigned long p
: 1; /* DAT-Protection Bit */
73 unsigned long tf
: 2; /* Segment-Table Offset */
74 unsigned long i
: 1; /* Region-Invalid Bit */
75 unsigned long cr
: 1; /* Common-Region Bit */
76 unsigned long tt
: 2; /* Table-Type Bits */
77 unsigned long tl
: 2; /* Segment-Table Length */
80 struct region3_table_entry_fc1
{
81 unsigned long rfaa
: 33; /* Region-Frame Absolute Address */
83 unsigned long av
: 1; /* ACCF-Validity Control */
84 unsigned long acc
: 4; /* Access-Control Bits */
85 unsigned long f
: 1; /* Fetch-Protection Bit */
86 unsigned long fc
: 1; /* Format-Control */
87 unsigned long p
: 1; /* DAT-Protection Bit */
88 unsigned long co
: 1; /* Change-Recording Override */
90 unsigned long i
: 1; /* Region-Invalid Bit */
91 unsigned long cr
: 1; /* Common-Region Bit */
92 unsigned long tt
: 2; /* Table-Type Bits */
96 union region3_table_entry
{
98 struct region3_table_entry_fc0 fc0
;
99 struct region3_table_entry_fc1 fc1
;
102 unsigned long fc
: 1; /* Format-Control */
104 unsigned long i
: 1; /* Region-Invalid Bit */
105 unsigned long cr
: 1; /* Common-Region Bit */
106 unsigned long tt
: 2; /* Table-Type Bits */
111 struct segment_entry_fc0
{
112 unsigned long pto
: 53;/* Page-Table Origin */
113 unsigned long fc
: 1; /* Format-Control */
114 unsigned long p
: 1; /* DAT-Protection Bit */
116 unsigned long i
: 1; /* Segment-Invalid Bit */
117 unsigned long cs
: 1; /* Common-Segment Bit */
118 unsigned long tt
: 2; /* Table-Type Bits */
122 struct segment_entry_fc1
{
123 unsigned long sfaa
: 44; /* Segment-Frame Absolute Address */
125 unsigned long av
: 1; /* ACCF-Validity Control */
126 unsigned long acc
: 4; /* Access-Control Bits */
127 unsigned long f
: 1; /* Fetch-Protection Bit */
128 unsigned long fc
: 1; /* Format-Control */
129 unsigned long p
: 1; /* DAT-Protection Bit */
130 unsigned long co
: 1; /* Change-Recording Override */
132 unsigned long i
: 1; /* Segment-Invalid Bit */
133 unsigned long cs
: 1; /* Common-Segment Bit */
134 unsigned long tt
: 2; /* Table-Type Bits */
138 union segment_table_entry
{
140 struct segment_entry_fc0 fc0
;
141 struct segment_entry_fc1 fc1
;
144 unsigned long fc
: 1; /* Format-Control */
146 unsigned long i
: 1; /* Segment-Invalid Bit */
147 unsigned long cs
: 1; /* Common-Segment Bit */
148 unsigned long tt
: 2; /* Table-Type Bits */
154 TABLE_TYPE_SEGMENT
= 0,
155 TABLE_TYPE_REGION3
= 1,
156 TABLE_TYPE_REGION2
= 2,
157 TABLE_TYPE_REGION1
= 3
160 union page_table_entry
{
163 unsigned long pfra
: 52; /* Page-Frame Real Address */
164 unsigned long z
: 1; /* Zero Bit */
165 unsigned long i
: 1; /* Page-Invalid Bit */
166 unsigned long p
: 1; /* DAT-Protection Bit */
167 unsigned long co
: 1; /* Change-Recording Override */
173 * vaddress union in order to easily decode a virtual address into its
174 * region first index, region second index etc. parts.
179 unsigned long rfx
: 11;
180 unsigned long rsx
: 11;
181 unsigned long rtx
: 11;
182 unsigned long sx
: 11;
183 unsigned long px
: 8;
184 unsigned long bx
: 12;
187 unsigned long rfx01
: 2;
189 unsigned long rsx01
: 2;
191 unsigned long rtx01
: 2;
193 unsigned long sx01
: 2;
199 * raddress union which will contain the result (real or absolute address)
200 * after a page table walk. The rfaa, sfaa and pfra members are used to
201 * simply assign them the value of a region, segment or page table entry.
205 unsigned long rfaa
: 33; /* Region-Frame Absolute Address */
206 unsigned long sfaa
: 44; /* Segment-Frame Absolute Address */
207 unsigned long pfra
: 52; /* Page-Frame Real Address */
211 int ipte_lock_held(struct kvm_vcpu
*vcpu
)
213 union ipte_control
*ic
= &vcpu
->kvm
->arch
.sca
->ipte_control
;
215 if (vcpu
->arch
.sie_block
->eca
& 1)
217 return vcpu
->kvm
->arch
.ipte_lock_count
!= 0;
220 static void ipte_lock_simple(struct kvm_vcpu
*vcpu
)
222 union ipte_control old
, new, *ic
;
224 mutex_lock(&vcpu
->kvm
->arch
.ipte_mutex
);
225 vcpu
->kvm
->arch
.ipte_lock_count
++;
226 if (vcpu
->kvm
->arch
.ipte_lock_count
> 1)
228 ic
= &vcpu
->kvm
->arch
.sca
->ipte_control
;
230 old
= READ_ONCE(*ic
);
233 old
= READ_ONCE(*ic
);
237 } while (cmpxchg(&ic
->val
, old
.val
, new.val
) != old
.val
);
239 mutex_unlock(&vcpu
->kvm
->arch
.ipte_mutex
);
242 static void ipte_unlock_simple(struct kvm_vcpu
*vcpu
)
244 union ipte_control old
, new, *ic
;
246 mutex_lock(&vcpu
->kvm
->arch
.ipte_mutex
);
247 vcpu
->kvm
->arch
.ipte_lock_count
--;
248 if (vcpu
->kvm
->arch
.ipte_lock_count
)
250 ic
= &vcpu
->kvm
->arch
.sca
->ipte_control
;
252 old
= READ_ONCE(*ic
);
255 } while (cmpxchg(&ic
->val
, old
.val
, new.val
) != old
.val
);
256 wake_up(&vcpu
->kvm
->arch
.ipte_wq
);
258 mutex_unlock(&vcpu
->kvm
->arch
.ipte_mutex
);
261 static void ipte_lock_siif(struct kvm_vcpu
*vcpu
)
263 union ipte_control old
, new, *ic
;
265 ic
= &vcpu
->kvm
->arch
.sca
->ipte_control
;
267 old
= READ_ONCE(*ic
);
270 old
= READ_ONCE(*ic
);
275 } while (cmpxchg(&ic
->val
, old
.val
, new.val
) != old
.val
);
278 static void ipte_unlock_siif(struct kvm_vcpu
*vcpu
)
280 union ipte_control old
, new, *ic
;
282 ic
= &vcpu
->kvm
->arch
.sca
->ipte_control
;
284 old
= READ_ONCE(*ic
);
289 } while (cmpxchg(&ic
->val
, old
.val
, new.val
) != old
.val
);
291 wake_up(&vcpu
->kvm
->arch
.ipte_wq
);
294 void ipte_lock(struct kvm_vcpu
*vcpu
)
296 if (vcpu
->arch
.sie_block
->eca
& 1)
297 ipte_lock_siif(vcpu
);
299 ipte_lock_simple(vcpu
);
302 void ipte_unlock(struct kvm_vcpu
*vcpu
)
304 if (vcpu
->arch
.sie_block
->eca
& 1)
305 ipte_unlock_siif(vcpu
);
307 ipte_unlock_simple(vcpu
);
310 static unsigned long get_vcpu_asce(struct kvm_vcpu
*vcpu
)
312 switch (psw_bits(vcpu
->arch
.sie_block
->gpsw
).as
) {
314 return vcpu
->arch
.sie_block
->gcr
[1];
315 case PSW_AS_SECONDARY
:
316 return vcpu
->arch
.sie_block
->gcr
[7];
318 return vcpu
->arch
.sie_block
->gcr
[13];
323 static int deref_table(struct kvm
*kvm
, unsigned long gpa
, unsigned long *val
)
325 return kvm_read_guest(kvm
, gpa
, val
, sizeof(*val
));
329 * guest_translate - translate a guest virtual into a guest absolute address
331 * @gva: guest virtual address
332 * @gpa: points to where guest physical (absolute) address should be stored
333 * @write: indicates if access is a write access
335 * Translate a guest virtual address into a guest absolute address by means
336 * of dynamic address translation as specified by the architecuture.
337 * If the resulting absolute address is not available in the configuration
338 * an addressing exception is indicated and @gpa will not be changed.
340 * Returns: - zero on success; @gpa contains the resulting absolute address
341 * - a negative value if guest access failed due to e.g. broken
343 * - a positve value if an access exception happened. In this case
344 * the returned value is the program interruption code as defined
345 * by the architecture
347 static unsigned long guest_translate(struct kvm_vcpu
*vcpu
, unsigned long gva
,
348 unsigned long *gpa
, int write
)
350 union vaddress vaddr
= {.addr
= gva
};
351 union raddress raddr
= {.addr
= gva
};
352 union page_table_entry pte
;
353 int dat_protection
= 0;
354 union ctlreg0 ctlreg0
;
359 ctlreg0
.val
= vcpu
->arch
.sie_block
->gcr
[0];
360 edat1
= ctlreg0
.edat
&& test_kvm_facility(vcpu
->kvm
, 8);
361 edat2
= edat1
&& test_kvm_facility(vcpu
->kvm
, 78);
362 asce
.val
= get_vcpu_asce(vcpu
);
365 ptr
= asce
.origin
* 4096;
367 case ASCE_TYPE_REGION1
:
368 if (vaddr
.rfx01
> asce
.tl
)
369 return PGM_REGION_FIRST_TRANS
;
370 ptr
+= vaddr
.rfx
* 8;
372 case ASCE_TYPE_REGION2
:
374 return PGM_ASCE_TYPE
;
375 if (vaddr
.rsx01
> asce
.tl
)
376 return PGM_REGION_SECOND_TRANS
;
377 ptr
+= vaddr
.rsx
* 8;
379 case ASCE_TYPE_REGION3
:
380 if (vaddr
.rfx
|| vaddr
.rsx
)
381 return PGM_ASCE_TYPE
;
382 if (vaddr
.rtx01
> asce
.tl
)
383 return PGM_REGION_THIRD_TRANS
;
384 ptr
+= vaddr
.rtx
* 8;
386 case ASCE_TYPE_SEGMENT
:
387 if (vaddr
.rfx
|| vaddr
.rsx
|| vaddr
.rtx
)
388 return PGM_ASCE_TYPE
;
389 if (vaddr
.sx01
> asce
.tl
)
390 return PGM_SEGMENT_TRANSLATION
;
395 case ASCE_TYPE_REGION1
: {
396 union region1_table_entry rfte
;
398 if (kvm_is_error_gpa(vcpu
->kvm
, ptr
))
399 return PGM_ADDRESSING
;
400 if (deref_table(vcpu
->kvm
, ptr
, &rfte
.val
))
403 return PGM_REGION_FIRST_TRANS
;
404 if (rfte
.tt
!= TABLE_TYPE_REGION1
)
405 return PGM_TRANSLATION_SPEC
;
406 if (vaddr
.rsx01
< rfte
.tf
|| vaddr
.rsx01
> rfte
.tl
)
407 return PGM_REGION_SECOND_TRANS
;
409 dat_protection
|= rfte
.p
;
410 ptr
= rfte
.rto
* 4096 + vaddr
.rsx
* 8;
413 case ASCE_TYPE_REGION2
: {
414 union region2_table_entry rste
;
416 if (kvm_is_error_gpa(vcpu
->kvm
, ptr
))
417 return PGM_ADDRESSING
;
418 if (deref_table(vcpu
->kvm
, ptr
, &rste
.val
))
421 return PGM_REGION_SECOND_TRANS
;
422 if (rste
.tt
!= TABLE_TYPE_REGION2
)
423 return PGM_TRANSLATION_SPEC
;
424 if (vaddr
.rtx01
< rste
.tf
|| vaddr
.rtx01
> rste
.tl
)
425 return PGM_REGION_THIRD_TRANS
;
427 dat_protection
|= rste
.p
;
428 ptr
= rste
.rto
* 4096 + vaddr
.rtx
* 8;
431 case ASCE_TYPE_REGION3
: {
432 union region3_table_entry rtte
;
434 if (kvm_is_error_gpa(vcpu
->kvm
, ptr
))
435 return PGM_ADDRESSING
;
436 if (deref_table(vcpu
->kvm
, ptr
, &rtte
.val
))
439 return PGM_REGION_THIRD_TRANS
;
440 if (rtte
.tt
!= TABLE_TYPE_REGION3
)
441 return PGM_TRANSLATION_SPEC
;
442 if (rtte
.cr
&& asce
.p
&& edat2
)
443 return PGM_TRANSLATION_SPEC
;
444 if (rtte
.fc
&& edat2
) {
445 dat_protection
|= rtte
.fc1
.p
;
446 raddr
.rfaa
= rtte
.fc1
.rfaa
;
447 goto absolute_address
;
449 if (vaddr
.sx01
< rtte
.fc0
.tf
)
450 return PGM_SEGMENT_TRANSLATION
;
451 if (vaddr
.sx01
> rtte
.fc0
.tl
)
452 return PGM_SEGMENT_TRANSLATION
;
454 dat_protection
|= rtte
.fc0
.p
;
455 ptr
= rtte
.fc0
.sto
* 4096 + vaddr
.sx
* 8;
458 case ASCE_TYPE_SEGMENT
: {
459 union segment_table_entry ste
;
461 if (kvm_is_error_gpa(vcpu
->kvm
, ptr
))
462 return PGM_ADDRESSING
;
463 if (deref_table(vcpu
->kvm
, ptr
, &ste
.val
))
466 return PGM_SEGMENT_TRANSLATION
;
467 if (ste
.tt
!= TABLE_TYPE_SEGMENT
)
468 return PGM_TRANSLATION_SPEC
;
469 if (ste
.cs
&& asce
.p
)
470 return PGM_TRANSLATION_SPEC
;
471 if (ste
.fc
&& edat1
) {
472 dat_protection
|= ste
.fc1
.p
;
473 raddr
.sfaa
= ste
.fc1
.sfaa
;
474 goto absolute_address
;
476 dat_protection
|= ste
.fc0
.p
;
477 ptr
= ste
.fc0
.pto
* 2048 + vaddr
.px
* 8;
480 if (kvm_is_error_gpa(vcpu
->kvm
, ptr
))
481 return PGM_ADDRESSING
;
482 if (deref_table(vcpu
->kvm
, ptr
, &pte
.val
))
485 return PGM_PAGE_TRANSLATION
;
487 return PGM_TRANSLATION_SPEC
;
488 if (pte
.co
&& !edat1
)
489 return PGM_TRANSLATION_SPEC
;
490 dat_protection
|= pte
.p
;
491 raddr
.pfra
= pte
.pfra
;
493 raddr
.addr
= kvm_s390_real_to_abs(vcpu
, raddr
.addr
);
495 if (write
&& dat_protection
)
496 return PGM_PROTECTION
;
497 if (kvm_is_error_gpa(vcpu
->kvm
, raddr
.addr
))
498 return PGM_ADDRESSING
;
503 static inline int is_low_address(unsigned long ga
)
505 /* Check for address ranges 0..511 and 4096..4607 */
506 return (ga
& ~0x11fful
) == 0;
509 static int low_address_protection_enabled(struct kvm_vcpu
*vcpu
)
511 union ctlreg0 ctlreg0
= {.val
= vcpu
->arch
.sie_block
->gcr
[0]};
512 psw_t
*psw
= &vcpu
->arch
.sie_block
->gpsw
;
517 asce
.val
= get_vcpu_asce(vcpu
);
518 if (psw_bits(*psw
).t
&& asce
.p
)
523 struct trans_exc_code_bits
{
524 unsigned long addr
: 52; /* Translation-exception Address */
525 unsigned long fsi
: 2; /* Access Exception Fetch/Store Indication */
527 unsigned long b61
: 1;
528 unsigned long as
: 2; /* ASCE Identifier */
532 FSI_UNKNOWN
= 0, /* Unknown wether fetch or store */
533 FSI_STORE
= 1, /* Exception was due to store operation */
534 FSI_FETCH
= 2 /* Exception was due to fetch operation */
537 static int guest_page_range(struct kvm_vcpu
*vcpu
, unsigned long ga
,
538 unsigned long *pages
, unsigned long nr_pages
,
541 struct kvm_s390_pgm_info
*pgm
= &vcpu
->arch
.pgm
;
542 psw_t
*psw
= &vcpu
->arch
.sie_block
->gpsw
;
543 struct trans_exc_code_bits
*tec_bits
;
546 memset(pgm
, 0, sizeof(*pgm
));
547 tec_bits
= (struct trans_exc_code_bits
*)&pgm
->trans_exc_code
;
548 tec_bits
->fsi
= write
? FSI_STORE
: FSI_FETCH
;
549 tec_bits
->as
= psw_bits(*psw
).as
;
550 lap_enabled
= low_address_protection_enabled(vcpu
);
552 ga
= kvm_s390_logical_to_effective(vcpu
, ga
);
553 tec_bits
->addr
= ga
>> PAGE_SHIFT
;
554 if (write
&& lap_enabled
&& is_low_address(ga
)) {
555 pgm
->code
= PGM_PROTECTION
;
559 if (psw_bits(*psw
).t
) {
560 rc
= guest_translate(vcpu
, ga
, pages
, write
);
563 if (rc
== PGM_PROTECTION
)
568 *pages
= kvm_s390_real_to_abs(vcpu
, ga
);
569 if (kvm_is_error_gpa(vcpu
->kvm
, *pages
))
570 pgm
->code
= PGM_ADDRESSING
;
581 int access_guest(struct kvm_vcpu
*vcpu
, unsigned long ga
, void *data
,
582 unsigned long len
, int write
)
584 psw_t
*psw
= &vcpu
->arch
.sie_block
->gpsw
;
585 unsigned long _len
, nr_pages
, gpa
, idx
;
586 unsigned long pages_array
[2];
587 unsigned long *pages
;
594 /* Access register mode is not supported yet. */
595 if (psw_bits(*psw
).t
&& psw_bits(*psw
).as
== PSW_AS_ACCREG
)
597 nr_pages
= (((ga
& ~PAGE_MASK
) + len
- 1) >> PAGE_SHIFT
) + 1;
599 if (nr_pages
> ARRAY_SIZE(pages_array
))
600 pages
= vmalloc(nr_pages
* sizeof(unsigned long));
603 asce
.val
= get_vcpu_asce(vcpu
);
604 need_ipte_lock
= psw_bits(*psw
).t
&& !asce
.r
;
607 rc
= guest_page_range(vcpu
, ga
, pages
, nr_pages
, write
);
608 for (idx
= 0; idx
< nr_pages
&& !rc
; idx
++) {
609 gpa
= *(pages
+ idx
) + (ga
& ~PAGE_MASK
);
610 _len
= min(PAGE_SIZE
- (gpa
& ~PAGE_MASK
), len
);
612 rc
= kvm_write_guest(vcpu
->kvm
, gpa
, data
, _len
);
614 rc
= kvm_read_guest(vcpu
->kvm
, gpa
, data
, _len
);
621 if (nr_pages
> ARRAY_SIZE(pages_array
))
626 int access_guest_real(struct kvm_vcpu
*vcpu
, unsigned long gra
,
627 void *data
, unsigned long len
, int write
)
629 unsigned long _len
, gpa
;
633 gpa
= kvm_s390_real_to_abs(vcpu
, gra
);
634 _len
= min(PAGE_SIZE
- (gpa
& ~PAGE_MASK
), len
);
636 rc
= write_guest_abs(vcpu
, gpa
, data
, _len
);
638 rc
= read_guest_abs(vcpu
, gpa
, data
, _len
);
647 * guest_translate_address - translate guest logical into guest absolute address
649 * Parameter semantics are the same as the ones from guest_translate.
650 * The memory contents at the guest address are not changed.
652 * Note: The IPTE lock is not taken during this function, so the caller
653 * has to take care of this.
655 int guest_translate_address(struct kvm_vcpu
*vcpu
, unsigned long gva
,
656 unsigned long *gpa
, int write
)
658 struct kvm_s390_pgm_info
*pgm
= &vcpu
->arch
.pgm
;
659 psw_t
*psw
= &vcpu
->arch
.sie_block
->gpsw
;
660 struct trans_exc_code_bits
*tec
;
664 /* Access register mode is not supported yet. */
665 if (psw_bits(*psw
).t
&& psw_bits(*psw
).as
== PSW_AS_ACCREG
)
668 gva
= kvm_s390_logical_to_effective(vcpu
, gva
);
669 memset(pgm
, 0, sizeof(*pgm
));
670 tec
= (struct trans_exc_code_bits
*)&pgm
->trans_exc_code
;
671 tec
->as
= psw_bits(*psw
).as
;
672 tec
->fsi
= write
? FSI_STORE
: FSI_FETCH
;
673 tec
->addr
= gva
>> PAGE_SHIFT
;
674 if (is_low_address(gva
) && low_address_protection_enabled(vcpu
)) {
676 rc
= pgm
->code
= PGM_PROTECTION
;
681 asce
.val
= get_vcpu_asce(vcpu
);
682 if (psw_bits(*psw
).t
&& !asce
.r
) { /* Use DAT? */
683 rc
= guest_translate(vcpu
, gva
, gpa
, write
);
685 if (rc
== PGM_PROTECTION
)
691 *gpa
= kvm_s390_real_to_abs(vcpu
, gva
);
692 if (kvm_is_error_gpa(vcpu
->kvm
, *gpa
))
693 rc
= pgm
->code
= PGM_ADDRESSING
;
700 * kvm_s390_check_low_addr_protection - check for low-address protection
703 * Checks whether an address is subject to low-address protection and set
704 * up vcpu->arch.pgm accordingly if necessary.
706 * Return: 0 if no protection exception, or PGM_PROTECTION if protected.
708 int kvm_s390_check_low_addr_protection(struct kvm_vcpu
*vcpu
, unsigned long ga
)
710 struct kvm_s390_pgm_info
*pgm
= &vcpu
->arch
.pgm
;
711 psw_t
*psw
= &vcpu
->arch
.sie_block
->gpsw
;
712 struct trans_exc_code_bits
*tec_bits
;
714 if (!is_low_address(ga
) || !low_address_protection_enabled(vcpu
))
717 memset(pgm
, 0, sizeof(*pgm
));
718 tec_bits
= (struct trans_exc_code_bits
*)&pgm
->trans_exc_code
;
719 tec_bits
->fsi
= FSI_STORE
;
720 tec_bits
->as
= psw_bits(*psw
).as
;
721 tec_bits
->addr
= ga
>> PAGE_SHIFT
;
722 pgm
->code
= PGM_PROTECTION
;