2 * guest access functions
4 * Copyright IBM Corp. 2014
8 #include <linux/vmalloc.h>
9 #include <linux/mm_types.h>
10 #include <linux/err.h>
12 #include <asm/pgtable.h>
16 #include <asm/switch_to.h>
21 unsigned long origin
: 52; /* Region- or Segment-Table Origin */
23 unsigned long g
: 1; /* Subspace Group Control */
24 unsigned long p
: 1; /* Private Space Control */
25 unsigned long s
: 1; /* Storage-Alteration-Event Control */
26 unsigned long x
: 1; /* Space-Switch-Event Control */
27 unsigned long r
: 1; /* Real-Space Control */
29 unsigned long dt
: 2; /* Designation-Type Control */
30 unsigned long tl
: 2; /* Region- or Segment-Table Length */
35 ASCE_TYPE_SEGMENT
= 0,
36 ASCE_TYPE_REGION3
= 1,
37 ASCE_TYPE_REGION2
= 2,
41 union region1_table_entry
{
44 unsigned long rto
: 52;/* Region-Table Origin */
46 unsigned long p
: 1; /* DAT-Protection Bit */
48 unsigned long tf
: 2; /* Region-Second-Table Offset */
49 unsigned long i
: 1; /* Region-Invalid Bit */
51 unsigned long tt
: 2; /* Table-Type Bits */
52 unsigned long tl
: 2; /* Region-Second-Table Length */
56 union region2_table_entry
{
59 unsigned long rto
: 52;/* Region-Table Origin */
61 unsigned long p
: 1; /* DAT-Protection Bit */
63 unsigned long tf
: 2; /* Region-Third-Table Offset */
64 unsigned long i
: 1; /* Region-Invalid Bit */
66 unsigned long tt
: 2; /* Table-Type Bits */
67 unsigned long tl
: 2; /* Region-Third-Table Length */
71 struct region3_table_entry_fc0
{
72 unsigned long sto
: 52;/* Segment-Table Origin */
74 unsigned long fc
: 1; /* Format-Control */
75 unsigned long p
: 1; /* DAT-Protection Bit */
77 unsigned long tf
: 2; /* Segment-Table Offset */
78 unsigned long i
: 1; /* Region-Invalid Bit */
79 unsigned long cr
: 1; /* Common-Region Bit */
80 unsigned long tt
: 2; /* Table-Type Bits */
81 unsigned long tl
: 2; /* Segment-Table Length */
84 struct region3_table_entry_fc1
{
85 unsigned long rfaa
: 33; /* Region-Frame Absolute Address */
87 unsigned long av
: 1; /* ACCF-Validity Control */
88 unsigned long acc
: 4; /* Access-Control Bits */
89 unsigned long f
: 1; /* Fetch-Protection Bit */
90 unsigned long fc
: 1; /* Format-Control */
91 unsigned long p
: 1; /* DAT-Protection Bit */
92 unsigned long iep
: 1; /* Instruction-Execution-Protection */
94 unsigned long i
: 1; /* Region-Invalid Bit */
95 unsigned long cr
: 1; /* Common-Region Bit */
96 unsigned long tt
: 2; /* Table-Type Bits */
100 union region3_table_entry
{
102 struct region3_table_entry_fc0 fc0
;
103 struct region3_table_entry_fc1 fc1
;
106 unsigned long fc
: 1; /* Format-Control */
108 unsigned long i
: 1; /* Region-Invalid Bit */
109 unsigned long cr
: 1; /* Common-Region Bit */
110 unsigned long tt
: 2; /* Table-Type Bits */
115 struct segment_entry_fc0
{
116 unsigned long pto
: 53;/* Page-Table Origin */
117 unsigned long fc
: 1; /* Format-Control */
118 unsigned long p
: 1; /* DAT-Protection Bit */
120 unsigned long i
: 1; /* Segment-Invalid Bit */
121 unsigned long cs
: 1; /* Common-Segment Bit */
122 unsigned long tt
: 2; /* Table-Type Bits */
126 struct segment_entry_fc1
{
127 unsigned long sfaa
: 44; /* Segment-Frame Absolute Address */
129 unsigned long av
: 1; /* ACCF-Validity Control */
130 unsigned long acc
: 4; /* Access-Control Bits */
131 unsigned long f
: 1; /* Fetch-Protection Bit */
132 unsigned long fc
: 1; /* Format-Control */
133 unsigned long p
: 1; /* DAT-Protection Bit */
134 unsigned long iep
: 1; /* Instruction-Execution-Protection */
136 unsigned long i
: 1; /* Segment-Invalid Bit */
137 unsigned long cs
: 1; /* Common-Segment Bit */
138 unsigned long tt
: 2; /* Table-Type Bits */
142 union segment_table_entry
{
144 struct segment_entry_fc0 fc0
;
145 struct segment_entry_fc1 fc1
;
148 unsigned long fc
: 1; /* Format-Control */
150 unsigned long i
: 1; /* Segment-Invalid Bit */
151 unsigned long cs
: 1; /* Common-Segment Bit */
152 unsigned long tt
: 2; /* Table-Type Bits */
158 TABLE_TYPE_SEGMENT
= 0,
159 TABLE_TYPE_REGION3
= 1,
160 TABLE_TYPE_REGION2
= 2,
161 TABLE_TYPE_REGION1
= 3
164 union page_table_entry
{
167 unsigned long pfra
: 52; /* Page-Frame Real Address */
168 unsigned long z
: 1; /* Zero Bit */
169 unsigned long i
: 1; /* Page-Invalid Bit */
170 unsigned long p
: 1; /* DAT-Protection Bit */
171 unsigned long iep
: 1; /* Instruction-Execution-Protection */
177 * vaddress union in order to easily decode a virtual address into its
178 * region first index, region second index etc. parts.
183 unsigned long rfx
: 11;
184 unsigned long rsx
: 11;
185 unsigned long rtx
: 11;
186 unsigned long sx
: 11;
187 unsigned long px
: 8;
188 unsigned long bx
: 12;
191 unsigned long rfx01
: 2;
193 unsigned long rsx01
: 2;
195 unsigned long rtx01
: 2;
197 unsigned long sx01
: 2;
203 * raddress union which will contain the result (real or absolute address)
204 * after a page table walk. The rfaa, sfaa and pfra members are used to
205 * simply assign them the value of a region, segment or page table entry.
209 unsigned long rfaa
: 33; /* Region-Frame Absolute Address */
210 unsigned long sfaa
: 44; /* Segment-Frame Absolute Address */
211 unsigned long pfra
: 52; /* Page-Frame Real Address */
234 unsigned long i
: 1; /* ALEN-Invalid Bit */
236 unsigned long fo
: 1; /* Fetch-Only Bit */
237 unsigned long p
: 1; /* Private Bit */
238 unsigned long alesn
: 8; /* Access-List-Entry Sequence Number */
239 unsigned long aleax
: 16; /* Access-List-Entry Authorization Index */
242 unsigned long asteo
: 25; /* ASN-Second-Table-Entry Origin */
244 unsigned long astesn
: 32; /* ASTE Sequence Number */
248 unsigned long i
: 1; /* ASX-Invalid Bit */
249 unsigned long ato
: 29; /* Authority-Table Origin */
251 unsigned long b
: 1; /* Base-Space Bit */
252 unsigned long ax
: 16; /* Authorization Index */
253 unsigned long atl
: 12; /* Authority-Table Length */
255 unsigned long ca
: 1; /* Controlled-ASN Bit */
256 unsigned long ra
: 1; /* Reusable-ASN Bit */
257 unsigned long asce
: 64; /* Address-Space-Control Element */
258 unsigned long ald
: 32;
259 unsigned long astesn
: 32;
260 /* .. more fields there */
263 int ipte_lock_held(struct kvm_vcpu
*vcpu
)
265 if (vcpu
->arch
.sie_block
->eca
& ECA_SII
) {
268 read_lock(&vcpu
->kvm
->arch
.sca_lock
);
269 rc
= kvm_s390_get_ipte_control(vcpu
->kvm
)->kh
!= 0;
270 read_unlock(&vcpu
->kvm
->arch
.sca_lock
);
273 return vcpu
->kvm
->arch
.ipte_lock_count
!= 0;
276 static void ipte_lock_simple(struct kvm_vcpu
*vcpu
)
278 union ipte_control old
, new, *ic
;
280 mutex_lock(&vcpu
->kvm
->arch
.ipte_mutex
);
281 vcpu
->kvm
->arch
.ipte_lock_count
++;
282 if (vcpu
->kvm
->arch
.ipte_lock_count
> 1)
285 read_lock(&vcpu
->kvm
->arch
.sca_lock
);
286 ic
= kvm_s390_get_ipte_control(vcpu
->kvm
);
288 old
= READ_ONCE(*ic
);
290 read_unlock(&vcpu
->kvm
->arch
.sca_lock
);
296 } while (cmpxchg(&ic
->val
, old
.val
, new.val
) != old
.val
);
297 read_unlock(&vcpu
->kvm
->arch
.sca_lock
);
299 mutex_unlock(&vcpu
->kvm
->arch
.ipte_mutex
);
302 static void ipte_unlock_simple(struct kvm_vcpu
*vcpu
)
304 union ipte_control old
, new, *ic
;
306 mutex_lock(&vcpu
->kvm
->arch
.ipte_mutex
);
307 vcpu
->kvm
->arch
.ipte_lock_count
--;
308 if (vcpu
->kvm
->arch
.ipte_lock_count
)
310 read_lock(&vcpu
->kvm
->arch
.sca_lock
);
311 ic
= kvm_s390_get_ipte_control(vcpu
->kvm
);
313 old
= READ_ONCE(*ic
);
316 } while (cmpxchg(&ic
->val
, old
.val
, new.val
) != old
.val
);
317 read_unlock(&vcpu
->kvm
->arch
.sca_lock
);
318 wake_up(&vcpu
->kvm
->arch
.ipte_wq
);
320 mutex_unlock(&vcpu
->kvm
->arch
.ipte_mutex
);
323 static void ipte_lock_siif(struct kvm_vcpu
*vcpu
)
325 union ipte_control old
, new, *ic
;
328 read_lock(&vcpu
->kvm
->arch
.sca_lock
);
329 ic
= kvm_s390_get_ipte_control(vcpu
->kvm
);
331 old
= READ_ONCE(*ic
);
333 read_unlock(&vcpu
->kvm
->arch
.sca_lock
);
340 } while (cmpxchg(&ic
->val
, old
.val
, new.val
) != old
.val
);
341 read_unlock(&vcpu
->kvm
->arch
.sca_lock
);
344 static void ipte_unlock_siif(struct kvm_vcpu
*vcpu
)
346 union ipte_control old
, new, *ic
;
348 read_lock(&vcpu
->kvm
->arch
.sca_lock
);
349 ic
= kvm_s390_get_ipte_control(vcpu
->kvm
);
351 old
= READ_ONCE(*ic
);
356 } while (cmpxchg(&ic
->val
, old
.val
, new.val
) != old
.val
);
357 read_unlock(&vcpu
->kvm
->arch
.sca_lock
);
359 wake_up(&vcpu
->kvm
->arch
.ipte_wq
);
362 void ipte_lock(struct kvm_vcpu
*vcpu
)
364 if (vcpu
->arch
.sie_block
->eca
& ECA_SII
)
365 ipte_lock_siif(vcpu
);
367 ipte_lock_simple(vcpu
);
370 void ipte_unlock(struct kvm_vcpu
*vcpu
)
372 if (vcpu
->arch
.sie_block
->eca
& ECA_SII
)
373 ipte_unlock_siif(vcpu
);
375 ipte_unlock_simple(vcpu
);
378 static int ar_translation(struct kvm_vcpu
*vcpu
, union asce
*asce
, u8 ar
,
384 unsigned long ald_addr
, authority_table_addr
;
392 save_access_regs(vcpu
->run
->s
.regs
.acrs
);
393 alet
.val
= vcpu
->run
->s
.regs
.acrs
[ar
];
395 if (ar
== 0 || alet
.val
== 0) {
396 asce
->val
= vcpu
->arch
.sie_block
->gcr
[1];
398 } else if (alet
.val
== 1) {
399 asce
->val
= vcpu
->arch
.sie_block
->gcr
[7];
404 return PGM_ALET_SPECIFICATION
;
407 ald_addr
= vcpu
->arch
.sie_block
->gcr
[5];
409 ald_addr
= vcpu
->arch
.sie_block
->gcr
[2];
410 ald_addr
&= 0x7fffffc0;
412 rc
= read_guest_real(vcpu
, ald_addr
+ 16, &ald
.val
, sizeof(union ald
));
416 if (alet
.alen
/ 8 > ald
.all
)
417 return PGM_ALEN_TRANSLATION
;
419 if (0x7fffffff - ald
.alo
* 128 < alet
.alen
* 16)
420 return PGM_ADDRESSING
;
422 rc
= read_guest_real(vcpu
, ald
.alo
* 128 + alet
.alen
* 16, &ale
,
428 return PGM_ALEN_TRANSLATION
;
429 if (ale
.alesn
!= alet
.alesn
)
430 return PGM_ALE_SEQUENCE
;
432 rc
= read_guest_real(vcpu
, ale
.asteo
* 64, &aste
, sizeof(struct aste
));
437 return PGM_ASTE_VALIDITY
;
438 if (aste
.astesn
!= ale
.astesn
)
439 return PGM_ASTE_SEQUENCE
;
442 eax
= (vcpu
->arch
.sie_block
->gcr
[8] >> 16) & 0xffff;
443 if (ale
.aleax
!= eax
) {
444 if (eax
/ 16 > aste
.atl
)
445 return PGM_EXTENDED_AUTHORITY
;
447 authority_table_addr
= aste
.ato
* 4 + eax
/ 4;
449 rc
= read_guest_real(vcpu
, authority_table_addr
,
455 if ((authority_table
& (0x40 >> ((eax
& 3) * 2))) == 0)
456 return PGM_EXTENDED_AUTHORITY
;
460 if (ale
.fo
== 1 && mode
== GACC_STORE
)
461 return PGM_PROTECTION
;
463 asce
->val
= aste
.asce
;
467 struct trans_exc_code_bits
{
468 unsigned long addr
: 52; /* Translation-exception Address */
469 unsigned long fsi
: 2; /* Access Exception Fetch/Store Indication */
471 unsigned long b56
: 1;
473 unsigned long b60
: 1;
474 unsigned long b61
: 1;
475 unsigned long as
: 2; /* ASCE Identifier */
479 FSI_UNKNOWN
= 0, /* Unknown wether fetch or store */
480 FSI_STORE
= 1, /* Exception was due to store operation */
481 FSI_FETCH
= 2 /* Exception was due to fetch operation */
492 static int trans_exc(struct kvm_vcpu
*vcpu
, int code
, unsigned long gva
,
493 u8 ar
, enum gacc_mode mode
, enum prot_type prot
)
495 struct kvm_s390_pgm_info
*pgm
= &vcpu
->arch
.pgm
;
496 struct trans_exc_code_bits
*tec
;
498 memset(pgm
, 0, sizeof(*pgm
));
500 tec
= (struct trans_exc_code_bits
*)&pgm
->trans_exc_code
;
523 case PGM_PAGE_TRANSLATION
:
524 case PGM_REGION_FIRST_TRANS
:
525 case PGM_REGION_SECOND_TRANS
:
526 case PGM_REGION_THIRD_TRANS
:
527 case PGM_SEGMENT_TRANSLATION
:
529 * op_access_id only applies to MOVE_PAGE -> set bit 61
530 * exc_access_id has to be set to 0 for some instructions. Both
531 * cases have to be handled by the caller.
533 tec
->addr
= gva
>> PAGE_SHIFT
;
534 tec
->fsi
= mode
== GACC_STORE
? FSI_STORE
: FSI_FETCH
;
535 tec
->as
= psw_bits(vcpu
->arch
.sie_block
->gpsw
).as
;
537 case PGM_ALEN_TRANSLATION
:
538 case PGM_ALE_SEQUENCE
:
539 case PGM_ASTE_VALIDITY
:
540 case PGM_ASTE_SEQUENCE
:
541 case PGM_EXTENDED_AUTHORITY
:
543 * We can always store exc_access_id, as it is
544 * undefined for non-ar cases. It is undefined for
545 * most DAT protection exceptions.
547 pgm
->exc_access_id
= ar
;
553 static int get_vcpu_asce(struct kvm_vcpu
*vcpu
, union asce
*asce
,
554 unsigned long ga
, u8 ar
, enum gacc_mode mode
)
557 struct psw_bits psw
= psw_bits(vcpu
->arch
.sie_block
->gpsw
);
565 if ((mode
== GACC_IFETCH
) && (psw
.as
!= PSW_BITS_AS_HOME
))
566 psw
.as
= PSW_BITS_AS_PRIMARY
;
569 case PSW_BITS_AS_PRIMARY
:
570 asce
->val
= vcpu
->arch
.sie_block
->gcr
[1];
572 case PSW_BITS_AS_SECONDARY
:
573 asce
->val
= vcpu
->arch
.sie_block
->gcr
[7];
575 case PSW_BITS_AS_HOME
:
576 asce
->val
= vcpu
->arch
.sie_block
->gcr
[13];
578 case PSW_BITS_AS_ACCREG
:
579 rc
= ar_translation(vcpu
, asce
, ar
, mode
);
581 return trans_exc(vcpu
, rc
, ga
, ar
, mode
, PROT_TYPE_ALC
);
587 static int deref_table(struct kvm
*kvm
, unsigned long gpa
, unsigned long *val
)
589 return kvm_read_guest(kvm
, gpa
, val
, sizeof(*val
));
593 * guest_translate - translate a guest virtual into a guest absolute address
595 * @gva: guest virtual address
596 * @gpa: points to where guest physical (absolute) address should be stored
597 * @asce: effective asce
598 * @mode: indicates the access mode to be used
599 * @prot: returns the type for protection exceptions
601 * Translate a guest virtual address into a guest absolute address by means
602 * of dynamic address translation as specified by the architecture.
603 * If the resulting absolute address is not available in the configuration
604 * an addressing exception is indicated and @gpa will not be changed.
606 * Returns: - zero on success; @gpa contains the resulting absolute address
607 * - a negative value if guest access failed due to e.g. broken
609 * - a positve value if an access exception happened. In this case
610 * the returned value is the program interruption code as defined
611 * by the architecture
613 static unsigned long guest_translate(struct kvm_vcpu
*vcpu
, unsigned long gva
,
614 unsigned long *gpa
, const union asce asce
,
615 enum gacc_mode mode
, enum prot_type
*prot
)
617 union vaddress vaddr
= {.addr
= gva
};
618 union raddress raddr
= {.addr
= gva
};
619 union page_table_entry pte
;
620 int dat_protection
= 0;
621 int iep_protection
= 0;
622 union ctlreg0 ctlreg0
;
624 int edat1
, edat2
, iep
;
626 ctlreg0
.val
= vcpu
->arch
.sie_block
->gcr
[0];
627 edat1
= ctlreg0
.edat
&& test_kvm_facility(vcpu
->kvm
, 8);
628 edat2
= edat1
&& test_kvm_facility(vcpu
->kvm
, 78);
629 iep
= ctlreg0
.iep
&& test_kvm_facility(vcpu
->kvm
, 130);
632 ptr
= asce
.origin
* PAGE_SIZE
;
634 case ASCE_TYPE_REGION1
:
635 if (vaddr
.rfx01
> asce
.tl
)
636 return PGM_REGION_FIRST_TRANS
;
637 ptr
+= vaddr
.rfx
* 8;
639 case ASCE_TYPE_REGION2
:
641 return PGM_ASCE_TYPE
;
642 if (vaddr
.rsx01
> asce
.tl
)
643 return PGM_REGION_SECOND_TRANS
;
644 ptr
+= vaddr
.rsx
* 8;
646 case ASCE_TYPE_REGION3
:
647 if (vaddr
.rfx
|| vaddr
.rsx
)
648 return PGM_ASCE_TYPE
;
649 if (vaddr
.rtx01
> asce
.tl
)
650 return PGM_REGION_THIRD_TRANS
;
651 ptr
+= vaddr
.rtx
* 8;
653 case ASCE_TYPE_SEGMENT
:
654 if (vaddr
.rfx
|| vaddr
.rsx
|| vaddr
.rtx
)
655 return PGM_ASCE_TYPE
;
656 if (vaddr
.sx01
> asce
.tl
)
657 return PGM_SEGMENT_TRANSLATION
;
662 case ASCE_TYPE_REGION1
: {
663 union region1_table_entry rfte
;
665 if (kvm_is_error_gpa(vcpu
->kvm
, ptr
))
666 return PGM_ADDRESSING
;
667 if (deref_table(vcpu
->kvm
, ptr
, &rfte
.val
))
670 return PGM_REGION_FIRST_TRANS
;
671 if (rfte
.tt
!= TABLE_TYPE_REGION1
)
672 return PGM_TRANSLATION_SPEC
;
673 if (vaddr
.rsx01
< rfte
.tf
|| vaddr
.rsx01
> rfte
.tl
)
674 return PGM_REGION_SECOND_TRANS
;
676 dat_protection
|= rfte
.p
;
677 ptr
= rfte
.rto
* PAGE_SIZE
+ vaddr
.rsx
* 8;
680 case ASCE_TYPE_REGION2
: {
681 union region2_table_entry rste
;
683 if (kvm_is_error_gpa(vcpu
->kvm
, ptr
))
684 return PGM_ADDRESSING
;
685 if (deref_table(vcpu
->kvm
, ptr
, &rste
.val
))
688 return PGM_REGION_SECOND_TRANS
;
689 if (rste
.tt
!= TABLE_TYPE_REGION2
)
690 return PGM_TRANSLATION_SPEC
;
691 if (vaddr
.rtx01
< rste
.tf
|| vaddr
.rtx01
> rste
.tl
)
692 return PGM_REGION_THIRD_TRANS
;
694 dat_protection
|= rste
.p
;
695 ptr
= rste
.rto
* PAGE_SIZE
+ vaddr
.rtx
* 8;
698 case ASCE_TYPE_REGION3
: {
699 union region3_table_entry rtte
;
701 if (kvm_is_error_gpa(vcpu
->kvm
, ptr
))
702 return PGM_ADDRESSING
;
703 if (deref_table(vcpu
->kvm
, ptr
, &rtte
.val
))
706 return PGM_REGION_THIRD_TRANS
;
707 if (rtte
.tt
!= TABLE_TYPE_REGION3
)
708 return PGM_TRANSLATION_SPEC
;
709 if (rtte
.cr
&& asce
.p
&& edat2
)
710 return PGM_TRANSLATION_SPEC
;
711 if (rtte
.fc
&& edat2
) {
712 dat_protection
|= rtte
.fc1
.p
;
713 iep_protection
= rtte
.fc1
.iep
;
714 raddr
.rfaa
= rtte
.fc1
.rfaa
;
715 goto absolute_address
;
717 if (vaddr
.sx01
< rtte
.fc0
.tf
)
718 return PGM_SEGMENT_TRANSLATION
;
719 if (vaddr
.sx01
> rtte
.fc0
.tl
)
720 return PGM_SEGMENT_TRANSLATION
;
722 dat_protection
|= rtte
.fc0
.p
;
723 ptr
= rtte
.fc0
.sto
* PAGE_SIZE
+ vaddr
.sx
* 8;
726 case ASCE_TYPE_SEGMENT
: {
727 union segment_table_entry ste
;
729 if (kvm_is_error_gpa(vcpu
->kvm
, ptr
))
730 return PGM_ADDRESSING
;
731 if (deref_table(vcpu
->kvm
, ptr
, &ste
.val
))
734 return PGM_SEGMENT_TRANSLATION
;
735 if (ste
.tt
!= TABLE_TYPE_SEGMENT
)
736 return PGM_TRANSLATION_SPEC
;
737 if (ste
.cs
&& asce
.p
)
738 return PGM_TRANSLATION_SPEC
;
739 if (ste
.fc
&& edat1
) {
740 dat_protection
|= ste
.fc1
.p
;
741 iep_protection
= ste
.fc1
.iep
;
742 raddr
.sfaa
= ste
.fc1
.sfaa
;
743 goto absolute_address
;
745 dat_protection
|= ste
.fc0
.p
;
746 ptr
= ste
.fc0
.pto
* (PAGE_SIZE
/ 2) + vaddr
.px
* 8;
749 if (kvm_is_error_gpa(vcpu
->kvm
, ptr
))
750 return PGM_ADDRESSING
;
751 if (deref_table(vcpu
->kvm
, ptr
, &pte
.val
))
754 return PGM_PAGE_TRANSLATION
;
756 return PGM_TRANSLATION_SPEC
;
757 dat_protection
|= pte
.p
;
758 iep_protection
= pte
.iep
;
759 raddr
.pfra
= pte
.pfra
;
761 raddr
.addr
= kvm_s390_real_to_abs(vcpu
, raddr
.addr
);
763 if (mode
== GACC_STORE
&& dat_protection
) {
764 *prot
= PROT_TYPE_DAT
;
765 return PGM_PROTECTION
;
767 if (mode
== GACC_IFETCH
&& iep_protection
&& iep
) {
768 *prot
= PROT_TYPE_IEP
;
769 return PGM_PROTECTION
;
771 if (kvm_is_error_gpa(vcpu
->kvm
, raddr
.addr
))
772 return PGM_ADDRESSING
;
777 static inline int is_low_address(unsigned long ga
)
779 /* Check for address ranges 0..511 and 4096..4607 */
780 return (ga
& ~0x11fful
) == 0;
783 static int low_address_protection_enabled(struct kvm_vcpu
*vcpu
,
784 const union asce asce
)
786 union ctlreg0 ctlreg0
= {.val
= vcpu
->arch
.sie_block
->gcr
[0]};
787 psw_t
*psw
= &vcpu
->arch
.sie_block
->gpsw
;
791 if (psw_bits(*psw
).dat
&& asce
.p
)
796 static int guest_page_range(struct kvm_vcpu
*vcpu
, unsigned long ga
, u8 ar
,
797 unsigned long *pages
, unsigned long nr_pages
,
798 const union asce asce
, enum gacc_mode mode
)
800 psw_t
*psw
= &vcpu
->arch
.sie_block
->gpsw
;
801 int lap_enabled
, rc
= 0;
804 lap_enabled
= low_address_protection_enabled(vcpu
, asce
);
806 ga
= kvm_s390_logical_to_effective(vcpu
, ga
);
807 if (mode
== GACC_STORE
&& lap_enabled
&& is_low_address(ga
))
808 return trans_exc(vcpu
, PGM_PROTECTION
, ga
, ar
, mode
,
811 if (psw_bits(*psw
).dat
) {
812 rc
= guest_translate(vcpu
, ga
, pages
, asce
, mode
, &prot
);
816 *pages
= kvm_s390_real_to_abs(vcpu
, ga
);
817 if (kvm_is_error_gpa(vcpu
->kvm
, *pages
))
821 return trans_exc(vcpu
, rc
, ga
, ar
, mode
, prot
);
829 int access_guest(struct kvm_vcpu
*vcpu
, unsigned long ga
, u8 ar
, void *data
,
830 unsigned long len
, enum gacc_mode mode
)
832 psw_t
*psw
= &vcpu
->arch
.sie_block
->gpsw
;
833 unsigned long _len
, nr_pages
, gpa
, idx
;
834 unsigned long pages_array
[2];
835 unsigned long *pages
;
842 ga
= kvm_s390_logical_to_effective(vcpu
, ga
);
843 rc
= get_vcpu_asce(vcpu
, &asce
, ga
, ar
, mode
);
846 nr_pages
= (((ga
& ~PAGE_MASK
) + len
- 1) >> PAGE_SHIFT
) + 1;
848 if (nr_pages
> ARRAY_SIZE(pages_array
))
849 pages
= vmalloc(nr_pages
* sizeof(unsigned long));
852 need_ipte_lock
= psw_bits(*psw
).dat
&& !asce
.r
;
855 rc
= guest_page_range(vcpu
, ga
, ar
, pages
, nr_pages
, asce
, mode
);
856 for (idx
= 0; idx
< nr_pages
&& !rc
; idx
++) {
857 gpa
= *(pages
+ idx
) + (ga
& ~PAGE_MASK
);
858 _len
= min(PAGE_SIZE
- (gpa
& ~PAGE_MASK
), len
);
859 if (mode
== GACC_STORE
)
860 rc
= kvm_write_guest(vcpu
->kvm
, gpa
, data
, _len
);
862 rc
= kvm_read_guest(vcpu
->kvm
, gpa
, data
, _len
);
869 if (nr_pages
> ARRAY_SIZE(pages_array
))
874 int access_guest_real(struct kvm_vcpu
*vcpu
, unsigned long gra
,
875 void *data
, unsigned long len
, enum gacc_mode mode
)
877 unsigned long _len
, gpa
;
881 gpa
= kvm_s390_real_to_abs(vcpu
, gra
);
882 _len
= min(PAGE_SIZE
- (gpa
& ~PAGE_MASK
), len
);
884 rc
= write_guest_abs(vcpu
, gpa
, data
, _len
);
886 rc
= read_guest_abs(vcpu
, gpa
, data
, _len
);
895 * guest_translate_address - translate guest logical into guest absolute address
897 * Parameter semantics are the same as the ones from guest_translate.
898 * The memory contents at the guest address are not changed.
900 * Note: The IPTE lock is not taken during this function, so the caller
901 * has to take care of this.
903 int guest_translate_address(struct kvm_vcpu
*vcpu
, unsigned long gva
, u8 ar
,
904 unsigned long *gpa
, enum gacc_mode mode
)
906 psw_t
*psw
= &vcpu
->arch
.sie_block
->gpsw
;
911 gva
= kvm_s390_logical_to_effective(vcpu
, gva
);
912 rc
= get_vcpu_asce(vcpu
, &asce
, gva
, ar
, mode
);
915 if (is_low_address(gva
) && low_address_protection_enabled(vcpu
, asce
)) {
916 if (mode
== GACC_STORE
)
917 return trans_exc(vcpu
, PGM_PROTECTION
, gva
, 0,
921 if (psw_bits(*psw
).dat
&& !asce
.r
) { /* Use DAT? */
922 rc
= guest_translate(vcpu
, gva
, gpa
, asce
, mode
, &prot
);
924 return trans_exc(vcpu
, rc
, gva
, 0, mode
, prot
);
926 *gpa
= kvm_s390_real_to_abs(vcpu
, gva
);
927 if (kvm_is_error_gpa(vcpu
->kvm
, *gpa
))
928 return trans_exc(vcpu
, rc
, gva
, PGM_ADDRESSING
, mode
, 0);
935 * check_gva_range - test a range of guest virtual addresses for accessibility
937 int check_gva_range(struct kvm_vcpu
*vcpu
, unsigned long gva
, u8 ar
,
938 unsigned long length
, enum gacc_mode mode
)
941 unsigned long currlen
;
945 while (length
> 0 && !rc
) {
946 currlen
= min(length
, PAGE_SIZE
- (gva
% PAGE_SIZE
));
947 rc
= guest_translate_address(vcpu
, gva
, ar
, &gpa
, mode
);
957 * kvm_s390_check_low_addr_prot_real - check for low-address protection
958 * @gra: Guest real address
960 * Checks whether an address is subject to low-address protection and set
961 * up vcpu->arch.pgm accordingly if necessary.
963 * Return: 0 if no protection exception, or PGM_PROTECTION if protected.
965 int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu
*vcpu
, unsigned long gra
)
967 union ctlreg0 ctlreg0
= {.val
= vcpu
->arch
.sie_block
->gcr
[0]};
969 if (!ctlreg0
.lap
|| !is_low_address(gra
))
971 return trans_exc(vcpu
, PGM_PROTECTION
, gra
, 0, GACC_STORE
, PROT_TYPE_LA
);
975 * kvm_s390_shadow_tables - walk the guest page table and create shadow tables
976 * @sg: pointer to the shadow guest address space structure
977 * @saddr: faulting address in the shadow gmap
978 * @pgt: pointer to the page table address result
979 * @fake: pgt references contiguous guest memory block, not a pgtable
981 static int kvm_s390_shadow_tables(struct gmap
*sg
, unsigned long saddr
,
982 unsigned long *pgt
, int *dat_protection
,
987 union vaddress vaddr
;
995 asce
.val
= sg
->orig_asce
;
996 ptr
= asce
.origin
* PAGE_SIZE
;
1000 asce
.dt
= ASCE_TYPE_REGION1
;
1003 case ASCE_TYPE_REGION1
:
1004 if (vaddr
.rfx01
> asce
.tl
&& !*fake
)
1005 return PGM_REGION_FIRST_TRANS
;
1007 case ASCE_TYPE_REGION2
:
1009 return PGM_ASCE_TYPE
;
1010 if (vaddr
.rsx01
> asce
.tl
)
1011 return PGM_REGION_SECOND_TRANS
;
1013 case ASCE_TYPE_REGION3
:
1014 if (vaddr
.rfx
|| vaddr
.rsx
)
1015 return PGM_ASCE_TYPE
;
1016 if (vaddr
.rtx01
> asce
.tl
)
1017 return PGM_REGION_THIRD_TRANS
;
1019 case ASCE_TYPE_SEGMENT
:
1020 if (vaddr
.rfx
|| vaddr
.rsx
|| vaddr
.rtx
)
1021 return PGM_ASCE_TYPE
;
1022 if (vaddr
.sx01
> asce
.tl
)
1023 return PGM_SEGMENT_TRANSLATION
;
1028 case ASCE_TYPE_REGION1
: {
1029 union region1_table_entry rfte
;
1032 ptr
+= vaddr
.rfx
* _REGION1_SIZE
;
1036 rc
= gmap_read_table(parent
, ptr
+ vaddr
.rfx
* 8, &rfte
.val
);
1040 return PGM_REGION_FIRST_TRANS
;
1041 if (rfte
.tt
!= TABLE_TYPE_REGION1
)
1042 return PGM_TRANSLATION_SPEC
;
1043 if (vaddr
.rsx01
< rfte
.tf
|| vaddr
.rsx01
> rfte
.tl
)
1044 return PGM_REGION_SECOND_TRANS
;
1045 if (sg
->edat_level
>= 1)
1046 *dat_protection
|= rfte
.p
;
1047 ptr
= rfte
.rto
* PAGE_SIZE
;
1049 rc
= gmap_shadow_r2t(sg
, saddr
, rfte
.val
, *fake
);
1054 case ASCE_TYPE_REGION2
: {
1055 union region2_table_entry rste
;
1058 ptr
+= vaddr
.rsx
* _REGION2_SIZE
;
1062 rc
= gmap_read_table(parent
, ptr
+ vaddr
.rsx
* 8, &rste
.val
);
1066 return PGM_REGION_SECOND_TRANS
;
1067 if (rste
.tt
!= TABLE_TYPE_REGION2
)
1068 return PGM_TRANSLATION_SPEC
;
1069 if (vaddr
.rtx01
< rste
.tf
|| vaddr
.rtx01
> rste
.tl
)
1070 return PGM_REGION_THIRD_TRANS
;
1071 if (sg
->edat_level
>= 1)
1072 *dat_protection
|= rste
.p
;
1073 ptr
= rste
.rto
* PAGE_SIZE
;
1075 rste
.p
|= *dat_protection
;
1076 rc
= gmap_shadow_r3t(sg
, saddr
, rste
.val
, *fake
);
1081 case ASCE_TYPE_REGION3
: {
1082 union region3_table_entry rtte
;
1085 ptr
+= vaddr
.rtx
* _REGION3_SIZE
;
1089 rc
= gmap_read_table(parent
, ptr
+ vaddr
.rtx
* 8, &rtte
.val
);
1093 return PGM_REGION_THIRD_TRANS
;
1094 if (rtte
.tt
!= TABLE_TYPE_REGION3
)
1095 return PGM_TRANSLATION_SPEC
;
1096 if (rtte
.cr
&& asce
.p
&& sg
->edat_level
>= 2)
1097 return PGM_TRANSLATION_SPEC
;
1098 if (rtte
.fc
&& sg
->edat_level
>= 2) {
1099 *dat_protection
|= rtte
.fc0
.p
;
1101 ptr
= rtte
.fc1
.rfaa
* _REGION3_SIZE
;
1105 if (vaddr
.sx01
< rtte
.fc0
.tf
|| vaddr
.sx01
> rtte
.fc0
.tl
)
1106 return PGM_SEGMENT_TRANSLATION
;
1107 if (sg
->edat_level
>= 1)
1108 *dat_protection
|= rtte
.fc0
.p
;
1109 ptr
= rtte
.fc0
.sto
* PAGE_SIZE
;
1111 rtte
.fc0
.p
|= *dat_protection
;
1112 rc
= gmap_shadow_sgt(sg
, saddr
, rtte
.val
, *fake
);
1117 case ASCE_TYPE_SEGMENT
: {
1118 union segment_table_entry ste
;
1121 ptr
+= vaddr
.sx
* _SEGMENT_SIZE
;
1125 rc
= gmap_read_table(parent
, ptr
+ vaddr
.sx
* 8, &ste
.val
);
1129 return PGM_SEGMENT_TRANSLATION
;
1130 if (ste
.tt
!= TABLE_TYPE_SEGMENT
)
1131 return PGM_TRANSLATION_SPEC
;
1132 if (ste
.cs
&& asce
.p
)
1133 return PGM_TRANSLATION_SPEC
;
1134 *dat_protection
|= ste
.fc0
.p
;
1135 if (ste
.fc
&& sg
->edat_level
>= 1) {
1137 ptr
= ste
.fc1
.sfaa
* _SEGMENT_SIZE
;
1141 ptr
= ste
.fc0
.pto
* (PAGE_SIZE
/ 2);
1143 ste
.fc0
.p
|= *dat_protection
;
1144 rc
= gmap_shadow_pgt(sg
, saddr
, ste
.val
, *fake
);
1149 /* Return the parent address of the page table */
1155 * kvm_s390_shadow_fault - handle fault on a shadow page table
1156 * @vcpu: virtual cpu
1157 * @sg: pointer to the shadow guest address space structure
1158 * @saddr: faulting address in the shadow gmap
1160 * Returns: - 0 if the shadow fault was successfully resolved
1161 * - > 0 (pgm exception code) on exceptions while faulting
1162 * - -EAGAIN if the caller can retry immediately
1163 * - -EFAULT when accessing invalid guest addresses
1164 * - -ENOMEM if out of memory
1166 int kvm_s390_shadow_fault(struct kvm_vcpu
*vcpu
, struct gmap
*sg
,
1167 unsigned long saddr
)
1169 union vaddress vaddr
;
1170 union page_table_entry pte
;
1172 int dat_protection
, fake
;
1175 down_read(&sg
->mm
->mmap_sem
);
1177 * We don't want any guest-2 tables to change - so the parent
1178 * tables/pointers we read stay valid - unshadowing is however
1179 * always possible - only guest_table_lock protects us.
1183 rc
= gmap_shadow_pgt_lookup(sg
, saddr
, &pgt
, &dat_protection
, &fake
);
1185 rc
= kvm_s390_shadow_tables(sg
, saddr
, &pgt
, &dat_protection
,
1190 pte
.val
= pgt
+ vaddr
.px
* PAGE_SIZE
;
1194 rc
= gmap_read_table(sg
->parent
, pgt
+ vaddr
.px
* 8, &pte
.val
);
1196 rc
= PGM_PAGE_TRANSLATION
;
1198 rc
= PGM_TRANSLATION_SPEC
;
1200 pte
.p
|= dat_protection
;
1202 rc
= gmap_shadow_page(sg
, saddr
, __pte(pte
.val
));
1204 up_read(&sg
->mm
->mmap_sem
);