2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 * GNU General Public License for more details.
11 * You should have received a copy of the GNU General Public License
12 * along with this program; if not, write to the Free Software
13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
15 * Copyright SUSE Linux Products GmbH 2009
17 * Authors: Alexander Graf <agraf@suse.de>
20 #include <linux/types.h>
21 #include <linux/string.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/highmem.h>
26 #include <asm/kvm_ppc.h>
27 #include <asm/kvm_book3s.h>
28 #include <asm/book3s/64/mmu-hash.h>
30 /* #define DEBUG_MMU */
33 #define dprintk(X...) printk(KERN_INFO X)
35 #define dprintk(X...) do { } while(0)
38 static void kvmppc_mmu_book3s_64_reset_msr(struct kvm_vcpu
*vcpu
)
40 unsigned long msr
= vcpu
->arch
.intr_msr
;
41 unsigned long cur_msr
= kvmppc_get_msr(vcpu
);
43 /* If transactional, change to suspend mode on IRQ delivery */
44 if (MSR_TM_TRANSACTIONAL(cur_msr
))
47 msr
|= cur_msr
& MSR_TS_MASK
;
49 kvmppc_set_msr(vcpu
, msr
);
52 static struct kvmppc_slb
*kvmppc_mmu_book3s_64_find_slbe(
53 struct kvm_vcpu
*vcpu
,
57 u64 esid
= GET_ESID(eaddr
);
58 u64 esid_1t
= GET_ESID_1T(eaddr
);
60 for (i
= 0; i
< vcpu
->arch
.slb_nr
; i
++) {
63 if (!vcpu
->arch
.slb
[i
].valid
)
66 if (vcpu
->arch
.slb
[i
].tb
)
69 if (vcpu
->arch
.slb
[i
].esid
== cmp_esid
)
70 return &vcpu
->arch
.slb
[i
];
73 dprintk("KVM: No SLB entry found for 0x%lx [%llx | %llx]\n",
74 eaddr
, esid
, esid_1t
);
75 for (i
= 0; i
< vcpu
->arch
.slb_nr
; i
++) {
76 if (vcpu
->arch
.slb
[i
].vsid
)
77 dprintk(" %d: %c%c%c %llx %llx\n", i
,
78 vcpu
->arch
.slb
[i
].valid
? 'v' : ' ',
79 vcpu
->arch
.slb
[i
].large
? 'l' : ' ',
80 vcpu
->arch
.slb
[i
].tb
? 't' : ' ',
81 vcpu
->arch
.slb
[i
].esid
,
82 vcpu
->arch
.slb
[i
].vsid
);
88 static int kvmppc_slb_sid_shift(struct kvmppc_slb
*slbe
)
90 return slbe
->tb
? SID_SHIFT_1T
: SID_SHIFT
;
93 static u64
kvmppc_slb_offset_mask(struct kvmppc_slb
*slbe
)
95 return (1ul << kvmppc_slb_sid_shift(slbe
)) - 1;
98 static u64
kvmppc_slb_calc_vpn(struct kvmppc_slb
*slb
, gva_t eaddr
)
100 eaddr
&= kvmppc_slb_offset_mask(slb
);
102 return (eaddr
>> VPN_SHIFT
) |
103 ((slb
->vsid
) << (kvmppc_slb_sid_shift(slb
) - VPN_SHIFT
));
106 static u64
kvmppc_mmu_book3s_64_ea_to_vp(struct kvm_vcpu
*vcpu
, gva_t eaddr
,
109 struct kvmppc_slb
*slb
;
111 slb
= kvmppc_mmu_book3s_64_find_slbe(vcpu
, eaddr
);
115 return kvmppc_slb_calc_vpn(slb
, eaddr
);
118 static int mmu_pagesize(int mmu_pg
)
129 static int kvmppc_mmu_book3s_64_get_pagesize(struct kvmppc_slb
*slbe
)
131 return mmu_pagesize(slbe
->base_page_size
);
134 static u32
kvmppc_mmu_book3s_64_get_page(struct kvmppc_slb
*slbe
, gva_t eaddr
)
136 int p
= kvmppc_mmu_book3s_64_get_pagesize(slbe
);
138 return ((eaddr
& kvmppc_slb_offset_mask(slbe
)) >> p
);
141 static hva_t
kvmppc_mmu_book3s_64_get_pteg(struct kvm_vcpu
*vcpu
,
142 struct kvmppc_slb
*slbe
, gva_t eaddr
,
145 struct kvmppc_vcpu_book3s
*vcpu_book3s
= to_book3s(vcpu
);
146 u64 hash
, pteg
, htabsize
;
151 htabsize
= ((1 << ((vcpu_book3s
->sdr1
& 0x1f) + 11)) - 1);
153 vpn
= kvmppc_slb_calc_vpn(slbe
, eaddr
);
154 ssize
= slbe
->tb
? MMU_SEGSIZE_1T
: MMU_SEGSIZE_256M
;
155 hash
= hpt_hash(vpn
, kvmppc_mmu_book3s_64_get_pagesize(slbe
), ssize
);
158 hash
&= ((1ULL << 39ULL) - 1ULL);
162 pteg
= vcpu_book3s
->sdr1
& 0xfffffffffffc0000ULL
;
165 dprintk("MMU: page=0x%x sdr1=0x%llx pteg=0x%llx vsid=0x%llx\n",
166 page
, vcpu_book3s
->sdr1
, pteg
, slbe
->vsid
);
168 /* When running a PAPR guest, SDR1 contains a HVA address instead
170 if (vcpu
->arch
.papr_enabled
)
173 r
= gfn_to_hva(vcpu
->kvm
, pteg
>> PAGE_SHIFT
);
175 if (kvm_is_error_hva(r
))
177 return r
| (pteg
& ~PAGE_MASK
);
180 static u64
kvmppc_mmu_book3s_64_get_avpn(struct kvmppc_slb
*slbe
, gva_t eaddr
)
182 int p
= kvmppc_mmu_book3s_64_get_pagesize(slbe
);
185 avpn
= kvmppc_mmu_book3s_64_get_page(slbe
, eaddr
);
186 avpn
|= slbe
->vsid
<< (kvmppc_slb_sid_shift(slbe
) - p
);
189 avpn
>>= ((80 - p
) - 56) - 8; /* 16 - p */
197 * Return page size encoded in the second word of a HPTE, or
198 * -1 for an invalid encoding for the base page size indicated by
199 * the SLB entry. This doesn't handle mixed pagesize segments yet.
201 static int decode_pagesize(struct kvmppc_slb
*slbe
, u64 r
)
203 switch (slbe
->base_page_size
) {
205 if ((r
& 0xf000) == 0x1000)
209 if ((r
& 0xff000) == 0)
216 static int kvmppc_mmu_book3s_64_xlate(struct kvm_vcpu
*vcpu
, gva_t eaddr
,
217 struct kvmppc_pte
*gpte
, bool data
,
220 struct kvmppc_slb
*slbe
;
232 ulong mp_ea
= vcpu
->arch
.magic_page_ea
;
234 /* Magic page override */
235 if (unlikely(mp_ea
) &&
236 unlikely((eaddr
& ~0xfffULL
) == (mp_ea
& ~0xfffULL
)) &&
237 !(kvmppc_get_msr(vcpu
) & MSR_PR
)) {
239 gpte
->vpage
= kvmppc_mmu_book3s_64_ea_to_vp(vcpu
, eaddr
, data
);
240 gpte
->raddr
= vcpu
->arch
.magic_page_pa
| (gpte
->raddr
& 0xfff);
241 gpte
->raddr
&= KVM_PAM
;
242 gpte
->may_execute
= true;
243 gpte
->may_read
= true;
244 gpte
->may_write
= true;
245 gpte
->page_size
= MMU_PAGE_4K
;
246 gpte
->wimg
= HPTE_R_M
;
251 slbe
= kvmppc_mmu_book3s_64_find_slbe(vcpu
, eaddr
);
255 avpn
= kvmppc_mmu_book3s_64_get_avpn(slbe
, eaddr
);
256 v_val
= avpn
& HPTE_V_AVPN
;
259 v_val
|= SLB_VSID_B_1T
;
261 v_val
|= HPTE_V_LARGE
;
262 v_val
|= HPTE_V_VALID
;
264 v_mask
= SLB_VSID_B
| HPTE_V_AVPN
| HPTE_V_LARGE
| HPTE_V_VALID
|
267 pgsize
= slbe
->large
? MMU_PAGE_16M
: MMU_PAGE_4K
;
269 mutex_lock(&vcpu
->kvm
->arch
.hpt_mutex
);
272 ptegp
= kvmppc_mmu_book3s_64_get_pteg(vcpu
, slbe
, eaddr
, second
);
273 if (kvm_is_error_hva(ptegp
))
276 if(copy_from_user(pteg
, (void __user
*)ptegp
, sizeof(pteg
))) {
277 printk_ratelimited(KERN_ERR
278 "KVM: Can't copy data from 0x%lx!\n", ptegp
);
282 if ((kvmppc_get_msr(vcpu
) & MSR_PR
) && slbe
->Kp
)
284 else if (!(kvmppc_get_msr(vcpu
) & MSR_PR
) && slbe
->Ks
)
287 for (i
=0; i
<16; i
+=2) {
288 u64 pte0
= be64_to_cpu(pteg
[i
]);
289 u64 pte1
= be64_to_cpu(pteg
[i
+ 1]);
291 /* Check all relevant fields of 1st dword */
292 if ((pte0
& v_mask
) == v_val
) {
293 /* If large page bit is set, check pgsize encoding */
295 (vcpu
->arch
.hflags
& BOOK3S_HFLAG_MULTI_PGSIZE
)) {
296 pgsize
= decode_pagesize(slbe
, pte1
);
308 v_val
|= HPTE_V_SECONDARY
;
313 v
= be64_to_cpu(pteg
[i
]);
314 r
= be64_to_cpu(pteg
[i
+1]);
315 pp
= (r
& HPTE_R_PP
) | key
;
320 gpte
->vpage
= kvmppc_mmu_book3s_64_ea_to_vp(vcpu
, eaddr
, data
);
322 eaddr_mask
= (1ull << mmu_pagesize(pgsize
)) - 1;
323 gpte
->raddr
= (r
& HPTE_R_RPN
& ~eaddr_mask
) | (eaddr
& eaddr_mask
);
324 gpte
->page_size
= pgsize
;
325 gpte
->may_execute
= ((r
& HPTE_R_N
) ? false : true);
326 if (unlikely(vcpu
->arch
.disable_kernel_nx
) &&
327 !(kvmppc_get_msr(vcpu
) & MSR_PR
))
328 gpte
->may_execute
= true;
329 gpte
->may_read
= false;
330 gpte
->may_write
= false;
331 gpte
->wimg
= r
& HPTE_R_WIMG
;
338 gpte
->may_write
= true;
344 gpte
->may_read
= true;
348 dprintk("KVM MMU: Translated 0x%lx [0x%llx] -> 0x%llx "
350 eaddr
, avpn
, gpte
->vpage
, gpte
->raddr
);
352 /* Update PTE R and C bits, so the guest's swapper knows we used the
354 if (gpte
->may_read
&& !(r
& HPTE_R_R
)) {
356 * Set the accessed flag.
357 * We have to write this back with a single byte write
358 * because another vcpu may be accessing this on
359 * non-PAPR platforms such as mac99, and this is
360 * what real hardware does.
362 char __user
*addr
= (char __user
*) (ptegp
+ (i
+ 1) * sizeof(u64
));
364 put_user(r
>> 8, addr
+ 6);
366 if (iswrite
&& gpte
->may_write
&& !(r
& HPTE_R_C
)) {
367 /* Set the dirty flag */
368 /* Use a single byte write */
369 char __user
*addr
= (char __user
*) (ptegp
+ (i
+ 1) * sizeof(u64
));
371 put_user(r
, addr
+ 7);
374 mutex_unlock(&vcpu
->kvm
->arch
.hpt_mutex
);
376 if (!gpte
->may_read
|| (iswrite
&& !gpte
->may_write
))
381 mutex_unlock(&vcpu
->kvm
->arch
.hpt_mutex
);
385 dprintk("KVM MMU: Trigger segment fault\n");
389 static void kvmppc_mmu_book3s_64_slbmte(struct kvm_vcpu
*vcpu
, u64 rs
, u64 rb
)
393 struct kvmppc_slb
*slbe
;
395 dprintk("KVM MMU: slbmte(0x%llx, 0x%llx)\n", rs
, rb
);
398 esid_1t
= GET_ESID_1T(rb
);
401 if (slb_nr
> vcpu
->arch
.slb_nr
)
404 slbe
= &vcpu
->arch
.slb
[slb_nr
];
406 slbe
->large
= (rs
& SLB_VSID_L
) ? 1 : 0;
407 slbe
->tb
= (rs
& SLB_VSID_B_1T
) ? 1 : 0;
408 slbe
->esid
= slbe
->tb
? esid_1t
: esid
;
409 slbe
->vsid
= (rs
& ~SLB_VSID_B
) >> (kvmppc_slb_sid_shift(slbe
) - 16);
410 slbe
->valid
= (rb
& SLB_ESID_V
) ? 1 : 0;
411 slbe
->Ks
= (rs
& SLB_VSID_KS
) ? 1 : 0;
412 slbe
->Kp
= (rs
& SLB_VSID_KP
) ? 1 : 0;
413 slbe
->nx
= (rs
& SLB_VSID_N
) ? 1 : 0;
414 slbe
->class = (rs
& SLB_VSID_C
) ? 1 : 0;
416 slbe
->base_page_size
= MMU_PAGE_4K
;
418 if (vcpu
->arch
.hflags
& BOOK3S_HFLAG_MULTI_PGSIZE
) {
419 switch (rs
& SLB_VSID_LP
) {
421 slbe
->base_page_size
= MMU_PAGE_16M
;
424 slbe
->base_page_size
= MMU_PAGE_64K
;
428 slbe
->base_page_size
= MMU_PAGE_16M
;
431 slbe
->orige
= rb
& (ESID_MASK
| SLB_ESID_V
);
434 /* Map the new segment */
435 kvmppc_mmu_map_segment(vcpu
, esid
<< SID_SHIFT
);
438 static u64
kvmppc_mmu_book3s_64_slbmfee(struct kvm_vcpu
*vcpu
, u64 slb_nr
)
440 struct kvmppc_slb
*slbe
;
442 if (slb_nr
> vcpu
->arch
.slb_nr
)
445 slbe
= &vcpu
->arch
.slb
[slb_nr
];
450 static u64
kvmppc_mmu_book3s_64_slbmfev(struct kvm_vcpu
*vcpu
, u64 slb_nr
)
452 struct kvmppc_slb
*slbe
;
454 if (slb_nr
> vcpu
->arch
.slb_nr
)
457 slbe
= &vcpu
->arch
.slb
[slb_nr
];
462 static void kvmppc_mmu_book3s_64_slbie(struct kvm_vcpu
*vcpu
, u64 ea
)
464 struct kvmppc_slb
*slbe
;
467 dprintk("KVM MMU: slbie(0x%llx)\n", ea
);
469 slbe
= kvmppc_mmu_book3s_64_find_slbe(vcpu
, ea
);
474 dprintk("KVM MMU: slbie(0x%llx, 0x%llx)\n", ea
, slbe
->esid
);
480 seg_size
= 1ull << kvmppc_slb_sid_shift(slbe
);
481 kvmppc_mmu_flush_segment(vcpu
, ea
& ~(seg_size
- 1), seg_size
);
484 static void kvmppc_mmu_book3s_64_slbia(struct kvm_vcpu
*vcpu
)
488 dprintk("KVM MMU: slbia()\n");
490 for (i
= 1; i
< vcpu
->arch
.slb_nr
; i
++) {
491 vcpu
->arch
.slb
[i
].valid
= false;
492 vcpu
->arch
.slb
[i
].orige
= 0;
493 vcpu
->arch
.slb
[i
].origv
= 0;
496 if (kvmppc_get_msr(vcpu
) & MSR_IR
) {
497 kvmppc_mmu_flush_segments(vcpu
);
498 kvmppc_mmu_map_segment(vcpu
, kvmppc_get_pc(vcpu
));
502 static void kvmppc_mmu_book3s_64_mtsrin(struct kvm_vcpu
*vcpu
, u32 srnum
,
508 * According to Book3 2.01 mtsrin is implemented as:
510 * The SLB entry specified by (RB)32:35 is loaded from register
513 * SLBE Bit Source SLB Field
515 * 0:31 0x0000_0000 ESID-0:31
516 * 32:35 (RB)32:35 ESID-32:35
518 * 37:61 0x00_0000|| 0b0 VSID-0:24
519 * 62:88 (RS)37:63 VSID-25:51
520 * 89:91 (RS)33:35 Ks Kp N
521 * 92 (RS)36 L ((RS)36 must be 0b0)
525 dprintk("KVM MMU: mtsrin(0x%x, 0x%lx)\n", srnum
, value
);
528 rb
|= (srnum
& 0xf) << 28;
529 /* Set the valid bit */
535 rs
|= (value
& 0xfffffff) << 12;
537 rs
|= ((value
>> 28) & 0x7) << 9;
539 kvmppc_mmu_book3s_64_slbmte(vcpu
, rs
, rb
);
542 static void kvmppc_mmu_book3s_64_tlbie(struct kvm_vcpu
*vcpu
, ulong va
,
545 u64 mask
= 0xFFFFFFFFFULL
;
549 dprintk("KVM MMU: tlbie(0x%lx)\n", va
);
552 * The tlbie instruction changed behaviour starting with
553 * POWER6. POWER6 and later don't have the large page flag
554 * in the instruction but in the RB value, along with bits
555 * indicating page and segment sizes.
557 if (vcpu
->arch
.hflags
& BOOK3S_HFLAG_NEW_TLBIE
) {
558 /* POWER6 or later */
559 if (va
& 1) { /* L bit */
560 if ((va
& 0xf000) == 0x1000)
561 mask
= 0xFFFFFFFF0ULL
; /* 64k page */
563 mask
= 0xFFFFFF000ULL
; /* 16M page */
566 /* older processors, e.g. PPC970 */
568 mask
= 0xFFFFFF000ULL
;
570 /* flush this VA on all vcpus */
571 kvm_for_each_vcpu(i
, v
, vcpu
->kvm
)
572 kvmppc_mmu_pte_vflush(v
, va
>> 12, mask
);
575 #ifdef CONFIG_PPC_64K_PAGES
576 static int segment_contains_magic_page(struct kvm_vcpu
*vcpu
, ulong esid
)
578 ulong mp_ea
= vcpu
->arch
.magic_page_ea
;
580 return mp_ea
&& !(kvmppc_get_msr(vcpu
) & MSR_PR
) &&
581 (mp_ea
>> SID_SHIFT
) == esid
;
585 static int kvmppc_mmu_book3s_64_esid_to_vsid(struct kvm_vcpu
*vcpu
, ulong esid
,
588 ulong ea
= esid
<< SID_SHIFT
;
589 struct kvmppc_slb
*slb
;
591 ulong mp_ea
= vcpu
->arch
.magic_page_ea
;
592 int pagesize
= MMU_PAGE_64K
;
593 u64 msr
= kvmppc_get_msr(vcpu
);
595 if (msr
& (MSR_DR
|MSR_IR
)) {
596 slb
= kvmppc_mmu_book3s_64_find_slbe(vcpu
, ea
);
599 pagesize
= slb
->base_page_size
;
601 gvsid
<<= SID_SHIFT_1T
- SID_SHIFT
;
602 gvsid
|= esid
& ((1ul << (SID_SHIFT_1T
- SID_SHIFT
)) - 1);
608 switch (msr
& (MSR_DR
|MSR_IR
)) {
610 gvsid
= VSID_REAL
| esid
;
613 gvsid
|= VSID_REAL_IR
;
616 gvsid
|= VSID_REAL_DR
;
628 #ifdef CONFIG_PPC_64K_PAGES
630 * Mark this as a 64k segment if the host is using
631 * 64k pages, the host MMU supports 64k pages and
632 * the guest segment page size is >= 64k,
633 * but not if this segment contains the magic page.
635 if (pagesize
>= MMU_PAGE_64K
&&
636 mmu_psize_defs
[MMU_PAGE_64K
].shift
&&
637 !segment_contains_magic_page(vcpu
, esid
))
641 if (kvmppc_get_msr(vcpu
) & MSR_PR
)
648 /* Catch magic page case */
649 if (unlikely(mp_ea
) &&
650 unlikely(esid
== (mp_ea
>> SID_SHIFT
)) &&
651 !(kvmppc_get_msr(vcpu
) & MSR_PR
)) {
652 *vsid
= VSID_REAL
| esid
;
659 static bool kvmppc_mmu_book3s_64_is_dcbz32(struct kvm_vcpu
*vcpu
)
661 return (to_book3s(vcpu
)->hid
[5] & 0x80);
664 void kvmppc_mmu_book3s_64_init(struct kvm_vcpu
*vcpu
)
666 struct kvmppc_mmu
*mmu
= &vcpu
->arch
.mmu
;
669 mmu
->mtsrin
= kvmppc_mmu_book3s_64_mtsrin
;
670 mmu
->slbmte
= kvmppc_mmu_book3s_64_slbmte
;
671 mmu
->slbmfee
= kvmppc_mmu_book3s_64_slbmfee
;
672 mmu
->slbmfev
= kvmppc_mmu_book3s_64_slbmfev
;
673 mmu
->slbie
= kvmppc_mmu_book3s_64_slbie
;
674 mmu
->slbia
= kvmppc_mmu_book3s_64_slbia
;
675 mmu
->xlate
= kvmppc_mmu_book3s_64_xlate
;
676 mmu
->reset_msr
= kvmppc_mmu_book3s_64_reset_msr
;
677 mmu
->tlbie
= kvmppc_mmu_book3s_64_tlbie
;
678 mmu
->esid_to_vsid
= kvmppc_mmu_book3s_64_esid_to_vsid
;
679 mmu
->ea_to_vp
= kvmppc_mmu_book3s_64_ea_to_vp
;
680 mmu
->is_dcbz32
= kvmppc_mmu_book3s_64_is_dcbz32
;
682 vcpu
->arch
.hflags
|= BOOK3S_HFLAG_SLB
;