2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 * GNU General Public License for more details.
11 * You should have received a copy of the GNU General Public License
12 * along with this program; if not, write to the Free Software
13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
15 * Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
18 #include <linux/types.h>
19 #include <linux/string.h>
20 #include <linux/kvm.h>
21 #include <linux/kvm_host.h>
22 #include <linux/highmem.h>
23 #include <linux/gfp.h>
24 #include <linux/slab.h>
25 #include <linux/hugetlb.h>
26 #include <linux/vmalloc.h>
28 #include <asm/tlbflush.h>
29 #include <asm/kvm_ppc.h>
30 #include <asm/kvm_book3s.h>
31 #include <asm/mmu-hash64.h>
32 #include <asm/hvcall.h>
33 #include <asm/synch.h>
34 #include <asm/ppc-opcode.h>
35 #include <asm/cputable.h>
37 /* POWER7 has 10-bit LPIDs, PPC970 has 6-bit LPIDs */
38 #define MAX_LPID_970 63
39 #define NR_LPIDS (LPID_RSVD + 1)
40 unsigned long lpid_inuse
[BITS_TO_LONGS(NR_LPIDS
)];
42 long kvmppc_alloc_hpt(struct kvm
*kvm
)
46 struct revmap_entry
*rev
;
47 struct kvmppc_linear_info
*li
;
49 /* Allocate guest's hashed page table */
52 /* using preallocated memory */
53 hpt
= (ulong
)li
->base_virt
;
54 kvm
->arch
.hpt_li
= li
;
56 /* using dynamic memory */
57 hpt
= __get_free_pages(GFP_KERNEL
|__GFP_ZERO
|__GFP_REPEAT
|
58 __GFP_NOWARN
, HPT_ORDER
- PAGE_SHIFT
);
62 pr_err("kvm_alloc_hpt: Couldn't alloc HPT\n");
65 kvm
->arch
.hpt_virt
= hpt
;
67 /* Allocate reverse map array */
68 rev
= vmalloc(sizeof(struct revmap_entry
) * HPT_NPTE
);
70 pr_err("kvmppc_alloc_hpt: Couldn't alloc reverse map array\n");
73 kvm
->arch
.revmap
= rev
;
75 /* Allocate the guest's logical partition ID */
77 lpid
= find_first_zero_bit(lpid_inuse
, NR_LPIDS
);
78 if (lpid
>= NR_LPIDS
) {
79 pr_err("kvm_alloc_hpt: No LPIDs free\n");
82 } while (test_and_set_bit(lpid
, lpid_inuse
));
84 kvm
->arch
.sdr1
= __pa(hpt
) | (HPT_ORDER
- 18);
85 kvm
->arch
.lpid
= lpid
;
87 pr_info("KVM guest htab at %lx, LPID %lx\n", hpt
, lpid
);
93 free_pages(hpt
, HPT_ORDER
- PAGE_SHIFT
);
97 void kvmppc_free_hpt(struct kvm
*kvm
)
99 clear_bit(kvm
->arch
.lpid
, lpid_inuse
);
100 vfree(kvm
->arch
.revmap
);
101 if (kvm
->arch
.hpt_li
)
102 kvm_release_hpt(kvm
->arch
.hpt_li
);
104 free_pages(kvm
->arch
.hpt_virt
, HPT_ORDER
- PAGE_SHIFT
);
107 /* Bits in first HPTE dword for pagesize 4k, 64k or 16M */
108 static inline unsigned long hpte0_pgsize_encoding(unsigned long pgsize
)
110 return (pgsize
> 0x1000) ? HPTE_V_LARGE
: 0;
113 /* Bits in second HPTE dword for pagesize 4k, 64k or 16M */
114 static inline unsigned long hpte1_pgsize_encoding(unsigned long pgsize
)
116 return (pgsize
== 0x10000) ? 0x1000 : 0;
119 void kvmppc_map_vrma(struct kvm_vcpu
*vcpu
, struct kvm_memory_slot
*memslot
,
120 unsigned long porder
)
123 unsigned long npages
;
124 unsigned long hp_v
, hp_r
;
125 unsigned long addr
, hash
;
127 unsigned long hp0
, hp1
;
130 psize
= 1ul << porder
;
131 npages
= memslot
->npages
>> (porder
- PAGE_SHIFT
);
133 /* VRMA can't be > 1TB */
134 if (npages
> 1ul << (40 - porder
))
135 npages
= 1ul << (40 - porder
);
136 /* Can't use more than 1 HPTE per HPTEG */
137 if (npages
> HPT_NPTEG
)
140 hp0
= HPTE_V_1TB_SEG
| (VRMA_VSID
<< (40 - 16)) |
141 HPTE_V_BOLTED
| hpte0_pgsize_encoding(psize
);
142 hp1
= hpte1_pgsize_encoding(psize
) |
143 HPTE_R_R
| HPTE_R_C
| HPTE_R_M
| PP_RWXX
;
145 for (i
= 0; i
< npages
; ++i
) {
147 /* can't use hpt_hash since va > 64 bits */
148 hash
= (i
^ (VRMA_VSID
^ (VRMA_VSID
<< 25))) & HPT_HASH_MASK
;
150 * We assume that the hash table is empty and no
151 * vcpus are using it at this stage. Since we create
152 * at most one HPTE per HPTEG, we just assume entry 7
153 * is available and use it.
155 hash
= (hash
<< 3) + 7;
156 hp_v
= hp0
| ((addr
>> 16) & ~0x7fUL
);
158 ret
= kvmppc_virtmode_h_enter(vcpu
, H_EXACT
, hash
, hp_v
, hp_r
);
159 if (ret
!= H_SUCCESS
) {
160 pr_err("KVM: map_vrma at %lx failed, ret=%ld\n",
167 int kvmppc_mmu_hv_init(void)
169 unsigned long host_lpid
, rsvd_lpid
;
171 if (!cpu_has_feature(CPU_FTR_HVMODE
))
174 memset(lpid_inuse
, 0, sizeof(lpid_inuse
));
176 if (cpu_has_feature(CPU_FTR_ARCH_206
)) {
177 host_lpid
= mfspr(SPRN_LPID
); /* POWER7 */
178 rsvd_lpid
= LPID_RSVD
;
180 host_lpid
= 0; /* PPC970 */
181 rsvd_lpid
= MAX_LPID_970
;
184 set_bit(host_lpid
, lpid_inuse
);
185 /* rsvd_lpid is reserved for use in partition switching */
186 set_bit(rsvd_lpid
, lpid_inuse
);
191 void kvmppc_mmu_destroy(struct kvm_vcpu
*vcpu
)
195 static void kvmppc_mmu_book3s_64_hv_reset_msr(struct kvm_vcpu
*vcpu
)
197 kvmppc_set_msr(vcpu
, MSR_SF
| MSR_ME
);
201 * This is called to get a reference to a guest page if there isn't
202 * one already in the kvm->arch.slot_phys[][] arrays.
204 static long kvmppc_get_guest_page(struct kvm
*kvm
, unsigned long gfn
,
205 struct kvm_memory_slot
*memslot
,
210 struct page
*page
, *hpage
, *pages
[1];
211 unsigned long s
, pgsize
;
212 unsigned long *physp
;
213 unsigned int is_io
, got
, pgorder
;
214 struct vm_area_struct
*vma
;
215 unsigned long pfn
, i
, npages
;
217 physp
= kvm
->arch
.slot_phys
[memslot
->id
];
220 if (physp
[gfn
- memslot
->base_gfn
])
228 start
= gfn_to_hva_memslot(memslot
, gfn
);
230 /* Instantiate and get the page we want access to */
231 np
= get_user_pages_fast(start
, 1, 1, pages
);
233 /* Look up the vma for the page */
234 down_read(¤t
->mm
->mmap_sem
);
235 vma
= find_vma(current
->mm
, start
);
236 if (!vma
|| vma
->vm_start
> start
||
237 start
+ psize
> vma
->vm_end
||
238 !(vma
->vm_flags
& VM_PFNMAP
))
240 is_io
= hpte_cache_bits(pgprot_val(vma
->vm_page_prot
));
241 pfn
= vma
->vm_pgoff
+ ((start
- vma
->vm_start
) >> PAGE_SHIFT
);
242 /* check alignment of pfn vs. requested page size */
243 if (psize
> PAGE_SIZE
&& (pfn
& ((psize
>> PAGE_SHIFT
) - 1)))
245 up_read(¤t
->mm
->mmap_sem
);
249 got
= KVMPPC_GOT_PAGE
;
251 /* See if this is a large page */
253 if (PageHuge(page
)) {
254 hpage
= compound_head(page
);
255 s
<<= compound_order(hpage
);
256 /* Get the whole large page if slot alignment is ok */
257 if (s
> psize
&& slot_is_aligned(memslot
, s
) &&
258 !(memslot
->userspace_addr
& (s
- 1))) {
266 pfn
= page_to_pfn(page
);
269 npages
= pgsize
>> PAGE_SHIFT
;
270 pgorder
= __ilog2(npages
);
271 physp
+= (gfn
- memslot
->base_gfn
) & ~(npages
- 1);
272 spin_lock(&kvm
->arch
.slot_phys_lock
);
273 for (i
= 0; i
< npages
; ++i
) {
275 physp
[i
] = ((pfn
+ i
) << PAGE_SHIFT
) +
276 got
+ is_io
+ pgorder
;
280 spin_unlock(&kvm
->arch
.slot_phys_lock
);
286 page
= compound_head(page
);
292 up_read(¤t
->mm
->mmap_sem
);
297 * We come here on a H_ENTER call from the guest when we are not
298 * using mmu notifiers and we don't have the requested page pinned
301 long kvmppc_virtmode_h_enter(struct kvm_vcpu
*vcpu
, unsigned long flags
,
302 long pte_index
, unsigned long pteh
, unsigned long ptel
)
304 struct kvm
*kvm
= vcpu
->kvm
;
305 unsigned long psize
, gpa
, gfn
;
306 struct kvm_memory_slot
*memslot
;
309 if (kvm
->arch
.using_mmu_notifiers
)
312 psize
= hpte_page_size(pteh
, ptel
);
316 pteh
&= ~(HPTE_V_HVLOCK
| HPTE_V_ABSENT
| HPTE_V_VALID
);
318 /* Find the memslot (if any) for this address */
319 gpa
= (ptel
& HPTE_R_RPN
) & ~(psize
- 1);
320 gfn
= gpa
>> PAGE_SHIFT
;
321 memslot
= gfn_to_memslot(kvm
, gfn
);
322 if (memslot
&& !(memslot
->flags
& KVM_MEMSLOT_INVALID
)) {
323 if (!slot_is_aligned(memslot
, psize
))
325 if (kvmppc_get_guest_page(kvm
, gfn
, memslot
, psize
) < 0)
330 /* Protect linux PTE lookup from page table destruction */
331 rcu_read_lock_sched(); /* this disables preemption too */
332 vcpu
->arch
.pgdir
= current
->mm
->pgd
;
333 ret
= kvmppc_h_enter(vcpu
, flags
, pte_index
, pteh
, ptel
);
334 rcu_read_unlock_sched();
335 if (ret
== H_TOO_HARD
) {
336 /* this can't happen */
337 pr_err("KVM: Oops, kvmppc_h_enter returned too hard!\n");
338 ret
= H_RESOURCE
; /* or something */
344 static struct kvmppc_slb
*kvmppc_mmu_book3s_hv_find_slbe(struct kvm_vcpu
*vcpu
,
350 for (i
= 0; i
< vcpu
->arch
.slb_nr
; i
++) {
351 if (!(vcpu
->arch
.slb
[i
].orige
& SLB_ESID_V
))
354 if (vcpu
->arch
.slb
[i
].origv
& SLB_VSID_B_1T
)
359 if (((vcpu
->arch
.slb
[i
].orige
^ eaddr
) & mask
) == 0)
360 return &vcpu
->arch
.slb
[i
];
365 static unsigned long kvmppc_mmu_get_real_addr(unsigned long v
, unsigned long r
,
368 unsigned long ra_mask
;
370 ra_mask
= hpte_page_size(v
, r
) - 1;
371 return (r
& HPTE_R_RPN
& ~ra_mask
) | (ea
& ra_mask
);
374 static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu
*vcpu
, gva_t eaddr
,
375 struct kvmppc_pte
*gpte
, bool data
)
377 struct kvm
*kvm
= vcpu
->kvm
;
378 struct kvmppc_slb
*slbe
;
380 unsigned long pp
, key
;
382 unsigned long *hptep
;
384 int virtmode
= vcpu
->arch
.shregs
.msr
& (data
? MSR_DR
: MSR_IR
);
388 slbe
= kvmppc_mmu_book3s_hv_find_slbe(vcpu
, eaddr
);
393 /* real mode access */
394 slb_v
= vcpu
->kvm
->arch
.vrma_slb_v
;
397 /* Find the HPTE in the hash table */
398 index
= kvmppc_hv_find_lock_hpte(kvm
, eaddr
, slb_v
,
399 HPTE_V_VALID
| HPTE_V_ABSENT
);
402 hptep
= (unsigned long *)(kvm
->arch
.hpt_virt
+ (index
<< 4));
403 v
= hptep
[0] & ~HPTE_V_HVLOCK
;
404 gr
= kvm
->arch
.revmap
[index
].guest_rpte
;
406 /* Unlock the HPTE */
407 asm volatile("lwsync" : : : "memory");
411 gpte
->vpage
= ((v
& HPTE_V_AVPN
) << 4) | ((eaddr
>> 12) & 0xfff);
413 /* Get PP bits and key for permission check */
414 pp
= gr
& (HPTE_R_PP0
| HPTE_R_PP
);
415 key
= (vcpu
->arch
.shregs
.msr
& MSR_PR
) ? SLB_VSID_KP
: SLB_VSID_KS
;
418 /* Calculate permissions */
419 gpte
->may_read
= hpte_read_permission(pp
, key
);
420 gpte
->may_write
= hpte_write_permission(pp
, key
);
421 gpte
->may_execute
= gpte
->may_read
&& !(gr
& (HPTE_R_N
| HPTE_R_G
));
423 /* Storage key permission check for POWER7 */
424 if (data
&& virtmode
&& cpu_has_feature(CPU_FTR_ARCH_206
)) {
425 int amrfield
= hpte_get_skey_perm(gr
, vcpu
->arch
.amr
);
432 /* Get the guest physical address */
433 gpte
->raddr
= kvmppc_mmu_get_real_addr(v
, gr
, eaddr
);
438 * Quick test for whether an instruction is a load or a store.
439 * If the instruction is a load or a store, then this will indicate
440 * which it is, at least on server processors. (Embedded processors
441 * have some external PID instructions that don't follow the rule
442 * embodied here.) If the instruction isn't a load or store, then
443 * this doesn't return anything useful.
445 static int instruction_is_store(unsigned int instr
)
450 if ((instr
& 0xfc000000) == 0x7c000000)
451 mask
= 0x100; /* major opcode 31 */
452 return (instr
& mask
) != 0;
455 static int kvmppc_hv_emulate_mmio(struct kvm_run
*run
, struct kvm_vcpu
*vcpu
,
456 unsigned long gpa
, int is_store
)
460 unsigned long srr0
= kvmppc_get_pc(vcpu
);
462 /* We try to load the last instruction. We don't let
463 * emulate_instruction do it as it doesn't check what
465 * If we fail, we just return to the guest and try executing it again.
467 if (vcpu
->arch
.last_inst
== KVM_INST_FETCH_FAILED
) {
468 ret
= kvmppc_ld(vcpu
, &srr0
, sizeof(u32
), &last_inst
, false);
469 if (ret
!= EMULATE_DONE
|| last_inst
== KVM_INST_FETCH_FAILED
)
471 vcpu
->arch
.last_inst
= last_inst
;
475 * WARNING: We do not know for sure whether the instruction we just
476 * read from memory is the same that caused the fault in the first
477 * place. If the instruction we read is neither an load or a store,
478 * then it can't access memory, so we don't need to worry about
479 * enforcing access permissions. So, assuming it is a load or
480 * store, we just check that its direction (load or store) is
481 * consistent with the original fault, since that's what we
482 * checked the access permissions against. If there is a mismatch
483 * we just return and retry the instruction.
486 if (instruction_is_store(vcpu
->arch
.last_inst
) != !!is_store
)
490 * Emulated accesses are emulated by looking at the hash for
491 * translation once, then performing the access later. The
492 * translation could be invalidated in the meantime in which
493 * point performing the subsequent memory access on the old
494 * physical address could possibly be a security hole for the
495 * guest (but not the host).
497 * This is less of an issue for MMIO stores since they aren't
498 * globally visible. It could be an issue for MMIO loads to
499 * a certain extent but we'll ignore it for now.
502 vcpu
->arch
.paddr_accessed
= gpa
;
503 return kvmppc_emulate_mmio(run
, vcpu
);
506 int kvmppc_book3s_hv_page_fault(struct kvm_run
*run
, struct kvm_vcpu
*vcpu
,
507 unsigned long ea
, unsigned long dsisr
)
509 struct kvm
*kvm
= vcpu
->kvm
;
510 unsigned long *hptep
, hpte
[3], r
;
511 unsigned long mmu_seq
, psize
, pte_size
;
512 unsigned long gfn
, hva
, pfn
;
513 struct kvm_memory_slot
*memslot
;
515 struct revmap_entry
*rev
;
516 struct page
*page
, *pages
[1];
517 long index
, ret
, npages
;
519 unsigned int writing
, write_ok
;
520 struct vm_area_struct
*vma
;
521 unsigned long rcbits
;
524 * Real-mode code has already searched the HPT and found the
525 * entry we're interested in. Lock the entry and check that
526 * it hasn't changed. If it has, just return and re-execute the
529 if (ea
!= vcpu
->arch
.pgfault_addr
)
531 index
= vcpu
->arch
.pgfault_index
;
532 hptep
= (unsigned long *)(kvm
->arch
.hpt_virt
+ (index
<< 4));
533 rev
= &kvm
->arch
.revmap
[index
];
535 while (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
))
537 hpte
[0] = hptep
[0] & ~HPTE_V_HVLOCK
;
539 hpte
[2] = r
= rev
->guest_rpte
;
540 asm volatile("lwsync" : : : "memory");
544 if (hpte
[0] != vcpu
->arch
.pgfault_hpte
[0] ||
545 hpte
[1] != vcpu
->arch
.pgfault_hpte
[1])
548 /* Translate the logical address and get the page */
549 psize
= hpte_page_size(hpte
[0], r
);
550 gfn
= hpte_rpn(r
, psize
);
551 memslot
= gfn_to_memslot(kvm
, gfn
);
553 /* No memslot means it's an emulated MMIO region */
554 if (!memslot
|| (memslot
->flags
& KVM_MEMSLOT_INVALID
)) {
555 unsigned long gpa
= (gfn
<< PAGE_SHIFT
) | (ea
& (psize
- 1));
556 return kvmppc_hv_emulate_mmio(run
, vcpu
, gpa
,
557 dsisr
& DSISR_ISSTORE
);
560 if (!kvm
->arch
.using_mmu_notifiers
)
561 return -EFAULT
; /* should never get here */
563 /* used to check for invalidations in progress */
564 mmu_seq
= kvm
->mmu_notifier_seq
;
570 pte_size
= PAGE_SIZE
;
571 writing
= (dsisr
& DSISR_ISSTORE
) != 0;
572 /* If writing != 0, then the HPTE must allow writing, if we get here */
574 hva
= gfn_to_hva_memslot(memslot
, gfn
);
575 npages
= get_user_pages_fast(hva
, 1, writing
, pages
);
577 /* Check if it's an I/O mapping */
578 down_read(¤t
->mm
->mmap_sem
);
579 vma
= find_vma(current
->mm
, hva
);
580 if (vma
&& vma
->vm_start
<= hva
&& hva
+ psize
<= vma
->vm_end
&&
581 (vma
->vm_flags
& VM_PFNMAP
)) {
582 pfn
= vma
->vm_pgoff
+
583 ((hva
- vma
->vm_start
) >> PAGE_SHIFT
);
585 is_io
= hpte_cache_bits(pgprot_val(vma
->vm_page_prot
));
586 write_ok
= vma
->vm_flags
& VM_WRITE
;
588 up_read(¤t
->mm
->mmap_sem
);
593 if (PageHuge(page
)) {
594 page
= compound_head(page
);
595 pte_size
<<= compound_order(page
);
597 /* if the guest wants write access, see if that is OK */
598 if (!writing
&& hpte_is_writable(r
)) {
602 * We need to protect against page table destruction
603 * while looking up and updating the pte.
605 rcu_read_lock_sched();
606 ptep
= find_linux_pte_or_hugepte(current
->mm
->pgd
,
608 if (ptep
&& pte_present(*ptep
)) {
609 pte
= kvmppc_read_update_linux_pte(ptep
, 1);
613 rcu_read_unlock_sched();
615 pfn
= page_to_pfn(page
);
619 if (psize
> pte_size
)
622 /* Check WIMG vs. the actual page we're accessing */
623 if (!hpte_cache_flags_ok(r
, is_io
)) {
627 * Allow guest to map emulated device memory as
628 * uncacheable, but actually make it cacheable.
630 r
= (r
& ~(HPTE_R_W
|HPTE_R_I
|HPTE_R_G
)) | HPTE_R_M
;
633 /* Set the HPTE to point to pfn */
634 r
= (r
& ~(HPTE_R_PP0
- pte_size
)) | (pfn
<< PAGE_SHIFT
);
635 if (hpte_is_writable(r
) && !write_ok
)
636 r
= hpte_make_readonly(r
);
639 while (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
))
641 if ((hptep
[0] & ~HPTE_V_HVLOCK
) != hpte
[0] || hptep
[1] != hpte
[1] ||
642 rev
->guest_rpte
!= hpte
[2])
643 /* HPTE has been changed under us; let the guest retry */
645 hpte
[0] = (hpte
[0] & ~HPTE_V_ABSENT
) | HPTE_V_VALID
;
647 rmap
= &memslot
->rmap
[gfn
- memslot
->base_gfn
];
650 /* Check if we might have been invalidated; let the guest retry if so */
652 if (mmu_notifier_retry(vcpu
, mmu_seq
)) {
657 /* Only set R/C in real HPTE if set in both *rmap and guest_rpte */
658 rcbits
= *rmap
>> KVMPPC_RMAP_RC_SHIFT
;
659 r
&= rcbits
| ~(HPTE_R_R
| HPTE_R_C
);
661 if (hptep
[0] & HPTE_V_VALID
) {
662 /* HPTE was previously valid, so we need to invalidate it */
664 hptep
[0] |= HPTE_V_ABSENT
;
665 kvmppc_invalidate_hpte(kvm
, hptep
, index
);
666 /* don't lose previous R and C bits */
667 r
|= hptep
[1] & (HPTE_R_R
| HPTE_R_C
);
669 kvmppc_add_revmap_chain(kvm
, rev
, rmap
, index
, 0);
675 asm volatile("ptesync" : : : "memory");
677 if (page
&& hpte_is_writable(r
))
686 hptep
[0] &= ~HPTE_V_HVLOCK
;
691 static int kvm_handle_hva(struct kvm
*kvm
, unsigned long hva
,
692 int (*handler
)(struct kvm
*kvm
, unsigned long *rmapp
,
697 struct kvm_memslots
*slots
;
698 struct kvm_memory_slot
*memslot
;
700 slots
= kvm_memslots(kvm
);
701 kvm_for_each_memslot(memslot
, slots
) {
702 unsigned long start
= memslot
->userspace_addr
;
705 end
= start
+ (memslot
->npages
<< PAGE_SHIFT
);
706 if (hva
>= start
&& hva
< end
) {
707 gfn_t gfn_offset
= (hva
- start
) >> PAGE_SHIFT
;
709 ret
= handler(kvm
, &memslot
->rmap
[gfn_offset
],
710 memslot
->base_gfn
+ gfn_offset
);
718 static int kvm_unmap_rmapp(struct kvm
*kvm
, unsigned long *rmapp
,
721 struct revmap_entry
*rev
= kvm
->arch
.revmap
;
722 unsigned long h
, i
, j
;
723 unsigned long *hptep
;
724 unsigned long ptel
, psize
, rcbits
;
728 if (!(*rmapp
& KVMPPC_RMAP_PRESENT
)) {
734 * To avoid an ABBA deadlock with the HPTE lock bit,
735 * we can't spin on the HPTE lock while holding the
738 i
= *rmapp
& KVMPPC_RMAP_INDEX
;
739 hptep
= (unsigned long *) (kvm
->arch
.hpt_virt
+ (i
<< 4));
740 if (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
)) {
741 /* unlock rmap before spinning on the HPTE lock */
743 while (hptep
[0] & HPTE_V_HVLOCK
)
749 /* chain is now empty */
750 *rmapp
&= ~(KVMPPC_RMAP_PRESENT
| KVMPPC_RMAP_INDEX
);
752 /* remove i from chain */
756 rev
[i
].forw
= rev
[i
].back
= i
;
757 *rmapp
= (*rmapp
& ~KVMPPC_RMAP_INDEX
) | j
;
760 /* Now check and modify the HPTE */
761 ptel
= rev
[i
].guest_rpte
;
762 psize
= hpte_page_size(hptep
[0], ptel
);
763 if ((hptep
[0] & HPTE_V_VALID
) &&
764 hpte_rpn(ptel
, psize
) == gfn
) {
765 hptep
[0] |= HPTE_V_ABSENT
;
766 kvmppc_invalidate_hpte(kvm
, hptep
, i
);
767 /* Harvest R and C */
768 rcbits
= hptep
[1] & (HPTE_R_R
| HPTE_R_C
);
769 *rmapp
|= rcbits
<< KVMPPC_RMAP_RC_SHIFT
;
770 rev
[i
].guest_rpte
= ptel
| rcbits
;
773 hptep
[0] &= ~HPTE_V_HVLOCK
;
778 int kvm_unmap_hva(struct kvm
*kvm
, unsigned long hva
)
780 if (kvm
->arch
.using_mmu_notifiers
)
781 kvm_handle_hva(kvm
, hva
, kvm_unmap_rmapp
);
785 static int kvm_age_rmapp(struct kvm
*kvm
, unsigned long *rmapp
,
788 struct revmap_entry
*rev
= kvm
->arch
.revmap
;
789 unsigned long head
, i
, j
;
790 unsigned long *hptep
;
795 if (*rmapp
& KVMPPC_RMAP_REFERENCED
) {
796 *rmapp
&= ~KVMPPC_RMAP_REFERENCED
;
799 if (!(*rmapp
& KVMPPC_RMAP_PRESENT
)) {
804 i
= head
= *rmapp
& KVMPPC_RMAP_INDEX
;
806 hptep
= (unsigned long *) (kvm
->arch
.hpt_virt
+ (i
<< 4));
809 /* If this HPTE isn't referenced, ignore it */
810 if (!(hptep
[1] & HPTE_R_R
))
813 if (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
)) {
814 /* unlock rmap before spinning on the HPTE lock */
816 while (hptep
[0] & HPTE_V_HVLOCK
)
821 /* Now check and modify the HPTE */
822 if ((hptep
[0] & HPTE_V_VALID
) && (hptep
[1] & HPTE_R_R
)) {
823 kvmppc_clear_ref_hpte(kvm
, hptep
, i
);
824 rev
[i
].guest_rpte
|= HPTE_R_R
;
827 hptep
[0] &= ~HPTE_V_HVLOCK
;
828 } while ((i
= j
) != head
);
834 int kvm_age_hva(struct kvm
*kvm
, unsigned long hva
)
836 if (!kvm
->arch
.using_mmu_notifiers
)
838 return kvm_handle_hva(kvm
, hva
, kvm_age_rmapp
);
841 static int kvm_test_age_rmapp(struct kvm
*kvm
, unsigned long *rmapp
,
844 struct revmap_entry
*rev
= kvm
->arch
.revmap
;
845 unsigned long head
, i
, j
;
849 if (*rmapp
& KVMPPC_RMAP_REFERENCED
)
853 if (*rmapp
& KVMPPC_RMAP_REFERENCED
)
856 if (*rmapp
& KVMPPC_RMAP_PRESENT
) {
857 i
= head
= *rmapp
& KVMPPC_RMAP_INDEX
;
859 hp
= (unsigned long *)(kvm
->arch
.hpt_virt
+ (i
<< 4));
861 if (hp
[1] & HPTE_R_R
)
863 } while ((i
= j
) != head
);
872 int kvm_test_age_hva(struct kvm
*kvm
, unsigned long hva
)
874 if (!kvm
->arch
.using_mmu_notifiers
)
876 return kvm_handle_hva(kvm
, hva
, kvm_test_age_rmapp
);
879 void kvm_set_spte_hva(struct kvm
*kvm
, unsigned long hva
, pte_t pte
)
881 if (!kvm
->arch
.using_mmu_notifiers
)
883 kvm_handle_hva(kvm
, hva
, kvm_unmap_rmapp
);
886 static int kvm_test_clear_dirty(struct kvm
*kvm
, unsigned long *rmapp
)
888 struct revmap_entry
*rev
= kvm
->arch
.revmap
;
889 unsigned long head
, i
, j
;
890 unsigned long *hptep
;
895 if (*rmapp
& KVMPPC_RMAP_CHANGED
) {
896 *rmapp
&= ~KVMPPC_RMAP_CHANGED
;
899 if (!(*rmapp
& KVMPPC_RMAP_PRESENT
)) {
904 i
= head
= *rmapp
& KVMPPC_RMAP_INDEX
;
906 hptep
= (unsigned long *) (kvm
->arch
.hpt_virt
+ (i
<< 4));
909 if (!(hptep
[1] & HPTE_R_C
))
912 if (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
)) {
913 /* unlock rmap before spinning on the HPTE lock */
915 while (hptep
[0] & HPTE_V_HVLOCK
)
920 /* Now check and modify the HPTE */
921 if ((hptep
[0] & HPTE_V_VALID
) && (hptep
[1] & HPTE_R_C
)) {
922 /* need to make it temporarily absent to clear C */
923 hptep
[0] |= HPTE_V_ABSENT
;
924 kvmppc_invalidate_hpte(kvm
, hptep
, i
);
925 hptep
[1] &= ~HPTE_R_C
;
927 hptep
[0] = (hptep
[0] & ~HPTE_V_ABSENT
) | HPTE_V_VALID
;
928 rev
[i
].guest_rpte
|= HPTE_R_C
;
931 hptep
[0] &= ~HPTE_V_HVLOCK
;
932 } while ((i
= j
) != head
);
938 long kvmppc_hv_get_dirty_log(struct kvm
*kvm
, struct kvm_memory_slot
*memslot
)
941 unsigned long *rmapp
, *map
;
944 rmapp
= memslot
->rmap
;
945 map
= memslot
->dirty_bitmap
;
946 for (i
= 0; i
< memslot
->npages
; ++i
) {
947 if (kvm_test_clear_dirty(kvm
, rmapp
))
948 __set_bit_le(i
, map
);
955 void *kvmppc_pin_guest_page(struct kvm
*kvm
, unsigned long gpa
,
956 unsigned long *nb_ret
)
958 struct kvm_memory_slot
*memslot
;
959 unsigned long gfn
= gpa
>> PAGE_SHIFT
;
960 struct page
*page
, *pages
[1];
962 unsigned long hva
, psize
, offset
;
964 unsigned long *physp
;
966 memslot
= gfn_to_memslot(kvm
, gfn
);
967 if (!memslot
|| (memslot
->flags
& KVM_MEMSLOT_INVALID
))
969 if (!kvm
->arch
.using_mmu_notifiers
) {
970 physp
= kvm
->arch
.slot_phys
[memslot
->id
];
973 physp
+= gfn
- memslot
->base_gfn
;
976 if (kvmppc_get_guest_page(kvm
, gfn
, memslot
,
981 page
= pfn_to_page(pa
>> PAGE_SHIFT
);
983 hva
= gfn_to_hva_memslot(memslot
, gfn
);
984 npages
= get_user_pages_fast(hva
, 1, 1, pages
);
990 if (PageHuge(page
)) {
991 page
= compound_head(page
);
992 psize
<<= compound_order(page
);
994 if (!kvm
->arch
.using_mmu_notifiers
)
996 offset
= gpa
& (psize
- 1);
998 *nb_ret
= psize
- offset
;
999 return page_address(page
) + offset
;
1002 void kvmppc_unpin_guest_page(struct kvm
*kvm
, void *va
)
1004 struct page
*page
= virt_to_page(va
);
1006 page
= compound_head(page
);
1010 void kvmppc_mmu_book3s_hv_init(struct kvm_vcpu
*vcpu
)
1012 struct kvmppc_mmu
*mmu
= &vcpu
->arch
.mmu
;
1014 if (cpu_has_feature(CPU_FTR_ARCH_206
))
1015 vcpu
->arch
.slb_nr
= 32; /* POWER7 */
1017 vcpu
->arch
.slb_nr
= 64;
1019 mmu
->xlate
= kvmppc_mmu_book3s_64_hv_xlate
;
1020 mmu
->reset_msr
= kvmppc_mmu_book3s_64_hv_reset_msr
;
1022 vcpu
->arch
.hflags
|= BOOK3S_HFLAG_SLB
;