2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 * GNU General Public License for more details.
11 * You should have received a copy of the GNU General Public License
12 * along with this program; if not, write to the Free Software
13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
15 * Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
18 #include <linux/types.h>
19 #include <linux/string.h>
20 #include <linux/kvm.h>
21 #include <linux/kvm_host.h>
22 #include <linux/highmem.h>
23 #include <linux/gfp.h>
24 #include <linux/slab.h>
25 #include <linux/hugetlb.h>
26 #include <linux/vmalloc.h>
27 #include <linux/srcu.h>
28 #include <linux/anon_inodes.h>
29 #include <linux/file.h>
30 #include <linux/debugfs.h>
32 #include <asm/tlbflush.h>
33 #include <asm/kvm_ppc.h>
34 #include <asm/kvm_book3s.h>
35 #include <asm/book3s/64/mmu-hash.h>
36 #include <asm/hvcall.h>
37 #include <asm/synch.h>
38 #include <asm/ppc-opcode.h>
39 #include <asm/cputable.h>
40 #include <asm/pte-walk.h>
44 //#define DEBUG_RESIZE_HPT 1
46 #ifdef DEBUG_RESIZE_HPT
47 #define resize_hpt_debug(resize, ...) \
49 printk(KERN_DEBUG "RESIZE HPT %p: ", resize); \
50 printk(__VA_ARGS__); \
53 #define resize_hpt_debug(resize, ...) \
57 static long kvmppc_virtmode_do_h_enter(struct kvm
*kvm
, unsigned long flags
,
58 long pte_index
, unsigned long pteh
,
59 unsigned long ptel
, unsigned long *pte_idx_ret
);
61 struct kvm_resize_hpt
{
62 /* These fields read-only after init */
64 struct work_struct work
;
67 /* These fields protected by kvm->lock */
71 /* Private to the work thread, until prepare_done is true,
72 * then protected by kvm->resize_hpt_sem */
73 struct kvm_hpt_info hpt
;
76 int kvmppc_allocate_hpt(struct kvm_hpt_info
*info
, u32 order
)
78 unsigned long hpt
= 0;
80 struct page
*page
= NULL
;
81 struct revmap_entry
*rev
;
84 if ((order
< PPC_MIN_HPT_ORDER
) || (order
> PPC_MAX_HPT_ORDER
))
87 page
= kvm_alloc_hpt_cma(1ul << (order
- PAGE_SHIFT
));
89 hpt
= (unsigned long)pfn_to_kaddr(page_to_pfn(page
));
90 memset((void *)hpt
, 0, (1ul << order
));
95 hpt
= __get_free_pages(GFP_KERNEL
|__GFP_ZERO
|__GFP_RETRY_MAYFAIL
96 |__GFP_NOWARN
, order
- PAGE_SHIFT
);
101 /* HPTEs are 2**4 bytes long */
102 npte
= 1ul << (order
- 4);
104 /* Allocate reverse map array */
105 rev
= vmalloc(sizeof(struct revmap_entry
) * npte
);
108 kvm_free_hpt_cma(page
, 1 << (order
- PAGE_SHIFT
));
110 free_pages(hpt
, order
- PAGE_SHIFT
);
122 void kvmppc_set_hpt(struct kvm
*kvm
, struct kvm_hpt_info
*info
)
124 atomic64_set(&kvm
->arch
.mmio_update
, 0);
125 kvm
->arch
.hpt
= *info
;
126 kvm
->arch
.sdr1
= __pa(info
->virt
) | (info
->order
- 18);
128 pr_debug("KVM guest htab at %lx (order %ld), LPID %x\n",
129 info
->virt
, (long)info
->order
, kvm
->arch
.lpid
);
132 long kvmppc_alloc_reset_hpt(struct kvm
*kvm
, int order
)
135 struct kvm_hpt_info info
;
137 mutex_lock(&kvm
->lock
);
138 if (kvm
->arch
.mmu_ready
) {
139 kvm
->arch
.mmu_ready
= 0;
140 /* order mmu_ready vs. vcpus_running */
142 if (atomic_read(&kvm
->arch
.vcpus_running
)) {
143 kvm
->arch
.mmu_ready
= 1;
147 if (kvm_is_radix(kvm
)) {
148 err
= kvmppc_switch_mmu_to_hpt(kvm
);
153 if (kvm
->arch
.hpt
.order
== order
) {
154 /* We already have a suitable HPT */
156 /* Set the entire HPT to 0, i.e. invalid HPTEs */
157 memset((void *)kvm
->arch
.hpt
.virt
, 0, 1ul << order
);
159 * Reset all the reverse-mapping chains for all memslots
161 kvmppc_rmap_reset(kvm
);
162 /* Ensure that each vcpu will flush its TLB on next entry. */
163 cpumask_setall(&kvm
->arch
.need_tlb_flush
);
168 if (kvm
->arch
.hpt
.virt
) {
169 kvmppc_free_hpt(&kvm
->arch
.hpt
);
170 kvmppc_rmap_reset(kvm
);
173 err
= kvmppc_allocate_hpt(&info
, order
);
176 kvmppc_set_hpt(kvm
, &info
);
179 mutex_unlock(&kvm
->lock
);
183 void kvmppc_free_hpt(struct kvm_hpt_info
*info
)
188 kvm_free_hpt_cma(virt_to_page(info
->virt
),
189 1 << (info
->order
- PAGE_SHIFT
));
191 free_pages(info
->virt
, info
->order
- PAGE_SHIFT
);
196 /* Bits in first HPTE dword for pagesize 4k, 64k or 16M */
197 static inline unsigned long hpte0_pgsize_encoding(unsigned long pgsize
)
199 return (pgsize
> 0x1000) ? HPTE_V_LARGE
: 0;
202 /* Bits in second HPTE dword for pagesize 4k, 64k or 16M */
203 static inline unsigned long hpte1_pgsize_encoding(unsigned long pgsize
)
205 return (pgsize
== 0x10000) ? 0x1000 : 0;
208 void kvmppc_map_vrma(struct kvm_vcpu
*vcpu
, struct kvm_memory_slot
*memslot
,
209 unsigned long porder
)
212 unsigned long npages
;
213 unsigned long hp_v
, hp_r
;
214 unsigned long addr
, hash
;
216 unsigned long hp0
, hp1
;
217 unsigned long idx_ret
;
219 struct kvm
*kvm
= vcpu
->kvm
;
221 psize
= 1ul << porder
;
222 npages
= memslot
->npages
>> (porder
- PAGE_SHIFT
);
224 /* VRMA can't be > 1TB */
225 if (npages
> 1ul << (40 - porder
))
226 npages
= 1ul << (40 - porder
);
227 /* Can't use more than 1 HPTE per HPTEG */
228 if (npages
> kvmppc_hpt_mask(&kvm
->arch
.hpt
) + 1)
229 npages
= kvmppc_hpt_mask(&kvm
->arch
.hpt
) + 1;
231 hp0
= HPTE_V_1TB_SEG
| (VRMA_VSID
<< (40 - 16)) |
232 HPTE_V_BOLTED
| hpte0_pgsize_encoding(psize
);
233 hp1
= hpte1_pgsize_encoding(psize
) |
234 HPTE_R_R
| HPTE_R_C
| HPTE_R_M
| PP_RWXX
;
236 for (i
= 0; i
< npages
; ++i
) {
238 /* can't use hpt_hash since va > 64 bits */
239 hash
= (i
^ (VRMA_VSID
^ (VRMA_VSID
<< 25)))
240 & kvmppc_hpt_mask(&kvm
->arch
.hpt
);
242 * We assume that the hash table is empty and no
243 * vcpus are using it at this stage. Since we create
244 * at most one HPTE per HPTEG, we just assume entry 7
245 * is available and use it.
247 hash
= (hash
<< 3) + 7;
248 hp_v
= hp0
| ((addr
>> 16) & ~0x7fUL
);
250 ret
= kvmppc_virtmode_do_h_enter(kvm
, H_EXACT
, hash
, hp_v
, hp_r
,
252 if (ret
!= H_SUCCESS
) {
253 pr_err("KVM: map_vrma at %lx failed, ret=%ld\n",
260 int kvmppc_mmu_hv_init(void)
262 unsigned long host_lpid
, rsvd_lpid
;
264 if (!cpu_has_feature(CPU_FTR_HVMODE
))
267 /* POWER7 has 10-bit LPIDs (12-bit in POWER8) */
268 host_lpid
= mfspr(SPRN_LPID
);
269 rsvd_lpid
= LPID_RSVD
;
271 kvmppc_init_lpid(rsvd_lpid
+ 1);
273 kvmppc_claim_lpid(host_lpid
);
274 /* rsvd_lpid is reserved for use in partition switching */
275 kvmppc_claim_lpid(rsvd_lpid
);
280 static void kvmppc_mmu_book3s_64_hv_reset_msr(struct kvm_vcpu
*vcpu
)
282 unsigned long msr
= vcpu
->arch
.intr_msr
;
284 /* If transactional, change to suspend mode on IRQ delivery */
285 if (MSR_TM_TRANSACTIONAL(vcpu
->arch
.shregs
.msr
))
288 msr
|= vcpu
->arch
.shregs
.msr
& MSR_TS_MASK
;
289 kvmppc_set_msr(vcpu
, msr
);
292 static long kvmppc_virtmode_do_h_enter(struct kvm
*kvm
, unsigned long flags
,
293 long pte_index
, unsigned long pteh
,
294 unsigned long ptel
, unsigned long *pte_idx_ret
)
298 /* Protect linux PTE lookup from page table destruction */
299 rcu_read_lock_sched(); /* this disables preemption too */
300 ret
= kvmppc_do_h_enter(kvm
, flags
, pte_index
, pteh
, ptel
,
301 current
->mm
->pgd
, false, pte_idx_ret
);
302 rcu_read_unlock_sched();
303 if (ret
== H_TOO_HARD
) {
304 /* this can't happen */
305 pr_err("KVM: Oops, kvmppc_h_enter returned too hard!\n");
306 ret
= H_RESOURCE
; /* or something */
312 static struct kvmppc_slb
*kvmppc_mmu_book3s_hv_find_slbe(struct kvm_vcpu
*vcpu
,
318 for (i
= 0; i
< vcpu
->arch
.slb_nr
; i
++) {
319 if (!(vcpu
->arch
.slb
[i
].orige
& SLB_ESID_V
))
322 if (vcpu
->arch
.slb
[i
].origv
& SLB_VSID_B_1T
)
327 if (((vcpu
->arch
.slb
[i
].orige
^ eaddr
) & mask
) == 0)
328 return &vcpu
->arch
.slb
[i
];
333 static unsigned long kvmppc_mmu_get_real_addr(unsigned long v
, unsigned long r
,
336 unsigned long ra_mask
;
338 ra_mask
= kvmppc_actual_pgsz(v
, r
) - 1;
339 return (r
& HPTE_R_RPN
& ~ra_mask
) | (ea
& ra_mask
);
342 static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu
*vcpu
, gva_t eaddr
,
343 struct kvmppc_pte
*gpte
, bool data
, bool iswrite
)
345 struct kvm
*kvm
= vcpu
->kvm
;
346 struct kvmppc_slb
*slbe
;
348 unsigned long pp
, key
;
349 unsigned long v
, orig_v
, gr
;
352 int virtmode
= vcpu
->arch
.shregs
.msr
& (data
? MSR_DR
: MSR_IR
);
354 if (kvm_is_radix(vcpu
->kvm
))
355 return kvmppc_mmu_radix_xlate(vcpu
, eaddr
, gpte
, data
, iswrite
);
359 slbe
= kvmppc_mmu_book3s_hv_find_slbe(vcpu
, eaddr
);
364 /* real mode access */
365 slb_v
= vcpu
->kvm
->arch
.vrma_slb_v
;
369 /* Find the HPTE in the hash table */
370 index
= kvmppc_hv_find_lock_hpte(kvm
, eaddr
, slb_v
,
371 HPTE_V_VALID
| HPTE_V_ABSENT
);
376 hptep
= (__be64
*)(kvm
->arch
.hpt
.virt
+ (index
<< 4));
377 v
= orig_v
= be64_to_cpu(hptep
[0]) & ~HPTE_V_HVLOCK
;
378 if (cpu_has_feature(CPU_FTR_ARCH_300
))
379 v
= hpte_new_to_old_v(v
, be64_to_cpu(hptep
[1]));
380 gr
= kvm
->arch
.hpt
.rev
[index
].guest_rpte
;
382 unlock_hpte(hptep
, orig_v
);
386 gpte
->vpage
= ((v
& HPTE_V_AVPN
) << 4) | ((eaddr
>> 12) & 0xfff);
388 /* Get PP bits and key for permission check */
389 pp
= gr
& (HPTE_R_PP0
| HPTE_R_PP
);
390 key
= (vcpu
->arch
.shregs
.msr
& MSR_PR
) ? SLB_VSID_KP
: SLB_VSID_KS
;
393 /* Calculate permissions */
394 gpte
->may_read
= hpte_read_permission(pp
, key
);
395 gpte
->may_write
= hpte_write_permission(pp
, key
);
396 gpte
->may_execute
= gpte
->may_read
&& !(gr
& (HPTE_R_N
| HPTE_R_G
));
398 /* Storage key permission check for POWER7 */
399 if (data
&& virtmode
) {
400 int amrfield
= hpte_get_skey_perm(gr
, vcpu
->arch
.amr
);
407 /* Get the guest physical address */
408 gpte
->raddr
= kvmppc_mmu_get_real_addr(v
, gr
, eaddr
);
413 * Quick test for whether an instruction is a load or a store.
414 * If the instruction is a load or a store, then this will indicate
415 * which it is, at least on server processors. (Embedded processors
416 * have some external PID instructions that don't follow the rule
417 * embodied here.) If the instruction isn't a load or store, then
418 * this doesn't return anything useful.
420 static int instruction_is_store(unsigned int instr
)
425 if ((instr
& 0xfc000000) == 0x7c000000)
426 mask
= 0x100; /* major opcode 31 */
427 return (instr
& mask
) != 0;
430 int kvmppc_hv_emulate_mmio(struct kvm_run
*run
, struct kvm_vcpu
*vcpu
,
431 unsigned long gpa
, gva_t ea
, int is_store
)
436 * If we fail, we just return to the guest and try executing it again.
438 if (kvmppc_get_last_inst(vcpu
, INST_GENERIC
, &last_inst
) !=
443 * WARNING: We do not know for sure whether the instruction we just
444 * read from memory is the same that caused the fault in the first
445 * place. If the instruction we read is neither an load or a store,
446 * then it can't access memory, so we don't need to worry about
447 * enforcing access permissions. So, assuming it is a load or
448 * store, we just check that its direction (load or store) is
449 * consistent with the original fault, since that's what we
450 * checked the access permissions against. If there is a mismatch
451 * we just return and retry the instruction.
454 if (instruction_is_store(last_inst
) != !!is_store
)
458 * Emulated accesses are emulated by looking at the hash for
459 * translation once, then performing the access later. The
460 * translation could be invalidated in the meantime in which
461 * point performing the subsequent memory access on the old
462 * physical address could possibly be a security hole for the
463 * guest (but not the host).
465 * This is less of an issue for MMIO stores since they aren't
466 * globally visible. It could be an issue for MMIO loads to
467 * a certain extent but we'll ignore it for now.
470 vcpu
->arch
.paddr_accessed
= gpa
;
471 vcpu
->arch
.vaddr_accessed
= ea
;
472 return kvmppc_emulate_mmio(run
, vcpu
);
475 int kvmppc_book3s_hv_page_fault(struct kvm_run
*run
, struct kvm_vcpu
*vcpu
,
476 unsigned long ea
, unsigned long dsisr
)
478 struct kvm
*kvm
= vcpu
->kvm
;
479 unsigned long hpte
[3], r
;
480 unsigned long hnow_v
, hnow_r
;
482 unsigned long mmu_seq
, psize
, pte_size
;
483 unsigned long gpa_base
, gfn_base
;
484 unsigned long gpa
, gfn
, hva
, pfn
;
485 struct kvm_memory_slot
*memslot
;
487 struct revmap_entry
*rev
;
488 struct page
*page
, *pages
[1];
489 long index
, ret
, npages
;
491 unsigned int writing
, write_ok
;
492 struct vm_area_struct
*vma
;
493 unsigned long rcbits
;
496 if (kvm_is_radix(kvm
))
497 return kvmppc_book3s_radix_page_fault(run
, vcpu
, ea
, dsisr
);
500 * Real-mode code has already searched the HPT and found the
501 * entry we're interested in. Lock the entry and check that
502 * it hasn't changed. If it has, just return and re-execute the
505 if (ea
!= vcpu
->arch
.pgfault_addr
)
508 if (vcpu
->arch
.pgfault_cache
) {
509 mmio_update
= atomic64_read(&kvm
->arch
.mmio_update
);
510 if (mmio_update
== vcpu
->arch
.pgfault_cache
->mmio_update
) {
511 r
= vcpu
->arch
.pgfault_cache
->rpte
;
512 psize
= kvmppc_actual_pgsz(vcpu
->arch
.pgfault_hpte
[0],
514 gpa_base
= r
& HPTE_R_RPN
& ~(psize
- 1);
515 gfn_base
= gpa_base
>> PAGE_SHIFT
;
516 gpa
= gpa_base
| (ea
& (psize
- 1));
517 return kvmppc_hv_emulate_mmio(run
, vcpu
, gpa
, ea
,
518 dsisr
& DSISR_ISSTORE
);
521 index
= vcpu
->arch
.pgfault_index
;
522 hptep
= (__be64
*)(kvm
->arch
.hpt
.virt
+ (index
<< 4));
523 rev
= &kvm
->arch
.hpt
.rev
[index
];
525 while (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
))
527 hpte
[0] = be64_to_cpu(hptep
[0]) & ~HPTE_V_HVLOCK
;
528 hpte
[1] = be64_to_cpu(hptep
[1]);
529 hpte
[2] = r
= rev
->guest_rpte
;
530 unlock_hpte(hptep
, hpte
[0]);
533 if (cpu_has_feature(CPU_FTR_ARCH_300
)) {
534 hpte
[0] = hpte_new_to_old_v(hpte
[0], hpte
[1]);
535 hpte
[1] = hpte_new_to_old_r(hpte
[1]);
537 if (hpte
[0] != vcpu
->arch
.pgfault_hpte
[0] ||
538 hpte
[1] != vcpu
->arch
.pgfault_hpte
[1])
541 /* Translate the logical address and get the page */
542 psize
= kvmppc_actual_pgsz(hpte
[0], r
);
543 gpa_base
= r
& HPTE_R_RPN
& ~(psize
- 1);
544 gfn_base
= gpa_base
>> PAGE_SHIFT
;
545 gpa
= gpa_base
| (ea
& (psize
- 1));
546 gfn
= gpa
>> PAGE_SHIFT
;
547 memslot
= gfn_to_memslot(kvm
, gfn
);
549 trace_kvm_page_fault_enter(vcpu
, hpte
, memslot
, ea
, dsisr
);
551 /* No memslot means it's an emulated MMIO region */
552 if (!memslot
|| (memslot
->flags
& KVM_MEMSLOT_INVALID
))
553 return kvmppc_hv_emulate_mmio(run
, vcpu
, gpa
, ea
,
554 dsisr
& DSISR_ISSTORE
);
557 * This should never happen, because of the slot_is_aligned()
558 * check in kvmppc_do_h_enter().
560 if (gfn_base
< memslot
->base_gfn
)
563 /* used to check for invalidations in progress */
564 mmu_seq
= kvm
->mmu_notifier_seq
;
571 pte_size
= PAGE_SIZE
;
572 writing
= (dsisr
& DSISR_ISSTORE
) != 0;
573 /* If writing != 0, then the HPTE must allow writing, if we get here */
575 hva
= gfn_to_hva_memslot(memslot
, gfn
);
576 npages
= get_user_pages_fast(hva
, 1, writing
, pages
);
578 /* Check if it's an I/O mapping */
579 down_read(¤t
->mm
->mmap_sem
);
580 vma
= find_vma(current
->mm
, hva
);
581 if (vma
&& vma
->vm_start
<= hva
&& hva
+ psize
<= vma
->vm_end
&&
582 (vma
->vm_flags
& VM_PFNMAP
)) {
583 pfn
= vma
->vm_pgoff
+
584 ((hva
- vma
->vm_start
) >> PAGE_SHIFT
);
586 is_ci
= pte_ci(__pte((pgprot_val(vma
->vm_page_prot
))));
587 write_ok
= vma
->vm_flags
& VM_WRITE
;
589 up_read(¤t
->mm
->mmap_sem
);
594 pfn
= page_to_pfn(page
);
595 if (PageHuge(page
)) {
596 page
= compound_head(page
);
597 pte_size
<<= compound_order(page
);
599 /* if the guest wants write access, see if that is OK */
600 if (!writing
&& hpte_is_writable(r
)) {
604 * We need to protect against page table destruction
605 * hugepage split and collapse.
607 local_irq_save(flags
);
608 ptep
= find_current_mm_pte(current
->mm
->pgd
,
611 pte
= kvmppc_read_update_linux_pte(ptep
, 1);
612 if (__pte_write(pte
))
615 local_irq_restore(flags
);
619 if (psize
> pte_size
)
622 /* Check WIMG vs. the actual page we're accessing */
623 if (!hpte_cache_flags_ok(r
, is_ci
)) {
627 * Allow guest to map emulated device memory as
628 * uncacheable, but actually make it cacheable.
630 r
= (r
& ~(HPTE_R_W
|HPTE_R_I
|HPTE_R_G
)) | HPTE_R_M
;
634 * Set the HPTE to point to pfn.
635 * Since the pfn is at PAGE_SIZE granularity, make sure we
636 * don't mask out lower-order bits if psize < PAGE_SIZE.
638 if (psize
< PAGE_SIZE
)
640 r
= (r
& HPTE_R_KEY_HI
) | (r
& ~(HPTE_R_PP0
- psize
)) |
641 ((pfn
<< PAGE_SHIFT
) & ~(psize
- 1));
642 if (hpte_is_writable(r
) && !write_ok
)
643 r
= hpte_make_readonly(r
);
646 while (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
))
648 hnow_v
= be64_to_cpu(hptep
[0]);
649 hnow_r
= be64_to_cpu(hptep
[1]);
650 if (cpu_has_feature(CPU_FTR_ARCH_300
)) {
651 hnow_v
= hpte_new_to_old_v(hnow_v
, hnow_r
);
652 hnow_r
= hpte_new_to_old_r(hnow_r
);
656 * If the HPT is being resized, don't update the HPTE,
657 * instead let the guest retry after the resize operation is complete.
658 * The synchronization for mmu_ready test vs. set is provided
661 if (!kvm
->arch
.mmu_ready
)
664 if ((hnow_v
& ~HPTE_V_HVLOCK
) != hpte
[0] || hnow_r
!= hpte
[1] ||
665 rev
->guest_rpte
!= hpte
[2])
666 /* HPTE has been changed under us; let the guest retry */
668 hpte
[0] = (hpte
[0] & ~HPTE_V_ABSENT
) | HPTE_V_VALID
;
670 /* Always put the HPTE in the rmap chain for the page base address */
671 rmap
= &memslot
->arch
.rmap
[gfn_base
- memslot
->base_gfn
];
674 /* Check if we might have been invalidated; let the guest retry if so */
676 if (mmu_notifier_retry(vcpu
->kvm
, mmu_seq
)) {
681 /* Only set R/C in real HPTE if set in both *rmap and guest_rpte */
682 rcbits
= *rmap
>> KVMPPC_RMAP_RC_SHIFT
;
683 r
&= rcbits
| ~(HPTE_R_R
| HPTE_R_C
);
685 if (be64_to_cpu(hptep
[0]) & HPTE_V_VALID
) {
686 /* HPTE was previously valid, so we need to invalidate it */
688 hptep
[0] |= cpu_to_be64(HPTE_V_ABSENT
);
689 kvmppc_invalidate_hpte(kvm
, hptep
, index
);
690 /* don't lose previous R and C bits */
691 r
|= be64_to_cpu(hptep
[1]) & (HPTE_R_R
| HPTE_R_C
);
693 kvmppc_add_revmap_chain(kvm
, rev
, rmap
, index
, 0);
696 if (cpu_has_feature(CPU_FTR_ARCH_300
)) {
697 r
= hpte_old_to_new_r(hpte
[0], r
);
698 hpte
[0] = hpte_old_to_new_v(hpte
[0]);
700 hptep
[1] = cpu_to_be64(r
);
702 __unlock_hpte(hptep
, hpte
[0]);
703 asm volatile("ptesync" : : : "memory");
705 if (page
&& hpte_is_writable(r
))
709 trace_kvm_page_fault_exit(vcpu
, hpte
, ret
);
713 * We drop pages[0] here, not page because page might
714 * have been set to the head page of a compound, but
715 * we have to drop the reference on the correct tail
716 * page to match the get inside gup()
723 __unlock_hpte(hptep
, be64_to_cpu(hptep
[0]));
728 void kvmppc_rmap_reset(struct kvm
*kvm
)
730 struct kvm_memslots
*slots
;
731 struct kvm_memory_slot
*memslot
;
734 srcu_idx
= srcu_read_lock(&kvm
->srcu
);
735 slots
= kvm_memslots(kvm
);
736 kvm_for_each_memslot(memslot
, slots
) {
738 * This assumes it is acceptable to lose reference and
739 * change bits across a reset.
741 memset(memslot
->arch
.rmap
, 0,
742 memslot
->npages
* sizeof(*memslot
->arch
.rmap
));
744 srcu_read_unlock(&kvm
->srcu
, srcu_idx
);
747 typedef int (*hva_handler_fn
)(struct kvm
*kvm
, struct kvm_memory_slot
*memslot
,
750 static int kvm_handle_hva_range(struct kvm
*kvm
,
753 hva_handler_fn handler
)
757 struct kvm_memslots
*slots
;
758 struct kvm_memory_slot
*memslot
;
760 slots
= kvm_memslots(kvm
);
761 kvm_for_each_memslot(memslot
, slots
) {
762 unsigned long hva_start
, hva_end
;
765 hva_start
= max(start
, memslot
->userspace_addr
);
766 hva_end
= min(end
, memslot
->userspace_addr
+
767 (memslot
->npages
<< PAGE_SHIFT
));
768 if (hva_start
>= hva_end
)
771 * {gfn(page) | page intersects with [hva_start, hva_end)} =
772 * {gfn, gfn+1, ..., gfn_end-1}.
774 gfn
= hva_to_gfn_memslot(hva_start
, memslot
);
775 gfn_end
= hva_to_gfn_memslot(hva_end
+ PAGE_SIZE
- 1, memslot
);
777 for (; gfn
< gfn_end
; ++gfn
) {
778 ret
= handler(kvm
, memslot
, gfn
);
786 static int kvm_handle_hva(struct kvm
*kvm
, unsigned long hva
,
787 hva_handler_fn handler
)
789 return kvm_handle_hva_range(kvm
, hva
, hva
+ 1, handler
);
792 /* Must be called with both HPTE and rmap locked */
793 static void kvmppc_unmap_hpte(struct kvm
*kvm
, unsigned long i
,
794 struct kvm_memory_slot
*memslot
,
795 unsigned long *rmapp
, unsigned long gfn
)
797 __be64
*hptep
= (__be64
*) (kvm
->arch
.hpt
.virt
+ (i
<< 4));
798 struct revmap_entry
*rev
= kvm
->arch
.hpt
.rev
;
800 unsigned long ptel
, psize
, rcbits
;
804 /* chain is now empty */
805 *rmapp
&= ~(KVMPPC_RMAP_PRESENT
| KVMPPC_RMAP_INDEX
);
807 /* remove i from chain */
811 rev
[i
].forw
= rev
[i
].back
= i
;
812 *rmapp
= (*rmapp
& ~KVMPPC_RMAP_INDEX
) | j
;
815 /* Now check and modify the HPTE */
816 ptel
= rev
[i
].guest_rpte
;
817 psize
= kvmppc_actual_pgsz(be64_to_cpu(hptep
[0]), ptel
);
818 if ((be64_to_cpu(hptep
[0]) & HPTE_V_VALID
) &&
819 hpte_rpn(ptel
, psize
) == gfn
) {
820 hptep
[0] |= cpu_to_be64(HPTE_V_ABSENT
);
821 kvmppc_invalidate_hpte(kvm
, hptep
, i
);
822 hptep
[1] &= ~cpu_to_be64(HPTE_R_KEY_HI
| HPTE_R_KEY_LO
);
823 /* Harvest R and C */
824 rcbits
= be64_to_cpu(hptep
[1]) & (HPTE_R_R
| HPTE_R_C
);
825 *rmapp
|= rcbits
<< KVMPPC_RMAP_RC_SHIFT
;
826 if ((rcbits
& HPTE_R_C
) && memslot
->dirty_bitmap
)
827 kvmppc_update_dirty_map(memslot
, gfn
, psize
);
828 if (rcbits
& ~rev
[i
].guest_rpte
) {
829 rev
[i
].guest_rpte
= ptel
| rcbits
;
830 note_hpte_modification(kvm
, &rev
[i
]);
835 static int kvm_unmap_rmapp(struct kvm
*kvm
, struct kvm_memory_slot
*memslot
,
840 unsigned long *rmapp
;
842 rmapp
= &memslot
->arch
.rmap
[gfn
- memslot
->base_gfn
];
845 if (!(*rmapp
& KVMPPC_RMAP_PRESENT
)) {
851 * To avoid an ABBA deadlock with the HPTE lock bit,
852 * we can't spin on the HPTE lock while holding the
855 i
= *rmapp
& KVMPPC_RMAP_INDEX
;
856 hptep
= (__be64
*) (kvm
->arch
.hpt
.virt
+ (i
<< 4));
857 if (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
)) {
858 /* unlock rmap before spinning on the HPTE lock */
860 while (be64_to_cpu(hptep
[0]) & HPTE_V_HVLOCK
)
865 kvmppc_unmap_hpte(kvm
, i
, memslot
, rmapp
, gfn
);
867 __unlock_hpte(hptep
, be64_to_cpu(hptep
[0]));
872 int kvm_unmap_hva_hv(struct kvm
*kvm
, unsigned long hva
)
874 hva_handler_fn handler
;
876 handler
= kvm_is_radix(kvm
) ? kvm_unmap_radix
: kvm_unmap_rmapp
;
877 kvm_handle_hva(kvm
, hva
, handler
);
881 int kvm_unmap_hva_range_hv(struct kvm
*kvm
, unsigned long start
, unsigned long end
)
883 hva_handler_fn handler
;
885 handler
= kvm_is_radix(kvm
) ? kvm_unmap_radix
: kvm_unmap_rmapp
;
886 kvm_handle_hva_range(kvm
, start
, end
, handler
);
890 void kvmppc_core_flush_memslot_hv(struct kvm
*kvm
,
891 struct kvm_memory_slot
*memslot
)
895 unsigned long *rmapp
;
897 gfn
= memslot
->base_gfn
;
898 rmapp
= memslot
->arch
.rmap
;
899 for (n
= memslot
->npages
; n
; --n
, ++gfn
) {
900 if (kvm_is_radix(kvm
)) {
901 kvm_unmap_radix(kvm
, memslot
, gfn
);
905 * Testing the present bit without locking is OK because
906 * the memslot has been marked invalid already, and hence
907 * no new HPTEs referencing this page can be created,
908 * thus the present bit can't go from 0 to 1.
910 if (*rmapp
& KVMPPC_RMAP_PRESENT
)
911 kvm_unmap_rmapp(kvm
, memslot
, gfn
);
916 static int kvm_age_rmapp(struct kvm
*kvm
, struct kvm_memory_slot
*memslot
,
919 struct revmap_entry
*rev
= kvm
->arch
.hpt
.rev
;
920 unsigned long head
, i
, j
;
923 unsigned long *rmapp
;
925 rmapp
= &memslot
->arch
.rmap
[gfn
- memslot
->base_gfn
];
928 if (*rmapp
& KVMPPC_RMAP_REFERENCED
) {
929 *rmapp
&= ~KVMPPC_RMAP_REFERENCED
;
932 if (!(*rmapp
& KVMPPC_RMAP_PRESENT
)) {
937 i
= head
= *rmapp
& KVMPPC_RMAP_INDEX
;
939 hptep
= (__be64
*) (kvm
->arch
.hpt
.virt
+ (i
<< 4));
942 /* If this HPTE isn't referenced, ignore it */
943 if (!(be64_to_cpu(hptep
[1]) & HPTE_R_R
))
946 if (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
)) {
947 /* unlock rmap before spinning on the HPTE lock */
949 while (be64_to_cpu(hptep
[0]) & HPTE_V_HVLOCK
)
954 /* Now check and modify the HPTE */
955 if ((be64_to_cpu(hptep
[0]) & HPTE_V_VALID
) &&
956 (be64_to_cpu(hptep
[1]) & HPTE_R_R
)) {
957 kvmppc_clear_ref_hpte(kvm
, hptep
, i
);
958 if (!(rev
[i
].guest_rpte
& HPTE_R_R
)) {
959 rev
[i
].guest_rpte
|= HPTE_R_R
;
960 note_hpte_modification(kvm
, &rev
[i
]);
964 __unlock_hpte(hptep
, be64_to_cpu(hptep
[0]));
965 } while ((i
= j
) != head
);
971 int kvm_age_hva_hv(struct kvm
*kvm
, unsigned long start
, unsigned long end
)
973 hva_handler_fn handler
;
975 handler
= kvm_is_radix(kvm
) ? kvm_age_radix
: kvm_age_rmapp
;
976 return kvm_handle_hva_range(kvm
, start
, end
, handler
);
979 static int kvm_test_age_rmapp(struct kvm
*kvm
, struct kvm_memory_slot
*memslot
,
982 struct revmap_entry
*rev
= kvm
->arch
.hpt
.rev
;
983 unsigned long head
, i
, j
;
986 unsigned long *rmapp
;
988 rmapp
= &memslot
->arch
.rmap
[gfn
- memslot
->base_gfn
];
989 if (*rmapp
& KVMPPC_RMAP_REFERENCED
)
993 if (*rmapp
& KVMPPC_RMAP_REFERENCED
)
996 if (*rmapp
& KVMPPC_RMAP_PRESENT
) {
997 i
= head
= *rmapp
& KVMPPC_RMAP_INDEX
;
999 hp
= (unsigned long *)(kvm
->arch
.hpt
.virt
+ (i
<< 4));
1001 if (be64_to_cpu(hp
[1]) & HPTE_R_R
)
1003 } while ((i
= j
) != head
);
1012 int kvm_test_age_hva_hv(struct kvm
*kvm
, unsigned long hva
)
1014 hva_handler_fn handler
;
1016 handler
= kvm_is_radix(kvm
) ? kvm_test_age_radix
: kvm_test_age_rmapp
;
1017 return kvm_handle_hva(kvm
, hva
, handler
);
1020 void kvm_set_spte_hva_hv(struct kvm
*kvm
, unsigned long hva
, pte_t pte
)
1022 hva_handler_fn handler
;
1024 handler
= kvm_is_radix(kvm
) ? kvm_unmap_radix
: kvm_unmap_rmapp
;
1025 kvm_handle_hva(kvm
, hva
, handler
);
1028 static int vcpus_running(struct kvm
*kvm
)
1030 return atomic_read(&kvm
->arch
.vcpus_running
) != 0;
1034 * Returns the number of system pages that are dirty.
1035 * This can be more than 1 if we find a huge-page HPTE.
1037 static int kvm_test_clear_dirty_npages(struct kvm
*kvm
, unsigned long *rmapp
)
1039 struct revmap_entry
*rev
= kvm
->arch
.hpt
.rev
;
1040 unsigned long head
, i
, j
;
1044 int npages_dirty
= 0;
1048 if (!(*rmapp
& KVMPPC_RMAP_PRESENT
)) {
1050 return npages_dirty
;
1053 i
= head
= *rmapp
& KVMPPC_RMAP_INDEX
;
1055 unsigned long hptep1
;
1056 hptep
= (__be64
*) (kvm
->arch
.hpt
.virt
+ (i
<< 4));
1060 * Checking the C (changed) bit here is racy since there
1061 * is no guarantee about when the hardware writes it back.
1062 * If the HPTE is not writable then it is stable since the
1063 * page can't be written to, and we would have done a tlbie
1064 * (which forces the hardware to complete any writeback)
1065 * when making the HPTE read-only.
1066 * If vcpus are running then this call is racy anyway
1067 * since the page could get dirtied subsequently, so we
1068 * expect there to be a further call which would pick up
1069 * any delayed C bit writeback.
1070 * Otherwise we need to do the tlbie even if C==0 in
1071 * order to pick up any delayed writeback of C.
1073 hptep1
= be64_to_cpu(hptep
[1]);
1074 if (!(hptep1
& HPTE_R_C
) &&
1075 (!hpte_is_writable(hptep1
) || vcpus_running(kvm
)))
1078 if (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
)) {
1079 /* unlock rmap before spinning on the HPTE lock */
1081 while (hptep
[0] & cpu_to_be64(HPTE_V_HVLOCK
))
1086 /* Now check and modify the HPTE */
1087 if (!(hptep
[0] & cpu_to_be64(HPTE_V_VALID
))) {
1088 __unlock_hpte(hptep
, be64_to_cpu(hptep
[0]));
1092 /* need to make it temporarily absent so C is stable */
1093 hptep
[0] |= cpu_to_be64(HPTE_V_ABSENT
);
1094 kvmppc_invalidate_hpte(kvm
, hptep
, i
);
1095 v
= be64_to_cpu(hptep
[0]);
1096 r
= be64_to_cpu(hptep
[1]);
1098 hptep
[1] = cpu_to_be64(r
& ~HPTE_R_C
);
1099 if (!(rev
[i
].guest_rpte
& HPTE_R_C
)) {
1100 rev
[i
].guest_rpte
|= HPTE_R_C
;
1101 note_hpte_modification(kvm
, &rev
[i
]);
1103 n
= kvmppc_actual_pgsz(v
, r
);
1104 n
= (n
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
1105 if (n
> npages_dirty
)
1109 v
&= ~HPTE_V_ABSENT
;
1111 __unlock_hpte(hptep
, v
);
1112 } while ((i
= j
) != head
);
1115 return npages_dirty
;
1118 void kvmppc_harvest_vpa_dirty(struct kvmppc_vpa
*vpa
,
1119 struct kvm_memory_slot
*memslot
,
1124 if (!vpa
->dirty
|| !vpa
->pinned_addr
)
1126 gfn
= vpa
->gpa
>> PAGE_SHIFT
;
1127 if (gfn
< memslot
->base_gfn
||
1128 gfn
>= memslot
->base_gfn
+ memslot
->npages
)
1133 __set_bit_le(gfn
- memslot
->base_gfn
, map
);
1136 long kvmppc_hv_get_dirty_log_hpt(struct kvm
*kvm
,
1137 struct kvm_memory_slot
*memslot
, unsigned long *map
)
1140 unsigned long *rmapp
;
1143 rmapp
= memslot
->arch
.rmap
;
1144 for (i
= 0; i
< memslot
->npages
; ++i
) {
1145 int npages
= kvm_test_clear_dirty_npages(kvm
, rmapp
);
1147 * Note that if npages > 0 then i must be a multiple of npages,
1148 * since we always put huge-page HPTEs in the rmap chain
1149 * corresponding to their page base address.
1152 set_dirty_bits(map
, i
, npages
);
1159 void *kvmppc_pin_guest_page(struct kvm
*kvm
, unsigned long gpa
,
1160 unsigned long *nb_ret
)
1162 struct kvm_memory_slot
*memslot
;
1163 unsigned long gfn
= gpa
>> PAGE_SHIFT
;
1164 struct page
*page
, *pages
[1];
1166 unsigned long hva
, offset
;
1169 srcu_idx
= srcu_read_lock(&kvm
->srcu
);
1170 memslot
= gfn_to_memslot(kvm
, gfn
);
1171 if (!memslot
|| (memslot
->flags
& KVM_MEMSLOT_INVALID
))
1173 hva
= gfn_to_hva_memslot(memslot
, gfn
);
1174 npages
= get_user_pages_fast(hva
, 1, 1, pages
);
1178 srcu_read_unlock(&kvm
->srcu
, srcu_idx
);
1180 offset
= gpa
& (PAGE_SIZE
- 1);
1182 *nb_ret
= PAGE_SIZE
- offset
;
1183 return page_address(page
) + offset
;
1186 srcu_read_unlock(&kvm
->srcu
, srcu_idx
);
1190 void kvmppc_unpin_guest_page(struct kvm
*kvm
, void *va
, unsigned long gpa
,
1193 struct page
*page
= virt_to_page(va
);
1194 struct kvm_memory_slot
*memslot
;
1203 /* We need to mark this page dirty in the memslot dirty_bitmap, if any */
1204 gfn
= gpa
>> PAGE_SHIFT
;
1205 srcu_idx
= srcu_read_lock(&kvm
->srcu
);
1206 memslot
= gfn_to_memslot(kvm
, gfn
);
1207 if (memslot
&& memslot
->dirty_bitmap
)
1208 set_bit_le(gfn
- memslot
->base_gfn
, memslot
->dirty_bitmap
);
1209 srcu_read_unlock(&kvm
->srcu
, srcu_idx
);
1215 static int resize_hpt_allocate(struct kvm_resize_hpt
*resize
)
1219 rc
= kvmppc_allocate_hpt(&resize
->hpt
, resize
->order
);
1223 resize_hpt_debug(resize
, "resize_hpt_allocate(): HPT @ 0x%lx\n",
1229 static unsigned long resize_hpt_rehash_hpte(struct kvm_resize_hpt
*resize
,
1232 struct kvm
*kvm
= resize
->kvm
;
1233 struct kvm_hpt_info
*old
= &kvm
->arch
.hpt
;
1234 struct kvm_hpt_info
*new = &resize
->hpt
;
1235 unsigned long old_hash_mask
= (1ULL << (old
->order
- 7)) - 1;
1236 unsigned long new_hash_mask
= (1ULL << (new->order
- 7)) - 1;
1237 __be64
*hptep
, *new_hptep
;
1238 unsigned long vpte
, rpte
, guest_rpte
;
1240 struct revmap_entry
*rev
;
1241 unsigned long apsize
, psize
, avpn
, pteg
, hash
;
1242 unsigned long new_idx
, new_pteg
, replace_vpte
;
1244 hptep
= (__be64
*)(old
->virt
+ (idx
<< 4));
1246 /* Guest is stopped, so new HPTEs can't be added or faulted
1247 * in, only unmapped or altered by host actions. So, it's
1248 * safe to check this before we take the HPTE lock */
1249 vpte
= be64_to_cpu(hptep
[0]);
1250 if (!(vpte
& HPTE_V_VALID
) && !(vpte
& HPTE_V_ABSENT
))
1251 return 0; /* nothing to do */
1253 while (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
))
1256 vpte
= be64_to_cpu(hptep
[0]);
1259 if (!(vpte
& HPTE_V_VALID
) && !(vpte
& HPTE_V_ABSENT
))
1264 rev
= &old
->rev
[idx
];
1265 guest_rpte
= rev
->guest_rpte
;
1268 apsize
= kvmppc_actual_pgsz(vpte
, guest_rpte
);
1272 if (vpte
& HPTE_V_VALID
) {
1273 unsigned long gfn
= hpte_rpn(guest_rpte
, apsize
);
1274 int srcu_idx
= srcu_read_lock(&kvm
->srcu
);
1275 struct kvm_memory_slot
*memslot
=
1276 __gfn_to_memslot(kvm_memslots(kvm
), gfn
);
1279 unsigned long *rmapp
;
1280 rmapp
= &memslot
->arch
.rmap
[gfn
- memslot
->base_gfn
];
1283 kvmppc_unmap_hpte(kvm
, idx
, memslot
, rmapp
, gfn
);
1287 srcu_read_unlock(&kvm
->srcu
, srcu_idx
);
1290 /* Reload PTE after unmap */
1291 vpte
= be64_to_cpu(hptep
[0]);
1293 BUG_ON(vpte
& HPTE_V_VALID
);
1294 BUG_ON(!(vpte
& HPTE_V_ABSENT
));
1297 if (!(vpte
& HPTE_V_BOLTED
))
1300 rpte
= be64_to_cpu(hptep
[1]);
1301 psize
= hpte_base_page_size(vpte
, rpte
);
1302 avpn
= HPTE_V_AVPN_VAL(vpte
) & ~((psize
- 1) >> 23);
1303 pteg
= idx
/ HPTES_PER_GROUP
;
1304 if (vpte
& HPTE_V_SECONDARY
)
1307 if (!(vpte
& HPTE_V_1TB_SEG
)) {
1308 unsigned long offset
, vsid
;
1310 /* We only have 28 - 23 bits of offset in avpn */
1311 offset
= (avpn
& 0x1f) << 23;
1313 /* We can find more bits from the pteg value */
1314 if (psize
< (1ULL << 23))
1315 offset
|= ((vsid
^ pteg
) & old_hash_mask
) * psize
;
1317 hash
= vsid
^ (offset
/ psize
);
1319 unsigned long offset
, vsid
;
1321 /* We only have 40 - 23 bits of seg_off in avpn */
1322 offset
= (avpn
& 0x1ffff) << 23;
1324 if (psize
< (1ULL << 23))
1325 offset
|= ((vsid
^ (vsid
<< 25) ^ pteg
) & old_hash_mask
) * psize
;
1327 hash
= vsid
^ (vsid
<< 25) ^ (offset
/ psize
);
1330 new_pteg
= hash
& new_hash_mask
;
1331 if (vpte
& HPTE_V_SECONDARY
) {
1332 BUG_ON(~pteg
!= (hash
& old_hash_mask
));
1333 new_pteg
= ~new_pteg
;
1335 BUG_ON(pteg
!= (hash
& old_hash_mask
));
1338 new_idx
= new_pteg
* HPTES_PER_GROUP
+ (idx
% HPTES_PER_GROUP
);
1339 new_hptep
= (__be64
*)(new->virt
+ (new_idx
<< 4));
1341 replace_vpte
= be64_to_cpu(new_hptep
[0]);
1343 if (replace_vpte
& (HPTE_V_VALID
| HPTE_V_ABSENT
)) {
1344 BUG_ON(new->order
>= old
->order
);
1346 if (replace_vpte
& HPTE_V_BOLTED
) {
1347 if (vpte
& HPTE_V_BOLTED
)
1348 /* Bolted collision, nothing we can do */
1350 /* Discard the new HPTE */
1354 /* Discard the previous HPTE */
1357 new_hptep
[1] = cpu_to_be64(rpte
);
1358 new->rev
[new_idx
].guest_rpte
= guest_rpte
;
1359 /* No need for a barrier, since new HPT isn't active */
1360 new_hptep
[0] = cpu_to_be64(vpte
);
1361 unlock_hpte(new_hptep
, vpte
);
1364 unlock_hpte(hptep
, vpte
);
1368 static int resize_hpt_rehash(struct kvm_resize_hpt
*resize
)
1370 struct kvm
*kvm
= resize
->kvm
;
1375 * resize_hpt_rehash_hpte() doesn't handle the new-format HPTEs
1376 * that POWER9 uses, and could well hit a BUG_ON on POWER9.
1378 if (cpu_has_feature(CPU_FTR_ARCH_300
))
1380 for (i
= 0; i
< kvmppc_hpt_npte(&kvm
->arch
.hpt
); i
++) {
1381 rc
= resize_hpt_rehash_hpte(resize
, i
);
1389 static void resize_hpt_pivot(struct kvm_resize_hpt
*resize
)
1391 struct kvm
*kvm
= resize
->kvm
;
1392 struct kvm_hpt_info hpt_tmp
;
1394 /* Exchange the pending tables in the resize structure with
1395 * the active tables */
1397 resize_hpt_debug(resize
, "resize_hpt_pivot()\n");
1399 spin_lock(&kvm
->mmu_lock
);
1400 asm volatile("ptesync" : : : "memory");
1402 hpt_tmp
= kvm
->arch
.hpt
;
1403 kvmppc_set_hpt(kvm
, &resize
->hpt
);
1404 resize
->hpt
= hpt_tmp
;
1406 spin_unlock(&kvm
->mmu_lock
);
1408 synchronize_srcu_expedited(&kvm
->srcu
);
1410 resize_hpt_debug(resize
, "resize_hpt_pivot() done\n");
1413 static void resize_hpt_release(struct kvm
*kvm
, struct kvm_resize_hpt
*resize
)
1415 BUG_ON(kvm
->arch
.resize_hpt
!= resize
);
1420 if (resize
->hpt
.virt
)
1421 kvmppc_free_hpt(&resize
->hpt
);
1423 kvm
->arch
.resize_hpt
= NULL
;
1427 static void resize_hpt_prepare_work(struct work_struct
*work
)
1429 struct kvm_resize_hpt
*resize
= container_of(work
,
1430 struct kvm_resize_hpt
,
1432 struct kvm
*kvm
= resize
->kvm
;
1435 resize_hpt_debug(resize
, "resize_hpt_prepare_work(): order = %d\n",
1438 err
= resize_hpt_allocate(resize
);
1440 mutex_lock(&kvm
->lock
);
1442 resize
->error
= err
;
1443 resize
->prepare_done
= true;
1445 mutex_unlock(&kvm
->lock
);
1448 long kvm_vm_ioctl_resize_hpt_prepare(struct kvm
*kvm
,
1449 struct kvm_ppc_resize_hpt
*rhpt
)
1451 unsigned long flags
= rhpt
->flags
;
1452 unsigned long shift
= rhpt
->shift
;
1453 struct kvm_resize_hpt
*resize
;
1456 if (flags
!= 0 || kvm_is_radix(kvm
))
1459 if (shift
&& ((shift
< 18) || (shift
> 46)))
1462 mutex_lock(&kvm
->lock
);
1464 resize
= kvm
->arch
.resize_hpt
;
1467 if (resize
->order
== shift
) {
1468 /* Suitable resize in progress */
1469 if (resize
->prepare_done
) {
1470 ret
= resize
->error
;
1472 resize_hpt_release(kvm
, resize
);
1474 ret
= 100; /* estimated time in ms */
1480 /* not suitable, cancel it */
1481 resize_hpt_release(kvm
, resize
);
1486 goto out
; /* nothing to do */
1488 /* start new resize */
1490 resize
= kzalloc(sizeof(*resize
), GFP_KERNEL
);
1495 resize
->order
= shift
;
1497 INIT_WORK(&resize
->work
, resize_hpt_prepare_work
);
1498 kvm
->arch
.resize_hpt
= resize
;
1500 schedule_work(&resize
->work
);
1502 ret
= 100; /* estimated time in ms */
1505 mutex_unlock(&kvm
->lock
);
1509 static void resize_hpt_boot_vcpu(void *opaque
)
1511 /* Nothing to do, just force a KVM exit */
1514 long kvm_vm_ioctl_resize_hpt_commit(struct kvm
*kvm
,
1515 struct kvm_ppc_resize_hpt
*rhpt
)
1517 unsigned long flags
= rhpt
->flags
;
1518 unsigned long shift
= rhpt
->shift
;
1519 struct kvm_resize_hpt
*resize
;
1522 if (flags
!= 0 || kvm_is_radix(kvm
))
1525 if (shift
&& ((shift
< 18) || (shift
> 46)))
1528 mutex_lock(&kvm
->lock
);
1530 resize
= kvm
->arch
.resize_hpt
;
1532 /* This shouldn't be possible */
1534 if (WARN_ON(!kvm
->arch
.mmu_ready
))
1537 /* Stop VCPUs from running while we mess with the HPT */
1538 kvm
->arch
.mmu_ready
= 0;
1541 /* Boot all CPUs out of the guest so they re-read
1543 on_each_cpu(resize_hpt_boot_vcpu
, NULL
, 1);
1546 if (!resize
|| (resize
->order
!= shift
))
1550 if (!resize
->prepare_done
)
1553 ret
= resize
->error
;
1557 ret
= resize_hpt_rehash(resize
);
1561 resize_hpt_pivot(resize
);
1564 /* Let VCPUs run again */
1565 kvm
->arch
.mmu_ready
= 1;
1568 resize_hpt_release(kvm
, resize
);
1569 mutex_unlock(&kvm
->lock
);
1574 * Functions for reading and writing the hash table via reads and
1575 * writes on a file descriptor.
1577 * Reads return the guest view of the hash table, which has to be
1578 * pieced together from the real hash table and the guest_rpte
1579 * values in the revmap array.
1581 * On writes, each HPTE written is considered in turn, and if it
1582 * is valid, it is written to the HPT as if an H_ENTER with the
1583 * exact flag set was done. When the invalid count is non-zero
1584 * in the header written to the stream, the kernel will make
1585 * sure that that many HPTEs are invalid, and invalidate them
1589 struct kvm_htab_ctx
{
1590 unsigned long index
;
1591 unsigned long flags
;
1596 #define HPTE_SIZE (2 * sizeof(unsigned long))
1599 * Returns 1 if this HPT entry has been modified or has pending
1602 static int hpte_dirty(struct revmap_entry
*revp
, __be64
*hptp
)
1604 unsigned long rcbits_unset
;
1606 if (revp
->guest_rpte
& HPTE_GR_MODIFIED
)
1609 /* Also need to consider changes in reference and changed bits */
1610 rcbits_unset
= ~revp
->guest_rpte
& (HPTE_R_R
| HPTE_R_C
);
1611 if ((be64_to_cpu(hptp
[0]) & HPTE_V_VALID
) &&
1612 (be64_to_cpu(hptp
[1]) & rcbits_unset
))
1618 static long record_hpte(unsigned long flags
, __be64
*hptp
,
1619 unsigned long *hpte
, struct revmap_entry
*revp
,
1620 int want_valid
, int first_pass
)
1622 unsigned long v
, r
, hr
;
1623 unsigned long rcbits_unset
;
1627 /* Unmodified entries are uninteresting except on the first pass */
1628 dirty
= hpte_dirty(revp
, hptp
);
1629 if (!first_pass
&& !dirty
)
1633 if (be64_to_cpu(hptp
[0]) & (HPTE_V_VALID
| HPTE_V_ABSENT
)) {
1635 if ((flags
& KVM_GET_HTAB_BOLTED_ONLY
) &&
1636 !(be64_to_cpu(hptp
[0]) & HPTE_V_BOLTED
))
1639 if (valid
!= want_valid
)
1643 if (valid
|| dirty
) {
1644 /* lock the HPTE so it's stable and read it */
1646 while (!try_lock_hpte(hptp
, HPTE_V_HVLOCK
))
1648 v
= be64_to_cpu(hptp
[0]);
1649 hr
= be64_to_cpu(hptp
[1]);
1650 if (cpu_has_feature(CPU_FTR_ARCH_300
)) {
1651 v
= hpte_new_to_old_v(v
, hr
);
1652 hr
= hpte_new_to_old_r(hr
);
1655 /* re-evaluate valid and dirty from synchronized HPTE value */
1656 valid
= !!(v
& HPTE_V_VALID
);
1657 dirty
= !!(revp
->guest_rpte
& HPTE_GR_MODIFIED
);
1659 /* Harvest R and C into guest view if necessary */
1660 rcbits_unset
= ~revp
->guest_rpte
& (HPTE_R_R
| HPTE_R_C
);
1661 if (valid
&& (rcbits_unset
& hr
)) {
1662 revp
->guest_rpte
|= (hr
&
1663 (HPTE_R_R
| HPTE_R_C
)) | HPTE_GR_MODIFIED
;
1667 if (v
& HPTE_V_ABSENT
) {
1668 v
&= ~HPTE_V_ABSENT
;
1672 if ((flags
& KVM_GET_HTAB_BOLTED_ONLY
) && !(v
& HPTE_V_BOLTED
))
1675 r
= revp
->guest_rpte
;
1676 /* only clear modified if this is the right sort of entry */
1677 if (valid
== want_valid
&& dirty
) {
1678 r
&= ~HPTE_GR_MODIFIED
;
1679 revp
->guest_rpte
= r
;
1681 unlock_hpte(hptp
, be64_to_cpu(hptp
[0]));
1683 if (!(valid
== want_valid
&& (first_pass
|| dirty
)))
1686 hpte
[0] = cpu_to_be64(v
);
1687 hpte
[1] = cpu_to_be64(r
);
1691 static ssize_t
kvm_htab_read(struct file
*file
, char __user
*buf
,
1692 size_t count
, loff_t
*ppos
)
1694 struct kvm_htab_ctx
*ctx
= file
->private_data
;
1695 struct kvm
*kvm
= ctx
->kvm
;
1696 struct kvm_get_htab_header hdr
;
1698 struct revmap_entry
*revp
;
1699 unsigned long i
, nb
, nw
;
1700 unsigned long __user
*lbuf
;
1701 struct kvm_get_htab_header __user
*hptr
;
1702 unsigned long flags
;
1704 unsigned long hpte
[2];
1706 if (!access_ok(VERIFY_WRITE
, buf
, count
))
1708 if (kvm_is_radix(kvm
))
1711 first_pass
= ctx
->first_pass
;
1715 hptp
= (__be64
*)(kvm
->arch
.hpt
.virt
+ (i
* HPTE_SIZE
));
1716 revp
= kvm
->arch
.hpt
.rev
+ i
;
1717 lbuf
= (unsigned long __user
*)buf
;
1720 while (nb
+ sizeof(hdr
) + HPTE_SIZE
< count
) {
1721 /* Initialize header */
1722 hptr
= (struct kvm_get_htab_header __user
*)buf
;
1727 lbuf
= (unsigned long __user
*)(buf
+ sizeof(hdr
));
1729 /* Skip uninteresting entries, i.e. clean on not-first pass */
1731 while (i
< kvmppc_hpt_npte(&kvm
->arch
.hpt
) &&
1732 !hpte_dirty(revp
, hptp
)) {
1740 /* Grab a series of valid entries */
1741 while (i
< kvmppc_hpt_npte(&kvm
->arch
.hpt
) &&
1742 hdr
.n_valid
< 0xffff &&
1743 nb
+ HPTE_SIZE
< count
&&
1744 record_hpte(flags
, hptp
, hpte
, revp
, 1, first_pass
)) {
1745 /* valid entry, write it out */
1747 if (__put_user(hpte
[0], lbuf
) ||
1748 __put_user(hpte
[1], lbuf
+ 1))
1756 /* Now skip invalid entries while we can */
1757 while (i
< kvmppc_hpt_npte(&kvm
->arch
.hpt
) &&
1758 hdr
.n_invalid
< 0xffff &&
1759 record_hpte(flags
, hptp
, hpte
, revp
, 0, first_pass
)) {
1760 /* found an invalid entry */
1767 if (hdr
.n_valid
|| hdr
.n_invalid
) {
1768 /* write back the header */
1769 if (__copy_to_user(hptr
, &hdr
, sizeof(hdr
)))
1772 buf
= (char __user
*)lbuf
;
1777 /* Check if we've wrapped around the hash table */
1778 if (i
>= kvmppc_hpt_npte(&kvm
->arch
.hpt
)) {
1780 ctx
->first_pass
= 0;
1790 static ssize_t
kvm_htab_write(struct file
*file
, const char __user
*buf
,
1791 size_t count
, loff_t
*ppos
)
1793 struct kvm_htab_ctx
*ctx
= file
->private_data
;
1794 struct kvm
*kvm
= ctx
->kvm
;
1795 struct kvm_get_htab_header hdr
;
1798 unsigned long __user
*lbuf
;
1800 unsigned long tmp
[2];
1805 if (!access_ok(VERIFY_READ
, buf
, count
))
1807 if (kvm_is_radix(kvm
))
1810 /* lock out vcpus from running while we're doing this */
1811 mutex_lock(&kvm
->lock
);
1812 mmu_ready
= kvm
->arch
.mmu_ready
;
1814 kvm
->arch
.mmu_ready
= 0; /* temporarily */
1815 /* order mmu_ready vs. vcpus_running */
1817 if (atomic_read(&kvm
->arch
.vcpus_running
)) {
1818 kvm
->arch
.mmu_ready
= 1;
1819 mutex_unlock(&kvm
->lock
);
1825 for (nb
= 0; nb
+ sizeof(hdr
) <= count
; ) {
1827 if (__copy_from_user(&hdr
, buf
, sizeof(hdr
)))
1831 if (nb
+ hdr
.n_valid
* HPTE_SIZE
> count
)
1839 if (i
>= kvmppc_hpt_npte(&kvm
->arch
.hpt
) ||
1840 i
+ hdr
.n_valid
+ hdr
.n_invalid
> kvmppc_hpt_npte(&kvm
->arch
.hpt
))
1843 hptp
= (__be64
*)(kvm
->arch
.hpt
.virt
+ (i
* HPTE_SIZE
));
1844 lbuf
= (unsigned long __user
*)buf
;
1845 for (j
= 0; j
< hdr
.n_valid
; ++j
) {
1850 if (__get_user(hpte_v
, lbuf
) ||
1851 __get_user(hpte_r
, lbuf
+ 1))
1853 v
= be64_to_cpu(hpte_v
);
1854 r
= be64_to_cpu(hpte_r
);
1856 if (!(v
& HPTE_V_VALID
))
1861 if (be64_to_cpu(hptp
[0]) & (HPTE_V_VALID
| HPTE_V_ABSENT
))
1862 kvmppc_do_h_remove(kvm
, 0, i
, 0, tmp
);
1864 ret
= kvmppc_virtmode_do_h_enter(kvm
, H_EXACT
, i
, v
, r
,
1866 if (ret
!= H_SUCCESS
) {
1867 pr_err("kvm_htab_write ret %ld i=%ld v=%lx "
1868 "r=%lx\n", ret
, i
, v
, r
);
1871 if (!mmu_ready
&& is_vrma_hpte(v
)) {
1872 unsigned long psize
= hpte_base_page_size(v
, r
);
1873 unsigned long senc
= slb_pgsize_encoding(psize
);
1876 kvm
->arch
.vrma_slb_v
= senc
| SLB_VSID_B_1T
|
1877 (VRMA_VSID
<< SLB_VSID_SHIFT_1T
);
1878 lpcr
= senc
<< (LPCR_VRMASD_SH
- 4);
1879 kvmppc_update_lpcr(kvm
, lpcr
, LPCR_VRMASD
);
1886 for (j
= 0; j
< hdr
.n_invalid
; ++j
) {
1887 if (be64_to_cpu(hptp
[0]) & (HPTE_V_VALID
| HPTE_V_ABSENT
))
1888 kvmppc_do_h_remove(kvm
, 0, i
, 0, tmp
);
1896 /* Order HPTE updates vs. mmu_ready */
1898 kvm
->arch
.mmu_ready
= mmu_ready
;
1899 mutex_unlock(&kvm
->lock
);
1906 static int kvm_htab_release(struct inode
*inode
, struct file
*filp
)
1908 struct kvm_htab_ctx
*ctx
= filp
->private_data
;
1910 filp
->private_data
= NULL
;
1911 if (!(ctx
->flags
& KVM_GET_HTAB_WRITE
))
1912 atomic_dec(&ctx
->kvm
->arch
.hpte_mod_interest
);
1913 kvm_put_kvm(ctx
->kvm
);
1918 static const struct file_operations kvm_htab_fops
= {
1919 .read
= kvm_htab_read
,
1920 .write
= kvm_htab_write
,
1921 .llseek
= default_llseek
,
1922 .release
= kvm_htab_release
,
1925 int kvm_vm_ioctl_get_htab_fd(struct kvm
*kvm
, struct kvm_get_htab_fd
*ghf
)
1928 struct kvm_htab_ctx
*ctx
;
1931 /* reject flags we don't recognize */
1932 if (ghf
->flags
& ~(KVM_GET_HTAB_BOLTED_ONLY
| KVM_GET_HTAB_WRITE
))
1934 ctx
= kzalloc(sizeof(*ctx
), GFP_KERNEL
);
1939 ctx
->index
= ghf
->start_index
;
1940 ctx
->flags
= ghf
->flags
;
1941 ctx
->first_pass
= 1;
1943 rwflag
= (ghf
->flags
& KVM_GET_HTAB_WRITE
) ? O_WRONLY
: O_RDONLY
;
1944 ret
= anon_inode_getfd("kvm-htab", &kvm_htab_fops
, ctx
, rwflag
| O_CLOEXEC
);
1951 if (rwflag
== O_RDONLY
) {
1952 mutex_lock(&kvm
->slots_lock
);
1953 atomic_inc(&kvm
->arch
.hpte_mod_interest
);
1954 /* make sure kvmppc_do_h_enter etc. see the increment */
1955 synchronize_srcu_expedited(&kvm
->srcu
);
1956 mutex_unlock(&kvm
->slots_lock
);
1962 struct debugfs_htab_state
{
1965 unsigned long hpt_index
;
1971 static int debugfs_htab_open(struct inode
*inode
, struct file
*file
)
1973 struct kvm
*kvm
= inode
->i_private
;
1974 struct debugfs_htab_state
*p
;
1976 p
= kzalloc(sizeof(*p
), GFP_KERNEL
);
1982 mutex_init(&p
->mutex
);
1983 file
->private_data
= p
;
1985 return nonseekable_open(inode
, file
);
1988 static int debugfs_htab_release(struct inode
*inode
, struct file
*file
)
1990 struct debugfs_htab_state
*p
= file
->private_data
;
1992 kvm_put_kvm(p
->kvm
);
1997 static ssize_t
debugfs_htab_read(struct file
*file
, char __user
*buf
,
1998 size_t len
, loff_t
*ppos
)
2000 struct debugfs_htab_state
*p
= file
->private_data
;
2003 unsigned long v
, hr
, gr
;
2008 if (kvm_is_radix(kvm
))
2011 ret
= mutex_lock_interruptible(&p
->mutex
);
2015 if (p
->chars_left
) {
2019 r
= copy_to_user(buf
, p
->buf
+ p
->buf_index
, n
);
2034 hptp
= (__be64
*)(kvm
->arch
.hpt
.virt
+ (i
* HPTE_SIZE
));
2035 for (; len
!= 0 && i
< kvmppc_hpt_npte(&kvm
->arch
.hpt
);
2037 if (!(be64_to_cpu(hptp
[0]) & (HPTE_V_VALID
| HPTE_V_ABSENT
)))
2040 /* lock the HPTE so it's stable and read it */
2042 while (!try_lock_hpte(hptp
, HPTE_V_HVLOCK
))
2044 v
= be64_to_cpu(hptp
[0]) & ~HPTE_V_HVLOCK
;
2045 hr
= be64_to_cpu(hptp
[1]);
2046 gr
= kvm
->arch
.hpt
.rev
[i
].guest_rpte
;
2047 unlock_hpte(hptp
, v
);
2050 if (!(v
& (HPTE_V_VALID
| HPTE_V_ABSENT
)))
2053 n
= scnprintf(p
->buf
, sizeof(p
->buf
),
2054 "%6lx %.16lx %.16lx %.16lx\n",
2059 r
= copy_to_user(buf
, p
->buf
, n
);
2075 mutex_unlock(&p
->mutex
);
2079 static ssize_t
debugfs_htab_write(struct file
*file
, const char __user
*buf
,
2080 size_t len
, loff_t
*ppos
)
2085 static const struct file_operations debugfs_htab_fops
= {
2086 .owner
= THIS_MODULE
,
2087 .open
= debugfs_htab_open
,
2088 .release
= debugfs_htab_release
,
2089 .read
= debugfs_htab_read
,
2090 .write
= debugfs_htab_write
,
2091 .llseek
= generic_file_llseek
,
2094 void kvmppc_mmu_debugfs_init(struct kvm
*kvm
)
2096 kvm
->arch
.htab_dentry
= debugfs_create_file("htab", 0400,
2097 kvm
->arch
.debugfs_dir
, kvm
,
2098 &debugfs_htab_fops
);
2101 void kvmppc_mmu_book3s_hv_init(struct kvm_vcpu
*vcpu
)
2103 struct kvmppc_mmu
*mmu
= &vcpu
->arch
.mmu
;
2105 vcpu
->arch
.slb_nr
= 32; /* POWER7/POWER8 */
2107 mmu
->xlate
= kvmppc_mmu_book3s_64_hv_xlate
;
2108 mmu
->reset_msr
= kvmppc_mmu_book3s_64_hv_reset_msr
;
2110 vcpu
->arch
.hflags
|= BOOK3S_HFLAG_SLB
;