2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 * GNU General Public License for more details.
11 * You should have received a copy of the GNU General Public License
12 * along with this program; if not, write to the Free Software
13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
15 * Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
18 #include <linux/types.h>
19 #include <linux/string.h>
20 #include <linux/kvm.h>
21 #include <linux/kvm_host.h>
22 #include <linux/highmem.h>
23 #include <linux/gfp.h>
24 #include <linux/slab.h>
25 #include <linux/hugetlb.h>
26 #include <linux/vmalloc.h>
27 #include <linux/srcu.h>
28 #include <linux/anon_inodes.h>
29 #include <linux/file.h>
30 #include <linux/debugfs.h>
32 #include <asm/tlbflush.h>
33 #include <asm/kvm_ppc.h>
34 #include <asm/kvm_book3s.h>
35 #include <asm/book3s/64/mmu-hash.h>
36 #include <asm/hvcall.h>
37 #include <asm/synch.h>
38 #include <asm/ppc-opcode.h>
39 #include <asm/cputable.h>
43 //#define DEBUG_RESIZE_HPT 1
45 #ifdef DEBUG_RESIZE_HPT
46 #define resize_hpt_debug(resize, ...) \
48 printk(KERN_DEBUG "RESIZE HPT %p: ", resize); \
49 printk(__VA_ARGS__); \
52 #define resize_hpt_debug(resize, ...) \
56 static long kvmppc_virtmode_do_h_enter(struct kvm
*kvm
, unsigned long flags
,
57 long pte_index
, unsigned long pteh
,
58 unsigned long ptel
, unsigned long *pte_idx_ret
);
60 struct kvm_resize_hpt
{
61 /* These fields read-only after init */
63 struct work_struct work
;
66 /* These fields protected by kvm->lock */
70 /* Private to the work thread, until prepare_done is true,
71 * then protected by kvm->resize_hpt_sem */
72 struct kvm_hpt_info hpt
;
75 static void kvmppc_rmap_reset(struct kvm
*kvm
);
77 int kvmppc_allocate_hpt(struct kvm_hpt_info
*info
, u32 order
)
79 unsigned long hpt
= 0;
81 struct page
*page
= NULL
;
82 struct revmap_entry
*rev
;
85 if ((order
< PPC_MIN_HPT_ORDER
) || (order
> PPC_MAX_HPT_ORDER
))
88 page
= kvm_alloc_hpt_cma(1ul << (order
- PAGE_SHIFT
));
90 hpt
= (unsigned long)pfn_to_kaddr(page_to_pfn(page
));
91 memset((void *)hpt
, 0, (1ul << order
));
96 hpt
= __get_free_pages(GFP_KERNEL
|__GFP_ZERO
|__GFP_RETRY_MAYFAIL
97 |__GFP_NOWARN
, order
- PAGE_SHIFT
);
102 /* HPTEs are 2**4 bytes long */
103 npte
= 1ul << (order
- 4);
105 /* Allocate reverse map array */
106 rev
= vmalloc(sizeof(struct revmap_entry
) * npte
);
108 pr_err("kvmppc_allocate_hpt: Couldn't alloc reverse map array\n");
110 kvm_free_hpt_cma(page
, 1 << (order
- PAGE_SHIFT
));
112 free_pages(hpt
, order
- PAGE_SHIFT
);
124 void kvmppc_set_hpt(struct kvm
*kvm
, struct kvm_hpt_info
*info
)
126 atomic64_set(&kvm
->arch
.mmio_update
, 0);
127 kvm
->arch
.hpt
= *info
;
128 kvm
->arch
.sdr1
= __pa(info
->virt
) | (info
->order
- 18);
130 pr_debug("KVM guest htab at %lx (order %ld), LPID %x\n",
131 info
->virt
, (long)info
->order
, kvm
->arch
.lpid
);
134 long kvmppc_alloc_reset_hpt(struct kvm
*kvm
, int order
)
137 struct kvm_hpt_info info
;
139 if (kvm_is_radix(kvm
))
142 mutex_lock(&kvm
->lock
);
143 if (kvm
->arch
.hpte_setup_done
) {
144 kvm
->arch
.hpte_setup_done
= 0;
145 /* order hpte_setup_done vs. vcpus_running */
147 if (atomic_read(&kvm
->arch
.vcpus_running
)) {
148 kvm
->arch
.hpte_setup_done
= 1;
152 if (kvm
->arch
.hpt
.order
== order
) {
153 /* We already have a suitable HPT */
155 /* Set the entire HPT to 0, i.e. invalid HPTEs */
156 memset((void *)kvm
->arch
.hpt
.virt
, 0, 1ul << order
);
158 * Reset all the reverse-mapping chains for all memslots
160 kvmppc_rmap_reset(kvm
);
161 /* Ensure that each vcpu will flush its TLB on next entry. */
162 cpumask_setall(&kvm
->arch
.need_tlb_flush
);
167 if (kvm
->arch
.hpt
.virt
) {
168 kvmppc_free_hpt(&kvm
->arch
.hpt
);
169 kvmppc_rmap_reset(kvm
);
172 err
= kvmppc_allocate_hpt(&info
, order
);
175 kvmppc_set_hpt(kvm
, &info
);
178 mutex_unlock(&kvm
->lock
);
182 void kvmppc_free_hpt(struct kvm_hpt_info
*info
)
186 kvm_free_hpt_cma(virt_to_page(info
->virt
),
187 1 << (info
->order
- PAGE_SHIFT
));
189 free_pages(info
->virt
, info
->order
- PAGE_SHIFT
);
194 /* Bits in first HPTE dword for pagesize 4k, 64k or 16M */
195 static inline unsigned long hpte0_pgsize_encoding(unsigned long pgsize
)
197 return (pgsize
> 0x1000) ? HPTE_V_LARGE
: 0;
200 /* Bits in second HPTE dword for pagesize 4k, 64k or 16M */
201 static inline unsigned long hpte1_pgsize_encoding(unsigned long pgsize
)
203 return (pgsize
== 0x10000) ? 0x1000 : 0;
206 void kvmppc_map_vrma(struct kvm_vcpu
*vcpu
, struct kvm_memory_slot
*memslot
,
207 unsigned long porder
)
210 unsigned long npages
;
211 unsigned long hp_v
, hp_r
;
212 unsigned long addr
, hash
;
214 unsigned long hp0
, hp1
;
215 unsigned long idx_ret
;
217 struct kvm
*kvm
= vcpu
->kvm
;
219 psize
= 1ul << porder
;
220 npages
= memslot
->npages
>> (porder
- PAGE_SHIFT
);
222 /* VRMA can't be > 1TB */
223 if (npages
> 1ul << (40 - porder
))
224 npages
= 1ul << (40 - porder
);
225 /* Can't use more than 1 HPTE per HPTEG */
226 if (npages
> kvmppc_hpt_mask(&kvm
->arch
.hpt
) + 1)
227 npages
= kvmppc_hpt_mask(&kvm
->arch
.hpt
) + 1;
229 hp0
= HPTE_V_1TB_SEG
| (VRMA_VSID
<< (40 - 16)) |
230 HPTE_V_BOLTED
| hpte0_pgsize_encoding(psize
);
231 hp1
= hpte1_pgsize_encoding(psize
) |
232 HPTE_R_R
| HPTE_R_C
| HPTE_R_M
| PP_RWXX
;
234 for (i
= 0; i
< npages
; ++i
) {
236 /* can't use hpt_hash since va > 64 bits */
237 hash
= (i
^ (VRMA_VSID
^ (VRMA_VSID
<< 25)))
238 & kvmppc_hpt_mask(&kvm
->arch
.hpt
);
240 * We assume that the hash table is empty and no
241 * vcpus are using it at this stage. Since we create
242 * at most one HPTE per HPTEG, we just assume entry 7
243 * is available and use it.
245 hash
= (hash
<< 3) + 7;
246 hp_v
= hp0
| ((addr
>> 16) & ~0x7fUL
);
248 ret
= kvmppc_virtmode_do_h_enter(kvm
, H_EXACT
, hash
, hp_v
, hp_r
,
250 if (ret
!= H_SUCCESS
) {
251 pr_err("KVM: map_vrma at %lx failed, ret=%ld\n",
258 int kvmppc_mmu_hv_init(void)
260 unsigned long host_lpid
, rsvd_lpid
;
262 if (!cpu_has_feature(CPU_FTR_HVMODE
))
265 /* POWER7 has 10-bit LPIDs (12-bit in POWER8) */
266 host_lpid
= mfspr(SPRN_LPID
);
267 rsvd_lpid
= LPID_RSVD
;
269 kvmppc_init_lpid(rsvd_lpid
+ 1);
271 kvmppc_claim_lpid(host_lpid
);
272 /* rsvd_lpid is reserved for use in partition switching */
273 kvmppc_claim_lpid(rsvd_lpid
);
278 static void kvmppc_mmu_book3s_64_hv_reset_msr(struct kvm_vcpu
*vcpu
)
280 unsigned long msr
= vcpu
->arch
.intr_msr
;
282 /* If transactional, change to suspend mode on IRQ delivery */
283 if (MSR_TM_TRANSACTIONAL(vcpu
->arch
.shregs
.msr
))
286 msr
|= vcpu
->arch
.shregs
.msr
& MSR_TS_MASK
;
287 kvmppc_set_msr(vcpu
, msr
);
290 static long kvmppc_virtmode_do_h_enter(struct kvm
*kvm
, unsigned long flags
,
291 long pte_index
, unsigned long pteh
,
292 unsigned long ptel
, unsigned long *pte_idx_ret
)
296 /* Protect linux PTE lookup from page table destruction */
297 rcu_read_lock_sched(); /* this disables preemption too */
298 ret
= kvmppc_do_h_enter(kvm
, flags
, pte_index
, pteh
, ptel
,
299 current
->mm
->pgd
, false, pte_idx_ret
);
300 rcu_read_unlock_sched();
301 if (ret
== H_TOO_HARD
) {
302 /* this can't happen */
303 pr_err("KVM: Oops, kvmppc_h_enter returned too hard!\n");
304 ret
= H_RESOURCE
; /* or something */
310 static struct kvmppc_slb
*kvmppc_mmu_book3s_hv_find_slbe(struct kvm_vcpu
*vcpu
,
316 for (i
= 0; i
< vcpu
->arch
.slb_nr
; i
++) {
317 if (!(vcpu
->arch
.slb
[i
].orige
& SLB_ESID_V
))
320 if (vcpu
->arch
.slb
[i
].origv
& SLB_VSID_B_1T
)
325 if (((vcpu
->arch
.slb
[i
].orige
^ eaddr
) & mask
) == 0)
326 return &vcpu
->arch
.slb
[i
];
331 static unsigned long kvmppc_mmu_get_real_addr(unsigned long v
, unsigned long r
,
334 unsigned long ra_mask
;
336 ra_mask
= hpte_page_size(v
, r
) - 1;
337 return (r
& HPTE_R_RPN
& ~ra_mask
) | (ea
& ra_mask
);
340 static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu
*vcpu
, gva_t eaddr
,
341 struct kvmppc_pte
*gpte
, bool data
, bool iswrite
)
343 struct kvm
*kvm
= vcpu
->kvm
;
344 struct kvmppc_slb
*slbe
;
346 unsigned long pp
, key
;
347 unsigned long v
, orig_v
, gr
;
350 int virtmode
= vcpu
->arch
.shregs
.msr
& (data
? MSR_DR
: MSR_IR
);
354 slbe
= kvmppc_mmu_book3s_hv_find_slbe(vcpu
, eaddr
);
359 /* real mode access */
360 slb_v
= vcpu
->kvm
->arch
.vrma_slb_v
;
364 /* Find the HPTE in the hash table */
365 index
= kvmppc_hv_find_lock_hpte(kvm
, eaddr
, slb_v
,
366 HPTE_V_VALID
| HPTE_V_ABSENT
);
371 hptep
= (__be64
*)(kvm
->arch
.hpt
.virt
+ (index
<< 4));
372 v
= orig_v
= be64_to_cpu(hptep
[0]) & ~HPTE_V_HVLOCK
;
373 if (cpu_has_feature(CPU_FTR_ARCH_300
))
374 v
= hpte_new_to_old_v(v
, be64_to_cpu(hptep
[1]));
375 gr
= kvm
->arch
.hpt
.rev
[index
].guest_rpte
;
377 unlock_hpte(hptep
, orig_v
);
381 gpte
->vpage
= ((v
& HPTE_V_AVPN
) << 4) | ((eaddr
>> 12) & 0xfff);
383 /* Get PP bits and key for permission check */
384 pp
= gr
& (HPTE_R_PP0
| HPTE_R_PP
);
385 key
= (vcpu
->arch
.shregs
.msr
& MSR_PR
) ? SLB_VSID_KP
: SLB_VSID_KS
;
388 /* Calculate permissions */
389 gpte
->may_read
= hpte_read_permission(pp
, key
);
390 gpte
->may_write
= hpte_write_permission(pp
, key
);
391 gpte
->may_execute
= gpte
->may_read
&& !(gr
& (HPTE_R_N
| HPTE_R_G
));
393 /* Storage key permission check for POWER7 */
394 if (data
&& virtmode
) {
395 int amrfield
= hpte_get_skey_perm(gr
, vcpu
->arch
.amr
);
402 /* Get the guest physical address */
403 gpte
->raddr
= kvmppc_mmu_get_real_addr(v
, gr
, eaddr
);
408 * Quick test for whether an instruction is a load or a store.
409 * If the instruction is a load or a store, then this will indicate
410 * which it is, at least on server processors. (Embedded processors
411 * have some external PID instructions that don't follow the rule
412 * embodied here.) If the instruction isn't a load or store, then
413 * this doesn't return anything useful.
415 static int instruction_is_store(unsigned int instr
)
420 if ((instr
& 0xfc000000) == 0x7c000000)
421 mask
= 0x100; /* major opcode 31 */
422 return (instr
& mask
) != 0;
425 int kvmppc_hv_emulate_mmio(struct kvm_run
*run
, struct kvm_vcpu
*vcpu
,
426 unsigned long gpa
, gva_t ea
, int is_store
)
431 * If we fail, we just return to the guest and try executing it again.
433 if (kvmppc_get_last_inst(vcpu
, INST_GENERIC
, &last_inst
) !=
438 * WARNING: We do not know for sure whether the instruction we just
439 * read from memory is the same that caused the fault in the first
440 * place. If the instruction we read is neither an load or a store,
441 * then it can't access memory, so we don't need to worry about
442 * enforcing access permissions. So, assuming it is a load or
443 * store, we just check that its direction (load or store) is
444 * consistent with the original fault, since that's what we
445 * checked the access permissions against. If there is a mismatch
446 * we just return and retry the instruction.
449 if (instruction_is_store(last_inst
) != !!is_store
)
453 * Emulated accesses are emulated by looking at the hash for
454 * translation once, then performing the access later. The
455 * translation could be invalidated in the meantime in which
456 * point performing the subsequent memory access on the old
457 * physical address could possibly be a security hole for the
458 * guest (but not the host).
460 * This is less of an issue for MMIO stores since they aren't
461 * globally visible. It could be an issue for MMIO loads to
462 * a certain extent but we'll ignore it for now.
465 vcpu
->arch
.paddr_accessed
= gpa
;
466 vcpu
->arch
.vaddr_accessed
= ea
;
467 return kvmppc_emulate_mmio(run
, vcpu
);
470 int kvmppc_book3s_hv_page_fault(struct kvm_run
*run
, struct kvm_vcpu
*vcpu
,
471 unsigned long ea
, unsigned long dsisr
)
473 struct kvm
*kvm
= vcpu
->kvm
;
474 unsigned long hpte
[3], r
;
475 unsigned long hnow_v
, hnow_r
;
477 unsigned long mmu_seq
, psize
, pte_size
;
478 unsigned long gpa_base
, gfn_base
;
479 unsigned long gpa
, gfn
, hva
, pfn
;
480 struct kvm_memory_slot
*memslot
;
482 struct revmap_entry
*rev
;
483 struct page
*page
, *pages
[1];
484 long index
, ret
, npages
;
486 unsigned int writing
, write_ok
;
487 struct vm_area_struct
*vma
;
488 unsigned long rcbits
;
491 if (kvm_is_radix(kvm
))
492 return kvmppc_book3s_radix_page_fault(run
, vcpu
, ea
, dsisr
);
495 * Real-mode code has already searched the HPT and found the
496 * entry we're interested in. Lock the entry and check that
497 * it hasn't changed. If it has, just return and re-execute the
500 if (ea
!= vcpu
->arch
.pgfault_addr
)
503 if (vcpu
->arch
.pgfault_cache
) {
504 mmio_update
= atomic64_read(&kvm
->arch
.mmio_update
);
505 if (mmio_update
== vcpu
->arch
.pgfault_cache
->mmio_update
) {
506 r
= vcpu
->arch
.pgfault_cache
->rpte
;
507 psize
= hpte_page_size(vcpu
->arch
.pgfault_hpte
[0], r
);
508 gpa_base
= r
& HPTE_R_RPN
& ~(psize
- 1);
509 gfn_base
= gpa_base
>> PAGE_SHIFT
;
510 gpa
= gpa_base
| (ea
& (psize
- 1));
511 return kvmppc_hv_emulate_mmio(run
, vcpu
, gpa
, ea
,
512 dsisr
& DSISR_ISSTORE
);
515 index
= vcpu
->arch
.pgfault_index
;
516 hptep
= (__be64
*)(kvm
->arch
.hpt
.virt
+ (index
<< 4));
517 rev
= &kvm
->arch
.hpt
.rev
[index
];
519 while (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
))
521 hpte
[0] = be64_to_cpu(hptep
[0]) & ~HPTE_V_HVLOCK
;
522 hpte
[1] = be64_to_cpu(hptep
[1]);
523 hpte
[2] = r
= rev
->guest_rpte
;
524 unlock_hpte(hptep
, hpte
[0]);
527 if (cpu_has_feature(CPU_FTR_ARCH_300
)) {
528 hpte
[0] = hpte_new_to_old_v(hpte
[0], hpte
[1]);
529 hpte
[1] = hpte_new_to_old_r(hpte
[1]);
531 if (hpte
[0] != vcpu
->arch
.pgfault_hpte
[0] ||
532 hpte
[1] != vcpu
->arch
.pgfault_hpte
[1])
535 /* Translate the logical address and get the page */
536 psize
= hpte_page_size(hpte
[0], r
);
537 gpa_base
= r
& HPTE_R_RPN
& ~(psize
- 1);
538 gfn_base
= gpa_base
>> PAGE_SHIFT
;
539 gpa
= gpa_base
| (ea
& (psize
- 1));
540 gfn
= gpa
>> PAGE_SHIFT
;
541 memslot
= gfn_to_memslot(kvm
, gfn
);
543 trace_kvm_page_fault_enter(vcpu
, hpte
, memslot
, ea
, dsisr
);
545 /* No memslot means it's an emulated MMIO region */
546 if (!memslot
|| (memslot
->flags
& KVM_MEMSLOT_INVALID
))
547 return kvmppc_hv_emulate_mmio(run
, vcpu
, gpa
, ea
,
548 dsisr
& DSISR_ISSTORE
);
551 * This should never happen, because of the slot_is_aligned()
552 * check in kvmppc_do_h_enter().
554 if (gfn_base
< memslot
->base_gfn
)
557 /* used to check for invalidations in progress */
558 mmu_seq
= kvm
->mmu_notifier_seq
;
565 pte_size
= PAGE_SIZE
;
566 writing
= (dsisr
& DSISR_ISSTORE
) != 0;
567 /* If writing != 0, then the HPTE must allow writing, if we get here */
569 hva
= gfn_to_hva_memslot(memslot
, gfn
);
570 npages
= get_user_pages_fast(hva
, 1, writing
, pages
);
572 /* Check if it's an I/O mapping */
573 down_read(¤t
->mm
->mmap_sem
);
574 vma
= find_vma(current
->mm
, hva
);
575 if (vma
&& vma
->vm_start
<= hva
&& hva
+ psize
<= vma
->vm_end
&&
576 (vma
->vm_flags
& VM_PFNMAP
)) {
577 pfn
= vma
->vm_pgoff
+
578 ((hva
- vma
->vm_start
) >> PAGE_SHIFT
);
580 is_ci
= pte_ci(__pte((pgprot_val(vma
->vm_page_prot
))));
581 write_ok
= vma
->vm_flags
& VM_WRITE
;
583 up_read(¤t
->mm
->mmap_sem
);
588 pfn
= page_to_pfn(page
);
589 if (PageHuge(page
)) {
590 page
= compound_head(page
);
591 pte_size
<<= compound_order(page
);
593 /* if the guest wants write access, see if that is OK */
594 if (!writing
&& hpte_is_writable(r
)) {
598 * We need to protect against page table destruction
599 * hugepage split and collapse.
601 local_irq_save(flags
);
602 ptep
= find_linux_pte_or_hugepte(current
->mm
->pgd
,
605 pte
= kvmppc_read_update_linux_pte(ptep
, 1);
606 if (__pte_write(pte
))
609 local_irq_restore(flags
);
613 if (psize
> pte_size
)
616 /* Check WIMG vs. the actual page we're accessing */
617 if (!hpte_cache_flags_ok(r
, is_ci
)) {
621 * Allow guest to map emulated device memory as
622 * uncacheable, but actually make it cacheable.
624 r
= (r
& ~(HPTE_R_W
|HPTE_R_I
|HPTE_R_G
)) | HPTE_R_M
;
628 * Set the HPTE to point to pfn.
629 * Since the pfn is at PAGE_SIZE granularity, make sure we
630 * don't mask out lower-order bits if psize < PAGE_SIZE.
632 if (psize
< PAGE_SIZE
)
634 r
= (r
& HPTE_R_KEY_HI
) | (r
& ~(HPTE_R_PP0
- psize
)) |
635 ((pfn
<< PAGE_SHIFT
) & ~(psize
- 1));
636 if (hpte_is_writable(r
) && !write_ok
)
637 r
= hpte_make_readonly(r
);
640 while (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
))
642 hnow_v
= be64_to_cpu(hptep
[0]);
643 hnow_r
= be64_to_cpu(hptep
[1]);
644 if (cpu_has_feature(CPU_FTR_ARCH_300
)) {
645 hnow_v
= hpte_new_to_old_v(hnow_v
, hnow_r
);
646 hnow_r
= hpte_new_to_old_r(hnow_r
);
648 if ((hnow_v
& ~HPTE_V_HVLOCK
) != hpte
[0] || hnow_r
!= hpte
[1] ||
649 rev
->guest_rpte
!= hpte
[2])
650 /* HPTE has been changed under us; let the guest retry */
652 hpte
[0] = (hpte
[0] & ~HPTE_V_ABSENT
) | HPTE_V_VALID
;
654 /* Always put the HPTE in the rmap chain for the page base address */
655 rmap
= &memslot
->arch
.rmap
[gfn_base
- memslot
->base_gfn
];
658 /* Check if we might have been invalidated; let the guest retry if so */
660 if (mmu_notifier_retry(vcpu
->kvm
, mmu_seq
)) {
665 /* Only set R/C in real HPTE if set in both *rmap and guest_rpte */
666 rcbits
= *rmap
>> KVMPPC_RMAP_RC_SHIFT
;
667 r
&= rcbits
| ~(HPTE_R_R
| HPTE_R_C
);
669 if (be64_to_cpu(hptep
[0]) & HPTE_V_VALID
) {
670 /* HPTE was previously valid, so we need to invalidate it */
672 hptep
[0] |= cpu_to_be64(HPTE_V_ABSENT
);
673 kvmppc_invalidate_hpte(kvm
, hptep
, index
);
674 /* don't lose previous R and C bits */
675 r
|= be64_to_cpu(hptep
[1]) & (HPTE_R_R
| HPTE_R_C
);
677 kvmppc_add_revmap_chain(kvm
, rev
, rmap
, index
, 0);
680 if (cpu_has_feature(CPU_FTR_ARCH_300
)) {
681 r
= hpte_old_to_new_r(hpte
[0], r
);
682 hpte
[0] = hpte_old_to_new_v(hpte
[0]);
684 hptep
[1] = cpu_to_be64(r
);
686 __unlock_hpte(hptep
, hpte
[0]);
687 asm volatile("ptesync" : : : "memory");
689 if (page
&& hpte_is_writable(r
))
693 trace_kvm_page_fault_exit(vcpu
, hpte
, ret
);
697 * We drop pages[0] here, not page because page might
698 * have been set to the head page of a compound, but
699 * we have to drop the reference on the correct tail
700 * page to match the get inside gup()
707 __unlock_hpte(hptep
, be64_to_cpu(hptep
[0]));
712 static void kvmppc_rmap_reset(struct kvm
*kvm
)
714 struct kvm_memslots
*slots
;
715 struct kvm_memory_slot
*memslot
;
718 srcu_idx
= srcu_read_lock(&kvm
->srcu
);
719 slots
= kvm_memslots(kvm
);
720 kvm_for_each_memslot(memslot
, slots
) {
722 * This assumes it is acceptable to lose reference and
723 * change bits across a reset.
725 memset(memslot
->arch
.rmap
, 0,
726 memslot
->npages
* sizeof(*memslot
->arch
.rmap
));
728 srcu_read_unlock(&kvm
->srcu
, srcu_idx
);
731 typedef int (*hva_handler_fn
)(struct kvm
*kvm
, struct kvm_memory_slot
*memslot
,
734 static int kvm_handle_hva_range(struct kvm
*kvm
,
737 hva_handler_fn handler
)
741 struct kvm_memslots
*slots
;
742 struct kvm_memory_slot
*memslot
;
744 slots
= kvm_memslots(kvm
);
745 kvm_for_each_memslot(memslot
, slots
) {
746 unsigned long hva_start
, hva_end
;
749 hva_start
= max(start
, memslot
->userspace_addr
);
750 hva_end
= min(end
, memslot
->userspace_addr
+
751 (memslot
->npages
<< PAGE_SHIFT
));
752 if (hva_start
>= hva_end
)
755 * {gfn(page) | page intersects with [hva_start, hva_end)} =
756 * {gfn, gfn+1, ..., gfn_end-1}.
758 gfn
= hva_to_gfn_memslot(hva_start
, memslot
);
759 gfn_end
= hva_to_gfn_memslot(hva_end
+ PAGE_SIZE
- 1, memslot
);
761 for (; gfn
< gfn_end
; ++gfn
) {
762 ret
= handler(kvm
, memslot
, gfn
);
770 static int kvm_handle_hva(struct kvm
*kvm
, unsigned long hva
,
771 hva_handler_fn handler
)
773 return kvm_handle_hva_range(kvm
, hva
, hva
+ 1, handler
);
776 /* Must be called with both HPTE and rmap locked */
777 static void kvmppc_unmap_hpte(struct kvm
*kvm
, unsigned long i
,
778 unsigned long *rmapp
, unsigned long gfn
)
780 __be64
*hptep
= (__be64
*) (kvm
->arch
.hpt
.virt
+ (i
<< 4));
781 struct revmap_entry
*rev
= kvm
->arch
.hpt
.rev
;
783 unsigned long ptel
, psize
, rcbits
;
787 /* chain is now empty */
788 *rmapp
&= ~(KVMPPC_RMAP_PRESENT
| KVMPPC_RMAP_INDEX
);
790 /* remove i from chain */
794 rev
[i
].forw
= rev
[i
].back
= i
;
795 *rmapp
= (*rmapp
& ~KVMPPC_RMAP_INDEX
) | j
;
798 /* Now check and modify the HPTE */
799 ptel
= rev
[i
].guest_rpte
;
800 psize
= hpte_page_size(be64_to_cpu(hptep
[0]), ptel
);
801 if ((be64_to_cpu(hptep
[0]) & HPTE_V_VALID
) &&
802 hpte_rpn(ptel
, psize
) == gfn
) {
803 hptep
[0] |= cpu_to_be64(HPTE_V_ABSENT
);
804 kvmppc_invalidate_hpte(kvm
, hptep
, i
);
805 hptep
[1] &= ~cpu_to_be64(HPTE_R_KEY_HI
| HPTE_R_KEY_LO
);
806 /* Harvest R and C */
807 rcbits
= be64_to_cpu(hptep
[1]) & (HPTE_R_R
| HPTE_R_C
);
808 *rmapp
|= rcbits
<< KVMPPC_RMAP_RC_SHIFT
;
809 if (rcbits
& HPTE_R_C
)
810 kvmppc_update_rmap_change(rmapp
, psize
);
811 if (rcbits
& ~rev
[i
].guest_rpte
) {
812 rev
[i
].guest_rpte
= ptel
| rcbits
;
813 note_hpte_modification(kvm
, &rev
[i
]);
818 static int kvm_unmap_rmapp(struct kvm
*kvm
, struct kvm_memory_slot
*memslot
,
823 unsigned long *rmapp
;
825 rmapp
= &memslot
->arch
.rmap
[gfn
- memslot
->base_gfn
];
828 if (!(*rmapp
& KVMPPC_RMAP_PRESENT
)) {
834 * To avoid an ABBA deadlock with the HPTE lock bit,
835 * we can't spin on the HPTE lock while holding the
838 i
= *rmapp
& KVMPPC_RMAP_INDEX
;
839 hptep
= (__be64
*) (kvm
->arch
.hpt
.virt
+ (i
<< 4));
840 if (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
)) {
841 /* unlock rmap before spinning on the HPTE lock */
843 while (be64_to_cpu(hptep
[0]) & HPTE_V_HVLOCK
)
848 kvmppc_unmap_hpte(kvm
, i
, rmapp
, gfn
);
850 __unlock_hpte(hptep
, be64_to_cpu(hptep
[0]));
855 int kvm_unmap_hva_hv(struct kvm
*kvm
, unsigned long hva
)
857 hva_handler_fn handler
;
859 handler
= kvm_is_radix(kvm
) ? kvm_unmap_radix
: kvm_unmap_rmapp
;
860 kvm_handle_hva(kvm
, hva
, handler
);
864 int kvm_unmap_hva_range_hv(struct kvm
*kvm
, unsigned long start
, unsigned long end
)
866 hva_handler_fn handler
;
868 handler
= kvm_is_radix(kvm
) ? kvm_unmap_radix
: kvm_unmap_rmapp
;
869 kvm_handle_hva_range(kvm
, start
, end
, handler
);
873 void kvmppc_core_flush_memslot_hv(struct kvm
*kvm
,
874 struct kvm_memory_slot
*memslot
)
878 unsigned long *rmapp
;
880 gfn
= memslot
->base_gfn
;
881 rmapp
= memslot
->arch
.rmap
;
882 for (n
= memslot
->npages
; n
; --n
, ++gfn
) {
883 if (kvm_is_radix(kvm
)) {
884 kvm_unmap_radix(kvm
, memslot
, gfn
);
888 * Testing the present bit without locking is OK because
889 * the memslot has been marked invalid already, and hence
890 * no new HPTEs referencing this page can be created,
891 * thus the present bit can't go from 0 to 1.
893 if (*rmapp
& KVMPPC_RMAP_PRESENT
)
894 kvm_unmap_rmapp(kvm
, memslot
, gfn
);
899 static int kvm_age_rmapp(struct kvm
*kvm
, struct kvm_memory_slot
*memslot
,
902 struct revmap_entry
*rev
= kvm
->arch
.hpt
.rev
;
903 unsigned long head
, i
, j
;
906 unsigned long *rmapp
;
908 rmapp
= &memslot
->arch
.rmap
[gfn
- memslot
->base_gfn
];
911 if (*rmapp
& KVMPPC_RMAP_REFERENCED
) {
912 *rmapp
&= ~KVMPPC_RMAP_REFERENCED
;
915 if (!(*rmapp
& KVMPPC_RMAP_PRESENT
)) {
920 i
= head
= *rmapp
& KVMPPC_RMAP_INDEX
;
922 hptep
= (__be64
*) (kvm
->arch
.hpt
.virt
+ (i
<< 4));
925 /* If this HPTE isn't referenced, ignore it */
926 if (!(be64_to_cpu(hptep
[1]) & HPTE_R_R
))
929 if (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
)) {
930 /* unlock rmap before spinning on the HPTE lock */
932 while (be64_to_cpu(hptep
[0]) & HPTE_V_HVLOCK
)
937 /* Now check and modify the HPTE */
938 if ((be64_to_cpu(hptep
[0]) & HPTE_V_VALID
) &&
939 (be64_to_cpu(hptep
[1]) & HPTE_R_R
)) {
940 kvmppc_clear_ref_hpte(kvm
, hptep
, i
);
941 if (!(rev
[i
].guest_rpte
& HPTE_R_R
)) {
942 rev
[i
].guest_rpte
|= HPTE_R_R
;
943 note_hpte_modification(kvm
, &rev
[i
]);
947 __unlock_hpte(hptep
, be64_to_cpu(hptep
[0]));
948 } while ((i
= j
) != head
);
954 int kvm_age_hva_hv(struct kvm
*kvm
, unsigned long start
, unsigned long end
)
956 hva_handler_fn handler
;
958 handler
= kvm_is_radix(kvm
) ? kvm_age_radix
: kvm_age_rmapp
;
959 return kvm_handle_hva_range(kvm
, start
, end
, handler
);
962 static int kvm_test_age_rmapp(struct kvm
*kvm
, struct kvm_memory_slot
*memslot
,
965 struct revmap_entry
*rev
= kvm
->arch
.hpt
.rev
;
966 unsigned long head
, i
, j
;
969 unsigned long *rmapp
;
971 rmapp
= &memslot
->arch
.rmap
[gfn
- memslot
->base_gfn
];
972 if (*rmapp
& KVMPPC_RMAP_REFERENCED
)
976 if (*rmapp
& KVMPPC_RMAP_REFERENCED
)
979 if (*rmapp
& KVMPPC_RMAP_PRESENT
) {
980 i
= head
= *rmapp
& KVMPPC_RMAP_INDEX
;
982 hp
= (unsigned long *)(kvm
->arch
.hpt
.virt
+ (i
<< 4));
984 if (be64_to_cpu(hp
[1]) & HPTE_R_R
)
986 } while ((i
= j
) != head
);
995 int kvm_test_age_hva_hv(struct kvm
*kvm
, unsigned long hva
)
997 hva_handler_fn handler
;
999 handler
= kvm_is_radix(kvm
) ? kvm_test_age_radix
: kvm_test_age_rmapp
;
1000 return kvm_handle_hva(kvm
, hva
, handler
);
1003 void kvm_set_spte_hva_hv(struct kvm
*kvm
, unsigned long hva
, pte_t pte
)
1005 hva_handler_fn handler
;
1007 handler
= kvm_is_radix(kvm
) ? kvm_unmap_radix
: kvm_unmap_rmapp
;
1008 kvm_handle_hva(kvm
, hva
, handler
);
1011 static int vcpus_running(struct kvm
*kvm
)
1013 return atomic_read(&kvm
->arch
.vcpus_running
) != 0;
1017 * Returns the number of system pages that are dirty.
1018 * This can be more than 1 if we find a huge-page HPTE.
1020 static int kvm_test_clear_dirty_npages(struct kvm
*kvm
, unsigned long *rmapp
)
1022 struct revmap_entry
*rev
= kvm
->arch
.hpt
.rev
;
1023 unsigned long head
, i
, j
;
1027 int npages_dirty
= 0;
1031 if (*rmapp
& KVMPPC_RMAP_CHANGED
) {
1032 long change_order
= (*rmapp
& KVMPPC_RMAP_CHG_ORDER
)
1033 >> KVMPPC_RMAP_CHG_SHIFT
;
1034 *rmapp
&= ~(KVMPPC_RMAP_CHANGED
| KVMPPC_RMAP_CHG_ORDER
);
1036 if (change_order
> PAGE_SHIFT
)
1037 npages_dirty
= 1ul << (change_order
- PAGE_SHIFT
);
1039 if (!(*rmapp
& KVMPPC_RMAP_PRESENT
)) {
1041 return npages_dirty
;
1044 i
= head
= *rmapp
& KVMPPC_RMAP_INDEX
;
1046 unsigned long hptep1
;
1047 hptep
= (__be64
*) (kvm
->arch
.hpt
.virt
+ (i
<< 4));
1051 * Checking the C (changed) bit here is racy since there
1052 * is no guarantee about when the hardware writes it back.
1053 * If the HPTE is not writable then it is stable since the
1054 * page can't be written to, and we would have done a tlbie
1055 * (which forces the hardware to complete any writeback)
1056 * when making the HPTE read-only.
1057 * If vcpus are running then this call is racy anyway
1058 * since the page could get dirtied subsequently, so we
1059 * expect there to be a further call which would pick up
1060 * any delayed C bit writeback.
1061 * Otherwise we need to do the tlbie even if C==0 in
1062 * order to pick up any delayed writeback of C.
1064 hptep1
= be64_to_cpu(hptep
[1]);
1065 if (!(hptep1
& HPTE_R_C
) &&
1066 (!hpte_is_writable(hptep1
) || vcpus_running(kvm
)))
1069 if (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
)) {
1070 /* unlock rmap before spinning on the HPTE lock */
1072 while (hptep
[0] & cpu_to_be64(HPTE_V_HVLOCK
))
1077 /* Now check and modify the HPTE */
1078 if (!(hptep
[0] & cpu_to_be64(HPTE_V_VALID
))) {
1079 __unlock_hpte(hptep
, be64_to_cpu(hptep
[0]));
1083 /* need to make it temporarily absent so C is stable */
1084 hptep
[0] |= cpu_to_be64(HPTE_V_ABSENT
);
1085 kvmppc_invalidate_hpte(kvm
, hptep
, i
);
1086 v
= be64_to_cpu(hptep
[0]);
1087 r
= be64_to_cpu(hptep
[1]);
1089 hptep
[1] = cpu_to_be64(r
& ~HPTE_R_C
);
1090 if (!(rev
[i
].guest_rpte
& HPTE_R_C
)) {
1091 rev
[i
].guest_rpte
|= HPTE_R_C
;
1092 note_hpte_modification(kvm
, &rev
[i
]);
1094 n
= hpte_page_size(v
, r
);
1095 n
= (n
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
1096 if (n
> npages_dirty
)
1100 v
&= ~HPTE_V_ABSENT
;
1102 __unlock_hpte(hptep
, v
);
1103 } while ((i
= j
) != head
);
1106 return npages_dirty
;
1109 void kvmppc_harvest_vpa_dirty(struct kvmppc_vpa
*vpa
,
1110 struct kvm_memory_slot
*memslot
,
1115 if (!vpa
->dirty
|| !vpa
->pinned_addr
)
1117 gfn
= vpa
->gpa
>> PAGE_SHIFT
;
1118 if (gfn
< memslot
->base_gfn
||
1119 gfn
>= memslot
->base_gfn
+ memslot
->npages
)
1124 __set_bit_le(gfn
- memslot
->base_gfn
, map
);
1127 long kvmppc_hv_get_dirty_log_hpt(struct kvm
*kvm
,
1128 struct kvm_memory_slot
*memslot
, unsigned long *map
)
1131 unsigned long *rmapp
;
1134 rmapp
= memslot
->arch
.rmap
;
1135 for (i
= 0; i
< memslot
->npages
; ++i
) {
1136 int npages
= kvm_test_clear_dirty_npages(kvm
, rmapp
);
1138 * Note that if npages > 0 then i must be a multiple of npages,
1139 * since we always put huge-page HPTEs in the rmap chain
1140 * corresponding to their page base address.
1143 for (j
= i
; npages
; ++j
, --npages
)
1144 __set_bit_le(j
, map
);
1151 void *kvmppc_pin_guest_page(struct kvm
*kvm
, unsigned long gpa
,
1152 unsigned long *nb_ret
)
1154 struct kvm_memory_slot
*memslot
;
1155 unsigned long gfn
= gpa
>> PAGE_SHIFT
;
1156 struct page
*page
, *pages
[1];
1158 unsigned long hva
, offset
;
1161 srcu_idx
= srcu_read_lock(&kvm
->srcu
);
1162 memslot
= gfn_to_memslot(kvm
, gfn
);
1163 if (!memslot
|| (memslot
->flags
& KVM_MEMSLOT_INVALID
))
1165 hva
= gfn_to_hva_memslot(memslot
, gfn
);
1166 npages
= get_user_pages_fast(hva
, 1, 1, pages
);
1170 srcu_read_unlock(&kvm
->srcu
, srcu_idx
);
1172 offset
= gpa
& (PAGE_SIZE
- 1);
1174 *nb_ret
= PAGE_SIZE
- offset
;
1175 return page_address(page
) + offset
;
1178 srcu_read_unlock(&kvm
->srcu
, srcu_idx
);
1182 void kvmppc_unpin_guest_page(struct kvm
*kvm
, void *va
, unsigned long gpa
,
1185 struct page
*page
= virt_to_page(va
);
1186 struct kvm_memory_slot
*memslot
;
1188 unsigned long *rmap
;
1196 /* We need to mark this page dirty in the rmap chain */
1197 gfn
= gpa
>> PAGE_SHIFT
;
1198 srcu_idx
= srcu_read_lock(&kvm
->srcu
);
1199 memslot
= gfn_to_memslot(kvm
, gfn
);
1201 if (!kvm_is_radix(kvm
)) {
1202 rmap
= &memslot
->arch
.rmap
[gfn
- memslot
->base_gfn
];
1204 *rmap
|= KVMPPC_RMAP_CHANGED
;
1206 } else if (memslot
->dirty_bitmap
) {
1207 mark_page_dirty(kvm
, gfn
);
1210 srcu_read_unlock(&kvm
->srcu
, srcu_idx
);
1216 static int resize_hpt_allocate(struct kvm_resize_hpt
*resize
)
1220 rc
= kvmppc_allocate_hpt(&resize
->hpt
, resize
->order
);
1224 resize_hpt_debug(resize
, "resize_hpt_allocate(): HPT @ 0x%lx\n",
1230 static unsigned long resize_hpt_rehash_hpte(struct kvm_resize_hpt
*resize
,
1233 struct kvm
*kvm
= resize
->kvm
;
1234 struct kvm_hpt_info
*old
= &kvm
->arch
.hpt
;
1235 struct kvm_hpt_info
*new = &resize
->hpt
;
1236 unsigned long old_hash_mask
= (1ULL << (old
->order
- 7)) - 1;
1237 unsigned long new_hash_mask
= (1ULL << (new->order
- 7)) - 1;
1238 __be64
*hptep
, *new_hptep
;
1239 unsigned long vpte
, rpte
, guest_rpte
;
1241 struct revmap_entry
*rev
;
1242 unsigned long apsize
, psize
, avpn
, pteg
, hash
;
1243 unsigned long new_idx
, new_pteg
, replace_vpte
;
1245 hptep
= (__be64
*)(old
->virt
+ (idx
<< 4));
1247 /* Guest is stopped, so new HPTEs can't be added or faulted
1248 * in, only unmapped or altered by host actions. So, it's
1249 * safe to check this before we take the HPTE lock */
1250 vpte
= be64_to_cpu(hptep
[0]);
1251 if (!(vpte
& HPTE_V_VALID
) && !(vpte
& HPTE_V_ABSENT
))
1252 return 0; /* nothing to do */
1254 while (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
))
1257 vpte
= be64_to_cpu(hptep
[0]);
1260 if (!(vpte
& HPTE_V_VALID
) && !(vpte
& HPTE_V_ABSENT
))
1265 rev
= &old
->rev
[idx
];
1266 guest_rpte
= rev
->guest_rpte
;
1269 apsize
= hpte_page_size(vpte
, guest_rpte
);
1273 if (vpte
& HPTE_V_VALID
) {
1274 unsigned long gfn
= hpte_rpn(guest_rpte
, apsize
);
1275 int srcu_idx
= srcu_read_lock(&kvm
->srcu
);
1276 struct kvm_memory_slot
*memslot
=
1277 __gfn_to_memslot(kvm_memslots(kvm
), gfn
);
1280 unsigned long *rmapp
;
1281 rmapp
= &memslot
->arch
.rmap
[gfn
- memslot
->base_gfn
];
1284 kvmppc_unmap_hpte(kvm
, idx
, rmapp
, gfn
);
1288 srcu_read_unlock(&kvm
->srcu
, srcu_idx
);
1291 /* Reload PTE after unmap */
1292 vpte
= be64_to_cpu(hptep
[0]);
1294 BUG_ON(vpte
& HPTE_V_VALID
);
1295 BUG_ON(!(vpte
& HPTE_V_ABSENT
));
1298 if (!(vpte
& HPTE_V_BOLTED
))
1301 rpte
= be64_to_cpu(hptep
[1]);
1302 psize
= hpte_base_page_size(vpte
, rpte
);
1303 avpn
= HPTE_V_AVPN_VAL(vpte
) & ~((psize
- 1) >> 23);
1304 pteg
= idx
/ HPTES_PER_GROUP
;
1305 if (vpte
& HPTE_V_SECONDARY
)
1308 if (!(vpte
& HPTE_V_1TB_SEG
)) {
1309 unsigned long offset
, vsid
;
1311 /* We only have 28 - 23 bits of offset in avpn */
1312 offset
= (avpn
& 0x1f) << 23;
1314 /* We can find more bits from the pteg value */
1315 if (psize
< (1ULL << 23))
1316 offset
|= ((vsid
^ pteg
) & old_hash_mask
) * psize
;
1318 hash
= vsid
^ (offset
/ psize
);
1320 unsigned long offset
, vsid
;
1322 /* We only have 40 - 23 bits of seg_off in avpn */
1323 offset
= (avpn
& 0x1ffff) << 23;
1325 if (psize
< (1ULL << 23))
1326 offset
|= ((vsid
^ (vsid
<< 25) ^ pteg
) & old_hash_mask
) * psize
;
1328 hash
= vsid
^ (vsid
<< 25) ^ (offset
/ psize
);
1331 new_pteg
= hash
& new_hash_mask
;
1332 if (vpte
& HPTE_V_SECONDARY
) {
1333 BUG_ON(~pteg
!= (hash
& old_hash_mask
));
1334 new_pteg
= ~new_pteg
;
1336 BUG_ON(pteg
!= (hash
& old_hash_mask
));
1339 new_idx
= new_pteg
* HPTES_PER_GROUP
+ (idx
% HPTES_PER_GROUP
);
1340 new_hptep
= (__be64
*)(new->virt
+ (new_idx
<< 4));
1342 replace_vpte
= be64_to_cpu(new_hptep
[0]);
1344 if (replace_vpte
& (HPTE_V_VALID
| HPTE_V_ABSENT
)) {
1345 BUG_ON(new->order
>= old
->order
);
1347 if (replace_vpte
& HPTE_V_BOLTED
) {
1348 if (vpte
& HPTE_V_BOLTED
)
1349 /* Bolted collision, nothing we can do */
1351 /* Discard the new HPTE */
1355 /* Discard the previous HPTE */
1358 new_hptep
[1] = cpu_to_be64(rpte
);
1359 new->rev
[new_idx
].guest_rpte
= guest_rpte
;
1360 /* No need for a barrier, since new HPT isn't active */
1361 new_hptep
[0] = cpu_to_be64(vpte
);
1362 unlock_hpte(new_hptep
, vpte
);
1365 unlock_hpte(hptep
, vpte
);
1369 static int resize_hpt_rehash(struct kvm_resize_hpt
*resize
)
1371 struct kvm
*kvm
= resize
->kvm
;
1376 * resize_hpt_rehash_hpte() doesn't handle the new-format HPTEs
1377 * that POWER9 uses, and could well hit a BUG_ON on POWER9.
1379 if (cpu_has_feature(CPU_FTR_ARCH_300
))
1381 for (i
= 0; i
< kvmppc_hpt_npte(&kvm
->arch
.hpt
); i
++) {
1382 rc
= resize_hpt_rehash_hpte(resize
, i
);
1390 static void resize_hpt_pivot(struct kvm_resize_hpt
*resize
)
1392 struct kvm
*kvm
= resize
->kvm
;
1393 struct kvm_hpt_info hpt_tmp
;
1395 /* Exchange the pending tables in the resize structure with
1396 * the active tables */
1398 resize_hpt_debug(resize
, "resize_hpt_pivot()\n");
1400 spin_lock(&kvm
->mmu_lock
);
1401 asm volatile("ptesync" : : : "memory");
1403 hpt_tmp
= kvm
->arch
.hpt
;
1404 kvmppc_set_hpt(kvm
, &resize
->hpt
);
1405 resize
->hpt
= hpt_tmp
;
1407 spin_unlock(&kvm
->mmu_lock
);
1409 synchronize_srcu_expedited(&kvm
->srcu
);
1411 resize_hpt_debug(resize
, "resize_hpt_pivot() done\n");
1414 static void resize_hpt_release(struct kvm
*kvm
, struct kvm_resize_hpt
*resize
)
1416 BUG_ON(kvm
->arch
.resize_hpt
!= resize
);
1421 if (resize
->hpt
.virt
)
1422 kvmppc_free_hpt(&resize
->hpt
);
1424 kvm
->arch
.resize_hpt
= NULL
;
1428 static void resize_hpt_prepare_work(struct work_struct
*work
)
1430 struct kvm_resize_hpt
*resize
= container_of(work
,
1431 struct kvm_resize_hpt
,
1433 struct kvm
*kvm
= resize
->kvm
;
1436 resize_hpt_debug(resize
, "resize_hpt_prepare_work(): order = %d\n",
1439 err
= resize_hpt_allocate(resize
);
1441 mutex_lock(&kvm
->lock
);
1443 resize
->error
= err
;
1444 resize
->prepare_done
= true;
1446 mutex_unlock(&kvm
->lock
);
1449 long kvm_vm_ioctl_resize_hpt_prepare(struct kvm
*kvm
,
1450 struct kvm_ppc_resize_hpt
*rhpt
)
1452 unsigned long flags
= rhpt
->flags
;
1453 unsigned long shift
= rhpt
->shift
;
1454 struct kvm_resize_hpt
*resize
;
1460 if (shift
&& ((shift
< 18) || (shift
> 46)))
1463 mutex_lock(&kvm
->lock
);
1465 resize
= kvm
->arch
.resize_hpt
;
1468 if (resize
->order
== shift
) {
1469 /* Suitable resize in progress */
1470 if (resize
->prepare_done
) {
1471 ret
= resize
->error
;
1473 resize_hpt_release(kvm
, resize
);
1475 ret
= 100; /* estimated time in ms */
1481 /* not suitable, cancel it */
1482 resize_hpt_release(kvm
, resize
);
1487 goto out
; /* nothing to do */
1489 /* start new resize */
1491 resize
= kzalloc(sizeof(*resize
), GFP_KERNEL
);
1496 resize
->order
= shift
;
1498 INIT_WORK(&resize
->work
, resize_hpt_prepare_work
);
1499 kvm
->arch
.resize_hpt
= resize
;
1501 schedule_work(&resize
->work
);
1503 ret
= 100; /* estimated time in ms */
1506 mutex_unlock(&kvm
->lock
);
1510 static void resize_hpt_boot_vcpu(void *opaque
)
1512 /* Nothing to do, just force a KVM exit */
1515 long kvm_vm_ioctl_resize_hpt_commit(struct kvm
*kvm
,
1516 struct kvm_ppc_resize_hpt
*rhpt
)
1518 unsigned long flags
= rhpt
->flags
;
1519 unsigned long shift
= rhpt
->shift
;
1520 struct kvm_resize_hpt
*resize
;
1526 if (shift
&& ((shift
< 18) || (shift
> 46)))
1529 mutex_lock(&kvm
->lock
);
1531 resize
= kvm
->arch
.resize_hpt
;
1533 /* This shouldn't be possible */
1535 if (WARN_ON(!kvm
->arch
.hpte_setup_done
))
1538 /* Stop VCPUs from running while we mess with the HPT */
1539 kvm
->arch
.hpte_setup_done
= 0;
1542 /* Boot all CPUs out of the guest so they re-read
1543 * hpte_setup_done */
1544 on_each_cpu(resize_hpt_boot_vcpu
, NULL
, 1);
1547 if (!resize
|| (resize
->order
!= shift
))
1551 if (!resize
->prepare_done
)
1554 ret
= resize
->error
;
1558 ret
= resize_hpt_rehash(resize
);
1562 resize_hpt_pivot(resize
);
1565 /* Let VCPUs run again */
1566 kvm
->arch
.hpte_setup_done
= 1;
1569 resize_hpt_release(kvm
, resize
);
1570 mutex_unlock(&kvm
->lock
);
1575 * Functions for reading and writing the hash table via reads and
1576 * writes on a file descriptor.
1578 * Reads return the guest view of the hash table, which has to be
1579 * pieced together from the real hash table and the guest_rpte
1580 * values in the revmap array.
1582 * On writes, each HPTE written is considered in turn, and if it
1583 * is valid, it is written to the HPT as if an H_ENTER with the
1584 * exact flag set was done. When the invalid count is non-zero
1585 * in the header written to the stream, the kernel will make
1586 * sure that that many HPTEs are invalid, and invalidate them
1590 struct kvm_htab_ctx
{
1591 unsigned long index
;
1592 unsigned long flags
;
1597 #define HPTE_SIZE (2 * sizeof(unsigned long))
1600 * Returns 1 if this HPT entry has been modified or has pending
1603 static int hpte_dirty(struct revmap_entry
*revp
, __be64
*hptp
)
1605 unsigned long rcbits_unset
;
1607 if (revp
->guest_rpte
& HPTE_GR_MODIFIED
)
1610 /* Also need to consider changes in reference and changed bits */
1611 rcbits_unset
= ~revp
->guest_rpte
& (HPTE_R_R
| HPTE_R_C
);
1612 if ((be64_to_cpu(hptp
[0]) & HPTE_V_VALID
) &&
1613 (be64_to_cpu(hptp
[1]) & rcbits_unset
))
1619 static long record_hpte(unsigned long flags
, __be64
*hptp
,
1620 unsigned long *hpte
, struct revmap_entry
*revp
,
1621 int want_valid
, int first_pass
)
1623 unsigned long v
, r
, hr
;
1624 unsigned long rcbits_unset
;
1628 /* Unmodified entries are uninteresting except on the first pass */
1629 dirty
= hpte_dirty(revp
, hptp
);
1630 if (!first_pass
&& !dirty
)
1634 if (be64_to_cpu(hptp
[0]) & (HPTE_V_VALID
| HPTE_V_ABSENT
)) {
1636 if ((flags
& KVM_GET_HTAB_BOLTED_ONLY
) &&
1637 !(be64_to_cpu(hptp
[0]) & HPTE_V_BOLTED
))
1640 if (valid
!= want_valid
)
1644 if (valid
|| dirty
) {
1645 /* lock the HPTE so it's stable and read it */
1647 while (!try_lock_hpte(hptp
, HPTE_V_HVLOCK
))
1649 v
= be64_to_cpu(hptp
[0]);
1650 hr
= be64_to_cpu(hptp
[1]);
1651 if (cpu_has_feature(CPU_FTR_ARCH_300
)) {
1652 v
= hpte_new_to_old_v(v
, hr
);
1653 hr
= hpte_new_to_old_r(hr
);
1656 /* re-evaluate valid and dirty from synchronized HPTE value */
1657 valid
= !!(v
& HPTE_V_VALID
);
1658 dirty
= !!(revp
->guest_rpte
& HPTE_GR_MODIFIED
);
1660 /* Harvest R and C into guest view if necessary */
1661 rcbits_unset
= ~revp
->guest_rpte
& (HPTE_R_R
| HPTE_R_C
);
1662 if (valid
&& (rcbits_unset
& hr
)) {
1663 revp
->guest_rpte
|= (hr
&
1664 (HPTE_R_R
| HPTE_R_C
)) | HPTE_GR_MODIFIED
;
1668 if (v
& HPTE_V_ABSENT
) {
1669 v
&= ~HPTE_V_ABSENT
;
1673 if ((flags
& KVM_GET_HTAB_BOLTED_ONLY
) && !(v
& HPTE_V_BOLTED
))
1676 r
= revp
->guest_rpte
;
1677 /* only clear modified if this is the right sort of entry */
1678 if (valid
== want_valid
&& dirty
) {
1679 r
&= ~HPTE_GR_MODIFIED
;
1680 revp
->guest_rpte
= r
;
1682 unlock_hpte(hptp
, be64_to_cpu(hptp
[0]));
1684 if (!(valid
== want_valid
&& (first_pass
|| dirty
)))
1687 hpte
[0] = cpu_to_be64(v
);
1688 hpte
[1] = cpu_to_be64(r
);
1692 static ssize_t
kvm_htab_read(struct file
*file
, char __user
*buf
,
1693 size_t count
, loff_t
*ppos
)
1695 struct kvm_htab_ctx
*ctx
= file
->private_data
;
1696 struct kvm
*kvm
= ctx
->kvm
;
1697 struct kvm_get_htab_header hdr
;
1699 struct revmap_entry
*revp
;
1700 unsigned long i
, nb
, nw
;
1701 unsigned long __user
*lbuf
;
1702 struct kvm_get_htab_header __user
*hptr
;
1703 unsigned long flags
;
1705 unsigned long hpte
[2];
1707 if (!access_ok(VERIFY_WRITE
, buf
, count
))
1710 first_pass
= ctx
->first_pass
;
1714 hptp
= (__be64
*)(kvm
->arch
.hpt
.virt
+ (i
* HPTE_SIZE
));
1715 revp
= kvm
->arch
.hpt
.rev
+ i
;
1716 lbuf
= (unsigned long __user
*)buf
;
1719 while (nb
+ sizeof(hdr
) + HPTE_SIZE
< count
) {
1720 /* Initialize header */
1721 hptr
= (struct kvm_get_htab_header __user
*)buf
;
1726 lbuf
= (unsigned long __user
*)(buf
+ sizeof(hdr
));
1728 /* Skip uninteresting entries, i.e. clean on not-first pass */
1730 while (i
< kvmppc_hpt_npte(&kvm
->arch
.hpt
) &&
1731 !hpte_dirty(revp
, hptp
)) {
1739 /* Grab a series of valid entries */
1740 while (i
< kvmppc_hpt_npte(&kvm
->arch
.hpt
) &&
1741 hdr
.n_valid
< 0xffff &&
1742 nb
+ HPTE_SIZE
< count
&&
1743 record_hpte(flags
, hptp
, hpte
, revp
, 1, first_pass
)) {
1744 /* valid entry, write it out */
1746 if (__put_user(hpte
[0], lbuf
) ||
1747 __put_user(hpte
[1], lbuf
+ 1))
1755 /* Now skip invalid entries while we can */
1756 while (i
< kvmppc_hpt_npte(&kvm
->arch
.hpt
) &&
1757 hdr
.n_invalid
< 0xffff &&
1758 record_hpte(flags
, hptp
, hpte
, revp
, 0, first_pass
)) {
1759 /* found an invalid entry */
1766 if (hdr
.n_valid
|| hdr
.n_invalid
) {
1767 /* write back the header */
1768 if (__copy_to_user(hptr
, &hdr
, sizeof(hdr
)))
1771 buf
= (char __user
*)lbuf
;
1776 /* Check if we've wrapped around the hash table */
1777 if (i
>= kvmppc_hpt_npte(&kvm
->arch
.hpt
)) {
1779 ctx
->first_pass
= 0;
1789 static ssize_t
kvm_htab_write(struct file
*file
, const char __user
*buf
,
1790 size_t count
, loff_t
*ppos
)
1792 struct kvm_htab_ctx
*ctx
= file
->private_data
;
1793 struct kvm
*kvm
= ctx
->kvm
;
1794 struct kvm_get_htab_header hdr
;
1797 unsigned long __user
*lbuf
;
1799 unsigned long tmp
[2];
1804 if (!access_ok(VERIFY_READ
, buf
, count
))
1807 /* lock out vcpus from running while we're doing this */
1808 mutex_lock(&kvm
->lock
);
1809 hpte_setup
= kvm
->arch
.hpte_setup_done
;
1811 kvm
->arch
.hpte_setup_done
= 0; /* temporarily */
1812 /* order hpte_setup_done vs. vcpus_running */
1814 if (atomic_read(&kvm
->arch
.vcpus_running
)) {
1815 kvm
->arch
.hpte_setup_done
= 1;
1816 mutex_unlock(&kvm
->lock
);
1822 for (nb
= 0; nb
+ sizeof(hdr
) <= count
; ) {
1824 if (__copy_from_user(&hdr
, buf
, sizeof(hdr
)))
1828 if (nb
+ hdr
.n_valid
* HPTE_SIZE
> count
)
1836 if (i
>= kvmppc_hpt_npte(&kvm
->arch
.hpt
) ||
1837 i
+ hdr
.n_valid
+ hdr
.n_invalid
> kvmppc_hpt_npte(&kvm
->arch
.hpt
))
1840 hptp
= (__be64
*)(kvm
->arch
.hpt
.virt
+ (i
* HPTE_SIZE
));
1841 lbuf
= (unsigned long __user
*)buf
;
1842 for (j
= 0; j
< hdr
.n_valid
; ++j
) {
1847 if (__get_user(hpte_v
, lbuf
) ||
1848 __get_user(hpte_r
, lbuf
+ 1))
1850 v
= be64_to_cpu(hpte_v
);
1851 r
= be64_to_cpu(hpte_r
);
1853 if (!(v
& HPTE_V_VALID
))
1858 if (be64_to_cpu(hptp
[0]) & (HPTE_V_VALID
| HPTE_V_ABSENT
))
1859 kvmppc_do_h_remove(kvm
, 0, i
, 0, tmp
);
1861 ret
= kvmppc_virtmode_do_h_enter(kvm
, H_EXACT
, i
, v
, r
,
1863 if (ret
!= H_SUCCESS
) {
1864 pr_err("kvm_htab_write ret %ld i=%ld v=%lx "
1865 "r=%lx\n", ret
, i
, v
, r
);
1868 if (!hpte_setup
&& is_vrma_hpte(v
)) {
1869 unsigned long psize
= hpte_base_page_size(v
, r
);
1870 unsigned long senc
= slb_pgsize_encoding(psize
);
1873 kvm
->arch
.vrma_slb_v
= senc
| SLB_VSID_B_1T
|
1874 (VRMA_VSID
<< SLB_VSID_SHIFT_1T
);
1875 lpcr
= senc
<< (LPCR_VRMASD_SH
- 4);
1876 kvmppc_update_lpcr(kvm
, lpcr
, LPCR_VRMASD
);
1883 for (j
= 0; j
< hdr
.n_invalid
; ++j
) {
1884 if (be64_to_cpu(hptp
[0]) & (HPTE_V_VALID
| HPTE_V_ABSENT
))
1885 kvmppc_do_h_remove(kvm
, 0, i
, 0, tmp
);
1893 /* Order HPTE updates vs. hpte_setup_done */
1895 kvm
->arch
.hpte_setup_done
= hpte_setup
;
1896 mutex_unlock(&kvm
->lock
);
1903 static int kvm_htab_release(struct inode
*inode
, struct file
*filp
)
1905 struct kvm_htab_ctx
*ctx
= filp
->private_data
;
1907 filp
->private_data
= NULL
;
1908 if (!(ctx
->flags
& KVM_GET_HTAB_WRITE
))
1909 atomic_dec(&ctx
->kvm
->arch
.hpte_mod_interest
);
1910 kvm_put_kvm(ctx
->kvm
);
1915 static const struct file_operations kvm_htab_fops
= {
1916 .read
= kvm_htab_read
,
1917 .write
= kvm_htab_write
,
1918 .llseek
= default_llseek
,
1919 .release
= kvm_htab_release
,
1922 int kvm_vm_ioctl_get_htab_fd(struct kvm
*kvm
, struct kvm_get_htab_fd
*ghf
)
1925 struct kvm_htab_ctx
*ctx
;
1928 /* reject flags we don't recognize */
1929 if (ghf
->flags
& ~(KVM_GET_HTAB_BOLTED_ONLY
| KVM_GET_HTAB_WRITE
))
1931 ctx
= kzalloc(sizeof(*ctx
), GFP_KERNEL
);
1936 ctx
->index
= ghf
->start_index
;
1937 ctx
->flags
= ghf
->flags
;
1938 ctx
->first_pass
= 1;
1940 rwflag
= (ghf
->flags
& KVM_GET_HTAB_WRITE
) ? O_WRONLY
: O_RDONLY
;
1941 ret
= anon_inode_getfd("kvm-htab", &kvm_htab_fops
, ctx
, rwflag
| O_CLOEXEC
);
1947 if (rwflag
== O_RDONLY
) {
1948 mutex_lock(&kvm
->slots_lock
);
1949 atomic_inc(&kvm
->arch
.hpte_mod_interest
);
1950 /* make sure kvmppc_do_h_enter etc. see the increment */
1951 synchronize_srcu_expedited(&kvm
->srcu
);
1952 mutex_unlock(&kvm
->slots_lock
);
1958 struct debugfs_htab_state
{
1961 unsigned long hpt_index
;
1967 static int debugfs_htab_open(struct inode
*inode
, struct file
*file
)
1969 struct kvm
*kvm
= inode
->i_private
;
1970 struct debugfs_htab_state
*p
;
1972 p
= kzalloc(sizeof(*p
), GFP_KERNEL
);
1978 mutex_init(&p
->mutex
);
1979 file
->private_data
= p
;
1981 return nonseekable_open(inode
, file
);
1984 static int debugfs_htab_release(struct inode
*inode
, struct file
*file
)
1986 struct debugfs_htab_state
*p
= file
->private_data
;
1988 kvm_put_kvm(p
->kvm
);
1993 static ssize_t
debugfs_htab_read(struct file
*file
, char __user
*buf
,
1994 size_t len
, loff_t
*ppos
)
1996 struct debugfs_htab_state
*p
= file
->private_data
;
1999 unsigned long v
, hr
, gr
;
2003 ret
= mutex_lock_interruptible(&p
->mutex
);
2007 if (p
->chars_left
) {
2011 r
= copy_to_user(buf
, p
->buf
+ p
->buf_index
, n
);
2027 hptp
= (__be64
*)(kvm
->arch
.hpt
.virt
+ (i
* HPTE_SIZE
));
2028 for (; len
!= 0 && i
< kvmppc_hpt_npte(&kvm
->arch
.hpt
);
2030 if (!(be64_to_cpu(hptp
[0]) & (HPTE_V_VALID
| HPTE_V_ABSENT
)))
2033 /* lock the HPTE so it's stable and read it */
2035 while (!try_lock_hpte(hptp
, HPTE_V_HVLOCK
))
2037 v
= be64_to_cpu(hptp
[0]) & ~HPTE_V_HVLOCK
;
2038 hr
= be64_to_cpu(hptp
[1]);
2039 gr
= kvm
->arch
.hpt
.rev
[i
].guest_rpte
;
2040 unlock_hpte(hptp
, v
);
2043 if (!(v
& (HPTE_V_VALID
| HPTE_V_ABSENT
)))
2046 n
= scnprintf(p
->buf
, sizeof(p
->buf
),
2047 "%6lx %.16lx %.16lx %.16lx\n",
2052 r
= copy_to_user(buf
, p
->buf
, n
);
2068 mutex_unlock(&p
->mutex
);
2072 static ssize_t
debugfs_htab_write(struct file
*file
, const char __user
*buf
,
2073 size_t len
, loff_t
*ppos
)
2078 static const struct file_operations debugfs_htab_fops
= {
2079 .owner
= THIS_MODULE
,
2080 .open
= debugfs_htab_open
,
2081 .release
= debugfs_htab_release
,
2082 .read
= debugfs_htab_read
,
2083 .write
= debugfs_htab_write
,
2084 .llseek
= generic_file_llseek
,
2087 void kvmppc_mmu_debugfs_init(struct kvm
*kvm
)
2089 kvm
->arch
.htab_dentry
= debugfs_create_file("htab", 0400,
2090 kvm
->arch
.debugfs_dir
, kvm
,
2091 &debugfs_htab_fops
);
2094 void kvmppc_mmu_book3s_hv_init(struct kvm_vcpu
*vcpu
)
2096 struct kvmppc_mmu
*mmu
= &vcpu
->arch
.mmu
;
2098 vcpu
->arch
.slb_nr
= 32; /* POWER7/POWER8 */
2100 if (kvm_is_radix(vcpu
->kvm
))
2101 mmu
->xlate
= kvmppc_mmu_radix_xlate
;
2103 mmu
->xlate
= kvmppc_mmu_book3s_64_hv_xlate
;
2104 mmu
->reset_msr
= kvmppc_mmu_book3s_64_hv_reset_msr
;
2106 vcpu
->arch
.hflags
|= BOOK3S_HFLAG_SLB
;