2 * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved.
5 * Alexander Graf <agraf@suse.de>
6 * Kevin Wolf <mail@kevin-wolf.de>
7 * Paul Mackerras <paulus@samba.org>
10 * Functions relating to running KVM on Book 3S processors where
11 * we don't have access to hypervisor mode, and we run the guest
12 * in problem state (user mode).
14 * This file is derived from arch/powerpc/kvm/44x.c,
15 * by Hollis Blanchard <hollisb@us.ibm.com>.
17 * This program is free software; you can redistribute it and/or modify
18 * it under the terms of the GNU General Public License, version 2, as
19 * published by the Free Software Foundation.
22 #include <linux/kvm_host.h>
23 #include <linux/export.h>
24 #include <linux/err.h>
25 #include <linux/slab.h>
28 #include <asm/cputable.h>
29 #include <asm/cacheflush.h>
30 #include <asm/tlbflush.h>
31 #include <linux/uaccess.h>
33 #include <asm/kvm_ppc.h>
34 #include <asm/kvm_book3s.h>
35 #include <asm/mmu_context.h>
36 #include <asm/switch_to.h>
37 #include <asm/firmware.h>
38 #include <asm/setup.h>
39 #include <linux/gfp.h>
40 #include <linux/sched.h>
41 #include <linux/vmalloc.h>
42 #include <linux/highmem.h>
43 #include <linux/module.h>
44 #include <linux/miscdevice.h>
48 #define CREATE_TRACE_POINTS
51 /* #define EXIT_DEBUG */
52 /* #define DEBUG_EXT */
54 static int kvmppc_handle_ext(struct kvm_vcpu
*vcpu
, unsigned int exit_nr
,
56 static void kvmppc_giveup_fac(struct kvm_vcpu
*vcpu
, ulong fac
);
58 /* Some compatibility defines */
59 #ifdef CONFIG_PPC_BOOK3S_32
60 #define MSR_USER32 MSR_USER
61 #define MSR_USER64 MSR_USER
62 #define HW_PAGE_SIZE PAGE_SIZE
65 static bool kvmppc_is_split_real(struct kvm_vcpu
*vcpu
)
67 ulong msr
= kvmppc_get_msr(vcpu
);
68 return (msr
& (MSR_IR
|MSR_DR
)) == MSR_DR
;
71 static void kvmppc_fixup_split_real(struct kvm_vcpu
*vcpu
)
73 ulong msr
= kvmppc_get_msr(vcpu
);
74 ulong pc
= kvmppc_get_pc(vcpu
);
76 /* We are in DR only split real mode */
77 if ((msr
& (MSR_IR
|MSR_DR
)) != MSR_DR
)
80 /* We have not fixed up the guest already */
81 if (vcpu
->arch
.hflags
& BOOK3S_HFLAG_SPLIT_HACK
)
84 /* The code is in fixupable address space */
85 if (pc
& SPLIT_HACK_MASK
)
88 vcpu
->arch
.hflags
|= BOOK3S_HFLAG_SPLIT_HACK
;
89 kvmppc_set_pc(vcpu
, pc
| SPLIT_HACK_OFFS
);
92 void kvmppc_unfixup_split_real(struct kvm_vcpu
*vcpu
);
94 static void kvmppc_core_vcpu_load_pr(struct kvm_vcpu
*vcpu
, int cpu
)
96 #ifdef CONFIG_PPC_BOOK3S_64
97 struct kvmppc_book3s_shadow_vcpu
*svcpu
= svcpu_get(vcpu
);
98 memcpy(svcpu
->slb
, to_book3s(vcpu
)->slb_shadow
, sizeof(svcpu
->slb
));
99 svcpu
->slb_max
= to_book3s(vcpu
)->slb_shadow_max
;
104 /* Disable AIL if supported */
105 if (cpu_has_feature(CPU_FTR_HVMODE
) &&
106 cpu_has_feature(CPU_FTR_ARCH_207S
))
107 mtspr(SPRN_LPCR
, mfspr(SPRN_LPCR
) & ~LPCR_AIL
);
109 vcpu
->cpu
= smp_processor_id();
110 #ifdef CONFIG_PPC_BOOK3S_32
111 current
->thread
.kvm_shadow_vcpu
= vcpu
->arch
.shadow_vcpu
;
114 if (kvmppc_is_split_real(vcpu
))
115 kvmppc_fixup_split_real(vcpu
);
118 static void kvmppc_core_vcpu_put_pr(struct kvm_vcpu
*vcpu
)
120 #ifdef CONFIG_PPC_BOOK3S_64
121 struct kvmppc_book3s_shadow_vcpu
*svcpu
= svcpu_get(vcpu
);
123 kvmppc_copy_from_svcpu(vcpu
, svcpu
);
125 memcpy(to_book3s(vcpu
)->slb_shadow
, svcpu
->slb
, sizeof(svcpu
->slb
));
126 to_book3s(vcpu
)->slb_shadow_max
= svcpu
->slb_max
;
130 if (kvmppc_is_split_real(vcpu
))
131 kvmppc_unfixup_split_real(vcpu
);
133 kvmppc_giveup_ext(vcpu
, MSR_FP
| MSR_VEC
| MSR_VSX
);
134 kvmppc_giveup_fac(vcpu
, FSCR_TAR_LG
);
136 /* Enable AIL if supported */
137 if (cpu_has_feature(CPU_FTR_HVMODE
) &&
138 cpu_has_feature(CPU_FTR_ARCH_207S
))
139 mtspr(SPRN_LPCR
, mfspr(SPRN_LPCR
) | LPCR_AIL_3
);
144 /* Copy data needed by real-mode code from vcpu to shadow vcpu */
145 void kvmppc_copy_to_svcpu(struct kvmppc_book3s_shadow_vcpu
*svcpu
,
146 struct kvm_vcpu
*vcpu
)
148 svcpu
->gpr
[0] = vcpu
->arch
.gpr
[0];
149 svcpu
->gpr
[1] = vcpu
->arch
.gpr
[1];
150 svcpu
->gpr
[2] = vcpu
->arch
.gpr
[2];
151 svcpu
->gpr
[3] = vcpu
->arch
.gpr
[3];
152 svcpu
->gpr
[4] = vcpu
->arch
.gpr
[4];
153 svcpu
->gpr
[5] = vcpu
->arch
.gpr
[5];
154 svcpu
->gpr
[6] = vcpu
->arch
.gpr
[6];
155 svcpu
->gpr
[7] = vcpu
->arch
.gpr
[7];
156 svcpu
->gpr
[8] = vcpu
->arch
.gpr
[8];
157 svcpu
->gpr
[9] = vcpu
->arch
.gpr
[9];
158 svcpu
->gpr
[10] = vcpu
->arch
.gpr
[10];
159 svcpu
->gpr
[11] = vcpu
->arch
.gpr
[11];
160 svcpu
->gpr
[12] = vcpu
->arch
.gpr
[12];
161 svcpu
->gpr
[13] = vcpu
->arch
.gpr
[13];
162 svcpu
->cr
= vcpu
->arch
.cr
;
163 svcpu
->xer
= vcpu
->arch
.xer
;
164 svcpu
->ctr
= vcpu
->arch
.ctr
;
165 svcpu
->lr
= vcpu
->arch
.lr
;
166 svcpu
->pc
= vcpu
->arch
.pc
;
167 #ifdef CONFIG_PPC_BOOK3S_64
168 svcpu
->shadow_fscr
= vcpu
->arch
.shadow_fscr
;
171 * Now also save the current time base value. We use this
172 * to find the guest purr and spurr value.
174 vcpu
->arch
.entry_tb
= get_tb();
175 vcpu
->arch
.entry_vtb
= get_vtb();
176 if (cpu_has_feature(CPU_FTR_ARCH_207S
))
177 vcpu
->arch
.entry_ic
= mfspr(SPRN_IC
);
178 svcpu
->in_use
= true;
181 /* Copy data touched by real-mode code from shadow vcpu back to vcpu */
182 void kvmppc_copy_from_svcpu(struct kvm_vcpu
*vcpu
,
183 struct kvmppc_book3s_shadow_vcpu
*svcpu
)
186 * vcpu_put would just call us again because in_use hasn't
192 * Maybe we were already preempted and synced the svcpu from
193 * our preempt notifiers. Don't bother touching this svcpu then.
198 vcpu
->arch
.gpr
[0] = svcpu
->gpr
[0];
199 vcpu
->arch
.gpr
[1] = svcpu
->gpr
[1];
200 vcpu
->arch
.gpr
[2] = svcpu
->gpr
[2];
201 vcpu
->arch
.gpr
[3] = svcpu
->gpr
[3];
202 vcpu
->arch
.gpr
[4] = svcpu
->gpr
[4];
203 vcpu
->arch
.gpr
[5] = svcpu
->gpr
[5];
204 vcpu
->arch
.gpr
[6] = svcpu
->gpr
[6];
205 vcpu
->arch
.gpr
[7] = svcpu
->gpr
[7];
206 vcpu
->arch
.gpr
[8] = svcpu
->gpr
[8];
207 vcpu
->arch
.gpr
[9] = svcpu
->gpr
[9];
208 vcpu
->arch
.gpr
[10] = svcpu
->gpr
[10];
209 vcpu
->arch
.gpr
[11] = svcpu
->gpr
[11];
210 vcpu
->arch
.gpr
[12] = svcpu
->gpr
[12];
211 vcpu
->arch
.gpr
[13] = svcpu
->gpr
[13];
212 vcpu
->arch
.cr
= svcpu
->cr
;
213 vcpu
->arch
.xer
= svcpu
->xer
;
214 vcpu
->arch
.ctr
= svcpu
->ctr
;
215 vcpu
->arch
.lr
= svcpu
->lr
;
216 vcpu
->arch
.pc
= svcpu
->pc
;
217 vcpu
->arch
.shadow_srr1
= svcpu
->shadow_srr1
;
218 vcpu
->arch
.fault_dar
= svcpu
->fault_dar
;
219 vcpu
->arch
.fault_dsisr
= svcpu
->fault_dsisr
;
220 vcpu
->arch
.last_inst
= svcpu
->last_inst
;
221 #ifdef CONFIG_PPC_BOOK3S_64
222 vcpu
->arch
.shadow_fscr
= svcpu
->shadow_fscr
;
225 * Update purr and spurr using time base on exit.
227 vcpu
->arch
.purr
+= get_tb() - vcpu
->arch
.entry_tb
;
228 vcpu
->arch
.spurr
+= get_tb() - vcpu
->arch
.entry_tb
;
229 to_book3s(vcpu
)->vtb
+= get_vtb() - vcpu
->arch
.entry_vtb
;
230 if (cpu_has_feature(CPU_FTR_ARCH_207S
))
231 vcpu
->arch
.ic
+= mfspr(SPRN_IC
) - vcpu
->arch
.entry_ic
;
232 svcpu
->in_use
= false;
238 static int kvmppc_core_check_requests_pr(struct kvm_vcpu
*vcpu
)
240 int r
= 1; /* Indicate we want to get back into the guest */
242 /* We misuse TLB_FLUSH to indicate that we want to clear
243 all shadow cache entries */
244 if (kvm_check_request(KVM_REQ_TLB_FLUSH
, vcpu
))
245 kvmppc_mmu_pte_flush(vcpu
, 0, 0);
250 /************* MMU Notifiers *************/
251 static void do_kvm_unmap_hva(struct kvm
*kvm
, unsigned long start
,
255 struct kvm_vcpu
*vcpu
;
256 struct kvm_memslots
*slots
;
257 struct kvm_memory_slot
*memslot
;
259 slots
= kvm_memslots(kvm
);
260 kvm_for_each_memslot(memslot
, slots
) {
261 unsigned long hva_start
, hva_end
;
264 hva_start
= max(start
, memslot
->userspace_addr
);
265 hva_end
= min(end
, memslot
->userspace_addr
+
266 (memslot
->npages
<< PAGE_SHIFT
));
267 if (hva_start
>= hva_end
)
270 * {gfn(page) | page intersects with [hva_start, hva_end)} =
271 * {gfn, gfn+1, ..., gfn_end-1}.
273 gfn
= hva_to_gfn_memslot(hva_start
, memslot
);
274 gfn_end
= hva_to_gfn_memslot(hva_end
+ PAGE_SIZE
- 1, memslot
);
275 kvm_for_each_vcpu(i
, vcpu
, kvm
)
276 kvmppc_mmu_pte_pflush(vcpu
, gfn
<< PAGE_SHIFT
,
277 gfn_end
<< PAGE_SHIFT
);
281 static int kvm_unmap_hva_pr(struct kvm
*kvm
, unsigned long hva
)
283 trace_kvm_unmap_hva(hva
);
285 do_kvm_unmap_hva(kvm
, hva
, hva
+ PAGE_SIZE
);
290 static int kvm_unmap_hva_range_pr(struct kvm
*kvm
, unsigned long start
,
293 do_kvm_unmap_hva(kvm
, start
, end
);
298 static int kvm_age_hva_pr(struct kvm
*kvm
, unsigned long start
,
301 /* XXX could be more clever ;) */
305 static int kvm_test_age_hva_pr(struct kvm
*kvm
, unsigned long hva
)
307 /* XXX could be more clever ;) */
311 static void kvm_set_spte_hva_pr(struct kvm
*kvm
, unsigned long hva
, pte_t pte
)
313 /* The page will get remapped properly on its next fault */
314 do_kvm_unmap_hva(kvm
, hva
, hva
+ PAGE_SIZE
);
317 /*****************************************/
319 static void kvmppc_recalc_shadow_msr(struct kvm_vcpu
*vcpu
)
321 ulong guest_msr
= kvmppc_get_msr(vcpu
);
322 ulong smsr
= guest_msr
;
324 /* Guest MSR values */
325 smsr
&= MSR_FE0
| MSR_FE1
| MSR_SF
| MSR_SE
| MSR_BE
| MSR_LE
;
326 /* Process MSR values */
327 smsr
|= MSR_ME
| MSR_RI
| MSR_IR
| MSR_DR
| MSR_PR
| MSR_EE
;
328 /* External providers the guest reserved */
329 smsr
|= (guest_msr
& vcpu
->arch
.guest_owned_ext
);
330 /* 64-bit Process MSR values */
331 #ifdef CONFIG_PPC_BOOK3S_64
332 smsr
|= MSR_ISF
| MSR_HV
;
334 vcpu
->arch
.shadow_msr
= smsr
;
337 static void kvmppc_set_msr_pr(struct kvm_vcpu
*vcpu
, u64 msr
)
339 ulong old_msr
= kvmppc_get_msr(vcpu
);
342 printk(KERN_INFO
"KVM: Set MSR to 0x%llx\n", msr
);
345 msr
&= to_book3s(vcpu
)->msr_mask
;
346 kvmppc_set_msr_fast(vcpu
, msr
);
347 kvmppc_recalc_shadow_msr(vcpu
);
350 if (!vcpu
->arch
.pending_exceptions
) {
351 kvm_vcpu_block(vcpu
);
352 clear_bit(KVM_REQ_UNHALT
, &vcpu
->requests
);
353 vcpu
->stat
.halt_wakeup
++;
355 /* Unset POW bit after we woke up */
357 kvmppc_set_msr_fast(vcpu
, msr
);
361 if (kvmppc_is_split_real(vcpu
))
362 kvmppc_fixup_split_real(vcpu
);
364 kvmppc_unfixup_split_real(vcpu
);
366 if ((kvmppc_get_msr(vcpu
) & (MSR_PR
|MSR_IR
|MSR_DR
)) !=
367 (old_msr
& (MSR_PR
|MSR_IR
|MSR_DR
))) {
368 kvmppc_mmu_flush_segments(vcpu
);
369 kvmppc_mmu_map_segment(vcpu
, kvmppc_get_pc(vcpu
));
371 /* Preload magic page segment when in kernel mode */
372 if (!(msr
& MSR_PR
) && vcpu
->arch
.magic_page_pa
) {
373 struct kvm_vcpu_arch
*a
= &vcpu
->arch
;
376 kvmppc_mmu_map_segment(vcpu
, a
->magic_page_ea
);
378 kvmppc_mmu_map_segment(vcpu
, a
->magic_page_pa
);
383 * When switching from 32 to 64-bit, we may have a stale 32-bit
384 * magic page around, we need to flush it. Typically 32-bit magic
385 * page will be instanciated when calling into RTAS. Note: We
386 * assume that such transition only happens while in kernel mode,
387 * ie, we never transition from user 32-bit to kernel 64-bit with
388 * a 32-bit magic page around.
390 if (vcpu
->arch
.magic_page_pa
&&
391 !(old_msr
& MSR_PR
) && !(old_msr
& MSR_SF
) && (msr
& MSR_SF
)) {
392 /* going from RTAS to normal kernel code */
393 kvmppc_mmu_pte_flush(vcpu
, (uint32_t)vcpu
->arch
.magic_page_pa
,
397 /* Preload FPU if it's enabled */
398 if (kvmppc_get_msr(vcpu
) & MSR_FP
)
399 kvmppc_handle_ext(vcpu
, BOOK3S_INTERRUPT_FP_UNAVAIL
, MSR_FP
);
402 void kvmppc_set_pvr_pr(struct kvm_vcpu
*vcpu
, u32 pvr
)
406 vcpu
->arch
.hflags
&= ~BOOK3S_HFLAG_SLB
;
407 vcpu
->arch
.pvr
= pvr
;
408 #ifdef CONFIG_PPC_BOOK3S_64
409 if ((pvr
>= 0x330000) && (pvr
< 0x70330000)) {
410 kvmppc_mmu_book3s_64_init(vcpu
);
411 if (!to_book3s(vcpu
)->hior_explicit
)
412 to_book3s(vcpu
)->hior
= 0xfff00000;
413 to_book3s(vcpu
)->msr_mask
= 0xffffffffffffffffULL
;
414 vcpu
->arch
.cpu_type
= KVM_CPU_3S_64
;
418 kvmppc_mmu_book3s_32_init(vcpu
);
419 if (!to_book3s(vcpu
)->hior_explicit
)
420 to_book3s(vcpu
)->hior
= 0;
421 to_book3s(vcpu
)->msr_mask
= 0xffffffffULL
;
422 vcpu
->arch
.cpu_type
= KVM_CPU_3S_32
;
425 kvmppc_sanity_check(vcpu
);
427 /* If we are in hypervisor level on 970, we can tell the CPU to
428 * treat DCBZ as 32 bytes store */
429 vcpu
->arch
.hflags
&= ~BOOK3S_HFLAG_DCBZ32
;
430 if (vcpu
->arch
.mmu
.is_dcbz32(vcpu
) && (mfmsr() & MSR_HV
) &&
431 !strcmp(cur_cpu_spec
->platform
, "ppc970"))
432 vcpu
->arch
.hflags
|= BOOK3S_HFLAG_DCBZ32
;
434 /* Cell performs badly if MSR_FEx are set. So let's hope nobody
435 really needs them in a VM on Cell and force disable them. */
436 if (!strcmp(cur_cpu_spec
->platform
, "ppc-cell-be"))
437 to_book3s(vcpu
)->msr_mask
&= ~(MSR_FE0
| MSR_FE1
);
440 * If they're asking for POWER6 or later, set the flag
441 * indicating that we can do multiple large page sizes
443 * Also set the flag that indicates that tlbie has the large
444 * page bit in the RB operand instead of the instruction.
446 switch (PVR_VER(pvr
)) {
453 vcpu
->arch
.hflags
|= BOOK3S_HFLAG_MULTI_PGSIZE
|
454 BOOK3S_HFLAG_NEW_TLBIE
;
458 #ifdef CONFIG_PPC_BOOK3S_32
459 /* 32 bit Book3S always has 32 byte dcbz */
460 vcpu
->arch
.hflags
|= BOOK3S_HFLAG_DCBZ32
;
463 /* On some CPUs we can execute paired single operations natively */
464 asm ( "mfpvr %0" : "=r"(host_pvr
));
466 case 0x00080200: /* lonestar 2.0 */
467 case 0x00088202: /* lonestar 2.2 */
468 case 0x70000100: /* gekko 1.0 */
469 case 0x00080100: /* gekko 2.0 */
470 case 0x00083203: /* gekko 2.3a */
471 case 0x00083213: /* gekko 2.3b */
472 case 0x00083204: /* gekko 2.4 */
473 case 0x00083214: /* gekko 2.4e (8SE) - retail HW2 */
474 case 0x00087200: /* broadway */
475 vcpu
->arch
.hflags
|= BOOK3S_HFLAG_NATIVE_PS
;
476 /* Enable HID2.PSE - in case we need it later */
477 mtspr(SPRN_HID2_GEKKO
, mfspr(SPRN_HID2_GEKKO
) | (1 << 29));
481 /* Book3s_32 CPUs always have 32 bytes cache line size, which Linux assumes. To
482 * make Book3s_32 Linux work on Book3s_64, we have to make sure we trap dcbz to
483 * emulate 32 bytes dcbz length.
485 * The Book3s_64 inventors also realized this case and implemented a special bit
486 * in the HID5 register, which is a hypervisor ressource. Thus we can't use it.
488 * My approach here is to patch the dcbz instruction on executing pages.
490 static void kvmppc_patch_dcbz(struct kvm_vcpu
*vcpu
, struct kvmppc_pte
*pte
)
497 hpage
= gfn_to_page(vcpu
->kvm
, pte
->raddr
>> PAGE_SHIFT
);
498 if (is_error_page(hpage
))
501 hpage_offset
= pte
->raddr
& ~PAGE_MASK
;
502 hpage_offset
&= ~0xFFFULL
;
506 page
= kmap_atomic(hpage
);
508 /* patch dcbz into reserved instruction, so we trap */
509 for (i
=hpage_offset
; i
< hpage_offset
+ (HW_PAGE_SIZE
/ 4); i
++)
510 if ((be32_to_cpu(page
[i
]) & 0xff0007ff) == INS_DCBZ
)
511 page
[i
] &= cpu_to_be32(0xfffffff7);
517 static bool kvmppc_visible_gpa(struct kvm_vcpu
*vcpu
, gpa_t gpa
)
519 ulong mp_pa
= vcpu
->arch
.magic_page_pa
;
521 if (!(kvmppc_get_msr(vcpu
) & MSR_SF
))
522 mp_pa
= (uint32_t)mp_pa
;
525 if (unlikely(mp_pa
) && unlikely((mp_pa
& KVM_PAM
) == (gpa
& KVM_PAM
))) {
529 return kvm_is_visible_gfn(vcpu
->kvm
, gpa
>> PAGE_SHIFT
);
532 int kvmppc_handle_pagefault(struct kvm_run
*run
, struct kvm_vcpu
*vcpu
,
533 ulong eaddr
, int vec
)
535 bool data
= (vec
== BOOK3S_INTERRUPT_DATA_STORAGE
);
536 bool iswrite
= false;
537 int r
= RESUME_GUEST
;
540 struct kvmppc_pte pte
;
541 bool is_mmio
= false;
542 bool dr
= (kvmppc_get_msr(vcpu
) & MSR_DR
) ? true : false;
543 bool ir
= (kvmppc_get_msr(vcpu
) & MSR_IR
) ? true : false;
546 relocated
= data
? dr
: ir
;
547 if (data
&& (vcpu
->arch
.fault_dsisr
& DSISR_ISSTORE
))
550 /* Resolve real address if translation turned on */
552 page_found
= vcpu
->arch
.mmu
.xlate(vcpu
, eaddr
, &pte
, data
, iswrite
);
554 pte
.may_execute
= true;
556 pte
.may_write
= true;
557 pte
.raddr
= eaddr
& KVM_PAM
;
559 pte
.vpage
= eaddr
>> 12;
560 pte
.page_size
= MMU_PAGE_64K
;
563 switch (kvmppc_get_msr(vcpu
) & (MSR_DR
|MSR_IR
)) {
565 pte
.vpage
|= ((u64
)VSID_REAL
<< (SID_SHIFT
- 12));
569 (vcpu
->arch
.hflags
& BOOK3S_HFLAG_SPLIT_HACK
) &&
570 ((pte
.raddr
& SPLIT_HACK_MASK
) == SPLIT_HACK_OFFS
))
571 pte
.raddr
&= ~SPLIT_HACK_MASK
;
574 vcpu
->arch
.mmu
.esid_to_vsid(vcpu
, eaddr
>> SID_SHIFT
, &vsid
);
576 if ((kvmppc_get_msr(vcpu
) & (MSR_DR
|MSR_IR
)) == MSR_DR
)
577 pte
.vpage
|= ((u64
)VSID_REAL_DR
<< (SID_SHIFT
- 12));
579 pte
.vpage
|= ((u64
)VSID_REAL_IR
<< (SID_SHIFT
- 12));
583 page_found
= -EINVAL
;
587 if (vcpu
->arch
.mmu
.is_dcbz32(vcpu
) &&
588 (!(vcpu
->arch
.hflags
& BOOK3S_HFLAG_DCBZ32
))) {
590 * If we do the dcbz hack, we have to NX on every execution,
591 * so we can patch the executing code. This renders our guest
594 pte
.may_execute
= !data
;
597 if (page_found
== -ENOENT
) {
598 /* Page not found in guest PTE entries */
599 u64 ssrr1
= vcpu
->arch
.shadow_srr1
;
600 u64 msr
= kvmppc_get_msr(vcpu
);
601 kvmppc_set_dar(vcpu
, kvmppc_get_fault_dar(vcpu
));
602 kvmppc_set_dsisr(vcpu
, vcpu
->arch
.fault_dsisr
);
603 kvmppc_set_msr_fast(vcpu
, msr
| (ssrr1
& 0xf8000000ULL
));
604 kvmppc_book3s_queue_irqprio(vcpu
, vec
);
605 } else if (page_found
== -EPERM
) {
606 /* Storage protection */
607 u32 dsisr
= vcpu
->arch
.fault_dsisr
;
608 u64 ssrr1
= vcpu
->arch
.shadow_srr1
;
609 u64 msr
= kvmppc_get_msr(vcpu
);
610 kvmppc_set_dar(vcpu
, kvmppc_get_fault_dar(vcpu
));
611 dsisr
= (dsisr
& ~DSISR_NOHPTE
) | DSISR_PROTFAULT
;
612 kvmppc_set_dsisr(vcpu
, dsisr
);
613 kvmppc_set_msr_fast(vcpu
, msr
| (ssrr1
& 0xf8000000ULL
));
614 kvmppc_book3s_queue_irqprio(vcpu
, vec
);
615 } else if (page_found
== -EINVAL
) {
616 /* Page not found in guest SLB */
617 kvmppc_set_dar(vcpu
, kvmppc_get_fault_dar(vcpu
));
618 kvmppc_book3s_queue_irqprio(vcpu
, vec
+ 0x80);
619 } else if (!is_mmio
&&
620 kvmppc_visible_gpa(vcpu
, pte
.raddr
)) {
621 if (data
&& !(vcpu
->arch
.fault_dsisr
& DSISR_NOHPTE
)) {
623 * There is already a host HPTE there, presumably
624 * a read-only one for a page the guest thinks
625 * is writable, so get rid of it first.
627 kvmppc_mmu_unmap_page(vcpu
, &pte
);
629 /* The guest's PTE is not mapped yet. Map on the host */
630 kvmppc_mmu_map_page(vcpu
, &pte
, iswrite
);
632 vcpu
->stat
.sp_storage
++;
633 else if (vcpu
->arch
.mmu
.is_dcbz32(vcpu
) &&
634 (!(vcpu
->arch
.hflags
& BOOK3S_HFLAG_DCBZ32
)))
635 kvmppc_patch_dcbz(vcpu
, &pte
);
638 vcpu
->stat
.mmio_exits
++;
639 vcpu
->arch
.paddr_accessed
= pte
.raddr
;
640 vcpu
->arch
.vaddr_accessed
= pte
.eaddr
;
641 r
= kvmppc_emulate_mmio(run
, vcpu
);
642 if ( r
== RESUME_HOST_NV
)
649 /* Give up external provider (FPU, Altivec, VSX) */
650 void kvmppc_giveup_ext(struct kvm_vcpu
*vcpu
, ulong msr
)
652 struct thread_struct
*t
= ¤t
->thread
;
655 * VSX instructions can access FP and vector registers, so if
656 * we are giving up VSX, make sure we give up FP and VMX as well.
659 msr
|= MSR_FP
| MSR_VEC
;
661 msr
&= vcpu
->arch
.guest_owned_ext
;
666 printk(KERN_INFO
"Giving up ext 0x%lx\n", msr
);
671 * Note that on CPUs with VSX, giveup_fpu stores
672 * both the traditional FP registers and the added VSX
673 * registers into thread.fp_state.fpr[].
675 if (t
->regs
->msr
& MSR_FP
)
677 t
->fp_save_area
= NULL
;
680 #ifdef CONFIG_ALTIVEC
682 if (current
->thread
.regs
->msr
& MSR_VEC
)
683 giveup_altivec(current
);
684 t
->vr_save_area
= NULL
;
688 vcpu
->arch
.guest_owned_ext
&= ~(msr
| MSR_VSX
);
689 kvmppc_recalc_shadow_msr(vcpu
);
692 /* Give up facility (TAR / EBB / DSCR) */
693 static void kvmppc_giveup_fac(struct kvm_vcpu
*vcpu
, ulong fac
)
695 #ifdef CONFIG_PPC_BOOK3S_64
696 if (!(vcpu
->arch
.shadow_fscr
& (1ULL << fac
))) {
697 /* Facility not available to the guest, ignore giveup request*/
703 vcpu
->arch
.tar
= mfspr(SPRN_TAR
);
704 mtspr(SPRN_TAR
, current
->thread
.tar
);
705 vcpu
->arch
.shadow_fscr
&= ~FSCR_TAR
;
711 /* Handle external providers (FPU, Altivec, VSX) */
712 static int kvmppc_handle_ext(struct kvm_vcpu
*vcpu
, unsigned int exit_nr
,
715 struct thread_struct
*t
= ¤t
->thread
;
717 /* When we have paired singles, we emulate in software */
718 if (vcpu
->arch
.hflags
& BOOK3S_HFLAG_PAIRED_SINGLE
)
721 if (!(kvmppc_get_msr(vcpu
) & msr
)) {
722 kvmppc_book3s_queue_irqprio(vcpu
, exit_nr
);
726 if (msr
== MSR_VSX
) {
727 /* No VSX? Give an illegal instruction interrupt */
729 if (!cpu_has_feature(CPU_FTR_VSX
))
732 kvmppc_core_queue_program(vcpu
, SRR1_PROGILL
);
737 * We have to load up all the FP and VMX registers before
738 * we can let the guest use VSX instructions.
740 msr
= MSR_FP
| MSR_VEC
| MSR_VSX
;
743 /* See if we already own all the ext(s) needed */
744 msr
&= ~vcpu
->arch
.guest_owned_ext
;
749 printk(KERN_INFO
"Loading up ext 0x%lx\n", msr
);
755 load_fp_state(&vcpu
->arch
.fp
);
757 t
->fp_save_area
= &vcpu
->arch
.fp
;
762 #ifdef CONFIG_ALTIVEC
764 enable_kernel_altivec();
765 load_vr_state(&vcpu
->arch
.vr
);
766 disable_kernel_altivec();
767 t
->vr_save_area
= &vcpu
->arch
.vr
;
773 vcpu
->arch
.guest_owned_ext
|= msr
;
774 kvmppc_recalc_shadow_msr(vcpu
);
780 * Kernel code using FP or VMX could have flushed guest state to
781 * the thread_struct; if so, get it back now.
783 static void kvmppc_handle_lost_ext(struct kvm_vcpu
*vcpu
)
785 unsigned long lost_ext
;
787 lost_ext
= vcpu
->arch
.guest_owned_ext
& ~current
->thread
.regs
->msr
;
791 if (lost_ext
& MSR_FP
) {
794 load_fp_state(&vcpu
->arch
.fp
);
798 #ifdef CONFIG_ALTIVEC
799 if (lost_ext
& MSR_VEC
) {
801 enable_kernel_altivec();
802 load_vr_state(&vcpu
->arch
.vr
);
803 disable_kernel_altivec();
807 current
->thread
.regs
->msr
|= lost_ext
;
810 #ifdef CONFIG_PPC_BOOK3S_64
812 static void kvmppc_trigger_fac_interrupt(struct kvm_vcpu
*vcpu
, ulong fac
)
814 /* Inject the Interrupt Cause field and trigger a guest interrupt */
815 vcpu
->arch
.fscr
&= ~(0xffULL
<< 56);
816 vcpu
->arch
.fscr
|= (fac
<< 56);
817 kvmppc_book3s_queue_irqprio(vcpu
, BOOK3S_INTERRUPT_FAC_UNAVAIL
);
820 static void kvmppc_emulate_fac(struct kvm_vcpu
*vcpu
, ulong fac
)
822 enum emulation_result er
= EMULATE_FAIL
;
824 if (!(kvmppc_get_msr(vcpu
) & MSR_PR
))
825 er
= kvmppc_emulate_instruction(vcpu
->run
, vcpu
);
827 if ((er
!= EMULATE_DONE
) && (er
!= EMULATE_AGAIN
)) {
828 /* Couldn't emulate, trigger interrupt in guest */
829 kvmppc_trigger_fac_interrupt(vcpu
, fac
);
833 /* Enable facilities (TAR, EBB, DSCR) for the guest */
834 static int kvmppc_handle_fac(struct kvm_vcpu
*vcpu
, ulong fac
)
836 bool guest_fac_enabled
;
837 BUG_ON(!cpu_has_feature(CPU_FTR_ARCH_207S
));
840 * Not every facility is enabled by FSCR bits, check whether the
841 * guest has this facility enabled at all.
846 guest_fac_enabled
= (vcpu
->arch
.fscr
& (1ULL << fac
));
849 guest_fac_enabled
= kvmppc_get_msr(vcpu
) & MSR_TM
;
852 guest_fac_enabled
= false;
856 if (!guest_fac_enabled
) {
857 /* Facility not enabled by the guest */
858 kvmppc_trigger_fac_interrupt(vcpu
, fac
);
864 /* TAR switching isn't lazy in Linux yet */
865 current
->thread
.tar
= mfspr(SPRN_TAR
);
866 mtspr(SPRN_TAR
, vcpu
->arch
.tar
);
867 vcpu
->arch
.shadow_fscr
|= FSCR_TAR
;
870 kvmppc_emulate_fac(vcpu
, fac
);
877 void kvmppc_set_fscr(struct kvm_vcpu
*vcpu
, u64 fscr
)
879 if ((vcpu
->arch
.fscr
& FSCR_TAR
) && !(fscr
& FSCR_TAR
)) {
880 /* TAR got dropped, drop it in shadow too */
881 kvmppc_giveup_fac(vcpu
, FSCR_TAR_LG
);
883 vcpu
->arch
.fscr
= fscr
;
887 static void kvmppc_setup_debug(struct kvm_vcpu
*vcpu
)
889 if (vcpu
->guest_debug
& KVM_GUESTDBG_SINGLESTEP
) {
890 u64 msr
= kvmppc_get_msr(vcpu
);
892 kvmppc_set_msr(vcpu
, msr
| MSR_SE
);
896 static void kvmppc_clear_debug(struct kvm_vcpu
*vcpu
)
898 if (vcpu
->guest_debug
& KVM_GUESTDBG_SINGLESTEP
) {
899 u64 msr
= kvmppc_get_msr(vcpu
);
901 kvmppc_set_msr(vcpu
, msr
& ~MSR_SE
);
905 int kvmppc_handle_exit_pr(struct kvm_run
*run
, struct kvm_vcpu
*vcpu
,
906 unsigned int exit_nr
)
911 vcpu
->stat
.sum_exits
++;
913 run
->exit_reason
= KVM_EXIT_UNKNOWN
;
914 run
->ready_for_interrupt_injection
= 1;
916 /* We get here with MSR.EE=1 */
918 trace_kvm_exit(exit_nr
, vcpu
);
922 case BOOK3S_INTERRUPT_INST_STORAGE
:
924 ulong shadow_srr1
= vcpu
->arch
.shadow_srr1
;
925 vcpu
->stat
.pf_instruc
++;
927 if (kvmppc_is_split_real(vcpu
))
928 kvmppc_fixup_split_real(vcpu
);
930 #ifdef CONFIG_PPC_BOOK3S_32
931 /* We set segments as unused segments when invalidating them. So
932 * treat the respective fault as segment fault. */
934 struct kvmppc_book3s_shadow_vcpu
*svcpu
;
937 svcpu
= svcpu_get(vcpu
);
938 sr
= svcpu
->sr
[kvmppc_get_pc(vcpu
) >> SID_SHIFT
];
940 if (sr
== SR_INVALID
) {
941 kvmppc_mmu_map_segment(vcpu
, kvmppc_get_pc(vcpu
));
948 /* only care about PTEG not found errors, but leave NX alone */
949 if (shadow_srr1
& 0x40000000) {
950 int idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
951 r
= kvmppc_handle_pagefault(run
, vcpu
, kvmppc_get_pc(vcpu
), exit_nr
);
952 srcu_read_unlock(&vcpu
->kvm
->srcu
, idx
);
953 vcpu
->stat
.sp_instruc
++;
954 } else if (vcpu
->arch
.mmu
.is_dcbz32(vcpu
) &&
955 (!(vcpu
->arch
.hflags
& BOOK3S_HFLAG_DCBZ32
))) {
957 * XXX If we do the dcbz hack we use the NX bit to flush&patch the page,
958 * so we can't use the NX bit inside the guest. Let's cross our fingers,
959 * that no guest that needs the dcbz hack does NX.
961 kvmppc_mmu_pte_flush(vcpu
, kvmppc_get_pc(vcpu
), ~0xFFFUL
);
964 u64 msr
= kvmppc_get_msr(vcpu
);
965 msr
|= shadow_srr1
& 0x58000000;
966 kvmppc_set_msr_fast(vcpu
, msr
);
967 kvmppc_book3s_queue_irqprio(vcpu
, exit_nr
);
972 case BOOK3S_INTERRUPT_DATA_STORAGE
:
974 ulong dar
= kvmppc_get_fault_dar(vcpu
);
975 u32 fault_dsisr
= vcpu
->arch
.fault_dsisr
;
976 vcpu
->stat
.pf_storage
++;
978 #ifdef CONFIG_PPC_BOOK3S_32
979 /* We set segments as unused segments when invalidating them. So
980 * treat the respective fault as segment fault. */
982 struct kvmppc_book3s_shadow_vcpu
*svcpu
;
985 svcpu
= svcpu_get(vcpu
);
986 sr
= svcpu
->sr
[dar
>> SID_SHIFT
];
988 if (sr
== SR_INVALID
) {
989 kvmppc_mmu_map_segment(vcpu
, dar
);
997 * We need to handle missing shadow PTEs, and
998 * protection faults due to us mapping a page read-only
999 * when the guest thinks it is writable.
1001 if (fault_dsisr
& (DSISR_NOHPTE
| DSISR_PROTFAULT
)) {
1002 int idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
1003 r
= kvmppc_handle_pagefault(run
, vcpu
, dar
, exit_nr
);
1004 srcu_read_unlock(&vcpu
->kvm
->srcu
, idx
);
1006 kvmppc_set_dar(vcpu
, dar
);
1007 kvmppc_set_dsisr(vcpu
, fault_dsisr
);
1008 kvmppc_book3s_queue_irqprio(vcpu
, exit_nr
);
1013 case BOOK3S_INTERRUPT_DATA_SEGMENT
:
1014 if (kvmppc_mmu_map_segment(vcpu
, kvmppc_get_fault_dar(vcpu
)) < 0) {
1015 kvmppc_set_dar(vcpu
, kvmppc_get_fault_dar(vcpu
));
1016 kvmppc_book3s_queue_irqprio(vcpu
,
1017 BOOK3S_INTERRUPT_DATA_SEGMENT
);
1021 case BOOK3S_INTERRUPT_INST_SEGMENT
:
1022 if (kvmppc_mmu_map_segment(vcpu
, kvmppc_get_pc(vcpu
)) < 0) {
1023 kvmppc_book3s_queue_irqprio(vcpu
,
1024 BOOK3S_INTERRUPT_INST_SEGMENT
);
1028 /* We're good on these - the host merely wanted to get our attention */
1029 case BOOK3S_INTERRUPT_DECREMENTER
:
1030 case BOOK3S_INTERRUPT_HV_DECREMENTER
:
1031 case BOOK3S_INTERRUPT_DOORBELL
:
1032 case BOOK3S_INTERRUPT_H_DOORBELL
:
1033 vcpu
->stat
.dec_exits
++;
1036 case BOOK3S_INTERRUPT_EXTERNAL
:
1037 case BOOK3S_INTERRUPT_EXTERNAL_LEVEL
:
1038 case BOOK3S_INTERRUPT_EXTERNAL_HV
:
1039 vcpu
->stat
.ext_intr_exits
++;
1042 case BOOK3S_INTERRUPT_PERFMON
:
1045 case BOOK3S_INTERRUPT_PROGRAM
:
1046 case BOOK3S_INTERRUPT_H_EMUL_ASSIST
:
1048 enum emulation_result er
;
1055 * shadow_srr1 only contains valid flags if we came here via
1056 * a program exception. The other exceptions (emulation assist,
1057 * FP unavailable, etc.) do not provide flags in SRR1, so use
1058 * an illegal-instruction exception when injecting a program
1059 * interrupt into the guest.
1061 if (exit_nr
== BOOK3S_INTERRUPT_PROGRAM
)
1062 flags
= vcpu
->arch
.shadow_srr1
& 0x1f0000ull
;
1064 flags
= SRR1_PROGILL
;
1066 emul
= kvmppc_get_last_inst(vcpu
, INST_GENERIC
, &last_inst
);
1067 if (emul
!= EMULATE_DONE
) {
1072 if (kvmppc_get_msr(vcpu
) & MSR_PR
) {
1074 pr_info("Userspace triggered 0x700 exception at\n 0x%lx (0x%x)\n",
1075 kvmppc_get_pc(vcpu
), last_inst
);
1077 if ((last_inst
& 0xff0007ff) !=
1078 (INS_DCBZ
& 0xfffffff7)) {
1079 kvmppc_core_queue_program(vcpu
, flags
);
1085 vcpu
->stat
.emulated_inst_exits
++;
1086 er
= kvmppc_emulate_instruction(run
, vcpu
);
1089 r
= RESUME_GUEST_NV
;
1095 printk(KERN_CRIT
"%s: emulation at %lx failed (%08x)\n",
1096 __func__
, kvmppc_get_pc(vcpu
), last_inst
);
1097 kvmppc_core_queue_program(vcpu
, flags
);
1100 case EMULATE_DO_MMIO
:
1101 run
->exit_reason
= KVM_EXIT_MMIO
;
1104 case EMULATE_EXIT_USER
:
1112 case BOOK3S_INTERRUPT_SYSCALL
:
1117 /* Get last sc for papr */
1118 if (vcpu
->arch
.papr_enabled
) {
1119 /* The sc instuction points SRR0 to the next inst */
1120 emul
= kvmppc_get_last_inst(vcpu
, INST_SC
, &last_sc
);
1121 if (emul
!= EMULATE_DONE
) {
1122 kvmppc_set_pc(vcpu
, kvmppc_get_pc(vcpu
) - 4);
1128 if (vcpu
->arch
.papr_enabled
&&
1129 (last_sc
== 0x44000022) &&
1130 !(kvmppc_get_msr(vcpu
) & MSR_PR
)) {
1131 /* SC 1 papr hypercalls */
1132 ulong cmd
= kvmppc_get_gpr(vcpu
, 3);
1135 #ifdef CONFIG_PPC_BOOK3S_64
1136 if (kvmppc_h_pr(vcpu
, cmd
) == EMULATE_DONE
) {
1142 run
->papr_hcall
.nr
= cmd
;
1143 for (i
= 0; i
< 9; ++i
) {
1144 ulong gpr
= kvmppc_get_gpr(vcpu
, 4 + i
);
1145 run
->papr_hcall
.args
[i
] = gpr
;
1147 run
->exit_reason
= KVM_EXIT_PAPR_HCALL
;
1148 vcpu
->arch
.hcall_needed
= 1;
1150 } else if (vcpu
->arch
.osi_enabled
&&
1151 (((u32
)kvmppc_get_gpr(vcpu
, 3)) == OSI_SC_MAGIC_R3
) &&
1152 (((u32
)kvmppc_get_gpr(vcpu
, 4)) == OSI_SC_MAGIC_R4
)) {
1153 /* MOL hypercalls */
1154 u64
*gprs
= run
->osi
.gprs
;
1157 run
->exit_reason
= KVM_EXIT_OSI
;
1158 for (i
= 0; i
< 32; i
++)
1159 gprs
[i
] = kvmppc_get_gpr(vcpu
, i
);
1160 vcpu
->arch
.osi_needed
= 1;
1162 } else if (!(kvmppc_get_msr(vcpu
) & MSR_PR
) &&
1163 (((u32
)kvmppc_get_gpr(vcpu
, 0)) == KVM_SC_MAGIC_R0
)) {
1164 /* KVM PV hypercalls */
1165 kvmppc_set_gpr(vcpu
, 3, kvmppc_kvm_pv(vcpu
));
1168 /* Guest syscalls */
1169 vcpu
->stat
.syscall_exits
++;
1170 kvmppc_book3s_queue_irqprio(vcpu
, exit_nr
);
1175 case BOOK3S_INTERRUPT_FP_UNAVAIL
:
1176 case BOOK3S_INTERRUPT_ALTIVEC
:
1177 case BOOK3S_INTERRUPT_VSX
:
1183 if (vcpu
->arch
.hflags
& BOOK3S_HFLAG_PAIRED_SINGLE
) {
1184 /* Do paired single instruction emulation */
1185 emul
= kvmppc_get_last_inst(vcpu
, INST_GENERIC
,
1187 if (emul
== EMULATE_DONE
)
1188 goto program_interrupt
;
1195 /* Enable external provider */
1197 case BOOK3S_INTERRUPT_FP_UNAVAIL
:
1201 case BOOK3S_INTERRUPT_ALTIVEC
:
1205 case BOOK3S_INTERRUPT_VSX
:
1210 r
= kvmppc_handle_ext(vcpu
, exit_nr
, ext_msr
);
1213 case BOOK3S_INTERRUPT_ALIGNMENT
:
1216 int emul
= kvmppc_get_last_inst(vcpu
, INST_GENERIC
, &last_inst
);
1218 if (emul
== EMULATE_DONE
) {
1222 dsisr
= kvmppc_alignment_dsisr(vcpu
, last_inst
);
1223 dar
= kvmppc_alignment_dar(vcpu
, last_inst
);
1225 kvmppc_set_dsisr(vcpu
, dsisr
);
1226 kvmppc_set_dar(vcpu
, dar
);
1228 kvmppc_book3s_queue_irqprio(vcpu
, exit_nr
);
1233 #ifdef CONFIG_PPC_BOOK3S_64
1234 case BOOK3S_INTERRUPT_FAC_UNAVAIL
:
1235 kvmppc_handle_fac(vcpu
, vcpu
->arch
.shadow_fscr
>> 56);
1239 case BOOK3S_INTERRUPT_MACHINE_CHECK
:
1240 kvmppc_book3s_queue_irqprio(vcpu
, exit_nr
);
1243 case BOOK3S_INTERRUPT_TRACE
:
1244 if (vcpu
->guest_debug
& KVM_GUESTDBG_SINGLESTEP
) {
1245 run
->exit_reason
= KVM_EXIT_DEBUG
;
1248 kvmppc_book3s_queue_irqprio(vcpu
, exit_nr
);
1254 ulong shadow_srr1
= vcpu
->arch
.shadow_srr1
;
1255 /* Ugh - bork here! What did we get? */
1256 printk(KERN_EMERG
"exit_nr=0x%x | pc=0x%lx | msr=0x%lx\n",
1257 exit_nr
, kvmppc_get_pc(vcpu
), shadow_srr1
);
1264 if (!(r
& RESUME_HOST
)) {
1265 /* To avoid clobbering exit_reason, only check for signals if
1266 * we aren't already exiting to userspace for some other
1270 * Interrupts could be timers for the guest which we have to
1271 * inject again, so let's postpone them until we're in the guest
1272 * and if we really did time things so badly, then we just exit
1273 * again due to a host external interrupt.
1275 s
= kvmppc_prepare_to_enter(vcpu
);
1279 /* interrupts now hard-disabled */
1280 kvmppc_fix_ee_before_entry();
1283 kvmppc_handle_lost_ext(vcpu
);
1286 trace_kvm_book3s_reenter(r
, vcpu
);
1291 static int kvm_arch_vcpu_ioctl_get_sregs_pr(struct kvm_vcpu
*vcpu
,
1292 struct kvm_sregs
*sregs
)
1294 struct kvmppc_vcpu_book3s
*vcpu3s
= to_book3s(vcpu
);
1297 sregs
->pvr
= vcpu
->arch
.pvr
;
1299 sregs
->u
.s
.sdr1
= to_book3s(vcpu
)->sdr1
;
1300 if (vcpu
->arch
.hflags
& BOOK3S_HFLAG_SLB
) {
1301 for (i
= 0; i
< 64; i
++) {
1302 sregs
->u
.s
.ppc64
.slb
[i
].slbe
= vcpu
->arch
.slb
[i
].orige
| i
;
1303 sregs
->u
.s
.ppc64
.slb
[i
].slbv
= vcpu
->arch
.slb
[i
].origv
;
1306 for (i
= 0; i
< 16; i
++)
1307 sregs
->u
.s
.ppc32
.sr
[i
] = kvmppc_get_sr(vcpu
, i
);
1309 for (i
= 0; i
< 8; i
++) {
1310 sregs
->u
.s
.ppc32
.ibat
[i
] = vcpu3s
->ibat
[i
].raw
;
1311 sregs
->u
.s
.ppc32
.dbat
[i
] = vcpu3s
->dbat
[i
].raw
;
1318 static int kvm_arch_vcpu_ioctl_set_sregs_pr(struct kvm_vcpu
*vcpu
,
1319 struct kvm_sregs
*sregs
)
1321 struct kvmppc_vcpu_book3s
*vcpu3s
= to_book3s(vcpu
);
1324 kvmppc_set_pvr_pr(vcpu
, sregs
->pvr
);
1326 vcpu3s
->sdr1
= sregs
->u
.s
.sdr1
;
1327 if (vcpu
->arch
.hflags
& BOOK3S_HFLAG_SLB
) {
1328 for (i
= 0; i
< 64; i
++) {
1329 vcpu
->arch
.mmu
.slbmte(vcpu
, sregs
->u
.s
.ppc64
.slb
[i
].slbv
,
1330 sregs
->u
.s
.ppc64
.slb
[i
].slbe
);
1333 for (i
= 0; i
< 16; i
++) {
1334 vcpu
->arch
.mmu
.mtsrin(vcpu
, i
, sregs
->u
.s
.ppc32
.sr
[i
]);
1336 for (i
= 0; i
< 8; i
++) {
1337 kvmppc_set_bat(vcpu
, &(vcpu3s
->ibat
[i
]), false,
1338 (u32
)sregs
->u
.s
.ppc32
.ibat
[i
]);
1339 kvmppc_set_bat(vcpu
, &(vcpu3s
->ibat
[i
]), true,
1340 (u32
)(sregs
->u
.s
.ppc32
.ibat
[i
] >> 32));
1341 kvmppc_set_bat(vcpu
, &(vcpu3s
->dbat
[i
]), false,
1342 (u32
)sregs
->u
.s
.ppc32
.dbat
[i
]);
1343 kvmppc_set_bat(vcpu
, &(vcpu3s
->dbat
[i
]), true,
1344 (u32
)(sregs
->u
.s
.ppc32
.dbat
[i
] >> 32));
1348 /* Flush the MMU after messing with the segments */
1349 kvmppc_mmu_pte_flush(vcpu
, 0, 0);
1354 static int kvmppc_get_one_reg_pr(struct kvm_vcpu
*vcpu
, u64 id
,
1355 union kvmppc_one_reg
*val
)
1360 case KVM_REG_PPC_DEBUG_INST
:
1361 *val
= get_reg_val(id
, KVMPPC_INST_SW_BREAKPOINT
);
1363 case KVM_REG_PPC_HIOR
:
1364 *val
= get_reg_val(id
, to_book3s(vcpu
)->hior
);
1366 case KVM_REG_PPC_VTB
:
1367 *val
= get_reg_val(id
, to_book3s(vcpu
)->vtb
);
1369 case KVM_REG_PPC_LPCR
:
1370 case KVM_REG_PPC_LPCR_64
:
1372 * We are only interested in the LPCR_ILE bit
1374 if (vcpu
->arch
.intr_msr
& MSR_LE
)
1375 *val
= get_reg_val(id
, LPCR_ILE
);
1377 *val
= get_reg_val(id
, 0);
1387 static void kvmppc_set_lpcr_pr(struct kvm_vcpu
*vcpu
, u64 new_lpcr
)
1389 if (new_lpcr
& LPCR_ILE
)
1390 vcpu
->arch
.intr_msr
|= MSR_LE
;
1392 vcpu
->arch
.intr_msr
&= ~MSR_LE
;
1395 static int kvmppc_set_one_reg_pr(struct kvm_vcpu
*vcpu
, u64 id
,
1396 union kvmppc_one_reg
*val
)
1401 case KVM_REG_PPC_HIOR
:
1402 to_book3s(vcpu
)->hior
= set_reg_val(id
, *val
);
1403 to_book3s(vcpu
)->hior_explicit
= true;
1405 case KVM_REG_PPC_VTB
:
1406 to_book3s(vcpu
)->vtb
= set_reg_val(id
, *val
);
1408 case KVM_REG_PPC_LPCR
:
1409 case KVM_REG_PPC_LPCR_64
:
1410 kvmppc_set_lpcr_pr(vcpu
, set_reg_val(id
, *val
));
1420 static struct kvm_vcpu
*kvmppc_core_vcpu_create_pr(struct kvm
*kvm
,
1423 struct kvmppc_vcpu_book3s
*vcpu_book3s
;
1424 struct kvm_vcpu
*vcpu
;
1428 vcpu
= kmem_cache_zalloc(kvm_vcpu_cache
, GFP_KERNEL
);
1432 vcpu_book3s
= vzalloc(sizeof(struct kvmppc_vcpu_book3s
));
1435 vcpu
->arch
.book3s
= vcpu_book3s
;
1437 #ifdef CONFIG_KVM_BOOK3S_32_HANDLER
1438 vcpu
->arch
.shadow_vcpu
=
1439 kzalloc(sizeof(*vcpu
->arch
.shadow_vcpu
), GFP_KERNEL
);
1440 if (!vcpu
->arch
.shadow_vcpu
)
1444 err
= kvm_vcpu_init(vcpu
, kvm
, id
);
1446 goto free_shadow_vcpu
;
1449 p
= __get_free_page(GFP_KERNEL
|__GFP_ZERO
);
1452 vcpu
->arch
.shared
= (void *)p
;
1453 #ifdef CONFIG_PPC_BOOK3S_64
1454 /* Always start the shared struct in native endian mode */
1455 #ifdef __BIG_ENDIAN__
1456 vcpu
->arch
.shared_big_endian
= true;
1458 vcpu
->arch
.shared_big_endian
= false;
1462 * Default to the same as the host if we're on sufficiently
1463 * recent machine that we have 1TB segments;
1464 * otherwise default to PPC970FX.
1466 vcpu
->arch
.pvr
= 0x3C0301;
1467 if (mmu_has_feature(MMU_FTR_1T_SEGMENT
))
1468 vcpu
->arch
.pvr
= mfspr(SPRN_PVR
);
1469 vcpu
->arch
.intr_msr
= MSR_SF
;
1471 /* default to book3s_32 (750) */
1472 vcpu
->arch
.pvr
= 0x84202;
1474 kvmppc_set_pvr_pr(vcpu
, vcpu
->arch
.pvr
);
1475 vcpu
->arch
.slb_nr
= 64;
1477 vcpu
->arch
.shadow_msr
= MSR_USER64
& ~MSR_LE
;
1479 err
= kvmppc_mmu_init(vcpu
);
1486 kvm_vcpu_uninit(vcpu
);
1488 #ifdef CONFIG_KVM_BOOK3S_32_HANDLER
1489 kfree(vcpu
->arch
.shadow_vcpu
);
1494 kmem_cache_free(kvm_vcpu_cache
, vcpu
);
1496 return ERR_PTR(err
);
1499 static void kvmppc_core_vcpu_free_pr(struct kvm_vcpu
*vcpu
)
1501 struct kvmppc_vcpu_book3s
*vcpu_book3s
= to_book3s(vcpu
);
1503 free_page((unsigned long)vcpu
->arch
.shared
& PAGE_MASK
);
1504 kvm_vcpu_uninit(vcpu
);
1505 #ifdef CONFIG_KVM_BOOK3S_32_HANDLER
1506 kfree(vcpu
->arch
.shadow_vcpu
);
1509 kmem_cache_free(kvm_vcpu_cache
, vcpu
);
1512 static int kvmppc_vcpu_run_pr(struct kvm_run
*kvm_run
, struct kvm_vcpu
*vcpu
)
1515 #ifdef CONFIG_ALTIVEC
1516 unsigned long uninitialized_var(vrsave
);
1519 /* Check if we can run the vcpu at all */
1520 if (!vcpu
->arch
.sane
) {
1521 kvm_run
->exit_reason
= KVM_EXIT_INTERNAL_ERROR
;
1526 kvmppc_setup_debug(vcpu
);
1529 * Interrupts could be timers for the guest which we have to inject
1530 * again, so let's postpone them until we're in the guest and if we
1531 * really did time things so badly, then we just exit again due to
1532 * a host external interrupt.
1534 ret
= kvmppc_prepare_to_enter(vcpu
);
1537 /* interrupts now hard-disabled */
1539 /* Save FPU, Altivec and VSX state */
1540 giveup_all(current
);
1542 /* Preload FPU if it's enabled */
1543 if (kvmppc_get_msr(vcpu
) & MSR_FP
)
1544 kvmppc_handle_ext(vcpu
, BOOK3S_INTERRUPT_FP_UNAVAIL
, MSR_FP
);
1546 kvmppc_fix_ee_before_entry();
1548 ret
= __kvmppc_vcpu_run(kvm_run
, vcpu
);
1550 kvmppc_clear_debug(vcpu
);
1552 /* No need for guest_exit. It's done in handle_exit.
1553 We also get here with interrupts enabled. */
1555 /* Make sure we save the guest FPU/Altivec/VSX state */
1556 kvmppc_giveup_ext(vcpu
, MSR_FP
| MSR_VEC
| MSR_VSX
);
1558 /* Make sure we save the guest TAR/EBB/DSCR state */
1559 kvmppc_giveup_fac(vcpu
, FSCR_TAR_LG
);
1562 vcpu
->mode
= OUTSIDE_GUEST_MODE
;
1567 * Get (and clear) the dirty memory log for a memory slot.
1569 static int kvm_vm_ioctl_get_dirty_log_pr(struct kvm
*kvm
,
1570 struct kvm_dirty_log
*log
)
1572 struct kvm_memslots
*slots
;
1573 struct kvm_memory_slot
*memslot
;
1574 struct kvm_vcpu
*vcpu
;
1580 mutex_lock(&kvm
->slots_lock
);
1582 r
= kvm_get_dirty_log(kvm
, log
, &is_dirty
);
1586 /* If nothing is dirty, don't bother messing with page tables. */
1588 slots
= kvm_memslots(kvm
);
1589 memslot
= id_to_memslot(slots
, log
->slot
);
1591 ga
= memslot
->base_gfn
<< PAGE_SHIFT
;
1592 ga_end
= ga
+ (memslot
->npages
<< PAGE_SHIFT
);
1594 kvm_for_each_vcpu(n
, vcpu
, kvm
)
1595 kvmppc_mmu_pte_pflush(vcpu
, ga
, ga_end
);
1597 n
= kvm_dirty_bitmap_bytes(memslot
);
1598 memset(memslot
->dirty_bitmap
, 0, n
);
1603 mutex_unlock(&kvm
->slots_lock
);
1607 static void kvmppc_core_flush_memslot_pr(struct kvm
*kvm
,
1608 struct kvm_memory_slot
*memslot
)
1613 static int kvmppc_core_prepare_memory_region_pr(struct kvm
*kvm
,
1614 struct kvm_memory_slot
*memslot
,
1615 const struct kvm_userspace_memory_region
*mem
)
1620 static void kvmppc_core_commit_memory_region_pr(struct kvm
*kvm
,
1621 const struct kvm_userspace_memory_region
*mem
,
1622 const struct kvm_memory_slot
*old
,
1623 const struct kvm_memory_slot
*new)
1628 static void kvmppc_core_free_memslot_pr(struct kvm_memory_slot
*free
,
1629 struct kvm_memory_slot
*dont
)
1634 static int kvmppc_core_create_memslot_pr(struct kvm_memory_slot
*slot
,
1635 unsigned long npages
)
1642 static int kvm_vm_ioctl_get_smmu_info_pr(struct kvm
*kvm
,
1643 struct kvm_ppc_smmu_info
*info
)
1646 struct kvm_vcpu
*vcpu
;
1650 /* SLB is always 64 entries */
1651 info
->slb_size
= 64;
1653 /* Standard 4k base page size segment */
1654 info
->sps
[0].page_shift
= 12;
1655 info
->sps
[0].slb_enc
= 0;
1656 info
->sps
[0].enc
[0].page_shift
= 12;
1657 info
->sps
[0].enc
[0].pte_enc
= 0;
1660 * 64k large page size.
1661 * We only want to put this in if the CPUs we're emulating
1662 * support it, but unfortunately we don't have a vcpu easily
1663 * to hand here to test. Just pick the first vcpu, and if
1664 * that doesn't exist yet, report the minimum capability,
1665 * i.e., no 64k pages.
1666 * 1T segment support goes along with 64k pages.
1669 vcpu
= kvm_get_vcpu(kvm
, 0);
1670 if (vcpu
&& (vcpu
->arch
.hflags
& BOOK3S_HFLAG_MULTI_PGSIZE
)) {
1671 info
->flags
= KVM_PPC_1T_SEGMENTS
;
1672 info
->sps
[i
].page_shift
= 16;
1673 info
->sps
[i
].slb_enc
= SLB_VSID_L
| SLB_VSID_LP_01
;
1674 info
->sps
[i
].enc
[0].page_shift
= 16;
1675 info
->sps
[i
].enc
[0].pte_enc
= 1;
1679 /* Standard 16M large page size segment */
1680 info
->sps
[i
].page_shift
= 24;
1681 info
->sps
[i
].slb_enc
= SLB_VSID_L
;
1682 info
->sps
[i
].enc
[0].page_shift
= 24;
1683 info
->sps
[i
].enc
[0].pte_enc
= 0;
1688 static int kvm_vm_ioctl_get_smmu_info_pr(struct kvm
*kvm
,
1689 struct kvm_ppc_smmu_info
*info
)
1691 /* We should not get called */
1694 #endif /* CONFIG_PPC64 */
1696 static unsigned int kvm_global_user_count
= 0;
1697 static DEFINE_SPINLOCK(kvm_global_user_count_lock
);
1699 static int kvmppc_core_init_vm_pr(struct kvm
*kvm
)
1701 mutex_init(&kvm
->arch
.hpt_mutex
);
1703 #ifdef CONFIG_PPC_BOOK3S_64
1704 /* Start out with the default set of hcalls enabled */
1705 kvmppc_pr_init_default_hcalls(kvm
);
1708 if (firmware_has_feature(FW_FEATURE_SET_MODE
)) {
1709 spin_lock(&kvm_global_user_count_lock
);
1710 if (++kvm_global_user_count
== 1)
1711 pseries_disable_reloc_on_exc();
1712 spin_unlock(&kvm_global_user_count_lock
);
1717 static void kvmppc_core_destroy_vm_pr(struct kvm
*kvm
)
1720 WARN_ON(!list_empty(&kvm
->arch
.spapr_tce_tables
));
1723 if (firmware_has_feature(FW_FEATURE_SET_MODE
)) {
1724 spin_lock(&kvm_global_user_count_lock
);
1725 BUG_ON(kvm_global_user_count
== 0);
1726 if (--kvm_global_user_count
== 0)
1727 pseries_enable_reloc_on_exc();
1728 spin_unlock(&kvm_global_user_count_lock
);
1732 static int kvmppc_core_check_processor_compat_pr(void)
1735 * Disable KVM for Power9 untill the required bits merged.
1737 if (cpu_has_feature(CPU_FTR_ARCH_300
))
1742 static long kvm_arch_vm_ioctl_pr(struct file
*filp
,
1743 unsigned int ioctl
, unsigned long arg
)
1748 static struct kvmppc_ops kvm_ops_pr
= {
1749 .get_sregs
= kvm_arch_vcpu_ioctl_get_sregs_pr
,
1750 .set_sregs
= kvm_arch_vcpu_ioctl_set_sregs_pr
,
1751 .get_one_reg
= kvmppc_get_one_reg_pr
,
1752 .set_one_reg
= kvmppc_set_one_reg_pr
,
1753 .vcpu_load
= kvmppc_core_vcpu_load_pr
,
1754 .vcpu_put
= kvmppc_core_vcpu_put_pr
,
1755 .set_msr
= kvmppc_set_msr_pr
,
1756 .vcpu_run
= kvmppc_vcpu_run_pr
,
1757 .vcpu_create
= kvmppc_core_vcpu_create_pr
,
1758 .vcpu_free
= kvmppc_core_vcpu_free_pr
,
1759 .check_requests
= kvmppc_core_check_requests_pr
,
1760 .get_dirty_log
= kvm_vm_ioctl_get_dirty_log_pr
,
1761 .flush_memslot
= kvmppc_core_flush_memslot_pr
,
1762 .prepare_memory_region
= kvmppc_core_prepare_memory_region_pr
,
1763 .commit_memory_region
= kvmppc_core_commit_memory_region_pr
,
1764 .unmap_hva
= kvm_unmap_hva_pr
,
1765 .unmap_hva_range
= kvm_unmap_hva_range_pr
,
1766 .age_hva
= kvm_age_hva_pr
,
1767 .test_age_hva
= kvm_test_age_hva_pr
,
1768 .set_spte_hva
= kvm_set_spte_hva_pr
,
1769 .mmu_destroy
= kvmppc_mmu_destroy_pr
,
1770 .free_memslot
= kvmppc_core_free_memslot_pr
,
1771 .create_memslot
= kvmppc_core_create_memslot_pr
,
1772 .init_vm
= kvmppc_core_init_vm_pr
,
1773 .destroy_vm
= kvmppc_core_destroy_vm_pr
,
1774 .get_smmu_info
= kvm_vm_ioctl_get_smmu_info_pr
,
1775 .emulate_op
= kvmppc_core_emulate_op_pr
,
1776 .emulate_mtspr
= kvmppc_core_emulate_mtspr_pr
,
1777 .emulate_mfspr
= kvmppc_core_emulate_mfspr_pr
,
1778 .fast_vcpu_kick
= kvm_vcpu_kick
,
1779 .arch_vm_ioctl
= kvm_arch_vm_ioctl_pr
,
1780 #ifdef CONFIG_PPC_BOOK3S_64
1781 .hcall_implemented
= kvmppc_hcall_impl_pr
,
1786 int kvmppc_book3s_init_pr(void)
1790 r
= kvmppc_core_check_processor_compat_pr();
1794 kvm_ops_pr
.owner
= THIS_MODULE
;
1795 kvmppc_pr_ops
= &kvm_ops_pr
;
1797 r
= kvmppc_mmu_hpte_sysinit();
1801 void kvmppc_book3s_exit_pr(void)
1803 kvmppc_pr_ops
= NULL
;
1804 kvmppc_mmu_hpte_sysexit();
1808 * We only support separate modules for book3s 64
1810 #ifdef CONFIG_PPC_BOOK3S_64
1812 module_init(kvmppc_book3s_init_pr
);
1813 module_exit(kvmppc_book3s_exit_pr
);
1815 MODULE_LICENSE("GPL");
1816 MODULE_ALIAS_MISCDEV(KVM_MINOR
);
1817 MODULE_ALIAS("devname:kvm");