2 * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved.
5 * Alexander Graf <agraf@suse.de>
6 * Kevin Wolf <mail@kevin-wolf.de>
7 * Paul Mackerras <paulus@samba.org>
10 * Functions relating to running KVM on Book 3S processors where
11 * we don't have access to hypervisor mode, and we run the guest
12 * in problem state (user mode).
14 * This file is derived from arch/powerpc/kvm/44x.c,
15 * by Hollis Blanchard <hollisb@us.ibm.com>.
17 * This program is free software; you can redistribute it and/or modify
18 * it under the terms of the GNU General Public License, version 2, as
19 * published by the Free Software Foundation.
22 #include <linux/kvm_host.h>
23 #include <linux/export.h>
24 #include <linux/err.h>
25 #include <linux/slab.h>
28 #include <asm/cputable.h>
29 #include <asm/cacheflush.h>
30 #include <asm/tlbflush.h>
31 #include <linux/uaccess.h>
33 #include <asm/kvm_ppc.h>
34 #include <asm/kvm_book3s.h>
35 #include <asm/mmu_context.h>
36 #include <asm/switch_to.h>
37 #include <asm/firmware.h>
38 #include <asm/setup.h>
39 #include <linux/gfp.h>
40 #include <linux/sched.h>
41 #include <linux/vmalloc.h>
42 #include <linux/highmem.h>
43 #include <linux/module.h>
44 #include <linux/miscdevice.h>
48 #define CREATE_TRACE_POINTS
51 /* #define EXIT_DEBUG */
52 /* #define DEBUG_EXT */
54 static int kvmppc_handle_ext(struct kvm_vcpu
*vcpu
, unsigned int exit_nr
,
56 static void kvmppc_giveup_fac(struct kvm_vcpu
*vcpu
, ulong fac
);
58 /* Some compatibility defines */
59 #ifdef CONFIG_PPC_BOOK3S_32
60 #define MSR_USER32 MSR_USER
61 #define MSR_USER64 MSR_USER
62 #define HW_PAGE_SIZE PAGE_SIZE
63 #define HPTE_R_M _PAGE_COHERENT
66 static bool kvmppc_is_split_real(struct kvm_vcpu
*vcpu
)
68 ulong msr
= kvmppc_get_msr(vcpu
);
69 return (msr
& (MSR_IR
|MSR_DR
)) == MSR_DR
;
72 static void kvmppc_fixup_split_real(struct kvm_vcpu
*vcpu
)
74 ulong msr
= kvmppc_get_msr(vcpu
);
75 ulong pc
= kvmppc_get_pc(vcpu
);
77 /* We are in DR only split real mode */
78 if ((msr
& (MSR_IR
|MSR_DR
)) != MSR_DR
)
81 /* We have not fixed up the guest already */
82 if (vcpu
->arch
.hflags
& BOOK3S_HFLAG_SPLIT_HACK
)
85 /* The code is in fixupable address space */
86 if (pc
& SPLIT_HACK_MASK
)
89 vcpu
->arch
.hflags
|= BOOK3S_HFLAG_SPLIT_HACK
;
90 kvmppc_set_pc(vcpu
, pc
| SPLIT_HACK_OFFS
);
93 void kvmppc_unfixup_split_real(struct kvm_vcpu
*vcpu
);
95 static void kvmppc_core_vcpu_load_pr(struct kvm_vcpu
*vcpu
, int cpu
)
97 #ifdef CONFIG_PPC_BOOK3S_64
98 struct kvmppc_book3s_shadow_vcpu
*svcpu
= svcpu_get(vcpu
);
99 memcpy(svcpu
->slb
, to_book3s(vcpu
)->slb_shadow
, sizeof(svcpu
->slb
));
100 svcpu
->slb_max
= to_book3s(vcpu
)->slb_shadow_max
;
105 /* Disable AIL if supported */
106 if (cpu_has_feature(CPU_FTR_HVMODE
) &&
107 cpu_has_feature(CPU_FTR_ARCH_207S
))
108 mtspr(SPRN_LPCR
, mfspr(SPRN_LPCR
) & ~LPCR_AIL
);
110 vcpu
->cpu
= smp_processor_id();
111 #ifdef CONFIG_PPC_BOOK3S_32
112 current
->thread
.kvm_shadow_vcpu
= vcpu
->arch
.shadow_vcpu
;
115 if (kvmppc_is_split_real(vcpu
))
116 kvmppc_fixup_split_real(vcpu
);
119 static void kvmppc_core_vcpu_put_pr(struct kvm_vcpu
*vcpu
)
121 #ifdef CONFIG_PPC_BOOK3S_64
122 struct kvmppc_book3s_shadow_vcpu
*svcpu
= svcpu_get(vcpu
);
124 kvmppc_copy_from_svcpu(vcpu
, svcpu
);
126 memcpy(to_book3s(vcpu
)->slb_shadow
, svcpu
->slb
, sizeof(svcpu
->slb
));
127 to_book3s(vcpu
)->slb_shadow_max
= svcpu
->slb_max
;
131 if (kvmppc_is_split_real(vcpu
))
132 kvmppc_unfixup_split_real(vcpu
);
134 kvmppc_giveup_ext(vcpu
, MSR_FP
| MSR_VEC
| MSR_VSX
);
135 kvmppc_giveup_fac(vcpu
, FSCR_TAR_LG
);
137 /* Enable AIL if supported */
138 if (cpu_has_feature(CPU_FTR_HVMODE
) &&
139 cpu_has_feature(CPU_FTR_ARCH_207S
))
140 mtspr(SPRN_LPCR
, mfspr(SPRN_LPCR
) | LPCR_AIL_3
);
145 /* Copy data needed by real-mode code from vcpu to shadow vcpu */
146 void kvmppc_copy_to_svcpu(struct kvmppc_book3s_shadow_vcpu
*svcpu
,
147 struct kvm_vcpu
*vcpu
)
149 svcpu
->gpr
[0] = vcpu
->arch
.gpr
[0];
150 svcpu
->gpr
[1] = vcpu
->arch
.gpr
[1];
151 svcpu
->gpr
[2] = vcpu
->arch
.gpr
[2];
152 svcpu
->gpr
[3] = vcpu
->arch
.gpr
[3];
153 svcpu
->gpr
[4] = vcpu
->arch
.gpr
[4];
154 svcpu
->gpr
[5] = vcpu
->arch
.gpr
[5];
155 svcpu
->gpr
[6] = vcpu
->arch
.gpr
[6];
156 svcpu
->gpr
[7] = vcpu
->arch
.gpr
[7];
157 svcpu
->gpr
[8] = vcpu
->arch
.gpr
[8];
158 svcpu
->gpr
[9] = vcpu
->arch
.gpr
[9];
159 svcpu
->gpr
[10] = vcpu
->arch
.gpr
[10];
160 svcpu
->gpr
[11] = vcpu
->arch
.gpr
[11];
161 svcpu
->gpr
[12] = vcpu
->arch
.gpr
[12];
162 svcpu
->gpr
[13] = vcpu
->arch
.gpr
[13];
163 svcpu
->cr
= vcpu
->arch
.cr
;
164 svcpu
->xer
= vcpu
->arch
.xer
;
165 svcpu
->ctr
= vcpu
->arch
.ctr
;
166 svcpu
->lr
= vcpu
->arch
.lr
;
167 svcpu
->pc
= vcpu
->arch
.pc
;
168 #ifdef CONFIG_PPC_BOOK3S_64
169 svcpu
->shadow_fscr
= vcpu
->arch
.shadow_fscr
;
172 * Now also save the current time base value. We use this
173 * to find the guest purr and spurr value.
175 vcpu
->arch
.entry_tb
= get_tb();
176 vcpu
->arch
.entry_vtb
= get_vtb();
177 if (cpu_has_feature(CPU_FTR_ARCH_207S
))
178 vcpu
->arch
.entry_ic
= mfspr(SPRN_IC
);
179 svcpu
->in_use
= true;
182 /* Copy data touched by real-mode code from shadow vcpu back to vcpu */
183 void kvmppc_copy_from_svcpu(struct kvm_vcpu
*vcpu
,
184 struct kvmppc_book3s_shadow_vcpu
*svcpu
)
187 * vcpu_put would just call us again because in_use hasn't
193 * Maybe we were already preempted and synced the svcpu from
194 * our preempt notifiers. Don't bother touching this svcpu then.
199 vcpu
->arch
.gpr
[0] = svcpu
->gpr
[0];
200 vcpu
->arch
.gpr
[1] = svcpu
->gpr
[1];
201 vcpu
->arch
.gpr
[2] = svcpu
->gpr
[2];
202 vcpu
->arch
.gpr
[3] = svcpu
->gpr
[3];
203 vcpu
->arch
.gpr
[4] = svcpu
->gpr
[4];
204 vcpu
->arch
.gpr
[5] = svcpu
->gpr
[5];
205 vcpu
->arch
.gpr
[6] = svcpu
->gpr
[6];
206 vcpu
->arch
.gpr
[7] = svcpu
->gpr
[7];
207 vcpu
->arch
.gpr
[8] = svcpu
->gpr
[8];
208 vcpu
->arch
.gpr
[9] = svcpu
->gpr
[9];
209 vcpu
->arch
.gpr
[10] = svcpu
->gpr
[10];
210 vcpu
->arch
.gpr
[11] = svcpu
->gpr
[11];
211 vcpu
->arch
.gpr
[12] = svcpu
->gpr
[12];
212 vcpu
->arch
.gpr
[13] = svcpu
->gpr
[13];
213 vcpu
->arch
.cr
= svcpu
->cr
;
214 vcpu
->arch
.xer
= svcpu
->xer
;
215 vcpu
->arch
.ctr
= svcpu
->ctr
;
216 vcpu
->arch
.lr
= svcpu
->lr
;
217 vcpu
->arch
.pc
= svcpu
->pc
;
218 vcpu
->arch
.shadow_srr1
= svcpu
->shadow_srr1
;
219 vcpu
->arch
.fault_dar
= svcpu
->fault_dar
;
220 vcpu
->arch
.fault_dsisr
= svcpu
->fault_dsisr
;
221 vcpu
->arch
.last_inst
= svcpu
->last_inst
;
222 #ifdef CONFIG_PPC_BOOK3S_64
223 vcpu
->arch
.shadow_fscr
= svcpu
->shadow_fscr
;
226 * Update purr and spurr using time base on exit.
228 vcpu
->arch
.purr
+= get_tb() - vcpu
->arch
.entry_tb
;
229 vcpu
->arch
.spurr
+= get_tb() - vcpu
->arch
.entry_tb
;
230 to_book3s(vcpu
)->vtb
+= get_vtb() - vcpu
->arch
.entry_vtb
;
231 if (cpu_has_feature(CPU_FTR_ARCH_207S
))
232 vcpu
->arch
.ic
+= mfspr(SPRN_IC
) - vcpu
->arch
.entry_ic
;
233 svcpu
->in_use
= false;
239 static int kvmppc_core_check_requests_pr(struct kvm_vcpu
*vcpu
)
241 int r
= 1; /* Indicate we want to get back into the guest */
243 /* We misuse TLB_FLUSH to indicate that we want to clear
244 all shadow cache entries */
245 if (kvm_check_request(KVM_REQ_TLB_FLUSH
, vcpu
))
246 kvmppc_mmu_pte_flush(vcpu
, 0, 0);
251 /************* MMU Notifiers *************/
252 static void do_kvm_unmap_hva(struct kvm
*kvm
, unsigned long start
,
256 struct kvm_vcpu
*vcpu
;
257 struct kvm_memslots
*slots
;
258 struct kvm_memory_slot
*memslot
;
260 slots
= kvm_memslots(kvm
);
261 kvm_for_each_memslot(memslot
, slots
) {
262 unsigned long hva_start
, hva_end
;
265 hva_start
= max(start
, memslot
->userspace_addr
);
266 hva_end
= min(end
, memslot
->userspace_addr
+
267 (memslot
->npages
<< PAGE_SHIFT
));
268 if (hva_start
>= hva_end
)
271 * {gfn(page) | page intersects with [hva_start, hva_end)} =
272 * {gfn, gfn+1, ..., gfn_end-1}.
274 gfn
= hva_to_gfn_memslot(hva_start
, memslot
);
275 gfn_end
= hva_to_gfn_memslot(hva_end
+ PAGE_SIZE
- 1, memslot
);
276 kvm_for_each_vcpu(i
, vcpu
, kvm
)
277 kvmppc_mmu_pte_pflush(vcpu
, gfn
<< PAGE_SHIFT
,
278 gfn_end
<< PAGE_SHIFT
);
282 static int kvm_unmap_hva_pr(struct kvm
*kvm
, unsigned long hva
)
284 trace_kvm_unmap_hva(hva
);
286 do_kvm_unmap_hva(kvm
, hva
, hva
+ PAGE_SIZE
);
291 static int kvm_unmap_hva_range_pr(struct kvm
*kvm
, unsigned long start
,
294 do_kvm_unmap_hva(kvm
, start
, end
);
299 static int kvm_age_hva_pr(struct kvm
*kvm
, unsigned long start
,
302 /* XXX could be more clever ;) */
306 static int kvm_test_age_hva_pr(struct kvm
*kvm
, unsigned long hva
)
308 /* XXX could be more clever ;) */
312 static void kvm_set_spte_hva_pr(struct kvm
*kvm
, unsigned long hva
, pte_t pte
)
314 /* The page will get remapped properly on its next fault */
315 do_kvm_unmap_hva(kvm
, hva
, hva
+ PAGE_SIZE
);
318 /*****************************************/
320 static void kvmppc_recalc_shadow_msr(struct kvm_vcpu
*vcpu
)
322 ulong guest_msr
= kvmppc_get_msr(vcpu
);
323 ulong smsr
= guest_msr
;
325 /* Guest MSR values */
326 smsr
&= MSR_FE0
| MSR_FE1
| MSR_SF
| MSR_SE
| MSR_BE
| MSR_LE
;
327 /* Process MSR values */
328 smsr
|= MSR_ME
| MSR_RI
| MSR_IR
| MSR_DR
| MSR_PR
| MSR_EE
;
329 /* External providers the guest reserved */
330 smsr
|= (guest_msr
& vcpu
->arch
.guest_owned_ext
);
331 /* 64-bit Process MSR values */
332 #ifdef CONFIG_PPC_BOOK3S_64
333 smsr
|= MSR_ISF
| MSR_HV
;
335 vcpu
->arch
.shadow_msr
= smsr
;
338 static void kvmppc_set_msr_pr(struct kvm_vcpu
*vcpu
, u64 msr
)
340 ulong old_msr
= kvmppc_get_msr(vcpu
);
343 printk(KERN_INFO
"KVM: Set MSR to 0x%llx\n", msr
);
346 msr
&= to_book3s(vcpu
)->msr_mask
;
347 kvmppc_set_msr_fast(vcpu
, msr
);
348 kvmppc_recalc_shadow_msr(vcpu
);
351 if (!vcpu
->arch
.pending_exceptions
) {
352 kvm_vcpu_block(vcpu
);
353 kvm_clear_request(KVM_REQ_UNHALT
, vcpu
);
354 vcpu
->stat
.halt_wakeup
++;
356 /* Unset POW bit after we woke up */
358 kvmppc_set_msr_fast(vcpu
, msr
);
362 if (kvmppc_is_split_real(vcpu
))
363 kvmppc_fixup_split_real(vcpu
);
365 kvmppc_unfixup_split_real(vcpu
);
367 if ((kvmppc_get_msr(vcpu
) & (MSR_PR
|MSR_IR
|MSR_DR
)) !=
368 (old_msr
& (MSR_PR
|MSR_IR
|MSR_DR
))) {
369 kvmppc_mmu_flush_segments(vcpu
);
370 kvmppc_mmu_map_segment(vcpu
, kvmppc_get_pc(vcpu
));
372 /* Preload magic page segment when in kernel mode */
373 if (!(msr
& MSR_PR
) && vcpu
->arch
.magic_page_pa
) {
374 struct kvm_vcpu_arch
*a
= &vcpu
->arch
;
377 kvmppc_mmu_map_segment(vcpu
, a
->magic_page_ea
);
379 kvmppc_mmu_map_segment(vcpu
, a
->magic_page_pa
);
384 * When switching from 32 to 64-bit, we may have a stale 32-bit
385 * magic page around, we need to flush it. Typically 32-bit magic
386 * page will be instanciated when calling into RTAS. Note: We
387 * assume that such transition only happens while in kernel mode,
388 * ie, we never transition from user 32-bit to kernel 64-bit with
389 * a 32-bit magic page around.
391 if (vcpu
->arch
.magic_page_pa
&&
392 !(old_msr
& MSR_PR
) && !(old_msr
& MSR_SF
) && (msr
& MSR_SF
)) {
393 /* going from RTAS to normal kernel code */
394 kvmppc_mmu_pte_flush(vcpu
, (uint32_t)vcpu
->arch
.magic_page_pa
,
398 /* Preload FPU if it's enabled */
399 if (kvmppc_get_msr(vcpu
) & MSR_FP
)
400 kvmppc_handle_ext(vcpu
, BOOK3S_INTERRUPT_FP_UNAVAIL
, MSR_FP
);
403 void kvmppc_set_pvr_pr(struct kvm_vcpu
*vcpu
, u32 pvr
)
407 vcpu
->arch
.hflags
&= ~BOOK3S_HFLAG_SLB
;
408 vcpu
->arch
.pvr
= pvr
;
409 #ifdef CONFIG_PPC_BOOK3S_64
410 if ((pvr
>= 0x330000) && (pvr
< 0x70330000)) {
411 kvmppc_mmu_book3s_64_init(vcpu
);
412 if (!to_book3s(vcpu
)->hior_explicit
)
413 to_book3s(vcpu
)->hior
= 0xfff00000;
414 to_book3s(vcpu
)->msr_mask
= 0xffffffffffffffffULL
;
415 vcpu
->arch
.cpu_type
= KVM_CPU_3S_64
;
419 kvmppc_mmu_book3s_32_init(vcpu
);
420 if (!to_book3s(vcpu
)->hior_explicit
)
421 to_book3s(vcpu
)->hior
= 0;
422 to_book3s(vcpu
)->msr_mask
= 0xffffffffULL
;
423 vcpu
->arch
.cpu_type
= KVM_CPU_3S_32
;
426 kvmppc_sanity_check(vcpu
);
428 /* If we are in hypervisor level on 970, we can tell the CPU to
429 * treat DCBZ as 32 bytes store */
430 vcpu
->arch
.hflags
&= ~BOOK3S_HFLAG_DCBZ32
;
431 if (vcpu
->arch
.mmu
.is_dcbz32(vcpu
) && (mfmsr() & MSR_HV
) &&
432 !strcmp(cur_cpu_spec
->platform
, "ppc970"))
433 vcpu
->arch
.hflags
|= BOOK3S_HFLAG_DCBZ32
;
435 /* Cell performs badly if MSR_FEx are set. So let's hope nobody
436 really needs them in a VM on Cell and force disable them. */
437 if (!strcmp(cur_cpu_spec
->platform
, "ppc-cell-be"))
438 to_book3s(vcpu
)->msr_mask
&= ~(MSR_FE0
| MSR_FE1
);
441 * If they're asking for POWER6 or later, set the flag
442 * indicating that we can do multiple large page sizes
444 * Also set the flag that indicates that tlbie has the large
445 * page bit in the RB operand instead of the instruction.
447 switch (PVR_VER(pvr
)) {
454 vcpu
->arch
.hflags
|= BOOK3S_HFLAG_MULTI_PGSIZE
|
455 BOOK3S_HFLAG_NEW_TLBIE
;
459 #ifdef CONFIG_PPC_BOOK3S_32
460 /* 32 bit Book3S always has 32 byte dcbz */
461 vcpu
->arch
.hflags
|= BOOK3S_HFLAG_DCBZ32
;
464 /* On some CPUs we can execute paired single operations natively */
465 asm ( "mfpvr %0" : "=r"(host_pvr
));
467 case 0x00080200: /* lonestar 2.0 */
468 case 0x00088202: /* lonestar 2.2 */
469 case 0x70000100: /* gekko 1.0 */
470 case 0x00080100: /* gekko 2.0 */
471 case 0x00083203: /* gekko 2.3a */
472 case 0x00083213: /* gekko 2.3b */
473 case 0x00083204: /* gekko 2.4 */
474 case 0x00083214: /* gekko 2.4e (8SE) - retail HW2 */
475 case 0x00087200: /* broadway */
476 vcpu
->arch
.hflags
|= BOOK3S_HFLAG_NATIVE_PS
;
477 /* Enable HID2.PSE - in case we need it later */
478 mtspr(SPRN_HID2_GEKKO
, mfspr(SPRN_HID2_GEKKO
) | (1 << 29));
482 /* Book3s_32 CPUs always have 32 bytes cache line size, which Linux assumes. To
483 * make Book3s_32 Linux work on Book3s_64, we have to make sure we trap dcbz to
484 * emulate 32 bytes dcbz length.
486 * The Book3s_64 inventors also realized this case and implemented a special bit
487 * in the HID5 register, which is a hypervisor ressource. Thus we can't use it.
489 * My approach here is to patch the dcbz instruction on executing pages.
491 static void kvmppc_patch_dcbz(struct kvm_vcpu
*vcpu
, struct kvmppc_pte
*pte
)
498 hpage
= gfn_to_page(vcpu
->kvm
, pte
->raddr
>> PAGE_SHIFT
);
499 if (is_error_page(hpage
))
502 hpage_offset
= pte
->raddr
& ~PAGE_MASK
;
503 hpage_offset
&= ~0xFFFULL
;
507 page
= kmap_atomic(hpage
);
509 /* patch dcbz into reserved instruction, so we trap */
510 for (i
=hpage_offset
; i
< hpage_offset
+ (HW_PAGE_SIZE
/ 4); i
++)
511 if ((be32_to_cpu(page
[i
]) & 0xff0007ff) == INS_DCBZ
)
512 page
[i
] &= cpu_to_be32(0xfffffff7);
518 static bool kvmppc_visible_gpa(struct kvm_vcpu
*vcpu
, gpa_t gpa
)
520 ulong mp_pa
= vcpu
->arch
.magic_page_pa
;
522 if (!(kvmppc_get_msr(vcpu
) & MSR_SF
))
523 mp_pa
= (uint32_t)mp_pa
;
526 if (unlikely(mp_pa
) && unlikely((mp_pa
& KVM_PAM
) == (gpa
& KVM_PAM
))) {
530 return kvm_is_visible_gfn(vcpu
->kvm
, gpa
>> PAGE_SHIFT
);
533 int kvmppc_handle_pagefault(struct kvm_run
*run
, struct kvm_vcpu
*vcpu
,
534 ulong eaddr
, int vec
)
536 bool data
= (vec
== BOOK3S_INTERRUPT_DATA_STORAGE
);
537 bool iswrite
= false;
538 int r
= RESUME_GUEST
;
541 struct kvmppc_pte pte
= { 0 };
542 bool dr
= (kvmppc_get_msr(vcpu
) & MSR_DR
) ? true : false;
543 bool ir
= (kvmppc_get_msr(vcpu
) & MSR_IR
) ? true : false;
546 relocated
= data
? dr
: ir
;
547 if (data
&& (vcpu
->arch
.fault_dsisr
& DSISR_ISSTORE
))
550 /* Resolve real address if translation turned on */
552 page_found
= vcpu
->arch
.mmu
.xlate(vcpu
, eaddr
, &pte
, data
, iswrite
);
554 pte
.may_execute
= true;
556 pte
.may_write
= true;
557 pte
.raddr
= eaddr
& KVM_PAM
;
559 pte
.vpage
= eaddr
>> 12;
560 pte
.page_size
= MMU_PAGE_64K
;
564 switch (kvmppc_get_msr(vcpu
) & (MSR_DR
|MSR_IR
)) {
566 pte
.vpage
|= ((u64
)VSID_REAL
<< (SID_SHIFT
- 12));
570 (vcpu
->arch
.hflags
& BOOK3S_HFLAG_SPLIT_HACK
) &&
571 ((pte
.raddr
& SPLIT_HACK_MASK
) == SPLIT_HACK_OFFS
))
572 pte
.raddr
&= ~SPLIT_HACK_MASK
;
575 vcpu
->arch
.mmu
.esid_to_vsid(vcpu
, eaddr
>> SID_SHIFT
, &vsid
);
577 if ((kvmppc_get_msr(vcpu
) & (MSR_DR
|MSR_IR
)) == MSR_DR
)
578 pte
.vpage
|= ((u64
)VSID_REAL_DR
<< (SID_SHIFT
- 12));
580 pte
.vpage
|= ((u64
)VSID_REAL_IR
<< (SID_SHIFT
- 12));
584 page_found
= -EINVAL
;
588 if (vcpu
->arch
.mmu
.is_dcbz32(vcpu
) &&
589 (!(vcpu
->arch
.hflags
& BOOK3S_HFLAG_DCBZ32
))) {
591 * If we do the dcbz hack, we have to NX on every execution,
592 * so we can patch the executing code. This renders our guest
595 pte
.may_execute
= !data
;
598 if (page_found
== -ENOENT
) {
599 /* Page not found in guest PTE entries */
600 u64 ssrr1
= vcpu
->arch
.shadow_srr1
;
601 u64 msr
= kvmppc_get_msr(vcpu
);
602 kvmppc_set_dar(vcpu
, kvmppc_get_fault_dar(vcpu
));
603 kvmppc_set_dsisr(vcpu
, vcpu
->arch
.fault_dsisr
);
604 kvmppc_set_msr_fast(vcpu
, msr
| (ssrr1
& 0xf8000000ULL
));
605 kvmppc_book3s_queue_irqprio(vcpu
, vec
);
606 } else if (page_found
== -EPERM
) {
607 /* Storage protection */
608 u32 dsisr
= vcpu
->arch
.fault_dsisr
;
609 u64 ssrr1
= vcpu
->arch
.shadow_srr1
;
610 u64 msr
= kvmppc_get_msr(vcpu
);
611 kvmppc_set_dar(vcpu
, kvmppc_get_fault_dar(vcpu
));
612 dsisr
= (dsisr
& ~DSISR_NOHPTE
) | DSISR_PROTFAULT
;
613 kvmppc_set_dsisr(vcpu
, dsisr
);
614 kvmppc_set_msr_fast(vcpu
, msr
| (ssrr1
& 0xf8000000ULL
));
615 kvmppc_book3s_queue_irqprio(vcpu
, vec
);
616 } else if (page_found
== -EINVAL
) {
617 /* Page not found in guest SLB */
618 kvmppc_set_dar(vcpu
, kvmppc_get_fault_dar(vcpu
));
619 kvmppc_book3s_queue_irqprio(vcpu
, vec
+ 0x80);
620 } else if (kvmppc_visible_gpa(vcpu
, pte
.raddr
)) {
621 if (data
&& !(vcpu
->arch
.fault_dsisr
& DSISR_NOHPTE
)) {
623 * There is already a host HPTE there, presumably
624 * a read-only one for a page the guest thinks
625 * is writable, so get rid of it first.
627 kvmppc_mmu_unmap_page(vcpu
, &pte
);
629 /* The guest's PTE is not mapped yet. Map on the host */
630 if (kvmppc_mmu_map_page(vcpu
, &pte
, iswrite
) == -EIO
) {
631 /* Exit KVM if mapping failed */
632 run
->exit_reason
= KVM_EXIT_INTERNAL_ERROR
;
636 vcpu
->stat
.sp_storage
++;
637 else if (vcpu
->arch
.mmu
.is_dcbz32(vcpu
) &&
638 (!(vcpu
->arch
.hflags
& BOOK3S_HFLAG_DCBZ32
)))
639 kvmppc_patch_dcbz(vcpu
, &pte
);
642 vcpu
->stat
.mmio_exits
++;
643 vcpu
->arch
.paddr_accessed
= pte
.raddr
;
644 vcpu
->arch
.vaddr_accessed
= pte
.eaddr
;
645 r
= kvmppc_emulate_mmio(run
, vcpu
);
646 if ( r
== RESUME_HOST_NV
)
653 /* Give up external provider (FPU, Altivec, VSX) */
654 void kvmppc_giveup_ext(struct kvm_vcpu
*vcpu
, ulong msr
)
656 struct thread_struct
*t
= ¤t
->thread
;
659 * VSX instructions can access FP and vector registers, so if
660 * we are giving up VSX, make sure we give up FP and VMX as well.
663 msr
|= MSR_FP
| MSR_VEC
;
665 msr
&= vcpu
->arch
.guest_owned_ext
;
670 printk(KERN_INFO
"Giving up ext 0x%lx\n", msr
);
675 * Note that on CPUs with VSX, giveup_fpu stores
676 * both the traditional FP registers and the added VSX
677 * registers into thread.fp_state.fpr[].
679 if (t
->regs
->msr
& MSR_FP
)
681 t
->fp_save_area
= NULL
;
684 #ifdef CONFIG_ALTIVEC
686 if (current
->thread
.regs
->msr
& MSR_VEC
)
687 giveup_altivec(current
);
688 t
->vr_save_area
= NULL
;
692 vcpu
->arch
.guest_owned_ext
&= ~(msr
| MSR_VSX
);
693 kvmppc_recalc_shadow_msr(vcpu
);
696 /* Give up facility (TAR / EBB / DSCR) */
697 static void kvmppc_giveup_fac(struct kvm_vcpu
*vcpu
, ulong fac
)
699 #ifdef CONFIG_PPC_BOOK3S_64
700 if (!(vcpu
->arch
.shadow_fscr
& (1ULL << fac
))) {
701 /* Facility not available to the guest, ignore giveup request*/
707 vcpu
->arch
.tar
= mfspr(SPRN_TAR
);
708 mtspr(SPRN_TAR
, current
->thread
.tar
);
709 vcpu
->arch
.shadow_fscr
&= ~FSCR_TAR
;
715 /* Handle external providers (FPU, Altivec, VSX) */
716 static int kvmppc_handle_ext(struct kvm_vcpu
*vcpu
, unsigned int exit_nr
,
719 struct thread_struct
*t
= ¤t
->thread
;
721 /* When we have paired singles, we emulate in software */
722 if (vcpu
->arch
.hflags
& BOOK3S_HFLAG_PAIRED_SINGLE
)
725 if (!(kvmppc_get_msr(vcpu
) & msr
)) {
726 kvmppc_book3s_queue_irqprio(vcpu
, exit_nr
);
730 if (msr
== MSR_VSX
) {
731 /* No VSX? Give an illegal instruction interrupt */
733 if (!cpu_has_feature(CPU_FTR_VSX
))
736 kvmppc_core_queue_program(vcpu
, SRR1_PROGILL
);
741 * We have to load up all the FP and VMX registers before
742 * we can let the guest use VSX instructions.
744 msr
= MSR_FP
| MSR_VEC
| MSR_VSX
;
747 /* See if we already own all the ext(s) needed */
748 msr
&= ~vcpu
->arch
.guest_owned_ext
;
753 printk(KERN_INFO
"Loading up ext 0x%lx\n", msr
);
759 load_fp_state(&vcpu
->arch
.fp
);
761 t
->fp_save_area
= &vcpu
->arch
.fp
;
766 #ifdef CONFIG_ALTIVEC
768 enable_kernel_altivec();
769 load_vr_state(&vcpu
->arch
.vr
);
770 disable_kernel_altivec();
771 t
->vr_save_area
= &vcpu
->arch
.vr
;
777 vcpu
->arch
.guest_owned_ext
|= msr
;
778 kvmppc_recalc_shadow_msr(vcpu
);
784 * Kernel code using FP or VMX could have flushed guest state to
785 * the thread_struct; if so, get it back now.
787 static void kvmppc_handle_lost_ext(struct kvm_vcpu
*vcpu
)
789 unsigned long lost_ext
;
791 lost_ext
= vcpu
->arch
.guest_owned_ext
& ~current
->thread
.regs
->msr
;
795 if (lost_ext
& MSR_FP
) {
798 load_fp_state(&vcpu
->arch
.fp
);
802 #ifdef CONFIG_ALTIVEC
803 if (lost_ext
& MSR_VEC
) {
805 enable_kernel_altivec();
806 load_vr_state(&vcpu
->arch
.vr
);
807 disable_kernel_altivec();
811 current
->thread
.regs
->msr
|= lost_ext
;
814 #ifdef CONFIG_PPC_BOOK3S_64
816 static void kvmppc_trigger_fac_interrupt(struct kvm_vcpu
*vcpu
, ulong fac
)
818 /* Inject the Interrupt Cause field and trigger a guest interrupt */
819 vcpu
->arch
.fscr
&= ~(0xffULL
<< 56);
820 vcpu
->arch
.fscr
|= (fac
<< 56);
821 kvmppc_book3s_queue_irqprio(vcpu
, BOOK3S_INTERRUPT_FAC_UNAVAIL
);
824 static void kvmppc_emulate_fac(struct kvm_vcpu
*vcpu
, ulong fac
)
826 enum emulation_result er
= EMULATE_FAIL
;
828 if (!(kvmppc_get_msr(vcpu
) & MSR_PR
))
829 er
= kvmppc_emulate_instruction(vcpu
->run
, vcpu
);
831 if ((er
!= EMULATE_DONE
) && (er
!= EMULATE_AGAIN
)) {
832 /* Couldn't emulate, trigger interrupt in guest */
833 kvmppc_trigger_fac_interrupt(vcpu
, fac
);
837 /* Enable facilities (TAR, EBB, DSCR) for the guest */
838 static int kvmppc_handle_fac(struct kvm_vcpu
*vcpu
, ulong fac
)
840 bool guest_fac_enabled
;
841 BUG_ON(!cpu_has_feature(CPU_FTR_ARCH_207S
));
844 * Not every facility is enabled by FSCR bits, check whether the
845 * guest has this facility enabled at all.
850 guest_fac_enabled
= (vcpu
->arch
.fscr
& (1ULL << fac
));
853 guest_fac_enabled
= kvmppc_get_msr(vcpu
) & MSR_TM
;
856 guest_fac_enabled
= false;
860 if (!guest_fac_enabled
) {
861 /* Facility not enabled by the guest */
862 kvmppc_trigger_fac_interrupt(vcpu
, fac
);
868 /* TAR switching isn't lazy in Linux yet */
869 current
->thread
.tar
= mfspr(SPRN_TAR
);
870 mtspr(SPRN_TAR
, vcpu
->arch
.tar
);
871 vcpu
->arch
.shadow_fscr
|= FSCR_TAR
;
874 kvmppc_emulate_fac(vcpu
, fac
);
881 void kvmppc_set_fscr(struct kvm_vcpu
*vcpu
, u64 fscr
)
883 if ((vcpu
->arch
.fscr
& FSCR_TAR
) && !(fscr
& FSCR_TAR
)) {
884 /* TAR got dropped, drop it in shadow too */
885 kvmppc_giveup_fac(vcpu
, FSCR_TAR_LG
);
887 vcpu
->arch
.fscr
= fscr
;
891 static void kvmppc_setup_debug(struct kvm_vcpu
*vcpu
)
893 if (vcpu
->guest_debug
& KVM_GUESTDBG_SINGLESTEP
) {
894 u64 msr
= kvmppc_get_msr(vcpu
);
896 kvmppc_set_msr(vcpu
, msr
| MSR_SE
);
900 static void kvmppc_clear_debug(struct kvm_vcpu
*vcpu
)
902 if (vcpu
->guest_debug
& KVM_GUESTDBG_SINGLESTEP
) {
903 u64 msr
= kvmppc_get_msr(vcpu
);
905 kvmppc_set_msr(vcpu
, msr
& ~MSR_SE
);
909 static int kvmppc_exit_pr_progint(struct kvm_run
*run
, struct kvm_vcpu
*vcpu
,
910 unsigned int exit_nr
)
912 enum emulation_result er
;
918 * shadow_srr1 only contains valid flags if we came here via a program
919 * exception. The other exceptions (emulation assist, FP unavailable,
920 * etc.) do not provide flags in SRR1, so use an illegal-instruction
921 * exception when injecting a program interrupt into the guest.
923 if (exit_nr
== BOOK3S_INTERRUPT_PROGRAM
)
924 flags
= vcpu
->arch
.shadow_srr1
& 0x1f0000ull
;
926 flags
= SRR1_PROGILL
;
928 emul
= kvmppc_get_last_inst(vcpu
, INST_GENERIC
, &last_inst
);
929 if (emul
!= EMULATE_DONE
)
932 if (kvmppc_get_msr(vcpu
) & MSR_PR
) {
934 pr_info("Userspace triggered 0x700 exception at\n 0x%lx (0x%x)\n",
935 kvmppc_get_pc(vcpu
), last_inst
);
937 if ((last_inst
& 0xff0007ff) != (INS_DCBZ
& 0xfffffff7)) {
938 kvmppc_core_queue_program(vcpu
, flags
);
943 vcpu
->stat
.emulated_inst_exits
++;
944 er
= kvmppc_emulate_instruction(run
, vcpu
);
953 pr_crit("%s: emulation at %lx failed (%08x)\n",
954 __func__
, kvmppc_get_pc(vcpu
), last_inst
);
955 kvmppc_core_queue_program(vcpu
, flags
);
958 case EMULATE_DO_MMIO
:
959 run
->exit_reason
= KVM_EXIT_MMIO
;
962 case EMULATE_EXIT_USER
:
972 int kvmppc_handle_exit_pr(struct kvm_run
*run
, struct kvm_vcpu
*vcpu
,
973 unsigned int exit_nr
)
978 vcpu
->stat
.sum_exits
++;
980 run
->exit_reason
= KVM_EXIT_UNKNOWN
;
981 run
->ready_for_interrupt_injection
= 1;
983 /* We get here with MSR.EE=1 */
985 trace_kvm_exit(exit_nr
, vcpu
);
989 case BOOK3S_INTERRUPT_INST_STORAGE
:
991 ulong shadow_srr1
= vcpu
->arch
.shadow_srr1
;
992 vcpu
->stat
.pf_instruc
++;
994 if (kvmppc_is_split_real(vcpu
))
995 kvmppc_fixup_split_real(vcpu
);
997 #ifdef CONFIG_PPC_BOOK3S_32
998 /* We set segments as unused segments when invalidating them. So
999 * treat the respective fault as segment fault. */
1001 struct kvmppc_book3s_shadow_vcpu
*svcpu
;
1004 svcpu
= svcpu_get(vcpu
);
1005 sr
= svcpu
->sr
[kvmppc_get_pc(vcpu
) >> SID_SHIFT
];
1007 if (sr
== SR_INVALID
) {
1008 kvmppc_mmu_map_segment(vcpu
, kvmppc_get_pc(vcpu
));
1015 /* only care about PTEG not found errors, but leave NX alone */
1016 if (shadow_srr1
& 0x40000000) {
1017 int idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
1018 r
= kvmppc_handle_pagefault(run
, vcpu
, kvmppc_get_pc(vcpu
), exit_nr
);
1019 srcu_read_unlock(&vcpu
->kvm
->srcu
, idx
);
1020 vcpu
->stat
.sp_instruc
++;
1021 } else if (vcpu
->arch
.mmu
.is_dcbz32(vcpu
) &&
1022 (!(vcpu
->arch
.hflags
& BOOK3S_HFLAG_DCBZ32
))) {
1024 * XXX If we do the dcbz hack we use the NX bit to flush&patch the page,
1025 * so we can't use the NX bit inside the guest. Let's cross our fingers,
1026 * that no guest that needs the dcbz hack does NX.
1028 kvmppc_mmu_pte_flush(vcpu
, kvmppc_get_pc(vcpu
), ~0xFFFUL
);
1031 u64 msr
= kvmppc_get_msr(vcpu
);
1032 msr
|= shadow_srr1
& 0x58000000;
1033 kvmppc_set_msr_fast(vcpu
, msr
);
1034 kvmppc_book3s_queue_irqprio(vcpu
, exit_nr
);
1039 case BOOK3S_INTERRUPT_DATA_STORAGE
:
1041 ulong dar
= kvmppc_get_fault_dar(vcpu
);
1042 u32 fault_dsisr
= vcpu
->arch
.fault_dsisr
;
1043 vcpu
->stat
.pf_storage
++;
1045 #ifdef CONFIG_PPC_BOOK3S_32
1046 /* We set segments as unused segments when invalidating them. So
1047 * treat the respective fault as segment fault. */
1049 struct kvmppc_book3s_shadow_vcpu
*svcpu
;
1052 svcpu
= svcpu_get(vcpu
);
1053 sr
= svcpu
->sr
[dar
>> SID_SHIFT
];
1055 if (sr
== SR_INVALID
) {
1056 kvmppc_mmu_map_segment(vcpu
, dar
);
1064 * We need to handle missing shadow PTEs, and
1065 * protection faults due to us mapping a page read-only
1066 * when the guest thinks it is writable.
1068 if (fault_dsisr
& (DSISR_NOHPTE
| DSISR_PROTFAULT
)) {
1069 int idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
1070 r
= kvmppc_handle_pagefault(run
, vcpu
, dar
, exit_nr
);
1071 srcu_read_unlock(&vcpu
->kvm
->srcu
, idx
);
1073 kvmppc_set_dar(vcpu
, dar
);
1074 kvmppc_set_dsisr(vcpu
, fault_dsisr
);
1075 kvmppc_book3s_queue_irqprio(vcpu
, exit_nr
);
1080 case BOOK3S_INTERRUPT_DATA_SEGMENT
:
1081 if (kvmppc_mmu_map_segment(vcpu
, kvmppc_get_fault_dar(vcpu
)) < 0) {
1082 kvmppc_set_dar(vcpu
, kvmppc_get_fault_dar(vcpu
));
1083 kvmppc_book3s_queue_irqprio(vcpu
,
1084 BOOK3S_INTERRUPT_DATA_SEGMENT
);
1088 case BOOK3S_INTERRUPT_INST_SEGMENT
:
1089 if (kvmppc_mmu_map_segment(vcpu
, kvmppc_get_pc(vcpu
)) < 0) {
1090 kvmppc_book3s_queue_irqprio(vcpu
,
1091 BOOK3S_INTERRUPT_INST_SEGMENT
);
1095 /* We're good on these - the host merely wanted to get our attention */
1096 case BOOK3S_INTERRUPT_DECREMENTER
:
1097 case BOOK3S_INTERRUPT_HV_DECREMENTER
:
1098 case BOOK3S_INTERRUPT_DOORBELL
:
1099 case BOOK3S_INTERRUPT_H_DOORBELL
:
1100 vcpu
->stat
.dec_exits
++;
1103 case BOOK3S_INTERRUPT_EXTERNAL
:
1104 case BOOK3S_INTERRUPT_EXTERNAL_LEVEL
:
1105 case BOOK3S_INTERRUPT_EXTERNAL_HV
:
1106 vcpu
->stat
.ext_intr_exits
++;
1109 case BOOK3S_INTERRUPT_PERFMON
:
1112 case BOOK3S_INTERRUPT_PROGRAM
:
1113 case BOOK3S_INTERRUPT_H_EMUL_ASSIST
:
1114 r
= kvmppc_exit_pr_progint(run
, vcpu
, exit_nr
);
1116 case BOOK3S_INTERRUPT_SYSCALL
:
1121 /* Get last sc for papr */
1122 if (vcpu
->arch
.papr_enabled
) {
1123 /* The sc instuction points SRR0 to the next inst */
1124 emul
= kvmppc_get_last_inst(vcpu
, INST_SC
, &last_sc
);
1125 if (emul
!= EMULATE_DONE
) {
1126 kvmppc_set_pc(vcpu
, kvmppc_get_pc(vcpu
) - 4);
1132 if (vcpu
->arch
.papr_enabled
&&
1133 (last_sc
== 0x44000022) &&
1134 !(kvmppc_get_msr(vcpu
) & MSR_PR
)) {
1135 /* SC 1 papr hypercalls */
1136 ulong cmd
= kvmppc_get_gpr(vcpu
, 3);
1139 #ifdef CONFIG_PPC_BOOK3S_64
1140 if (kvmppc_h_pr(vcpu
, cmd
) == EMULATE_DONE
) {
1146 run
->papr_hcall
.nr
= cmd
;
1147 for (i
= 0; i
< 9; ++i
) {
1148 ulong gpr
= kvmppc_get_gpr(vcpu
, 4 + i
);
1149 run
->papr_hcall
.args
[i
] = gpr
;
1151 run
->exit_reason
= KVM_EXIT_PAPR_HCALL
;
1152 vcpu
->arch
.hcall_needed
= 1;
1154 } else if (vcpu
->arch
.osi_enabled
&&
1155 (((u32
)kvmppc_get_gpr(vcpu
, 3)) == OSI_SC_MAGIC_R3
) &&
1156 (((u32
)kvmppc_get_gpr(vcpu
, 4)) == OSI_SC_MAGIC_R4
)) {
1157 /* MOL hypercalls */
1158 u64
*gprs
= run
->osi
.gprs
;
1161 run
->exit_reason
= KVM_EXIT_OSI
;
1162 for (i
= 0; i
< 32; i
++)
1163 gprs
[i
] = kvmppc_get_gpr(vcpu
, i
);
1164 vcpu
->arch
.osi_needed
= 1;
1166 } else if (!(kvmppc_get_msr(vcpu
) & MSR_PR
) &&
1167 (((u32
)kvmppc_get_gpr(vcpu
, 0)) == KVM_SC_MAGIC_R0
)) {
1168 /* KVM PV hypercalls */
1169 kvmppc_set_gpr(vcpu
, 3, kvmppc_kvm_pv(vcpu
));
1172 /* Guest syscalls */
1173 vcpu
->stat
.syscall_exits
++;
1174 kvmppc_book3s_queue_irqprio(vcpu
, exit_nr
);
1179 case BOOK3S_INTERRUPT_FP_UNAVAIL
:
1180 case BOOK3S_INTERRUPT_ALTIVEC
:
1181 case BOOK3S_INTERRUPT_VSX
:
1187 if (vcpu
->arch
.hflags
& BOOK3S_HFLAG_PAIRED_SINGLE
) {
1188 /* Do paired single instruction emulation */
1189 emul
= kvmppc_get_last_inst(vcpu
, INST_GENERIC
,
1191 if (emul
== EMULATE_DONE
)
1192 r
= kvmppc_exit_pr_progint(run
, vcpu
, exit_nr
);
1199 /* Enable external provider */
1201 case BOOK3S_INTERRUPT_FP_UNAVAIL
:
1205 case BOOK3S_INTERRUPT_ALTIVEC
:
1209 case BOOK3S_INTERRUPT_VSX
:
1214 r
= kvmppc_handle_ext(vcpu
, exit_nr
, ext_msr
);
1217 case BOOK3S_INTERRUPT_ALIGNMENT
:
1220 int emul
= kvmppc_get_last_inst(vcpu
, INST_GENERIC
, &last_inst
);
1222 if (emul
== EMULATE_DONE
) {
1226 dsisr
= kvmppc_alignment_dsisr(vcpu
, last_inst
);
1227 dar
= kvmppc_alignment_dar(vcpu
, last_inst
);
1229 kvmppc_set_dsisr(vcpu
, dsisr
);
1230 kvmppc_set_dar(vcpu
, dar
);
1232 kvmppc_book3s_queue_irqprio(vcpu
, exit_nr
);
1237 #ifdef CONFIG_PPC_BOOK3S_64
1238 case BOOK3S_INTERRUPT_FAC_UNAVAIL
:
1239 kvmppc_handle_fac(vcpu
, vcpu
->arch
.shadow_fscr
>> 56);
1243 case BOOK3S_INTERRUPT_MACHINE_CHECK
:
1244 kvmppc_book3s_queue_irqprio(vcpu
, exit_nr
);
1247 case BOOK3S_INTERRUPT_TRACE
:
1248 if (vcpu
->guest_debug
& KVM_GUESTDBG_SINGLESTEP
) {
1249 run
->exit_reason
= KVM_EXIT_DEBUG
;
1252 kvmppc_book3s_queue_irqprio(vcpu
, exit_nr
);
1258 ulong shadow_srr1
= vcpu
->arch
.shadow_srr1
;
1259 /* Ugh - bork here! What did we get? */
1260 printk(KERN_EMERG
"exit_nr=0x%x | pc=0x%lx | msr=0x%lx\n",
1261 exit_nr
, kvmppc_get_pc(vcpu
), shadow_srr1
);
1268 if (!(r
& RESUME_HOST
)) {
1269 /* To avoid clobbering exit_reason, only check for signals if
1270 * we aren't already exiting to userspace for some other
1274 * Interrupts could be timers for the guest which we have to
1275 * inject again, so let's postpone them until we're in the guest
1276 * and if we really did time things so badly, then we just exit
1277 * again due to a host external interrupt.
1279 s
= kvmppc_prepare_to_enter(vcpu
);
1283 /* interrupts now hard-disabled */
1284 kvmppc_fix_ee_before_entry();
1287 kvmppc_handle_lost_ext(vcpu
);
1290 trace_kvm_book3s_reenter(r
, vcpu
);
1295 static int kvm_arch_vcpu_ioctl_get_sregs_pr(struct kvm_vcpu
*vcpu
,
1296 struct kvm_sregs
*sregs
)
1298 struct kvmppc_vcpu_book3s
*vcpu3s
= to_book3s(vcpu
);
1301 sregs
->pvr
= vcpu
->arch
.pvr
;
1303 sregs
->u
.s
.sdr1
= to_book3s(vcpu
)->sdr1
;
1304 if (vcpu
->arch
.hflags
& BOOK3S_HFLAG_SLB
) {
1305 for (i
= 0; i
< 64; i
++) {
1306 sregs
->u
.s
.ppc64
.slb
[i
].slbe
= vcpu
->arch
.slb
[i
].orige
| i
;
1307 sregs
->u
.s
.ppc64
.slb
[i
].slbv
= vcpu
->arch
.slb
[i
].origv
;
1310 for (i
= 0; i
< 16; i
++)
1311 sregs
->u
.s
.ppc32
.sr
[i
] = kvmppc_get_sr(vcpu
, i
);
1313 for (i
= 0; i
< 8; i
++) {
1314 sregs
->u
.s
.ppc32
.ibat
[i
] = vcpu3s
->ibat
[i
].raw
;
1315 sregs
->u
.s
.ppc32
.dbat
[i
] = vcpu3s
->dbat
[i
].raw
;
1322 static int kvm_arch_vcpu_ioctl_set_sregs_pr(struct kvm_vcpu
*vcpu
,
1323 struct kvm_sregs
*sregs
)
1325 struct kvmppc_vcpu_book3s
*vcpu3s
= to_book3s(vcpu
);
1328 kvmppc_set_pvr_pr(vcpu
, sregs
->pvr
);
1330 vcpu3s
->sdr1
= sregs
->u
.s
.sdr1
;
1331 #ifdef CONFIG_PPC_BOOK3S_64
1332 if (vcpu
->arch
.hflags
& BOOK3S_HFLAG_SLB
) {
1333 /* Flush all SLB entries */
1334 vcpu
->arch
.mmu
.slbmte(vcpu
, 0, 0);
1335 vcpu
->arch
.mmu
.slbia(vcpu
);
1337 for (i
= 0; i
< 64; i
++) {
1338 u64 rb
= sregs
->u
.s
.ppc64
.slb
[i
].slbe
;
1339 u64 rs
= sregs
->u
.s
.ppc64
.slb
[i
].slbv
;
1341 if (rb
& SLB_ESID_V
)
1342 vcpu
->arch
.mmu
.slbmte(vcpu
, rs
, rb
);
1347 for (i
= 0; i
< 16; i
++) {
1348 vcpu
->arch
.mmu
.mtsrin(vcpu
, i
, sregs
->u
.s
.ppc32
.sr
[i
]);
1350 for (i
= 0; i
< 8; i
++) {
1351 kvmppc_set_bat(vcpu
, &(vcpu3s
->ibat
[i
]), false,
1352 (u32
)sregs
->u
.s
.ppc32
.ibat
[i
]);
1353 kvmppc_set_bat(vcpu
, &(vcpu3s
->ibat
[i
]), true,
1354 (u32
)(sregs
->u
.s
.ppc32
.ibat
[i
] >> 32));
1355 kvmppc_set_bat(vcpu
, &(vcpu3s
->dbat
[i
]), false,
1356 (u32
)sregs
->u
.s
.ppc32
.dbat
[i
]);
1357 kvmppc_set_bat(vcpu
, &(vcpu3s
->dbat
[i
]), true,
1358 (u32
)(sregs
->u
.s
.ppc32
.dbat
[i
] >> 32));
1362 /* Flush the MMU after messing with the segments */
1363 kvmppc_mmu_pte_flush(vcpu
, 0, 0);
1368 static int kvmppc_get_one_reg_pr(struct kvm_vcpu
*vcpu
, u64 id
,
1369 union kvmppc_one_reg
*val
)
1374 case KVM_REG_PPC_DEBUG_INST
:
1375 *val
= get_reg_val(id
, KVMPPC_INST_SW_BREAKPOINT
);
1377 case KVM_REG_PPC_HIOR
:
1378 *val
= get_reg_val(id
, to_book3s(vcpu
)->hior
);
1380 case KVM_REG_PPC_VTB
:
1381 *val
= get_reg_val(id
, to_book3s(vcpu
)->vtb
);
1383 case KVM_REG_PPC_LPCR
:
1384 case KVM_REG_PPC_LPCR_64
:
1386 * We are only interested in the LPCR_ILE bit
1388 if (vcpu
->arch
.intr_msr
& MSR_LE
)
1389 *val
= get_reg_val(id
, LPCR_ILE
);
1391 *val
= get_reg_val(id
, 0);
1401 static void kvmppc_set_lpcr_pr(struct kvm_vcpu
*vcpu
, u64 new_lpcr
)
1403 if (new_lpcr
& LPCR_ILE
)
1404 vcpu
->arch
.intr_msr
|= MSR_LE
;
1406 vcpu
->arch
.intr_msr
&= ~MSR_LE
;
1409 static int kvmppc_set_one_reg_pr(struct kvm_vcpu
*vcpu
, u64 id
,
1410 union kvmppc_one_reg
*val
)
1415 case KVM_REG_PPC_HIOR
:
1416 to_book3s(vcpu
)->hior
= set_reg_val(id
, *val
);
1417 to_book3s(vcpu
)->hior_explicit
= true;
1419 case KVM_REG_PPC_VTB
:
1420 to_book3s(vcpu
)->vtb
= set_reg_val(id
, *val
);
1422 case KVM_REG_PPC_LPCR
:
1423 case KVM_REG_PPC_LPCR_64
:
1424 kvmppc_set_lpcr_pr(vcpu
, set_reg_val(id
, *val
));
1434 static struct kvm_vcpu
*kvmppc_core_vcpu_create_pr(struct kvm
*kvm
,
1437 struct kvmppc_vcpu_book3s
*vcpu_book3s
;
1438 struct kvm_vcpu
*vcpu
;
1442 vcpu
= kmem_cache_zalloc(kvm_vcpu_cache
, GFP_KERNEL
);
1446 vcpu_book3s
= vzalloc(sizeof(struct kvmppc_vcpu_book3s
));
1449 vcpu
->arch
.book3s
= vcpu_book3s
;
1451 #ifdef CONFIG_KVM_BOOK3S_32_HANDLER
1452 vcpu
->arch
.shadow_vcpu
=
1453 kzalloc(sizeof(*vcpu
->arch
.shadow_vcpu
), GFP_KERNEL
);
1454 if (!vcpu
->arch
.shadow_vcpu
)
1458 err
= kvm_vcpu_init(vcpu
, kvm
, id
);
1460 goto free_shadow_vcpu
;
1463 p
= __get_free_page(GFP_KERNEL
|__GFP_ZERO
);
1466 vcpu
->arch
.shared
= (void *)p
;
1467 #ifdef CONFIG_PPC_BOOK3S_64
1468 /* Always start the shared struct in native endian mode */
1469 #ifdef __BIG_ENDIAN__
1470 vcpu
->arch
.shared_big_endian
= true;
1472 vcpu
->arch
.shared_big_endian
= false;
1476 * Default to the same as the host if we're on sufficiently
1477 * recent machine that we have 1TB segments;
1478 * otherwise default to PPC970FX.
1480 vcpu
->arch
.pvr
= 0x3C0301;
1481 if (mmu_has_feature(MMU_FTR_1T_SEGMENT
))
1482 vcpu
->arch
.pvr
= mfspr(SPRN_PVR
);
1483 vcpu
->arch
.intr_msr
= MSR_SF
;
1485 /* default to book3s_32 (750) */
1486 vcpu
->arch
.pvr
= 0x84202;
1488 kvmppc_set_pvr_pr(vcpu
, vcpu
->arch
.pvr
);
1489 vcpu
->arch
.slb_nr
= 64;
1491 vcpu
->arch
.shadow_msr
= MSR_USER64
& ~MSR_LE
;
1493 err
= kvmppc_mmu_init(vcpu
);
1500 kvm_vcpu_uninit(vcpu
);
1502 #ifdef CONFIG_KVM_BOOK3S_32_HANDLER
1503 kfree(vcpu
->arch
.shadow_vcpu
);
1508 kmem_cache_free(kvm_vcpu_cache
, vcpu
);
1510 return ERR_PTR(err
);
1513 static void kvmppc_core_vcpu_free_pr(struct kvm_vcpu
*vcpu
)
1515 struct kvmppc_vcpu_book3s
*vcpu_book3s
= to_book3s(vcpu
);
1517 free_page((unsigned long)vcpu
->arch
.shared
& PAGE_MASK
);
1518 kvm_vcpu_uninit(vcpu
);
1519 #ifdef CONFIG_KVM_BOOK3S_32_HANDLER
1520 kfree(vcpu
->arch
.shadow_vcpu
);
1523 kmem_cache_free(kvm_vcpu_cache
, vcpu
);
1526 static int kvmppc_vcpu_run_pr(struct kvm_run
*kvm_run
, struct kvm_vcpu
*vcpu
)
1529 #ifdef CONFIG_ALTIVEC
1530 unsigned long uninitialized_var(vrsave
);
1533 /* Check if we can run the vcpu at all */
1534 if (!vcpu
->arch
.sane
) {
1535 kvm_run
->exit_reason
= KVM_EXIT_INTERNAL_ERROR
;
1540 kvmppc_setup_debug(vcpu
);
1543 * Interrupts could be timers for the guest which we have to inject
1544 * again, so let's postpone them until we're in the guest and if we
1545 * really did time things so badly, then we just exit again due to
1546 * a host external interrupt.
1548 ret
= kvmppc_prepare_to_enter(vcpu
);
1551 /* interrupts now hard-disabled */
1553 /* Save FPU, Altivec and VSX state */
1554 giveup_all(current
);
1556 /* Preload FPU if it's enabled */
1557 if (kvmppc_get_msr(vcpu
) & MSR_FP
)
1558 kvmppc_handle_ext(vcpu
, BOOK3S_INTERRUPT_FP_UNAVAIL
, MSR_FP
);
1560 kvmppc_fix_ee_before_entry();
1562 ret
= __kvmppc_vcpu_run(kvm_run
, vcpu
);
1564 kvmppc_clear_debug(vcpu
);
1566 /* No need for guest_exit. It's done in handle_exit.
1567 We also get here with interrupts enabled. */
1569 /* Make sure we save the guest FPU/Altivec/VSX state */
1570 kvmppc_giveup_ext(vcpu
, MSR_FP
| MSR_VEC
| MSR_VSX
);
1572 /* Make sure we save the guest TAR/EBB/DSCR state */
1573 kvmppc_giveup_fac(vcpu
, FSCR_TAR_LG
);
1576 vcpu
->mode
= OUTSIDE_GUEST_MODE
;
1581 * Get (and clear) the dirty memory log for a memory slot.
1583 static int kvm_vm_ioctl_get_dirty_log_pr(struct kvm
*kvm
,
1584 struct kvm_dirty_log
*log
)
1586 struct kvm_memslots
*slots
;
1587 struct kvm_memory_slot
*memslot
;
1588 struct kvm_vcpu
*vcpu
;
1594 mutex_lock(&kvm
->slots_lock
);
1596 r
= kvm_get_dirty_log(kvm
, log
, &is_dirty
);
1600 /* If nothing is dirty, don't bother messing with page tables. */
1602 slots
= kvm_memslots(kvm
);
1603 memslot
= id_to_memslot(slots
, log
->slot
);
1605 ga
= memslot
->base_gfn
<< PAGE_SHIFT
;
1606 ga_end
= ga
+ (memslot
->npages
<< PAGE_SHIFT
);
1608 kvm_for_each_vcpu(n
, vcpu
, kvm
)
1609 kvmppc_mmu_pte_pflush(vcpu
, ga
, ga_end
);
1611 n
= kvm_dirty_bitmap_bytes(memslot
);
1612 memset(memslot
->dirty_bitmap
, 0, n
);
1617 mutex_unlock(&kvm
->slots_lock
);
1621 static void kvmppc_core_flush_memslot_pr(struct kvm
*kvm
,
1622 struct kvm_memory_slot
*memslot
)
1627 static int kvmppc_core_prepare_memory_region_pr(struct kvm
*kvm
,
1628 struct kvm_memory_slot
*memslot
,
1629 const struct kvm_userspace_memory_region
*mem
)
1634 static void kvmppc_core_commit_memory_region_pr(struct kvm
*kvm
,
1635 const struct kvm_userspace_memory_region
*mem
,
1636 const struct kvm_memory_slot
*old
,
1637 const struct kvm_memory_slot
*new)
1642 static void kvmppc_core_free_memslot_pr(struct kvm_memory_slot
*free
,
1643 struct kvm_memory_slot
*dont
)
1648 static int kvmppc_core_create_memslot_pr(struct kvm_memory_slot
*slot
,
1649 unsigned long npages
)
1656 static int kvm_vm_ioctl_get_smmu_info_pr(struct kvm
*kvm
,
1657 struct kvm_ppc_smmu_info
*info
)
1660 struct kvm_vcpu
*vcpu
;
1664 /* SLB is always 64 entries */
1665 info
->slb_size
= 64;
1667 /* Standard 4k base page size segment */
1668 info
->sps
[0].page_shift
= 12;
1669 info
->sps
[0].slb_enc
= 0;
1670 info
->sps
[0].enc
[0].page_shift
= 12;
1671 info
->sps
[0].enc
[0].pte_enc
= 0;
1674 * 64k large page size.
1675 * We only want to put this in if the CPUs we're emulating
1676 * support it, but unfortunately we don't have a vcpu easily
1677 * to hand here to test. Just pick the first vcpu, and if
1678 * that doesn't exist yet, report the minimum capability,
1679 * i.e., no 64k pages.
1680 * 1T segment support goes along with 64k pages.
1683 vcpu
= kvm_get_vcpu(kvm
, 0);
1684 if (vcpu
&& (vcpu
->arch
.hflags
& BOOK3S_HFLAG_MULTI_PGSIZE
)) {
1685 info
->flags
= KVM_PPC_1T_SEGMENTS
;
1686 info
->sps
[i
].page_shift
= 16;
1687 info
->sps
[i
].slb_enc
= SLB_VSID_L
| SLB_VSID_LP_01
;
1688 info
->sps
[i
].enc
[0].page_shift
= 16;
1689 info
->sps
[i
].enc
[0].pte_enc
= 1;
1693 /* Standard 16M large page size segment */
1694 info
->sps
[i
].page_shift
= 24;
1695 info
->sps
[i
].slb_enc
= SLB_VSID_L
;
1696 info
->sps
[i
].enc
[0].page_shift
= 24;
1697 info
->sps
[i
].enc
[0].pte_enc
= 0;
1702 static int kvm_vm_ioctl_get_smmu_info_pr(struct kvm
*kvm
,
1703 struct kvm_ppc_smmu_info
*info
)
1705 /* We should not get called */
1708 #endif /* CONFIG_PPC64 */
1710 static unsigned int kvm_global_user_count
= 0;
1711 static DEFINE_SPINLOCK(kvm_global_user_count_lock
);
1713 static int kvmppc_core_init_vm_pr(struct kvm
*kvm
)
1715 mutex_init(&kvm
->arch
.hpt_mutex
);
1717 #ifdef CONFIG_PPC_BOOK3S_64
1718 /* Start out with the default set of hcalls enabled */
1719 kvmppc_pr_init_default_hcalls(kvm
);
1722 if (firmware_has_feature(FW_FEATURE_SET_MODE
)) {
1723 spin_lock(&kvm_global_user_count_lock
);
1724 if (++kvm_global_user_count
== 1)
1725 pseries_disable_reloc_on_exc();
1726 spin_unlock(&kvm_global_user_count_lock
);
1731 static void kvmppc_core_destroy_vm_pr(struct kvm
*kvm
)
1734 WARN_ON(!list_empty(&kvm
->arch
.spapr_tce_tables
));
1737 if (firmware_has_feature(FW_FEATURE_SET_MODE
)) {
1738 spin_lock(&kvm_global_user_count_lock
);
1739 BUG_ON(kvm_global_user_count
== 0);
1740 if (--kvm_global_user_count
== 0)
1741 pseries_enable_reloc_on_exc();
1742 spin_unlock(&kvm_global_user_count_lock
);
1746 static int kvmppc_core_check_processor_compat_pr(void)
1749 * Disable KVM for Power9 untill the required bits merged.
1751 if (cpu_has_feature(CPU_FTR_ARCH_300
))
1756 static long kvm_arch_vm_ioctl_pr(struct file
*filp
,
1757 unsigned int ioctl
, unsigned long arg
)
1762 static struct kvmppc_ops kvm_ops_pr
= {
1763 .get_sregs
= kvm_arch_vcpu_ioctl_get_sregs_pr
,
1764 .set_sregs
= kvm_arch_vcpu_ioctl_set_sregs_pr
,
1765 .get_one_reg
= kvmppc_get_one_reg_pr
,
1766 .set_one_reg
= kvmppc_set_one_reg_pr
,
1767 .vcpu_load
= kvmppc_core_vcpu_load_pr
,
1768 .vcpu_put
= kvmppc_core_vcpu_put_pr
,
1769 .set_msr
= kvmppc_set_msr_pr
,
1770 .vcpu_run
= kvmppc_vcpu_run_pr
,
1771 .vcpu_create
= kvmppc_core_vcpu_create_pr
,
1772 .vcpu_free
= kvmppc_core_vcpu_free_pr
,
1773 .check_requests
= kvmppc_core_check_requests_pr
,
1774 .get_dirty_log
= kvm_vm_ioctl_get_dirty_log_pr
,
1775 .flush_memslot
= kvmppc_core_flush_memslot_pr
,
1776 .prepare_memory_region
= kvmppc_core_prepare_memory_region_pr
,
1777 .commit_memory_region
= kvmppc_core_commit_memory_region_pr
,
1778 .unmap_hva
= kvm_unmap_hva_pr
,
1779 .unmap_hva_range
= kvm_unmap_hva_range_pr
,
1780 .age_hva
= kvm_age_hva_pr
,
1781 .test_age_hva
= kvm_test_age_hva_pr
,
1782 .set_spte_hva
= kvm_set_spte_hva_pr
,
1783 .mmu_destroy
= kvmppc_mmu_destroy_pr
,
1784 .free_memslot
= kvmppc_core_free_memslot_pr
,
1785 .create_memslot
= kvmppc_core_create_memslot_pr
,
1786 .init_vm
= kvmppc_core_init_vm_pr
,
1787 .destroy_vm
= kvmppc_core_destroy_vm_pr
,
1788 .get_smmu_info
= kvm_vm_ioctl_get_smmu_info_pr
,
1789 .emulate_op
= kvmppc_core_emulate_op_pr
,
1790 .emulate_mtspr
= kvmppc_core_emulate_mtspr_pr
,
1791 .emulate_mfspr
= kvmppc_core_emulate_mfspr_pr
,
1792 .fast_vcpu_kick
= kvm_vcpu_kick
,
1793 .arch_vm_ioctl
= kvm_arch_vm_ioctl_pr
,
1794 #ifdef CONFIG_PPC_BOOK3S_64
1795 .hcall_implemented
= kvmppc_hcall_impl_pr
,
1800 int kvmppc_book3s_init_pr(void)
1804 r
= kvmppc_core_check_processor_compat_pr();
1808 kvm_ops_pr
.owner
= THIS_MODULE
;
1809 kvmppc_pr_ops
= &kvm_ops_pr
;
1811 r
= kvmppc_mmu_hpte_sysinit();
1815 void kvmppc_book3s_exit_pr(void)
1817 kvmppc_pr_ops
= NULL
;
1818 kvmppc_mmu_hpte_sysexit();
1822 * We only support separate modules for book3s 64
1824 #ifdef CONFIG_PPC_BOOK3S_64
1826 module_init(kvmppc_book3s_init_pr
);
1827 module_exit(kvmppc_book3s_exit_pr
);
1829 MODULE_LICENSE("GPL");
1830 MODULE_ALIAS_MISCDEV(KVM_MINOR
);
1831 MODULE_ALIAS("devname:kvm");