[mirror_ubuntu-artful-kernel.git] / arch / powerpc / kernel / kvm.c

/*
 * Copyright (C) 2010 SUSE Linux Products GmbH. All rights reserved.
 *
 * Authors:
 *     Alexander Graf <agraf@suse.de>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2, as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

#include <linux/kvm_host.h>
#include <linux/init.h>
#include <linux/kvm_para.h>
#include <linux/slab.h>
#include <linux/of.h>

#include <asm/reg.h>
#include <asm/sections.h>
#include <asm/cacheflush.h>
#include <asm/disassemble.h>

#define KVM_MAGIC_PAGE		(-4096L)
#define magic_var(x) KVM_MAGIC_PAGE + offsetof(struct kvm_vcpu_arch_shared, x)

#define KVM_INST_LWZ		0x80000000
#define KVM_INST_STW		0x90000000
#define KVM_INST_LD		0xe8000000
#define KVM_INST_STD		0xf8000000
#define KVM_INST_NOP		0x60000000
#define KVM_INST_B		0x48000000
#define KVM_INST_B_MASK		0x03ffffff
#define KVM_INST_B_MAX		0x01ffffff

#define KVM_MASK_RT		0x03e00000
#define KVM_RT_30		0x03c00000
#define KVM_MASK_RB		0x0000f800
#define KVM_INST_MFMSR		0x7c0000a6
#define KVM_INST_MFSPR_SPRG0	0x7c1042a6
#define KVM_INST_MFSPR_SPRG1	0x7c1142a6
#define KVM_INST_MFSPR_SPRG2	0x7c1242a6
#define KVM_INST_MFSPR_SPRG3	0x7c1342a6
#define KVM_INST_MFSPR_SRR0	0x7c1a02a6
#define KVM_INST_MFSPR_SRR1	0x7c1b02a6
#define KVM_INST_MFSPR_DAR	0x7c1302a6
#define KVM_INST_MFSPR_DSISR	0x7c1202a6

#define KVM_INST_MTSPR_SPRG0	0x7c1043a6
#define KVM_INST_MTSPR_SPRG1	0x7c1143a6
#define KVM_INST_MTSPR_SPRG2	0x7c1243a6
#define KVM_INST_MTSPR_SPRG3	0x7c1343a6
#define KVM_INST_MTSPR_SRR0	0x7c1a03a6
#define KVM_INST_MTSPR_SRR1	0x7c1b03a6
#define KVM_INST_MTSPR_DAR	0x7c1303a6
#define KVM_INST_MTSPR_DSISR	0x7c1203a6

#define KVM_INST_TLBSYNC	0x7c00046c
#define KVM_INST_MTMSRD_L0	0x7c000164
#define KVM_INST_MTMSRD_L1	0x7c010164
#define KVM_INST_MTMSR		0x7c000124

#define KVM_INST_WRTEEI_0	0x7c000146
#define KVM_INST_WRTEEI_1	0x7c008146

#define KVM_INST_MTSRIN		0x7c0001e4

static bool kvm_patching_worked = true;
static char kvm_tmp[1024 * 1024];
static int kvm_tmp_index;

static inline void kvm_patch_ins(u32 *inst, u32 new_inst)
{
	*inst = new_inst;
	flush_icache_range((ulong)inst, (ulong)inst + 4);
}

static void kvm_patch_ins_ll(u32 *inst, long addr, u32 rt)
{
#ifdef CONFIG_64BIT
	kvm_patch_ins(inst, KVM_INST_LD | rt | (addr & 0x0000fffc));
#else
	kvm_patch_ins(inst, KVM_INST_LWZ | rt | (addr & 0x0000fffc));
#endif
}

static void kvm_patch_ins_ld(u32 *inst, long addr, u32 rt)
{
#ifdef CONFIG_64BIT
	kvm_patch_ins(inst, KVM_INST_LD | rt | (addr & 0x0000fffc));
#else
	kvm_patch_ins(inst, KVM_INST_LWZ | rt | ((addr + 4) & 0x0000fffc));
#endif
}

static void kvm_patch_ins_lwz(u32 *inst, long addr, u32 rt)
{
	kvm_patch_ins(inst, KVM_INST_LWZ | rt | (addr & 0x0000ffff));
}

static void kvm_patch_ins_std(u32 *inst, long addr, u32 rt)
{
#ifdef CONFIG_64BIT
	kvm_patch_ins(inst, KVM_INST_STD | rt | (addr & 0x0000fffc));
#else
	kvm_patch_ins(inst, KVM_INST_STW | rt | ((addr + 4) & 0x0000fffc));
#endif
}

static void kvm_patch_ins_stw(u32 *inst, long addr, u32 rt)
{
	kvm_patch_ins(inst, KVM_INST_STW | rt | (addr & 0x0000fffc));
}

static void kvm_patch_ins_nop(u32 *inst)
{
	kvm_patch_ins(inst, KVM_INST_NOP);
}

static void kvm_patch_ins_b(u32 *inst, int addr)
{
#if defined(CONFIG_RELOCATABLE) && defined(CONFIG_PPC_BOOK3S)
	/* On relocatable kernels interrupts handlers and our code
	   can be in different regions, so we don't patch them */

	extern u32 __end_interrupts;
	if ((ulong)inst < (ulong)&__end_interrupts)
		return;
#endif

	kvm_patch_ins(inst, KVM_INST_B | (addr & KVM_INST_B_MASK));
}

static u32 *kvm_alloc(int len)
{
	u32 *p;

	if ((kvm_tmp_index + len) > ARRAY_SIZE(kvm_tmp)) {
		printk(KERN_ERR "KVM: No more space (%d + %d)\n",
				kvm_tmp_index, len);
		kvm_patching_worked = false;
		return NULL;
	}

	p = (void*)&kvm_tmp[kvm_tmp_index];
	kvm_tmp_index += len;

	return p;
}

extern u32 kvm_emulate_mtmsrd_branch_offs;
extern u32 kvm_emulate_mtmsrd_reg_offs;
extern u32 kvm_emulate_mtmsrd_orig_ins_offs;
extern u32 kvm_emulate_mtmsrd_len;
extern u32 kvm_emulate_mtmsrd[];

static void kvm_patch_ins_mtmsrd(u32 *inst, u32 rt)
{
	u32 *p;
	int distance_start;
	int distance_end;
	ulong next_inst;

	p = kvm_alloc(kvm_emulate_mtmsrd_len * 4);
	if (!p)
		return;

	/* Find out where we are and put everything there */
	distance_start = (ulong)p - (ulong)inst;
	next_inst = ((ulong)inst + 4);
	distance_end = next_inst - (ulong)&p[kvm_emulate_mtmsrd_branch_offs];

	/* Make sure we only write valid b instructions */
	if (distance_start > KVM_INST_B_MAX) {
		kvm_patching_worked = false;
		return;
	}

	/* Modify the chunk to fit the invocation */
	memcpy(p, kvm_emulate_mtmsrd, kvm_emulate_mtmsrd_len * 4);
	p[kvm_emulate_mtmsrd_branch_offs] |= distance_end & KVM_INST_B_MASK;
	switch (get_rt(rt)) {
	case 30:
		kvm_patch_ins_ll(&p[kvm_emulate_mtmsrd_reg_offs],
				 magic_var(scratch2), KVM_RT_30);
		break;
	case 31:
		kvm_patch_ins_ll(&p[kvm_emulate_mtmsrd_reg_offs],
				 magic_var(scratch1), KVM_RT_30);
		break;
	default:
		p[kvm_emulate_mtmsrd_reg_offs] |= rt;
		break;
	}

	p[kvm_emulate_mtmsrd_orig_ins_offs] = *inst;
	flush_icache_range((ulong)p, (ulong)p + kvm_emulate_mtmsrd_len * 4);

	/* Patch the invocation */
	kvm_patch_ins_b(inst, distance_start);
}

extern u32 kvm_emulate_mtmsr_branch_offs;
extern u32 kvm_emulate_mtmsr_reg1_offs;
extern u32 kvm_emulate_mtmsr_reg2_offs;
extern u32 kvm_emulate_mtmsr_orig_ins_offs;
extern u32 kvm_emulate_mtmsr_len;
extern u32 kvm_emulate_mtmsr[];

static void kvm_patch_ins_mtmsr(u32 *inst, u32 rt)
{
	u32 *p;
	int distance_start;
	int distance_end;
	ulong next_inst;

	p = kvm_alloc(kvm_emulate_mtmsr_len * 4);
	if (!p)
		return;

	/* Find out where we are and put everything there */
	distance_start = (ulong)p - (ulong)inst;
	next_inst = ((ulong)inst + 4);
	distance_end = next_inst - (ulong)&p[kvm_emulate_mtmsr_branch_offs];

	/* Make sure we only write valid b instructions */
	if (distance_start > KVM_INST_B_MAX) {
		kvm_patching_worked = false;
		return;
	}

	/* Modify the chunk to fit the invocation */
	memcpy(p, kvm_emulate_mtmsr, kvm_emulate_mtmsr_len * 4);
	p[kvm_emulate_mtmsr_branch_offs] |= distance_end & KVM_INST_B_MASK;

	/* Make clobbered registers work too */
	switch (get_rt(rt)) {
	case 30:
		kvm_patch_ins_ll(&p[kvm_emulate_mtmsr_reg1_offs],
				 magic_var(scratch2), KVM_RT_30);
		kvm_patch_ins_ll(&p[kvm_emulate_mtmsr_reg2_offs],
				 magic_var(scratch2), KVM_RT_30);
		break;
	case 31:
		kvm_patch_ins_ll(&p[kvm_emulate_mtmsr_reg1_offs],
				 magic_var(scratch1), KVM_RT_30);
		kvm_patch_ins_ll(&p[kvm_emulate_mtmsr_reg2_offs],
				 magic_var(scratch1), KVM_RT_30);
		break;
	default:
		p[kvm_emulate_mtmsr_reg1_offs] |= rt;
		p[kvm_emulate_mtmsr_reg2_offs] |= rt;
		break;
	}

	p[kvm_emulate_mtmsr_orig_ins_offs] = *inst;
	flush_icache_range((ulong)p, (ulong)p + kvm_emulate_mtmsr_len * 4);

	/* Patch the invocation */
	kvm_patch_ins_b(inst, distance_start);
}

#ifdef CONFIG_BOOKE

extern u32 kvm_emulate_wrteei_branch_offs;
extern u32 kvm_emulate_wrteei_ee_offs;
extern u32 kvm_emulate_wrteei_len;
extern u32 kvm_emulate_wrteei[];

static void kvm_patch_ins_wrteei(u32 *inst)
{
	u32 *p;
	int distance_start;
	int distance_end;
	ulong next_inst;

	p = kvm_alloc(kvm_emulate_wrteei_len * 4);
	if (!p)
		return;

	/* Find out where we are and put everything there */
	distance_start = (ulong)p - (ulong)inst;
	next_inst = ((ulong)inst + 4);
	distance_end = next_inst - (ulong)&p[kvm_emulate_wrteei_branch_offs];

	/* Make sure we only write valid b instructions */
	if (distance_start > KVM_INST_B_MAX) {
		kvm_patching_worked = false;
		return;
	}

	/* Modify the chunk to fit the invocation */
	memcpy(p, kvm_emulate_wrteei, kvm_emulate_wrteei_len * 4);
	p[kvm_emulate_wrteei_branch_offs] |= distance_end & KVM_INST_B_MASK;
	p[kvm_emulate_wrteei_ee_offs] |= (*inst & MSR_EE);
	flush_icache_range((ulong)p, (ulong)p + kvm_emulate_wrteei_len * 4);

	/* Patch the invocation */
	kvm_patch_ins_b(inst, distance_start);
}

#endif

#ifdef CONFIG_PPC_BOOK3S_32

extern u32 kvm_emulate_mtsrin_branch_offs;
extern u32 kvm_emulate_mtsrin_reg1_offs;
extern u32 kvm_emulate_mtsrin_reg2_offs;
extern u32 kvm_emulate_mtsrin_orig_ins_offs;
extern u32 kvm_emulate_mtsrin_len;
extern u32 kvm_emulate_mtsrin[];

static void kvm_patch_ins_mtsrin(u32 *inst, u32 rt, u32 rb)
{
	u32 *p;
	int distance_start;
	int distance_end;
	ulong next_inst;

	p = kvm_alloc(kvm_emulate_mtsrin_len * 4);
	if (!p)
		return;

	/* Find out where we are and put everything there */
	distance_start = (ulong)p - (ulong)inst;
	next_inst = ((ulong)inst + 4);
	distance_end = next_inst - (ulong)&p[kvm_emulate_mtsrin_branch_offs];

	/* Make sure we only write valid b instructions */
	if (distance_start > KVM_INST_B_MAX) {
		kvm_patching_worked = false;
		return;
	}

	/* Modify the chunk to fit the invocation */
	memcpy(p, kvm_emulate_mtsrin, kvm_emulate_mtsrin_len * 4);
	p[kvm_emulate_mtsrin_branch_offs] |= distance_end & KVM_INST_B_MASK;
	p[kvm_emulate_mtsrin_reg1_offs] |= (rb << 10);
	p[kvm_emulate_mtsrin_reg2_offs] |= rt;
	p[kvm_emulate_mtsrin_orig_ins_offs] = *inst;
	flush_icache_range((ulong)p, (ulong)p + kvm_emulate_mtsrin_len * 4);

	/* Patch the invocation */
	kvm_patch_ins_b(inst, distance_start);
}

#endif

static void kvm_map_magic_page(void *data)
{
	u32 *features = data;

	ulong in[8];
	ulong out[8];

	in[0] = KVM_MAGIC_PAGE;
	in[1] = KVM_MAGIC_PAGE;

	kvm_hypercall(in, out, HC_VENDOR_KVM | KVM_HC_PPC_MAP_MAGIC_PAGE);

	*features = out[0];
}

static void kvm_check_ins(u32 *inst, u32 features)
{
	u32 _inst = *inst;
	u32 inst_no_rt = _inst & ~KVM_MASK_RT;
	u32 inst_rt = _inst & KVM_MASK_RT;

	switch (inst_no_rt) {
	/* Loads */
	case KVM_INST_MFMSR:
		kvm_patch_ins_ld(inst, magic_var(msr), inst_rt);
		break;
	case KVM_INST_MFSPR_SPRG0:
		kvm_patch_ins_ld(inst, magic_var(sprg0), inst_rt);
		break;
	case KVM_INST_MFSPR_SPRG1:
		kvm_patch_ins_ld(inst, magic_var(sprg1), inst_rt);
		break;
	case KVM_INST_MFSPR_SPRG2:
		kvm_patch_ins_ld(inst, magic_var(sprg2), inst_rt);
		break;
	case KVM_INST_MFSPR_SPRG3:
		kvm_patch_ins_ld(inst, magic_var(sprg3), inst_rt);
		break;
	case KVM_INST_MFSPR_SRR0:
		kvm_patch_ins_ld(inst, magic_var(srr0), inst_rt);
		break;
	case KVM_INST_MFSPR_SRR1:
		kvm_patch_ins_ld(inst, magic_var(srr1), inst_rt);
		break;
	case KVM_INST_MFSPR_DAR:
		kvm_patch_ins_ld(inst, magic_var(dar), inst_rt);
		break;
	case KVM_INST_MFSPR_DSISR:
		kvm_patch_ins_lwz(inst, magic_var(dsisr), inst_rt);
		break;

	/* Stores */
	case KVM_INST_MTSPR_SPRG0:
		kvm_patch_ins_std(inst, magic_var(sprg0), inst_rt);
		break;
	case KVM_INST_MTSPR_SPRG1:
		kvm_patch_ins_std(inst, magic_var(sprg1), inst_rt);
		break;
	case KVM_INST_MTSPR_SPRG2:
		kvm_patch_ins_std(inst, magic_var(sprg2), inst_rt);
		break;
	case KVM_INST_MTSPR_SPRG3:
		kvm_patch_ins_std(inst, magic_var(sprg3), inst_rt);
		break;
	case KVM_INST_MTSPR_SRR0:
		kvm_patch_ins_std(inst, magic_var(srr0), inst_rt);
		break;
	case KVM_INST_MTSPR_SRR1:
		kvm_patch_ins_std(inst, magic_var(srr1), inst_rt);
		break;
	case KVM_INST_MTSPR_DAR:
		kvm_patch_ins_std(inst, magic_var(dar), inst_rt);
		break;
	case KVM_INST_MTSPR_DSISR:
		kvm_patch_ins_stw(inst, magic_var(dsisr), inst_rt);
		break;

	/* Nops */
	case KVM_INST_TLBSYNC:
		kvm_patch_ins_nop(inst);
		break;

	/* Rewrites */
	case KVM_INST_MTMSRD_L1:
		kvm_patch_ins_mtmsrd(inst, inst_rt);
		break;
	case KVM_INST_MTMSR:
	case KVM_INST_MTMSRD_L0:
		kvm_patch_ins_mtmsr(inst, inst_rt);
		break;
	}

	switch (inst_no_rt & ~KVM_MASK_RB) {
#ifdef CONFIG_PPC_BOOK3S_32
	case KVM_INST_MTSRIN:
		if (features & KVM_MAGIC_FEAT_SR) {
			u32 inst_rb = _inst & KVM_MASK_RB;
			kvm_patch_ins_mtsrin(inst, inst_rt, inst_rb);
		}
		break;
		break;
#endif
	}

	switch (_inst) {
#ifdef CONFIG_BOOKE
	case KVM_INST_WRTEEI_0:
	case KVM_INST_WRTEEI_1:
		kvm_patch_ins_wrteei(inst);
		break;
#endif
	}
}

static void kvm_use_magic_page(void)
{
	u32 *p;
	u32 *start, *end;
	u32 tmp;
	u32 features;

	/* Tell the host to map the magic page to -4096 on all CPUs */
	on_each_cpu(kvm_map_magic_page, &features, 1);

	/* Quick self-test to see if the mapping works */
	if (__get_user(tmp, (u32*)KVM_MAGIC_PAGE)) {
		kvm_patching_worked = false;
		return;
	}

	/* Now loop through all code and find instructions */
	start = (void*)_stext;
	end = (void*)_etext;

	for (p = start; p < end; p++)
		kvm_check_ins(p, features);

	printk(KERN_INFO "KVM: Live patching for a fast VM %s\n",
			 kvm_patching_worked ? "worked" : "failed");
}

unsigned long kvm_hypercall(unsigned long *in,
			    unsigned long *out,
			    unsigned long nr)
{
	unsigned long register r0 asm("r0");
	unsigned long register r3 asm("r3") = in[0];
	unsigned long register r4 asm("r4") = in[1];
	unsigned long register r5 asm("r5") = in[2];
	unsigned long register r6 asm("r6") = in[3];
	unsigned long register r7 asm("r7") = in[4];
	unsigned long register r8 asm("r8") = in[5];
	unsigned long register r9 asm("r9") = in[6];
	unsigned long register r10 asm("r10") = in[7];
	unsigned long register r11 asm("r11") = nr;
	unsigned long register r12 asm("r12");

	asm volatile("bl	kvm_hypercall_start"
		     : "=r"(r0), "=r"(r3), "=r"(r4), "=r"(r5), "=r"(r6),
		       "=r"(r7), "=r"(r8), "=r"(r9), "=r"(r10), "=r"(r11),
		       "=r"(r12)
		     : "r"(r3), "r"(r4), "r"(r5), "r"(r6), "r"(r7), "r"(r8),
		       "r"(r9), "r"(r10), "r"(r11)
		     : "memory", "cc", "xer", "ctr", "lr");

	out[0] = r4;
	out[1] = r5;
	out[2] = r6;
	out[3] = r7;
	out[4] = r8;
	out[5] = r9;
	out[6] = r10;
	out[7] = r11;

	return r3;
}
EXPORT_SYMBOL_GPL(kvm_hypercall);

static int kvm_para_setup(void)
{
	extern u32 kvm_hypercall_start;
	struct device_node *hyper_node;
	u32 *insts;
	int len, i;

	hyper_node = of_find_node_by_path("/hypervisor");
	if (!hyper_node)
		return -1;

	insts = (u32*)of_get_property(hyper_node, "hcall-instructions", &len);
	if (len % 4)
		return -1;
	if (len > (4 * 4))
		return -1;

	for (i = 0; i < (len / 4); i++)
		kvm_patch_ins(&(&kvm_hypercall_start)[i], insts[i]);

	return 0;
}

static __init void kvm_free_tmp(void)
{
	unsigned long start, end;

	start = (ulong)&kvm_tmp[kvm_tmp_index + (PAGE_SIZE - 1)] & PAGE_MASK;
	end = (ulong)&kvm_tmp[ARRAY_SIZE(kvm_tmp)] & PAGE_MASK;

	/* Free the tmp space we don't need */
	for (; start < end; start += PAGE_SIZE) {
		ClearPageReserved(virt_to_page(start));
		init_page_count(virt_to_page(start));
		free_page(start);
		totalram_pages++;
	}
}

static int __init kvm_guest_init(void)
{
	if (!kvm_para_available())
		goto free_tmp;

	if (kvm_para_setup())
		goto free_tmp;

	if (kvm_para_has_feature(KVM_FEATURE_MAGIC_PAGE))
		kvm_use_magic_page();

#ifdef CONFIG_PPC_BOOK3S_64
	/* Enable napping */
	powersave_nap = 1;
#endif

free_tmp:
	kvm_free_tmp();

	return 0;
}

postcore_initcall(kvm_guest_init);
Commit	Line	Data
2a342ed5 AG	1	/*
	2	* Copyright (C) 2010 SUSE Linux Products GmbH. All rights reserved.
	3	*
	4	* Authors:
	5	* Alexander Graf <agraf@suse.de>
	6	*
	7	* This program is free software; you can redistribute it and/or modify
	8	* it under the terms of the GNU General Public License, version 2, as
	9	* published by the Free Software Foundation.
	10	*
	11	* This program is distributed in the hope that it will be useful,
	12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	14	* GNU General Public License for more details.
	15	*
	16	* You should have received a copy of the GNU General Public License
	17	* along with this program; if not, write to the Free Software
	18	* Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
	19	*/
	20
	21	#include <linux/kvm_host.h>
	22	#include <linux/init.h>
	23	#include <linux/kvm_para.h>
	24	#include <linux/slab.h>
	25	#include <linux/of.h>
	26
	27	#include <asm/reg.h>
2a342ed5 AG	28	#include <asm/sections.h>
	29	#include <asm/cacheflush.h>
	30	#include <asm/disassemble.h>
	31
d17051cb AG	32	#define KVM_MAGIC_PAGE (-4096L)
	33	#define magic_var(x) KVM_MAGIC_PAGE + offsetof(struct kvm_vcpu_arch_shared, x)
	34
d1293c92 AG	35	#define KVM_INST_LWZ 0x80000000
	36	#define KVM_INST_STW 0x90000000
	37	#define KVM_INST_LD 0xe8000000
	38	#define KVM_INST_STD 0xf8000000
	39	#define KVM_INST_NOP 0x60000000
	40	#define KVM_INST_B 0x48000000
	41	#define KVM_INST_B_MASK 0x03ffffff
	42	#define KVM_INST_B_MAX 0x01ffffff
	43
73a18109	44	#define KVM_MASK_RT 0x03e00000
512ba59e	45	#define KVM_RT_30 0x03c00000
cbe487fa	46	#define KVM_MASK_RB 0x0000f800
d1293c92 AG	47	#define KVM_INST_MFMSR 0x7c0000a6
	48	#define KVM_INST_MFSPR_SPRG0 0x7c1042a6
	49	#define KVM_INST_MFSPR_SPRG1 0x7c1142a6
	50	#define KVM_INST_MFSPR_SPRG2 0x7c1242a6
	51	#define KVM_INST_MFSPR_SPRG3 0x7c1342a6
	52	#define KVM_INST_MFSPR_SRR0 0x7c1a02a6
	53	#define KVM_INST_MFSPR_SRR1 0x7c1b02a6
	54	#define KVM_INST_MFSPR_DAR 0x7c1302a6
	55	#define KVM_INST_MFSPR_DSISR 0x7c1202a6
	56
	57	#define KVM_INST_MTSPR_SPRG0 0x7c1043a6
	58	#define KVM_INST_MTSPR_SPRG1 0x7c1143a6
	59	#define KVM_INST_MTSPR_SPRG2 0x7c1243a6
	60	#define KVM_INST_MTSPR_SPRG3 0x7c1343a6
	61	#define KVM_INST_MTSPR_SRR0 0x7c1a03a6
	62	#define KVM_INST_MTSPR_SRR1 0x7c1b03a6
	63	#define KVM_INST_MTSPR_DAR 0x7c1303a6
	64	#define KVM_INST_MTSPR_DSISR 0x7c1203a6
73a18109	65
d1290b15	66	#define KVM_INST_TLBSYNC 0x7c00046c
78109277	67	#define KVM_INST_MTMSRD_L0 0x7c000164
819a63dc	68	#define KVM_INST_MTMSRD_L1 0x7c010164
78109277	69	#define KVM_INST_MTMSR 0x7c000124
d1290b15	70
644bfa01 AG	71	#define KVM_INST_WRTEEI_0 0x7c000146
	72	#define KVM_INST_WRTEEI_1 0x7c008146
	73
cbe487fa AG	74	#define KVM_INST_MTSRIN 0x7c0001e4
cbe487fa AG	75
73a18109	76	static bool kvm_patching_worked = true;
2d4f5671 AG	77	static char kvm_tmp[1024 * 1024];
2d4f5671 AG	78	static int kvm_tmp_index;
73a18109 AG	79
	80	static inline void kvm_patch_ins(u32 *inst, u32 new_inst)
	81	{
	82	*inst = new_inst;
	83	flush_icache_range((ulong)inst, (ulong)inst + 4);
	84	}
	85
512ba59e AG	86	static void kvm_patch_ins_ll(u32 *inst, long addr, u32 rt)
	87	{
	88	#ifdef CONFIG_64BIT
	89	kvm_patch_ins(inst, KVM_INST_LD \| rt \| (addr & 0x0000fffc));
	90	#else
	91	kvm_patch_ins(inst, KVM_INST_LWZ \| rt \| (addr & 0x0000fffc));
	92	#endif
	93	}
	94
d1293c92 AG	95	static void kvm_patch_ins_ld(u32 *inst, long addr, u32 rt)
	96	{
	97	#ifdef CONFIG_64BIT
	98	kvm_patch_ins(inst, KVM_INST_LD \| rt \| (addr & 0x0000fffc));
	99	#else
	100	kvm_patch_ins(inst, KVM_INST_LWZ \| rt \| ((addr + 4) & 0x0000fffc));
	101	#endif
	102	}
	103
	104	static void kvm_patch_ins_lwz(u32 *inst, long addr, u32 rt)
	105	{
	106	kvm_patch_ins(inst, KVM_INST_LWZ \| rt \| (addr & 0x0000ffff));
	107	}
	108
	109	static void kvm_patch_ins_std(u32 *inst, long addr, u32 rt)
	110	{
	111	#ifdef CONFIG_64BIT
	112	kvm_patch_ins(inst, KVM_INST_STD \| rt \| (addr & 0x0000fffc));
	113	#else
	114	kvm_patch_ins(inst, KVM_INST_STW \| rt \| ((addr + 4) & 0x0000fffc));
	115	#endif
	116	}
	117
	118	static void kvm_patch_ins_stw(u32 *inst, long addr, u32 rt)
	119	{
	120	kvm_patch_ins(inst, KVM_INST_STW \| rt \| (addr & 0x0000fffc));
	121	}
	122
d1290b15 AG	123	static void kvm_patch_ins_nop(u32 *inst)
	124	{
	125	kvm_patch_ins(inst, KVM_INST_NOP);
	126	}
	127
71ee8e34 AG	128	static void kvm_patch_ins_b(u32 *inst, int addr)
71ee8e34 AG	129	{
a36be100	130	#if defined(CONFIG_RELOCATABLE) && defined(CONFIG_PPC_BOOK3S)
71ee8e34 AG	131	/* On relocatable kernels interrupts handlers and our code
	132	can be in different regions, so we don't patch them */
	133
	134	extern u32 __end_interrupts;
	135	if ((ulong)inst < (ulong)&__end_interrupts)
	136	return;
	137	#endif
	138
	139	kvm_patch_ins(inst, KVM_INST_B \| (addr & KVM_INST_B_MASK));
	140	}
	141
2d4f5671 AG	142	static u32 *kvm_alloc(int len)
	143	{
	144	u32 *p;
	145
	146	if ((kvm_tmp_index + len) > ARRAY_SIZE(kvm_tmp)) {
	147	printk(KERN_ERR "KVM: No more space (%d + %d)\n",
	148	kvm_tmp_index, len);
	149	kvm_patching_worked = false;
	150	return NULL;
	151	}
	152
	153	p = (void*)&kvm_tmp[kvm_tmp_index];
	154	kvm_tmp_index += len;
	155
	156	return p;
	157	}
	158
819a63dc AG	159	extern u32 kvm_emulate_mtmsrd_branch_offs;
819a63dc AG	160	extern u32 kvm_emulate_mtmsrd_reg_offs;
df08bd10	161	extern u32 kvm_emulate_mtmsrd_orig_ins_offs;
819a63dc AG	162	extern u32 kvm_emulate_mtmsrd_len;
	163	extern u32 kvm_emulate_mtmsrd[];
	164
	165	static void kvm_patch_ins_mtmsrd(u32 *inst, u32 rt)
	166	{
	167	u32 *p;
	168	int distance_start;
	169	int distance_end;
	170	ulong next_inst;
	171
	172	p = kvm_alloc(kvm_emulate_mtmsrd_len * 4);
	173	if (!p)
	174	return;
	175
	176	/* Find out where we are and put everything there */
	177	distance_start = (ulong)p - (ulong)inst;
	178	next_inst = ((ulong)inst + 4);
	179	distance_end = next_inst - (ulong)&p[kvm_emulate_mtmsrd_branch_offs];
	180
	181	/* Make sure we only write valid b instructions */
	182	if (distance_start > KVM_INST_B_MAX) {
	183	kvm_patching_worked = false;
	184	return;
	185	}
	186
	187	/* Modify the chunk to fit the invocation */
	188	memcpy(p, kvm_emulate_mtmsrd, kvm_emulate_mtmsrd_len * 4);
	189	p[kvm_emulate_mtmsrd_branch_offs] \|= distance_end & KVM_INST_B_MASK;
df08bd10 AG	190	switch (get_rt(rt)) {
	191	case 30:
	192	kvm_patch_ins_ll(&p[kvm_emulate_mtmsrd_reg_offs],
	193	magic_var(scratch2), KVM_RT_30);
	194	break;
	195	case 31:
	196	kvm_patch_ins_ll(&p[kvm_emulate_mtmsrd_reg_offs],
	197	magic_var(scratch1), KVM_RT_30);
	198	break;
	199	default:
	200	p[kvm_emulate_mtmsrd_reg_offs] \|= rt;
	201	break;
	202	}
	203
	204	p[kvm_emulate_mtmsrd_orig_ins_offs] = *inst;
819a63dc AG	205	flush_icache_range((ulong)p, (ulong)p + kvm_emulate_mtmsrd_len * 4);
	206
	207	/* Patch the invocation */
	208	kvm_patch_ins_b(inst, distance_start);
	209	}
	210
78109277 AG	211	extern u32 kvm_emulate_mtmsr_branch_offs;
	212	extern u32 kvm_emulate_mtmsr_reg1_offs;
	213	extern u32 kvm_emulate_mtmsr_reg2_offs;
78109277 AG	214	extern u32 kvm_emulate_mtmsr_orig_ins_offs;
	215	extern u32 kvm_emulate_mtmsr_len;
	216	extern u32 kvm_emulate_mtmsr[];
	217
	218	static void kvm_patch_ins_mtmsr(u32 *inst, u32 rt)
	219	{
	220	u32 *p;
	221	int distance_start;
	222	int distance_end;
	223	ulong next_inst;
	224
	225	p = kvm_alloc(kvm_emulate_mtmsr_len * 4);
	226	if (!p)
	227	return;
	228
	229	/* Find out where we are and put everything there */
	230	distance_start = (ulong)p - (ulong)inst;
	231	next_inst = ((ulong)inst + 4);
	232	distance_end = next_inst - (ulong)&p[kvm_emulate_mtmsr_branch_offs];
	233
	234	/* Make sure we only write valid b instructions */
	235	if (distance_start > KVM_INST_B_MAX) {
	236	kvm_patching_worked = false;
	237	return;
	238	}
	239
	240	/* Modify the chunk to fit the invocation */
	241	memcpy(p, kvm_emulate_mtmsr, kvm_emulate_mtmsr_len * 4);
	242	p[kvm_emulate_mtmsr_branch_offs] \|= distance_end & KVM_INST_B_MASK;
512ba59e AG	243
	244	/* Make clobbered registers work too */
	245	switch (get_rt(rt)) {
	246	case 30:
	247	kvm_patch_ins_ll(&p[kvm_emulate_mtmsr_reg1_offs],
	248	magic_var(scratch2), KVM_RT_30);
	249	kvm_patch_ins_ll(&p[kvm_emulate_mtmsr_reg2_offs],
	250	magic_var(scratch2), KVM_RT_30);
	251	break;
	252	case 31:
	253	kvm_patch_ins_ll(&p[kvm_emulate_mtmsr_reg1_offs],
	254	magic_var(scratch1), KVM_RT_30);
	255	kvm_patch_ins_ll(&p[kvm_emulate_mtmsr_reg2_offs],
	256	magic_var(scratch1), KVM_RT_30);
	257	break;
	258	default:
	259	p[kvm_emulate_mtmsr_reg1_offs] \|= rt;
	260	p[kvm_emulate_mtmsr_reg2_offs] \|= rt;
	261	break;
	262	}
	263
78109277 AG	264	p[kvm_emulate_mtmsr_orig_ins_offs] = *inst;
	265	flush_icache_range((ulong)p, (ulong)p + kvm_emulate_mtmsr_len * 4);
	266
	267	/* Patch the invocation */
	268	kvm_patch_ins_b(inst, distance_start);
	269	}
	270
644bfa01 AG	271	#ifdef CONFIG_BOOKE
	272
	273	extern u32 kvm_emulate_wrteei_branch_offs;
	274	extern u32 kvm_emulate_wrteei_ee_offs;
	275	extern u32 kvm_emulate_wrteei_len;
	276	extern u32 kvm_emulate_wrteei[];
	277
	278	static void kvm_patch_ins_wrteei(u32 *inst)
	279	{
	280	u32 *p;
	281	int distance_start;
	282	int distance_end;
	283	ulong next_inst;
	284
	285	p = kvm_alloc(kvm_emulate_wrteei_len * 4);
	286	if (!p)
	287	return;
	288
	289	/* Find out where we are and put everything there */
	290	distance_start = (ulong)p - (ulong)inst;
	291	next_inst = ((ulong)inst + 4);
	292	distance_end = next_inst - (ulong)&p[kvm_emulate_wrteei_branch_offs];
	293
	294	/* Make sure we only write valid b instructions */
	295	if (distance_start > KVM_INST_B_MAX) {
	296	kvm_patching_worked = false;
	297	return;
	298	}
	299
	300	/* Modify the chunk to fit the invocation */
	301	memcpy(p, kvm_emulate_wrteei, kvm_emulate_wrteei_len * 4);
	302	p[kvm_emulate_wrteei_branch_offs] \|= distance_end & KVM_INST_B_MASK;
	303	p[kvm_emulate_wrteei_ee_offs] \|= (*inst & MSR_EE);
	304	flush_icache_range((ulong)p, (ulong)p + kvm_emulate_wrteei_len * 4);
	305
	306	/* Patch the invocation */
	307	kvm_patch_ins_b(inst, distance_start);
	308	}
	309
	310	#endif
	311
cbe487fa AG	312	#ifdef CONFIG_PPC_BOOK3S_32
	313
	314	extern u32 kvm_emulate_mtsrin_branch_offs;
	315	extern u32 kvm_emulate_mtsrin_reg1_offs;
	316	extern u32 kvm_emulate_mtsrin_reg2_offs;
	317	extern u32 kvm_emulate_mtsrin_orig_ins_offs;
	318	extern u32 kvm_emulate_mtsrin_len;
	319	extern u32 kvm_emulate_mtsrin[];
	320
	321	static void kvm_patch_ins_mtsrin(u32 *inst, u32 rt, u32 rb)
	322	{
	323	u32 *p;
	324	int distance_start;
	325	int distance_end;
	326	ulong next_inst;
	327
	328	p = kvm_alloc(kvm_emulate_mtsrin_len * 4);
	329	if (!p)
	330	return;
	331
	332	/* Find out where we are and put everything there */
	333	distance_start = (ulong)p - (ulong)inst;
	334	next_inst = ((ulong)inst + 4);
	335	distance_end = next_inst - (ulong)&p[kvm_emulate_mtsrin_branch_offs];
	336
	337	/* Make sure we only write valid b instructions */
	338	if (distance_start > KVM_INST_B_MAX) {
	339	kvm_patching_worked = false;
	340	return;
	341	}
	342
	343	/* Modify the chunk to fit the invocation */
	344	memcpy(p, kvm_emulate_mtsrin, kvm_emulate_mtsrin_len * 4);
	345	p[kvm_emulate_mtsrin_branch_offs] \|= distance_end & KVM_INST_B_MASK;
	346	p[kvm_emulate_mtsrin_reg1_offs] \|= (rb << 10);
	347	p[kvm_emulate_mtsrin_reg2_offs] \|= rt;
	348	p[kvm_emulate_mtsrin_orig_ins_offs] = *inst;
	349	flush_icache_range((ulong)p, (ulong)p + kvm_emulate_mtsrin_len * 4);
	350
	351	/* Patch the invocation */
	352	kvm_patch_ins_b(inst, distance_start);
	353	}
	354
	355	#endif
	356
73a18109 AG	357	static void kvm_map_magic_page(void *data)
73a18109 AG	358	{
7508e16c AG	359	u32 *features = data;
	360
	361	ulong in[8];
	362	ulong out[8];
	363
	364	in[0] = KVM_MAGIC_PAGE;
	365	in[1] = KVM_MAGIC_PAGE;
	366
	367	kvm_hypercall(in, out, HC_VENDOR_KVM \| KVM_HC_PPC_MAP_MAGIC_PAGE);
	368
	369	*features = out[0];
73a18109 AG	370	}
73a18109 AG	371
7508e16c	372	static void kvm_check_ins(u32 *inst, u32 features)
73a18109 AG	373	{
	374	u32 _inst = *inst;
	375	u32 inst_no_rt = _inst & ~KVM_MASK_RT;
	376	u32 inst_rt = _inst & KVM_MASK_RT;
	377
	378	switch (inst_no_rt) {
d1293c92 AG	379	/* Loads */
	380	case KVM_INST_MFMSR:
	381	kvm_patch_ins_ld(inst, magic_var(msr), inst_rt);
	382	break;
	383	case KVM_INST_MFSPR_SPRG0:
	384	kvm_patch_ins_ld(inst, magic_var(sprg0), inst_rt);
	385	break;
	386	case KVM_INST_MFSPR_SPRG1:
	387	kvm_patch_ins_ld(inst, magic_var(sprg1), inst_rt);
	388	break;
	389	case KVM_INST_MFSPR_SPRG2:
	390	kvm_patch_ins_ld(inst, magic_var(sprg2), inst_rt);
	391	break;
	392	case KVM_INST_MFSPR_SPRG3:
	393	kvm_patch_ins_ld(inst, magic_var(sprg3), inst_rt);
	394	break;
	395	case KVM_INST_MFSPR_SRR0:
	396	kvm_patch_ins_ld(inst, magic_var(srr0), inst_rt);
	397	break;
	398	case KVM_INST_MFSPR_SRR1:
	399	kvm_patch_ins_ld(inst, magic_var(srr1), inst_rt);
	400	break;
	401	case KVM_INST_MFSPR_DAR:
	402	kvm_patch_ins_ld(inst, magic_var(dar), inst_rt);
	403	break;
	404	case KVM_INST_MFSPR_DSISR:
	405	kvm_patch_ins_lwz(inst, magic_var(dsisr), inst_rt);
	406	break;
	407
	408	/* Stores */
	409	case KVM_INST_MTSPR_SPRG0:
	410	kvm_patch_ins_std(inst, magic_var(sprg0), inst_rt);
	411	break;
	412	case KVM_INST_MTSPR_SPRG1:
	413	kvm_patch_ins_std(inst, magic_var(sprg1), inst_rt);
	414	break;
	415	case KVM_INST_MTSPR_SPRG2:
	416	kvm_patch_ins_std(inst, magic_var(sprg2), inst_rt);
	417	break;
	418	case KVM_INST_MTSPR_SPRG3:
	419	kvm_patch_ins_std(inst, magic_var(sprg3), inst_rt);
	420	break;
	421	case KVM_INST_MTSPR_SRR0:
	422	kvm_patch_ins_std(inst, magic_var(srr0), inst_rt);
	423	break;
	424	case KVM_INST_MTSPR_SRR1:
	425	kvm_patch_ins_std(inst, magic_var(srr1), inst_rt);
	426	break;
	427	case KVM_INST_MTSPR_DAR:
	428	kvm_patch_ins_std(inst, magic_var(dar), inst_rt);
	429	break;
	430	case KVM_INST_MTSPR_DSISR:
	431	kvm_patch_ins_stw(inst, magic_var(dsisr), inst_rt);
	432	break;
d1290b15 AG	433
	434	/* Nops */
	435	case KVM_INST_TLBSYNC:
	436	kvm_patch_ins_nop(inst);
	437	break;
819a63dc AG	438
	439	/* Rewrites */
	440	case KVM_INST_MTMSRD_L1:
df08bd10	441	kvm_patch_ins_mtmsrd(inst, inst_rt);
819a63dc	442	break;
78109277 AG	443	case KVM_INST_MTMSR:
78109277 AG	444	case KVM_INST_MTMSRD_L0:
512ba59e	445	kvm_patch_ins_mtmsr(inst, inst_rt);
78109277	446	break;
73a18109 AG	447	}
73a18109 AG	448
cbe487fa AG	449	switch (inst_no_rt & ~KVM_MASK_RB) {
	450	#ifdef CONFIG_PPC_BOOK3S_32
	451	case KVM_INST_MTSRIN:
	452	if (features & KVM_MAGIC_FEAT_SR) {
	453	u32 inst_rb = _inst & KVM_MASK_RB;
	454	kvm_patch_ins_mtsrin(inst, inst_rt, inst_rb);
	455	}
	456	break;
	457	break;
	458	#endif
	459	}
	460
73a18109	461	switch (_inst) {
644bfa01 AG	462	#ifdef CONFIG_BOOKE
	463	case KVM_INST_WRTEEI_0:
	464	case KVM_INST_WRTEEI_1:
	465	kvm_patch_ins_wrteei(inst);
	466	break;
	467	#endif
73a18109 AG	468	}
	469	}
	470
	471	static void kvm_use_magic_page(void)
	472	{
	473	u32 *p;
	474	u32 start, end;
	475	u32 tmp;
7508e16c	476	u32 features;
73a18109 AG	477
73a18109 AG	478	/* Tell the host to map the magic page to -4096 on all CPUs */
7508e16c	479	on_each_cpu(kvm_map_magic_page, &features, 1);
73a18109 AG	480
	481	/* Quick self-test to see if the mapping works */
	482	if (__get_user(tmp, (u32*)KVM_MAGIC_PAGE)) {
	483	kvm_patching_worked = false;
	484	return;
	485	}
	486
	487	/* Now loop through all code and find instructions */
	488	start = (void*)_stext;
	489	end = (void*)_etext;
	490
	491	for (p = start; p < end; p++)
7508e16c	492	kvm_check_ins(p, features);
73a18109 AG	493
	494	printk(KERN_INFO "KVM: Live patching for a fast VM %s\n",
	495	kvm_patching_worked ? "worked" : "failed");
	496	}
	497
2a342ed5 AG	498	unsigned long kvm_hypercall(unsigned long *in,
	499	unsigned long *out,
	500	unsigned long nr)
	501	{
	502	unsigned long register r0 asm("r0");
	503	unsigned long register r3 asm("r3") = in[0];
	504	unsigned long register r4 asm("r4") = in[1];
	505	unsigned long register r5 asm("r5") = in[2];
	506	unsigned long register r6 asm("r6") = in[3];
	507	unsigned long register r7 asm("r7") = in[4];
	508	unsigned long register r8 asm("r8") = in[5];
	509	unsigned long register r9 asm("r9") = in[6];
	510	unsigned long register r10 asm("r10") = in[7];
	511	unsigned long register r11 asm("r11") = nr;
	512	unsigned long register r12 asm("r12");
	513
	514	asm volatile("bl kvm_hypercall_start"
	515	: "=r"(r0), "=r"(r3), "=r"(r4), "=r"(r5), "=r"(r6),
	516	"=r"(r7), "=r"(r8), "=r"(r9), "=r"(r10), "=r"(r11),
	517	"=r"(r12)
	518	: "r"(r3), "r"(r4), "r"(r5), "r"(r6), "r"(r7), "r"(r8),
	519	"r"(r9), "r"(r10), "r"(r11)
	520	: "memory", "cc", "xer", "ctr", "lr");
	521
	522	out[0] = r4;
	523	out[1] = r5;
	524	out[2] = r6;
	525	out[3] = r7;
	526	out[4] = r8;
	527	out[5] = r9;
	528	out[6] = r10;
	529	out[7] = r11;
	530
	531	return r3;
	532	}
	533	EXPORT_SYMBOL_GPL(kvm_hypercall);
73a18109 AG	534
	535	static int kvm_para_setup(void)
	536	{
	537	extern u32 kvm_hypercall_start;
	538	struct device_node *hyper_node;
	539	u32 *insts;
	540	int len, i;
	541
	542	hyper_node = of_find_node_by_path("/hypervisor");
	543	if (!hyper_node)
	544	return -1;
	545
	546	insts = (u32*)of_get_property(hyper_node, "hcall-instructions", &len);
	547	if (len % 4)
	548	return -1;
	549	if (len > (4 * 4))
	550	return -1;
	551
	552	for (i = 0; i < (len / 4); i++)
	553	kvm_patch_ins(&(&kvm_hypercall_start)[i], insts[i]);
	554
	555	return 0;
	556	}
	557
2d4f5671 AG	558	static __init void kvm_free_tmp(void)
	559	{
	560	unsigned long start, end;
	561
	562	start = (ulong)&kvm_tmp[kvm_tmp_index + (PAGE_SIZE - 1)] & PAGE_MASK;
	563	end = (ulong)&kvm_tmp[ARRAY_SIZE(kvm_tmp)] & PAGE_MASK;
	564
	565	/* Free the tmp space we don't need */
	566	for (; start < end; start += PAGE_SIZE) {
	567	ClearPageReserved(virt_to_page(start));
	568	init_page_count(virt_to_page(start));
	569	free_page(start);
	570	totalram_pages++;
	571	}
	572	}
	573
73a18109 AG	574	static int __init kvm_guest_init(void)
	575	{
	576	if (!kvm_para_available())
2d4f5671	577	goto free_tmp;
73a18109 AG	578
73a18109 AG	579	if (kvm_para_setup())
2d4f5671	580	goto free_tmp;
73a18109 AG	581
	582	if (kvm_para_has_feature(KVM_FEATURE_MAGIC_PAGE))
	583	kvm_use_magic_page();
	584
591bd8e7	585	#ifdef CONFIG_PPC_BOOK3S_64
ad087376 AG	586	/* Enable napping */
ad087376 AG	587	powersave_nap = 1;
591bd8e7	588	#endif
ad087376	589
2d4f5671 AG	590	free_tmp:
	591	kvm_free_tmp();
	592
73a18109 AG	593	return 0;
	594	}
	595
	596	postcore_initcall(kvm_guest_init);