[mirror_ubuntu-zesty-kernel.git] / arch / powerpc / mm / tlb-radix.c

/*
 * TLB flush routines for radix kernels.
 *
 * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/memblock.h>
#include <asm/ppc-opcode.h>

#include <asm/tlb.h>
#include <asm/tlbflush.h>

static DEFINE_RAW_SPINLOCK(native_tlbie_lock);

#define RIC_FLUSH_TLB 0
#define RIC_FLUSH_PWC 1
#define RIC_FLUSH_ALL 2

static inline void __tlbiel_pid(unsigned long pid, int set,
				unsigned long ric)
{
	unsigned long rb,rs,prs,r;

	rb = PPC_BIT(53); /* IS = 1 */
	rb |= set << PPC_BITLSHIFT(51);
	rs = ((unsigned long)pid) << PPC_BITLSHIFT(31);
	prs = 1; /* process scoped */
	r = 1;   /* raidx format */

	asm volatile("ptesync": : :"memory");
	asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	asm volatile("ptesync": : :"memory");
}

/*
 * We use 128 set in radix mode and 256 set in hpt mode.
 */
static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
{
	int set;

	for (set = 0; set < POWER9_TLB_SETS_RADIX ; set++) {
		__tlbiel_pid(pid, set, ric);
	}
	if (cpu_has_feature(CPU_FTR_POWER9_DD1))
		asm volatile(PPC_INVALIDATE_ERAT : : :"memory");
	return;
}

static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
{
	unsigned long rb,rs,prs,r;

	rb = PPC_BIT(53); /* IS = 1 */
	rs = pid << PPC_BITLSHIFT(31);
	prs = 1; /* process scoped */
	r = 1;   /* raidx format */

	asm volatile("ptesync": : :"memory");
	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	asm volatile("eieio; tlbsync; ptesync": : :"memory");
}

static inline void _tlbiel_va(unsigned long va, unsigned long pid,
			      unsigned long ap, unsigned long ric)
{
	unsigned long rb,rs,prs,r;

	rb = va & ~(PPC_BITMASK(52, 63));
	rb |= ap << PPC_BITLSHIFT(58);
	rs = pid << PPC_BITLSHIFT(31);
	prs = 1; /* process scoped */
	r = 1;   /* raidx format */

	asm volatile("ptesync": : :"memory");
	asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	asm volatile("ptesync": : :"memory");
	if (cpu_has_feature(CPU_FTR_POWER9_DD1))
		asm volatile(PPC_INVALIDATE_ERAT : : :"memory");
}

static inline void _tlbie_va(unsigned long va, unsigned long pid,
			     unsigned long ap, unsigned long ric)
{
	unsigned long rb,rs,prs,r;

	rb = va & ~(PPC_BITMASK(52, 63));
	rb |= ap << PPC_BITLSHIFT(58);
	rs = pid << PPC_BITLSHIFT(31);
	prs = 1; /* process scoped */
	r = 1;   /* raidx format */

	asm volatile("ptesync": : :"memory");
	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	asm volatile("eieio; tlbsync; ptesync": : :"memory");
}

/*
 * Base TLB flushing operations:
 *
 *  - flush_tlb_mm(mm) flushes the specified mm context TLB's
 *  - flush_tlb_page(vma, vmaddr) flushes one page
 *  - flush_tlb_range(vma, start, end) flushes a range of pages
 *  - flush_tlb_kernel_range(start, end) flushes kernel pages
 *
 *  - local_* variants of page and mm only apply to the current
 *    processor
 */
void radix__local_flush_tlb_mm(struct mm_struct *mm)
{
	unsigned long pid;

	preempt_disable();
	pid = mm->context.id;
	if (pid != MMU_NO_CONTEXT)
		_tlbiel_pid(pid, RIC_FLUSH_ALL);
	preempt_enable();
}
EXPORT_SYMBOL(radix__local_flush_tlb_mm);

void radix__local_flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
{
	unsigned long pid;
	struct mm_struct *mm = tlb->mm;

	preempt_disable();

	pid = mm->context.id;
	if (pid != MMU_NO_CONTEXT)
		_tlbiel_pid(pid, RIC_FLUSH_PWC);

	preempt_enable();
}
EXPORT_SYMBOL(radix__local_flush_tlb_pwc);

void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
				       int psize)
{
	unsigned long pid;
	unsigned long ap = mmu_get_ap(psize);

	preempt_disable();
	pid = mm ? mm->context.id : 0;
	if (pid != MMU_NO_CONTEXT)
		_tlbiel_va(vmaddr, pid, ap, RIC_FLUSH_TLB);
	preempt_enable();
}

void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
{
#ifdef CONFIG_HUGETLB_PAGE
	/* need the return fix for nohash.c */
	if (vma && is_vm_hugetlb_page(vma))
		return __local_flush_hugetlb_page(vma, vmaddr);
#endif
	radix__local_flush_tlb_page_psize(vma ? vma->vm_mm : NULL, vmaddr,
					  mmu_virtual_psize);
}
EXPORT_SYMBOL(radix__local_flush_tlb_page);

#ifdef CONFIG_SMP
void radix__flush_tlb_mm(struct mm_struct *mm)
{
	unsigned long pid;

	preempt_disable();
	pid = mm->context.id;
	if (unlikely(pid == MMU_NO_CONTEXT))
		goto no_context;

	if (!mm_is_thread_local(mm)) {
		int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);

		if (lock_tlbie)
			raw_spin_lock(&native_tlbie_lock);
		_tlbie_pid(pid, RIC_FLUSH_ALL);
		if (lock_tlbie)
			raw_spin_unlock(&native_tlbie_lock);
	} else
		_tlbiel_pid(pid, RIC_FLUSH_ALL);
no_context:
	preempt_enable();
}
EXPORT_SYMBOL(radix__flush_tlb_mm);

void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
{
	unsigned long pid;
	struct mm_struct *mm = tlb->mm;

	preempt_disable();

	pid = mm->context.id;
	if (unlikely(pid == MMU_NO_CONTEXT))
		goto no_context;

	if (!mm_is_thread_local(mm)) {
		int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);

		if (lock_tlbie)
			raw_spin_lock(&native_tlbie_lock);
		_tlbie_pid(pid, RIC_FLUSH_PWC);
		if (lock_tlbie)
			raw_spin_unlock(&native_tlbie_lock);
	} else
		_tlbiel_pid(pid, RIC_FLUSH_PWC);
no_context:
	preempt_enable();
}
EXPORT_SYMBOL(radix__flush_tlb_pwc);

void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
				 int psize)
{
	unsigned long pid;
	unsigned long ap = mmu_get_ap(psize);

	preempt_disable();
	pid = mm ? mm->context.id : 0;
	if (unlikely(pid == MMU_NO_CONTEXT))
		goto bail;
	if (!mm_is_thread_local(mm)) {
		int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);

		if (lock_tlbie)
			raw_spin_lock(&native_tlbie_lock);
		_tlbie_va(vmaddr, pid, ap, RIC_FLUSH_TLB);
		if (lock_tlbie)
			raw_spin_unlock(&native_tlbie_lock);
	} else
		_tlbiel_va(vmaddr, pid, ap, RIC_FLUSH_TLB);
bail:
	preempt_enable();
}

void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
{
#ifdef CONFIG_HUGETLB_PAGE
	if (vma && is_vm_hugetlb_page(vma))
		return flush_hugetlb_page(vma, vmaddr);
#endif
	radix__flush_tlb_page_psize(vma ? vma->vm_mm : NULL, vmaddr,
				    mmu_virtual_psize);
}
EXPORT_SYMBOL(radix__flush_tlb_page);

#endif /* CONFIG_SMP */

void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
	int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);

	if (lock_tlbie)
		raw_spin_lock(&native_tlbie_lock);
	_tlbie_pid(0, RIC_FLUSH_ALL);
	if (lock_tlbie)
		raw_spin_unlock(&native_tlbie_lock);
}
EXPORT_SYMBOL(radix__flush_tlb_kernel_range);

/*
 * Currently, for range flushing, we just do a full mm flush. Because
 * we use this in code path where we don' track the page size.
 */
void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
		     unsigned long end)

{
	struct mm_struct *mm = vma->vm_mm;
	radix__flush_tlb_mm(mm);
}
EXPORT_SYMBOL(radix__flush_tlb_range);

static int radix_get_mmu_psize(int page_size)
{
	int psize;

	if (page_size == (1UL << mmu_psize_defs[mmu_virtual_psize].shift))
		psize = mmu_virtual_psize;
	else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_2M].shift))
		psize = MMU_PAGE_2M;
	else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_1G].shift))
		psize = MMU_PAGE_1G;
	else
		return -1;
	return psize;
}

void radix__tlb_flush(struct mmu_gather *tlb)
{
	int psize = 0;
	struct mm_struct *mm = tlb->mm;
	int page_size = tlb->page_size;

	psize = radix_get_mmu_psize(page_size);
	/*
	 * if page size is not something we understand, do a full mm flush
	 */
	if (psize != -1 && !tlb->fullmm && !tlb->need_flush_all)
		radix__flush_tlb_range_psize(mm, tlb->start, tlb->end, psize);
	else
		radix__flush_tlb_mm(mm);
}

#define TLB_FLUSH_ALL -1UL
/*
 * Number of pages above which we will do a bcast tlbie. Just a
 * number at this point copied from x86
 */
static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33;

void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
				  unsigned long end, int psize)
{
	unsigned long pid;
	unsigned long addr;
	int local = mm_is_thread_local(mm);
	unsigned long ap = mmu_get_ap(psize);
	int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
	unsigned long page_size = 1UL << mmu_psize_defs[psize].shift;


	preempt_disable();
	pid = mm ? mm->context.id : 0;
	if (unlikely(pid == MMU_NO_CONTEXT))
		goto err_out;

	if (end == TLB_FLUSH_ALL ||
	    (end - start) > tlb_single_page_flush_ceiling * page_size) {
		if (local)
			_tlbiel_pid(pid, RIC_FLUSH_TLB);
		else
			_tlbie_pid(pid, RIC_FLUSH_TLB);
		goto err_out;
	}
	for (addr = start; addr < end; addr += page_size) {

		if (local)
			_tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB);
		else {
			if (lock_tlbie)
				raw_spin_lock(&native_tlbie_lock);
			_tlbie_va(addr, pid, ap, RIC_FLUSH_TLB);
			if (lock_tlbie)
				raw_spin_unlock(&native_tlbie_lock);
		}
	}
err_out:
	preempt_enable();
}

void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
			      unsigned long page_size)
{
	unsigned long rb,rs,prs,r;
	unsigned long ap;
	unsigned long ric = RIC_FLUSH_TLB;

	ap = mmu_get_ap(radix_get_mmu_psize(page_size));
	rb = gpa & ~(PPC_BITMASK(52, 63));
	rb |= ap << PPC_BITLSHIFT(58);
	rs = lpid & ((1UL << 32) - 1);
	prs = 0; /* process scoped */
	r = 1;   /* raidx format */

	asm volatile("ptesync": : :"memory");
	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	asm volatile("eieio; tlbsync; ptesync": : :"memory");
}
EXPORT_SYMBOL(radix__flush_tlb_lpid_va);

void radix__flush_tlb_lpid(unsigned long lpid)
{
	unsigned long rb,rs,prs,r;
	unsigned long ric = RIC_FLUSH_ALL;

	rb = 0x2 << PPC_BITLSHIFT(53); /* IS = 2 */
	rs = lpid & ((1UL << 32) - 1);
	prs = 0; /* partition scoped */
	r = 1;   /* raidx format */

	asm volatile("ptesync": : :"memory");
	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	asm volatile("eieio; tlbsync; ptesync": : :"memory");
}
EXPORT_SYMBOL(radix__flush_tlb_lpid);

void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
				unsigned long start, unsigned long end)
{
	radix__flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M);
}
EXPORT_SYMBOL(radix__flush_pmd_tlb_range);

void radix__flush_tlb_all(void)
{
	unsigned long rb,prs,r,rs;
	unsigned long ric = RIC_FLUSH_ALL;

	rb = 0x3 << PPC_BITLSHIFT(53); /* IS = 3 */
	prs = 0; /* partition scoped */
	r = 1;   /* raidx format */
	rs = 1 & ((1UL << 32) - 1); /* any LPID value to flush guest mappings */

	asm volatile("ptesync": : :"memory");
	/*
	 * now flush guest entries by passing PRS = 1 and LPID != 0
	 */
	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
		     : : "r"(rb), "i"(r), "i"(1), "i"(ric), "r"(rs) : "memory");
	/*
	 * now flush host entires by passing PRS = 0 and LPID == 0
	 */
	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(0) : "memory");
	asm volatile("eieio; tlbsync; ptesync": : :"memory");
}
Commit	Line	Data
1a472c9d AK	1	/*
	2	* TLB flush routines for radix kernels.
	3	*
	4	* Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
	5	*
	6	* This program is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU General Public License
	8	* as published by the Free Software Foundation; either version
	9	* 2 of the License, or (at your option) any later version.
	10	*/
	11
	12	#include <linux/mm.h>
	13	#include <linux/hugetlb.h>
	14	#include <linux/memblock.h>
8cd6d3c2	15	#include <asm/ppc-opcode.h>
1a472c9d AK	16
	17	#include <asm/tlb.h>
	18	#include <asm/tlbflush.h>
	19
	20	static DEFINE_RAW_SPINLOCK(native_tlbie_lock);
	21
36194812 AK	22	#define RIC_FLUSH_TLB 0
	23	#define RIC_FLUSH_PWC 1
	24	#define RIC_FLUSH_ALL 2
	25
	26	static inline void __tlbiel_pid(unsigned long pid, int set,
	27	unsigned long ric)
1a472c9d	28	{
36194812	29	unsigned long rb,rs,prs,r;
1a472c9d AK	30
	31	rb = PPC_BIT(53); /* IS = 1 */
	32	rb \|= set << PPC_BITLSHIFT(51);
	33	rs = ((unsigned long)pid) << PPC_BITLSHIFT(31);
	34	prs = 1; /* process scoped */
	35	r = 1; /* raidx format */
1a472c9d AK	36
1a472c9d AK	37	asm volatile("ptesync": : :"memory");
8cd6d3c2	38	asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
1a472c9d AK	39	: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	40	asm volatile("ptesync": : :"memory");
	41	}
	42
	43	/*
	44	* We use 128 set in radix mode and 256 set in hpt mode.
	45	*/
36194812	46	static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
1a472c9d AK	47	{
	48	int set;
	49
	50	for (set = 0; set < POWER9_TLB_SETS_RADIX ; set++) {
36194812	51	__tlbiel_pid(pid, set, ric);
1a472c9d	52	}
96ed1fe5 MN	53	if (cpu_has_feature(CPU_FTR_POWER9_DD1))
96ed1fe5 MN	54	asm volatile(PPC_INVALIDATE_ERAT : : :"memory");
1a472c9d AK	55	return;
	56	}
	57
36194812	58	static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
1a472c9d	59	{
36194812	60	unsigned long rb,rs,prs,r;
1a472c9d AK	61
	62	rb = PPC_BIT(53); /* IS = 1 */
	63	rs = pid << PPC_BITLSHIFT(31);
	64	prs = 1; /* process scoped */
	65	r = 1; /* raidx format */
1a472c9d AK	66
1a472c9d AK	67	asm volatile("ptesync": : :"memory");
8cd6d3c2	68	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
1a472c9d AK	69	: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	70	asm volatile("eieio; tlbsync; ptesync": : :"memory");
	71	}
	72
	73	static inline void _tlbiel_va(unsigned long va, unsigned long pid,
36194812	74	unsigned long ap, unsigned long ric)
1a472c9d	75	{
36194812	76	unsigned long rb,rs,prs,r;
1a472c9d AK	77
	78	rb = va & ~(PPC_BITMASK(52, 63));
	79	rb \|= ap << PPC_BITLSHIFT(58);
	80	rs = pid << PPC_BITLSHIFT(31);
	81	prs = 1; /* process scoped */
	82	r = 1; /* raidx format */
1a472c9d AK	83
1a472c9d AK	84	asm volatile("ptesync": : :"memory");
8cd6d3c2	85	asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
1a472c9d AK	86	: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
1a472c9d AK	87	asm volatile("ptesync": : :"memory");
96ed1fe5 MN	88	if (cpu_has_feature(CPU_FTR_POWER9_DD1))
96ed1fe5 MN	89	asm volatile(PPC_INVALIDATE_ERAT : : :"memory");
1a472c9d AK	90	}
	91
	92	static inline void _tlbie_va(unsigned long va, unsigned long pid,
36194812	93	unsigned long ap, unsigned long ric)
1a472c9d	94	{
36194812	95	unsigned long rb,rs,prs,r;
1a472c9d AK	96
	97	rb = va & ~(PPC_BITMASK(52, 63));
	98	rb \|= ap << PPC_BITLSHIFT(58);
	99	rs = pid << PPC_BITLSHIFT(31);
	100	prs = 1; /* process scoped */
	101	r = 1; /* raidx format */
1a472c9d AK	102
1a472c9d AK	103	asm volatile("ptesync": : :"memory");
8cd6d3c2	104	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
1a472c9d AK	105	: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	106	asm volatile("eieio; tlbsync; ptesync": : :"memory");
	107	}
	108
	109	/*
	110	* Base TLB flushing operations:
	111	*
	112	* - flush_tlb_mm(mm) flushes the specified mm context TLB's
	113	* - flush_tlb_page(vma, vmaddr) flushes one page
	114	* - flush_tlb_range(vma, start, end) flushes a range of pages
	115	* - flush_tlb_kernel_range(start, end) flushes kernel pages
	116	*
	117	* - local_* variants of page and mm only apply to the current
	118	* processor
	119	*/
	120	void radix__local_flush_tlb_mm(struct mm_struct *mm)
	121	{
9690c157	122	unsigned long pid;
1a472c9d AK	123
	124	preempt_disable();
	125	pid = mm->context.id;
	126	if (pid != MMU_NO_CONTEXT)
36194812	127	_tlbiel_pid(pid, RIC_FLUSH_ALL);
1a472c9d AK	128	preempt_enable();
	129	}
	130	EXPORT_SYMBOL(radix__local_flush_tlb_mm);
	131
a145abf1 AK	132	void radix__local_flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
	133	{
	134	unsigned long pid;
	135	struct mm_struct *mm = tlb->mm;
	136
	137	preempt_disable();
	138
	139	pid = mm->context.id;
	140	if (pid != MMU_NO_CONTEXT)
	141	_tlbiel_pid(pid, RIC_FLUSH_PWC);
	142
	143	preempt_enable();
	144	}
	145	EXPORT_SYMBOL(radix__local_flush_tlb_pwc);
	146
f22dfc91	147	void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
fbfa26d8	148	int psize)
1a472c9d	149	{
9690c157	150	unsigned long pid;
fbfa26d8	151	unsigned long ap = mmu_get_ap(psize);
1a472c9d AK	152
	153	preempt_disable();
	154	pid = mm ? mm->context.id : 0;
	155	if (pid != MMU_NO_CONTEXT)
36194812	156	_tlbiel_va(vmaddr, pid, ap, RIC_FLUSH_TLB);
1a472c9d AK	157	preempt_enable();
	158	}
	159
	160	void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
	161	{
48483760 AK	162	#ifdef CONFIG_HUGETLB_PAGE
	163	/* need the return fix for nohash.c */
	164	if (vma && is_vm_hugetlb_page(vma))
	165	return __local_flush_hugetlb_page(vma, vmaddr);
	166	#endif
f22dfc91	167	radix__local_flush_tlb_page_psize(vma ? vma->vm_mm : NULL, vmaddr,
fbfa26d8	168	mmu_virtual_psize);
1a472c9d AK	169	}
	170	EXPORT_SYMBOL(radix__local_flush_tlb_page);
	171
	172	#ifdef CONFIG_SMP
1a472c9d AK	173	void radix__flush_tlb_mm(struct mm_struct *mm)
1a472c9d AK	174	{
9690c157	175	unsigned long pid;
1a472c9d AK	176
	177	preempt_disable();
	178	pid = mm->context.id;
	179	if (unlikely(pid == MMU_NO_CONTEXT))
	180	goto no_context;
	181
bd77c449	182	if (!mm_is_thread_local(mm)) {
1a472c9d AK	183	int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
	184
	185	if (lock_tlbie)
	186	raw_spin_lock(&native_tlbie_lock);
36194812	187	_tlbie_pid(pid, RIC_FLUSH_ALL);
1a472c9d AK	188	if (lock_tlbie)
	189	raw_spin_unlock(&native_tlbie_lock);
	190	} else
36194812	191	_tlbiel_pid(pid, RIC_FLUSH_ALL);
1a472c9d AK	192	no_context:
	193	preempt_enable();
	194	}
	195	EXPORT_SYMBOL(radix__flush_tlb_mm);
	196
a145abf1 AK	197	void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
	198	{
	199	unsigned long pid;
	200	struct mm_struct *mm = tlb->mm;
	201
	202	preempt_disable();
	203
	204	pid = mm->context.id;
	205	if (unlikely(pid == MMU_NO_CONTEXT))
	206	goto no_context;
	207
bd77c449	208	if (!mm_is_thread_local(mm)) {
a145abf1 AK	209	int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
	210
	211	if (lock_tlbie)
	212	raw_spin_lock(&native_tlbie_lock);
	213	_tlbie_pid(pid, RIC_FLUSH_PWC);
	214	if (lock_tlbie)
	215	raw_spin_unlock(&native_tlbie_lock);
	216	} else
	217	_tlbiel_pid(pid, RIC_FLUSH_PWC);
	218	no_context:
	219	preempt_enable();
	220	}
	221	EXPORT_SYMBOL(radix__flush_tlb_pwc);
	222
f22dfc91	223	void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
fbfa26d8	224	int psize)
1a472c9d	225	{
9690c157	226	unsigned long pid;
fbfa26d8	227	unsigned long ap = mmu_get_ap(psize);
1a472c9d AK	228
	229	preempt_disable();
	230	pid = mm ? mm->context.id : 0;
	231	if (unlikely(pid == MMU_NO_CONTEXT))
	232	goto bail;
bd77c449	233	if (!mm_is_thread_local(mm)) {
1a472c9d AK	234	int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
	235
	236	if (lock_tlbie)
	237	raw_spin_lock(&native_tlbie_lock);
36194812	238	_tlbie_va(vmaddr, pid, ap, RIC_FLUSH_TLB);
1a472c9d AK	239	if (lock_tlbie)
	240	raw_spin_unlock(&native_tlbie_lock);
	241	} else
36194812	242	_tlbiel_va(vmaddr, pid, ap, RIC_FLUSH_TLB);
1a472c9d AK	243	bail:
	244	preempt_enable();
	245	}
	246
	247	void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
	248	{
48483760 AK	249	#ifdef CONFIG_HUGETLB_PAGE
	250	if (vma && is_vm_hugetlb_page(vma))
	251	return flush_hugetlb_page(vma, vmaddr);
	252	#endif
f22dfc91	253	radix__flush_tlb_page_psize(vma ? vma->vm_mm : NULL, vmaddr,
fbfa26d8	254	mmu_virtual_psize);
1a472c9d AK	255	}
	256	EXPORT_SYMBOL(radix__flush_tlb_page);
	257
	258	#endif /* CONFIG_SMP */
	259
	260	void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end)
	261	{
	262	int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
	263
	264	if (lock_tlbie)
	265	raw_spin_lock(&native_tlbie_lock);
36194812	266	_tlbie_pid(0, RIC_FLUSH_ALL);
1a472c9d AK	267	if (lock_tlbie)
	268	raw_spin_unlock(&native_tlbie_lock);
	269	}
	270	EXPORT_SYMBOL(radix__flush_tlb_kernel_range);
	271
	272	/*
	273	* Currently, for range flushing, we just do a full mm flush. Because
	274	* we use this in code path where we don' track the page size.
	275	*/
	276	void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
	277	unsigned long end)
	278
	279	{
	280	struct mm_struct *mm = vma->vm_mm;
	281	radix__flush_tlb_mm(mm);
	282	}
	283	EXPORT_SYMBOL(radix__flush_tlb_range);
	284
912cc87a AK	285	static int radix_get_mmu_psize(int page_size)
	286	{
	287	int psize;
	288
	289	if (page_size == (1UL << mmu_psize_defs[mmu_virtual_psize].shift))
	290	psize = mmu_virtual_psize;
	291	else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_2M].shift))
	292	psize = MMU_PAGE_2M;
	293	else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_1G].shift))
	294	psize = MMU_PAGE_1G;
	295	else
	296	return -1;
	297	return psize;
	298	}
1a472c9d AK	299
	300	void radix__tlb_flush(struct mmu_gather *tlb)
	301	{
8cb8140c	302	int psize = 0;
1a472c9d	303	struct mm_struct *mm = tlb->mm;
8cb8140c AK	304	int page_size = tlb->page_size;
	305
	306	psize = radix_get_mmu_psize(page_size);
	307	/*
	308	* if page size is not something we understand, do a full mm flush
	309	*/
	310	if (psize != -1 && !tlb->fullmm && !tlb->need_flush_all)
	311	radix__flush_tlb_range_psize(mm, tlb->start, tlb->end, psize);
	312	else
	313	radix__flush_tlb_mm(mm);
	314	}
	315
	316	#define TLB_FLUSH_ALL -1UL
	317	/*
	318	* Number of pages above which we will do a bcast tlbie. Just a
	319	* number at this point copied from x86
	320	*/
	321	static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33;
	322
	323	void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
	324	unsigned long end, int psize)
	325	{
	326	unsigned long pid;
	327	unsigned long addr;
bd77c449	328	int local = mm_is_thread_local(mm);
8cb8140c AK	329	unsigned long ap = mmu_get_ap(psize);
	330	int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
	331	unsigned long page_size = 1UL << mmu_psize_defs[psize].shift;
	332
	333
	334	preempt_disable();
	335	pid = mm ? mm->context.id : 0;
	336	if (unlikely(pid == MMU_NO_CONTEXT))
	337	goto err_out;
	338
	339	if (end == TLB_FLUSH_ALL \|\|
	340	(end - start) > tlb_single_page_flush_ceiling * page_size) {
	341	if (local)
	342	_tlbiel_pid(pid, RIC_FLUSH_TLB);
	343	else
	344	_tlbie_pid(pid, RIC_FLUSH_TLB);
	345	goto err_out;
	346	}
	347	for (addr = start; addr < end; addr += page_size) {
	348
	349	if (local)
	350	_tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB);
	351	else {
	352	if (lock_tlbie)
	353	raw_spin_lock(&native_tlbie_lock);
	354	_tlbie_va(addr, pid, ap, RIC_FLUSH_TLB);
	355	if (lock_tlbie)
	356	raw_spin_unlock(&native_tlbie_lock);
	357	}
	358	}
	359	err_out:
	360	preempt_enable();
1a472c9d	361	}
912cc87a AK	362
	363	void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
	364	unsigned long page_size)
	365	{
	366	unsigned long rb,rs,prs,r;
	367	unsigned long ap;
	368	unsigned long ric = RIC_FLUSH_TLB;
	369
	370	ap = mmu_get_ap(radix_get_mmu_psize(page_size));
	371	rb = gpa & ~(PPC_BITMASK(52, 63));
	372	rb \|= ap << PPC_BITLSHIFT(58);
	373	rs = lpid & ((1UL << 32) - 1);
	374	prs = 0; /* process scoped */
	375	r = 1; /* raidx format */
	376
	377	asm volatile("ptesync": : :"memory");
	378	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
	379	: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	380	asm volatile("eieio; tlbsync; ptesync": : :"memory");
	381	}
	382	EXPORT_SYMBOL(radix__flush_tlb_lpid_va);
	383
	384	void radix__flush_tlb_lpid(unsigned long lpid)
	385	{
	386	unsigned long rb,rs,prs,r;
	387	unsigned long ric = RIC_FLUSH_ALL;
	388
	389	rb = 0x2 << PPC_BITLSHIFT(53); /* IS = 2 */
	390	rs = lpid & ((1UL << 32) - 1);
	391	prs = 0; /* partition scoped */
	392	r = 1; /* raidx format */
	393
	394	asm volatile("ptesync": : :"memory");
	395	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
	396	: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	397	asm volatile("eieio; tlbsync; ptesync": : :"memory");
	398	}
	399	EXPORT_SYMBOL(radix__flush_tlb_lpid);
d8e91e93 AK	400
	401	void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
	402	unsigned long start, unsigned long end)
	403	{
	404	radix__flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M);
	405	}
	406	EXPORT_SYMBOL(radix__flush_pmd_tlb_range);
be34d300 AK	407
	408	void radix__flush_tlb_all(void)
	409	{
	410	unsigned long rb,prs,r,rs;
	411	unsigned long ric = RIC_FLUSH_ALL;
	412
	413	rb = 0x3 << PPC_BITLSHIFT(53); /* IS = 3 */
	414	prs = 0; /* partition scoped */
	415	r = 1; /* raidx format */
	416	rs = 1 & ((1UL << 32) - 1); /* any LPID value to flush guest mappings */
	417
	418	asm volatile("ptesync": : :"memory");
	419	/*
	420	* now flush guest entries by passing PRS = 1 and LPID != 0
	421	*/
	422	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
	423	: : "r"(rb), "i"(r), "i"(1), "i"(ric), "r"(rs) : "memory");
	424	/*
	425	* now flush host entires by passing PRS = 0 and LPID == 0
	426	*/
	427	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
	428	: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(0) : "memory");
	429	asm volatile("eieio; tlbsync; ptesync": : :"memory");
	430	}