[mirror_ubuntu-zesty-kernel.git] / arch / powerpc / mm / tlb-radix.c

/*
 * TLB flush routines for radix kernels.
 *
 * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/memblock.h>

#include <asm/tlb.h>
#include <asm/tlbflush.h>

static DEFINE_RAW_SPINLOCK(native_tlbie_lock);

#define RIC_FLUSH_TLB 0
#define RIC_FLUSH_PWC 1
#define RIC_FLUSH_ALL 2

static inline void __tlbiel_pid(unsigned long pid, int set,
				unsigned long ric)
{
	unsigned long rb,rs,prs,r;

	rb = PPC_BIT(53); /* IS = 1 */
	rb |= set << PPC_BITLSHIFT(51);
	rs = ((unsigned long)pid) << PPC_BITLSHIFT(31);
	prs = 1; /* process scoped */
	r = 1;   /* raidx format */

	asm volatile("ptesync": : :"memory");
	asm volatile(".long 0x7c000224 | (%0 << 11) | (%1 << 16) |"
		     "(%2 << 17) | (%3 << 18) | (%4 << 21)"
		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	asm volatile("ptesync": : :"memory");
}

/*
 * We use 128 set in radix mode and 256 set in hpt mode.
 */
static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
{
	int set;

	for (set = 0; set < POWER9_TLB_SETS_RADIX ; set++) {
		__tlbiel_pid(pid, set, ric);
	}
	return;
}

static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
{
	unsigned long rb,rs,prs,r;

	rb = PPC_BIT(53); /* IS = 1 */
	rs = pid << PPC_BITLSHIFT(31);
	prs = 1; /* process scoped */
	r = 1;   /* raidx format */

	asm volatile("ptesync": : :"memory");
	asm volatile(".long 0x7c000264 | (%0 << 11) | (%1 << 16) |"
		     "(%2 << 17) | (%3 << 18) | (%4 << 21)"
		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	asm volatile("eieio; tlbsync; ptesync": : :"memory");
}

static inline void _tlbiel_va(unsigned long va, unsigned long pid,
			      unsigned long ap, unsigned long ric)
{
	unsigned long rb,rs,prs,r;

	rb = va & ~(PPC_BITMASK(52, 63));
	rb |= ap << PPC_BITLSHIFT(58);
	rs = pid << PPC_BITLSHIFT(31);
	prs = 1; /* process scoped */
	r = 1;   /* raidx format */

	asm volatile("ptesync": : :"memory");
	asm volatile(".long 0x7c000224 | (%0 << 11) | (%1 << 16) |"
		     "(%2 << 17) | (%3 << 18) | (%4 << 21)"
		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	asm volatile("ptesync": : :"memory");
}

static inline void _tlbie_va(unsigned long va, unsigned long pid,
			     unsigned long ap, unsigned long ric)
{
	unsigned long rb,rs,prs,r;

	rb = va & ~(PPC_BITMASK(52, 63));
	rb |= ap << PPC_BITLSHIFT(58);
	rs = pid << PPC_BITLSHIFT(31);
	prs = 1; /* process scoped */
	r = 1;   /* raidx format */

	asm volatile("ptesync": : :"memory");
	asm volatile(".long 0x7c000264 | (%0 << 11) | (%1 << 16) |"
		     "(%2 << 17) | (%3 << 18) | (%4 << 21)"
		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	asm volatile("eieio; tlbsync; ptesync": : :"memory");
}

/*
 * Base TLB flushing operations:
 *
 *  - flush_tlb_mm(mm) flushes the specified mm context TLB's
 *  - flush_tlb_page(vma, vmaddr) flushes one page
 *  - flush_tlb_range(vma, start, end) flushes a range of pages
 *  - flush_tlb_kernel_range(start, end) flushes kernel pages
 *
 *  - local_* variants of page and mm only apply to the current
 *    processor
 */
void radix__local_flush_tlb_mm(struct mm_struct *mm)
{
	unsigned long pid;

	preempt_disable();
	pid = mm->context.id;
	if (pid != MMU_NO_CONTEXT)
		_tlbiel_pid(pid, RIC_FLUSH_ALL);
	preempt_enable();
}
EXPORT_SYMBOL(radix__local_flush_tlb_mm);

void radix__local_flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
{
	unsigned long pid;
	struct mm_struct *mm = tlb->mm;

	preempt_disable();

	pid = mm->context.id;
	if (pid != MMU_NO_CONTEXT)
		_tlbiel_pid(pid, RIC_FLUSH_PWC);

	preempt_enable();
}
EXPORT_SYMBOL(radix__local_flush_tlb_pwc);

void radix___local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
			    unsigned long ap, int nid)
{
	unsigned long pid;

	preempt_disable();
	pid = mm ? mm->context.id : 0;
	if (pid != MMU_NO_CONTEXT)
		_tlbiel_va(vmaddr, pid, ap, RIC_FLUSH_TLB);
	preempt_enable();
}

void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
{
#ifdef CONFIG_HUGETLB_PAGE
	/* need the return fix for nohash.c */
	if (vma && is_vm_hugetlb_page(vma))
		return __local_flush_hugetlb_page(vma, vmaddr);
#endif
	radix___local_flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
			       mmu_get_ap(mmu_virtual_psize), 0);
}
EXPORT_SYMBOL(radix__local_flush_tlb_page);

#ifdef CONFIG_SMP
static int mm_is_core_local(struct mm_struct *mm)
{
	return cpumask_subset(mm_cpumask(mm),
			      topology_sibling_cpumask(smp_processor_id()));
}

void radix__flush_tlb_mm(struct mm_struct *mm)
{
	unsigned long pid;

	preempt_disable();
	pid = mm->context.id;
	if (unlikely(pid == MMU_NO_CONTEXT))
		goto no_context;

	if (!mm_is_core_local(mm)) {
		int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);

		if (lock_tlbie)
			raw_spin_lock(&native_tlbie_lock);
		_tlbie_pid(pid, RIC_FLUSH_ALL);
		if (lock_tlbie)
			raw_spin_unlock(&native_tlbie_lock);
	} else
		_tlbiel_pid(pid, RIC_FLUSH_ALL);
no_context:
	preempt_enable();
}
EXPORT_SYMBOL(radix__flush_tlb_mm);

void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
{
	unsigned long pid;
	struct mm_struct *mm = tlb->mm;

	preempt_disable();

	pid = mm->context.id;
	if (unlikely(pid == MMU_NO_CONTEXT))
		goto no_context;

	if (!mm_is_core_local(mm)) {
		int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);

		if (lock_tlbie)
			raw_spin_lock(&native_tlbie_lock);
		_tlbie_pid(pid, RIC_FLUSH_PWC);
		if (lock_tlbie)
			raw_spin_unlock(&native_tlbie_lock);
	} else
		_tlbiel_pid(pid, RIC_FLUSH_PWC);
no_context:
	preempt_enable();
}
EXPORT_SYMBOL(radix__flush_tlb_pwc);

void radix___flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
		       unsigned long ap, int nid)
{
	unsigned long pid;

	preempt_disable();
	pid = mm ? mm->context.id : 0;
	if (unlikely(pid == MMU_NO_CONTEXT))
		goto bail;
	if (!mm_is_core_local(mm)) {
		int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);

		if (lock_tlbie)
			raw_spin_lock(&native_tlbie_lock);
		_tlbie_va(vmaddr, pid, ap, RIC_FLUSH_TLB);
		if (lock_tlbie)
			raw_spin_unlock(&native_tlbie_lock);
	} else
		_tlbiel_va(vmaddr, pid, ap, RIC_FLUSH_TLB);
bail:
	preempt_enable();
}

void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
{
#ifdef CONFIG_HUGETLB_PAGE
	if (vma && is_vm_hugetlb_page(vma))
		return flush_hugetlb_page(vma, vmaddr);
#endif
	radix___flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
			 mmu_get_ap(mmu_virtual_psize), 0);
}
EXPORT_SYMBOL(radix__flush_tlb_page);

#endif /* CONFIG_SMP */

void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
	int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);

	if (lock_tlbie)
		raw_spin_lock(&native_tlbie_lock);
	_tlbie_pid(0, RIC_FLUSH_ALL);
	if (lock_tlbie)
		raw_spin_unlock(&native_tlbie_lock);
}
EXPORT_SYMBOL(radix__flush_tlb_kernel_range);

/*
 * Currently, for range flushing, we just do a full mm flush. Because
 * we use this in code path where we don' track the page size.
 */
void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
		     unsigned long end)

{
	struct mm_struct *mm = vma->vm_mm;
	radix__flush_tlb_mm(mm);
}
EXPORT_SYMBOL(radix__flush_tlb_range);


void radix__tlb_flush(struct mmu_gather *tlb)
{
	struct mm_struct *mm = tlb->mm;
	radix__flush_tlb_mm(mm);
}
Commit	Line	Data
1a472c9d AK	1	/*
	2	* TLB flush routines for radix kernels.
	3	*
	4	* Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
	5	*
	6	* This program is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU General Public License
	8	* as published by the Free Software Foundation; either version
	9	* 2 of the License, or (at your option) any later version.
	10	*/
	11
	12	#include <linux/mm.h>
	13	#include <linux/hugetlb.h>
	14	#include <linux/memblock.h>
	15
	16	#include <asm/tlb.h>
	17	#include <asm/tlbflush.h>
	18
	19	static DEFINE_RAW_SPINLOCK(native_tlbie_lock);
	20
36194812 AK	21	#define RIC_FLUSH_TLB 0
	22	#define RIC_FLUSH_PWC 1
	23	#define RIC_FLUSH_ALL 2
	24
	25	static inline void __tlbiel_pid(unsigned long pid, int set,
	26	unsigned long ric)
1a472c9d	27	{
36194812	28	unsigned long rb,rs,prs,r;
1a472c9d AK	29
	30	rb = PPC_BIT(53); /* IS = 1 */
	31	rb \|= set << PPC_BITLSHIFT(51);
	32	rs = ((unsigned long)pid) << PPC_BITLSHIFT(31);
	33	prs = 1; /* process scoped */
	34	r = 1; /* raidx format */
1a472c9d AK	35
	36	asm volatile("ptesync": : :"memory");
	37	asm volatile(".long 0x7c000224 \| (%0 << 11) \| (%1 << 16) \|"
	38	"(%2 << 17) \| (%3 << 18) \| (%4 << 21)"
	39	: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	40	asm volatile("ptesync": : :"memory");
	41	}
	42
	43	/*
	44	* We use 128 set in radix mode and 256 set in hpt mode.
	45	*/
36194812	46	static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
1a472c9d AK	47	{
	48	int set;
	49
	50	for (set = 0; set < POWER9_TLB_SETS_RADIX ; set++) {
36194812	51	__tlbiel_pid(pid, set, ric);
1a472c9d AK	52	}
	53	return;
	54	}
	55
36194812	56	static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
1a472c9d	57	{
36194812	58	unsigned long rb,rs,prs,r;
1a472c9d AK	59
	60	rb = PPC_BIT(53); /* IS = 1 */
	61	rs = pid << PPC_BITLSHIFT(31);
	62	prs = 1; /* process scoped */
	63	r = 1; /* raidx format */
1a472c9d AK	64
	65	asm volatile("ptesync": : :"memory");
	66	asm volatile(".long 0x7c000264 \| (%0 << 11) \| (%1 << 16) \|"
	67	"(%2 << 17) \| (%3 << 18) \| (%4 << 21)"
	68	: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	69	asm volatile("eieio; tlbsync; ptesync": : :"memory");
	70	}
	71
	72	static inline void _tlbiel_va(unsigned long va, unsigned long pid,
36194812	73	unsigned long ap, unsigned long ric)
1a472c9d	74	{
36194812	75	unsigned long rb,rs,prs,r;
1a472c9d AK	76
	77	rb = va & ~(PPC_BITMASK(52, 63));
	78	rb \|= ap << PPC_BITLSHIFT(58);
	79	rs = pid << PPC_BITLSHIFT(31);
	80	prs = 1; /* process scoped */
	81	r = 1; /* raidx format */
1a472c9d AK	82
	83	asm volatile("ptesync": : :"memory");
	84	asm volatile(".long 0x7c000224 \| (%0 << 11) \| (%1 << 16) \|"
	85	"(%2 << 17) \| (%3 << 18) \| (%4 << 21)"
	86	: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	87	asm volatile("ptesync": : :"memory");
	88	}
	89
	90	static inline void _tlbie_va(unsigned long va, unsigned long pid,
36194812	91	unsigned long ap, unsigned long ric)
1a472c9d	92	{
36194812	93	unsigned long rb,rs,prs,r;
1a472c9d AK	94
	95	rb = va & ~(PPC_BITMASK(52, 63));
	96	rb \|= ap << PPC_BITLSHIFT(58);
	97	rs = pid << PPC_BITLSHIFT(31);
	98	prs = 1; /* process scoped */
	99	r = 1; /* raidx format */
1a472c9d AK	100
	101	asm volatile("ptesync": : :"memory");
	102	asm volatile(".long 0x7c000264 \| (%0 << 11) \| (%1 << 16) \|"
	103	"(%2 << 17) \| (%3 << 18) \| (%4 << 21)"
	104	: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	105	asm volatile("eieio; tlbsync; ptesync": : :"memory");
	106	}
	107
	108	/*
	109	* Base TLB flushing operations:
	110	*
	111	* - flush_tlb_mm(mm) flushes the specified mm context TLB's
	112	* - flush_tlb_page(vma, vmaddr) flushes one page
	113	* - flush_tlb_range(vma, start, end) flushes a range of pages
	114	* - flush_tlb_kernel_range(start, end) flushes kernel pages
	115	*
	116	* - local_* variants of page and mm only apply to the current
	117	* processor
	118	*/
	119	void radix__local_flush_tlb_mm(struct mm_struct *mm)
	120	{
9690c157	121	unsigned long pid;
1a472c9d AK	122
	123	preempt_disable();
	124	pid = mm->context.id;
	125	if (pid != MMU_NO_CONTEXT)
36194812	126	_tlbiel_pid(pid, RIC_FLUSH_ALL);
1a472c9d AK	127	preempt_enable();
	128	}
	129	EXPORT_SYMBOL(radix__local_flush_tlb_mm);
	130
a145abf1 AK	131	void radix__local_flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
	132	{
	133	unsigned long pid;
	134	struct mm_struct *mm = tlb->mm;
	135
	136	preempt_disable();
	137
	138	pid = mm->context.id;
	139	if (pid != MMU_NO_CONTEXT)
	140	_tlbiel_pid(pid, RIC_FLUSH_PWC);
	141
	142	preempt_enable();
	143	}
	144	EXPORT_SYMBOL(radix__local_flush_tlb_pwc);
	145
1a472c9d AK	146	void radix___local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
	147	unsigned long ap, int nid)
	148	{
9690c157	149	unsigned long pid;
1a472c9d AK	150
	151	preempt_disable();
	152	pid = mm ? mm->context.id : 0;
	153	if (pid != MMU_NO_CONTEXT)
36194812	154	_tlbiel_va(vmaddr, pid, ap, RIC_FLUSH_TLB);
1a472c9d AK	155	preempt_enable();
	156	}
	157
	158	void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
	159	{
48483760 AK	160	#ifdef CONFIG_HUGETLB_PAGE
	161	/* need the return fix for nohash.c */
	162	if (vma && is_vm_hugetlb_page(vma))
	163	return __local_flush_hugetlb_page(vma, vmaddr);
	164	#endif
1a472c9d AK	165	radix___local_flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
	166	mmu_get_ap(mmu_virtual_psize), 0);
	167	}
	168	EXPORT_SYMBOL(radix__local_flush_tlb_page);
	169
	170	#ifdef CONFIG_SMP
	171	static int mm_is_core_local(struct mm_struct *mm)
	172	{
	173	return cpumask_subset(mm_cpumask(mm),
	174	topology_sibling_cpumask(smp_processor_id()));
	175	}
	176
	177	void radix__flush_tlb_mm(struct mm_struct *mm)
	178	{
9690c157	179	unsigned long pid;
1a472c9d AK	180
	181	preempt_disable();
	182	pid = mm->context.id;
	183	if (unlikely(pid == MMU_NO_CONTEXT))
	184	goto no_context;
	185
	186	if (!mm_is_core_local(mm)) {
	187	int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
	188
	189	if (lock_tlbie)
	190	raw_spin_lock(&native_tlbie_lock);
36194812	191	_tlbie_pid(pid, RIC_FLUSH_ALL);
1a472c9d AK	192	if (lock_tlbie)
	193	raw_spin_unlock(&native_tlbie_lock);
	194	} else
36194812	195	_tlbiel_pid(pid, RIC_FLUSH_ALL);
1a472c9d AK	196	no_context:
	197	preempt_enable();
	198	}
	199	EXPORT_SYMBOL(radix__flush_tlb_mm);
	200
a145abf1 AK	201	void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
	202	{
	203	unsigned long pid;
	204	struct mm_struct *mm = tlb->mm;
	205
	206	preempt_disable();
	207
	208	pid = mm->context.id;
	209	if (unlikely(pid == MMU_NO_CONTEXT))
	210	goto no_context;
	211
	212	if (!mm_is_core_local(mm)) {
	213	int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
	214
	215	if (lock_tlbie)
	216	raw_spin_lock(&native_tlbie_lock);
	217	_tlbie_pid(pid, RIC_FLUSH_PWC);
	218	if (lock_tlbie)
	219	raw_spin_unlock(&native_tlbie_lock);
	220	} else
	221	_tlbiel_pid(pid, RIC_FLUSH_PWC);
	222	no_context:
	223	preempt_enable();
	224	}
	225	EXPORT_SYMBOL(radix__flush_tlb_pwc);
	226
1a472c9d AK	227	void radix___flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
	228	unsigned long ap, int nid)
	229	{
9690c157	230	unsigned long pid;
1a472c9d AK	231
	232	preempt_disable();
	233	pid = mm ? mm->context.id : 0;
	234	if (unlikely(pid == MMU_NO_CONTEXT))
	235	goto bail;
	236	if (!mm_is_core_local(mm)) {
	237	int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
	238
	239	if (lock_tlbie)
	240	raw_spin_lock(&native_tlbie_lock);
36194812	241	_tlbie_va(vmaddr, pid, ap, RIC_FLUSH_TLB);
1a472c9d AK	242	if (lock_tlbie)
	243	raw_spin_unlock(&native_tlbie_lock);
	244	} else
36194812	245	_tlbiel_va(vmaddr, pid, ap, RIC_FLUSH_TLB);
1a472c9d AK	246	bail:
	247	preempt_enable();
	248	}
	249
	250	void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
	251	{
48483760 AK	252	#ifdef CONFIG_HUGETLB_PAGE
	253	if (vma && is_vm_hugetlb_page(vma))
	254	return flush_hugetlb_page(vma, vmaddr);
	255	#endif
1a472c9d AK	256	radix___flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
	257	mmu_get_ap(mmu_virtual_psize), 0);
	258	}
	259	EXPORT_SYMBOL(radix__flush_tlb_page);
	260
	261	#endif /* CONFIG_SMP */
	262
	263	void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end)
	264	{
	265	int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
	266
	267	if (lock_tlbie)
	268	raw_spin_lock(&native_tlbie_lock);
36194812	269	_tlbie_pid(0, RIC_FLUSH_ALL);
1a472c9d AK	270	if (lock_tlbie)
	271	raw_spin_unlock(&native_tlbie_lock);
	272	}
	273	EXPORT_SYMBOL(radix__flush_tlb_kernel_range);
	274
	275	/*
	276	* Currently, for range flushing, we just do a full mm flush. Because
	277	* we use this in code path where we don' track the page size.
	278	*/
	279	void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
	280	unsigned long end)
	281
	282	{
	283	struct mm_struct *mm = vma->vm_mm;
	284	radix__flush_tlb_mm(mm);
	285	}
	286	EXPORT_SYMBOL(radix__flush_tlb_range);
	287
	288
	289	void radix__tlb_flush(struct mmu_gather *tlb)
	290	{
	291	struct mm_struct *mm = tlb->mm;
	292	radix__flush_tlb_mm(mm);
	293	}