[mirror_ubuntu-bionic-kernel.git] / arch / arm / kernel / smp_tlb.c

/*
 *  linux/arch/arm/kernel/smp_tlb.c
 *
 *  Copyright (C) 2002 ARM Limited, All Rights Reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */
#include <linux/preempt.h>
#include <linux/smp.h>
#include <linux/uaccess.h>

#include <asm/smp_plat.h>
#include <asm/tlbflush.h>
#include <asm/mmu_context.h>

/**********************************************************************/

/*
 * TLB operations
 */
struct tlb_args {
	struct vm_area_struct *ta_vma;
	unsigned long ta_start;
	unsigned long ta_end;
};

static inline void ipi_flush_tlb_all(void *ignored)
{
	local_flush_tlb_all();
}

static inline void ipi_flush_tlb_mm(void *arg)
{
	struct mm_struct *mm = (struct mm_struct *)arg;

	local_flush_tlb_mm(mm);
}

static inline void ipi_flush_tlb_page(void *arg)
{
	struct tlb_args *ta = (struct tlb_args *)arg;
	unsigned int __ua_flags = uaccess_save_and_enable();

	local_flush_tlb_page(ta->ta_vma, ta->ta_start);

	uaccess_restore(__ua_flags);
}

static inline void ipi_flush_tlb_kernel_page(void *arg)
{
	struct tlb_args *ta = (struct tlb_args *)arg;

	local_flush_tlb_kernel_page(ta->ta_start);
}

static inline void ipi_flush_tlb_range(void *arg)
{
	struct tlb_args *ta = (struct tlb_args *)arg;
	unsigned int __ua_flags = uaccess_save_and_enable();

	local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);

	uaccess_restore(__ua_flags);
}

static inline void ipi_flush_tlb_kernel_range(void *arg)
{
	struct tlb_args *ta = (struct tlb_args *)arg;

	local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end);
}

static inline void ipi_flush_bp_all(void *ignored)
{
	local_flush_bp_all();
}

#ifdef CONFIG_ARM_ERRATA_798181
bool (*erratum_a15_798181_handler)(void);

static bool erratum_a15_798181_partial(void)
{
	asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0));
	dsb(ish);
	return false;
}

static bool erratum_a15_798181_broadcast(void)
{
	asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0));
	dsb(ish);
	return true;
}

void erratum_a15_798181_init(void)
{
	unsigned int midr = read_cpuid_id();
	unsigned int revidr = read_cpuid(CPUID_REVIDR);

	/* Brahma-B15 r0p0..r0p2 affected
	 * Cortex-A15 r0p0..r3p3 w/o ECO fix affected
	 * Fixes applied to A15 with respect to the revision and revidr are:
	 *
	 * r0p0-r2p1: No fixes applied
	 * r2p2,r2p3:
	 *	REVIDR[4]: 798181 Moving a virtual page that is being accessed
	 *		   by an active process can lead to unexpected behavior
	 *	REVIDR[9]: Not defined
	 * r2p4,r3p0,r3p1,r3p2:
	 *	REVIDR[4]: 798181 Moving a virtual page that is being accessed
	 *		   by an active process can lead to unexpected behavior
	 *	REVIDR[9]: 798181 Moving a virtual page that is being accessed
	 *		   by an active process can lead to unexpected behavior
	 *		   - This is an update to a previously released ECO.
	 * r3p3:
	 *	REVIDR[4]: Reserved
	 *	REVIDR[9]: 798181 Moving a virtual page that is being accessed
	 *		   by an active process can lead to unexpected behavior
	 *		   - This is an update to a previously released ECO.
	 *
	 * Handling:
	 *	REVIDR[9] set -> No WA
	 *	REVIDR[4] set, REVIDR[9] cleared -> Partial WA
	 *	Both cleared -> Full WA
	 */
	if ((midr & 0xff0ffff0) == 0x420f00f0 && midr <= 0x420f00f2) {
		erratum_a15_798181_handler = erratum_a15_798181_broadcast;
	} else if ((midr & 0xff0ffff0) == 0x410fc0f0 && midr < 0x412fc0f2) {
		erratum_a15_798181_handler = erratum_a15_798181_broadcast;
	} else if ((midr & 0xff0ffff0) == 0x410fc0f0 && midr < 0x412fc0f4) {
		if (revidr & 0x10)
			erratum_a15_798181_handler =
				erratum_a15_798181_partial;
		else
			erratum_a15_798181_handler =
				erratum_a15_798181_broadcast;
	} else if ((midr & 0xff0ffff0) == 0x410fc0f0 && midr < 0x413fc0f3) {
		if ((revidr & 0x210) == 0)
			erratum_a15_798181_handler =
				erratum_a15_798181_broadcast;
		else if (revidr & 0x10)
			erratum_a15_798181_handler =
				erratum_a15_798181_partial;
	} else if ((midr & 0xff0ffff0) == 0x410fc0f0 && midr < 0x414fc0f0) {
		if ((revidr & 0x200) == 0)
			erratum_a15_798181_handler =
				erratum_a15_798181_partial;
	}
}
#endif

static void ipi_flush_tlb_a15_erratum(void *arg)
{
	dmb();
}

static void broadcast_tlb_a15_erratum(void)
{
	if (!erratum_a15_798181())
		return;

	smp_call_function(ipi_flush_tlb_a15_erratum, NULL, 1);
}

static void broadcast_tlb_mm_a15_erratum(struct mm_struct *mm)
{
	int this_cpu;
	cpumask_t mask = { CPU_BITS_NONE };

	if (!erratum_a15_798181())
		return;

	this_cpu = get_cpu();
	a15_erratum_get_cpumask(this_cpu, mm, &mask);
	smp_call_function_many(&mask, ipi_flush_tlb_a15_erratum, NULL, 1);
	put_cpu();
}

void flush_tlb_all(void)
{
	if (tlb_ops_need_broadcast())
		on_each_cpu(ipi_flush_tlb_all, NULL, 1);
	else
		__flush_tlb_all();
	broadcast_tlb_a15_erratum();
}

void flush_tlb_mm(struct mm_struct *mm)
{
	if (tlb_ops_need_broadcast())
		on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_mm, mm, 1);
	else
		__flush_tlb_mm(mm);
	broadcast_tlb_mm_a15_erratum(mm);
}

void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
{
	if (tlb_ops_need_broadcast()) {
		struct tlb_args ta;
		ta.ta_vma = vma;
		ta.ta_start = uaddr;
		on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_page,
					&ta, 1);
	} else
		__flush_tlb_page(vma, uaddr);
	broadcast_tlb_mm_a15_erratum(vma->vm_mm);
}

void flush_tlb_kernel_page(unsigned long kaddr)
{
	if (tlb_ops_need_broadcast()) {
		struct tlb_args ta;
		ta.ta_start = kaddr;
		on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1);
	} else
		__flush_tlb_kernel_page(kaddr);
	broadcast_tlb_a15_erratum();
}

void flush_tlb_range(struct vm_area_struct *vma,
                     unsigned long start, unsigned long end)
{
	if (tlb_ops_need_broadcast()) {
		struct tlb_args ta;
		ta.ta_vma = vma;
		ta.ta_start = start;
		ta.ta_end = end;
		on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_range,
					&ta, 1);
	} else
		local_flush_tlb_range(vma, start, end);
	broadcast_tlb_mm_a15_erratum(vma->vm_mm);
}

void flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
	if (tlb_ops_need_broadcast()) {
		struct tlb_args ta;
		ta.ta_start = start;
		ta.ta_end = end;
		on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
	} else
		local_flush_tlb_kernel_range(start, end);
	broadcast_tlb_a15_erratum();
}

void flush_bp_all(void)
{
	if (tlb_ops_need_broadcast())
		on_each_cpu(ipi_flush_bp_all, NULL, 1);
	else
		__flush_bp_all();
}
Commit	Line	Data
03b505ea RK	1	/*
	2	* linux/arch/arm/kernel/smp_tlb.c
	3	*
	4	* Copyright (C) 2002 ARM Limited, All Rights Reserved.
	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License version 2 as
	8	* published by the Free Software Foundation.
	9	*/
	10	#include <linux/preempt.h>
	11	#include <linux/smp.h>
90f92c63	12	#include <linux/uaccess.h>
03b505ea RK	13
	14	#include <asm/smp_plat.h>
	15	#include <asm/tlbflush.h>
93dc6887	16	#include <asm/mmu_context.h>
03b505ea	17
03b505ea RK	18	/**********************************************************************/
	19
	20	/*
	21	* TLB operations
	22	*/
	23	struct tlb_args {
	24	struct vm_area_struct *ta_vma;
	25	unsigned long ta_start;
	26	unsigned long ta_end;
	27	};
	28
	29	static inline void ipi_flush_tlb_all(void *ignored)
	30	{
	31	local_flush_tlb_all();
	32	}
	33
	34	static inline void ipi_flush_tlb_mm(void *arg)
	35	{
	36	struct mm_struct mm = (struct mm_struct )arg;
	37
	38	local_flush_tlb_mm(mm);
	39	}
	40
	41	static inline void ipi_flush_tlb_page(void *arg)
	42	{
	43	struct tlb_args ta = (struct tlb_args )arg;
90f92c63	44	unsigned int __ua_flags = uaccess_save_and_enable();
03b505ea RK	45
03b505ea RK	46	local_flush_tlb_page(ta->ta_vma, ta->ta_start);
90f92c63 LW	47
90f92c63 LW	48	uaccess_restore(__ua_flags);
03b505ea RK	49	}
	50
	51	static inline void ipi_flush_tlb_kernel_page(void *arg)
	52	{
	53	struct tlb_args ta = (struct tlb_args )arg;
	54
	55	local_flush_tlb_kernel_page(ta->ta_start);
	56	}
	57
	58	static inline void ipi_flush_tlb_range(void *arg)
	59	{
	60	struct tlb_args ta = (struct tlb_args )arg;
90f92c63	61	unsigned int __ua_flags = uaccess_save_and_enable();
03b505ea RK	62
03b505ea RK	63	local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
90f92c63 LW	64
90f92c63 LW	65	uaccess_restore(__ua_flags);
03b505ea RK	66	}
	67
	68	static inline void ipi_flush_tlb_kernel_range(void *arg)
	69	{
	70	struct tlb_args ta = (struct tlb_args )arg;
	71
	72	local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end);
	73	}
	74
862c588f WD	75	static inline void ipi_flush_bp_all(void *ignored)
	76	{
	77	local_flush_bp_all();
	78	}
	79
92871b94 RH	80	#ifdef CONFIG_ARM_ERRATA_798181
	81	bool (*erratum_a15_798181_handler)(void);
	82
	83	static bool erratum_a15_798181_partial(void)
	84	{
	85	asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0));
	86	dsb(ish);
	87	return false;
	88	}
	89
	90	static bool erratum_a15_798181_broadcast(void)
	91	{
	92	asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0));
	93	dsb(ish);
	94	return true;
	95	}
	96
	97	void erratum_a15_798181_init(void)
	98	{
	99	unsigned int midr = read_cpuid_id();
	100	unsigned int revidr = read_cpuid(CPUID_REVIDR);
	101
04fcab32	102	/* Brahma-B15 r0p0..r0p2 affected
f6492164 MGP	103	* Cortex-A15 r0p0..r3p3 w/o ECO fix affected
	104	* Fixes applied to A15 with respect to the revision and revidr are:
	105	*
	106	* r0p0-r2p1: No fixes applied
	107	* r2p2,r2p3:
	108	* REVIDR[4]: 798181 Moving a virtual page that is being accessed
	109	* by an active process can lead to unexpected behavior
	110	* REVIDR[9]: Not defined
	111	* r2p4,r3p0,r3p1,r3p2:
	112	* REVIDR[4]: 798181 Moving a virtual page that is being accessed
	113	* by an active process can lead to unexpected behavior
	114	* REVIDR[9]: 798181 Moving a virtual page that is being accessed
	115	* by an active process can lead to unexpected behavior
	116	* - This is an update to a previously released ECO.
	117	* r3p3:
	118	* REVIDR[4]: Reserved
	119	* REVIDR[9]: 798181 Moving a virtual page that is being accessed
	120	* by an active process can lead to unexpected behavior
	121	* - This is an update to a previously released ECO.
	122	*
	123	* Handling:
	124	* REVIDR[9] set -> No WA
	125	* REVIDR[4] set, REVIDR[9] cleared -> Partial WA
	126	* Both cleared -> Full WA
	127	*/
	128	if ((midr & 0xff0ffff0) == 0x420f00f0 && midr <= 0x420f00f2) {
92871b94	129	erratum_a15_798181_handler = erratum_a15_798181_broadcast;
f6492164 MGP	130	} else if ((midr & 0xff0ffff0) == 0x410fc0f0 && midr < 0x412fc0f2) {
	131	erratum_a15_798181_handler = erratum_a15_798181_broadcast;
	132	} else if ((midr & 0xff0ffff0) == 0x410fc0f0 && midr < 0x412fc0f4) {
04fcab32 GF	133	if (revidr & 0x10)
	134	erratum_a15_798181_handler =
	135	erratum_a15_798181_partial;
	136	else
	137	erratum_a15_798181_handler =
	138	erratum_a15_798181_broadcast;
f6492164 MGP	139	} else if ((midr & 0xff0ffff0) == 0x410fc0f0 && midr < 0x413fc0f3) {
	140	if ((revidr & 0x210) == 0)
	141	erratum_a15_798181_handler =
	142	erratum_a15_798181_broadcast;
	143	else if (revidr & 0x10)
	144	erratum_a15_798181_handler =
	145	erratum_a15_798181_partial;
	146	} else if ((midr & 0xff0ffff0) == 0x410fc0f0 && midr < 0x414fc0f0) {
	147	if ((revidr & 0x200) == 0)
	148	erratum_a15_798181_handler =
	149	erratum_a15_798181_partial;
04fcab32	150	}
92871b94 RH	151	}
	152	#endif
	153
93dc6887 CM	154	static void ipi_flush_tlb_a15_erratum(void *arg)
	155	{
	156	dmb();
	157	}
	158
	159	static void broadcast_tlb_a15_erratum(void)
	160	{
	161	if (!erratum_a15_798181())
	162	return;
	163
3eb0be30	164	smp_call_function(ipi_flush_tlb_a15_erratum, NULL, 1);
93dc6887 CM	165	}
	166
	167	static void broadcast_tlb_mm_a15_erratum(struct mm_struct *mm)
	168	{
0d0752bc	169	int this_cpu;
93dc6887 CM	170	cpumask_t mask = { CPU_BITS_NONE };
	171
	172	if (!erratum_a15_798181())
	173	return;
	174
3eb0be30	175	this_cpu = get_cpu();
0d0752bc	176	a15_erratum_get_cpumask(this_cpu, mm, &mask);
93dc6887	177	smp_call_function_many(&mask, ipi_flush_tlb_a15_erratum, NULL, 1);
3eb0be30	178	put_cpu();
93dc6887 CM	179	}
93dc6887 CM	180
03b505ea RK	181	void flush_tlb_all(void)
	182	{
	183	if (tlb_ops_need_broadcast())
	184	on_each_cpu(ipi_flush_tlb_all, NULL, 1);
	185	else
f0915781	186	__flush_tlb_all();
93dc6887	187	broadcast_tlb_a15_erratum();
03b505ea RK	188	}
	189
	190	void flush_tlb_mm(struct mm_struct *mm)
	191	{
	192	if (tlb_ops_need_broadcast())
3fc498f1	193	on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_mm, mm, 1);
03b505ea	194	else
f0915781	195	__flush_tlb_mm(mm);
93dc6887	196	broadcast_tlb_mm_a15_erratum(mm);
03b505ea RK	197	}
	198
	199	void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
	200	{
	201	if (tlb_ops_need_broadcast()) {
	202	struct tlb_args ta;
	203	ta.ta_vma = vma;
	204	ta.ta_start = uaddr;
3fc498f1 GBY	205	on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_page,
3fc498f1 GBY	206	&ta, 1);
03b505ea	207	} else
f0915781	208	__flush_tlb_page(vma, uaddr);
93dc6887	209	broadcast_tlb_mm_a15_erratum(vma->vm_mm);
03b505ea RK	210	}
	211
	212	void flush_tlb_kernel_page(unsigned long kaddr)
	213	{
	214	if (tlb_ops_need_broadcast()) {
	215	struct tlb_args ta;
	216	ta.ta_start = kaddr;
	217	on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1);
	218	} else
f0915781	219	__flush_tlb_kernel_page(kaddr);
93dc6887	220	broadcast_tlb_a15_erratum();
03b505ea RK	221	}
	222
	223	void flush_tlb_range(struct vm_area_struct *vma,
	224	unsigned long start, unsigned long end)
	225	{
	226	if (tlb_ops_need_broadcast()) {
	227	struct tlb_args ta;
	228	ta.ta_vma = vma;
	229	ta.ta_start = start;
	230	ta.ta_end = end;
3fc498f1 GBY	231	on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_range,
3fc498f1 GBY	232	&ta, 1);
03b505ea RK	233	} else
03b505ea RK	234	local_flush_tlb_range(vma, start, end);
93dc6887	235	broadcast_tlb_mm_a15_erratum(vma->vm_mm);
03b505ea RK	236	}
	237
	238	void flush_tlb_kernel_range(unsigned long start, unsigned long end)
	239	{
	240	if (tlb_ops_need_broadcast()) {
	241	struct tlb_args ta;
	242	ta.ta_start = start;
	243	ta.ta_end = end;
	244	on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
	245	} else
	246	local_flush_tlb_kernel_range(start, end);
93dc6887	247	broadcast_tlb_a15_erratum();
03b505ea RK	248	}
03b505ea RK	249
862c588f WD	250	void flush_bp_all(void)
	251	{
	252	if (tlb_ops_need_broadcast())
	253	on_each_cpu(ipi_flush_bp_all, NULL, 1);
	254	else
2c813980	255	__flush_bp_all();
862c588f	256	}