[mirror_ubuntu-focal-kernel.git] / arch / mips / mm / cache.c

/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) 1994 - 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org)
 * Copyright (C) 2007 MIPS Technologies, Inc.
 */
#include <linux/fs.h>
#include <linux/fcntl.h>
#include <linux/kernel.h>
#include <linux/linkage.h>
#include <linux/export.h>
#include <linux/sched.h>
#include <linux/syscalls.h>
#include <linux/mm.h>

#include <asm/cacheflush.h>
#include <asm/highmem.h>
#include <asm/processor.h>
#include <asm/cpu.h>
#include <asm/cpu-features.h>
#include <asm/setup.h>

/* Cache operations. */
void (*flush_cache_all)(void);
void (*__flush_cache_all)(void);
EXPORT_SYMBOL_GPL(__flush_cache_all);
void (*flush_cache_mm)(struct mm_struct *mm);
void (*flush_cache_range)(struct vm_area_struct *vma, unsigned long start,
	unsigned long end);
void (*flush_cache_page)(struct vm_area_struct *vma, unsigned long page,
	unsigned long pfn);
void (*flush_icache_range)(unsigned long start, unsigned long end);
EXPORT_SYMBOL_GPL(flush_icache_range);
void (*local_flush_icache_range)(unsigned long start, unsigned long end);
EXPORT_SYMBOL_GPL(local_flush_icache_range);
void (*__flush_icache_user_range)(unsigned long start, unsigned long end);
EXPORT_SYMBOL_GPL(__flush_icache_user_range);
void (*__local_flush_icache_user_range)(unsigned long start, unsigned long end);
EXPORT_SYMBOL_GPL(__local_flush_icache_user_range);

void (*__flush_cache_vmap)(void);
void (*__flush_cache_vunmap)(void);

void (*__flush_kernel_vmap_range)(unsigned long vaddr, int size);
EXPORT_SYMBOL_GPL(__flush_kernel_vmap_range);

/* MIPS specific cache operations */
void (*flush_cache_sigtramp)(unsigned long addr);
void (*local_flush_data_cache_page)(void * addr);
void (*flush_data_cache_page)(unsigned long addr);
void (*flush_icache_all)(void);

EXPORT_SYMBOL_GPL(local_flush_data_cache_page);
EXPORT_SYMBOL(flush_data_cache_page);
EXPORT_SYMBOL(flush_icache_all);

#ifdef CONFIG_DMA_NONCOHERENT

/* DMA cache operations. */
void (*_dma_cache_wback_inv)(unsigned long start, unsigned long size);
void (*_dma_cache_wback)(unsigned long start, unsigned long size);
void (*_dma_cache_inv)(unsigned long start, unsigned long size);

EXPORT_SYMBOL(_dma_cache_wback_inv);

#endif /* CONFIG_DMA_NONCOHERENT */

/*
 * We could optimize the case where the cache argument is not BCACHE but
 * that seems very atypical use ...
 */
SYSCALL_DEFINE3(cacheflush, unsigned long, addr, unsigned long, bytes,
	unsigned int, cache)
{
	if (bytes == 0)
		return 0;
	if (!access_ok((void __user *) addr, bytes))
		return -EFAULT;

	__flush_icache_user_range(addr, addr + bytes);

	return 0;
}

void __flush_dcache_page(struct page *page)
{
	struct address_space *mapping = page_mapping_file(page);
	unsigned long addr;

	if (mapping && !mapping_mapped(mapping)) {
		SetPageDcacheDirty(page);
		return;
	}

	/*
	 * We could delay the flush for the !page_mapping case too.  But that
	 * case is for exec env/arg pages and those are %99 certainly going to
	 * get faulted into the tlb (and thus flushed) anyways.
	 */
	if (PageHighMem(page))
		addr = (unsigned long)kmap_atomic(page);
	else
		addr = (unsigned long)page_address(page);

	flush_data_cache_page(addr);

	if (PageHighMem(page))
		__kunmap_atomic((void *)addr);
}

EXPORT_SYMBOL(__flush_dcache_page);

void __flush_anon_page(struct page *page, unsigned long vmaddr)
{
	unsigned long addr = (unsigned long) page_address(page);

	if (pages_do_alias(addr, vmaddr)) {
		if (page_mapcount(page) && !Page_dcache_dirty(page)) {
			void *kaddr;

			kaddr = kmap_coherent(page, vmaddr);
			flush_data_cache_page((unsigned long)kaddr);
			kunmap_coherent();
		} else
			flush_data_cache_page(addr);
	}
}

EXPORT_SYMBOL(__flush_anon_page);

void __update_cache(unsigned long address, pte_t pte)
{
	struct page *page;
	unsigned long pfn, addr;
	int exec = !pte_no_exec(pte) && !cpu_has_ic_fills_f_dc;

	pfn = pte_pfn(pte);
	if (unlikely(!pfn_valid(pfn)))
		return;
	page = pfn_to_page(pfn);
	if (Page_dcache_dirty(page)) {
		if (PageHighMem(page))
			addr = (unsigned long)kmap_atomic(page);
		else
			addr = (unsigned long)page_address(page);

		if (exec || pages_do_alias(addr, address & PAGE_MASK))
			flush_data_cache_page(addr);

		if (PageHighMem(page))
			__kunmap_atomic((void *)addr);

		ClearPageDcacheDirty(page);
	}
}

unsigned long _page_cachable_default;
EXPORT_SYMBOL(_page_cachable_default);

static inline void setup_protection_map(void)
{
	if (cpu_has_rixi) {
		protection_map[0]  = __pgprot(_page_cachable_default | _PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_NO_READ);
		protection_map[1]  = __pgprot(_page_cachable_default | _PAGE_PRESENT | _PAGE_NO_EXEC);
		protection_map[2]  = __pgprot(_page_cachable_default | _PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_NO_READ);
		protection_map[3]  = __pgprot(_page_cachable_default | _PAGE_PRESENT | _PAGE_NO_EXEC);
		protection_map[4]  = __pgprot(_page_cachable_default | _PAGE_PRESENT);
		protection_map[5]  = __pgprot(_page_cachable_default | _PAGE_PRESENT);
		protection_map[6]  = __pgprot(_page_cachable_default | _PAGE_PRESENT);
		protection_map[7]  = __pgprot(_page_cachable_default | _PAGE_PRESENT);

		protection_map[8]  = __pgprot(_page_cachable_default | _PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_NO_READ);
		protection_map[9]  = __pgprot(_page_cachable_default | _PAGE_PRESENT | _PAGE_NO_EXEC);
		protection_map[10] = __pgprot(_page_cachable_default | _PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE | _PAGE_NO_READ);
		protection_map[11] = __pgprot(_page_cachable_default | _PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE);
		protection_map[12] = __pgprot(_page_cachable_default | _PAGE_PRESENT);
		protection_map[13] = __pgprot(_page_cachable_default | _PAGE_PRESENT);
		protection_map[14] = __pgprot(_page_cachable_default | _PAGE_PRESENT | _PAGE_WRITE);
		protection_map[15] = __pgprot(_page_cachable_default | _PAGE_PRESENT | _PAGE_WRITE);

	} else {
		protection_map[0] = PAGE_NONE;
		protection_map[1] = PAGE_READONLY;
		protection_map[2] = PAGE_COPY;
		protection_map[3] = PAGE_COPY;
		protection_map[4] = PAGE_READONLY;
		protection_map[5] = PAGE_READONLY;
		protection_map[6] = PAGE_COPY;
		protection_map[7] = PAGE_COPY;
		protection_map[8] = PAGE_NONE;
		protection_map[9] = PAGE_READONLY;
		protection_map[10] = PAGE_SHARED;
		protection_map[11] = PAGE_SHARED;
		protection_map[12] = PAGE_READONLY;
		protection_map[13] = PAGE_READONLY;
		protection_map[14] = PAGE_SHARED;
		protection_map[15] = PAGE_SHARED;
	}
}

void cpu_cache_init(void)
{
	if (cpu_has_3k_cache) {
		extern void __weak r3k_cache_init(void);

		r3k_cache_init();
	}
	if (cpu_has_6k_cache) {
		extern void __weak r6k_cache_init(void);

		r6k_cache_init();
	}
	if (cpu_has_4k_cache) {
		extern void __weak r4k_cache_init(void);

		r4k_cache_init();
	}
	if (cpu_has_8k_cache) {
		extern void __weak r8k_cache_init(void);

		r8k_cache_init();
	}
	if (cpu_has_tx39_cache) {
		extern void __weak tx39_cache_init(void);

		tx39_cache_init();
	}

	if (cpu_has_octeon_cache) {
		extern void __weak octeon_cache_init(void);

		octeon_cache_init();
	}

	setup_protection_map();
}

int __weak __uncached_access(struct file *file, unsigned long addr)
{
	if (file->f_flags & O_DSYNC)
		return 1;

	return addr >= __pa(high_memory);
}
Commit	Line	Data
1da177e4 LT	1	/*
	2	* This file is subject to the terms and conditions of the GNU General Public
	3	* License. See the file "COPYING" in the main directory of this archive
	4	* for more details.
	5	*
641e97f3	6	* Copyright (C) 1994 - 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org)
7575a49f	7	* Copyright (C) 2007 MIPS Technologies, Inc.
1da177e4	8	*/
24e9d0b9 RB	9	#include <linux/fs.h>
24e9d0b9 RB	10	#include <linux/fcntl.h>
1da177e4	11	#include <linux/kernel.h>
641e97f3	12	#include <linux/linkage.h>
d9ba5778	13	#include <linux/export.h>
1da177e4	14	#include <linux/sched.h>
dbda6ac0	15	#include <linux/syscalls.h>
1da177e4 LT	16	#include <linux/mm.h>
	17
	18	#include <asm/cacheflush.h>
234859e4	19	#include <asm/highmem.h>
1da177e4 LT	20	#include <asm/processor.h>
	21	#include <asm/cpu.h>
	22	#include <asm/cpu-features.h>
69939424	23	#include <asm/setup.h>
1da177e4 LT	24
	25	/* Cache operations. */
	26	void (*flush_cache_all)(void);
	27	void (*__flush_cache_all)(void);
4fa9de5a	28	EXPORT_SYMBOL_GPL(__flush_cache_all);
1da177e4 LT	29	void (flush_cache_mm)(struct mm_struct mm);
	30	void (flush_cache_range)(struct vm_area_struct vma, unsigned long start,
	31	unsigned long end);
53de0d47 RB	32	void (flush_cache_page)(struct vm_area_struct vma, unsigned long page,
53de0d47 RB	33	unsigned long pfn);
d4264f18	34	void (*flush_icache_range)(unsigned long start, unsigned long end);
8229f1a0	35	EXPORT_SYMBOL_GPL(flush_icache_range);
e0cee3ee	36	void (*local_flush_icache_range)(unsigned long start, unsigned long end);
90f91356	37	EXPORT_SYMBOL_GPL(local_flush_icache_range);
01882b4d JH	38	void (*__flush_icache_user_range)(unsigned long start, unsigned long end);
	39	EXPORT_SYMBOL_GPL(__flush_icache_user_range);
	40	void (*__local_flush_icache_user_range)(unsigned long start, unsigned long end);
	41	EXPORT_SYMBOL_GPL(__local_flush_icache_user_range);
1da177e4	42
9c5a3d72 RB	43	void (*__flush_cache_vmap)(void);
	44	void (*__flush_cache_vunmap)(void);
	45
d9cdc901	46	void (*__flush_kernel_vmap_range)(unsigned long vaddr, int size);
d9cdc901 RB	47	EXPORT_SYMBOL_GPL(__flush_kernel_vmap_range);
d9cdc901 RB	48
1da177e4 LT	49	/* MIPS specific cache operations */
1da177e4 LT	50	void (*flush_cache_sigtramp)(unsigned long addr);
7e3bfc7c	51	void (local_flush_data_cache_page)(void addr);
1da177e4 LT	52	void (*flush_data_cache_page)(unsigned long addr);
	53	void (*flush_icache_all)(void);
	54
9202f325	55	EXPORT_SYMBOL_GPL(local_flush_data_cache_page);
9ff77c46	56	EXPORT_SYMBOL(flush_data_cache_page);
f2e3656d	57	EXPORT_SYMBOL(flush_icache_all);
9ff77c46	58
972dc3b7	59	#ifdef CONFIG_DMA_NONCOHERENT
1da177e4 LT	60
	61	/* DMA cache operations. */
	62	void (*_dma_cache_wback_inv)(unsigned long start, unsigned long size);
	63	void (*_dma_cache_wback)(unsigned long start, unsigned long size);
	64	void (*_dma_cache_inv)(unsigned long start, unsigned long size);
	65
	66	EXPORT_SYMBOL(_dma_cache_wback_inv);
1da177e4	67
972dc3b7	68	#endif /* CONFIG_DMA_NONCOHERENT */
1da177e4 LT	69
	70	/*
	71	* We could optimize the case where the cache argument is not BCACHE but
	72	* that seems very atypical use ...
	73	*/
dbda6ac0 RB	74	SYSCALL_DEFINE3(cacheflush, unsigned long, addr, unsigned long, bytes,
dbda6ac0 RB	75	unsigned int, cache)
1da177e4	76	{
750ccf68 AN	77	if (bytes == 0)
750ccf68 AN	78	return 0;
96d4f267	79	if (!access_ok((void __user *) addr, bytes))
1da177e4 LT	80	return -EFAULT;
1da177e4 LT	81
8e3a9f4c	82	__flush_icache_user_range(addr, addr + bytes);
1da177e4 LT	83
	84	return 0;
	85	}
	86
	87	void __flush_dcache_page(struct page *page)
	88	{
cb9f753a	89	struct address_space *mapping = page_mapping_file(page);
1da177e4 LT	90	unsigned long addr;
	91
	92	if (mapping && !mapping_mapped(mapping)) {
	93	SetPageDcacheDirty(page);
	94	return;
	95	}
	96
	97	/*
	98	* We could delay the flush for the !page_mapping case too. But that
	99	* case is for exec env/arg pages and those are %99 certainly going to
	100	* get faulted into the tlb (and thus flushed) anyways.
	101	*/
234859e4 PB	102	if (PageHighMem(page))
	103	addr = (unsigned long)kmap_atomic(page);
	104	else
	105	addr = (unsigned long)page_address(page);
	106
1da177e4	107	flush_data_cache_page(addr);
234859e4 PB	108
	109	if (PageHighMem(page))
	110	__kunmap_atomic((void *)addr);
1da177e4 LT	111	}
	112
	113	EXPORT_SYMBOL(__flush_dcache_page);
	114
7575a49f RB	115	void __flush_anon_page(struct page *page, unsigned long vmaddr)
7575a49f RB	116	{
9a74b3eb	117	unsigned long addr = (unsigned long) page_address(page);
7575a49f	118
9a74b3eb	119	if (pages_do_alias(addr, vmaddr)) {
e1534ae9	120	if (page_mapcount(page) && !Page_dcache_dirty(page)) {
9a74b3eb RB	121	void *kaddr;
	122
	123	kaddr = kmap_coherent(page, vmaddr);
	124	flush_data_cache_page((unsigned long)kaddr);
	125	kunmap_coherent();
	126	} else
	127	flush_data_cache_page(addr);
7575a49f RB	128	}
	129	}
	130
	131	EXPORT_SYMBOL(__flush_anon_page);
	132
37d22a0d	133	void __update_cache(unsigned long address, pte_t pte)
1da177e4 LT	134	{
1da177e4 LT	135	struct page *page;
5b9593f3	136	unsigned long pfn, addr;
37d22a0d	137	int exec = !pte_no_exec(pte) && !cpu_has_ic_fills_f_dc;
1da177e4	138
5b9593f3	139	pfn = pte_pfn(pte);
585fa724 RB	140	if (unlikely(!pfn_valid(pfn)))
	141	return;
	142	page = pfn_to_page(pfn);
37d22a0d	143	if (Page_dcache_dirty(page)) {
f4281bba PB	144	if (PageHighMem(page))
	145	addr = (unsigned long)kmap_atomic(page);
	146	else
	147	addr = (unsigned long)page_address(page);
	148
5b9593f3 LP	149	if (exec \|\| pages_do_alias(addr, address & PAGE_MASK))
5b9593f3 LP	150	flush_data_cache_page(addr);
f4281bba PB	151
	152	if (PageHighMem(page))
	153	__kunmap_atomic((void *)addr);
	154
1da177e4 LT	155	ClearPageDcacheDirty(page);
	156	}
	157	}
	158
35133692	159	unsigned long _page_cachable_default;
7b3e543d	160	EXPORT_SYMBOL(_page_cachable_default);
35133692 CD	161
	162	static inline void setup_protection_map(void)
	163	{
05857c64	164	if (cpu_has_rixi) {
6dd9344c DD	165	protection_map[0] = __pgprot(_page_cachable_default \| _PAGE_PRESENT \| _PAGE_NO_EXEC \| _PAGE_NO_READ);
	166	protection_map[1] = __pgprot(_page_cachable_default \| _PAGE_PRESENT \| _PAGE_NO_EXEC);
	167	protection_map[2] = __pgprot(_page_cachable_default \| _PAGE_PRESENT \| _PAGE_NO_EXEC \| _PAGE_NO_READ);
	168	protection_map[3] = __pgprot(_page_cachable_default \| _PAGE_PRESENT \| _PAGE_NO_EXEC);
55fdcb2d	169	protection_map[4] = __pgprot(_page_cachable_default \| _PAGE_PRESENT);
6dd9344c	170	protection_map[5] = __pgprot(_page_cachable_default \| _PAGE_PRESENT);
55fdcb2d	171	protection_map[6] = __pgprot(_page_cachable_default \| _PAGE_PRESENT);
6dd9344c DD	172	protection_map[7] = __pgprot(_page_cachable_default \| _PAGE_PRESENT);
	173
	174	protection_map[8] = __pgprot(_page_cachable_default \| _PAGE_PRESENT \| _PAGE_NO_EXEC \| _PAGE_NO_READ);
	175	protection_map[9] = __pgprot(_page_cachable_default \| _PAGE_PRESENT \| _PAGE_NO_EXEC);
	176	protection_map[10] = __pgprot(_page_cachable_default \| _PAGE_PRESENT \| _PAGE_NO_EXEC \| _PAGE_WRITE \| _PAGE_NO_READ);
	177	protection_map[11] = __pgprot(_page_cachable_default \| _PAGE_PRESENT \| _PAGE_NO_EXEC \| _PAGE_WRITE);
55fdcb2d	178	protection_map[12] = __pgprot(_page_cachable_default \| _PAGE_PRESENT);
6dd9344c	179	protection_map[13] = __pgprot(_page_cachable_default \| _PAGE_PRESENT);
55fdcb2d	180	protection_map[14] = __pgprot(_page_cachable_default \| _PAGE_PRESENT \| _PAGE_WRITE);
6dd9344c DD	181	protection_map[15] = __pgprot(_page_cachable_default \| _PAGE_PRESENT \| _PAGE_WRITE);
	182
	183	} else {
	184	protection_map[0] = PAGE_NONE;
	185	protection_map[1] = PAGE_READONLY;
	186	protection_map[2] = PAGE_COPY;
	187	protection_map[3] = PAGE_COPY;
	188	protection_map[4] = PAGE_READONLY;
	189	protection_map[5] = PAGE_READONLY;
	190	protection_map[6] = PAGE_COPY;
	191	protection_map[7] = PAGE_COPY;
	192	protection_map[8] = PAGE_NONE;
	193	protection_map[9] = PAGE_READONLY;
	194	protection_map[10] = PAGE_SHARED;
	195	protection_map[11] = PAGE_SHARED;
	196	protection_map[12] = PAGE_READONLY;
	197	protection_map[13] = PAGE_READONLY;
	198	protection_map[14] = PAGE_SHARED;
	199	protection_map[15] = PAGE_SHARED;
	200	}
35133692	201	}
1da177e4	202
078a55fc	203	void cpu_cache_init(void)
1da177e4	204	{
02cf2119 RB	205	if (cpu_has_3k_cache) {
	206	extern void __weak r3k_cache_init(void);
	207
	208	r3k_cache_init();
02cf2119 RB	209	}
	210	if (cpu_has_6k_cache) {
	211	extern void __weak r6k_cache_init(void);
	212
	213	r6k_cache_init();
1da177e4	214	}
02cf2119 RB	215	if (cpu_has_4k_cache) {
	216	extern void __weak r4k_cache_init(void);
	217
	218	r4k_cache_init();
02cf2119 RB	219	}
	220	if (cpu_has_8k_cache) {
	221	extern void __weak r8k_cache_init(void);
	222
	223	r8k_cache_init();
02cf2119 RB	224	}
	225	if (cpu_has_tx39_cache) {
	226	extern void __weak tx39_cache_init(void);
	227
	228	tx39_cache_init();
02cf2119	229	}
02cf2119	230
47d979ec DD	231	if (cpu_has_octeon_cache) {
	232	extern void __weak octeon_cache_init(void);
	233
	234	octeon_cache_init();
	235	}
	236
35133692	237	setup_protection_map();
1da177e4	238	}
24e9d0b9 RB	239
	240	int __weak __uncached_access(struct file *file, unsigned long addr)
	241	{
6b2f3d1f	242	if (file->f_flags & O_DSYNC)
24e9d0b9 RB	243	return 1;
	244
	245	return addr >= __pa(high_memory);
	246	}