]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * arch/sh/mm/cache-sh4.c | |
3 | * | |
4 | * Copyright (C) 1999, 2000, 2002 Niibe Yutaka | |
deaef20e | 5 | * Copyright (C) 2001 - 2009 Paul Mundt |
1da177e4 | 6 | * Copyright (C) 2003 Richard Curnow |
09b5a10c | 7 | * Copyright (c) 2007 STMicroelectronics (R&D) Ltd. |
1da177e4 LT |
8 | * |
9 | * This file is subject to the terms and conditions of the GNU General Public | |
10 | * License. See the file "COPYING" in the main directory of this archive | |
11 | * for more details. | |
12 | */ | |
1da177e4 | 13 | #include <linux/init.h> |
1da177e4 | 14 | #include <linux/mm.h> |
52e27782 PM |
15 | #include <linux/io.h> |
16 | #include <linux/mutex.h> | |
2277ab4a | 17 | #include <linux/fs.h> |
deaef20e | 18 | #include <linux/highmem.h> |
1da177e4 | 19 | #include <asm/mmu_context.h> |
f03c4866 | 20 | #include <asm/cache_insns.h> |
1da177e4 LT |
21 | #include <asm/cacheflush.h> |
22 | ||
28ccf7f9 PM |
23 | /* |
24 | * The maximum number of pages we support up to when doing ranged dcache | |
25 | * flushing. Anything exceeding this will simply flush the dcache in its | |
26 | * entirety. | |
27 | */ | |
09b5a10c | 28 | #define MAX_ICACHE_PAGES 32 |
28ccf7f9 | 29 | |
a7a7c0e1 | 30 | static void __flush_cache_one(unsigned long addr, unsigned long phys, |
a252710f | 31 | unsigned long exec_offset); |
b638d0b9 | 32 | |
1da177e4 LT |
33 | /* |
34 | * Write back the range of D-cache, and purge the I-cache. | |
35 | * | |
09b5a10c CS |
36 | * Called from kernel/module.c:sys_init_module and routine for a.out format, |
37 | * signal handler code and kprobes code | |
1da177e4 | 38 | */ |
2dc2f8e0 | 39 | static void sh4_flush_icache_range(void *args) |
1da177e4 | 40 | { |
f26b2a56 | 41 | struct flusher_data *data = args; |
f26b2a56 | 42 | unsigned long start, end; |
983f4c51 | 43 | unsigned long flags, v; |
1da177e4 LT |
44 | int i; |
45 | ||
f26b2a56 PM |
46 | start = data->addr1; |
47 | end = data->addr2; | |
48 | ||
682f88ab PM |
49 | /* If there are too many pages then just blow away the caches */ |
50 | if (((end - start) >> PAGE_SHIFT) >= MAX_ICACHE_PAGES) { | |
51 | local_flush_cache_all(NULL); | |
52 | return; | |
53 | } | |
54 | ||
55 | /* | |
56 | * Selectively flush d-cache then invalidate the i-cache. | |
57 | * This is inefficient, so only use this for small ranges. | |
58 | */ | |
59 | start &= ~(L1_CACHE_BYTES-1); | |
60 | end += L1_CACHE_BYTES-1; | |
61 | end &= ~(L1_CACHE_BYTES-1); | |
983f4c51 | 62 | |
682f88ab PM |
63 | local_irq_save(flags); |
64 | jump_to_uncached(); | |
983f4c51 | 65 | |
682f88ab PM |
66 | for (v = start; v < end; v += L1_CACHE_BYTES) { |
67 | unsigned long icacheaddr; | |
a9d244a2 | 68 | int j, n; |
983f4c51 | 69 | |
682f88ab | 70 | __ocbwb(v); |
983f4c51 | 71 | |
682f88ab PM |
72 | icacheaddr = CACHE_IC_ADDRESS_ARRAY | (v & |
73 | cpu_data->icache.entry_mask); | |
09b5a10c | 74 | |
682f88ab | 75 | /* Clear i-cache line valid-bit */ |
a9d244a2 | 76 | n = boot_cpu_data.icache.n_aliases; |
682f88ab | 77 | for (i = 0; i < cpu_data->icache.ways; i++) { |
a9d244a2 MF |
78 | for (j = 0; j < n; j++) |
79 | __raw_writel(0, icacheaddr + (j * PAGE_SIZE)); | |
682f88ab PM |
80 | icacheaddr += cpu_data->icache.way_incr; |
81 | } | |
09b5a10c | 82 | } |
682f88ab PM |
83 | |
84 | back_to_cached(); | |
85 | local_irq_restore(flags); | |
1da177e4 LT |
86 | } |
87 | ||
a7a7c0e1 | 88 | static inline void flush_cache_one(unsigned long start, unsigned long phys) |
1da177e4 | 89 | { |
983f4c51 | 90 | unsigned long flags, exec_offset = 0; |
33573c0e | 91 | |
1da177e4 | 92 | /* |
1f69b6af MF |
93 | * All types of SH-4 require PC to be uncached to operate on the I-cache. |
94 | * Some types of SH-4 require PC to be uncached to operate on the D-cache. | |
1da177e4 | 95 | */ |
7ec9d6f8 | 96 | if ((boot_cpu_data.flags & CPU_HAS_P2_FLUSH_BUG) || |
33573c0e | 97 | (start < CACHE_OC_ADDRESS_ARRAY)) |
1f69b6af | 98 | exec_offset = cached_to_uncached; |
33573c0e | 99 | |
983f4c51 | 100 | local_irq_save(flags); |
a781d1e5 | 101 | __flush_cache_one(start, phys, exec_offset); |
983f4c51 | 102 | local_irq_restore(flags); |
1da177e4 LT |
103 | } |
104 | ||
105 | /* | |
106 | * Write back & invalidate the D-cache of the page. | |
107 | * (To avoid "alias" issues) | |
108 | */ | |
e76a0136 | 109 | static void sh4_flush_dcache_page(void *arg) |
1da177e4 | 110 | { |
e76a0136 | 111 | struct page *page = arg; |
b4c89276 | 112 | unsigned long addr = (unsigned long)page_address(page); |
c139a595 | 113 | #ifndef CONFIG_SMP |
cb9f753a | 114 | struct address_space *mapping = page_mapping_file(page); |
2277ab4a | 115 | |
2277ab4a | 116 | if (mapping && !mapping_mapped(mapping)) |
55661fc1 | 117 | clear_bit(PG_dcache_clean, &page->flags); |
2277ab4a PM |
118 | else |
119 | #endif | |
b4c89276 MF |
120 | flush_cache_one(CACHE_OC_ADDRESS_ARRAY | |
121 | (addr & shm_align_mask), page_to_phys(page)); | |
fdfc74f9 PM |
122 | |
123 | wmb(); | |
1da177e4 LT |
124 | } |
125 | ||
28ccf7f9 | 126 | /* TODO: Selective icache invalidation through IC address array.. */ |
2dc2f8e0 | 127 | static void flush_icache_all(void) |
1da177e4 | 128 | { |
983f4c51 | 129 | unsigned long flags, ccr; |
1da177e4 | 130 | |
983f4c51 | 131 | local_irq_save(flags); |
cbaa118e | 132 | jump_to_uncached(); |
1da177e4 LT |
133 | |
134 | /* Flush I-cache */ | |
a5f6ea29 | 135 | ccr = __raw_readl(SH_CCR); |
1da177e4 | 136 | ccr |= CCR_CACHE_ICI; |
a5f6ea29 | 137 | __raw_writel(ccr, SH_CCR); |
1da177e4 | 138 | |
29847622 | 139 | /* |
cbaa118e | 140 | * back_to_cached() will take care of the barrier for us, don't add |
29847622 PM |
141 | * another one! |
142 | */ | |
983f4c51 | 143 | |
cbaa118e | 144 | back_to_cached(); |
983f4c51 | 145 | local_irq_restore(flags); |
1da177e4 LT |
146 | } |
147 | ||
bd6df574 | 148 | static void flush_dcache_all(void) |
1da177e4 | 149 | { |
bd6df574 PM |
150 | unsigned long addr, end_addr, entry_offset; |
151 | ||
152 | end_addr = CACHE_OC_ADDRESS_ARRAY + | |
153 | (current_cpu_data.dcache.sets << | |
154 | current_cpu_data.dcache.entry_shift) * | |
155 | current_cpu_data.dcache.ways; | |
156 | ||
157 | entry_offset = 1 << current_cpu_data.dcache.entry_shift; | |
158 | ||
159 | for (addr = CACHE_OC_ADDRESS_ARRAY; addr < end_addr; ) { | |
160 | __raw_writel(0, addr); addr += entry_offset; | |
161 | __raw_writel(0, addr); addr += entry_offset; | |
162 | __raw_writel(0, addr); addr += entry_offset; | |
163 | __raw_writel(0, addr); addr += entry_offset; | |
164 | __raw_writel(0, addr); addr += entry_offset; | |
165 | __raw_writel(0, addr); addr += entry_offset; | |
166 | __raw_writel(0, addr); addr += entry_offset; | |
167 | __raw_writel(0, addr); addr += entry_offset; | |
168 | } | |
a252710f PM |
169 | } |
170 | ||
f26b2a56 | 171 | static void sh4_flush_cache_all(void *unused) |
a252710f PM |
172 | { |
173 | flush_dcache_all(); | |
1da177e4 LT |
174 | flush_icache_all(); |
175 | } | |
176 | ||
28ccf7f9 PM |
177 | /* |
178 | * Note : (RPC) since the caches are physically tagged, the only point | |
179 | * of flush_cache_mm for SH-4 is to get rid of aliases from the | |
180 | * D-cache. The assumption elsewhere, e.g. flush_cache_range, is that | |
181 | * lines can stay resident so long as the virtual address they were | |
182 | * accessed with (hence cache set) is in accord with the physical | |
654d364e | 183 | * address (i.e. tag). It's no different here. |
28ccf7f9 | 184 | * |
c1e8d7c6 | 185 | * Caller takes mm->mmap_lock. |
28ccf7f9 | 186 | */ |
f26b2a56 | 187 | static void sh4_flush_cache_mm(void *arg) |
1da177e4 | 188 | { |
f26b2a56 PM |
189 | struct mm_struct *mm = arg; |
190 | ||
e7b8b7f1 PM |
191 | if (cpu_context(smp_processor_id(), mm) == NO_CONTEXT) |
192 | return; | |
193 | ||
654d364e | 194 | flush_dcache_all(); |
1da177e4 LT |
195 | } |
196 | ||
197 | /* | |
198 | * Write back and invalidate I/D-caches for the page. | |
199 | * | |
200 | * ADDR: Virtual Address (U0 address) | |
201 | * PFN: Physical page number | |
202 | */ | |
f26b2a56 | 203 | static void sh4_flush_cache_page(void *args) |
1da177e4 | 204 | { |
f26b2a56 PM |
205 | struct flusher_data *data = args; |
206 | struct vm_area_struct *vma; | |
deaef20e | 207 | struct page *page; |
f26b2a56 | 208 | unsigned long address, pfn, phys; |
deaef20e | 209 | int map_coherent = 0; |
deaef20e PM |
210 | pmd_t *pmd; |
211 | pte_t *pte; | |
212 | void *vaddr; | |
b638d0b9 | 213 | |
f26b2a56 | 214 | vma = data->vma; |
abeaf33a | 215 | address = data->addr1 & PAGE_MASK; |
f26b2a56 PM |
216 | pfn = data->addr2; |
217 | phys = pfn << PAGE_SHIFT; | |
deaef20e | 218 | page = pfn_to_page(pfn); |
f26b2a56 | 219 | |
e7b8b7f1 PM |
220 | if (cpu_context(smp_processor_id(), vma->vm_mm) == NO_CONTEXT) |
221 | return; | |
222 | ||
e05c7b1f | 223 | pmd = pmd_off(vma->vm_mm, address); |
deaef20e PM |
224 | pte = pte_offset_kernel(pmd, address); |
225 | ||
226 | /* If the page isn't present, there is nothing to do here. */ | |
227 | if (!(pte_val(*pte) & _PAGE_PRESENT)) | |
228 | return; | |
1da177e4 | 229 | |
deaef20e PM |
230 | if ((vma->vm_mm == current->active_mm)) |
231 | vaddr = NULL; | |
232 | else { | |
b638d0b9 | 233 | /* |
deaef20e PM |
234 | * Use kmap_coherent or kmap_atomic to do flushes for |
235 | * another ASID than the current one. | |
b638d0b9 | 236 | */ |
deaef20e | 237 | map_coherent = (current_cpu_data.dcache.n_aliases && |
55661fc1 | 238 | test_bit(PG_dcache_clean, &page->flags) && |
e1534ae9 | 239 | page_mapcount(page)); |
deaef20e PM |
240 | if (map_coherent) |
241 | vaddr = kmap_coherent(page, address); | |
242 | else | |
bc3e11be | 243 | vaddr = kmap_atomic(page); |
deaef20e PM |
244 | |
245 | address = (unsigned long)vaddr; | |
246 | } | |
247 | ||
e717cc6c | 248 | flush_cache_one(CACHE_OC_ADDRESS_ARRAY | |
deaef20e PM |
249 | (address & shm_align_mask), phys); |
250 | ||
251 | if (vma->vm_flags & VM_EXEC) | |
252 | flush_icache_all(); | |
253 | ||
254 | if (vaddr) { | |
255 | if (map_coherent) | |
256 | kunmap_coherent(vaddr); | |
257 | else | |
bc3e11be | 258 | kunmap_atomic(vaddr); |
b638d0b9 | 259 | } |
1da177e4 LT |
260 | } |
261 | ||
262 | /* | |
263 | * Write back and invalidate D-caches. | |
264 | * | |
265 | * START, END: Virtual Address (U0 address) | |
266 | * | |
267 | * NOTE: We need to flush the _physical_ page entry. | |
268 | * Flushing the cache lines for U0 only isn't enough. | |
269 | * We need to flush for P1 too, which may contain aliases. | |
270 | */ | |
f26b2a56 | 271 | static void sh4_flush_cache_range(void *args) |
1da177e4 | 272 | { |
f26b2a56 PM |
273 | struct flusher_data *data = args; |
274 | struct vm_area_struct *vma; | |
275 | unsigned long start, end; | |
276 | ||
277 | vma = data->vma; | |
278 | start = data->addr1; | |
279 | end = data->addr2; | |
280 | ||
e7b8b7f1 PM |
281 | if (cpu_context(smp_processor_id(), vma->vm_mm) == NO_CONTEXT) |
282 | return; | |
283 | ||
b638d0b9 RC |
284 | /* |
285 | * If cache is only 4k-per-way, there are never any 'aliases'. Since | |
286 | * the cache is physically tagged, the data can just be left in there. | |
287 | */ | |
7ec9d6f8 | 288 | if (boot_cpu_data.dcache.n_aliases == 0) |
b638d0b9 RC |
289 | return; |
290 | ||
654d364e | 291 | flush_dcache_all(); |
b638d0b9 | 292 | |
654d364e | 293 | if (vma->vm_flags & VM_EXEC) |
1da177e4 LT |
294 | flush_icache_all(); |
295 | } | |
296 | ||
b638d0b9 | 297 | /** |
a7a7c0e1 | 298 | * __flush_cache_one |
b638d0b9 RC |
299 | * |
300 | * @addr: address in memory mapped cache array | |
301 | * @phys: P1 address to flush (has to match tags if addr has 'A' bit | |
302 | * set i.e. associative write) | |
303 | * @exec_offset: set to 0x20000000 if flush has to be executed from P2 | |
304 | * region else 0x0 | |
305 | * | |
306 | * The offset into the cache array implied by 'addr' selects the | |
307 | * 'colour' of the virtual address range that will be flushed. The | |
308 | * operation (purge/write-back) is selected by the lower 2 bits of | |
309 | * 'phys'. | |
310 | */ | |
a7a7c0e1 | 311 | static void __flush_cache_one(unsigned long addr, unsigned long phys, |
b638d0b9 RC |
312 | unsigned long exec_offset) |
313 | { | |
314 | int way_count; | |
315 | unsigned long base_addr = addr; | |
316 | struct cache_info *dcache; | |
317 | unsigned long way_incr; | |
318 | unsigned long a, ea, p; | |
319 | unsigned long temp_pc; | |
320 | ||
7ec9d6f8 | 321 | dcache = &boot_cpu_data.dcache; |
b638d0b9 RC |
322 | /* Write this way for better assembly. */ |
323 | way_count = dcache->ways; | |
324 | way_incr = dcache->way_incr; | |
325 | ||
326 | /* | |
327 | * Apply exec_offset (i.e. branch to P2 if required.). | |
328 | * | |
329 | * FIXME: | |
330 | * | |
331 | * If I write "=r" for the (temp_pc), it puts this in r6 hence | |
332 | * trashing exec_offset before it's been added on - why? Hence | |
333 | * "=&r" as a 'workaround' | |
334 | */ | |
335 | asm volatile("mov.l 1f, %0\n\t" | |
336 | "add %1, %0\n\t" | |
337 | "jmp @%0\n\t" | |
338 | "nop\n\t" | |
339 | ".balign 4\n\t" | |
340 | "1: .long 2f\n\t" | |
341 | "2:\n" : "=&r" (temp_pc) : "r" (exec_offset)); | |
342 | ||
343 | /* | |
344 | * We know there will be >=1 iteration, so write as do-while to avoid | |
345 | * pointless nead-of-loop check for 0 iterations. | |
346 | */ | |
347 | do { | |
348 | ea = base_addr + PAGE_SIZE; | |
349 | a = base_addr; | |
350 | p = phys; | |
351 | ||
352 | do { | |
353 | *(volatile unsigned long *)a = p; | |
354 | /* | |
355 | * Next line: intentionally not p+32, saves an add, p | |
356 | * will do since only the cache tag bits need to | |
357 | * match. | |
358 | */ | |
359 | *(volatile unsigned long *)(a+32) = p; | |
360 | a += 64; | |
361 | p += 64; | |
362 | } while (a < ea); | |
363 | ||
364 | base_addr += way_incr; | |
365 | } while (--way_count != 0); | |
366 | } | |
367 | ||
37443ef3 PM |
368 | extern void __weak sh4__flush_region_init(void); |
369 | ||
370 | /* | |
371 | * SH-4 has virtually indexed and physically tagged cache. | |
372 | */ | |
373 | void __init sh4_cache_init(void) | |
374 | { | |
375 | printk("PVR=%08x CVR=%08x PRR=%08x\n", | |
9d56dd3b PM |
376 | __raw_readl(CCN_PVR), |
377 | __raw_readl(CCN_CVR), | |
378 | __raw_readl(CCN_PRR)); | |
37443ef3 | 379 | |
f26b2a56 PM |
380 | local_flush_icache_range = sh4_flush_icache_range; |
381 | local_flush_dcache_page = sh4_flush_dcache_page; | |
382 | local_flush_cache_all = sh4_flush_cache_all; | |
383 | local_flush_cache_mm = sh4_flush_cache_mm; | |
384 | local_flush_cache_dup_mm = sh4_flush_cache_mm; | |
385 | local_flush_cache_page = sh4_flush_cache_page; | |
386 | local_flush_cache_range = sh4_flush_cache_range; | |
37443ef3 PM |
387 | |
388 | sh4__flush_region_init(); | |
389 | } |