]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * arch/sh/mm/cache-sh4.c | |
3 | * | |
4 | * Copyright (C) 1999, 2000, 2002 Niibe Yutaka | |
deaef20e | 5 | * Copyright (C) 2001 - 2009 Paul Mundt |
1da177e4 | 6 | * Copyright (C) 2003 Richard Curnow |
09b5a10c | 7 | * Copyright (c) 2007 STMicroelectronics (R&D) Ltd. |
1da177e4 LT |
8 | * |
9 | * This file is subject to the terms and conditions of the GNU General Public | |
10 | * License. See the file "COPYING" in the main directory of this archive | |
11 | * for more details. | |
12 | */ | |
1da177e4 | 13 | #include <linux/init.h> |
1da177e4 | 14 | #include <linux/mm.h> |
52e27782 PM |
15 | #include <linux/io.h> |
16 | #include <linux/mutex.h> | |
2277ab4a | 17 | #include <linux/fs.h> |
deaef20e PM |
18 | #include <linux/highmem.h> |
19 | #include <asm/pgtable.h> | |
1da177e4 LT |
20 | #include <asm/mmu_context.h> |
21 | #include <asm/cacheflush.h> | |
22 | ||
28ccf7f9 PM |
23 | /* |
24 | * The maximum number of pages we support up to when doing ranged dcache | |
25 | * flushing. Anything exceeding this will simply flush the dcache in its | |
26 | * entirety. | |
27 | */ | |
09b5a10c | 28 | #define MAX_ICACHE_PAGES 32 |
28ccf7f9 | 29 | |
a7a7c0e1 | 30 | static void __flush_cache_one(unsigned long addr, unsigned long phys, |
a252710f | 31 | unsigned long exec_offset); |
b638d0b9 | 32 | |
1da177e4 LT |
33 | /* |
34 | * Write back the range of D-cache, and purge the I-cache. | |
35 | * | |
09b5a10c CS |
36 | * Called from kernel/module.c:sys_init_module and routine for a.out format, |
37 | * signal handler code and kprobes code | |
1da177e4 | 38 | */ |
2dc2f8e0 | 39 | static void sh4_flush_icache_range(void *args) |
1da177e4 | 40 | { |
f26b2a56 | 41 | struct flusher_data *data = args; |
f26b2a56 | 42 | unsigned long start, end; |
983f4c51 | 43 | unsigned long flags, v; |
1da177e4 LT |
44 | int i; |
45 | ||
f26b2a56 PM |
46 | start = data->addr1; |
47 | end = data->addr2; | |
48 | ||
682f88ab PM |
49 | /* If there are too many pages then just blow away the caches */ |
50 | if (((end - start) >> PAGE_SHIFT) >= MAX_ICACHE_PAGES) { | |
51 | local_flush_cache_all(NULL); | |
52 | return; | |
53 | } | |
54 | ||
55 | /* | |
56 | * Selectively flush d-cache then invalidate the i-cache. | |
57 | * This is inefficient, so only use this for small ranges. | |
58 | */ | |
59 | start &= ~(L1_CACHE_BYTES-1); | |
60 | end += L1_CACHE_BYTES-1; | |
61 | end &= ~(L1_CACHE_BYTES-1); | |
983f4c51 | 62 | |
682f88ab PM |
63 | local_irq_save(flags); |
64 | jump_to_uncached(); | |
983f4c51 | 65 | |
682f88ab PM |
66 | for (v = start; v < end; v += L1_CACHE_BYTES) { |
67 | unsigned long icacheaddr; | |
a9d244a2 | 68 | int j, n; |
983f4c51 | 69 | |
682f88ab | 70 | __ocbwb(v); |
983f4c51 | 71 | |
682f88ab PM |
72 | icacheaddr = CACHE_IC_ADDRESS_ARRAY | (v & |
73 | cpu_data->icache.entry_mask); | |
09b5a10c | 74 | |
682f88ab | 75 | /* Clear i-cache line valid-bit */ |
a9d244a2 | 76 | n = boot_cpu_data.icache.n_aliases; |
682f88ab | 77 | for (i = 0; i < cpu_data->icache.ways; i++) { |
a9d244a2 MF |
78 | for (j = 0; j < n; j++) |
79 | __raw_writel(0, icacheaddr + (j * PAGE_SIZE)); | |
682f88ab PM |
80 | icacheaddr += cpu_data->icache.way_incr; |
81 | } | |
09b5a10c | 82 | } |
682f88ab PM |
83 | |
84 | back_to_cached(); | |
85 | local_irq_restore(flags); | |
1da177e4 LT |
86 | } |
87 | ||
a7a7c0e1 | 88 | static inline void flush_cache_one(unsigned long start, unsigned long phys) |
1da177e4 | 89 | { |
983f4c51 | 90 | unsigned long flags, exec_offset = 0; |
33573c0e | 91 | |
1da177e4 | 92 | /* |
1f69b6af MF |
93 | * All types of SH-4 require PC to be uncached to operate on the I-cache. |
94 | * Some types of SH-4 require PC to be uncached to operate on the D-cache. | |
1da177e4 | 95 | */ |
7ec9d6f8 | 96 | if ((boot_cpu_data.flags & CPU_HAS_P2_FLUSH_BUG) || |
33573c0e | 97 | (start < CACHE_OC_ADDRESS_ARRAY)) |
1f69b6af | 98 | exec_offset = cached_to_uncached; |
33573c0e | 99 | |
983f4c51 | 100 | local_irq_save(flags); |
a781d1e5 | 101 | __flush_cache_one(start, phys, exec_offset); |
983f4c51 | 102 | local_irq_restore(flags); |
1da177e4 LT |
103 | } |
104 | ||
105 | /* | |
106 | * Write back & invalidate the D-cache of the page. | |
107 | * (To avoid "alias" issues) | |
108 | */ | |
e76a0136 | 109 | static void sh4_flush_dcache_page(void *arg) |
1da177e4 | 110 | { |
e76a0136 | 111 | struct page *page = arg; |
b4c89276 | 112 | unsigned long addr = (unsigned long)page_address(page); |
c139a595 | 113 | #ifndef CONFIG_SMP |
2277ab4a PM |
114 | struct address_space *mapping = page_mapping(page); |
115 | ||
2277ab4a | 116 | if (mapping && !mapping_mapped(mapping)) |
55661fc1 | 117 | clear_bit(PG_dcache_clean, &page->flags); |
2277ab4a PM |
118 | else |
119 | #endif | |
b4c89276 MF |
120 | flush_cache_one(CACHE_OC_ADDRESS_ARRAY | |
121 | (addr & shm_align_mask), page_to_phys(page)); | |
fdfc74f9 PM |
122 | |
123 | wmb(); | |
1da177e4 LT |
124 | } |
125 | ||
28ccf7f9 | 126 | /* TODO: Selective icache invalidation through IC address array.. */ |
2dc2f8e0 | 127 | static void flush_icache_all(void) |
1da177e4 | 128 | { |
983f4c51 | 129 | unsigned long flags, ccr; |
1da177e4 | 130 | |
983f4c51 | 131 | local_irq_save(flags); |
cbaa118e | 132 | jump_to_uncached(); |
1da177e4 LT |
133 | |
134 | /* Flush I-cache */ | |
9d56dd3b | 135 | ccr = __raw_readl(CCR); |
1da177e4 | 136 | ccr |= CCR_CACHE_ICI; |
9d56dd3b | 137 | __raw_writel(ccr, CCR); |
1da177e4 | 138 | |
29847622 | 139 | /* |
cbaa118e | 140 | * back_to_cached() will take care of the barrier for us, don't add |
29847622 PM |
141 | * another one! |
142 | */ | |
983f4c51 | 143 | |
cbaa118e | 144 | back_to_cached(); |
983f4c51 | 145 | local_irq_restore(flags); |
1da177e4 LT |
146 | } |
147 | ||
bd6df574 | 148 | static void flush_dcache_all(void) |
1da177e4 | 149 | { |
bd6df574 PM |
150 | unsigned long addr, end_addr, entry_offset; |
151 | ||
152 | end_addr = CACHE_OC_ADDRESS_ARRAY + | |
153 | (current_cpu_data.dcache.sets << | |
154 | current_cpu_data.dcache.entry_shift) * | |
155 | current_cpu_data.dcache.ways; | |
156 | ||
157 | entry_offset = 1 << current_cpu_data.dcache.entry_shift; | |
158 | ||
159 | for (addr = CACHE_OC_ADDRESS_ARRAY; addr < end_addr; ) { | |
160 | __raw_writel(0, addr); addr += entry_offset; | |
161 | __raw_writel(0, addr); addr += entry_offset; | |
162 | __raw_writel(0, addr); addr += entry_offset; | |
163 | __raw_writel(0, addr); addr += entry_offset; | |
164 | __raw_writel(0, addr); addr += entry_offset; | |
165 | __raw_writel(0, addr); addr += entry_offset; | |
166 | __raw_writel(0, addr); addr += entry_offset; | |
167 | __raw_writel(0, addr); addr += entry_offset; | |
168 | } | |
a252710f PM |
169 | } |
170 | ||
f26b2a56 | 171 | static void sh4_flush_cache_all(void *unused) |
a252710f PM |
172 | { |
173 | flush_dcache_all(); | |
1da177e4 LT |
174 | flush_icache_all(); |
175 | } | |
176 | ||
28ccf7f9 PM |
177 | /* |
178 | * Note : (RPC) since the caches are physically tagged, the only point | |
179 | * of flush_cache_mm for SH-4 is to get rid of aliases from the | |
180 | * D-cache. The assumption elsewhere, e.g. flush_cache_range, is that | |
181 | * lines can stay resident so long as the virtual address they were | |
182 | * accessed with (hence cache set) is in accord with the physical | |
654d364e | 183 | * address (i.e. tag). It's no different here. |
28ccf7f9 PM |
184 | * |
185 | * Caller takes mm->mmap_sem. | |
186 | */ | |
f26b2a56 | 187 | static void sh4_flush_cache_mm(void *arg) |
1da177e4 | 188 | { |
f26b2a56 PM |
189 | struct mm_struct *mm = arg; |
190 | ||
e7b8b7f1 PM |
191 | if (cpu_context(smp_processor_id(), mm) == NO_CONTEXT) |
192 | return; | |
193 | ||
654d364e | 194 | flush_dcache_all(); |
1da177e4 LT |
195 | } |
196 | ||
197 | /* | |
198 | * Write back and invalidate I/D-caches for the page. | |
199 | * | |
200 | * ADDR: Virtual Address (U0 address) | |
201 | * PFN: Physical page number | |
202 | */ | |
f26b2a56 | 203 | static void sh4_flush_cache_page(void *args) |
1da177e4 | 204 | { |
f26b2a56 PM |
205 | struct flusher_data *data = args; |
206 | struct vm_area_struct *vma; | |
deaef20e | 207 | struct page *page; |
f26b2a56 | 208 | unsigned long address, pfn, phys; |
deaef20e PM |
209 | int map_coherent = 0; |
210 | pgd_t *pgd; | |
211 | pud_t *pud; | |
212 | pmd_t *pmd; | |
213 | pte_t *pte; | |
214 | void *vaddr; | |
b638d0b9 | 215 | |
f26b2a56 | 216 | vma = data->vma; |
abeaf33a | 217 | address = data->addr1 & PAGE_MASK; |
f26b2a56 PM |
218 | pfn = data->addr2; |
219 | phys = pfn << PAGE_SHIFT; | |
deaef20e | 220 | page = pfn_to_page(pfn); |
f26b2a56 | 221 | |
e7b8b7f1 PM |
222 | if (cpu_context(smp_processor_id(), vma->vm_mm) == NO_CONTEXT) |
223 | return; | |
224 | ||
deaef20e PM |
225 | pgd = pgd_offset(vma->vm_mm, address); |
226 | pud = pud_offset(pgd, address); | |
227 | pmd = pmd_offset(pud, address); | |
228 | pte = pte_offset_kernel(pmd, address); | |
229 | ||
230 | /* If the page isn't present, there is nothing to do here. */ | |
231 | if (!(pte_val(*pte) & _PAGE_PRESENT)) | |
232 | return; | |
1da177e4 | 233 | |
deaef20e PM |
234 | if ((vma->vm_mm == current->active_mm)) |
235 | vaddr = NULL; | |
236 | else { | |
b638d0b9 | 237 | /* |
deaef20e PM |
238 | * Use kmap_coherent or kmap_atomic to do flushes for |
239 | * another ASID than the current one. | |
b638d0b9 | 240 | */ |
deaef20e | 241 | map_coherent = (current_cpu_data.dcache.n_aliases && |
55661fc1 | 242 | test_bit(PG_dcache_clean, &page->flags) && |
deaef20e PM |
243 | page_mapped(page)); |
244 | if (map_coherent) | |
245 | vaddr = kmap_coherent(page, address); | |
246 | else | |
bc3e11be | 247 | vaddr = kmap_atomic(page); |
deaef20e PM |
248 | |
249 | address = (unsigned long)vaddr; | |
250 | } | |
251 | ||
e717cc6c | 252 | flush_cache_one(CACHE_OC_ADDRESS_ARRAY | |
deaef20e PM |
253 | (address & shm_align_mask), phys); |
254 | ||
255 | if (vma->vm_flags & VM_EXEC) | |
256 | flush_icache_all(); | |
257 | ||
258 | if (vaddr) { | |
259 | if (map_coherent) | |
260 | kunmap_coherent(vaddr); | |
261 | else | |
bc3e11be | 262 | kunmap_atomic(vaddr); |
b638d0b9 | 263 | } |
1da177e4 LT |
264 | } |
265 | ||
266 | /* | |
267 | * Write back and invalidate D-caches. | |
268 | * | |
269 | * START, END: Virtual Address (U0 address) | |
270 | * | |
271 | * NOTE: We need to flush the _physical_ page entry. | |
272 | * Flushing the cache lines for U0 only isn't enough. | |
273 | * We need to flush for P1 too, which may contain aliases. | |
274 | */ | |
f26b2a56 | 275 | static void sh4_flush_cache_range(void *args) |
1da177e4 | 276 | { |
f26b2a56 PM |
277 | struct flusher_data *data = args; |
278 | struct vm_area_struct *vma; | |
279 | unsigned long start, end; | |
280 | ||
281 | vma = data->vma; | |
282 | start = data->addr1; | |
283 | end = data->addr2; | |
284 | ||
e7b8b7f1 PM |
285 | if (cpu_context(smp_processor_id(), vma->vm_mm) == NO_CONTEXT) |
286 | return; | |
287 | ||
b638d0b9 RC |
288 | /* |
289 | * If cache is only 4k-per-way, there are never any 'aliases'. Since | |
290 | * the cache is physically tagged, the data can just be left in there. | |
291 | */ | |
7ec9d6f8 | 292 | if (boot_cpu_data.dcache.n_aliases == 0) |
b638d0b9 RC |
293 | return; |
294 | ||
654d364e | 295 | flush_dcache_all(); |
b638d0b9 | 296 | |
654d364e | 297 | if (vma->vm_flags & VM_EXEC) |
1da177e4 LT |
298 | flush_icache_all(); |
299 | } | |
300 | ||
b638d0b9 | 301 | /** |
a7a7c0e1 | 302 | * __flush_cache_one |
b638d0b9 RC |
303 | * |
304 | * @addr: address in memory mapped cache array | |
305 | * @phys: P1 address to flush (has to match tags if addr has 'A' bit | |
306 | * set i.e. associative write) | |
307 | * @exec_offset: set to 0x20000000 if flush has to be executed from P2 | |
308 | * region else 0x0 | |
309 | * | |
310 | * The offset into the cache array implied by 'addr' selects the | |
311 | * 'colour' of the virtual address range that will be flushed. The | |
312 | * operation (purge/write-back) is selected by the lower 2 bits of | |
313 | * 'phys'. | |
314 | */ | |
a7a7c0e1 | 315 | static void __flush_cache_one(unsigned long addr, unsigned long phys, |
b638d0b9 RC |
316 | unsigned long exec_offset) |
317 | { | |
318 | int way_count; | |
319 | unsigned long base_addr = addr; | |
320 | struct cache_info *dcache; | |
321 | unsigned long way_incr; | |
322 | unsigned long a, ea, p; | |
323 | unsigned long temp_pc; | |
324 | ||
7ec9d6f8 | 325 | dcache = &boot_cpu_data.dcache; |
b638d0b9 RC |
326 | /* Write this way for better assembly. */ |
327 | way_count = dcache->ways; | |
328 | way_incr = dcache->way_incr; | |
329 | ||
330 | /* | |
331 | * Apply exec_offset (i.e. branch to P2 if required.). | |
332 | * | |
333 | * FIXME: | |
334 | * | |
335 | * If I write "=r" for the (temp_pc), it puts this in r6 hence | |
336 | * trashing exec_offset before it's been added on - why? Hence | |
337 | * "=&r" as a 'workaround' | |
338 | */ | |
339 | asm volatile("mov.l 1f, %0\n\t" | |
340 | "add %1, %0\n\t" | |
341 | "jmp @%0\n\t" | |
342 | "nop\n\t" | |
343 | ".balign 4\n\t" | |
344 | "1: .long 2f\n\t" | |
345 | "2:\n" : "=&r" (temp_pc) : "r" (exec_offset)); | |
346 | ||
347 | /* | |
348 | * We know there will be >=1 iteration, so write as do-while to avoid | |
349 | * pointless nead-of-loop check for 0 iterations. | |
350 | */ | |
351 | do { | |
352 | ea = base_addr + PAGE_SIZE; | |
353 | a = base_addr; | |
354 | p = phys; | |
355 | ||
356 | do { | |
357 | *(volatile unsigned long *)a = p; | |
358 | /* | |
359 | * Next line: intentionally not p+32, saves an add, p | |
360 | * will do since only the cache tag bits need to | |
361 | * match. | |
362 | */ | |
363 | *(volatile unsigned long *)(a+32) = p; | |
364 | a += 64; | |
365 | p += 64; | |
366 | } while (a < ea); | |
367 | ||
368 | base_addr += way_incr; | |
369 | } while (--way_count != 0); | |
370 | } | |
371 | ||
37443ef3 PM |
372 | extern void __weak sh4__flush_region_init(void); |
373 | ||
374 | /* | |
375 | * SH-4 has virtually indexed and physically tagged cache. | |
376 | */ | |
377 | void __init sh4_cache_init(void) | |
378 | { | |
379 | printk("PVR=%08x CVR=%08x PRR=%08x\n", | |
9d56dd3b PM |
380 | __raw_readl(CCN_PVR), |
381 | __raw_readl(CCN_CVR), | |
382 | __raw_readl(CCN_PRR)); | |
37443ef3 | 383 | |
f26b2a56 PM |
384 | local_flush_icache_range = sh4_flush_icache_range; |
385 | local_flush_dcache_page = sh4_flush_dcache_page; | |
386 | local_flush_cache_all = sh4_flush_cache_all; | |
387 | local_flush_cache_mm = sh4_flush_cache_mm; | |
388 | local_flush_cache_dup_mm = sh4_flush_cache_mm; | |
389 | local_flush_cache_page = sh4_flush_cache_page; | |
390 | local_flush_cache_range = sh4_flush_cache_range; | |
37443ef3 PM |
391 | |
392 | sh4__flush_region_init(); | |
393 | } |