]>
Commit | Line | Data |
---|---|---|
c78095bd CM |
1 | /* |
2 | * Copyright 2010 Tilera Corporation. All Rights Reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation, version 2. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but | |
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or | |
11 | * NON INFRINGEMENT. See the GNU General Public License for | |
12 | * more details. | |
13 | */ | |
14 | ||
15 | #include <asm/page.h> | |
16 | #include <asm/cacheflush.h> | |
17 | #include <arch/icache.h> | |
18 | ||
19 | ||
20 | void __flush_icache_range(unsigned long start, unsigned long end) | |
21 | { | |
22 | invalidate_icache((const void *)start, end - start, PAGE_SIZE); | |
23 | } | |
63b7ca6b CM |
24 | |
25 | ||
26 | /* Force a load instruction to issue. */ | |
27 | static inline void force_load(char *p) | |
28 | { | |
29 | *(volatile char *)p; | |
30 | } | |
31 | ||
32 | /* | |
33 | * Flush and invalidate a VA range that is homed remotely on a single | |
34 | * core (if "!hfh") or homed via hash-for-home (if "hfh"), waiting | |
35 | * until the memory controller holds the flushed values. | |
36 | */ | |
37 | void finv_buffer_remote(void *buffer, size_t size, int hfh) | |
38 | { | |
39 | char *p, *base; | |
40 | size_t step_size, load_count; | |
41 | const unsigned long STRIPE_WIDTH = 8192; | |
42 | ||
43 | /* | |
44 | * Flush and invalidate the buffer out of the local L1/L2 | |
45 | * and request the home cache to flush and invalidate as well. | |
46 | */ | |
47 | __finv_buffer(buffer, size); | |
48 | ||
49 | /* | |
50 | * Wait for the home cache to acknowledge that it has processed | |
51 | * all the flush-and-invalidate requests. This does not mean | |
52 | * that the flushed data has reached the memory controller yet, | |
53 | * but it does mean the home cache is processing the flushes. | |
54 | */ | |
55 | __insn_mf(); | |
56 | ||
57 | /* | |
58 | * Issue a load to the last cache line, which can't complete | |
59 | * until all the previously-issued flushes to the same memory | |
60 | * controller have also completed. If we weren't striping | |
61 | * memory, that one load would be sufficient, but since we may | |
62 | * be, we also need to back up to the last load issued to | |
63 | * another memory controller, which would be the point where | |
64 | * we crossed an 8KB boundary (the granularity of striping | |
65 | * across memory controllers). Keep backing up and doing this | |
66 | * until we are before the beginning of the buffer, or have | |
67 | * hit all the controllers. | |
68 | * | |
69 | * If we are flushing a hash-for-home buffer, it's even worse. | |
70 | * Each line may be homed on a different tile, and each tile | |
71 | * may have up to four lines that are on different | |
72 | * controllers. So as we walk backwards, we have to touch | |
73 | * enough cache lines to satisfy these constraints. In | |
74 | * practice this ends up being close enough to "load from | |
75 | * every cache line on a full memory stripe on each | |
76 | * controller" that we simply do that, to simplify the logic. | |
77 | * | |
78 | * FIXME: See bug 9535 for some issues with this code. | |
79 | */ | |
80 | if (hfh) { | |
81 | step_size = L2_CACHE_BYTES; | |
82 | load_count = (STRIPE_WIDTH / L2_CACHE_BYTES) * | |
83 | (1 << CHIP_LOG_NUM_MSHIMS()); | |
84 | } else { | |
85 | step_size = STRIPE_WIDTH; | |
86 | load_count = (1 << CHIP_LOG_NUM_MSHIMS()); | |
87 | } | |
88 | ||
89 | /* Load the last byte of the buffer. */ | |
90 | p = (char *)buffer + size - 1; | |
91 | force_load(p); | |
92 | ||
93 | /* Bump down to the end of the previous stripe or cache line. */ | |
94 | p -= step_size; | |
95 | p = (char *)((unsigned long)p | (step_size - 1)); | |
96 | ||
97 | /* Figure out how far back we need to go. */ | |
98 | base = p - (step_size * (load_count - 2)); | |
99 | if ((long)base < (long)buffer) | |
100 | base = buffer; | |
101 | ||
102 | /* | |
103 | * Fire all the loads we need. The MAF only has eight entries | |
104 | * so we can have at most eight outstanding loads, so we | |
105 | * unroll by that amount. | |
106 | */ | |
107 | #pragma unroll 8 | |
108 | for (; p >= base; p -= step_size) | |
109 | force_load(p); | |
110 | ||
111 | /* | |
112 | * Repeat, but with inv's instead of loads, to get rid of the | |
113 | * data we just loaded into our own cache and the old home L3. | |
114 | * No need to unroll since inv's don't target a register. | |
115 | */ | |
116 | p = (char *)buffer + size - 1; | |
117 | __insn_inv(p); | |
118 | p -= step_size; | |
119 | p = (char *)((unsigned long)p | (step_size - 1)); | |
120 | for (; p >= base; p -= step_size) | |
121 | __insn_inv(p); | |
122 | ||
123 | /* Wait for the load+inv's (and thus finvs) to have completed. */ | |
124 | __insn_mf(); | |
125 | } |