]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * TLB support routines. | |
3 | * | |
4 | * Copyright (C) 1998-2001, 2003 Hewlett-Packard Co | |
5 | * David Mosberger-Tang <davidm@hpl.hp.com> | |
6 | * | |
7 | * 08/02/00 A. Mallick <asit.k.mallick@intel.com> | |
8 | * Modified RID allocation for SMP | |
9 | * Goutham Rao <goutham.rao@intel.com> | |
10 | * IPI based ptc implementation and A-step IPI implementation. | |
dcc17d1b PK |
11 | * Rohit Seth <rohit.seth@intel.com> |
12 | * Ken Chen <kenneth.w.chen@intel.com> | |
aec103bf | 13 | * Christophe de Dinechin <ddd@hp.com>: Avoid ptc.e on memory allocation |
2046b94e FY |
14 | * Copyright (C) 2007 Intel Corp |
15 | * Fenghua Yu <fenghua.yu@intel.com> | |
16 | * Add multiple ptc.g/ptc.ga instruction support in global tlb purge. | |
1da177e4 | 17 | */ |
1da177e4 LT |
18 | #include <linux/module.h> |
19 | #include <linux/init.h> | |
20 | #include <linux/kernel.h> | |
21 | #include <linux/sched.h> | |
22 | #include <linux/smp.h> | |
23 | #include <linux/mm.h> | |
dcc17d1b | 24 | #include <linux/bootmem.h> |
1da177e4 LT |
25 | |
26 | #include <asm/delay.h> | |
27 | #include <asm/mmu_context.h> | |
28 | #include <asm/pgalloc.h> | |
29 | #include <asm/pal.h> | |
30 | #include <asm/tlbflush.h> | |
dcc17d1b | 31 | #include <asm/dma.h> |
2046b94e | 32 | #include <asm/sal.h> |
1da177e4 LT |
33 | |
34 | static struct { | |
35 | unsigned long mask; /* mask of supported purge page-sizes */ | |
58cd9082 | 36 | unsigned long max_bits; /* log2 of largest supported purge page-size */ |
1da177e4 LT |
37 | } purge; |
38 | ||
39 | struct ia64_ctx ia64_ctx = { | |
8737d595 MAC |
40 | .lock = __SPIN_LOCK_UNLOCKED(ia64_ctx.lock), |
41 | .next = 1, | |
42 | .max_ctx = ~0U | |
1da177e4 LT |
43 | }; |
44 | ||
45 | DEFINE_PER_CPU(u8, ia64_need_tlb_flush); | |
46 | ||
dcc17d1b PK |
47 | /* |
48 | * Initializes the ia64_ctx.bitmap array based on max_ctx+1. | |
49 | * Called after cpu_init() has setup ia64_ctx.max_ctx based on | |
50 | * maximum RID that is supported by boot CPU. | |
51 | */ | |
52 | void __init | |
53 | mmu_context_init (void) | |
54 | { | |
55 | ia64_ctx.bitmap = alloc_bootmem((ia64_ctx.max_ctx+1)>>3); | |
56 | ia64_ctx.flushmap = alloc_bootmem((ia64_ctx.max_ctx+1)>>3); | |
57 | } | |
58 | ||
1da177e4 LT |
59 | /* |
60 | * Acquire the ia64_ctx.lock before calling this function! | |
61 | */ | |
62 | void | |
63 | wrap_mmu_context (struct mm_struct *mm) | |
64 | { | |
58cd9082 | 65 | int i, cpu; |
dcc17d1b | 66 | unsigned long flush_bit; |
1da177e4 | 67 | |
dcc17d1b PK |
68 | for (i=0; i <= ia64_ctx.max_ctx / BITS_PER_LONG; i++) { |
69 | flush_bit = xchg(&ia64_ctx.flushmap[i], 0); | |
70 | ia64_ctx.bitmap[i] ^= flush_bit; | |
1da177e4 | 71 | } |
dcc17d1b PK |
72 | |
73 | /* use offset at 300 to skip daemons */ | |
74 | ia64_ctx.next = find_next_zero_bit(ia64_ctx.bitmap, | |
75 | ia64_ctx.max_ctx, 300); | |
76 | ia64_ctx.limit = find_next_bit(ia64_ctx.bitmap, | |
77 | ia64_ctx.max_ctx, ia64_ctx.next); | |
78 | ||
58cd9082 KC |
79 | /* |
80 | * can't call flush_tlb_all() here because of race condition | |
81 | * with O(1) scheduler [EF] | |
82 | */ | |
83 | cpu = get_cpu(); /* prevent preemption/migration */ | |
84 | for_each_online_cpu(i) | |
85 | if (i != cpu) | |
86 | per_cpu(ia64_need_tlb_flush, i) = 1; | |
87 | put_cpu(); | |
1da177e4 LT |
88 | local_flush_tlb_all(); |
89 | } | |
90 | ||
2046b94e FY |
91 | /* |
92 | * Implement "spinaphores" ... like counting semaphores, but they | |
93 | * spin instead of sleeping. If there are ever any other users for | |
94 | * this primitive it can be moved up to a spinaphore.h header. | |
95 | */ | |
96 | struct spinaphore { | |
97 | atomic_t cur; | |
98 | }; | |
99 | ||
100 | static inline void spinaphore_init(struct spinaphore *ss, int val) | |
101 | { | |
102 | atomic_set(&ss->cur, val); | |
103 | } | |
104 | ||
105 | static inline void down_spin(struct spinaphore *ss) | |
106 | { | |
107 | while (unlikely(!atomic_add_unless(&ss->cur, -1, 0))) | |
108 | while (atomic_read(&ss->cur) == 0) | |
109 | cpu_relax(); | |
110 | } | |
111 | ||
112 | static inline void up_spin(struct spinaphore *ss) | |
113 | { | |
114 | atomic_add(1, &ss->cur); | |
115 | } | |
116 | ||
117 | static struct spinaphore ptcg_sem; | |
118 | static u16 nptcg = 1; | |
119 | static int need_ptcg_sem = 1; | |
120 | static int toolatetochangeptcgsem = 0; | |
121 | ||
122 | /* | |
123 | * Maximum number of simultaneous ptc.g purges in the system can | |
124 | * be defined by PAL_VM_SUMMARY (in which case we should take | |
125 | * the smallest value for any cpu in the system) or by the PAL | |
126 | * override table (in which case we should ignore the value from | |
127 | * PAL_VM_SUMMARY). | |
128 | * | |
129 | * Complicating the logic here is the fact that num_possible_cpus() | |
130 | * isn't fully setup until we start bringing cpus online. | |
131 | */ | |
132 | void | |
133 | setup_ptcg_sem(int max_purges, int from_palo) | |
134 | { | |
135 | static int have_palo; | |
136 | static int firstcpu = 1; | |
137 | ||
138 | if (toolatetochangeptcgsem) { | |
139 | BUG_ON(max_purges < nptcg); | |
140 | return; | |
141 | } | |
142 | ||
143 | if (from_palo) { | |
144 | have_palo = 1; | |
145 | ||
146 | /* In PALO max_purges == 0 really means it! */ | |
147 | if (max_purges == 0) | |
148 | panic("Whoa! Platform does not support global TLB purges.\n"); | |
149 | nptcg = max_purges; | |
150 | if (nptcg == PALO_MAX_TLB_PURGES) { | |
151 | need_ptcg_sem = 0; | |
152 | return; | |
153 | } | |
154 | goto resetsema; | |
155 | } | |
156 | if (have_palo) { | |
157 | if (nptcg != PALO_MAX_TLB_PURGES) | |
158 | need_ptcg_sem = (num_possible_cpus() > nptcg); | |
159 | return; | |
160 | } | |
161 | ||
162 | /* In PAL_VM_SUMMARY max_purges == 0 actually means 1 */ | |
163 | if (max_purges == 0) max_purges = 1; | |
164 | ||
165 | if (firstcpu) { | |
166 | nptcg = max_purges; | |
167 | firstcpu = 0; | |
168 | } | |
169 | if (max_purges < nptcg) | |
170 | nptcg = max_purges; | |
171 | if (nptcg == PAL_MAX_PURGES) { | |
172 | need_ptcg_sem = 0; | |
173 | return; | |
174 | } else | |
175 | need_ptcg_sem = (num_possible_cpus() > nptcg); | |
176 | ||
177 | resetsema: | |
178 | spinaphore_init(&ptcg_sem, max_purges); | |
179 | } | |
180 | ||
1da177e4 | 181 | void |
58cd9082 KC |
182 | ia64_global_tlb_purge (struct mm_struct *mm, unsigned long start, |
183 | unsigned long end, unsigned long nbits) | |
1da177e4 | 184 | { |
aec103bf DCIV |
185 | struct mm_struct *active_mm = current->active_mm; |
186 | ||
2046b94e FY |
187 | toolatetochangeptcgsem = 1; |
188 | ||
aec103bf DCIV |
189 | if (mm != active_mm) { |
190 | /* Restore region IDs for mm */ | |
191 | if (mm && active_mm) { | |
192 | activate_context(mm); | |
193 | } else { | |
194 | flush_tlb_all(); | |
195 | return; | |
196 | } | |
c1902aae DR |
197 | } |
198 | ||
2046b94e FY |
199 | if (need_ptcg_sem) |
200 | down_spin(&ptcg_sem); | |
201 | ||
202 | do { | |
203 | /* | |
204 | * Flush ALAT entries also. | |
205 | */ | |
206 | ia64_ptcga(start, (nbits << 2)); | |
207 | ia64_srlz_i(); | |
208 | start += (1UL << nbits); | |
209 | } while (start < end); | |
210 | ||
211 | if (need_ptcg_sem) | |
212 | up_spin(&ptcg_sem); | |
aec103bf DCIV |
213 | |
214 | if (mm != active_mm) { | |
215 | activate_context(active_mm); | |
216 | } | |
1da177e4 LT |
217 | } |
218 | ||
219 | void | |
220 | local_flush_tlb_all (void) | |
221 | { | |
222 | unsigned long i, j, flags, count0, count1, stride0, stride1, addr; | |
223 | ||
224 | addr = local_cpu_data->ptce_base; | |
225 | count0 = local_cpu_data->ptce_count[0]; | |
226 | count1 = local_cpu_data->ptce_count[1]; | |
227 | stride0 = local_cpu_data->ptce_stride[0]; | |
228 | stride1 = local_cpu_data->ptce_stride[1]; | |
229 | ||
230 | local_irq_save(flags); | |
231 | for (i = 0; i < count0; ++i) { | |
232 | for (j = 0; j < count1; ++j) { | |
233 | ia64_ptce(addr); | |
234 | addr += stride1; | |
235 | } | |
236 | addr += stride0; | |
237 | } | |
238 | local_irq_restore(flags); | |
239 | ia64_srlz_i(); /* srlz.i implies srlz.d */ | |
240 | } | |
241 | ||
242 | void | |
58cd9082 KC |
243 | flush_tlb_range (struct vm_area_struct *vma, unsigned long start, |
244 | unsigned long end) | |
1da177e4 LT |
245 | { |
246 | struct mm_struct *mm = vma->vm_mm; | |
247 | unsigned long size = end - start; | |
248 | unsigned long nbits; | |
249 | ||
c1902aae | 250 | #ifndef CONFIG_SMP |
1da177e4 | 251 | if (mm != current->active_mm) { |
1da177e4 | 252 | mm->context = 0; |
1da177e4 LT |
253 | return; |
254 | } | |
c1902aae | 255 | #endif |
1da177e4 LT |
256 | |
257 | nbits = ia64_fls(size + 0xfff); | |
58cd9082 KC |
258 | while (unlikely (((1UL << nbits) & purge.mask) == 0) && |
259 | (nbits < purge.max_bits)) | |
1da177e4 LT |
260 | ++nbits; |
261 | if (nbits > purge.max_bits) | |
262 | nbits = purge.max_bits; | |
263 | start &= ~((1UL << nbits) - 1); | |
264 | ||
663b97f7 | 265 | preempt_disable(); |
ce9eed5a KC |
266 | #ifdef CONFIG_SMP |
267 | if (mm != current->active_mm || cpus_weight(mm->cpu_vm_mask) != 1) { | |
268 | platform_global_tlb_purge(mm, start, end, nbits); | |
269 | preempt_enable(); | |
270 | return; | |
271 | } | |
272 | #endif | |
1da177e4 LT |
273 | do { |
274 | ia64_ptcl(start, (nbits<<2)); | |
275 | start += (1UL << nbits); | |
276 | } while (start < end); | |
663b97f7 | 277 | preempt_enable(); |
1da177e4 LT |
278 | ia64_srlz_i(); /* srlz.i implies srlz.d */ |
279 | } | |
280 | EXPORT_SYMBOL(flush_tlb_range); | |
281 | ||
282 | void __devinit | |
283 | ia64_tlb_init (void) | |
284 | { | |
256a7e09 | 285 | ia64_ptce_info_t uninitialized_var(ptce_info); /* GCC be quiet */ |
1da177e4 LT |
286 | unsigned long tr_pgbits; |
287 | long status; | |
288 | ||
289 | if ((status = ia64_pal_vm_page_size(&tr_pgbits, &purge.mask)) != 0) { | |
c2eeb321 | 290 | printk(KERN_ERR "PAL_VM_PAGE_SIZE failed with status=%ld; " |
1da177e4 LT |
291 | "defaulting to architected purge page-sizes.\n", status); |
292 | purge.mask = 0x115557000UL; | |
293 | } | |
294 | purge.max_bits = ia64_fls(purge.mask); | |
295 | ||
296 | ia64_get_ptce(&ptce_info); | |
297 | local_cpu_data->ptce_base = ptce_info.base; | |
298 | local_cpu_data->ptce_count[0] = ptce_info.count[0]; | |
299 | local_cpu_data->ptce_count[1] = ptce_info.count[1]; | |
300 | local_cpu_data->ptce_stride[0] = ptce_info.stride[0]; | |
301 | local_cpu_data->ptce_stride[1] = ptce_info.stride[1]; | |
302 | ||
58cd9082 | 303 | local_flush_tlb_all(); /* nuke left overs from bootstrapping... */ |
1da177e4 | 304 | } |