]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * native hashtable management. | |
3 | * | |
4 | * SMP scalability work: | |
5 | * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or | |
8 | * modify it under the terms of the GNU General Public License | |
9 | * as published by the Free Software Foundation; either version | |
10 | * 2 of the License, or (at your option) any later version. | |
11 | */ | |
12 | #include <linux/spinlock.h> | |
13 | #include <linux/bitops.h> | |
14 | #include <linux/threads.h> | |
15 | #include <linux/smp.h> | |
16 | ||
17 | #include <asm/abs_addr.h> | |
18 | #include <asm/machdep.h> | |
19 | #include <asm/mmu.h> | |
20 | #include <asm/mmu_context.h> | |
21 | #include <asm/pgtable.h> | |
22 | #include <asm/tlbflush.h> | |
23 | #include <asm/tlb.h> | |
24 | #include <asm/cputable.h> | |
25 | ||
26 | #define HPTE_LOCK_BIT 3 | |
27 | ||
28 | static DEFINE_SPINLOCK(native_tlbie_lock); | |
29 | ||
96e28449 | 30 | static inline void native_lock_hpte(hpte_t *hptep) |
1da177e4 | 31 | { |
96e28449 | 32 | unsigned long *word = &hptep->v; |
1da177e4 LT |
33 | |
34 | while (1) { | |
35 | if (!test_and_set_bit(HPTE_LOCK_BIT, word)) | |
36 | break; | |
37 | while(test_bit(HPTE_LOCK_BIT, word)) | |
38 | cpu_relax(); | |
39 | } | |
40 | } | |
41 | ||
96e28449 | 42 | static inline void native_unlock_hpte(hpte_t *hptep) |
1da177e4 | 43 | { |
96e28449 | 44 | unsigned long *word = &hptep->v; |
1da177e4 LT |
45 | |
46 | asm volatile("lwsync":::"memory"); | |
47 | clear_bit(HPTE_LOCK_BIT, word); | |
48 | } | |
49 | ||
50 | long native_hpte_insert(unsigned long hpte_group, unsigned long va, | |
96e28449 DG |
51 | unsigned long prpn, unsigned long vflags, |
52 | unsigned long rflags) | |
1da177e4 | 53 | { |
96e28449 DG |
54 | hpte_t *hptep = htab_address + hpte_group; |
55 | unsigned long hpte_v, hpte_r; | |
1da177e4 LT |
56 | int i; |
57 | ||
58 | for (i = 0; i < HPTES_PER_GROUP; i++) { | |
96e28449 | 59 | if (! (hptep->v & HPTE_V_VALID)) { |
1da177e4 LT |
60 | /* retry with lock held */ |
61 | native_lock_hpte(hptep); | |
96e28449 | 62 | if (! (hptep->v & HPTE_V_VALID)) |
1da177e4 LT |
63 | break; |
64 | native_unlock_hpte(hptep); | |
65 | } | |
66 | ||
67 | hptep++; | |
68 | } | |
69 | ||
70 | if (i == HPTES_PER_GROUP) | |
71 | return -1; | |
72 | ||
96e28449 DG |
73 | hpte_v = (va >> 23) << HPTE_V_AVPN_SHIFT | vflags | HPTE_V_VALID; |
74 | if (vflags & HPTE_V_LARGE) | |
75 | va &= ~(1UL << HPTE_V_AVPN_SHIFT); | |
aefd16b0 | 76 | hpte_r = (prpn << HPTE_R_RPN_SHIFT) | rflags; |
1da177e4 | 77 | |
96e28449 | 78 | hptep->r = hpte_r; |
1da177e4 LT |
79 | /* Guarantee the second dword is visible before the valid bit */ |
80 | __asm__ __volatile__ ("eieio" : : : "memory"); | |
1da177e4 LT |
81 | /* |
82 | * Now set the first dword including the valid bit | |
83 | * NOTE: this also unlocks the hpte | |
84 | */ | |
96e28449 | 85 | hptep->v = hpte_v; |
1da177e4 LT |
86 | |
87 | __asm__ __volatile__ ("ptesync" : : : "memory"); | |
88 | ||
96e28449 | 89 | return i | (!!(vflags & HPTE_V_SECONDARY) << 3); |
1da177e4 LT |
90 | } |
91 | ||
92 | static long native_hpte_remove(unsigned long hpte_group) | |
93 | { | |
96e28449 | 94 | hpte_t *hptep; |
1da177e4 LT |
95 | int i; |
96 | int slot_offset; | |
96e28449 | 97 | unsigned long hpte_v; |
1da177e4 LT |
98 | |
99 | /* pick a random entry to start at */ | |
100 | slot_offset = mftb() & 0x7; | |
101 | ||
102 | for (i = 0; i < HPTES_PER_GROUP; i++) { | |
103 | hptep = htab_address + hpte_group + slot_offset; | |
96e28449 | 104 | hpte_v = hptep->v; |
1da177e4 | 105 | |
96e28449 | 106 | if ((hpte_v & HPTE_V_VALID) && !(hpte_v & HPTE_V_BOLTED)) { |
1da177e4 LT |
107 | /* retry with lock held */ |
108 | native_lock_hpte(hptep); | |
96e28449 DG |
109 | hpte_v = hptep->v; |
110 | if ((hpte_v & HPTE_V_VALID) | |
111 | && !(hpte_v & HPTE_V_BOLTED)) | |
1da177e4 LT |
112 | break; |
113 | native_unlock_hpte(hptep); | |
114 | } | |
115 | ||
116 | slot_offset++; | |
117 | slot_offset &= 0x7; | |
118 | } | |
119 | ||
120 | if (i == HPTES_PER_GROUP) | |
121 | return -1; | |
122 | ||
123 | /* Invalidate the hpte. NOTE: this also unlocks it */ | |
96e28449 | 124 | hptep->v = 0; |
1da177e4 LT |
125 | |
126 | return i; | |
127 | } | |
128 | ||
96e28449 | 129 | static inline void set_pp_bit(unsigned long pp, hpte_t *addr) |
1da177e4 LT |
130 | { |
131 | unsigned long old; | |
96e28449 | 132 | unsigned long *p = &addr->r; |
1da177e4 LT |
133 | |
134 | __asm__ __volatile__( | |
135 | "1: ldarx %0,0,%3\n\ | |
136 | rldimi %0,%2,0,61\n\ | |
137 | stdcx. %0,0,%3\n\ | |
138 | bne 1b" | |
139 | : "=&r" (old), "=m" (*p) | |
140 | : "r" (pp), "r" (p), "m" (*p) | |
141 | : "cc"); | |
142 | } | |
143 | ||
144 | /* | |
145 | * Only works on small pages. Yes its ugly to have to check each slot in | |
146 | * the group but we only use this during bootup. | |
147 | */ | |
148 | static long native_hpte_find(unsigned long vpn) | |
149 | { | |
96e28449 | 150 | hpte_t *hptep; |
1da177e4 LT |
151 | unsigned long hash; |
152 | unsigned long i, j; | |
153 | long slot; | |
96e28449 | 154 | unsigned long hpte_v; |
1da177e4 LT |
155 | |
156 | hash = hpt_hash(vpn, 0); | |
157 | ||
158 | for (j = 0; j < 2; j++) { | |
159 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | |
160 | for (i = 0; i < HPTES_PER_GROUP; i++) { | |
161 | hptep = htab_address + slot; | |
96e28449 | 162 | hpte_v = hptep->v; |
1da177e4 | 163 | |
96e28449 DG |
164 | if ((HPTE_V_AVPN_VAL(hpte_v) == (vpn >> 11)) |
165 | && (hpte_v & HPTE_V_VALID) | |
166 | && ( !!(hpte_v & HPTE_V_SECONDARY) == j)) { | |
1da177e4 LT |
167 | /* HPTE matches */ |
168 | if (j) | |
169 | slot = -slot; | |
170 | return slot; | |
171 | } | |
172 | ++slot; | |
173 | } | |
174 | hash = ~hash; | |
175 | } | |
176 | ||
177 | return -1; | |
178 | } | |
179 | ||
180 | static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, | |
181 | unsigned long va, int large, int local) | |
182 | { | |
96e28449 DG |
183 | hpte_t *hptep = htab_address + slot; |
184 | unsigned long hpte_v; | |
1da177e4 LT |
185 | unsigned long avpn = va >> 23; |
186 | int ret = 0; | |
187 | ||
188 | if (large) | |
96e28449 | 189 | avpn &= ~1; |
1da177e4 LT |
190 | |
191 | native_lock_hpte(hptep); | |
192 | ||
96e28449 | 193 | hpte_v = hptep->v; |
1da177e4 LT |
194 | |
195 | /* Even if we miss, we need to invalidate the TLB */ | |
96e28449 DG |
196 | if ((HPTE_V_AVPN_VAL(hpte_v) != avpn) |
197 | || !(hpte_v & HPTE_V_VALID)) { | |
1da177e4 LT |
198 | native_unlock_hpte(hptep); |
199 | ret = -1; | |
200 | } else { | |
201 | set_pp_bit(newpp, hptep); | |
202 | native_unlock_hpte(hptep); | |
203 | } | |
204 | ||
205 | /* Ensure it is out of the tlb too */ | |
206 | if (cpu_has_feature(CPU_FTR_TLBIEL) && !large && local) { | |
207 | tlbiel(va); | |
208 | } else { | |
209 | int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE); | |
210 | ||
211 | if (lock_tlbie) | |
212 | spin_lock(&native_tlbie_lock); | |
213 | tlbie(va, large); | |
214 | if (lock_tlbie) | |
215 | spin_unlock(&native_tlbie_lock); | |
216 | } | |
217 | ||
218 | return ret; | |
219 | } | |
220 | ||
221 | /* | |
222 | * Update the page protection bits. Intended to be used to create | |
223 | * guard pages for kernel data structures on pages which are bolted | |
224 | * in the HPT. Assumes pages being operated on will not be stolen. | |
225 | * Does not work on large pages. | |
226 | * | |
227 | * No need to lock here because we should be the only user. | |
228 | */ | |
229 | static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea) | |
230 | { | |
231 | unsigned long vsid, va, vpn, flags = 0; | |
232 | long slot; | |
96e28449 | 233 | hpte_t *hptep; |
1da177e4 LT |
234 | int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE); |
235 | ||
236 | vsid = get_kernel_vsid(ea); | |
237 | va = (vsid << 28) | (ea & 0x0fffffff); | |
238 | vpn = va >> PAGE_SHIFT; | |
239 | ||
240 | slot = native_hpte_find(vpn); | |
241 | if (slot == -1) | |
242 | panic("could not find page to bolt\n"); | |
243 | hptep = htab_address + slot; | |
244 | ||
245 | set_pp_bit(newpp, hptep); | |
246 | ||
247 | /* Ensure it is out of the tlb too */ | |
248 | if (lock_tlbie) | |
249 | spin_lock_irqsave(&native_tlbie_lock, flags); | |
250 | tlbie(va, 0); | |
251 | if (lock_tlbie) | |
252 | spin_unlock_irqrestore(&native_tlbie_lock, flags); | |
253 | } | |
254 | ||
255 | static void native_hpte_invalidate(unsigned long slot, unsigned long va, | |
256 | int large, int local) | |
257 | { | |
96e28449 DG |
258 | hpte_t *hptep = htab_address + slot; |
259 | unsigned long hpte_v; | |
1da177e4 LT |
260 | unsigned long avpn = va >> 23; |
261 | unsigned long flags; | |
262 | int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE); | |
263 | ||
264 | if (large) | |
96e28449 | 265 | avpn &= ~1; |
1da177e4 LT |
266 | |
267 | local_irq_save(flags); | |
268 | native_lock_hpte(hptep); | |
269 | ||
96e28449 | 270 | hpte_v = hptep->v; |
1da177e4 LT |
271 | |
272 | /* Even if we miss, we need to invalidate the TLB */ | |
96e28449 DG |
273 | if ((HPTE_V_AVPN_VAL(hpte_v) != avpn) |
274 | || !(hpte_v & HPTE_V_VALID)) { | |
1da177e4 LT |
275 | native_unlock_hpte(hptep); |
276 | } else { | |
277 | /* Invalidate the hpte. NOTE: this also unlocks it */ | |
96e28449 | 278 | hptep->v = 0; |
1da177e4 LT |
279 | } |
280 | ||
281 | /* Invalidate the tlb */ | |
282 | if (cpu_has_feature(CPU_FTR_TLBIEL) && !large && local) { | |
283 | tlbiel(va); | |
284 | } else { | |
285 | if (lock_tlbie) | |
286 | spin_lock(&native_tlbie_lock); | |
287 | tlbie(va, large); | |
288 | if (lock_tlbie) | |
289 | spin_unlock(&native_tlbie_lock); | |
290 | } | |
291 | local_irq_restore(flags); | |
292 | } | |
293 | ||
f4c82d51 S |
294 | /* |
295 | * clear all mappings on kexec. All cpus are in real mode (or they will | |
296 | * be when they isi), and we are the only one left. We rely on our kernel | |
297 | * mapping being 0xC0's and the hardware ignoring those two real bits. | |
298 | * | |
299 | * TODO: add batching support when enabled. remember, no dynamic memory here, | |
300 | * athough there is the control page available... | |
301 | */ | |
302 | static void native_hpte_clear(void) | |
303 | { | |
304 | unsigned long slot, slots, flags; | |
96e28449 DG |
305 | hpte_t *hptep = htab_address; |
306 | unsigned long hpte_v; | |
f4c82d51 S |
307 | unsigned long pteg_count; |
308 | ||
309 | pteg_count = htab_hash_mask + 1; | |
310 | ||
311 | local_irq_save(flags); | |
312 | ||
313 | /* we take the tlbie lock and hold it. Some hardware will | |
314 | * deadlock if we try to tlbie from two processors at once. | |
315 | */ | |
316 | spin_lock(&native_tlbie_lock); | |
317 | ||
318 | slots = pteg_count * HPTES_PER_GROUP; | |
319 | ||
320 | for (slot = 0; slot < slots; slot++, hptep++) { | |
321 | /* | |
322 | * we could lock the pte here, but we are the only cpu | |
323 | * running, right? and for crash dump, we probably | |
324 | * don't want to wait for a maybe bad cpu. | |
325 | */ | |
96e28449 | 326 | hpte_v = hptep->v; |
f4c82d51 | 327 | |
96e28449 DG |
328 | if (hpte_v & HPTE_V_VALID) { |
329 | hptep->v = 0; | |
330 | tlbie(slot2va(hpte_v, slot), hpte_v & HPTE_V_LARGE); | |
f4c82d51 S |
331 | } |
332 | } | |
333 | ||
334 | spin_unlock(&native_tlbie_lock); | |
335 | local_irq_restore(flags); | |
336 | } | |
337 | ||
1da177e4 LT |
338 | static void native_flush_hash_range(unsigned long context, |
339 | unsigned long number, int local) | |
340 | { | |
341 | unsigned long vsid, vpn, va, hash, secondary, slot, flags, avpn; | |
342 | int i, j; | |
96e28449 DG |
343 | hpte_t *hptep; |
344 | unsigned long hpte_v; | |
1da177e4 LT |
345 | struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch); |
346 | ||
347 | /* XXX fix for large ptes */ | |
348 | unsigned long large = 0; | |
349 | ||
350 | local_irq_save(flags); | |
351 | ||
352 | j = 0; | |
353 | for (i = 0; i < number; i++) { | |
1f8d419e | 354 | if (batch->addr[i] < KERNELBASE) |
1da177e4 LT |
355 | vsid = get_vsid(context, batch->addr[i]); |
356 | else | |
357 | vsid = get_kernel_vsid(batch->addr[i]); | |
358 | ||
359 | va = (vsid << 28) | (batch->addr[i] & 0x0fffffff); | |
360 | batch->vaddr[j] = va; | |
361 | if (large) | |
362 | vpn = va >> HPAGE_SHIFT; | |
363 | else | |
364 | vpn = va >> PAGE_SHIFT; | |
365 | hash = hpt_hash(vpn, large); | |
366 | secondary = (pte_val(batch->pte[i]) & _PAGE_SECONDARY) >> 15; | |
367 | if (secondary) | |
368 | hash = ~hash; | |
369 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | |
370 | slot += (pte_val(batch->pte[i]) & _PAGE_GROUP_IX) >> 12; | |
371 | ||
372 | hptep = htab_address + slot; | |
373 | ||
374 | avpn = va >> 23; | |
375 | if (large) | |
376 | avpn &= ~0x1UL; | |
377 | ||
378 | native_lock_hpte(hptep); | |
379 | ||
96e28449 | 380 | hpte_v = hptep->v; |
1da177e4 LT |
381 | |
382 | /* Even if we miss, we need to invalidate the TLB */ | |
96e28449 DG |
383 | if ((HPTE_V_AVPN_VAL(hpte_v) != avpn) |
384 | || !(hpte_v & HPTE_V_VALID)) { | |
1da177e4 LT |
385 | native_unlock_hpte(hptep); |
386 | } else { | |
387 | /* Invalidate the hpte. NOTE: this also unlocks it */ | |
96e28449 | 388 | hptep->v = 0; |
1da177e4 LT |
389 | } |
390 | ||
391 | j++; | |
392 | } | |
393 | ||
394 | if (cpu_has_feature(CPU_FTR_TLBIEL) && !large && local) { | |
395 | asm volatile("ptesync":::"memory"); | |
396 | ||
397 | for (i = 0; i < j; i++) | |
398 | __tlbiel(batch->vaddr[i]); | |
399 | ||
400 | asm volatile("ptesync":::"memory"); | |
401 | } else { | |
402 | int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE); | |
403 | ||
404 | if (lock_tlbie) | |
405 | spin_lock(&native_tlbie_lock); | |
406 | ||
407 | asm volatile("ptesync":::"memory"); | |
408 | ||
409 | for (i = 0; i < j; i++) | |
410 | __tlbie(batch->vaddr[i], 0); | |
411 | ||
412 | asm volatile("eieio; tlbsync; ptesync":::"memory"); | |
413 | ||
414 | if (lock_tlbie) | |
415 | spin_unlock(&native_tlbie_lock); | |
416 | } | |
417 | ||
418 | local_irq_restore(flags); | |
419 | } | |
420 | ||
421 | #ifdef CONFIG_PPC_PSERIES | |
422 | /* Disable TLB batching on nighthawk */ | |
423 | static inline int tlb_batching_enabled(void) | |
424 | { | |
425 | struct device_node *root = of_find_node_by_path("/"); | |
426 | int enabled = 1; | |
427 | ||
428 | if (root) { | |
429 | const char *model = get_property(root, "model", NULL); | |
430 | if (model && !strcmp(model, "IBM,9076-N81")) | |
431 | enabled = 0; | |
432 | of_node_put(root); | |
433 | } | |
434 | ||
435 | return enabled; | |
436 | } | |
437 | #else | |
438 | static inline int tlb_batching_enabled(void) | |
439 | { | |
440 | return 1; | |
441 | } | |
442 | #endif | |
443 | ||
444 | void hpte_init_native(void) | |
445 | { | |
446 | ppc_md.hpte_invalidate = native_hpte_invalidate; | |
447 | ppc_md.hpte_updatepp = native_hpte_updatepp; | |
448 | ppc_md.hpte_updateboltedpp = native_hpte_updateboltedpp; | |
449 | ppc_md.hpte_insert = native_hpte_insert; | |
f4c82d51 S |
450 | ppc_md.hpte_remove = native_hpte_remove; |
451 | ppc_md.hpte_clear_all = native_hpte_clear; | |
1da177e4 LT |
452 | if (tlb_batching_enabled()) |
453 | ppc_md.flush_hash_range = native_flush_hash_range; | |
454 | htab_finish_init(); | |
455 | } |