]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * native hashtable management. | |
3 | * | |
4 | * SMP scalability work: | |
5 | * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or | |
8 | * modify it under the terms of the GNU General Public License | |
9 | * as published by the Free Software Foundation; either version | |
10 | * 2 of the License, or (at your option) any later version. | |
11 | */ | |
3c726f8d BH |
12 | |
13 | #undef DEBUG_LOW | |
14 | ||
1da177e4 LT |
15 | #include <linux/spinlock.h> |
16 | #include <linux/bitops.h> | |
17 | #include <linux/threads.h> | |
18 | #include <linux/smp.h> | |
19 | ||
20 | #include <asm/abs_addr.h> | |
21 | #include <asm/machdep.h> | |
22 | #include <asm/mmu.h> | |
23 | #include <asm/mmu_context.h> | |
24 | #include <asm/pgtable.h> | |
25 | #include <asm/tlbflush.h> | |
26 | #include <asm/tlb.h> | |
27 | #include <asm/cputable.h> | |
3c726f8d | 28 | #include <asm/udbg.h> |
71bf08b6 | 29 | #include <asm/kexec.h> |
3c726f8d BH |
30 | |
31 | #ifdef DEBUG_LOW | |
32 | #define DBG_LOW(fmt...) udbg_printf(fmt) | |
33 | #else | |
34 | #define DBG_LOW(fmt...) | |
35 | #endif | |
1da177e4 LT |
36 | |
37 | #define HPTE_LOCK_BIT 3 | |
38 | ||
39 | static DEFINE_SPINLOCK(native_tlbie_lock); | |
40 | ||
3c726f8d BH |
41 | static inline void __tlbie(unsigned long va, unsigned int psize) |
42 | { | |
43 | unsigned int penc; | |
44 | ||
45 | /* clear top 16 bits, non SLS segment */ | |
46 | va &= ~(0xffffULL << 48); | |
47 | ||
48 | switch (psize) { | |
49 | case MMU_PAGE_4K: | |
50 | va &= ~0xffful; | |
51 | asm volatile("tlbie %0,0" : : "r" (va) : "memory"); | |
52 | break; | |
53 | default: | |
54 | penc = mmu_psize_defs[psize].penc; | |
55 | va &= ~((1ul << mmu_psize_defs[psize].shift) - 1); | |
19242b24 | 56 | va |= penc << 12; |
3c726f8d BH |
57 | asm volatile("tlbie %0,1" : : "r" (va) : "memory"); |
58 | break; | |
59 | } | |
60 | } | |
61 | ||
62 | static inline void __tlbiel(unsigned long va, unsigned int psize) | |
63 | { | |
64 | unsigned int penc; | |
65 | ||
66 | /* clear top 16 bits, non SLS segment */ | |
67 | va &= ~(0xffffULL << 48); | |
68 | ||
69 | switch (psize) { | |
70 | case MMU_PAGE_4K: | |
71 | va &= ~0xffful; | |
72 | asm volatile(".long 0x7c000224 | (%0 << 11) | (0 << 21)" | |
73 | : : "r"(va) : "memory"); | |
74 | break; | |
75 | default: | |
76 | penc = mmu_psize_defs[psize].penc; | |
77 | va &= ~((1ul << mmu_psize_defs[psize].shift) - 1); | |
19242b24 | 78 | va |= penc << 12; |
3c726f8d BH |
79 | asm volatile(".long 0x7c000224 | (%0 << 11) | (1 << 21)" |
80 | : : "r"(va) : "memory"); | |
81 | break; | |
82 | } | |
83 | ||
84 | } | |
85 | ||
86 | static inline void tlbie(unsigned long va, int psize, int local) | |
87 | { | |
88 | unsigned int use_local = local && cpu_has_feature(CPU_FTR_TLBIEL); | |
89 | int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE); | |
90 | ||
91 | if (use_local) | |
92 | use_local = mmu_psize_defs[psize].tlbiel; | |
93 | if (lock_tlbie && !use_local) | |
94 | spin_lock(&native_tlbie_lock); | |
95 | asm volatile("ptesync": : :"memory"); | |
96 | if (use_local) { | |
97 | __tlbiel(va, psize); | |
98 | asm volatile("ptesync": : :"memory"); | |
99 | } else { | |
100 | __tlbie(va, psize); | |
101 | asm volatile("eieio; tlbsync; ptesync": : :"memory"); | |
102 | } | |
103 | if (lock_tlbie && !use_local) | |
104 | spin_unlock(&native_tlbie_lock); | |
105 | } | |
106 | ||
96e28449 | 107 | static inline void native_lock_hpte(hpte_t *hptep) |
1da177e4 | 108 | { |
96e28449 | 109 | unsigned long *word = &hptep->v; |
1da177e4 LT |
110 | |
111 | while (1) { | |
112 | if (!test_and_set_bit(HPTE_LOCK_BIT, word)) | |
113 | break; | |
114 | while(test_bit(HPTE_LOCK_BIT, word)) | |
115 | cpu_relax(); | |
116 | } | |
117 | } | |
118 | ||
96e28449 | 119 | static inline void native_unlock_hpte(hpte_t *hptep) |
1da177e4 | 120 | { |
96e28449 | 121 | unsigned long *word = &hptep->v; |
1da177e4 LT |
122 | |
123 | asm volatile("lwsync":::"memory"); | |
124 | clear_bit(HPTE_LOCK_BIT, word); | |
125 | } | |
126 | ||
035223fb | 127 | static long native_hpte_insert(unsigned long hpte_group, unsigned long va, |
3c726f8d BH |
128 | unsigned long pa, unsigned long rflags, |
129 | unsigned long vflags, int psize) | |
1da177e4 | 130 | { |
96e28449 DG |
131 | hpte_t *hptep = htab_address + hpte_group; |
132 | unsigned long hpte_v, hpte_r; | |
1da177e4 LT |
133 | int i; |
134 | ||
3c726f8d BH |
135 | if (!(vflags & HPTE_V_BOLTED)) { |
136 | DBG_LOW(" insert(group=%lx, va=%016lx, pa=%016lx," | |
137 | " rflags=%lx, vflags=%lx, psize=%d)\n", | |
138 | hpte_group, va, pa, rflags, vflags, psize); | |
139 | } | |
140 | ||
1da177e4 | 141 | for (i = 0; i < HPTES_PER_GROUP; i++) { |
96e28449 | 142 | if (! (hptep->v & HPTE_V_VALID)) { |
1da177e4 LT |
143 | /* retry with lock held */ |
144 | native_lock_hpte(hptep); | |
96e28449 | 145 | if (! (hptep->v & HPTE_V_VALID)) |
1da177e4 LT |
146 | break; |
147 | native_unlock_hpte(hptep); | |
148 | } | |
149 | ||
150 | hptep++; | |
151 | } | |
152 | ||
153 | if (i == HPTES_PER_GROUP) | |
154 | return -1; | |
155 | ||
3c726f8d BH |
156 | hpte_v = hpte_encode_v(va, psize) | vflags | HPTE_V_VALID; |
157 | hpte_r = hpte_encode_r(pa, psize) | rflags; | |
158 | ||
159 | if (!(vflags & HPTE_V_BOLTED)) { | |
160 | DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n", | |
161 | i, hpte_v, hpte_r); | |
162 | } | |
1da177e4 | 163 | |
96e28449 | 164 | hptep->r = hpte_r; |
1da177e4 LT |
165 | /* Guarantee the second dword is visible before the valid bit */ |
166 | __asm__ __volatile__ ("eieio" : : : "memory"); | |
1da177e4 LT |
167 | /* |
168 | * Now set the first dword including the valid bit | |
169 | * NOTE: this also unlocks the hpte | |
170 | */ | |
96e28449 | 171 | hptep->v = hpte_v; |
1da177e4 LT |
172 | |
173 | __asm__ __volatile__ ("ptesync" : : : "memory"); | |
174 | ||
96e28449 | 175 | return i | (!!(vflags & HPTE_V_SECONDARY) << 3); |
1da177e4 LT |
176 | } |
177 | ||
178 | static long native_hpte_remove(unsigned long hpte_group) | |
179 | { | |
96e28449 | 180 | hpte_t *hptep; |
1da177e4 LT |
181 | int i; |
182 | int slot_offset; | |
96e28449 | 183 | unsigned long hpte_v; |
1da177e4 | 184 | |
3c726f8d BH |
185 | DBG_LOW(" remove(group=%lx)\n", hpte_group); |
186 | ||
1da177e4 LT |
187 | /* pick a random entry to start at */ |
188 | slot_offset = mftb() & 0x7; | |
189 | ||
190 | for (i = 0; i < HPTES_PER_GROUP; i++) { | |
191 | hptep = htab_address + hpte_group + slot_offset; | |
96e28449 | 192 | hpte_v = hptep->v; |
1da177e4 | 193 | |
96e28449 | 194 | if ((hpte_v & HPTE_V_VALID) && !(hpte_v & HPTE_V_BOLTED)) { |
1da177e4 LT |
195 | /* retry with lock held */ |
196 | native_lock_hpte(hptep); | |
96e28449 DG |
197 | hpte_v = hptep->v; |
198 | if ((hpte_v & HPTE_V_VALID) | |
199 | && !(hpte_v & HPTE_V_BOLTED)) | |
1da177e4 LT |
200 | break; |
201 | native_unlock_hpte(hptep); | |
202 | } | |
203 | ||
204 | slot_offset++; | |
205 | slot_offset &= 0x7; | |
206 | } | |
207 | ||
208 | if (i == HPTES_PER_GROUP) | |
209 | return -1; | |
210 | ||
211 | /* Invalidate the hpte. NOTE: this also unlocks it */ | |
96e28449 | 212 | hptep->v = 0; |
1da177e4 LT |
213 | |
214 | return i; | |
215 | } | |
216 | ||
3c726f8d BH |
217 | static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, |
218 | unsigned long va, int psize, int local) | |
1da177e4 | 219 | { |
3c726f8d BH |
220 | hpte_t *hptep = htab_address + slot; |
221 | unsigned long hpte_v, want_v; | |
222 | int ret = 0; | |
223 | ||
224 | want_v = hpte_encode_v(va, psize); | |
225 | ||
226 | DBG_LOW(" update(va=%016lx, avpnv=%016lx, hash=%016lx, newpp=%x)", | |
227 | va, want_v & HPTE_V_AVPN, slot, newpp); | |
228 | ||
229 | native_lock_hpte(hptep); | |
230 | ||
231 | hpte_v = hptep->v; | |
232 | ||
233 | /* Even if we miss, we need to invalidate the TLB */ | |
234 | if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) { | |
235 | DBG_LOW(" -> miss\n"); | |
236 | native_unlock_hpte(hptep); | |
237 | ret = -1; | |
238 | } else { | |
239 | DBG_LOW(" -> hit\n"); | |
240 | /* Update the HPTE */ | |
241 | hptep->r = (hptep->r & ~(HPTE_R_PP | HPTE_R_N)) | | |
c5cf0e30 | 242 | (newpp & (HPTE_R_PP | HPTE_R_N | HPTE_R_C)); |
3c726f8d BH |
243 | native_unlock_hpte(hptep); |
244 | } | |
245 | ||
246 | /* Ensure it is out of the tlb too. */ | |
247 | tlbie(va, psize, local); | |
248 | ||
249 | return ret; | |
1da177e4 LT |
250 | } |
251 | ||
3c726f8d | 252 | static long native_hpte_find(unsigned long va, int psize) |
1da177e4 | 253 | { |
96e28449 | 254 | hpte_t *hptep; |
1da177e4 LT |
255 | unsigned long hash; |
256 | unsigned long i, j; | |
257 | long slot; | |
3c726f8d | 258 | unsigned long want_v, hpte_v; |
1da177e4 | 259 | |
3c726f8d BH |
260 | hash = hpt_hash(va, mmu_psize_defs[psize].shift); |
261 | want_v = hpte_encode_v(va, psize); | |
1da177e4 LT |
262 | |
263 | for (j = 0; j < 2; j++) { | |
264 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | |
265 | for (i = 0; i < HPTES_PER_GROUP; i++) { | |
266 | hptep = htab_address + slot; | |
96e28449 | 267 | hpte_v = hptep->v; |
1da177e4 | 268 | |
3c726f8d | 269 | if (HPTE_V_COMPARE(hpte_v, want_v) |
96e28449 DG |
270 | && (hpte_v & HPTE_V_VALID) |
271 | && ( !!(hpte_v & HPTE_V_SECONDARY) == j)) { | |
1da177e4 LT |
272 | /* HPTE matches */ |
273 | if (j) | |
274 | slot = -slot; | |
275 | return slot; | |
276 | } | |
277 | ++slot; | |
278 | } | |
279 | hash = ~hash; | |
280 | } | |
281 | ||
282 | return -1; | |
283 | } | |
284 | ||
1da177e4 LT |
285 | /* |
286 | * Update the page protection bits. Intended to be used to create | |
287 | * guard pages for kernel data structures on pages which are bolted | |
288 | * in the HPT. Assumes pages being operated on will not be stolen. | |
1da177e4 LT |
289 | * |
290 | * No need to lock here because we should be the only user. | |
291 | */ | |
3c726f8d BH |
292 | static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea, |
293 | int psize) | |
1da177e4 | 294 | { |
3c726f8d | 295 | unsigned long vsid, va; |
1da177e4 | 296 | long slot; |
96e28449 | 297 | hpte_t *hptep; |
1da177e4 LT |
298 | |
299 | vsid = get_kernel_vsid(ea); | |
300 | va = (vsid << 28) | (ea & 0x0fffffff); | |
1da177e4 | 301 | |
3c726f8d | 302 | slot = native_hpte_find(va, psize); |
1da177e4 LT |
303 | if (slot == -1) |
304 | panic("could not find page to bolt\n"); | |
305 | hptep = htab_address + slot; | |
306 | ||
3c726f8d BH |
307 | /* Update the HPTE */ |
308 | hptep->r = (hptep->r & ~(HPTE_R_PP | HPTE_R_N)) | | |
309 | (newpp & (HPTE_R_PP | HPTE_R_N)); | |
1da177e4 | 310 | |
3c726f8d BH |
311 | /* Ensure it is out of the tlb too. */ |
312 | tlbie(va, psize, 0); | |
1da177e4 LT |
313 | } |
314 | ||
315 | static void native_hpte_invalidate(unsigned long slot, unsigned long va, | |
3c726f8d | 316 | int psize, int local) |
1da177e4 | 317 | { |
96e28449 DG |
318 | hpte_t *hptep = htab_address + slot; |
319 | unsigned long hpte_v; | |
3c726f8d | 320 | unsigned long want_v; |
1da177e4 | 321 | unsigned long flags; |
1da177e4 LT |
322 | |
323 | local_irq_save(flags); | |
1da177e4 | 324 | |
3c726f8d BH |
325 | DBG_LOW(" invalidate(va=%016lx, hash: %x)\n", va, slot); |
326 | ||
327 | want_v = hpte_encode_v(va, psize); | |
328 | native_lock_hpte(hptep); | |
96e28449 | 329 | hpte_v = hptep->v; |
1da177e4 LT |
330 | |
331 | /* Even if we miss, we need to invalidate the TLB */ | |
3c726f8d | 332 | if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) |
1da177e4 | 333 | native_unlock_hpte(hptep); |
3c726f8d | 334 | else |
1da177e4 | 335 | /* Invalidate the hpte. NOTE: this also unlocks it */ |
96e28449 | 336 | hptep->v = 0; |
1da177e4 | 337 | |
3c726f8d BH |
338 | /* Invalidate the TLB */ |
339 | tlbie(va, psize, local); | |
340 | ||
1da177e4 LT |
341 | local_irq_restore(flags); |
342 | } | |
343 | ||
71bf08b6 LB |
344 | #define LP_SHIFT 12 |
345 | #define LP_BITS 8 | |
346 | #define LP_MASK(i) ((0xFF >> (i)) << LP_SHIFT) | |
3c726f8d | 347 | |
71bf08b6 LB |
348 | static void hpte_decode(hpte_t *hpte, unsigned long slot, |
349 | int *psize, unsigned long *va) | |
350 | { | |
351 | unsigned long hpte_r = hpte->r; | |
352 | unsigned long hpte_v = hpte->v; | |
353 | unsigned long avpn; | |
354 | int i, size, shift, penc, avpnm_bits; | |
355 | ||
356 | if (!(hpte_v & HPTE_V_LARGE)) | |
357 | size = MMU_PAGE_4K; | |
358 | else { | |
359 | for (i = 0; i < LP_BITS; i++) { | |
360 | if ((hpte_r & LP_MASK(i+1)) == LP_MASK(i+1)) | |
361 | break; | |
362 | } | |
363 | penc = LP_MASK(i+1) >> LP_SHIFT; | |
364 | for (size = 0; size < MMU_PAGE_COUNT; size++) { | |
3c726f8d | 365 | |
71bf08b6 LB |
366 | /* 4K pages are not represented by LP */ |
367 | if (size == MMU_PAGE_4K) | |
368 | continue; | |
3c726f8d | 369 | |
71bf08b6 LB |
370 | /* valid entries have a shift value */ |
371 | if (!mmu_psize_defs[size].shift) | |
372 | continue; | |
3c726f8d | 373 | |
71bf08b6 LB |
374 | if (penc == mmu_psize_defs[size].penc) |
375 | break; | |
376 | } | |
377 | } | |
3c726f8d | 378 | |
71bf08b6 LB |
379 | /* |
380 | * FIXME, the code below works for 16M, 64K, and 4K pages as these | |
381 | * fall under the p<=23 rules for calculating the virtual address. | |
382 | * In the case of 16M pages, an extra bit is stolen from the AVPN | |
383 | * field to achieve the requisite 24 bits. | |
384 | * | |
385 | * Does not work for 16G pages or 1 TB segments. | |
386 | */ | |
387 | shift = mmu_psize_defs[size].shift; | |
388 | if (mmu_psize_defs[size].avpnm) | |
389 | avpnm_bits = __ilog2_u64(mmu_psize_defs[size].avpnm) + 1; | |
390 | else | |
391 | avpnm_bits = 0; | |
392 | if (shift - avpnm_bits <= 23) { | |
393 | avpn = HPTE_V_AVPN_VAL(hpte_v) << 23; | |
394 | ||
395 | if (shift < 23) { | |
396 | unsigned long vpi, pteg; | |
397 | ||
398 | pteg = slot / HPTES_PER_GROUP; | |
399 | if (hpte_v & HPTE_V_SECONDARY) | |
400 | pteg = ~pteg; | |
401 | vpi = ((avpn >> 28) ^ pteg) & htab_hash_mask; | |
402 | avpn |= (vpi << mmu_psize_defs[size].shift); | |
403 | } | |
3c726f8d BH |
404 | } |
405 | ||
71bf08b6 LB |
406 | *va = avpn; |
407 | *psize = size; | |
3c726f8d BH |
408 | } |
409 | ||
f4c82d51 S |
410 | /* |
411 | * clear all mappings on kexec. All cpus are in real mode (or they will | |
412 | * be when they isi), and we are the only one left. We rely on our kernel | |
413 | * mapping being 0xC0's and the hardware ignoring those two real bits. | |
414 | * | |
415 | * TODO: add batching support when enabled. remember, no dynamic memory here, | |
416 | * athough there is the control page available... | |
417 | */ | |
418 | static void native_hpte_clear(void) | |
419 | { | |
420 | unsigned long slot, slots, flags; | |
96e28449 | 421 | hpte_t *hptep = htab_address; |
71bf08b6 | 422 | unsigned long hpte_v, va; |
f4c82d51 | 423 | unsigned long pteg_count; |
71bf08b6 | 424 | int psize; |
f4c82d51 S |
425 | |
426 | pteg_count = htab_hash_mask + 1; | |
427 | ||
428 | local_irq_save(flags); | |
429 | ||
430 | /* we take the tlbie lock and hold it. Some hardware will | |
431 | * deadlock if we try to tlbie from two processors at once. | |
432 | */ | |
433 | spin_lock(&native_tlbie_lock); | |
434 | ||
435 | slots = pteg_count * HPTES_PER_GROUP; | |
436 | ||
437 | for (slot = 0; slot < slots; slot++, hptep++) { | |
438 | /* | |
439 | * we could lock the pte here, but we are the only cpu | |
440 | * running, right? and for crash dump, we probably | |
441 | * don't want to wait for a maybe bad cpu. | |
442 | */ | |
96e28449 | 443 | hpte_v = hptep->v; |
f4c82d51 | 444 | |
47f78a49 S |
445 | /* |
446 | * Call __tlbie() here rather than tlbie() since we | |
447 | * already hold the native_tlbie_lock. | |
448 | */ | |
96e28449 | 449 | if (hpte_v & HPTE_V_VALID) { |
71bf08b6 | 450 | hpte_decode(hptep, slot, &psize, &va); |
96e28449 | 451 | hptep->v = 0; |
71bf08b6 | 452 | __tlbie(va, psize); |
f4c82d51 S |
453 | } |
454 | } | |
455 | ||
47f78a49 | 456 | asm volatile("eieio; tlbsync; ptesync":::"memory"); |
f4c82d51 S |
457 | spin_unlock(&native_tlbie_lock); |
458 | local_irq_restore(flags); | |
459 | } | |
460 | ||
3c726f8d BH |
461 | /* |
462 | * Batched hash table flush, we batch the tlbie's to avoid taking/releasing | |
463 | * the lock all the time | |
464 | */ | |
61b1a942 | 465 | static void native_flush_hash_range(unsigned long number, int local) |
1da177e4 | 466 | { |
3c726f8d | 467 | unsigned long va, hash, index, hidx, shift, slot; |
96e28449 DG |
468 | hpte_t *hptep; |
469 | unsigned long hpte_v; | |
3c726f8d BH |
470 | unsigned long want_v; |
471 | unsigned long flags; | |
472 | real_pte_t pte; | |
1da177e4 | 473 | struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch); |
3c726f8d BH |
474 | unsigned long psize = batch->psize; |
475 | int i; | |
1da177e4 LT |
476 | |
477 | local_irq_save(flags); | |
478 | ||
1da177e4 | 479 | for (i = 0; i < number; i++) { |
3c726f8d BH |
480 | va = batch->vaddr[i]; |
481 | pte = batch->pte[i]; | |
482 | ||
483 | pte_iterate_hashed_subpages(pte, psize, va, index, shift) { | |
484 | hash = hpt_hash(va, shift); | |
485 | hidx = __rpte_to_hidx(pte, index); | |
486 | if (hidx & _PTEIDX_SECONDARY) | |
487 | hash = ~hash; | |
488 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | |
489 | slot += hidx & _PTEIDX_GROUP_IX; | |
490 | hptep = htab_address + slot; | |
491 | want_v = hpte_encode_v(va, psize); | |
492 | native_lock_hpte(hptep); | |
493 | hpte_v = hptep->v; | |
494 | if (!HPTE_V_COMPARE(hpte_v, want_v) || | |
495 | !(hpte_v & HPTE_V_VALID)) | |
496 | native_unlock_hpte(hptep); | |
497 | else | |
498 | hptep->v = 0; | |
499 | } pte_iterate_hashed_end(); | |
1da177e4 LT |
500 | } |
501 | ||
3c726f8d BH |
502 | if (cpu_has_feature(CPU_FTR_TLBIEL) && |
503 | mmu_psize_defs[psize].tlbiel && local) { | |
1da177e4 | 504 | asm volatile("ptesync":::"memory"); |
3c726f8d BH |
505 | for (i = 0; i < number; i++) { |
506 | va = batch->vaddr[i]; | |
507 | pte = batch->pte[i]; | |
508 | ||
509 | pte_iterate_hashed_subpages(pte, psize, va, index, | |
510 | shift) { | |
511 | __tlbiel(va, psize); | |
512 | } pte_iterate_hashed_end(); | |
513 | } | |
1da177e4 LT |
514 | asm volatile("ptesync":::"memory"); |
515 | } else { | |
516 | int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE); | |
517 | ||
518 | if (lock_tlbie) | |
519 | spin_lock(&native_tlbie_lock); | |
520 | ||
521 | asm volatile("ptesync":::"memory"); | |
3c726f8d BH |
522 | for (i = 0; i < number; i++) { |
523 | va = batch->vaddr[i]; | |
524 | pte = batch->pte[i]; | |
525 | ||
526 | pte_iterate_hashed_subpages(pte, psize, va, index, | |
527 | shift) { | |
528 | __tlbie(va, psize); | |
529 | } pte_iterate_hashed_end(); | |
530 | } | |
1da177e4 LT |
531 | asm volatile("eieio; tlbsync; ptesync":::"memory"); |
532 | ||
533 | if (lock_tlbie) | |
534 | spin_unlock(&native_tlbie_lock); | |
535 | } | |
536 | ||
537 | local_irq_restore(flags); | |
538 | } | |
539 | ||
540 | #ifdef CONFIG_PPC_PSERIES | |
541 | /* Disable TLB batching on nighthawk */ | |
542 | static inline int tlb_batching_enabled(void) | |
543 | { | |
544 | struct device_node *root = of_find_node_by_path("/"); | |
545 | int enabled = 1; | |
546 | ||
547 | if (root) { | |
e2eb6392 | 548 | const char *model = of_get_property(root, "model", NULL); |
1da177e4 LT |
549 | if (model && !strcmp(model, "IBM,9076-N81")) |
550 | enabled = 0; | |
551 | of_node_put(root); | |
552 | } | |
553 | ||
554 | return enabled; | |
555 | } | |
556 | #else | |
557 | static inline int tlb_batching_enabled(void) | |
558 | { | |
559 | return 1; | |
560 | } | |
561 | #endif | |
562 | ||
7d0daae4 | 563 | void __init hpte_init_native(void) |
1da177e4 LT |
564 | { |
565 | ppc_md.hpte_invalidate = native_hpte_invalidate; | |
566 | ppc_md.hpte_updatepp = native_hpte_updatepp; | |
567 | ppc_md.hpte_updateboltedpp = native_hpte_updateboltedpp; | |
568 | ppc_md.hpte_insert = native_hpte_insert; | |
f4c82d51 S |
569 | ppc_md.hpte_remove = native_hpte_remove; |
570 | ppc_md.hpte_clear_all = native_hpte_clear; | |
1da177e4 LT |
571 | if (tlb_batching_enabled()) |
572 | ppc_md.flush_hash_range = native_flush_hash_range; | |
1da177e4 | 573 | } |