]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * native hashtable management. | |
3 | * | |
4 | * SMP scalability work: | |
5 | * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or | |
8 | * modify it under the terms of the GNU General Public License | |
9 | * as published by the Free Software Foundation; either version | |
10 | * 2 of the License, or (at your option) any later version. | |
11 | */ | |
3c726f8d BH |
12 | |
13 | #undef DEBUG_LOW | |
14 | ||
1da177e4 LT |
15 | #include <linux/spinlock.h> |
16 | #include <linux/bitops.h> | |
17 | #include <linux/threads.h> | |
18 | #include <linux/smp.h> | |
19 | ||
20 | #include <asm/abs_addr.h> | |
21 | #include <asm/machdep.h> | |
22 | #include <asm/mmu.h> | |
23 | #include <asm/mmu_context.h> | |
24 | #include <asm/pgtable.h> | |
25 | #include <asm/tlbflush.h> | |
26 | #include <asm/tlb.h> | |
27 | #include <asm/cputable.h> | |
3c726f8d | 28 | #include <asm/udbg.h> |
71bf08b6 | 29 | #include <asm/kexec.h> |
60dbf438 | 30 | #include <asm/ppc-opcode.h> |
3c726f8d BH |
31 | |
32 | #ifdef DEBUG_LOW | |
33 | #define DBG_LOW(fmt...) udbg_printf(fmt) | |
34 | #else | |
35 | #define DBG_LOW(fmt...) | |
36 | #endif | |
1da177e4 LT |
37 | |
38 | #define HPTE_LOCK_BIT 3 | |
39 | ||
6b9c9b8a | 40 | static DEFINE_RAW_SPINLOCK(native_tlbie_lock); |
1da177e4 | 41 | |
1189be65 | 42 | static inline void __tlbie(unsigned long va, int psize, int ssize) |
3c726f8d BH |
43 | { |
44 | unsigned int penc; | |
45 | ||
46 | /* clear top 16 bits, non SLS segment */ | |
47 | va &= ~(0xffffULL << 48); | |
48 | ||
49 | switch (psize) { | |
50 | case MMU_PAGE_4K: | |
51 | va &= ~0xffful; | |
1189be65 | 52 | va |= ssize << 8; |
a32e252f | 53 | asm volatile(ASM_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0), %2) |
969391c5 | 54 | : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206) |
60dbf438 | 55 | : "memory"); |
3c726f8d BH |
56 | break; |
57 | default: | |
58 | penc = mmu_psize_defs[psize].penc; | |
59 | va &= ~((1ul << mmu_psize_defs[psize].shift) - 1); | |
19242b24 | 60 | va |= penc << 12; |
1189be65 | 61 | va |= ssize << 8; |
60dbf438 | 62 | va |= 1; /* L */ |
a32e252f | 63 | asm volatile(ASM_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0), %2) |
969391c5 | 64 | : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206) |
60dbf438 | 65 | : "memory"); |
3c726f8d BH |
66 | break; |
67 | } | |
68 | } | |
69 | ||
1189be65 | 70 | static inline void __tlbiel(unsigned long va, int psize, int ssize) |
3c726f8d BH |
71 | { |
72 | unsigned int penc; | |
73 | ||
74 | /* clear top 16 bits, non SLS segment */ | |
75 | va &= ~(0xffffULL << 48); | |
76 | ||
77 | switch (psize) { | |
78 | case MMU_PAGE_4K: | |
79 | va &= ~0xffful; | |
1189be65 | 80 | va |= ssize << 8; |
3c726f8d BH |
81 | asm volatile(".long 0x7c000224 | (%0 << 11) | (0 << 21)" |
82 | : : "r"(va) : "memory"); | |
83 | break; | |
84 | default: | |
85 | penc = mmu_psize_defs[psize].penc; | |
86 | va &= ~((1ul << mmu_psize_defs[psize].shift) - 1); | |
19242b24 | 87 | va |= penc << 12; |
1189be65 | 88 | va |= ssize << 8; |
60dbf438 | 89 | va |= 1; /* L */ |
3c726f8d BH |
90 | asm volatile(".long 0x7c000224 | (%0 << 11) | (1 << 21)" |
91 | : : "r"(va) : "memory"); | |
92 | break; | |
93 | } | |
94 | ||
95 | } | |
96 | ||
1189be65 | 97 | static inline void tlbie(unsigned long va, int psize, int ssize, int local) |
3c726f8d | 98 | { |
44ae3ab3 ME |
99 | unsigned int use_local = local && mmu_has_feature(MMU_FTR_TLBIEL); |
100 | int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); | |
3c726f8d BH |
101 | |
102 | if (use_local) | |
103 | use_local = mmu_psize_defs[psize].tlbiel; | |
104 | if (lock_tlbie && !use_local) | |
6b9c9b8a | 105 | raw_spin_lock(&native_tlbie_lock); |
3c726f8d BH |
106 | asm volatile("ptesync": : :"memory"); |
107 | if (use_local) { | |
1189be65 | 108 | __tlbiel(va, psize, ssize); |
3c726f8d BH |
109 | asm volatile("ptesync": : :"memory"); |
110 | } else { | |
1189be65 | 111 | __tlbie(va, psize, ssize); |
3c726f8d BH |
112 | asm volatile("eieio; tlbsync; ptesync": : :"memory"); |
113 | } | |
114 | if (lock_tlbie && !use_local) | |
6b9c9b8a | 115 | raw_spin_unlock(&native_tlbie_lock); |
3c726f8d BH |
116 | } |
117 | ||
8e561e7e | 118 | static inline void native_lock_hpte(struct hash_pte *hptep) |
1da177e4 | 119 | { |
96e28449 | 120 | unsigned long *word = &hptep->v; |
1da177e4 LT |
121 | |
122 | while (1) { | |
66d99b88 | 123 | if (!test_and_set_bit_lock(HPTE_LOCK_BIT, word)) |
1da177e4 LT |
124 | break; |
125 | while(test_bit(HPTE_LOCK_BIT, word)) | |
126 | cpu_relax(); | |
127 | } | |
128 | } | |
129 | ||
8e561e7e | 130 | static inline void native_unlock_hpte(struct hash_pte *hptep) |
1da177e4 | 131 | { |
96e28449 | 132 | unsigned long *word = &hptep->v; |
1da177e4 | 133 | |
66d99b88 | 134 | clear_bit_unlock(HPTE_LOCK_BIT, word); |
1da177e4 LT |
135 | } |
136 | ||
035223fb | 137 | static long native_hpte_insert(unsigned long hpte_group, unsigned long va, |
3c726f8d | 138 | unsigned long pa, unsigned long rflags, |
1189be65 | 139 | unsigned long vflags, int psize, int ssize) |
1da177e4 | 140 | { |
8e561e7e | 141 | struct hash_pte *hptep = htab_address + hpte_group; |
96e28449 | 142 | unsigned long hpte_v, hpte_r; |
1da177e4 LT |
143 | int i; |
144 | ||
3c726f8d BH |
145 | if (!(vflags & HPTE_V_BOLTED)) { |
146 | DBG_LOW(" insert(group=%lx, va=%016lx, pa=%016lx," | |
147 | " rflags=%lx, vflags=%lx, psize=%d)\n", | |
148 | hpte_group, va, pa, rflags, vflags, psize); | |
149 | } | |
150 | ||
1da177e4 | 151 | for (i = 0; i < HPTES_PER_GROUP; i++) { |
96e28449 | 152 | if (! (hptep->v & HPTE_V_VALID)) { |
1da177e4 LT |
153 | /* retry with lock held */ |
154 | native_lock_hpte(hptep); | |
96e28449 | 155 | if (! (hptep->v & HPTE_V_VALID)) |
1da177e4 LT |
156 | break; |
157 | native_unlock_hpte(hptep); | |
158 | } | |
159 | ||
160 | hptep++; | |
161 | } | |
162 | ||
163 | if (i == HPTES_PER_GROUP) | |
164 | return -1; | |
165 | ||
1189be65 | 166 | hpte_v = hpte_encode_v(va, psize, ssize) | vflags | HPTE_V_VALID; |
3c726f8d BH |
167 | hpte_r = hpte_encode_r(pa, psize) | rflags; |
168 | ||
169 | if (!(vflags & HPTE_V_BOLTED)) { | |
170 | DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n", | |
171 | i, hpte_v, hpte_r); | |
172 | } | |
1da177e4 | 173 | |
96e28449 | 174 | hptep->r = hpte_r; |
1da177e4 | 175 | /* Guarantee the second dword is visible before the valid bit */ |
74a0ba61 | 176 | eieio(); |
1da177e4 LT |
177 | /* |
178 | * Now set the first dword including the valid bit | |
179 | * NOTE: this also unlocks the hpte | |
180 | */ | |
96e28449 | 181 | hptep->v = hpte_v; |
1da177e4 LT |
182 | |
183 | __asm__ __volatile__ ("ptesync" : : : "memory"); | |
184 | ||
96e28449 | 185 | return i | (!!(vflags & HPTE_V_SECONDARY) << 3); |
1da177e4 LT |
186 | } |
187 | ||
188 | static long native_hpte_remove(unsigned long hpte_group) | |
189 | { | |
8e561e7e | 190 | struct hash_pte *hptep; |
1da177e4 LT |
191 | int i; |
192 | int slot_offset; | |
96e28449 | 193 | unsigned long hpte_v; |
1da177e4 | 194 | |
3c726f8d BH |
195 | DBG_LOW(" remove(group=%lx)\n", hpte_group); |
196 | ||
1da177e4 LT |
197 | /* pick a random entry to start at */ |
198 | slot_offset = mftb() & 0x7; | |
199 | ||
200 | for (i = 0; i < HPTES_PER_GROUP; i++) { | |
201 | hptep = htab_address + hpte_group + slot_offset; | |
96e28449 | 202 | hpte_v = hptep->v; |
1da177e4 | 203 | |
96e28449 | 204 | if ((hpte_v & HPTE_V_VALID) && !(hpte_v & HPTE_V_BOLTED)) { |
1da177e4 LT |
205 | /* retry with lock held */ |
206 | native_lock_hpte(hptep); | |
96e28449 DG |
207 | hpte_v = hptep->v; |
208 | if ((hpte_v & HPTE_V_VALID) | |
209 | && !(hpte_v & HPTE_V_BOLTED)) | |
1da177e4 LT |
210 | break; |
211 | native_unlock_hpte(hptep); | |
212 | } | |
213 | ||
214 | slot_offset++; | |
215 | slot_offset &= 0x7; | |
216 | } | |
217 | ||
218 | if (i == HPTES_PER_GROUP) | |
219 | return -1; | |
220 | ||
221 | /* Invalidate the hpte. NOTE: this also unlocks it */ | |
96e28449 | 222 | hptep->v = 0; |
1da177e4 LT |
223 | |
224 | return i; | |
225 | } | |
226 | ||
3c726f8d | 227 | static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, |
1189be65 PM |
228 | unsigned long va, int psize, int ssize, |
229 | int local) | |
1da177e4 | 230 | { |
8e561e7e | 231 | struct hash_pte *hptep = htab_address + slot; |
3c726f8d BH |
232 | unsigned long hpte_v, want_v; |
233 | int ret = 0; | |
234 | ||
1189be65 | 235 | want_v = hpte_encode_v(va, psize, ssize); |
3c726f8d BH |
236 | |
237 | DBG_LOW(" update(va=%016lx, avpnv=%016lx, hash=%016lx, newpp=%x)", | |
238 | va, want_v & HPTE_V_AVPN, slot, newpp); | |
239 | ||
240 | native_lock_hpte(hptep); | |
241 | ||
242 | hpte_v = hptep->v; | |
243 | ||
244 | /* Even if we miss, we need to invalidate the TLB */ | |
245 | if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) { | |
246 | DBG_LOW(" -> miss\n"); | |
3c726f8d BH |
247 | ret = -1; |
248 | } else { | |
249 | DBG_LOW(" -> hit\n"); | |
250 | /* Update the HPTE */ | |
251 | hptep->r = (hptep->r & ~(HPTE_R_PP | HPTE_R_N)) | | |
c5cf0e30 | 252 | (newpp & (HPTE_R_PP | HPTE_R_N | HPTE_R_C)); |
3c726f8d | 253 | } |
3f1df7a2 | 254 | native_unlock_hpte(hptep); |
3c726f8d BH |
255 | |
256 | /* Ensure it is out of the tlb too. */ | |
1189be65 | 257 | tlbie(va, psize, ssize, local); |
3c726f8d BH |
258 | |
259 | return ret; | |
1da177e4 LT |
260 | } |
261 | ||
1189be65 | 262 | static long native_hpte_find(unsigned long va, int psize, int ssize) |
1da177e4 | 263 | { |
8e561e7e | 264 | struct hash_pte *hptep; |
1da177e4 | 265 | unsigned long hash; |
1189be65 | 266 | unsigned long i; |
1da177e4 | 267 | long slot; |
3c726f8d | 268 | unsigned long want_v, hpte_v; |
1da177e4 | 269 | |
1189be65 PM |
270 | hash = hpt_hash(va, mmu_psize_defs[psize].shift, ssize); |
271 | want_v = hpte_encode_v(va, psize, ssize); | |
1da177e4 | 272 | |
1189be65 PM |
273 | /* Bolted mappings are only ever in the primary group */ |
274 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | |
275 | for (i = 0; i < HPTES_PER_GROUP; i++) { | |
276 | hptep = htab_address + slot; | |
277 | hpte_v = hptep->v; | |
1da177e4 | 278 | |
1189be65 PM |
279 | if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) |
280 | /* HPTE matches */ | |
281 | return slot; | |
282 | ++slot; | |
1da177e4 LT |
283 | } |
284 | ||
285 | return -1; | |
286 | } | |
287 | ||
1da177e4 LT |
288 | /* |
289 | * Update the page protection bits. Intended to be used to create | |
290 | * guard pages for kernel data structures on pages which are bolted | |
291 | * in the HPT. Assumes pages being operated on will not be stolen. | |
1da177e4 LT |
292 | * |
293 | * No need to lock here because we should be the only user. | |
294 | */ | |
3c726f8d | 295 | static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea, |
1189be65 | 296 | int psize, int ssize) |
1da177e4 | 297 | { |
3c726f8d | 298 | unsigned long vsid, va; |
1da177e4 | 299 | long slot; |
8e561e7e | 300 | struct hash_pte *hptep; |
1da177e4 | 301 | |
1189be65 PM |
302 | vsid = get_kernel_vsid(ea, ssize); |
303 | va = hpt_va(ea, vsid, ssize); | |
1da177e4 | 304 | |
1189be65 | 305 | slot = native_hpte_find(va, psize, ssize); |
1da177e4 LT |
306 | if (slot == -1) |
307 | panic("could not find page to bolt\n"); | |
308 | hptep = htab_address + slot; | |
309 | ||
3c726f8d BH |
310 | /* Update the HPTE */ |
311 | hptep->r = (hptep->r & ~(HPTE_R_PP | HPTE_R_N)) | | |
312 | (newpp & (HPTE_R_PP | HPTE_R_N)); | |
1da177e4 | 313 | |
3c726f8d | 314 | /* Ensure it is out of the tlb too. */ |
1189be65 | 315 | tlbie(va, psize, ssize, 0); |
1da177e4 LT |
316 | } |
317 | ||
318 | static void native_hpte_invalidate(unsigned long slot, unsigned long va, | |
1189be65 | 319 | int psize, int ssize, int local) |
1da177e4 | 320 | { |
8e561e7e | 321 | struct hash_pte *hptep = htab_address + slot; |
96e28449 | 322 | unsigned long hpte_v; |
3c726f8d | 323 | unsigned long want_v; |
1da177e4 | 324 | unsigned long flags; |
1da177e4 LT |
325 | |
326 | local_irq_save(flags); | |
1da177e4 | 327 | |
3c726f8d BH |
328 | DBG_LOW(" invalidate(va=%016lx, hash: %x)\n", va, slot); |
329 | ||
1189be65 | 330 | want_v = hpte_encode_v(va, psize, ssize); |
3c726f8d | 331 | native_lock_hpte(hptep); |
96e28449 | 332 | hpte_v = hptep->v; |
1da177e4 LT |
333 | |
334 | /* Even if we miss, we need to invalidate the TLB */ | |
3c726f8d | 335 | if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) |
1da177e4 | 336 | native_unlock_hpte(hptep); |
3c726f8d | 337 | else |
1da177e4 | 338 | /* Invalidate the hpte. NOTE: this also unlocks it */ |
96e28449 | 339 | hptep->v = 0; |
1da177e4 | 340 | |
3c726f8d | 341 | /* Invalidate the TLB */ |
1189be65 | 342 | tlbie(va, psize, ssize, local); |
3c726f8d | 343 | |
1da177e4 LT |
344 | local_irq_restore(flags); |
345 | } | |
346 | ||
71bf08b6 LB |
347 | #define LP_SHIFT 12 |
348 | #define LP_BITS 8 | |
349 | #define LP_MASK(i) ((0xFF >> (i)) << LP_SHIFT) | |
3c726f8d | 350 | |
8e561e7e | 351 | static void hpte_decode(struct hash_pte *hpte, unsigned long slot, |
1189be65 | 352 | int *psize, int *ssize, unsigned long *va) |
71bf08b6 LB |
353 | { |
354 | unsigned long hpte_r = hpte->r; | |
355 | unsigned long hpte_v = hpte->v; | |
356 | unsigned long avpn; | |
8980ae86 | 357 | int i, size, shift, penc; |
71bf08b6 LB |
358 | |
359 | if (!(hpte_v & HPTE_V_LARGE)) | |
360 | size = MMU_PAGE_4K; | |
361 | else { | |
362 | for (i = 0; i < LP_BITS; i++) { | |
363 | if ((hpte_r & LP_MASK(i+1)) == LP_MASK(i+1)) | |
364 | break; | |
365 | } | |
366 | penc = LP_MASK(i+1) >> LP_SHIFT; | |
367 | for (size = 0; size < MMU_PAGE_COUNT; size++) { | |
3c726f8d | 368 | |
71bf08b6 LB |
369 | /* 4K pages are not represented by LP */ |
370 | if (size == MMU_PAGE_4K) | |
371 | continue; | |
3c726f8d | 372 | |
71bf08b6 LB |
373 | /* valid entries have a shift value */ |
374 | if (!mmu_psize_defs[size].shift) | |
375 | continue; | |
3c726f8d | 376 | |
71bf08b6 LB |
377 | if (penc == mmu_psize_defs[size].penc) |
378 | break; | |
379 | } | |
380 | } | |
3c726f8d | 381 | |
2454c7e9 | 382 | /* This works for all page sizes, and for 256M and 1T segments */ |
71bf08b6 | 383 | shift = mmu_psize_defs[size].shift; |
2454c7e9 | 384 | avpn = (HPTE_V_AVPN_VAL(hpte_v) & ~mmu_psize_defs[size].avpnm) << 23; |
71bf08b6 | 385 | |
2454c7e9 PM |
386 | if (shift < 23) { |
387 | unsigned long vpi, vsid, pteg; | |
71bf08b6 | 388 | |
2454c7e9 PM |
389 | pteg = slot / HPTES_PER_GROUP; |
390 | if (hpte_v & HPTE_V_SECONDARY) | |
391 | pteg = ~pteg; | |
392 | switch (hpte_v >> HPTE_V_SSIZE_SHIFT) { | |
393 | case MMU_SEGSIZE_256M: | |
71bf08b6 | 394 | vpi = ((avpn >> 28) ^ pteg) & htab_hash_mask; |
2454c7e9 PM |
395 | break; |
396 | case MMU_SEGSIZE_1T: | |
397 | vsid = avpn >> 40; | |
398 | vpi = (vsid ^ (vsid << 25) ^ pteg) & htab_hash_mask; | |
399 | break; | |
400 | default: | |
0c12fe56 | 401 | avpn = vpi = size = 0; |
71bf08b6 | 402 | } |
2454c7e9 | 403 | avpn |= (vpi << mmu_psize_defs[size].shift); |
3c726f8d BH |
404 | } |
405 | ||
71bf08b6 LB |
406 | *va = avpn; |
407 | *psize = size; | |
1189be65 | 408 | *ssize = hpte_v >> HPTE_V_SSIZE_SHIFT; |
3c726f8d BH |
409 | } |
410 | ||
f4c82d51 S |
411 | /* |
412 | * clear all mappings on kexec. All cpus are in real mode (or they will | |
413 | * be when they isi), and we are the only one left. We rely on our kernel | |
414 | * mapping being 0xC0's and the hardware ignoring those two real bits. | |
415 | * | |
416 | * TODO: add batching support when enabled. remember, no dynamic memory here, | |
417 | * athough there is the control page available... | |
418 | */ | |
419 | static void native_hpte_clear(void) | |
420 | { | |
421 | unsigned long slot, slots, flags; | |
8e561e7e | 422 | struct hash_pte *hptep = htab_address; |
71bf08b6 | 423 | unsigned long hpte_v, va; |
f4c82d51 | 424 | unsigned long pteg_count; |
1189be65 | 425 | int psize, ssize; |
f4c82d51 S |
426 | |
427 | pteg_count = htab_hash_mask + 1; | |
428 | ||
429 | local_irq_save(flags); | |
430 | ||
431 | /* we take the tlbie lock and hold it. Some hardware will | |
432 | * deadlock if we try to tlbie from two processors at once. | |
433 | */ | |
6b9c9b8a | 434 | raw_spin_lock(&native_tlbie_lock); |
f4c82d51 S |
435 | |
436 | slots = pteg_count * HPTES_PER_GROUP; | |
437 | ||
438 | for (slot = 0; slot < slots; slot++, hptep++) { | |
439 | /* | |
440 | * we could lock the pte here, but we are the only cpu | |
441 | * running, right? and for crash dump, we probably | |
442 | * don't want to wait for a maybe bad cpu. | |
443 | */ | |
96e28449 | 444 | hpte_v = hptep->v; |
f4c82d51 | 445 | |
47f78a49 S |
446 | /* |
447 | * Call __tlbie() here rather than tlbie() since we | |
448 | * already hold the native_tlbie_lock. | |
449 | */ | |
96e28449 | 450 | if (hpte_v & HPTE_V_VALID) { |
1189be65 | 451 | hpte_decode(hptep, slot, &psize, &ssize, &va); |
96e28449 | 452 | hptep->v = 0; |
1189be65 | 453 | __tlbie(va, psize, ssize); |
f4c82d51 S |
454 | } |
455 | } | |
456 | ||
47f78a49 | 457 | asm volatile("eieio; tlbsync; ptesync":::"memory"); |
6b9c9b8a | 458 | raw_spin_unlock(&native_tlbie_lock); |
f4c82d51 S |
459 | local_irq_restore(flags); |
460 | } | |
461 | ||
3c726f8d BH |
462 | /* |
463 | * Batched hash table flush, we batch the tlbie's to avoid taking/releasing | |
464 | * the lock all the time | |
465 | */ | |
61b1a942 | 466 | static void native_flush_hash_range(unsigned long number, int local) |
1da177e4 | 467 | { |
3c726f8d | 468 | unsigned long va, hash, index, hidx, shift, slot; |
8e561e7e | 469 | struct hash_pte *hptep; |
96e28449 | 470 | unsigned long hpte_v; |
3c726f8d BH |
471 | unsigned long want_v; |
472 | unsigned long flags; | |
473 | real_pte_t pte; | |
1da177e4 | 474 | struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch); |
3c726f8d | 475 | unsigned long psize = batch->psize; |
1189be65 | 476 | int ssize = batch->ssize; |
3c726f8d | 477 | int i; |
1da177e4 LT |
478 | |
479 | local_irq_save(flags); | |
480 | ||
1da177e4 | 481 | for (i = 0; i < number; i++) { |
3c726f8d BH |
482 | va = batch->vaddr[i]; |
483 | pte = batch->pte[i]; | |
484 | ||
485 | pte_iterate_hashed_subpages(pte, psize, va, index, shift) { | |
1189be65 | 486 | hash = hpt_hash(va, shift, ssize); |
3c726f8d BH |
487 | hidx = __rpte_to_hidx(pte, index); |
488 | if (hidx & _PTEIDX_SECONDARY) | |
489 | hash = ~hash; | |
490 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | |
491 | slot += hidx & _PTEIDX_GROUP_IX; | |
492 | hptep = htab_address + slot; | |
1189be65 | 493 | want_v = hpte_encode_v(va, psize, ssize); |
3c726f8d BH |
494 | native_lock_hpte(hptep); |
495 | hpte_v = hptep->v; | |
496 | if (!HPTE_V_COMPARE(hpte_v, want_v) || | |
497 | !(hpte_v & HPTE_V_VALID)) | |
498 | native_unlock_hpte(hptep); | |
499 | else | |
500 | hptep->v = 0; | |
501 | } pte_iterate_hashed_end(); | |
1da177e4 LT |
502 | } |
503 | ||
44ae3ab3 | 504 | if (mmu_has_feature(MMU_FTR_TLBIEL) && |
3c726f8d | 505 | mmu_psize_defs[psize].tlbiel && local) { |
1da177e4 | 506 | asm volatile("ptesync":::"memory"); |
3c726f8d BH |
507 | for (i = 0; i < number; i++) { |
508 | va = batch->vaddr[i]; | |
509 | pte = batch->pte[i]; | |
510 | ||
511 | pte_iterate_hashed_subpages(pte, psize, va, index, | |
512 | shift) { | |
1189be65 | 513 | __tlbiel(va, psize, ssize); |
3c726f8d BH |
514 | } pte_iterate_hashed_end(); |
515 | } | |
1da177e4 LT |
516 | asm volatile("ptesync":::"memory"); |
517 | } else { | |
44ae3ab3 | 518 | int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); |
1da177e4 LT |
519 | |
520 | if (lock_tlbie) | |
6b9c9b8a | 521 | raw_spin_lock(&native_tlbie_lock); |
1da177e4 LT |
522 | |
523 | asm volatile("ptesync":::"memory"); | |
3c726f8d BH |
524 | for (i = 0; i < number; i++) { |
525 | va = batch->vaddr[i]; | |
526 | pte = batch->pte[i]; | |
527 | ||
528 | pte_iterate_hashed_subpages(pte, psize, va, index, | |
529 | shift) { | |
1189be65 | 530 | __tlbie(va, psize, ssize); |
3c726f8d BH |
531 | } pte_iterate_hashed_end(); |
532 | } | |
1da177e4 LT |
533 | asm volatile("eieio; tlbsync; ptesync":::"memory"); |
534 | ||
535 | if (lock_tlbie) | |
6b9c9b8a | 536 | raw_spin_unlock(&native_tlbie_lock); |
1da177e4 LT |
537 | } |
538 | ||
539 | local_irq_restore(flags); | |
540 | } | |
541 | ||
542 | #ifdef CONFIG_PPC_PSERIES | |
543 | /* Disable TLB batching on nighthawk */ | |
544 | static inline int tlb_batching_enabled(void) | |
545 | { | |
546 | struct device_node *root = of_find_node_by_path("/"); | |
547 | int enabled = 1; | |
548 | ||
549 | if (root) { | |
e2eb6392 | 550 | const char *model = of_get_property(root, "model", NULL); |
1da177e4 LT |
551 | if (model && !strcmp(model, "IBM,9076-N81")) |
552 | enabled = 0; | |
553 | of_node_put(root); | |
554 | } | |
555 | ||
556 | return enabled; | |
557 | } | |
558 | #else | |
559 | static inline int tlb_batching_enabled(void) | |
560 | { | |
561 | return 1; | |
562 | } | |
563 | #endif | |
564 | ||
7d0daae4 | 565 | void __init hpte_init_native(void) |
1da177e4 LT |
566 | { |
567 | ppc_md.hpte_invalidate = native_hpte_invalidate; | |
568 | ppc_md.hpte_updatepp = native_hpte_updatepp; | |
569 | ppc_md.hpte_updateboltedpp = native_hpte_updateboltedpp; | |
570 | ppc_md.hpte_insert = native_hpte_insert; | |
f4c82d51 S |
571 | ppc_md.hpte_remove = native_hpte_remove; |
572 | ppc_md.hpte_clear_all = native_hpte_clear; | |
1da177e4 LT |
573 | if (tlb_batching_enabled()) |
574 | ppc_md.flush_hash_range = native_flush_hash_range; | |
1da177e4 | 575 | } |