]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * native hashtable management. | |
3 | * | |
4 | * SMP scalability work: | |
5 | * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or | |
8 | * modify it under the terms of the GNU General Public License | |
9 | * as published by the Free Software Foundation; either version | |
10 | * 2 of the License, or (at your option) any later version. | |
11 | */ | |
3c726f8d BH |
12 | |
13 | #undef DEBUG_LOW | |
14 | ||
1da177e4 LT |
15 | #include <linux/spinlock.h> |
16 | #include <linux/bitops.h> | |
beacc6da | 17 | #include <linux/of.h> |
4e287e65 | 18 | #include <linux/processor.h> |
1da177e4 LT |
19 | #include <linux/threads.h> |
20 | #include <linux/smp.h> | |
21 | ||
1da177e4 LT |
22 | #include <asm/machdep.h> |
23 | #include <asm/mmu.h> | |
24 | #include <asm/mmu_context.h> | |
25 | #include <asm/pgtable.h> | |
26 | #include <asm/tlbflush.h> | |
0428491c | 27 | #include <asm/trace.h> |
1da177e4 LT |
28 | #include <asm/tlb.h> |
29 | #include <asm/cputable.h> | |
3c726f8d | 30 | #include <asm/udbg.h> |
71bf08b6 | 31 | #include <asm/kexec.h> |
60dbf438 | 32 | #include <asm/ppc-opcode.h> |
3c726f8d | 33 | |
ec249dd8 | 34 | #include <misc/cxl-base.h> |
4c6d9acc | 35 | |
3c726f8d BH |
36 | #ifdef DEBUG_LOW |
37 | #define DBG_LOW(fmt...) udbg_printf(fmt) | |
38 | #else | |
39 | #define DBG_LOW(fmt...) | |
40 | #endif | |
1da177e4 | 41 | |
12f04f2b | 42 | #ifdef __BIG_ENDIAN__ |
1da177e4 | 43 | #define HPTE_LOCK_BIT 3 |
12f04f2b AB |
44 | #else |
45 | #define HPTE_LOCK_BIT (56+3) | |
46 | #endif | |
1da177e4 | 47 | |
9e368f29 | 48 | DEFINE_RAW_SPINLOCK(native_tlbie_lock); |
1da177e4 | 49 | |
a3961f82 MS |
50 | static inline unsigned long ___tlbie(unsigned long vpn, int psize, |
51 | int apsize, int ssize) | |
3c726f8d | 52 | { |
5524a27d | 53 | unsigned long va; |
3c726f8d | 54 | unsigned int penc; |
de640959 | 55 | unsigned long sllp; |
3c726f8d | 56 | |
5524a27d AK |
57 | /* |
58 | * We need 14 to 65 bits of va for a tlibe of 4K page | |
59 | * With vpn we ignore the lower VPN_SHIFT bits already. | |
60 | * And top two bits are already ignored because we can | |
027dfac6 | 61 | * only accomodate 76 bits in a 64 bit vpn with a VPN_SHIFT |
5524a27d AK |
62 | * of 12. |
63 | */ | |
64 | va = vpn << VPN_SHIFT; | |
65 | /* | |
66 | * clear top 16 bits of 64bit va, non SLS segment | |
67 | * Older versions of the architecture (2.02 and earler) require the | |
68 | * masking of the top 16 bits. | |
69 | */ | |
accfad7d AK |
70 | if (mmu_has_feature(MMU_FTR_TLBIE_CROP_VA)) |
71 | va &= ~(0xffffULL << 48); | |
3c726f8d BH |
72 | |
73 | switch (psize) { | |
74 | case MMU_PAGE_4K: | |
1f6aaacc AK |
75 | /* clear out bits after (52) [0....52.....63] */ |
76 | va &= ~((1ul << (64 - 52)) - 1); | |
1189be65 | 77 | va |= ssize << 8; |
138ee7ee | 78 | sllp = get_sllp_encoding(apsize); |
de640959 | 79 | va |= sllp << 5; |
a32e252f | 80 | asm volatile(ASM_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0), %2) |
969391c5 | 81 | : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206) |
60dbf438 | 82 | : "memory"); |
3c726f8d BH |
83 | break; |
84 | default: | |
5524a27d | 85 | /* We need 14 to 14 + i bits of va */ |
b1022fbd | 86 | penc = mmu_psize_defs[psize].penc[apsize]; |
1f6aaacc | 87 | va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1); |
19242b24 | 88 | va |= penc << 12; |
1189be65 | 89 | va |= ssize << 8; |
29ef7a3e AK |
90 | /* |
91 | * AVAL bits: | |
92 | * We don't need all the bits, but rest of the bits | |
93 | * must be ignored by the processor. | |
94 | * vpn cover upto 65 bits of va. (0...65) and we need | |
95 | * 58..64 bits of va. | |
96 | */ | |
97 | va |= (vpn & 0xfe); /* AVAL */ | |
60dbf438 | 98 | va |= 1; /* L */ |
a32e252f | 99 | asm volatile(ASM_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0), %2) |
969391c5 | 100 | : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206) |
60dbf438 | 101 | : "memory"); |
3c726f8d BH |
102 | break; |
103 | } | |
a3961f82 MS |
104 | return va; |
105 | } | |
106 | ||
107 | static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize) | |
108 | { | |
109 | unsigned long rb; | |
110 | ||
111 | rb = ___tlbie(vpn, psize, apsize, ssize); | |
112 | trace_tlbie(0, 0, rb, 0, 0, 0, 0); | |
3c726f8d BH |
113 | } |
114 | ||
b1022fbd | 115 | static inline void __tlbiel(unsigned long vpn, int psize, int apsize, int ssize) |
3c726f8d | 116 | { |
5524a27d | 117 | unsigned long va; |
3c726f8d | 118 | unsigned int penc; |
de640959 | 119 | unsigned long sllp; |
3c726f8d | 120 | |
5524a27d AK |
121 | /* VPN_SHIFT can be atmost 12 */ |
122 | va = vpn << VPN_SHIFT; | |
123 | /* | |
124 | * clear top 16 bits of 64 bit va, non SLS segment | |
125 | * Older versions of the architecture (2.02 and earler) require the | |
126 | * masking of the top 16 bits. | |
127 | */ | |
accfad7d AK |
128 | if (mmu_has_feature(MMU_FTR_TLBIE_CROP_VA)) |
129 | va &= ~(0xffffULL << 48); | |
3c726f8d BH |
130 | |
131 | switch (psize) { | |
132 | case MMU_PAGE_4K: | |
1f6aaacc AK |
133 | /* clear out bits after(52) [0....52.....63] */ |
134 | va &= ~((1ul << (64 - 52)) - 1); | |
1189be65 | 135 | va |= ssize << 8; |
138ee7ee | 136 | sllp = get_sllp_encoding(apsize); |
de640959 | 137 | va |= sllp << 5; |
f923efbc BS |
138 | asm volatile(ASM_FTR_IFSET("tlbiel %0", "tlbiel %0,0", %1) |
139 | : : "r" (va), "i" (CPU_FTR_ARCH_206) | |
140 | : "memory"); | |
3c726f8d BH |
141 | break; |
142 | default: | |
5524a27d | 143 | /* We need 14 to 14 + i bits of va */ |
b1022fbd | 144 | penc = mmu_psize_defs[psize].penc[apsize]; |
1f6aaacc | 145 | va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1); |
19242b24 | 146 | va |= penc << 12; |
1189be65 | 147 | va |= ssize << 8; |
29ef7a3e AK |
148 | /* |
149 | * AVAL bits: | |
150 | * We don't need all the bits, but rest of the bits | |
151 | * must be ignored by the processor. | |
152 | * vpn cover upto 65 bits of va. (0...65) and we need | |
153 | * 58..64 bits of va. | |
154 | */ | |
155 | va |= (vpn & 0xfe); | |
60dbf438 | 156 | va |= 1; /* L */ |
f923efbc BS |
157 | asm volatile(ASM_FTR_IFSET("tlbiel %0", "tlbiel %0,1", %1) |
158 | : : "r" (va), "i" (CPU_FTR_ARCH_206) | |
159 | : "memory"); | |
3c726f8d BH |
160 | break; |
161 | } | |
0428491c | 162 | trace_tlbie(0, 1, va, 0, 0, 0, 0); |
3c726f8d BH |
163 | |
164 | } | |
165 | ||
b1022fbd AK |
166 | static inline void tlbie(unsigned long vpn, int psize, int apsize, |
167 | int ssize, int local) | |
3c726f8d | 168 | { |
4c6d9acc | 169 | unsigned int use_local; |
44ae3ab3 | 170 | int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); |
3c726f8d | 171 | |
4c6d9acc IM |
172 | use_local = local && mmu_has_feature(MMU_FTR_TLBIEL) && !cxl_ctx_in_use(); |
173 | ||
3c726f8d BH |
174 | if (use_local) |
175 | use_local = mmu_psize_defs[psize].tlbiel; | |
176 | if (lock_tlbie && !use_local) | |
6b9c9b8a | 177 | raw_spin_lock(&native_tlbie_lock); |
3c726f8d BH |
178 | asm volatile("ptesync": : :"memory"); |
179 | if (use_local) { | |
b1022fbd | 180 | __tlbiel(vpn, psize, apsize, ssize); |
3c726f8d BH |
181 | asm volatile("ptesync": : :"memory"); |
182 | } else { | |
b1022fbd | 183 | __tlbie(vpn, psize, apsize, ssize); |
3c726f8d BH |
184 | asm volatile("eieio; tlbsync; ptesync": : :"memory"); |
185 | } | |
186 | if (lock_tlbie && !use_local) | |
6b9c9b8a | 187 | raw_spin_unlock(&native_tlbie_lock); |
3c726f8d BH |
188 | } |
189 | ||
8e561e7e | 190 | static inline void native_lock_hpte(struct hash_pte *hptep) |
1da177e4 | 191 | { |
12f04f2b | 192 | unsigned long *word = (unsigned long *)&hptep->v; |
1da177e4 LT |
193 | |
194 | while (1) { | |
66d99b88 | 195 | if (!test_and_set_bit_lock(HPTE_LOCK_BIT, word)) |
1da177e4 | 196 | break; |
4e287e65 | 197 | spin_begin(); |
1da177e4 | 198 | while(test_bit(HPTE_LOCK_BIT, word)) |
4e287e65 NP |
199 | spin_cpu_relax(); |
200 | spin_end(); | |
1da177e4 LT |
201 | } |
202 | } | |
203 | ||
8e561e7e | 204 | static inline void native_unlock_hpte(struct hash_pte *hptep) |
1da177e4 | 205 | { |
12f04f2b | 206 | unsigned long *word = (unsigned long *)&hptep->v; |
1da177e4 | 207 | |
66d99b88 | 208 | clear_bit_unlock(HPTE_LOCK_BIT, word); |
1da177e4 LT |
209 | } |
210 | ||
5524a27d | 211 | static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn, |
3c726f8d | 212 | unsigned long pa, unsigned long rflags, |
b1022fbd | 213 | unsigned long vflags, int psize, int apsize, int ssize) |
1da177e4 | 214 | { |
8e561e7e | 215 | struct hash_pte *hptep = htab_address + hpte_group; |
96e28449 | 216 | unsigned long hpte_v, hpte_r; |
1da177e4 LT |
217 | int i; |
218 | ||
3c726f8d | 219 | if (!(vflags & HPTE_V_BOLTED)) { |
5524a27d | 220 | DBG_LOW(" insert(group=%lx, vpn=%016lx, pa=%016lx," |
3c726f8d | 221 | " rflags=%lx, vflags=%lx, psize=%d)\n", |
5524a27d | 222 | hpte_group, vpn, pa, rflags, vflags, psize); |
3c726f8d BH |
223 | } |
224 | ||
1da177e4 | 225 | for (i = 0; i < HPTES_PER_GROUP; i++) { |
12f04f2b | 226 | if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID)) { |
1da177e4 LT |
227 | /* retry with lock held */ |
228 | native_lock_hpte(hptep); | |
12f04f2b | 229 | if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID)) |
1da177e4 LT |
230 | break; |
231 | native_unlock_hpte(hptep); | |
232 | } | |
233 | ||
234 | hptep++; | |
235 | } | |
236 | ||
237 | if (i == HPTES_PER_GROUP) | |
238 | return -1; | |
239 | ||
b1022fbd | 240 | hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID; |
6b243fcf | 241 | hpte_r = hpte_encode_r(pa, psize, apsize) | rflags; |
3c726f8d BH |
242 | |
243 | if (!(vflags & HPTE_V_BOLTED)) { | |
244 | DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n", | |
245 | i, hpte_v, hpte_r); | |
246 | } | |
1da177e4 | 247 | |
6b243fcf PM |
248 | if (cpu_has_feature(CPU_FTR_ARCH_300)) { |
249 | hpte_r = hpte_old_to_new_r(hpte_v, hpte_r); | |
250 | hpte_v = hpte_old_to_new_v(hpte_v); | |
251 | } | |
252 | ||
12f04f2b | 253 | hptep->r = cpu_to_be64(hpte_r); |
1da177e4 | 254 | /* Guarantee the second dword is visible before the valid bit */ |
74a0ba61 | 255 | eieio(); |
1da177e4 LT |
256 | /* |
257 | * Now set the first dword including the valid bit | |
258 | * NOTE: this also unlocks the hpte | |
259 | */ | |
12f04f2b | 260 | hptep->v = cpu_to_be64(hpte_v); |
1da177e4 LT |
261 | |
262 | __asm__ __volatile__ ("ptesync" : : : "memory"); | |
263 | ||
96e28449 | 264 | return i | (!!(vflags & HPTE_V_SECONDARY) << 3); |
1da177e4 LT |
265 | } |
266 | ||
267 | static long native_hpte_remove(unsigned long hpte_group) | |
268 | { | |
8e561e7e | 269 | struct hash_pte *hptep; |
1da177e4 LT |
270 | int i; |
271 | int slot_offset; | |
96e28449 | 272 | unsigned long hpte_v; |
1da177e4 | 273 | |
3c726f8d BH |
274 | DBG_LOW(" remove(group=%lx)\n", hpte_group); |
275 | ||
1da177e4 LT |
276 | /* pick a random entry to start at */ |
277 | slot_offset = mftb() & 0x7; | |
278 | ||
279 | for (i = 0; i < HPTES_PER_GROUP; i++) { | |
280 | hptep = htab_address + hpte_group + slot_offset; | |
12f04f2b | 281 | hpte_v = be64_to_cpu(hptep->v); |
1da177e4 | 282 | |
96e28449 | 283 | if ((hpte_v & HPTE_V_VALID) && !(hpte_v & HPTE_V_BOLTED)) { |
1da177e4 LT |
284 | /* retry with lock held */ |
285 | native_lock_hpte(hptep); | |
12f04f2b | 286 | hpte_v = be64_to_cpu(hptep->v); |
96e28449 DG |
287 | if ((hpte_v & HPTE_V_VALID) |
288 | && !(hpte_v & HPTE_V_BOLTED)) | |
1da177e4 LT |
289 | break; |
290 | native_unlock_hpte(hptep); | |
291 | } | |
292 | ||
293 | slot_offset++; | |
294 | slot_offset &= 0x7; | |
295 | } | |
296 | ||
297 | if (i == HPTES_PER_GROUP) | |
298 | return -1; | |
299 | ||
300 | /* Invalidate the hpte. NOTE: this also unlocks it */ | |
96e28449 | 301 | hptep->v = 0; |
1da177e4 LT |
302 | |
303 | return i; | |
304 | } | |
305 | ||
3c726f8d | 306 | static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, |
db3d8534 | 307 | unsigned long vpn, int bpsize, |
aefa5688 | 308 | int apsize, int ssize, unsigned long flags) |
1da177e4 | 309 | { |
8e561e7e | 310 | struct hash_pte *hptep = htab_address + slot; |
3c726f8d | 311 | unsigned long hpte_v, want_v; |
aefa5688 | 312 | int ret = 0, local = 0; |
3c726f8d | 313 | |
db3d8534 | 314 | want_v = hpte_encode_avpn(vpn, bpsize, ssize); |
3c726f8d | 315 | |
5524a27d AK |
316 | DBG_LOW(" update(vpn=%016lx, avpnv=%016lx, group=%lx, newpp=%lx)", |
317 | vpn, want_v & HPTE_V_AVPN, slot, newpp); | |
3c726f8d | 318 | |
12f04f2b | 319 | hpte_v = be64_to_cpu(hptep->v); |
6b243fcf PM |
320 | if (cpu_has_feature(CPU_FTR_ARCH_300)) |
321 | hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); | |
0608d692 AK |
322 | /* |
323 | * We need to invalidate the TLB always because hpte_remove doesn't do | |
324 | * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less | |
325 | * random entry from it. When we do that we don't invalidate the TLB | |
326 | * (hpte_remove) because we assume the old translation is still | |
327 | * technically "valid". | |
328 | */ | |
db3d8534 | 329 | if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) { |
3c726f8d | 330 | DBG_LOW(" -> miss\n"); |
3c726f8d BH |
331 | ret = -1; |
332 | } else { | |
0ec2698f AK |
333 | native_lock_hpte(hptep); |
334 | /* recheck with locks held */ | |
335 | hpte_v = be64_to_cpu(hptep->v); | |
6b243fcf PM |
336 | if (cpu_has_feature(CPU_FTR_ARCH_300)) |
337 | hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); | |
0ec2698f AK |
338 | if (unlikely(!HPTE_V_COMPARE(hpte_v, want_v) || |
339 | !(hpte_v & HPTE_V_VALID))) { | |
340 | ret = -1; | |
341 | } else { | |
342 | DBG_LOW(" -> hit\n"); | |
343 | /* Update the HPTE */ | |
344 | hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) & | |
8550e2fa AK |
345 | ~(HPTE_R_PPP | HPTE_R_N)) | |
346 | (newpp & (HPTE_R_PPP | HPTE_R_N | | |
0ec2698f AK |
347 | HPTE_R_C))); |
348 | } | |
349 | native_unlock_hpte(hptep); | |
3c726f8d | 350 | } |
aefa5688 AK |
351 | |
352 | if (flags & HPTE_LOCAL_UPDATE) | |
353 | local = 1; | |
354 | /* | |
355 | * Ensure it is out of the tlb too if it is not a nohpte fault | |
356 | */ | |
357 | if (!(flags & HPTE_NOHPTE_UPDATE)) | |
358 | tlbie(vpn, bpsize, apsize, ssize, local); | |
359 | ||
3c726f8d | 360 | return ret; |
1da177e4 LT |
361 | } |
362 | ||
5524a27d | 363 | static long native_hpte_find(unsigned long vpn, int psize, int ssize) |
1da177e4 | 364 | { |
8e561e7e | 365 | struct hash_pte *hptep; |
1da177e4 | 366 | unsigned long hash; |
1189be65 | 367 | unsigned long i; |
1da177e4 | 368 | long slot; |
3c726f8d | 369 | unsigned long want_v, hpte_v; |
1da177e4 | 370 | |
5524a27d | 371 | hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, ssize); |
74f227b2 | 372 | want_v = hpte_encode_avpn(vpn, psize, ssize); |
1da177e4 | 373 | |
1189be65 PM |
374 | /* Bolted mappings are only ever in the primary group */ |
375 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | |
376 | for (i = 0; i < HPTES_PER_GROUP; i++) { | |
377 | hptep = htab_address + slot; | |
12f04f2b | 378 | hpte_v = be64_to_cpu(hptep->v); |
6b243fcf PM |
379 | if (cpu_has_feature(CPU_FTR_ARCH_300)) |
380 | hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); | |
1da177e4 | 381 | |
1189be65 PM |
382 | if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) |
383 | /* HPTE matches */ | |
384 | return slot; | |
385 | ++slot; | |
1da177e4 LT |
386 | } |
387 | ||
388 | return -1; | |
389 | } | |
390 | ||
1da177e4 LT |
391 | /* |
392 | * Update the page protection bits. Intended to be used to create | |
393 | * guard pages for kernel data structures on pages which are bolted | |
394 | * in the HPT. Assumes pages being operated on will not be stolen. | |
1da177e4 LT |
395 | * |
396 | * No need to lock here because we should be the only user. | |
397 | */ | |
3c726f8d | 398 | static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea, |
1189be65 | 399 | int psize, int ssize) |
1da177e4 | 400 | { |
5524a27d AK |
401 | unsigned long vpn; |
402 | unsigned long vsid; | |
1da177e4 | 403 | long slot; |
8e561e7e | 404 | struct hash_pte *hptep; |
1da177e4 | 405 | |
1189be65 | 406 | vsid = get_kernel_vsid(ea, ssize); |
5524a27d | 407 | vpn = hpt_vpn(ea, vsid, ssize); |
1da177e4 | 408 | |
5524a27d | 409 | slot = native_hpte_find(vpn, psize, ssize); |
1da177e4 LT |
410 | if (slot == -1) |
411 | panic("could not find page to bolt\n"); | |
412 | hptep = htab_address + slot; | |
413 | ||
3c726f8d | 414 | /* Update the HPTE */ |
12f04f2b | 415 | hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) & |
8550e2fa AK |
416 | ~(HPTE_R_PPP | HPTE_R_N)) | |
417 | (newpp & (HPTE_R_PPP | HPTE_R_N))); | |
db3d8534 AK |
418 | /* |
419 | * Ensure it is out of the tlb too. Bolted entries base and | |
420 | * actual page size will be same. | |
421 | */ | |
422 | tlbie(vpn, psize, psize, ssize, 0); | |
1da177e4 LT |
423 | } |
424 | ||
1b644f57 AB |
425 | /* |
426 | * Remove a bolted kernel entry. Memory hotplug uses this. | |
427 | * | |
428 | * No need to lock here because we should be the only user. | |
429 | */ | |
430 | static int native_hpte_removebolted(unsigned long ea, int psize, int ssize) | |
431 | { | |
432 | unsigned long vpn; | |
433 | unsigned long vsid; | |
434 | long slot; | |
435 | struct hash_pte *hptep; | |
436 | ||
437 | vsid = get_kernel_vsid(ea, ssize); | |
438 | vpn = hpt_vpn(ea, vsid, ssize); | |
439 | ||
440 | slot = native_hpte_find(vpn, psize, ssize); | |
441 | if (slot == -1) | |
442 | return -ENOENT; | |
443 | ||
444 | hptep = htab_address + slot; | |
445 | ||
446 | VM_WARN_ON(!(be64_to_cpu(hptep->v) & HPTE_V_BOLTED)); | |
447 | ||
448 | /* Invalidate the hpte */ | |
449 | hptep->v = 0; | |
450 | ||
451 | /* Invalidate the TLB */ | |
452 | tlbie(vpn, psize, psize, ssize, 0); | |
453 | return 0; | |
454 | } | |
455 | ||
456 | ||
5524a27d | 457 | static void native_hpte_invalidate(unsigned long slot, unsigned long vpn, |
db3d8534 | 458 | int bpsize, int apsize, int ssize, int local) |
1da177e4 | 459 | { |
8e561e7e | 460 | struct hash_pte *hptep = htab_address + slot; |
96e28449 | 461 | unsigned long hpte_v; |
3c726f8d | 462 | unsigned long want_v; |
1da177e4 | 463 | unsigned long flags; |
1da177e4 LT |
464 | |
465 | local_irq_save(flags); | |
1da177e4 | 466 | |
5524a27d | 467 | DBG_LOW(" invalidate(vpn=%016lx, hash: %lx)\n", vpn, slot); |
3c726f8d | 468 | |
db3d8534 | 469 | want_v = hpte_encode_avpn(vpn, bpsize, ssize); |
3c726f8d | 470 | native_lock_hpte(hptep); |
12f04f2b | 471 | hpte_v = be64_to_cpu(hptep->v); |
6b243fcf PM |
472 | if (cpu_has_feature(CPU_FTR_ARCH_300)) |
473 | hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); | |
1da177e4 | 474 | |
0608d692 AK |
475 | /* |
476 | * We need to invalidate the TLB always because hpte_remove doesn't do | |
477 | * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less | |
478 | * random entry from it. When we do that we don't invalidate the TLB | |
479 | * (hpte_remove) because we assume the old translation is still | |
480 | * technically "valid". | |
481 | */ | |
db3d8534 | 482 | if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) |
1da177e4 | 483 | native_unlock_hpte(hptep); |
3c726f8d | 484 | else |
1da177e4 | 485 | /* Invalidate the hpte. NOTE: this also unlocks it */ |
96e28449 | 486 | hptep->v = 0; |
1da177e4 | 487 | |
3c726f8d | 488 | /* Invalidate the TLB */ |
db3d8534 AK |
489 | tlbie(vpn, bpsize, apsize, ssize, local); |
490 | ||
1da177e4 LT |
491 | local_irq_restore(flags); |
492 | } | |
493 | ||
e34aa03c | 494 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
fa1f8ae8 AK |
495 | static void native_hugepage_invalidate(unsigned long vsid, |
496 | unsigned long addr, | |
1a527286 | 497 | unsigned char *hpte_slot_array, |
d557b098 | 498 | int psize, int ssize, int local) |
1a527286 | 499 | { |
969b7b20 | 500 | int i; |
1a527286 AK |
501 | struct hash_pte *hptep; |
502 | int actual_psize = MMU_PAGE_16M; | |
503 | unsigned int max_hpte_count, valid; | |
504 | unsigned long flags, s_addr = addr; | |
505 | unsigned long hpte_v, want_v, shift; | |
fa1f8ae8 | 506 | unsigned long hidx, vpn = 0, hash, slot; |
1a527286 AK |
507 | |
508 | shift = mmu_psize_defs[psize].shift; | |
509 | max_hpte_count = 1U << (PMD_SHIFT - shift); | |
510 | ||
511 | local_irq_save(flags); | |
512 | for (i = 0; i < max_hpte_count; i++) { | |
513 | valid = hpte_valid(hpte_slot_array, i); | |
514 | if (!valid) | |
515 | continue; | |
516 | hidx = hpte_hash_index(hpte_slot_array, i); | |
517 | ||
518 | /* get the vpn */ | |
519 | addr = s_addr + (i * (1ul << shift)); | |
1a527286 AK |
520 | vpn = hpt_vpn(addr, vsid, ssize); |
521 | hash = hpt_hash(vpn, shift, ssize); | |
522 | if (hidx & _PTEIDX_SECONDARY) | |
523 | hash = ~hash; | |
524 | ||
525 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | |
526 | slot += hidx & _PTEIDX_GROUP_IX; | |
527 | ||
528 | hptep = htab_address + slot; | |
529 | want_v = hpte_encode_avpn(vpn, psize, ssize); | |
530 | native_lock_hpte(hptep); | |
12f04f2b | 531 | hpte_v = be64_to_cpu(hptep->v); |
6b243fcf PM |
532 | if (cpu_has_feature(CPU_FTR_ARCH_300)) |
533 | hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); | |
1a527286 AK |
534 | |
535 | /* Even if we miss, we need to invalidate the TLB */ | |
536 | if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) | |
537 | native_unlock_hpte(hptep); | |
538 | else | |
539 | /* Invalidate the hpte. NOTE: this also unlocks it */ | |
540 | hptep->v = 0; | |
969b7b20 AK |
541 | /* |
542 | * We need to do tlb invalidate for all the address, tlbie | |
543 | * instruction compares entry_VA in tlb with the VA specified | |
544 | * here | |
545 | */ | |
d557b098 | 546 | tlbie(vpn, psize, actual_psize, ssize, local); |
1a527286 | 547 | } |
1a527286 AK |
548 | local_irq_restore(flags); |
549 | } | |
e34aa03c AK |
550 | #else |
551 | static void native_hugepage_invalidate(unsigned long vsid, | |
552 | unsigned long addr, | |
553 | unsigned char *hpte_slot_array, | |
554 | int psize, int ssize, int local) | |
555 | { | |
556 | WARN(1, "%s called without THP support\n", __func__); | |
557 | } | |
558 | #endif | |
1a527286 | 559 | |
8e561e7e | 560 | static void hpte_decode(struct hash_pte *hpte, unsigned long slot, |
b1022fbd | 561 | int *psize, int *apsize, int *ssize, unsigned long *vpn) |
71bf08b6 | 562 | { |
dcda287a | 563 | unsigned long avpn, pteg, vpi; |
12f04f2b AB |
564 | unsigned long hpte_v = be64_to_cpu(hpte->v); |
565 | unsigned long hpte_r = be64_to_cpu(hpte->r); | |
dcda287a | 566 | unsigned long vsid, seg_off; |
7e74c392 AK |
567 | int size, a_size, shift; |
568 | /* Look at the 8 bit LP value */ | |
12f04f2b | 569 | unsigned int lp = (hpte_r >> LP_SHIFT) & ((1 << LP_BITS) - 1); |
71bf08b6 | 570 | |
6b243fcf PM |
571 | if (cpu_has_feature(CPU_FTR_ARCH_300)) { |
572 | hpte_v = hpte_new_to_old_v(hpte_v, hpte_r); | |
573 | hpte_r = hpte_new_to_old_r(hpte_r); | |
574 | } | |
b1022fbd AK |
575 | if (!(hpte_v & HPTE_V_LARGE)) { |
576 | size = MMU_PAGE_4K; | |
577 | a_size = MMU_PAGE_4K; | |
578 | } else { | |
0eeede0c PM |
579 | size = hpte_page_sizes[lp] & 0xf; |
580 | a_size = hpte_page_sizes[lp] >> 4; | |
71bf08b6 | 581 | } |
2454c7e9 | 582 | /* This works for all page sizes, and for 256M and 1T segments */ |
6b243fcf | 583 | *ssize = hpte_v >> HPTE_V_SSIZE_SHIFT; |
71bf08b6 | 584 | shift = mmu_psize_defs[size].shift; |
71bf08b6 | 585 | |
dcda287a AK |
586 | avpn = (HPTE_V_AVPN_VAL(hpte_v) & ~mmu_psize_defs[size].avpnm); |
587 | pteg = slot / HPTES_PER_GROUP; | |
588 | if (hpte_v & HPTE_V_SECONDARY) | |
589 | pteg = ~pteg; | |
590 | ||
591 | switch (*ssize) { | |
592 | case MMU_SEGSIZE_256M: | |
593 | /* We only have 28 - 23 bits of seg_off in avpn */ | |
594 | seg_off = (avpn & 0x1f) << 23; | |
595 | vsid = avpn >> 5; | |
596 | /* We can find more bits from the pteg value */ | |
597 | if (shift < 23) { | |
598 | vpi = (vsid ^ pteg) & htab_hash_mask; | |
599 | seg_off |= vpi << shift; | |
600 | } | |
5524a27d | 601 | *vpn = vsid << (SID_SHIFT - VPN_SHIFT) | seg_off >> VPN_SHIFT; |
83383b73 | 602 | break; |
dcda287a AK |
603 | case MMU_SEGSIZE_1T: |
604 | /* We only have 40 - 23 bits of seg_off in avpn */ | |
605 | seg_off = (avpn & 0x1ffff) << 23; | |
606 | vsid = avpn >> 17; | |
607 | if (shift < 23) { | |
2454c7e9 | 608 | vpi = (vsid ^ (vsid << 25) ^ pteg) & htab_hash_mask; |
dcda287a | 609 | seg_off |= vpi << shift; |
71bf08b6 | 610 | } |
5524a27d | 611 | *vpn = vsid << (SID_SHIFT_1T - VPN_SHIFT) | seg_off >> VPN_SHIFT; |
83383b73 | 612 | break; |
dcda287a | 613 | default: |
5524a27d | 614 | *vpn = size = 0; |
3c726f8d | 615 | } |
b1022fbd AK |
616 | *psize = size; |
617 | *apsize = a_size; | |
3c726f8d BH |
618 | } |
619 | ||
f4c82d51 S |
620 | /* |
621 | * clear all mappings on kexec. All cpus are in real mode (or they will | |
622 | * be when they isi), and we are the only one left. We rely on our kernel | |
623 | * mapping being 0xC0's and the hardware ignoring those two real bits. | |
624 | * | |
fdf880a6 CB |
625 | * This must be called with interrupts disabled. |
626 | * | |
627 | * Taking the native_tlbie_lock is unsafe here due to the possibility of | |
628 | * lockdep being on. On pre POWER5 hardware, not taking the lock could | |
629 | * cause deadlock. POWER5 and newer not taking the lock is fine. This only | |
630 | * gets called during boot before secondary CPUs have come up and during | |
631 | * crashdump and all bets are off anyway. | |
632 | * | |
f4c82d51 | 633 | * TODO: add batching support when enabled. remember, no dynamic memory here, |
027dfac6 | 634 | * although there is the control page available... |
f4c82d51 S |
635 | */ |
636 | static void native_hpte_clear(void) | |
637 | { | |
5524a27d | 638 | unsigned long vpn = 0; |
fdf880a6 | 639 | unsigned long slot, slots; |
8e561e7e | 640 | struct hash_pte *hptep = htab_address; |
5524a27d | 641 | unsigned long hpte_v; |
f4c82d51 | 642 | unsigned long pteg_count; |
b1022fbd | 643 | int psize, apsize, ssize; |
f4c82d51 S |
644 | |
645 | pteg_count = htab_hash_mask + 1; | |
646 | ||
f4c82d51 S |
647 | slots = pteg_count * HPTES_PER_GROUP; |
648 | ||
649 | for (slot = 0; slot < slots; slot++, hptep++) { | |
650 | /* | |
651 | * we could lock the pte here, but we are the only cpu | |
652 | * running, right? and for crash dump, we probably | |
653 | * don't want to wait for a maybe bad cpu. | |
654 | */ | |
12f04f2b | 655 | hpte_v = be64_to_cpu(hptep->v); |
f4c82d51 | 656 | |
47f78a49 | 657 | /* |
fdf880a6 CB |
658 | * Call __tlbie() here rather than tlbie() since we can't take the |
659 | * native_tlbie_lock. | |
47f78a49 | 660 | */ |
96e28449 | 661 | if (hpte_v & HPTE_V_VALID) { |
b1022fbd | 662 | hpte_decode(hptep, slot, &psize, &apsize, &ssize, &vpn); |
96e28449 | 663 | hptep->v = 0; |
a3961f82 | 664 | ___tlbie(vpn, psize, apsize, ssize); |
f4c82d51 S |
665 | } |
666 | } | |
667 | ||
47f78a49 | 668 | asm volatile("eieio; tlbsync; ptesync":::"memory"); |
f4c82d51 S |
669 | } |
670 | ||
3c726f8d BH |
671 | /* |
672 | * Batched hash table flush, we batch the tlbie's to avoid taking/releasing | |
673 | * the lock all the time | |
674 | */ | |
61b1a942 | 675 | static void native_flush_hash_range(unsigned long number, int local) |
1da177e4 | 676 | { |
5524a27d AK |
677 | unsigned long vpn; |
678 | unsigned long hash, index, hidx, shift, slot; | |
8e561e7e | 679 | struct hash_pte *hptep; |
96e28449 | 680 | unsigned long hpte_v; |
3c726f8d BH |
681 | unsigned long want_v; |
682 | unsigned long flags; | |
683 | real_pte_t pte; | |
69111bac | 684 | struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch); |
3c726f8d | 685 | unsigned long psize = batch->psize; |
1189be65 | 686 | int ssize = batch->ssize; |
3c726f8d | 687 | int i; |
88b1bf72 FB |
688 | unsigned int use_local; |
689 | ||
690 | use_local = local && mmu_has_feature(MMU_FTR_TLBIEL) && | |
691 | mmu_psize_defs[psize].tlbiel && !cxl_ctx_in_use(); | |
1da177e4 LT |
692 | |
693 | local_irq_save(flags); | |
694 | ||
1da177e4 | 695 | for (i = 0; i < number; i++) { |
5524a27d | 696 | vpn = batch->vpn[i]; |
3c726f8d BH |
697 | pte = batch->pte[i]; |
698 | ||
5524a27d AK |
699 | pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) { |
700 | hash = hpt_hash(vpn, shift, ssize); | |
3c726f8d BH |
701 | hidx = __rpte_to_hidx(pte, index); |
702 | if (hidx & _PTEIDX_SECONDARY) | |
703 | hash = ~hash; | |
704 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | |
705 | slot += hidx & _PTEIDX_GROUP_IX; | |
706 | hptep = htab_address + slot; | |
74f227b2 | 707 | want_v = hpte_encode_avpn(vpn, psize, ssize); |
3c726f8d | 708 | native_lock_hpte(hptep); |
12f04f2b | 709 | hpte_v = be64_to_cpu(hptep->v); |
6b243fcf PM |
710 | if (cpu_has_feature(CPU_FTR_ARCH_300)) |
711 | hpte_v = hpte_new_to_old_v(hpte_v, | |
712 | be64_to_cpu(hptep->r)); | |
3c726f8d BH |
713 | if (!HPTE_V_COMPARE(hpte_v, want_v) || |
714 | !(hpte_v & HPTE_V_VALID)) | |
715 | native_unlock_hpte(hptep); | |
716 | else | |
717 | hptep->v = 0; | |
718 | } pte_iterate_hashed_end(); | |
1da177e4 LT |
719 | } |
720 | ||
88b1bf72 | 721 | if (use_local) { |
1da177e4 | 722 | asm volatile("ptesync":::"memory"); |
3c726f8d | 723 | for (i = 0; i < number; i++) { |
5524a27d | 724 | vpn = batch->vpn[i]; |
3c726f8d BH |
725 | pte = batch->pte[i]; |
726 | ||
5524a27d AK |
727 | pte_iterate_hashed_subpages(pte, psize, |
728 | vpn, index, shift) { | |
b1022fbd | 729 | __tlbiel(vpn, psize, psize, ssize); |
3c726f8d BH |
730 | } pte_iterate_hashed_end(); |
731 | } | |
1da177e4 LT |
732 | asm volatile("ptesync":::"memory"); |
733 | } else { | |
44ae3ab3 | 734 | int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); |
1da177e4 LT |
735 | |
736 | if (lock_tlbie) | |
6b9c9b8a | 737 | raw_spin_lock(&native_tlbie_lock); |
1da177e4 LT |
738 | |
739 | asm volatile("ptesync":::"memory"); | |
3c726f8d | 740 | for (i = 0; i < number; i++) { |
5524a27d | 741 | vpn = batch->vpn[i]; |
3c726f8d BH |
742 | pte = batch->pte[i]; |
743 | ||
5524a27d AK |
744 | pte_iterate_hashed_subpages(pte, psize, |
745 | vpn, index, shift) { | |
b1022fbd | 746 | __tlbie(vpn, psize, psize, ssize); |
3c726f8d BH |
747 | } pte_iterate_hashed_end(); |
748 | } | |
1da177e4 LT |
749 | asm volatile("eieio; tlbsync; ptesync":::"memory"); |
750 | ||
751 | if (lock_tlbie) | |
6b9c9b8a | 752 | raw_spin_unlock(&native_tlbie_lock); |
1da177e4 LT |
753 | } |
754 | ||
755 | local_irq_restore(flags); | |
756 | } | |
757 | ||
83209bc8 AK |
758 | static int native_register_proc_table(unsigned long base, unsigned long page_size, |
759 | unsigned long table_size) | |
50de596d | 760 | { |
83209bc8 AK |
761 | unsigned long patb1 = base << 25; /* VSID */ |
762 | ||
763 | patb1 |= (page_size << 5); /* sllp */ | |
764 | patb1 |= table_size; | |
765 | ||
50de596d AK |
766 | partition_tb->patb1 = cpu_to_be64(patb1); |
767 | return 0; | |
768 | } | |
769 | ||
7d0daae4 | 770 | void __init hpte_init_native(void) |
1da177e4 | 771 | { |
7025776e BH |
772 | mmu_hash_ops.hpte_invalidate = native_hpte_invalidate; |
773 | mmu_hash_ops.hpte_updatepp = native_hpte_updatepp; | |
774 | mmu_hash_ops.hpte_updateboltedpp = native_hpte_updateboltedpp; | |
1b644f57 | 775 | mmu_hash_ops.hpte_removebolted = native_hpte_removebolted; |
7025776e BH |
776 | mmu_hash_ops.hpte_insert = native_hpte_insert; |
777 | mmu_hash_ops.hpte_remove = native_hpte_remove; | |
778 | mmu_hash_ops.hpte_clear_all = native_hpte_clear; | |
779 | mmu_hash_ops.flush_hash_range = native_flush_hash_range; | |
780 | mmu_hash_ops.hugepage_invalidate = native_hugepage_invalidate; | |
50de596d AK |
781 | |
782 | if (cpu_has_feature(CPU_FTR_ARCH_300)) | |
eea8148c | 783 | register_process_table = native_register_proc_table; |
1da177e4 | 784 | } |