]>
Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
1da177e4 LT |
2 | /* |
3 | * native hashtable management. | |
4 | * | |
5 | * SMP scalability work: | |
6 | * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM | |
1da177e4 | 7 | */ |
3c726f8d BH |
8 | |
9 | #undef DEBUG_LOW | |
10 | ||
1da177e4 LT |
11 | #include <linux/spinlock.h> |
12 | #include <linux/bitops.h> | |
beacc6da | 13 | #include <linux/of.h> |
4e287e65 | 14 | #include <linux/processor.h> |
1da177e4 LT |
15 | #include <linux/threads.h> |
16 | #include <linux/smp.h> | |
17 | ||
1da177e4 LT |
18 | #include <asm/machdep.h> |
19 | #include <asm/mmu.h> | |
20 | #include <asm/mmu_context.h> | |
ca5999fd | 21 | #include <linux/pgtable.h> |
0428491c | 22 | #include <asm/trace.h> |
1da177e4 LT |
23 | #include <asm/tlb.h> |
24 | #include <asm/cputable.h> | |
3c726f8d | 25 | #include <asm/udbg.h> |
71bf08b6 | 26 | #include <asm/kexec.h> |
60dbf438 | 27 | #include <asm/ppc-opcode.h> |
2c86cd18 | 28 | #include <asm/feature-fixups.h> |
3c726f8d | 29 | |
ec249dd8 | 30 | #include <misc/cxl-base.h> |
4c6d9acc | 31 | |
3c726f8d BH |
32 | #ifdef DEBUG_LOW |
33 | #define DBG_LOW(fmt...) udbg_printf(fmt) | |
34 | #else | |
35 | #define DBG_LOW(fmt...) | |
36 | #endif | |
1da177e4 | 37 | |
12f04f2b | 38 | #ifdef __BIG_ENDIAN__ |
1da177e4 | 39 | #define HPTE_LOCK_BIT 3 |
12f04f2b AB |
40 | #else |
41 | #define HPTE_LOCK_BIT (56+3) | |
42 | #endif | |
1da177e4 | 43 | |
d667edc0 | 44 | static DEFINE_RAW_SPINLOCK(native_tlbie_lock); |
1da177e4 | 45 | |
d4748276 NP |
46 | static inline void tlbiel_hash_set_isa206(unsigned int set, unsigned int is) |
47 | { | |
48 | unsigned long rb; | |
49 | ||
50 | rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53)); | |
51 | ||
52 | asm volatile("tlbiel %0" : : "r" (rb)); | |
53 | } | |
54 | ||
55 | /* | |
56 | * tlbiel instruction for hash, set invalidation | |
57 | * i.e., r=1 and is=01 or is=10 or is=11 | |
58 | */ | |
6d3ca7e7 | 59 | static __always_inline void tlbiel_hash_set_isa300(unsigned int set, unsigned int is, |
d4748276 NP |
60 | unsigned int pid, |
61 | unsigned int ric, unsigned int prs) | |
62 | { | |
63 | unsigned long rb; | |
64 | unsigned long rs; | |
65 | unsigned int r = 0; /* hash format */ | |
66 | ||
67 | rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53)); | |
68 | rs = ((unsigned long)pid << PPC_BITLSHIFT(31)); | |
69 | ||
70 | asm volatile(PPC_TLBIEL(%0, %1, %2, %3, %4) | |
71 | : : "r"(rb), "r"(rs), "i"(ric), "i"(prs), "r"(r) | |
72 | : "memory"); | |
73 | } | |
74 | ||
75 | ||
76 | static void tlbiel_all_isa206(unsigned int num_sets, unsigned int is) | |
77 | { | |
78 | unsigned int set; | |
79 | ||
80 | asm volatile("ptesync": : :"memory"); | |
81 | ||
82 | for (set = 0; set < num_sets; set++) | |
83 | tlbiel_hash_set_isa206(set, is); | |
84 | ||
85 | asm volatile("ptesync": : :"memory"); | |
86 | } | |
87 | ||
88 | static void tlbiel_all_isa300(unsigned int num_sets, unsigned int is) | |
89 | { | |
90 | unsigned int set; | |
91 | ||
92 | asm volatile("ptesync": : :"memory"); | |
93 | ||
94 | /* | |
95 | * Flush the first set of the TLB, and any caching of partition table | |
96 | * entries. Then flush the remaining sets of the TLB. Hash mode uses | |
97 | * partition scoped TLB translations. | |
98 | */ | |
99 | tlbiel_hash_set_isa300(0, is, 0, 2, 0); | |
100 | for (set = 1; set < num_sets; set++) | |
101 | tlbiel_hash_set_isa300(set, is, 0, 0, 0); | |
102 | ||
103 | /* | |
104 | * Now invalidate the process table cache. | |
105 | * | |
106 | * From ISA v3.0B p. 1078: | |
107 | * The following forms are invalid. | |
108 | * * PRS=1, R=0, and RIC!=2 (The only process-scoped | |
109 | * HPT caching is of the Process Table.) | |
110 | */ | |
111 | tlbiel_hash_set_isa300(0, is, 0, 2, 1); | |
112 | ||
113 | asm volatile("ptesync": : :"memory"); | |
bc276ecb | 114 | |
fe7946ce | 115 | asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory"); |
d4748276 NP |
116 | } |
117 | ||
118 | void hash__tlbiel_all(unsigned int action) | |
119 | { | |
120 | unsigned int is; | |
121 | ||
122 | switch (action) { | |
123 | case TLB_INVAL_SCOPE_GLOBAL: | |
124 | is = 3; | |
125 | break; | |
126 | case TLB_INVAL_SCOPE_LPID: | |
127 | is = 2; | |
128 | break; | |
129 | default: | |
130 | BUG(); | |
131 | } | |
132 | ||
133 | if (early_cpu_has_feature(CPU_FTR_ARCH_300)) | |
134 | tlbiel_all_isa300(POWER9_TLB_SETS_HASH, is); | |
135 | else if (early_cpu_has_feature(CPU_FTR_ARCH_207S)) | |
136 | tlbiel_all_isa206(POWER8_TLB_SETS, is); | |
137 | else if (early_cpu_has_feature(CPU_FTR_ARCH_206)) | |
138 | tlbiel_all_isa206(POWER7_TLB_SETS, is); | |
139 | else | |
140 | WARN(1, "%s called on pre-POWER7 CPU\n", __func__); | |
d4748276 NP |
141 | } |
142 | ||
a3961f82 MS |
143 | static inline unsigned long ___tlbie(unsigned long vpn, int psize, |
144 | int apsize, int ssize) | |
3c726f8d | 145 | { |
5524a27d | 146 | unsigned long va; |
3c726f8d | 147 | unsigned int penc; |
de640959 | 148 | unsigned long sllp; |
3c726f8d | 149 | |
5524a27d AK |
150 | /* |
151 | * We need 14 to 65 bits of va for a tlibe of 4K page | |
152 | * With vpn we ignore the lower VPN_SHIFT bits already. | |
153 | * And top two bits are already ignored because we can | |
027dfac6 | 154 | * only accomodate 76 bits in a 64 bit vpn with a VPN_SHIFT |
5524a27d AK |
155 | * of 12. |
156 | */ | |
157 | va = vpn << VPN_SHIFT; | |
158 | /* | |
159 | * clear top 16 bits of 64bit va, non SLS segment | |
160 | * Older versions of the architecture (2.02 and earler) require the | |
161 | * masking of the top 16 bits. | |
162 | */ | |
accfad7d AK |
163 | if (mmu_has_feature(MMU_FTR_TLBIE_CROP_VA)) |
164 | va &= ~(0xffffULL << 48); | |
3c726f8d BH |
165 | |
166 | switch (psize) { | |
167 | case MMU_PAGE_4K: | |
1f6aaacc AK |
168 | /* clear out bits after (52) [0....52.....63] */ |
169 | va &= ~((1ul << (64 - 52)) - 1); | |
1189be65 | 170 | va |= ssize << 8; |
138ee7ee | 171 | sllp = get_sllp_encoding(apsize); |
de640959 | 172 | va |= sllp << 5; |
a32e252f | 173 | asm volatile(ASM_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0), %2) |
969391c5 | 174 | : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206) |
60dbf438 | 175 | : "memory"); |
3c726f8d BH |
176 | break; |
177 | default: | |
5524a27d | 178 | /* We need 14 to 14 + i bits of va */ |
b1022fbd | 179 | penc = mmu_psize_defs[psize].penc[apsize]; |
1f6aaacc | 180 | va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1); |
19242b24 | 181 | va |= penc << 12; |
1189be65 | 182 | va |= ssize << 8; |
29ef7a3e AK |
183 | /* |
184 | * AVAL bits: | |
185 | * We don't need all the bits, but rest of the bits | |
186 | * must be ignored by the processor. | |
187 | * vpn cover upto 65 bits of va. (0...65) and we need | |
188 | * 58..64 bits of va. | |
189 | */ | |
190 | va |= (vpn & 0xfe); /* AVAL */ | |
60dbf438 | 191 | va |= 1; /* L */ |
a32e252f | 192 | asm volatile(ASM_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0), %2) |
969391c5 | 193 | : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206) |
60dbf438 | 194 | : "memory"); |
3c726f8d BH |
195 | break; |
196 | } | |
a3961f82 MS |
197 | return va; |
198 | } | |
199 | ||
047e6575 AK |
200 | static inline void fixup_tlbie_vpn(unsigned long vpn, int psize, |
201 | int apsize, int ssize) | |
a5d4b589 | 202 | { |
047e6575 AK |
203 | if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { |
204 | /* Radix flush for a hash guest */ | |
205 | ||
206 | unsigned long rb,rs,prs,r,ric; | |
207 | ||
208 | rb = PPC_BIT(52); /* IS = 2 */ | |
209 | rs = 0; /* lpid = 0 */ | |
210 | prs = 0; /* partition scoped */ | |
211 | r = 1; /* radix format */ | |
212 | ric = 0; /* RIC_FLSUH_TLB */ | |
213 | ||
214 | /* | |
215 | * Need the extra ptesync to make sure we don't | |
216 | * re-order the tlbie | |
217 | */ | |
218 | asm volatile("ptesync": : :"memory"); | |
219 | asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) | |
220 | : : "r"(rb), "i"(r), "i"(prs), | |
221 | "i"(ric), "r"(rs) : "memory"); | |
222 | } | |
223 | ||
224 | ||
09ce98ca | 225 | if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { |
a5d4b589 AK |
226 | /* Need the extra ptesync to ensure we don't reorder tlbie*/ |
227 | asm volatile("ptesync": : :"memory"); | |
228 | ___tlbie(vpn, psize, apsize, ssize); | |
229 | } | |
230 | } | |
231 | ||
a3961f82 MS |
232 | static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize) |
233 | { | |
234 | unsigned long rb; | |
235 | ||
236 | rb = ___tlbie(vpn, psize, apsize, ssize); | |
237 | trace_tlbie(0, 0, rb, 0, 0, 0, 0); | |
3c726f8d BH |
238 | } |
239 | ||
b1022fbd | 240 | static inline void __tlbiel(unsigned long vpn, int psize, int apsize, int ssize) |
3c726f8d | 241 | { |
5524a27d | 242 | unsigned long va; |
3c726f8d | 243 | unsigned int penc; |
de640959 | 244 | unsigned long sllp; |
3c726f8d | 245 | |
5524a27d AK |
246 | /* VPN_SHIFT can be atmost 12 */ |
247 | va = vpn << VPN_SHIFT; | |
248 | /* | |
249 | * clear top 16 bits of 64 bit va, non SLS segment | |
250 | * Older versions of the architecture (2.02 and earler) require the | |
251 | * masking of the top 16 bits. | |
252 | */ | |
accfad7d AK |
253 | if (mmu_has_feature(MMU_FTR_TLBIE_CROP_VA)) |
254 | va &= ~(0xffffULL << 48); | |
3c726f8d BH |
255 | |
256 | switch (psize) { | |
257 | case MMU_PAGE_4K: | |
1f6aaacc AK |
258 | /* clear out bits after(52) [0....52.....63] */ |
259 | va &= ~((1ul << (64 - 52)) - 1); | |
1189be65 | 260 | va |= ssize << 8; |
138ee7ee | 261 | sllp = get_sllp_encoding(apsize); |
de640959 | 262 | va |= sllp << 5; |
f923efbc BS |
263 | asm volatile(ASM_FTR_IFSET("tlbiel %0", "tlbiel %0,0", %1) |
264 | : : "r" (va), "i" (CPU_FTR_ARCH_206) | |
265 | : "memory"); | |
3c726f8d BH |
266 | break; |
267 | default: | |
5524a27d | 268 | /* We need 14 to 14 + i bits of va */ |
b1022fbd | 269 | penc = mmu_psize_defs[psize].penc[apsize]; |
1f6aaacc | 270 | va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1); |
19242b24 | 271 | va |= penc << 12; |
1189be65 | 272 | va |= ssize << 8; |
29ef7a3e AK |
273 | /* |
274 | * AVAL bits: | |
275 | * We don't need all the bits, but rest of the bits | |
276 | * must be ignored by the processor. | |
277 | * vpn cover upto 65 bits of va. (0...65) and we need | |
278 | * 58..64 bits of va. | |
279 | */ | |
280 | va |= (vpn & 0xfe); | |
60dbf438 | 281 | va |= 1; /* L */ |
f923efbc BS |
282 | asm volatile(ASM_FTR_IFSET("tlbiel %0", "tlbiel %0,1", %1) |
283 | : : "r" (va), "i" (CPU_FTR_ARCH_206) | |
284 | : "memory"); | |
3c726f8d BH |
285 | break; |
286 | } | |
0428491c | 287 | trace_tlbie(0, 1, va, 0, 0, 0, 0); |
3c726f8d BH |
288 | |
289 | } | |
290 | ||
b1022fbd AK |
291 | static inline void tlbie(unsigned long vpn, int psize, int apsize, |
292 | int ssize, int local) | |
3c726f8d | 293 | { |
4c6d9acc | 294 | unsigned int use_local; |
44ae3ab3 | 295 | int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); |
3c726f8d | 296 | |
4c6d9acc IM |
297 | use_local = local && mmu_has_feature(MMU_FTR_TLBIEL) && !cxl_ctx_in_use(); |
298 | ||
3c726f8d BH |
299 | if (use_local) |
300 | use_local = mmu_psize_defs[psize].tlbiel; | |
301 | if (lock_tlbie && !use_local) | |
6b9c9b8a | 302 | raw_spin_lock(&native_tlbie_lock); |
3c726f8d BH |
303 | asm volatile("ptesync": : :"memory"); |
304 | if (use_local) { | |
b1022fbd | 305 | __tlbiel(vpn, psize, apsize, ssize); |
3c726f8d BH |
306 | asm volatile("ptesync": : :"memory"); |
307 | } else { | |
b1022fbd | 308 | __tlbie(vpn, psize, apsize, ssize); |
047e6575 | 309 | fixup_tlbie_vpn(vpn, psize, apsize, ssize); |
3c726f8d BH |
310 | asm volatile("eieio; tlbsync; ptesync": : :"memory"); |
311 | } | |
312 | if (lock_tlbie && !use_local) | |
6b9c9b8a | 313 | raw_spin_unlock(&native_tlbie_lock); |
3c726f8d BH |
314 | } |
315 | ||
8e561e7e | 316 | static inline void native_lock_hpte(struct hash_pte *hptep) |
1da177e4 | 317 | { |
12f04f2b | 318 | unsigned long *word = (unsigned long *)&hptep->v; |
1da177e4 LT |
319 | |
320 | while (1) { | |
66d99b88 | 321 | if (!test_and_set_bit_lock(HPTE_LOCK_BIT, word)) |
1da177e4 | 322 | break; |
4e287e65 | 323 | spin_begin(); |
1da177e4 | 324 | while(test_bit(HPTE_LOCK_BIT, word)) |
4e287e65 NP |
325 | spin_cpu_relax(); |
326 | spin_end(); | |
1da177e4 LT |
327 | } |
328 | } | |
329 | ||
8e561e7e | 330 | static inline void native_unlock_hpte(struct hash_pte *hptep) |
1da177e4 | 331 | { |
12f04f2b | 332 | unsigned long *word = (unsigned long *)&hptep->v; |
1da177e4 | 333 | |
66d99b88 | 334 | clear_bit_unlock(HPTE_LOCK_BIT, word); |
1da177e4 LT |
335 | } |
336 | ||
5524a27d | 337 | static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn, |
3c726f8d | 338 | unsigned long pa, unsigned long rflags, |
b1022fbd | 339 | unsigned long vflags, int psize, int apsize, int ssize) |
1da177e4 | 340 | { |
8e561e7e | 341 | struct hash_pte *hptep = htab_address + hpte_group; |
96e28449 | 342 | unsigned long hpte_v, hpte_r; |
1da177e4 LT |
343 | int i; |
344 | ||
3c726f8d | 345 | if (!(vflags & HPTE_V_BOLTED)) { |
5524a27d | 346 | DBG_LOW(" insert(group=%lx, vpn=%016lx, pa=%016lx," |
3c726f8d | 347 | " rflags=%lx, vflags=%lx, psize=%d)\n", |
5524a27d | 348 | hpte_group, vpn, pa, rflags, vflags, psize); |
3c726f8d BH |
349 | } |
350 | ||
1da177e4 | 351 | for (i = 0; i < HPTES_PER_GROUP; i++) { |
12f04f2b | 352 | if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID)) { |
1da177e4 LT |
353 | /* retry with lock held */ |
354 | native_lock_hpte(hptep); | |
12f04f2b | 355 | if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID)) |
1da177e4 LT |
356 | break; |
357 | native_unlock_hpte(hptep); | |
358 | } | |
359 | ||
360 | hptep++; | |
361 | } | |
362 | ||
363 | if (i == HPTES_PER_GROUP) | |
364 | return -1; | |
365 | ||
b1022fbd | 366 | hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID; |
6b243fcf | 367 | hpte_r = hpte_encode_r(pa, psize, apsize) | rflags; |
3c726f8d BH |
368 | |
369 | if (!(vflags & HPTE_V_BOLTED)) { | |
370 | DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n", | |
371 | i, hpte_v, hpte_r); | |
372 | } | |
1da177e4 | 373 | |
6b243fcf PM |
374 | if (cpu_has_feature(CPU_FTR_ARCH_300)) { |
375 | hpte_r = hpte_old_to_new_r(hpte_v, hpte_r); | |
376 | hpte_v = hpte_old_to_new_v(hpte_v); | |
377 | } | |
378 | ||
12f04f2b | 379 | hptep->r = cpu_to_be64(hpte_r); |
1da177e4 | 380 | /* Guarantee the second dword is visible before the valid bit */ |
74a0ba61 | 381 | eieio(); |
1da177e4 LT |
382 | /* |
383 | * Now set the first dword including the valid bit | |
384 | * NOTE: this also unlocks the hpte | |
385 | */ | |
12f04f2b | 386 | hptep->v = cpu_to_be64(hpte_v); |
1da177e4 LT |
387 | |
388 | __asm__ __volatile__ ("ptesync" : : : "memory"); | |
389 | ||
96e28449 | 390 | return i | (!!(vflags & HPTE_V_SECONDARY) << 3); |
1da177e4 LT |
391 | } |
392 | ||
393 | static long native_hpte_remove(unsigned long hpte_group) | |
394 | { | |
8e561e7e | 395 | struct hash_pte *hptep; |
1da177e4 LT |
396 | int i; |
397 | int slot_offset; | |
96e28449 | 398 | unsigned long hpte_v; |
1da177e4 | 399 | |
3c726f8d BH |
400 | DBG_LOW(" remove(group=%lx)\n", hpte_group); |
401 | ||
1da177e4 LT |
402 | /* pick a random entry to start at */ |
403 | slot_offset = mftb() & 0x7; | |
404 | ||
405 | for (i = 0; i < HPTES_PER_GROUP; i++) { | |
406 | hptep = htab_address + hpte_group + slot_offset; | |
12f04f2b | 407 | hpte_v = be64_to_cpu(hptep->v); |
1da177e4 | 408 | |
96e28449 | 409 | if ((hpte_v & HPTE_V_VALID) && !(hpte_v & HPTE_V_BOLTED)) { |
1da177e4 LT |
410 | /* retry with lock held */ |
411 | native_lock_hpte(hptep); | |
12f04f2b | 412 | hpte_v = be64_to_cpu(hptep->v); |
96e28449 DG |
413 | if ((hpte_v & HPTE_V_VALID) |
414 | && !(hpte_v & HPTE_V_BOLTED)) | |
1da177e4 LT |
415 | break; |
416 | native_unlock_hpte(hptep); | |
417 | } | |
418 | ||
419 | slot_offset++; | |
420 | slot_offset &= 0x7; | |
421 | } | |
422 | ||
423 | if (i == HPTES_PER_GROUP) | |
424 | return -1; | |
425 | ||
426 | /* Invalidate the hpte. NOTE: this also unlocks it */ | |
96e28449 | 427 | hptep->v = 0; |
1da177e4 LT |
428 | |
429 | return i; | |
430 | } | |
431 | ||
3c726f8d | 432 | static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, |
db3d8534 | 433 | unsigned long vpn, int bpsize, |
aefa5688 | 434 | int apsize, int ssize, unsigned long flags) |
1da177e4 | 435 | { |
8e561e7e | 436 | struct hash_pte *hptep = htab_address + slot; |
3c726f8d | 437 | unsigned long hpte_v, want_v; |
aefa5688 | 438 | int ret = 0, local = 0; |
3c726f8d | 439 | |
db3d8534 | 440 | want_v = hpte_encode_avpn(vpn, bpsize, ssize); |
3c726f8d | 441 | |
5524a27d AK |
442 | DBG_LOW(" update(vpn=%016lx, avpnv=%016lx, group=%lx, newpp=%lx)", |
443 | vpn, want_v & HPTE_V_AVPN, slot, newpp); | |
3c726f8d | 444 | |
a833280b | 445 | hpte_v = hpte_get_old_v(hptep); |
0608d692 AK |
446 | /* |
447 | * We need to invalidate the TLB always because hpte_remove doesn't do | |
448 | * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less | |
449 | * random entry from it. When we do that we don't invalidate the TLB | |
450 | * (hpte_remove) because we assume the old translation is still | |
451 | * technically "valid". | |
452 | */ | |
db3d8534 | 453 | if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) { |
3c726f8d | 454 | DBG_LOW(" -> miss\n"); |
3c726f8d BH |
455 | ret = -1; |
456 | } else { | |
0ec2698f AK |
457 | native_lock_hpte(hptep); |
458 | /* recheck with locks held */ | |
a833280b | 459 | hpte_v = hpte_get_old_v(hptep); |
0ec2698f AK |
460 | if (unlikely(!HPTE_V_COMPARE(hpte_v, want_v) || |
461 | !(hpte_v & HPTE_V_VALID))) { | |
462 | ret = -1; | |
463 | } else { | |
464 | DBG_LOW(" -> hit\n"); | |
465 | /* Update the HPTE */ | |
466 | hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) & | |
8550e2fa AK |
467 | ~(HPTE_R_PPP | HPTE_R_N)) | |
468 | (newpp & (HPTE_R_PPP | HPTE_R_N | | |
0ec2698f AK |
469 | HPTE_R_C))); |
470 | } | |
471 | native_unlock_hpte(hptep); | |
3c726f8d | 472 | } |
aefa5688 AK |
473 | |
474 | if (flags & HPTE_LOCAL_UPDATE) | |
475 | local = 1; | |
476 | /* | |
477 | * Ensure it is out of the tlb too if it is not a nohpte fault | |
478 | */ | |
479 | if (!(flags & HPTE_NOHPTE_UPDATE)) | |
480 | tlbie(vpn, bpsize, apsize, ssize, local); | |
481 | ||
3c726f8d | 482 | return ret; |
1da177e4 LT |
483 | } |
484 | ||
d78d5dac | 485 | static long __native_hpte_find(unsigned long want_v, unsigned long slot) |
1da177e4 | 486 | { |
8e561e7e | 487 | struct hash_pte *hptep; |
d78d5dac | 488 | unsigned long hpte_v; |
1189be65 | 489 | unsigned long i; |
1da177e4 | 490 | |
1189be65 | 491 | for (i = 0; i < HPTES_PER_GROUP; i++) { |
1da177e4 | 492 | |
a833280b AK |
493 | hptep = htab_address + slot; |
494 | hpte_v = hpte_get_old_v(hptep); | |
1189be65 PM |
495 | if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) |
496 | /* HPTE matches */ | |
497 | return slot; | |
498 | ++slot; | |
1da177e4 LT |
499 | } |
500 | ||
501 | return -1; | |
502 | } | |
503 | ||
d78d5dac AK |
504 | static long native_hpte_find(unsigned long vpn, int psize, int ssize) |
505 | { | |
506 | unsigned long hpte_group; | |
507 | unsigned long want_v; | |
508 | unsigned long hash; | |
509 | long slot; | |
510 | ||
511 | hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, ssize); | |
512 | want_v = hpte_encode_avpn(vpn, psize, ssize); | |
513 | ||
514 | /* | |
515 | * We try to keep bolted entries always in primary hash | |
516 | * But in some case we can find them in secondary too. | |
517 | */ | |
518 | hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP; | |
519 | slot = __native_hpte_find(want_v, hpte_group); | |
520 | if (slot < 0) { | |
521 | /* Try in secondary */ | |
522 | hpte_group = (~hash & htab_hash_mask) * HPTES_PER_GROUP; | |
523 | slot = __native_hpte_find(want_v, hpte_group); | |
524 | if (slot < 0) | |
525 | return -1; | |
526 | } | |
527 | ||
528 | return slot; | |
529 | } | |
530 | ||
1da177e4 LT |
531 | /* |
532 | * Update the page protection bits. Intended to be used to create | |
533 | * guard pages for kernel data structures on pages which are bolted | |
534 | * in the HPT. Assumes pages being operated on will not be stolen. | |
1da177e4 LT |
535 | * |
536 | * No need to lock here because we should be the only user. | |
537 | */ | |
3c726f8d | 538 | static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea, |
1189be65 | 539 | int psize, int ssize) |
1da177e4 | 540 | { |
5524a27d AK |
541 | unsigned long vpn; |
542 | unsigned long vsid; | |
1da177e4 | 543 | long slot; |
8e561e7e | 544 | struct hash_pte *hptep; |
1da177e4 | 545 | |
1189be65 | 546 | vsid = get_kernel_vsid(ea, ssize); |
5524a27d | 547 | vpn = hpt_vpn(ea, vsid, ssize); |
1da177e4 | 548 | |
5524a27d | 549 | slot = native_hpte_find(vpn, psize, ssize); |
1da177e4 LT |
550 | if (slot == -1) |
551 | panic("could not find page to bolt\n"); | |
552 | hptep = htab_address + slot; | |
553 | ||
3c726f8d | 554 | /* Update the HPTE */ |
12f04f2b | 555 | hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) & |
8550e2fa AK |
556 | ~(HPTE_R_PPP | HPTE_R_N)) | |
557 | (newpp & (HPTE_R_PPP | HPTE_R_N))); | |
db3d8534 AK |
558 | /* |
559 | * Ensure it is out of the tlb too. Bolted entries base and | |
560 | * actual page size will be same. | |
561 | */ | |
562 | tlbie(vpn, psize, psize, ssize, 0); | |
1da177e4 LT |
563 | } |
564 | ||
1b644f57 AB |
565 | /* |
566 | * Remove a bolted kernel entry. Memory hotplug uses this. | |
567 | * | |
568 | * No need to lock here because we should be the only user. | |
569 | */ | |
570 | static int native_hpte_removebolted(unsigned long ea, int psize, int ssize) | |
571 | { | |
572 | unsigned long vpn; | |
573 | unsigned long vsid; | |
574 | long slot; | |
575 | struct hash_pte *hptep; | |
576 | ||
577 | vsid = get_kernel_vsid(ea, ssize); | |
578 | vpn = hpt_vpn(ea, vsid, ssize); | |
579 | ||
580 | slot = native_hpte_find(vpn, psize, ssize); | |
581 | if (slot == -1) | |
582 | return -ENOENT; | |
583 | ||
584 | hptep = htab_address + slot; | |
585 | ||
586 | VM_WARN_ON(!(be64_to_cpu(hptep->v) & HPTE_V_BOLTED)); | |
587 | ||
588 | /* Invalidate the hpte */ | |
589 | hptep->v = 0; | |
590 | ||
591 | /* Invalidate the TLB */ | |
592 | tlbie(vpn, psize, psize, ssize, 0); | |
593 | return 0; | |
594 | } | |
595 | ||
596 | ||
5524a27d | 597 | static void native_hpte_invalidate(unsigned long slot, unsigned long vpn, |
db3d8534 | 598 | int bpsize, int apsize, int ssize, int local) |
1da177e4 | 599 | { |
8e561e7e | 600 | struct hash_pte *hptep = htab_address + slot; |
96e28449 | 601 | unsigned long hpte_v; |
3c726f8d | 602 | unsigned long want_v; |
1da177e4 | 603 | unsigned long flags; |
1da177e4 LT |
604 | |
605 | local_irq_save(flags); | |
1da177e4 | 606 | |
5524a27d | 607 | DBG_LOW(" invalidate(vpn=%016lx, hash: %lx)\n", vpn, slot); |
3c726f8d | 608 | |
db3d8534 | 609 | want_v = hpte_encode_avpn(vpn, bpsize, ssize); |
a833280b | 610 | hpte_v = hpte_get_old_v(hptep); |
1da177e4 | 611 | |
27d8959d AK |
612 | if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) { |
613 | native_lock_hpte(hptep); | |
614 | /* recheck with locks held */ | |
615 | hpte_v = hpte_get_old_v(hptep); | |
616 | ||
617 | if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) | |
618 | /* Invalidate the hpte. NOTE: this also unlocks it */ | |
619 | hptep->v = 0; | |
620 | else | |
621 | native_unlock_hpte(hptep); | |
622 | } | |
0608d692 AK |
623 | /* |
624 | * We need to invalidate the TLB always because hpte_remove doesn't do | |
625 | * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less | |
626 | * random entry from it. When we do that we don't invalidate the TLB | |
627 | * (hpte_remove) because we assume the old translation is still | |
628 | * technically "valid". | |
629 | */ | |
db3d8534 AK |
630 | tlbie(vpn, bpsize, apsize, ssize, local); |
631 | ||
1da177e4 LT |
632 | local_irq_restore(flags); |
633 | } | |
634 | ||
e34aa03c | 635 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
fa1f8ae8 AK |
636 | static void native_hugepage_invalidate(unsigned long vsid, |
637 | unsigned long addr, | |
1a527286 | 638 | unsigned char *hpte_slot_array, |
d557b098 | 639 | int psize, int ssize, int local) |
1a527286 | 640 | { |
969b7b20 | 641 | int i; |
1a527286 AK |
642 | struct hash_pte *hptep; |
643 | int actual_psize = MMU_PAGE_16M; | |
644 | unsigned int max_hpte_count, valid; | |
645 | unsigned long flags, s_addr = addr; | |
646 | unsigned long hpte_v, want_v, shift; | |
fa1f8ae8 | 647 | unsigned long hidx, vpn = 0, hash, slot; |
1a527286 AK |
648 | |
649 | shift = mmu_psize_defs[psize].shift; | |
650 | max_hpte_count = 1U << (PMD_SHIFT - shift); | |
651 | ||
652 | local_irq_save(flags); | |
653 | for (i = 0; i < max_hpte_count; i++) { | |
654 | valid = hpte_valid(hpte_slot_array, i); | |
655 | if (!valid) | |
656 | continue; | |
657 | hidx = hpte_hash_index(hpte_slot_array, i); | |
658 | ||
659 | /* get the vpn */ | |
660 | addr = s_addr + (i * (1ul << shift)); | |
1a527286 AK |
661 | vpn = hpt_vpn(addr, vsid, ssize); |
662 | hash = hpt_hash(vpn, shift, ssize); | |
663 | if (hidx & _PTEIDX_SECONDARY) | |
664 | hash = ~hash; | |
665 | ||
666 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | |
667 | slot += hidx & _PTEIDX_GROUP_IX; | |
668 | ||
669 | hptep = htab_address + slot; | |
670 | want_v = hpte_encode_avpn(vpn, psize, ssize); | |
a833280b | 671 | hpte_v = hpte_get_old_v(hptep); |
1a527286 AK |
672 | |
673 | /* Even if we miss, we need to invalidate the TLB */ | |
27d8959d AK |
674 | if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) { |
675 | /* recheck with locks held */ | |
676 | native_lock_hpte(hptep); | |
677 | hpte_v = hpte_get_old_v(hptep); | |
678 | ||
679 | if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) { | |
680 | /* | |
681 | * Invalidate the hpte. NOTE: this also unlocks it | |
682 | */ | |
683 | ||
684 | hptep->v = 0; | |
685 | } else | |
686 | native_unlock_hpte(hptep); | |
687 | } | |
969b7b20 AK |
688 | /* |
689 | * We need to do tlb invalidate for all the address, tlbie | |
690 | * instruction compares entry_VA in tlb with the VA specified | |
691 | * here | |
692 | */ | |
d557b098 | 693 | tlbie(vpn, psize, actual_psize, ssize, local); |
1a527286 | 694 | } |
1a527286 AK |
695 | local_irq_restore(flags); |
696 | } | |
e34aa03c AK |
697 | #else |
698 | static void native_hugepage_invalidate(unsigned long vsid, | |
699 | unsigned long addr, | |
700 | unsigned char *hpte_slot_array, | |
701 | int psize, int ssize, int local) | |
702 | { | |
703 | WARN(1, "%s called without THP support\n", __func__); | |
704 | } | |
705 | #endif | |
1a527286 | 706 | |
8e561e7e | 707 | static void hpte_decode(struct hash_pte *hpte, unsigned long slot, |
b1022fbd | 708 | int *psize, int *apsize, int *ssize, unsigned long *vpn) |
71bf08b6 | 709 | { |
dcda287a | 710 | unsigned long avpn, pteg, vpi; |
12f04f2b AB |
711 | unsigned long hpte_v = be64_to_cpu(hpte->v); |
712 | unsigned long hpte_r = be64_to_cpu(hpte->r); | |
dcda287a | 713 | unsigned long vsid, seg_off; |
7e74c392 AK |
714 | int size, a_size, shift; |
715 | /* Look at the 8 bit LP value */ | |
12f04f2b | 716 | unsigned int lp = (hpte_r >> LP_SHIFT) & ((1 << LP_BITS) - 1); |
71bf08b6 | 717 | |
6b243fcf PM |
718 | if (cpu_has_feature(CPU_FTR_ARCH_300)) { |
719 | hpte_v = hpte_new_to_old_v(hpte_v, hpte_r); | |
720 | hpte_r = hpte_new_to_old_r(hpte_r); | |
721 | } | |
b1022fbd AK |
722 | if (!(hpte_v & HPTE_V_LARGE)) { |
723 | size = MMU_PAGE_4K; | |
724 | a_size = MMU_PAGE_4K; | |
725 | } else { | |
0eeede0c PM |
726 | size = hpte_page_sizes[lp] & 0xf; |
727 | a_size = hpte_page_sizes[lp] >> 4; | |
71bf08b6 | 728 | } |
2454c7e9 | 729 | /* This works for all page sizes, and for 256M and 1T segments */ |
6b243fcf | 730 | *ssize = hpte_v >> HPTE_V_SSIZE_SHIFT; |
71bf08b6 | 731 | shift = mmu_psize_defs[size].shift; |
71bf08b6 | 732 | |
dcda287a AK |
733 | avpn = (HPTE_V_AVPN_VAL(hpte_v) & ~mmu_psize_defs[size].avpnm); |
734 | pteg = slot / HPTES_PER_GROUP; | |
735 | if (hpte_v & HPTE_V_SECONDARY) | |
736 | pteg = ~pteg; | |
737 | ||
738 | switch (*ssize) { | |
739 | case MMU_SEGSIZE_256M: | |
740 | /* We only have 28 - 23 bits of seg_off in avpn */ | |
741 | seg_off = (avpn & 0x1f) << 23; | |
742 | vsid = avpn >> 5; | |
743 | /* We can find more bits from the pteg value */ | |
744 | if (shift < 23) { | |
745 | vpi = (vsid ^ pteg) & htab_hash_mask; | |
746 | seg_off |= vpi << shift; | |
747 | } | |
5524a27d | 748 | *vpn = vsid << (SID_SHIFT - VPN_SHIFT) | seg_off >> VPN_SHIFT; |
83383b73 | 749 | break; |
dcda287a AK |
750 | case MMU_SEGSIZE_1T: |
751 | /* We only have 40 - 23 bits of seg_off in avpn */ | |
752 | seg_off = (avpn & 0x1ffff) << 23; | |
753 | vsid = avpn >> 17; | |
754 | if (shift < 23) { | |
2454c7e9 | 755 | vpi = (vsid ^ (vsid << 25) ^ pteg) & htab_hash_mask; |
dcda287a | 756 | seg_off |= vpi << shift; |
71bf08b6 | 757 | } |
5524a27d | 758 | *vpn = vsid << (SID_SHIFT_1T - VPN_SHIFT) | seg_off >> VPN_SHIFT; |
83383b73 | 759 | break; |
dcda287a | 760 | default: |
5524a27d | 761 | *vpn = size = 0; |
3c726f8d | 762 | } |
b1022fbd AK |
763 | *psize = size; |
764 | *apsize = a_size; | |
3c726f8d BH |
765 | } |
766 | ||
f4c82d51 S |
767 | /* |
768 | * clear all mappings on kexec. All cpus are in real mode (or they will | |
769 | * be when they isi), and we are the only one left. We rely on our kernel | |
770 | * mapping being 0xC0's and the hardware ignoring those two real bits. | |
771 | * | |
fdf880a6 CB |
772 | * This must be called with interrupts disabled. |
773 | * | |
774 | * Taking the native_tlbie_lock is unsafe here due to the possibility of | |
775 | * lockdep being on. On pre POWER5 hardware, not taking the lock could | |
776 | * cause deadlock. POWER5 and newer not taking the lock is fine. This only | |
777 | * gets called during boot before secondary CPUs have come up and during | |
778 | * crashdump and all bets are off anyway. | |
779 | * | |
f4c82d51 | 780 | * TODO: add batching support when enabled. remember, no dynamic memory here, |
027dfac6 | 781 | * although there is the control page available... |
f4c82d51 S |
782 | */ |
783 | static void native_hpte_clear(void) | |
784 | { | |
5524a27d | 785 | unsigned long vpn = 0; |
fdf880a6 | 786 | unsigned long slot, slots; |
8e561e7e | 787 | struct hash_pte *hptep = htab_address; |
5524a27d | 788 | unsigned long hpte_v; |
f4c82d51 | 789 | unsigned long pteg_count; |
b1022fbd | 790 | int psize, apsize, ssize; |
f4c82d51 S |
791 | |
792 | pteg_count = htab_hash_mask + 1; | |
793 | ||
f4c82d51 S |
794 | slots = pteg_count * HPTES_PER_GROUP; |
795 | ||
796 | for (slot = 0; slot < slots; slot++, hptep++) { | |
797 | /* | |
798 | * we could lock the pte here, but we are the only cpu | |
799 | * running, right? and for crash dump, we probably | |
800 | * don't want to wait for a maybe bad cpu. | |
801 | */ | |
12f04f2b | 802 | hpte_v = be64_to_cpu(hptep->v); |
f4c82d51 | 803 | |
47f78a49 | 804 | /* |
fdf880a6 CB |
805 | * Call __tlbie() here rather than tlbie() since we can't take the |
806 | * native_tlbie_lock. | |
47f78a49 | 807 | */ |
96e28449 | 808 | if (hpte_v & HPTE_V_VALID) { |
b1022fbd | 809 | hpte_decode(hptep, slot, &psize, &apsize, &ssize, &vpn); |
96e28449 | 810 | hptep->v = 0; |
a3961f82 | 811 | ___tlbie(vpn, psize, apsize, ssize); |
f4c82d51 S |
812 | } |
813 | } | |
814 | ||
47f78a49 | 815 | asm volatile("eieio; tlbsync; ptesync":::"memory"); |
f4c82d51 S |
816 | } |
817 | ||
3c726f8d BH |
818 | /* |
819 | * Batched hash table flush, we batch the tlbie's to avoid taking/releasing | |
820 | * the lock all the time | |
821 | */ | |
61b1a942 | 822 | static void native_flush_hash_range(unsigned long number, int local) |
1da177e4 | 823 | { |
a5d4b589 | 824 | unsigned long vpn = 0; |
5524a27d | 825 | unsigned long hash, index, hidx, shift, slot; |
8e561e7e | 826 | struct hash_pte *hptep; |
96e28449 | 827 | unsigned long hpte_v; |
3c726f8d BH |
828 | unsigned long want_v; |
829 | unsigned long flags; | |
830 | real_pte_t pte; | |
69111bac | 831 | struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch); |
3c726f8d | 832 | unsigned long psize = batch->psize; |
1189be65 | 833 | int ssize = batch->ssize; |
3c726f8d | 834 | int i; |
88b1bf72 FB |
835 | unsigned int use_local; |
836 | ||
837 | use_local = local && mmu_has_feature(MMU_FTR_TLBIEL) && | |
838 | mmu_psize_defs[psize].tlbiel && !cxl_ctx_in_use(); | |
1da177e4 LT |
839 | |
840 | local_irq_save(flags); | |
841 | ||
1da177e4 | 842 | for (i = 0; i < number; i++) { |
5524a27d | 843 | vpn = batch->vpn[i]; |
3c726f8d BH |
844 | pte = batch->pte[i]; |
845 | ||
5524a27d AK |
846 | pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) { |
847 | hash = hpt_hash(vpn, shift, ssize); | |
3c726f8d BH |
848 | hidx = __rpte_to_hidx(pte, index); |
849 | if (hidx & _PTEIDX_SECONDARY) | |
850 | hash = ~hash; | |
851 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | |
852 | slot += hidx & _PTEIDX_GROUP_IX; | |
853 | hptep = htab_address + slot; | |
74f227b2 | 854 | want_v = hpte_encode_avpn(vpn, psize, ssize); |
27d8959d AK |
855 | hpte_v = hpte_get_old_v(hptep); |
856 | ||
857 | if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) | |
858 | continue; | |
859 | /* lock and try again */ | |
3c726f8d | 860 | native_lock_hpte(hptep); |
a833280b | 861 | hpte_v = hpte_get_old_v(hptep); |
27d8959d AK |
862 | |
863 | if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) | |
3c726f8d BH |
864 | native_unlock_hpte(hptep); |
865 | else | |
866 | hptep->v = 0; | |
27d8959d | 867 | |
3c726f8d | 868 | } pte_iterate_hashed_end(); |
1da177e4 LT |
869 | } |
870 | ||
88b1bf72 | 871 | if (use_local) { |
1da177e4 | 872 | asm volatile("ptesync":::"memory"); |
3c726f8d | 873 | for (i = 0; i < number; i++) { |
5524a27d | 874 | vpn = batch->vpn[i]; |
3c726f8d BH |
875 | pte = batch->pte[i]; |
876 | ||
5524a27d AK |
877 | pte_iterate_hashed_subpages(pte, psize, |
878 | vpn, index, shift) { | |
b1022fbd | 879 | __tlbiel(vpn, psize, psize, ssize); |
3c726f8d BH |
880 | } pte_iterate_hashed_end(); |
881 | } | |
1da177e4 LT |
882 | asm volatile("ptesync":::"memory"); |
883 | } else { | |
44ae3ab3 | 884 | int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); |
1da177e4 LT |
885 | |
886 | if (lock_tlbie) | |
6b9c9b8a | 887 | raw_spin_lock(&native_tlbie_lock); |
1da177e4 LT |
888 | |
889 | asm volatile("ptesync":::"memory"); | |
3c726f8d | 890 | for (i = 0; i < number; i++) { |
5524a27d | 891 | vpn = batch->vpn[i]; |
3c726f8d BH |
892 | pte = batch->pte[i]; |
893 | ||
5524a27d AK |
894 | pte_iterate_hashed_subpages(pte, psize, |
895 | vpn, index, shift) { | |
b1022fbd | 896 | __tlbie(vpn, psize, psize, ssize); |
3c726f8d BH |
897 | } pte_iterate_hashed_end(); |
898 | } | |
a5d4b589 AK |
899 | /* |
900 | * Just do one more with the last used values. | |
901 | */ | |
047e6575 | 902 | fixup_tlbie_vpn(vpn, psize, psize, ssize); |
1da177e4 LT |
903 | asm volatile("eieio; tlbsync; ptesync":::"memory"); |
904 | ||
905 | if (lock_tlbie) | |
6b9c9b8a | 906 | raw_spin_unlock(&native_tlbie_lock); |
1da177e4 LT |
907 | } |
908 | ||
909 | local_irq_restore(flags); | |
910 | } | |
911 | ||
7d0daae4 | 912 | void __init hpte_init_native(void) |
1da177e4 | 913 | { |
7025776e BH |
914 | mmu_hash_ops.hpte_invalidate = native_hpte_invalidate; |
915 | mmu_hash_ops.hpte_updatepp = native_hpte_updatepp; | |
916 | mmu_hash_ops.hpte_updateboltedpp = native_hpte_updateboltedpp; | |
1b644f57 | 917 | mmu_hash_ops.hpte_removebolted = native_hpte_removebolted; |
7025776e BH |
918 | mmu_hash_ops.hpte_insert = native_hpte_insert; |
919 | mmu_hash_ops.hpte_remove = native_hpte_remove; | |
920 | mmu_hash_ops.hpte_clear_all = native_hpte_clear; | |
921 | mmu_hash_ops.flush_hash_range = native_flush_hash_range; | |
922 | mmu_hash_ops.hugepage_invalidate = native_hugepage_invalidate; | |
1da177e4 | 923 | } |