]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * ppc64 MMU hashtable management routines | |
3 | * | |
3c726f8d | 4 | * (c) Copyright IBM Corp. 2003, 2005 |
1da177e4 LT |
5 | * |
6 | * Maintained by: Benjamin Herrenschmidt | |
7 | * <benh@kernel.crashing.org> | |
8 | * | |
9 | * This file is covered by the GNU Public Licence v2 as | |
10 | * described in the kernel's COPYING file. | |
11 | */ | |
12 | ||
3c726f8d | 13 | #include <linux/config.h> |
ab1f9dac | 14 | #include <asm/reg.h> |
1da177e4 LT |
15 | #include <asm/pgtable.h> |
16 | #include <asm/mmu.h> | |
17 | #include <asm/page.h> | |
18 | #include <asm/types.h> | |
19 | #include <asm/ppc_asm.h> | |
0013a854 | 20 | #include <asm/asm-offsets.h> |
1da177e4 LT |
21 | #include <asm/cputable.h> |
22 | ||
23 | .text | |
24 | ||
25 | /* | |
26 | * Stackframe: | |
27 | * | |
28 | * +-> Back chain (SP + 256) | |
29 | * | General register save area (SP + 112) | |
30 | * | Parameter save area (SP + 48) | |
31 | * | TOC save area (SP + 40) | |
32 | * | link editor doubleword (SP + 32) | |
33 | * | compiler doubleword (SP + 24) | |
34 | * | LR save area (SP + 16) | |
35 | * | CR save area (SP + 8) | |
36 | * SP ---> +-- Back chain (SP + 0) | |
37 | */ | |
38 | #define STACKFRAMESIZE 256 | |
39 | ||
40 | /* Save parameters offsets */ | |
41 | #define STK_PARM(i) (STACKFRAMESIZE + 48 + ((i)-3)*8) | |
42 | ||
43 | /* Save non-volatile offsets */ | |
44 | #define STK_REG(i) (112 + ((i)-14)*8) | |
45 | ||
3c726f8d BH |
46 | |
47 | #ifndef CONFIG_PPC_64K_PAGES | |
48 | ||
49 | /***************************************************************************** | |
50 | * * | |
51 | * 4K SW & 4K HW pages implementation * | |
52 | * * | |
53 | *****************************************************************************/ | |
54 | ||
55 | ||
1da177e4 | 56 | /* |
3c726f8d BH |
57 | * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, |
58 | * pte_t *ptep, unsigned long trap, int local) | |
1da177e4 | 59 | * |
3c726f8d | 60 | * Adds a 4K page to the hash table in a segment of 4K pages only |
1da177e4 LT |
61 | */ |
62 | ||
3c726f8d | 63 | _GLOBAL(__hash_page_4K) |
1da177e4 LT |
64 | mflr r0 |
65 | std r0,16(r1) | |
66 | stdu r1,-STACKFRAMESIZE(r1) | |
67 | /* Save all params that we need after a function call */ | |
68 | std r6,STK_PARM(r6)(r1) | |
69 | std r8,STK_PARM(r8)(r1) | |
70 | ||
71 | /* Add _PAGE_PRESENT to access */ | |
72 | ori r4,r4,_PAGE_PRESENT | |
73 | ||
74 | /* Save non-volatile registers. | |
75 | * r31 will hold "old PTE" | |
76 | * r30 is "new PTE" | |
77 | * r29 is "va" | |
78 | * r28 is a hash value | |
79 | * r27 is hashtab mask (maybe dynamic patched instead ?) | |
80 | */ | |
81 | std r27,STK_REG(r27)(r1) | |
82 | std r28,STK_REG(r28)(r1) | |
83 | std r29,STK_REG(r29)(r1) | |
84 | std r30,STK_REG(r30)(r1) | |
85 | std r31,STK_REG(r31)(r1) | |
86 | ||
87 | /* Step 1: | |
88 | * | |
89 | * Check permissions, atomically mark the linux PTE busy | |
90 | * and hashed. | |
91 | */ | |
92 | 1: | |
93 | ldarx r31,0,r6 | |
94 | /* Check access rights (access & ~(pte_val(*ptep))) */ | |
95 | andc. r0,r4,r31 | |
96 | bne- htab_wrong_access | |
97 | /* Check if PTE is busy */ | |
98 | andi. r0,r31,_PAGE_BUSY | |
d03853d5 OJ |
99 | /* If so, just bail out and refault if needed. Someone else |
100 | * is changing this PTE anyway and might hash it. | |
101 | */ | |
3c726f8d BH |
102 | bne- htab_bail_ok |
103 | ||
1da177e4 LT |
104 | /* Prepare new PTE value (turn access RW into DIRTY, then |
105 | * add BUSY,HASHPTE and ACCESSED) | |
106 | */ | |
107 | rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ | |
108 | or r30,r30,r31 | |
109 | ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE | |
110 | /* Write the linux PTE atomically (setting busy) */ | |
111 | stdcx. r30,0,r6 | |
112 | bne- 1b | |
113 | isync | |
114 | ||
115 | /* Step 2: | |
116 | * | |
117 | * Insert/Update the HPTE in the hash table. At this point, | |
118 | * r4 (access) is re-useable, we use it for the new HPTE flags | |
119 | */ | |
120 | ||
121 | /* Calc va and put it in r29 */ | |
122 | rldicr r29,r5,28,63-28 | |
123 | rldicl r3,r3,0,36 | |
124 | or r29,r3,r29 | |
125 | ||
126 | /* Calculate hash value for primary slot and store it in r28 */ | |
127 | rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ | |
128 | rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */ | |
129 | xor r28,r5,r0 | |
130 | ||
131 | /* Convert linux PTE bits into HW equivalents */ | |
132 | andi. r3,r30,0x1fe /* Get basic set of flags */ | |
3c726f8d | 133 | xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ |
1da177e4 LT |
134 | rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ |
135 | rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ | |
3c726f8d | 136 | and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ |
1da177e4 LT |
137 | andc r0,r30,r0 /* r0 = pte & ~r0 */ |
138 | rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ | |
139 | ||
140 | /* We eventually do the icache sync here (maybe inline that | |
141 | * code rather than call a C function...) | |
142 | */ | |
1da177e4 LT |
143 | BEGIN_FTR_SECTION |
144 | mr r4,r30 | |
145 | mr r5,r7 | |
146 | bl .hash_page_do_lazy_icache | |
8913ca1c | 147 | END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) |
1da177e4 LT |
148 | |
149 | /* At this point, r3 contains new PP bits, save them in | |
150 | * place of "access" in the param area (sic) | |
151 | */ | |
152 | std r3,STK_PARM(r4)(r1) | |
153 | ||
154 | /* Get htab_hash_mask */ | |
155 | ld r4,htab_hash_mask@got(2) | |
156 | ld r27,0(r4) /* htab_hash_mask -> r27 */ | |
157 | ||
158 | /* Check if we may already be in the hashtable, in this case, we | |
159 | * go to out-of-line code to try to modify the HPTE | |
160 | */ | |
161 | andi. r0,r31,_PAGE_HASHPTE | |
162 | bne htab_modify_pte | |
163 | ||
164 | htab_insert_pte: | |
165 | /* Clear hpte bits in new pte (we also clear BUSY btw) and | |
166 | * add _PAGE_HASHPTE | |
167 | */ | |
168 | lis r0,_PAGE_HPTEFLAGS@h | |
169 | ori r0,r0,_PAGE_HPTEFLAGS@l | |
170 | andc r30,r30,r0 | |
171 | ori r30,r30,_PAGE_HASHPTE | |
172 | ||
3c726f8d BH |
173 | /* physical address r5 */ |
174 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
175 | sldi r5,r5,PAGE_SHIFT | |
1da177e4 LT |
176 | |
177 | /* Calculate primary group hash */ | |
178 | and r0,r28,r27 | |
3c726f8d | 179 | rldicr r3,r0,3,63-3 /* r3 = (hash & mask) << 3 */ |
1da177e4 LT |
180 | |
181 | /* Call ppc_md.hpte_insert */ | |
3c726f8d | 182 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ |
1da177e4 | 183 | mr r4,r29 /* Retreive va */ |
3c726f8d BH |
184 | li r7,0 /* !bolted, !secondary */ |
185 | li r8,MMU_PAGE_4K /* page size */ | |
1da177e4 | 186 | _GLOBAL(htab_call_hpte_insert1) |
3c726f8d | 187 | bl . /* Patched by htab_finish_init() */ |
1da177e4 LT |
188 | cmpdi 0,r3,0 |
189 | bge htab_pte_insert_ok /* Insertion successful */ | |
190 | cmpdi 0,r3,-2 /* Critical failure */ | |
191 | beq- htab_pte_insert_failure | |
192 | ||
193 | /* Now try secondary slot */ | |
194 | ||
3c726f8d BH |
195 | /* physical address r5 */ |
196 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
197 | sldi r5,r5,PAGE_SHIFT | |
1da177e4 LT |
198 | |
199 | /* Calculate secondary group hash */ | |
200 | andc r0,r27,r28 | |
201 | rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ | |
202 | ||
203 | /* Call ppc_md.hpte_insert */ | |
3c726f8d | 204 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ |
1da177e4 | 205 | mr r4,r29 /* Retreive va */ |
3c726f8d BH |
206 | li r7,HPTE_V_SECONDARY /* !bolted, secondary */ |
207 | li r8,MMU_PAGE_4K /* page size */ | |
1da177e4 | 208 | _GLOBAL(htab_call_hpte_insert2) |
3c726f8d | 209 | bl . /* Patched by htab_finish_init() */ |
1da177e4 LT |
210 | cmpdi 0,r3,0 |
211 | bge+ htab_pte_insert_ok /* Insertion successful */ | |
212 | cmpdi 0,r3,-2 /* Critical failure */ | |
213 | beq- htab_pte_insert_failure | |
214 | ||
215 | /* Both are full, we need to evict something */ | |
216 | mftb r0 | |
217 | /* Pick a random group based on TB */ | |
218 | andi. r0,r0,1 | |
219 | mr r5,r28 | |
220 | bne 2f | |
221 | not r5,r5 | |
222 | 2: and r0,r5,r27 | |
223 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
224 | /* Call ppc_md.hpte_remove */ | |
225 | _GLOBAL(htab_call_hpte_remove) | |
3c726f8d | 226 | bl . /* Patched by htab_finish_init() */ |
1da177e4 LT |
227 | |
228 | /* Try all again */ | |
229 | b htab_insert_pte | |
230 | ||
3c726f8d | 231 | htab_bail_ok: |
d03853d5 | 232 | li r3,0 |
3c726f8d | 233 | b htab_bail |
d03853d5 | 234 | |
1da177e4 LT |
235 | htab_pte_insert_ok: |
236 | /* Insert slot number & secondary bit in PTE */ | |
237 | rldimi r30,r3,12,63-15 | |
238 | ||
239 | /* Write out the PTE with a normal write | |
240 | * (maybe add eieio may be good still ?) | |
241 | */ | |
242 | htab_write_out_pte: | |
243 | ld r6,STK_PARM(r6)(r1) | |
244 | std r30,0(r6) | |
245 | li r3, 0 | |
3c726f8d | 246 | htab_bail: |
1da177e4 LT |
247 | ld r27,STK_REG(r27)(r1) |
248 | ld r28,STK_REG(r28)(r1) | |
249 | ld r29,STK_REG(r29)(r1) | |
250 | ld r30,STK_REG(r30)(r1) | |
251 | ld r31,STK_REG(r31)(r1) | |
252 | addi r1,r1,STACKFRAMESIZE | |
253 | ld r0,16(r1) | |
254 | mtlr r0 | |
255 | blr | |
256 | ||
257 | htab_modify_pte: | |
258 | /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ | |
259 | mr r4,r3 | |
260 | rlwinm r3,r31,32-12,29,31 | |
261 | ||
262 | /* Secondary group ? if yes, get a inverted hash value */ | |
263 | mr r5,r28 | |
264 | andi. r0,r31,_PAGE_SECONDARY | |
265 | beq 1f | |
266 | not r5,r5 | |
267 | 1: | |
268 | /* Calculate proper slot value for ppc_md.hpte_updatepp */ | |
269 | and r0,r5,r27 | |
270 | rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
271 | add r3,r0,r3 /* add slot idx */ | |
272 | ||
273 | /* Call ppc_md.hpte_updatepp */ | |
274 | mr r5,r29 /* va */ | |
3c726f8d | 275 | li r6,MMU_PAGE_4K /* page size */ |
1da177e4 LT |
276 | ld r7,STK_PARM(r8)(r1) /* get "local" param */ |
277 | _GLOBAL(htab_call_hpte_updatepp) | |
3c726f8d | 278 | bl . /* Patched by htab_finish_init() */ |
1da177e4 LT |
279 | |
280 | /* if we failed because typically the HPTE wasn't really here | |
281 | * we try an insertion. | |
282 | */ | |
283 | cmpdi 0,r3,-1 | |
284 | beq- htab_insert_pte | |
285 | ||
286 | /* Clear the BUSY bit and Write out the PTE */ | |
287 | li r0,_PAGE_BUSY | |
288 | andc r30,r30,r0 | |
289 | b htab_write_out_pte | |
290 | ||
291 | htab_wrong_access: | |
292 | /* Bail out clearing reservation */ | |
293 | stdcx. r31,0,r6 | |
294 | li r3,1 | |
3c726f8d BH |
295 | b htab_bail |
296 | ||
297 | htab_pte_insert_failure: | |
298 | /* Bail out restoring old PTE */ | |
299 | ld r6,STK_PARM(r6)(r1) | |
300 | std r31,0(r6) | |
301 | li r3,-1 | |
302 | b htab_bail | |
303 | ||
304 | ||
305 | #else /* CONFIG_PPC_64K_PAGES */ | |
306 | ||
307 | ||
308 | /***************************************************************************** | |
309 | * * | |
310 | * 64K SW & 4K or 64K HW in a 4K segment pages implementation * | |
311 | * * | |
312 | *****************************************************************************/ | |
313 | ||
314 | /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, | |
315 | * pte_t *ptep, unsigned long trap, int local) | |
316 | */ | |
317 | ||
318 | /* | |
319 | * For now, we do NOT implement Admixed pages | |
320 | */ | |
321 | _GLOBAL(__hash_page_4K) | |
322 | mflr r0 | |
323 | std r0,16(r1) | |
324 | stdu r1,-STACKFRAMESIZE(r1) | |
325 | /* Save all params that we need after a function call */ | |
326 | std r6,STK_PARM(r6)(r1) | |
327 | std r8,STK_PARM(r8)(r1) | |
328 | ||
329 | /* Add _PAGE_PRESENT to access */ | |
330 | ori r4,r4,_PAGE_PRESENT | |
331 | ||
332 | /* Save non-volatile registers. | |
333 | * r31 will hold "old PTE" | |
334 | * r30 is "new PTE" | |
335 | * r29 is "va" | |
336 | * r28 is a hash value | |
337 | * r27 is hashtab mask (maybe dynamic patched instead ?) | |
338 | * r26 is the hidx mask | |
339 | * r25 is the index in combo page | |
340 | */ | |
341 | std r25,STK_REG(r25)(r1) | |
342 | std r26,STK_REG(r26)(r1) | |
343 | std r27,STK_REG(r27)(r1) | |
344 | std r28,STK_REG(r28)(r1) | |
345 | std r29,STK_REG(r29)(r1) | |
346 | std r30,STK_REG(r30)(r1) | |
347 | std r31,STK_REG(r31)(r1) | |
348 | ||
349 | /* Step 1: | |
350 | * | |
351 | * Check permissions, atomically mark the linux PTE busy | |
352 | * and hashed. | |
353 | */ | |
354 | 1: | |
355 | ldarx r31,0,r6 | |
356 | /* Check access rights (access & ~(pte_val(*ptep))) */ | |
357 | andc. r0,r4,r31 | |
358 | bne- htab_wrong_access | |
359 | /* Check if PTE is busy */ | |
360 | andi. r0,r31,_PAGE_BUSY | |
361 | /* If so, just bail out and refault if needed. Someone else | |
362 | * is changing this PTE anyway and might hash it. | |
363 | */ | |
364 | bne- htab_bail_ok | |
365 | /* Prepare new PTE value (turn access RW into DIRTY, then | |
366 | * add BUSY and ACCESSED) | |
367 | */ | |
368 | rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ | |
369 | or r30,r30,r31 | |
370 | ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE | |
371 | /* Write the linux PTE atomically (setting busy) */ | |
372 | stdcx. r30,0,r6 | |
373 | bne- 1b | |
374 | isync | |
375 | ||
376 | /* Step 2: | |
377 | * | |
378 | * Insert/Update the HPTE in the hash table. At this point, | |
379 | * r4 (access) is re-useable, we use it for the new HPTE flags | |
380 | */ | |
381 | ||
382 | /* Load the hidx index */ | |
383 | rldicl r25,r3,64-12,60 | |
384 | ||
385 | /* Calc va and put it in r29 */ | |
386 | rldicr r29,r5,28,63-28 /* r29 = (vsid << 28) */ | |
387 | rldicl r3,r3,0,36 /* r3 = (ea & 0x0fffffff) */ | |
388 | or r29,r3,r29 /* r29 = va | |
389 | ||
390 | /* Calculate hash value for primary slot and store it in r28 */ | |
391 | rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ | |
392 | rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */ | |
393 | xor r28,r5,r0 | |
394 | ||
395 | /* Convert linux PTE bits into HW equivalents */ | |
396 | andi. r3,r30,0x1fe /* Get basic set of flags */ | |
397 | xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ | |
398 | rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ | |
399 | rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ | |
400 | and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ | |
401 | andc r0,r30,r0 /* r0 = pte & ~r0 */ | |
402 | rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ | |
403 | ||
404 | /* We eventually do the icache sync here (maybe inline that | |
405 | * code rather than call a C function...) | |
406 | */ | |
407 | BEGIN_FTR_SECTION | |
408 | mr r4,r30 | |
409 | mr r5,r7 | |
410 | bl .hash_page_do_lazy_icache | |
411 | END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) | |
412 | ||
413 | /* At this point, r3 contains new PP bits, save them in | |
414 | * place of "access" in the param area (sic) | |
415 | */ | |
416 | std r3,STK_PARM(r4)(r1) | |
417 | ||
418 | /* Get htab_hash_mask */ | |
419 | ld r4,htab_hash_mask@got(2) | |
420 | ld r27,0(r4) /* htab_hash_mask -> r27 */ | |
421 | ||
422 | /* Check if we may already be in the hashtable, in this case, we | |
423 | * go to out-of-line code to try to modify the HPTE. We look for | |
424 | * the bit at (1 >> (index + 32)) | |
425 | */ | |
426 | andi. r0,r31,_PAGE_HASHPTE | |
427 | li r26,0 /* Default hidx */ | |
428 | beq htab_insert_pte | |
429 | ld r6,STK_PARM(r6)(r1) | |
430 | ori r26,r6,0x8000 /* Load the hidx mask */ | |
431 | ld r26,0(r26) | |
432 | addi r5,r25,36 /* Check actual HPTE_SUB bit, this */ | |
433 | rldcr. r0,r31,r5,0 /* must match pgtable.h definition */ | |
434 | bne htab_modify_pte | |
435 | ||
436 | htab_insert_pte: | |
437 | /* real page number in r5, PTE RPN value + index */ | |
438 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
439 | sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT | |
440 | add r5,r5,r25 | |
441 | sldi r5,r5,HW_PAGE_SHIFT | |
442 | ||
443 | /* Calculate primary group hash */ | |
444 | and r0,r28,r27 | |
445 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
446 | ||
447 | /* Call ppc_md.hpte_insert */ | |
448 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ | |
449 | mr r4,r29 /* Retreive va */ | |
450 | li r7,0 /* !bolted, !secondary */ | |
451 | li r8,MMU_PAGE_4K /* page size */ | |
452 | _GLOBAL(htab_call_hpte_insert1) | |
453 | bl . /* patched by htab_finish_init() */ | |
454 | cmpdi 0,r3,0 | |
455 | bge htab_pte_insert_ok /* Insertion successful */ | |
456 | cmpdi 0,r3,-2 /* Critical failure */ | |
457 | beq- htab_pte_insert_failure | |
458 | ||
459 | /* Now try secondary slot */ | |
460 | ||
461 | /* real page number in r5, PTE RPN value + index */ | |
462 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
463 | sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT | |
464 | add r5,r5,r25 | |
465 | sldi r5,r5,HW_PAGE_SHIFT | |
466 | ||
467 | /* Calculate secondary group hash */ | |
468 | andc r0,r27,r28 | |
469 | rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ | |
470 | ||
471 | /* Call ppc_md.hpte_insert */ | |
472 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ | |
473 | mr r4,r29 /* Retreive va */ | |
474 | li r7,HPTE_V_SECONDARY /* !bolted, secondary */ | |
475 | li r8,MMU_PAGE_4K /* page size */ | |
476 | _GLOBAL(htab_call_hpte_insert2) | |
477 | bl . /* patched by htab_finish_init() */ | |
478 | cmpdi 0,r3,0 | |
479 | bge+ htab_pte_insert_ok /* Insertion successful */ | |
480 | cmpdi 0,r3,-2 /* Critical failure */ | |
481 | beq- htab_pte_insert_failure | |
482 | ||
483 | /* Both are full, we need to evict something */ | |
484 | mftb r0 | |
485 | /* Pick a random group based on TB */ | |
486 | andi. r0,r0,1 | |
487 | mr r5,r28 | |
488 | bne 2f | |
489 | not r5,r5 | |
490 | 2: and r0,r5,r27 | |
491 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
492 | /* Call ppc_md.hpte_remove */ | |
493 | _GLOBAL(htab_call_hpte_remove) | |
494 | bl . /* patched by htab_finish_init() */ | |
495 | ||
496 | /* Try all again */ | |
497 | b htab_insert_pte | |
498 | ||
499 | htab_bail_ok: | |
500 | li r3,0 | |
501 | b htab_bail | |
502 | ||
503 | htab_pte_insert_ok: | |
504 | /* Insert slot number & secondary bit in PTE second half, | |
505 | * clear _PAGE_BUSY and set approriate HPTE slot bit | |
506 | */ | |
507 | ld r6,STK_PARM(r6)(r1) | |
508 | li r0,_PAGE_BUSY | |
509 | andc r30,r30,r0 | |
510 | /* HPTE SUB bit */ | |
511 | li r0,1 | |
512 | subfic r5,r25,27 /* Must match bit position in */ | |
513 | sld r0,r0,r5 /* pgtable.h */ | |
514 | or r30,r30,r0 | |
515 | /* hindx */ | |
516 | sldi r5,r25,2 | |
517 | sld r3,r3,r5 | |
518 | li r4,0xf | |
519 | sld r4,r4,r5 | |
520 | andc r26,r26,r4 | |
521 | or r26,r26,r3 | |
522 | ori r5,r6,0x8000 | |
523 | std r26,0(r5) | |
524 | lwsync | |
525 | std r30,0(r6) | |
526 | li r3, 0 | |
527 | htab_bail: | |
528 | ld r25,STK_REG(r25)(r1) | |
529 | ld r26,STK_REG(r26)(r1) | |
530 | ld r27,STK_REG(r27)(r1) | |
531 | ld r28,STK_REG(r28)(r1) | |
532 | ld r29,STK_REG(r29)(r1) | |
533 | ld r30,STK_REG(r30)(r1) | |
534 | ld r31,STK_REG(r31)(r1) | |
535 | addi r1,r1,STACKFRAMESIZE | |
536 | ld r0,16(r1) | |
537 | mtlr r0 | |
538 | blr | |
539 | ||
540 | htab_modify_pte: | |
541 | /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ | |
542 | mr r4,r3 | |
543 | sldi r5,r25,2 | |
544 | srd r3,r26,r5 | |
545 | ||
546 | /* Secondary group ? if yes, get a inverted hash value */ | |
547 | mr r5,r28 | |
548 | andi. r0,r3,0x8 /* page secondary ? */ | |
549 | beq 1f | |
550 | not r5,r5 | |
551 | 1: andi. r3,r3,0x7 /* extract idx alone */ | |
552 | ||
553 | /* Calculate proper slot value for ppc_md.hpte_updatepp */ | |
554 | and r0,r5,r27 | |
555 | rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
556 | add r3,r0,r3 /* add slot idx */ | |
557 | ||
558 | /* Call ppc_md.hpte_updatepp */ | |
559 | mr r5,r29 /* va */ | |
560 | li r6,MMU_PAGE_4K /* page size */ | |
561 | ld r7,STK_PARM(r8)(r1) /* get "local" param */ | |
562 | _GLOBAL(htab_call_hpte_updatepp) | |
563 | bl . /* patched by htab_finish_init() */ | |
564 | ||
565 | /* if we failed because typically the HPTE wasn't really here | |
566 | * we try an insertion. | |
567 | */ | |
568 | cmpdi 0,r3,-1 | |
569 | beq- htab_insert_pte | |
570 | ||
571 | /* Clear the BUSY bit and Write out the PTE */ | |
572 | li r0,_PAGE_BUSY | |
573 | andc r30,r30,r0 | |
574 | ld r6,STK_PARM(r6)(r1) | |
575 | std r30,0(r6) | |
576 | li r3,0 | |
577 | b htab_bail | |
578 | ||
579 | htab_wrong_access: | |
580 | /* Bail out clearing reservation */ | |
581 | stdcx. r31,0,r6 | |
582 | li r3,1 | |
583 | b htab_bail | |
1da177e4 LT |
584 | |
585 | htab_pte_insert_failure: | |
586 | /* Bail out restoring old PTE */ | |
587 | ld r6,STK_PARM(r6)(r1) | |
588 | std r31,0(r6) | |
589 | li r3,-1 | |
3c726f8d BH |
590 | b htab_bail |
591 | ||
592 | ||
593 | /***************************************************************************** | |
594 | * * | |
595 | * 64K SW & 64K HW in a 64K segment pages implementation * | |
596 | * * | |
597 | *****************************************************************************/ | |
598 | ||
599 | _GLOBAL(__hash_page_64K) | |
600 | mflr r0 | |
601 | std r0,16(r1) | |
602 | stdu r1,-STACKFRAMESIZE(r1) | |
603 | /* Save all params that we need after a function call */ | |
604 | std r6,STK_PARM(r6)(r1) | |
605 | std r8,STK_PARM(r8)(r1) | |
606 | ||
607 | /* Add _PAGE_PRESENT to access */ | |
608 | ori r4,r4,_PAGE_PRESENT | |
609 | ||
610 | /* Save non-volatile registers. | |
611 | * r31 will hold "old PTE" | |
612 | * r30 is "new PTE" | |
613 | * r29 is "va" | |
614 | * r28 is a hash value | |
615 | * r27 is hashtab mask (maybe dynamic patched instead ?) | |
616 | */ | |
617 | std r27,STK_REG(r27)(r1) | |
618 | std r28,STK_REG(r28)(r1) | |
619 | std r29,STK_REG(r29)(r1) | |
620 | std r30,STK_REG(r30)(r1) | |
621 | std r31,STK_REG(r31)(r1) | |
622 | ||
623 | /* Step 1: | |
624 | * | |
625 | * Check permissions, atomically mark the linux PTE busy | |
626 | * and hashed. | |
627 | */ | |
628 | 1: | |
629 | ldarx r31,0,r6 | |
630 | /* Check access rights (access & ~(pte_val(*ptep))) */ | |
631 | andc. r0,r4,r31 | |
632 | bne- ht64_wrong_access | |
633 | /* Check if PTE is busy */ | |
634 | andi. r0,r31,_PAGE_BUSY | |
635 | /* If so, just bail out and refault if needed. Someone else | |
636 | * is changing this PTE anyway and might hash it. | |
637 | */ | |
638 | bne- ht64_bail_ok | |
639 | /* Prepare new PTE value (turn access RW into DIRTY, then | |
640 | * add BUSY,HASHPTE and ACCESSED) | |
641 | */ | |
642 | rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ | |
643 | or r30,r30,r31 | |
644 | ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE | |
645 | /* Write the linux PTE atomically (setting busy) */ | |
646 | stdcx. r30,0,r6 | |
647 | bne- 1b | |
648 | isync | |
649 | ||
650 | /* Step 2: | |
651 | * | |
652 | * Insert/Update the HPTE in the hash table. At this point, | |
653 | * r4 (access) is re-useable, we use it for the new HPTE flags | |
654 | */ | |
655 | ||
656 | /* Calc va and put it in r29 */ | |
657 | rldicr r29,r5,28,63-28 | |
658 | rldicl r3,r3,0,36 | |
659 | or r29,r3,r29 | |
660 | ||
661 | /* Calculate hash value for primary slot and store it in r28 */ | |
662 | rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ | |
663 | rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */ | |
664 | xor r28,r5,r0 | |
665 | ||
666 | /* Convert linux PTE bits into HW equivalents */ | |
667 | andi. r3,r30,0x1fe /* Get basic set of flags */ | |
668 | xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ | |
669 | rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ | |
670 | rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ | |
671 | and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ | |
672 | andc r0,r30,r0 /* r0 = pte & ~r0 */ | |
673 | rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ | |
674 | ||
675 | /* We eventually do the icache sync here (maybe inline that | |
676 | * code rather than call a C function...) | |
677 | */ | |
678 | BEGIN_FTR_SECTION | |
679 | mr r4,r30 | |
680 | mr r5,r7 | |
681 | bl .hash_page_do_lazy_icache | |
682 | END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) | |
683 | ||
684 | /* At this point, r3 contains new PP bits, save them in | |
685 | * place of "access" in the param area (sic) | |
686 | */ | |
687 | std r3,STK_PARM(r4)(r1) | |
688 | ||
689 | /* Get htab_hash_mask */ | |
690 | ld r4,htab_hash_mask@got(2) | |
691 | ld r27,0(r4) /* htab_hash_mask -> r27 */ | |
692 | ||
693 | /* Check if we may already be in the hashtable, in this case, we | |
694 | * go to out-of-line code to try to modify the HPTE | |
695 | */ | |
696 | andi. r0,r31,_PAGE_HASHPTE | |
697 | bne ht64_modify_pte | |
698 | ||
699 | ht64_insert_pte: | |
700 | /* Clear hpte bits in new pte (we also clear BUSY btw) and | |
701 | * add _PAGE_HASHPTE | |
702 | */ | |
703 | lis r0,_PAGE_HPTEFLAGS@h | |
704 | ori r0,r0,_PAGE_HPTEFLAGS@l | |
705 | andc r30,r30,r0 | |
706 | ori r30,r30,_PAGE_HASHPTE | |
707 | ||
708 | /* Phyical address in r5 */ | |
709 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
710 | sldi r5,r5,PAGE_SHIFT | |
711 | ||
712 | /* Calculate primary group hash */ | |
713 | and r0,r28,r27 | |
714 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
715 | ||
716 | /* Call ppc_md.hpte_insert */ | |
717 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ | |
718 | mr r4,r29 /* Retreive va */ | |
719 | li r7,0 /* !bolted, !secondary */ | |
720 | li r8,MMU_PAGE_64K | |
721 | _GLOBAL(ht64_call_hpte_insert1) | |
722 | bl . /* patched by htab_finish_init() */ | |
723 | cmpdi 0,r3,0 | |
724 | bge ht64_pte_insert_ok /* Insertion successful */ | |
725 | cmpdi 0,r3,-2 /* Critical failure */ | |
726 | beq- ht64_pte_insert_failure | |
727 | ||
728 | /* Now try secondary slot */ | |
729 | ||
730 | /* Phyical address in r5 */ | |
731 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
732 | sldi r5,r5,PAGE_SHIFT | |
733 | ||
734 | /* Calculate secondary group hash */ | |
735 | andc r0,r27,r28 | |
736 | rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ | |
737 | ||
738 | /* Call ppc_md.hpte_insert */ | |
739 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ | |
740 | mr r4,r29 /* Retreive va */ | |
741 | li r7,HPTE_V_SECONDARY /* !bolted, secondary */ | |
742 | li r8,MMU_PAGE_64K | |
743 | _GLOBAL(ht64_call_hpte_insert2) | |
744 | bl . /* patched by htab_finish_init() */ | |
745 | cmpdi 0,r3,0 | |
746 | bge+ ht64_pte_insert_ok /* Insertion successful */ | |
747 | cmpdi 0,r3,-2 /* Critical failure */ | |
748 | beq- ht64_pte_insert_failure | |
749 | ||
750 | /* Both are full, we need to evict something */ | |
751 | mftb r0 | |
752 | /* Pick a random group based on TB */ | |
753 | andi. r0,r0,1 | |
754 | mr r5,r28 | |
755 | bne 2f | |
756 | not r5,r5 | |
757 | 2: and r0,r5,r27 | |
758 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
759 | /* Call ppc_md.hpte_remove */ | |
760 | _GLOBAL(ht64_call_hpte_remove) | |
761 | bl . /* patched by htab_finish_init() */ | |
762 | ||
763 | /* Try all again */ | |
764 | b ht64_insert_pte | |
765 | ||
766 | ht64_bail_ok: | |
767 | li r3,0 | |
768 | b ht64_bail | |
769 | ||
770 | ht64_pte_insert_ok: | |
771 | /* Insert slot number & secondary bit in PTE */ | |
772 | rldimi r30,r3,12,63-15 | |
773 | ||
774 | /* Write out the PTE with a normal write | |
775 | * (maybe add eieio may be good still ?) | |
776 | */ | |
777 | ht64_write_out_pte: | |
778 | ld r6,STK_PARM(r6)(r1) | |
779 | std r30,0(r6) | |
780 | li r3, 0 | |
781 | ht64_bail: | |
782 | ld r27,STK_REG(r27)(r1) | |
783 | ld r28,STK_REG(r28)(r1) | |
784 | ld r29,STK_REG(r29)(r1) | |
785 | ld r30,STK_REG(r30)(r1) | |
786 | ld r31,STK_REG(r31)(r1) | |
787 | addi r1,r1,STACKFRAMESIZE | |
788 | ld r0,16(r1) | |
789 | mtlr r0 | |
790 | blr | |
791 | ||
792 | ht64_modify_pte: | |
793 | /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ | |
794 | mr r4,r3 | |
795 | rlwinm r3,r31,32-12,29,31 | |
796 | ||
797 | /* Secondary group ? if yes, get a inverted hash value */ | |
798 | mr r5,r28 | |
799 | andi. r0,r31,_PAGE_F_SECOND | |
800 | beq 1f | |
801 | not r5,r5 | |
802 | 1: | |
803 | /* Calculate proper slot value for ppc_md.hpte_updatepp */ | |
804 | and r0,r5,r27 | |
805 | rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
806 | add r3,r0,r3 /* add slot idx */ | |
807 | ||
808 | /* Call ppc_md.hpte_updatepp */ | |
809 | mr r5,r29 /* va */ | |
810 | li r6,MMU_PAGE_64K | |
811 | ld r7,STK_PARM(r8)(r1) /* get "local" param */ | |
812 | _GLOBAL(ht64_call_hpte_updatepp) | |
813 | bl . /* patched by htab_finish_init() */ | |
814 | ||
815 | /* if we failed because typically the HPTE wasn't really here | |
816 | * we try an insertion. | |
817 | */ | |
818 | cmpdi 0,r3,-1 | |
819 | beq- ht64_insert_pte | |
820 | ||
821 | /* Clear the BUSY bit and Write out the PTE */ | |
822 | li r0,_PAGE_BUSY | |
823 | andc r30,r30,r0 | |
824 | b ht64_write_out_pte | |
825 | ||
826 | ht64_wrong_access: | |
827 | /* Bail out clearing reservation */ | |
828 | stdcx. r31,0,r6 | |
829 | li r3,1 | |
830 | b ht64_bail | |
831 | ||
832 | ht64_pte_insert_failure: | |
833 | /* Bail out restoring old PTE */ | |
834 | ld r6,STK_PARM(r6)(r1) | |
835 | std r31,0(r6) | |
836 | li r3,-1 | |
837 | b ht64_bail | |
838 | ||
839 | ||
840 | #endif /* CONFIG_PPC_64K_PAGES */ | |
1da177e4 LT |
841 | |
842 | ||
3c726f8d BH |
843 | /***************************************************************************** |
844 | * * | |
845 | * Huge pages implementation is in hugetlbpage.c * | |
846 | * * | |
847 | *****************************************************************************/ |