]>
Commit | Line | Data |
---|---|---|
fe4a0cf1 IK |
1 | /* |
2 | * "Cell Reference Set" HTAB support. | |
3 | * | |
4 | * (C) Copyright 2006-2007 TOSHIBA CORPORATION | |
5 | * | |
6 | * This code is based on arch/powerpc/platforms/pseries/lpar.c: | |
7 | * Copyright (C) 2001 Todd Inglett, IBM Corporation | |
8 | * | |
9 | * This program is free software; you can redistribute it and/or modify | |
10 | * it under the terms of the GNU General Public License as published by | |
11 | * the Free Software Foundation; either version 2 of the License, or | |
12 | * (at your option) any later version. | |
13 | * | |
14 | * This program is distributed in the hope that it will be useful, | |
15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
17 | * GNU General Public License for more details. | |
18 | * | |
19 | * You should have received a copy of the GNU General Public License along | |
20 | * with this program; if not, write to the Free Software Foundation, Inc., | |
21 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | |
22 | */ | |
23 | ||
24 | #undef DEBUG_LOW | |
25 | ||
26 | #include <linux/kernel.h> | |
27 | #include <linux/spinlock.h> | |
28 | ||
29 | #include <asm/mmu.h> | |
30 | #include <asm/page.h> | |
31 | #include <asm/pgtable.h> | |
32 | #include <asm/machdep.h> | |
33 | #include <asm/udbg.h> | |
34 | ||
8ae6e30d | 35 | #include "beat_wrapper.h" |
fe4a0cf1 IK |
36 | |
37 | #ifdef DEBUG_LOW | |
2fe37a6e | 38 | #define DBG_LOW(fmt...) do { udbg_printf(fmt); } while (0) |
fe4a0cf1 | 39 | #else |
2fe37a6e | 40 | #define DBG_LOW(fmt...) do { } while (0) |
fe4a0cf1 IK |
41 | #endif |
42 | ||
43 | static DEFINE_SPINLOCK(beat_htab_lock); | |
44 | ||
45 | static inline unsigned int beat_read_mask(unsigned hpte_group) | |
46 | { | |
fe4a0cf1 | 47 | unsigned long rmask = 0; |
19b0bd02 | 48 | u64 hpte_v[5]; |
fe4a0cf1 IK |
49 | |
50 | beat_read_htab_entries(0, hpte_group + 0, hpte_v); | |
51 | if (!(hpte_v[0] & HPTE_V_BOLTED)) | |
52 | rmask |= 0x8000; | |
53 | if (!(hpte_v[1] & HPTE_V_BOLTED)) | |
54 | rmask |= 0x4000; | |
55 | if (!(hpte_v[2] & HPTE_V_BOLTED)) | |
56 | rmask |= 0x2000; | |
57 | if (!(hpte_v[3] & HPTE_V_BOLTED)) | |
58 | rmask |= 0x1000; | |
59 | beat_read_htab_entries(0, hpte_group + 4, hpte_v); | |
60 | if (!(hpte_v[0] & HPTE_V_BOLTED)) | |
61 | rmask |= 0x0800; | |
62 | if (!(hpte_v[1] & HPTE_V_BOLTED)) | |
63 | rmask |= 0x0400; | |
64 | if (!(hpte_v[2] & HPTE_V_BOLTED)) | |
65 | rmask |= 0x0200; | |
66 | if (!(hpte_v[3] & HPTE_V_BOLTED)) | |
67 | rmask |= 0x0100; | |
68 | hpte_group = ~hpte_group & (htab_hash_mask * HPTES_PER_GROUP); | |
69 | beat_read_htab_entries(0, hpte_group + 0, hpte_v); | |
70 | if (!(hpte_v[0] & HPTE_V_BOLTED)) | |
71 | rmask |= 0x80; | |
72 | if (!(hpte_v[1] & HPTE_V_BOLTED)) | |
73 | rmask |= 0x40; | |
74 | if (!(hpte_v[2] & HPTE_V_BOLTED)) | |
75 | rmask |= 0x20; | |
76 | if (!(hpte_v[3] & HPTE_V_BOLTED)) | |
77 | rmask |= 0x10; | |
78 | beat_read_htab_entries(0, hpte_group + 4, hpte_v); | |
79 | if (!(hpte_v[0] & HPTE_V_BOLTED)) | |
80 | rmask |= 0x08; | |
81 | if (!(hpte_v[1] & HPTE_V_BOLTED)) | |
82 | rmask |= 0x04; | |
83 | if (!(hpte_v[2] & HPTE_V_BOLTED)) | |
84 | rmask |= 0x02; | |
85 | if (!(hpte_v[3] & HPTE_V_BOLTED)) | |
86 | rmask |= 0x01; | |
87 | return rmask; | |
88 | } | |
89 | ||
90 | static long beat_lpar_hpte_insert(unsigned long hpte_group, | |
91 | unsigned long va, unsigned long pa, | |
92 | unsigned long rflags, unsigned long vflags, | |
1189be65 | 93 | int psize, int ssize) |
fe4a0cf1 IK |
94 | { |
95 | unsigned long lpar_rc; | |
19b0bd02 | 96 | u64 hpte_v, hpte_r, slot; |
fe4a0cf1 IK |
97 | |
98 | /* same as iseries */ | |
99 | if (vflags & HPTE_V_SECONDARY) | |
100 | return -1; | |
101 | ||
102 | if (!(vflags & HPTE_V_BOLTED)) | |
103 | DBG_LOW("hpte_insert(group=%lx, va=%016lx, pa=%016lx, " | |
104 | "rflags=%lx, vflags=%lx, psize=%d)\n", | |
105 | hpte_group, va, pa, rflags, vflags, psize); | |
106 | ||
1189be65 PM |
107 | hpte_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M) | |
108 | vflags | HPTE_V_VALID; | |
fe4a0cf1 IK |
109 | hpte_r = hpte_encode_r(pa, psize) | rflags; |
110 | ||
111 | if (!(vflags & HPTE_V_BOLTED)) | |
112 | DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r); | |
113 | ||
443dcac4 | 114 | if (rflags & _PAGE_NO_CACHE) |
fe4a0cf1 IK |
115 | hpte_r &= ~_PAGE_COHERENT; |
116 | ||
cbca567e | 117 | spin_lock(&beat_htab_lock); |
2fe37a6e IK |
118 | lpar_rc = beat_read_mask(hpte_group); |
119 | if (lpar_rc == 0) { | |
fe4a0cf1 IK |
120 | if (!(vflags & HPTE_V_BOLTED)) |
121 | DBG_LOW(" full\n"); | |
cbca567e | 122 | spin_unlock(&beat_htab_lock); |
fe4a0cf1 IK |
123 | return -1; |
124 | } | |
125 | ||
126 | lpar_rc = beat_insert_htab_entry(0, hpte_group, lpar_rc << 48, | |
127 | hpte_v, hpte_r, &slot); | |
cbca567e | 128 | spin_unlock(&beat_htab_lock); |
fe4a0cf1 IK |
129 | |
130 | /* | |
131 | * Since we try and ioremap PHBs we don't own, the pte insert | |
132 | * will fail. However we must catch the failure in hash_page | |
133 | * or we will loop forever, so return -2 in this case. | |
134 | */ | |
135 | if (unlikely(lpar_rc != 0)) { | |
136 | if (!(vflags & HPTE_V_BOLTED)) | |
137 | DBG_LOW(" lpar err %lx\n", lpar_rc); | |
138 | return -2; | |
139 | } | |
140 | if (!(vflags & HPTE_V_BOLTED)) | |
141 | DBG_LOW(" -> slot: %lx\n", slot); | |
142 | ||
143 | /* We have to pass down the secondary bucket bit here as well */ | |
144 | return (slot ^ hpte_group) & 15; | |
145 | } | |
146 | ||
147 | static long beat_lpar_hpte_remove(unsigned long hpte_group) | |
148 | { | |
149 | DBG_LOW("hpte_remove(group=%lx)\n", hpte_group); | |
150 | return -1; | |
151 | } | |
152 | ||
153 | static unsigned long beat_lpar_hpte_getword0(unsigned long slot) | |
154 | { | |
19b0bd02 | 155 | unsigned long dword0; |
fe4a0cf1 | 156 | unsigned long lpar_rc; |
19b0bd02 | 157 | u64 dword[5]; |
fe4a0cf1 IK |
158 | |
159 | lpar_rc = beat_read_htab_entries(0, slot & ~3UL, dword); | |
160 | ||
161 | dword0 = dword[slot&3]; | |
162 | ||
163 | BUG_ON(lpar_rc != 0); | |
164 | ||
165 | return dword0; | |
166 | } | |
167 | ||
168 | static void beat_lpar_hptab_clear(void) | |
169 | { | |
170 | unsigned long size_bytes = 1UL << ppc64_pft_size; | |
171 | unsigned long hpte_count = size_bytes >> 4; | |
172 | int i; | |
19b0bd02 | 173 | u64 dummy0, dummy1; |
fe4a0cf1 IK |
174 | |
175 | /* TODO: Use bulk call */ | |
176 | for (i = 0; i < hpte_count; i++) | |
177 | beat_write_htab_entry(0, i, 0, 0, -1UL, -1UL, &dummy0, &dummy1); | |
178 | } | |
179 | ||
180 | /* | |
181 | * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and | |
182 | * the low 3 bits of flags happen to line up. So no transform is needed. | |
183 | * We can probably optimize here and assume the high bits of newpp are | |
184 | * already zero. For now I am paranoid. | |
185 | */ | |
186 | static long beat_lpar_hpte_updatepp(unsigned long slot, | |
187 | unsigned long newpp, | |
188 | unsigned long va, | |
1189be65 | 189 | int psize, int ssize, int local) |
fe4a0cf1 IK |
190 | { |
191 | unsigned long lpar_rc; | |
19b0bd02 IM |
192 | u64 dummy0, dummy1; |
193 | unsigned long want_v; | |
fe4a0cf1 | 194 | |
1189be65 | 195 | want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M); |
fe4a0cf1 IK |
196 | |
197 | DBG_LOW(" update: " | |
198 | "avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ", | |
199 | want_v & HPTE_V_AVPN, slot, psize, newpp); | |
200 | ||
cbca567e | 201 | spin_lock(&beat_htab_lock); |
fe4a0cf1 IK |
202 | dummy0 = beat_lpar_hpte_getword0(slot); |
203 | if ((dummy0 & ~0x7FUL) != (want_v & ~0x7FUL)) { | |
204 | DBG_LOW("not found !\n"); | |
cbca567e | 205 | spin_unlock(&beat_htab_lock); |
fe4a0cf1 IK |
206 | return -1; |
207 | } | |
208 | ||
209 | lpar_rc = beat_write_htab_entry(0, slot, 0, newpp, 0, 7, &dummy0, | |
210 | &dummy1); | |
cbca567e | 211 | spin_unlock(&beat_htab_lock); |
fe4a0cf1 IK |
212 | if (lpar_rc != 0 || dummy0 == 0) { |
213 | DBG_LOW("not found !\n"); | |
214 | return -1; | |
215 | } | |
216 | ||
217 | DBG_LOW("ok %lx %lx\n", dummy0, dummy1); | |
218 | ||
219 | BUG_ON(lpar_rc != 0); | |
220 | ||
221 | return 0; | |
222 | } | |
223 | ||
224 | static long beat_lpar_hpte_find(unsigned long va, int psize) | |
225 | { | |
226 | unsigned long hash; | |
227 | unsigned long i, j; | |
228 | long slot; | |
229 | unsigned long want_v, hpte_v; | |
230 | ||
1189be65 PM |
231 | hash = hpt_hash(va, mmu_psize_defs[psize].shift, MMU_SEGSIZE_256M); |
232 | want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M); | |
fe4a0cf1 IK |
233 | |
234 | for (j = 0; j < 2; j++) { | |
235 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | |
236 | for (i = 0; i < HPTES_PER_GROUP; i++) { | |
237 | hpte_v = beat_lpar_hpte_getword0(slot); | |
238 | ||
239 | if (HPTE_V_COMPARE(hpte_v, want_v) | |
240 | && (hpte_v & HPTE_V_VALID) | |
241 | && (!!(hpte_v & HPTE_V_SECONDARY) == j)) { | |
242 | /* HPTE matches */ | |
243 | if (j) | |
244 | slot = -slot; | |
245 | return slot; | |
246 | } | |
247 | ++slot; | |
248 | } | |
249 | hash = ~hash; | |
250 | } | |
251 | ||
252 | return -1; | |
253 | } | |
254 | ||
255 | static void beat_lpar_hpte_updateboltedpp(unsigned long newpp, | |
256 | unsigned long ea, | |
1189be65 | 257 | int psize, int ssize) |
fe4a0cf1 | 258 | { |
19b0bd02 IM |
259 | unsigned long lpar_rc, slot, vsid, va; |
260 | u64 dummy0, dummy1; | |
fe4a0cf1 | 261 | |
1189be65 | 262 | vsid = get_kernel_vsid(ea, MMU_SEGSIZE_256M); |
fe4a0cf1 IK |
263 | va = (vsid << 28) | (ea & 0x0fffffff); |
264 | ||
cbca567e | 265 | spin_lock(&beat_htab_lock); |
fe4a0cf1 IK |
266 | slot = beat_lpar_hpte_find(va, psize); |
267 | BUG_ON(slot == -1); | |
268 | ||
269 | lpar_rc = beat_write_htab_entry(0, slot, 0, newpp, 0, 7, | |
270 | &dummy0, &dummy1); | |
cbca567e | 271 | spin_unlock(&beat_htab_lock); |
fe4a0cf1 IK |
272 | |
273 | BUG_ON(lpar_rc != 0); | |
274 | } | |
275 | ||
276 | static void beat_lpar_hpte_invalidate(unsigned long slot, unsigned long va, | |
1189be65 | 277 | int psize, int ssize, int local) |
fe4a0cf1 IK |
278 | { |
279 | unsigned long want_v; | |
280 | unsigned long lpar_rc; | |
19b0bd02 | 281 | u64 dummy1, dummy2; |
fe4a0cf1 IK |
282 | unsigned long flags; |
283 | ||
284 | DBG_LOW(" inval : slot=%lx, va=%016lx, psize: %d, local: %d\n", | |
285 | slot, va, psize, local); | |
1189be65 | 286 | want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M); |
fe4a0cf1 IK |
287 | |
288 | spin_lock_irqsave(&beat_htab_lock, flags); | |
289 | dummy1 = beat_lpar_hpte_getword0(slot); | |
290 | ||
291 | if ((dummy1 & ~0x7FUL) != (want_v & ~0x7FUL)) { | |
292 | DBG_LOW("not found !\n"); | |
293 | spin_unlock_irqrestore(&beat_htab_lock, flags); | |
294 | return; | |
295 | } | |
296 | ||
297 | lpar_rc = beat_write_htab_entry(0, slot, 0, 0, HPTE_V_VALID, 0, | |
298 | &dummy1, &dummy2); | |
299 | spin_unlock_irqrestore(&beat_htab_lock, flags); | |
300 | ||
301 | BUG_ON(lpar_rc != 0); | |
302 | } | |
303 | ||
304 | void __init hpte_init_beat(void) | |
305 | { | |
306 | ppc_md.hpte_invalidate = beat_lpar_hpte_invalidate; | |
307 | ppc_md.hpte_updatepp = beat_lpar_hpte_updatepp; | |
308 | ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp; | |
309 | ppc_md.hpte_insert = beat_lpar_hpte_insert; | |
310 | ppc_md.hpte_remove = beat_lpar_hpte_remove; | |
311 | ppc_md.hpte_clear_all = beat_lpar_hptab_clear; | |
312 | } | |
7f2c8577 IK |
313 | |
314 | static long beat_lpar_hpte_insert_v3(unsigned long hpte_group, | |
315 | unsigned long va, unsigned long pa, | |
316 | unsigned long rflags, unsigned long vflags, | |
1189be65 | 317 | int psize, int ssize) |
7f2c8577 IK |
318 | { |
319 | unsigned long lpar_rc; | |
19b0bd02 | 320 | u64 hpte_v, hpte_r, slot; |
7f2c8577 IK |
321 | |
322 | /* same as iseries */ | |
323 | if (vflags & HPTE_V_SECONDARY) | |
324 | return -1; | |
325 | ||
326 | if (!(vflags & HPTE_V_BOLTED)) | |
327 | DBG_LOW("hpte_insert(group=%lx, va=%016lx, pa=%016lx, " | |
328 | "rflags=%lx, vflags=%lx, psize=%d)\n", | |
329 | hpte_group, va, pa, rflags, vflags, psize); | |
330 | ||
1189be65 PM |
331 | hpte_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M) | |
332 | vflags | HPTE_V_VALID; | |
7f2c8577 IK |
333 | hpte_r = hpte_encode_r(pa, psize) | rflags; |
334 | ||
335 | if (!(vflags & HPTE_V_BOLTED)) | |
336 | DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r); | |
337 | ||
443dcac4 | 338 | if (rflags & _PAGE_NO_CACHE) |
7f2c8577 IK |
339 | hpte_r &= ~_PAGE_COHERENT; |
340 | ||
341 | /* insert into not-volted entry */ | |
342 | lpar_rc = beat_insert_htab_entry3(0, hpte_group, hpte_v, hpte_r, | |
343 | HPTE_V_BOLTED, 0, &slot); | |
344 | /* | |
345 | * Since we try and ioremap PHBs we don't own, the pte insert | |
346 | * will fail. However we must catch the failure in hash_page | |
347 | * or we will loop forever, so return -2 in this case. | |
348 | */ | |
349 | if (unlikely(lpar_rc != 0)) { | |
350 | if (!(vflags & HPTE_V_BOLTED)) | |
351 | DBG_LOW(" lpar err %lx\n", lpar_rc); | |
352 | return -2; | |
353 | } | |
354 | if (!(vflags & HPTE_V_BOLTED)) | |
355 | DBG_LOW(" -> slot: %lx\n", slot); | |
356 | ||
357 | /* We have to pass down the secondary bucket bit here as well */ | |
358 | return (slot ^ hpte_group) & 15; | |
359 | } | |
360 | ||
361 | /* | |
362 | * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and | |
363 | * the low 3 bits of flags happen to line up. So no transform is needed. | |
364 | * We can probably optimize here and assume the high bits of newpp are | |
365 | * already zero. For now I am paranoid. | |
366 | */ | |
367 | static long beat_lpar_hpte_updatepp_v3(unsigned long slot, | |
368 | unsigned long newpp, | |
369 | unsigned long va, | |
1189be65 | 370 | int psize, int ssize, int local) |
7f2c8577 IK |
371 | { |
372 | unsigned long lpar_rc; | |
373 | unsigned long want_v; | |
374 | unsigned long pss; | |
375 | ||
1189be65 | 376 | want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M); |
7f2c8577 IK |
377 | pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc; |
378 | ||
379 | DBG_LOW(" update: " | |
380 | "avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ", | |
381 | want_v & HPTE_V_AVPN, slot, psize, newpp); | |
382 | ||
383 | lpar_rc = beat_update_htab_permission3(0, slot, want_v, pss, 7, newpp); | |
384 | ||
385 | if (lpar_rc == 0xfffffff7) { | |
386 | DBG_LOW("not found !\n"); | |
387 | return -1; | |
388 | } | |
389 | ||
390 | DBG_LOW("ok\n"); | |
391 | ||
392 | BUG_ON(lpar_rc != 0); | |
393 | ||
394 | return 0; | |
395 | } | |
396 | ||
397 | static void beat_lpar_hpte_invalidate_v3(unsigned long slot, unsigned long va, | |
1189be65 | 398 | int psize, int ssize, int local) |
7f2c8577 IK |
399 | { |
400 | unsigned long want_v; | |
401 | unsigned long lpar_rc; | |
402 | unsigned long pss; | |
403 | ||
404 | DBG_LOW(" inval : slot=%lx, va=%016lx, psize: %d, local: %d\n", | |
405 | slot, va, psize, local); | |
1189be65 | 406 | want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M); |
7f2c8577 IK |
407 | pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc; |
408 | ||
409 | lpar_rc = beat_invalidate_htab_entry3(0, slot, want_v, pss); | |
410 | ||
411 | /* E_busy can be valid output: page may be already replaced */ | |
412 | BUG_ON(lpar_rc != 0 && lpar_rc != 0xfffffff7); | |
413 | } | |
414 | ||
415 | static int64_t _beat_lpar_hptab_clear_v3(void) | |
416 | { | |
417 | return beat_clear_htab3(0); | |
418 | } | |
419 | ||
420 | static void beat_lpar_hptab_clear_v3(void) | |
421 | { | |
422 | _beat_lpar_hptab_clear_v3(); | |
423 | } | |
424 | ||
425 | void __init hpte_init_beat_v3(void) | |
426 | { | |
427 | if (_beat_lpar_hptab_clear_v3() == 0) { | |
428 | ppc_md.hpte_invalidate = beat_lpar_hpte_invalidate_v3; | |
429 | ppc_md.hpte_updatepp = beat_lpar_hpte_updatepp_v3; | |
430 | ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp; | |
431 | ppc_md.hpte_insert = beat_lpar_hpte_insert_v3; | |
432 | ppc_md.hpte_remove = beat_lpar_hpte_remove; | |
433 | ppc_md.hpte_clear_all = beat_lpar_hptab_clear_v3; | |
434 | } else { | |
435 | ppc_md.hpte_invalidate = beat_lpar_hpte_invalidate; | |
436 | ppc_md.hpte_updatepp = beat_lpar_hpte_updatepp; | |
437 | ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp; | |
438 | ppc_md.hpte_insert = beat_lpar_hpte_insert; | |
439 | ppc_md.hpte_remove = beat_lpar_hpte_remove; | |
440 | ppc_md.hpte_clear_all = beat_lpar_hptab_clear; | |
441 | } | |
442 | } |