2 * "Cell Reference Set" HTAB support.
4 * (C) Copyright 2006-2007 TOSHIBA CORPORATION
6 * This code is based on arch/powerpc/platforms/pseries/lpar.c:
7 * Copyright (C) 2001 Todd Inglett, IBM Corporation
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
19 * You should have received a copy of the GNU General Public License along
20 * with this program; if not, write to the Free Software Foundation, Inc.,
21 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
26 #include <linux/kernel.h>
27 #include <linux/spinlock.h>
31 #include <asm/pgtable.h>
32 #include <asm/machdep.h>
35 #include "beat_wrapper.h"
38 #define DBG_LOW(fmt...) do { udbg_printf(fmt); } while (0)
40 #define DBG_LOW(fmt...) do { } while (0)
43 static DEFINE_SPINLOCK(beat_htab_lock
);
45 static inline unsigned int beat_read_mask(unsigned hpte_group
)
47 unsigned long hpte_v
[5];
48 unsigned long rmask
= 0;
50 beat_read_htab_entries(0, hpte_group
+ 0, hpte_v
);
51 if (!(hpte_v
[0] & HPTE_V_BOLTED
))
53 if (!(hpte_v
[1] & HPTE_V_BOLTED
))
55 if (!(hpte_v
[2] & HPTE_V_BOLTED
))
57 if (!(hpte_v
[3] & HPTE_V_BOLTED
))
59 beat_read_htab_entries(0, hpte_group
+ 4, hpte_v
);
60 if (!(hpte_v
[0] & HPTE_V_BOLTED
))
62 if (!(hpte_v
[1] & HPTE_V_BOLTED
))
64 if (!(hpte_v
[2] & HPTE_V_BOLTED
))
66 if (!(hpte_v
[3] & HPTE_V_BOLTED
))
68 hpte_group
= ~hpte_group
& (htab_hash_mask
* HPTES_PER_GROUP
);
69 beat_read_htab_entries(0, hpte_group
+ 0, hpte_v
);
70 if (!(hpte_v
[0] & HPTE_V_BOLTED
))
72 if (!(hpte_v
[1] & HPTE_V_BOLTED
))
74 if (!(hpte_v
[2] & HPTE_V_BOLTED
))
76 if (!(hpte_v
[3] & HPTE_V_BOLTED
))
78 beat_read_htab_entries(0, hpte_group
+ 4, hpte_v
);
79 if (!(hpte_v
[0] & HPTE_V_BOLTED
))
81 if (!(hpte_v
[1] & HPTE_V_BOLTED
))
83 if (!(hpte_v
[2] & HPTE_V_BOLTED
))
85 if (!(hpte_v
[3] & HPTE_V_BOLTED
))
90 static long beat_lpar_hpte_insert(unsigned long hpte_group
,
91 unsigned long va
, unsigned long pa
,
92 unsigned long rflags
, unsigned long vflags
,
95 unsigned long lpar_rc
;
97 unsigned long hpte_v
, hpte_r
;
100 if (vflags
& HPTE_V_SECONDARY
)
103 if (!(vflags
& HPTE_V_BOLTED
))
104 DBG_LOW("hpte_insert(group=%lx, va=%016lx, pa=%016lx, "
105 "rflags=%lx, vflags=%lx, psize=%d)\n",
106 hpte_group
, va
, pa
, rflags
, vflags
, psize
);
108 hpte_v
= hpte_encode_v(va
, psize
, MMU_SEGSIZE_256M
) |
109 vflags
| HPTE_V_VALID
;
110 hpte_r
= hpte_encode_r(pa
, psize
) | rflags
;
112 if (!(vflags
& HPTE_V_BOLTED
))
113 DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v
, hpte_r
);
115 if (rflags
& (_PAGE_GUARDED
|_PAGE_NO_CACHE
))
116 hpte_r
&= ~_PAGE_COHERENT
;
118 spin_lock(&beat_htab_lock
);
119 lpar_rc
= beat_read_mask(hpte_group
);
121 if (!(vflags
& HPTE_V_BOLTED
))
123 spin_unlock(&beat_htab_lock
);
127 lpar_rc
= beat_insert_htab_entry(0, hpte_group
, lpar_rc
<< 48,
128 hpte_v
, hpte_r
, &slot
);
129 spin_unlock(&beat_htab_lock
);
132 * Since we try and ioremap PHBs we don't own, the pte insert
133 * will fail. However we must catch the failure in hash_page
134 * or we will loop forever, so return -2 in this case.
136 if (unlikely(lpar_rc
!= 0)) {
137 if (!(vflags
& HPTE_V_BOLTED
))
138 DBG_LOW(" lpar err %lx\n", lpar_rc
);
141 if (!(vflags
& HPTE_V_BOLTED
))
142 DBG_LOW(" -> slot: %lx\n", slot
);
144 /* We have to pass down the secondary bucket bit here as well */
145 return (slot
^ hpte_group
) & 15;
148 static long beat_lpar_hpte_remove(unsigned long hpte_group
)
150 DBG_LOW("hpte_remove(group=%lx)\n", hpte_group
);
154 static unsigned long beat_lpar_hpte_getword0(unsigned long slot
)
156 unsigned long dword0
, dword
[5];
157 unsigned long lpar_rc
;
159 lpar_rc
= beat_read_htab_entries(0, slot
& ~3UL, dword
);
161 dword0
= dword
[slot
&3];
163 BUG_ON(lpar_rc
!= 0);
168 static void beat_lpar_hptab_clear(void)
170 unsigned long size_bytes
= 1UL << ppc64_pft_size
;
171 unsigned long hpte_count
= size_bytes
>> 4;
173 unsigned long dummy0
, dummy1
;
175 /* TODO: Use bulk call */
176 for (i
= 0; i
< hpte_count
; i
++)
177 beat_write_htab_entry(0, i
, 0, 0, -1UL, -1UL, &dummy0
, &dummy1
);
181 * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
182 * the low 3 bits of flags happen to line up. So no transform is needed.
183 * We can probably optimize here and assume the high bits of newpp are
184 * already zero. For now I am paranoid.
186 static long beat_lpar_hpte_updatepp(unsigned long slot
,
189 int psize
, int ssize
, int local
)
191 unsigned long lpar_rc
;
192 unsigned long dummy0
, dummy1
, want_v
;
194 want_v
= hpte_encode_v(va
, psize
, MMU_SEGSIZE_256M
);
197 "avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ",
198 want_v
& HPTE_V_AVPN
, slot
, psize
, newpp
);
200 spin_lock(&beat_htab_lock
);
201 dummy0
= beat_lpar_hpte_getword0(slot
);
202 if ((dummy0
& ~0x7FUL
) != (want_v
& ~0x7FUL
)) {
203 DBG_LOW("not found !\n");
204 spin_unlock(&beat_htab_lock
);
208 lpar_rc
= beat_write_htab_entry(0, slot
, 0, newpp
, 0, 7, &dummy0
,
210 spin_unlock(&beat_htab_lock
);
211 if (lpar_rc
!= 0 || dummy0
== 0) {
212 DBG_LOW("not found !\n");
216 DBG_LOW("ok %lx %lx\n", dummy0
, dummy1
);
218 BUG_ON(lpar_rc
!= 0);
223 static long beat_lpar_hpte_find(unsigned long va
, int psize
)
228 unsigned long want_v
, hpte_v
;
230 hash
= hpt_hash(va
, mmu_psize_defs
[psize
].shift
, MMU_SEGSIZE_256M
);
231 want_v
= hpte_encode_v(va
, psize
, MMU_SEGSIZE_256M
);
233 for (j
= 0; j
< 2; j
++) {
234 slot
= (hash
& htab_hash_mask
) * HPTES_PER_GROUP
;
235 for (i
= 0; i
< HPTES_PER_GROUP
; i
++) {
236 hpte_v
= beat_lpar_hpte_getword0(slot
);
238 if (HPTE_V_COMPARE(hpte_v
, want_v
)
239 && (hpte_v
& HPTE_V_VALID
)
240 && (!!(hpte_v
& HPTE_V_SECONDARY
) == j
)) {
254 static void beat_lpar_hpte_updateboltedpp(unsigned long newpp
,
256 int psize
, int ssize
)
258 unsigned long lpar_rc
, slot
, vsid
, va
, dummy0
, dummy1
;
260 vsid
= get_kernel_vsid(ea
, MMU_SEGSIZE_256M
);
261 va
= (vsid
<< 28) | (ea
& 0x0fffffff);
263 spin_lock(&beat_htab_lock
);
264 slot
= beat_lpar_hpte_find(va
, psize
);
267 lpar_rc
= beat_write_htab_entry(0, slot
, 0, newpp
, 0, 7,
269 spin_unlock(&beat_htab_lock
);
271 BUG_ON(lpar_rc
!= 0);
274 static void beat_lpar_hpte_invalidate(unsigned long slot
, unsigned long va
,
275 int psize
, int ssize
, int local
)
277 unsigned long want_v
;
278 unsigned long lpar_rc
;
279 unsigned long dummy1
, dummy2
;
282 DBG_LOW(" inval : slot=%lx, va=%016lx, psize: %d, local: %d\n",
283 slot
, va
, psize
, local
);
284 want_v
= hpte_encode_v(va
, psize
, MMU_SEGSIZE_256M
);
286 spin_lock_irqsave(&beat_htab_lock
, flags
);
287 dummy1
= beat_lpar_hpte_getword0(slot
);
289 if ((dummy1
& ~0x7FUL
) != (want_v
& ~0x7FUL
)) {
290 DBG_LOW("not found !\n");
291 spin_unlock_irqrestore(&beat_htab_lock
, flags
);
295 lpar_rc
= beat_write_htab_entry(0, slot
, 0, 0, HPTE_V_VALID
, 0,
297 spin_unlock_irqrestore(&beat_htab_lock
, flags
);
299 BUG_ON(lpar_rc
!= 0);
302 void __init
hpte_init_beat(void)
304 ppc_md
.hpte_invalidate
= beat_lpar_hpte_invalidate
;
305 ppc_md
.hpte_updatepp
= beat_lpar_hpte_updatepp
;
306 ppc_md
.hpte_updateboltedpp
= beat_lpar_hpte_updateboltedpp
;
307 ppc_md
.hpte_insert
= beat_lpar_hpte_insert
;
308 ppc_md
.hpte_remove
= beat_lpar_hpte_remove
;
309 ppc_md
.hpte_clear_all
= beat_lpar_hptab_clear
;
312 static long beat_lpar_hpte_insert_v3(unsigned long hpte_group
,
313 unsigned long va
, unsigned long pa
,
314 unsigned long rflags
, unsigned long vflags
,
315 int psize
, int ssize
)
317 unsigned long lpar_rc
;
319 unsigned long hpte_v
, hpte_r
;
321 /* same as iseries */
322 if (vflags
& HPTE_V_SECONDARY
)
325 if (!(vflags
& HPTE_V_BOLTED
))
326 DBG_LOW("hpte_insert(group=%lx, va=%016lx, pa=%016lx, "
327 "rflags=%lx, vflags=%lx, psize=%d)\n",
328 hpte_group
, va
, pa
, rflags
, vflags
, psize
);
330 hpte_v
= hpte_encode_v(va
, psize
, MMU_SEGSIZE_256M
) |
331 vflags
| HPTE_V_VALID
;
332 hpte_r
= hpte_encode_r(pa
, psize
) | rflags
;
334 if (!(vflags
& HPTE_V_BOLTED
))
335 DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v
, hpte_r
);
337 if (rflags
& (_PAGE_GUARDED
|_PAGE_NO_CACHE
))
338 hpte_r
&= ~_PAGE_COHERENT
;
340 /* insert into not-volted entry */
341 lpar_rc
= beat_insert_htab_entry3(0, hpte_group
, hpte_v
, hpte_r
,
342 HPTE_V_BOLTED
, 0, &slot
);
344 * Since we try and ioremap PHBs we don't own, the pte insert
345 * will fail. However we must catch the failure in hash_page
346 * or we will loop forever, so return -2 in this case.
348 if (unlikely(lpar_rc
!= 0)) {
349 if (!(vflags
& HPTE_V_BOLTED
))
350 DBG_LOW(" lpar err %lx\n", lpar_rc
);
353 if (!(vflags
& HPTE_V_BOLTED
))
354 DBG_LOW(" -> slot: %lx\n", slot
);
356 /* We have to pass down the secondary bucket bit here as well */
357 return (slot
^ hpte_group
) & 15;
361 * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
362 * the low 3 bits of flags happen to line up. So no transform is needed.
363 * We can probably optimize here and assume the high bits of newpp are
364 * already zero. For now I am paranoid.
366 static long beat_lpar_hpte_updatepp_v3(unsigned long slot
,
369 int psize
, int ssize
, int local
)
371 unsigned long lpar_rc
;
372 unsigned long want_v
;
375 want_v
= hpte_encode_v(va
, psize
, MMU_SEGSIZE_256M
);
376 pss
= (psize
== MMU_PAGE_4K
) ? -1UL : mmu_psize_defs
[psize
].penc
;
379 "avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ",
380 want_v
& HPTE_V_AVPN
, slot
, psize
, newpp
);
382 lpar_rc
= beat_update_htab_permission3(0, slot
, want_v
, pss
, 7, newpp
);
384 if (lpar_rc
== 0xfffffff7) {
385 DBG_LOW("not found !\n");
391 BUG_ON(lpar_rc
!= 0);
396 static void beat_lpar_hpte_invalidate_v3(unsigned long slot
, unsigned long va
,
397 int psize
, int ssize
, int local
)
399 unsigned long want_v
;
400 unsigned long lpar_rc
;
403 DBG_LOW(" inval : slot=%lx, va=%016lx, psize: %d, local: %d\n",
404 slot
, va
, psize
, local
);
405 want_v
= hpte_encode_v(va
, psize
, MMU_SEGSIZE_256M
);
406 pss
= (psize
== MMU_PAGE_4K
) ? -1UL : mmu_psize_defs
[psize
].penc
;
408 lpar_rc
= beat_invalidate_htab_entry3(0, slot
, want_v
, pss
);
410 /* E_busy can be valid output: page may be already replaced */
411 BUG_ON(lpar_rc
!= 0 && lpar_rc
!= 0xfffffff7);
414 static int64_t _beat_lpar_hptab_clear_v3(void)
416 return beat_clear_htab3(0);
419 static void beat_lpar_hptab_clear_v3(void)
421 _beat_lpar_hptab_clear_v3();
424 void __init
hpte_init_beat_v3(void)
426 if (_beat_lpar_hptab_clear_v3() == 0) {
427 ppc_md
.hpte_invalidate
= beat_lpar_hpte_invalidate_v3
;
428 ppc_md
.hpte_updatepp
= beat_lpar_hpte_updatepp_v3
;
429 ppc_md
.hpte_updateboltedpp
= beat_lpar_hpte_updateboltedpp
;
430 ppc_md
.hpte_insert
= beat_lpar_hpte_insert_v3
;
431 ppc_md
.hpte_remove
= beat_lpar_hpte_remove
;
432 ppc_md
.hpte_clear_all
= beat_lpar_hptab_clear_v3
;
434 ppc_md
.hpte_invalidate
= beat_lpar_hpte_invalidate
;
435 ppc_md
.hpte_updatepp
= beat_lpar_hpte_updatepp
;
436 ppc_md
.hpte_updateboltedpp
= beat_lpar_hpte_updateboltedpp
;
437 ppc_md
.hpte_insert
= beat_lpar_hpte_insert
;
438 ppc_md
.hpte_remove
= beat_lpar_hpte_remove
;
439 ppc_md
.hpte_clear_all
= beat_lpar_hptab_clear
;