]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - arch/sparc/mm/sun4c.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
sparc: drop CONFIG_SUN_AUXIO
[mirror_ubuntu-artful-kernel.git] / arch / sparc / mm / sun4c.c
CommitLineData
64d329ee 1/* sun4c.c: Doing in software what should be done in hardware.
1da177e4 2 *
64d329ee 3 * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
1da177e4
LT		 4 * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be)
5 * Copyright (C) 1996 Andrew Tridgell (Andrew.Tridgell@anu.edu.au)
6 * Copyright (C) 1997-2000 Anton Blanchard (anton@samba.org)
7 * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
8 */
9
10#define NR_TASK_BUCKETS 512
11
1da177e4
LT		 12#include <linux/kernel.h>
13#include <linux/mm.h>
14#include <linux/init.h>
15#include <linux/bootmem.h>
16#include <linux/highmem.h>
17#include <linux/fs.h>
18#include <linux/seq_file.h>
0912a5db 19#include <linux/scatterlist.h>
1da177e4 20
1da177e4
LT		 21#include <asm/page.h>
22#include <asm/pgalloc.h>
23#include <asm/pgtable.h>
24#include <asm/vaddrs.h>
25#include <asm/idprom.h>
26#include <asm/machines.h>
27#include <asm/memreg.h>
28#include <asm/processor.h>
29#include <asm/auxio.h>
30#include <asm/io.h>
31#include <asm/oplib.h>
32#include <asm/openprom.h>
33#include <asm/mmu_context.h>
1da177e4
LT		 34#include <asm/highmem.h>
35#include <asm/btfixup.h>
36#include <asm/cacheflush.h>
37#include <asm/tlbflush.h>
38
39/* Because of our dynamic kernel TLB miss strategy, and how
40 * our DVMA mapping allocation works, you _MUST_:
41 *
42 * 1) Disable interrupts _and_ not touch any dynamic kernel
43 * memory while messing with kernel MMU state. By
44 * dynamic memory I mean any object which is not in
45 * the kernel image itself or a thread_union (both of
46 * which are locked into the MMU).
47 * 2) Disable interrupts while messing with user MMU state.
48 */
49
50extern int num_segmaps, num_contexts;
51
52extern unsigned long page_kernel;
53
1da177e4
LT		 54/* That's it, we prom_halt() on sun4c if the cache size is something other than 65536.
55 * So let's save some cycles and just use that everywhere except for that bootup
56 * sanity check.
57 */
58#define SUN4C_VAC_SIZE 65536
1da177e4
LT		 59
60#define SUN4C_KERNEL_BUCKETS 32
61
62/* Flushing the cache. */
63struct sun4c_vac_props sun4c_vacinfo;
64unsigned long sun4c_kernel_faults;
65
66/* Invalidate every sun4c cache line tag. */
67static void __init sun4c_flush_all(void)
68{
69 unsigned long begin, end;
70
71 if (sun4c_vacinfo.on)
72 panic("SUN4C: AIEEE, trying to invalidate vac while it is on.");
73
74 /* Clear 'valid' bit in all cache line tags */
75 begin = AC_CACHETAGS;
76 end = (AC_CACHETAGS + SUN4C_VAC_SIZE);
77 while (begin < end) {
78 __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
79 "r" (begin), "i" (ASI_CONTROL));
80 begin += sun4c_vacinfo.linesize;
81 }
82}
83
84static void sun4c_flush_context_hw(void)
85{
86 unsigned long end = SUN4C_VAC_SIZE;
87
88 __asm__ __volatile__(
89 "1: addcc %0, -4096, %0\n\t"
90 " bne 1b\n\t"
91 " sta %%g0, [%0] %2"
92 : "=&r" (end)
93 : "0" (end), "i" (ASI_HWFLUSHCONTEXT)
94 : "cc");
95}
96
97/* Must be called minimally with IRQs disabled. */
98static void sun4c_flush_segment_hw(unsigned long addr)
99{
100 if (sun4c_get_segmap(addr) != invalid_segment) {
101 unsigned long vac_size = SUN4C_VAC_SIZE;
102
103 __asm__ __volatile__(
104 "1: addcc %0, -4096, %0\n\t"
105 " bne 1b\n\t"
106 " sta %%g0, [%2 + %0] %3"
107 : "=&r" (vac_size)
108 : "0" (vac_size), "r" (addr), "i" (ASI_HWFLUSHSEG)
109 : "cc");
110 }
111}
112
113/* File local boot time fixups. */
114BTFIXUPDEF_CALL(void, sun4c_flush_page, unsigned long)
115BTFIXUPDEF_CALL(void, sun4c_flush_segment, unsigned long)
116BTFIXUPDEF_CALL(void, sun4c_flush_context, void)
117
118#define sun4c_flush_page(addr) BTFIXUP_CALL(sun4c_flush_page)(addr)
119#define sun4c_flush_segment(addr) BTFIXUP_CALL(sun4c_flush_segment)(addr)
120#define sun4c_flush_context() BTFIXUP_CALL(sun4c_flush_context)()
121
122/* Must be called minimally with interrupts disabled. */
123static void sun4c_flush_page_hw(unsigned long addr)
124{
125 addr &= PAGE_MASK;
126 if ((int)sun4c_get_pte(addr) < 0)
127 __asm__ __volatile__("sta %%g0, [%0] %1"
128 : : "r" (addr), "i" (ASI_HWFLUSHPAGE));
129}
130
131/* Don't inline the software version as it eats too many cache lines if expanded. */
132static void sun4c_flush_context_sw(void)
133{
134 unsigned long nbytes = SUN4C_VAC_SIZE;
135 unsigned long lsize = sun4c_vacinfo.linesize;
136
137 __asm__ __volatile__(
138 "add %2, %2, %%g1\n\t"
139 "add %2, %%g1, %%g2\n\t"
140 "add %2, %%g2, %%g3\n\t"
141 "add %2, %%g3, %%g4\n\t"
142 "add %2, %%g4, %%g5\n\t"
143 "add %2, %%g5, %%o4\n\t"
144 "add %2, %%o4, %%o5\n"
145 "1:\n\t"
146 "subcc %0, %%o5, %0\n\t"
147 "sta %%g0, [%0] %3\n\t"
148 "sta %%g0, [%0 + %2] %3\n\t"
149 "sta %%g0, [%0 + %%g1] %3\n\t"
150 "sta %%g0, [%0 + %%g2] %3\n\t"
151 "sta %%g0, [%0 + %%g3] %3\n\t"
152 "sta %%g0, [%0 + %%g4] %3\n\t"
153 "sta %%g0, [%0 + %%g5] %3\n\t"
154 "bg 1b\n\t"
155 " sta %%g0, [%1 + %%o4] %3\n"
156 : "=&r" (nbytes)
157 : "0" (nbytes), "r" (lsize), "i" (ASI_FLUSHCTX)
158 : "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc");
159}
160
161/* Don't inline the software version as it eats too many cache lines if expanded. */
162static void sun4c_flush_segment_sw(unsigned long addr)
163{
164 if (sun4c_get_segmap(addr) != invalid_segment) {
165 unsigned long nbytes = SUN4C_VAC_SIZE;
166 unsigned long lsize = sun4c_vacinfo.linesize;
167
168 __asm__ __volatile__(
169 "add %2, %2, %%g1\n\t"
170 "add %2, %%g1, %%g2\n\t"
171 "add %2, %%g2, %%g3\n\t"
172 "add %2, %%g3, %%g4\n\t"
173 "add %2, %%g4, %%g5\n\t"
174 "add %2, %%g5, %%o4\n\t"
175 "add %2, %%o4, %%o5\n"
176 "1:\n\t"
177 "subcc %1, %%o5, %1\n\t"
178 "sta %%g0, [%0] %6\n\t"
179 "sta %%g0, [%0 + %2] %6\n\t"
180 "sta %%g0, [%0 + %%g1] %6\n\t"
181 "sta %%g0, [%0 + %%g2] %6\n\t"
182 "sta %%g0, [%0 + %%g3] %6\n\t"
183 "sta %%g0, [%0 + %%g4] %6\n\t"
184 "sta %%g0, [%0 + %%g5] %6\n\t"
185 "sta %%g0, [%0 + %%o4] %6\n\t"
186 "bg 1b\n\t"
187 " add %0, %%o5, %0\n"
188 : "=&r" (addr), "=&r" (nbytes), "=&r" (lsize)
189 : "0" (addr), "1" (nbytes), "2" (lsize),
190 "i" (ASI_FLUSHSEG)
191 : "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc");
192 }
193}
194
195/* Don't inline the software version as it eats too many cache lines if expanded. */
196static void sun4c_flush_page_sw(unsigned long addr)
197{
198 addr &= PAGE_MASK;
199 if ((sun4c_get_pte(addr) & (_SUN4C_PAGE_NOCACHE | _SUN4C_PAGE_VALID)) ==
200 _SUN4C_PAGE_VALID) {
201 unsigned long left = PAGE_SIZE;
202 unsigned long lsize = sun4c_vacinfo.linesize;
203
204 __asm__ __volatile__(
205 "add %2, %2, %%g1\n\t"
206 "add %2, %%g1, %%g2\n\t"
207 "add %2, %%g2, %%g3\n\t"
208 "add %2, %%g3, %%g4\n\t"
209 "add %2, %%g4, %%g5\n\t"
210 "add %2, %%g5, %%o4\n\t"
211 "add %2, %%o4, %%o5\n"
212 "1:\n\t"
213 "subcc %1, %%o5, %1\n\t"
214 "sta %%g0, [%0] %6\n\t"
215 "sta %%g0, [%0 + %2] %6\n\t"
216 "sta %%g0, [%0 + %%g1] %6\n\t"
217 "sta %%g0, [%0 + %%g2] %6\n\t"
218 "sta %%g0, [%0 + %%g3] %6\n\t"
219 "sta %%g0, [%0 + %%g4] %6\n\t"
220 "sta %%g0, [%0 + %%g5] %6\n\t"
221 "sta %%g0, [%0 + %%o4] %6\n\t"
222 "bg 1b\n\t"
223 " add %0, %%o5, %0\n"
224 : "=&r" (addr), "=&r" (left), "=&r" (lsize)
225 : "0" (addr), "1" (left), "2" (lsize),
226 "i" (ASI_FLUSHPG)
227 : "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc");
228 }
229}
230
231/* The sun4c's do have an on chip store buffer. And the way you
232 * clear them out isn't so obvious. The only way I can think of
233 * to accomplish this is to read the current context register,
234 * store the same value there, then read an external hardware
235 * register.
236 */
237void sun4c_complete_all_stores(void)
238{
239 volatile int _unused;
240
241 _unused = sun4c_get_context();
242 sun4c_set_context(_unused);
1da177e4 243 _unused = get_auxio();
1da177e4
LT		 244}
245
246/* Bootup utility functions. */
247static inline void sun4c_init_clean_segmap(unsigned char pseg)
248{
249 unsigned long vaddr;
250
251 sun4c_put_segmap(0, pseg);
252 for (vaddr = 0; vaddr < SUN4C_REAL_PGDIR_SIZE; vaddr += PAGE_SIZE)
253 sun4c_put_pte(vaddr, 0);
254 sun4c_put_segmap(0, invalid_segment);
255}
256
257static inline void sun4c_init_clean_mmu(unsigned long kernel_end)
258{
259 unsigned long vaddr;
260 unsigned char savectx, ctx;
261
262 savectx = sun4c_get_context();
1da177e4
LT		 263 for (ctx = 0; ctx < num_contexts; ctx++) {
264 sun4c_set_context(ctx);
265 for (vaddr = 0; vaddr < 0x20000000; vaddr += SUN4C_REAL_PGDIR_SIZE)
266 sun4c_put_segmap(vaddr, invalid_segment);
267 for (vaddr = 0xe0000000; vaddr < KERNBASE; vaddr += SUN4C_REAL_PGDIR_SIZE)
268 sun4c_put_segmap(vaddr, invalid_segment);
269 for (vaddr = kernel_end; vaddr < KADB_DEBUGGER_BEGVM; vaddr += SUN4C_REAL_PGDIR_SIZE)
270 sun4c_put_segmap(vaddr, invalid_segment);
271 for (vaddr = LINUX_OPPROM_ENDVM; vaddr; vaddr += SUN4C_REAL_PGDIR_SIZE)
272 sun4c_put_segmap(vaddr, invalid_segment);
273 }
274 sun4c_set_context(savectx);
275}
276
277void __init sun4c_probe_vac(void)
278{
279 sun4c_disable_vac();
280
5110bd21
AB		 281 if ((idprom->id_machtype == (SM_SUN4C | SM_4C_SS1)) ||
282 (idprom->id_machtype == (SM_SUN4C | SM_4C_SS1PLUS))) {
283 /* PROM on SS1 lacks this info, to be super safe we
284 * hard code it here since this arch is cast in stone.
285 */
286 sun4c_vacinfo.num_bytes = 65536;
287 sun4c_vacinfo.linesize = 16;
1da177e4 288 } else {
5110bd21
AB		 289 sun4c_vacinfo.num_bytes =
290 prom_getintdefault(prom_root_node, "vac-size", 65536);
291 sun4c_vacinfo.linesize =
292 prom_getintdefault(prom_root_node, "vac-linesize", 16);
293 }
294 sun4c_vacinfo.do_hwflushes =
295 prom_getintdefault(prom_root_node, "vac-hwflush", 0);
1da177e4 296
5110bd21
AB		 297 if (sun4c_vacinfo.do_hwflushes == 0)
298 sun4c_vacinfo.do_hwflushes =
299 prom_getintdefault(prom_root_node, "vac_hwflush", 0);
1da177e4 300
5110bd21
AB		 301 if (sun4c_vacinfo.num_bytes != 65536) {
302 prom_printf("WEIRD Sun4C VAC cache size, "
303 "tell sparclinux@vger.kernel.org");
304 prom_halt();
1da177e4
LT		 305 }
306
1da177e4
LT		 307 switch (sun4c_vacinfo.linesize) {
308 case 16:
309 sun4c_vacinfo.log2lsize = 4;
310 break;
311 case 32:
312 sun4c_vacinfo.log2lsize = 5;
313 break;
314 default:
315 prom_printf("probe_vac: Didn't expect vac-linesize of %d, halting\n",
316 sun4c_vacinfo.linesize);
317 prom_halt();
318 };
319
320 sun4c_flush_all();
321 sun4c_enable_vac();
322}
323
324/* Patch instructions for the low level kernel fault handler. */
325extern unsigned long invalid_segment_patch1, invalid_segment_patch1_ff;
326extern unsigned long invalid_segment_patch2, invalid_segment_patch2_ff;
327extern unsigned long invalid_segment_patch1_1ff, invalid_segment_patch2_1ff;
328extern unsigned long num_context_patch1, num_context_patch1_16;
329extern unsigned long num_context_patch2_16;
330extern unsigned long vac_linesize_patch, vac_linesize_patch_32;
331extern unsigned long vac_hwflush_patch1, vac_hwflush_patch1_on;
332extern unsigned long vac_hwflush_patch2, vac_hwflush_patch2_on;
333
334#define PATCH_INSN(src, dst) do { \
335 daddr = &(dst); \
336 iaddr = &(src); \
337 *daddr = *iaddr; \
338 } while (0)
339
340static void __init patch_kernel_fault_handler(void)
341{
342 unsigned long *iaddr, *daddr;
343
344 switch (num_segmaps) {
345 case 128:
346 /* Default, nothing to do. */
347 break;
348 case 256:
349 PATCH_INSN(invalid_segment_patch1_ff,
350 invalid_segment_patch1);
351 PATCH_INSN(invalid_segment_patch2_ff,
352 invalid_segment_patch2);
353 break;
354 case 512:
355 PATCH_INSN(invalid_segment_patch1_1ff,
356 invalid_segment_patch1);
357 PATCH_INSN(invalid_segment_patch2_1ff,
358 invalid_segment_patch2);
359 break;
360 default:
361 prom_printf("Unhandled number of segmaps: %d\n",
362 num_segmaps);
363 prom_halt();
364 };
365 switch (num_contexts) {
366 case 8:
367 /* Default, nothing to do. */
368 break;
369 case 16:
370 PATCH_INSN(num_context_patch1_16,
371 num_context_patch1);
372 break;
373 default:
374 prom_printf("Unhandled number of contexts: %d\n",
375 num_contexts);
376 prom_halt();
377 };
378
379 if (sun4c_vacinfo.do_hwflushes != 0) {
380 PATCH_INSN(vac_hwflush_patch1_on, vac_hwflush_patch1);
381 PATCH_INSN(vac_hwflush_patch2_on, vac_hwflush_patch2);
382 } else {
383 switch (sun4c_vacinfo.linesize) {
384 case 16:
385 /* Default, nothing to do. */
386 break;
387 case 32:
388 PATCH_INSN(vac_linesize_patch_32, vac_linesize_patch);
389 break;
390 default:
391 prom_printf("Impossible VAC linesize %d, halting...\n",
392 sun4c_vacinfo.linesize);
393 prom_halt();
394 };
395 }
396}
397
398static void __init sun4c_probe_mmu(void)
399{
5110bd21
AB		 400 if ((idprom->id_machtype == (SM_SUN4C | SM_4C_SS1)) ||
401 (idprom->id_machtype == (SM_SUN4C | SM_4C_SS1PLUS))) {
402 /* Hardcode these just to be safe, PROM on SS1 does
403 * not have this info available in the root node.
404 */
405 num_segmaps = 128;
406 num_contexts = 8;
1da177e4 407 } else {
5110bd21
AB		 408 num_segmaps =
409 prom_getintdefault(prom_root_node, "mmu-npmg", 128);
410 num_contexts =
411 prom_getintdefault(prom_root_node, "mmu-nctx", 0x8);
1da177e4
LT		 412 }
413 patch_kernel_fault_handler();
414}
415
a3297296 416volatile unsigned long __iomem *sun4c_memerr_reg = NULL;
1da177e4
LT		 417
418void __init sun4c_probe_memerr_reg(void)
419{
420 int node;
421 struct linux_prom_registers regs[1];
422
5110bd21
AB		 423 node = prom_getchild(prom_root_node);
424 node = prom_searchsiblings(prom_root_node, "memory-error");
425 if (!node)
426 return;
427 if (prom_getproperty(node, "reg", (char *)regs, sizeof(regs)) <= 0)
428 return;
429 /* hmm I think regs[0].which_io is zero here anyways */
430 sun4c_memerr_reg = ioremap(regs[0].phys_addr, regs[0].reg_size);
1da177e4
LT		 431}
432
433static inline void sun4c_init_ss2_cache_bug(void)
434{
435 extern unsigned long start;
436
437 if ((idprom->id_machtype == (SM_SUN4C | SM_4C_SS2)) ||
438 (idprom->id_machtype == (SM_SUN4C | SM_4C_IPX)) ||
1da177e4
LT		 439 (idprom->id_machtype == (SM_SUN4C | SM_4C_ELC))) {
440 /* Whee.. */
441 printk("SS2 cache bug detected, uncaching trap table page\n");
442 sun4c_flush_page((unsigned int) &start);
443 sun4c_put_pte(((unsigned long) &start),
444 (sun4c_get_pte((unsigned long) &start) | _SUN4C_PAGE_NOCACHE));
445 }
446}
447
448/* Addr is always aligned on a page boundary for us already. */
4b1c5df2
DM		 449static int sun4c_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned long va,
450 unsigned long addr, int len)
1da177e4
LT		 451{
452 unsigned long page, end;
453
454 *pba = addr;
455
456 end = PAGE_ALIGN((addr + len));
457 while (addr < end) {
458 page = va;
459 sun4c_flush_page(page);
460 page -= PAGE_OFFSET;
461 page >>= PAGE_SHIFT;
462 page |= (_SUN4C_PAGE_VALID | _SUN4C_PAGE_DIRTY |
463 _SUN4C_PAGE_NOCACHE | _SUN4C_PAGE_PRIV);
464 sun4c_put_pte(addr, page);
465 addr += PAGE_SIZE;
466 va += PAGE_SIZE;
467 }
468
469 return 0;
470}
471
4b1c5df2 472static void sun4c_unmap_dma_area(struct device *dev, unsigned long busa, int len)
1da177e4
LT		 473{
474 /* Fortunately for us, bus_addr == uncached_virt in sun4c. */
475 /* XXX Implement this */
476}
477
478/* TLB management. */
479
480/* Don't change this struct without changing entry.S. This is used
481 * in the in-window kernel fault handler, and you don't want to mess
482 * with that. (See sun4c_fault in entry.S).
483 */
484struct sun4c_mmu_entry {
485 struct sun4c_mmu_entry *next;
486 struct sun4c_mmu_entry *prev;
487 unsigned long vaddr;
488 unsigned char pseg;
489 unsigned char locked;
490
491 /* For user mappings only, and completely hidden from kernel
492 * TLB miss code.
493 */
494 unsigned char ctx;
495 struct sun4c_mmu_entry *lru_next;
496 struct sun4c_mmu_entry *lru_prev;
497};
498
499static struct sun4c_mmu_entry mmu_entry_pool[SUN4C_MAX_SEGMAPS];
500
501static void __init sun4c_init_mmu_entry_pool(void)
502{
503 int i;
504
505 for (i=0; i < SUN4C_MAX_SEGMAPS; i++) {
506 mmu_entry_pool[i].pseg = i;
507 mmu_entry_pool[i].next = NULL;
508 mmu_entry_pool[i].prev = NULL;
509 mmu_entry_pool[i].vaddr = 0;
510 mmu_entry_pool[i].locked = 0;
511 mmu_entry_pool[i].ctx = 0;
512 mmu_entry_pool[i].lru_next = NULL;
513 mmu_entry_pool[i].lru_prev = NULL;
514 }
515 mmu_entry_pool[invalid_segment].locked = 1;
516}
517
518static inline void fix_permissions(unsigned long vaddr, unsigned long bits_on,
519 unsigned long bits_off)
520{
521 unsigned long start, end;
522
523 end = vaddr + SUN4C_REAL_PGDIR_SIZE;
524 for (start = vaddr; start < end; start += PAGE_SIZE)
525 if (sun4c_get_pte(start) & _SUN4C_PAGE_VALID)
526 sun4c_put_pte(start, (sun4c_get_pte(start) | bits_on) &
527 ~bits_off);
528}
529
530static inline void sun4c_init_map_kernelprom(unsigned long kernel_end)
531{
532 unsigned long vaddr;
533 unsigned char pseg, ctx;
5110bd21 534
1da177e4
LT		 535 for (vaddr = KADB_DEBUGGER_BEGVM;
536 vaddr < LINUX_OPPROM_ENDVM;
537 vaddr += SUN4C_REAL_PGDIR_SIZE) {
538 pseg = sun4c_get_segmap(vaddr);
539 if (pseg != invalid_segment) {
540 mmu_entry_pool[pseg].locked = 1;
541 for (ctx = 0; ctx < num_contexts; ctx++)
542 prom_putsegment(ctx, vaddr, pseg);
543 fix_permissions(vaddr, _SUN4C_PAGE_PRIV, 0);
544 }
545 }
5110bd21 546
1da177e4
LT		 547 for (vaddr = KERNBASE; vaddr < kernel_end; vaddr += SUN4C_REAL_PGDIR_SIZE) {
548 pseg = sun4c_get_segmap(vaddr);
549 mmu_entry_pool[pseg].locked = 1;
550 for (ctx = 0; ctx < num_contexts; ctx++)
551 prom_putsegment(ctx, vaddr, pseg);
552 fix_permissions(vaddr, _SUN4C_PAGE_PRIV, _SUN4C_PAGE_NOCACHE);
553 }
554}
555
556static void __init sun4c_init_lock_area(unsigned long start, unsigned long end)
557{
558 int i, ctx;
559
560 while (start < end) {
561 for (i = 0; i < invalid_segment; i++)
562 if (!mmu_entry_pool[i].locked)
563 break;
564 mmu_entry_pool[i].locked = 1;
565 sun4c_init_clean_segmap(i);
566 for (ctx = 0; ctx < num_contexts; ctx++)
567 prom_putsegment(ctx, start, mmu_entry_pool[i].pseg);
568 start += SUN4C_REAL_PGDIR_SIZE;
569 }
570}
571
572/* Don't change this struct without changing entry.S. This is used
573 * in the in-window kernel fault handler, and you don't want to mess
574 * with that. (See sun4c_fault in entry.S).
575 */
576struct sun4c_mmu_ring {
577 struct sun4c_mmu_entry ringhd;
578 int num_entries;
579};
580
581static struct sun4c_mmu_ring sun4c_context_ring[SUN4C_MAX_CONTEXTS]; /* used user entries */
582static struct sun4c_mmu_ring sun4c_ufree_ring; /* free user entries */
583static struct sun4c_mmu_ring sun4c_ulru_ring; /* LRU user entries */
584struct sun4c_mmu_ring sun4c_kernel_ring; /* used kernel entries */
585struct sun4c_mmu_ring sun4c_kfree_ring; /* free kernel entries */
586
587static inline void sun4c_init_rings(void)
588{
589 int i;
590
591 for (i = 0; i < SUN4C_MAX_CONTEXTS; i++) {
592 sun4c_context_ring[i].ringhd.next =
593 sun4c_context_ring[i].ringhd.prev =
594 &sun4c_context_ring[i].ringhd;
595 sun4c_context_ring[i].num_entries = 0;
596 }
597 sun4c_ufree_ring.ringhd.next = sun4c_ufree_ring.ringhd.prev =
598 &sun4c_ufree_ring.ringhd;
599 sun4c_ufree_ring.num_entries = 0;
600 sun4c_ulru_ring.ringhd.lru_next = sun4c_ulru_ring.ringhd.lru_prev =
601 &sun4c_ulru_ring.ringhd;
602 sun4c_ulru_ring.num_entries = 0;
603 sun4c_kernel_ring.ringhd.next = sun4c_kernel_ring.ringhd.prev =
604 &sun4c_kernel_ring.ringhd;
605 sun4c_kernel_ring.num_entries = 0;
606 sun4c_kfree_ring.ringhd.next = sun4c_kfree_ring.ringhd.prev =
607 &sun4c_kfree_ring.ringhd;
608 sun4c_kfree_ring.num_entries = 0;
609}
610
611static void add_ring(struct sun4c_mmu_ring *ring,
612 struct sun4c_mmu_entry *entry)
613{
614 struct sun4c_mmu_entry *head = &ring->ringhd;
615
616 entry->prev = head;
617 (entry->next = head->next)->prev = entry;
618 head->next = entry;
619 ring->num_entries++;
620}
621
64d329ee 622static inline void add_lru(struct sun4c_mmu_entry *entry)
1da177e4
LT		 623{
624 struct sun4c_mmu_ring *ring = &sun4c_ulru_ring;
625 struct sun4c_mmu_entry *head = &ring->ringhd;
626
627 entry->lru_next = head;
628 (entry->lru_prev = head->lru_prev)->lru_next = entry;
629 head->lru_prev = entry;
630}
631
632static void add_ring_ordered(struct sun4c_mmu_ring *ring,
633 struct sun4c_mmu_entry *entry)
634{
635 struct sun4c_mmu_entry *head = &ring->ringhd;
636 unsigned long addr = entry->vaddr;
637
638 while ((head->next != &ring->ringhd) && (head->next->vaddr < addr))
639 head = head->next;
640
641 entry->prev = head;
642 (entry->next = head->next)->prev = entry;
643 head->next = entry;
644 ring->num_entries++;
645
646 add_lru(entry);
647}
648
64d329ee 649static inline void remove_ring(struct sun4c_mmu_ring *ring,
1da177e4
LT		 650 struct sun4c_mmu_entry *entry)
651{
652 struct sun4c_mmu_entry *next = entry->next;
653
654 (next->prev = entry->prev)->next = next;
655 ring->num_entries--;
656}
657
658static void remove_lru(struct sun4c_mmu_entry *entry)
659{
660 struct sun4c_mmu_entry *next = entry->lru_next;
661
662 (next->lru_prev = entry->lru_prev)->lru_next = next;
663}
664
665static void free_user_entry(int ctx, struct sun4c_mmu_entry *entry)
666{
667 remove_ring(sun4c_context_ring+ctx, entry);
668 remove_lru(entry);
669 add_ring(&sun4c_ufree_ring, entry);
670}
671
672static void free_kernel_entry(struct sun4c_mmu_entry *entry,
673 struct sun4c_mmu_ring *ring)
674{
675 remove_ring(ring, entry);
676 add_ring(&sun4c_kfree_ring, entry);
677}
678
679static void __init sun4c_init_fill_kernel_ring(int howmany)
680{
681 int i;
682
683 while (howmany) {
684 for (i = 0; i < invalid_segment; i++)
685 if (!mmu_entry_pool[i].locked)
686 break;
687 mmu_entry_pool[i].locked = 1;
688 sun4c_init_clean_segmap(i);
689 add_ring(&sun4c_kfree_ring, &mmu_entry_pool[i]);
690 howmany--;
691 }
692}
693
694static void __init sun4c_init_fill_user_ring(void)
695{
696 int i;
697
698 for (i = 0; i < invalid_segment; i++) {
699 if (mmu_entry_pool[i].locked)
700 continue;
701 sun4c_init_clean_segmap(i);
702 add_ring(&sun4c_ufree_ring, &mmu_entry_pool[i]);
703 }
704}
705
706static void sun4c_kernel_unmap(struct sun4c_mmu_entry *kentry)
707{
708 int savectx, ctx;
709
710 savectx = sun4c_get_context();
711 for (ctx = 0; ctx < num_contexts; ctx++) {
712 sun4c_set_context(ctx);
713 sun4c_put_segmap(kentry->vaddr, invalid_segment);
714 }
715 sun4c_set_context(savectx);
716}
717
718static void sun4c_kernel_map(struct sun4c_mmu_entry *kentry)
719{
720 int savectx, ctx;
721
722 savectx = sun4c_get_context();
723 for (ctx = 0; ctx < num_contexts; ctx++) {
724 sun4c_set_context(ctx);
725 sun4c_put_segmap(kentry->vaddr, kentry->pseg);
726 }
727 sun4c_set_context(savectx);
728}
729
730#define sun4c_user_unmap(__entry) \
731 sun4c_put_segmap((__entry)->vaddr, invalid_segment)
732
733static void sun4c_demap_context(struct sun4c_mmu_ring *crp, unsigned char ctx)
734{
735 struct sun4c_mmu_entry *head = &crp->ringhd;
736 unsigned long flags;
737
738 local_irq_save(flags);
739 if (head->next != head) {
740 struct sun4c_mmu_entry *entry = head->next;
741 int savectx = sun4c_get_context();
742
743 flush_user_windows();
744 sun4c_set_context(ctx);
745 sun4c_flush_context();
746 do {
747 struct sun4c_mmu_entry *next = entry->next;
748
749 sun4c_user_unmap(entry);
750 free_user_entry(ctx, entry);
751
752 entry = next;
753 } while (entry != head);
754 sun4c_set_context(savectx);
755 }
756 local_irq_restore(flags);
757}
758
759static int sun4c_user_taken_entries; /* This is how much we have. */
760static int max_user_taken_entries; /* This limits us and prevents deadlock. */
761
762static struct sun4c_mmu_entry *sun4c_kernel_strategy(void)
763{
764 struct sun4c_mmu_entry *this_entry;
765
766 /* If some are free, return first one. */
767 if (sun4c_kfree_ring.num_entries) {
768 this_entry = sun4c_kfree_ring.ringhd.next;
769 return this_entry;
770 }
771
772 /* Else free one up. */
773 this_entry = sun4c_kernel_ring.ringhd.prev;
774 sun4c_flush_segment(this_entry->vaddr);
775 sun4c_kernel_unmap(this_entry);
776 free_kernel_entry(this_entry, &sun4c_kernel_ring);
777 this_entry = sun4c_kfree_ring.ringhd.next;
778
779 return this_entry;
780}
781
782/* Using this method to free up mmu entries eliminates a lot of
783 * potential races since we have a kernel that incurs tlb
784 * replacement faults. There may be performance penalties.
785 *
786 * NOTE: Must be called with interrupts disabled.
787 */
788static struct sun4c_mmu_entry *sun4c_user_strategy(void)
789{
790 struct sun4c_mmu_entry *entry;
791 unsigned char ctx;
792 int savectx;
793
794 /* If some are free, return first one. */
795 if (sun4c_ufree_ring.num_entries) {
796 entry = sun4c_ufree_ring.ringhd.next;
797 goto unlink_out;
798 }
799
800 if (sun4c_user_taken_entries) {
801 entry = sun4c_kernel_strategy();
802 sun4c_user_taken_entries--;
803 goto kunlink_out;
804 }
805
806 /* Grab from the beginning of the LRU list. */
807 entry = sun4c_ulru_ring.ringhd.lru_next;
808 ctx = entry->ctx;
809
810 savectx = sun4c_get_context();
811 flush_user_windows();
812 sun4c_set_context(ctx);
813 sun4c_flush_segment(entry->vaddr);
814 sun4c_user_unmap(entry);
815 remove_ring(sun4c_context_ring + ctx, entry);
816 remove_lru(entry);
817 sun4c_set_context(savectx);
818
819 return entry;
820
821unlink_out:
822 remove_ring(&sun4c_ufree_ring, entry);
823 return entry;
824kunlink_out:
825 remove_ring(&sun4c_kfree_ring, entry);
826 return entry;
827}
828
829/* NOTE: Must be called with interrupts disabled. */
830void sun4c_grow_kernel_ring(void)
831{
832 struct sun4c_mmu_entry *entry;
833
834 /* Prevent deadlock condition. */
835 if (sun4c_user_taken_entries >= max_user_taken_entries)
836 return;
837
838 if (sun4c_ufree_ring.num_entries) {
839 entry = sun4c_ufree_ring.ringhd.next;
840 remove_ring(&sun4c_ufree_ring, entry);
841 add_ring(&sun4c_kfree_ring, entry);
842 sun4c_user_taken_entries++;
843 }
844}
845
846/* 2 page buckets for task struct and kernel stack allocation.
847 *
848 * TASK_STACK_BEGIN
849 * bucket[0]
850 * bucket[1]
851 * [ ... ]
852 * bucket[NR_TASK_BUCKETS-1]
853 * TASK_STACK_BEGIN + (sizeof(struct task_bucket) * NR_TASK_BUCKETS)
854 *
855 * Each slot looks like:
856 *
857 * page 1 -- task struct + beginning of kernel stack
858 * page 2 -- rest of kernel stack
859 */
860
861union task_union *sun4c_bucket[NR_TASK_BUCKETS];
862
863static int sun4c_lowbucket_avail;
864
865#define BUCKET_EMPTY ((union task_union *) 0)
866#define BUCKET_SHIFT (PAGE_SHIFT + 1) /* log2(sizeof(struct task_bucket)) */
867#define BUCKET_SIZE (1 << BUCKET_SHIFT)
868#define BUCKET_NUM(addr) ((((addr) - SUN4C_LOCK_VADDR) >> BUCKET_SHIFT))
869#define BUCKET_ADDR(num) (((num) << BUCKET_SHIFT) + SUN4C_LOCK_VADDR)
870#define BUCKET_PTE(page) \
871 ((((page) - PAGE_OFFSET) >> PAGE_SHIFT) | pgprot_val(SUN4C_PAGE_KERNEL))
872#define BUCKET_PTE_PAGE(pte) \
873 (PAGE_OFFSET + (((pte) & SUN4C_PFN_MASK) << PAGE_SHIFT))
874
875static void get_locked_segment(unsigned long addr)
876{
877 struct sun4c_mmu_entry *stolen;
878 unsigned long flags;
879
880 local_irq_save(flags);
881 addr &= SUN4C_REAL_PGDIR_MASK;
882 stolen = sun4c_user_strategy();
883 max_user_taken_entries--;
884 stolen->vaddr = addr;
885 flush_user_windows();
886 sun4c_kernel_map(stolen);
887 local_irq_restore(flags);
888}
889
890static void free_locked_segment(unsigned long addr)
891{
892 struct sun4c_mmu_entry *entry;
893 unsigned long flags;
894 unsigned char pseg;
895
896 local_irq_save(flags);
897 addr &= SUN4C_REAL_PGDIR_MASK;
898 pseg = sun4c_get_segmap(addr);
899 entry = &mmu_entry_pool[pseg];
900
901 flush_user_windows();
902 sun4c_flush_segment(addr);
903 sun4c_kernel_unmap(entry);
904 add_ring(&sun4c_ufree_ring, entry);
905 max_user_taken_entries++;
906 local_irq_restore(flags);
907}
908
909static inline void garbage_collect(int entry)
910{
911 int start, end;
912
913 /* 32 buckets per segment... */
914 entry &= ~31;
915 start = entry;
916 for (end = (start + 32); start < end; start++)
917 if (sun4c_bucket[start] != BUCKET_EMPTY)
918 return;
919
920 /* Entire segment empty, release it. */
921 free_locked_segment(BUCKET_ADDR(entry));
922}
923
924static struct thread_info *sun4c_alloc_thread_info(void)
925{
926 unsigned long addr, pages;
927 int entry;
928
929 pages = __get_free_pages(GFP_KERNEL, THREAD_INFO_ORDER);
930 if (!pages)
931 return NULL;
932
933 for (entry = sun4c_lowbucket_avail; entry < NR_TASK_BUCKETS; entry++)
934 if (sun4c_bucket[entry] == BUCKET_EMPTY)
935 break;
936 if (entry == NR_TASK_BUCKETS) {
937 free_pages(pages, THREAD_INFO_ORDER);
938 return NULL;
939 }
940 if (entry >= sun4c_lowbucket_avail)
941 sun4c_lowbucket_avail = entry + 1;
942
943 addr = BUCKET_ADDR(entry);
944 sun4c_bucket[entry] = (union task_union *) addr;
945 if(sun4c_get_segmap(addr) == invalid_segment)
946 get_locked_segment(addr);
947
948 /* We are changing the virtual color of the page(s)
949 * so we must flush the cache to guarantee consistency.
950 */
951 sun4c_flush_page(pages);
1da177e4 952 sun4c_flush_page(pages + PAGE_SIZE);
1da177e4
LT		 953
954 sun4c_put_pte(addr, BUCKET_PTE(pages));
1da177e4 955 sun4c_put_pte(addr + PAGE_SIZE, BUCKET_PTE(pages + PAGE_SIZE));
1da177e4
LT		 956
957#ifdef CONFIG_DEBUG_STACK_USAGE
958 memset((void *)addr, 0, PAGE_SIZE << THREAD_INFO_ORDER);
959#endif /* DEBUG_STACK_USAGE */
960
961 return (struct thread_info *) addr;
962}
963
964static void sun4c_free_thread_info(struct thread_info *ti)
965{
966 unsigned long tiaddr = (unsigned long) ti;
967 unsigned long pages = BUCKET_PTE_PAGE(sun4c_get_pte(tiaddr));
968 int entry = BUCKET_NUM(tiaddr);
969
970 /* We are deleting a mapping, so the flush here is mandatory. */
971 sun4c_flush_page(tiaddr);
1da177e4 972 sun4c_flush_page(tiaddr + PAGE_SIZE);
5110bd21 973
1da177e4 974 sun4c_put_pte(tiaddr, 0);
1da177e4 975 sun4c_put_pte(tiaddr + PAGE_SIZE, 0);
5110bd21 976
1da177e4
LT		 977 sun4c_bucket[entry] = BUCKET_EMPTY;
978 if (entry < sun4c_lowbucket_avail)
979 sun4c_lowbucket_avail = entry;
980
981 free_pages(pages, THREAD_INFO_ORDER);
982 garbage_collect(entry);
983}
984
985static void __init sun4c_init_buckets(void)
986{
987 int entry;
988
989 if (sizeof(union thread_union) != (PAGE_SIZE << THREAD_INFO_ORDER)) {
990 extern void thread_info_size_is_bolixed_pete(void);
991 thread_info_size_is_bolixed_pete();
992 }
993
994 for (entry = 0; entry < NR_TASK_BUCKETS; entry++)
995 sun4c_bucket[entry] = BUCKET_EMPTY;
996 sun4c_lowbucket_avail = 0;
997}
998
999static unsigned long sun4c_iobuffer_start;
1000static unsigned long sun4c_iobuffer_end;
1001static unsigned long sun4c_iobuffer_high;
1002static unsigned long *sun4c_iobuffer_map;
1003static int iobuffer_map_size;
1004
1005/*
1006 * Alias our pages so they do not cause a trap.
1007 * Also one page may be aliased into several I/O areas and we may
1008 * finish these I/O separately.
1009 */
1010static char *sun4c_lockarea(char *vaddr, unsigned long size)
1011{
1012 unsigned long base, scan;
1013 unsigned long npages;
1014 unsigned long vpage;
1015 unsigned long pte;
1016 unsigned long apage;
1017 unsigned long high;
1018 unsigned long flags;
1019
1020 npages = (((unsigned long)vaddr & ~PAGE_MASK) +
1021 size + (PAGE_SIZE-1)) >> PAGE_SHIFT;
1022
1023 scan = 0;
1024 local_irq_save(flags);
1025 for (;;) {
1026 scan = find_next_zero_bit(sun4c_iobuffer_map,
1027 iobuffer_map_size, scan);
1028 if ((base = scan) + npages > iobuffer_map_size) goto abend;
1029 for (;;) {
1030 if (scan >= base + npages) goto found;
1031 if (test_bit(scan, sun4c_iobuffer_map)) break;
1032 scan++;
1033 }
1034 }
1035
1036found:
1037 high = ((base + npages) << PAGE_SHIFT) + sun4c_iobuffer_start;
1038 high = SUN4C_REAL_PGDIR_ALIGN(high);
1039 while (high > sun4c_iobuffer_high) {
1040 get_locked_segment(sun4c_iobuffer_high);
1041 sun4c_iobuffer_high += SUN4C_REAL_PGDIR_SIZE;
1042 }
1043
1044 vpage = ((unsigned long) vaddr) & PAGE_MASK;
1045 for (scan = base; scan < base+npages; scan++) {
1046 pte = ((vpage-PAGE_OFFSET) >> PAGE_SHIFT);
1047 pte |= pgprot_val(SUN4C_PAGE_KERNEL);
1048 pte |= _SUN4C_PAGE_NOCACHE;
1049 set_bit(scan, sun4c_iobuffer_map);
1050 apage = (scan << PAGE_SHIFT) + sun4c_iobuffer_start;
1051
1052 /* Flush original mapping so we see the right things later. */
1053 sun4c_flush_page(vpage);
1054
1055 sun4c_put_pte(apage, pte);
1056 vpage += PAGE_SIZE;
1057 }
1058 local_irq_restore(flags);
1059 return (char *) ((base << PAGE_SHIFT) + sun4c_iobuffer_start +
1060 (((unsigned long) vaddr) & ~PAGE_MASK));
1061
1062abend:
1063 local_irq_restore(flags);
1064 printk("DMA vaddr=0x%p size=%08lx\n", vaddr, size);
1065 panic("Out of iobuffer table");
1066 return NULL;
1067}
1068
1069static void sun4c_unlockarea(char *vaddr, unsigned long size)
1070{
1071 unsigned long vpage, npages;
1072 unsigned long flags;
1073 int scan, high;
1074
1075 vpage = (unsigned long)vaddr & PAGE_MASK;
1076 npages = (((unsigned long)vaddr & ~PAGE_MASK) +
1077 size + (PAGE_SIZE-1)) >> PAGE_SHIFT;
1078
1079 local_irq_save(flags);
1080 while (npages != 0) {
1081 --npages;
1082
1083 /* This mapping is marked non-cachable, no flush necessary. */
1084 sun4c_put_pte(vpage, 0);
1085 clear_bit((vpage - sun4c_iobuffer_start) >> PAGE_SHIFT,
1086 sun4c_iobuffer_map);
1087 vpage += PAGE_SIZE;
1088 }
1089
1090 /* garbage collect */
1091 scan = (sun4c_iobuffer_high - sun4c_iobuffer_start) >> PAGE_SHIFT;
1092 while (scan >= 0 && !sun4c_iobuffer_map[scan >> 5])
1093 scan -= 32;
1094 scan += 32;
1095 high = sun4c_iobuffer_start + (scan << PAGE_SHIFT);
1096 high = SUN4C_REAL_PGDIR_ALIGN(high) + SUN4C_REAL_PGDIR_SIZE;
1097 while (high < sun4c_iobuffer_high) {
1098 sun4c_iobuffer_high -= SUN4C_REAL_PGDIR_SIZE;
1099 free_locked_segment(sun4c_iobuffer_high);
1100 }
1101 local_irq_restore(flags);
1102}
1103
1104/* Note the scsi code at init time passes to here buffers
1105 * which sit on the kernel stack, those are already locked
1106 * by implication and fool the page locking code above
1107 * if passed to by mistake.
1108 */
9723f38e 1109static __u32 sun4c_get_scsi_one(struct device *dev, char *bufptr, unsigned long len)
1da177e4
LT		 1110{
1111 unsigned long page;
1112
1113 page = ((unsigned long)bufptr) & PAGE_MASK;
1114 if (!virt_addr_valid(page)) {
1115 sun4c_flush_page(page);
1116 return (__u32)bufptr; /* already locked */
1117 }
1118 return (__u32)sun4c_lockarea(bufptr, len);
1119}
1120
9723f38e 1121static void sun4c_get_scsi_sgl(struct device *dev, struct scatterlist *sg, int sz)
1da177e4
LT		 1122{
1123 while (sz != 0) {
1124 --sz;
58b053e4 1125 sg->dvma_address = (__u32)sun4c_lockarea(sg_virt(sg), sg->length);
0912a5db
JA		 1126 sg->dvma_length = sg->length;
1127 sg = sg_next(sg);
1da177e4
LT		 1128 }
1129}
1130
9723f38e 1131static void sun4c_release_scsi_one(struct device *dev, __u32 bufptr, unsigned long len)
1da177e4
LT		 1132{
1133 if (bufptr < sun4c_iobuffer_start)
1134 return; /* On kernel stack or similar, see above */
1135 sun4c_unlockarea((char *)bufptr, len);
1136}
1137
9723f38e 1138static void sun4c_release_scsi_sgl(struct device *dev, struct scatterlist *sg, int sz)
1da177e4
LT		 1139{
1140 while (sz != 0) {
1141 --sz;
0912a5db
JA		 1142 sun4c_unlockarea((char *)sg->dvma_address, sg->length);
1143 sg = sg_next(sg);
1da177e4
LT		 1144 }
1145}
1146
1147#define TASK_ENTRY_SIZE BUCKET_SIZE /* see above */
1148#define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1))
1149
1150struct vm_area_struct sun4c_kstack_vma;
1151
1152static void __init sun4c_init_lock_areas(void)
1153{
1154 unsigned long sun4c_taskstack_start;
1155 unsigned long sun4c_taskstack_end;
1156 int bitmap_size;
1157
1158 sun4c_init_buckets();
1159 sun4c_taskstack_start = SUN4C_LOCK_VADDR;
1160 sun4c_taskstack_end = (sun4c_taskstack_start +
1161 (TASK_ENTRY_SIZE * NR_TASK_BUCKETS));
1162 if (sun4c_taskstack_end >= SUN4C_LOCK_END) {
1163 prom_printf("Too many tasks, decrease NR_TASK_BUCKETS please.\n");
1164 prom_halt();
1165 }
1166
1167 sun4c_iobuffer_start = sun4c_iobuffer_high =
1168 SUN4C_REAL_PGDIR_ALIGN(sun4c_taskstack_end);
1169 sun4c_iobuffer_end = SUN4C_LOCK_END;
1170 bitmap_size = (sun4c_iobuffer_end - sun4c_iobuffer_start) >> PAGE_SHIFT;
1171 bitmap_size = (bitmap_size + 7) >> 3;
1172 bitmap_size = LONG_ALIGN(bitmap_size);
1173 iobuffer_map_size = bitmap_size << 3;
1174 sun4c_iobuffer_map = __alloc_bootmem(bitmap_size, SMP_CACHE_BYTES, 0UL);
1175 memset((void *) sun4c_iobuffer_map, 0, bitmap_size);
1176
1177 sun4c_kstack_vma.vm_mm = &init_mm;
1178 sun4c_kstack_vma.vm_start = sun4c_taskstack_start;
1179 sun4c_kstack_vma.vm_end = sun4c_taskstack_end;
1180 sun4c_kstack_vma.vm_page_prot = PAGE_SHARED;
1181 sun4c_kstack_vma.vm_flags = VM_READ | VM_WRITE | VM_EXEC;
1182 insert_vm_struct(&init_mm, &sun4c_kstack_vma);
1183}
1184
1185/* Cache flushing on the sun4c. */
1186static void sun4c_flush_cache_all(void)
1187{
1188 unsigned long begin, end;
1189
1190 flush_user_windows();
1191 begin = (KERNBASE + SUN4C_REAL_PGDIR_SIZE);
1192 end = (begin + SUN4C_VAC_SIZE);
1193
1194 if (sun4c_vacinfo.linesize == 32) {
1195 while (begin < end) {
1196 __asm__ __volatile__(
1197 "ld [%0 + 0x00], %%g0\n\t"
1198 "ld [%0 + 0x20], %%g0\n\t"
1199 "ld [%0 + 0x40], %%g0\n\t"
1200 "ld [%0 + 0x60], %%g0\n\t"
1201 "ld [%0 + 0x80], %%g0\n\t"
1202 "ld [%0 + 0xa0], %%g0\n\t"
1203 "ld [%0 + 0xc0], %%g0\n\t"
1204 "ld [%0 + 0xe0], %%g0\n\t"
1205 "ld [%0 + 0x100], %%g0\n\t"
1206 "ld [%0 + 0x120], %%g0\n\t"
1207 "ld [%0 + 0x140], %%g0\n\t"
1208 "ld [%0 + 0x160], %%g0\n\t"
1209 "ld [%0 + 0x180], %%g0\n\t"
1210 "ld [%0 + 0x1a0], %%g0\n\t"
1211 "ld [%0 + 0x1c0], %%g0\n\t"
1212 "ld [%0 + 0x1e0], %%g0\n"
1213 : : "r" (begin));
1214 begin += 512;
1215 }
1216 } else {
1217 while (begin < end) {
1218 __asm__ __volatile__(
1219 "ld [%0 + 0x00], %%g0\n\t"
1220 "ld [%0 + 0x10], %%g0\n\t"
1221 "ld [%0 + 0x20], %%g0\n\t"
1222 "ld [%0 + 0x30], %%g0\n\t"
1223 "ld [%0 + 0x40], %%g0\n\t"
1224 "ld [%0 + 0x50], %%g0\n\t"
1225 "ld [%0 + 0x60], %%g0\n\t"
1226 "ld [%0 + 0x70], %%g0\n\t"
1227 "ld [%0 + 0x80], %%g0\n\t"
1228 "ld [%0 + 0x90], %%g0\n\t"
1229 "ld [%0 + 0xa0], %%g0\n\t"
1230 "ld [%0 + 0xb0], %%g0\n\t"
1231 "ld [%0 + 0xc0], %%g0\n\t"
1232 "ld [%0 + 0xd0], %%g0\n\t"
1233 "ld [%0 + 0xe0], %%g0\n\t"
1234 "ld [%0 + 0xf0], %%g0\n"
1235 : : "r" (begin));
1236 begin += 256;
1237 }
1238 }
1239}
1240
1241static void sun4c_flush_cache_mm(struct mm_struct *mm)
1242{
1243 int new_ctx = mm->context;
1244
1245 if (new_ctx != NO_CONTEXT) {
1246 flush_user_windows();
1247
1248 if (sun4c_context_ring[new_ctx].num_entries) {
1249 struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
1250 unsigned long flags;
1251
1252 local_irq_save(flags);
1253 if (head->next != head) {
1254 struct sun4c_mmu_entry *entry = head->next;
1255 int savectx = sun4c_get_context();
1256
1257 sun4c_set_context(new_ctx);
1258 sun4c_flush_context();
1259 do {
1260 struct sun4c_mmu_entry *next = entry->next;
1261
1262 sun4c_user_unmap(entry);
1263 free_user_entry(new_ctx, entry);
1264
1265 entry = next;
1266 } while (entry != head);
1267 sun4c_set_context(savectx);
1268 }
1269 local_irq_restore(flags);
1270 }
1271 }
1272}
1273
1274static void sun4c_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1275{
1276 struct mm_struct *mm = vma->vm_mm;
1277 int new_ctx = mm->context;
1278
1279 if (new_ctx != NO_CONTEXT) {
1280 struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
1281 struct sun4c_mmu_entry *entry;
1282 unsigned long flags;
1283
1284 flush_user_windows();
1285
1286 local_irq_save(flags);
1287 /* All user segmap chains are ordered on entry->vaddr. */
1288 for (entry = head->next;
1289 (entry != head) && ((entry->vaddr+SUN4C_REAL_PGDIR_SIZE) < start);
1290 entry = entry->next)
1291 ;
1292
1293 /* Tracing various job mixtures showed that this conditional
1294 * only passes ~35% of the time for most worse case situations,
1295 * therefore we avoid all of this gross overhead ~65% of the time.
1296 */
1297 if ((entry != head) && (entry->vaddr < end)) {
1298 int octx = sun4c_get_context();
1299 sun4c_set_context(new_ctx);
1300
1301 /* At this point, always, (start >= entry->vaddr) and
1302 * (entry->vaddr < end), once the latter condition
1303 * ceases to hold, or we hit the end of the list, we
1304 * exit the loop. The ordering of all user allocated
1305 * segmaps makes this all work out so beautifully.
1306 */
1307 do {
1308 struct sun4c_mmu_entry *next = entry->next;
1309 unsigned long realend;
1310
1311 /* "realstart" is always >= entry->vaddr */
1312 realend = entry->vaddr + SUN4C_REAL_PGDIR_SIZE;
1313 if (end < realend)
1314 realend = end;
1315 if ((realend - entry->vaddr) <= (PAGE_SIZE << 3)) {
1316 unsigned long page = entry->vaddr;
1317 while (page < realend) {
1318 sun4c_flush_page(page);
1319 page += PAGE_SIZE;
1320 }
1321 } else {
1322 sun4c_flush_segment(entry->vaddr);
1323 sun4c_user_unmap(entry);
1324 free_user_entry(new_ctx, entry);
1325 }
1326 entry = next;
1327 } while ((entry != head) && (entry->vaddr < end));
1328 sun4c_set_context(octx);
1329 }
1330 local_irq_restore(flags);
1331 }
1332}
1333
1334static void sun4c_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
1335{
1336 struct mm_struct *mm = vma->vm_mm;
1337 int new_ctx = mm->context;
1338
1339 /* Sun4c has no separate I/D caches so cannot optimize for non
1340 * text page flushes.
1341 */
1342 if (new_ctx != NO_CONTEXT) {
1343 int octx = sun4c_get_context();
1344 unsigned long flags;
1345
1346 flush_user_windows();
1347 local_irq_save(flags);
1348 sun4c_set_context(new_ctx);
1349 sun4c_flush_page(page);
1350 sun4c_set_context(octx);
1351 local_irq_restore(flags);
1352 }
1353}
1354
1355static void sun4c_flush_page_to_ram(unsigned long page)
1356{
1357 unsigned long flags;
1358
1359 local_irq_save(flags);
1360 sun4c_flush_page(page);
1361 local_irq_restore(flags);
1362}
1363
1364/* Sun4c cache is unified, both instructions and data live there, so
1365 * no need to flush the on-stack instructions for new signal handlers.
1366 */
1367static void sun4c_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr)
1368{
1369}
1370
1371/* TLB flushing on the sun4c. These routines count on the cache
1372 * flushing code to flush the user register windows so that we need
1373 * not do so when we get here.
1374 */
1375
1376static void sun4c_flush_tlb_all(void)
1377{
1378 struct sun4c_mmu_entry *this_entry, *next_entry;
1379 unsigned long flags;
1380 int savectx, ctx;
1381
1382 local_irq_save(flags);
1383 this_entry = sun4c_kernel_ring.ringhd.next;
1384 savectx = sun4c_get_context();
1385 flush_user_windows();
1386 while (sun4c_kernel_ring.num_entries) {
1387 next_entry = this_entry->next;
1388 sun4c_flush_segment(this_entry->vaddr);
1389 for (ctx = 0; ctx < num_contexts; ctx++) {
1390 sun4c_set_context(ctx);
1391 sun4c_put_segmap(this_entry->vaddr, invalid_segment);
1392 }
1393 free_kernel_entry(this_entry, &sun4c_kernel_ring);
1394 this_entry = next_entry;
1395 }
1396 sun4c_set_context(savectx);
1397 local_irq_restore(flags);
1398}
1399
1400static void sun4c_flush_tlb_mm(struct mm_struct *mm)
1401{
1402 int new_ctx = mm->context;
1403
1404 if (new_ctx != NO_CONTEXT) {
1405 struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
1406 unsigned long flags;
1407
1408 local_irq_save(flags);
1409 if (head->next != head) {
1410 struct sun4c_mmu_entry *entry = head->next;
1411 int savectx = sun4c_get_context();
1412
1413 sun4c_set_context(new_ctx);
1414 sun4c_flush_context();
1415 do {
1416 struct sun4c_mmu_entry *next = entry->next;
1417
1418 sun4c_user_unmap(entry);
1419 free_user_entry(new_ctx, entry);
1420
1421 entry = next;
1422 } while (entry != head);
1423 sun4c_set_context(savectx);
1424 }
1425 local_irq_restore(flags);
1426 }
1427}
1428
1429static void sun4c_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1430{
1431 struct mm_struct *mm = vma->vm_mm;
1432 int new_ctx = mm->context;
1433
1434 if (new_ctx != NO_CONTEXT) {
1435 struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
1436 struct sun4c_mmu_entry *entry;
1437 unsigned long flags;
1438
1439 local_irq_save(flags);
1440 /* See commentary in sun4c_flush_cache_range(). */
1441 for (entry = head->next;
1442 (entry != head) && ((entry->vaddr+SUN4C_REAL_PGDIR_SIZE) < start);
1443 entry = entry->next)
1444 ;
1445
1446 if ((entry != head) && (entry->vaddr < end)) {
1447 int octx = sun4c_get_context();
1448
1449 sun4c_set_context(new_ctx);
1450 do {
1451 struct sun4c_mmu_entry *next = entry->next;
1452
1453 sun4c_flush_segment(entry->vaddr);
1454 sun4c_user_unmap(entry);
1455 free_user_entry(new_ctx, entry);
1456
1457 entry = next;
1458 } while ((entry != head) && (entry->vaddr < end));
1459 sun4c_set_context(octx);
1460 }
1461 local_irq_restore(flags);
1462 }
1463}
1464
1465static void sun4c_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
1466{
1467 struct mm_struct *mm = vma->vm_mm;
1468 int new_ctx = mm->context;
1469
1470 if (new_ctx != NO_CONTEXT) {
1471 int savectx = sun4c_get_context();
1472 unsigned long flags;
1473
1474 local_irq_save(flags);
1475 sun4c_set_context(new_ctx);
1476 page &= PAGE_MASK;
1477 sun4c_flush_page(page);
1478 sun4c_put_pte(page, 0);
1479 sun4c_set_context(savectx);
1480 local_irq_restore(flags);
1481 }
1482}
1483
1484static inline void sun4c_mapioaddr(unsigned long physaddr, unsigned long virt_addr)
1485{
14778d90
DM		 1486 unsigned long page_entry, pg_iobits;
1487
1488 pg_iobits = _SUN4C_PAGE_PRESENT | _SUN4C_READABLE | _SUN4C_WRITEABLE |
1489 _SUN4C_PAGE_IO | _SUN4C_PAGE_NOCACHE;
1da177e4
LT		 1490
1491 page_entry = ((physaddr >> PAGE_SHIFT) & SUN4C_PFN_MASK);
1492 page_entry |= ((pg_iobits | _SUN4C_PAGE_PRIV) & ~(_SUN4C_PAGE_PRESENT));
1493 sun4c_put_pte(virt_addr, page_entry);
1494}
1495
1496static void sun4c_mapiorange(unsigned int bus, unsigned long xpa,
1497 unsigned long xva, unsigned int len)
1498{
1499 while (len != 0) {
1500 len -= PAGE_SIZE;
1501 sun4c_mapioaddr(xpa, xva);
1502 xva += PAGE_SIZE;
1503 xpa += PAGE_SIZE;
1504 }
1505}
1506
1507static void sun4c_unmapiorange(unsigned long virt_addr, unsigned int len)
1508{
1509 while (len != 0) {
1510 len -= PAGE_SIZE;
1511 sun4c_put_pte(virt_addr, 0);
1512 virt_addr += PAGE_SIZE;
1513 }
1514}
1515
1516static void sun4c_alloc_context(struct mm_struct *old_mm, struct mm_struct *mm)
1517{
1518 struct ctx_list *ctxp;
1519
1520 ctxp = ctx_free.next;
1521 if (ctxp != &ctx_free) {
1522 remove_from_ctx_list(ctxp);
1523 add_to_used_ctxlist(ctxp);
1524 mm->context = ctxp->ctx_number;
1525 ctxp->ctx_mm = mm;
1526 return;
1527 }
1528 ctxp = ctx_used.next;
1529 if (ctxp->ctx_mm == old_mm)
1530 ctxp = ctxp->next;
1531 remove_from_ctx_list(ctxp);
1532 add_to_used_ctxlist(ctxp);
1533 ctxp->ctx_mm->context = NO_CONTEXT;
1534 ctxp->ctx_mm = mm;
1535 mm->context = ctxp->ctx_number;
1536 sun4c_demap_context(&sun4c_context_ring[ctxp->ctx_number],
1537 ctxp->ctx_number);
1538}
1539
1540/* Switch the current MM context. */
1541static void sun4c_switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, struct task_struct *tsk, int cpu)
1542{
1543 struct ctx_list *ctx;
1544 int dirty = 0;
1545
1546 if (mm->context == NO_CONTEXT) {
1547 dirty = 1;
1548 sun4c_alloc_context(old_mm, mm);
1549 } else {
1550 /* Update the LRU ring of contexts. */
1551 ctx = ctx_list_pool + mm->context;
1552 remove_from_ctx_list(ctx);
1553 add_to_used_ctxlist(ctx);
1554 }
1555 if (dirty || old_mm != mm)
1556 sun4c_set_context(mm->context);
1557}
1558
1559static void sun4c_destroy_context(struct mm_struct *mm)
1560{
1561 struct ctx_list *ctx_old;
1562
1563 if (mm->context != NO_CONTEXT) {
1564 sun4c_demap_context(&sun4c_context_ring[mm->context], mm->context);
1565 ctx_old = ctx_list_pool + mm->context;
1566 remove_from_ctx_list(ctx_old);
1567 add_to_free_ctxlist(ctx_old);
1568 mm->context = NO_CONTEXT;
1569 }
1570}
1571
1572static void sun4c_mmu_info(struct seq_file *m)
1573{
1574 int used_user_entries, i;
1575
1576 used_user_entries = 0;
1577 for (i = 0; i < num_contexts; i++)
1578 used_user_entries += sun4c_context_ring[i].num_entries;
1579
1580 seq_printf(m,
1581 "vacsize\t\t: %d bytes\n"
1582 "vachwflush\t: %s\n"
1583 "vaclinesize\t: %d bytes\n"
1584 "mmuctxs\t\t: %d\n"
1585 "mmupsegs\t: %d\n"
1586 "kernelpsegs\t: %d\n"
1587 "kfreepsegs\t: %d\n"
1588 "usedpsegs\t: %d\n"
1589 "ufreepsegs\t: %d\n"
1590 "user_taken\t: %d\n"
1591 "max_taken\t: %d\n",
1592 sun4c_vacinfo.num_bytes,
1593 (sun4c_vacinfo.do_hwflushes ? "yes" : "no"),
1594 sun4c_vacinfo.linesize,
1595 num_contexts,
1596 (invalid_segment + 1),
1597 sun4c_kernel_ring.num_entries,
1598 sun4c_kfree_ring.num_entries,
1599 used_user_entries,
1600 sun4c_ufree_ring.num_entries,
1601 sun4c_user_taken_entries,
1602 max_user_taken_entries);
1603}
1604
1605/* Nothing below here should touch the mmu hardware nor the mmu_entry
1606 * data structures.
1607 */
1608
1609/* First the functions which the mid-level code uses to directly
1610 * manipulate the software page tables. Some defines since we are
1611 * emulating the i386 page directory layout.
1612 */
1613#define PGD_PRESENT 0x001
1614#define PGD_RW 0x002
1615#define PGD_USER 0x004
1616#define PGD_ACCESSED 0x020
1617#define PGD_DIRTY 0x040
1618#define PGD_TABLE (PGD_PRESENT | PGD_RW | PGD_USER | PGD_ACCESSED | PGD_DIRTY)
1619
1620static void sun4c_set_pte(pte_t *ptep, pte_t pte)
1621{
1622 *ptep = pte;
1623}
1624
1625static void sun4c_pgd_set(pgd_t * pgdp, pmd_t * pmdp)
1626{
1627}
1628
1629static void sun4c_pmd_set(pmd_t * pmdp, pte_t * ptep)
1630{
1631 pmdp->pmdv[0] = PGD_TABLE | (unsigned long) ptep;
1632}
1633
1634static void sun4c_pmd_populate(pmd_t * pmdp, struct page * ptep)
1635{
1636 if (page_address(ptep) == NULL) BUG(); /* No highmem on sun4c */
1637 pmdp->pmdv[0] = PGD_TABLE | (unsigned long) page_address(ptep);
1638}
1639
1640static int sun4c_pte_present(pte_t pte)
1641{
1642 return ((pte_val(pte) & (_SUN4C_PAGE_PRESENT | _SUN4C_PAGE_PRIV)) != 0);
1643}
1644static void sun4c_pte_clear(pte_t *ptep) { *ptep = __pte(0); }
1645
1da177e4
LT		 1646static int sun4c_pmd_bad(pmd_t pmd)
1647{
1648 return (((pmd_val(pmd) & ~PAGE_MASK) != PGD_TABLE) ||
1649 (!virt_addr_valid(pmd_val(pmd))));
1650}
1651
1652static int sun4c_pmd_present(pmd_t pmd)
1653{
1654 return ((pmd_val(pmd) & PGD_PRESENT) != 0);
1655}
1656
1657#if 0 /* if PMD takes one word */
1658static void sun4c_pmd_clear(pmd_t *pmdp) { *pmdp = __pmd(0); }
1659#else /* if pmd_t is a longish aggregate */
1660static void sun4c_pmd_clear(pmd_t *pmdp) {
1661 memset((void *)pmdp, 0, sizeof(pmd_t));
1662}
1663#endif
1664
1665static int sun4c_pgd_none(pgd_t pgd) { return 0; }
1666static int sun4c_pgd_bad(pgd_t pgd) { return 0; }
1667static int sun4c_pgd_present(pgd_t pgd) { return 1; }
1668static void sun4c_pgd_clear(pgd_t * pgdp) { }
1669
1670/*
1671 * The following only work if pte_present() is true.
1672 * Undefined behaviour if not..
1673 */
1674static pte_t sun4c_pte_mkwrite(pte_t pte)
1675{
1676 pte = __pte(pte_val(pte) | _SUN4C_PAGE_WRITE);
1677 if (pte_val(pte) & _SUN4C_PAGE_MODIFIED)
1678 pte = __pte(pte_val(pte) | _SUN4C_PAGE_SILENT_WRITE);
1679 return pte;
1680}
1681
1682static pte_t sun4c_pte_mkdirty(pte_t pte)
1683{
1684 pte = __pte(pte_val(pte) | _SUN4C_PAGE_MODIFIED);
1685 if (pte_val(pte) & _SUN4C_PAGE_WRITE)
1686 pte = __pte(pte_val(pte) | _SUN4C_PAGE_SILENT_WRITE);
1687 return pte;
1688}
1689
1690static pte_t sun4c_pte_mkyoung(pte_t pte)
1691{
1692 pte = __pte(pte_val(pte) | _SUN4C_PAGE_ACCESSED);
1693 if (pte_val(pte) & _SUN4C_PAGE_READ)
1694 pte = __pte(pte_val(pte) | _SUN4C_PAGE_SILENT_READ);
1695 return pte;
1696}
1697
1698/*
1699 * Conversion functions: convert a page and protection to a page entry,
1700 * and a page entry and page directory to the page they refer to.
1701 */
1702static pte_t sun4c_mk_pte(struct page *page, pgprot_t pgprot)
1703{
1704 return __pte(page_to_pfn(page) | pgprot_val(pgprot));
1705}
1706
1707static pte_t sun4c_mk_pte_phys(unsigned long phys_page, pgprot_t pgprot)
1708{
1709 return __pte((phys_page >> PAGE_SHIFT) | pgprot_val(pgprot));
1710}
1711
1712static pte_t sun4c_mk_pte_io(unsigned long page, pgprot_t pgprot, int space)
1713{
1714 return __pte(((page - PAGE_OFFSET) >> PAGE_SHIFT) | pgprot_val(pgprot));
1715}
1716
1717static unsigned long sun4c_pte_pfn(pte_t pte)
1718{
1719 return pte_val(pte) & SUN4C_PFN_MASK;
1720}
1721
1722static pte_t sun4c_pgoff_to_pte(unsigned long pgoff)
1723{
1724 return __pte(pgoff | _SUN4C_PAGE_FILE);
1725}
1726
1727static unsigned long sun4c_pte_to_pgoff(pte_t pte)
1728{
1729 return pte_val(pte) & ((1UL << PTE_FILE_MAX_BITS) - 1);
1730}
1731
1732
64d329ee 1733static inline unsigned long sun4c_pmd_page_v(pmd_t pmd)
1da177e4
LT		 1734{
1735 return (pmd_val(pmd) & PAGE_MASK);
1736}
1737
1738static struct page *sun4c_pmd_page(pmd_t pmd)
1739{
1740 return virt_to_page(sun4c_pmd_page_v(pmd));
1741}
1742
1743static unsigned long sun4c_pgd_page(pgd_t pgd) { return 0; }
1744
1745/* to find an entry in a page-table-directory */
1746static inline pgd_t *sun4c_pgd_offset(struct mm_struct * mm, unsigned long address)
1747{
1748 return mm->pgd + (address >> SUN4C_PGDIR_SHIFT);
1749}
1750
1751/* Find an entry in the second-level page table.. */
1752static pmd_t *sun4c_pmd_offset(pgd_t * dir, unsigned long address)
1753{
1754 return (pmd_t *) dir;
1755}
1756
1757/* Find an entry in the third-level page table.. */
1758pte_t *sun4c_pte_offset_kernel(pmd_t * dir, unsigned long address)
1759{
1760 return (pte_t *) sun4c_pmd_page_v(*dir) +
1761 ((address >> PAGE_SHIFT) & (SUN4C_PTRS_PER_PTE - 1));
1762}
1763
1764static unsigned long sun4c_swp_type(swp_entry_t entry)
1765{
1766 return (entry.val & SUN4C_SWP_TYPE_MASK);
1767}
1768
1769static unsigned long sun4c_swp_offset(swp_entry_t entry)
1770{
1771 return (entry.val >> SUN4C_SWP_OFF_SHIFT) & SUN4C_SWP_OFF_MASK;
1772}
1773
1774static swp_entry_t sun4c_swp_entry(unsigned long type, unsigned long offset)
1775{
1776 return (swp_entry_t) {
1777 (offset & SUN4C_SWP_OFF_MASK) << SUN4C_SWP_OFF_SHIFT
1778 | (type & SUN4C_SWP_TYPE_MASK) };
1779}
1780
1781static void sun4c_free_pte_slow(pte_t *pte)
1782{
1783 free_page((unsigned long)pte);
1784}
1785
1786static void sun4c_free_pgd_slow(pgd_t *pgd)
1787{
1788 free_page((unsigned long)pgd);
1789}
1790
1791static pgd_t *sun4c_get_pgd_fast(void)
1792{
1793 unsigned long *ret;
1794
1795 if ((ret = pgd_quicklist) != NULL) {
1796 pgd_quicklist = (unsigned long *)(*ret);
1797 ret[0] = ret[1];
1798 pgtable_cache_size--;
1799 } else {
1800 pgd_t *init;
1801
1802 ret = (unsigned long *)__get_free_page(GFP_KERNEL);
1803 memset (ret, 0, (KERNBASE / SUN4C_PGDIR_SIZE) * sizeof(pgd_t));
1804 init = sun4c_pgd_offset(&init_mm, 0);
1805 memcpy (((pgd_t *)ret) + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,
1806 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
1807 }
1808 return (pgd_t *)ret;
1809}
1810
1811static void sun4c_free_pgd_fast(pgd_t *pgd)
1812{
1813 *(unsigned long *)pgd = (unsigned long) pgd_quicklist;
1814 pgd_quicklist = (unsigned long *) pgd;
1815 pgtable_cache_size++;
1816}
1817
1818
64d329ee 1819static inline pte_t *
1da177e4
LT		 1820sun4c_pte_alloc_one_fast(struct mm_struct *mm, unsigned long address)
1821{
1822 unsigned long *ret;
1823
1824 if ((ret = (unsigned long *)pte_quicklist) != NULL) {
1825 pte_quicklist = (unsigned long *)(*ret);
1826 ret[0] = ret[1];
1827 pgtable_cache_size--;
1828 }
1829 return (pte_t *)ret;
1830}
1831
1832static pte_t *sun4c_pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
1833{
1834 pte_t *pte;
1835
1836 if ((pte = sun4c_pte_alloc_one_fast(mm, address)) != NULL)
1837 return pte;
1838
b3dd5b82 1839 pte = (pte_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
1da177e4
LT		 1840 return pte;
1841}
1842
2f569afd 1843static pgtable_t sun4c_pte_alloc_one(struct mm_struct *mm, unsigned long address)
1da177e4 1844{
2f569afd
MS		 1845 pte_t *pte;
1846 struct page *page;
1847
1848 pte = sun4c_pte_alloc_one_kernel(mm, address);
1da177e4
LT		 1849 if (pte == NULL)
1850 return NULL;
2f569afd
MS		 1851 page = virt_to_page(pte);
1852 pgtable_page_ctor(page);
1853 return page;
1da177e4
LT		 1854}
1855
64d329ee 1856static inline void sun4c_free_pte_fast(pte_t *pte)
1da177e4
LT		 1857{
1858 *(unsigned long *)pte = (unsigned long) pte_quicklist;
1859 pte_quicklist = (unsigned long *) pte;
1860 pgtable_cache_size++;
1861}
1862
2f569afd 1863static void sun4c_pte_free(pgtable_t pte)
1da177e4 1864{
2f569afd 1865 pgtable_page_dtor(pte);
1da177e4
LT		 1866 sun4c_free_pte_fast(page_address(pte));
1867}
1868
1869/*
1870 * allocating and freeing a pmd is trivial: the 1-entry pmd is
1871 * inside the pgd, so has no extra memory associated with it.
1872 */
1873static pmd_t *sun4c_pmd_alloc_one(struct mm_struct *mm, unsigned long address)
1874{
1875 BUG();
1876 return NULL;
1877}
1878
1879static void sun4c_free_pmd_fast(pmd_t * pmd) { }
1880
1881static void sun4c_check_pgt_cache(int low, int high)
1882{
1883 if (pgtable_cache_size > high) {
1884 do {
1885 if (pgd_quicklist)
1886 sun4c_free_pgd_slow(sun4c_get_pgd_fast());
1887 if (pte_quicklist)
1888 sun4c_free_pte_slow(sun4c_pte_alloc_one_fast(NULL, 0));
1889 } while (pgtable_cache_size > low);
1890 }
1891}
1892
1893/* An experiment, turn off by default for now... -DaveM */
1894#define SUN4C_PRELOAD_PSEG
1895
1896void sun4c_update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t pte)
1897{
1898 unsigned long flags;
1899 int pseg;
1900
da4e9fea
MF		 1901 if (vma->vm_mm->context == NO_CONTEXT)
1902 return;
1903
1da177e4
LT		 1904 local_irq_save(flags);
1905 address &= PAGE_MASK;
1906 if ((pseg = sun4c_get_segmap(address)) == invalid_segment) {
1907 struct sun4c_mmu_entry *entry = sun4c_user_strategy();
1908 struct mm_struct *mm = vma->vm_mm;
1909 unsigned long start, end;
1910
1911 entry->vaddr = start = (address & SUN4C_REAL_PGDIR_MASK);
1912 entry->ctx = mm->context;
1913 add_ring_ordered(sun4c_context_ring + mm->context, entry);
1914 sun4c_put_segmap(entry->vaddr, entry->pseg);
1915 end = start + SUN4C_REAL_PGDIR_SIZE;
1916 while (start < end) {
1917#ifdef SUN4C_PRELOAD_PSEG
1918 pgd_t *pgdp = sun4c_pgd_offset(mm, start);
1919 pte_t *ptep;
1920
1921 if (!pgdp)
1922 goto no_mapping;
1923 ptep = sun4c_pte_offset_kernel((pmd_t *) pgdp, start);
1924 if (!ptep || !(pte_val(*ptep) & _SUN4C_PAGE_PRESENT))
1925 goto no_mapping;
1926 sun4c_put_pte(start, pte_val(*ptep));
1927 goto next;
1928
1929 no_mapping:
1930#endif
1931 sun4c_put_pte(start, 0);
1932#ifdef SUN4C_PRELOAD_PSEG
1933 next:
1934#endif
1935 start += PAGE_SIZE;
1936 }
1937#ifndef SUN4C_PRELOAD_PSEG
1938 sun4c_put_pte(address, pte_val(pte));
1939#endif
1940 local_irq_restore(flags);
1941 return;
1942 } else {
1943 struct sun4c_mmu_entry *entry = &mmu_entry_pool[pseg];
1944
1945 remove_lru(entry);
1946 add_lru(entry);
1947 }
1948
1949 sun4c_put_pte(address, pte_val(pte));
1950 local_irq_restore(flags);
1951}
1952
1953extern void sparc_context_init(int);
1954extern unsigned long end;
1955extern unsigned long bootmem_init(unsigned long *pages_avail);
1956extern unsigned long last_valid_pfn;
1957
1958void __init sun4c_paging_init(void)
1959{
1960 int i, cnt;
1961 unsigned long kernel_end, vaddr;
1962 extern struct resource sparc_iomap;
1963 unsigned long end_pfn, pages_avail;
1964
1965 kernel_end = (unsigned long) &end;
1da177e4
LT		 1966 kernel_end = SUN4C_REAL_PGDIR_ALIGN(kernel_end);
1967
1968 pages_avail = 0;
1969 last_valid_pfn = bootmem_init(&pages_avail);
1970 end_pfn = last_valid_pfn;
1971
1972 sun4c_probe_mmu();
1973 invalid_segment = (num_segmaps - 1);
1974 sun4c_init_mmu_entry_pool();
1975 sun4c_init_rings();
1976 sun4c_init_map_kernelprom(kernel_end);
1977 sun4c_init_clean_mmu(kernel_end);
1978 sun4c_init_fill_kernel_ring(SUN4C_KERNEL_BUCKETS);
1979 sun4c_init_lock_area(sparc_iomap.start, IOBASE_END);
1980 sun4c_init_lock_area(DVMA_VADDR, DVMA_END);
1981 sun4c_init_lock_areas();
1982 sun4c_init_fill_user_ring();
1983
1984 sun4c_set_context(0);
1985 memset(swapper_pg_dir, 0, PAGE_SIZE);
1986 memset(pg0, 0, PAGE_SIZE);
1987 memset(pg1, 0, PAGE_SIZE);
1988 memset(pg2, 0, PAGE_SIZE);
1989 memset(pg3, 0, PAGE_SIZE);
1990
1991 /* Save work later. */
1992 vaddr = VMALLOC_START;
1993 swapper_pg_dir[vaddr>>SUN4C_PGDIR_SHIFT] = __pgd(PGD_TABLE | (unsigned long) pg0);
1994 vaddr += SUN4C_PGDIR_SIZE;
1995 swapper_pg_dir[vaddr>>SUN4C_PGDIR_SHIFT] = __pgd(PGD_TABLE | (unsigned long) pg1);
1996 vaddr += SUN4C_PGDIR_SIZE;
1997 swapper_pg_dir[vaddr>>SUN4C_PGDIR_SHIFT] = __pgd(PGD_TABLE | (unsigned long) pg2);
1998 vaddr += SUN4C_PGDIR_SIZE;
1999 swapper_pg_dir[vaddr>>SUN4C_PGDIR_SHIFT] = __pgd(PGD_TABLE | (unsigned long) pg3);
2000 sun4c_init_ss2_cache_bug();
2001 sparc_context_init(num_contexts);
2002
2003 {
2004 unsigned long zones_size[MAX_NR_ZONES];
2005 unsigned long zholes_size[MAX_NR_ZONES];
2006 unsigned long npages;
2007 int znum;
2008
2009 for (znum = 0; znum < MAX_NR_ZONES; znum++)
2010 zones_size[znum] = zholes_size[znum] = 0;
2011
2012 npages = max_low_pfn - pfn_base;
2013
2014 zones_size[ZONE_DMA] = npages;
2015 zholes_size[ZONE_DMA] = npages - pages_avail;
2016
2017 npages = highend_pfn - max_low_pfn;
2018 zones_size[ZONE_HIGHMEM] = npages;
2019 zholes_size[ZONE_HIGHMEM] = npages - calc_highpages();
2020
9109fb7b 2021 free_area_init_node(0, zones_size, pfn_base, zholes_size);
1da177e4
LT		 2022 }
2023
2024 cnt = 0;
2025 for (i = 0; i < num_segmaps; i++)
2026 if (mmu_entry_pool[i].locked)
2027 cnt++;
2028
2029 max_user_taken_entries = num_segmaps - cnt - 40 - 1;
2030
2031 printk("SUN4C: %d mmu entries for the kernel\n", cnt);
2032}
2033
14778d90
DM		 2034static pgprot_t sun4c_pgprot_noncached(pgprot_t prot)
2035{
2036 prot |= __pgprot(_SUN4C_PAGE_IO | _SUN4C_PAGE_NOCACHE);
2037
2038 return prot;
2039}
2040
1da177e4
LT		 2041/* Load up routines and constants for sun4c mmu */
2042void __init ld_mmu_sun4c(void)
2043{
2044 extern void ___xchg32_sun4c(void);
2045
2046 printk("Loading sun4c MMU routines\n");
2047
2048 /* First the constants */
2049 BTFIXUPSET_SIMM13(pgdir_shift, SUN4C_PGDIR_SHIFT);
2050 BTFIXUPSET_SETHI(pgdir_size, SUN4C_PGDIR_SIZE);
2051 BTFIXUPSET_SETHI(pgdir_mask, SUN4C_PGDIR_MASK);
2052
2053 BTFIXUPSET_SIMM13(ptrs_per_pmd, SUN4C_PTRS_PER_PMD);
2054 BTFIXUPSET_SIMM13(ptrs_per_pgd, SUN4C_PTRS_PER_PGD);
2055 BTFIXUPSET_SIMM13(user_ptrs_per_pgd, KERNBASE / SUN4C_PGDIR_SIZE);
2056
2057 BTFIXUPSET_INT(page_none, pgprot_val(SUN4C_PAGE_NONE));
378e515c 2058 PAGE_SHARED = pgprot_val(SUN4C_PAGE_SHARED);
1da177e4
LT		 2059 BTFIXUPSET_INT(page_copy, pgprot_val(SUN4C_PAGE_COPY));
2060 BTFIXUPSET_INT(page_readonly, pgprot_val(SUN4C_PAGE_READONLY));
2061 BTFIXUPSET_INT(page_kernel, pgprot_val(SUN4C_PAGE_KERNEL));
2062 page_kernel = pgprot_val(SUN4C_PAGE_KERNEL);
1da177e4
LT		 2063
2064 /* Functions */
14778d90 2065 BTFIXUPSET_CALL(pgprot_noncached, sun4c_pgprot_noncached, BTFIXUPCALL_NORM);
1da177e4
LT		 2066 BTFIXUPSET_CALL(___xchg32, ___xchg32_sun4c, BTFIXUPCALL_NORM);
2067 BTFIXUPSET_CALL(do_check_pgt_cache, sun4c_check_pgt_cache, BTFIXUPCALL_NORM);
2068
2069 BTFIXUPSET_CALL(flush_cache_all, sun4c_flush_cache_all, BTFIXUPCALL_NORM);
2070
2071 if (sun4c_vacinfo.do_hwflushes) {
2072 BTFIXUPSET_CALL(sun4c_flush_page, sun4c_flush_page_hw, BTFIXUPCALL_NORM);
2073 BTFIXUPSET_CALL(sun4c_flush_segment, sun4c_flush_segment_hw, BTFIXUPCALL_NORM);
2074 BTFIXUPSET_CALL(sun4c_flush_context, sun4c_flush_context_hw, BTFIXUPCALL_NORM);
2075 } else {
2076 BTFIXUPSET_CALL(sun4c_flush_page, sun4c_flush_page_sw, BTFIXUPCALL_NORM);
2077 BTFIXUPSET_CALL(sun4c_flush_segment, sun4c_flush_segment_sw, BTFIXUPCALL_NORM);
2078 BTFIXUPSET_CALL(sun4c_flush_context, sun4c_flush_context_sw, BTFIXUPCALL_NORM);
2079 }
2080
2081 BTFIXUPSET_CALL(flush_tlb_mm, sun4c_flush_tlb_mm, BTFIXUPCALL_NORM);
2082 BTFIXUPSET_CALL(flush_cache_mm, sun4c_flush_cache_mm, BTFIXUPCALL_NORM);
2083 BTFIXUPSET_CALL(destroy_context, sun4c_destroy_context, BTFIXUPCALL_NORM);
2084 BTFIXUPSET_CALL(switch_mm, sun4c_switch_mm, BTFIXUPCALL_NORM);
2085 BTFIXUPSET_CALL(flush_cache_page, sun4c_flush_cache_page, BTFIXUPCALL_NORM);
2086 BTFIXUPSET_CALL(flush_tlb_page, sun4c_flush_tlb_page, BTFIXUPCALL_NORM);
2087 BTFIXUPSET_CALL(flush_tlb_range, sun4c_flush_tlb_range, BTFIXUPCALL_NORM);
2088 BTFIXUPSET_CALL(flush_cache_range, sun4c_flush_cache_range, BTFIXUPCALL_NORM);
2089 BTFIXUPSET_CALL(__flush_page_to_ram, sun4c_flush_page_to_ram, BTFIXUPCALL_NORM);
2090 BTFIXUPSET_CALL(flush_tlb_all, sun4c_flush_tlb_all, BTFIXUPCALL_NORM);
2091
2092 BTFIXUPSET_CALL(flush_sig_insns, sun4c_flush_sig_insns, BTFIXUPCALL_NOP);
2093
2094 BTFIXUPSET_CALL(set_pte, sun4c_set_pte, BTFIXUPCALL_STO1O0);
2095
2096 /* The 2.4.18 code does not set this on sun4c, how does it work? XXX */
2097 /* BTFIXUPSET_SETHI(none_mask, 0x00000000); */ /* Defaults to zero? */
2098
2099 BTFIXUPSET_CALL(pte_pfn, sun4c_pte_pfn, BTFIXUPCALL_NORM);
2100#if 0 /* PAGE_SHIFT <= 12 */ /* Eek. Investigate. XXX */
2101 BTFIXUPSET_CALL(pmd_page, sun4c_pmd_page, BTFIXUPCALL_ANDNINT(PAGE_SIZE - 1));
2102#else
2103 BTFIXUPSET_CALL(pmd_page, sun4c_pmd_page, BTFIXUPCALL_NORM);
2104#endif
2105 BTFIXUPSET_CALL(pmd_set, sun4c_pmd_set, BTFIXUPCALL_NORM);
2106 BTFIXUPSET_CALL(pmd_populate, sun4c_pmd_populate, BTFIXUPCALL_NORM);
2107
2108 BTFIXUPSET_CALL(pte_present, sun4c_pte_present, BTFIXUPCALL_NORM);
2109 BTFIXUPSET_CALL(pte_clear, sun4c_pte_clear, BTFIXUPCALL_STG0O0);
1da177e4
LT		 2110
2111 BTFIXUPSET_CALL(pmd_bad, sun4c_pmd_bad, BTFIXUPCALL_NORM);
2112 BTFIXUPSET_CALL(pmd_present, sun4c_pmd_present, BTFIXUPCALL_NORM);
2113 BTFIXUPSET_CALL(pmd_clear, sun4c_pmd_clear, BTFIXUPCALL_STG0O0);
2114
2115 BTFIXUPSET_CALL(pgd_none, sun4c_pgd_none, BTFIXUPCALL_RETINT(0));
2116 BTFIXUPSET_CALL(pgd_bad, sun4c_pgd_bad, BTFIXUPCALL_RETINT(0));
2117 BTFIXUPSET_CALL(pgd_present, sun4c_pgd_present, BTFIXUPCALL_RETINT(1));
2118 BTFIXUPSET_CALL(pgd_clear, sun4c_pgd_clear, BTFIXUPCALL_NOP);
2119
2120 BTFIXUPSET_CALL(mk_pte, sun4c_mk_pte, BTFIXUPCALL_NORM);
2121 BTFIXUPSET_CALL(mk_pte_phys, sun4c_mk_pte_phys, BTFIXUPCALL_NORM);
2122 BTFIXUPSET_CALL(mk_pte_io, sun4c_mk_pte_io, BTFIXUPCALL_NORM);
2123
2124 BTFIXUPSET_INT(pte_modify_mask, _SUN4C_PAGE_CHG_MASK);
2125 BTFIXUPSET_CALL(pmd_offset, sun4c_pmd_offset, BTFIXUPCALL_NORM);
2126 BTFIXUPSET_CALL(pte_offset_kernel, sun4c_pte_offset_kernel, BTFIXUPCALL_NORM);
2127 BTFIXUPSET_CALL(free_pte_fast, sun4c_free_pte_fast, BTFIXUPCALL_NORM);
2128 BTFIXUPSET_CALL(pte_free, sun4c_pte_free, BTFIXUPCALL_NORM);
2129 BTFIXUPSET_CALL(pte_alloc_one_kernel, sun4c_pte_alloc_one_kernel, BTFIXUPCALL_NORM);
2130 BTFIXUPSET_CALL(pte_alloc_one, sun4c_pte_alloc_one, BTFIXUPCALL_NORM);
2131 BTFIXUPSET_CALL(free_pmd_fast, sun4c_free_pmd_fast, BTFIXUPCALL_NOP);
2132 BTFIXUPSET_CALL(pmd_alloc_one, sun4c_pmd_alloc_one, BTFIXUPCALL_RETO0);
2133 BTFIXUPSET_CALL(free_pgd_fast, sun4c_free_pgd_fast, BTFIXUPCALL_NORM);
2134 BTFIXUPSET_CALL(get_pgd_fast, sun4c_get_pgd_fast, BTFIXUPCALL_NORM);
2135
2136 BTFIXUPSET_HALF(pte_writei, _SUN4C_PAGE_WRITE);
2137 BTFIXUPSET_HALF(pte_dirtyi, _SUN4C_PAGE_MODIFIED);
2138 BTFIXUPSET_HALF(pte_youngi, _SUN4C_PAGE_ACCESSED);
2139 BTFIXUPSET_HALF(pte_filei, _SUN4C_PAGE_FILE);
2140 BTFIXUPSET_HALF(pte_wrprotecti, _SUN4C_PAGE_WRITE|_SUN4C_PAGE_SILENT_WRITE);
2141 BTFIXUPSET_HALF(pte_mkcleani, _SUN4C_PAGE_MODIFIED|_SUN4C_PAGE_SILENT_WRITE);
2142 BTFIXUPSET_HALF(pte_mkoldi, _SUN4C_PAGE_ACCESSED|_SUN4C_PAGE_SILENT_READ);
2143 BTFIXUPSET_CALL(pte_mkwrite, sun4c_pte_mkwrite, BTFIXUPCALL_NORM);
2144 BTFIXUPSET_CALL(pte_mkdirty, sun4c_pte_mkdirty, BTFIXUPCALL_NORM);
2145 BTFIXUPSET_CALL(pte_mkyoung, sun4c_pte_mkyoung, BTFIXUPCALL_NORM);
2146 BTFIXUPSET_CALL(update_mmu_cache, sun4c_update_mmu_cache, BTFIXUPCALL_NORM);
2147
2148 BTFIXUPSET_CALL(pte_to_pgoff, sun4c_pte_to_pgoff, BTFIXUPCALL_NORM);
2149 BTFIXUPSET_CALL(pgoff_to_pte, sun4c_pgoff_to_pte, BTFIXUPCALL_NORM);
2150
2151 BTFIXUPSET_CALL(mmu_lockarea, sun4c_lockarea, BTFIXUPCALL_NORM);
2152 BTFIXUPSET_CALL(mmu_unlockarea, sun4c_unlockarea, BTFIXUPCALL_NORM);
2153
2154 BTFIXUPSET_CALL(mmu_get_scsi_one, sun4c_get_scsi_one, BTFIXUPCALL_NORM);
2155 BTFIXUPSET_CALL(mmu_get_scsi_sgl, sun4c_get_scsi_sgl, BTFIXUPCALL_NORM);
2156 BTFIXUPSET_CALL(mmu_release_scsi_one, sun4c_release_scsi_one, BTFIXUPCALL_NORM);
2157 BTFIXUPSET_CALL(mmu_release_scsi_sgl, sun4c_release_scsi_sgl, BTFIXUPCALL_NORM);
2158
2159 BTFIXUPSET_CALL(mmu_map_dma_area, sun4c_map_dma_area, BTFIXUPCALL_NORM);
2160 BTFIXUPSET_CALL(mmu_unmap_dma_area, sun4c_unmap_dma_area, BTFIXUPCALL_NORM);
1da177e4
LT		 2161
2162 BTFIXUPSET_CALL(sparc_mapiorange, sun4c_mapiorange, BTFIXUPCALL_NORM);
2163 BTFIXUPSET_CALL(sparc_unmapiorange, sun4c_unmapiorange, BTFIXUPCALL_NORM);
2164
2165 BTFIXUPSET_CALL(__swp_type, sun4c_swp_type, BTFIXUPCALL_NORM);
2166 BTFIXUPSET_CALL(__swp_offset, sun4c_swp_offset, BTFIXUPCALL_NORM);
2167 BTFIXUPSET_CALL(__swp_entry, sun4c_swp_entry, BTFIXUPCALL_NORM);
2168
2169 BTFIXUPSET_CALL(alloc_thread_info, sun4c_alloc_thread_info, BTFIXUPCALL_NORM);
2170 BTFIXUPSET_CALL(free_thread_info, sun4c_free_thread_info, BTFIXUPCALL_NORM);
2171
2172 BTFIXUPSET_CALL(mmu_info, sun4c_mmu_info, BTFIXUPCALL_NORM);
2173
2174 /* These should _never_ get called with two level tables. */
2175 BTFIXUPSET_CALL(pgd_set, sun4c_pgd_set, BTFIXUPCALL_NOP);
46a82b2d 2176 BTFIXUPSET_CALL(pgd_page_vaddr, sun4c_pgd_page, BTFIXUPCALL_RETO0);
1da177e4 2177}
Ubuntu Artful kernel mirror