]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blob - arch/sparc64/mm/init.c
projects / mirror_ubuntu-zesty-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
[PATCH] add poison.h and patch primary users
[mirror_ubuntu-zesty-kernel.git] / arch / sparc64 / mm / init.c
1 /* $Id: init.c,v 1.209 2002/02/09 19:49:31 davem Exp $
2 * arch/sparc64/mm/init.c
3 *
4 * Copyright (C) 1996-1999 David S. Miller (davem@caip.rutgers.edu)
5 * Copyright (C) 1997-1999 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
6 */
7
8 #include <linux/config.h>
9 #include <linux/module.h>
10 #include <linux/kernel.h>
11 #include <linux/sched.h>
12 #include <linux/string.h>
13 #include <linux/init.h>
14 #include <linux/bootmem.h>
15 #include <linux/mm.h>
16 #include <linux/hugetlb.h>
17 #include <linux/slab.h>
18 #include <linux/initrd.h>
19 #include <linux/swap.h>
20 #include <linux/pagemap.h>
21 #include <linux/poison.h>
22 #include <linux/fs.h>
23 #include <linux/seq_file.h>
24 #include <linux/kprobes.h>
25 #include <linux/cache.h>
26 #include <linux/sort.h>
27
28 #include <asm/head.h>
29 #include <asm/system.h>
30 #include <asm/page.h>
31 #include <asm/pgalloc.h>
32 #include <asm/pgtable.h>
33 #include <asm/oplib.h>
34 #include <asm/iommu.h>
35 #include <asm/io.h>
36 #include <asm/uaccess.h>
37 #include <asm/mmu_context.h>
38 #include <asm/tlbflush.h>
39 #include <asm/dma.h>
40 #include <asm/starfire.h>
41 #include <asm/tlb.h>
42 #include <asm/spitfire.h>
43 #include <asm/sections.h>
44 #include <asm/tsb.h>
45 #include <asm/hypervisor.h>
46 #include <asm/prom.h>
47
48 extern void device_scan(void);
49
50 #define MAX_PHYS_ADDRESS (1UL << 42UL)
51 #define KPTE_BITMAP_CHUNK_SZ (256UL * 1024UL * 1024UL)
52 #define KPTE_BITMAP_BYTES \
53 ((MAX_PHYS_ADDRESS / KPTE_BITMAP_CHUNK_SZ) / 8)
54
55 unsigned long kern_linear_pte_xor[2] __read_mostly;
56
57 /* A bitmap, one bit for every 256MB of physical memory. If the bit
58 * is clear, we should use a 4MB page (via kern_linear_pte_xor[0]) else
59 * if set we should use a 256MB page (via kern_linear_pte_xor[1]).
60 */
61 unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)];
62
63 /* A special kernel TSB for 4MB and 256MB linear mappings. */
64 struct tsb swapper_4m_tsb[KERNEL_TSB4M_NENTRIES];
65
66 #define MAX_BANKS 32
67
68 static struct linux_prom64_registers pavail[MAX_BANKS] __initdata;
69 static struct linux_prom64_registers pavail_rescan[MAX_BANKS] __initdata;
70 static int pavail_ents __initdata;
71 static int pavail_rescan_ents __initdata;
72
73 static int cmp_p64(const void *a, const void *b)
74 {
75 const struct linux_prom64_registers *x = a, *y = b;
76
77 if (x->phys_addr > y->phys_addr)
78 return 1;
79 if (x->phys_addr < y->phys_addr)
80 return -1;
81 return 0;
82 }
83
84 static void __init read_obp_memory(const char *property,
85 struct linux_prom64_registers *regs,
86 int *num_ents)
87 {
88 int node = prom_finddevice("/memory");
89 int prop_size = prom_getproplen(node, property);
90 int ents, ret, i;
91
92 ents = prop_size / sizeof(struct linux_prom64_registers);
93 if (ents > MAX_BANKS) {
94 prom_printf("The machine has more %s property entries than "
95 "this kernel can support (%d).\n",
96 property, MAX_BANKS);
97 prom_halt();
98 }
99
100 ret = prom_getproperty(node, property, (char *) regs, prop_size);
101 if (ret == -1) {
102 prom_printf("Couldn't get %s property from /memory.\n");
103 prom_halt();
104 }
105
106 /* Sanitize what we got from the firmware, by page aligning
107 * everything.
108 */
109 for (i = 0; i < ents; i++) {
110 unsigned long base, size;
111
112 base = regs[i].phys_addr;
113 size = regs[i].reg_size;
114
115 size &= PAGE_MASK;
116 if (base & ~PAGE_MASK) {
117 unsigned long new_base = PAGE_ALIGN(base);
118
119 size -= new_base - base;
120 if ((long) size < 0L)
121 size = 0UL;
122 base = new_base;
123 }
124 regs[i].phys_addr = base;
125 regs[i].reg_size = size;
126 }
127
128 for (i = 0; i < ents; i++) {
129 if (regs[i].reg_size == 0UL) {
130 int j;
131
132 for (j = i; j < ents - 1; j++) {
133 regs[j].phys_addr =
134 regs[j+1].phys_addr;
135 regs[j].reg_size =
136 regs[j+1].reg_size;
137 }
138
139 ents--;
140 i--;
141 }
142 }
143
144 *num_ents = ents;
145
146 sort(regs, ents, sizeof(struct linux_prom64_registers),
147 cmp_p64, NULL);
148 }
149
150 unsigned long *sparc64_valid_addr_bitmap __read_mostly;
151
152 /* Kernel physical address base and size in bytes. */
153 unsigned long kern_base __read_mostly;
154 unsigned long kern_size __read_mostly;
155
156 /* get_new_mmu_context() uses "cache + 1". */
157 DEFINE_SPINLOCK(ctx_alloc_lock);
158 unsigned long tlb_context_cache = CTX_FIRST_VERSION - 1;
159 #define CTX_BMAP_SLOTS (1UL << (CTX_NR_BITS - 6))
160 unsigned long mmu_context_bmap[CTX_BMAP_SLOTS];
161
162 /* References to special section boundaries */
163 extern char _start[], _end[];
164
165 /* Initial ramdisk setup */
166 extern unsigned long sparc_ramdisk_image64;
167 extern unsigned int sparc_ramdisk_image;
168 extern unsigned int sparc_ramdisk_size;
169
170 struct page *mem_map_zero __read_mostly;
171
172 unsigned int sparc64_highest_unlocked_tlb_ent __read_mostly;
173
174 unsigned long sparc64_kern_pri_context __read_mostly;
175 unsigned long sparc64_kern_pri_nuc_bits __read_mostly;
176 unsigned long sparc64_kern_sec_context __read_mostly;
177
178 int bigkernel = 0;
179
180 kmem_cache_t *pgtable_cache __read_mostly;
181
182 static void zero_ctor(void *addr, kmem_cache_t *cache, unsigned long flags)
183 {
184 clear_page(addr);
185 }
186
187 extern void tsb_cache_init(void);
188
189 void pgtable_cache_init(void)
190 {
191 pgtable_cache = kmem_cache_create("pgtable_cache",
192 PAGE_SIZE, PAGE_SIZE,
193 SLAB_HWCACHE_ALIGN |
194 SLAB_MUST_HWCACHE_ALIGN,
195 zero_ctor,
196 NULL);
197 if (!pgtable_cache) {
198 prom_printf("Could not create pgtable_cache\n");
199 prom_halt();
200 }
201 tsb_cache_init();
202 }
203
204 #ifdef CONFIG_DEBUG_DCFLUSH
205 atomic_t dcpage_flushes = ATOMIC_INIT(0);
206 #ifdef CONFIG_SMP
207 atomic_t dcpage_flushes_xcall = ATOMIC_INIT(0);
208 #endif
209 #endif
210
211 inline void flush_dcache_page_impl(struct page *page)
212 {
213 BUG_ON(tlb_type == hypervisor);
214 #ifdef CONFIG_DEBUG_DCFLUSH
215 atomic_inc(&dcpage_flushes);
216 #endif
217
218 #ifdef DCACHE_ALIASING_POSSIBLE
219 __flush_dcache_page(page_address(page),
220 ((tlb_type == spitfire) &&
221 page_mapping(page) != NULL));
222 #else
223 if (page_mapping(page) != NULL &&
224 tlb_type == spitfire)
225 __flush_icache_page(__pa(page_address(page)));
226 #endif
227 }
228
229 #define PG_dcache_dirty PG_arch_1
230 #define PG_dcache_cpu_shift 24UL
231 #define PG_dcache_cpu_mask (256UL - 1UL)
232
233 #if NR_CPUS > 256
234 #error D-cache dirty tracking and thread_info->cpu need fixing for > 256 cpus
235 #endif
236
237 #define dcache_dirty_cpu(page) \
238 (((page)->flags >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask)
239
240 static __inline__ void set_dcache_dirty(struct page *page, int this_cpu)
241 {
242 unsigned long mask = this_cpu;
243 unsigned long non_cpu_bits;
244
245 non_cpu_bits = ~(PG_dcache_cpu_mask << PG_dcache_cpu_shift);
246 mask = (mask << PG_dcache_cpu_shift) | (1UL << PG_dcache_dirty);
247
248 __asm__ __volatile__("1:\n\t"
249 "ldx [%2], %%g7\n\t"
250 "and %%g7, %1, %%g1\n\t"
251 "or %%g1, %0, %%g1\n\t"
252 "casx [%2], %%g7, %%g1\n\t"
253 "cmp %%g7, %%g1\n\t"
254 "membar #StoreLoad | #StoreStore\n\t"
255 "bne,pn %%xcc, 1b\n\t"
256 " nop"
257 : /* no outputs */
258 : "r" (mask), "r" (non_cpu_bits), "r" (&page->flags)
259 : "g1", "g7");
260 }
261
262 static __inline__ void clear_dcache_dirty_cpu(struct page *page, unsigned long cpu)
263 {
264 unsigned long mask = (1UL << PG_dcache_dirty);
265
266 __asm__ __volatile__("! test_and_clear_dcache_dirty\n"
267 "1:\n\t"
268 "ldx [%2], %%g7\n\t"
269 "srlx %%g7, %4, %%g1\n\t"
270 "and %%g1, %3, %%g1\n\t"
271 "cmp %%g1, %0\n\t"
272 "bne,pn %%icc, 2f\n\t"
273 " andn %%g7, %1, %%g1\n\t"
274 "casx [%2], %%g7, %%g1\n\t"
275 "cmp %%g7, %%g1\n\t"
276 "membar #StoreLoad | #StoreStore\n\t"
277 "bne,pn %%xcc, 1b\n\t"
278 " nop\n"
279 "2:"
280 : /* no outputs */
281 : "r" (cpu), "r" (mask), "r" (&page->flags),
282 "i" (PG_dcache_cpu_mask),
283 "i" (PG_dcache_cpu_shift)
284 : "g1", "g7");
285 }
286
287 static inline void tsb_insert(struct tsb *ent, unsigned long tag, unsigned long pte)
288 {
289 unsigned long tsb_addr = (unsigned long) ent;
290
291 if (tlb_type == cheetah_plus || tlb_type == hypervisor)
292 tsb_addr = __pa(tsb_addr);
293
294 __tsb_insert(tsb_addr, tag, pte);
295 }
296
297 unsigned long _PAGE_ALL_SZ_BITS __read_mostly;
298 unsigned long _PAGE_SZBITS __read_mostly;
299
300 void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t pte)
301 {
302 struct mm_struct *mm;
303 struct tsb *tsb;
304 unsigned long tag, flags;
305 unsigned long tsb_index, tsb_hash_shift;
306
307 if (tlb_type != hypervisor) {
308 unsigned long pfn = pte_pfn(pte);
309 unsigned long pg_flags;
310 struct page *page;
311
312 if (pfn_valid(pfn) &&
313 (page = pfn_to_page(pfn), page_mapping(page)) &&
314 ((pg_flags = page->flags) & (1UL << PG_dcache_dirty))) {
315 int cpu = ((pg_flags >> PG_dcache_cpu_shift) &
316 PG_dcache_cpu_mask);
317 int this_cpu = get_cpu();
318
319 /* This is just to optimize away some function calls
320 * in the SMP case.
321 */
322 if (cpu == this_cpu)
323 flush_dcache_page_impl(page);
324 else
325 smp_flush_dcache_page_impl(page, cpu);
326
327 clear_dcache_dirty_cpu(page, cpu);
328
329 put_cpu();
330 }
331 }
332
333 mm = vma->vm_mm;
334
335 tsb_index = MM_TSB_BASE;
336 tsb_hash_shift = PAGE_SHIFT;
337
338 spin_lock_irqsave(&mm->context.lock, flags);
339
340 #ifdef CONFIG_HUGETLB_PAGE
341 if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL) {
342 if ((tlb_type == hypervisor &&
343 (pte_val(pte) & _PAGE_SZALL_4V) == _PAGE_SZHUGE_4V) ||
344 (tlb_type != hypervisor &&
345 (pte_val(pte) & _PAGE_SZALL_4U) == _PAGE_SZHUGE_4U)) {
346 tsb_index = MM_TSB_HUGE;
347 tsb_hash_shift = HPAGE_SHIFT;
348 }
349 }
350 #endif
351
352 tsb = mm->context.tsb_block[tsb_index].tsb;
353 tsb += ((address >> tsb_hash_shift) &
354 (mm->context.tsb_block[tsb_index].tsb_nentries - 1UL));
355 tag = (address >> 22UL);
356 tsb_insert(tsb, tag, pte_val(pte));
357
358 spin_unlock_irqrestore(&mm->context.lock, flags);
359 }
360
361 void flush_dcache_page(struct page *page)
362 {
363 struct address_space *mapping;
364 int this_cpu;
365
366 if (tlb_type == hypervisor)
367 return;
368
369 /* Do not bother with the expensive D-cache flush if it
370 * is merely the zero page. The 'bigcore' testcase in GDB
371 * causes this case to run millions of times.
372 */
373 if (page == ZERO_PAGE(0))
374 return;
375
376 this_cpu = get_cpu();
377
378 mapping = page_mapping(page);
379 if (mapping && !mapping_mapped(mapping)) {
380 int dirty = test_bit(PG_dcache_dirty, &page->flags);
381 if (dirty) {
382 int dirty_cpu = dcache_dirty_cpu(page);
383
384 if (dirty_cpu == this_cpu)
385 goto out;
386 smp_flush_dcache_page_impl(page, dirty_cpu);
387 }
388 set_dcache_dirty(page, this_cpu);
389 } else {
390 /* We could delay the flush for the !page_mapping
391 * case too. But that case is for exec env/arg
392 * pages and those are %99 certainly going to get
393 * faulted into the tlb (and thus flushed) anyways.
394 */
395 flush_dcache_page_impl(page);
396 }
397
398 out:
399 put_cpu();
400 }
401
402 void __kprobes flush_icache_range(unsigned long start, unsigned long end)
403 {
404 /* Cheetah and Hypervisor platform cpus have coherent I-cache. */
405 if (tlb_type == spitfire) {
406 unsigned long kaddr;
407
408 for (kaddr = start; kaddr < end; kaddr += PAGE_SIZE)
409 __flush_icache_page(__get_phys(kaddr));
410 }
411 }
412
413 void show_mem(void)
414 {
415 printk("Mem-info:\n");
416 show_free_areas();
417 printk("Free swap: %6ldkB\n",
418 nr_swap_pages << (PAGE_SHIFT-10));
419 printk("%ld pages of RAM\n", num_physpages);
420 printk("%d free pages\n", nr_free_pages());
421 }
422
423 void mmu_info(struct seq_file *m)
424 {
425 if (tlb_type == cheetah)
426 seq_printf(m, "MMU Type\t: Cheetah\n");
427 else if (tlb_type == cheetah_plus)
428 seq_printf(m, "MMU Type\t: Cheetah+\n");
429 else if (tlb_type == spitfire)
430 seq_printf(m, "MMU Type\t: Spitfire\n");
431 else if (tlb_type == hypervisor)
432 seq_printf(m, "MMU Type\t: Hypervisor (sun4v)\n");
433 else
434 seq_printf(m, "MMU Type\t: ???\n");
435
436 #ifdef CONFIG_DEBUG_DCFLUSH
437 seq_printf(m, "DCPageFlushes\t: %d\n",
438 atomic_read(&dcpage_flushes));
439 #ifdef CONFIG_SMP
440 seq_printf(m, "DCPageFlushesXC\t: %d\n",
441 atomic_read(&dcpage_flushes_xcall));
442 #endif /* CONFIG_SMP */
443 #endif /* CONFIG_DEBUG_DCFLUSH */
444 }
445
446 struct linux_prom_translation {
447 unsigned long virt;
448 unsigned long size;
449 unsigned long data;
450 };
451
452 /* Exported for kernel TLB miss handling in ktlb.S */
453 struct linux_prom_translation prom_trans[512] __read_mostly;
454 unsigned int prom_trans_ents __read_mostly;
455
456 /* Exported for SMP bootup purposes. */
457 unsigned long kern_locked_tte_data;
458
459 /* The obp translations are saved based on 8k pagesize, since obp can
460 * use a mixture of pagesizes. Misses to the LOW_OBP_ADDRESS ->
461 * HI_OBP_ADDRESS range are handled in ktlb.S.
462 */
463 static inline int in_obp_range(unsigned long vaddr)
464 {
465 return (vaddr >= LOW_OBP_ADDRESS &&
466 vaddr < HI_OBP_ADDRESS);
467 }
468
469 static int cmp_ptrans(const void *a, const void *b)
470 {
471 const struct linux_prom_translation *x = a, *y = b;
472
473 if (x->virt > y->virt)
474 return 1;
475 if (x->virt < y->virt)
476 return -1;
477 return 0;
478 }
479
480 /* Read OBP translations property into 'prom_trans[]'. */
481 static void __init read_obp_translations(void)
482 {
483 int n, node, ents, first, last, i;
484
485 node = prom_finddevice("/virtual-memory");
486 n = prom_getproplen(node, "translations");
487 if (unlikely(n == 0 || n == -1)) {
488 prom_printf("prom_mappings: Couldn't get size.\n");
489 prom_halt();
490 }
491 if (unlikely(n > sizeof(prom_trans))) {
492 prom_printf("prom_mappings: Size %Zd is too big.\n", n);
493 prom_halt();
494 }
495
496 if ((n = prom_getproperty(node, "translations",
497 (char *)&prom_trans[0],
498 sizeof(prom_trans))) == -1) {
499 prom_printf("prom_mappings: Couldn't get property.\n");
500 prom_halt();
501 }
502
503 n = n / sizeof(struct linux_prom_translation);
504
505 ents = n;
506
507 sort(prom_trans, ents, sizeof(struct linux_prom_translation),
508 cmp_ptrans, NULL);
509
510 /* Now kick out all the non-OBP entries. */
511 for (i = 0; i < ents; i++) {
512 if (in_obp_range(prom_trans[i].virt))
513 break;
514 }
515 first = i;
516 for (; i < ents; i++) {
517 if (!in_obp_range(prom_trans[i].virt))
518 break;
519 }
520 last = i;
521
522 for (i = 0; i < (last - first); i++) {
523 struct linux_prom_translation *src = &prom_trans[i + first];
524 struct linux_prom_translation *dest = &prom_trans[i];
525
526 *dest = *src;
527 }
528 for (; i < ents; i++) {
529 struct linux_prom_translation *dest = &prom_trans[i];
530 dest->virt = dest->size = dest->data = 0x0UL;
531 }
532
533 prom_trans_ents = last - first;
534
535 if (tlb_type == spitfire) {
536 /* Clear diag TTE bits. */
537 for (i = 0; i < prom_trans_ents; i++)
538 prom_trans[i].data &= ~0x0003fe0000000000UL;
539 }
540 }
541
542 static void __init hypervisor_tlb_lock(unsigned long vaddr,
543 unsigned long pte,
544 unsigned long mmu)
545 {
546 register unsigned long func asm("%o5");
547 register unsigned long arg0 asm("%o0");
548 register unsigned long arg1 asm("%o1");
549 register unsigned long arg2 asm("%o2");
550 register unsigned long arg3 asm("%o3");
551
552 func = HV_FAST_MMU_MAP_PERM_ADDR;
553 arg0 = vaddr;
554 arg1 = 0;
555 arg2 = pte;
556 arg3 = mmu;
557 __asm__ __volatile__("ta 0x80"
558 : "=&r" (func), "=&r" (arg0),
559 "=&r" (arg1), "=&r" (arg2),
560 "=&r" (arg3)
561 : "0" (func), "1" (arg0), "2" (arg1),
562 "3" (arg2), "4" (arg3));
563 if (arg0 != 0) {
564 prom_printf("hypervisor_tlb_lock[%lx:%lx:%lx:%lx]: "
565 "errors with %lx\n", vaddr, 0, pte, mmu, arg0);
566 prom_halt();
567 }
568 }
569
570 static unsigned long kern_large_tte(unsigned long paddr);
571
572 static void __init remap_kernel(void)
573 {
574 unsigned long phys_page, tte_vaddr, tte_data;
575 int tlb_ent = sparc64_highest_locked_tlbent();
576
577 tte_vaddr = (unsigned long) KERNBASE;
578 phys_page = (prom_boot_mapping_phys_low >> 22UL) << 22UL;
579 tte_data = kern_large_tte(phys_page);
580
581 kern_locked_tte_data = tte_data;
582
583 /* Now lock us into the TLBs via Hypervisor or OBP. */
584 if (tlb_type == hypervisor) {
585 hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_DMMU);
586 hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_IMMU);
587 if (bigkernel) {
588 tte_vaddr += 0x400000;
589 tte_data += 0x400000;
590 hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_DMMU);
591 hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_IMMU);
592 }
593 } else {
594 prom_dtlb_load(tlb_ent, tte_data, tte_vaddr);
595 prom_itlb_load(tlb_ent, tte_data, tte_vaddr);
596 if (bigkernel) {
597 tlb_ent -= 1;
598 prom_dtlb_load(tlb_ent,
599 tte_data + 0x400000,
600 tte_vaddr + 0x400000);
601 prom_itlb_load(tlb_ent,
602 tte_data + 0x400000,
603 tte_vaddr + 0x400000);
604 }
605 sparc64_highest_unlocked_tlb_ent = tlb_ent - 1;
606 }
607 if (tlb_type == cheetah_plus) {
608 sparc64_kern_pri_context = (CTX_CHEETAH_PLUS_CTX0 |
609 CTX_CHEETAH_PLUS_NUC);
610 sparc64_kern_pri_nuc_bits = CTX_CHEETAH_PLUS_NUC;
611 sparc64_kern_sec_context = CTX_CHEETAH_PLUS_CTX0;
612 }
613 }
614
615
616 static void __init inherit_prom_mappings(void)
617 {
618 read_obp_translations();
619
620 /* Now fixup OBP's idea about where we really are mapped. */
621 prom_printf("Remapping the kernel... ");
622 remap_kernel();
623 prom_printf("done.\n");
624 }
625
626 void prom_world(int enter)
627 {
628 if (!enter)
629 set_fs((mm_segment_t) { get_thread_current_ds() });
630
631 __asm__ __volatile__("flushw");
632 }
633
634 #ifdef DCACHE_ALIASING_POSSIBLE
635 void __flush_dcache_range(unsigned long start, unsigned long end)
636 {
637 unsigned long va;
638
639 if (tlb_type == spitfire) {
640 int n = 0;
641
642 for (va = start; va < end; va += 32) {
643 spitfire_put_dcache_tag(va & 0x3fe0, 0x0);
644 if (++n >= 512)
645 break;
646 }
647 } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
648 start = __pa(start);
649 end = __pa(end);
650 for (va = start; va < end; va += 32)
651 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
652 "membar #Sync"
653 : /* no outputs */
654 : "r" (va),
655 "i" (ASI_DCACHE_INVALIDATE));
656 }
657 }
658 #endif /* DCACHE_ALIASING_POSSIBLE */
659
660 /* Caller does TLB context flushing on local CPU if necessary.
661 * The caller also ensures that CTX_VALID(mm->context) is false.
662 *
663 * We must be careful about boundary cases so that we never
664 * let the user have CTX 0 (nucleus) or we ever use a CTX
665 * version of zero (and thus NO_CONTEXT would not be caught
666 * by version mis-match tests in mmu_context.h).
667 *
668 * Always invoked with interrupts disabled.
669 */
670 void get_new_mmu_context(struct mm_struct *mm)
671 {
672 unsigned long ctx, new_ctx;
673 unsigned long orig_pgsz_bits;
674 unsigned long flags;
675 int new_version;
676
677 spin_lock_irqsave(&ctx_alloc_lock, flags);
678 orig_pgsz_bits = (mm->context.sparc64_ctx_val & CTX_PGSZ_MASK);
679 ctx = (tlb_context_cache + 1) & CTX_NR_MASK;
680 new_ctx = find_next_zero_bit(mmu_context_bmap, 1 << CTX_NR_BITS, ctx);
681 new_version = 0;
682 if (new_ctx >= (1 << CTX_NR_BITS)) {
683 new_ctx = find_next_zero_bit(mmu_context_bmap, ctx, 1);
684 if (new_ctx >= ctx) {
685 int i;
686 new_ctx = (tlb_context_cache & CTX_VERSION_MASK) +
687 CTX_FIRST_VERSION;
688 if (new_ctx == 1)
689 new_ctx = CTX_FIRST_VERSION;
690
691 /* Don't call memset, for 16 entries that's just
692 * plain silly...
693 */
694 mmu_context_bmap[0] = 3;
695 mmu_context_bmap[1] = 0;
696 mmu_context_bmap[2] = 0;
697 mmu_context_bmap[3] = 0;
698 for (i = 4; i < CTX_BMAP_SLOTS; i += 4) {
699 mmu_context_bmap[i + 0] = 0;
700 mmu_context_bmap[i + 1] = 0;
701 mmu_context_bmap[i + 2] = 0;
702 mmu_context_bmap[i + 3] = 0;
703 }
704 new_version = 1;
705 goto out;
706 }
707 }
708 mmu_context_bmap[new_ctx>>6] |= (1UL << (new_ctx & 63));
709 new_ctx |= (tlb_context_cache & CTX_VERSION_MASK);
710 out:
711 tlb_context_cache = new_ctx;
712 mm->context.sparc64_ctx_val = new_ctx | orig_pgsz_bits;
713 spin_unlock_irqrestore(&ctx_alloc_lock, flags);
714
715 if (unlikely(new_version))
716 smp_new_mmu_context_version();
717 }
718
719 void sparc_ultra_dump_itlb(void)
720 {
721 int slot;
722
723 if (tlb_type == spitfire) {
724 printk ("Contents of itlb: ");
725 for (slot = 0; slot < 14; slot++) printk (" ");
726 printk ("%2x:%016lx,%016lx\n",
727 0,
728 spitfire_get_itlb_tag(0), spitfire_get_itlb_data(0));
729 for (slot = 1; slot < 64; slot+=3) {
730 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx %2x:%016lx,%016lx\n",
731 slot,
732 spitfire_get_itlb_tag(slot), spitfire_get_itlb_data(slot),
733 slot+1,
734 spitfire_get_itlb_tag(slot+1), spitfire_get_itlb_data(slot+1),
735 slot+2,
736 spitfire_get_itlb_tag(slot+2), spitfire_get_itlb_data(slot+2));
737 }
738 } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
739 printk ("Contents of itlb0:\n");
740 for (slot = 0; slot < 16; slot+=2) {
741 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
742 slot,
743 cheetah_get_litlb_tag(slot), cheetah_get_litlb_data(slot),
744 slot+1,
745 cheetah_get_litlb_tag(slot+1), cheetah_get_litlb_data(slot+1));
746 }
747 printk ("Contents of itlb2:\n");
748 for (slot = 0; slot < 128; slot+=2) {
749 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
750 slot,
751 cheetah_get_itlb_tag(slot), cheetah_get_itlb_data(slot),
752 slot+1,
753 cheetah_get_itlb_tag(slot+1), cheetah_get_itlb_data(slot+1));
754 }
755 }
756 }
757
758 void sparc_ultra_dump_dtlb(void)
759 {
760 int slot;
761
762 if (tlb_type == spitfire) {
763 printk ("Contents of dtlb: ");
764 for (slot = 0; slot < 14; slot++) printk (" ");
765 printk ("%2x:%016lx,%016lx\n", 0,
766 spitfire_get_dtlb_tag(0), spitfire_get_dtlb_data(0));
767 for (slot = 1; slot < 64; slot+=3) {
768 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx %2x:%016lx,%016lx\n",
769 slot,
770 spitfire_get_dtlb_tag(slot), spitfire_get_dtlb_data(slot),
771 slot+1,
772 spitfire_get_dtlb_tag(slot+1), spitfire_get_dtlb_data(slot+1),
773 slot+2,
774 spitfire_get_dtlb_tag(slot+2), spitfire_get_dtlb_data(slot+2));
775 }
776 } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
777 printk ("Contents of dtlb0:\n");
778 for (slot = 0; slot < 16; slot+=2) {
779 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
780 slot,
781 cheetah_get_ldtlb_tag(slot), cheetah_get_ldtlb_data(slot),
782 slot+1,
783 cheetah_get_ldtlb_tag(slot+1), cheetah_get_ldtlb_data(slot+1));
784 }
785 printk ("Contents of dtlb2:\n");
786 for (slot = 0; slot < 512; slot+=2) {
787 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
788 slot,
789 cheetah_get_dtlb_tag(slot, 2), cheetah_get_dtlb_data(slot, 2),
790 slot+1,
791 cheetah_get_dtlb_tag(slot+1, 2), cheetah_get_dtlb_data(slot+1, 2));
792 }
793 if (tlb_type == cheetah_plus) {
794 printk ("Contents of dtlb3:\n");
795 for (slot = 0; slot < 512; slot+=2) {
796 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
797 slot,
798 cheetah_get_dtlb_tag(slot, 3), cheetah_get_dtlb_data(slot, 3),
799 slot+1,
800 cheetah_get_dtlb_tag(slot+1, 3), cheetah_get_dtlb_data(slot+1, 3));
801 }
802 }
803 }
804 }
805
806 extern unsigned long cmdline_memory_size;
807
808 /* Find a free area for the bootmem map, avoiding the kernel image
809 * and the initial ramdisk.
810 */
811 static unsigned long __init choose_bootmap_pfn(unsigned long start_pfn,
812 unsigned long end_pfn)
813 {
814 unsigned long avoid_start, avoid_end, bootmap_size;
815 int i;
816
817 bootmap_size = ((end_pfn - start_pfn) + 7) / 8;
818 bootmap_size = ALIGN(bootmap_size, sizeof(long));
819
820 avoid_start = avoid_end = 0;
821 #ifdef CONFIG_BLK_DEV_INITRD
822 avoid_start = initrd_start;
823 avoid_end = PAGE_ALIGN(initrd_end);
824 #endif
825
826 #ifdef CONFIG_DEBUG_BOOTMEM
827 prom_printf("choose_bootmap_pfn: kern[%lx:%lx] avoid[%lx:%lx]\n",
828 kern_base, PAGE_ALIGN(kern_base + kern_size),
829 avoid_start, avoid_end);
830 #endif
831 for (i = 0; i < pavail_ents; i++) {
832 unsigned long start, end;
833
834 start = pavail[i].phys_addr;
835 end = start + pavail[i].reg_size;
836
837 while (start < end) {
838 if (start >= kern_base &&
839 start < PAGE_ALIGN(kern_base + kern_size)) {
840 start = PAGE_ALIGN(kern_base + kern_size);
841 continue;
842 }
843 if (start >= avoid_start && start < avoid_end) {
844 start = avoid_end;
845 continue;
846 }
847
848 if ((end - start) < bootmap_size)
849 break;
850
851 if (start < kern_base &&
852 (start + bootmap_size) > kern_base) {
853 start = PAGE_ALIGN(kern_base + kern_size);
854 continue;
855 }
856
857 if (start < avoid_start &&
858 (start + bootmap_size) > avoid_start) {
859 start = avoid_end;
860 continue;
861 }
862
863 /* OK, it doesn't overlap anything, use it. */
864 #ifdef CONFIG_DEBUG_BOOTMEM
865 prom_printf("choose_bootmap_pfn: Using %lx [%lx]\n",
866 start >> PAGE_SHIFT, start);
867 #endif
868 return start >> PAGE_SHIFT;
869 }
870 }
871
872 prom_printf("Cannot find free area for bootmap, aborting.\n");
873 prom_halt();
874 }
875
876 static unsigned long __init bootmem_init(unsigned long *pages_avail,
877 unsigned long phys_base)
878 {
879 unsigned long bootmap_size, end_pfn;
880 unsigned long end_of_phys_memory = 0UL;
881 unsigned long bootmap_pfn, bytes_avail, size;
882 int i;
883
884 #ifdef CONFIG_DEBUG_BOOTMEM
885 prom_printf("bootmem_init: Scan pavail, ");
886 #endif
887
888 bytes_avail = 0UL;
889 for (i = 0; i < pavail_ents; i++) {
890 end_of_phys_memory = pavail[i].phys_addr +
891 pavail[i].reg_size;
892 bytes_avail += pavail[i].reg_size;
893 if (cmdline_memory_size) {
894 if (bytes_avail > cmdline_memory_size) {
895 unsigned long slack = bytes_avail - cmdline_memory_size;
896
897 bytes_avail -= slack;
898 end_of_phys_memory -= slack;
899
900 pavail[i].reg_size -= slack;
901 if ((long)pavail[i].reg_size <= 0L) {
902 pavail[i].phys_addr = 0xdeadbeefUL;
903 pavail[i].reg_size = 0UL;
904 pavail_ents = i;
905 } else {
906 pavail[i+1].reg_size = 0Ul;
907 pavail[i+1].phys_addr = 0xdeadbeefUL;
908 pavail_ents = i + 1;
909 }
910 break;
911 }
912 }
913 }
914
915 *pages_avail = bytes_avail >> PAGE_SHIFT;
916
917 end_pfn = end_of_phys_memory >> PAGE_SHIFT;
918
919 #ifdef CONFIG_BLK_DEV_INITRD
920 /* Now have to check initial ramdisk, so that bootmap does not overwrite it */
921 if (sparc_ramdisk_image || sparc_ramdisk_image64) {
922 unsigned long ramdisk_image = sparc_ramdisk_image ?
923 sparc_ramdisk_image : sparc_ramdisk_image64;
924 if (ramdisk_image >= (unsigned long)_end - 2 * PAGE_SIZE)
925 ramdisk_image -= KERNBASE;
926 initrd_start = ramdisk_image + phys_base;
927 initrd_end = initrd_start + sparc_ramdisk_size;
928 if (initrd_end > end_of_phys_memory) {
929 printk(KERN_CRIT "initrd extends beyond end of memory "
930 "(0x%016lx > 0x%016lx)\ndisabling initrd\n",
931 initrd_end, end_of_phys_memory);
932 initrd_start = 0;
933 initrd_end = 0;
934 }
935 }
936 #endif
937 /* Initialize the boot-time allocator. */
938 max_pfn = max_low_pfn = end_pfn;
939 min_low_pfn = (phys_base >> PAGE_SHIFT);
940
941 bootmap_pfn = choose_bootmap_pfn(min_low_pfn, end_pfn);
942
943 #ifdef CONFIG_DEBUG_BOOTMEM
944 prom_printf("init_bootmem(min[%lx], bootmap[%lx], max[%lx])\n",
945 min_low_pfn, bootmap_pfn, max_low_pfn);
946 #endif
947 bootmap_size = init_bootmem_node(NODE_DATA(0), bootmap_pfn,
948 min_low_pfn, end_pfn);
949
950 /* Now register the available physical memory with the
951 * allocator.
952 */
953 for (i = 0; i < pavail_ents; i++) {
954 #ifdef CONFIG_DEBUG_BOOTMEM
955 prom_printf("free_bootmem(pavail:%d): base[%lx] size[%lx]\n",
956 i, pavail[i].phys_addr, pavail[i].reg_size);
957 #endif
958 free_bootmem(pavail[i].phys_addr, pavail[i].reg_size);
959 }
960
961 #ifdef CONFIG_BLK_DEV_INITRD
962 if (initrd_start) {
963 size = initrd_end - initrd_start;
964
965 /* Resert the initrd image area. */
966 #ifdef CONFIG_DEBUG_BOOTMEM
967 prom_printf("reserve_bootmem(initrd): base[%llx] size[%lx]\n",
968 initrd_start, initrd_end);
969 #endif
970 reserve_bootmem(initrd_start, size);
971 *pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT;
972
973 initrd_start += PAGE_OFFSET;
974 initrd_end += PAGE_OFFSET;
975 }
976 #endif
977 /* Reserve the kernel text/data/bss. */
978 #ifdef CONFIG_DEBUG_BOOTMEM
979 prom_printf("reserve_bootmem(kernel): base[%lx] size[%lx]\n", kern_base, kern_size);
980 #endif
981 reserve_bootmem(kern_base, kern_size);
982 *pages_avail -= PAGE_ALIGN(kern_size) >> PAGE_SHIFT;
983
984 /* Reserve the bootmem map. We do not account for it
985 * in pages_avail because we will release that memory
986 * in free_all_bootmem.
987 */
988 size = bootmap_size;
989 #ifdef CONFIG_DEBUG_BOOTMEM
990 prom_printf("reserve_bootmem(bootmap): base[%lx] size[%lx]\n",
991 (bootmap_pfn << PAGE_SHIFT), size);
992 #endif
993 reserve_bootmem((bootmap_pfn << PAGE_SHIFT), size);
994 *pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT;
995
996 for (i = 0; i < pavail_ents; i++) {
997 unsigned long start_pfn, end_pfn;
998
999 start_pfn = pavail[i].phys_addr >> PAGE_SHIFT;
1000 end_pfn = (start_pfn + (pavail[i].reg_size >> PAGE_SHIFT));
1001 #ifdef CONFIG_DEBUG_BOOTMEM
1002 prom_printf("memory_present(0, %lx, %lx)\n",
1003 start_pfn, end_pfn);
1004 #endif
1005 memory_present(0, start_pfn, end_pfn);
1006 }
1007
1008 sparse_init();
1009
1010 return end_pfn;
1011 }
1012
1013 static struct linux_prom64_registers pall[MAX_BANKS] __initdata;
1014 static int pall_ents __initdata;
1015
1016 #ifdef CONFIG_DEBUG_PAGEALLOC
1017 static unsigned long kernel_map_range(unsigned long pstart, unsigned long pend, pgprot_t prot)
1018 {
1019 unsigned long vstart = PAGE_OFFSET + pstart;
1020 unsigned long vend = PAGE_OFFSET + pend;
1021 unsigned long alloc_bytes = 0UL;
1022
1023 if ((vstart & ~PAGE_MASK) || (vend & ~PAGE_MASK)) {
1024 prom_printf("kernel_map: Unaligned physmem[%lx:%lx]\n",
1025 vstart, vend);
1026 prom_halt();
1027 }
1028
1029 while (vstart < vend) {
1030 unsigned long this_end, paddr = __pa(vstart);
1031 pgd_t *pgd = pgd_offset_k(vstart);
1032 pud_t *pud;
1033 pmd_t *pmd;
1034 pte_t *pte;
1035
1036 pud = pud_offset(pgd, vstart);
1037 if (pud_none(*pud)) {
1038 pmd_t *new;
1039
1040 new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
1041 alloc_bytes += PAGE_SIZE;
1042 pud_populate(&init_mm, pud, new);
1043 }
1044
1045 pmd = pmd_offset(pud, vstart);
1046 if (!pmd_present(*pmd)) {
1047 pte_t *new;
1048
1049 new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
1050 alloc_bytes += PAGE_SIZE;
1051 pmd_populate_kernel(&init_mm, pmd, new);
1052 }
1053
1054 pte = pte_offset_kernel(pmd, vstart);
1055 this_end = (vstart + PMD_SIZE) & PMD_MASK;
1056 if (this_end > vend)
1057 this_end = vend;
1058
1059 while (vstart < this_end) {
1060 pte_val(*pte) = (paddr | pgprot_val(prot));
1061
1062 vstart += PAGE_SIZE;
1063 paddr += PAGE_SIZE;
1064 pte++;
1065 }
1066 }
1067
1068 return alloc_bytes;
1069 }
1070
1071 extern unsigned int kvmap_linear_patch[1];
1072 #endif /* CONFIG_DEBUG_PAGEALLOC */
1073
1074 static void __init mark_kpte_bitmap(unsigned long start, unsigned long end)
1075 {
1076 const unsigned long shift_256MB = 28;
1077 const unsigned long mask_256MB = ((1UL << shift_256MB) - 1UL);
1078 const unsigned long size_256MB = (1UL << shift_256MB);
1079
1080 while (start < end) {
1081 long remains;
1082
1083 remains = end - start;
1084 if (remains < size_256MB)
1085 break;
1086
1087 if (start & mask_256MB) {
1088 start = (start + size_256MB) & ~mask_256MB;
1089 continue;
1090 }
1091
1092 while (remains >= size_256MB) {
1093 unsigned long index = start >> shift_256MB;
1094
1095 __set_bit(index, kpte_linear_bitmap);
1096
1097 start += size_256MB;
1098 remains -= size_256MB;
1099 }
1100 }
1101 }
1102
1103 static void __init kernel_physical_mapping_init(void)
1104 {
1105 unsigned long i;
1106 #ifdef CONFIG_DEBUG_PAGEALLOC
1107 unsigned long mem_alloced = 0UL;
1108 #endif
1109
1110 read_obp_memory("reg", &pall[0], &pall_ents);
1111
1112 for (i = 0; i < pall_ents; i++) {
1113 unsigned long phys_start, phys_end;
1114
1115 phys_start = pall[i].phys_addr;
1116 phys_end = phys_start + pall[i].reg_size;
1117
1118 mark_kpte_bitmap(phys_start, phys_end);
1119
1120 #ifdef CONFIG_DEBUG_PAGEALLOC
1121 mem_alloced += kernel_map_range(phys_start, phys_end,
1122 PAGE_KERNEL);
1123 #endif
1124 }
1125
1126 #ifdef CONFIG_DEBUG_PAGEALLOC
1127 printk("Allocated %ld bytes for kernel page tables.\n",
1128 mem_alloced);
1129
1130 kvmap_linear_patch[0] = 0x01000000; /* nop */
1131 flushi(&kvmap_linear_patch[0]);
1132
1133 __flush_tlb_all();
1134 #endif
1135 }
1136
1137 #ifdef CONFIG_DEBUG_PAGEALLOC
1138 void kernel_map_pages(struct page *page, int numpages, int enable)
1139 {
1140 unsigned long phys_start = page_to_pfn(page) << PAGE_SHIFT;
1141 unsigned long phys_end = phys_start + (numpages * PAGE_SIZE);
1142
1143 kernel_map_range(phys_start, phys_end,
1144 (enable ? PAGE_KERNEL : __pgprot(0)));
1145
1146 flush_tsb_kernel_range(PAGE_OFFSET + phys_start,
1147 PAGE_OFFSET + phys_end);
1148
1149 /* we should perform an IPI and flush all tlbs,
1150 * but that can deadlock->flush only current cpu.
1151 */
1152 __flush_tlb_kernel_range(PAGE_OFFSET + phys_start,
1153 PAGE_OFFSET + phys_end);
1154 }
1155 #endif
1156
1157 unsigned long __init find_ecache_flush_span(unsigned long size)
1158 {
1159 int i;
1160
1161 for (i = 0; i < pavail_ents; i++) {
1162 if (pavail[i].reg_size >= size)
1163 return pavail[i].phys_addr;
1164 }
1165
1166 return ~0UL;
1167 }
1168
1169 static void __init tsb_phys_patch(void)
1170 {
1171 struct tsb_ldquad_phys_patch_entry *pquad;
1172 struct tsb_phys_patch_entry *p;
1173
1174 pquad = &__tsb_ldquad_phys_patch;
1175 while (pquad < &__tsb_ldquad_phys_patch_end) {
1176 unsigned long addr = pquad->addr;
1177
1178 if (tlb_type == hypervisor)
1179 *(unsigned int *) addr = pquad->sun4v_insn;
1180 else
1181 *(unsigned int *) addr = pquad->sun4u_insn;
1182 wmb();
1183 __asm__ __volatile__("flush %0"
1184 : /* no outputs */
1185 : "r" (addr));
1186
1187 pquad++;
1188 }
1189
1190 p = &__tsb_phys_patch;
1191 while (p < &__tsb_phys_patch_end) {
1192 unsigned long addr = p->addr;
1193
1194 *(unsigned int *) addr = p->insn;
1195 wmb();
1196 __asm__ __volatile__("flush %0"
1197 : /* no outputs */
1198 : "r" (addr));
1199
1200 p++;
1201 }
1202 }
1203
1204 /* Don't mark as init, we give this to the Hypervisor. */
1205 static struct hv_tsb_descr ktsb_descr[2];
1206 extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];
1207
1208 static void __init sun4v_ktsb_init(void)
1209 {
1210 unsigned long ktsb_pa;
1211
1212 /* First KTSB for PAGE_SIZE mappings. */
1213 ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE);
1214
1215 switch (PAGE_SIZE) {
1216 case 8 * 1024:
1217 default:
1218 ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_8K;
1219 ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_8K;
1220 break;
1221
1222 case 64 * 1024:
1223 ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_64K;
1224 ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_64K;
1225 break;
1226
1227 case 512 * 1024:
1228 ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_512K;
1229 ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_512K;
1230 break;
1231
1232 case 4 * 1024 * 1024:
1233 ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_4MB;
1234 ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_4MB;
1235 break;
1236 };
1237
1238 ktsb_descr[0].assoc = 1;
1239 ktsb_descr[0].num_ttes = KERNEL_TSB_NENTRIES;
1240 ktsb_descr[0].ctx_idx = 0;
1241 ktsb_descr[0].tsb_base = ktsb_pa;
1242 ktsb_descr[0].resv = 0;
1243
1244 /* Second KTSB for 4MB/256MB mappings. */
1245 ktsb_pa = (kern_base +
1246 ((unsigned long)&swapper_4m_tsb[0] - KERNBASE));
1247
1248 ktsb_descr[1].pgsz_idx = HV_PGSZ_IDX_4MB;
1249 ktsb_descr[1].pgsz_mask = (HV_PGSZ_MASK_4MB |
1250 HV_PGSZ_MASK_256MB);
1251 ktsb_descr[1].assoc = 1;
1252 ktsb_descr[1].num_ttes = KERNEL_TSB4M_NENTRIES;
1253 ktsb_descr[1].ctx_idx = 0;
1254 ktsb_descr[1].tsb_base = ktsb_pa;
1255 ktsb_descr[1].resv = 0;
1256 }
1257
1258 void __cpuinit sun4v_ktsb_register(void)
1259 {
1260 register unsigned long func asm("%o5");
1261 register unsigned long arg0 asm("%o0");
1262 register unsigned long arg1 asm("%o1");
1263 unsigned long pa;
1264
1265 pa = kern_base + ((unsigned long)&ktsb_descr[0] - KERNBASE);
1266
1267 func = HV_FAST_MMU_TSB_CTX0;
1268 arg0 = 2;
1269 arg1 = pa;
1270 __asm__ __volatile__("ta %6"
1271 : "=&r" (func), "=&r" (arg0), "=&r" (arg1)
1272 : "0" (func), "1" (arg0), "2" (arg1),
1273 "i" (HV_FAST_TRAP));
1274 }
1275
1276 /* paging_init() sets up the page tables */
1277
1278 extern void cheetah_ecache_flush_init(void);
1279 extern void sun4v_patch_tlb_handlers(void);
1280
1281 static unsigned long last_valid_pfn;
1282 pgd_t swapper_pg_dir[2048];
1283
1284 static void sun4u_pgprot_init(void);
1285 static void sun4v_pgprot_init(void);
1286
1287 void __init paging_init(void)
1288 {
1289 unsigned long end_pfn, pages_avail, shift, phys_base;
1290 unsigned long real_end, i;
1291
1292 kern_base = (prom_boot_mapping_phys_low >> 22UL) << 22UL;
1293 kern_size = (unsigned long)&_end - (unsigned long)KERNBASE;
1294
1295 /* Invalidate both kernel TSBs. */
1296 memset(swapper_tsb, 0x40, sizeof(swapper_tsb));
1297 memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb));
1298
1299 if (tlb_type == hypervisor)
1300 sun4v_pgprot_init();
1301 else
1302 sun4u_pgprot_init();
1303
1304 if (tlb_type == cheetah_plus ||
1305 tlb_type == hypervisor)
1306 tsb_phys_patch();
1307
1308 if (tlb_type == hypervisor) {
1309 sun4v_patch_tlb_handlers();
1310 sun4v_ktsb_init();
1311 }
1312
1313 /* Find available physical memory... */
1314 read_obp_memory("available", &pavail[0], &pavail_ents);
1315
1316 phys_base = 0xffffffffffffffffUL;
1317 for (i = 0; i < pavail_ents; i++)
1318 phys_base = min(phys_base, pavail[i].phys_addr);
1319
1320 set_bit(0, mmu_context_bmap);
1321
1322 shift = kern_base + PAGE_OFFSET - ((unsigned long)KERNBASE);
1323
1324 real_end = (unsigned long)_end;
1325 if ((real_end > ((unsigned long)KERNBASE + 0x400000)))
1326 bigkernel = 1;
1327 if ((real_end > ((unsigned long)KERNBASE + 0x800000))) {
1328 prom_printf("paging_init: Kernel > 8MB, too large.\n");
1329 prom_halt();
1330 }
1331
1332 /* Set kernel pgd to upper alias so physical page computations
1333 * work.
1334 */
1335 init_mm.pgd += ((shift) / (sizeof(pgd_t)));
1336
1337 memset(swapper_low_pmd_dir, 0, sizeof(swapper_low_pmd_dir));
1338
1339 /* Now can init the kernel/bad page tables. */
1340 pud_set(pud_offset(&swapper_pg_dir[0], 0),
1341 swapper_low_pmd_dir + (shift / sizeof(pgd_t)));
1342
1343 inherit_prom_mappings();
1344
1345 /* Ok, we can use our TLB miss and window trap handlers safely. */
1346 setup_tba();
1347
1348 __flush_tlb_all();
1349
1350 if (tlb_type == hypervisor)
1351 sun4v_ktsb_register();
1352
1353 /* Setup bootmem... */
1354 pages_avail = 0;
1355 last_valid_pfn = end_pfn = bootmem_init(&pages_avail, phys_base);
1356
1357 max_mapnr = last_valid_pfn;
1358
1359 kernel_physical_mapping_init();
1360
1361 prom_build_devicetree();
1362
1363 {
1364 unsigned long zones_size[MAX_NR_ZONES];
1365 unsigned long zholes_size[MAX_NR_ZONES];
1366 int znum;
1367
1368 for (znum = 0; znum < MAX_NR_ZONES; znum++)
1369 zones_size[znum] = zholes_size[znum] = 0;
1370
1371 zones_size[ZONE_DMA] = end_pfn;
1372 zholes_size[ZONE_DMA] = end_pfn - pages_avail;
1373
1374 free_area_init_node(0, &contig_page_data, zones_size,
1375 __pa(PAGE_OFFSET) >> PAGE_SHIFT,
1376 zholes_size);
1377 }
1378
1379 device_scan();
1380 }
1381
1382 static void __init taint_real_pages(void)
1383 {
1384 int i;
1385
1386 read_obp_memory("available", &pavail_rescan[0], &pavail_rescan_ents);
1387
1388 /* Find changes discovered in the physmem available rescan and
1389 * reserve the lost portions in the bootmem maps.
1390 */
1391 for (i = 0; i < pavail_ents; i++) {
1392 unsigned long old_start, old_end;
1393
1394 old_start = pavail[i].phys_addr;
1395 old_end = old_start +
1396 pavail[i].reg_size;
1397 while (old_start < old_end) {
1398 int n;
1399
1400 for (n = 0; n < pavail_rescan_ents; n++) {
1401 unsigned long new_start, new_end;
1402
1403 new_start = pavail_rescan[n].phys_addr;
1404 new_end = new_start +
1405 pavail_rescan[n].reg_size;
1406
1407 if (new_start <= old_start &&
1408 new_end >= (old_start + PAGE_SIZE)) {
1409 set_bit(old_start >> 22,
1410 sparc64_valid_addr_bitmap);
1411 goto do_next_page;
1412 }
1413 }
1414 reserve_bootmem(old_start, PAGE_SIZE);
1415
1416 do_next_page:
1417 old_start += PAGE_SIZE;
1418 }
1419 }
1420 }
1421
1422 int __init page_in_phys_avail(unsigned long paddr)
1423 {
1424 int i;
1425
1426 paddr &= PAGE_MASK;
1427
1428 for (i = 0; i < pavail_rescan_ents; i++) {
1429 unsigned long start, end;
1430
1431 start = pavail_rescan[i].phys_addr;
1432 end = start + pavail_rescan[i].reg_size;
1433
1434 if (paddr >= start && paddr < end)
1435 return 1;
1436 }
1437 if (paddr >= kern_base && paddr < (kern_base + kern_size))
1438 return 1;
1439 #ifdef CONFIG_BLK_DEV_INITRD
1440 if (paddr >= __pa(initrd_start) &&
1441 paddr < __pa(PAGE_ALIGN(initrd_end)))
1442 return 1;
1443 #endif
1444
1445 return 0;
1446 }
1447
1448 void __init mem_init(void)
1449 {
1450 unsigned long codepages, datapages, initpages;
1451 unsigned long addr, last;
1452 int i;
1453
1454 i = last_valid_pfn >> ((22 - PAGE_SHIFT) + 6);
1455 i += 1;
1456 sparc64_valid_addr_bitmap = (unsigned long *) alloc_bootmem(i << 3);
1457 if (sparc64_valid_addr_bitmap == NULL) {
1458 prom_printf("mem_init: Cannot alloc valid_addr_bitmap.\n");
1459 prom_halt();
1460 }
1461 memset(sparc64_valid_addr_bitmap, 0, i << 3);
1462
1463 addr = PAGE_OFFSET + kern_base;
1464 last = PAGE_ALIGN(kern_size) + addr;
1465 while (addr < last) {
1466 set_bit(__pa(addr) >> 22, sparc64_valid_addr_bitmap);
1467 addr += PAGE_SIZE;
1468 }
1469
1470 taint_real_pages();
1471
1472 high_memory = __va(last_valid_pfn << PAGE_SHIFT);
1473
1474 #ifdef CONFIG_DEBUG_BOOTMEM
1475 prom_printf("mem_init: Calling free_all_bootmem().\n");
1476 #endif
1477 totalram_pages = num_physpages = free_all_bootmem() - 1;
1478
1479 /*
1480 * Set up the zero page, mark it reserved, so that page count
1481 * is not manipulated when freeing the page from user ptes.
1482 */
1483 mem_map_zero = alloc_pages(GFP_KERNEL|__GFP_ZERO, 0);
1484 if (mem_map_zero == NULL) {
1485 prom_printf("paging_init: Cannot alloc zero page.\n");
1486 prom_halt();
1487 }
1488 SetPageReserved(mem_map_zero);
1489
1490 codepages = (((unsigned long) _etext) - ((unsigned long) _start));
1491 codepages = PAGE_ALIGN(codepages) >> PAGE_SHIFT;
1492 datapages = (((unsigned long) _edata) - ((unsigned long) _etext));
1493 datapages = PAGE_ALIGN(datapages) >> PAGE_SHIFT;
1494 initpages = (((unsigned long) __init_end) - ((unsigned long) __init_begin));
1495 initpages = PAGE_ALIGN(initpages) >> PAGE_SHIFT;
1496
1497 printk("Memory: %uk available (%ldk kernel code, %ldk data, %ldk init) [%016lx,%016lx]\n",
1498 nr_free_pages() << (PAGE_SHIFT-10),
1499 codepages << (PAGE_SHIFT-10),
1500 datapages << (PAGE_SHIFT-10),
1501 initpages << (PAGE_SHIFT-10),
1502 PAGE_OFFSET, (last_valid_pfn << PAGE_SHIFT));
1503
1504 if (tlb_type == cheetah || tlb_type == cheetah_plus)
1505 cheetah_ecache_flush_init();
1506 }
1507
1508 void free_initmem(void)
1509 {
1510 unsigned long addr, initend;
1511
1512 /*
1513 * The init section is aligned to 8k in vmlinux.lds. Page align for >8k pagesizes.
1514 */
1515 addr = PAGE_ALIGN((unsigned long)(__init_begin));
1516 initend = (unsigned long)(__init_end) & PAGE_MASK;
1517 for (; addr < initend; addr += PAGE_SIZE) {
1518 unsigned long page;
1519 struct page *p;
1520
1521 page = (addr +
1522 ((unsigned long) __va(kern_base)) -
1523 ((unsigned long) KERNBASE));
1524 memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
1525 p = virt_to_page(page);
1526
1527 ClearPageReserved(p);
1528 init_page_count(p);
1529 __free_page(p);
1530 num_physpages++;
1531 totalram_pages++;
1532 }
1533 }
1534
1535 #ifdef CONFIG_BLK_DEV_INITRD
1536 void free_initrd_mem(unsigned long start, unsigned long end)
1537 {
1538 if (start < end)
1539 printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10);
1540 for (; start < end; start += PAGE_SIZE) {
1541 struct page *p = virt_to_page(start);
1542
1543 ClearPageReserved(p);
1544 init_page_count(p);
1545 __free_page(p);
1546 num_physpages++;
1547 totalram_pages++;
1548 }
1549 }
1550 #endif
1551
1552 #define _PAGE_CACHE_4U (_PAGE_CP_4U | _PAGE_CV_4U)
1553 #define _PAGE_CACHE_4V (_PAGE_CP_4V | _PAGE_CV_4V)
1554 #define __DIRTY_BITS_4U (_PAGE_MODIFIED_4U | _PAGE_WRITE_4U | _PAGE_W_4U)
1555 #define __DIRTY_BITS_4V (_PAGE_MODIFIED_4V | _PAGE_WRITE_4V | _PAGE_W_4V)
1556 #define __ACCESS_BITS_4U (_PAGE_ACCESSED_4U | _PAGE_READ_4U | _PAGE_R)
1557 #define __ACCESS_BITS_4V (_PAGE_ACCESSED_4V | _PAGE_READ_4V | _PAGE_R)
1558
1559 pgprot_t PAGE_KERNEL __read_mostly;
1560 EXPORT_SYMBOL(PAGE_KERNEL);
1561
1562 pgprot_t PAGE_KERNEL_LOCKED __read_mostly;
1563 pgprot_t PAGE_COPY __read_mostly;
1564
1565 pgprot_t PAGE_SHARED __read_mostly;
1566 EXPORT_SYMBOL(PAGE_SHARED);
1567
1568 pgprot_t PAGE_EXEC __read_mostly;
1569 unsigned long pg_iobits __read_mostly;
1570
1571 unsigned long _PAGE_IE __read_mostly;
1572 EXPORT_SYMBOL(_PAGE_IE);
1573
1574 unsigned long _PAGE_E __read_mostly;
1575 EXPORT_SYMBOL(_PAGE_E);
1576
1577 unsigned long _PAGE_CACHE __read_mostly;
1578 EXPORT_SYMBOL(_PAGE_CACHE);
1579
1580 static void prot_init_common(unsigned long page_none,
1581 unsigned long page_shared,
1582 unsigned long page_copy,
1583 unsigned long page_readonly,
1584 unsigned long page_exec_bit)
1585 {
1586 PAGE_COPY = __pgprot(page_copy);
1587 PAGE_SHARED = __pgprot(page_shared);
1588
1589 protection_map[0x0] = __pgprot(page_none);
1590 protection_map[0x1] = __pgprot(page_readonly & ~page_exec_bit);
1591 protection_map[0x2] = __pgprot(page_copy & ~page_exec_bit);
1592 protection_map[0x3] = __pgprot(page_copy & ~page_exec_bit);
1593 protection_map[0x4] = __pgprot(page_readonly);
1594 protection_map[0x5] = __pgprot(page_readonly);
1595 protection_map[0x6] = __pgprot(page_copy);
1596 protection_map[0x7] = __pgprot(page_copy);
1597 protection_map[0x8] = __pgprot(page_none);
1598 protection_map[0x9] = __pgprot(page_readonly & ~page_exec_bit);
1599 protection_map[0xa] = __pgprot(page_shared & ~page_exec_bit);
1600 protection_map[0xb] = __pgprot(page_shared & ~page_exec_bit);
1601 protection_map[0xc] = __pgprot(page_readonly);
1602 protection_map[0xd] = __pgprot(page_readonly);
1603 protection_map[0xe] = __pgprot(page_shared);
1604 protection_map[0xf] = __pgprot(page_shared);
1605 }
1606
1607 static void __init sun4u_pgprot_init(void)
1608 {
1609 unsigned long page_none, page_shared, page_copy, page_readonly;
1610 unsigned long page_exec_bit;
1611
1612 PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID |
1613 _PAGE_CACHE_4U | _PAGE_P_4U |
1614 __ACCESS_BITS_4U | __DIRTY_BITS_4U |
1615 _PAGE_EXEC_4U);
1616 PAGE_KERNEL_LOCKED = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID |
1617 _PAGE_CACHE_4U | _PAGE_P_4U |
1618 __ACCESS_BITS_4U | __DIRTY_BITS_4U |
1619 _PAGE_EXEC_4U | _PAGE_L_4U);
1620 PAGE_EXEC = __pgprot(_PAGE_EXEC_4U);
1621
1622 _PAGE_IE = _PAGE_IE_4U;
1623 _PAGE_E = _PAGE_E_4U;
1624 _PAGE_CACHE = _PAGE_CACHE_4U;
1625
1626 pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4U | __DIRTY_BITS_4U |
1627 __ACCESS_BITS_4U | _PAGE_E_4U);
1628
1629 kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4U) ^
1630 0xfffff80000000000;
1631 kern_linear_pte_xor[0] |= (_PAGE_CP_4U | _PAGE_CV_4U |
1632 _PAGE_P_4U | _PAGE_W_4U);
1633
1634 /* XXX Should use 256MB on Panther. XXX */
1635 kern_linear_pte_xor[1] = kern_linear_pte_xor[0];
1636
1637 _PAGE_SZBITS = _PAGE_SZBITS_4U;
1638 _PAGE_ALL_SZ_BITS = (_PAGE_SZ4MB_4U | _PAGE_SZ512K_4U |
1639 _PAGE_SZ64K_4U | _PAGE_SZ8K_4U |
1640 _PAGE_SZ32MB_4U | _PAGE_SZ256MB_4U);
1641
1642
1643 page_none = _PAGE_PRESENT_4U | _PAGE_ACCESSED_4U | _PAGE_CACHE_4U;
1644 page_shared = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
1645 __ACCESS_BITS_4U | _PAGE_WRITE_4U | _PAGE_EXEC_4U);
1646 page_copy = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
1647 __ACCESS_BITS_4U | _PAGE_EXEC_4U);
1648 page_readonly = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
1649 __ACCESS_BITS_4U | _PAGE_EXEC_4U);
1650
1651 page_exec_bit = _PAGE_EXEC_4U;
1652
1653 prot_init_common(page_none, page_shared, page_copy, page_readonly,
1654 page_exec_bit);
1655 }
1656
1657 static void __init sun4v_pgprot_init(void)
1658 {
1659 unsigned long page_none, page_shared, page_copy, page_readonly;
1660 unsigned long page_exec_bit;
1661
1662 PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4V | _PAGE_VALID |
1663 _PAGE_CACHE_4V | _PAGE_P_4V |
1664 __ACCESS_BITS_4V | __DIRTY_BITS_4V |
1665 _PAGE_EXEC_4V);
1666 PAGE_KERNEL_LOCKED = PAGE_KERNEL;
1667 PAGE_EXEC = __pgprot(_PAGE_EXEC_4V);
1668
1669 _PAGE_IE = _PAGE_IE_4V;
1670 _PAGE_E = _PAGE_E_4V;
1671 _PAGE_CACHE = _PAGE_CACHE_4V;
1672
1673 kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4V) ^
1674 0xfffff80000000000;
1675 kern_linear_pte_xor[0] |= (_PAGE_CP_4V | _PAGE_CV_4V |
1676 _PAGE_P_4V | _PAGE_W_4V);
1677
1678 kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^
1679 0xfffff80000000000;
1680 kern_linear_pte_xor[1] |= (_PAGE_CP_4V | _PAGE_CV_4V |
1681 _PAGE_P_4V | _PAGE_W_4V);
1682
1683 pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4V | __DIRTY_BITS_4V |
1684 __ACCESS_BITS_4V | _PAGE_E_4V);
1685
1686 _PAGE_SZBITS = _PAGE_SZBITS_4V;
1687 _PAGE_ALL_SZ_BITS = (_PAGE_SZ16GB_4V | _PAGE_SZ2GB_4V |
1688 _PAGE_SZ256MB_4V | _PAGE_SZ32MB_4V |
1689 _PAGE_SZ4MB_4V | _PAGE_SZ512K_4V |
1690 _PAGE_SZ64K_4V | _PAGE_SZ8K_4V);
1691
1692 page_none = _PAGE_PRESENT_4V | _PAGE_ACCESSED_4V | _PAGE_CACHE_4V;
1693 page_shared = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V |
1694 __ACCESS_BITS_4V | _PAGE_WRITE_4V | _PAGE_EXEC_4V);
1695 page_copy = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V |
1696 __ACCESS_BITS_4V | _PAGE_EXEC_4V);
1697 page_readonly = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V |
1698 __ACCESS_BITS_4V | _PAGE_EXEC_4V);
1699
1700 page_exec_bit = _PAGE_EXEC_4V;
1701
1702 prot_init_common(page_none, page_shared, page_copy, page_readonly,
1703 page_exec_bit);
1704 }
1705
1706 unsigned long pte_sz_bits(unsigned long sz)
1707 {
1708 if (tlb_type == hypervisor) {
1709 switch (sz) {
1710 case 8 * 1024:
1711 default:
1712 return _PAGE_SZ8K_4V;
1713 case 64 * 1024:
1714 return _PAGE_SZ64K_4V;
1715 case 512 * 1024:
1716 return _PAGE_SZ512K_4V;
1717 case 4 * 1024 * 1024:
1718 return _PAGE_SZ4MB_4V;
1719 };
1720 } else {
1721 switch (sz) {
1722 case 8 * 1024:
1723 default:
1724 return _PAGE_SZ8K_4U;
1725 case 64 * 1024:
1726 return _PAGE_SZ64K_4U;
1727 case 512 * 1024:
1728 return _PAGE_SZ512K_4U;
1729 case 4 * 1024 * 1024:
1730 return _PAGE_SZ4MB_4U;
1731 };
1732 }
1733 }
1734
1735 pte_t mk_pte_io(unsigned long page, pgprot_t prot, int space, unsigned long page_size)
1736 {
1737 pte_t pte;
1738
1739 pte_val(pte) = page | pgprot_val(pgprot_noncached(prot));
1740 pte_val(pte) |= (((unsigned long)space) << 32);
1741 pte_val(pte) |= pte_sz_bits(page_size);
1742
1743 return pte;
1744 }
1745
1746 static unsigned long kern_large_tte(unsigned long paddr)
1747 {
1748 unsigned long val;
1749
1750 val = (_PAGE_VALID | _PAGE_SZ4MB_4U |
1751 _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_P_4U |
1752 _PAGE_EXEC_4U | _PAGE_L_4U | _PAGE_W_4U);
1753 if (tlb_type == hypervisor)
1754 val = (_PAGE_VALID | _PAGE_SZ4MB_4V |
1755 _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_P_4V |
1756 _PAGE_EXEC_4V | _PAGE_W_4V);
1757
1758 return val | paddr;
1759 }
1760
1761 /*
1762 * Translate PROM's mapping we capture at boot time into physical address.
1763 * The second parameter is only set from prom_callback() invocations.
1764 */
1765 unsigned long prom_virt_to_phys(unsigned long promva, int *error)
1766 {
1767 unsigned long mask;
1768 int i;
1769
1770 mask = _PAGE_PADDR_4U;
1771 if (tlb_type == hypervisor)
1772 mask = _PAGE_PADDR_4V;
1773
1774 for (i = 0; i < prom_trans_ents; i++) {
1775 struct linux_prom_translation *p = &prom_trans[i];
1776
1777 if (promva >= p->virt &&
1778 promva < (p->virt + p->size)) {
1779 unsigned long base = p->data & mask;
1780
1781 if (error)
1782 *error = 0;
1783 return base + (promva & (8192 - 1));
1784 }
1785 }
1786 if (error)
1787 *error = 1;
1788 return 0UL;
1789 }
1790
1791 /* XXX We should kill off this ugly thing at so me point. XXX */
1792 unsigned long sun4u_get_pte(unsigned long addr)
1793 {
1794 pgd_t *pgdp;
1795 pud_t *pudp;
1796 pmd_t *pmdp;
1797 pte_t *ptep;
1798 unsigned long mask = _PAGE_PADDR_4U;
1799
1800 if (tlb_type == hypervisor)
1801 mask = _PAGE_PADDR_4V;
1802
1803 if (addr >= PAGE_OFFSET)
1804 return addr & mask;
1805
1806 if ((addr >= LOW_OBP_ADDRESS) && (addr < HI_OBP_ADDRESS))
1807 return prom_virt_to_phys(addr, NULL);
1808
1809 pgdp = pgd_offset_k(addr);
1810 pudp = pud_offset(pgdp, addr);
1811 pmdp = pmd_offset(pudp, addr);
1812 ptep = pte_offset_kernel(pmdp, addr);
1813
1814 return pte_val(*ptep) & mask;
1815 }
1816
1817 /* If not locked, zap it. */
1818 void __flush_tlb_all(void)
1819 {
1820 unsigned long pstate;
1821 int i;
1822
1823 __asm__ __volatile__("flushw\n\t"
1824 "rdpr %%pstate, %0\n\t"
1825 "wrpr %0, %1, %%pstate"
1826 : "=r" (pstate)
1827 : "i" (PSTATE_IE));
1828 if (tlb_type == spitfire) {
1829 for (i = 0; i < 64; i++) {
1830 /* Spitfire Errata #32 workaround */
1831 /* NOTE: Always runs on spitfire, so no
1832 * cheetah+ page size encodings.
1833 */
1834 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
1835 "flush %%g6"
1836 : /* No outputs */
1837 : "r" (0),
1838 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
1839
1840 if (!(spitfire_get_dtlb_data(i) & _PAGE_L_4U)) {
1841 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
1842 "membar #Sync"
1843 : /* no outputs */
1844 : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
1845 spitfire_put_dtlb_data(i, 0x0UL);
1846 }
1847
1848 /* Spitfire Errata #32 workaround */
1849 /* NOTE: Always runs on spitfire, so no
1850 * cheetah+ page size encodings.
1851 */
1852 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
1853 "flush %%g6"
1854 : /* No outputs */
1855 : "r" (0),
1856 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
1857
1858 if (!(spitfire_get_itlb_data(i) & _PAGE_L_4U)) {
1859 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
1860 "membar #Sync"
1861 : /* no outputs */
1862 : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
1863 spitfire_put_itlb_data(i, 0x0UL);
1864 }
1865 }
1866 } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
1867 cheetah_flush_dtlb_all();
1868 cheetah_flush_itlb_all();
1869 }
1870 __asm__ __volatile__("wrpr %0, 0, %%pstate"
1871 : : "r" (pstate));
1872 }
1873
1874 #ifdef CONFIG_MEMORY_HOTPLUG
1875
1876 void online_page(struct page *page)
1877 {
1878 ClearPageReserved(page);
1879 init_page_count(page);
1880 __free_page(page);
1881 totalram_pages++;
1882 num_physpages++;
1883 }
1884
1885 int remove_memory(u64 start, u64 size)
1886 {
1887 return -EINVAL;
1888 }
1889
1890 #endif /* CONFIG_MEMORY_HOTPLUG */
ubuntu-zesty-kernel mirror