]>
Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
14cf11af PM |
2 | /* |
3 | * This file contains the routines setting up the linux page tables. | |
4 | * -- paulus | |
5 | * | |
6 | * Derived from arch/ppc/mm/init.c: | |
7 | * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) | |
8 | * | |
9 | * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au) | |
10 | * and Cort Dougan (PReP) (cort@cs.nmt.edu) | |
11 | * Copyright (C) 1996 Paul Mackerras | |
14cf11af PM |
12 | * |
13 | * Derived from "arch/i386/mm/init.c" | |
14 | * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds | |
14cf11af PM |
15 | */ |
16 | ||
14cf11af PM |
17 | #include <linux/kernel.h> |
18 | #include <linux/module.h> | |
19 | #include <linux/types.h> | |
20 | #include <linux/mm.h> | |
21 | #include <linux/vmalloc.h> | |
22 | #include <linux/init.h> | |
23 | #include <linux/highmem.h> | |
95f72d1e | 24 | #include <linux/memblock.h> |
5a0e3ad6 | 25 | #include <linux/slab.h> |
14cf11af PM |
26 | |
27 | #include <asm/pgtable.h> | |
28 | #include <asm/pgalloc.h> | |
2c419bde | 29 | #include <asm/fixmap.h> |
14cf11af | 30 | #include <asm/io.h> |
ae3a197e | 31 | #include <asm/setup.h> |
95902e6c | 32 | #include <asm/sections.h> |
14cf11af | 33 | |
9d9f2ccc | 34 | #include <mm/mmu_decl.h> |
14cf11af | 35 | |
14cf11af | 36 | unsigned long ioremap_bot; |
920573bd | 37 | EXPORT_SYMBOL(ioremap_bot); /* aka VMALLOC_END */ |
14cf11af | 38 | |
060ef9d8 | 39 | extern char etext[], _stext[], _sinittext[], _einittext[]; |
14cf11af | 40 | |
14cf11af PM |
41 | void __iomem * |
42 | ioremap(phys_addr_t addr, unsigned long size) | |
43 | { | |
c766ee72 | 44 | pgprot_t prot = pgprot_noncached(PAGE_KERNEL); |
56f3c141 | 45 | |
c766ee72 | 46 | return __ioremap_caller(addr, size, prot, __builtin_return_address(0)); |
14cf11af | 47 | } |
920573bd | 48 | EXPORT_SYMBOL(ioremap); |
14cf11af | 49 | |
be135f40 AB |
50 | void __iomem * |
51 | ioremap_wc(phys_addr_t addr, unsigned long size) | |
52 | { | |
c766ee72 | 53 | pgprot_t prot = pgprot_noncached_wc(PAGE_KERNEL); |
56f3c141 | 54 | |
c766ee72 | 55 | return __ioremap_caller(addr, size, prot, __builtin_return_address(0)); |
be135f40 AB |
56 | } |
57 | EXPORT_SYMBOL(ioremap_wc); | |
58 | ||
86c391bd CL |
59 | void __iomem * |
60 | ioremap_wt(phys_addr_t addr, unsigned long size) | |
61 | { | |
c766ee72 | 62 | pgprot_t prot = pgprot_cached_wthru(PAGE_KERNEL); |
56f3c141 | 63 | |
c766ee72 | 64 | return __ioremap_caller(addr, size, prot, __builtin_return_address(0)); |
86c391bd CL |
65 | } |
66 | EXPORT_SYMBOL(ioremap_wt); | |
67 | ||
68 | void __iomem * | |
69 | ioremap_coherent(phys_addr_t addr, unsigned long size) | |
70 | { | |
c766ee72 | 71 | pgprot_t prot = pgprot_cached(PAGE_KERNEL); |
56f3c141 | 72 | |
c766ee72 | 73 | return __ioremap_caller(addr, size, prot, __builtin_return_address(0)); |
86c391bd CL |
74 | } |
75 | EXPORT_SYMBOL(ioremap_coherent); | |
76 | ||
68a64357 | 77 | void __iomem * |
40f1ce7f | 78 | ioremap_prot(phys_addr_t addr, unsigned long size, unsigned long flags) |
68a64357 | 79 | { |
26973fa5 CL |
80 | pte_t pte = __pte(flags); |
81 | ||
a1f242ff | 82 | /* writeable implies dirty for kernel addresses */ |
26973fa5 CL |
83 | if (pte_write(pte)) |
84 | pte = pte_mkdirty(pte); | |
a1f242ff BH |
85 | |
86 | /* we don't want to let _PAGE_USER and _PAGE_EXEC leak out */ | |
26973fa5 CL |
87 | pte = pte_exprotect(pte); |
88 | pte = pte_mkprivileged(pte); | |
55052eec | 89 | |
26973fa5 | 90 | return __ioremap_caller(addr, size, pte_pgprot(pte), __builtin_return_address(0)); |
68a64357 | 91 | } |
40f1ce7f | 92 | EXPORT_SYMBOL(ioremap_prot); |
68a64357 | 93 | |
14cf11af PM |
94 | void __iomem * |
95 | __ioremap(phys_addr_t addr, unsigned long size, unsigned long flags) | |
1cdab55d | 96 | { |
c766ee72 | 97 | return __ioremap_caller(addr, size, __pgprot(flags), __builtin_return_address(0)); |
1cdab55d BH |
98 | } |
99 | ||
100 | void __iomem * | |
c766ee72 | 101 | __ioremap_caller(phys_addr_t addr, unsigned long size, pgprot_t prot, void *caller) |
14cf11af PM |
102 | { |
103 | unsigned long v, i; | |
104 | phys_addr_t p; | |
105 | int err; | |
106 | ||
107 | /* | |
108 | * Choose an address to map it to. | |
109 | * Once the vmalloc system is running, we use it. | |
e974cd4b | 110 | * Before then, we use space going down from IOREMAP_TOP |
14cf11af PM |
111 | * (ioremap_bot records where we're up to). |
112 | */ | |
113 | p = addr & PAGE_MASK; | |
114 | size = PAGE_ALIGN(addr + size) - p; | |
115 | ||
116 | /* | |
117 | * If the address lies within the first 16 MB, assume it's in ISA | |
118 | * memory space | |
119 | */ | |
120 | if (p < 16*1024*1024) | |
121 | p += _ISA_MEM_BASE; | |
122 | ||
01695a96 | 123 | #ifndef CONFIG_CRASH_DUMP |
14cf11af PM |
124 | /* |
125 | * Don't allow anybody to remap normal RAM that we're using. | |
126 | * mem_init() sets high_memory so only do the check after that. | |
127 | */ | |
6bf752da | 128 | if (slab_is_available() && p <= virt_to_phys(high_memory - 1) && |
7e140591 | 129 | page_is_ram(__phys_to_pfn(p))) { |
cc83458d | 130 | printk("__ioremap(): phys addr 0x%llx is RAM lr %ps\n", |
37f01d64 | 131 | (unsigned long long)p, __builtin_return_address(0)); |
14cf11af PM |
132 | return NULL; |
133 | } | |
01695a96 | 134 | #endif |
14cf11af PM |
135 | |
136 | if (size == 0) | |
137 | return NULL; | |
138 | ||
139 | /* | |
140 | * Is it already mapped? Perhaps overlapped by a previous | |
3084cdb7 | 141 | * mapping. |
14cf11af | 142 | */ |
3084cdb7 CL |
143 | v = p_block_mapped(p); |
144 | if (v) | |
14cf11af PM |
145 | goto out; |
146 | ||
f691fa10 | 147 | if (slab_is_available()) { |
14cf11af | 148 | struct vm_struct *area; |
1cdab55d | 149 | area = get_vm_area_caller(size, VM_IOREMAP, caller); |
14cf11af PM |
150 | if (area == 0) |
151 | return NULL; | |
7a9d1256 | 152 | area->phys_addr = p; |
14cf11af PM |
153 | v = (unsigned long) area->addr; |
154 | } else { | |
155 | v = (ioremap_bot -= size); | |
156 | } | |
157 | ||
14cf11af PM |
158 | /* |
159 | * Should check if it is a candidate for a BAT mapping | |
160 | */ | |
161 | ||
162 | err = 0; | |
163 | for (i = 0; i < size && err == 0; i += PAGE_SIZE) | |
c766ee72 | 164 | err = map_kernel_page(v + i, p + i, prot); |
14cf11af | 165 | if (err) { |
f691fa10 | 166 | if (slab_is_available()) |
14cf11af PM |
167 | vunmap((void *)v); |
168 | return NULL; | |
169 | } | |
170 | ||
171 | out: | |
172 | return (void __iomem *) (v + ((unsigned long)addr & ~PAGE_MASK)); | |
173 | } | |
920573bd | 174 | EXPORT_SYMBOL(__ioremap); |
14cf11af PM |
175 | |
176 | void iounmap(volatile void __iomem *addr) | |
177 | { | |
178 | /* | |
179 | * If mapped by BATs then there is nothing to do. | |
180 | * Calling vfree() generates a benign warning. | |
181 | */ | |
3084cdb7 CL |
182 | if (v_block_mapped((unsigned long)addr)) |
183 | return; | |
14cf11af PM |
184 | |
185 | if (addr > high_memory && (unsigned long) addr < ioremap_bot) | |
186 | vunmap((void *) (PAGE_MASK & (unsigned long)addr)); | |
187 | } | |
920573bd | 188 | EXPORT_SYMBOL(iounmap); |
14cf11af | 189 | |
4a6d8cf9 CL |
190 | static void __init *early_alloc_pgtable(unsigned long size) |
191 | { | |
192 | void *ptr = memblock_alloc(size, size); | |
193 | ||
194 | if (!ptr) | |
195 | panic("%s: Failed to allocate %lu bytes align=0x%lx\n", | |
196 | __func__, size, size); | |
197 | ||
198 | return ptr; | |
199 | } | |
200 | ||
201 | static pte_t __init *early_pte_alloc_kernel(pmd_t *pmdp, unsigned long va) | |
202 | { | |
203 | if (pmd_none(*pmdp)) { | |
204 | pte_t *ptep = early_alloc_pgtable(PTE_FRAG_SIZE); | |
205 | ||
206 | pmd_populate_kernel(&init_mm, pmdp, ptep); | |
207 | } | |
208 | return pte_offset_kernel(pmdp, va); | |
209 | } | |
210 | ||
211 | ||
212 | int __ref map_kernel_page(unsigned long va, phys_addr_t pa, pgprot_t prot) | |
14cf11af PM |
213 | { |
214 | pmd_t *pd; | |
215 | pte_t *pg; | |
216 | int err = -ENOMEM; | |
217 | ||
14cf11af | 218 | /* Use upper 10 bits of VA to index the first level map */ |
d1953c88 | 219 | pd = pmd_offset(pud_offset(pgd_offset_k(va), va), va); |
14cf11af | 220 | /* Use middle 10 bits of VA to index the second-level map */ |
4a6d8cf9 CL |
221 | if (likely(slab_is_available())) |
222 | pg = pte_alloc_kernel(pd, va); | |
223 | else | |
224 | pg = early_pte_alloc_kernel(pd, va); | |
14cf11af PM |
225 | if (pg != 0) { |
226 | err = 0; | |
3be4e699 BH |
227 | /* The PTE should never be already set nor present in the |
228 | * hash table | |
229 | */ | |
26973fa5 | 230 | BUG_ON((pte_present(*pg) | pte_hashpte(*pg)) && pgprot_val(prot)); |
c766ee72 | 231 | set_pte_at(&init_mm, va, pg, pfn_pte(pa >> PAGE_SHIFT, prot)); |
14cf11af | 232 | } |
47ce8af4 | 233 | smp_wmb(); |
14cf11af PM |
234 | return err; |
235 | } | |
236 | ||
237 | /* | |
de32400d | 238 | * Map in a chunk of physical memory starting at start. |
14cf11af | 239 | */ |
86b19520 | 240 | static void __init __mapin_ram_chunk(unsigned long offset, unsigned long top) |
14cf11af | 241 | { |
c766ee72 | 242 | unsigned long v, s; |
99c62dd7 | 243 | phys_addr_t p; |
ee4f2ea4 | 244 | int ktext; |
14cf11af | 245 | |
de32400d | 246 | s = offset; |
ccdcef72 | 247 | v = PAGE_OFFSET + s; |
99c62dd7 | 248 | p = memstart_addr + s; |
de32400d | 249 | for (; s < top; s += PAGE_SIZE) { |
060ef9d8 CL |
250 | ktext = ((char *)v >= _stext && (char *)v < etext) || |
251 | ((char *)v >= _sinittext && (char *)v < _einittext); | |
c766ee72 | 252 | map_kernel_page(v, p, ktext ? PAGE_KERNEL_TEXT : PAGE_KERNEL); |
68289ae9 | 253 | #ifdef CONFIG_PPC_BOOK3S_32 |
ee4f2ea4 | 254 | if (ktext) |
34eb138e | 255 | hash_preload(&init_mm, v, false, 0x300); |
ee4f2ea4 | 256 | #endif |
14cf11af PM |
257 | v += PAGE_SIZE; |
258 | p += PAGE_SIZE; | |
259 | } | |
260 | } | |
261 | ||
de32400d AH |
262 | void __init mapin_ram(void) |
263 | { | |
9e849f23 CL |
264 | struct memblock_region *reg; |
265 | ||
266 | for_each_memblock(memory, reg) { | |
267 | phys_addr_t base = reg->base; | |
268 | phys_addr_t top = min(base + reg->size, total_lowmem); | |
de32400d | 269 | |
9e849f23 CL |
270 | if (base >= top) |
271 | continue; | |
272 | base = mmu_mapin_ram(base, top); | |
d2f15e09 CL |
273 | if (IS_ENABLED(CONFIG_BDI_SWITCH)) |
274 | __mapin_ram_chunk(reg->base, top); | |
275 | else | |
276 | __mapin_ram_chunk(base, top); | |
de32400d | 277 | } |
de32400d AH |
278 | } |
279 | ||
14cf11af PM |
280 | /* Scan the real Linux page tables and return a PTE pointer for |
281 | * a virtual address in a context. | |
282 | * Returns true (1) if PTE was found, zero otherwise. The pointer to | |
283 | * the PTE pointer is unmodified if PTE is not found. | |
284 | */ | |
86b19520 | 285 | static int |
bab70a4a | 286 | get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep, pmd_t **pmdp) |
14cf11af PM |
287 | { |
288 | pgd_t *pgd; | |
d1953c88 | 289 | pud_t *pud; |
14cf11af PM |
290 | pmd_t *pmd; |
291 | pte_t *pte; | |
292 | int retval = 0; | |
293 | ||
294 | pgd = pgd_offset(mm, addr & PAGE_MASK); | |
295 | if (pgd) { | |
d1953c88 DG |
296 | pud = pud_offset(pgd, addr & PAGE_MASK); |
297 | if (pud && pud_present(*pud)) { | |
298 | pmd = pmd_offset(pud, addr & PAGE_MASK); | |
299 | if (pmd_present(*pmd)) { | |
300 | pte = pte_offset_map(pmd, addr & PAGE_MASK); | |
301 | if (pte) { | |
302 | retval = 1; | |
303 | *ptep = pte; | |
304 | if (pmdp) | |
305 | *pmdp = pmd; | |
306 | /* XXX caller needs to do pte_unmap, yuck */ | |
307 | } | |
308 | } | |
309 | } | |
14cf11af PM |
310 | } |
311 | return(retval); | |
312 | } | |
313 | ||
e611939f | 314 | static int __change_page_attr_noflush(struct page *page, pgprot_t prot) |
88df6e90 BH |
315 | { |
316 | pte_t *kpte; | |
317 | pmd_t *kpmd; | |
318 | unsigned long address; | |
319 | ||
320 | BUG_ON(PageHighMem(page)); | |
321 | address = (unsigned long)page_address(page); | |
322 | ||
3084cdb7 | 323 | if (v_block_mapped(address)) |
88df6e90 BH |
324 | return 0; |
325 | if (!get_pteptr(&init_mm, address, &kpte, &kpmd)) | |
326 | return -EINVAL; | |
50891457 | 327 | __set_pte_at(&init_mm, address, kpte, mk_pte(page, prot), 0); |
88df6e90 BH |
328 | pte_unmap(kpte); |
329 | ||
330 | return 0; | |
331 | } | |
332 | ||
333 | /* | |
334 | * Change the page attributes of an page in the linear mapping. | |
335 | * | |
3184cc4b | 336 | * THIS DOES NOTHING WITH BAT MAPPINGS, DEBUG USE ONLY |
88df6e90 BH |
337 | */ |
338 | static int change_page_attr(struct page *page, int numpages, pgprot_t prot) | |
339 | { | |
340 | int i, err = 0; | |
341 | unsigned long flags; | |
e611939f | 342 | struct page *start = page; |
88df6e90 BH |
343 | |
344 | local_irq_save(flags); | |
345 | for (i = 0; i < numpages; i++, page++) { | |
e611939f | 346 | err = __change_page_attr_noflush(page, prot); |
88df6e90 BH |
347 | if (err) |
348 | break; | |
349 | } | |
e611939f | 350 | wmb(); |
7c6a4f3b | 351 | local_irq_restore(flags); |
e611939f CL |
352 | flush_tlb_kernel_range((unsigned long)page_address(start), |
353 | (unsigned long)page_address(page)); | |
88df6e90 BH |
354 | return err; |
355 | } | |
356 | ||
3184cc4b CL |
357 | void mark_initmem_nx(void) |
358 | { | |
359 | struct page *page = virt_to_page(_sinittext); | |
360 | unsigned long numpages = PFN_UP((unsigned long)_einittext) - | |
361 | PFN_DOWN((unsigned long)_sinittext); | |
362 | ||
63b2bc61 CL |
363 | if (v_block_mapped((unsigned long)_stext) + 1) |
364 | mmu_mark_initmem_nx(); | |
365 | else | |
366 | change_page_attr(page, numpages, PAGE_KERNEL); | |
3184cc4b | 367 | } |
88df6e90 | 368 | |
95902e6c CL |
369 | #ifdef CONFIG_STRICT_KERNEL_RWX |
370 | void mark_rodata_ro(void) | |
371 | { | |
372 | struct page *page; | |
373 | unsigned long numpages; | |
374 | ||
63b2bc61 CL |
375 | if (v_block_mapped((unsigned long)_sinittext)) { |
376 | mmu_mark_rodata_ro(); | |
377 | return; | |
378 | } | |
379 | ||
95902e6c CL |
380 | page = virt_to_page(_stext); |
381 | numpages = PFN_UP((unsigned long)_etext) - | |
382 | PFN_DOWN((unsigned long)_stext); | |
383 | ||
384 | change_page_attr(page, numpages, PAGE_KERNEL_ROX); | |
385 | /* | |
386 | * mark .rodata as read only. Use __init_begin rather than __end_rodata | |
387 | * to cover NOTES and EXCEPTION_TABLE. | |
388 | */ | |
389 | page = virt_to_page(__start_rodata); | |
390 | numpages = PFN_UP((unsigned long)__init_begin) - | |
391 | PFN_DOWN((unsigned long)__start_rodata); | |
392 | ||
393 | change_page_attr(page, numpages, PAGE_KERNEL_RO); | |
453d87f6 RC |
394 | |
395 | // mark_initmem_nx() should have already run by now | |
396 | ptdump_check_wx(); | |
95902e6c CL |
397 | } |
398 | #endif | |
399 | ||
3184cc4b | 400 | #ifdef CONFIG_DEBUG_PAGEALLOC |
031bc574 | 401 | void __kernel_map_pages(struct page *page, int numpages, int enable) |
88df6e90 BH |
402 | { |
403 | if (PageHighMem(page)) | |
404 | return; | |
405 | ||
406 | change_page_attr(page, numpages, enable ? PAGE_KERNEL : __pgprot(0)); | |
407 | } | |
408 | #endif /* CONFIG_DEBUG_PAGEALLOC */ |