]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
9e9a367c | 2 | #include <linux/module.h> |
614dd058 | 3 | #include <linux/kernel.h> |
9e9a367c | 4 | #include <linux/slab.h> |
589ee628 | 5 | #include <linux/mm_types.h> |
614dd058 RK |
6 | |
7 | #include <asm/cputype.h> | |
8903826d | 8 | #include <asm/idmap.h> |
614dd058 RK |
9 | #include <asm/pgalloc.h> |
10 | #include <asm/pgtable.h> | |
8903826d | 11 | #include <asm/sections.h> |
9f97da78 | 12 | #include <asm/system_info.h> |
8903826d | 13 | |
c5cc87fa RK |
14 | /* |
15 | * Note: accesses outside of the kernel image and the identity map area | |
16 | * are not supported on any CPU using the idmap tables as its current | |
17 | * page tables. | |
18 | */ | |
8903826d | 19 | pgd_t *idmap_pgd; |
981b6714 | 20 | long long arch_phys_to_idmap_offset; |
614dd058 | 21 | |
ae2de101 CM |
22 | #ifdef CONFIG_ARM_LPAE |
23 | static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end, | |
24 | unsigned long prot) | |
25 | { | |
26 | pmd_t *pmd; | |
27 | unsigned long next; | |
28 | ||
29 | if (pud_none_or_clear_bad(pud) || (pud_val(*pud) & L_PGD_SWAPPER)) { | |
30 | pmd = pmd_alloc_one(&init_mm, addr); | |
31 | if (!pmd) { | |
8b521cb2 | 32 | pr_warn("Failed to allocate identity pmd.\n"); |
ae2de101 CM |
33 | return; |
34 | } | |
811a2407 KK |
35 | /* |
36 | * Copy the original PMD to ensure that the PMD entries for | |
37 | * the kernel image are preserved. | |
38 | */ | |
39 | if (!pud_none(*pud)) | |
40 | memcpy(pmd, pmd_offset(pud, 0), | |
41 | PTRS_PER_PMD * sizeof(pmd_t)); | |
ae2de101 CM |
42 | pud_populate(&init_mm, pud, pmd); |
43 | pmd += pmd_index(addr); | |
44 | } else | |
45 | pmd = pmd_offset(pud, addr); | |
46 | ||
47 | do { | |
48 | next = pmd_addr_end(addr, end); | |
49 | *pmd = __pmd((addr & PMD_MASK) | prot); | |
50 | flush_pmd_entry(pmd); | |
51 | } while (pmd++, addr = next, addr != end); | |
52 | } | |
53 | #else /* !CONFIG_ARM_LPAE */ | |
516295e5 | 54 | static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end, |
af3813d6 RK |
55 | unsigned long prot) |
56 | { | |
516295e5 | 57 | pmd_t *pmd = pmd_offset(pud, addr); |
af3813d6 RK |
58 | |
59 | addr = (addr & PMD_MASK) | prot; | |
60 | pmd[0] = __pmd(addr); | |
61 | addr += SECTION_SIZE; | |
62 | pmd[1] = __pmd(addr); | |
63 | flush_pmd_entry(pmd); | |
64 | } | |
ae2de101 | 65 | #endif /* CONFIG_ARM_LPAE */ |
af3813d6 | 66 | |
516295e5 RK |
67 | static void idmap_add_pud(pgd_t *pgd, unsigned long addr, unsigned long end, |
68 | unsigned long prot) | |
69 | { | |
70 | pud_t *pud = pud_offset(pgd, addr); | |
71 | unsigned long next; | |
72 | ||
73 | do { | |
74 | next = pud_addr_end(addr, end); | |
75 | idmap_add_pmd(pud, addr, next, prot); | |
76 | } while (pud++, addr = next, addr != end); | |
77 | } | |
78 | ||
9e9a367c CD |
79 | static void identity_mapping_add(pgd_t *pgd, const char *text_start, |
80 | const char *text_end, unsigned long prot) | |
614dd058 | 81 | { |
9e9a367c CD |
82 | unsigned long addr, end; |
83 | unsigned long next; | |
84 | ||
4dc9a817 SS |
85 | addr = virt_to_idmap(text_start); |
86 | end = virt_to_idmap(text_end); | |
c1a5f4f6 | 87 | pr_info("Setting up static identity map for 0x%lx - 0x%lx\n", addr, end); |
9e9a367c CD |
88 | |
89 | prot |= PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AF; | |
614dd058 | 90 | |
d33c43ac | 91 | if (cpu_architecture() <= CPU_ARCH_ARMv5TEJ && !cpu_is_xscale_family()) |
614dd058 RK |
92 | prot |= PMD_BIT4; |
93 | ||
af3813d6 RK |
94 | pgd += pgd_index(addr); |
95 | do { | |
96 | next = pgd_addr_end(addr, end); | |
516295e5 | 97 | idmap_add_pud(pgd, addr, next, prot); |
af3813d6 | 98 | } while (pgd++, addr = next, addr != end); |
614dd058 RK |
99 | } |
100 | ||
8903826d WD |
101 | extern char __idmap_text_start[], __idmap_text_end[]; |
102 | ||
103 | static int __init init_static_idmap(void) | |
104 | { | |
8903826d WD |
105 | idmap_pgd = pgd_alloc(&init_mm); |
106 | if (!idmap_pgd) | |
107 | return -ENOMEM; | |
108 | ||
9e9a367c CD |
109 | identity_mapping_add(idmap_pgd, __idmap_text_start, |
110 | __idmap_text_end, 0); | |
8903826d | 111 | |
e4067855 NP |
112 | /* Flush L1 for the hardware to see this page table content */ |
113 | flush_cache_louis(); | |
114 | ||
2fb41059 | 115 | return 0; |
8903826d | 116 | } |
4e8ee7de | 117 | early_initcall(init_static_idmap); |
8903826d | 118 | |
614dd058 | 119 | /* |
2c8951ab WD |
120 | * In order to soft-boot, we need to switch to a 1:1 mapping for the |
121 | * cpu_reset functions. This will then ensure that we have predictable | |
122 | * results when turning off the mmu. | |
614dd058 | 123 | */ |
5aafec15 | 124 | void setup_mm_for_reboot(void) |
614dd058 | 125 | { |
2c8951ab WD |
126 | /* Switch to the identity mapping. */ |
127 | cpu_switch_mm(idmap_pgd, &init_mm); | |
89c7e4b8 | 128 | local_flush_bp_all(); |
2c8951ab | 129 | |
e4067855 NP |
130 | #ifdef CONFIG_CPU_HAS_ASID |
131 | /* | |
132 | * We don't have a clean ASID for the identity mapping, which | |
133 | * may clash with virtual addresses of the previous page tables | |
134 | * and therefore potentially in the TLB. | |
135 | */ | |
614dd058 | 136 | local_flush_tlb_all(); |
e4067855 | 137 | #endif |
614dd058 | 138 | } |