]>
Commit | Line | Data |
---|---|---|
d2912cb1 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
1da177e4 | 2 | /* |
0c668984 | 3 | * linux/arch/arm/mm/pgd.c |
1da177e4 | 4 | * |
90072059 | 5 | * Copyright (C) 1998-2005 Russell King |
1da177e4 | 6 | */ |
1da177e4 | 7 | #include <linux/mm.h> |
5a0e3ad6 | 8 | #include <linux/gfp.h> |
1da177e4 | 9 | #include <linux/highmem.h> |
da028779 | 10 | #include <linux/slab.h> |
1da177e4 | 11 | |
15d07dc9 | 12 | #include <asm/cp15.h> |
1da177e4 LT |
13 | #include <asm/pgalloc.h> |
14 | #include <asm/page.h> | |
1da177e4 LT |
15 | #include <asm/tlbflush.h> |
16 | ||
1b2e2b73 RK |
17 | #include "mm.h" |
18 | ||
da028779 | 19 | #ifdef CONFIG_ARM_LPAE |
6da2ec56 | 20 | #define __pgd_alloc() kmalloc_array(PTRS_PER_PGD, sizeof(pgd_t), GFP_KERNEL) |
da028779 CM |
21 | #define __pgd_free(pgd) kfree(pgd) |
22 | #else | |
397b080b | 23 | #define __pgd_alloc() (pgd_t *)__get_free_pages(GFP_KERNEL, 2) |
da028779 CM |
24 | #define __pgd_free(pgd) free_pages((unsigned long)pgd, 2) |
25 | #endif | |
26 | ||
1da177e4 LT |
27 | /* |
28 | * need to get a 16k page for level 1 | |
29 | */ | |
b0d03745 | 30 | pgd_t *pgd_alloc(struct mm_struct *mm) |
1da177e4 LT |
31 | { |
32 | pgd_t *new_pgd, *init_pgd; | |
516295e5 | 33 | pud_t *new_pud, *init_pud; |
1da177e4 LT |
34 | pmd_t *new_pmd, *init_pmd; |
35 | pte_t *new_pte, *init_pte; | |
36 | ||
da028779 | 37 | new_pgd = __pgd_alloc(); |
1da177e4 LT |
38 | if (!new_pgd) |
39 | goto no_pgd; | |
40 | ||
e926f449 | 41 | memset(new_pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); |
1da177e4 | 42 | |
a343e607 RK |
43 | /* |
44 | * Copy over the kernel and IO PGD entries | |
45 | */ | |
1da177e4 | 46 | init_pgd = pgd_offset_k(0); |
e926f449 RK |
47 | memcpy(new_pgd + USER_PTRS_PER_PGD, init_pgd + USER_PTRS_PER_PGD, |
48 | (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); | |
a343e607 RK |
49 | |
50 | clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t)); | |
1da177e4 | 51 | |
da028779 CM |
52 | #ifdef CONFIG_ARM_LPAE |
53 | /* | |
54 | * Allocate PMD table for modules and pkmap mappings. | |
55 | */ | |
56 | new_pud = pud_alloc(mm, new_pgd + pgd_index(MODULES_VADDR), | |
57 | MODULES_VADDR); | |
58 | if (!new_pud) | |
59 | goto no_pud; | |
60 | ||
61 | new_pmd = pmd_alloc(mm, new_pud, 0); | |
62 | if (!new_pmd) | |
63 | goto no_pmd; | |
64 | #endif | |
65 | ||
1da177e4 | 66 | if (!vectors_high()) { |
1da177e4 LT |
67 | /* |
68 | * On ARM, first page must always be allocated since it | |
da028779 CM |
69 | * contains the machine vectors. The vectors are always high |
70 | * with LPAE. | |
1da177e4 | 71 | */ |
516295e5 RK |
72 | new_pud = pud_alloc(mm, new_pgd, 0); |
73 | if (!new_pud) | |
74 | goto no_pud; | |
75 | ||
76 | new_pmd = pmd_alloc(mm, new_pud, 0); | |
1da177e4 LT |
77 | if (!new_pmd) |
78 | goto no_pmd; | |
79 | ||
3ed3a4f0 | 80 | new_pte = pte_alloc_map(mm, new_pmd, 0); |
1da177e4 LT |
81 | if (!new_pte) |
82 | goto no_pte; | |
83 | ||
a02d8dfd RK |
84 | #ifndef CONFIG_ARM_LPAE |
85 | /* | |
86 | * Modify the PTE pointer to have the correct domain. This | |
87 | * needs to be the vectors domain to avoid the low vectors | |
88 | * being unmapped. | |
89 | */ | |
90 | pmd_val(*new_pmd) &= ~PMD_DOMAIN_MASK; | |
91 | pmd_val(*new_pmd) |= PMD_DOMAIN(DOMAIN_VECTORS); | |
92 | #endif | |
93 | ||
516295e5 RK |
94 | init_pud = pud_offset(init_pgd, 0); |
95 | init_pmd = pmd_offset(init_pud, 0); | |
ece0e2b6 | 96 | init_pte = pte_offset_map(init_pmd, 0); |
d8aa712c RK |
97 | set_pte_ext(new_pte + 0, init_pte[0], 0); |
98 | set_pte_ext(new_pte + 1, init_pte[1], 0); | |
ece0e2b6 | 99 | pte_unmap(init_pte); |
1da177e4 | 100 | pte_unmap(new_pte); |
1da177e4 LT |
101 | } |
102 | ||
1da177e4 LT |
103 | return new_pgd; |
104 | ||
105 | no_pte: | |
5e541973 | 106 | pmd_free(mm, new_pmd); |
b30fe6c7 | 107 | mm_dec_nr_pmds(mm); |
1da177e4 | 108 | no_pmd: |
516295e5 RK |
109 | pud_free(mm, new_pud); |
110 | no_pud: | |
da028779 | 111 | __pgd_free(new_pgd); |
1da177e4 LT |
112 | no_pgd: |
113 | return NULL; | |
114 | } | |
115 | ||
6e4beb5e | 116 | void pgd_free(struct mm_struct *mm, pgd_t *pgd_base) |
1da177e4 | 117 | { |
6e4beb5e | 118 | pgd_t *pgd; |
516295e5 | 119 | pud_t *pud; |
1da177e4 | 120 | pmd_t *pmd; |
0c82d83c | 121 | pgtable_t pte; |
1da177e4 | 122 | |
6e4beb5e | 123 | if (!pgd_base) |
1da177e4 LT |
124 | return; |
125 | ||
6e4beb5e RK |
126 | pgd = pgd_base + pgd_index(0); |
127 | if (pgd_none_or_clear_bad(pgd)) | |
128 | goto no_pgd; | |
129 | ||
516295e5 RK |
130 | pud = pud_offset(pgd, 0); |
131 | if (pud_none_or_clear_bad(pud)) | |
132 | goto no_pud; | |
133 | ||
134 | pmd = pmd_offset(pud, 0); | |
6e4beb5e RK |
135 | if (pmd_none_or_clear_bad(pmd)) |
136 | goto no_pmd; | |
1da177e4 | 137 | |
0c82d83c | 138 | pte = pmd_pgtable(*pmd); |
1da177e4 | 139 | pmd_clear(pmd); |
5e541973 | 140 | pte_free(mm, pte); |
c4812909 | 141 | mm_dec_nr_ptes(mm); |
6e4beb5e | 142 | no_pmd: |
516295e5 | 143 | pud_clear(pud); |
5e541973 | 144 | pmd_free(mm, pmd); |
b30fe6c7 | 145 | mm_dec_nr_pmds(mm); |
516295e5 RK |
146 | no_pud: |
147 | pgd_clear(pgd); | |
148 | pud_free(mm, pud); | |
6e4beb5e | 149 | no_pgd: |
da028779 CM |
150 | #ifdef CONFIG_ARM_LPAE |
151 | /* | |
152 | * Free modules/pkmap or identity pmd tables. | |
153 | */ | |
154 | for (pgd = pgd_base; pgd < pgd_base + PTRS_PER_PGD; pgd++) { | |
155 | if (pgd_none_or_clear_bad(pgd)) | |
156 | continue; | |
157 | if (pgd_val(*pgd) & L_PGD_SWAPPER) | |
158 | continue; | |
159 | pud = pud_offset(pgd, 0); | |
160 | if (pud_none_or_clear_bad(pud)) | |
161 | continue; | |
162 | pmd = pmd_offset(pud, 0); | |
163 | pud_clear(pud); | |
164 | pmd_free(mm, pmd); | |
b30fe6c7 | 165 | mm_dec_nr_pmds(mm); |
da028779 CM |
166 | pgd_clear(pgd); |
167 | pud_free(mm, pud); | |
168 | } | |
169 | #endif | |
170 | __pgd_free(pgd_base); | |
1da177e4 | 171 | } |