]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
74588d8b HS |
2 | /* |
3 | * Re-map IO memory to kernel address space so that we can access it. | |
4 | * This is needed for high PCI addresses that aren't mapped in the | |
5 | * 640k-1MB IO memory area on PC's | |
6 | * | |
7 | * (C) Copyright 1995 1996 Linus Torvalds | |
8 | */ | |
74588d8b HS |
9 | #include <linux/vmalloc.h> |
10 | #include <linux/mm.h> | |
e8edc6e0 | 11 | #include <linux/sched.h> |
53fa6645 | 12 | #include <linux/io.h> |
8bc3bcc9 | 13 | #include <linux/export.h> |
74588d8b HS |
14 | #include <asm/cacheflush.h> |
15 | #include <asm/pgtable.h> | |
16 | ||
0ddab1d2 | 17 | #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP |
c2febafc | 18 | static int __read_mostly ioremap_p4d_capable; |
6b637835 TK |
19 | static int __read_mostly ioremap_pud_capable; |
20 | static int __read_mostly ioremap_pmd_capable; | |
21 | static int __read_mostly ioremap_huge_disabled; | |
0ddab1d2 TK |
22 | |
23 | static int __init set_nohugeiomap(char *str) | |
24 | { | |
25 | ioremap_huge_disabled = 1; | |
26 | return 0; | |
27 | } | |
28 | early_param("nohugeiomap", set_nohugeiomap); | |
29 | ||
30 | void __init ioremap_huge_init(void) | |
31 | { | |
32 | if (!ioremap_huge_disabled) { | |
0f472d04 AK |
33 | if (arch_ioremap_p4d_supported()) |
34 | ioremap_p4d_capable = 1; | |
0ddab1d2 TK |
35 | if (arch_ioremap_pud_supported()) |
36 | ioremap_pud_capable = 1; | |
37 | if (arch_ioremap_pmd_supported()) | |
38 | ioremap_pmd_capable = 1; | |
39 | } | |
40 | } | |
41 | ||
c2febafc KS |
42 | static inline int ioremap_p4d_enabled(void) |
43 | { | |
44 | return ioremap_p4d_capable; | |
45 | } | |
46 | ||
0ddab1d2 TK |
47 | static inline int ioremap_pud_enabled(void) |
48 | { | |
49 | return ioremap_pud_capable; | |
50 | } | |
51 | ||
52 | static inline int ioremap_pmd_enabled(void) | |
53 | { | |
54 | return ioremap_pmd_capable; | |
55 | } | |
56 | ||
57 | #else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */ | |
c2febafc | 58 | static inline int ioremap_p4d_enabled(void) { return 0; } |
0ddab1d2 TK |
59 | static inline int ioremap_pud_enabled(void) { return 0; } |
60 | static inline int ioremap_pmd_enabled(void) { return 0; } | |
61 | #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ | |
62 | ||
74588d8b | 63 | static int ioremap_pte_range(pmd_t *pmd, unsigned long addr, |
6c0c7d2b JR |
64 | unsigned long end, phys_addr_t phys_addr, pgprot_t prot, |
65 | pgtbl_mod_mask *mask) | |
74588d8b HS |
66 | { |
67 | pte_t *pte; | |
ffa71f33 | 68 | u64 pfn; |
74588d8b HS |
69 | |
70 | pfn = phys_addr >> PAGE_SHIFT; | |
6c0c7d2b | 71 | pte = pte_alloc_kernel_track(pmd, addr, mask); |
74588d8b HS |
72 | if (!pte) |
73 | return -ENOMEM; | |
74 | do { | |
75 | BUG_ON(!pte_none(*pte)); | |
76 | set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot)); | |
77 | pfn++; | |
78 | } while (pte++, addr += PAGE_SIZE, addr != end); | |
6c0c7d2b | 79 | *mask |= PGTBL_PTE_MODIFIED; |
74588d8b HS |
80 | return 0; |
81 | } | |
82 | ||
d239865a WD |
83 | static int ioremap_try_huge_pmd(pmd_t *pmd, unsigned long addr, |
84 | unsigned long end, phys_addr_t phys_addr, | |
85 | pgprot_t prot) | |
86 | { | |
87 | if (!ioremap_pmd_enabled()) | |
88 | return 0; | |
89 | ||
90 | if ((end - addr) != PMD_SIZE) | |
91 | return 0; | |
92 | ||
6b95ab42 AK |
93 | if (!IS_ALIGNED(addr, PMD_SIZE)) |
94 | return 0; | |
95 | ||
d239865a WD |
96 | if (!IS_ALIGNED(phys_addr, PMD_SIZE)) |
97 | return 0; | |
98 | ||
99 | if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr)) | |
100 | return 0; | |
101 | ||
102 | return pmd_set_huge(pmd, phys_addr, prot); | |
103 | } | |
104 | ||
74588d8b | 105 | static inline int ioremap_pmd_range(pud_t *pud, unsigned long addr, |
6c0c7d2b JR |
106 | unsigned long end, phys_addr_t phys_addr, pgprot_t prot, |
107 | pgtbl_mod_mask *mask) | |
74588d8b HS |
108 | { |
109 | pmd_t *pmd; | |
110 | unsigned long next; | |
111 | ||
6c0c7d2b | 112 | pmd = pmd_alloc_track(&init_mm, pud, addr, mask); |
74588d8b HS |
113 | if (!pmd) |
114 | return -ENOMEM; | |
115 | do { | |
116 | next = pmd_addr_end(addr, end); | |
e61ce6ad | 117 | |
6c0c7d2b JR |
118 | if (ioremap_try_huge_pmd(pmd, addr, next, phys_addr, prot)) { |
119 | *mask |= PGTBL_PMD_MODIFIED; | |
d239865a | 120 | continue; |
6c0c7d2b | 121 | } |
e61ce6ad | 122 | |
6c0c7d2b | 123 | if (ioremap_pte_range(pmd, addr, next, phys_addr, prot, mask)) |
74588d8b | 124 | return -ENOMEM; |
36ddc5a7 | 125 | } while (pmd++, phys_addr += (next - addr), addr = next, addr != end); |
74588d8b HS |
126 | return 0; |
127 | } | |
128 | ||
d239865a WD |
129 | static int ioremap_try_huge_pud(pud_t *pud, unsigned long addr, |
130 | unsigned long end, phys_addr_t phys_addr, | |
131 | pgprot_t prot) | |
132 | { | |
133 | if (!ioremap_pud_enabled()) | |
134 | return 0; | |
135 | ||
136 | if ((end - addr) != PUD_SIZE) | |
137 | return 0; | |
138 | ||
6b95ab42 AK |
139 | if (!IS_ALIGNED(addr, PUD_SIZE)) |
140 | return 0; | |
141 | ||
d239865a WD |
142 | if (!IS_ALIGNED(phys_addr, PUD_SIZE)) |
143 | return 0; | |
144 | ||
145 | if (pud_present(*pud) && !pud_free_pmd_page(pud, addr)) | |
146 | return 0; | |
147 | ||
148 | return pud_set_huge(pud, phys_addr, prot); | |
149 | } | |
150 | ||
c2febafc | 151 | static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr, |
6c0c7d2b JR |
152 | unsigned long end, phys_addr_t phys_addr, pgprot_t prot, |
153 | pgtbl_mod_mask *mask) | |
74588d8b HS |
154 | { |
155 | pud_t *pud; | |
156 | unsigned long next; | |
157 | ||
6c0c7d2b | 158 | pud = pud_alloc_track(&init_mm, p4d, addr, mask); |
74588d8b HS |
159 | if (!pud) |
160 | return -ENOMEM; | |
161 | do { | |
162 | next = pud_addr_end(addr, end); | |
e61ce6ad | 163 | |
6c0c7d2b JR |
164 | if (ioremap_try_huge_pud(pud, addr, next, phys_addr, prot)) { |
165 | *mask |= PGTBL_PUD_MODIFIED; | |
d239865a | 166 | continue; |
6c0c7d2b | 167 | } |
e61ce6ad | 168 | |
6c0c7d2b | 169 | if (ioremap_pmd_range(pud, addr, next, phys_addr, prot, mask)) |
74588d8b | 170 | return -ENOMEM; |
36ddc5a7 | 171 | } while (pud++, phys_addr += (next - addr), addr = next, addr != end); |
74588d8b HS |
172 | return 0; |
173 | } | |
174 | ||
8e2d4340 WD |
175 | static int ioremap_try_huge_p4d(p4d_t *p4d, unsigned long addr, |
176 | unsigned long end, phys_addr_t phys_addr, | |
177 | pgprot_t prot) | |
178 | { | |
179 | if (!ioremap_p4d_enabled()) | |
180 | return 0; | |
181 | ||
182 | if ((end - addr) != P4D_SIZE) | |
183 | return 0; | |
184 | ||
6b95ab42 AK |
185 | if (!IS_ALIGNED(addr, P4D_SIZE)) |
186 | return 0; | |
187 | ||
8e2d4340 WD |
188 | if (!IS_ALIGNED(phys_addr, P4D_SIZE)) |
189 | return 0; | |
190 | ||
191 | if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr)) | |
192 | return 0; | |
193 | ||
194 | return p4d_set_huge(p4d, phys_addr, prot); | |
195 | } | |
196 | ||
c2febafc | 197 | static inline int ioremap_p4d_range(pgd_t *pgd, unsigned long addr, |
6c0c7d2b JR |
198 | unsigned long end, phys_addr_t phys_addr, pgprot_t prot, |
199 | pgtbl_mod_mask *mask) | |
c2febafc KS |
200 | { |
201 | p4d_t *p4d; | |
202 | unsigned long next; | |
203 | ||
6c0c7d2b | 204 | p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); |
c2febafc KS |
205 | if (!p4d) |
206 | return -ENOMEM; | |
207 | do { | |
208 | next = p4d_addr_end(addr, end); | |
209 | ||
6c0c7d2b JR |
210 | if (ioremap_try_huge_p4d(p4d, addr, next, phys_addr, prot)) { |
211 | *mask |= PGTBL_P4D_MODIFIED; | |
8e2d4340 | 212 | continue; |
6c0c7d2b | 213 | } |
c2febafc | 214 | |
6c0c7d2b | 215 | if (ioremap_pud_range(p4d, addr, next, phys_addr, prot, mask)) |
c2febafc | 216 | return -ENOMEM; |
36ddc5a7 | 217 | } while (p4d++, phys_addr += (next - addr), addr = next, addr != end); |
c2febafc KS |
218 | return 0; |
219 | } | |
220 | ||
74588d8b | 221 | int ioremap_page_range(unsigned long addr, |
ffa71f33 | 222 | unsigned long end, phys_addr_t phys_addr, pgprot_t prot) |
74588d8b HS |
223 | { |
224 | pgd_t *pgd; | |
225 | unsigned long start; | |
226 | unsigned long next; | |
227 | int err; | |
6c0c7d2b | 228 | pgtbl_mod_mask mask = 0; |
74588d8b | 229 | |
b39ab98e | 230 | might_sleep(); |
74588d8b HS |
231 | BUG_ON(addr >= end); |
232 | ||
74588d8b | 233 | start = addr; |
74588d8b HS |
234 | pgd = pgd_offset_k(addr); |
235 | do { | |
236 | next = pgd_addr_end(addr, end); | |
6c0c7d2b JR |
237 | err = ioremap_p4d_range(pgd, addr, next, phys_addr, prot, |
238 | &mask); | |
74588d8b HS |
239 | if (err) |
240 | break; | |
36ddc5a7 | 241 | } while (pgd++, phys_addr += (next - addr), addr = next, addr != end); |
74588d8b | 242 | |
db71daab | 243 | flush_cache_vmap(start, end); |
74588d8b | 244 | |
6c0c7d2b JR |
245 | if (mask & ARCH_PAGE_TABLE_SYNC_MASK) |
246 | arch_sync_kernel_mappings(start, end); | |
247 | ||
74588d8b HS |
248 | return err; |
249 | } | |
80b0ca98 CH |
250 | |
251 | #ifdef CONFIG_GENERIC_IOREMAP | |
252 | void __iomem *ioremap_prot(phys_addr_t addr, size_t size, unsigned long prot) | |
253 | { | |
254 | unsigned long offset, vaddr; | |
255 | phys_addr_t last_addr; | |
256 | struct vm_struct *area; | |
257 | ||
258 | /* Disallow wrap-around or zero size */ | |
259 | last_addr = addr + size - 1; | |
260 | if (!size || last_addr < addr) | |
261 | return NULL; | |
262 | ||
263 | /* Page-align mappings */ | |
264 | offset = addr & (~PAGE_MASK); | |
265 | addr -= offset; | |
266 | size = PAGE_ALIGN(size + offset); | |
267 | ||
268 | area = get_vm_area_caller(size, VM_IOREMAP, | |
269 | __builtin_return_address(0)); | |
270 | if (!area) | |
271 | return NULL; | |
272 | vaddr = (unsigned long)area->addr; | |
273 | ||
274 | if (ioremap_page_range(vaddr, vaddr + size, addr, __pgprot(prot))) { | |
275 | free_vm_area(area); | |
276 | return NULL; | |
277 | } | |
278 | ||
279 | return (void __iomem *)(vaddr + offset); | |
280 | } | |
281 | EXPORT_SYMBOL(ioremap_prot); | |
282 | ||
283 | void iounmap(volatile void __iomem *addr) | |
284 | { | |
285 | vunmap((void *)((unsigned long)addr & PAGE_MASK)); | |
286 | } | |
287 | EXPORT_SYMBOL(iounmap); | |
288 | #endif /* CONFIG_GENERIC_IOREMAP */ |