]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
74588d8b HS |
2 | /* |
3 | * Re-map IO memory to kernel address space so that we can access it. | |
4 | * This is needed for high PCI addresses that aren't mapped in the | |
5 | * 640k-1MB IO memory area on PC's | |
6 | * | |
7 | * (C) Copyright 1995 1996 Linus Torvalds | |
8 | */ | |
74588d8b HS |
9 | #include <linux/vmalloc.h> |
10 | #include <linux/mm.h> | |
e8edc6e0 | 11 | #include <linux/sched.h> |
53fa6645 | 12 | #include <linux/io.h> |
8bc3bcc9 | 13 | #include <linux/export.h> |
74588d8b | 14 | #include <asm/cacheflush.h> |
74588d8b | 15 | |
2a681cfa JR |
16 | #include "pgalloc-track.h" |
17 | ||
0ddab1d2 | 18 | #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP |
c2febafc | 19 | static int __read_mostly ioremap_p4d_capable; |
6b637835 TK |
20 | static int __read_mostly ioremap_pud_capable; |
21 | static int __read_mostly ioremap_pmd_capable; | |
22 | static int __read_mostly ioremap_huge_disabled; | |
0ddab1d2 TK |
23 | |
24 | static int __init set_nohugeiomap(char *str) | |
25 | { | |
26 | ioremap_huge_disabled = 1; | |
27 | return 0; | |
28 | } | |
29 | early_param("nohugeiomap", set_nohugeiomap); | |
30 | ||
31 | void __init ioremap_huge_init(void) | |
32 | { | |
33 | if (!ioremap_huge_disabled) { | |
0f472d04 AK |
34 | if (arch_ioremap_p4d_supported()) |
35 | ioremap_p4d_capable = 1; | |
0ddab1d2 TK |
36 | if (arch_ioremap_pud_supported()) |
37 | ioremap_pud_capable = 1; | |
38 | if (arch_ioremap_pmd_supported()) | |
39 | ioremap_pmd_capable = 1; | |
40 | } | |
41 | } | |
42 | ||
c2febafc KS |
43 | static inline int ioremap_p4d_enabled(void) |
44 | { | |
45 | return ioremap_p4d_capable; | |
46 | } | |
47 | ||
0ddab1d2 TK |
48 | static inline int ioremap_pud_enabled(void) |
49 | { | |
50 | return ioremap_pud_capable; | |
51 | } | |
52 | ||
53 | static inline int ioremap_pmd_enabled(void) | |
54 | { | |
55 | return ioremap_pmd_capable; | |
56 | } | |
57 | ||
58 | #else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */ | |
c2febafc | 59 | static inline int ioremap_p4d_enabled(void) { return 0; } |
0ddab1d2 TK |
60 | static inline int ioremap_pud_enabled(void) { return 0; } |
61 | static inline int ioremap_pmd_enabled(void) { return 0; } | |
62 | #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ | |
63 | ||
74588d8b | 64 | static int ioremap_pte_range(pmd_t *pmd, unsigned long addr, |
6c0c7d2b JR |
65 | unsigned long end, phys_addr_t phys_addr, pgprot_t prot, |
66 | pgtbl_mod_mask *mask) | |
74588d8b HS |
67 | { |
68 | pte_t *pte; | |
ffa71f33 | 69 | u64 pfn; |
74588d8b HS |
70 | |
71 | pfn = phys_addr >> PAGE_SHIFT; | |
6c0c7d2b | 72 | pte = pte_alloc_kernel_track(pmd, addr, mask); |
74588d8b HS |
73 | if (!pte) |
74 | return -ENOMEM; | |
75 | do { | |
76 | BUG_ON(!pte_none(*pte)); | |
77 | set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot)); | |
78 | pfn++; | |
79 | } while (pte++, addr += PAGE_SIZE, addr != end); | |
6c0c7d2b | 80 | *mask |= PGTBL_PTE_MODIFIED; |
74588d8b HS |
81 | return 0; |
82 | } | |
83 | ||
d239865a WD |
84 | static int ioremap_try_huge_pmd(pmd_t *pmd, unsigned long addr, |
85 | unsigned long end, phys_addr_t phys_addr, | |
86 | pgprot_t prot) | |
87 | { | |
88 | if (!ioremap_pmd_enabled()) | |
89 | return 0; | |
90 | ||
91 | if ((end - addr) != PMD_SIZE) | |
92 | return 0; | |
93 | ||
6b95ab42 AK |
94 | if (!IS_ALIGNED(addr, PMD_SIZE)) |
95 | return 0; | |
96 | ||
d239865a WD |
97 | if (!IS_ALIGNED(phys_addr, PMD_SIZE)) |
98 | return 0; | |
99 | ||
100 | if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr)) | |
101 | return 0; | |
102 | ||
103 | return pmd_set_huge(pmd, phys_addr, prot); | |
104 | } | |
105 | ||
74588d8b | 106 | static inline int ioremap_pmd_range(pud_t *pud, unsigned long addr, |
6c0c7d2b JR |
107 | unsigned long end, phys_addr_t phys_addr, pgprot_t prot, |
108 | pgtbl_mod_mask *mask) | |
74588d8b HS |
109 | { |
110 | pmd_t *pmd; | |
111 | unsigned long next; | |
112 | ||
6c0c7d2b | 113 | pmd = pmd_alloc_track(&init_mm, pud, addr, mask); |
74588d8b HS |
114 | if (!pmd) |
115 | return -ENOMEM; | |
116 | do { | |
117 | next = pmd_addr_end(addr, end); | |
e61ce6ad | 118 | |
6c0c7d2b JR |
119 | if (ioremap_try_huge_pmd(pmd, addr, next, phys_addr, prot)) { |
120 | *mask |= PGTBL_PMD_MODIFIED; | |
d239865a | 121 | continue; |
6c0c7d2b | 122 | } |
e61ce6ad | 123 | |
6c0c7d2b | 124 | if (ioremap_pte_range(pmd, addr, next, phys_addr, prot, mask)) |
74588d8b | 125 | return -ENOMEM; |
36ddc5a7 | 126 | } while (pmd++, phys_addr += (next - addr), addr = next, addr != end); |
74588d8b HS |
127 | return 0; |
128 | } | |
129 | ||
d239865a WD |
130 | static int ioremap_try_huge_pud(pud_t *pud, unsigned long addr, |
131 | unsigned long end, phys_addr_t phys_addr, | |
132 | pgprot_t prot) | |
133 | { | |
134 | if (!ioremap_pud_enabled()) | |
135 | return 0; | |
136 | ||
137 | if ((end - addr) != PUD_SIZE) | |
138 | return 0; | |
139 | ||
6b95ab42 AK |
140 | if (!IS_ALIGNED(addr, PUD_SIZE)) |
141 | return 0; | |
142 | ||
d239865a WD |
143 | if (!IS_ALIGNED(phys_addr, PUD_SIZE)) |
144 | return 0; | |
145 | ||
146 | if (pud_present(*pud) && !pud_free_pmd_page(pud, addr)) | |
147 | return 0; | |
148 | ||
149 | return pud_set_huge(pud, phys_addr, prot); | |
150 | } | |
151 | ||
c2febafc | 152 | static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr, |
6c0c7d2b JR |
153 | unsigned long end, phys_addr_t phys_addr, pgprot_t prot, |
154 | pgtbl_mod_mask *mask) | |
74588d8b HS |
155 | { |
156 | pud_t *pud; | |
157 | unsigned long next; | |
158 | ||
6c0c7d2b | 159 | pud = pud_alloc_track(&init_mm, p4d, addr, mask); |
74588d8b HS |
160 | if (!pud) |
161 | return -ENOMEM; | |
162 | do { | |
163 | next = pud_addr_end(addr, end); | |
e61ce6ad | 164 | |
6c0c7d2b JR |
165 | if (ioremap_try_huge_pud(pud, addr, next, phys_addr, prot)) { |
166 | *mask |= PGTBL_PUD_MODIFIED; | |
d239865a | 167 | continue; |
6c0c7d2b | 168 | } |
e61ce6ad | 169 | |
6c0c7d2b | 170 | if (ioremap_pmd_range(pud, addr, next, phys_addr, prot, mask)) |
74588d8b | 171 | return -ENOMEM; |
36ddc5a7 | 172 | } while (pud++, phys_addr += (next - addr), addr = next, addr != end); |
74588d8b HS |
173 | return 0; |
174 | } | |
175 | ||
8e2d4340 WD |
176 | static int ioremap_try_huge_p4d(p4d_t *p4d, unsigned long addr, |
177 | unsigned long end, phys_addr_t phys_addr, | |
178 | pgprot_t prot) | |
179 | { | |
180 | if (!ioremap_p4d_enabled()) | |
181 | return 0; | |
182 | ||
183 | if ((end - addr) != P4D_SIZE) | |
184 | return 0; | |
185 | ||
6b95ab42 AK |
186 | if (!IS_ALIGNED(addr, P4D_SIZE)) |
187 | return 0; | |
188 | ||
8e2d4340 WD |
189 | if (!IS_ALIGNED(phys_addr, P4D_SIZE)) |
190 | return 0; | |
191 | ||
192 | if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr)) | |
193 | return 0; | |
194 | ||
195 | return p4d_set_huge(p4d, phys_addr, prot); | |
196 | } | |
197 | ||
c2febafc | 198 | static inline int ioremap_p4d_range(pgd_t *pgd, unsigned long addr, |
6c0c7d2b JR |
199 | unsigned long end, phys_addr_t phys_addr, pgprot_t prot, |
200 | pgtbl_mod_mask *mask) | |
c2febafc KS |
201 | { |
202 | p4d_t *p4d; | |
203 | unsigned long next; | |
204 | ||
6c0c7d2b | 205 | p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); |
c2febafc KS |
206 | if (!p4d) |
207 | return -ENOMEM; | |
208 | do { | |
209 | next = p4d_addr_end(addr, end); | |
210 | ||
6c0c7d2b JR |
211 | if (ioremap_try_huge_p4d(p4d, addr, next, phys_addr, prot)) { |
212 | *mask |= PGTBL_P4D_MODIFIED; | |
8e2d4340 | 213 | continue; |
6c0c7d2b | 214 | } |
c2febafc | 215 | |
6c0c7d2b | 216 | if (ioremap_pud_range(p4d, addr, next, phys_addr, prot, mask)) |
c2febafc | 217 | return -ENOMEM; |
36ddc5a7 | 218 | } while (p4d++, phys_addr += (next - addr), addr = next, addr != end); |
c2febafc KS |
219 | return 0; |
220 | } | |
221 | ||
74588d8b | 222 | int ioremap_page_range(unsigned long addr, |
ffa71f33 | 223 | unsigned long end, phys_addr_t phys_addr, pgprot_t prot) |
74588d8b HS |
224 | { |
225 | pgd_t *pgd; | |
226 | unsigned long start; | |
227 | unsigned long next; | |
228 | int err; | |
6c0c7d2b | 229 | pgtbl_mod_mask mask = 0; |
74588d8b | 230 | |
b39ab98e | 231 | might_sleep(); |
74588d8b HS |
232 | BUG_ON(addr >= end); |
233 | ||
74588d8b | 234 | start = addr; |
74588d8b HS |
235 | pgd = pgd_offset_k(addr); |
236 | do { | |
237 | next = pgd_addr_end(addr, end); | |
6c0c7d2b JR |
238 | err = ioremap_p4d_range(pgd, addr, next, phys_addr, prot, |
239 | &mask); | |
74588d8b HS |
240 | if (err) |
241 | break; | |
36ddc5a7 | 242 | } while (pgd++, phys_addr += (next - addr), addr = next, addr != end); |
74588d8b | 243 | |
db71daab | 244 | flush_cache_vmap(start, end); |
74588d8b | 245 | |
6c0c7d2b JR |
246 | if (mask & ARCH_PAGE_TABLE_SYNC_MASK) |
247 | arch_sync_kernel_mappings(start, end); | |
248 | ||
74588d8b HS |
249 | return err; |
250 | } | |
80b0ca98 CH |
251 | |
252 | #ifdef CONFIG_GENERIC_IOREMAP | |
253 | void __iomem *ioremap_prot(phys_addr_t addr, size_t size, unsigned long prot) | |
254 | { | |
255 | unsigned long offset, vaddr; | |
256 | phys_addr_t last_addr; | |
257 | struct vm_struct *area; | |
258 | ||
259 | /* Disallow wrap-around or zero size */ | |
260 | last_addr = addr + size - 1; | |
261 | if (!size || last_addr < addr) | |
262 | return NULL; | |
263 | ||
264 | /* Page-align mappings */ | |
265 | offset = addr & (~PAGE_MASK); | |
266 | addr -= offset; | |
267 | size = PAGE_ALIGN(size + offset); | |
268 | ||
269 | area = get_vm_area_caller(size, VM_IOREMAP, | |
270 | __builtin_return_address(0)); | |
271 | if (!area) | |
272 | return NULL; | |
273 | vaddr = (unsigned long)area->addr; | |
274 | ||
275 | if (ioremap_page_range(vaddr, vaddr + size, addr, __pgprot(prot))) { | |
276 | free_vm_area(area); | |
277 | return NULL; | |
278 | } | |
279 | ||
280 | return (void __iomem *)(vaddr + offset); | |
281 | } | |
282 | EXPORT_SYMBOL(ioremap_prot); | |
283 | ||
284 | void iounmap(volatile void __iomem *addr) | |
285 | { | |
286 | vunmap((void *)((unsigned long)addr & PAGE_MASK)); | |
287 | } | |
288 | EXPORT_SYMBOL(iounmap); | |
289 | #endif /* CONFIG_GENERIC_IOREMAP */ |