]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
74588d8b HS |
2 | /* |
3 | * Re-map IO memory to kernel address space so that we can access it. | |
4 | * This is needed for high PCI addresses that aren't mapped in the | |
5 | * 640k-1MB IO memory area on PC's | |
6 | * | |
7 | * (C) Copyright 1995 1996 Linus Torvalds | |
8 | */ | |
74588d8b HS |
9 | #include <linux/vmalloc.h> |
10 | #include <linux/mm.h> | |
e8edc6e0 | 11 | #include <linux/sched.h> |
53fa6645 | 12 | #include <linux/io.h> |
8bc3bcc9 | 13 | #include <linux/export.h> |
74588d8b | 14 | #include <asm/cacheflush.h> |
74588d8b | 15 | |
0ddab1d2 | 16 | #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP |
c2febafc | 17 | static int __read_mostly ioremap_p4d_capable; |
6b637835 TK |
18 | static int __read_mostly ioremap_pud_capable; |
19 | static int __read_mostly ioremap_pmd_capable; | |
20 | static int __read_mostly ioremap_huge_disabled; | |
0ddab1d2 TK |
21 | |
22 | static int __init set_nohugeiomap(char *str) | |
23 | { | |
24 | ioremap_huge_disabled = 1; | |
25 | return 0; | |
26 | } | |
27 | early_param("nohugeiomap", set_nohugeiomap); | |
28 | ||
29 | void __init ioremap_huge_init(void) | |
30 | { | |
31 | if (!ioremap_huge_disabled) { | |
0f472d04 AK |
32 | if (arch_ioremap_p4d_supported()) |
33 | ioremap_p4d_capable = 1; | |
0ddab1d2 TK |
34 | if (arch_ioremap_pud_supported()) |
35 | ioremap_pud_capable = 1; | |
36 | if (arch_ioremap_pmd_supported()) | |
37 | ioremap_pmd_capable = 1; | |
38 | } | |
39 | } | |
40 | ||
c2febafc KS |
41 | static inline int ioremap_p4d_enabled(void) |
42 | { | |
43 | return ioremap_p4d_capable; | |
44 | } | |
45 | ||
0ddab1d2 TK |
46 | static inline int ioremap_pud_enabled(void) |
47 | { | |
48 | return ioremap_pud_capable; | |
49 | } | |
50 | ||
51 | static inline int ioremap_pmd_enabled(void) | |
52 | { | |
53 | return ioremap_pmd_capable; | |
54 | } | |
55 | ||
56 | #else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */ | |
c2febafc | 57 | static inline int ioremap_p4d_enabled(void) { return 0; } |
0ddab1d2 TK |
58 | static inline int ioremap_pud_enabled(void) { return 0; } |
59 | static inline int ioremap_pmd_enabled(void) { return 0; } | |
60 | #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ | |
61 | ||
74588d8b | 62 | static int ioremap_pte_range(pmd_t *pmd, unsigned long addr, |
6c0c7d2b JR |
63 | unsigned long end, phys_addr_t phys_addr, pgprot_t prot, |
64 | pgtbl_mod_mask *mask) | |
74588d8b HS |
65 | { |
66 | pte_t *pte; | |
ffa71f33 | 67 | u64 pfn; |
74588d8b HS |
68 | |
69 | pfn = phys_addr >> PAGE_SHIFT; | |
6c0c7d2b | 70 | pte = pte_alloc_kernel_track(pmd, addr, mask); |
74588d8b HS |
71 | if (!pte) |
72 | return -ENOMEM; | |
73 | do { | |
74 | BUG_ON(!pte_none(*pte)); | |
75 | set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot)); | |
76 | pfn++; | |
77 | } while (pte++, addr += PAGE_SIZE, addr != end); | |
6c0c7d2b | 78 | *mask |= PGTBL_PTE_MODIFIED; |
74588d8b HS |
79 | return 0; |
80 | } | |
81 | ||
d239865a WD |
82 | static int ioremap_try_huge_pmd(pmd_t *pmd, unsigned long addr, |
83 | unsigned long end, phys_addr_t phys_addr, | |
84 | pgprot_t prot) | |
85 | { | |
86 | if (!ioremap_pmd_enabled()) | |
87 | return 0; | |
88 | ||
89 | if ((end - addr) != PMD_SIZE) | |
90 | return 0; | |
91 | ||
6b95ab42 AK |
92 | if (!IS_ALIGNED(addr, PMD_SIZE)) |
93 | return 0; | |
94 | ||
d239865a WD |
95 | if (!IS_ALIGNED(phys_addr, PMD_SIZE)) |
96 | return 0; | |
97 | ||
98 | if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr)) | |
99 | return 0; | |
100 | ||
101 | return pmd_set_huge(pmd, phys_addr, prot); | |
102 | } | |
103 | ||
74588d8b | 104 | static inline int ioremap_pmd_range(pud_t *pud, unsigned long addr, |
6c0c7d2b JR |
105 | unsigned long end, phys_addr_t phys_addr, pgprot_t prot, |
106 | pgtbl_mod_mask *mask) | |
74588d8b HS |
107 | { |
108 | pmd_t *pmd; | |
109 | unsigned long next; | |
110 | ||
6c0c7d2b | 111 | pmd = pmd_alloc_track(&init_mm, pud, addr, mask); |
74588d8b HS |
112 | if (!pmd) |
113 | return -ENOMEM; | |
114 | do { | |
115 | next = pmd_addr_end(addr, end); | |
e61ce6ad | 116 | |
6c0c7d2b JR |
117 | if (ioremap_try_huge_pmd(pmd, addr, next, phys_addr, prot)) { |
118 | *mask |= PGTBL_PMD_MODIFIED; | |
d239865a | 119 | continue; |
6c0c7d2b | 120 | } |
e61ce6ad | 121 | |
6c0c7d2b | 122 | if (ioremap_pte_range(pmd, addr, next, phys_addr, prot, mask)) |
74588d8b | 123 | return -ENOMEM; |
36ddc5a7 | 124 | } while (pmd++, phys_addr += (next - addr), addr = next, addr != end); |
74588d8b HS |
125 | return 0; |
126 | } | |
127 | ||
d239865a WD |
128 | static int ioremap_try_huge_pud(pud_t *pud, unsigned long addr, |
129 | unsigned long end, phys_addr_t phys_addr, | |
130 | pgprot_t prot) | |
131 | { | |
132 | if (!ioremap_pud_enabled()) | |
133 | return 0; | |
134 | ||
135 | if ((end - addr) != PUD_SIZE) | |
136 | return 0; | |
137 | ||
6b95ab42 AK |
138 | if (!IS_ALIGNED(addr, PUD_SIZE)) |
139 | return 0; | |
140 | ||
d239865a WD |
141 | if (!IS_ALIGNED(phys_addr, PUD_SIZE)) |
142 | return 0; | |
143 | ||
144 | if (pud_present(*pud) && !pud_free_pmd_page(pud, addr)) | |
145 | return 0; | |
146 | ||
147 | return pud_set_huge(pud, phys_addr, prot); | |
148 | } | |
149 | ||
c2febafc | 150 | static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr, |
6c0c7d2b JR |
151 | unsigned long end, phys_addr_t phys_addr, pgprot_t prot, |
152 | pgtbl_mod_mask *mask) | |
74588d8b HS |
153 | { |
154 | pud_t *pud; | |
155 | unsigned long next; | |
156 | ||
6c0c7d2b | 157 | pud = pud_alloc_track(&init_mm, p4d, addr, mask); |
74588d8b HS |
158 | if (!pud) |
159 | return -ENOMEM; | |
160 | do { | |
161 | next = pud_addr_end(addr, end); | |
e61ce6ad | 162 | |
6c0c7d2b JR |
163 | if (ioremap_try_huge_pud(pud, addr, next, phys_addr, prot)) { |
164 | *mask |= PGTBL_PUD_MODIFIED; | |
d239865a | 165 | continue; |
6c0c7d2b | 166 | } |
e61ce6ad | 167 | |
6c0c7d2b | 168 | if (ioremap_pmd_range(pud, addr, next, phys_addr, prot, mask)) |
74588d8b | 169 | return -ENOMEM; |
36ddc5a7 | 170 | } while (pud++, phys_addr += (next - addr), addr = next, addr != end); |
74588d8b HS |
171 | return 0; |
172 | } | |
173 | ||
8e2d4340 WD |
174 | static int ioremap_try_huge_p4d(p4d_t *p4d, unsigned long addr, |
175 | unsigned long end, phys_addr_t phys_addr, | |
176 | pgprot_t prot) | |
177 | { | |
178 | if (!ioremap_p4d_enabled()) | |
179 | return 0; | |
180 | ||
181 | if ((end - addr) != P4D_SIZE) | |
182 | return 0; | |
183 | ||
6b95ab42 AK |
184 | if (!IS_ALIGNED(addr, P4D_SIZE)) |
185 | return 0; | |
186 | ||
8e2d4340 WD |
187 | if (!IS_ALIGNED(phys_addr, P4D_SIZE)) |
188 | return 0; | |
189 | ||
190 | if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr)) | |
191 | return 0; | |
192 | ||
193 | return p4d_set_huge(p4d, phys_addr, prot); | |
194 | } | |
195 | ||
c2febafc | 196 | static inline int ioremap_p4d_range(pgd_t *pgd, unsigned long addr, |
6c0c7d2b JR |
197 | unsigned long end, phys_addr_t phys_addr, pgprot_t prot, |
198 | pgtbl_mod_mask *mask) | |
c2febafc KS |
199 | { |
200 | p4d_t *p4d; | |
201 | unsigned long next; | |
202 | ||
6c0c7d2b | 203 | p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); |
c2febafc KS |
204 | if (!p4d) |
205 | return -ENOMEM; | |
206 | do { | |
207 | next = p4d_addr_end(addr, end); | |
208 | ||
6c0c7d2b JR |
209 | if (ioremap_try_huge_p4d(p4d, addr, next, phys_addr, prot)) { |
210 | *mask |= PGTBL_P4D_MODIFIED; | |
8e2d4340 | 211 | continue; |
6c0c7d2b | 212 | } |
c2febafc | 213 | |
6c0c7d2b | 214 | if (ioremap_pud_range(p4d, addr, next, phys_addr, prot, mask)) |
c2febafc | 215 | return -ENOMEM; |
36ddc5a7 | 216 | } while (p4d++, phys_addr += (next - addr), addr = next, addr != end); |
c2febafc KS |
217 | return 0; |
218 | } | |
219 | ||
74588d8b | 220 | int ioremap_page_range(unsigned long addr, |
ffa71f33 | 221 | unsigned long end, phys_addr_t phys_addr, pgprot_t prot) |
74588d8b HS |
222 | { |
223 | pgd_t *pgd; | |
224 | unsigned long start; | |
225 | unsigned long next; | |
226 | int err; | |
6c0c7d2b | 227 | pgtbl_mod_mask mask = 0; |
74588d8b | 228 | |
b39ab98e | 229 | might_sleep(); |
74588d8b HS |
230 | BUG_ON(addr >= end); |
231 | ||
74588d8b | 232 | start = addr; |
74588d8b HS |
233 | pgd = pgd_offset_k(addr); |
234 | do { | |
235 | next = pgd_addr_end(addr, end); | |
6c0c7d2b JR |
236 | err = ioremap_p4d_range(pgd, addr, next, phys_addr, prot, |
237 | &mask); | |
74588d8b HS |
238 | if (err) |
239 | break; | |
36ddc5a7 | 240 | } while (pgd++, phys_addr += (next - addr), addr = next, addr != end); |
74588d8b | 241 | |
db71daab | 242 | flush_cache_vmap(start, end); |
74588d8b | 243 | |
6c0c7d2b JR |
244 | if (mask & ARCH_PAGE_TABLE_SYNC_MASK) |
245 | arch_sync_kernel_mappings(start, end); | |
246 | ||
74588d8b HS |
247 | return err; |
248 | } | |
80b0ca98 CH |
249 | |
250 | #ifdef CONFIG_GENERIC_IOREMAP | |
251 | void __iomem *ioremap_prot(phys_addr_t addr, size_t size, unsigned long prot) | |
252 | { | |
253 | unsigned long offset, vaddr; | |
254 | phys_addr_t last_addr; | |
255 | struct vm_struct *area; | |
256 | ||
257 | /* Disallow wrap-around or zero size */ | |
258 | last_addr = addr + size - 1; | |
259 | if (!size || last_addr < addr) | |
260 | return NULL; | |
261 | ||
262 | /* Page-align mappings */ | |
263 | offset = addr & (~PAGE_MASK); | |
264 | addr -= offset; | |
265 | size = PAGE_ALIGN(size + offset); | |
266 | ||
267 | area = get_vm_area_caller(size, VM_IOREMAP, | |
268 | __builtin_return_address(0)); | |
269 | if (!area) | |
270 | return NULL; | |
271 | vaddr = (unsigned long)area->addr; | |
272 | ||
273 | if (ioremap_page_range(vaddr, vaddr + size, addr, __pgprot(prot))) { | |
274 | free_vm_area(area); | |
275 | return NULL; | |
276 | } | |
277 | ||
278 | return (void __iomem *)(vaddr + offset); | |
279 | } | |
280 | EXPORT_SYMBOL(ioremap_prot); | |
281 | ||
282 | void iounmap(volatile void __iomem *addr) | |
283 | { | |
284 | vunmap((void *)((unsigned long)addr & PAGE_MASK)); | |
285 | } | |
286 | EXPORT_SYMBOL(iounmap); | |
287 | #endif /* CONFIG_GENERIC_IOREMAP */ |