]>
Commit | Line | Data |
---|---|---|
e1d3c0fd WD |
1 | /* |
2 | * CPU-agnostic ARM page table allocator. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License version 2 as | |
6 | * published by the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, | |
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
11 | * GNU General Public License for more details. | |
12 | * | |
13 | * You should have received a copy of the GNU General Public License | |
14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
15 | * | |
16 | * Copyright (C) 2014 ARM Limited | |
17 | * | |
18 | * Author: Will Deacon <will.deacon@arm.com> | |
19 | */ | |
20 | ||
21 | #define pr_fmt(fmt) "arm-lpae io-pgtable: " fmt | |
22 | ||
2c3d273e | 23 | #include <linux/atomic.h> |
e1d3c0fd WD |
24 | #include <linux/iommu.h> |
25 | #include <linux/kernel.h> | |
26 | #include <linux/sizes.h> | |
27 | #include <linux/slab.h> | |
28 | #include <linux/types.h> | |
8f6aff98 | 29 | #include <linux/dma-mapping.h> |
e1d3c0fd | 30 | |
87a91b15 RM |
31 | #include <asm/barrier.h> |
32 | ||
e1d3c0fd WD |
33 | #include "io-pgtable.h" |
34 | ||
35 | #define ARM_LPAE_MAX_ADDR_BITS 48 | |
36 | #define ARM_LPAE_S2_MAX_CONCAT_PAGES 16 | |
37 | #define ARM_LPAE_MAX_LEVELS 4 | |
38 | ||
39 | /* Struct accessors */ | |
40 | #define io_pgtable_to_data(x) \ | |
41 | container_of((x), struct arm_lpae_io_pgtable, iop) | |
42 | ||
e1d3c0fd WD |
43 | #define io_pgtable_ops_to_data(x) \ |
44 | io_pgtable_to_data(io_pgtable_ops_to_pgtable(x)) | |
45 | ||
46 | /* | |
47 | * For consistency with the architecture, we always consider | |
48 | * ARM_LPAE_MAX_LEVELS levels, with the walk starting at level n >=0 | |
49 | */ | |
50 | #define ARM_LPAE_START_LVL(d) (ARM_LPAE_MAX_LEVELS - (d)->levels) | |
51 | ||
52 | /* | |
53 | * Calculate the right shift amount to get to the portion describing level l | |
54 | * in a virtual address mapped by the pagetable in d. | |
55 | */ | |
56 | #define ARM_LPAE_LVL_SHIFT(l,d) \ | |
57 | ((((d)->levels - ((l) - ARM_LPAE_START_LVL(d) + 1)) \ | |
58 | * (d)->bits_per_level) + (d)->pg_shift) | |
59 | ||
06c610e8 RM |
60 | #define ARM_LPAE_GRANULE(d) (1UL << (d)->pg_shift) |
61 | ||
367bd978 | 62 | #define ARM_LPAE_PAGES_PER_PGD(d) \ |
06c610e8 | 63 | DIV_ROUND_UP((d)->pgd_size, ARM_LPAE_GRANULE(d)) |
e1d3c0fd WD |
64 | |
65 | /* | |
66 | * Calculate the index at level l used to map virtual address a using the | |
67 | * pagetable in d. | |
68 | */ | |
69 | #define ARM_LPAE_PGD_IDX(l,d) \ | |
70 | ((l) == ARM_LPAE_START_LVL(d) ? ilog2(ARM_LPAE_PAGES_PER_PGD(d)) : 0) | |
71 | ||
72 | #define ARM_LPAE_LVL_IDX(a,l,d) \ | |
367bd978 | 73 | (((u64)(a) >> ARM_LPAE_LVL_SHIFT(l,d)) & \ |
e1d3c0fd WD |
74 | ((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1)) |
75 | ||
76 | /* Calculate the block/page mapping size at level l for pagetable in d. */ | |
77 | #define ARM_LPAE_BLOCK_SIZE(l,d) \ | |
022f4e4f | 78 | (1ULL << (ilog2(sizeof(arm_lpae_iopte)) + \ |
e1d3c0fd WD |
79 | ((ARM_LPAE_MAX_LEVELS - (l)) * (d)->bits_per_level))) |
80 | ||
81 | /* Page table bits */ | |
82 | #define ARM_LPAE_PTE_TYPE_SHIFT 0 | |
83 | #define ARM_LPAE_PTE_TYPE_MASK 0x3 | |
84 | ||
85 | #define ARM_LPAE_PTE_TYPE_BLOCK 1 | |
86 | #define ARM_LPAE_PTE_TYPE_TABLE 3 | |
87 | #define ARM_LPAE_PTE_TYPE_PAGE 3 | |
88 | ||
c896c132 | 89 | #define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63) |
e1d3c0fd WD |
90 | #define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53) |
91 | #define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10) | |
92 | #define ARM_LPAE_PTE_SH_NS (((arm_lpae_iopte)0) << 8) | |
93 | #define ARM_LPAE_PTE_SH_OS (((arm_lpae_iopte)2) << 8) | |
94 | #define ARM_LPAE_PTE_SH_IS (((arm_lpae_iopte)3) << 8) | |
c896c132 | 95 | #define ARM_LPAE_PTE_NS (((arm_lpae_iopte)1) << 5) |
e1d3c0fd WD |
96 | #define ARM_LPAE_PTE_VALID (((arm_lpae_iopte)1) << 0) |
97 | ||
98 | #define ARM_LPAE_PTE_ATTR_LO_MASK (((arm_lpae_iopte)0x3ff) << 2) | |
99 | /* Ignore the contiguous bit for block splitting */ | |
100 | #define ARM_LPAE_PTE_ATTR_HI_MASK (((arm_lpae_iopte)6) << 52) | |
101 | #define ARM_LPAE_PTE_ATTR_MASK (ARM_LPAE_PTE_ATTR_LO_MASK | \ | |
102 | ARM_LPAE_PTE_ATTR_HI_MASK) | |
2c3d273e RM |
103 | /* Software bit for solving coherency races */ |
104 | #define ARM_LPAE_PTE_SW_SYNC (((arm_lpae_iopte)1) << 55) | |
e1d3c0fd WD |
105 | |
106 | /* Stage-1 PTE */ | |
107 | #define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6) | |
108 | #define ARM_LPAE_PTE_AP_RDONLY (((arm_lpae_iopte)2) << 6) | |
109 | #define ARM_LPAE_PTE_ATTRINDX_SHIFT 2 | |
110 | #define ARM_LPAE_PTE_nG (((arm_lpae_iopte)1) << 11) | |
111 | ||
112 | /* Stage-2 PTE */ | |
113 | #define ARM_LPAE_PTE_HAP_FAULT (((arm_lpae_iopte)0) << 6) | |
114 | #define ARM_LPAE_PTE_HAP_READ (((arm_lpae_iopte)1) << 6) | |
115 | #define ARM_LPAE_PTE_HAP_WRITE (((arm_lpae_iopte)2) << 6) | |
116 | #define ARM_LPAE_PTE_MEMATTR_OIWB (((arm_lpae_iopte)0xf) << 2) | |
117 | #define ARM_LPAE_PTE_MEMATTR_NC (((arm_lpae_iopte)0x5) << 2) | |
118 | #define ARM_LPAE_PTE_MEMATTR_DEV (((arm_lpae_iopte)0x1) << 2) | |
119 | ||
120 | /* Register bits */ | |
121 | #define ARM_32_LPAE_TCR_EAE (1 << 31) | |
122 | #define ARM_64_LPAE_S2_TCR_RES1 (1 << 31) | |
123 | ||
63979b8d WD |
124 | #define ARM_LPAE_TCR_EPD1 (1 << 23) |
125 | ||
e1d3c0fd WD |
126 | #define ARM_LPAE_TCR_TG0_4K (0 << 14) |
127 | #define ARM_LPAE_TCR_TG0_64K (1 << 14) | |
128 | #define ARM_LPAE_TCR_TG0_16K (2 << 14) | |
129 | ||
130 | #define ARM_LPAE_TCR_SH0_SHIFT 12 | |
131 | #define ARM_LPAE_TCR_SH0_MASK 0x3 | |
132 | #define ARM_LPAE_TCR_SH_NS 0 | |
133 | #define ARM_LPAE_TCR_SH_OS 2 | |
134 | #define ARM_LPAE_TCR_SH_IS 3 | |
135 | ||
136 | #define ARM_LPAE_TCR_ORGN0_SHIFT 10 | |
137 | #define ARM_LPAE_TCR_IRGN0_SHIFT 8 | |
138 | #define ARM_LPAE_TCR_RGN_MASK 0x3 | |
139 | #define ARM_LPAE_TCR_RGN_NC 0 | |
140 | #define ARM_LPAE_TCR_RGN_WBWA 1 | |
141 | #define ARM_LPAE_TCR_RGN_WT 2 | |
142 | #define ARM_LPAE_TCR_RGN_WB 3 | |
143 | ||
144 | #define ARM_LPAE_TCR_SL0_SHIFT 6 | |
145 | #define ARM_LPAE_TCR_SL0_MASK 0x3 | |
146 | ||
147 | #define ARM_LPAE_TCR_T0SZ_SHIFT 0 | |
148 | #define ARM_LPAE_TCR_SZ_MASK 0xf | |
149 | ||
150 | #define ARM_LPAE_TCR_PS_SHIFT 16 | |
151 | #define ARM_LPAE_TCR_PS_MASK 0x7 | |
152 | ||
153 | #define ARM_LPAE_TCR_IPS_SHIFT 32 | |
154 | #define ARM_LPAE_TCR_IPS_MASK 0x7 | |
155 | ||
156 | #define ARM_LPAE_TCR_PS_32_BIT 0x0ULL | |
157 | #define ARM_LPAE_TCR_PS_36_BIT 0x1ULL | |
158 | #define ARM_LPAE_TCR_PS_40_BIT 0x2ULL | |
159 | #define ARM_LPAE_TCR_PS_42_BIT 0x3ULL | |
160 | #define ARM_LPAE_TCR_PS_44_BIT 0x4ULL | |
161 | #define ARM_LPAE_TCR_PS_48_BIT 0x5ULL | |
162 | ||
163 | #define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3) | |
164 | #define ARM_LPAE_MAIR_ATTR_MASK 0xff | |
165 | #define ARM_LPAE_MAIR_ATTR_DEVICE 0x04 | |
166 | #define ARM_LPAE_MAIR_ATTR_NC 0x44 | |
167 | #define ARM_LPAE_MAIR_ATTR_WBRWA 0xff | |
168 | #define ARM_LPAE_MAIR_ATTR_IDX_NC 0 | |
169 | #define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1 | |
170 | #define ARM_LPAE_MAIR_ATTR_IDX_DEV 2 | |
171 | ||
172 | /* IOPTE accessors */ | |
173 | #define iopte_deref(pte,d) \ | |
174 | (__va((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1) \ | |
06c610e8 | 175 | & ~(ARM_LPAE_GRANULE(d) - 1ULL))) |
e1d3c0fd WD |
176 | |
177 | #define iopte_type(pte,l) \ | |
178 | (((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK) | |
179 | ||
180 | #define iopte_prot(pte) ((pte) & ARM_LPAE_PTE_ATTR_MASK) | |
181 | ||
182 | #define iopte_leaf(pte,l) \ | |
183 | (l == (ARM_LPAE_MAX_LEVELS - 1) ? \ | |
184 | (iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_PAGE) : \ | |
185 | (iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_BLOCK)) | |
186 | ||
187 | #define iopte_to_pfn(pte,d) \ | |
188 | (((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1)) >> (d)->pg_shift) | |
189 | ||
190 | #define pfn_to_iopte(pfn,d) \ | |
191 | (((pfn) << (d)->pg_shift) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1)) | |
192 | ||
193 | struct arm_lpae_io_pgtable { | |
194 | struct io_pgtable iop; | |
195 | ||
196 | int levels; | |
197 | size_t pgd_size; | |
198 | unsigned long pg_shift; | |
199 | unsigned long bits_per_level; | |
200 | ||
201 | void *pgd; | |
202 | }; | |
203 | ||
204 | typedef u64 arm_lpae_iopte; | |
205 | ||
fe4b991d WD |
206 | static bool selftest_running = false; |
207 | ||
ffcb6d16 | 208 | static dma_addr_t __arm_lpae_dma_addr(void *pages) |
f8d54961 | 209 | { |
ffcb6d16 | 210 | return (dma_addr_t)virt_to_phys(pages); |
f8d54961 RM |
211 | } |
212 | ||
213 | static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, | |
214 | struct io_pgtable_cfg *cfg) | |
215 | { | |
216 | struct device *dev = cfg->iommu_dev; | |
217 | dma_addr_t dma; | |
218 | void *pages = alloc_pages_exact(size, gfp | __GFP_ZERO); | |
219 | ||
220 | if (!pages) | |
221 | return NULL; | |
222 | ||
81b3c252 | 223 | if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) { |
f8d54961 RM |
224 | dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE); |
225 | if (dma_mapping_error(dev, dma)) | |
226 | goto out_free; | |
227 | /* | |
228 | * We depend on the IOMMU being able to work with any physical | |
ffcb6d16 RM |
229 | * address directly, so if the DMA layer suggests otherwise by |
230 | * translating or truncating them, that bodes very badly... | |
f8d54961 | 231 | */ |
ffcb6d16 | 232 | if (dma != virt_to_phys(pages)) |
f8d54961 RM |
233 | goto out_unmap; |
234 | } | |
235 | ||
236 | return pages; | |
237 | ||
238 | out_unmap: | |
239 | dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n"); | |
240 | dma_unmap_single(dev, dma, size, DMA_TO_DEVICE); | |
241 | out_free: | |
242 | free_pages_exact(pages, size); | |
243 | return NULL; | |
244 | } | |
245 | ||
246 | static void __arm_lpae_free_pages(void *pages, size_t size, | |
247 | struct io_pgtable_cfg *cfg) | |
248 | { | |
81b3c252 | 249 | if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) |
ffcb6d16 | 250 | dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages), |
f8d54961 RM |
251 | size, DMA_TO_DEVICE); |
252 | free_pages_exact(pages, size); | |
253 | } | |
254 | ||
2c3d273e RM |
255 | static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, |
256 | struct io_pgtable_cfg *cfg) | |
257 | { | |
258 | dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep), | |
259 | sizeof(*ptep), DMA_TO_DEVICE); | |
260 | } | |
261 | ||
f8d54961 | 262 | static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte, |
87a91b15 | 263 | struct io_pgtable_cfg *cfg) |
f8d54961 | 264 | { |
f8d54961 RM |
265 | *ptep = pte; |
266 | ||
81b3c252 | 267 | if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) |
2c3d273e | 268 | __arm_lpae_sync_pte(ptep, cfg); |
f8d54961 RM |
269 | } |
270 | ||
cf27ec93 WD |
271 | static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, |
272 | unsigned long iova, size_t size, int lvl, | |
273 | arm_lpae_iopte *ptep); | |
274 | ||
fb3a9579 RM |
275 | static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, |
276 | phys_addr_t paddr, arm_lpae_iopte prot, | |
277 | int lvl, arm_lpae_iopte *ptep) | |
278 | { | |
279 | arm_lpae_iopte pte = prot; | |
280 | ||
281 | if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) | |
282 | pte |= ARM_LPAE_PTE_NS; | |
283 | ||
284 | if (lvl == ARM_LPAE_MAX_LEVELS - 1) | |
285 | pte |= ARM_LPAE_PTE_TYPE_PAGE; | |
286 | else | |
287 | pte |= ARM_LPAE_PTE_TYPE_BLOCK; | |
288 | ||
289 | pte |= ARM_LPAE_PTE_AF | ARM_LPAE_PTE_SH_IS; | |
290 | pte |= pfn_to_iopte(paddr >> data->pg_shift, data); | |
291 | ||
292 | __arm_lpae_set_pte(ptep, pte, &data->iop.cfg); | |
293 | } | |
294 | ||
e1d3c0fd WD |
295 | static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, |
296 | unsigned long iova, phys_addr_t paddr, | |
297 | arm_lpae_iopte prot, int lvl, | |
298 | arm_lpae_iopte *ptep) | |
299 | { | |
fb3a9579 | 300 | arm_lpae_iopte pte = *ptep; |
e1d3c0fd | 301 | |
fb3a9579 | 302 | if (iopte_leaf(pte, lvl)) { |
cf27ec93 | 303 | /* We require an unmap first */ |
fe4b991d | 304 | WARN_ON(!selftest_running); |
e1d3c0fd | 305 | return -EEXIST; |
fb3a9579 | 306 | } else if (iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_TABLE) { |
cf27ec93 WD |
307 | /* |
308 | * We need to unmap and free the old table before | |
309 | * overwriting it with a block entry. | |
310 | */ | |
311 | arm_lpae_iopte *tblp; | |
312 | size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data); | |
313 | ||
314 | tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data); | |
315 | if (WARN_ON(__arm_lpae_unmap(data, iova, sz, lvl, tblp) != sz)) | |
316 | return -EINVAL; | |
fe4b991d | 317 | } |
e1d3c0fd | 318 | |
fb3a9579 RM |
319 | __arm_lpae_init_pte(data, paddr, prot, lvl, ptep); |
320 | return 0; | |
321 | } | |
c896c132 | 322 | |
fb3a9579 RM |
323 | static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table, |
324 | arm_lpae_iopte *ptep, | |
2c3d273e | 325 | arm_lpae_iopte curr, |
fb3a9579 RM |
326 | struct io_pgtable_cfg *cfg) |
327 | { | |
2c3d273e | 328 | arm_lpae_iopte old, new; |
e1d3c0fd | 329 | |
fb3a9579 RM |
330 | new = __pa(table) | ARM_LPAE_PTE_TYPE_TABLE; |
331 | if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) | |
332 | new |= ARM_LPAE_PTE_NSTABLE; | |
e1d3c0fd | 333 | |
77f34458 WD |
334 | /* |
335 | * Ensure the table itself is visible before its PTE can be. | |
336 | * Whilst we could get away with cmpxchg64_release below, this | |
337 | * doesn't have any ordering semantics when !CONFIG_SMP. | |
338 | */ | |
339 | dma_wmb(); | |
2c3d273e RM |
340 | |
341 | old = cmpxchg64_relaxed(ptep, curr, new); | |
342 | ||
343 | if ((cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA) || | |
344 | (old & ARM_LPAE_PTE_SW_SYNC)) | |
345 | return old; | |
346 | ||
347 | /* Even if it's not ours, there's no point waiting; just kick it */ | |
348 | __arm_lpae_sync_pte(ptep, cfg); | |
349 | if (old == curr) | |
350 | WRITE_ONCE(*ptep, new | ARM_LPAE_PTE_SW_SYNC); | |
351 | ||
352 | return old; | |
e1d3c0fd WD |
353 | } |
354 | ||
355 | static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, | |
356 | phys_addr_t paddr, size_t size, arm_lpae_iopte prot, | |
357 | int lvl, arm_lpae_iopte *ptep) | |
358 | { | |
359 | arm_lpae_iopte *cptep, pte; | |
e1d3c0fd | 360 | size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); |
2c3d273e | 361 | size_t tblsz = ARM_LPAE_GRANULE(data); |
f8d54961 | 362 | struct io_pgtable_cfg *cfg = &data->iop.cfg; |
e1d3c0fd WD |
363 | |
364 | /* Find our entry at the current level */ | |
365 | ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); | |
366 | ||
367 | /* If we can install a leaf entry at this level, then do so */ | |
f8d54961 | 368 | if (size == block_size && (size & cfg->pgsize_bitmap)) |
e1d3c0fd WD |
369 | return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep); |
370 | ||
371 | /* We can't allocate tables at the final level */ | |
372 | if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1)) | |
373 | return -EINVAL; | |
374 | ||
375 | /* Grab a pointer to the next level */ | |
2c3d273e | 376 | pte = READ_ONCE(*ptep); |
e1d3c0fd | 377 | if (!pte) { |
2c3d273e | 378 | cptep = __arm_lpae_alloc_pages(tblsz, GFP_ATOMIC, cfg); |
e1d3c0fd WD |
379 | if (!cptep) |
380 | return -ENOMEM; | |
381 | ||
2c3d273e RM |
382 | pte = arm_lpae_install_table(cptep, ptep, 0, cfg); |
383 | if (pte) | |
384 | __arm_lpae_free_pages(cptep, tblsz, cfg); | |
385 | } else if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA) && | |
386 | !(pte & ARM_LPAE_PTE_SW_SYNC)) { | |
387 | __arm_lpae_sync_pte(ptep, cfg); | |
388 | } | |
389 | ||
390 | if (pte && !iopte_leaf(pte, lvl)) { | |
e1d3c0fd | 391 | cptep = iopte_deref(pte, data); |
2c3d273e | 392 | } else if (pte) { |
ed46e66c OT |
393 | /* We require an unmap first */ |
394 | WARN_ON(!selftest_running); | |
395 | return -EEXIST; | |
e1d3c0fd WD |
396 | } |
397 | ||
398 | /* Rinse, repeat */ | |
399 | return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep); | |
400 | } | |
401 | ||
402 | static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, | |
403 | int prot) | |
404 | { | |
405 | arm_lpae_iopte pte; | |
406 | ||
407 | if (data->iop.fmt == ARM_64_LPAE_S1 || | |
408 | data->iop.fmt == ARM_32_LPAE_S1) { | |
e7468a23 | 409 | pte = ARM_LPAE_PTE_nG; |
e1d3c0fd WD |
410 | |
411 | if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ)) | |
412 | pte |= ARM_LPAE_PTE_AP_RDONLY; | |
413 | ||
e7468a23 JG |
414 | if (!(prot & IOMMU_PRIV)) |
415 | pte |= ARM_LPAE_PTE_AP_UNPRIV; | |
416 | ||
fb948251 RM |
417 | if (prot & IOMMU_MMIO) |
418 | pte |= (ARM_LPAE_MAIR_ATTR_IDX_DEV | |
419 | << ARM_LPAE_PTE_ATTRINDX_SHIFT); | |
420 | else if (prot & IOMMU_CACHE) | |
e1d3c0fd WD |
421 | pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE |
422 | << ARM_LPAE_PTE_ATTRINDX_SHIFT); | |
423 | } else { | |
424 | pte = ARM_LPAE_PTE_HAP_FAULT; | |
425 | if (prot & IOMMU_READ) | |
426 | pte |= ARM_LPAE_PTE_HAP_READ; | |
427 | if (prot & IOMMU_WRITE) | |
428 | pte |= ARM_LPAE_PTE_HAP_WRITE; | |
fb948251 RM |
429 | if (prot & IOMMU_MMIO) |
430 | pte |= ARM_LPAE_PTE_MEMATTR_DEV; | |
431 | else if (prot & IOMMU_CACHE) | |
e1d3c0fd WD |
432 | pte |= ARM_LPAE_PTE_MEMATTR_OIWB; |
433 | else | |
434 | pte |= ARM_LPAE_PTE_MEMATTR_NC; | |
435 | } | |
436 | ||
437 | if (prot & IOMMU_NOEXEC) | |
438 | pte |= ARM_LPAE_PTE_XN; | |
439 | ||
440 | return pte; | |
441 | } | |
442 | ||
443 | static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova, | |
444 | phys_addr_t paddr, size_t size, int iommu_prot) | |
445 | { | |
446 | struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); | |
447 | arm_lpae_iopte *ptep = data->pgd; | |
87a91b15 | 448 | int ret, lvl = ARM_LPAE_START_LVL(data); |
e1d3c0fd WD |
449 | arm_lpae_iopte prot; |
450 | ||
451 | /* If no access, then nothing to do */ | |
452 | if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) | |
453 | return 0; | |
454 | ||
76557391 RM |
455 | if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias) || |
456 | paddr >= (1ULL << data->iop.cfg.oas))) | |
457 | return -ERANGE; | |
458 | ||
e1d3c0fd | 459 | prot = arm_lpae_prot_to_pte(data, iommu_prot); |
87a91b15 RM |
460 | ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep); |
461 | /* | |
462 | * Synchronise all PTE updates for the new mapping before there's | |
463 | * a chance for anything to kick off a table walk for the new iova. | |
464 | */ | |
465 | wmb(); | |
466 | ||
467 | return ret; | |
e1d3c0fd WD |
468 | } |
469 | ||
470 | static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, | |
471 | arm_lpae_iopte *ptep) | |
472 | { | |
473 | arm_lpae_iopte *start, *end; | |
474 | unsigned long table_size; | |
475 | ||
e1d3c0fd WD |
476 | if (lvl == ARM_LPAE_START_LVL(data)) |
477 | table_size = data->pgd_size; | |
478 | else | |
06c610e8 | 479 | table_size = ARM_LPAE_GRANULE(data); |
e1d3c0fd WD |
480 | |
481 | start = ptep; | |
12c2ab09 WD |
482 | |
483 | /* Only leaf entries at the last level */ | |
484 | if (lvl == ARM_LPAE_MAX_LEVELS - 1) | |
485 | end = ptep; | |
486 | else | |
487 | end = (void *)ptep + table_size; | |
e1d3c0fd WD |
488 | |
489 | while (ptep != end) { | |
490 | arm_lpae_iopte pte = *ptep++; | |
491 | ||
492 | if (!pte || iopte_leaf(pte, lvl)) | |
493 | continue; | |
494 | ||
495 | __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); | |
496 | } | |
497 | ||
f8d54961 | 498 | __arm_lpae_free_pages(start, table_size, &data->iop.cfg); |
e1d3c0fd WD |
499 | } |
500 | ||
501 | static void arm_lpae_free_pgtable(struct io_pgtable *iop) | |
502 | { | |
503 | struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop); | |
504 | ||
505 | __arm_lpae_free_pgtable(data, ARM_LPAE_START_LVL(data), data->pgd); | |
506 | kfree(data); | |
507 | } | |
508 | ||
509 | static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, | |
510 | unsigned long iova, size_t size, | |
fb3a9579 RM |
511 | arm_lpae_iopte blk_pte, int lvl, |
512 | arm_lpae_iopte *ptep) | |
e1d3c0fd | 513 | { |
fb3a9579 RM |
514 | struct io_pgtable_cfg *cfg = &data->iop.cfg; |
515 | arm_lpae_iopte pte, *tablep; | |
e1d3c0fd | 516 | phys_addr_t blk_paddr; |
fb3a9579 RM |
517 | size_t tablesz = ARM_LPAE_GRANULE(data); |
518 | size_t split_sz = ARM_LPAE_BLOCK_SIZE(lvl, data); | |
519 | int i, unmap_idx = -1; | |
520 | ||
521 | if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) | |
522 | return 0; | |
e1d3c0fd | 523 | |
fb3a9579 RM |
524 | tablep = __arm_lpae_alloc_pages(tablesz, GFP_ATOMIC, cfg); |
525 | if (!tablep) | |
526 | return 0; /* Bytes unmapped */ | |
e1d3c0fd | 527 | |
fb3a9579 RM |
528 | if (size == split_sz) |
529 | unmap_idx = ARM_LPAE_LVL_IDX(iova, lvl, data); | |
e1d3c0fd | 530 | |
fb3a9579 RM |
531 | blk_paddr = iopte_to_pfn(blk_pte, data) << data->pg_shift; |
532 | pte = iopte_prot(blk_pte); | |
533 | ||
534 | for (i = 0; i < tablesz / sizeof(pte); i++, blk_paddr += split_sz) { | |
e1d3c0fd | 535 | /* Unmap! */ |
fb3a9579 | 536 | if (i == unmap_idx) |
e1d3c0fd WD |
537 | continue; |
538 | ||
fb3a9579 | 539 | __arm_lpae_init_pte(data, blk_paddr, pte, lvl, &tablep[i]); |
e1d3c0fd WD |
540 | } |
541 | ||
2c3d273e RM |
542 | pte = arm_lpae_install_table(tablep, ptep, blk_pte, cfg); |
543 | if (pte != blk_pte) { | |
544 | __arm_lpae_free_pages(tablep, tablesz, cfg); | |
545 | /* | |
546 | * We may race against someone unmapping another part of this | |
547 | * block, but anything else is invalid. We can't misinterpret | |
548 | * a page entry here since we're never at the last level. | |
549 | */ | |
550 | if (iopte_type(pte, lvl - 1) != ARM_LPAE_PTE_TYPE_TABLE) | |
551 | return 0; | |
552 | ||
553 | tablep = iopte_deref(pte, data); | |
554 | } | |
fb3a9579 RM |
555 | |
556 | if (unmap_idx < 0) | |
557 | return __arm_lpae_unmap(data, iova, size, lvl, tablep); | |
558 | ||
559 | io_pgtable_tlb_add_flush(&data->iop, iova, size, size, true); | |
e1d3c0fd WD |
560 | return size; |
561 | } | |
562 | ||
563 | static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, | |
564 | unsigned long iova, size_t size, int lvl, | |
565 | arm_lpae_iopte *ptep) | |
566 | { | |
567 | arm_lpae_iopte pte; | |
507e4c9d | 568 | struct io_pgtable *iop = &data->iop; |
e1d3c0fd | 569 | |
2eb97c78 RM |
570 | /* Something went horribly wrong and we ran out of page table */ |
571 | if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) | |
572 | return 0; | |
573 | ||
e1d3c0fd | 574 | ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); |
2c3d273e | 575 | pte = READ_ONCE(*ptep); |
2eb97c78 | 576 | if (WARN_ON(!pte)) |
e1d3c0fd WD |
577 | return 0; |
578 | ||
579 | /* If the size matches this level, we're in the right place */ | |
fb3a9579 | 580 | if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) { |
507e4c9d | 581 | __arm_lpae_set_pte(ptep, 0, &iop->cfg); |
e1d3c0fd WD |
582 | |
583 | if (!iopte_leaf(pte, lvl)) { | |
584 | /* Also flush any partial walks */ | |
507e4c9d RM |
585 | io_pgtable_tlb_add_flush(iop, iova, size, |
586 | ARM_LPAE_GRANULE(data), false); | |
587 | io_pgtable_tlb_sync(iop); | |
e1d3c0fd WD |
588 | ptep = iopte_deref(pte, data); |
589 | __arm_lpae_free_pgtable(data, lvl + 1, ptep); | |
590 | } else { | |
507e4c9d | 591 | io_pgtable_tlb_add_flush(iop, iova, size, size, true); |
e1d3c0fd WD |
592 | } |
593 | ||
594 | return size; | |
595 | } else if (iopte_leaf(pte, lvl)) { | |
596 | /* | |
597 | * Insert a table at the next level to map the old region, | |
598 | * minus the part we want to unmap | |
599 | */ | |
fb3a9579 RM |
600 | return arm_lpae_split_blk_unmap(data, iova, size, pte, |
601 | lvl + 1, ptep); | |
e1d3c0fd WD |
602 | } |
603 | ||
604 | /* Keep on walkin' */ | |
605 | ptep = iopte_deref(pte, data); | |
606 | return __arm_lpae_unmap(data, iova, size, lvl + 1, ptep); | |
607 | } | |
608 | ||
609 | static int arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova, | |
610 | size_t size) | |
611 | { | |
e1d3c0fd | 612 | struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); |
e1d3c0fd WD |
613 | arm_lpae_iopte *ptep = data->pgd; |
614 | int lvl = ARM_LPAE_START_LVL(data); | |
615 | ||
76557391 RM |
616 | if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias))) |
617 | return 0; | |
618 | ||
32b12449 | 619 | return __arm_lpae_unmap(data, iova, size, lvl, ptep); |
e1d3c0fd WD |
620 | } |
621 | ||
622 | static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops, | |
623 | unsigned long iova) | |
624 | { | |
625 | struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); | |
626 | arm_lpae_iopte pte, *ptep = data->pgd; | |
627 | int lvl = ARM_LPAE_START_LVL(data); | |
628 | ||
629 | do { | |
630 | /* Valid IOPTE pointer? */ | |
631 | if (!ptep) | |
632 | return 0; | |
633 | ||
634 | /* Grab the IOPTE we're interested in */ | |
2c3d273e RM |
635 | ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); |
636 | pte = READ_ONCE(*ptep); | |
e1d3c0fd WD |
637 | |
638 | /* Valid entry? */ | |
639 | if (!pte) | |
640 | return 0; | |
641 | ||
642 | /* Leaf entry? */ | |
643 | if (iopte_leaf(pte,lvl)) | |
644 | goto found_translation; | |
645 | ||
646 | /* Take it to the next level */ | |
647 | ptep = iopte_deref(pte, data); | |
648 | } while (++lvl < ARM_LPAE_MAX_LEVELS); | |
649 | ||
650 | /* Ran out of page tables to walk */ | |
651 | return 0; | |
652 | ||
653 | found_translation: | |
7c6d90e2 | 654 | iova &= (ARM_LPAE_BLOCK_SIZE(lvl, data) - 1); |
e1d3c0fd WD |
655 | return ((phys_addr_t)iopte_to_pfn(pte,data) << data->pg_shift) | iova; |
656 | } | |
657 | ||
658 | static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg) | |
659 | { | |
660 | unsigned long granule; | |
661 | ||
662 | /* | |
663 | * We need to restrict the supported page sizes to match the | |
664 | * translation regime for a particular granule. Aim to match | |
665 | * the CPU page size if possible, otherwise prefer smaller sizes. | |
666 | * While we're at it, restrict the block sizes to match the | |
667 | * chosen granule. | |
668 | */ | |
669 | if (cfg->pgsize_bitmap & PAGE_SIZE) | |
670 | granule = PAGE_SIZE; | |
671 | else if (cfg->pgsize_bitmap & ~PAGE_MASK) | |
672 | granule = 1UL << __fls(cfg->pgsize_bitmap & ~PAGE_MASK); | |
673 | else if (cfg->pgsize_bitmap & PAGE_MASK) | |
674 | granule = 1UL << __ffs(cfg->pgsize_bitmap & PAGE_MASK); | |
675 | else | |
676 | granule = 0; | |
677 | ||
678 | switch (granule) { | |
679 | case SZ_4K: | |
680 | cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); | |
681 | break; | |
682 | case SZ_16K: | |
683 | cfg->pgsize_bitmap &= (SZ_16K | SZ_32M); | |
684 | break; | |
685 | case SZ_64K: | |
686 | cfg->pgsize_bitmap &= (SZ_64K | SZ_512M); | |
687 | break; | |
688 | default: | |
689 | cfg->pgsize_bitmap = 0; | |
690 | } | |
691 | } | |
692 | ||
693 | static struct arm_lpae_io_pgtable * | |
694 | arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg) | |
695 | { | |
696 | unsigned long va_bits, pgd_bits; | |
697 | struct arm_lpae_io_pgtable *data; | |
698 | ||
699 | arm_lpae_restrict_pgsizes(cfg); | |
700 | ||
701 | if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K))) | |
702 | return NULL; | |
703 | ||
704 | if (cfg->ias > ARM_LPAE_MAX_ADDR_BITS) | |
705 | return NULL; | |
706 | ||
707 | if (cfg->oas > ARM_LPAE_MAX_ADDR_BITS) | |
708 | return NULL; | |
709 | ||
ffcb6d16 RM |
710 | if (!selftest_running && cfg->iommu_dev->dma_pfn_offset) { |
711 | dev_err(cfg->iommu_dev, "Cannot accommodate DMA offset for IOMMU page tables\n"); | |
712 | return NULL; | |
713 | } | |
714 | ||
e1d3c0fd WD |
715 | data = kmalloc(sizeof(*data), GFP_KERNEL); |
716 | if (!data) | |
717 | return NULL; | |
718 | ||
719 | data->pg_shift = __ffs(cfg->pgsize_bitmap); | |
720 | data->bits_per_level = data->pg_shift - ilog2(sizeof(arm_lpae_iopte)); | |
721 | ||
722 | va_bits = cfg->ias - data->pg_shift; | |
723 | data->levels = DIV_ROUND_UP(va_bits, data->bits_per_level); | |
724 | ||
725 | /* Calculate the actual size of our pgd (without concatenation) */ | |
726 | pgd_bits = va_bits - (data->bits_per_level * (data->levels - 1)); | |
727 | data->pgd_size = 1UL << (pgd_bits + ilog2(sizeof(arm_lpae_iopte))); | |
728 | ||
729 | data->iop.ops = (struct io_pgtable_ops) { | |
730 | .map = arm_lpae_map, | |
731 | .unmap = arm_lpae_unmap, | |
732 | .iova_to_phys = arm_lpae_iova_to_phys, | |
733 | }; | |
734 | ||
735 | return data; | |
736 | } | |
737 | ||
738 | static struct io_pgtable * | |
739 | arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) | |
740 | { | |
741 | u64 reg; | |
3850db49 RM |
742 | struct arm_lpae_io_pgtable *data; |
743 | ||
81b3c252 | 744 | if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | IO_PGTABLE_QUIRK_NO_DMA)) |
3850db49 | 745 | return NULL; |
e1d3c0fd | 746 | |
3850db49 | 747 | data = arm_lpae_alloc_pgtable(cfg); |
e1d3c0fd WD |
748 | if (!data) |
749 | return NULL; | |
750 | ||
751 | /* TCR */ | |
752 | reg = (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) | | |
753 | (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) | | |
754 | (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT); | |
755 | ||
06c610e8 | 756 | switch (ARM_LPAE_GRANULE(data)) { |
e1d3c0fd WD |
757 | case SZ_4K: |
758 | reg |= ARM_LPAE_TCR_TG0_4K; | |
759 | break; | |
760 | case SZ_16K: | |
761 | reg |= ARM_LPAE_TCR_TG0_16K; | |
762 | break; | |
763 | case SZ_64K: | |
764 | reg |= ARM_LPAE_TCR_TG0_64K; | |
765 | break; | |
766 | } | |
767 | ||
768 | switch (cfg->oas) { | |
769 | case 32: | |
770 | reg |= (ARM_LPAE_TCR_PS_32_BIT << ARM_LPAE_TCR_IPS_SHIFT); | |
771 | break; | |
772 | case 36: | |
773 | reg |= (ARM_LPAE_TCR_PS_36_BIT << ARM_LPAE_TCR_IPS_SHIFT); | |
774 | break; | |
775 | case 40: | |
776 | reg |= (ARM_LPAE_TCR_PS_40_BIT << ARM_LPAE_TCR_IPS_SHIFT); | |
777 | break; | |
778 | case 42: | |
779 | reg |= (ARM_LPAE_TCR_PS_42_BIT << ARM_LPAE_TCR_IPS_SHIFT); | |
780 | break; | |
781 | case 44: | |
782 | reg |= (ARM_LPAE_TCR_PS_44_BIT << ARM_LPAE_TCR_IPS_SHIFT); | |
783 | break; | |
784 | case 48: | |
785 | reg |= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_IPS_SHIFT); | |
786 | break; | |
787 | default: | |
788 | goto out_free_data; | |
789 | } | |
790 | ||
791 | reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT; | |
63979b8d WD |
792 | |
793 | /* Disable speculative walks through TTBR1 */ | |
794 | reg |= ARM_LPAE_TCR_EPD1; | |
e1d3c0fd WD |
795 | cfg->arm_lpae_s1_cfg.tcr = reg; |
796 | ||
797 | /* MAIRs */ | |
798 | reg = (ARM_LPAE_MAIR_ATTR_NC | |
799 | << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) | | |
800 | (ARM_LPAE_MAIR_ATTR_WBRWA | |
801 | << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) | | |
802 | (ARM_LPAE_MAIR_ATTR_DEVICE | |
803 | << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)); | |
804 | ||
805 | cfg->arm_lpae_s1_cfg.mair[0] = reg; | |
806 | cfg->arm_lpae_s1_cfg.mair[1] = 0; | |
807 | ||
808 | /* Looking good; allocate a pgd */ | |
f8d54961 | 809 | data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg); |
e1d3c0fd WD |
810 | if (!data->pgd) |
811 | goto out_free_data; | |
812 | ||
87a91b15 RM |
813 | /* Ensure the empty pgd is visible before any actual TTBR write */ |
814 | wmb(); | |
e1d3c0fd WD |
815 | |
816 | /* TTBRs */ | |
817 | cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd); | |
818 | cfg->arm_lpae_s1_cfg.ttbr[1] = 0; | |
819 | return &data->iop; | |
820 | ||
821 | out_free_data: | |
822 | kfree(data); | |
823 | return NULL; | |
824 | } | |
825 | ||
826 | static struct io_pgtable * | |
827 | arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) | |
828 | { | |
829 | u64 reg, sl; | |
3850db49 RM |
830 | struct arm_lpae_io_pgtable *data; |
831 | ||
832 | /* The NS quirk doesn't apply at stage 2 */ | |
81b3c252 | 833 | if (cfg->quirks & ~IO_PGTABLE_QUIRK_NO_DMA) |
3850db49 | 834 | return NULL; |
e1d3c0fd | 835 | |
3850db49 | 836 | data = arm_lpae_alloc_pgtable(cfg); |
e1d3c0fd WD |
837 | if (!data) |
838 | return NULL; | |
839 | ||
840 | /* | |
841 | * Concatenate PGDs at level 1 if possible in order to reduce | |
842 | * the depth of the stage-2 walk. | |
843 | */ | |
844 | if (data->levels == ARM_LPAE_MAX_LEVELS) { | |
845 | unsigned long pgd_pages; | |
846 | ||
847 | pgd_pages = data->pgd_size >> ilog2(sizeof(arm_lpae_iopte)); | |
848 | if (pgd_pages <= ARM_LPAE_S2_MAX_CONCAT_PAGES) { | |
849 | data->pgd_size = pgd_pages << data->pg_shift; | |
850 | data->levels--; | |
851 | } | |
852 | } | |
853 | ||
854 | /* VTCR */ | |
855 | reg = ARM_64_LPAE_S2_TCR_RES1 | | |
856 | (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) | | |
857 | (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) | | |
858 | (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT); | |
859 | ||
860 | sl = ARM_LPAE_START_LVL(data); | |
861 | ||
06c610e8 | 862 | switch (ARM_LPAE_GRANULE(data)) { |
e1d3c0fd WD |
863 | case SZ_4K: |
864 | reg |= ARM_LPAE_TCR_TG0_4K; | |
865 | sl++; /* SL0 format is different for 4K granule size */ | |
866 | break; | |
867 | case SZ_16K: | |
868 | reg |= ARM_LPAE_TCR_TG0_16K; | |
869 | break; | |
870 | case SZ_64K: | |
871 | reg |= ARM_LPAE_TCR_TG0_64K; | |
872 | break; | |
873 | } | |
874 | ||
875 | switch (cfg->oas) { | |
876 | case 32: | |
877 | reg |= (ARM_LPAE_TCR_PS_32_BIT << ARM_LPAE_TCR_PS_SHIFT); | |
878 | break; | |
879 | case 36: | |
880 | reg |= (ARM_LPAE_TCR_PS_36_BIT << ARM_LPAE_TCR_PS_SHIFT); | |
881 | break; | |
882 | case 40: | |
883 | reg |= (ARM_LPAE_TCR_PS_40_BIT << ARM_LPAE_TCR_PS_SHIFT); | |
884 | break; | |
885 | case 42: | |
886 | reg |= (ARM_LPAE_TCR_PS_42_BIT << ARM_LPAE_TCR_PS_SHIFT); | |
887 | break; | |
888 | case 44: | |
889 | reg |= (ARM_LPAE_TCR_PS_44_BIT << ARM_LPAE_TCR_PS_SHIFT); | |
890 | break; | |
891 | case 48: | |
892 | reg |= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_PS_SHIFT); | |
893 | break; | |
894 | default: | |
895 | goto out_free_data; | |
896 | } | |
897 | ||
898 | reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT; | |
899 | reg |= (~sl & ARM_LPAE_TCR_SL0_MASK) << ARM_LPAE_TCR_SL0_SHIFT; | |
900 | cfg->arm_lpae_s2_cfg.vtcr = reg; | |
901 | ||
902 | /* Allocate pgd pages */ | |
f8d54961 | 903 | data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg); |
e1d3c0fd WD |
904 | if (!data->pgd) |
905 | goto out_free_data; | |
906 | ||
87a91b15 RM |
907 | /* Ensure the empty pgd is visible before any actual TTBR write */ |
908 | wmb(); | |
e1d3c0fd WD |
909 | |
910 | /* VTTBR */ | |
911 | cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd); | |
912 | return &data->iop; | |
913 | ||
914 | out_free_data: | |
915 | kfree(data); | |
916 | return NULL; | |
917 | } | |
918 | ||
919 | static struct io_pgtable * | |
920 | arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) | |
921 | { | |
922 | struct io_pgtable *iop; | |
923 | ||
924 | if (cfg->ias > 32 || cfg->oas > 40) | |
925 | return NULL; | |
926 | ||
927 | cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); | |
928 | iop = arm_64_lpae_alloc_pgtable_s1(cfg, cookie); | |
929 | if (iop) { | |
930 | cfg->arm_lpae_s1_cfg.tcr |= ARM_32_LPAE_TCR_EAE; | |
931 | cfg->arm_lpae_s1_cfg.tcr &= 0xffffffff; | |
932 | } | |
933 | ||
934 | return iop; | |
935 | } | |
936 | ||
937 | static struct io_pgtable * | |
938 | arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) | |
939 | { | |
940 | struct io_pgtable *iop; | |
941 | ||
942 | if (cfg->ias > 40 || cfg->oas > 40) | |
943 | return NULL; | |
944 | ||
945 | cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); | |
946 | iop = arm_64_lpae_alloc_pgtable_s2(cfg, cookie); | |
947 | if (iop) | |
948 | cfg->arm_lpae_s2_cfg.vtcr &= 0xffffffff; | |
949 | ||
950 | return iop; | |
951 | } | |
952 | ||
953 | struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns = { | |
954 | .alloc = arm_64_lpae_alloc_pgtable_s1, | |
955 | .free = arm_lpae_free_pgtable, | |
956 | }; | |
957 | ||
958 | struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns = { | |
959 | .alloc = arm_64_lpae_alloc_pgtable_s2, | |
960 | .free = arm_lpae_free_pgtable, | |
961 | }; | |
962 | ||
963 | struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns = { | |
964 | .alloc = arm_32_lpae_alloc_pgtable_s1, | |
965 | .free = arm_lpae_free_pgtable, | |
966 | }; | |
967 | ||
968 | struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns = { | |
969 | .alloc = arm_32_lpae_alloc_pgtable_s2, | |
970 | .free = arm_lpae_free_pgtable, | |
971 | }; | |
fe4b991d WD |
972 | |
973 | #ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE_SELFTEST | |
974 | ||
975 | static struct io_pgtable_cfg *cfg_cookie; | |
976 | ||
977 | static void dummy_tlb_flush_all(void *cookie) | |
978 | { | |
979 | WARN_ON(cookie != cfg_cookie); | |
980 | } | |
981 | ||
06c610e8 RM |
982 | static void dummy_tlb_add_flush(unsigned long iova, size_t size, |
983 | size_t granule, bool leaf, void *cookie) | |
fe4b991d WD |
984 | { |
985 | WARN_ON(cookie != cfg_cookie); | |
986 | WARN_ON(!(size & cfg_cookie->pgsize_bitmap)); | |
987 | } | |
988 | ||
989 | static void dummy_tlb_sync(void *cookie) | |
990 | { | |
991 | WARN_ON(cookie != cfg_cookie); | |
992 | } | |
993 | ||
dfed5f01 | 994 | static const struct iommu_gather_ops dummy_tlb_ops __initconst = { |
fe4b991d WD |
995 | .tlb_flush_all = dummy_tlb_flush_all, |
996 | .tlb_add_flush = dummy_tlb_add_flush, | |
997 | .tlb_sync = dummy_tlb_sync, | |
fe4b991d WD |
998 | }; |
999 | ||
1000 | static void __init arm_lpae_dump_ops(struct io_pgtable_ops *ops) | |
1001 | { | |
1002 | struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); | |
1003 | struct io_pgtable_cfg *cfg = &data->iop.cfg; | |
1004 | ||
1005 | pr_err("cfg: pgsize_bitmap 0x%lx, ias %u-bit\n", | |
1006 | cfg->pgsize_bitmap, cfg->ias); | |
1007 | pr_err("data: %d levels, 0x%zx pgd_size, %lu pg_shift, %lu bits_per_level, pgd @ %p\n", | |
1008 | data->levels, data->pgd_size, data->pg_shift, | |
1009 | data->bits_per_level, data->pgd); | |
1010 | } | |
1011 | ||
1012 | #define __FAIL(ops, i) ({ \ | |
1013 | WARN(1, "selftest: test failed for fmt idx %d\n", (i)); \ | |
1014 | arm_lpae_dump_ops(ops); \ | |
1015 | selftest_running = false; \ | |
1016 | -EFAULT; \ | |
1017 | }) | |
1018 | ||
1019 | static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) | |
1020 | { | |
1021 | static const enum io_pgtable_fmt fmts[] = { | |
1022 | ARM_64_LPAE_S1, | |
1023 | ARM_64_LPAE_S2, | |
1024 | }; | |
1025 | ||
1026 | int i, j; | |
1027 | unsigned long iova; | |
1028 | size_t size; | |
1029 | struct io_pgtable_ops *ops; | |
1030 | ||
1031 | selftest_running = true; | |
1032 | ||
1033 | for (i = 0; i < ARRAY_SIZE(fmts); ++i) { | |
1034 | cfg_cookie = cfg; | |
1035 | ops = alloc_io_pgtable_ops(fmts[i], cfg, cfg); | |
1036 | if (!ops) { | |
1037 | pr_err("selftest: failed to allocate io pgtable ops\n"); | |
1038 | return -ENOMEM; | |
1039 | } | |
1040 | ||
1041 | /* | |
1042 | * Initial sanity checks. | |
1043 | * Empty page tables shouldn't provide any translations. | |
1044 | */ | |
1045 | if (ops->iova_to_phys(ops, 42)) | |
1046 | return __FAIL(ops, i); | |
1047 | ||
1048 | if (ops->iova_to_phys(ops, SZ_1G + 42)) | |
1049 | return __FAIL(ops, i); | |
1050 | ||
1051 | if (ops->iova_to_phys(ops, SZ_2G + 42)) | |
1052 | return __FAIL(ops, i); | |
1053 | ||
1054 | /* | |
1055 | * Distinct mappings of different granule sizes. | |
1056 | */ | |
1057 | iova = 0; | |
4ae8a5c5 | 1058 | for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) { |
fe4b991d WD |
1059 | size = 1UL << j; |
1060 | ||
1061 | if (ops->map(ops, iova, iova, size, IOMMU_READ | | |
1062 | IOMMU_WRITE | | |
1063 | IOMMU_NOEXEC | | |
1064 | IOMMU_CACHE)) | |
1065 | return __FAIL(ops, i); | |
1066 | ||
1067 | /* Overlapping mappings */ | |
1068 | if (!ops->map(ops, iova, iova + size, size, | |
1069 | IOMMU_READ | IOMMU_NOEXEC)) | |
1070 | return __FAIL(ops, i); | |
1071 | ||
1072 | if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) | |
1073 | return __FAIL(ops, i); | |
1074 | ||
1075 | iova += SZ_1G; | |
fe4b991d WD |
1076 | } |
1077 | ||
1078 | /* Partial unmap */ | |
1079 | size = 1UL << __ffs(cfg->pgsize_bitmap); | |
1080 | if (ops->unmap(ops, SZ_1G + size, size) != size) | |
1081 | return __FAIL(ops, i); | |
1082 | ||
1083 | /* Remap of partial unmap */ | |
1084 | if (ops->map(ops, SZ_1G + size, size, size, IOMMU_READ)) | |
1085 | return __FAIL(ops, i); | |
1086 | ||
1087 | if (ops->iova_to_phys(ops, SZ_1G + size + 42) != (size + 42)) | |
1088 | return __FAIL(ops, i); | |
1089 | ||
1090 | /* Full unmap */ | |
1091 | iova = 0; | |
1092 | j = find_first_bit(&cfg->pgsize_bitmap, BITS_PER_LONG); | |
1093 | while (j != BITS_PER_LONG) { | |
1094 | size = 1UL << j; | |
1095 | ||
1096 | if (ops->unmap(ops, iova, size) != size) | |
1097 | return __FAIL(ops, i); | |
1098 | ||
1099 | if (ops->iova_to_phys(ops, iova + 42)) | |
1100 | return __FAIL(ops, i); | |
1101 | ||
1102 | /* Remap full block */ | |
1103 | if (ops->map(ops, iova, iova, size, IOMMU_WRITE)) | |
1104 | return __FAIL(ops, i); | |
1105 | ||
1106 | if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) | |
1107 | return __FAIL(ops, i); | |
1108 | ||
1109 | iova += SZ_1G; | |
1110 | j++; | |
1111 | j = find_next_bit(&cfg->pgsize_bitmap, BITS_PER_LONG, j); | |
1112 | } | |
1113 | ||
1114 | free_io_pgtable_ops(ops); | |
1115 | } | |
1116 | ||
1117 | selftest_running = false; | |
1118 | return 0; | |
1119 | } | |
1120 | ||
1121 | static int __init arm_lpae_do_selftests(void) | |
1122 | { | |
1123 | static const unsigned long pgsize[] = { | |
1124 | SZ_4K | SZ_2M | SZ_1G, | |
1125 | SZ_16K | SZ_32M, | |
1126 | SZ_64K | SZ_512M, | |
1127 | }; | |
1128 | ||
1129 | static const unsigned int ias[] = { | |
1130 | 32, 36, 40, 42, 44, 48, | |
1131 | }; | |
1132 | ||
1133 | int i, j, pass = 0, fail = 0; | |
1134 | struct io_pgtable_cfg cfg = { | |
1135 | .tlb = &dummy_tlb_ops, | |
1136 | .oas = 48, | |
81b3c252 | 1137 | .quirks = IO_PGTABLE_QUIRK_NO_DMA, |
fe4b991d WD |
1138 | }; |
1139 | ||
1140 | for (i = 0; i < ARRAY_SIZE(pgsize); ++i) { | |
1141 | for (j = 0; j < ARRAY_SIZE(ias); ++j) { | |
1142 | cfg.pgsize_bitmap = pgsize[i]; | |
1143 | cfg.ias = ias[j]; | |
1144 | pr_info("selftest: pgsize_bitmap 0x%08lx, IAS %u\n", | |
1145 | pgsize[i], ias[j]); | |
1146 | if (arm_lpae_run_tests(&cfg)) | |
1147 | fail++; | |
1148 | else | |
1149 | pass++; | |
1150 | } | |
1151 | } | |
1152 | ||
1153 | pr_info("selftest: completed with %d PASS %d FAIL\n", pass, fail); | |
1154 | return fail ? -EFAULT : 0; | |
1155 | } | |
1156 | subsys_initcall(arm_lpae_do_selftests); | |
1157 | #endif |