]> git.proxmox.com Git - mirror_ubuntu-focal-kernel.git/blame - drivers/iommu/io-pgtable-arm.c
projects / mirror_ubuntu-focal-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
iommu/ipmmu-vmsa: Clean up DMA API usage
[mirror_ubuntu-focal-kernel.git] / drivers / iommu / io-pgtable-arm.c
CommitLineData
e1d3c0fd
WD		 1/*
2 * CPU-agnostic ARM page table allocator.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 *
16 * Copyright (C) 2014 ARM Limited
17 *
18 * Author: Will Deacon <will.deacon@arm.com>
19 */
20
21#define pr_fmt(fmt) "arm-lpae io-pgtable: " fmt
22
23#include <linux/iommu.h>
24#include <linux/kernel.h>
25#include <linux/sizes.h>
26#include <linux/slab.h>
27#include <linux/types.h>
28
29#include "io-pgtable.h"
30
31#define ARM_LPAE_MAX_ADDR_BITS 48
32#define ARM_LPAE_S2_MAX_CONCAT_PAGES 16
33#define ARM_LPAE_MAX_LEVELS 4
34
35/* Struct accessors */
36#define io_pgtable_to_data(x) \
37 container_of((x), struct arm_lpae_io_pgtable, iop)
38
39#define io_pgtable_ops_to_pgtable(x) \
40 container_of((x), struct io_pgtable, ops)
41
42#define io_pgtable_ops_to_data(x) \
43 io_pgtable_to_data(io_pgtable_ops_to_pgtable(x))
44
45/*
46 * For consistency with the architecture, we always consider
47 * ARM_LPAE_MAX_LEVELS levels, with the walk starting at level n >=0
48 */
49#define ARM_LPAE_START_LVL(d) (ARM_LPAE_MAX_LEVELS - (d)->levels)
50
51/*
52 * Calculate the right shift amount to get to the portion describing level l
53 * in a virtual address mapped by the pagetable in d.
54 */
55#define ARM_LPAE_LVL_SHIFT(l,d) \
56 ((((d)->levels - ((l) - ARM_LPAE_START_LVL(d) + 1)) \
57 * (d)->bits_per_level) + (d)->pg_shift)
58
367bd978
WD		 59#define ARM_LPAE_PAGES_PER_PGD(d) \
60 DIV_ROUND_UP((d)->pgd_size, 1UL << (d)->pg_shift)
e1d3c0fd
WD		 61
62/*
63 * Calculate the index at level l used to map virtual address a using the
64 * pagetable in d.
65 */
66#define ARM_LPAE_PGD_IDX(l,d) \
67 ((l) == ARM_LPAE_START_LVL(d) ? ilog2(ARM_LPAE_PAGES_PER_PGD(d)) : 0)
68
69#define ARM_LPAE_LVL_IDX(a,l,d) \
367bd978 70 (((u64)(a) >> ARM_LPAE_LVL_SHIFT(l,d)) & \
e1d3c0fd
WD		 71 ((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1))
72
73/* Calculate the block/page mapping size at level l for pagetable in d. */
74#define ARM_LPAE_BLOCK_SIZE(l,d) \
75 (1 << (ilog2(sizeof(arm_lpae_iopte)) + \
76 ((ARM_LPAE_MAX_LEVELS - (l)) * (d)->bits_per_level)))
77
78/* Page table bits */
79#define ARM_LPAE_PTE_TYPE_SHIFT 0
80#define ARM_LPAE_PTE_TYPE_MASK 0x3
81
82#define ARM_LPAE_PTE_TYPE_BLOCK 1
83#define ARM_LPAE_PTE_TYPE_TABLE 3
84#define ARM_LPAE_PTE_TYPE_PAGE 3
85
c896c132 86#define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63)
e1d3c0fd
WD		 87#define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53)
88#define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10)
89#define ARM_LPAE_PTE_SH_NS (((arm_lpae_iopte)0) << 8)
90#define ARM_LPAE_PTE_SH_OS (((arm_lpae_iopte)2) << 8)
91#define ARM_LPAE_PTE_SH_IS (((arm_lpae_iopte)3) << 8)
c896c132 92#define ARM_LPAE_PTE_NS (((arm_lpae_iopte)1) << 5)
e1d3c0fd
WD		 93#define ARM_LPAE_PTE_VALID (((arm_lpae_iopte)1) << 0)
94
95#define ARM_LPAE_PTE_ATTR_LO_MASK (((arm_lpae_iopte)0x3ff) << 2)
96/* Ignore the contiguous bit for block splitting */
97#define ARM_LPAE_PTE_ATTR_HI_MASK (((arm_lpae_iopte)6) << 52)
98#define ARM_LPAE_PTE_ATTR_MASK (ARM_LPAE_PTE_ATTR_LO_MASK | \
99 ARM_LPAE_PTE_ATTR_HI_MASK)
100
101/* Stage-1 PTE */
102#define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6)
103#define ARM_LPAE_PTE_AP_RDONLY (((arm_lpae_iopte)2) << 6)
104#define ARM_LPAE_PTE_ATTRINDX_SHIFT 2
105#define ARM_LPAE_PTE_nG (((arm_lpae_iopte)1) << 11)
106
107/* Stage-2 PTE */
108#define ARM_LPAE_PTE_HAP_FAULT (((arm_lpae_iopte)0) << 6)
109#define ARM_LPAE_PTE_HAP_READ (((arm_lpae_iopte)1) << 6)
110#define ARM_LPAE_PTE_HAP_WRITE (((arm_lpae_iopte)2) << 6)
111#define ARM_LPAE_PTE_MEMATTR_OIWB (((arm_lpae_iopte)0xf) << 2)
112#define ARM_LPAE_PTE_MEMATTR_NC (((arm_lpae_iopte)0x5) << 2)
113#define ARM_LPAE_PTE_MEMATTR_DEV (((arm_lpae_iopte)0x1) << 2)
114
115/* Register bits */
116#define ARM_32_LPAE_TCR_EAE (1 << 31)
117#define ARM_64_LPAE_S2_TCR_RES1 (1 << 31)
118
63979b8d
WD		 119#define ARM_LPAE_TCR_EPD1 (1 << 23)
120
e1d3c0fd
WD		 121#define ARM_LPAE_TCR_TG0_4K (0 << 14)
122#define ARM_LPAE_TCR_TG0_64K (1 << 14)
123#define ARM_LPAE_TCR_TG0_16K (2 << 14)
124
125#define ARM_LPAE_TCR_SH0_SHIFT 12
126#define ARM_LPAE_TCR_SH0_MASK 0x3
127#define ARM_LPAE_TCR_SH_NS 0
128#define ARM_LPAE_TCR_SH_OS 2
129#define ARM_LPAE_TCR_SH_IS 3
130
131#define ARM_LPAE_TCR_ORGN0_SHIFT 10
132#define ARM_LPAE_TCR_IRGN0_SHIFT 8
133#define ARM_LPAE_TCR_RGN_MASK 0x3
134#define ARM_LPAE_TCR_RGN_NC 0
135#define ARM_LPAE_TCR_RGN_WBWA 1
136#define ARM_LPAE_TCR_RGN_WT 2
137#define ARM_LPAE_TCR_RGN_WB 3
138
139#define ARM_LPAE_TCR_SL0_SHIFT 6
140#define ARM_LPAE_TCR_SL0_MASK 0x3
141
142#define ARM_LPAE_TCR_T0SZ_SHIFT 0
143#define ARM_LPAE_TCR_SZ_MASK 0xf
144
145#define ARM_LPAE_TCR_PS_SHIFT 16
146#define ARM_LPAE_TCR_PS_MASK 0x7
147
148#define ARM_LPAE_TCR_IPS_SHIFT 32
149#define ARM_LPAE_TCR_IPS_MASK 0x7
150
151#define ARM_LPAE_TCR_PS_32_BIT 0x0ULL
152#define ARM_LPAE_TCR_PS_36_BIT 0x1ULL
153#define ARM_LPAE_TCR_PS_40_BIT 0x2ULL
154#define ARM_LPAE_TCR_PS_42_BIT 0x3ULL
155#define ARM_LPAE_TCR_PS_44_BIT 0x4ULL
156#define ARM_LPAE_TCR_PS_48_BIT 0x5ULL
157
158#define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3)
159#define ARM_LPAE_MAIR_ATTR_MASK 0xff
160#define ARM_LPAE_MAIR_ATTR_DEVICE 0x04
161#define ARM_LPAE_MAIR_ATTR_NC 0x44
162#define ARM_LPAE_MAIR_ATTR_WBRWA 0xff
163#define ARM_LPAE_MAIR_ATTR_IDX_NC 0
164#define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1
165#define ARM_LPAE_MAIR_ATTR_IDX_DEV 2
166
167/* IOPTE accessors */
168#define iopte_deref(pte,d) \
169 (__va((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1) \
170 & ~((1ULL << (d)->pg_shift) - 1)))
171
172#define iopte_type(pte,l) \
173 (((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK)
174
175#define iopte_prot(pte) ((pte) & ARM_LPAE_PTE_ATTR_MASK)
176
177#define iopte_leaf(pte,l) \
178 (l == (ARM_LPAE_MAX_LEVELS - 1) ? \
179 (iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_PAGE) : \
180 (iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_BLOCK))
181
182#define iopte_to_pfn(pte,d) \
183 (((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1)) >> (d)->pg_shift)
184
185#define pfn_to_iopte(pfn,d) \
186 (((pfn) << (d)->pg_shift) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1))
187
188struct arm_lpae_io_pgtable {
189 struct io_pgtable iop;
190
191 int levels;
192 size_t pgd_size;
193 unsigned long pg_shift;
194 unsigned long bits_per_level;
195
196 void *pgd;
197};
198
199typedef u64 arm_lpae_iopte;
200
fe4b991d
WD		 201static bool selftest_running = false;
202
f8d54961
RM		 203static dma_addr_t __arm_lpae_dma_addr(struct device *dev, void *pages)
204{
205 return phys_to_dma(dev, virt_to_phys(pages));
206}
207
208static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,
209 struct io_pgtable_cfg *cfg)
210{
211 struct device *dev = cfg->iommu_dev;
212 dma_addr_t dma;
213 void *pages = alloc_pages_exact(size, gfp | __GFP_ZERO);
214
215 if (!pages)
216 return NULL;
217
218 if (dev) {
219 dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE);
220 if (dma_mapping_error(dev, dma))
221 goto out_free;
222 /*
223 * We depend on the IOMMU being able to work with any physical
224 * address directly, so if the DMA layer suggests it can't by
225 * giving us back some translation, that bodes very badly...
226 */
227 if (dma != __arm_lpae_dma_addr(dev, pages))
228 goto out_unmap;
229 }
230
231 return pages;
232
233out_unmap:
234 dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n");
235 dma_unmap_single(dev, dma, size, DMA_TO_DEVICE);
236out_free:
237 free_pages_exact(pages, size);
238 return NULL;
239}
240
241static void __arm_lpae_free_pages(void *pages, size_t size,
242 struct io_pgtable_cfg *cfg)
243{
244 struct device *dev = cfg->iommu_dev;
245
246 if (dev)
247 dma_unmap_single(dev, __arm_lpae_dma_addr(dev, pages),
248 size, DMA_TO_DEVICE);
249 free_pages_exact(pages, size);
250}
251
252static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte,
253 struct io_pgtable_cfg *cfg, void *cookie)
254{
255 struct device *dev = cfg->iommu_dev;
256
257 *ptep = pte;
258
259 if (dev)
260 dma_sync_single_for_device(dev, __arm_lpae_dma_addr(dev, ptep),
261 sizeof(pte), DMA_TO_DEVICE);
262 else if (cfg->tlb->flush_pgtable)
263 cfg->tlb->flush_pgtable(ptep, sizeof(pte), cookie);
264}
265
e1d3c0fd
WD		 266static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
267 unsigned long iova, phys_addr_t paddr,
268 arm_lpae_iopte prot, int lvl,
269 arm_lpae_iopte *ptep)
270{
271 arm_lpae_iopte pte = prot;
f8d54961 272 struct io_pgtable_cfg *cfg = &data->iop.cfg;
e1d3c0fd
WD		 273
274 /* We require an unmap first */
fe4b991d
WD		 275 if (iopte_leaf(*ptep, lvl)) {
276 WARN_ON(!selftest_running);
e1d3c0fd 277 return -EEXIST;
fe4b991d 278 }
e1d3c0fd 279
f8d54961 280 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)
c896c132
LP		 281 pte |= ARM_LPAE_PTE_NS;
282
e1d3c0fd
WD		 283 if (lvl == ARM_LPAE_MAX_LEVELS - 1)
284 pte |= ARM_LPAE_PTE_TYPE_PAGE;
285 else
286 pte |= ARM_LPAE_PTE_TYPE_BLOCK;
287
288 pte |= ARM_LPAE_PTE_AF | ARM_LPAE_PTE_SH_IS;
289 pte |= pfn_to_iopte(paddr >> data->pg_shift, data);
290
f8d54961 291 __arm_lpae_set_pte(ptep, pte, cfg, data->iop.cookie);
e1d3c0fd
WD		 292 return 0;
293}
294
295static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
296 phys_addr_t paddr, size_t size, arm_lpae_iopte prot,
297 int lvl, arm_lpae_iopte *ptep)
298{
299 arm_lpae_iopte *cptep, pte;
300 void *cookie = data->iop.cookie;
301 size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data);
f8d54961 302 struct io_pgtable_cfg *cfg = &data->iop.cfg;
e1d3c0fd
WD		 303
304 /* Find our entry at the current level */
305 ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);
306
307 /* If we can install a leaf entry at this level, then do so */
f8d54961 308 if (size == block_size && (size & cfg->pgsize_bitmap))
e1d3c0fd
WD		 309 return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep);
310
311 /* We can't allocate tables at the final level */
312 if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1))
313 return -EINVAL;
314
315 /* Grab a pointer to the next level */
316 pte = *ptep;
317 if (!pte) {
f8d54961
RM		 318 cptep = __arm_lpae_alloc_pages(1UL << data->pg_shift,
319 GFP_ATOMIC, cfg);
e1d3c0fd
WD		 320 if (!cptep)
321 return -ENOMEM;
322
e1d3c0fd 323 pte = __pa(cptep) | ARM_LPAE_PTE_TYPE_TABLE;
f8d54961 324 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)
c896c132 325 pte |= ARM_LPAE_PTE_NSTABLE;
f8d54961 326 __arm_lpae_set_pte(ptep, pte, cfg, cookie);
e1d3c0fd
WD		 327 } else {
328 cptep = iopte_deref(pte, data);
329 }
330
331 /* Rinse, repeat */
332 return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep);
333}
334
335static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data,
336 int prot)
337{
338 arm_lpae_iopte pte;
339
340 if (data->iop.fmt == ARM_64_LPAE_S1 ||
341 data->iop.fmt == ARM_32_LPAE_S1) {
342 pte = ARM_LPAE_PTE_AP_UNPRIV | ARM_LPAE_PTE_nG;
343
344 if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ))
345 pte |= ARM_LPAE_PTE_AP_RDONLY;
346
347 if (prot & IOMMU_CACHE)
348 pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE
349 << ARM_LPAE_PTE_ATTRINDX_SHIFT);
350 } else {
351 pte = ARM_LPAE_PTE_HAP_FAULT;
352 if (prot & IOMMU_READ)
353 pte |= ARM_LPAE_PTE_HAP_READ;
354 if (prot & IOMMU_WRITE)
355 pte |= ARM_LPAE_PTE_HAP_WRITE;
356 if (prot & IOMMU_CACHE)
357 pte |= ARM_LPAE_PTE_MEMATTR_OIWB;
358 else
359 pte |= ARM_LPAE_PTE_MEMATTR_NC;
360 }
361
362 if (prot & IOMMU_NOEXEC)
363 pte |= ARM_LPAE_PTE_XN;
364
365 return pte;
366}
367
368static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova,
369 phys_addr_t paddr, size_t size, int iommu_prot)
370{
371 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
372 arm_lpae_iopte *ptep = data->pgd;
373 int lvl = ARM_LPAE_START_LVL(data);
374 arm_lpae_iopte prot;
375
376 /* If no access, then nothing to do */
377 if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE)))
378 return 0;
379
380 prot = arm_lpae_prot_to_pte(data, iommu_prot);
381 return __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep);
382}
383
384static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
385 arm_lpae_iopte *ptep)
386{
387 arm_lpae_iopte *start, *end;
388 unsigned long table_size;
389
390 /* Only leaf entries at the last level */
391 if (lvl == ARM_LPAE_MAX_LEVELS - 1)
392 return;
393
394 if (lvl == ARM_LPAE_START_LVL(data))
395 table_size = data->pgd_size;
396 else
397 table_size = 1UL << data->pg_shift;
398
399 start = ptep;
400 end = (void *)ptep + table_size;
401
402 while (ptep != end) {
403 arm_lpae_iopte pte = *ptep++;
404
405 if (!pte || iopte_leaf(pte, lvl))
406 continue;
407
408 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));
409 }
410
f8d54961 411 __arm_lpae_free_pages(start, table_size, &data->iop.cfg);
e1d3c0fd
WD		 412}
413
414static void arm_lpae_free_pgtable(struct io_pgtable *iop)
415{
416 struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop);
417
418 __arm_lpae_free_pgtable(data, ARM_LPAE_START_LVL(data), data->pgd);
419 kfree(data);
420}
421
422static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
423 unsigned long iova, size_t size,
424 arm_lpae_iopte prot, int lvl,
425 arm_lpae_iopte *ptep, size_t blk_size)
426{
427 unsigned long blk_start, blk_end;
428 phys_addr_t blk_paddr;
429 arm_lpae_iopte table = 0;
f8d54961 430 struct io_pgtable_cfg *cfg = &data->iop.cfg;
e1d3c0fd 431 void *cookie = data->iop.cookie;
e1d3c0fd
WD		 432
433 blk_start = iova & ~(blk_size - 1);
434 blk_end = blk_start + blk_size;
435 blk_paddr = iopte_to_pfn(*ptep, data) << data->pg_shift;
436
437 for (; blk_start < blk_end; blk_start += size, blk_paddr += size) {
438 arm_lpae_iopte *tablep;
439
440 /* Unmap! */
441 if (blk_start == iova)
442 continue;
443
444 /* __arm_lpae_map expects a pointer to the start of the table */
445 tablep = &table - ARM_LPAE_LVL_IDX(blk_start, lvl, data);
446 if (__arm_lpae_map(data, blk_start, blk_paddr, size, prot, lvl,
447 tablep) < 0) {
448 if (table) {
449 /* Free the table we allocated */
450 tablep = iopte_deref(table, data);
451 __arm_lpae_free_pgtable(data, lvl + 1, tablep);
452 }
453 return 0; /* Bytes unmapped */
454 }
455 }
456
f8d54961 457 __arm_lpae_set_pte(ptep, table, cfg, cookie);
e1d3c0fd 458 iova &= ~(blk_size - 1);
f8d54961 459 cfg->tlb->tlb_add_flush(iova, blk_size, true, cookie);
e1d3c0fd
WD		 460 return size;
461}
462
463static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
464 unsigned long iova, size_t size, int lvl,
465 arm_lpae_iopte *ptep)
466{
467 arm_lpae_iopte pte;
468 const struct iommu_gather_ops *tlb = data->iop.cfg.tlb;
469 void *cookie = data->iop.cookie;
470 size_t blk_size = ARM_LPAE_BLOCK_SIZE(lvl, data);
471
472 ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);
473 pte = *ptep;
474
475 /* Something went horribly wrong and we ran out of page table */
476 if (WARN_ON(!pte || (lvl == ARM_LPAE_MAX_LEVELS)))
477 return 0;
478
479 /* If the size matches this level, we're in the right place */
480 if (size == blk_size) {
f8d54961 481 __arm_lpae_set_pte(ptep, 0, &data->iop.cfg, cookie);
e1d3c0fd
WD		 482
483 if (!iopte_leaf(pte, lvl)) {
484 /* Also flush any partial walks */
485 tlb->tlb_add_flush(iova, size, false, cookie);
f8d54961 486 tlb->tlb_sync(cookie);
e1d3c0fd
WD		 487 ptep = iopte_deref(pte, data);
488 __arm_lpae_free_pgtable(data, lvl + 1, ptep);
489 } else {
490 tlb->tlb_add_flush(iova, size, true, cookie);
491 }
492
493 return size;
494 } else if (iopte_leaf(pte, lvl)) {
495 /*
496 * Insert a table at the next level to map the old region,
497 * minus the part we want to unmap
498 */
499 return arm_lpae_split_blk_unmap(data, iova, size,
500 iopte_prot(pte), lvl, ptep,
501 blk_size);
502 }
503
504 /* Keep on walkin' */
505 ptep = iopte_deref(pte, data);
506 return __arm_lpae_unmap(data, iova, size, lvl + 1, ptep);
507}
508
509static int arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova,
510 size_t size)
511{
512 size_t unmapped;
513 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
514 struct io_pgtable *iop = &data->iop;
515 arm_lpae_iopte *ptep = data->pgd;
516 int lvl = ARM_LPAE_START_LVL(data);
517
518 unmapped = __arm_lpae_unmap(data, iova, size, lvl, ptep);
519 if (unmapped)
520 iop->cfg.tlb->tlb_sync(iop->cookie);
521
522 return unmapped;
523}
524
525static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops,
526 unsigned long iova)
527{
528 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
529 arm_lpae_iopte pte, *ptep = data->pgd;
530 int lvl = ARM_LPAE_START_LVL(data);
531
532 do {
533 /* Valid IOPTE pointer? */
534 if (!ptep)
535 return 0;
536
537 /* Grab the IOPTE we're interested in */
538 pte = *(ptep + ARM_LPAE_LVL_IDX(iova, lvl, data));
539
540 /* Valid entry? */
541 if (!pte)
542 return 0;
543
544 /* Leaf entry? */
545 if (iopte_leaf(pte,lvl))
546 goto found_translation;
547
548 /* Take it to the next level */
549 ptep = iopte_deref(pte, data);
550 } while (++lvl < ARM_LPAE_MAX_LEVELS);
551
552 /* Ran out of page tables to walk */
553 return 0;
554
555found_translation:
556 iova &= ((1 << data->pg_shift) - 1);
557 return ((phys_addr_t)iopte_to_pfn(pte,data) << data->pg_shift) | iova;
558}
559
560static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg)
561{
562 unsigned long granule;
563
564 /*
565 * We need to restrict the supported page sizes to match the
566 * translation regime for a particular granule. Aim to match
567 * the CPU page size if possible, otherwise prefer smaller sizes.
568 * While we're at it, restrict the block sizes to match the
569 * chosen granule.
570 */
571 if (cfg->pgsize_bitmap & PAGE_SIZE)
572 granule = PAGE_SIZE;
573 else if (cfg->pgsize_bitmap & ~PAGE_MASK)
574 granule = 1UL << __fls(cfg->pgsize_bitmap & ~PAGE_MASK);
575 else if (cfg->pgsize_bitmap & PAGE_MASK)
576 granule = 1UL << __ffs(cfg->pgsize_bitmap & PAGE_MASK);
577 else
578 granule = 0;
579
580 switch (granule) {
581 case SZ_4K:
582 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
583 break;
584 case SZ_16K:
585 cfg->pgsize_bitmap &= (SZ_16K | SZ_32M);
586 break;
587 case SZ_64K:
588 cfg->pgsize_bitmap &= (SZ_64K | SZ_512M);
589 break;
590 default:
591 cfg->pgsize_bitmap = 0;
592 }
593}
594
595static struct arm_lpae_io_pgtable *
596arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg)
597{
598 unsigned long va_bits, pgd_bits;
599 struct arm_lpae_io_pgtable *data;
600
601 arm_lpae_restrict_pgsizes(cfg);
602
603 if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K)))
604 return NULL;
605
606 if (cfg->ias > ARM_LPAE_MAX_ADDR_BITS)
607 return NULL;
608
609 if (cfg->oas > ARM_LPAE_MAX_ADDR_BITS)
610 return NULL;
611
612 data = kmalloc(sizeof(*data), GFP_KERNEL);
613 if (!data)
614 return NULL;
615
616 data->pg_shift = __ffs(cfg->pgsize_bitmap);
617 data->bits_per_level = data->pg_shift - ilog2(sizeof(arm_lpae_iopte));
618
619 va_bits = cfg->ias - data->pg_shift;
620 data->levels = DIV_ROUND_UP(va_bits, data->bits_per_level);
621
622 /* Calculate the actual size of our pgd (without concatenation) */
623 pgd_bits = va_bits - (data->bits_per_level * (data->levels - 1));
624 data->pgd_size = 1UL << (pgd_bits + ilog2(sizeof(arm_lpae_iopte)));
625
626 data->iop.ops = (struct io_pgtable_ops) {
627 .map = arm_lpae_map,
628 .unmap = arm_lpae_unmap,
629 .iova_to_phys = arm_lpae_iova_to_phys,
630 };
631
632 return data;
633}
634
635static struct io_pgtable *
636arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
637{
638 u64 reg;
639 struct arm_lpae_io_pgtable *data = arm_lpae_alloc_pgtable(cfg);
640
641 if (!data)
642 return NULL;
643
644 /* TCR */
645 reg = (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) |
646 (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) |
647 (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT);
648
649 switch (1 << data->pg_shift) {
650 case SZ_4K:
651 reg |= ARM_LPAE_TCR_TG0_4K;
652 break;
653 case SZ_16K:
654 reg |= ARM_LPAE_TCR_TG0_16K;
655 break;
656 case SZ_64K:
657 reg |= ARM_LPAE_TCR_TG0_64K;
658 break;
659 }
660
661 switch (cfg->oas) {
662 case 32:
663 reg |= (ARM_LPAE_TCR_PS_32_BIT << ARM_LPAE_TCR_IPS_SHIFT);
664 break;
665 case 36:
666 reg |= (ARM_LPAE_TCR_PS_36_BIT << ARM_LPAE_TCR_IPS_SHIFT);
667 break;
668 case 40:
669 reg |= (ARM_LPAE_TCR_PS_40_BIT << ARM_LPAE_TCR_IPS_SHIFT);
670 break;
671 case 42:
672 reg |= (ARM_LPAE_TCR_PS_42_BIT << ARM_LPAE_TCR_IPS_SHIFT);
673 break;
674 case 44:
675 reg |= (ARM_LPAE_TCR_PS_44_BIT << ARM_LPAE_TCR_IPS_SHIFT);
676 break;
677 case 48:
678 reg |= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_IPS_SHIFT);
679 break;
680 default:
681 goto out_free_data;
682 }
683
684 reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT;
63979b8d
WD		 685
686 /* Disable speculative walks through TTBR1 */
687 reg |= ARM_LPAE_TCR_EPD1;
e1d3c0fd
WD		 688 cfg->arm_lpae_s1_cfg.tcr = reg;
689
690 /* MAIRs */
691 reg = (ARM_LPAE_MAIR_ATTR_NC
692 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) |
693 (ARM_LPAE_MAIR_ATTR_WBRWA
694 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) |
695 (ARM_LPAE_MAIR_ATTR_DEVICE
696 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV));
697
698 cfg->arm_lpae_s1_cfg.mair[0] = reg;
699 cfg->arm_lpae_s1_cfg.mair[1] = 0;
700
701 /* Looking good; allocate a pgd */
f8d54961 702 data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg);
e1d3c0fd
WD		 703 if (!data->pgd)
704 goto out_free_data;
705
f8d54961
RM		 706 if (cfg->tlb->flush_pgtable)
707 cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie);
e1d3c0fd
WD		 708
709 /* TTBRs */
710 cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd);
711 cfg->arm_lpae_s1_cfg.ttbr[1] = 0;
712 return &data->iop;
713
714out_free_data:
715 kfree(data);
716 return NULL;
717}
718
719static struct io_pgtable *
720arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
721{
722 u64 reg, sl;
723 struct arm_lpae_io_pgtable *data = arm_lpae_alloc_pgtable(cfg);
724
725 if (!data)
726 return NULL;
727
728 /*
729 * Concatenate PGDs at level 1 if possible in order to reduce
730 * the depth of the stage-2 walk.
731 */
732 if (data->levels == ARM_LPAE_MAX_LEVELS) {
733 unsigned long pgd_pages;
734
735 pgd_pages = data->pgd_size >> ilog2(sizeof(arm_lpae_iopte));
736 if (pgd_pages <= ARM_LPAE_S2_MAX_CONCAT_PAGES) {
737 data->pgd_size = pgd_pages << data->pg_shift;
738 data->levels--;
739 }
740 }
741
742 /* VTCR */
743 reg = ARM_64_LPAE_S2_TCR_RES1 |
744 (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) |
745 (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) |
746 (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT);
747
748 sl = ARM_LPAE_START_LVL(data);
749
750 switch (1 << data->pg_shift) {
751 case SZ_4K:
752 reg |= ARM_LPAE_TCR_TG0_4K;
753 sl++; /* SL0 format is different for 4K granule size */
754 break;
755 case SZ_16K:
756 reg |= ARM_LPAE_TCR_TG0_16K;
757 break;
758 case SZ_64K:
759 reg |= ARM_LPAE_TCR_TG0_64K;
760 break;
761 }
762
763 switch (cfg->oas) {
764 case 32:
765 reg |= (ARM_LPAE_TCR_PS_32_BIT << ARM_LPAE_TCR_PS_SHIFT);
766 break;
767 case 36:
768 reg |= (ARM_LPAE_TCR_PS_36_BIT << ARM_LPAE_TCR_PS_SHIFT);
769 break;
770 case 40:
771 reg |= (ARM_LPAE_TCR_PS_40_BIT << ARM_LPAE_TCR_PS_SHIFT);
772 break;
773 case 42:
774 reg |= (ARM_LPAE_TCR_PS_42_BIT << ARM_LPAE_TCR_PS_SHIFT);
775 break;
776 case 44:
777 reg |= (ARM_LPAE_TCR_PS_44_BIT << ARM_LPAE_TCR_PS_SHIFT);
778 break;
779 case 48:
780 reg |= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_PS_SHIFT);
781 break;
782 default:
783 goto out_free_data;
784 }
785
786 reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT;
787 reg |= (~sl & ARM_LPAE_TCR_SL0_MASK) << ARM_LPAE_TCR_SL0_SHIFT;
788 cfg->arm_lpae_s2_cfg.vtcr = reg;
789
790 /* Allocate pgd pages */
f8d54961 791 data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg);
e1d3c0fd
WD		 792 if (!data->pgd)
793 goto out_free_data;
794
f8d54961
RM		 795 if (cfg->tlb->flush_pgtable)
796 cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie);
e1d3c0fd
WD		 797
798 /* VTTBR */
799 cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd);
800 return &data->iop;
801
802out_free_data:
803 kfree(data);
804 return NULL;
805}
806
807static struct io_pgtable *
808arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
809{
810 struct io_pgtable *iop;
811
812 if (cfg->ias > 32 || cfg->oas > 40)
813 return NULL;
814
815 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
816 iop = arm_64_lpae_alloc_pgtable_s1(cfg, cookie);
817 if (iop) {
818 cfg->arm_lpae_s1_cfg.tcr |= ARM_32_LPAE_TCR_EAE;
819 cfg->arm_lpae_s1_cfg.tcr &= 0xffffffff;
820 }
821
822 return iop;
823}
824
825static struct io_pgtable *
826arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
827{
828 struct io_pgtable *iop;
829
830 if (cfg->ias > 40 || cfg->oas > 40)
831 return NULL;
832
833 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
834 iop = arm_64_lpae_alloc_pgtable_s2(cfg, cookie);
835 if (iop)
836 cfg->arm_lpae_s2_cfg.vtcr &= 0xffffffff;
837
838 return iop;
839}
840
841struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns = {
842 .alloc = arm_64_lpae_alloc_pgtable_s1,
843 .free = arm_lpae_free_pgtable,
844};
845
846struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns = {
847 .alloc = arm_64_lpae_alloc_pgtable_s2,
848 .free = arm_lpae_free_pgtable,
849};
850
851struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns = {
852 .alloc = arm_32_lpae_alloc_pgtable_s1,
853 .free = arm_lpae_free_pgtable,
854};
855
856struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns = {
857 .alloc = arm_32_lpae_alloc_pgtable_s2,
858 .free = arm_lpae_free_pgtable,
859};
fe4b991d
WD		 860
861#ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE_SELFTEST
862
863static struct io_pgtable_cfg *cfg_cookie;
864
865static void dummy_tlb_flush_all(void *cookie)
866{
867 WARN_ON(cookie != cfg_cookie);
868}
869
870static void dummy_tlb_add_flush(unsigned long iova, size_t size, bool leaf,
871 void *cookie)
872{
873 WARN_ON(cookie != cfg_cookie);
874 WARN_ON(!(size & cfg_cookie->pgsize_bitmap));
875}
876
877static void dummy_tlb_sync(void *cookie)
878{
879 WARN_ON(cookie != cfg_cookie);
880}
881
882static void dummy_flush_pgtable(void *ptr, size_t size, void *cookie)
883{
884 WARN_ON(cookie != cfg_cookie);
885}
886
887static struct iommu_gather_ops dummy_tlb_ops __initdata = {
888 .tlb_flush_all = dummy_tlb_flush_all,
889 .tlb_add_flush = dummy_tlb_add_flush,
890 .tlb_sync = dummy_tlb_sync,
891 .flush_pgtable = dummy_flush_pgtable,
892};
893
894static void __init arm_lpae_dump_ops(struct io_pgtable_ops *ops)
895{
896 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
897 struct io_pgtable_cfg *cfg = &data->iop.cfg;
898
899 pr_err("cfg: pgsize_bitmap 0x%lx, ias %u-bit\n",
900 cfg->pgsize_bitmap, cfg->ias);
901 pr_err("data: %d levels, 0x%zx pgd_size, %lu pg_shift, %lu bits_per_level, pgd @ %p\n",
902 data->levels, data->pgd_size, data->pg_shift,
903 data->bits_per_level, data->pgd);
904}
905
906#define __FAIL(ops, i) ({ \
907 WARN(1, "selftest: test failed for fmt idx %d\n", (i)); \
908 arm_lpae_dump_ops(ops); \
909 selftest_running = false; \
910 -EFAULT; \
911})
912
913static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
914{
915 static const enum io_pgtable_fmt fmts[] = {
916 ARM_64_LPAE_S1,
917 ARM_64_LPAE_S2,
918 };
919
920 int i, j;
921 unsigned long iova;
922 size_t size;
923 struct io_pgtable_ops *ops;
924
925 selftest_running = true;
926
927 for (i = 0; i < ARRAY_SIZE(fmts); ++i) {
928 cfg_cookie = cfg;
929 ops = alloc_io_pgtable_ops(fmts[i], cfg, cfg);
930 if (!ops) {
931 pr_err("selftest: failed to allocate io pgtable ops\n");
932 return -ENOMEM;
933 }
934
935 /*
936 * Initial sanity checks.
937 * Empty page tables shouldn't provide any translations.
938 */
939 if (ops->iova_to_phys(ops, 42))
940 return __FAIL(ops, i);
941
942 if (ops->iova_to_phys(ops, SZ_1G + 42))
943 return __FAIL(ops, i);
944
945 if (ops->iova_to_phys(ops, SZ_2G + 42))
946 return __FAIL(ops, i);
947
948 /*
949 * Distinct mappings of different granule sizes.
950 */
951 iova = 0;
952 j = find_first_bit(&cfg->pgsize_bitmap, BITS_PER_LONG);
953 while (j != BITS_PER_LONG) {
954 size = 1UL << j;
955
956 if (ops->map(ops, iova, iova, size, IOMMU_READ |
957 IOMMU_WRITE |
958 IOMMU_NOEXEC |
959 IOMMU_CACHE))
960 return __FAIL(ops, i);
961
962 /* Overlapping mappings */
963 if (!ops->map(ops, iova, iova + size, size,
964 IOMMU_READ | IOMMU_NOEXEC))
965 return __FAIL(ops, i);
966
967 if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))
968 return __FAIL(ops, i);
969
970 iova += SZ_1G;
971 j++;
972 j = find_next_bit(&cfg->pgsize_bitmap, BITS_PER_LONG, j);
973 }
974
975 /* Partial unmap */
976 size = 1UL << __ffs(cfg->pgsize_bitmap);
977 if (ops->unmap(ops, SZ_1G + size, size) != size)
978 return __FAIL(ops, i);
979
980 /* Remap of partial unmap */
981 if (ops->map(ops, SZ_1G + size, size, size, IOMMU_READ))
982 return __FAIL(ops, i);
983
984 if (ops->iova_to_phys(ops, SZ_1G + size + 42) != (size + 42))
985 return __FAIL(ops, i);
986
987 /* Full unmap */
988 iova = 0;
989 j = find_first_bit(&cfg->pgsize_bitmap, BITS_PER_LONG);
990 while (j != BITS_PER_LONG) {
991 size = 1UL << j;
992
993 if (ops->unmap(ops, iova, size) != size)
994 return __FAIL(ops, i);
995
996 if (ops->iova_to_phys(ops, iova + 42))
997 return __FAIL(ops, i);
998
999 /* Remap full block */
1000 if (ops->map(ops, iova, iova, size, IOMMU_WRITE))
1001 return __FAIL(ops, i);
1002
1003 if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))
1004 return __FAIL(ops, i);
1005
1006 iova += SZ_1G;
1007 j++;
1008 j = find_next_bit(&cfg->pgsize_bitmap, BITS_PER_LONG, j);
1009 }
1010
1011 free_io_pgtable_ops(ops);
1012 }
1013
1014 selftest_running = false;
1015 return 0;
1016}
1017
1018static int __init arm_lpae_do_selftests(void)
1019{
1020 static const unsigned long pgsize[] = {
1021 SZ_4K | SZ_2M | SZ_1G,
1022 SZ_16K | SZ_32M,
1023 SZ_64K | SZ_512M,
1024 };
1025
1026 static const unsigned int ias[] = {
1027 32, 36, 40, 42, 44, 48,
1028 };
1029
1030 int i, j, pass = 0, fail = 0;
1031 struct io_pgtable_cfg cfg = {
1032 .tlb = &dummy_tlb_ops,
1033 .oas = 48,
1034 };
1035
1036 for (i = 0; i < ARRAY_SIZE(pgsize); ++i) {
1037 for (j = 0; j < ARRAY_SIZE(ias); ++j) {
1038 cfg.pgsize_bitmap = pgsize[i];
1039 cfg.ias = ias[j];
1040 pr_info("selftest: pgsize_bitmap 0x%08lx, IAS %u\n",
1041 pgsize[i], ias[j]);
1042 if (arm_lpae_run_tests(&cfg))
1043 fail++;
1044 else
1045 pass++;
1046 }
1047 }
1048
1049 pr_info("selftest: completed with %d PASS %d FAIL\n", pass, fail);
1050 return fail ? -EFAULT : 0;
1051}
1052subsys_initcall(arm_lpae_do_selftests);
1053#endif
Ubuntu Focal Linux kernel mirror