]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - drivers/iommu/arm-smmu.c
projects / mirror_ubuntu-jammy-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
iommu/amd: Fix amd_iommu=force_isolation
[mirror_ubuntu-jammy-kernel.git] / drivers / iommu / arm-smmu.c
1 /*
2 * IOMMU API for ARM architected SMMU implementations.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
16 *
17 * Copyright (C) 2013 ARM Limited
18 *
19 * Author: Will Deacon <will.deacon@arm.com>
20 *
21 * This driver currently supports:
22 * - SMMUv1 and v2 implementations
23 * - Stream-matching and stream-indexing
24 * - v7/v8 long-descriptor format
25 * - Non-secure access to the SMMU
26 * - Context fault reporting
27 * - Extended Stream ID (16 bit)
28 */
29
30 #define pr_fmt(fmt) "arm-smmu: " fmt
31
32 #include <linux/acpi.h>
33 #include <linux/acpi_iort.h>
34 #include <linux/atomic.h>
35 #include <linux/delay.h>
36 #include <linux/dma-iommu.h>
37 #include <linux/dma-mapping.h>
38 #include <linux/err.h>
39 #include <linux/interrupt.h>
40 #include <linux/io.h>
41 #include <linux/io-64-nonatomic-hi-lo.h>
42 #include <linux/iommu.h>
43 #include <linux/iopoll.h>
44 #include <linux/module.h>
45 #include <linux/of.h>
46 #include <linux/of_address.h>
47 #include <linux/of_device.h>
48 #include <linux/of_iommu.h>
49 #include <linux/pci.h>
50 #include <linux/platform_device.h>
51 #include <linux/slab.h>
52 #include <linux/spinlock.h>
53
54 #include <linux/amba/bus.h>
55 #include <linux/fsl/mc.h>
56
57 #include "io-pgtable.h"
58 #include "arm-smmu-regs.h"
59
60 #define ARM_MMU500_ACTLR_CPRE (1 << 1)
61
62 #define ARM_MMU500_ACR_CACHE_LOCK (1 << 26)
63 #define ARM_MMU500_ACR_S2CRB_TLBEN (1 << 10)
64 #define ARM_MMU500_ACR_SMTNMB_TLBEN (1 << 8)
65
66 #define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
67 #define TLB_SPIN_COUNT 10
68
69 /* Maximum number of context banks per SMMU */
70 #define ARM_SMMU_MAX_CBS 128
71
72 /* SMMU global address space */
73 #define ARM_SMMU_GR0(smmu) ((smmu)->base)
74 #define ARM_SMMU_GR1(smmu) ((smmu)->base + (1 << (smmu)->pgshift))
75
76 /*
77 * SMMU global address space with conditional offset to access secure
78 * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
79 * nsGFSYNR0: 0x450)
80 */
81 #define ARM_SMMU_GR0_NS(smmu) \
82 ((smmu)->base + \
83 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \
84 ? 0x400 : 0))
85
86 /*
87 * Some 64-bit registers only make sense to write atomically, but in such
88 * cases all the data relevant to AArch32 formats lies within the lower word,
89 * therefore this actually makes more sense than it might first appear.
90 */
91 #ifdef CONFIG_64BIT
92 #define smmu_write_atomic_lq writeq_relaxed
93 #else
94 #define smmu_write_atomic_lq writel_relaxed
95 #endif
96
97 /* Translation context bank */
98 #define ARM_SMMU_CB(smmu, n) ((smmu)->cb_base + ((n) << (smmu)->pgshift))
99
100 #define MSI_IOVA_BASE 0x8000000
101 #define MSI_IOVA_LENGTH 0x100000
102
103 static int force_stage;
104 module_param(force_stage, int, S_IRUGO);
105 MODULE_PARM_DESC(force_stage,
106 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
107 static bool disable_bypass;
108 module_param(disable_bypass, bool, S_IRUGO);
109 MODULE_PARM_DESC(disable_bypass,
110 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
111
112 enum arm_smmu_arch_version {
113 ARM_SMMU_V1,
114 ARM_SMMU_V1_64K,
115 ARM_SMMU_V2,
116 };
117
118 enum arm_smmu_implementation {
119 GENERIC_SMMU,
120 ARM_MMU500,
121 CAVIUM_SMMUV2,
122 };
123
124 struct arm_smmu_s2cr {
125 struct iommu_group *group;
126 int count;
127 enum arm_smmu_s2cr_type type;
128 enum arm_smmu_s2cr_privcfg privcfg;
129 u8 cbndx;
130 };
131
132 #define s2cr_init_val (struct arm_smmu_s2cr){ \
133 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
134 }
135
136 struct arm_smmu_smr {
137 u16 mask;
138 u16 id;
139 bool valid;
140 };
141
142 struct arm_smmu_cb {
143 u64 ttbr[2];
144 u32 tcr[2];
145 u32 mair[2];
146 struct arm_smmu_cfg *cfg;
147 };
148
149 struct arm_smmu_master_cfg {
150 struct arm_smmu_device *smmu;
151 s16 smendx[];
152 };
153 #define INVALID_SMENDX -1
154 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
155 #define fwspec_smmu(fw) (__fwspec_cfg(fw)->smmu)
156 #define fwspec_smendx(fw, i) \
157 (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
158 #define for_each_cfg_sme(fw, i, idx) \
159 for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
160
161 struct arm_smmu_device {
162 struct device *dev;
163
164 void __iomem *base;
165 void __iomem *cb_base;
166 unsigned long pgshift;
167
168 #define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0)
169 #define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1)
170 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 2)
171 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 3)
172 #define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4)
173 #define ARM_SMMU_FEAT_TRANS_OPS (1 << 5)
174 #define ARM_SMMU_FEAT_VMID16 (1 << 6)
175 #define ARM_SMMU_FEAT_FMT_AARCH64_4K (1 << 7)
176 #define ARM_SMMU_FEAT_FMT_AARCH64_16K (1 << 8)
177 #define ARM_SMMU_FEAT_FMT_AARCH64_64K (1 << 9)
178 #define ARM_SMMU_FEAT_FMT_AARCH32_L (1 << 10)
179 #define ARM_SMMU_FEAT_FMT_AARCH32_S (1 << 11)
180 #define ARM_SMMU_FEAT_EXIDS (1 << 12)
181 u32 features;
182
183 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
184 u32 options;
185 enum arm_smmu_arch_version version;
186 enum arm_smmu_implementation model;
187
188 u32 num_context_banks;
189 u32 num_s2_context_banks;
190 DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
191 struct arm_smmu_cb *cbs;
192 atomic_t irptndx;
193
194 u32 num_mapping_groups;
195 u16 streamid_mask;
196 u16 smr_mask_mask;
197 struct arm_smmu_smr *smrs;
198 struct arm_smmu_s2cr *s2crs;
199 struct mutex stream_map_mutex;
200
201 unsigned long va_size;
202 unsigned long ipa_size;
203 unsigned long pa_size;
204 unsigned long pgsize_bitmap;
205
206 u32 num_global_irqs;
207 u32 num_context_irqs;
208 unsigned int *irqs;
209
210 u32 cavium_id_base; /* Specific to Cavium */
211
212 spinlock_t global_sync_lock;
213
214 /* IOMMU core code handle */
215 struct iommu_device iommu;
216 };
217
218 enum arm_smmu_context_fmt {
219 ARM_SMMU_CTX_FMT_NONE,
220 ARM_SMMU_CTX_FMT_AARCH64,
221 ARM_SMMU_CTX_FMT_AARCH32_L,
222 ARM_SMMU_CTX_FMT_AARCH32_S,
223 };
224
225 struct arm_smmu_cfg {
226 u8 cbndx;
227 u8 irptndx;
228 union {
229 u16 asid;
230 u16 vmid;
231 };
232 u32 cbar;
233 enum arm_smmu_context_fmt fmt;
234 };
235 #define INVALID_IRPTNDX 0xff
236
237 enum arm_smmu_domain_stage {
238 ARM_SMMU_DOMAIN_S1 = 0,
239 ARM_SMMU_DOMAIN_S2,
240 ARM_SMMU_DOMAIN_NESTED,
241 ARM_SMMU_DOMAIN_BYPASS,
242 };
243
244 struct arm_smmu_domain {
245 struct arm_smmu_device *smmu;
246 struct io_pgtable_ops *pgtbl_ops;
247 const struct iommu_gather_ops *tlb_ops;
248 struct arm_smmu_cfg cfg;
249 enum arm_smmu_domain_stage stage;
250 bool non_strict;
251 struct mutex init_mutex; /* Protects smmu pointer */
252 spinlock_t cb_lock; /* Serialises ATS1* ops and TLB syncs */
253 struct iommu_domain domain;
254 };
255
256 struct arm_smmu_option_prop {
257 u32 opt;
258 const char *prop;
259 };
260
261 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
262
263 static bool using_legacy_binding, using_generic_binding;
264
265 static struct arm_smmu_option_prop arm_smmu_options[] = {
266 { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
267 { 0, NULL},
268 };
269
270 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
271 {
272 return container_of(dom, struct arm_smmu_domain, domain);
273 }
274
275 static void parse_driver_options(struct arm_smmu_device *smmu)
276 {
277 int i = 0;
278
279 do {
280 if (of_property_read_bool(smmu->dev->of_node,
281 arm_smmu_options[i].prop)) {
282 smmu->options |= arm_smmu_options[i].opt;
283 dev_notice(smmu->dev, "option %s\n",
284 arm_smmu_options[i].prop);
285 }
286 } while (arm_smmu_options[++i].opt);
287 }
288
289 static struct device_node *dev_get_dev_node(struct device *dev)
290 {
291 if (dev_is_pci(dev)) {
292 struct pci_bus *bus = to_pci_dev(dev)->bus;
293
294 while (!pci_is_root_bus(bus))
295 bus = bus->parent;
296 return of_node_get(bus->bridge->parent->of_node);
297 }
298
299 return of_node_get(dev->of_node);
300 }
301
302 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
303 {
304 *((__be32 *)data) = cpu_to_be32(alias);
305 return 0; /* Continue walking */
306 }
307
308 static int __find_legacy_master_phandle(struct device *dev, void *data)
309 {
310 struct of_phandle_iterator *it = *(void **)data;
311 struct device_node *np = it->node;
312 int err;
313
314 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
315 "#stream-id-cells", 0)
316 if (it->node == np) {
317 *(void **)data = dev;
318 return 1;
319 }
320 it->node = np;
321 return err == -ENOENT ? 0 : err;
322 }
323
324 static struct platform_driver arm_smmu_driver;
325 static struct iommu_ops arm_smmu_ops;
326
327 static int arm_smmu_register_legacy_master(struct device *dev,
328 struct arm_smmu_device **smmu)
329 {
330 struct device *smmu_dev;
331 struct device_node *np;
332 struct of_phandle_iterator it;
333 void *data = &it;
334 u32 *sids;
335 __be32 pci_sid;
336 int err;
337
338 np = dev_get_dev_node(dev);
339 if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
340 of_node_put(np);
341 return -ENODEV;
342 }
343
344 it.node = np;
345 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
346 __find_legacy_master_phandle);
347 smmu_dev = data;
348 of_node_put(np);
349 if (err == 0)
350 return -ENODEV;
351 if (err < 0)
352 return err;
353
354 if (dev_is_pci(dev)) {
355 /* "mmu-masters" assumes Stream ID == Requester ID */
356 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
357 &pci_sid);
358 it.cur = &pci_sid;
359 it.cur_count = 1;
360 }
361
362 err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
363 &arm_smmu_ops);
364 if (err)
365 return err;
366
367 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
368 if (!sids)
369 return -ENOMEM;
370
371 *smmu = dev_get_drvdata(smmu_dev);
372 of_phandle_iterator_args(&it, sids, it.cur_count);
373 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
374 kfree(sids);
375 return err;
376 }
377
378 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
379 {
380 int idx;
381
382 do {
383 idx = find_next_zero_bit(map, end, start);
384 if (idx == end)
385 return -ENOSPC;
386 } while (test_and_set_bit(idx, map));
387
388 return idx;
389 }
390
391 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
392 {
393 clear_bit(idx, map);
394 }
395
396 /* Wait for any pending TLB invalidations to complete */
397 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu,
398 void __iomem *sync, void __iomem *status)
399 {
400 unsigned int spin_cnt, delay;
401
402 writel_relaxed(0, sync);
403 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
404 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
405 if (!(readl_relaxed(status) & sTLBGSTATUS_GSACTIVE))
406 return;
407 cpu_relax();
408 }
409 udelay(delay);
410 }
411 dev_err_ratelimited(smmu->dev,
412 "TLB sync timed out -- SMMU may be deadlocked\n");
413 }
414
415 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
416 {
417 void __iomem *base = ARM_SMMU_GR0(smmu);
418 unsigned long flags;
419
420 spin_lock_irqsave(&smmu->global_sync_lock, flags);
421 __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_GR0_sTLBGSYNC,
422 base + ARM_SMMU_GR0_sTLBGSTATUS);
423 spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
424 }
425
426 static void arm_smmu_tlb_sync_context(void *cookie)
427 {
428 struct arm_smmu_domain *smmu_domain = cookie;
429 struct arm_smmu_device *smmu = smmu_domain->smmu;
430 void __iomem *base = ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx);
431 unsigned long flags;
432
433 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
434 __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_CB_TLBSYNC,
435 base + ARM_SMMU_CB_TLBSTATUS);
436 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
437 }
438
439 static void arm_smmu_tlb_sync_vmid(void *cookie)
440 {
441 struct arm_smmu_domain *smmu_domain = cookie;
442
443 arm_smmu_tlb_sync_global(smmu_domain->smmu);
444 }
445
446 static void arm_smmu_tlb_inv_context_s1(void *cookie)
447 {
448 struct arm_smmu_domain *smmu_domain = cookie;
449 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
450 void __iomem *base = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx);
451
452 /*
453 * NOTE: this is not a relaxed write; it needs to guarantee that PTEs
454 * cleared by the current CPU are visible to the SMMU before the TLBI.
455 */
456 writel(cfg->asid, base + ARM_SMMU_CB_S1_TLBIASID);
457 arm_smmu_tlb_sync_context(cookie);
458 }
459
460 static void arm_smmu_tlb_inv_context_s2(void *cookie)
461 {
462 struct arm_smmu_domain *smmu_domain = cookie;
463 struct arm_smmu_device *smmu = smmu_domain->smmu;
464 void __iomem *base = ARM_SMMU_GR0(smmu);
465
466 /* NOTE: see above */
467 writel(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID);
468 arm_smmu_tlb_sync_global(smmu);
469 }
470
471 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
472 size_t granule, bool leaf, void *cookie)
473 {
474 struct arm_smmu_domain *smmu_domain = cookie;
475 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
476 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
477 void __iomem *reg = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx);
478
479 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
480 wmb();
481
482 if (stage1) {
483 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
484
485 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
486 iova &= ~12UL;
487 iova |= cfg->asid;
488 do {
489 writel_relaxed(iova, reg);
490 iova += granule;
491 } while (size -= granule);
492 } else {
493 iova >>= 12;
494 iova |= (u64)cfg->asid << 48;
495 do {
496 writeq_relaxed(iova, reg);
497 iova += granule >> 12;
498 } while (size -= granule);
499 }
500 } else {
501 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
502 ARM_SMMU_CB_S2_TLBIIPAS2;
503 iova >>= 12;
504 do {
505 smmu_write_atomic_lq(iova, reg);
506 iova += granule >> 12;
507 } while (size -= granule);
508 }
509 }
510
511 /*
512 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
513 * almost negligible, but the benefit of getting the first one in as far ahead
514 * of the sync as possible is significant, hence we don't just make this a
515 * no-op and set .tlb_sync to arm_smmu_inv_context_s2() as you might think.
516 */
517 static void arm_smmu_tlb_inv_vmid_nosync(unsigned long iova, size_t size,
518 size_t granule, bool leaf, void *cookie)
519 {
520 struct arm_smmu_domain *smmu_domain = cookie;
521 void __iomem *base = ARM_SMMU_GR0(smmu_domain->smmu);
522
523 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
524 wmb();
525
526 writel_relaxed(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID);
527 }
528
529 static const struct iommu_gather_ops arm_smmu_s1_tlb_ops = {
530 .tlb_flush_all = arm_smmu_tlb_inv_context_s1,
531 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
532 .tlb_sync = arm_smmu_tlb_sync_context,
533 };
534
535 static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v2 = {
536 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
537 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
538 .tlb_sync = arm_smmu_tlb_sync_context,
539 };
540
541 static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v1 = {
542 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
543 .tlb_add_flush = arm_smmu_tlb_inv_vmid_nosync,
544 .tlb_sync = arm_smmu_tlb_sync_vmid,
545 };
546
547 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
548 {
549 u32 fsr, fsynr;
550 unsigned long iova;
551 struct iommu_domain *domain = dev;
552 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
553 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
554 struct arm_smmu_device *smmu = smmu_domain->smmu;
555 void __iomem *cb_base;
556
557 cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
558 fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
559
560 if (!(fsr & FSR_FAULT))
561 return IRQ_NONE;
562
563 fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
564 iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
565
566 dev_err_ratelimited(smmu->dev,
567 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
568 fsr, iova, fsynr, cfg->cbndx);
569
570 writel(fsr, cb_base + ARM_SMMU_CB_FSR);
571 return IRQ_HANDLED;
572 }
573
574 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
575 {
576 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
577 struct arm_smmu_device *smmu = dev;
578 void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
579
580 gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
581 gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
582 gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
583 gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
584
585 if (!gfsr)
586 return IRQ_NONE;
587
588 dev_err_ratelimited(smmu->dev,
589 "Unexpected global fault, this could be serious\n");
590 dev_err_ratelimited(smmu->dev,
591 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
592 gfsr, gfsynr0, gfsynr1, gfsynr2);
593
594 writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
595 return IRQ_HANDLED;
596 }
597
598 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
599 struct io_pgtable_cfg *pgtbl_cfg)
600 {
601 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
602 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
603 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
604
605 cb->cfg = cfg;
606
607 /* TTBCR */
608 if (stage1) {
609 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
610 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
611 } else {
612 cb->tcr[0] = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
613 cb->tcr[1] = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
614 cb->tcr[1] |= TTBCR2_SEP_UPSTREAM;
615 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
616 cb->tcr[1] |= TTBCR2_AS;
617 }
618 } else {
619 cb->tcr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
620 }
621
622 /* TTBRs */
623 if (stage1) {
624 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
625 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
626 cb->ttbr[1] = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
627 } else {
628 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
629 cb->ttbr[0] |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
630 cb->ttbr[1] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
631 cb->ttbr[1] |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
632 }
633 } else {
634 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
635 }
636
637 /* MAIRs (stage-1 only) */
638 if (stage1) {
639 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
640 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
641 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
642 } else {
643 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
644 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
645 }
646 }
647 }
648
649 static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
650 {
651 u32 reg;
652 bool stage1;
653 struct arm_smmu_cb *cb = &smmu->cbs[idx];
654 struct arm_smmu_cfg *cfg = cb->cfg;
655 void __iomem *cb_base, *gr1_base;
656
657 cb_base = ARM_SMMU_CB(smmu, idx);
658
659 /* Unassigned context banks only need disabling */
660 if (!cfg) {
661 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
662 return;
663 }
664
665 gr1_base = ARM_SMMU_GR1(smmu);
666 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
667
668 /* CBA2R */
669 if (smmu->version > ARM_SMMU_V1) {
670 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
671 reg = CBA2R_RW64_64BIT;
672 else
673 reg = CBA2R_RW64_32BIT;
674 /* 16-bit VMIDs live in CBA2R */
675 if (smmu->features & ARM_SMMU_FEAT_VMID16)
676 reg |= cfg->vmid << CBA2R_VMID_SHIFT;
677
678 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(idx));
679 }
680
681 /* CBAR */
682 reg = cfg->cbar;
683 if (smmu->version < ARM_SMMU_V2)
684 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
685
686 /*
687 * Use the weakest shareability/memory types, so they are
688 * overridden by the ttbcr/pte.
689 */
690 if (stage1) {
691 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
692 (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
693 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
694 /* 8-bit VMIDs live in CBAR */
695 reg |= cfg->vmid << CBAR_VMID_SHIFT;
696 }
697 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(idx));
698
699 /*
700 * TTBCR
701 * We must write this before the TTBRs, since it determines the
702 * access behaviour of some fields (in particular, ASID[15:8]).
703 */
704 if (stage1 && smmu->version > ARM_SMMU_V1)
705 writel_relaxed(cb->tcr[1], cb_base + ARM_SMMU_CB_TTBCR2);
706 writel_relaxed(cb->tcr[0], cb_base + ARM_SMMU_CB_TTBCR);
707
708 /* TTBRs */
709 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
710 writel_relaxed(cfg->asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
711 writel_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0);
712 writel_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1);
713 } else {
714 writeq_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0);
715 if (stage1)
716 writeq_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1);
717 }
718
719 /* MAIRs (stage-1 only) */
720 if (stage1) {
721 writel_relaxed(cb->mair[0], cb_base + ARM_SMMU_CB_S1_MAIR0);
722 writel_relaxed(cb->mair[1], cb_base + ARM_SMMU_CB_S1_MAIR1);
723 }
724
725 /* SCTLR */
726 reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
727 if (stage1)
728 reg |= SCTLR_S1_ASIDPNE;
729 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
730 reg |= SCTLR_E;
731
732 writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
733 }
734
735 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
736 struct arm_smmu_device *smmu)
737 {
738 int irq, start, ret = 0;
739 unsigned long ias, oas;
740 struct io_pgtable_ops *pgtbl_ops;
741 struct io_pgtable_cfg pgtbl_cfg;
742 enum io_pgtable_fmt fmt;
743 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
744 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
745
746 mutex_lock(&smmu_domain->init_mutex);
747 if (smmu_domain->smmu)
748 goto out_unlock;
749
750 if (domain->type == IOMMU_DOMAIN_IDENTITY) {
751 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
752 smmu_domain->smmu = smmu;
753 goto out_unlock;
754 }
755
756 /*
757 * Mapping the requested stage onto what we support is surprisingly
758 * complicated, mainly because the spec allows S1+S2 SMMUs without
759 * support for nested translation. That means we end up with the
760 * following table:
761 *
762 * Requested Supported Actual
763 * S1 N S1
764 * S1 S1+S2 S1
765 * S1 S2 S2
766 * S1 S1 S1
767 * N N N
768 * N S1+S2 S2
769 * N S2 S2
770 * N S1 S1
771 *
772 * Note that you can't actually request stage-2 mappings.
773 */
774 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
775 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
776 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
777 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
778
779 /*
780 * Choosing a suitable context format is even more fiddly. Until we
781 * grow some way for the caller to express a preference, and/or move
782 * the decision into the io-pgtable code where it arguably belongs,
783 * just aim for the closest thing to the rest of the system, and hope
784 * that the hardware isn't esoteric enough that we can't assume AArch64
785 * support to be a superset of AArch32 support...
786 */
787 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
788 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
789 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
790 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
791 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
792 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
793 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
794 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
795 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
796 ARM_SMMU_FEAT_FMT_AARCH64_16K |
797 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
798 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
799
800 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
801 ret = -EINVAL;
802 goto out_unlock;
803 }
804
805 switch (smmu_domain->stage) {
806 case ARM_SMMU_DOMAIN_S1:
807 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
808 start = smmu->num_s2_context_banks;
809 ias = smmu->va_size;
810 oas = smmu->ipa_size;
811 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
812 fmt = ARM_64_LPAE_S1;
813 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
814 fmt = ARM_32_LPAE_S1;
815 ias = min(ias, 32UL);
816 oas = min(oas, 40UL);
817 } else {
818 fmt = ARM_V7S;
819 ias = min(ias, 32UL);
820 oas = min(oas, 32UL);
821 }
822 smmu_domain->tlb_ops = &arm_smmu_s1_tlb_ops;
823 break;
824 case ARM_SMMU_DOMAIN_NESTED:
825 /*
826 * We will likely want to change this if/when KVM gets
827 * involved.
828 */
829 case ARM_SMMU_DOMAIN_S2:
830 cfg->cbar = CBAR_TYPE_S2_TRANS;
831 start = 0;
832 ias = smmu->ipa_size;
833 oas = smmu->pa_size;
834 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
835 fmt = ARM_64_LPAE_S2;
836 } else {
837 fmt = ARM_32_LPAE_S2;
838 ias = min(ias, 40UL);
839 oas = min(oas, 40UL);
840 }
841 if (smmu->version == ARM_SMMU_V2)
842 smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v2;
843 else
844 smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v1;
845 break;
846 default:
847 ret = -EINVAL;
848 goto out_unlock;
849 }
850 ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
851 smmu->num_context_banks);
852 if (ret < 0)
853 goto out_unlock;
854
855 cfg->cbndx = ret;
856 if (smmu->version < ARM_SMMU_V2) {
857 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
858 cfg->irptndx %= smmu->num_context_irqs;
859 } else {
860 cfg->irptndx = cfg->cbndx;
861 }
862
863 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
864 cfg->vmid = cfg->cbndx + 1 + smmu->cavium_id_base;
865 else
866 cfg->asid = cfg->cbndx + smmu->cavium_id_base;
867
868 pgtbl_cfg = (struct io_pgtable_cfg) {
869 .pgsize_bitmap = smmu->pgsize_bitmap,
870 .ias = ias,
871 .oas = oas,
872 .tlb = smmu_domain->tlb_ops,
873 .iommu_dev = smmu->dev,
874 };
875
876 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
877 pgtbl_cfg.quirks = IO_PGTABLE_QUIRK_NO_DMA;
878
879 if (smmu_domain->non_strict)
880 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
881
882 smmu_domain->smmu = smmu;
883 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
884 if (!pgtbl_ops) {
885 ret = -ENOMEM;
886 goto out_clear_smmu;
887 }
888
889 /* Update the domain's page sizes to reflect the page table format */
890 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
891 domain->geometry.aperture_end = (1UL << ias) - 1;
892 domain->geometry.force_aperture = true;
893
894 /* Initialise the context bank with our page table cfg */
895 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
896 arm_smmu_write_context_bank(smmu, cfg->cbndx);
897
898 /*
899 * Request context fault interrupt. Do this last to avoid the
900 * handler seeing a half-initialised domain state.
901 */
902 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
903 ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
904 IRQF_SHARED, "arm-smmu-context-fault", domain);
905 if (ret < 0) {
906 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
907 cfg->irptndx, irq);
908 cfg->irptndx = INVALID_IRPTNDX;
909 }
910
911 mutex_unlock(&smmu_domain->init_mutex);
912
913 /* Publish page table ops for map/unmap */
914 smmu_domain->pgtbl_ops = pgtbl_ops;
915 return 0;
916
917 out_clear_smmu:
918 smmu_domain->smmu = NULL;
919 out_unlock:
920 mutex_unlock(&smmu_domain->init_mutex);
921 return ret;
922 }
923
924 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
925 {
926 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
927 struct arm_smmu_device *smmu = smmu_domain->smmu;
928 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
929 int irq;
930
931 if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
932 return;
933
934 /*
935 * Disable the context bank and free the page tables before freeing
936 * it.
937 */
938 smmu->cbs[cfg->cbndx].cfg = NULL;
939 arm_smmu_write_context_bank(smmu, cfg->cbndx);
940
941 if (cfg->irptndx != INVALID_IRPTNDX) {
942 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
943 devm_free_irq(smmu->dev, irq, domain);
944 }
945
946 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
947 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
948 }
949
950 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
951 {
952 struct arm_smmu_domain *smmu_domain;
953
954 if (type != IOMMU_DOMAIN_UNMANAGED &&
955 type != IOMMU_DOMAIN_DMA &&
956 type != IOMMU_DOMAIN_IDENTITY)
957 return NULL;
958 /*
959 * Allocate the domain and initialise some of its data structures.
960 * We can't really do anything meaningful until we've added a
961 * master.
962 */
963 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
964 if (!smmu_domain)
965 return NULL;
966
967 if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
968 iommu_get_dma_cookie(&smmu_domain->domain))) {
969 kfree(smmu_domain);
970 return NULL;
971 }
972
973 mutex_init(&smmu_domain->init_mutex);
974 spin_lock_init(&smmu_domain->cb_lock);
975
976 return &smmu_domain->domain;
977 }
978
979 static void arm_smmu_domain_free(struct iommu_domain *domain)
980 {
981 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
982
983 /*
984 * Free the domain resources. We assume that all devices have
985 * already been detached.
986 */
987 iommu_put_dma_cookie(domain);
988 arm_smmu_destroy_domain_context(domain);
989 kfree(smmu_domain);
990 }
991
992 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
993 {
994 struct arm_smmu_smr *smr = smmu->smrs + idx;
995 u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT;
996
997 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
998 reg |= SMR_VALID;
999 writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1000 }
1001
1002 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
1003 {
1004 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1005 u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT |
1006 (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT |
1007 (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT;
1008
1009 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
1010 smmu->smrs[idx].valid)
1011 reg |= S2CR_EXIDVALID;
1012 writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
1013 }
1014
1015 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1016 {
1017 arm_smmu_write_s2cr(smmu, idx);
1018 if (smmu->smrs)
1019 arm_smmu_write_smr(smmu, idx);
1020 }
1021
1022 /*
1023 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
1024 * should be called after sCR0 is written.
1025 */
1026 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
1027 {
1028 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1029 u32 smr;
1030
1031 if (!smmu->smrs)
1032 return;
1033
1034 /*
1035 * SMR.ID bits may not be preserved if the corresponding MASK
1036 * bits are set, so check each one separately. We can reject
1037 * masters later if they try to claim IDs outside these masks.
1038 */
1039 smr = smmu->streamid_mask << SMR_ID_SHIFT;
1040 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1041 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1042 smmu->streamid_mask = smr >> SMR_ID_SHIFT;
1043
1044 smr = smmu->streamid_mask << SMR_MASK_SHIFT;
1045 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1046 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1047 smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT;
1048 }
1049
1050 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1051 {
1052 struct arm_smmu_smr *smrs = smmu->smrs;
1053 int i, free_idx = -ENOSPC;
1054
1055 /* Stream indexing is blissfully easy */
1056 if (!smrs)
1057 return id;
1058
1059 /* Validating SMRs is... less so */
1060 for (i = 0; i < smmu->num_mapping_groups; ++i) {
1061 if (!smrs[i].valid) {
1062 /*
1063 * Note the first free entry we come across, which
1064 * we'll claim in the end if nothing else matches.
1065 */
1066 if (free_idx < 0)
1067 free_idx = i;
1068 continue;
1069 }
1070 /*
1071 * If the new entry is _entirely_ matched by an existing entry,
1072 * then reuse that, with the guarantee that there also cannot
1073 * be any subsequent conflicting entries. In normal use we'd
1074 * expect simply identical entries for this case, but there's
1075 * no harm in accommodating the generalisation.
1076 */
1077 if ((mask & smrs[i].mask) == mask &&
1078 !((id ^ smrs[i].id) & ~smrs[i].mask))
1079 return i;
1080 /*
1081 * If the new entry has any other overlap with an existing one,
1082 * though, then there always exists at least one stream ID
1083 * which would cause a conflict, and we can't allow that risk.
1084 */
1085 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1086 return -EINVAL;
1087 }
1088
1089 return free_idx;
1090 }
1091
1092 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1093 {
1094 if (--smmu->s2crs[idx].count)
1095 return false;
1096
1097 smmu->s2crs[idx] = s2cr_init_val;
1098 if (smmu->smrs)
1099 smmu->smrs[idx].valid = false;
1100
1101 return true;
1102 }
1103
1104 static int arm_smmu_master_alloc_smes(struct device *dev)
1105 {
1106 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1107 struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1108 struct arm_smmu_device *smmu = cfg->smmu;
1109 struct arm_smmu_smr *smrs = smmu->smrs;
1110 struct iommu_group *group;
1111 int i, idx, ret;
1112
1113 mutex_lock(&smmu->stream_map_mutex);
1114 /* Figure out a viable stream map entry allocation */
1115 for_each_cfg_sme(fwspec, i, idx) {
1116 u16 sid = fwspec->ids[i];
1117 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1118
1119 if (idx != INVALID_SMENDX) {
1120 ret = -EEXIST;
1121 goto out_err;
1122 }
1123
1124 ret = arm_smmu_find_sme(smmu, sid, mask);
1125 if (ret < 0)
1126 goto out_err;
1127
1128 idx = ret;
1129 if (smrs && smmu->s2crs[idx].count == 0) {
1130 smrs[idx].id = sid;
1131 smrs[idx].mask = mask;
1132 smrs[idx].valid = true;
1133 }
1134 smmu->s2crs[idx].count++;
1135 cfg->smendx[i] = (s16)idx;
1136 }
1137
1138 group = iommu_group_get_for_dev(dev);
1139 if (!group)
1140 group = ERR_PTR(-ENOMEM);
1141 if (IS_ERR(group)) {
1142 ret = PTR_ERR(group);
1143 goto out_err;
1144 }
1145 iommu_group_put(group);
1146
1147 /* It worked! Now, poke the actual hardware */
1148 for_each_cfg_sme(fwspec, i, idx) {
1149 arm_smmu_write_sme(smmu, idx);
1150 smmu->s2crs[idx].group = group;
1151 }
1152
1153 mutex_unlock(&smmu->stream_map_mutex);
1154 return 0;
1155
1156 out_err:
1157 while (i--) {
1158 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1159 cfg->smendx[i] = INVALID_SMENDX;
1160 }
1161 mutex_unlock(&smmu->stream_map_mutex);
1162 return ret;
1163 }
1164
1165 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1166 {
1167 struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1168 struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1169 int i, idx;
1170
1171 mutex_lock(&smmu->stream_map_mutex);
1172 for_each_cfg_sme(fwspec, i, idx) {
1173 if (arm_smmu_free_sme(smmu, idx))
1174 arm_smmu_write_sme(smmu, idx);
1175 cfg->smendx[i] = INVALID_SMENDX;
1176 }
1177 mutex_unlock(&smmu->stream_map_mutex);
1178 }
1179
1180 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1181 struct iommu_fwspec *fwspec)
1182 {
1183 struct arm_smmu_device *smmu = smmu_domain->smmu;
1184 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1185 u8 cbndx = smmu_domain->cfg.cbndx;
1186 enum arm_smmu_s2cr_type type;
1187 int i, idx;
1188
1189 if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1190 type = S2CR_TYPE_BYPASS;
1191 else
1192 type = S2CR_TYPE_TRANS;
1193
1194 for_each_cfg_sme(fwspec, i, idx) {
1195 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1196 continue;
1197
1198 s2cr[idx].type = type;
1199 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1200 s2cr[idx].cbndx = cbndx;
1201 arm_smmu_write_s2cr(smmu, idx);
1202 }
1203 return 0;
1204 }
1205
1206 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1207 {
1208 int ret;
1209 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1210 struct arm_smmu_device *smmu;
1211 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1212
1213 if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1214 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1215 return -ENXIO;
1216 }
1217
1218 /*
1219 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1220 * domains between of_xlate() and add_device() - we have no way to cope
1221 * with that, so until ARM gets converted to rely on groups and default
1222 * domains, just say no (but more politely than by dereferencing NULL).
1223 * This should be at least a WARN_ON once that's sorted.
1224 */
1225 if (!fwspec->iommu_priv)
1226 return -ENODEV;
1227
1228 smmu = fwspec_smmu(fwspec);
1229 /* Ensure that the domain is finalised */
1230 ret = arm_smmu_init_domain_context(domain, smmu);
1231 if (ret < 0)
1232 return ret;
1233
1234 /*
1235 * Sanity check the domain. We don't support domains across
1236 * different SMMUs.
1237 */
1238 if (smmu_domain->smmu != smmu) {
1239 dev_err(dev,
1240 "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1241 dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1242 return -EINVAL;
1243 }
1244
1245 /* Looks ok, so add the device to the domain */
1246 return arm_smmu_domain_add_master(smmu_domain, fwspec);
1247 }
1248
1249 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1250 phys_addr_t paddr, size_t size, int prot)
1251 {
1252 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1253
1254 if (!ops)
1255 return -ENODEV;
1256
1257 return ops->map(ops, iova, paddr, size, prot);
1258 }
1259
1260 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1261 size_t size)
1262 {
1263 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1264
1265 if (!ops)
1266 return 0;
1267
1268 return ops->unmap(ops, iova, size);
1269 }
1270
1271 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1272 {
1273 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1274
1275 if (smmu_domain->tlb_ops)
1276 smmu_domain->tlb_ops->tlb_flush_all(smmu_domain);
1277 }
1278
1279 static void arm_smmu_iotlb_sync(struct iommu_domain *domain)
1280 {
1281 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1282
1283 if (smmu_domain->tlb_ops)
1284 smmu_domain->tlb_ops->tlb_sync(smmu_domain);
1285 }
1286
1287 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1288 dma_addr_t iova)
1289 {
1290 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1291 struct arm_smmu_device *smmu = smmu_domain->smmu;
1292 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1293 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1294 struct device *dev = smmu->dev;
1295 void __iomem *cb_base;
1296 u32 tmp;
1297 u64 phys;
1298 unsigned long va, flags;
1299
1300 cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
1301
1302 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1303 /* ATS1 registers can only be written atomically */
1304 va = iova & ~0xfffUL;
1305 if (smmu->version == ARM_SMMU_V2)
1306 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1307 else /* Register is only 32-bit in v1 */
1308 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1309
1310 if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1311 !(tmp & ATSR_ACTIVE), 5, 50)) {
1312 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1313 dev_err(dev,
1314 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1315 &iova);
1316 return ops->iova_to_phys(ops, iova);
1317 }
1318
1319 phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1320 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1321 if (phys & CB_PAR_F) {
1322 dev_err(dev, "translation fault!\n");
1323 dev_err(dev, "PAR = 0x%llx\n", phys);
1324 return 0;
1325 }
1326
1327 return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1328 }
1329
1330 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1331 dma_addr_t iova)
1332 {
1333 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1334 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1335
1336 if (domain->type == IOMMU_DOMAIN_IDENTITY)
1337 return iova;
1338
1339 if (!ops)
1340 return 0;
1341
1342 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1343 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1344 return arm_smmu_iova_to_phys_hard(domain, iova);
1345
1346 return ops->iova_to_phys(ops, iova);
1347 }
1348
1349 static bool arm_smmu_capable(enum iommu_cap cap)
1350 {
1351 switch (cap) {
1352 case IOMMU_CAP_CACHE_COHERENCY:
1353 /*
1354 * Return true here as the SMMU can always send out coherent
1355 * requests.
1356 */
1357 return true;
1358 case IOMMU_CAP_NOEXEC:
1359 return true;
1360 default:
1361 return false;
1362 }
1363 }
1364
1365 static int arm_smmu_match_node(struct device *dev, void *data)
1366 {
1367 return dev->fwnode == data;
1368 }
1369
1370 static
1371 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1372 {
1373 struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1374 fwnode, arm_smmu_match_node);
1375 put_device(dev);
1376 return dev ? dev_get_drvdata(dev) : NULL;
1377 }
1378
1379 static int arm_smmu_add_device(struct device *dev)
1380 {
1381 struct arm_smmu_device *smmu;
1382 struct arm_smmu_master_cfg *cfg;
1383 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1384 int i, ret;
1385
1386 if (using_legacy_binding) {
1387 ret = arm_smmu_register_legacy_master(dev, &smmu);
1388
1389 /*
1390 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1391 * will allocate/initialise a new one. Thus we need to update fwspec for
1392 * later use.
1393 */
1394 fwspec = dev->iommu_fwspec;
1395 if (ret)
1396 goto out_free;
1397 } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1398 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1399 } else {
1400 return -ENODEV;
1401 }
1402
1403 ret = -EINVAL;
1404 for (i = 0; i < fwspec->num_ids; i++) {
1405 u16 sid = fwspec->ids[i];
1406 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1407
1408 if (sid & ~smmu->streamid_mask) {
1409 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1410 sid, smmu->streamid_mask);
1411 goto out_free;
1412 }
1413 if (mask & ~smmu->smr_mask_mask) {
1414 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1415 mask, smmu->smr_mask_mask);
1416 goto out_free;
1417 }
1418 }
1419
1420 ret = -ENOMEM;
1421 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1422 GFP_KERNEL);
1423 if (!cfg)
1424 goto out_free;
1425
1426 cfg->smmu = smmu;
1427 fwspec->iommu_priv = cfg;
1428 while (i--)
1429 cfg->smendx[i] = INVALID_SMENDX;
1430
1431 ret = arm_smmu_master_alloc_smes(dev);
1432 if (ret)
1433 goto out_cfg_free;
1434
1435 iommu_device_link(&smmu->iommu, dev);
1436
1437 return 0;
1438
1439 out_cfg_free:
1440 kfree(cfg);
1441 out_free:
1442 iommu_fwspec_free(dev);
1443 return ret;
1444 }
1445
1446 static void arm_smmu_remove_device(struct device *dev)
1447 {
1448 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1449 struct arm_smmu_master_cfg *cfg;
1450 struct arm_smmu_device *smmu;
1451
1452
1453 if (!fwspec || fwspec->ops != &arm_smmu_ops)
1454 return;
1455
1456 cfg = fwspec->iommu_priv;
1457 smmu = cfg->smmu;
1458
1459 iommu_device_unlink(&smmu->iommu, dev);
1460 arm_smmu_master_free_smes(fwspec);
1461 iommu_group_remove_device(dev);
1462 kfree(fwspec->iommu_priv);
1463 iommu_fwspec_free(dev);
1464 }
1465
1466 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1467 {
1468 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1469 struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1470 struct iommu_group *group = NULL;
1471 int i, idx;
1472
1473 for_each_cfg_sme(fwspec, i, idx) {
1474 if (group && smmu->s2crs[idx].group &&
1475 group != smmu->s2crs[idx].group)
1476 return ERR_PTR(-EINVAL);
1477
1478 group = smmu->s2crs[idx].group;
1479 }
1480
1481 if (group)
1482 return iommu_group_ref_get(group);
1483
1484 if (dev_is_pci(dev))
1485 group = pci_device_group(dev);
1486 else if (dev_is_fsl_mc(dev))
1487 group = fsl_mc_device_group(dev);
1488 else
1489 group = generic_device_group(dev);
1490
1491 return group;
1492 }
1493
1494 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1495 enum iommu_attr attr, void *data)
1496 {
1497 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1498
1499 switch(domain->type) {
1500 case IOMMU_DOMAIN_UNMANAGED:
1501 switch (attr) {
1502 case DOMAIN_ATTR_NESTING:
1503 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1504 return 0;
1505 default:
1506 return -ENODEV;
1507 }
1508 break;
1509 case IOMMU_DOMAIN_DMA:
1510 switch (attr) {
1511 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1512 *(int *)data = smmu_domain->non_strict;
1513 return 0;
1514 default:
1515 return -ENODEV;
1516 }
1517 break;
1518 default:
1519 return -EINVAL;
1520 }
1521 }
1522
1523 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1524 enum iommu_attr attr, void *data)
1525 {
1526 int ret = 0;
1527 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1528
1529 mutex_lock(&smmu_domain->init_mutex);
1530
1531 switch(domain->type) {
1532 case IOMMU_DOMAIN_UNMANAGED:
1533 switch (attr) {
1534 case DOMAIN_ATTR_NESTING:
1535 if (smmu_domain->smmu) {
1536 ret = -EPERM;
1537 goto out_unlock;
1538 }
1539
1540 if (*(int *)data)
1541 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1542 else
1543 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1544 break;
1545 default:
1546 ret = -ENODEV;
1547 }
1548 break;
1549 case IOMMU_DOMAIN_DMA:
1550 switch (attr) {
1551 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1552 smmu_domain->non_strict = *(int *)data;
1553 break;
1554 default:
1555 ret = -ENODEV;
1556 }
1557 break;
1558 default:
1559 ret = -EINVAL;
1560 }
1561 out_unlock:
1562 mutex_unlock(&smmu_domain->init_mutex);
1563 return ret;
1564 }
1565
1566 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1567 {
1568 u32 mask, fwid = 0;
1569
1570 if (args->args_count > 0)
1571 fwid |= (u16)args->args[0];
1572
1573 if (args->args_count > 1)
1574 fwid |= (u16)args->args[1] << SMR_MASK_SHIFT;
1575 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1576 fwid |= (u16)mask << SMR_MASK_SHIFT;
1577
1578 return iommu_fwspec_add_ids(dev, &fwid, 1);
1579 }
1580
1581 static void arm_smmu_get_resv_regions(struct device *dev,
1582 struct list_head *head)
1583 {
1584 struct iommu_resv_region *region;
1585 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1586
1587 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1588 prot, IOMMU_RESV_SW_MSI);
1589 if (!region)
1590 return;
1591
1592 list_add_tail(&region->list, head);
1593
1594 iommu_dma_get_resv_regions(dev, head);
1595 }
1596
1597 static void arm_smmu_put_resv_regions(struct device *dev,
1598 struct list_head *head)
1599 {
1600 struct iommu_resv_region *entry, *next;
1601
1602 list_for_each_entry_safe(entry, next, head, list)
1603 kfree(entry);
1604 }
1605
1606 static struct iommu_ops arm_smmu_ops = {
1607 .capable = arm_smmu_capable,
1608 .domain_alloc = arm_smmu_domain_alloc,
1609 .domain_free = arm_smmu_domain_free,
1610 .attach_dev = arm_smmu_attach_dev,
1611 .map = arm_smmu_map,
1612 .unmap = arm_smmu_unmap,
1613 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
1614 .iotlb_sync = arm_smmu_iotlb_sync,
1615 .iova_to_phys = arm_smmu_iova_to_phys,
1616 .add_device = arm_smmu_add_device,
1617 .remove_device = arm_smmu_remove_device,
1618 .device_group = arm_smmu_device_group,
1619 .domain_get_attr = arm_smmu_domain_get_attr,
1620 .domain_set_attr = arm_smmu_domain_set_attr,
1621 .of_xlate = arm_smmu_of_xlate,
1622 .get_resv_regions = arm_smmu_get_resv_regions,
1623 .put_resv_regions = arm_smmu_put_resv_regions,
1624 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1625 };
1626
1627 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1628 {
1629 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1630 int i;
1631 u32 reg, major;
1632
1633 /* clear global FSR */
1634 reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1635 writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1636
1637 /*
1638 * Reset stream mapping groups: Initial values mark all SMRn as
1639 * invalid and all S2CRn as bypass unless overridden.
1640 */
1641 for (i = 0; i < smmu->num_mapping_groups; ++i)
1642 arm_smmu_write_sme(smmu, i);
1643
1644 if (smmu->model == ARM_MMU500) {
1645 /*
1646 * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1647 * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1648 * bit is only present in MMU-500r2 onwards.
1649 */
1650 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1651 major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1652 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1653 if (major >= 2)
1654 reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1655 /*
1656 * Allow unmatched Stream IDs to allocate bypass
1657 * TLB entries for reduced latency.
1658 */
1659 reg |= ARM_MMU500_ACR_SMTNMB_TLBEN | ARM_MMU500_ACR_S2CRB_TLBEN;
1660 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1661 }
1662
1663 /* Make sure all context banks are disabled and clear CB_FSR */
1664 for (i = 0; i < smmu->num_context_banks; ++i) {
1665 void __iomem *cb_base = ARM_SMMU_CB(smmu, i);
1666
1667 arm_smmu_write_context_bank(smmu, i);
1668 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1669 /*
1670 * Disable MMU-500's not-particularly-beneficial next-page
1671 * prefetcher for the sake of errata #841119 and #826419.
1672 */
1673 if (smmu->model == ARM_MMU500) {
1674 reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1675 reg &= ~ARM_MMU500_ACTLR_CPRE;
1676 writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1677 }
1678 }
1679
1680 /* Invalidate the TLB, just in case */
1681 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1682 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1683
1684 reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1685
1686 /* Enable fault reporting */
1687 reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1688
1689 /* Disable TLB broadcasting. */
1690 reg |= (sCR0_VMIDPNE | sCR0_PTM);
1691
1692 /* Enable client access, handling unmatched streams as appropriate */
1693 reg &= ~sCR0_CLIENTPD;
1694 if (disable_bypass)
1695 reg |= sCR0_USFCFG;
1696 else
1697 reg &= ~sCR0_USFCFG;
1698
1699 /* Disable forced broadcasting */
1700 reg &= ~sCR0_FB;
1701
1702 /* Don't upgrade barriers */
1703 reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1704
1705 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1706 reg |= sCR0_VMID16EN;
1707
1708 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1709 reg |= sCR0_EXIDENABLE;
1710
1711 /* Push the button */
1712 arm_smmu_tlb_sync_global(smmu);
1713 writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1714 }
1715
1716 static int arm_smmu_id_size_to_bits(int size)
1717 {
1718 switch (size) {
1719 case 0:
1720 return 32;
1721 case 1:
1722 return 36;
1723 case 2:
1724 return 40;
1725 case 3:
1726 return 42;
1727 case 4:
1728 return 44;
1729 case 5:
1730 default:
1731 return 48;
1732 }
1733 }
1734
1735 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1736 {
1737 unsigned long size;
1738 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1739 u32 id;
1740 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1741 int i;
1742
1743 dev_notice(smmu->dev, "probing hardware configuration...\n");
1744 dev_notice(smmu->dev, "SMMUv%d with:\n",
1745 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1746
1747 /* ID0 */
1748 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1749
1750 /* Restrict available stages based on module parameter */
1751 if (force_stage == 1)
1752 id &= ~(ID0_S2TS | ID0_NTS);
1753 else if (force_stage == 2)
1754 id &= ~(ID0_S1TS | ID0_NTS);
1755
1756 if (id & ID0_S1TS) {
1757 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1758 dev_notice(smmu->dev, "\tstage 1 translation\n");
1759 }
1760
1761 if (id & ID0_S2TS) {
1762 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1763 dev_notice(smmu->dev, "\tstage 2 translation\n");
1764 }
1765
1766 if (id & ID0_NTS) {
1767 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1768 dev_notice(smmu->dev, "\tnested translation\n");
1769 }
1770
1771 if (!(smmu->features &
1772 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1773 dev_err(smmu->dev, "\tno translation support!\n");
1774 return -ENODEV;
1775 }
1776
1777 if ((id & ID0_S1TS) &&
1778 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1779 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1780 dev_notice(smmu->dev, "\taddress translation ops\n");
1781 }
1782
1783 /*
1784 * In order for DMA API calls to work properly, we must defer to what
1785 * the FW says about coherency, regardless of what the hardware claims.
1786 * Fortunately, this also opens up a workaround for systems where the
1787 * ID register value has ended up configured incorrectly.
1788 */
1789 cttw_reg = !!(id & ID0_CTTW);
1790 if (cttw_fw || cttw_reg)
1791 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1792 cttw_fw ? "" : "non-");
1793 if (cttw_fw != cttw_reg)
1794 dev_notice(smmu->dev,
1795 "\t(IDR0.CTTW overridden by FW configuration)\n");
1796
1797 /* Max. number of entries we have for stream matching/indexing */
1798 if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) {
1799 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1800 size = 1 << 16;
1801 } else {
1802 size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
1803 }
1804 smmu->streamid_mask = size - 1;
1805 if (id & ID0_SMS) {
1806 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1807 size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK;
1808 if (size == 0) {
1809 dev_err(smmu->dev,
1810 "stream-matching supported, but no SMRs present!\n");
1811 return -ENODEV;
1812 }
1813
1814 /* Zero-initialised to mark as invalid */
1815 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1816 GFP_KERNEL);
1817 if (!smmu->smrs)
1818 return -ENOMEM;
1819
1820 dev_notice(smmu->dev,
1821 "\tstream matching with %lu register groups", size);
1822 }
1823 /* s2cr->type == 0 means translation, so initialise explicitly */
1824 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1825 GFP_KERNEL);
1826 if (!smmu->s2crs)
1827 return -ENOMEM;
1828 for (i = 0; i < size; i++)
1829 smmu->s2crs[i] = s2cr_init_val;
1830
1831 smmu->num_mapping_groups = size;
1832 mutex_init(&smmu->stream_map_mutex);
1833 spin_lock_init(&smmu->global_sync_lock);
1834
1835 if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1836 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1837 if (!(id & ID0_PTFS_NO_AARCH32S))
1838 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1839 }
1840
1841 /* ID1 */
1842 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1843 smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1844
1845 /* Check for size mismatch of SMMU address space from mapped region */
1846 size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1847 size <<= smmu->pgshift;
1848 if (smmu->cb_base != gr0_base + size)
1849 dev_warn(smmu->dev,
1850 "SMMU address space size (0x%lx) differs from mapped region size (0x%tx)!\n",
1851 size * 2, (smmu->cb_base - gr0_base) * 2);
1852
1853 smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1854 smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1855 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1856 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1857 return -ENODEV;
1858 }
1859 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1860 smmu->num_context_banks, smmu->num_s2_context_banks);
1861 /*
1862 * Cavium CN88xx erratum #27704.
1863 * Ensure ASID and VMID allocation is unique across all SMMUs in
1864 * the system.
1865 */
1866 if (smmu->model == CAVIUM_SMMUV2) {
1867 smmu->cavium_id_base =
1868 atomic_add_return(smmu->num_context_banks,
1869 &cavium_smmu_context_count);
1870 smmu->cavium_id_base -= smmu->num_context_banks;
1871 dev_notice(smmu->dev, "\tenabling workaround for Cavium erratum 27704\n");
1872 }
1873 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1874 sizeof(*smmu->cbs), GFP_KERNEL);
1875 if (!smmu->cbs)
1876 return -ENOMEM;
1877
1878 /* ID2 */
1879 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1880 size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1881 smmu->ipa_size = size;
1882
1883 /* The output mask is also applied for bypass */
1884 size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1885 smmu->pa_size = size;
1886
1887 if (id & ID2_VMID16)
1888 smmu->features |= ARM_SMMU_FEAT_VMID16;
1889
1890 /*
1891 * What the page table walker can address actually depends on which
1892 * descriptor format is in use, but since a) we don't know that yet,
1893 * and b) it can vary per context bank, this will have to do...
1894 */
1895 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1896 dev_warn(smmu->dev,
1897 "failed to set DMA mask for table walker\n");
1898
1899 if (smmu->version < ARM_SMMU_V2) {
1900 smmu->va_size = smmu->ipa_size;
1901 if (smmu->version == ARM_SMMU_V1_64K)
1902 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1903 } else {
1904 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1905 smmu->va_size = arm_smmu_id_size_to_bits(size);
1906 if (id & ID2_PTFS_4K)
1907 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1908 if (id & ID2_PTFS_16K)
1909 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1910 if (id & ID2_PTFS_64K)
1911 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1912 }
1913
1914 /* Now we've corralled the various formats, what'll it do? */
1915 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1916 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1917 if (smmu->features &
1918 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1919 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1920 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1921 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1922 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1923 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1924
1925 if (arm_smmu_ops.pgsize_bitmap == -1UL)
1926 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1927 else
1928 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1929 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1930 smmu->pgsize_bitmap);
1931
1932
1933 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1934 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1935 smmu->va_size, smmu->ipa_size);
1936
1937 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1938 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1939 smmu->ipa_size, smmu->pa_size);
1940
1941 return 0;
1942 }
1943
1944 struct arm_smmu_match_data {
1945 enum arm_smmu_arch_version version;
1946 enum arm_smmu_implementation model;
1947 };
1948
1949 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1950 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1951
1952 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1953 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1954 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1955 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1956 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1957
1958 static const struct of_device_id arm_smmu_of_match[] = {
1959 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1960 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1961 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1962 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1963 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1964 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1965 { },
1966 };
1967 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1968
1969 #ifdef CONFIG_ACPI
1970 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1971 {
1972 int ret = 0;
1973
1974 switch (model) {
1975 case ACPI_IORT_SMMU_V1:
1976 case ACPI_IORT_SMMU_CORELINK_MMU400:
1977 smmu->version = ARM_SMMU_V1;
1978 smmu->model = GENERIC_SMMU;
1979 break;
1980 case ACPI_IORT_SMMU_CORELINK_MMU401:
1981 smmu->version = ARM_SMMU_V1_64K;
1982 smmu->model = GENERIC_SMMU;
1983 break;
1984 case ACPI_IORT_SMMU_V2:
1985 smmu->version = ARM_SMMU_V2;
1986 smmu->model = GENERIC_SMMU;
1987 break;
1988 case ACPI_IORT_SMMU_CORELINK_MMU500:
1989 smmu->version = ARM_SMMU_V2;
1990 smmu->model = ARM_MMU500;
1991 break;
1992 case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1993 smmu->version = ARM_SMMU_V2;
1994 smmu->model = CAVIUM_SMMUV2;
1995 break;
1996 default:
1997 ret = -ENODEV;
1998 }
1999
2000 return ret;
2001 }
2002
2003 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2004 struct arm_smmu_device *smmu)
2005 {
2006 struct device *dev = smmu->dev;
2007 struct acpi_iort_node *node =
2008 *(struct acpi_iort_node **)dev_get_platdata(dev);
2009 struct acpi_iort_smmu *iort_smmu;
2010 int ret;
2011
2012 /* Retrieve SMMU1/2 specific data */
2013 iort_smmu = (struct acpi_iort_smmu *)node->node_data;
2014
2015 ret = acpi_smmu_get_data(iort_smmu->model, smmu);
2016 if (ret < 0)
2017 return ret;
2018
2019 /* Ignore the configuration access interrupt */
2020 smmu->num_global_irqs = 1;
2021
2022 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2023 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2024
2025 return 0;
2026 }
2027 #else
2028 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2029 struct arm_smmu_device *smmu)
2030 {
2031 return -ENODEV;
2032 }
2033 #endif
2034
2035 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2036 struct arm_smmu_device *smmu)
2037 {
2038 const struct arm_smmu_match_data *data;
2039 struct device *dev = &pdev->dev;
2040 bool legacy_binding;
2041
2042 if (of_property_read_u32(dev->of_node, "#global-interrupts",
2043 &smmu->num_global_irqs)) {
2044 dev_err(dev, "missing #global-interrupts property\n");
2045 return -ENODEV;
2046 }
2047
2048 data = of_device_get_match_data(dev);
2049 smmu->version = data->version;
2050 smmu->model = data->model;
2051
2052 parse_driver_options(smmu);
2053
2054 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2055 if (legacy_binding && !using_generic_binding) {
2056 if (!using_legacy_binding)
2057 pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
2058 using_legacy_binding = true;
2059 } else if (!legacy_binding && !using_legacy_binding) {
2060 using_generic_binding = true;
2061 } else {
2062 dev_err(dev, "not probing due to mismatched DT properties\n");
2063 return -ENODEV;
2064 }
2065
2066 if (of_dma_is_coherent(dev->of_node))
2067 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2068
2069 return 0;
2070 }
2071
2072 static void arm_smmu_bus_init(void)
2073 {
2074 /* Oh, for a proper bus abstraction */
2075 if (!iommu_present(&platform_bus_type))
2076 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2077 #ifdef CONFIG_ARM_AMBA
2078 if (!iommu_present(&amba_bustype))
2079 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2080 #endif
2081 #ifdef CONFIG_PCI
2082 if (!iommu_present(&pci_bus_type)) {
2083 pci_request_acs();
2084 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2085 }
2086 #endif
2087 #ifdef CONFIG_FSL_MC_BUS
2088 if (!iommu_present(&fsl_mc_bus_type))
2089 bus_set_iommu(&fsl_mc_bus_type, &arm_smmu_ops);
2090 #endif
2091 }
2092
2093 static int arm_smmu_device_probe(struct platform_device *pdev)
2094 {
2095 struct resource *res;
2096 resource_size_t ioaddr;
2097 struct arm_smmu_device *smmu;
2098 struct device *dev = &pdev->dev;
2099 int num_irqs, i, err;
2100
2101 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2102 if (!smmu) {
2103 dev_err(dev, "failed to allocate arm_smmu_device\n");
2104 return -ENOMEM;
2105 }
2106 smmu->dev = dev;
2107
2108 if (dev->of_node)
2109 err = arm_smmu_device_dt_probe(pdev, smmu);
2110 else
2111 err = arm_smmu_device_acpi_probe(pdev, smmu);
2112
2113 if (err)
2114 return err;
2115
2116 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2117 ioaddr = res->start;
2118 smmu->base = devm_ioremap_resource(dev, res);
2119 if (IS_ERR(smmu->base))
2120 return PTR_ERR(smmu->base);
2121 smmu->cb_base = smmu->base + resource_size(res) / 2;
2122
2123 num_irqs = 0;
2124 while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2125 num_irqs++;
2126 if (num_irqs > smmu->num_global_irqs)
2127 smmu->num_context_irqs++;
2128 }
2129
2130 if (!smmu->num_context_irqs) {
2131 dev_err(dev, "found %d interrupts but expected at least %d\n",
2132 num_irqs, smmu->num_global_irqs + 1);
2133 return -ENODEV;
2134 }
2135
2136 smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),
2137 GFP_KERNEL);
2138 if (!smmu->irqs) {
2139 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2140 return -ENOMEM;
2141 }
2142
2143 for (i = 0; i < num_irqs; ++i) {
2144 int irq = platform_get_irq(pdev, i);
2145
2146 if (irq < 0) {
2147 dev_err(dev, "failed to get irq index %d\n", i);
2148 return -ENODEV;
2149 }
2150 smmu->irqs[i] = irq;
2151 }
2152
2153 err = arm_smmu_device_cfg_probe(smmu);
2154 if (err)
2155 return err;
2156
2157 if (smmu->version == ARM_SMMU_V2) {
2158 if (smmu->num_context_banks > smmu->num_context_irqs) {
2159 dev_err(dev,
2160 "found only %d context irq(s) but %d required\n",
2161 smmu->num_context_irqs, smmu->num_context_banks);
2162 return -ENODEV;
2163 }
2164
2165 /* Ignore superfluous interrupts */
2166 smmu->num_context_irqs = smmu->num_context_banks;
2167 }
2168
2169 for (i = 0; i < smmu->num_global_irqs; ++i) {
2170 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2171 arm_smmu_global_fault,
2172 IRQF_SHARED,
2173 "arm-smmu global fault",
2174 smmu);
2175 if (err) {
2176 dev_err(dev, "failed to request global IRQ %d (%u)\n",
2177 i, smmu->irqs[i]);
2178 return err;
2179 }
2180 }
2181
2182 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2183 "smmu.%pa", &ioaddr);
2184 if (err) {
2185 dev_err(dev, "Failed to register iommu in sysfs\n");
2186 return err;
2187 }
2188
2189 iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2190 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2191
2192 err = iommu_device_register(&smmu->iommu);
2193 if (err) {
2194 dev_err(dev, "Failed to register iommu\n");
2195 return err;
2196 }
2197
2198 platform_set_drvdata(pdev, smmu);
2199 arm_smmu_device_reset(smmu);
2200 arm_smmu_test_smr_masks(smmu);
2201
2202 /*
2203 * For ACPI and generic DT bindings, an SMMU will be probed before
2204 * any device which might need it, so we want the bus ops in place
2205 * ready to handle default domain setup as soon as any SMMU exists.
2206 */
2207 if (!using_legacy_binding)
2208 arm_smmu_bus_init();
2209
2210 return 0;
2211 }
2212
2213 /*
2214 * With the legacy DT binding in play, though, we have no guarantees about
2215 * probe order, but then we're also not doing default domains, so we can
2216 * delay setting bus ops until we're sure every possible SMMU is ready,
2217 * and that way ensure that no add_device() calls get missed.
2218 */
2219 static int arm_smmu_legacy_bus_init(void)
2220 {
2221 if (using_legacy_binding)
2222 arm_smmu_bus_init();
2223 return 0;
2224 }
2225 device_initcall_sync(arm_smmu_legacy_bus_init);
2226
2227 static int arm_smmu_device_remove(struct platform_device *pdev)
2228 {
2229 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2230
2231 if (!smmu)
2232 return -ENODEV;
2233
2234 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2235 dev_err(&pdev->dev, "removing device with active domains!\n");
2236
2237 /* Turn the thing off */
2238 writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2239 return 0;
2240 }
2241
2242 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2243 {
2244 arm_smmu_device_remove(pdev);
2245 }
2246
2247 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2248 {
2249 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2250
2251 arm_smmu_device_reset(smmu);
2252 return 0;
2253 }
2254
2255 static SIMPLE_DEV_PM_OPS(arm_smmu_pm_ops, NULL, arm_smmu_pm_resume);
2256
2257 static struct platform_driver arm_smmu_driver = {
2258 .driver = {
2259 .name = "arm-smmu",
2260 .of_match_table = of_match_ptr(arm_smmu_of_match),
2261 .pm = &arm_smmu_pm_ops,
2262 },
2263 .probe = arm_smmu_device_probe,
2264 .remove = arm_smmu_device_remove,
2265 .shutdown = arm_smmu_device_shutdown,
2266 };
2267 module_platform_driver(arm_smmu_driver);
2268
2269 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2270 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2271 MODULE_LICENSE("GPL v2");
Ubuntu Jammy Linux kernel mirror