]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - drivers/iommu/arm-smmu.c
projects / mirror_ubuntu-jammy-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'for-linus-5.6-1' of https://github.com/cminyard/linux-ipmi
[mirror_ubuntu-jammy-kernel.git] / drivers / iommu / arm-smmu.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * IOMMU API for ARM architected SMMU implementations.
4 *
5 * Copyright (C) 2013 ARM Limited
6 *
7 * Author: Will Deacon <will.deacon@arm.com>
8 *
9 * This driver currently supports:
10 * - SMMUv1 and v2 implementations
11 * - Stream-matching and stream-indexing
12 * - v7/v8 long-descriptor format
13 * - Non-secure access to the SMMU
14 * - Context fault reporting
15 * - Extended Stream ID (16 bit)
16 */
17
18 #define pr_fmt(fmt) "arm-smmu: " fmt
19
20 #include <linux/acpi.h>
21 #include <linux/acpi_iort.h>
22 #include <linux/bitfield.h>
23 #include <linux/delay.h>
24 #include <linux/dma-iommu.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/err.h>
27 #include <linux/interrupt.h>
28 #include <linux/io.h>
29 #include <linux/iopoll.h>
30 #include <linux/module.h>
31 #include <linux/of.h>
32 #include <linux/of_address.h>
33 #include <linux/of_device.h>
34 #include <linux/of_iommu.h>
35 #include <linux/pci.h>
36 #include <linux/platform_device.h>
37 #include <linux/pm_runtime.h>
38 #include <linux/ratelimit.h>
39 #include <linux/slab.h>
40
41 #include <linux/amba/bus.h>
42 #include <linux/fsl/mc.h>
43
44 #include "arm-smmu.h"
45
46 /*
47 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
48 * global register space are still, in fact, using a hypervisor to mediate it
49 * by trapping and emulating register accesses. Sadly, some deployed versions
50 * of said trapping code have bugs wherein they go horribly wrong for stores
51 * using r31 (i.e. XZR/WZR) as the source register.
52 */
53 #define QCOM_DUMMY_VAL -1
54
55 #define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
56 #define TLB_SPIN_COUNT 10
57
58 #define MSI_IOVA_BASE 0x8000000
59 #define MSI_IOVA_LENGTH 0x100000
60
61 static int force_stage;
62 module_param(force_stage, int, S_IRUGO);
63 MODULE_PARM_DESC(force_stage,
64 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
65 static bool disable_bypass =
66 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
67 module_param(disable_bypass, bool, S_IRUGO);
68 MODULE_PARM_DESC(disable_bypass,
69 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
70
71 struct arm_smmu_s2cr {
72 struct iommu_group *group;
73 int count;
74 enum arm_smmu_s2cr_type type;
75 enum arm_smmu_s2cr_privcfg privcfg;
76 u8 cbndx;
77 };
78
79 #define s2cr_init_val (struct arm_smmu_s2cr){ \
80 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
81 }
82
83 struct arm_smmu_smr {
84 u16 mask;
85 u16 id;
86 bool valid;
87 };
88
89 struct arm_smmu_cb {
90 u64 ttbr[2];
91 u32 tcr[2];
92 u32 mair[2];
93 struct arm_smmu_cfg *cfg;
94 };
95
96 struct arm_smmu_master_cfg {
97 struct arm_smmu_device *smmu;
98 s16 smendx[];
99 };
100 #define INVALID_SMENDX -1
101 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
102 #define fwspec_smmu(fw) (__fwspec_cfg(fw)->smmu)
103 #define fwspec_smendx(fw, i) \
104 (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
105 #define for_each_cfg_sme(fw, i, idx) \
106 for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
107
108 static bool using_legacy_binding, using_generic_binding;
109
110 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
111 {
112 if (pm_runtime_enabled(smmu->dev))
113 return pm_runtime_get_sync(smmu->dev);
114
115 return 0;
116 }
117
118 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
119 {
120 if (pm_runtime_enabled(smmu->dev))
121 pm_runtime_put_autosuspend(smmu->dev);
122 }
123
124 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
125 {
126 return container_of(dom, struct arm_smmu_domain, domain);
127 }
128
129 static struct platform_driver arm_smmu_driver;
130 static struct iommu_ops arm_smmu_ops;
131
132 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
133 static int arm_smmu_bus_init(struct iommu_ops *ops);
134
135 static struct device_node *dev_get_dev_node(struct device *dev)
136 {
137 if (dev_is_pci(dev)) {
138 struct pci_bus *bus = to_pci_dev(dev)->bus;
139
140 while (!pci_is_root_bus(bus))
141 bus = bus->parent;
142 return of_node_get(bus->bridge->parent->of_node);
143 }
144
145 return of_node_get(dev->of_node);
146 }
147
148 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
149 {
150 *((__be32 *)data) = cpu_to_be32(alias);
151 return 0; /* Continue walking */
152 }
153
154 static int __find_legacy_master_phandle(struct device *dev, void *data)
155 {
156 struct of_phandle_iterator *it = *(void **)data;
157 struct device_node *np = it->node;
158 int err;
159
160 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
161 "#stream-id-cells", -1)
162 if (it->node == np) {
163 *(void **)data = dev;
164 return 1;
165 }
166 it->node = np;
167 return err == -ENOENT ? 0 : err;
168 }
169
170 static int arm_smmu_register_legacy_master(struct device *dev,
171 struct arm_smmu_device **smmu)
172 {
173 struct device *smmu_dev;
174 struct device_node *np;
175 struct of_phandle_iterator it;
176 void *data = &it;
177 u32 *sids;
178 __be32 pci_sid;
179 int err;
180
181 np = dev_get_dev_node(dev);
182 if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
183 of_node_put(np);
184 return -ENODEV;
185 }
186
187 it.node = np;
188 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
189 __find_legacy_master_phandle);
190 smmu_dev = data;
191 of_node_put(np);
192 if (err == 0)
193 return -ENODEV;
194 if (err < 0)
195 return err;
196
197 if (dev_is_pci(dev)) {
198 /* "mmu-masters" assumes Stream ID == Requester ID */
199 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
200 &pci_sid);
201 it.cur = &pci_sid;
202 it.cur_count = 1;
203 }
204
205 err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
206 &arm_smmu_ops);
207 if (err)
208 return err;
209
210 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
211 if (!sids)
212 return -ENOMEM;
213
214 *smmu = dev_get_drvdata(smmu_dev);
215 of_phandle_iterator_args(&it, sids, it.cur_count);
216 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
217 kfree(sids);
218 return err;
219 }
220
221 /*
222 * With the legacy DT binding in play, we have no guarantees about
223 * probe order, but then we're also not doing default domains, so we can
224 * delay setting bus ops until we're sure every possible SMMU is ready,
225 * and that way ensure that no add_device() calls get missed.
226 */
227 static int arm_smmu_legacy_bus_init(void)
228 {
229 if (using_legacy_binding)
230 return arm_smmu_bus_init(&arm_smmu_ops);
231 return 0;
232 }
233 device_initcall_sync(arm_smmu_legacy_bus_init);
234 #else
235 static int arm_smmu_register_legacy_master(struct device *dev,
236 struct arm_smmu_device **smmu)
237 {
238 return -ENODEV;
239 }
240 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
241
242 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
243 {
244 int idx;
245
246 do {
247 idx = find_next_zero_bit(map, end, start);
248 if (idx == end)
249 return -ENOSPC;
250 } while (test_and_set_bit(idx, map));
251
252 return idx;
253 }
254
255 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
256 {
257 clear_bit(idx, map);
258 }
259
260 /* Wait for any pending TLB invalidations to complete */
261 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
262 int sync, int status)
263 {
264 unsigned int spin_cnt, delay;
265 u32 reg;
266
267 if (smmu->impl && unlikely(smmu->impl->tlb_sync))
268 return smmu->impl->tlb_sync(smmu, page, sync, status);
269
270 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
271 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
272 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
273 reg = arm_smmu_readl(smmu, page, status);
274 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
275 return;
276 cpu_relax();
277 }
278 udelay(delay);
279 }
280 dev_err_ratelimited(smmu->dev,
281 "TLB sync timed out -- SMMU may be deadlocked\n");
282 }
283
284 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
285 {
286 unsigned long flags;
287
288 spin_lock_irqsave(&smmu->global_sync_lock, flags);
289 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
290 ARM_SMMU_GR0_sTLBGSTATUS);
291 spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
292 }
293
294 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
295 {
296 struct arm_smmu_device *smmu = smmu_domain->smmu;
297 unsigned long flags;
298
299 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
300 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
301 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
302 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
303 }
304
305 static void arm_smmu_tlb_inv_context_s1(void *cookie)
306 {
307 struct arm_smmu_domain *smmu_domain = cookie;
308 /*
309 * The TLBI write may be relaxed, so ensure that PTEs cleared by the
310 * current CPU are visible beforehand.
311 */
312 wmb();
313 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
314 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
315 arm_smmu_tlb_sync_context(smmu_domain);
316 }
317
318 static void arm_smmu_tlb_inv_context_s2(void *cookie)
319 {
320 struct arm_smmu_domain *smmu_domain = cookie;
321 struct arm_smmu_device *smmu = smmu_domain->smmu;
322
323 /* See above */
324 wmb();
325 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
326 arm_smmu_tlb_sync_global(smmu);
327 }
328
329 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
330 size_t granule, void *cookie, int reg)
331 {
332 struct arm_smmu_domain *smmu_domain = cookie;
333 struct arm_smmu_device *smmu = smmu_domain->smmu;
334 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
335 int idx = cfg->cbndx;
336
337 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
338 wmb();
339
340 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
341 iova = (iova >> 12) << 12;
342 iova |= cfg->asid;
343 do {
344 arm_smmu_cb_write(smmu, idx, reg, iova);
345 iova += granule;
346 } while (size -= granule);
347 } else {
348 iova >>= 12;
349 iova |= (u64)cfg->asid << 48;
350 do {
351 arm_smmu_cb_writeq(smmu, idx, reg, iova);
352 iova += granule >> 12;
353 } while (size -= granule);
354 }
355 }
356
357 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
358 size_t granule, void *cookie, int reg)
359 {
360 struct arm_smmu_domain *smmu_domain = cookie;
361 struct arm_smmu_device *smmu = smmu_domain->smmu;
362 int idx = smmu_domain->cfg.cbndx;
363
364 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
365 wmb();
366
367 iova >>= 12;
368 do {
369 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
370 arm_smmu_cb_writeq(smmu, idx, reg, iova);
371 else
372 arm_smmu_cb_write(smmu, idx, reg, iova);
373 iova += granule >> 12;
374 } while (size -= granule);
375 }
376
377 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
378 size_t granule, void *cookie)
379 {
380 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
381 ARM_SMMU_CB_S1_TLBIVA);
382 arm_smmu_tlb_sync_context(cookie);
383 }
384
385 static void arm_smmu_tlb_inv_leaf_s1(unsigned long iova, size_t size,
386 size_t granule, void *cookie)
387 {
388 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
389 ARM_SMMU_CB_S1_TLBIVAL);
390 arm_smmu_tlb_sync_context(cookie);
391 }
392
393 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
394 unsigned long iova, size_t granule,
395 void *cookie)
396 {
397 arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
398 ARM_SMMU_CB_S1_TLBIVAL);
399 }
400
401 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
402 size_t granule, void *cookie)
403 {
404 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
405 ARM_SMMU_CB_S2_TLBIIPAS2);
406 arm_smmu_tlb_sync_context(cookie);
407 }
408
409 static void arm_smmu_tlb_inv_leaf_s2(unsigned long iova, size_t size,
410 size_t granule, void *cookie)
411 {
412 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
413 ARM_SMMU_CB_S2_TLBIIPAS2L);
414 arm_smmu_tlb_sync_context(cookie);
415 }
416
417 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
418 unsigned long iova, size_t granule,
419 void *cookie)
420 {
421 arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
422 ARM_SMMU_CB_S2_TLBIIPAS2L);
423 }
424
425 static void arm_smmu_tlb_inv_any_s2_v1(unsigned long iova, size_t size,
426 size_t granule, void *cookie)
427 {
428 arm_smmu_tlb_inv_context_s2(cookie);
429 }
430 /*
431 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
432 * almost negligible, but the benefit of getting the first one in as far ahead
433 * of the sync as possible is significant, hence we don't just make this a
434 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
435 * think.
436 */
437 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
438 unsigned long iova, size_t granule,
439 void *cookie)
440 {
441 struct arm_smmu_domain *smmu_domain = cookie;
442 struct arm_smmu_device *smmu = smmu_domain->smmu;
443
444 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
445 wmb();
446
447 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
448 }
449
450 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
451 .tlb_flush_all = arm_smmu_tlb_inv_context_s1,
452 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
453 .tlb_flush_leaf = arm_smmu_tlb_inv_leaf_s1,
454 .tlb_add_page = arm_smmu_tlb_add_page_s1,
455 };
456
457 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
458 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
459 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
460 .tlb_flush_leaf = arm_smmu_tlb_inv_leaf_s2,
461 .tlb_add_page = arm_smmu_tlb_add_page_s2,
462 };
463
464 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
465 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
466 .tlb_flush_walk = arm_smmu_tlb_inv_any_s2_v1,
467 .tlb_flush_leaf = arm_smmu_tlb_inv_any_s2_v1,
468 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1,
469 };
470
471 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
472 {
473 u32 fsr, fsynr, cbfrsynra;
474 unsigned long iova;
475 struct iommu_domain *domain = dev;
476 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
477 struct arm_smmu_device *smmu = smmu_domain->smmu;
478 int idx = smmu_domain->cfg.cbndx;
479
480 fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
481 if (!(fsr & ARM_SMMU_FSR_FAULT))
482 return IRQ_NONE;
483
484 fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
485 iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
486 cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
487
488 dev_err_ratelimited(smmu->dev,
489 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
490 fsr, iova, fsynr, cbfrsynra, idx);
491
492 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
493 return IRQ_HANDLED;
494 }
495
496 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
497 {
498 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
499 struct arm_smmu_device *smmu = dev;
500 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
501 DEFAULT_RATELIMIT_BURST);
502
503 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
504 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
505 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
506 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
507
508 if (!gfsr)
509 return IRQ_NONE;
510
511 if (__ratelimit(&rs)) {
512 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
513 (gfsr & ARM_SMMU_sGFSR_USF))
514 dev_err(smmu->dev,
515 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
516 (u16)gfsynr1);
517 else
518 dev_err(smmu->dev,
519 "Unexpected global fault, this could be serious\n");
520 dev_err(smmu->dev,
521 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
522 gfsr, gfsynr0, gfsynr1, gfsynr2);
523 }
524
525 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
526 return IRQ_HANDLED;
527 }
528
529 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
530 struct io_pgtable_cfg *pgtbl_cfg)
531 {
532 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
533 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
534 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
535
536 cb->cfg = cfg;
537
538 /* TCR */
539 if (stage1) {
540 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
541 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
542 } else {
543 cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
544 cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
545 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
546 cb->tcr[1] |= ARM_SMMU_TCR2_AS;
547 else
548 cb->tcr[0] |= ARM_SMMU_TCR_EAE;
549 }
550 } else {
551 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
552 }
553
554 /* TTBRs */
555 if (stage1) {
556 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
557 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
558 cb->ttbr[1] = 0;
559 } else {
560 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
561 cb->ttbr[0] |= FIELD_PREP(ARM_SMMU_TTBRn_ASID,
562 cfg->asid);
563 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
564 cfg->asid);
565 }
566 } else {
567 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
568 }
569
570 /* MAIRs (stage-1 only) */
571 if (stage1) {
572 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
573 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
574 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
575 } else {
576 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
577 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
578 }
579 }
580 }
581
582 static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
583 {
584 u32 reg;
585 bool stage1;
586 struct arm_smmu_cb *cb = &smmu->cbs[idx];
587 struct arm_smmu_cfg *cfg = cb->cfg;
588
589 /* Unassigned context banks only need disabling */
590 if (!cfg) {
591 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
592 return;
593 }
594
595 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
596
597 /* CBA2R */
598 if (smmu->version > ARM_SMMU_V1) {
599 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
600 reg = ARM_SMMU_CBA2R_VA64;
601 else
602 reg = 0;
603 /* 16-bit VMIDs live in CBA2R */
604 if (smmu->features & ARM_SMMU_FEAT_VMID16)
605 reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
606
607 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
608 }
609
610 /* CBAR */
611 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
612 if (smmu->version < ARM_SMMU_V2)
613 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
614
615 /*
616 * Use the weakest shareability/memory types, so they are
617 * overridden by the ttbcr/pte.
618 */
619 if (stage1) {
620 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
621 ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
622 FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
623 ARM_SMMU_CBAR_S1_MEMATTR_WB);
624 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
625 /* 8-bit VMIDs live in CBAR */
626 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
627 }
628 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
629
630 /*
631 * TCR
632 * We must write this before the TTBRs, since it determines the
633 * access behaviour of some fields (in particular, ASID[15:8]).
634 */
635 if (stage1 && smmu->version > ARM_SMMU_V1)
636 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
637 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
638
639 /* TTBRs */
640 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
641 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
642 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
643 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
644 } else {
645 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
646 if (stage1)
647 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
648 cb->ttbr[1]);
649 }
650
651 /* MAIRs (stage-1 only) */
652 if (stage1) {
653 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
654 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
655 }
656
657 /* SCTLR */
658 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
659 ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
660 if (stage1)
661 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
662 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
663 reg |= ARM_SMMU_SCTLR_E;
664
665 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
666 }
667
668 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
669 struct arm_smmu_device *smmu)
670 {
671 int irq, start, ret = 0;
672 unsigned long ias, oas;
673 struct io_pgtable_ops *pgtbl_ops;
674 struct io_pgtable_cfg pgtbl_cfg;
675 enum io_pgtable_fmt fmt;
676 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
677 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
678
679 mutex_lock(&smmu_domain->init_mutex);
680 if (smmu_domain->smmu)
681 goto out_unlock;
682
683 if (domain->type == IOMMU_DOMAIN_IDENTITY) {
684 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
685 smmu_domain->smmu = smmu;
686 goto out_unlock;
687 }
688
689 /*
690 * Mapping the requested stage onto what we support is surprisingly
691 * complicated, mainly because the spec allows S1+S2 SMMUs without
692 * support for nested translation. That means we end up with the
693 * following table:
694 *
695 * Requested Supported Actual
696 * S1 N S1
697 * S1 S1+S2 S1
698 * S1 S2 S2
699 * S1 S1 S1
700 * N N N
701 * N S1+S2 S2
702 * N S2 S2
703 * N S1 S1
704 *
705 * Note that you can't actually request stage-2 mappings.
706 */
707 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
708 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
709 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
710 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
711
712 /*
713 * Choosing a suitable context format is even more fiddly. Until we
714 * grow some way for the caller to express a preference, and/or move
715 * the decision into the io-pgtable code where it arguably belongs,
716 * just aim for the closest thing to the rest of the system, and hope
717 * that the hardware isn't esoteric enough that we can't assume AArch64
718 * support to be a superset of AArch32 support...
719 */
720 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
721 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
722 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
723 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
724 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
725 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
726 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
727 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
728 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
729 ARM_SMMU_FEAT_FMT_AARCH64_16K |
730 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
731 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
732
733 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
734 ret = -EINVAL;
735 goto out_unlock;
736 }
737
738 switch (smmu_domain->stage) {
739 case ARM_SMMU_DOMAIN_S1:
740 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
741 start = smmu->num_s2_context_banks;
742 ias = smmu->va_size;
743 oas = smmu->ipa_size;
744 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
745 fmt = ARM_64_LPAE_S1;
746 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
747 fmt = ARM_32_LPAE_S1;
748 ias = min(ias, 32UL);
749 oas = min(oas, 40UL);
750 } else {
751 fmt = ARM_V7S;
752 ias = min(ias, 32UL);
753 oas = min(oas, 32UL);
754 }
755 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
756 break;
757 case ARM_SMMU_DOMAIN_NESTED:
758 /*
759 * We will likely want to change this if/when KVM gets
760 * involved.
761 */
762 case ARM_SMMU_DOMAIN_S2:
763 cfg->cbar = CBAR_TYPE_S2_TRANS;
764 start = 0;
765 ias = smmu->ipa_size;
766 oas = smmu->pa_size;
767 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
768 fmt = ARM_64_LPAE_S2;
769 } else {
770 fmt = ARM_32_LPAE_S2;
771 ias = min(ias, 40UL);
772 oas = min(oas, 40UL);
773 }
774 if (smmu->version == ARM_SMMU_V2)
775 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
776 else
777 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
778 break;
779 default:
780 ret = -EINVAL;
781 goto out_unlock;
782 }
783 ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
784 smmu->num_context_banks);
785 if (ret < 0)
786 goto out_unlock;
787
788 cfg->cbndx = ret;
789 if (smmu->version < ARM_SMMU_V2) {
790 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
791 cfg->irptndx %= smmu->num_context_irqs;
792 } else {
793 cfg->irptndx = cfg->cbndx;
794 }
795
796 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
797 cfg->vmid = cfg->cbndx + 1;
798 else
799 cfg->asid = cfg->cbndx;
800
801 smmu_domain->smmu = smmu;
802 if (smmu->impl && smmu->impl->init_context) {
803 ret = smmu->impl->init_context(smmu_domain);
804 if (ret)
805 goto out_unlock;
806 }
807
808 pgtbl_cfg = (struct io_pgtable_cfg) {
809 .pgsize_bitmap = smmu->pgsize_bitmap,
810 .ias = ias,
811 .oas = oas,
812 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
813 .tlb = smmu_domain->flush_ops,
814 .iommu_dev = smmu->dev,
815 };
816
817 if (smmu_domain->non_strict)
818 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
819
820 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
821 if (!pgtbl_ops) {
822 ret = -ENOMEM;
823 goto out_clear_smmu;
824 }
825
826 /* Update the domain's page sizes to reflect the page table format */
827 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
828 domain->geometry.aperture_end = (1UL << ias) - 1;
829 domain->geometry.force_aperture = true;
830
831 /* Initialise the context bank with our page table cfg */
832 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
833 arm_smmu_write_context_bank(smmu, cfg->cbndx);
834
835 /*
836 * Request context fault interrupt. Do this last to avoid the
837 * handler seeing a half-initialised domain state.
838 */
839 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
840 ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
841 IRQF_SHARED, "arm-smmu-context-fault", domain);
842 if (ret < 0) {
843 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
844 cfg->irptndx, irq);
845 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
846 }
847
848 mutex_unlock(&smmu_domain->init_mutex);
849
850 /* Publish page table ops for map/unmap */
851 smmu_domain->pgtbl_ops = pgtbl_ops;
852 return 0;
853
854 out_clear_smmu:
855 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
856 smmu_domain->smmu = NULL;
857 out_unlock:
858 mutex_unlock(&smmu_domain->init_mutex);
859 return ret;
860 }
861
862 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
863 {
864 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
865 struct arm_smmu_device *smmu = smmu_domain->smmu;
866 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
867 int ret, irq;
868
869 if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
870 return;
871
872 ret = arm_smmu_rpm_get(smmu);
873 if (ret < 0)
874 return;
875
876 /*
877 * Disable the context bank and free the page tables before freeing
878 * it.
879 */
880 smmu->cbs[cfg->cbndx].cfg = NULL;
881 arm_smmu_write_context_bank(smmu, cfg->cbndx);
882
883 if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
884 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
885 devm_free_irq(smmu->dev, irq, domain);
886 }
887
888 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
889 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
890
891 arm_smmu_rpm_put(smmu);
892 }
893
894 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
895 {
896 struct arm_smmu_domain *smmu_domain;
897
898 if (type != IOMMU_DOMAIN_UNMANAGED &&
899 type != IOMMU_DOMAIN_DMA &&
900 type != IOMMU_DOMAIN_IDENTITY)
901 return NULL;
902 /*
903 * Allocate the domain and initialise some of its data structures.
904 * We can't really do anything meaningful until we've added a
905 * master.
906 */
907 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
908 if (!smmu_domain)
909 return NULL;
910
911 if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
912 iommu_get_dma_cookie(&smmu_domain->domain))) {
913 kfree(smmu_domain);
914 return NULL;
915 }
916
917 mutex_init(&smmu_domain->init_mutex);
918 spin_lock_init(&smmu_domain->cb_lock);
919
920 return &smmu_domain->domain;
921 }
922
923 static void arm_smmu_domain_free(struct iommu_domain *domain)
924 {
925 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
926
927 /*
928 * Free the domain resources. We assume that all devices have
929 * already been detached.
930 */
931 iommu_put_dma_cookie(domain);
932 arm_smmu_destroy_domain_context(domain);
933 kfree(smmu_domain);
934 }
935
936 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
937 {
938 struct arm_smmu_smr *smr = smmu->smrs + idx;
939 u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
940 FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
941
942 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
943 reg |= ARM_SMMU_SMR_VALID;
944 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
945 }
946
947 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
948 {
949 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
950 u32 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
951 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
952 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
953
954 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
955 smmu->smrs[idx].valid)
956 reg |= ARM_SMMU_S2CR_EXIDVALID;
957 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
958 }
959
960 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
961 {
962 arm_smmu_write_s2cr(smmu, idx);
963 if (smmu->smrs)
964 arm_smmu_write_smr(smmu, idx);
965 }
966
967 /*
968 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
969 * should be called after sCR0 is written.
970 */
971 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
972 {
973 u32 smr;
974 int i;
975
976 if (!smmu->smrs)
977 return;
978 /*
979 * If we've had to accommodate firmware memory regions, we may
980 * have live SMRs by now; tread carefully...
981 *
982 * Somewhat perversely, not having a free SMR for this test implies we
983 * can get away without it anyway, as we'll only be able to 'allocate'
984 * these SMRs for the ID/mask values we're already trusting to be OK.
985 */
986 for (i = 0; i < smmu->num_mapping_groups; i++)
987 if (!smmu->smrs[i].valid)
988 goto smr_ok;
989 return;
990 smr_ok:
991 /*
992 * SMR.ID bits may not be preserved if the corresponding MASK
993 * bits are set, so check each one separately. We can reject
994 * masters later if they try to claim IDs outside these masks.
995 */
996 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
997 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
998 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
999 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
1000
1001 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
1002 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
1003 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
1004 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
1005 }
1006
1007 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1008 {
1009 struct arm_smmu_smr *smrs = smmu->smrs;
1010 int i, free_idx = -ENOSPC;
1011
1012 /* Stream indexing is blissfully easy */
1013 if (!smrs)
1014 return id;
1015
1016 /* Validating SMRs is... less so */
1017 for (i = 0; i < smmu->num_mapping_groups; ++i) {
1018 if (!smrs[i].valid) {
1019 /*
1020 * Note the first free entry we come across, which
1021 * we'll claim in the end if nothing else matches.
1022 */
1023 if (free_idx < 0)
1024 free_idx = i;
1025 continue;
1026 }
1027 /*
1028 * If the new entry is _entirely_ matched by an existing entry,
1029 * then reuse that, with the guarantee that there also cannot
1030 * be any subsequent conflicting entries. In normal use we'd
1031 * expect simply identical entries for this case, but there's
1032 * no harm in accommodating the generalisation.
1033 */
1034 if ((mask & smrs[i].mask) == mask &&
1035 !((id ^ smrs[i].id) & ~smrs[i].mask))
1036 return i;
1037 /*
1038 * If the new entry has any other overlap with an existing one,
1039 * though, then there always exists at least one stream ID
1040 * which would cause a conflict, and we can't allow that risk.
1041 */
1042 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1043 return -EINVAL;
1044 }
1045
1046 return free_idx;
1047 }
1048
1049 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1050 {
1051 if (--smmu->s2crs[idx].count)
1052 return false;
1053
1054 smmu->s2crs[idx] = s2cr_init_val;
1055 if (smmu->smrs)
1056 smmu->smrs[idx].valid = false;
1057
1058 return true;
1059 }
1060
1061 static int arm_smmu_master_alloc_smes(struct device *dev)
1062 {
1063 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1064 struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1065 struct arm_smmu_device *smmu = cfg->smmu;
1066 struct arm_smmu_smr *smrs = smmu->smrs;
1067 struct iommu_group *group;
1068 int i, idx, ret;
1069
1070 mutex_lock(&smmu->stream_map_mutex);
1071 /* Figure out a viable stream map entry allocation */
1072 for_each_cfg_sme(fwspec, i, idx) {
1073 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1074 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1075
1076 if (idx != INVALID_SMENDX) {
1077 ret = -EEXIST;
1078 goto out_err;
1079 }
1080
1081 ret = arm_smmu_find_sme(smmu, sid, mask);
1082 if (ret < 0)
1083 goto out_err;
1084
1085 idx = ret;
1086 if (smrs && smmu->s2crs[idx].count == 0) {
1087 smrs[idx].id = sid;
1088 smrs[idx].mask = mask;
1089 smrs[idx].valid = true;
1090 }
1091 smmu->s2crs[idx].count++;
1092 cfg->smendx[i] = (s16)idx;
1093 }
1094
1095 group = iommu_group_get_for_dev(dev);
1096 if (IS_ERR(group)) {
1097 ret = PTR_ERR(group);
1098 goto out_err;
1099 }
1100 iommu_group_put(group);
1101
1102 /* It worked! Now, poke the actual hardware */
1103 for_each_cfg_sme(fwspec, i, idx) {
1104 arm_smmu_write_sme(smmu, idx);
1105 smmu->s2crs[idx].group = group;
1106 }
1107
1108 mutex_unlock(&smmu->stream_map_mutex);
1109 return 0;
1110
1111 out_err:
1112 while (i--) {
1113 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1114 cfg->smendx[i] = INVALID_SMENDX;
1115 }
1116 mutex_unlock(&smmu->stream_map_mutex);
1117 return ret;
1118 }
1119
1120 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1121 {
1122 struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1123 struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1124 int i, idx;
1125
1126 mutex_lock(&smmu->stream_map_mutex);
1127 for_each_cfg_sme(fwspec, i, idx) {
1128 if (arm_smmu_free_sme(smmu, idx))
1129 arm_smmu_write_sme(smmu, idx);
1130 cfg->smendx[i] = INVALID_SMENDX;
1131 }
1132 mutex_unlock(&smmu->stream_map_mutex);
1133 }
1134
1135 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1136 struct iommu_fwspec *fwspec)
1137 {
1138 struct arm_smmu_device *smmu = smmu_domain->smmu;
1139 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1140 u8 cbndx = smmu_domain->cfg.cbndx;
1141 enum arm_smmu_s2cr_type type;
1142 int i, idx;
1143
1144 if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1145 type = S2CR_TYPE_BYPASS;
1146 else
1147 type = S2CR_TYPE_TRANS;
1148
1149 for_each_cfg_sme(fwspec, i, idx) {
1150 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1151 continue;
1152
1153 s2cr[idx].type = type;
1154 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1155 s2cr[idx].cbndx = cbndx;
1156 arm_smmu_write_s2cr(smmu, idx);
1157 }
1158 return 0;
1159 }
1160
1161 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1162 {
1163 int ret;
1164 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1165 struct arm_smmu_device *smmu;
1166 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1167
1168 if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1169 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1170 return -ENXIO;
1171 }
1172
1173 /*
1174 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1175 * domains between of_xlate() and add_device() - we have no way to cope
1176 * with that, so until ARM gets converted to rely on groups and default
1177 * domains, just say no (but more politely than by dereferencing NULL).
1178 * This should be at least a WARN_ON once that's sorted.
1179 */
1180 if (!fwspec->iommu_priv)
1181 return -ENODEV;
1182
1183 smmu = fwspec_smmu(fwspec);
1184
1185 ret = arm_smmu_rpm_get(smmu);
1186 if (ret < 0)
1187 return ret;
1188
1189 /* Ensure that the domain is finalised */
1190 ret = arm_smmu_init_domain_context(domain, smmu);
1191 if (ret < 0)
1192 goto rpm_put;
1193
1194 /*
1195 * Sanity check the domain. We don't support domains across
1196 * different SMMUs.
1197 */
1198 if (smmu_domain->smmu != smmu) {
1199 dev_err(dev,
1200 "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1201 dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1202 ret = -EINVAL;
1203 goto rpm_put;
1204 }
1205
1206 /* Looks ok, so add the device to the domain */
1207 ret = arm_smmu_domain_add_master(smmu_domain, fwspec);
1208
1209 /*
1210 * Setup an autosuspend delay to avoid bouncing runpm state.
1211 * Otherwise, if a driver for a suspended consumer device
1212 * unmaps buffers, it will runpm resume/suspend for each one.
1213 *
1214 * For example, when used by a GPU device, when an application
1215 * or game exits, it can trigger unmapping 100s or 1000s of
1216 * buffers. With a runpm cycle for each buffer, that adds up
1217 * to 5-10sec worth of reprogramming the context bank, while
1218 * the system appears to be locked up to the user.
1219 */
1220 pm_runtime_set_autosuspend_delay(smmu->dev, 20);
1221 pm_runtime_use_autosuspend(smmu->dev);
1222
1223 rpm_put:
1224 arm_smmu_rpm_put(smmu);
1225 return ret;
1226 }
1227
1228 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1229 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
1230 {
1231 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1232 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1233 int ret;
1234
1235 if (!ops)
1236 return -ENODEV;
1237
1238 arm_smmu_rpm_get(smmu);
1239 ret = ops->map(ops, iova, paddr, size, prot);
1240 arm_smmu_rpm_put(smmu);
1241
1242 return ret;
1243 }
1244
1245 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1246 size_t size, struct iommu_iotlb_gather *gather)
1247 {
1248 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1249 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1250 size_t ret;
1251
1252 if (!ops)
1253 return 0;
1254
1255 arm_smmu_rpm_get(smmu);
1256 ret = ops->unmap(ops, iova, size, gather);
1257 arm_smmu_rpm_put(smmu);
1258
1259 return ret;
1260 }
1261
1262 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1263 {
1264 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1265 struct arm_smmu_device *smmu = smmu_domain->smmu;
1266
1267 if (smmu_domain->flush_ops) {
1268 arm_smmu_rpm_get(smmu);
1269 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1270 arm_smmu_rpm_put(smmu);
1271 }
1272 }
1273
1274 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1275 struct iommu_iotlb_gather *gather)
1276 {
1277 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1278 struct arm_smmu_device *smmu = smmu_domain->smmu;
1279
1280 if (!smmu)
1281 return;
1282
1283 arm_smmu_rpm_get(smmu);
1284 if (smmu->version == ARM_SMMU_V2 ||
1285 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1286 arm_smmu_tlb_sync_context(smmu_domain);
1287 else
1288 arm_smmu_tlb_sync_global(smmu);
1289 arm_smmu_rpm_put(smmu);
1290 }
1291
1292 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1293 dma_addr_t iova)
1294 {
1295 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1296 struct arm_smmu_device *smmu = smmu_domain->smmu;
1297 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1298 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1299 struct device *dev = smmu->dev;
1300 void __iomem *reg;
1301 u32 tmp;
1302 u64 phys;
1303 unsigned long va, flags;
1304 int ret, idx = cfg->cbndx;
1305
1306 ret = arm_smmu_rpm_get(smmu);
1307 if (ret < 0)
1308 return 0;
1309
1310 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1311 va = iova & ~0xfffUL;
1312 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1313 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1314 else
1315 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1316
1317 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1318 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE),
1319 5, 50)) {
1320 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1321 dev_err(dev,
1322 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1323 &iova);
1324 return ops->iova_to_phys(ops, iova);
1325 }
1326
1327 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1328 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1329 if (phys & ARM_SMMU_CB_PAR_F) {
1330 dev_err(dev, "translation fault!\n");
1331 dev_err(dev, "PAR = 0x%llx\n", phys);
1332 return 0;
1333 }
1334
1335 arm_smmu_rpm_put(smmu);
1336
1337 return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1338 }
1339
1340 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1341 dma_addr_t iova)
1342 {
1343 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1344 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1345
1346 if (domain->type == IOMMU_DOMAIN_IDENTITY)
1347 return iova;
1348
1349 if (!ops)
1350 return 0;
1351
1352 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1353 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1354 return arm_smmu_iova_to_phys_hard(domain, iova);
1355
1356 return ops->iova_to_phys(ops, iova);
1357 }
1358
1359 static bool arm_smmu_capable(enum iommu_cap cap)
1360 {
1361 switch (cap) {
1362 case IOMMU_CAP_CACHE_COHERENCY:
1363 /*
1364 * Return true here as the SMMU can always send out coherent
1365 * requests.
1366 */
1367 return true;
1368 case IOMMU_CAP_NOEXEC:
1369 return true;
1370 default:
1371 return false;
1372 }
1373 }
1374
1375 static
1376 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1377 {
1378 struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
1379 fwnode);
1380 put_device(dev);
1381 return dev ? dev_get_drvdata(dev) : NULL;
1382 }
1383
1384 static int arm_smmu_add_device(struct device *dev)
1385 {
1386 struct arm_smmu_device *smmu;
1387 struct arm_smmu_master_cfg *cfg;
1388 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1389 int i, ret;
1390
1391 if (using_legacy_binding) {
1392 ret = arm_smmu_register_legacy_master(dev, &smmu);
1393
1394 /*
1395 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1396 * will allocate/initialise a new one. Thus we need to update fwspec for
1397 * later use.
1398 */
1399 fwspec = dev_iommu_fwspec_get(dev);
1400 if (ret)
1401 goto out_free;
1402 } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1403 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1404 } else {
1405 return -ENODEV;
1406 }
1407
1408 ret = -EINVAL;
1409 for (i = 0; i < fwspec->num_ids; i++) {
1410 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1411 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1412
1413 if (sid & ~smmu->streamid_mask) {
1414 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1415 sid, smmu->streamid_mask);
1416 goto out_free;
1417 }
1418 if (mask & ~smmu->smr_mask_mask) {
1419 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1420 mask, smmu->smr_mask_mask);
1421 goto out_free;
1422 }
1423 }
1424
1425 ret = -ENOMEM;
1426 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1427 GFP_KERNEL);
1428 if (!cfg)
1429 goto out_free;
1430
1431 cfg->smmu = smmu;
1432 fwspec->iommu_priv = cfg;
1433 while (i--)
1434 cfg->smendx[i] = INVALID_SMENDX;
1435
1436 ret = arm_smmu_rpm_get(smmu);
1437 if (ret < 0)
1438 goto out_cfg_free;
1439
1440 ret = arm_smmu_master_alloc_smes(dev);
1441 arm_smmu_rpm_put(smmu);
1442
1443 if (ret)
1444 goto out_cfg_free;
1445
1446 iommu_device_link(&smmu->iommu, dev);
1447
1448 device_link_add(dev, smmu->dev,
1449 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1450
1451 return 0;
1452
1453 out_cfg_free:
1454 kfree(cfg);
1455 out_free:
1456 iommu_fwspec_free(dev);
1457 return ret;
1458 }
1459
1460 static void arm_smmu_remove_device(struct device *dev)
1461 {
1462 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1463 struct arm_smmu_master_cfg *cfg;
1464 struct arm_smmu_device *smmu;
1465 int ret;
1466
1467 if (!fwspec || fwspec->ops != &arm_smmu_ops)
1468 return;
1469
1470 cfg = fwspec->iommu_priv;
1471 smmu = cfg->smmu;
1472
1473 ret = arm_smmu_rpm_get(smmu);
1474 if (ret < 0)
1475 return;
1476
1477 iommu_device_unlink(&smmu->iommu, dev);
1478 arm_smmu_master_free_smes(fwspec);
1479
1480 arm_smmu_rpm_put(smmu);
1481
1482 iommu_group_remove_device(dev);
1483 kfree(fwspec->iommu_priv);
1484 iommu_fwspec_free(dev);
1485 }
1486
1487 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1488 {
1489 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1490 struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1491 struct iommu_group *group = NULL;
1492 int i, idx;
1493
1494 for_each_cfg_sme(fwspec, i, idx) {
1495 if (group && smmu->s2crs[idx].group &&
1496 group != smmu->s2crs[idx].group)
1497 return ERR_PTR(-EINVAL);
1498
1499 group = smmu->s2crs[idx].group;
1500 }
1501
1502 if (group)
1503 return iommu_group_ref_get(group);
1504
1505 if (dev_is_pci(dev))
1506 group = pci_device_group(dev);
1507 else if (dev_is_fsl_mc(dev))
1508 group = fsl_mc_device_group(dev);
1509 else
1510 group = generic_device_group(dev);
1511
1512 return group;
1513 }
1514
1515 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1516 enum iommu_attr attr, void *data)
1517 {
1518 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1519
1520 switch(domain->type) {
1521 case IOMMU_DOMAIN_UNMANAGED:
1522 switch (attr) {
1523 case DOMAIN_ATTR_NESTING:
1524 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1525 return 0;
1526 default:
1527 return -ENODEV;
1528 }
1529 break;
1530 case IOMMU_DOMAIN_DMA:
1531 switch (attr) {
1532 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1533 *(int *)data = smmu_domain->non_strict;
1534 return 0;
1535 default:
1536 return -ENODEV;
1537 }
1538 break;
1539 default:
1540 return -EINVAL;
1541 }
1542 }
1543
1544 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1545 enum iommu_attr attr, void *data)
1546 {
1547 int ret = 0;
1548 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1549
1550 mutex_lock(&smmu_domain->init_mutex);
1551
1552 switch(domain->type) {
1553 case IOMMU_DOMAIN_UNMANAGED:
1554 switch (attr) {
1555 case DOMAIN_ATTR_NESTING:
1556 if (smmu_domain->smmu) {
1557 ret = -EPERM;
1558 goto out_unlock;
1559 }
1560
1561 if (*(int *)data)
1562 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1563 else
1564 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1565 break;
1566 default:
1567 ret = -ENODEV;
1568 }
1569 break;
1570 case IOMMU_DOMAIN_DMA:
1571 switch (attr) {
1572 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1573 smmu_domain->non_strict = *(int *)data;
1574 break;
1575 default:
1576 ret = -ENODEV;
1577 }
1578 break;
1579 default:
1580 ret = -EINVAL;
1581 }
1582 out_unlock:
1583 mutex_unlock(&smmu_domain->init_mutex);
1584 return ret;
1585 }
1586
1587 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1588 {
1589 u32 mask, fwid = 0;
1590
1591 if (args->args_count > 0)
1592 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1593
1594 if (args->args_count > 1)
1595 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1596 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1597 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1598
1599 return iommu_fwspec_add_ids(dev, &fwid, 1);
1600 }
1601
1602 static void arm_smmu_get_resv_regions(struct device *dev,
1603 struct list_head *head)
1604 {
1605 struct iommu_resv_region *region;
1606 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1607
1608 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1609 prot, IOMMU_RESV_SW_MSI);
1610 if (!region)
1611 return;
1612
1613 list_add_tail(&region->list, head);
1614
1615 iommu_dma_get_resv_regions(dev, head);
1616 }
1617
1618 static struct iommu_ops arm_smmu_ops = {
1619 .capable = arm_smmu_capable,
1620 .domain_alloc = arm_smmu_domain_alloc,
1621 .domain_free = arm_smmu_domain_free,
1622 .attach_dev = arm_smmu_attach_dev,
1623 .map = arm_smmu_map,
1624 .unmap = arm_smmu_unmap,
1625 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
1626 .iotlb_sync = arm_smmu_iotlb_sync,
1627 .iova_to_phys = arm_smmu_iova_to_phys,
1628 .add_device = arm_smmu_add_device,
1629 .remove_device = arm_smmu_remove_device,
1630 .device_group = arm_smmu_device_group,
1631 .domain_get_attr = arm_smmu_domain_get_attr,
1632 .domain_set_attr = arm_smmu_domain_set_attr,
1633 .of_xlate = arm_smmu_of_xlate,
1634 .get_resv_regions = arm_smmu_get_resv_regions,
1635 .put_resv_regions = generic_iommu_put_resv_regions,
1636 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1637 };
1638
1639 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1640 {
1641 int i;
1642 u32 reg;
1643
1644 /* clear global FSR */
1645 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1646 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1647
1648 /*
1649 * Reset stream mapping groups: Initial values mark all SMRn as
1650 * invalid and all S2CRn as bypass unless overridden.
1651 */
1652 for (i = 0; i < smmu->num_mapping_groups; ++i)
1653 arm_smmu_write_sme(smmu, i);
1654
1655 /* Make sure all context banks are disabled and clear CB_FSR */
1656 for (i = 0; i < smmu->num_context_banks; ++i) {
1657 arm_smmu_write_context_bank(smmu, i);
1658 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT);
1659 }
1660
1661 /* Invalidate the TLB, just in case */
1662 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1663 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1664
1665 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1666
1667 /* Enable fault reporting */
1668 reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1669 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1670
1671 /* Disable TLB broadcasting. */
1672 reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1673
1674 /* Enable client access, handling unmatched streams as appropriate */
1675 reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1676 if (disable_bypass)
1677 reg |= ARM_SMMU_sCR0_USFCFG;
1678 else
1679 reg &= ~ARM_SMMU_sCR0_USFCFG;
1680
1681 /* Disable forced broadcasting */
1682 reg &= ~ARM_SMMU_sCR0_FB;
1683
1684 /* Don't upgrade barriers */
1685 reg &= ~(ARM_SMMU_sCR0_BSU);
1686
1687 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1688 reg |= ARM_SMMU_sCR0_VMID16EN;
1689
1690 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1691 reg |= ARM_SMMU_sCR0_EXIDENABLE;
1692
1693 if (smmu->impl && smmu->impl->reset)
1694 smmu->impl->reset(smmu);
1695
1696 /* Push the button */
1697 arm_smmu_tlb_sync_global(smmu);
1698 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1699 }
1700
1701 static int arm_smmu_id_size_to_bits(int size)
1702 {
1703 switch (size) {
1704 case 0:
1705 return 32;
1706 case 1:
1707 return 36;
1708 case 2:
1709 return 40;
1710 case 3:
1711 return 42;
1712 case 4:
1713 return 44;
1714 case 5:
1715 default:
1716 return 48;
1717 }
1718 }
1719
1720 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1721 {
1722 unsigned int size;
1723 u32 id;
1724 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1725 int i;
1726
1727 dev_notice(smmu->dev, "probing hardware configuration...\n");
1728 dev_notice(smmu->dev, "SMMUv%d with:\n",
1729 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1730
1731 /* ID0 */
1732 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1733
1734 /* Restrict available stages based on module parameter */
1735 if (force_stage == 1)
1736 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1737 else if (force_stage == 2)
1738 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1739
1740 if (id & ARM_SMMU_ID0_S1TS) {
1741 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1742 dev_notice(smmu->dev, "\tstage 1 translation\n");
1743 }
1744
1745 if (id & ARM_SMMU_ID0_S2TS) {
1746 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1747 dev_notice(smmu->dev, "\tstage 2 translation\n");
1748 }
1749
1750 if (id & ARM_SMMU_ID0_NTS) {
1751 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1752 dev_notice(smmu->dev, "\tnested translation\n");
1753 }
1754
1755 if (!(smmu->features &
1756 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1757 dev_err(smmu->dev, "\tno translation support!\n");
1758 return -ENODEV;
1759 }
1760
1761 if ((id & ARM_SMMU_ID0_S1TS) &&
1762 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1763 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1764 dev_notice(smmu->dev, "\taddress translation ops\n");
1765 }
1766
1767 /*
1768 * In order for DMA API calls to work properly, we must defer to what
1769 * the FW says about coherency, regardless of what the hardware claims.
1770 * Fortunately, this also opens up a workaround for systems where the
1771 * ID register value has ended up configured incorrectly.
1772 */
1773 cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1774 if (cttw_fw || cttw_reg)
1775 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1776 cttw_fw ? "" : "non-");
1777 if (cttw_fw != cttw_reg)
1778 dev_notice(smmu->dev,
1779 "\t(IDR0.CTTW overridden by FW configuration)\n");
1780
1781 /* Max. number of entries we have for stream matching/indexing */
1782 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1783 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1784 size = 1 << 16;
1785 } else {
1786 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1787 }
1788 smmu->streamid_mask = size - 1;
1789 if (id & ARM_SMMU_ID0_SMS) {
1790 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1791 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1792 if (size == 0) {
1793 dev_err(smmu->dev,
1794 "stream-matching supported, but no SMRs present!\n");
1795 return -ENODEV;
1796 }
1797
1798 /* Zero-initialised to mark as invalid */
1799 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1800 GFP_KERNEL);
1801 if (!smmu->smrs)
1802 return -ENOMEM;
1803
1804 dev_notice(smmu->dev,
1805 "\tstream matching with %u register groups", size);
1806 }
1807 /* s2cr->type == 0 means translation, so initialise explicitly */
1808 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1809 GFP_KERNEL);
1810 if (!smmu->s2crs)
1811 return -ENOMEM;
1812 for (i = 0; i < size; i++)
1813 smmu->s2crs[i] = s2cr_init_val;
1814
1815 smmu->num_mapping_groups = size;
1816 mutex_init(&smmu->stream_map_mutex);
1817 spin_lock_init(&smmu->global_sync_lock);
1818
1819 if (smmu->version < ARM_SMMU_V2 ||
1820 !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1821 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1822 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1823 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1824 }
1825
1826 /* ID1 */
1827 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1828 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1829
1830 /* Check for size mismatch of SMMU address space from mapped region */
1831 size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1832 if (smmu->numpage != 2 * size << smmu->pgshift)
1833 dev_warn(smmu->dev,
1834 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1835 2 * size << smmu->pgshift, smmu->numpage);
1836 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1837 smmu->numpage = size;
1838
1839 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1840 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1841 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1842 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1843 return -ENODEV;
1844 }
1845 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1846 smmu->num_context_banks, smmu->num_s2_context_banks);
1847 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1848 sizeof(*smmu->cbs), GFP_KERNEL);
1849 if (!smmu->cbs)
1850 return -ENOMEM;
1851
1852 /* ID2 */
1853 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1854 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1855 smmu->ipa_size = size;
1856
1857 /* The output mask is also applied for bypass */
1858 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1859 smmu->pa_size = size;
1860
1861 if (id & ARM_SMMU_ID2_VMID16)
1862 smmu->features |= ARM_SMMU_FEAT_VMID16;
1863
1864 /*
1865 * What the page table walker can address actually depends on which
1866 * descriptor format is in use, but since a) we don't know that yet,
1867 * and b) it can vary per context bank, this will have to do...
1868 */
1869 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1870 dev_warn(smmu->dev,
1871 "failed to set DMA mask for table walker\n");
1872
1873 if (smmu->version < ARM_SMMU_V2) {
1874 smmu->va_size = smmu->ipa_size;
1875 if (smmu->version == ARM_SMMU_V1_64K)
1876 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1877 } else {
1878 size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1879 smmu->va_size = arm_smmu_id_size_to_bits(size);
1880 if (id & ARM_SMMU_ID2_PTFS_4K)
1881 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1882 if (id & ARM_SMMU_ID2_PTFS_16K)
1883 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1884 if (id & ARM_SMMU_ID2_PTFS_64K)
1885 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1886 }
1887
1888 /* Now we've corralled the various formats, what'll it do? */
1889 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1890 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1891 if (smmu->features &
1892 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1893 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1894 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1895 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1896 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1897 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1898
1899 if (arm_smmu_ops.pgsize_bitmap == -1UL)
1900 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1901 else
1902 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1903 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1904 smmu->pgsize_bitmap);
1905
1906
1907 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1908 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1909 smmu->va_size, smmu->ipa_size);
1910
1911 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1912 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1913 smmu->ipa_size, smmu->pa_size);
1914
1915 if (smmu->impl && smmu->impl->cfg_probe)
1916 return smmu->impl->cfg_probe(smmu);
1917
1918 return 0;
1919 }
1920
1921 struct arm_smmu_match_data {
1922 enum arm_smmu_arch_version version;
1923 enum arm_smmu_implementation model;
1924 };
1925
1926 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1927 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1928
1929 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1930 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1931 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1932 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1933 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1934 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1935
1936 static const struct of_device_id arm_smmu_of_match[] = {
1937 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1938 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1939 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1940 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1941 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1942 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1943 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1944 { },
1945 };
1946 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1947
1948 #ifdef CONFIG_ACPI
1949 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1950 {
1951 int ret = 0;
1952
1953 switch (model) {
1954 case ACPI_IORT_SMMU_V1:
1955 case ACPI_IORT_SMMU_CORELINK_MMU400:
1956 smmu->version = ARM_SMMU_V1;
1957 smmu->model = GENERIC_SMMU;
1958 break;
1959 case ACPI_IORT_SMMU_CORELINK_MMU401:
1960 smmu->version = ARM_SMMU_V1_64K;
1961 smmu->model = GENERIC_SMMU;
1962 break;
1963 case ACPI_IORT_SMMU_V2:
1964 smmu->version = ARM_SMMU_V2;
1965 smmu->model = GENERIC_SMMU;
1966 break;
1967 case ACPI_IORT_SMMU_CORELINK_MMU500:
1968 smmu->version = ARM_SMMU_V2;
1969 smmu->model = ARM_MMU500;
1970 break;
1971 case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1972 smmu->version = ARM_SMMU_V2;
1973 smmu->model = CAVIUM_SMMUV2;
1974 break;
1975 default:
1976 ret = -ENODEV;
1977 }
1978
1979 return ret;
1980 }
1981
1982 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1983 struct arm_smmu_device *smmu)
1984 {
1985 struct device *dev = smmu->dev;
1986 struct acpi_iort_node *node =
1987 *(struct acpi_iort_node **)dev_get_platdata(dev);
1988 struct acpi_iort_smmu *iort_smmu;
1989 int ret;
1990
1991 /* Retrieve SMMU1/2 specific data */
1992 iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1993
1994 ret = acpi_smmu_get_data(iort_smmu->model, smmu);
1995 if (ret < 0)
1996 return ret;
1997
1998 /* Ignore the configuration access interrupt */
1999 smmu->num_global_irqs = 1;
2000
2001 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2002 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2003
2004 return 0;
2005 }
2006 #else
2007 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2008 struct arm_smmu_device *smmu)
2009 {
2010 return -ENODEV;
2011 }
2012 #endif
2013
2014 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2015 struct arm_smmu_device *smmu)
2016 {
2017 const struct arm_smmu_match_data *data;
2018 struct device *dev = &pdev->dev;
2019 bool legacy_binding;
2020
2021 if (of_property_read_u32(dev->of_node, "#global-interrupts",
2022 &smmu->num_global_irqs)) {
2023 dev_err(dev, "missing #global-interrupts property\n");
2024 return -ENODEV;
2025 }
2026
2027 data = of_device_get_match_data(dev);
2028 smmu->version = data->version;
2029 smmu->model = data->model;
2030
2031 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2032 if (legacy_binding && !using_generic_binding) {
2033 if (!using_legacy_binding) {
2034 pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
2035 IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
2036 }
2037 using_legacy_binding = true;
2038 } else if (!legacy_binding && !using_legacy_binding) {
2039 using_generic_binding = true;
2040 } else {
2041 dev_err(dev, "not probing due to mismatched DT properties\n");
2042 return -ENODEV;
2043 }
2044
2045 if (of_dma_is_coherent(dev->of_node))
2046 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2047
2048 return 0;
2049 }
2050
2051 static int arm_smmu_bus_init(struct iommu_ops *ops)
2052 {
2053 int err;
2054
2055 /* Oh, for a proper bus abstraction */
2056 if (!iommu_present(&platform_bus_type)) {
2057 err = bus_set_iommu(&platform_bus_type, ops);
2058 if (err)
2059 return err;
2060 }
2061 #ifdef CONFIG_ARM_AMBA
2062 if (!iommu_present(&amba_bustype)) {
2063 err = bus_set_iommu(&amba_bustype, ops);
2064 if (err)
2065 goto err_reset_platform_ops;
2066 }
2067 #endif
2068 #ifdef CONFIG_PCI
2069 if (!iommu_present(&pci_bus_type)) {
2070 err = bus_set_iommu(&pci_bus_type, ops);
2071 if (err)
2072 goto err_reset_amba_ops;
2073 }
2074 #endif
2075 #ifdef CONFIG_FSL_MC_BUS
2076 if (!iommu_present(&fsl_mc_bus_type)) {
2077 err = bus_set_iommu(&fsl_mc_bus_type, ops);
2078 if (err)
2079 goto err_reset_pci_ops;
2080 }
2081 #endif
2082 return 0;
2083
2084 err_reset_pci_ops: __maybe_unused;
2085 #ifdef CONFIG_PCI
2086 bus_set_iommu(&pci_bus_type, NULL);
2087 #endif
2088 err_reset_amba_ops: __maybe_unused;
2089 #ifdef CONFIG_ARM_AMBA
2090 bus_set_iommu(&amba_bustype, NULL);
2091 #endif
2092 err_reset_platform_ops: __maybe_unused;
2093 bus_set_iommu(&platform_bus_type, NULL);
2094 return err;
2095 }
2096
2097 static int arm_smmu_device_probe(struct platform_device *pdev)
2098 {
2099 struct resource *res;
2100 resource_size_t ioaddr;
2101 struct arm_smmu_device *smmu;
2102 struct device *dev = &pdev->dev;
2103 int num_irqs, i, err;
2104
2105 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2106 if (!smmu) {
2107 dev_err(dev, "failed to allocate arm_smmu_device\n");
2108 return -ENOMEM;
2109 }
2110 smmu->dev = dev;
2111
2112 if (dev->of_node)
2113 err = arm_smmu_device_dt_probe(pdev, smmu);
2114 else
2115 err = arm_smmu_device_acpi_probe(pdev, smmu);
2116
2117 if (err)
2118 return err;
2119
2120 smmu = arm_smmu_impl_init(smmu);
2121 if (IS_ERR(smmu))
2122 return PTR_ERR(smmu);
2123
2124 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2125 ioaddr = res->start;
2126 smmu->base = devm_ioremap_resource(dev, res);
2127 if (IS_ERR(smmu->base))
2128 return PTR_ERR(smmu->base);
2129 /*
2130 * The resource size should effectively match the value of SMMU_TOP;
2131 * stash that temporarily until we know PAGESIZE to validate it with.
2132 */
2133 smmu->numpage = resource_size(res);
2134
2135 num_irqs = 0;
2136 while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2137 num_irqs++;
2138 if (num_irqs > smmu->num_global_irqs)
2139 smmu->num_context_irqs++;
2140 }
2141
2142 if (!smmu->num_context_irqs) {
2143 dev_err(dev, "found %d interrupts but expected at least %d\n",
2144 num_irqs, smmu->num_global_irqs + 1);
2145 return -ENODEV;
2146 }
2147
2148 smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),
2149 GFP_KERNEL);
2150 if (!smmu->irqs) {
2151 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2152 return -ENOMEM;
2153 }
2154
2155 for (i = 0; i < num_irqs; ++i) {
2156 int irq = platform_get_irq(pdev, i);
2157
2158 if (irq < 0)
2159 return -ENODEV;
2160 smmu->irqs[i] = irq;
2161 }
2162
2163 err = devm_clk_bulk_get_all(dev, &smmu->clks);
2164 if (err < 0) {
2165 dev_err(dev, "failed to get clocks %d\n", err);
2166 return err;
2167 }
2168 smmu->num_clks = err;
2169
2170 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2171 if (err)
2172 return err;
2173
2174 err = arm_smmu_device_cfg_probe(smmu);
2175 if (err)
2176 return err;
2177
2178 if (smmu->version == ARM_SMMU_V2) {
2179 if (smmu->num_context_banks > smmu->num_context_irqs) {
2180 dev_err(dev,
2181 "found only %d context irq(s) but %d required\n",
2182 smmu->num_context_irqs, smmu->num_context_banks);
2183 return -ENODEV;
2184 }
2185
2186 /* Ignore superfluous interrupts */
2187 smmu->num_context_irqs = smmu->num_context_banks;
2188 }
2189
2190 for (i = 0; i < smmu->num_global_irqs; ++i) {
2191 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2192 arm_smmu_global_fault,
2193 IRQF_SHARED,
2194 "arm-smmu global fault",
2195 smmu);
2196 if (err) {
2197 dev_err(dev, "failed to request global IRQ %d (%u)\n",
2198 i, smmu->irqs[i]);
2199 return err;
2200 }
2201 }
2202
2203 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2204 "smmu.%pa", &ioaddr);
2205 if (err) {
2206 dev_err(dev, "Failed to register iommu in sysfs\n");
2207 return err;
2208 }
2209
2210 iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2211 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2212
2213 err = iommu_device_register(&smmu->iommu);
2214 if (err) {
2215 dev_err(dev, "Failed to register iommu\n");
2216 return err;
2217 }
2218
2219 platform_set_drvdata(pdev, smmu);
2220 arm_smmu_device_reset(smmu);
2221 arm_smmu_test_smr_masks(smmu);
2222
2223 /*
2224 * We want to avoid touching dev->power.lock in fastpaths unless
2225 * it's really going to do something useful - pm_runtime_enabled()
2226 * can serve as an ideal proxy for that decision. So, conditionally
2227 * enable pm_runtime.
2228 */
2229 if (dev->pm_domain) {
2230 pm_runtime_set_active(dev);
2231 pm_runtime_enable(dev);
2232 }
2233
2234 /*
2235 * For ACPI and generic DT bindings, an SMMU will be probed before
2236 * any device which might need it, so we want the bus ops in place
2237 * ready to handle default domain setup as soon as any SMMU exists.
2238 */
2239 if (!using_legacy_binding)
2240 return arm_smmu_bus_init(&arm_smmu_ops);
2241
2242 return 0;
2243 }
2244
2245 static int arm_smmu_device_remove(struct platform_device *pdev)
2246 {
2247 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2248
2249 if (!smmu)
2250 return -ENODEV;
2251
2252 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2253 dev_err(&pdev->dev, "removing device with active domains!\n");
2254
2255 arm_smmu_bus_init(NULL);
2256 iommu_device_unregister(&smmu->iommu);
2257 iommu_device_sysfs_remove(&smmu->iommu);
2258
2259 arm_smmu_rpm_get(smmu);
2260 /* Turn the thing off */
2261 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2262 arm_smmu_rpm_put(smmu);
2263
2264 if (pm_runtime_enabled(smmu->dev))
2265 pm_runtime_force_suspend(smmu->dev);
2266 else
2267 clk_bulk_disable(smmu->num_clks, smmu->clks);
2268
2269 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2270 return 0;
2271 }
2272
2273 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2274 {
2275 arm_smmu_device_remove(pdev);
2276 }
2277
2278 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2279 {
2280 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2281 int ret;
2282
2283 ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2284 if (ret)
2285 return ret;
2286
2287 arm_smmu_device_reset(smmu);
2288
2289 return 0;
2290 }
2291
2292 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2293 {
2294 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2295
2296 clk_bulk_disable(smmu->num_clks, smmu->clks);
2297
2298 return 0;
2299 }
2300
2301 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2302 {
2303 if (pm_runtime_suspended(dev))
2304 return 0;
2305
2306 return arm_smmu_runtime_resume(dev);
2307 }
2308
2309 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2310 {
2311 if (pm_runtime_suspended(dev))
2312 return 0;
2313
2314 return arm_smmu_runtime_suspend(dev);
2315 }
2316
2317 static const struct dev_pm_ops arm_smmu_pm_ops = {
2318 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2319 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2320 arm_smmu_runtime_resume, NULL)
2321 };
2322
2323 static struct platform_driver arm_smmu_driver = {
2324 .driver = {
2325 .name = "arm-smmu",
2326 .of_match_table = arm_smmu_of_match,
2327 .pm = &arm_smmu_pm_ops,
2328 .suppress_bind_attrs = true,
2329 },
2330 .probe = arm_smmu_device_probe,
2331 .remove = arm_smmu_device_remove,
2332 .shutdown = arm_smmu_device_shutdown,
2333 };
2334 module_platform_driver(arm_smmu_driver);
2335
2336 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2337 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2338 MODULE_ALIAS("platform:arm-smmu");
2339 MODULE_LICENSE("GPL v2");
Ubuntu Jammy Linux kernel mirror