]>
Commit | Line | Data |
---|---|---|
45051539 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
45ae7cff WD |
2 | /* |
3 | * IOMMU API for ARM architected SMMU implementations. | |
4 | * | |
45ae7cff WD |
5 | * Copyright (C) 2013 ARM Limited |
6 | * | |
7 | * Author: Will Deacon <will.deacon@arm.com> | |
8 | * | |
9 | * This driver currently supports: | |
10 | * - SMMUv1 and v2 implementations | |
11 | * - Stream-matching and stream-indexing | |
12 | * - v7/v8 long-descriptor format | |
13 | * - Non-secure access to the SMMU | |
45ae7cff | 14 | * - Context fault reporting |
dc0eaa4e | 15 | * - Extended Stream ID (16 bit) |
45ae7cff WD |
16 | */ |
17 | ||
18 | #define pr_fmt(fmt) "arm-smmu: " fmt | |
19 | ||
d6fcd3b1 LP |
20 | #include <linux/acpi.h> |
21 | #include <linux/acpi_iort.h> | |
1f3d5ca4 | 22 | #include <linux/atomic.h> |
45ae7cff | 23 | #include <linux/delay.h> |
9adb9594 | 24 | #include <linux/dma-iommu.h> |
45ae7cff WD |
25 | #include <linux/dma-mapping.h> |
26 | #include <linux/err.h> | |
27 | #include <linux/interrupt.h> | |
28 | #include <linux/io.h> | |
f9a05f05 | 29 | #include <linux/io-64-nonatomic-hi-lo.h> |
b77cf11f | 30 | #include <linux/io-pgtable.h> |
45ae7cff | 31 | #include <linux/iommu.h> |
859a732e | 32 | #include <linux/iopoll.h> |
addb672f PG |
33 | #include <linux/init.h> |
34 | #include <linux/moduleparam.h> | |
45ae7cff | 35 | #include <linux/of.h> |
bae2c2d4 | 36 | #include <linux/of_address.h> |
d6fc5d97 | 37 | #include <linux/of_device.h> |
adfec2e7 | 38 | #include <linux/of_iommu.h> |
a9a1b0b5 | 39 | #include <linux/pci.h> |
45ae7cff | 40 | #include <linux/platform_device.h> |
96a299d2 | 41 | #include <linux/pm_runtime.h> |
45ae7cff WD |
42 | #include <linux/slab.h> |
43 | #include <linux/spinlock.h> | |
44 | ||
45 | #include <linux/amba/bus.h> | |
eab03e2a | 46 | #include <linux/fsl/mc.h> |
45ae7cff | 47 | |
2b03774b RC |
48 | #include "arm-smmu-regs.h" |
49 | ||
4e4abae3 RM |
50 | /* |
51 | * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU | |
52 | * global register space are still, in fact, using a hypervisor to mediate it | |
53 | * by trapping and emulating register accesses. Sadly, some deployed versions | |
54 | * of said trapping code have bugs wherein they go horribly wrong for stores | |
55 | * using r31 (i.e. XZR/WZR) as the source register. | |
56 | */ | |
57 | #define QCOM_DUMMY_VAL -1 | |
58 | ||
2b03774b RC |
59 | #define ARM_MMU500_ACTLR_CPRE (1 << 1) |
60 | ||
61 | #define ARM_MMU500_ACR_CACHE_LOCK (1 << 26) | |
74f55d34 | 62 | #define ARM_MMU500_ACR_S2CRB_TLBEN (1 << 10) |
2b03774b RC |
63 | #define ARM_MMU500_ACR_SMTNMB_TLBEN (1 << 8) |
64 | ||
65 | #define TLB_LOOP_TIMEOUT 1000000 /* 1s! */ | |
66 | #define TLB_SPIN_COUNT 10 | |
45ae7cff | 67 | |
45ae7cff WD |
68 | /* Maximum number of context banks per SMMU */ |
69 | #define ARM_SMMU_MAX_CBS 128 | |
70 | ||
45ae7cff WD |
71 | /* SMMU global address space */ |
72 | #define ARM_SMMU_GR0(smmu) ((smmu)->base) | |
c757e852 | 73 | #define ARM_SMMU_GR1(smmu) ((smmu)->base + (1 << (smmu)->pgshift)) |
45ae7cff | 74 | |
3a5df8ff AH |
75 | /* |
76 | * SMMU global address space with conditional offset to access secure | |
77 | * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448, | |
78 | * nsGFSYNR0: 0x450) | |
79 | */ | |
80 | #define ARM_SMMU_GR0_NS(smmu) \ | |
81 | ((smmu)->base + \ | |
82 | ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \ | |
83 | ? 0x400 : 0)) | |
84 | ||
f9a05f05 RM |
85 | /* |
86 | * Some 64-bit registers only make sense to write atomically, but in such | |
87 | * cases all the data relevant to AArch32 formats lies within the lower word, | |
88 | * therefore this actually makes more sense than it might first appear. | |
89 | */ | |
668b4ada | 90 | #ifdef CONFIG_64BIT |
f9a05f05 | 91 | #define smmu_write_atomic_lq writeq_relaxed |
668b4ada | 92 | #else |
f9a05f05 | 93 | #define smmu_write_atomic_lq writel_relaxed |
668b4ada TC |
94 | #endif |
95 | ||
45ae7cff | 96 | /* Translation context bank */ |
452107c7 | 97 | #define ARM_SMMU_CB(smmu, n) ((smmu)->cb_base + ((n) << (smmu)->pgshift)) |
45ae7cff | 98 | |
f3ebee80 EA |
99 | #define MSI_IOVA_BASE 0x8000000 |
100 | #define MSI_IOVA_LENGTH 0x100000 | |
101 | ||
4cf740b0 | 102 | static int force_stage; |
addb672f PG |
103 | /* |
104 | * not really modular, but the easiest way to keep compat with existing | |
105 | * bootargs behaviour is to continue using module_param() here. | |
106 | */ | |
25a1c96c | 107 | module_param(force_stage, int, S_IRUGO); |
4cf740b0 WD |
108 | MODULE_PARM_DESC(force_stage, |
109 | "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation."); | |
954a03be DA |
110 | static bool disable_bypass = |
111 | IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT); | |
25a1c96c RM |
112 | module_param(disable_bypass, bool, S_IRUGO); |
113 | MODULE_PARM_DESC(disable_bypass, | |
114 | "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU."); | |
4cf740b0 | 115 | |
09360403 | 116 | enum arm_smmu_arch_version { |
b7862e35 RM |
117 | ARM_SMMU_V1, |
118 | ARM_SMMU_V1_64K, | |
09360403 RM |
119 | ARM_SMMU_V2, |
120 | }; | |
121 | ||
67b65a3f RM |
122 | enum arm_smmu_implementation { |
123 | GENERIC_SMMU, | |
f0cfffc4 | 124 | ARM_MMU500, |
e086d912 | 125 | CAVIUM_SMMUV2, |
89cddc56 | 126 | QCOM_SMMUV2, |
67b65a3f RM |
127 | }; |
128 | ||
8e8b203e | 129 | struct arm_smmu_s2cr { |
588888a7 RM |
130 | struct iommu_group *group; |
131 | int count; | |
8e8b203e RM |
132 | enum arm_smmu_s2cr_type type; |
133 | enum arm_smmu_s2cr_privcfg privcfg; | |
134 | u8 cbndx; | |
135 | }; | |
136 | ||
137 | #define s2cr_init_val (struct arm_smmu_s2cr){ \ | |
138 | .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \ | |
139 | } | |
140 | ||
45ae7cff | 141 | struct arm_smmu_smr { |
45ae7cff WD |
142 | u16 mask; |
143 | u16 id; | |
1f3d5ca4 | 144 | bool valid; |
45ae7cff WD |
145 | }; |
146 | ||
90df373c RM |
147 | struct arm_smmu_cb { |
148 | u64 ttbr[2]; | |
149 | u32 tcr[2]; | |
150 | u32 mair[2]; | |
151 | struct arm_smmu_cfg *cfg; | |
152 | }; | |
153 | ||
a9a1b0b5 | 154 | struct arm_smmu_master_cfg { |
f80cd885 | 155 | struct arm_smmu_device *smmu; |
adfec2e7 | 156 | s16 smendx[]; |
45ae7cff | 157 | }; |
1f3d5ca4 | 158 | #define INVALID_SMENDX -1 |
adfec2e7 RM |
159 | #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv) |
160 | #define fwspec_smmu(fw) (__fwspec_cfg(fw)->smmu) | |
8c82d6ec RM |
161 | #define fwspec_smendx(fw, i) \ |
162 | (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i]) | |
adfec2e7 | 163 | #define for_each_cfg_sme(fw, i, idx) \ |
8c82d6ec | 164 | for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i) |
45ae7cff WD |
165 | |
166 | struct arm_smmu_device { | |
167 | struct device *dev; | |
45ae7cff WD |
168 | |
169 | void __iomem *base; | |
452107c7 | 170 | void __iomem *cb_base; |
c757e852 | 171 | unsigned long pgshift; |
45ae7cff WD |
172 | |
173 | #define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0) | |
174 | #define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1) | |
175 | #define ARM_SMMU_FEAT_TRANS_S1 (1 << 2) | |
176 | #define ARM_SMMU_FEAT_TRANS_S2 (1 << 3) | |
177 | #define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4) | |
859a732e | 178 | #define ARM_SMMU_FEAT_TRANS_OPS (1 << 5) |
4e3e9b69 | 179 | #define ARM_SMMU_FEAT_VMID16 (1 << 6) |
7602b871 RM |
180 | #define ARM_SMMU_FEAT_FMT_AARCH64_4K (1 << 7) |
181 | #define ARM_SMMU_FEAT_FMT_AARCH64_16K (1 << 8) | |
182 | #define ARM_SMMU_FEAT_FMT_AARCH64_64K (1 << 9) | |
183 | #define ARM_SMMU_FEAT_FMT_AARCH32_L (1 << 10) | |
184 | #define ARM_SMMU_FEAT_FMT_AARCH32_S (1 << 11) | |
dc0eaa4e | 185 | #define ARM_SMMU_FEAT_EXIDS (1 << 12) |
45ae7cff | 186 | u32 features; |
3a5df8ff AH |
187 | |
188 | #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0) | |
189 | u32 options; | |
09360403 | 190 | enum arm_smmu_arch_version version; |
67b65a3f | 191 | enum arm_smmu_implementation model; |
45ae7cff WD |
192 | |
193 | u32 num_context_banks; | |
194 | u32 num_s2_context_banks; | |
195 | DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS); | |
90df373c | 196 | struct arm_smmu_cb *cbs; |
45ae7cff WD |
197 | atomic_t irptndx; |
198 | ||
199 | u32 num_mapping_groups; | |
21174240 RM |
200 | u16 streamid_mask; |
201 | u16 smr_mask_mask; | |
1f3d5ca4 | 202 | struct arm_smmu_smr *smrs; |
8e8b203e | 203 | struct arm_smmu_s2cr *s2crs; |
588888a7 | 204 | struct mutex stream_map_mutex; |
45ae7cff | 205 | |
518f7136 WD |
206 | unsigned long va_size; |
207 | unsigned long ipa_size; | |
208 | unsigned long pa_size; | |
d5466357 | 209 | unsigned long pgsize_bitmap; |
45ae7cff WD |
210 | |
211 | u32 num_global_irqs; | |
212 | u32 num_context_irqs; | |
213 | unsigned int *irqs; | |
96a299d2 S |
214 | struct clk_bulk_data *clks; |
215 | int num_clks; | |
45ae7cff | 216 | |
1bd37a68 | 217 | u32 cavium_id_base; /* Specific to Cavium */ |
9648cbc9 | 218 | |
8e517e76 WD |
219 | spinlock_t global_sync_lock; |
220 | ||
9648cbc9 JR |
221 | /* IOMMU core code handle */ |
222 | struct iommu_device iommu; | |
45ae7cff WD |
223 | }; |
224 | ||
7602b871 RM |
225 | enum arm_smmu_context_fmt { |
226 | ARM_SMMU_CTX_FMT_NONE, | |
227 | ARM_SMMU_CTX_FMT_AARCH64, | |
228 | ARM_SMMU_CTX_FMT_AARCH32_L, | |
229 | ARM_SMMU_CTX_FMT_AARCH32_S, | |
45ae7cff WD |
230 | }; |
231 | ||
232 | struct arm_smmu_cfg { | |
45ae7cff WD |
233 | u8 cbndx; |
234 | u8 irptndx; | |
280b683c RM |
235 | union { |
236 | u16 asid; | |
237 | u16 vmid; | |
238 | }; | |
45ae7cff | 239 | u32 cbar; |
7602b871 | 240 | enum arm_smmu_context_fmt fmt; |
45ae7cff | 241 | }; |
faea13b7 | 242 | #define INVALID_IRPTNDX 0xff |
45ae7cff | 243 | |
c752ce45 WD |
244 | enum arm_smmu_domain_stage { |
245 | ARM_SMMU_DOMAIN_S1 = 0, | |
246 | ARM_SMMU_DOMAIN_S2, | |
247 | ARM_SMMU_DOMAIN_NESTED, | |
61bc6711 | 248 | ARM_SMMU_DOMAIN_BYPASS, |
c752ce45 WD |
249 | }; |
250 | ||
45ae7cff | 251 | struct arm_smmu_domain { |
44680eed | 252 | struct arm_smmu_device *smmu; |
518f7136 | 253 | struct io_pgtable_ops *pgtbl_ops; |
32b12449 | 254 | const struct iommu_gather_ops *tlb_ops; |
44680eed | 255 | struct arm_smmu_cfg cfg; |
c752ce45 | 256 | enum arm_smmu_domain_stage stage; |
44f6876a | 257 | bool non_strict; |
518f7136 | 258 | struct mutex init_mutex; /* Protects smmu pointer */ |
8e517e76 | 259 | spinlock_t cb_lock; /* Serialises ATS1* ops and TLB syncs */ |
1d672638 | 260 | struct iommu_domain domain; |
45ae7cff WD |
261 | }; |
262 | ||
3a5df8ff AH |
263 | struct arm_smmu_option_prop { |
264 | u32 opt; | |
265 | const char *prop; | |
266 | }; | |
267 | ||
1bd37a68 TC |
268 | static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0); |
269 | ||
021bb842 RM |
270 | static bool using_legacy_binding, using_generic_binding; |
271 | ||
2907320d | 272 | static struct arm_smmu_option_prop arm_smmu_options[] = { |
3a5df8ff AH |
273 | { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" }, |
274 | { 0, NULL}, | |
275 | }; | |
276 | ||
d4a44f07 S |
277 | static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu) |
278 | { | |
279 | if (pm_runtime_enabled(smmu->dev)) | |
280 | return pm_runtime_get_sync(smmu->dev); | |
281 | ||
282 | return 0; | |
283 | } | |
284 | ||
285 | static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu) | |
286 | { | |
287 | if (pm_runtime_enabled(smmu->dev)) | |
288 | pm_runtime_put(smmu->dev); | |
289 | } | |
290 | ||
1d672638 JR |
291 | static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) |
292 | { | |
293 | return container_of(dom, struct arm_smmu_domain, domain); | |
294 | } | |
295 | ||
3a5df8ff AH |
296 | static void parse_driver_options(struct arm_smmu_device *smmu) |
297 | { | |
298 | int i = 0; | |
2907320d | 299 | |
3a5df8ff AH |
300 | do { |
301 | if (of_property_read_bool(smmu->dev->of_node, | |
302 | arm_smmu_options[i].prop)) { | |
303 | smmu->options |= arm_smmu_options[i].opt; | |
304 | dev_notice(smmu->dev, "option %s\n", | |
305 | arm_smmu_options[i].prop); | |
306 | } | |
307 | } while (arm_smmu_options[++i].opt); | |
308 | } | |
309 | ||
8f68f8e2 | 310 | static struct device_node *dev_get_dev_node(struct device *dev) |
a9a1b0b5 WD |
311 | { |
312 | if (dev_is_pci(dev)) { | |
313 | struct pci_bus *bus = to_pci_dev(dev)->bus; | |
2907320d | 314 | |
a9a1b0b5 WD |
315 | while (!pci_is_root_bus(bus)) |
316 | bus = bus->parent; | |
f80cd885 | 317 | return of_node_get(bus->bridge->parent->of_node); |
a9a1b0b5 WD |
318 | } |
319 | ||
f80cd885 | 320 | return of_node_get(dev->of_node); |
a9a1b0b5 WD |
321 | } |
322 | ||
f80cd885 | 323 | static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data) |
45ae7cff | 324 | { |
f80cd885 RM |
325 | *((__be32 *)data) = cpu_to_be32(alias); |
326 | return 0; /* Continue walking */ | |
45ae7cff WD |
327 | } |
328 | ||
f80cd885 | 329 | static int __find_legacy_master_phandle(struct device *dev, void *data) |
a9a1b0b5 | 330 | { |
f80cd885 RM |
331 | struct of_phandle_iterator *it = *(void **)data; |
332 | struct device_node *np = it->node; | |
333 | int err; | |
334 | ||
335 | of_for_each_phandle(it, err, dev->of_node, "mmu-masters", | |
336 | "#stream-id-cells", 0) | |
337 | if (it->node == np) { | |
338 | *(void **)data = dev; | |
339 | return 1; | |
340 | } | |
341 | it->node = np; | |
342 | return err == -ENOENT ? 0 : err; | |
a9a1b0b5 WD |
343 | } |
344 | ||
d6fc5d97 | 345 | static struct platform_driver arm_smmu_driver; |
adfec2e7 | 346 | static struct iommu_ops arm_smmu_ops; |
d6fc5d97 | 347 | |
adfec2e7 RM |
348 | static int arm_smmu_register_legacy_master(struct device *dev, |
349 | struct arm_smmu_device **smmu) | |
45ae7cff | 350 | { |
adfec2e7 | 351 | struct device *smmu_dev; |
f80cd885 RM |
352 | struct device_node *np; |
353 | struct of_phandle_iterator it; | |
354 | void *data = ⁢ | |
adfec2e7 | 355 | u32 *sids; |
f80cd885 RM |
356 | __be32 pci_sid; |
357 | int err; | |
45ae7cff | 358 | |
f80cd885 RM |
359 | np = dev_get_dev_node(dev); |
360 | if (!np || !of_find_property(np, "#stream-id-cells", NULL)) { | |
361 | of_node_put(np); | |
362 | return -ENODEV; | |
363 | } | |
45ae7cff | 364 | |
f80cd885 | 365 | it.node = np; |
d6fc5d97 RM |
366 | err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data, |
367 | __find_legacy_master_phandle); | |
adfec2e7 | 368 | smmu_dev = data; |
f80cd885 RM |
369 | of_node_put(np); |
370 | if (err == 0) | |
371 | return -ENODEV; | |
372 | if (err < 0) | |
373 | return err; | |
45ae7cff | 374 | |
f80cd885 RM |
375 | if (dev_is_pci(dev)) { |
376 | /* "mmu-masters" assumes Stream ID == Requester ID */ | |
377 | pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid, | |
378 | &pci_sid); | |
379 | it.cur = &pci_sid; | |
380 | it.cur_count = 1; | |
381 | } | |
45ae7cff | 382 | |
adfec2e7 RM |
383 | err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode, |
384 | &arm_smmu_ops); | |
385 | if (err) | |
386 | return err; | |
45ae7cff | 387 | |
adfec2e7 RM |
388 | sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL); |
389 | if (!sids) | |
390 | return -ENOMEM; | |
44680eed | 391 | |
adfec2e7 RM |
392 | *smmu = dev_get_drvdata(smmu_dev); |
393 | of_phandle_iterator_args(&it, sids, it.cur_count); | |
394 | err = iommu_fwspec_add_ids(dev, sids, it.cur_count); | |
395 | kfree(sids); | |
396 | return err; | |
45ae7cff WD |
397 | } |
398 | ||
399 | static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end) | |
400 | { | |
401 | int idx; | |
402 | ||
403 | do { | |
404 | idx = find_next_zero_bit(map, end, start); | |
405 | if (idx == end) | |
406 | return -ENOSPC; | |
407 | } while (test_and_set_bit(idx, map)); | |
408 | ||
409 | return idx; | |
410 | } | |
411 | ||
412 | static void __arm_smmu_free_bitmap(unsigned long *map, int idx) | |
413 | { | |
414 | clear_bit(idx, map); | |
415 | } | |
416 | ||
417 | /* Wait for any pending TLB invalidations to complete */ | |
11febfca RM |
418 | static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, |
419 | void __iomem *sync, void __iomem *status) | |
45ae7cff | 420 | { |
8513c893 | 421 | unsigned int spin_cnt, delay; |
45ae7cff | 422 | |
4e4abae3 | 423 | writel_relaxed(QCOM_DUMMY_VAL, sync); |
8513c893 RM |
424 | for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) { |
425 | for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) { | |
426 | if (!(readl_relaxed(status) & sTLBGSTATUS_GSACTIVE)) | |
427 | return; | |
428 | cpu_relax(); | |
45ae7cff | 429 | } |
8513c893 | 430 | udelay(delay); |
45ae7cff | 431 | } |
8513c893 RM |
432 | dev_err_ratelimited(smmu->dev, |
433 | "TLB sync timed out -- SMMU may be deadlocked\n"); | |
45ae7cff WD |
434 | } |
435 | ||
11febfca RM |
436 | static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu) |
437 | { | |
438 | void __iomem *base = ARM_SMMU_GR0(smmu); | |
8e517e76 | 439 | unsigned long flags; |
11febfca | 440 | |
8e517e76 | 441 | spin_lock_irqsave(&smmu->global_sync_lock, flags); |
11febfca RM |
442 | __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_GR0_sTLBGSYNC, |
443 | base + ARM_SMMU_GR0_sTLBGSTATUS); | |
8e517e76 | 444 | spin_unlock_irqrestore(&smmu->global_sync_lock, flags); |
11febfca RM |
445 | } |
446 | ||
447 | static void arm_smmu_tlb_sync_context(void *cookie) | |
518f7136 WD |
448 | { |
449 | struct arm_smmu_domain *smmu_domain = cookie; | |
11febfca RM |
450 | struct arm_smmu_device *smmu = smmu_domain->smmu; |
451 | void __iomem *base = ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx); | |
8e517e76 | 452 | unsigned long flags; |
11febfca | 453 | |
8e517e76 | 454 | spin_lock_irqsave(&smmu_domain->cb_lock, flags); |
11febfca RM |
455 | __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_CB_TLBSYNC, |
456 | base + ARM_SMMU_CB_TLBSTATUS); | |
8e517e76 | 457 | spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); |
518f7136 WD |
458 | } |
459 | ||
11febfca | 460 | static void arm_smmu_tlb_sync_vmid(void *cookie) |
518f7136 WD |
461 | { |
462 | struct arm_smmu_domain *smmu_domain = cookie; | |
11febfca RM |
463 | |
464 | arm_smmu_tlb_sync_global(smmu_domain->smmu); | |
518f7136 WD |
465 | } |
466 | ||
11febfca | 467 | static void arm_smmu_tlb_inv_context_s1(void *cookie) |
1463fe44 | 468 | { |
518f7136 | 469 | struct arm_smmu_domain *smmu_domain = cookie; |
44680eed | 470 | struct arm_smmu_cfg *cfg = &smmu_domain->cfg; |
11febfca | 471 | void __iomem *base = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx); |
1463fe44 | 472 | |
44f6876a RM |
473 | /* |
474 | * NOTE: this is not a relaxed write; it needs to guarantee that PTEs | |
475 | * cleared by the current CPU are visible to the SMMU before the TLBI. | |
476 | */ | |
477 | writel(cfg->asid, base + ARM_SMMU_CB_S1_TLBIASID); | |
11febfca RM |
478 | arm_smmu_tlb_sync_context(cookie); |
479 | } | |
1463fe44 | 480 | |
11febfca RM |
481 | static void arm_smmu_tlb_inv_context_s2(void *cookie) |
482 | { | |
483 | struct arm_smmu_domain *smmu_domain = cookie; | |
484 | struct arm_smmu_device *smmu = smmu_domain->smmu; | |
485 | void __iomem *base = ARM_SMMU_GR0(smmu); | |
1463fe44 | 486 | |
44f6876a RM |
487 | /* NOTE: see above */ |
488 | writel(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID); | |
11febfca | 489 | arm_smmu_tlb_sync_global(smmu); |
518f7136 WD |
490 | } |
491 | ||
492 | static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, | |
06c610e8 | 493 | size_t granule, bool leaf, void *cookie) |
518f7136 WD |
494 | { |
495 | struct arm_smmu_domain *smmu_domain = cookie; | |
496 | struct arm_smmu_cfg *cfg = &smmu_domain->cfg; | |
518f7136 | 497 | bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; |
11febfca | 498 | void __iomem *reg = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx); |
518f7136 | 499 | |
7d321bd3 WD |
500 | if (smmu_domain->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) |
501 | wmb(); | |
502 | ||
518f7136 | 503 | if (stage1) { |
518f7136 WD |
504 | reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA; |
505 | ||
7602b871 | 506 | if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) { |
518f7136 | 507 | iova &= ~12UL; |
280b683c | 508 | iova |= cfg->asid; |
75df1386 RM |
509 | do { |
510 | writel_relaxed(iova, reg); | |
511 | iova += granule; | |
512 | } while (size -= granule); | |
518f7136 WD |
513 | } else { |
514 | iova >>= 12; | |
280b683c | 515 | iova |= (u64)cfg->asid << 48; |
75df1386 RM |
516 | do { |
517 | writeq_relaxed(iova, reg); | |
518 | iova += granule >> 12; | |
519 | } while (size -= granule); | |
518f7136 | 520 | } |
11febfca | 521 | } else { |
518f7136 WD |
522 | reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L : |
523 | ARM_SMMU_CB_S2_TLBIIPAS2; | |
75df1386 RM |
524 | iova >>= 12; |
525 | do { | |
f9a05f05 | 526 | smmu_write_atomic_lq(iova, reg); |
75df1386 RM |
527 | iova += granule >> 12; |
528 | } while (size -= granule); | |
518f7136 WD |
529 | } |
530 | } | |
531 | ||
11febfca RM |
532 | /* |
533 | * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears | |
534 | * almost negligible, but the benefit of getting the first one in as far ahead | |
535 | * of the sync as possible is significant, hence we don't just make this a | |
536 | * no-op and set .tlb_sync to arm_smmu_inv_context_s2() as you might think. | |
537 | */ | |
538 | static void arm_smmu_tlb_inv_vmid_nosync(unsigned long iova, size_t size, | |
539 | size_t granule, bool leaf, void *cookie) | |
540 | { | |
541 | struct arm_smmu_domain *smmu_domain = cookie; | |
542 | void __iomem *base = ARM_SMMU_GR0(smmu_domain->smmu); | |
543 | ||
7d321bd3 WD |
544 | if (smmu_domain->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) |
545 | wmb(); | |
546 | ||
11febfca RM |
547 | writel_relaxed(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID); |
548 | } | |
549 | ||
550 | static const struct iommu_gather_ops arm_smmu_s1_tlb_ops = { | |
551 | .tlb_flush_all = arm_smmu_tlb_inv_context_s1, | |
518f7136 | 552 | .tlb_add_flush = arm_smmu_tlb_inv_range_nosync, |
11febfca RM |
553 | .tlb_sync = arm_smmu_tlb_sync_context, |
554 | }; | |
555 | ||
556 | static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v2 = { | |
557 | .tlb_flush_all = arm_smmu_tlb_inv_context_s2, | |
518f7136 | 558 | .tlb_add_flush = arm_smmu_tlb_inv_range_nosync, |
11febfca RM |
559 | .tlb_sync = arm_smmu_tlb_sync_context, |
560 | }; | |
561 | ||
562 | static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v1 = { | |
563 | .tlb_flush_all = arm_smmu_tlb_inv_context_s2, | |
564 | .tlb_add_flush = arm_smmu_tlb_inv_vmid_nosync, | |
565 | .tlb_sync = arm_smmu_tlb_sync_vmid, | |
518f7136 WD |
566 | }; |
567 | ||
45ae7cff WD |
568 | static irqreturn_t arm_smmu_context_fault(int irq, void *dev) |
569 | { | |
bc580b56 | 570 | u32 fsr, fsynr, cbfrsynra; |
45ae7cff WD |
571 | unsigned long iova; |
572 | struct iommu_domain *domain = dev; | |
1d672638 | 573 | struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); |
44680eed WD |
574 | struct arm_smmu_cfg *cfg = &smmu_domain->cfg; |
575 | struct arm_smmu_device *smmu = smmu_domain->smmu; | |
bc580b56 | 576 | void __iomem *gr1_base = ARM_SMMU_GR1(smmu); |
45ae7cff WD |
577 | void __iomem *cb_base; |
578 | ||
452107c7 | 579 | cb_base = ARM_SMMU_CB(smmu, cfg->cbndx); |
45ae7cff WD |
580 | fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR); |
581 | ||
582 | if (!(fsr & FSR_FAULT)) | |
583 | return IRQ_NONE; | |
584 | ||
45ae7cff | 585 | fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0); |
f9a05f05 | 586 | iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR); |
bc580b56 | 587 | cbfrsynra = readl_relaxed(gr1_base + ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx)); |
45ae7cff | 588 | |
3714ce1d | 589 | dev_err_ratelimited(smmu->dev, |
bc580b56 VG |
590 | "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n", |
591 | fsr, iova, fsynr, cbfrsynra, cfg->cbndx); | |
45ae7cff | 592 | |
3714ce1d WD |
593 | writel(fsr, cb_base + ARM_SMMU_CB_FSR); |
594 | return IRQ_HANDLED; | |
45ae7cff WD |
595 | } |
596 | ||
597 | static irqreturn_t arm_smmu_global_fault(int irq, void *dev) | |
598 | { | |
599 | u32 gfsr, gfsynr0, gfsynr1, gfsynr2; | |
600 | struct arm_smmu_device *smmu = dev; | |
3a5df8ff | 601 | void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu); |
45ae7cff WD |
602 | |
603 | gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR); | |
604 | gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0); | |
605 | gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1); | |
606 | gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2); | |
607 | ||
3a5df8ff AH |
608 | if (!gfsr) |
609 | return IRQ_NONE; | |
610 | ||
45ae7cff WD |
611 | dev_err_ratelimited(smmu->dev, |
612 | "Unexpected global fault, this could be serious\n"); | |
613 | dev_err_ratelimited(smmu->dev, | |
614 | "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n", | |
615 | gfsr, gfsynr0, gfsynr1, gfsynr2); | |
616 | ||
617 | writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR); | |
adaba320 | 618 | return IRQ_HANDLED; |
45ae7cff WD |
619 | } |
620 | ||
518f7136 WD |
621 | static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, |
622 | struct io_pgtable_cfg *pgtbl_cfg) | |
45ae7cff | 623 | { |
44680eed | 624 | struct arm_smmu_cfg *cfg = &smmu_domain->cfg; |
90df373c RM |
625 | struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx]; |
626 | bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; | |
627 | ||
628 | cb->cfg = cfg; | |
629 | ||
630 | /* TTBCR */ | |
631 | if (stage1) { | |
632 | if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { | |
633 | cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr; | |
634 | } else { | |
635 | cb->tcr[0] = pgtbl_cfg->arm_lpae_s1_cfg.tcr; | |
636 | cb->tcr[1] = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32; | |
637 | cb->tcr[1] |= TTBCR2_SEP_UPSTREAM; | |
638 | if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) | |
639 | cb->tcr[1] |= TTBCR2_AS; | |
640 | } | |
641 | } else { | |
642 | cb->tcr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vtcr; | |
643 | } | |
644 | ||
645 | /* TTBRs */ | |
646 | if (stage1) { | |
647 | if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { | |
648 | cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr[0]; | |
649 | cb->ttbr[1] = pgtbl_cfg->arm_v7s_cfg.ttbr[1]; | |
650 | } else { | |
651 | cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0]; | |
652 | cb->ttbr[0] |= (u64)cfg->asid << TTBRn_ASID_SHIFT; | |
653 | cb->ttbr[1] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1]; | |
654 | cb->ttbr[1] |= (u64)cfg->asid << TTBRn_ASID_SHIFT; | |
655 | } | |
656 | } else { | |
657 | cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; | |
658 | } | |
659 | ||
660 | /* MAIRs (stage-1 only) */ | |
661 | if (stage1) { | |
662 | if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { | |
663 | cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr; | |
664 | cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr; | |
665 | } else { | |
666 | cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair[0]; | |
667 | cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair[1]; | |
668 | } | |
669 | } | |
670 | } | |
671 | ||
672 | static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) | |
673 | { | |
674 | u32 reg; | |
675 | bool stage1; | |
676 | struct arm_smmu_cb *cb = &smmu->cbs[idx]; | |
677 | struct arm_smmu_cfg *cfg = cb->cfg; | |
c88ae5de | 678 | void __iomem *cb_base, *gr1_base; |
45ae7cff | 679 | |
90df373c RM |
680 | cb_base = ARM_SMMU_CB(smmu, idx); |
681 | ||
682 | /* Unassigned context banks only need disabling */ | |
683 | if (!cfg) { | |
684 | writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR); | |
685 | return; | |
686 | } | |
687 | ||
45ae7cff | 688 | gr1_base = ARM_SMMU_GR1(smmu); |
44680eed | 689 | stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; |
45ae7cff | 690 | |
90df373c | 691 | /* CBA2R */ |
4a1c93cb | 692 | if (smmu->version > ARM_SMMU_V1) { |
7602b871 RM |
693 | if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) |
694 | reg = CBA2R_RW64_64BIT; | |
695 | else | |
696 | reg = CBA2R_RW64_32BIT; | |
4e3e9b69 TC |
697 | /* 16-bit VMIDs live in CBA2R */ |
698 | if (smmu->features & ARM_SMMU_FEAT_VMID16) | |
280b683c | 699 | reg |= cfg->vmid << CBA2R_VMID_SHIFT; |
4e3e9b69 | 700 | |
90df373c | 701 | writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(idx)); |
4a1c93cb WD |
702 | } |
703 | ||
45ae7cff | 704 | /* CBAR */ |
44680eed | 705 | reg = cfg->cbar; |
b7862e35 | 706 | if (smmu->version < ARM_SMMU_V2) |
2907320d | 707 | reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT; |
45ae7cff | 708 | |
57ca90f6 WD |
709 | /* |
710 | * Use the weakest shareability/memory types, so they are | |
711 | * overridden by the ttbcr/pte. | |
712 | */ | |
713 | if (stage1) { | |
714 | reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) | | |
715 | (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT); | |
4e3e9b69 TC |
716 | } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) { |
717 | /* 8-bit VMIDs live in CBAR */ | |
280b683c | 718 | reg |= cfg->vmid << CBAR_VMID_SHIFT; |
57ca90f6 | 719 | } |
90df373c | 720 | writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(idx)); |
45ae7cff | 721 | |
125458ab SG |
722 | /* |
723 | * TTBCR | |
724 | * We must write this before the TTBRs, since it determines the | |
725 | * access behaviour of some fields (in particular, ASID[15:8]). | |
726 | */ | |
90df373c RM |
727 | if (stage1 && smmu->version > ARM_SMMU_V1) |
728 | writel_relaxed(cb->tcr[1], cb_base + ARM_SMMU_CB_TTBCR2); | |
729 | writel_relaxed(cb->tcr[0], cb_base + ARM_SMMU_CB_TTBCR); | |
45ae7cff | 730 | |
518f7136 | 731 | /* TTBRs */ |
90df373c RM |
732 | if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { |
733 | writel_relaxed(cfg->asid, cb_base + ARM_SMMU_CB_CONTEXTIDR); | |
734 | writel_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0); | |
735 | writel_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1); | |
518f7136 | 736 | } else { |
90df373c RM |
737 | writeq_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0); |
738 | if (stage1) | |
739 | writeq_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1); | |
518f7136 | 740 | } |
a65217a4 | 741 | |
518f7136 | 742 | /* MAIRs (stage-1 only) */ |
45ae7cff | 743 | if (stage1) { |
90df373c RM |
744 | writel_relaxed(cb->mair[0], cb_base + ARM_SMMU_CB_S1_MAIR0); |
745 | writel_relaxed(cb->mair[1], cb_base + ARM_SMMU_CB_S1_MAIR1); | |
45ae7cff WD |
746 | } |
747 | ||
45ae7cff | 748 | /* SCTLR */ |
6070529b | 749 | reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M; |
45ae7cff WD |
750 | if (stage1) |
751 | reg |= SCTLR_S1_ASIDPNE; | |
90df373c RM |
752 | if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) |
753 | reg |= SCTLR_E; | |
754 | ||
25724841 | 755 | writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR); |
45ae7cff WD |
756 | } |
757 | ||
758 | static int arm_smmu_init_domain_context(struct iommu_domain *domain, | |
44680eed | 759 | struct arm_smmu_device *smmu) |
45ae7cff | 760 | { |
a18037b2 | 761 | int irq, start, ret = 0; |
518f7136 WD |
762 | unsigned long ias, oas; |
763 | struct io_pgtable_ops *pgtbl_ops; | |
764 | struct io_pgtable_cfg pgtbl_cfg; | |
765 | enum io_pgtable_fmt fmt; | |
1d672638 | 766 | struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); |
44680eed | 767 | struct arm_smmu_cfg *cfg = &smmu_domain->cfg; |
45ae7cff | 768 | |
518f7136 | 769 | mutex_lock(&smmu_domain->init_mutex); |
a18037b2 MH |
770 | if (smmu_domain->smmu) |
771 | goto out_unlock; | |
772 | ||
61bc6711 WD |
773 | if (domain->type == IOMMU_DOMAIN_IDENTITY) { |
774 | smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS; | |
775 | smmu_domain->smmu = smmu; | |
776 | goto out_unlock; | |
777 | } | |
778 | ||
c752ce45 WD |
779 | /* |
780 | * Mapping the requested stage onto what we support is surprisingly | |
781 | * complicated, mainly because the spec allows S1+S2 SMMUs without | |
782 | * support for nested translation. That means we end up with the | |
783 | * following table: | |
784 | * | |
785 | * Requested Supported Actual | |
786 | * S1 N S1 | |
787 | * S1 S1+S2 S1 | |
788 | * S1 S2 S2 | |
789 | * S1 S1 S1 | |
790 | * N N N | |
791 | * N S1+S2 S2 | |
792 | * N S2 S2 | |
793 | * N S1 S1 | |
794 | * | |
795 | * Note that you can't actually request stage-2 mappings. | |
796 | */ | |
797 | if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) | |
798 | smmu_domain->stage = ARM_SMMU_DOMAIN_S2; | |
799 | if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) | |
800 | smmu_domain->stage = ARM_SMMU_DOMAIN_S1; | |
801 | ||
7602b871 RM |
802 | /* |
803 | * Choosing a suitable context format is even more fiddly. Until we | |
804 | * grow some way for the caller to express a preference, and/or move | |
805 | * the decision into the io-pgtable code where it arguably belongs, | |
806 | * just aim for the closest thing to the rest of the system, and hope | |
807 | * that the hardware isn't esoteric enough that we can't assume AArch64 | |
808 | * support to be a superset of AArch32 support... | |
809 | */ | |
810 | if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L) | |
811 | cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L; | |
6070529b RM |
812 | if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) && |
813 | !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) && | |
814 | (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) && | |
815 | (smmu_domain->stage == ARM_SMMU_DOMAIN_S1)) | |
816 | cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S; | |
7602b871 RM |
817 | if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) && |
818 | (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K | | |
819 | ARM_SMMU_FEAT_FMT_AARCH64_16K | | |
820 | ARM_SMMU_FEAT_FMT_AARCH64_4K))) | |
821 | cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64; | |
822 | ||
823 | if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) { | |
824 | ret = -EINVAL; | |
825 | goto out_unlock; | |
826 | } | |
827 | ||
c752ce45 WD |
828 | switch (smmu_domain->stage) { |
829 | case ARM_SMMU_DOMAIN_S1: | |
830 | cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS; | |
831 | start = smmu->num_s2_context_banks; | |
518f7136 WD |
832 | ias = smmu->va_size; |
833 | oas = smmu->ipa_size; | |
7602b871 | 834 | if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { |
518f7136 | 835 | fmt = ARM_64_LPAE_S1; |
6070529b | 836 | } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) { |
518f7136 | 837 | fmt = ARM_32_LPAE_S1; |
7602b871 RM |
838 | ias = min(ias, 32UL); |
839 | oas = min(oas, 40UL); | |
6070529b RM |
840 | } else { |
841 | fmt = ARM_V7S; | |
842 | ias = min(ias, 32UL); | |
843 | oas = min(oas, 32UL); | |
7602b871 | 844 | } |
32b12449 | 845 | smmu_domain->tlb_ops = &arm_smmu_s1_tlb_ops; |
c752ce45 WD |
846 | break; |
847 | case ARM_SMMU_DOMAIN_NESTED: | |
45ae7cff WD |
848 | /* |
849 | * We will likely want to change this if/when KVM gets | |
850 | * involved. | |
851 | */ | |
c752ce45 | 852 | case ARM_SMMU_DOMAIN_S2: |
9c5c92e3 WD |
853 | cfg->cbar = CBAR_TYPE_S2_TRANS; |
854 | start = 0; | |
518f7136 WD |
855 | ias = smmu->ipa_size; |
856 | oas = smmu->pa_size; | |
7602b871 | 857 | if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { |
518f7136 | 858 | fmt = ARM_64_LPAE_S2; |
7602b871 | 859 | } else { |
518f7136 | 860 | fmt = ARM_32_LPAE_S2; |
7602b871 RM |
861 | ias = min(ias, 40UL); |
862 | oas = min(oas, 40UL); | |
863 | } | |
11febfca | 864 | if (smmu->version == ARM_SMMU_V2) |
32b12449 | 865 | smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v2; |
11febfca | 866 | else |
32b12449 | 867 | smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v1; |
c752ce45 WD |
868 | break; |
869 | default: | |
870 | ret = -EINVAL; | |
871 | goto out_unlock; | |
45ae7cff | 872 | } |
45ae7cff WD |
873 | ret = __arm_smmu_alloc_bitmap(smmu->context_map, start, |
874 | smmu->num_context_banks); | |
287980e4 | 875 | if (ret < 0) |
a18037b2 | 876 | goto out_unlock; |
45ae7cff | 877 | |
44680eed | 878 | cfg->cbndx = ret; |
b7862e35 | 879 | if (smmu->version < ARM_SMMU_V2) { |
44680eed WD |
880 | cfg->irptndx = atomic_inc_return(&smmu->irptndx); |
881 | cfg->irptndx %= smmu->num_context_irqs; | |
45ae7cff | 882 | } else { |
44680eed | 883 | cfg->irptndx = cfg->cbndx; |
45ae7cff WD |
884 | } |
885 | ||
280b683c RM |
886 | if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2) |
887 | cfg->vmid = cfg->cbndx + 1 + smmu->cavium_id_base; | |
888 | else | |
889 | cfg->asid = cfg->cbndx + smmu->cavium_id_base; | |
890 | ||
518f7136 | 891 | pgtbl_cfg = (struct io_pgtable_cfg) { |
d5466357 | 892 | .pgsize_bitmap = smmu->pgsize_bitmap, |
518f7136 WD |
893 | .ias = ias, |
894 | .oas = oas, | |
32b12449 | 895 | .tlb = smmu_domain->tlb_ops, |
2df7a25c | 896 | .iommu_dev = smmu->dev, |
518f7136 WD |
897 | }; |
898 | ||
81b3c252 RM |
899 | if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) |
900 | pgtbl_cfg.quirks = IO_PGTABLE_QUIRK_NO_DMA; | |
901 | ||
44f6876a RM |
902 | if (smmu_domain->non_strict) |
903 | pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT; | |
904 | ||
518f7136 WD |
905 | smmu_domain->smmu = smmu; |
906 | pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); | |
907 | if (!pgtbl_ops) { | |
908 | ret = -ENOMEM; | |
909 | goto out_clear_smmu; | |
910 | } | |
911 | ||
d5466357 RM |
912 | /* Update the domain's page sizes to reflect the page table format */ |
913 | domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; | |
455eb7d3 RM |
914 | domain->geometry.aperture_end = (1UL << ias) - 1; |
915 | domain->geometry.force_aperture = true; | |
a18037b2 | 916 | |
518f7136 WD |
917 | /* Initialise the context bank with our page table cfg */ |
918 | arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg); | |
90df373c | 919 | arm_smmu_write_context_bank(smmu, cfg->cbndx); |
518f7136 WD |
920 | |
921 | /* | |
922 | * Request context fault interrupt. Do this last to avoid the | |
923 | * handler seeing a half-initialised domain state. | |
924 | */ | |
44680eed | 925 | irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; |
bee14004 PF |
926 | ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault, |
927 | IRQF_SHARED, "arm-smmu-context-fault", domain); | |
287980e4 | 928 | if (ret < 0) { |
45ae7cff | 929 | dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", |
44680eed WD |
930 | cfg->irptndx, irq); |
931 | cfg->irptndx = INVALID_IRPTNDX; | |
45ae7cff WD |
932 | } |
933 | ||
518f7136 WD |
934 | mutex_unlock(&smmu_domain->init_mutex); |
935 | ||
936 | /* Publish page table ops for map/unmap */ | |
937 | smmu_domain->pgtbl_ops = pgtbl_ops; | |
a9a1b0b5 | 938 | return 0; |
45ae7cff | 939 | |
518f7136 WD |
940 | out_clear_smmu: |
941 | smmu_domain->smmu = NULL; | |
a18037b2 | 942 | out_unlock: |
518f7136 | 943 | mutex_unlock(&smmu_domain->init_mutex); |
45ae7cff WD |
944 | return ret; |
945 | } | |
946 | ||
947 | static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) | |
948 | { | |
1d672638 | 949 | struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); |
44680eed WD |
950 | struct arm_smmu_device *smmu = smmu_domain->smmu; |
951 | struct arm_smmu_cfg *cfg = &smmu_domain->cfg; | |
d4a44f07 | 952 | int ret, irq; |
45ae7cff | 953 | |
61bc6711 | 954 | if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY) |
45ae7cff WD |
955 | return; |
956 | ||
d4a44f07 S |
957 | ret = arm_smmu_rpm_get(smmu); |
958 | if (ret < 0) | |
959 | return; | |
960 | ||
518f7136 WD |
961 | /* |
962 | * Disable the context bank and free the page tables before freeing | |
963 | * it. | |
964 | */ | |
90df373c RM |
965 | smmu->cbs[cfg->cbndx].cfg = NULL; |
966 | arm_smmu_write_context_bank(smmu, cfg->cbndx); | |
1463fe44 | 967 | |
44680eed WD |
968 | if (cfg->irptndx != INVALID_IRPTNDX) { |
969 | irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; | |
bee14004 | 970 | devm_free_irq(smmu->dev, irq, domain); |
45ae7cff WD |
971 | } |
972 | ||
44830b0c | 973 | free_io_pgtable_ops(smmu_domain->pgtbl_ops); |
44680eed | 974 | __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); |
d4a44f07 S |
975 | |
976 | arm_smmu_rpm_put(smmu); | |
45ae7cff WD |
977 | } |
978 | ||
1d672638 | 979 | static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) |
45ae7cff WD |
980 | { |
981 | struct arm_smmu_domain *smmu_domain; | |
45ae7cff | 982 | |
61bc6711 WD |
983 | if (type != IOMMU_DOMAIN_UNMANAGED && |
984 | type != IOMMU_DOMAIN_DMA && | |
985 | type != IOMMU_DOMAIN_IDENTITY) | |
1d672638 | 986 | return NULL; |
45ae7cff WD |
987 | /* |
988 | * Allocate the domain and initialise some of its data structures. | |
989 | * We can't really do anything meaningful until we've added a | |
990 | * master. | |
991 | */ | |
992 | smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL); | |
993 | if (!smmu_domain) | |
1d672638 | 994 | return NULL; |
45ae7cff | 995 | |
021bb842 RM |
996 | if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding || |
997 | iommu_get_dma_cookie(&smmu_domain->domain))) { | |
9adb9594 RM |
998 | kfree(smmu_domain); |
999 | return NULL; | |
1000 | } | |
1001 | ||
518f7136 | 1002 | mutex_init(&smmu_domain->init_mutex); |
523d7423 | 1003 | spin_lock_init(&smmu_domain->cb_lock); |
1d672638 JR |
1004 | |
1005 | return &smmu_domain->domain; | |
45ae7cff WD |
1006 | } |
1007 | ||
1d672638 | 1008 | static void arm_smmu_domain_free(struct iommu_domain *domain) |
45ae7cff | 1009 | { |
1d672638 | 1010 | struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); |
1463fe44 WD |
1011 | |
1012 | /* | |
1013 | * Free the domain resources. We assume that all devices have | |
1014 | * already been detached. | |
1015 | */ | |
9adb9594 | 1016 | iommu_put_dma_cookie(domain); |
45ae7cff | 1017 | arm_smmu_destroy_domain_context(domain); |
45ae7cff WD |
1018 | kfree(smmu_domain); |
1019 | } | |
1020 | ||
1f3d5ca4 RM |
1021 | static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx) |
1022 | { | |
1023 | struct arm_smmu_smr *smr = smmu->smrs + idx; | |
f80cd885 | 1024 | u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT; |
1f3d5ca4 | 1025 | |
dc0eaa4e | 1026 | if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid) |
1f3d5ca4 RM |
1027 | reg |= SMR_VALID; |
1028 | writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx)); | |
1029 | } | |
1030 | ||
8e8b203e RM |
1031 | static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx) |
1032 | { | |
1033 | struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx; | |
1034 | u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT | | |
1035 | (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT | | |
1036 | (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT; | |
1037 | ||
dc0eaa4e AM |
1038 | if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs && |
1039 | smmu->smrs[idx].valid) | |
1040 | reg |= S2CR_EXIDVALID; | |
8e8b203e RM |
1041 | writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx)); |
1042 | } | |
1043 | ||
1044 | static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx) | |
1045 | { | |
1046 | arm_smmu_write_s2cr(smmu, idx); | |
1047 | if (smmu->smrs) | |
1048 | arm_smmu_write_smr(smmu, idx); | |
1049 | } | |
1050 | ||
dc0eaa4e AM |
1051 | /* |
1052 | * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function | |
1053 | * should be called after sCR0 is written. | |
1054 | */ | |
1055 | static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu) | |
1056 | { | |
1057 | void __iomem *gr0_base = ARM_SMMU_GR0(smmu); | |
1058 | u32 smr; | |
1059 | ||
1060 | if (!smmu->smrs) | |
1061 | return; | |
1062 | ||
1063 | /* | |
1064 | * SMR.ID bits may not be preserved if the corresponding MASK | |
1065 | * bits are set, so check each one separately. We can reject | |
1066 | * masters later if they try to claim IDs outside these masks. | |
1067 | */ | |
1068 | smr = smmu->streamid_mask << SMR_ID_SHIFT; | |
1069 | writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0)); | |
1070 | smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0)); | |
1071 | smmu->streamid_mask = smr >> SMR_ID_SHIFT; | |
1072 | ||
1073 | smr = smmu->streamid_mask << SMR_MASK_SHIFT; | |
1074 | writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0)); | |
1075 | smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0)); | |
1076 | smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT; | |
1077 | } | |
1078 | ||
588888a7 | 1079 | static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask) |
1f3d5ca4 RM |
1080 | { |
1081 | struct arm_smmu_smr *smrs = smmu->smrs; | |
588888a7 | 1082 | int i, free_idx = -ENOSPC; |
1f3d5ca4 | 1083 | |
588888a7 RM |
1084 | /* Stream indexing is blissfully easy */ |
1085 | if (!smrs) | |
1086 | return id; | |
1087 | ||
1088 | /* Validating SMRs is... less so */ | |
1089 | for (i = 0; i < smmu->num_mapping_groups; ++i) { | |
1090 | if (!smrs[i].valid) { | |
1091 | /* | |
1092 | * Note the first free entry we come across, which | |
1093 | * we'll claim in the end if nothing else matches. | |
1094 | */ | |
1095 | if (free_idx < 0) | |
1096 | free_idx = i; | |
1f3d5ca4 RM |
1097 | continue; |
1098 | } | |
588888a7 RM |
1099 | /* |
1100 | * If the new entry is _entirely_ matched by an existing entry, | |
1101 | * then reuse that, with the guarantee that there also cannot | |
1102 | * be any subsequent conflicting entries. In normal use we'd | |
1103 | * expect simply identical entries for this case, but there's | |
1104 | * no harm in accommodating the generalisation. | |
1105 | */ | |
1106 | if ((mask & smrs[i].mask) == mask && | |
1107 | !((id ^ smrs[i].id) & ~smrs[i].mask)) | |
1108 | return i; | |
1109 | /* | |
1110 | * If the new entry has any other overlap with an existing one, | |
1111 | * though, then there always exists at least one stream ID | |
1112 | * which would cause a conflict, and we can't allow that risk. | |
1113 | */ | |
1114 | if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask))) | |
1115 | return -EINVAL; | |
1116 | } | |
1f3d5ca4 | 1117 | |
588888a7 RM |
1118 | return free_idx; |
1119 | } | |
1120 | ||
1121 | static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx) | |
1122 | { | |
1123 | if (--smmu->s2crs[idx].count) | |
1124 | return false; | |
1125 | ||
1126 | smmu->s2crs[idx] = s2cr_init_val; | |
1127 | if (smmu->smrs) | |
1128 | smmu->smrs[idx].valid = false; | |
1129 | ||
1130 | return true; | |
1131 | } | |
1132 | ||
1133 | static int arm_smmu_master_alloc_smes(struct device *dev) | |
1134 | { | |
9b468f7d | 1135 | struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); |
adfec2e7 | 1136 | struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv; |
588888a7 RM |
1137 | struct arm_smmu_device *smmu = cfg->smmu; |
1138 | struct arm_smmu_smr *smrs = smmu->smrs; | |
1139 | struct iommu_group *group; | |
1140 | int i, idx, ret; | |
1141 | ||
1142 | mutex_lock(&smmu->stream_map_mutex); | |
1143 | /* Figure out a viable stream map entry allocation */ | |
adfec2e7 | 1144 | for_each_cfg_sme(fwspec, i, idx) { |
021bb842 RM |
1145 | u16 sid = fwspec->ids[i]; |
1146 | u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT; | |
1147 | ||
588888a7 RM |
1148 | if (idx != INVALID_SMENDX) { |
1149 | ret = -EEXIST; | |
1150 | goto out_err; | |
45ae7cff WD |
1151 | } |
1152 | ||
021bb842 | 1153 | ret = arm_smmu_find_sme(smmu, sid, mask); |
588888a7 RM |
1154 | if (ret < 0) |
1155 | goto out_err; | |
1156 | ||
1157 | idx = ret; | |
1158 | if (smrs && smmu->s2crs[idx].count == 0) { | |
021bb842 RM |
1159 | smrs[idx].id = sid; |
1160 | smrs[idx].mask = mask; | |
588888a7 RM |
1161 | smrs[idx].valid = true; |
1162 | } | |
1163 | smmu->s2crs[idx].count++; | |
1164 | cfg->smendx[i] = (s16)idx; | |
45ae7cff WD |
1165 | } |
1166 | ||
588888a7 RM |
1167 | group = iommu_group_get_for_dev(dev); |
1168 | if (!group) | |
1169 | group = ERR_PTR(-ENOMEM); | |
1170 | if (IS_ERR(group)) { | |
1171 | ret = PTR_ERR(group); | |
1172 | goto out_err; | |
1173 | } | |
1174 | iommu_group_put(group); | |
1f3d5ca4 | 1175 | |
45ae7cff | 1176 | /* It worked! Now, poke the actual hardware */ |
adfec2e7 | 1177 | for_each_cfg_sme(fwspec, i, idx) { |
588888a7 RM |
1178 | arm_smmu_write_sme(smmu, idx); |
1179 | smmu->s2crs[idx].group = group; | |
1180 | } | |
45ae7cff | 1181 | |
588888a7 | 1182 | mutex_unlock(&smmu->stream_map_mutex); |
45ae7cff WD |
1183 | return 0; |
1184 | ||
588888a7 | 1185 | out_err: |
1f3d5ca4 | 1186 | while (i--) { |
588888a7 | 1187 | arm_smmu_free_sme(smmu, cfg->smendx[i]); |
1f3d5ca4 RM |
1188 | cfg->smendx[i] = INVALID_SMENDX; |
1189 | } | |
588888a7 RM |
1190 | mutex_unlock(&smmu->stream_map_mutex); |
1191 | return ret; | |
45ae7cff WD |
1192 | } |
1193 | ||
adfec2e7 | 1194 | static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec) |
45ae7cff | 1195 | { |
adfec2e7 RM |
1196 | struct arm_smmu_device *smmu = fwspec_smmu(fwspec); |
1197 | struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv; | |
d3097e39 | 1198 | int i, idx; |
43b412be | 1199 | |
588888a7 | 1200 | mutex_lock(&smmu->stream_map_mutex); |
adfec2e7 | 1201 | for_each_cfg_sme(fwspec, i, idx) { |
588888a7 RM |
1202 | if (arm_smmu_free_sme(smmu, idx)) |
1203 | arm_smmu_write_sme(smmu, idx); | |
1f3d5ca4 | 1204 | cfg->smendx[i] = INVALID_SMENDX; |
45ae7cff | 1205 | } |
588888a7 | 1206 | mutex_unlock(&smmu->stream_map_mutex); |
45ae7cff WD |
1207 | } |
1208 | ||
45ae7cff | 1209 | static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, |
adfec2e7 | 1210 | struct iommu_fwspec *fwspec) |
45ae7cff | 1211 | { |
44680eed | 1212 | struct arm_smmu_device *smmu = smmu_domain->smmu; |
8e8b203e | 1213 | struct arm_smmu_s2cr *s2cr = smmu->s2crs; |
8e8b203e | 1214 | u8 cbndx = smmu_domain->cfg.cbndx; |
61bc6711 | 1215 | enum arm_smmu_s2cr_type type; |
588888a7 | 1216 | int i, idx; |
45ae7cff | 1217 | |
61bc6711 WD |
1218 | if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS) |
1219 | type = S2CR_TYPE_BYPASS; | |
1220 | else | |
1221 | type = S2CR_TYPE_TRANS; | |
1222 | ||
adfec2e7 | 1223 | for_each_cfg_sme(fwspec, i, idx) { |
8e8b203e | 1224 | if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx) |
588888a7 | 1225 | continue; |
1f3d5ca4 | 1226 | |
8e8b203e | 1227 | s2cr[idx].type = type; |
e1989807 | 1228 | s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT; |
8e8b203e RM |
1229 | s2cr[idx].cbndx = cbndx; |
1230 | arm_smmu_write_s2cr(smmu, idx); | |
43b412be | 1231 | } |
8e8b203e | 1232 | return 0; |
bc7f2ce0 WD |
1233 | } |
1234 | ||
45ae7cff WD |
1235 | static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) |
1236 | { | |
a18037b2 | 1237 | int ret; |
9b468f7d | 1238 | struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); |
adfec2e7 | 1239 | struct arm_smmu_device *smmu; |
1d672638 | 1240 | struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); |
45ae7cff | 1241 | |
adfec2e7 | 1242 | if (!fwspec || fwspec->ops != &arm_smmu_ops) { |
45ae7cff WD |
1243 | dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n"); |
1244 | return -ENXIO; | |
1245 | } | |
1246 | ||
fba4f8e5 RM |
1247 | /* |
1248 | * FIXME: The arch/arm DMA API code tries to attach devices to its own | |
1249 | * domains between of_xlate() and add_device() - we have no way to cope | |
1250 | * with that, so until ARM gets converted to rely on groups and default | |
1251 | * domains, just say no (but more politely than by dereferencing NULL). | |
1252 | * This should be at least a WARN_ON once that's sorted. | |
1253 | */ | |
1254 | if (!fwspec->iommu_priv) | |
1255 | return -ENODEV; | |
1256 | ||
adfec2e7 | 1257 | smmu = fwspec_smmu(fwspec); |
d4a44f07 S |
1258 | |
1259 | ret = arm_smmu_rpm_get(smmu); | |
1260 | if (ret < 0) | |
1261 | return ret; | |
1262 | ||
518f7136 | 1263 | /* Ensure that the domain is finalised */ |
adfec2e7 | 1264 | ret = arm_smmu_init_domain_context(domain, smmu); |
287980e4 | 1265 | if (ret < 0) |
d4a44f07 | 1266 | goto rpm_put; |
518f7136 | 1267 | |
45ae7cff | 1268 | /* |
44680eed WD |
1269 | * Sanity check the domain. We don't support domains across |
1270 | * different SMMUs. | |
45ae7cff | 1271 | */ |
adfec2e7 | 1272 | if (smmu_domain->smmu != smmu) { |
45ae7cff WD |
1273 | dev_err(dev, |
1274 | "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n", | |
adfec2e7 | 1275 | dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev)); |
d4a44f07 S |
1276 | ret = -EINVAL; |
1277 | goto rpm_put; | |
45ae7cff | 1278 | } |
45ae7cff WD |
1279 | |
1280 | /* Looks ok, so add the device to the domain */ | |
d4a44f07 S |
1281 | ret = arm_smmu_domain_add_master(smmu_domain, fwspec); |
1282 | ||
1283 | rpm_put: | |
1284 | arm_smmu_rpm_put(smmu); | |
1285 | return ret; | |
45ae7cff WD |
1286 | } |
1287 | ||
45ae7cff | 1288 | static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova, |
b410aed9 | 1289 | phys_addr_t paddr, size_t size, int prot) |
45ae7cff | 1290 | { |
523d7423 | 1291 | struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; |
d4a44f07 S |
1292 | struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; |
1293 | int ret; | |
45ae7cff | 1294 | |
518f7136 | 1295 | if (!ops) |
45ae7cff WD |
1296 | return -ENODEV; |
1297 | ||
d4a44f07 S |
1298 | arm_smmu_rpm_get(smmu); |
1299 | ret = ops->map(ops, iova, paddr, size, prot); | |
1300 | arm_smmu_rpm_put(smmu); | |
1301 | ||
1302 | return ret; | |
45ae7cff WD |
1303 | } |
1304 | ||
1305 | static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, | |
1306 | size_t size) | |
1307 | { | |
523d7423 | 1308 | struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; |
d4a44f07 S |
1309 | struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; |
1310 | size_t ret; | |
45ae7cff | 1311 | |
518f7136 WD |
1312 | if (!ops) |
1313 | return 0; | |
1314 | ||
d4a44f07 S |
1315 | arm_smmu_rpm_get(smmu); |
1316 | ret = ops->unmap(ops, iova, size); | |
1317 | arm_smmu_rpm_put(smmu); | |
1318 | ||
1319 | return ret; | |
45ae7cff WD |
1320 | } |
1321 | ||
44f6876a RM |
1322 | static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain) |
1323 | { | |
1324 | struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); | |
d4a44f07 | 1325 | struct arm_smmu_device *smmu = smmu_domain->smmu; |
44f6876a | 1326 | |
d4a44f07 S |
1327 | if (smmu_domain->tlb_ops) { |
1328 | arm_smmu_rpm_get(smmu); | |
44f6876a | 1329 | smmu_domain->tlb_ops->tlb_flush_all(smmu_domain); |
d4a44f07 S |
1330 | arm_smmu_rpm_put(smmu); |
1331 | } | |
44f6876a RM |
1332 | } |
1333 | ||
32b12449 RM |
1334 | static void arm_smmu_iotlb_sync(struct iommu_domain *domain) |
1335 | { | |
1336 | struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); | |
d4a44f07 | 1337 | struct arm_smmu_device *smmu = smmu_domain->smmu; |
32b12449 | 1338 | |
d4a44f07 S |
1339 | if (smmu_domain->tlb_ops) { |
1340 | arm_smmu_rpm_get(smmu); | |
32b12449 | 1341 | smmu_domain->tlb_ops->tlb_sync(smmu_domain); |
d4a44f07 S |
1342 | arm_smmu_rpm_put(smmu); |
1343 | } | |
32b12449 RM |
1344 | } |
1345 | ||
859a732e MH |
1346 | static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, |
1347 | dma_addr_t iova) | |
1348 | { | |
1d672638 | 1349 | struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); |
859a732e MH |
1350 | struct arm_smmu_device *smmu = smmu_domain->smmu; |
1351 | struct arm_smmu_cfg *cfg = &smmu_domain->cfg; | |
1352 | struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; | |
1353 | struct device *dev = smmu->dev; | |
1354 | void __iomem *cb_base; | |
1355 | u32 tmp; | |
1356 | u64 phys; | |
523d7423 | 1357 | unsigned long va, flags; |
d4a44f07 S |
1358 | int ret; |
1359 | ||
1360 | ret = arm_smmu_rpm_get(smmu); | |
1361 | if (ret < 0) | |
1362 | return 0; | |
859a732e | 1363 | |
452107c7 | 1364 | cb_base = ARM_SMMU_CB(smmu, cfg->cbndx); |
859a732e | 1365 | |
523d7423 | 1366 | spin_lock_irqsave(&smmu_domain->cb_lock, flags); |
661d962f RM |
1367 | /* ATS1 registers can only be written atomically */ |
1368 | va = iova & ~0xfffUL; | |
661d962f | 1369 | if (smmu->version == ARM_SMMU_V2) |
f9a05f05 RM |
1370 | smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR); |
1371 | else /* Register is only 32-bit in v1 */ | |
661d962f | 1372 | writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR); |
859a732e MH |
1373 | |
1374 | if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp, | |
1375 | !(tmp & ATSR_ACTIVE), 5, 50)) { | |
523d7423 | 1376 | spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); |
859a732e | 1377 | dev_err(dev, |
077124c9 | 1378 | "iova to phys timed out on %pad. Falling back to software table walk.\n", |
859a732e MH |
1379 | &iova); |
1380 | return ops->iova_to_phys(ops, iova); | |
1381 | } | |
1382 | ||
f9a05f05 | 1383 | phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR); |
523d7423 | 1384 | spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); |
859a732e MH |
1385 | if (phys & CB_PAR_F) { |
1386 | dev_err(dev, "translation fault!\n"); | |
1387 | dev_err(dev, "PAR = 0x%llx\n", phys); | |
1388 | return 0; | |
1389 | } | |
1390 | ||
d4a44f07 S |
1391 | arm_smmu_rpm_put(smmu); |
1392 | ||
859a732e MH |
1393 | return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff); |
1394 | } | |
1395 | ||
45ae7cff | 1396 | static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, |
859a732e | 1397 | dma_addr_t iova) |
45ae7cff | 1398 | { |
1d672638 | 1399 | struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); |
523d7423 | 1400 | struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; |
45ae7cff | 1401 | |
bdf95923 SG |
1402 | if (domain->type == IOMMU_DOMAIN_IDENTITY) |
1403 | return iova; | |
1404 | ||
518f7136 | 1405 | if (!ops) |
a44a9791 | 1406 | return 0; |
45ae7cff | 1407 | |
83a60ed8 | 1408 | if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS && |
523d7423 RM |
1409 | smmu_domain->stage == ARM_SMMU_DOMAIN_S1) |
1410 | return arm_smmu_iova_to_phys_hard(domain, iova); | |
859a732e | 1411 | |
523d7423 | 1412 | return ops->iova_to_phys(ops, iova); |
45ae7cff WD |
1413 | } |
1414 | ||
1fd0c775 | 1415 | static bool arm_smmu_capable(enum iommu_cap cap) |
45ae7cff | 1416 | { |
d0948945 WD |
1417 | switch (cap) { |
1418 | case IOMMU_CAP_CACHE_COHERENCY: | |
1fd0c775 JR |
1419 | /* |
1420 | * Return true here as the SMMU can always send out coherent | |
1421 | * requests. | |
1422 | */ | |
1423 | return true; | |
0029a8dd AM |
1424 | case IOMMU_CAP_NOEXEC: |
1425 | return true; | |
d0948945 | 1426 | default: |
1fd0c775 | 1427 | return false; |
d0948945 | 1428 | } |
45ae7cff | 1429 | } |
45ae7cff | 1430 | |
021bb842 RM |
1431 | static int arm_smmu_match_node(struct device *dev, void *data) |
1432 | { | |
ce9babe5 | 1433 | return dev->fwnode == data; |
021bb842 RM |
1434 | } |
1435 | ||
ce9babe5 LP |
1436 | static |
1437 | struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode) | |
021bb842 RM |
1438 | { |
1439 | struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL, | |
ce9babe5 | 1440 | fwnode, arm_smmu_match_node); |
021bb842 RM |
1441 | put_device(dev); |
1442 | return dev ? dev_get_drvdata(dev) : NULL; | |
1443 | } | |
1444 | ||
f80cd885 | 1445 | static int arm_smmu_add_device(struct device *dev) |
45ae7cff | 1446 | { |
adfec2e7 | 1447 | struct arm_smmu_device *smmu; |
03edb226 | 1448 | struct arm_smmu_master_cfg *cfg; |
9b468f7d | 1449 | struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); |
f80cd885 | 1450 | int i, ret; |
8f68f8e2 | 1451 | |
021bb842 RM |
1452 | if (using_legacy_binding) { |
1453 | ret = arm_smmu_register_legacy_master(dev, &smmu); | |
a7990c64 AS |
1454 | |
1455 | /* | |
1456 | * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master() | |
1457 | * will allocate/initialise a new one. Thus we need to update fwspec for | |
1458 | * later use. | |
1459 | */ | |
9b468f7d | 1460 | fwspec = dev_iommu_fwspec_get(dev); |
021bb842 RM |
1461 | if (ret) |
1462 | goto out_free; | |
3c117b54 | 1463 | } else if (fwspec && fwspec->ops == &arm_smmu_ops) { |
ce9babe5 | 1464 | smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode); |
021bb842 RM |
1465 | } else { |
1466 | return -ENODEV; | |
1467 | } | |
a9a1b0b5 | 1468 | |
f80cd885 | 1469 | ret = -EINVAL; |
adfec2e7 RM |
1470 | for (i = 0; i < fwspec->num_ids; i++) { |
1471 | u16 sid = fwspec->ids[i]; | |
021bb842 | 1472 | u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT; |
03edb226 | 1473 | |
adfec2e7 | 1474 | if (sid & ~smmu->streamid_mask) { |
f80cd885 | 1475 | dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n", |
021bb842 RM |
1476 | sid, smmu->streamid_mask); |
1477 | goto out_free; | |
1478 | } | |
1479 | if (mask & ~smmu->smr_mask_mask) { | |
1480 | dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n", | |
6323f474 | 1481 | mask, smmu->smr_mask_mask); |
f80cd885 RM |
1482 | goto out_free; |
1483 | } | |
1f3d5ca4 | 1484 | } |
5fc63a7c | 1485 | |
adfec2e7 RM |
1486 | ret = -ENOMEM; |
1487 | cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]), | |
1488 | GFP_KERNEL); | |
1489 | if (!cfg) | |
1490 | goto out_free; | |
1491 | ||
1492 | cfg->smmu = smmu; | |
1493 | fwspec->iommu_priv = cfg; | |
1494 | while (i--) | |
1495 | cfg->smendx[i] = INVALID_SMENDX; | |
1496 | ||
d4a44f07 S |
1497 | ret = arm_smmu_rpm_get(smmu); |
1498 | if (ret < 0) | |
1499 | goto out_cfg_free; | |
1500 | ||
588888a7 | 1501 | ret = arm_smmu_master_alloc_smes(dev); |
d4a44f07 S |
1502 | arm_smmu_rpm_put(smmu); |
1503 | ||
adfec2e7 | 1504 | if (ret) |
c54451a5 | 1505 | goto out_cfg_free; |
adfec2e7 | 1506 | |
9648cbc9 JR |
1507 | iommu_device_link(&smmu->iommu, dev); |
1508 | ||
655e3643 S |
1509 | device_link_add(dev, smmu->dev, |
1510 | DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER); | |
1511 | ||
adfec2e7 | 1512 | return 0; |
f80cd885 | 1513 | |
c54451a5 VG |
1514 | out_cfg_free: |
1515 | kfree(cfg); | |
f80cd885 | 1516 | out_free: |
adfec2e7 | 1517 | iommu_fwspec_free(dev); |
f80cd885 | 1518 | return ret; |
03edb226 WD |
1519 | } |
1520 | ||
45ae7cff WD |
1521 | static void arm_smmu_remove_device(struct device *dev) |
1522 | { | |
9b468f7d | 1523 | struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); |
9648cbc9 JR |
1524 | struct arm_smmu_master_cfg *cfg; |
1525 | struct arm_smmu_device *smmu; | |
d4a44f07 | 1526 | int ret; |
8e8b203e | 1527 | |
adfec2e7 | 1528 | if (!fwspec || fwspec->ops != &arm_smmu_ops) |
f80cd885 | 1529 | return; |
8e8b203e | 1530 | |
9648cbc9 JR |
1531 | cfg = fwspec->iommu_priv; |
1532 | smmu = cfg->smmu; | |
1533 | ||
d4a44f07 S |
1534 | ret = arm_smmu_rpm_get(smmu); |
1535 | if (ret < 0) | |
1536 | return; | |
1537 | ||
9648cbc9 | 1538 | iommu_device_unlink(&smmu->iommu, dev); |
adfec2e7 | 1539 | arm_smmu_master_free_smes(fwspec); |
d4a44f07 S |
1540 | |
1541 | arm_smmu_rpm_put(smmu); | |
1542 | ||
5fc63a7c | 1543 | iommu_group_remove_device(dev); |
adfec2e7 RM |
1544 | kfree(fwspec->iommu_priv); |
1545 | iommu_fwspec_free(dev); | |
45ae7cff WD |
1546 | } |
1547 | ||
af659932 JR |
1548 | static struct iommu_group *arm_smmu_device_group(struct device *dev) |
1549 | { | |
9b468f7d | 1550 | struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); |
adfec2e7 | 1551 | struct arm_smmu_device *smmu = fwspec_smmu(fwspec); |
588888a7 RM |
1552 | struct iommu_group *group = NULL; |
1553 | int i, idx; | |
1554 | ||
adfec2e7 | 1555 | for_each_cfg_sme(fwspec, i, idx) { |
588888a7 RM |
1556 | if (group && smmu->s2crs[idx].group && |
1557 | group != smmu->s2crs[idx].group) | |
1558 | return ERR_PTR(-EINVAL); | |
1559 | ||
1560 | group = smmu->s2crs[idx].group; | |
1561 | } | |
1562 | ||
1563 | if (group) | |
e1b44cbe | 1564 | return iommu_group_ref_get(group); |
af659932 JR |
1565 | |
1566 | if (dev_is_pci(dev)) | |
1567 | group = pci_device_group(dev); | |
eab03e2a NG |
1568 | else if (dev_is_fsl_mc(dev)) |
1569 | group = fsl_mc_device_group(dev); | |
af659932 JR |
1570 | else |
1571 | group = generic_device_group(dev); | |
1572 | ||
af659932 JR |
1573 | return group; |
1574 | } | |
1575 | ||
c752ce45 WD |
1576 | static int arm_smmu_domain_get_attr(struct iommu_domain *domain, |
1577 | enum iommu_attr attr, void *data) | |
1578 | { | |
1d672638 | 1579 | struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); |
c752ce45 | 1580 | |
44f6876a RM |
1581 | switch(domain->type) { |
1582 | case IOMMU_DOMAIN_UNMANAGED: | |
1583 | switch (attr) { | |
1584 | case DOMAIN_ATTR_NESTING: | |
1585 | *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED); | |
1586 | return 0; | |
1587 | default: | |
1588 | return -ENODEV; | |
1589 | } | |
1590 | break; | |
1591 | case IOMMU_DOMAIN_DMA: | |
1592 | switch (attr) { | |
1593 | case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE: | |
1594 | *(int *)data = smmu_domain->non_strict; | |
1595 | return 0; | |
1596 | default: | |
1597 | return -ENODEV; | |
1598 | } | |
1599 | break; | |
c752ce45 | 1600 | default: |
44f6876a | 1601 | return -EINVAL; |
c752ce45 WD |
1602 | } |
1603 | } | |
1604 | ||
1605 | static int arm_smmu_domain_set_attr(struct iommu_domain *domain, | |
1606 | enum iommu_attr attr, void *data) | |
1607 | { | |
518f7136 | 1608 | int ret = 0; |
1d672638 | 1609 | struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); |
c752ce45 | 1610 | |
518f7136 WD |
1611 | mutex_lock(&smmu_domain->init_mutex); |
1612 | ||
44f6876a RM |
1613 | switch(domain->type) { |
1614 | case IOMMU_DOMAIN_UNMANAGED: | |
1615 | switch (attr) { | |
1616 | case DOMAIN_ATTR_NESTING: | |
1617 | if (smmu_domain->smmu) { | |
1618 | ret = -EPERM; | |
1619 | goto out_unlock; | |
1620 | } | |
1621 | ||
1622 | if (*(int *)data) | |
1623 | smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED; | |
1624 | else | |
1625 | smmu_domain->stage = ARM_SMMU_DOMAIN_S1; | |
1626 | break; | |
1627 | default: | |
1628 | ret = -ENODEV; | |
1629 | } | |
1630 | break; | |
1631 | case IOMMU_DOMAIN_DMA: | |
1632 | switch (attr) { | |
1633 | case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE: | |
1634 | smmu_domain->non_strict = *(int *)data; | |
1635 | break; | |
1636 | default: | |
1637 | ret = -ENODEV; | |
518f7136 | 1638 | } |
518f7136 | 1639 | break; |
c752ce45 | 1640 | default: |
44f6876a | 1641 | ret = -EINVAL; |
c752ce45 | 1642 | } |
518f7136 WD |
1643 | out_unlock: |
1644 | mutex_unlock(&smmu_domain->init_mutex); | |
1645 | return ret; | |
c752ce45 WD |
1646 | } |
1647 | ||
021bb842 RM |
1648 | static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args) |
1649 | { | |
56fbf600 | 1650 | u32 mask, fwid = 0; |
021bb842 RM |
1651 | |
1652 | if (args->args_count > 0) | |
1653 | fwid |= (u16)args->args[0]; | |
1654 | ||
1655 | if (args->args_count > 1) | |
1656 | fwid |= (u16)args->args[1] << SMR_MASK_SHIFT; | |
56fbf600 RM |
1657 | else if (!of_property_read_u32(args->np, "stream-match-mask", &mask)) |
1658 | fwid |= (u16)mask << SMR_MASK_SHIFT; | |
021bb842 RM |
1659 | |
1660 | return iommu_fwspec_add_ids(dev, &fwid, 1); | |
1661 | } | |
1662 | ||
f3ebee80 EA |
1663 | static void arm_smmu_get_resv_regions(struct device *dev, |
1664 | struct list_head *head) | |
1665 | { | |
1666 | struct iommu_resv_region *region; | |
1667 | int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; | |
1668 | ||
1669 | region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH, | |
9d3a4de4 | 1670 | prot, IOMMU_RESV_SW_MSI); |
f3ebee80 EA |
1671 | if (!region) |
1672 | return; | |
1673 | ||
1674 | list_add_tail(®ion->list, head); | |
273df963 RM |
1675 | |
1676 | iommu_dma_get_resv_regions(dev, head); | |
f3ebee80 EA |
1677 | } |
1678 | ||
1679 | static void arm_smmu_put_resv_regions(struct device *dev, | |
1680 | struct list_head *head) | |
1681 | { | |
1682 | struct iommu_resv_region *entry, *next; | |
1683 | ||
1684 | list_for_each_entry_safe(entry, next, head, list) | |
1685 | kfree(entry); | |
1686 | } | |
1687 | ||
518f7136 | 1688 | static struct iommu_ops arm_smmu_ops = { |
c752ce45 | 1689 | .capable = arm_smmu_capable, |
1d672638 JR |
1690 | .domain_alloc = arm_smmu_domain_alloc, |
1691 | .domain_free = arm_smmu_domain_free, | |
c752ce45 | 1692 | .attach_dev = arm_smmu_attach_dev, |
c752ce45 WD |
1693 | .map = arm_smmu_map, |
1694 | .unmap = arm_smmu_unmap, | |
44f6876a | 1695 | .flush_iotlb_all = arm_smmu_flush_iotlb_all, |
32b12449 | 1696 | .iotlb_sync = arm_smmu_iotlb_sync, |
c752ce45 WD |
1697 | .iova_to_phys = arm_smmu_iova_to_phys, |
1698 | .add_device = arm_smmu_add_device, | |
1699 | .remove_device = arm_smmu_remove_device, | |
af659932 | 1700 | .device_group = arm_smmu_device_group, |
c752ce45 WD |
1701 | .domain_get_attr = arm_smmu_domain_get_attr, |
1702 | .domain_set_attr = arm_smmu_domain_set_attr, | |
021bb842 | 1703 | .of_xlate = arm_smmu_of_xlate, |
f3ebee80 EA |
1704 | .get_resv_regions = arm_smmu_get_resv_regions, |
1705 | .put_resv_regions = arm_smmu_put_resv_regions, | |
518f7136 | 1706 | .pgsize_bitmap = -1UL, /* Restricted during device attach */ |
45ae7cff WD |
1707 | }; |
1708 | ||
1709 | static void arm_smmu_device_reset(struct arm_smmu_device *smmu) | |
1710 | { | |
1711 | void __iomem *gr0_base = ARM_SMMU_GR0(smmu); | |
1f3d5ca4 | 1712 | int i; |
3ca3712a | 1713 | u32 reg, major; |
659db6f6 | 1714 | |
3a5df8ff AH |
1715 | /* clear global FSR */ |
1716 | reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR); | |
1717 | writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR); | |
45ae7cff | 1718 | |
1f3d5ca4 RM |
1719 | /* |
1720 | * Reset stream mapping groups: Initial values mark all SMRn as | |
1721 | * invalid and all S2CRn as bypass unless overridden. | |
1722 | */ | |
8e8b203e RM |
1723 | for (i = 0; i < smmu->num_mapping_groups; ++i) |
1724 | arm_smmu_write_sme(smmu, i); | |
45ae7cff | 1725 | |
6eb18d4a NG |
1726 | if (smmu->model == ARM_MMU500) { |
1727 | /* | |
1728 | * Before clearing ARM_MMU500_ACTLR_CPRE, need to | |
1729 | * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK | |
1730 | * bit is only present in MMU-500r2 onwards. | |
1731 | */ | |
1732 | reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7); | |
1733 | major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK; | |
3ca3712a | 1734 | reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR); |
6eb18d4a NG |
1735 | if (major >= 2) |
1736 | reg &= ~ARM_MMU500_ACR_CACHE_LOCK; | |
1737 | /* | |
1738 | * Allow unmatched Stream IDs to allocate bypass | |
1739 | * TLB entries for reduced latency. | |
1740 | */ | |
74f55d34 | 1741 | reg |= ARM_MMU500_ACR_SMTNMB_TLBEN | ARM_MMU500_ACR_S2CRB_TLBEN; |
3ca3712a PF |
1742 | writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR); |
1743 | } | |
1744 | ||
659db6f6 AH |
1745 | /* Make sure all context banks are disabled and clear CB_FSR */ |
1746 | for (i = 0; i < smmu->num_context_banks; ++i) { | |
90df373c RM |
1747 | void __iomem *cb_base = ARM_SMMU_CB(smmu, i); |
1748 | ||
1749 | arm_smmu_write_context_bank(smmu, i); | |
659db6f6 | 1750 | writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR); |
f0cfffc4 RM |
1751 | /* |
1752 | * Disable MMU-500's not-particularly-beneficial next-page | |
1753 | * prefetcher for the sake of errata #841119 and #826419. | |
1754 | */ | |
1755 | if (smmu->model == ARM_MMU500) { | |
1756 | reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR); | |
1757 | reg &= ~ARM_MMU500_ACTLR_CPRE; | |
1758 | writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR); | |
1759 | } | |
659db6f6 | 1760 | } |
1463fe44 | 1761 | |
45ae7cff | 1762 | /* Invalidate the TLB, just in case */ |
4e4abae3 RM |
1763 | writel_relaxed(QCOM_DUMMY_VAL, gr0_base + ARM_SMMU_GR0_TLBIALLH); |
1764 | writel_relaxed(QCOM_DUMMY_VAL, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH); | |
45ae7cff | 1765 | |
3a5df8ff | 1766 | reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); |
659db6f6 | 1767 | |
45ae7cff | 1768 | /* Enable fault reporting */ |
659db6f6 | 1769 | reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE); |
45ae7cff WD |
1770 | |
1771 | /* Disable TLB broadcasting. */ | |
659db6f6 | 1772 | reg |= (sCR0_VMIDPNE | sCR0_PTM); |
45ae7cff | 1773 | |
25a1c96c RM |
1774 | /* Enable client access, handling unmatched streams as appropriate */ |
1775 | reg &= ~sCR0_CLIENTPD; | |
1776 | if (disable_bypass) | |
1777 | reg |= sCR0_USFCFG; | |
1778 | else | |
1779 | reg &= ~sCR0_USFCFG; | |
45ae7cff WD |
1780 | |
1781 | /* Disable forced broadcasting */ | |
659db6f6 | 1782 | reg &= ~sCR0_FB; |
45ae7cff WD |
1783 | |
1784 | /* Don't upgrade barriers */ | |
659db6f6 | 1785 | reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT); |
45ae7cff | 1786 | |
4e3e9b69 TC |
1787 | if (smmu->features & ARM_SMMU_FEAT_VMID16) |
1788 | reg |= sCR0_VMID16EN; | |
1789 | ||
dc0eaa4e AM |
1790 | if (smmu->features & ARM_SMMU_FEAT_EXIDS) |
1791 | reg |= sCR0_EXIDENABLE; | |
1792 | ||
45ae7cff | 1793 | /* Push the button */ |
11febfca | 1794 | arm_smmu_tlb_sync_global(smmu); |
3a5df8ff | 1795 | writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); |
45ae7cff WD |
1796 | } |
1797 | ||
1798 | static int arm_smmu_id_size_to_bits(int size) | |
1799 | { | |
1800 | switch (size) { | |
1801 | case 0: | |
1802 | return 32; | |
1803 | case 1: | |
1804 | return 36; | |
1805 | case 2: | |
1806 | return 40; | |
1807 | case 3: | |
1808 | return 42; | |
1809 | case 4: | |
1810 | return 44; | |
1811 | case 5: | |
1812 | default: | |
1813 | return 48; | |
1814 | } | |
1815 | } | |
1816 | ||
1817 | static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) | |
1818 | { | |
1819 | unsigned long size; | |
1820 | void __iomem *gr0_base = ARM_SMMU_GR0(smmu); | |
1821 | u32 id; | |
bbb8a184 | 1822 | bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK; |
8e8b203e | 1823 | int i; |
45ae7cff WD |
1824 | |
1825 | dev_notice(smmu->dev, "probing hardware configuration...\n"); | |
b7862e35 RM |
1826 | dev_notice(smmu->dev, "SMMUv%d with:\n", |
1827 | smmu->version == ARM_SMMU_V2 ? 2 : 1); | |
45ae7cff WD |
1828 | |
1829 | /* ID0 */ | |
1830 | id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0); | |
4cf740b0 WD |
1831 | |
1832 | /* Restrict available stages based on module parameter */ | |
1833 | if (force_stage == 1) | |
1834 | id &= ~(ID0_S2TS | ID0_NTS); | |
1835 | else if (force_stage == 2) | |
1836 | id &= ~(ID0_S1TS | ID0_NTS); | |
1837 | ||
45ae7cff WD |
1838 | if (id & ID0_S1TS) { |
1839 | smmu->features |= ARM_SMMU_FEAT_TRANS_S1; | |
1840 | dev_notice(smmu->dev, "\tstage 1 translation\n"); | |
1841 | } | |
1842 | ||
1843 | if (id & ID0_S2TS) { | |
1844 | smmu->features |= ARM_SMMU_FEAT_TRANS_S2; | |
1845 | dev_notice(smmu->dev, "\tstage 2 translation\n"); | |
1846 | } | |
1847 | ||
1848 | if (id & ID0_NTS) { | |
1849 | smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED; | |
1850 | dev_notice(smmu->dev, "\tnested translation\n"); | |
1851 | } | |
1852 | ||
1853 | if (!(smmu->features & | |
4cf740b0 | 1854 | (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) { |
45ae7cff WD |
1855 | dev_err(smmu->dev, "\tno translation support!\n"); |
1856 | return -ENODEV; | |
1857 | } | |
1858 | ||
b7862e35 RM |
1859 | if ((id & ID0_S1TS) && |
1860 | ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) { | |
859a732e MH |
1861 | smmu->features |= ARM_SMMU_FEAT_TRANS_OPS; |
1862 | dev_notice(smmu->dev, "\taddress translation ops\n"); | |
1863 | } | |
1864 | ||
bae2c2d4 RM |
1865 | /* |
1866 | * In order for DMA API calls to work properly, we must defer to what | |
bbb8a184 | 1867 | * the FW says about coherency, regardless of what the hardware claims. |
bae2c2d4 RM |
1868 | * Fortunately, this also opens up a workaround for systems where the |
1869 | * ID register value has ended up configured incorrectly. | |
1870 | */ | |
bae2c2d4 | 1871 | cttw_reg = !!(id & ID0_CTTW); |
bbb8a184 | 1872 | if (cttw_fw || cttw_reg) |
bae2c2d4 | 1873 | dev_notice(smmu->dev, "\t%scoherent table walk\n", |
bbb8a184 LP |
1874 | cttw_fw ? "" : "non-"); |
1875 | if (cttw_fw != cttw_reg) | |
bae2c2d4 | 1876 | dev_notice(smmu->dev, |
bbb8a184 | 1877 | "\t(IDR0.CTTW overridden by FW configuration)\n"); |
45ae7cff | 1878 | |
21174240 | 1879 | /* Max. number of entries we have for stream matching/indexing */ |
dc0eaa4e AM |
1880 | if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) { |
1881 | smmu->features |= ARM_SMMU_FEAT_EXIDS; | |
1882 | size = 1 << 16; | |
1883 | } else { | |
1884 | size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK); | |
1885 | } | |
21174240 | 1886 | smmu->streamid_mask = size - 1; |
45ae7cff | 1887 | if (id & ID0_SMS) { |
45ae7cff | 1888 | smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH; |
21174240 RM |
1889 | size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK; |
1890 | if (size == 0) { | |
45ae7cff WD |
1891 | dev_err(smmu->dev, |
1892 | "stream-matching supported, but no SMRs present!\n"); | |
1893 | return -ENODEV; | |
1894 | } | |
1895 | ||
1f3d5ca4 RM |
1896 | /* Zero-initialised to mark as invalid */ |
1897 | smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs), | |
1898 | GFP_KERNEL); | |
1899 | if (!smmu->smrs) | |
1900 | return -ENOMEM; | |
1901 | ||
45ae7cff | 1902 | dev_notice(smmu->dev, |
dc0eaa4e | 1903 | "\tstream matching with %lu register groups", size); |
45ae7cff | 1904 | } |
8e8b203e RM |
1905 | /* s2cr->type == 0 means translation, so initialise explicitly */ |
1906 | smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs), | |
1907 | GFP_KERNEL); | |
1908 | if (!smmu->s2crs) | |
1909 | return -ENOMEM; | |
1910 | for (i = 0; i < size; i++) | |
1911 | smmu->s2crs[i] = s2cr_init_val; | |
1912 | ||
21174240 | 1913 | smmu->num_mapping_groups = size; |
588888a7 | 1914 | mutex_init(&smmu->stream_map_mutex); |
8e517e76 | 1915 | spin_lock_init(&smmu->global_sync_lock); |
45ae7cff | 1916 | |
7602b871 RM |
1917 | if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) { |
1918 | smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L; | |
1919 | if (!(id & ID0_PTFS_NO_AARCH32S)) | |
1920 | smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S; | |
1921 | } | |
1922 | ||
45ae7cff WD |
1923 | /* ID1 */ |
1924 | id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1); | |
c757e852 | 1925 | smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12; |
45ae7cff | 1926 | |
c55af7f7 | 1927 | /* Check for size mismatch of SMMU address space from mapped region */ |
518f7136 | 1928 | size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1); |
452107c7 RM |
1929 | size <<= smmu->pgshift; |
1930 | if (smmu->cb_base != gr0_base + size) | |
2907320d | 1931 | dev_warn(smmu->dev, |
452107c7 RM |
1932 | "SMMU address space size (0x%lx) differs from mapped region size (0x%tx)!\n", |
1933 | size * 2, (smmu->cb_base - gr0_base) * 2); | |
45ae7cff | 1934 | |
518f7136 | 1935 | smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK; |
45ae7cff WD |
1936 | smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK; |
1937 | if (smmu->num_s2_context_banks > smmu->num_context_banks) { | |
1938 | dev_err(smmu->dev, "impossible number of S2 context banks!\n"); | |
1939 | return -ENODEV; | |
1940 | } | |
1941 | dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n", | |
1942 | smmu->num_context_banks, smmu->num_s2_context_banks); | |
e086d912 RM |
1943 | /* |
1944 | * Cavium CN88xx erratum #27704. | |
1945 | * Ensure ASID and VMID allocation is unique across all SMMUs in | |
1946 | * the system. | |
1947 | */ | |
1948 | if (smmu->model == CAVIUM_SMMUV2) { | |
1949 | smmu->cavium_id_base = | |
1950 | atomic_add_return(smmu->num_context_banks, | |
1951 | &cavium_smmu_context_count); | |
1952 | smmu->cavium_id_base -= smmu->num_context_banks; | |
53c35dce | 1953 | dev_notice(smmu->dev, "\tenabling workaround for Cavium erratum 27704\n"); |
e086d912 | 1954 | } |
90df373c RM |
1955 | smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks, |
1956 | sizeof(*smmu->cbs), GFP_KERNEL); | |
1957 | if (!smmu->cbs) | |
1958 | return -ENOMEM; | |
45ae7cff WD |
1959 | |
1960 | /* ID2 */ | |
1961 | id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2); | |
1962 | size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK); | |
518f7136 | 1963 | smmu->ipa_size = size; |
45ae7cff | 1964 | |
518f7136 | 1965 | /* The output mask is also applied for bypass */ |
45ae7cff | 1966 | size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK); |
518f7136 | 1967 | smmu->pa_size = size; |
45ae7cff | 1968 | |
4e3e9b69 TC |
1969 | if (id & ID2_VMID16) |
1970 | smmu->features |= ARM_SMMU_FEAT_VMID16; | |
1971 | ||
f1d84548 RM |
1972 | /* |
1973 | * What the page table walker can address actually depends on which | |
1974 | * descriptor format is in use, but since a) we don't know that yet, | |
1975 | * and b) it can vary per context bank, this will have to do... | |
1976 | */ | |
1977 | if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size))) | |
1978 | dev_warn(smmu->dev, | |
1979 | "failed to set DMA mask for table walker\n"); | |
1980 | ||
b7862e35 | 1981 | if (smmu->version < ARM_SMMU_V2) { |
518f7136 | 1982 | smmu->va_size = smmu->ipa_size; |
b7862e35 RM |
1983 | if (smmu->version == ARM_SMMU_V1_64K) |
1984 | smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; | |
45ae7cff | 1985 | } else { |
45ae7cff | 1986 | size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK; |
518f7136 | 1987 | smmu->va_size = arm_smmu_id_size_to_bits(size); |
518f7136 | 1988 | if (id & ID2_PTFS_4K) |
7602b871 | 1989 | smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K; |
518f7136 | 1990 | if (id & ID2_PTFS_16K) |
7602b871 | 1991 | smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K; |
518f7136 | 1992 | if (id & ID2_PTFS_64K) |
7602b871 | 1993 | smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; |
45ae7cff WD |
1994 | } |
1995 | ||
7602b871 | 1996 | /* Now we've corralled the various formats, what'll it do? */ |
7602b871 | 1997 | if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) |
d5466357 | 1998 | smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M; |
7602b871 RM |
1999 | if (smmu->features & |
2000 | (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K)) | |
d5466357 | 2001 | smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G; |
7602b871 | 2002 | if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K) |
d5466357 | 2003 | smmu->pgsize_bitmap |= SZ_16K | SZ_32M; |
7602b871 | 2004 | if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K) |
d5466357 RM |
2005 | smmu->pgsize_bitmap |= SZ_64K | SZ_512M; |
2006 | ||
2007 | if (arm_smmu_ops.pgsize_bitmap == -1UL) | |
2008 | arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap; | |
2009 | else | |
2010 | arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; | |
2011 | dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", | |
2012 | smmu->pgsize_bitmap); | |
7602b871 | 2013 | |
518f7136 | 2014 | |
28d6007b WD |
2015 | if (smmu->features & ARM_SMMU_FEAT_TRANS_S1) |
2016 | dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n", | |
518f7136 | 2017 | smmu->va_size, smmu->ipa_size); |
28d6007b WD |
2018 | |
2019 | if (smmu->features & ARM_SMMU_FEAT_TRANS_S2) | |
2020 | dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n", | |
518f7136 | 2021 | smmu->ipa_size, smmu->pa_size); |
28d6007b | 2022 | |
45ae7cff WD |
2023 | return 0; |
2024 | } | |
2025 | ||
67b65a3f RM |
2026 | struct arm_smmu_match_data { |
2027 | enum arm_smmu_arch_version version; | |
2028 | enum arm_smmu_implementation model; | |
2029 | }; | |
2030 | ||
2031 | #define ARM_SMMU_MATCH_DATA(name, ver, imp) \ | |
96a299d2 | 2032 | static const struct arm_smmu_match_data name = { .version = ver, .model = imp } |
67b65a3f RM |
2033 | |
2034 | ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU); | |
2035 | ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU); | |
b7862e35 | 2036 | ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU); |
f0cfffc4 | 2037 | ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500); |
e086d912 | 2038 | ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2); |
89cddc56 | 2039 | ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2); |
67b65a3f | 2040 | |
09b5269a | 2041 | static const struct of_device_id arm_smmu_of_match[] = { |
67b65a3f RM |
2042 | { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 }, |
2043 | { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 }, | |
2044 | { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 }, | |
b7862e35 | 2045 | { .compatible = "arm,mmu-401", .data = &arm_mmu401 }, |
f0cfffc4 | 2046 | { .compatible = "arm,mmu-500", .data = &arm_mmu500 }, |
e086d912 | 2047 | { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 }, |
89cddc56 | 2048 | { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 }, |
09360403 RM |
2049 | { }, |
2050 | }; | |
09360403 | 2051 | |
d6fcd3b1 LP |
2052 | #ifdef CONFIG_ACPI |
2053 | static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu) | |
2054 | { | |
2055 | int ret = 0; | |
2056 | ||
2057 | switch (model) { | |
2058 | case ACPI_IORT_SMMU_V1: | |
2059 | case ACPI_IORT_SMMU_CORELINK_MMU400: | |
2060 | smmu->version = ARM_SMMU_V1; | |
2061 | smmu->model = GENERIC_SMMU; | |
2062 | break; | |
84c24379 RM |
2063 | case ACPI_IORT_SMMU_CORELINK_MMU401: |
2064 | smmu->version = ARM_SMMU_V1_64K; | |
2065 | smmu->model = GENERIC_SMMU; | |
2066 | break; | |
d6fcd3b1 LP |
2067 | case ACPI_IORT_SMMU_V2: |
2068 | smmu->version = ARM_SMMU_V2; | |
2069 | smmu->model = GENERIC_SMMU; | |
2070 | break; | |
2071 | case ACPI_IORT_SMMU_CORELINK_MMU500: | |
2072 | smmu->version = ARM_SMMU_V2; | |
2073 | smmu->model = ARM_MMU500; | |
2074 | break; | |
84c24379 RM |
2075 | case ACPI_IORT_SMMU_CAVIUM_THUNDERX: |
2076 | smmu->version = ARM_SMMU_V2; | |
2077 | smmu->model = CAVIUM_SMMUV2; | |
2078 | break; | |
d6fcd3b1 LP |
2079 | default: |
2080 | ret = -ENODEV; | |
2081 | } | |
2082 | ||
2083 | return ret; | |
2084 | } | |
2085 | ||
2086 | static int arm_smmu_device_acpi_probe(struct platform_device *pdev, | |
2087 | struct arm_smmu_device *smmu) | |
2088 | { | |
2089 | struct device *dev = smmu->dev; | |
2090 | struct acpi_iort_node *node = | |
2091 | *(struct acpi_iort_node **)dev_get_platdata(dev); | |
2092 | struct acpi_iort_smmu *iort_smmu; | |
2093 | int ret; | |
2094 | ||
2095 | /* Retrieve SMMU1/2 specific data */ | |
2096 | iort_smmu = (struct acpi_iort_smmu *)node->node_data; | |
2097 | ||
2098 | ret = acpi_smmu_get_data(iort_smmu->model, smmu); | |
2099 | if (ret < 0) | |
2100 | return ret; | |
2101 | ||
2102 | /* Ignore the configuration access interrupt */ | |
2103 | smmu->num_global_irqs = 1; | |
2104 | ||
2105 | if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK) | |
2106 | smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; | |
2107 | ||
2108 | return 0; | |
2109 | } | |
2110 | #else | |
2111 | static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev, | |
2112 | struct arm_smmu_device *smmu) | |
2113 | { | |
2114 | return -ENODEV; | |
2115 | } | |
2116 | #endif | |
2117 | ||
bbb8a184 LP |
2118 | static int arm_smmu_device_dt_probe(struct platform_device *pdev, |
2119 | struct arm_smmu_device *smmu) | |
45ae7cff | 2120 | { |
67b65a3f | 2121 | const struct arm_smmu_match_data *data; |
45ae7cff | 2122 | struct device *dev = &pdev->dev; |
021bb842 RM |
2123 | bool legacy_binding; |
2124 | ||
bbb8a184 LP |
2125 | if (of_property_read_u32(dev->of_node, "#global-interrupts", |
2126 | &smmu->num_global_irqs)) { | |
2127 | dev_err(dev, "missing #global-interrupts property\n"); | |
2128 | return -ENODEV; | |
2129 | } | |
2130 | ||
2131 | data = of_device_get_match_data(dev); | |
2132 | smmu->version = data->version; | |
2133 | smmu->model = data->model; | |
2134 | ||
2135 | parse_driver_options(smmu); | |
2136 | ||
021bb842 RM |
2137 | legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL); |
2138 | if (legacy_binding && !using_generic_binding) { | |
2139 | if (!using_legacy_binding) | |
2140 | pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n"); | |
2141 | using_legacy_binding = true; | |
2142 | } else if (!legacy_binding && !using_legacy_binding) { | |
2143 | using_generic_binding = true; | |
2144 | } else { | |
2145 | dev_err(dev, "not probing due to mismatched DT properties\n"); | |
2146 | return -ENODEV; | |
2147 | } | |
45ae7cff | 2148 | |
bbb8a184 LP |
2149 | if (of_dma_is_coherent(dev->of_node)) |
2150 | smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; | |
2151 | ||
2152 | return 0; | |
2153 | } | |
2154 | ||
f6810c15 RM |
2155 | static void arm_smmu_bus_init(void) |
2156 | { | |
2157 | /* Oh, for a proper bus abstraction */ | |
2158 | if (!iommu_present(&platform_bus_type)) | |
2159 | bus_set_iommu(&platform_bus_type, &arm_smmu_ops); | |
2160 | #ifdef CONFIG_ARM_AMBA | |
2161 | if (!iommu_present(&amba_bustype)) | |
2162 | bus_set_iommu(&amba_bustype, &arm_smmu_ops); | |
2163 | #endif | |
2164 | #ifdef CONFIG_PCI | |
2165 | if (!iommu_present(&pci_bus_type)) { | |
2166 | pci_request_acs(); | |
2167 | bus_set_iommu(&pci_bus_type, &arm_smmu_ops); | |
2168 | } | |
2169 | #endif | |
eab03e2a NG |
2170 | #ifdef CONFIG_FSL_MC_BUS |
2171 | if (!iommu_present(&fsl_mc_bus_type)) | |
2172 | bus_set_iommu(&fsl_mc_bus_type, &arm_smmu_ops); | |
2173 | #endif | |
f6810c15 RM |
2174 | } |
2175 | ||
bbb8a184 LP |
2176 | static int arm_smmu_device_probe(struct platform_device *pdev) |
2177 | { | |
2178 | struct resource *res; | |
9648cbc9 | 2179 | resource_size_t ioaddr; |
bbb8a184 LP |
2180 | struct arm_smmu_device *smmu; |
2181 | struct device *dev = &pdev->dev; | |
2182 | int num_irqs, i, err; | |
2183 | ||
45ae7cff WD |
2184 | smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); |
2185 | if (!smmu) { | |
2186 | dev_err(dev, "failed to allocate arm_smmu_device\n"); | |
2187 | return -ENOMEM; | |
2188 | } | |
2189 | smmu->dev = dev; | |
2190 | ||
d6fcd3b1 LP |
2191 | if (dev->of_node) |
2192 | err = arm_smmu_device_dt_probe(pdev, smmu); | |
2193 | else | |
2194 | err = arm_smmu_device_acpi_probe(pdev, smmu); | |
2195 | ||
bbb8a184 LP |
2196 | if (err) |
2197 | return err; | |
09360403 | 2198 | |
45ae7cff | 2199 | res = platform_get_resource(pdev, IORESOURCE_MEM, 0); |
9648cbc9 | 2200 | ioaddr = res->start; |
8a7f4312 JL |
2201 | smmu->base = devm_ioremap_resource(dev, res); |
2202 | if (IS_ERR(smmu->base)) | |
2203 | return PTR_ERR(smmu->base); | |
452107c7 | 2204 | smmu->cb_base = smmu->base + resource_size(res) / 2; |
45ae7cff | 2205 | |
45ae7cff WD |
2206 | num_irqs = 0; |
2207 | while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) { | |
2208 | num_irqs++; | |
2209 | if (num_irqs > smmu->num_global_irqs) | |
2210 | smmu->num_context_irqs++; | |
2211 | } | |
2212 | ||
44a08de2 AH |
2213 | if (!smmu->num_context_irqs) { |
2214 | dev_err(dev, "found %d interrupts but expected at least %d\n", | |
2215 | num_irqs, smmu->num_global_irqs + 1); | |
2216 | return -ENODEV; | |
45ae7cff | 2217 | } |
45ae7cff | 2218 | |
a86854d0 | 2219 | smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs), |
45ae7cff WD |
2220 | GFP_KERNEL); |
2221 | if (!smmu->irqs) { | |
2222 | dev_err(dev, "failed to allocate %d irqs\n", num_irqs); | |
2223 | return -ENOMEM; | |
2224 | } | |
2225 | ||
2226 | for (i = 0; i < num_irqs; ++i) { | |
2227 | int irq = platform_get_irq(pdev, i); | |
2907320d | 2228 | |
45ae7cff WD |
2229 | if (irq < 0) { |
2230 | dev_err(dev, "failed to get irq index %d\n", i); | |
2231 | return -ENODEV; | |
2232 | } | |
2233 | smmu->irqs[i] = irq; | |
2234 | } | |
2235 | ||
96a299d2 S |
2236 | err = devm_clk_bulk_get_all(dev, &smmu->clks); |
2237 | if (err < 0) { | |
2238 | dev_err(dev, "failed to get clocks %d\n", err); | |
2239 | return err; | |
2240 | } | |
2241 | smmu->num_clks = err; | |
2242 | ||
2243 | err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks); | |
2244 | if (err) | |
2245 | return err; | |
2246 | ||
3c8766d0 OH |
2247 | err = arm_smmu_device_cfg_probe(smmu); |
2248 | if (err) | |
2249 | return err; | |
2250 | ||
d1e20222 VG |
2251 | if (smmu->version == ARM_SMMU_V2) { |
2252 | if (smmu->num_context_banks > smmu->num_context_irqs) { | |
2253 | dev_err(dev, | |
2254 | "found only %d context irq(s) but %d required\n", | |
2255 | smmu->num_context_irqs, smmu->num_context_banks); | |
2256 | return -ENODEV; | |
2257 | } | |
2258 | ||
2259 | /* Ignore superfluous interrupts */ | |
2260 | smmu->num_context_irqs = smmu->num_context_banks; | |
45ae7cff WD |
2261 | } |
2262 | ||
45ae7cff | 2263 | for (i = 0; i < smmu->num_global_irqs; ++i) { |
bee14004 PF |
2264 | err = devm_request_irq(smmu->dev, smmu->irqs[i], |
2265 | arm_smmu_global_fault, | |
2266 | IRQF_SHARED, | |
2267 | "arm-smmu global fault", | |
2268 | smmu); | |
45ae7cff WD |
2269 | if (err) { |
2270 | dev_err(dev, "failed to request global IRQ %d (%u)\n", | |
2271 | i, smmu->irqs[i]); | |
f80cd885 | 2272 | return err; |
45ae7cff WD |
2273 | } |
2274 | } | |
2275 | ||
9648cbc9 JR |
2276 | err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL, |
2277 | "smmu.%pa", &ioaddr); | |
2278 | if (err) { | |
2279 | dev_err(dev, "Failed to register iommu in sysfs\n"); | |
2280 | return err; | |
2281 | } | |
2282 | ||
2283 | iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops); | |
2284 | iommu_device_set_fwnode(&smmu->iommu, dev->fwnode); | |
2285 | ||
2286 | err = iommu_device_register(&smmu->iommu); | |
2287 | if (err) { | |
2288 | dev_err(dev, "Failed to register iommu\n"); | |
2289 | return err; | |
2290 | } | |
2291 | ||
d6fc5d97 | 2292 | platform_set_drvdata(pdev, smmu); |
fd90cecb | 2293 | arm_smmu_device_reset(smmu); |
dc0eaa4e | 2294 | arm_smmu_test_smr_masks(smmu); |
021bb842 | 2295 | |
d4a44f07 S |
2296 | /* |
2297 | * We want to avoid touching dev->power.lock in fastpaths unless | |
2298 | * it's really going to do something useful - pm_runtime_enabled() | |
2299 | * can serve as an ideal proxy for that decision. So, conditionally | |
2300 | * enable pm_runtime. | |
2301 | */ | |
2302 | if (dev->pm_domain) { | |
2303 | pm_runtime_set_active(dev); | |
2304 | pm_runtime_enable(dev); | |
2305 | } | |
2306 | ||
f6810c15 RM |
2307 | /* |
2308 | * For ACPI and generic DT bindings, an SMMU will be probed before | |
2309 | * any device which might need it, so we want the bus ops in place | |
2310 | * ready to handle default domain setup as soon as any SMMU exists. | |
2311 | */ | |
2312 | if (!using_legacy_binding) | |
2313 | arm_smmu_bus_init(); | |
2314 | ||
45ae7cff | 2315 | return 0; |
45ae7cff WD |
2316 | } |
2317 | ||
f6810c15 RM |
2318 | /* |
2319 | * With the legacy DT binding in play, though, we have no guarantees about | |
2320 | * probe order, but then we're also not doing default domains, so we can | |
2321 | * delay setting bus ops until we're sure every possible SMMU is ready, | |
2322 | * and that way ensure that no add_device() calls get missed. | |
2323 | */ | |
2324 | static int arm_smmu_legacy_bus_init(void) | |
2325 | { | |
2326 | if (using_legacy_binding) | |
2327 | arm_smmu_bus_init(); | |
45ae7cff | 2328 | return 0; |
45ae7cff | 2329 | } |
f6810c15 | 2330 | device_initcall_sync(arm_smmu_legacy_bus_init); |
45ae7cff | 2331 | |
addb672f | 2332 | static void arm_smmu_device_shutdown(struct platform_device *pdev) |
45ae7cff | 2333 | { |
d6fc5d97 | 2334 | struct arm_smmu_device *smmu = platform_get_drvdata(pdev); |
45ae7cff WD |
2335 | |
2336 | if (!smmu) | |
addb672f | 2337 | return; |
45ae7cff | 2338 | |
ecfadb6e | 2339 | if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS)) |
d6fc5d97 | 2340 | dev_err(&pdev->dev, "removing device with active domains!\n"); |
45ae7cff | 2341 | |
d4a44f07 | 2342 | arm_smmu_rpm_get(smmu); |
45ae7cff | 2343 | /* Turn the thing off */ |
2907320d | 2344 | writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); |
d4a44f07 S |
2345 | arm_smmu_rpm_put(smmu); |
2346 | ||
2347 | if (pm_runtime_enabled(smmu->dev)) | |
2348 | pm_runtime_force_suspend(smmu->dev); | |
2349 | else | |
2350 | clk_bulk_disable(smmu->num_clks, smmu->clks); | |
96a299d2 | 2351 | |
d4a44f07 | 2352 | clk_bulk_unprepare(smmu->num_clks, smmu->clks); |
45ae7cff WD |
2353 | } |
2354 | ||
96a299d2 | 2355 | static int __maybe_unused arm_smmu_runtime_resume(struct device *dev) |
7aa8619a | 2356 | { |
a2d866f7 | 2357 | struct arm_smmu_device *smmu = dev_get_drvdata(dev); |
96a299d2 S |
2358 | int ret; |
2359 | ||
2360 | ret = clk_bulk_enable(smmu->num_clks, smmu->clks); | |
2361 | if (ret) | |
2362 | return ret; | |
a2d866f7 RM |
2363 | |
2364 | arm_smmu_device_reset(smmu); | |
96a299d2 | 2365 | |
a2d866f7 | 2366 | return 0; |
7aa8619a NW |
2367 | } |
2368 | ||
96a299d2 | 2369 | static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev) |
a2d866f7 RM |
2370 | { |
2371 | struct arm_smmu_device *smmu = dev_get_drvdata(dev); | |
2372 | ||
96a299d2 S |
2373 | clk_bulk_disable(smmu->num_clks, smmu->clks); |
2374 | ||
a2d866f7 RM |
2375 | return 0; |
2376 | } | |
2377 | ||
96a299d2 S |
2378 | static int __maybe_unused arm_smmu_pm_resume(struct device *dev) |
2379 | { | |
2380 | if (pm_runtime_suspended(dev)) | |
2381 | return 0; | |
2382 | ||
2383 | return arm_smmu_runtime_resume(dev); | |
2384 | } | |
2385 | ||
2386 | static int __maybe_unused arm_smmu_pm_suspend(struct device *dev) | |
2387 | { | |
2388 | if (pm_runtime_suspended(dev)) | |
2389 | return 0; | |
2390 | ||
2391 | return arm_smmu_runtime_suspend(dev); | |
2392 | } | |
2393 | ||
2394 | static const struct dev_pm_ops arm_smmu_pm_ops = { | |
2395 | SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume) | |
2396 | SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend, | |
2397 | arm_smmu_runtime_resume, NULL) | |
2398 | }; | |
a2d866f7 | 2399 | |
45ae7cff WD |
2400 | static struct platform_driver arm_smmu_driver = { |
2401 | .driver = { | |
addb672f PG |
2402 | .name = "arm-smmu", |
2403 | .of_match_table = of_match_ptr(arm_smmu_of_match), | |
2404 | .pm = &arm_smmu_pm_ops, | |
2405 | .suppress_bind_attrs = true, | |
45ae7cff | 2406 | }, |
bbb8a184 | 2407 | .probe = arm_smmu_device_probe, |
7aa8619a | 2408 | .shutdown = arm_smmu_device_shutdown, |
45ae7cff | 2409 | }; |
addb672f | 2410 | builtin_platform_driver(arm_smmu_driver); |