]> git.proxmox.com Git - mirror_ubuntu-focal-kernel.git/blob - drivers/iommu/arm-smmu.c
projects / mirror_ubuntu-focal-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/dledford/rdma
[mirror_ubuntu-focal-kernel.git] / drivers / iommu / arm-smmu.c
1 /*
2 * IOMMU API for ARM architected SMMU implementations.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
16 *
17 * Copyright (C) 2013 ARM Limited
18 *
19 * Author: Will Deacon <will.deacon@arm.com>
20 *
21 * This driver currently supports:
22 * - SMMUv1 and v2 implementations
23 * - Stream-matching and stream-indexing
24 * - v7/v8 long-descriptor format
25 * - Non-secure access to the SMMU
26 * - Context fault reporting
27 */
28
29 #define pr_fmt(fmt) "arm-smmu: " fmt
30
31 #include <linux/delay.h>
32 #include <linux/dma-iommu.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/err.h>
35 #include <linux/interrupt.h>
36 #include <linux/io.h>
37 #include <linux/io-64-nonatomic-hi-lo.h>
38 #include <linux/iommu.h>
39 #include <linux/iopoll.h>
40 #include <linux/module.h>
41 #include <linux/of.h>
42 #include <linux/of_address.h>
43 #include <linux/pci.h>
44 #include <linux/platform_device.h>
45 #include <linux/slab.h>
46 #include <linux/spinlock.h>
47
48 #include <linux/amba/bus.h>
49
50 #include "io-pgtable.h"
51
52 /* Maximum number of stream IDs assigned to a single device */
53 #define MAX_MASTER_STREAMIDS MAX_PHANDLE_ARGS
54
55 /* Maximum number of context banks per SMMU */
56 #define ARM_SMMU_MAX_CBS 128
57
58 /* Maximum number of mapping groups per SMMU */
59 #define ARM_SMMU_MAX_SMRS 128
60
61 /* SMMU global address space */
62 #define ARM_SMMU_GR0(smmu) ((smmu)->base)
63 #define ARM_SMMU_GR1(smmu) ((smmu)->base + (1 << (smmu)->pgshift))
64
65 /*
66 * SMMU global address space with conditional offset to access secure
67 * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
68 * nsGFSYNR0: 0x450)
69 */
70 #define ARM_SMMU_GR0_NS(smmu) \
71 ((smmu)->base + \
72 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \
73 ? 0x400 : 0))
74
75 /*
76 * Some 64-bit registers only make sense to write atomically, but in such
77 * cases all the data relevant to AArch32 formats lies within the lower word,
78 * therefore this actually makes more sense than it might first appear.
79 */
80 #ifdef CONFIG_64BIT
81 #define smmu_write_atomic_lq writeq_relaxed
82 #else
83 #define smmu_write_atomic_lq writel_relaxed
84 #endif
85
86 /* Configuration registers */
87 #define ARM_SMMU_GR0_sCR0 0x0
88 #define sCR0_CLIENTPD (1 << 0)
89 #define sCR0_GFRE (1 << 1)
90 #define sCR0_GFIE (1 << 2)
91 #define sCR0_GCFGFRE (1 << 4)
92 #define sCR0_GCFGFIE (1 << 5)
93 #define sCR0_USFCFG (1 << 10)
94 #define sCR0_VMIDPNE (1 << 11)
95 #define sCR0_PTM (1 << 12)
96 #define sCR0_FB (1 << 13)
97 #define sCR0_VMID16EN (1 << 31)
98 #define sCR0_BSU_SHIFT 14
99 #define sCR0_BSU_MASK 0x3
100
101 /* Auxiliary Configuration register */
102 #define ARM_SMMU_GR0_sACR 0x10
103
104 /* Identification registers */
105 #define ARM_SMMU_GR0_ID0 0x20
106 #define ARM_SMMU_GR0_ID1 0x24
107 #define ARM_SMMU_GR0_ID2 0x28
108 #define ARM_SMMU_GR0_ID3 0x2c
109 #define ARM_SMMU_GR0_ID4 0x30
110 #define ARM_SMMU_GR0_ID5 0x34
111 #define ARM_SMMU_GR0_ID6 0x38
112 #define ARM_SMMU_GR0_ID7 0x3c
113 #define ARM_SMMU_GR0_sGFSR 0x48
114 #define ARM_SMMU_GR0_sGFSYNR0 0x50
115 #define ARM_SMMU_GR0_sGFSYNR1 0x54
116 #define ARM_SMMU_GR0_sGFSYNR2 0x58
117
118 #define ID0_S1TS (1 << 30)
119 #define ID0_S2TS (1 << 29)
120 #define ID0_NTS (1 << 28)
121 #define ID0_SMS (1 << 27)
122 #define ID0_ATOSNS (1 << 26)
123 #define ID0_PTFS_NO_AARCH32 (1 << 25)
124 #define ID0_PTFS_NO_AARCH32S (1 << 24)
125 #define ID0_CTTW (1 << 14)
126 #define ID0_NUMIRPT_SHIFT 16
127 #define ID0_NUMIRPT_MASK 0xff
128 #define ID0_NUMSIDB_SHIFT 9
129 #define ID0_NUMSIDB_MASK 0xf
130 #define ID0_NUMSMRG_SHIFT 0
131 #define ID0_NUMSMRG_MASK 0xff
132
133 #define ID1_PAGESIZE (1 << 31)
134 #define ID1_NUMPAGENDXB_SHIFT 28
135 #define ID1_NUMPAGENDXB_MASK 7
136 #define ID1_NUMS2CB_SHIFT 16
137 #define ID1_NUMS2CB_MASK 0xff
138 #define ID1_NUMCB_SHIFT 0
139 #define ID1_NUMCB_MASK 0xff
140
141 #define ID2_OAS_SHIFT 4
142 #define ID2_OAS_MASK 0xf
143 #define ID2_IAS_SHIFT 0
144 #define ID2_IAS_MASK 0xf
145 #define ID2_UBS_SHIFT 8
146 #define ID2_UBS_MASK 0xf
147 #define ID2_PTFS_4K (1 << 12)
148 #define ID2_PTFS_16K (1 << 13)
149 #define ID2_PTFS_64K (1 << 14)
150 #define ID2_VMID16 (1 << 15)
151
152 #define ID7_MAJOR_SHIFT 4
153 #define ID7_MAJOR_MASK 0xf
154
155 /* Global TLB invalidation */
156 #define ARM_SMMU_GR0_TLBIVMID 0x64
157 #define ARM_SMMU_GR0_TLBIALLNSNH 0x68
158 #define ARM_SMMU_GR0_TLBIALLH 0x6c
159 #define ARM_SMMU_GR0_sTLBGSYNC 0x70
160 #define ARM_SMMU_GR0_sTLBGSTATUS 0x74
161 #define sTLBGSTATUS_GSACTIVE (1 << 0)
162 #define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
163
164 /* Stream mapping registers */
165 #define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
166 #define SMR_VALID (1 << 31)
167 #define SMR_MASK_SHIFT 16
168 #define SMR_MASK_MASK 0x7fff
169 #define SMR_ID_SHIFT 0
170 #define SMR_ID_MASK 0x7fff
171
172 #define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
173 #define S2CR_CBNDX_SHIFT 0
174 #define S2CR_CBNDX_MASK 0xff
175 #define S2CR_TYPE_SHIFT 16
176 #define S2CR_TYPE_MASK 0x3
177 #define S2CR_TYPE_TRANS (0 << S2CR_TYPE_SHIFT)
178 #define S2CR_TYPE_BYPASS (1 << S2CR_TYPE_SHIFT)
179 #define S2CR_TYPE_FAULT (2 << S2CR_TYPE_SHIFT)
180
181 #define S2CR_PRIVCFG_SHIFT 24
182 #define S2CR_PRIVCFG_UNPRIV (2 << S2CR_PRIVCFG_SHIFT)
183
184 /* Context bank attribute registers */
185 #define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
186 #define CBAR_VMID_SHIFT 0
187 #define CBAR_VMID_MASK 0xff
188 #define CBAR_S1_BPSHCFG_SHIFT 8
189 #define CBAR_S1_BPSHCFG_MASK 3
190 #define CBAR_S1_BPSHCFG_NSH 3
191 #define CBAR_S1_MEMATTR_SHIFT 12
192 #define CBAR_S1_MEMATTR_MASK 0xf
193 #define CBAR_S1_MEMATTR_WB 0xf
194 #define CBAR_TYPE_SHIFT 16
195 #define CBAR_TYPE_MASK 0x3
196 #define CBAR_TYPE_S2_TRANS (0 << CBAR_TYPE_SHIFT)
197 #define CBAR_TYPE_S1_TRANS_S2_BYPASS (1 << CBAR_TYPE_SHIFT)
198 #define CBAR_TYPE_S1_TRANS_S2_FAULT (2 << CBAR_TYPE_SHIFT)
199 #define CBAR_TYPE_S1_TRANS_S2_TRANS (3 << CBAR_TYPE_SHIFT)
200 #define CBAR_IRPTNDX_SHIFT 24
201 #define CBAR_IRPTNDX_MASK 0xff
202
203 #define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
204 #define CBA2R_RW64_32BIT (0 << 0)
205 #define CBA2R_RW64_64BIT (1 << 0)
206 #define CBA2R_VMID_SHIFT 16
207 #define CBA2R_VMID_MASK 0xffff
208
209 /* Translation context bank */
210 #define ARM_SMMU_CB_BASE(smmu) ((smmu)->base + ((smmu)->size >> 1))
211 #define ARM_SMMU_CB(smmu, n) ((n) * (1 << (smmu)->pgshift))
212
213 #define ARM_SMMU_CB_SCTLR 0x0
214 #define ARM_SMMU_CB_ACTLR 0x4
215 #define ARM_SMMU_CB_RESUME 0x8
216 #define ARM_SMMU_CB_TTBCR2 0x10
217 #define ARM_SMMU_CB_TTBR0 0x20
218 #define ARM_SMMU_CB_TTBR1 0x28
219 #define ARM_SMMU_CB_TTBCR 0x30
220 #define ARM_SMMU_CB_S1_MAIR0 0x38
221 #define ARM_SMMU_CB_S1_MAIR1 0x3c
222 #define ARM_SMMU_CB_PAR 0x50
223 #define ARM_SMMU_CB_FSR 0x58
224 #define ARM_SMMU_CB_FAR 0x60
225 #define ARM_SMMU_CB_FSYNR0 0x68
226 #define ARM_SMMU_CB_S1_TLBIVA 0x600
227 #define ARM_SMMU_CB_S1_TLBIASID 0x610
228 #define ARM_SMMU_CB_S1_TLBIVAL 0x620
229 #define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
230 #define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
231 #define ARM_SMMU_CB_ATS1PR 0x800
232 #define ARM_SMMU_CB_ATSR 0x8f0
233
234 #define SCTLR_S1_ASIDPNE (1 << 12)
235 #define SCTLR_CFCFG (1 << 7)
236 #define SCTLR_CFIE (1 << 6)
237 #define SCTLR_CFRE (1 << 5)
238 #define SCTLR_E (1 << 4)
239 #define SCTLR_AFE (1 << 2)
240 #define SCTLR_TRE (1 << 1)
241 #define SCTLR_M (1 << 0)
242 #define SCTLR_EAE_SBOP (SCTLR_AFE | SCTLR_TRE)
243
244 #define ARM_MMU500_ACTLR_CPRE (1 << 1)
245
246 #define ARM_MMU500_ACR_CACHE_LOCK (1 << 26)
247
248 #define CB_PAR_F (1 << 0)
249
250 #define ATSR_ACTIVE (1 << 0)
251
252 #define RESUME_RETRY (0 << 0)
253 #define RESUME_TERMINATE (1 << 0)
254
255 #define TTBCR2_SEP_SHIFT 15
256 #define TTBCR2_SEP_UPSTREAM (0x7 << TTBCR2_SEP_SHIFT)
257
258 #define TTBRn_ASID_SHIFT 48
259
260 #define FSR_MULTI (1 << 31)
261 #define FSR_SS (1 << 30)
262 #define FSR_UUT (1 << 8)
263 #define FSR_ASF (1 << 7)
264 #define FSR_TLBLKF (1 << 6)
265 #define FSR_TLBMCF (1 << 5)
266 #define FSR_EF (1 << 4)
267 #define FSR_PF (1 << 3)
268 #define FSR_AFF (1 << 2)
269 #define FSR_TF (1 << 1)
270
271 #define FSR_IGN (FSR_AFF | FSR_ASF | \
272 FSR_TLBMCF | FSR_TLBLKF)
273 #define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \
274 FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
275
276 #define FSYNR0_WNR (1 << 4)
277
278 static int force_stage;
279 module_param(force_stage, int, S_IRUGO);
280 MODULE_PARM_DESC(force_stage,
281 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
282 static bool disable_bypass;
283 module_param(disable_bypass, bool, S_IRUGO);
284 MODULE_PARM_DESC(disable_bypass,
285 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
286
287 enum arm_smmu_arch_version {
288 ARM_SMMU_V1,
289 ARM_SMMU_V1_64K,
290 ARM_SMMU_V2,
291 };
292
293 enum arm_smmu_implementation {
294 GENERIC_SMMU,
295 ARM_MMU500,
296 CAVIUM_SMMUV2,
297 };
298
299 struct arm_smmu_smr {
300 u8 idx;
301 u16 mask;
302 u16 id;
303 };
304
305 struct arm_smmu_master_cfg {
306 int num_streamids;
307 u16 streamids[MAX_MASTER_STREAMIDS];
308 struct arm_smmu_smr *smrs;
309 };
310
311 struct arm_smmu_master {
312 struct device_node *of_node;
313 struct rb_node node;
314 struct arm_smmu_master_cfg cfg;
315 };
316
317 struct arm_smmu_device {
318 struct device *dev;
319
320 void __iomem *base;
321 unsigned long size;
322 unsigned long pgshift;
323
324 #define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0)
325 #define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1)
326 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 2)
327 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 3)
328 #define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4)
329 #define ARM_SMMU_FEAT_TRANS_OPS (1 << 5)
330 #define ARM_SMMU_FEAT_VMID16 (1 << 6)
331 #define ARM_SMMU_FEAT_FMT_AARCH64_4K (1 << 7)
332 #define ARM_SMMU_FEAT_FMT_AARCH64_16K (1 << 8)
333 #define ARM_SMMU_FEAT_FMT_AARCH64_64K (1 << 9)
334 #define ARM_SMMU_FEAT_FMT_AARCH32_L (1 << 10)
335 #define ARM_SMMU_FEAT_FMT_AARCH32_S (1 << 11)
336 u32 features;
337
338 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
339 u32 options;
340 enum arm_smmu_arch_version version;
341 enum arm_smmu_implementation model;
342
343 u32 num_context_banks;
344 u32 num_s2_context_banks;
345 DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
346 atomic_t irptndx;
347
348 u32 num_mapping_groups;
349 DECLARE_BITMAP(smr_map, ARM_SMMU_MAX_SMRS);
350
351 unsigned long va_size;
352 unsigned long ipa_size;
353 unsigned long pa_size;
354 unsigned long pgsize_bitmap;
355
356 u32 num_global_irqs;
357 u32 num_context_irqs;
358 unsigned int *irqs;
359
360 struct list_head list;
361 struct rb_root masters;
362
363 u32 cavium_id_base; /* Specific to Cavium */
364 };
365
366 enum arm_smmu_context_fmt {
367 ARM_SMMU_CTX_FMT_NONE,
368 ARM_SMMU_CTX_FMT_AARCH64,
369 ARM_SMMU_CTX_FMT_AARCH32_L,
370 ARM_SMMU_CTX_FMT_AARCH32_S,
371 };
372
373 struct arm_smmu_cfg {
374 u8 cbndx;
375 u8 irptndx;
376 u32 cbar;
377 enum arm_smmu_context_fmt fmt;
378 };
379 #define INVALID_IRPTNDX 0xff
380
381 #define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
382 #define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
383
384 enum arm_smmu_domain_stage {
385 ARM_SMMU_DOMAIN_S1 = 0,
386 ARM_SMMU_DOMAIN_S2,
387 ARM_SMMU_DOMAIN_NESTED,
388 };
389
390 struct arm_smmu_domain {
391 struct arm_smmu_device *smmu;
392 struct io_pgtable_ops *pgtbl_ops;
393 spinlock_t pgtbl_lock;
394 struct arm_smmu_cfg cfg;
395 enum arm_smmu_domain_stage stage;
396 struct mutex init_mutex; /* Protects smmu pointer */
397 struct iommu_domain domain;
398 };
399
400 static DEFINE_SPINLOCK(arm_smmu_devices_lock);
401 static LIST_HEAD(arm_smmu_devices);
402
403 struct arm_smmu_option_prop {
404 u32 opt;
405 const char *prop;
406 };
407
408 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
409
410 static struct arm_smmu_option_prop arm_smmu_options[] = {
411 { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
412 { 0, NULL},
413 };
414
415 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
416 {
417 return container_of(dom, struct arm_smmu_domain, domain);
418 }
419
420 static void parse_driver_options(struct arm_smmu_device *smmu)
421 {
422 int i = 0;
423
424 do {
425 if (of_property_read_bool(smmu->dev->of_node,
426 arm_smmu_options[i].prop)) {
427 smmu->options |= arm_smmu_options[i].opt;
428 dev_notice(smmu->dev, "option %s\n",
429 arm_smmu_options[i].prop);
430 }
431 } while (arm_smmu_options[++i].opt);
432 }
433
434 static struct device_node *dev_get_dev_node(struct device *dev)
435 {
436 if (dev_is_pci(dev)) {
437 struct pci_bus *bus = to_pci_dev(dev)->bus;
438
439 while (!pci_is_root_bus(bus))
440 bus = bus->parent;
441 return bus->bridge->parent->of_node;
442 }
443
444 return dev->of_node;
445 }
446
447 static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu,
448 struct device_node *dev_node)
449 {
450 struct rb_node *node = smmu->masters.rb_node;
451
452 while (node) {
453 struct arm_smmu_master *master;
454
455 master = container_of(node, struct arm_smmu_master, node);
456
457 if (dev_node < master->of_node)
458 node = node->rb_left;
459 else if (dev_node > master->of_node)
460 node = node->rb_right;
461 else
462 return master;
463 }
464
465 return NULL;
466 }
467
468 static struct arm_smmu_master_cfg *
469 find_smmu_master_cfg(struct device *dev)
470 {
471 struct arm_smmu_master_cfg *cfg = NULL;
472 struct iommu_group *group = iommu_group_get(dev);
473
474 if (group) {
475 cfg = iommu_group_get_iommudata(group);
476 iommu_group_put(group);
477 }
478
479 return cfg;
480 }
481
482 static int insert_smmu_master(struct arm_smmu_device *smmu,
483 struct arm_smmu_master *master)
484 {
485 struct rb_node **new, *parent;
486
487 new = &smmu->masters.rb_node;
488 parent = NULL;
489 while (*new) {
490 struct arm_smmu_master *this
491 = container_of(*new, struct arm_smmu_master, node);
492
493 parent = *new;
494 if (master->of_node < this->of_node)
495 new = &((*new)->rb_left);
496 else if (master->of_node > this->of_node)
497 new = &((*new)->rb_right);
498 else
499 return -EEXIST;
500 }
501
502 rb_link_node(&master->node, parent, new);
503 rb_insert_color(&master->node, &smmu->masters);
504 return 0;
505 }
506
507 static int register_smmu_master(struct arm_smmu_device *smmu,
508 struct device *dev,
509 struct of_phandle_args *masterspec)
510 {
511 int i;
512 struct arm_smmu_master *master;
513
514 master = find_smmu_master(smmu, masterspec->np);
515 if (master) {
516 dev_err(dev,
517 "rejecting multiple registrations for master device %s\n",
518 masterspec->np->name);
519 return -EBUSY;
520 }
521
522 if (masterspec->args_count > MAX_MASTER_STREAMIDS) {
523 dev_err(dev,
524 "reached maximum number (%d) of stream IDs for master device %s\n",
525 MAX_MASTER_STREAMIDS, masterspec->np->name);
526 return -ENOSPC;
527 }
528
529 master = devm_kzalloc(dev, sizeof(*master), GFP_KERNEL);
530 if (!master)
531 return -ENOMEM;
532
533 master->of_node = masterspec->np;
534 master->cfg.num_streamids = masterspec->args_count;
535
536 for (i = 0; i < master->cfg.num_streamids; ++i) {
537 u16 streamid = masterspec->args[i];
538
539 if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) &&
540 (streamid >= smmu->num_mapping_groups)) {
541 dev_err(dev,
542 "stream ID for master device %s greater than maximum allowed (%d)\n",
543 masterspec->np->name, smmu->num_mapping_groups);
544 return -ERANGE;
545 }
546 master->cfg.streamids[i] = streamid;
547 }
548 return insert_smmu_master(smmu, master);
549 }
550
551 static struct arm_smmu_device *find_smmu_for_device(struct device *dev)
552 {
553 struct arm_smmu_device *smmu;
554 struct arm_smmu_master *master = NULL;
555 struct device_node *dev_node = dev_get_dev_node(dev);
556
557 spin_lock(&arm_smmu_devices_lock);
558 list_for_each_entry(smmu, &arm_smmu_devices, list) {
559 master = find_smmu_master(smmu, dev_node);
560 if (master)
561 break;
562 }
563 spin_unlock(&arm_smmu_devices_lock);
564
565 return master ? smmu : NULL;
566 }
567
568 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
569 {
570 int idx;
571
572 do {
573 idx = find_next_zero_bit(map, end, start);
574 if (idx == end)
575 return -ENOSPC;
576 } while (test_and_set_bit(idx, map));
577
578 return idx;
579 }
580
581 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
582 {
583 clear_bit(idx, map);
584 }
585
586 /* Wait for any pending TLB invalidations to complete */
587 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
588 {
589 int count = 0;
590 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
591
592 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
593 while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
594 & sTLBGSTATUS_GSACTIVE) {
595 cpu_relax();
596 if (++count == TLB_LOOP_TIMEOUT) {
597 dev_err_ratelimited(smmu->dev,
598 "TLB sync timed out -- SMMU may be deadlocked\n");
599 return;
600 }
601 udelay(1);
602 }
603 }
604
605 static void arm_smmu_tlb_sync(void *cookie)
606 {
607 struct arm_smmu_domain *smmu_domain = cookie;
608 __arm_smmu_tlb_sync(smmu_domain->smmu);
609 }
610
611 static void arm_smmu_tlb_inv_context(void *cookie)
612 {
613 struct arm_smmu_domain *smmu_domain = cookie;
614 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
615 struct arm_smmu_device *smmu = smmu_domain->smmu;
616 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
617 void __iomem *base;
618
619 if (stage1) {
620 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
621 writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg),
622 base + ARM_SMMU_CB_S1_TLBIASID);
623 } else {
624 base = ARM_SMMU_GR0(smmu);
625 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg),
626 base + ARM_SMMU_GR0_TLBIVMID);
627 }
628
629 __arm_smmu_tlb_sync(smmu);
630 }
631
632 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
633 size_t granule, bool leaf, void *cookie)
634 {
635 struct arm_smmu_domain *smmu_domain = cookie;
636 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
637 struct arm_smmu_device *smmu = smmu_domain->smmu;
638 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
639 void __iomem *reg;
640
641 if (stage1) {
642 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
643 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
644
645 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
646 iova &= ~12UL;
647 iova |= ARM_SMMU_CB_ASID(smmu, cfg);
648 do {
649 writel_relaxed(iova, reg);
650 iova += granule;
651 } while (size -= granule);
652 } else {
653 iova >>= 12;
654 iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48;
655 do {
656 writeq_relaxed(iova, reg);
657 iova += granule >> 12;
658 } while (size -= granule);
659 }
660 } else if (smmu->version == ARM_SMMU_V2) {
661 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
662 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
663 ARM_SMMU_CB_S2_TLBIIPAS2;
664 iova >>= 12;
665 do {
666 smmu_write_atomic_lq(iova, reg);
667 iova += granule >> 12;
668 } while (size -= granule);
669 } else {
670 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
671 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg);
672 }
673 }
674
675 static struct iommu_gather_ops arm_smmu_gather_ops = {
676 .tlb_flush_all = arm_smmu_tlb_inv_context,
677 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
678 .tlb_sync = arm_smmu_tlb_sync,
679 };
680
681 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
682 {
683 int flags, ret;
684 u32 fsr, fsynr, resume;
685 unsigned long iova;
686 struct iommu_domain *domain = dev;
687 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
688 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
689 struct arm_smmu_device *smmu = smmu_domain->smmu;
690 void __iomem *cb_base;
691
692 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
693 fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
694
695 if (!(fsr & FSR_FAULT))
696 return IRQ_NONE;
697
698 if (fsr & FSR_IGN)
699 dev_err_ratelimited(smmu->dev,
700 "Unexpected context fault (fsr 0x%x)\n",
701 fsr);
702
703 fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
704 flags = fsynr & FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ;
705
706 iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
707 if (!report_iommu_fault(domain, smmu->dev, iova, flags)) {
708 ret = IRQ_HANDLED;
709 resume = RESUME_RETRY;
710 } else {
711 dev_err_ratelimited(smmu->dev,
712 "Unhandled context fault: iova=0x%08lx, fsynr=0x%x, cb=%d\n",
713 iova, fsynr, cfg->cbndx);
714 ret = IRQ_NONE;
715 resume = RESUME_TERMINATE;
716 }
717
718 /* Clear the faulting FSR */
719 writel(fsr, cb_base + ARM_SMMU_CB_FSR);
720
721 /* Retry or terminate any stalled transactions */
722 if (fsr & FSR_SS)
723 writel_relaxed(resume, cb_base + ARM_SMMU_CB_RESUME);
724
725 return ret;
726 }
727
728 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
729 {
730 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
731 struct arm_smmu_device *smmu = dev;
732 void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
733
734 gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
735 gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
736 gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
737 gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
738
739 if (!gfsr)
740 return IRQ_NONE;
741
742 dev_err_ratelimited(smmu->dev,
743 "Unexpected global fault, this could be serious\n");
744 dev_err_ratelimited(smmu->dev,
745 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
746 gfsr, gfsynr0, gfsynr1, gfsynr2);
747
748 writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
749 return IRQ_HANDLED;
750 }
751
752 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
753 struct io_pgtable_cfg *pgtbl_cfg)
754 {
755 u32 reg;
756 u64 reg64;
757 bool stage1;
758 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
759 struct arm_smmu_device *smmu = smmu_domain->smmu;
760 void __iomem *cb_base, *gr1_base;
761
762 gr1_base = ARM_SMMU_GR1(smmu);
763 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
764 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
765
766 if (smmu->version > ARM_SMMU_V1) {
767 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
768 reg = CBA2R_RW64_64BIT;
769 else
770 reg = CBA2R_RW64_32BIT;
771 /* 16-bit VMIDs live in CBA2R */
772 if (smmu->features & ARM_SMMU_FEAT_VMID16)
773 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT;
774
775 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
776 }
777
778 /* CBAR */
779 reg = cfg->cbar;
780 if (smmu->version < ARM_SMMU_V2)
781 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
782
783 /*
784 * Use the weakest shareability/memory types, so they are
785 * overridden by the ttbcr/pte.
786 */
787 if (stage1) {
788 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
789 (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
790 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
791 /* 8-bit VMIDs live in CBAR */
792 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT;
793 }
794 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
795
796 /* TTBRs */
797 if (stage1) {
798 reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
799
800 reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT;
801 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
802
803 reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
804 reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT;
805 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
806 } else {
807 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
808 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
809 }
810
811 /* TTBCR */
812 if (stage1) {
813 reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
814 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
815 if (smmu->version > ARM_SMMU_V1) {
816 reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
817 reg |= TTBCR2_SEP_UPSTREAM;
818 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR2);
819 }
820 } else {
821 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
822 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
823 }
824
825 /* MAIRs (stage-1 only) */
826 if (stage1) {
827 reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
828 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
829 reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
830 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR1);
831 }
832
833 /* SCTLR */
834 reg = SCTLR_CFCFG | SCTLR_CFIE | SCTLR_CFRE | SCTLR_M | SCTLR_EAE_SBOP;
835 if (stage1)
836 reg |= SCTLR_S1_ASIDPNE;
837 #ifdef __BIG_ENDIAN
838 reg |= SCTLR_E;
839 #endif
840 writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
841 }
842
843 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
844 struct arm_smmu_device *smmu)
845 {
846 int irq, start, ret = 0;
847 unsigned long ias, oas;
848 struct io_pgtable_ops *pgtbl_ops;
849 struct io_pgtable_cfg pgtbl_cfg;
850 enum io_pgtable_fmt fmt;
851 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
852 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
853
854 mutex_lock(&smmu_domain->init_mutex);
855 if (smmu_domain->smmu)
856 goto out_unlock;
857
858 /* We're bypassing these SIDs, so don't allocate an actual context */
859 if (domain->type == IOMMU_DOMAIN_DMA) {
860 smmu_domain->smmu = smmu;
861 goto out_unlock;
862 }
863
864 /*
865 * Mapping the requested stage onto what we support is surprisingly
866 * complicated, mainly because the spec allows S1+S2 SMMUs without
867 * support for nested translation. That means we end up with the
868 * following table:
869 *
870 * Requested Supported Actual
871 * S1 N S1
872 * S1 S1+S2 S1
873 * S1 S2 S2
874 * S1 S1 S1
875 * N N N
876 * N S1+S2 S2
877 * N S2 S2
878 * N S1 S1
879 *
880 * Note that you can't actually request stage-2 mappings.
881 */
882 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
883 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
884 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
885 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
886
887 /*
888 * Choosing a suitable context format is even more fiddly. Until we
889 * grow some way for the caller to express a preference, and/or move
890 * the decision into the io-pgtable code where it arguably belongs,
891 * just aim for the closest thing to the rest of the system, and hope
892 * that the hardware isn't esoteric enough that we can't assume AArch64
893 * support to be a superset of AArch32 support...
894 */
895 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
896 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
897 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
898 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
899 ARM_SMMU_FEAT_FMT_AARCH64_16K |
900 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
901 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
902
903 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
904 ret = -EINVAL;
905 goto out_unlock;
906 }
907
908 switch (smmu_domain->stage) {
909 case ARM_SMMU_DOMAIN_S1:
910 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
911 start = smmu->num_s2_context_banks;
912 ias = smmu->va_size;
913 oas = smmu->ipa_size;
914 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
915 fmt = ARM_64_LPAE_S1;
916 } else {
917 fmt = ARM_32_LPAE_S1;
918 ias = min(ias, 32UL);
919 oas = min(oas, 40UL);
920 }
921 break;
922 case ARM_SMMU_DOMAIN_NESTED:
923 /*
924 * We will likely want to change this if/when KVM gets
925 * involved.
926 */
927 case ARM_SMMU_DOMAIN_S2:
928 cfg->cbar = CBAR_TYPE_S2_TRANS;
929 start = 0;
930 ias = smmu->ipa_size;
931 oas = smmu->pa_size;
932 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
933 fmt = ARM_64_LPAE_S2;
934 } else {
935 fmt = ARM_32_LPAE_S2;
936 ias = min(ias, 40UL);
937 oas = min(oas, 40UL);
938 }
939 break;
940 default:
941 ret = -EINVAL;
942 goto out_unlock;
943 }
944
945 ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
946 smmu->num_context_banks);
947 if (IS_ERR_VALUE(ret))
948 goto out_unlock;
949
950 cfg->cbndx = ret;
951 if (smmu->version < ARM_SMMU_V2) {
952 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
953 cfg->irptndx %= smmu->num_context_irqs;
954 } else {
955 cfg->irptndx = cfg->cbndx;
956 }
957
958 pgtbl_cfg = (struct io_pgtable_cfg) {
959 .pgsize_bitmap = smmu->pgsize_bitmap,
960 .ias = ias,
961 .oas = oas,
962 .tlb = &arm_smmu_gather_ops,
963 .iommu_dev = smmu->dev,
964 };
965
966 smmu_domain->smmu = smmu;
967 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
968 if (!pgtbl_ops) {
969 ret = -ENOMEM;
970 goto out_clear_smmu;
971 }
972
973 /* Update the domain's page sizes to reflect the page table format */
974 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
975
976 /* Initialise the context bank with our page table cfg */
977 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
978
979 /*
980 * Request context fault interrupt. Do this last to avoid the
981 * handler seeing a half-initialised domain state.
982 */
983 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
984 ret = request_irq(irq, arm_smmu_context_fault, IRQF_SHARED,
985 "arm-smmu-context-fault", domain);
986 if (IS_ERR_VALUE(ret)) {
987 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
988 cfg->irptndx, irq);
989 cfg->irptndx = INVALID_IRPTNDX;
990 }
991
992 mutex_unlock(&smmu_domain->init_mutex);
993
994 /* Publish page table ops for map/unmap */
995 smmu_domain->pgtbl_ops = pgtbl_ops;
996 return 0;
997
998 out_clear_smmu:
999 smmu_domain->smmu = NULL;
1000 out_unlock:
1001 mutex_unlock(&smmu_domain->init_mutex);
1002 return ret;
1003 }
1004
1005 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
1006 {
1007 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1008 struct arm_smmu_device *smmu = smmu_domain->smmu;
1009 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1010 void __iomem *cb_base;
1011 int irq;
1012
1013 if (!smmu || domain->type == IOMMU_DOMAIN_DMA)
1014 return;
1015
1016 /*
1017 * Disable the context bank and free the page tables before freeing
1018 * it.
1019 */
1020 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1021 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1022
1023 if (cfg->irptndx != INVALID_IRPTNDX) {
1024 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
1025 free_irq(irq, domain);
1026 }
1027
1028 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1029 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
1030 }
1031
1032 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1033 {
1034 struct arm_smmu_domain *smmu_domain;
1035
1036 if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
1037 return NULL;
1038 /*
1039 * Allocate the domain and initialise some of its data structures.
1040 * We can't really do anything meaningful until we've added a
1041 * master.
1042 */
1043 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1044 if (!smmu_domain)
1045 return NULL;
1046
1047 if (type == IOMMU_DOMAIN_DMA &&
1048 iommu_get_dma_cookie(&smmu_domain->domain)) {
1049 kfree(smmu_domain);
1050 return NULL;
1051 }
1052
1053 mutex_init(&smmu_domain->init_mutex);
1054 spin_lock_init(&smmu_domain->pgtbl_lock);
1055
1056 return &smmu_domain->domain;
1057 }
1058
1059 static void arm_smmu_domain_free(struct iommu_domain *domain)
1060 {
1061 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1062
1063 /*
1064 * Free the domain resources. We assume that all devices have
1065 * already been detached.
1066 */
1067 iommu_put_dma_cookie(domain);
1068 arm_smmu_destroy_domain_context(domain);
1069 kfree(smmu_domain);
1070 }
1071
1072 static int arm_smmu_master_configure_smrs(struct arm_smmu_device *smmu,
1073 struct arm_smmu_master_cfg *cfg)
1074 {
1075 int i;
1076 struct arm_smmu_smr *smrs;
1077 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1078
1079 if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH))
1080 return 0;
1081
1082 if (cfg->smrs)
1083 return -EEXIST;
1084
1085 smrs = kmalloc_array(cfg->num_streamids, sizeof(*smrs), GFP_KERNEL);
1086 if (!smrs) {
1087 dev_err(smmu->dev, "failed to allocate %d SMRs\n",
1088 cfg->num_streamids);
1089 return -ENOMEM;
1090 }
1091
1092 /* Allocate the SMRs on the SMMU */
1093 for (i = 0; i < cfg->num_streamids; ++i) {
1094 int idx = __arm_smmu_alloc_bitmap(smmu->smr_map, 0,
1095 smmu->num_mapping_groups);
1096 if (IS_ERR_VALUE(idx)) {
1097 dev_err(smmu->dev, "failed to allocate free SMR\n");
1098 goto err_free_smrs;
1099 }
1100
1101 smrs[i] = (struct arm_smmu_smr) {
1102 .idx = idx,
1103 .mask = 0, /* We don't currently share SMRs */
1104 .id = cfg->streamids[i],
1105 };
1106 }
1107
1108 /* It worked! Now, poke the actual hardware */
1109 for (i = 0; i < cfg->num_streamids; ++i) {
1110 u32 reg = SMR_VALID | smrs[i].id << SMR_ID_SHIFT |
1111 smrs[i].mask << SMR_MASK_SHIFT;
1112 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_SMR(smrs[i].idx));
1113 }
1114
1115 cfg->smrs = smrs;
1116 return 0;
1117
1118 err_free_smrs:
1119 while (--i >= 0)
1120 __arm_smmu_free_bitmap(smmu->smr_map, smrs[i].idx);
1121 kfree(smrs);
1122 return -ENOSPC;
1123 }
1124
1125 static void arm_smmu_master_free_smrs(struct arm_smmu_device *smmu,
1126 struct arm_smmu_master_cfg *cfg)
1127 {
1128 int i;
1129 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1130 struct arm_smmu_smr *smrs = cfg->smrs;
1131
1132 if (!smrs)
1133 return;
1134
1135 /* Invalidate the SMRs before freeing back to the allocator */
1136 for (i = 0; i < cfg->num_streamids; ++i) {
1137 u8 idx = smrs[i].idx;
1138
1139 writel_relaxed(~SMR_VALID, gr0_base + ARM_SMMU_GR0_SMR(idx));
1140 __arm_smmu_free_bitmap(smmu->smr_map, idx);
1141 }
1142
1143 cfg->smrs = NULL;
1144 kfree(smrs);
1145 }
1146
1147 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1148 struct arm_smmu_master_cfg *cfg)
1149 {
1150 int i, ret;
1151 struct arm_smmu_device *smmu = smmu_domain->smmu;
1152 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1153
1154 /*
1155 * FIXME: This won't be needed once we have IOMMU-backed DMA ops
1156 * for all devices behind the SMMU. Note that we need to take
1157 * care configuring SMRs for devices both a platform_device and
1158 * and a PCI device (i.e. a PCI host controller)
1159 */
1160 if (smmu_domain->domain.type == IOMMU_DOMAIN_DMA)
1161 return 0;
1162
1163 /* Devices in an IOMMU group may already be configured */
1164 ret = arm_smmu_master_configure_smrs(smmu, cfg);
1165 if (ret)
1166 return ret == -EEXIST ? 0 : ret;
1167
1168 for (i = 0; i < cfg->num_streamids; ++i) {
1169 u32 idx, s2cr;
1170
1171 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i];
1172 s2cr = S2CR_TYPE_TRANS | S2CR_PRIVCFG_UNPRIV |
1173 (smmu_domain->cfg.cbndx << S2CR_CBNDX_SHIFT);
1174 writel_relaxed(s2cr, gr0_base + ARM_SMMU_GR0_S2CR(idx));
1175 }
1176
1177 return 0;
1178 }
1179
1180 static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain,
1181 struct arm_smmu_master_cfg *cfg)
1182 {
1183 int i;
1184 struct arm_smmu_device *smmu = smmu_domain->smmu;
1185 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1186
1187 /* An IOMMU group is torn down by the first device to be removed */
1188 if ((smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) && !cfg->smrs)
1189 return;
1190
1191 /*
1192 * We *must* clear the S2CR first, because freeing the SMR means
1193 * that it can be re-allocated immediately.
1194 */
1195 for (i = 0; i < cfg->num_streamids; ++i) {
1196 u32 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i];
1197 u32 reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS;
1198
1199 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(idx));
1200 }
1201
1202 arm_smmu_master_free_smrs(smmu, cfg);
1203 }
1204
1205 static void arm_smmu_detach_dev(struct device *dev,
1206 struct arm_smmu_master_cfg *cfg)
1207 {
1208 struct iommu_domain *domain = dev->archdata.iommu;
1209 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1210
1211 dev->archdata.iommu = NULL;
1212 arm_smmu_domain_remove_master(smmu_domain, cfg);
1213 }
1214
1215 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1216 {
1217 int ret;
1218 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1219 struct arm_smmu_device *smmu;
1220 struct arm_smmu_master_cfg *cfg;
1221
1222 smmu = find_smmu_for_device(dev);
1223 if (!smmu) {
1224 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1225 return -ENXIO;
1226 }
1227
1228 /* Ensure that the domain is finalised */
1229 ret = arm_smmu_init_domain_context(domain, smmu);
1230 if (IS_ERR_VALUE(ret))
1231 return ret;
1232
1233 /*
1234 * Sanity check the domain. We don't support domains across
1235 * different SMMUs.
1236 */
1237 if (smmu_domain->smmu != smmu) {
1238 dev_err(dev,
1239 "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1240 dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1241 return -EINVAL;
1242 }
1243
1244 /* Looks ok, so add the device to the domain */
1245 cfg = find_smmu_master_cfg(dev);
1246 if (!cfg)
1247 return -ENODEV;
1248
1249 /* Detach the dev from its current domain */
1250 if (dev->archdata.iommu)
1251 arm_smmu_detach_dev(dev, cfg);
1252
1253 ret = arm_smmu_domain_add_master(smmu_domain, cfg);
1254 if (!ret)
1255 dev->archdata.iommu = domain;
1256 return ret;
1257 }
1258
1259 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1260 phys_addr_t paddr, size_t size, int prot)
1261 {
1262 int ret;
1263 unsigned long flags;
1264 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1265 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1266
1267 if (!ops)
1268 return -ENODEV;
1269
1270 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1271 ret = ops->map(ops, iova, paddr, size, prot);
1272 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1273 return ret;
1274 }
1275
1276 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1277 size_t size)
1278 {
1279 size_t ret;
1280 unsigned long flags;
1281 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1282 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1283
1284 if (!ops)
1285 return 0;
1286
1287 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1288 ret = ops->unmap(ops, iova, size);
1289 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1290 return ret;
1291 }
1292
1293 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1294 dma_addr_t iova)
1295 {
1296 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1297 struct arm_smmu_device *smmu = smmu_domain->smmu;
1298 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1299 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1300 struct device *dev = smmu->dev;
1301 void __iomem *cb_base;
1302 u32 tmp;
1303 u64 phys;
1304 unsigned long va;
1305
1306 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1307
1308 /* ATS1 registers can only be written atomically */
1309 va = iova & ~0xfffUL;
1310 if (smmu->version == ARM_SMMU_V2)
1311 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1312 else /* Register is only 32-bit in v1 */
1313 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1314
1315 if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1316 !(tmp & ATSR_ACTIVE), 5, 50)) {
1317 dev_err(dev,
1318 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1319 &iova);
1320 return ops->iova_to_phys(ops, iova);
1321 }
1322
1323 phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1324 if (phys & CB_PAR_F) {
1325 dev_err(dev, "translation fault!\n");
1326 dev_err(dev, "PAR = 0x%llx\n", phys);
1327 return 0;
1328 }
1329
1330 return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1331 }
1332
1333 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1334 dma_addr_t iova)
1335 {
1336 phys_addr_t ret;
1337 unsigned long flags;
1338 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1339 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1340
1341 if (!ops)
1342 return 0;
1343
1344 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1345 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1346 smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1347 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1348 } else {
1349 ret = ops->iova_to_phys(ops, iova);
1350 }
1351
1352 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1353
1354 return ret;
1355 }
1356
1357 static bool arm_smmu_capable(enum iommu_cap cap)
1358 {
1359 switch (cap) {
1360 case IOMMU_CAP_CACHE_COHERENCY:
1361 /*
1362 * Return true here as the SMMU can always send out coherent
1363 * requests.
1364 */
1365 return true;
1366 case IOMMU_CAP_INTR_REMAP:
1367 return true; /* MSIs are just memory writes */
1368 case IOMMU_CAP_NOEXEC:
1369 return true;
1370 default:
1371 return false;
1372 }
1373 }
1374
1375 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
1376 {
1377 *((u16 *)data) = alias;
1378 return 0; /* Continue walking */
1379 }
1380
1381 static void __arm_smmu_release_pci_iommudata(void *data)
1382 {
1383 kfree(data);
1384 }
1385
1386 static int arm_smmu_init_pci_device(struct pci_dev *pdev,
1387 struct iommu_group *group)
1388 {
1389 struct arm_smmu_master_cfg *cfg;
1390 u16 sid;
1391 int i;
1392
1393 cfg = iommu_group_get_iommudata(group);
1394 if (!cfg) {
1395 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
1396 if (!cfg)
1397 return -ENOMEM;
1398
1399 iommu_group_set_iommudata(group, cfg,
1400 __arm_smmu_release_pci_iommudata);
1401 }
1402
1403 if (cfg->num_streamids >= MAX_MASTER_STREAMIDS)
1404 return -ENOSPC;
1405
1406 /*
1407 * Assume Stream ID == Requester ID for now.
1408 * We need a way to describe the ID mappings in FDT.
1409 */
1410 pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid, &sid);
1411 for (i = 0; i < cfg->num_streamids; ++i)
1412 if (cfg->streamids[i] == sid)
1413 break;
1414
1415 /* Avoid duplicate SIDs, as this can lead to SMR conflicts */
1416 if (i == cfg->num_streamids)
1417 cfg->streamids[cfg->num_streamids++] = sid;
1418
1419 return 0;
1420 }
1421
1422 static int arm_smmu_init_platform_device(struct device *dev,
1423 struct iommu_group *group)
1424 {
1425 struct arm_smmu_device *smmu = find_smmu_for_device(dev);
1426 struct arm_smmu_master *master;
1427
1428 if (!smmu)
1429 return -ENODEV;
1430
1431 master = find_smmu_master(smmu, dev->of_node);
1432 if (!master)
1433 return -ENODEV;
1434
1435 iommu_group_set_iommudata(group, &master->cfg, NULL);
1436
1437 return 0;
1438 }
1439
1440 static int arm_smmu_add_device(struct device *dev)
1441 {
1442 struct iommu_group *group;
1443
1444 group = iommu_group_get_for_dev(dev);
1445 if (IS_ERR(group))
1446 return PTR_ERR(group);
1447
1448 iommu_group_put(group);
1449 return 0;
1450 }
1451
1452 static void arm_smmu_remove_device(struct device *dev)
1453 {
1454 iommu_group_remove_device(dev);
1455 }
1456
1457 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1458 {
1459 struct iommu_group *group;
1460 int ret;
1461
1462 if (dev_is_pci(dev))
1463 group = pci_device_group(dev);
1464 else
1465 group = generic_device_group(dev);
1466
1467 if (IS_ERR(group))
1468 return group;
1469
1470 if (dev_is_pci(dev))
1471 ret = arm_smmu_init_pci_device(to_pci_dev(dev), group);
1472 else
1473 ret = arm_smmu_init_platform_device(dev, group);
1474
1475 if (ret) {
1476 iommu_group_put(group);
1477 group = ERR_PTR(ret);
1478 }
1479
1480 return group;
1481 }
1482
1483 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1484 enum iommu_attr attr, void *data)
1485 {
1486 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1487
1488 switch (attr) {
1489 case DOMAIN_ATTR_NESTING:
1490 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1491 return 0;
1492 default:
1493 return -ENODEV;
1494 }
1495 }
1496
1497 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1498 enum iommu_attr attr, void *data)
1499 {
1500 int ret = 0;
1501 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1502
1503 mutex_lock(&smmu_domain->init_mutex);
1504
1505 switch (attr) {
1506 case DOMAIN_ATTR_NESTING:
1507 if (smmu_domain->smmu) {
1508 ret = -EPERM;
1509 goto out_unlock;
1510 }
1511
1512 if (*(int *)data)
1513 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1514 else
1515 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1516
1517 break;
1518 default:
1519 ret = -ENODEV;
1520 }
1521
1522 out_unlock:
1523 mutex_unlock(&smmu_domain->init_mutex);
1524 return ret;
1525 }
1526
1527 static struct iommu_ops arm_smmu_ops = {
1528 .capable = arm_smmu_capable,
1529 .domain_alloc = arm_smmu_domain_alloc,
1530 .domain_free = arm_smmu_domain_free,
1531 .attach_dev = arm_smmu_attach_dev,
1532 .map = arm_smmu_map,
1533 .unmap = arm_smmu_unmap,
1534 .map_sg = default_iommu_map_sg,
1535 .iova_to_phys = arm_smmu_iova_to_phys,
1536 .add_device = arm_smmu_add_device,
1537 .remove_device = arm_smmu_remove_device,
1538 .device_group = arm_smmu_device_group,
1539 .domain_get_attr = arm_smmu_domain_get_attr,
1540 .domain_set_attr = arm_smmu_domain_set_attr,
1541 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1542 };
1543
1544 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1545 {
1546 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1547 void __iomem *cb_base;
1548 int i = 0;
1549 u32 reg, major;
1550
1551 /* clear global FSR */
1552 reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1553 writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1554
1555 /* Mark all SMRn as invalid and all S2CRn as bypass unless overridden */
1556 reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS;
1557 for (i = 0; i < smmu->num_mapping_groups; ++i) {
1558 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_SMR(i));
1559 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(i));
1560 }
1561
1562 /*
1563 * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1564 * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1565 * bit is only present in MMU-500r2 onwards.
1566 */
1567 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1568 major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1569 if ((smmu->model == ARM_MMU500) && (major >= 2)) {
1570 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1571 reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1572 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1573 }
1574
1575 /* Make sure all context banks are disabled and clear CB_FSR */
1576 for (i = 0; i < smmu->num_context_banks; ++i) {
1577 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
1578 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1579 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1580 /*
1581 * Disable MMU-500's not-particularly-beneficial next-page
1582 * prefetcher for the sake of errata #841119 and #826419.
1583 */
1584 if (smmu->model == ARM_MMU500) {
1585 reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1586 reg &= ~ARM_MMU500_ACTLR_CPRE;
1587 writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1588 }
1589 }
1590
1591 /* Invalidate the TLB, just in case */
1592 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1593 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1594
1595 reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1596
1597 /* Enable fault reporting */
1598 reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1599
1600 /* Disable TLB broadcasting. */
1601 reg |= (sCR0_VMIDPNE | sCR0_PTM);
1602
1603 /* Enable client access, handling unmatched streams as appropriate */
1604 reg &= ~sCR0_CLIENTPD;
1605 if (disable_bypass)
1606 reg |= sCR0_USFCFG;
1607 else
1608 reg &= ~sCR0_USFCFG;
1609
1610 /* Disable forced broadcasting */
1611 reg &= ~sCR0_FB;
1612
1613 /* Don't upgrade barriers */
1614 reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1615
1616 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1617 reg |= sCR0_VMID16EN;
1618
1619 /* Push the button */
1620 __arm_smmu_tlb_sync(smmu);
1621 writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1622 }
1623
1624 static int arm_smmu_id_size_to_bits(int size)
1625 {
1626 switch (size) {
1627 case 0:
1628 return 32;
1629 case 1:
1630 return 36;
1631 case 2:
1632 return 40;
1633 case 3:
1634 return 42;
1635 case 4:
1636 return 44;
1637 case 5:
1638 default:
1639 return 48;
1640 }
1641 }
1642
1643 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1644 {
1645 unsigned long size;
1646 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1647 u32 id;
1648 bool cttw_dt, cttw_reg;
1649
1650 dev_notice(smmu->dev, "probing hardware configuration...\n");
1651 dev_notice(smmu->dev, "SMMUv%d with:\n",
1652 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1653
1654 /* ID0 */
1655 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1656
1657 /* Restrict available stages based on module parameter */
1658 if (force_stage == 1)
1659 id &= ~(ID0_S2TS | ID0_NTS);
1660 else if (force_stage == 2)
1661 id &= ~(ID0_S1TS | ID0_NTS);
1662
1663 if (id & ID0_S1TS) {
1664 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1665 dev_notice(smmu->dev, "\tstage 1 translation\n");
1666 }
1667
1668 if (id & ID0_S2TS) {
1669 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1670 dev_notice(smmu->dev, "\tstage 2 translation\n");
1671 }
1672
1673 if (id & ID0_NTS) {
1674 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1675 dev_notice(smmu->dev, "\tnested translation\n");
1676 }
1677
1678 if (!(smmu->features &
1679 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1680 dev_err(smmu->dev, "\tno translation support!\n");
1681 return -ENODEV;
1682 }
1683
1684 if ((id & ID0_S1TS) &&
1685 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1686 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1687 dev_notice(smmu->dev, "\taddress translation ops\n");
1688 }
1689
1690 /*
1691 * In order for DMA API calls to work properly, we must defer to what
1692 * the DT says about coherency, regardless of what the hardware claims.
1693 * Fortunately, this also opens up a workaround for systems where the
1694 * ID register value has ended up configured incorrectly.
1695 */
1696 cttw_dt = of_dma_is_coherent(smmu->dev->of_node);
1697 cttw_reg = !!(id & ID0_CTTW);
1698 if (cttw_dt)
1699 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1700 if (cttw_dt || cttw_reg)
1701 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1702 cttw_dt ? "" : "non-");
1703 if (cttw_dt != cttw_reg)
1704 dev_notice(smmu->dev,
1705 "\t(IDR0.CTTW overridden by dma-coherent property)\n");
1706
1707 if (id & ID0_SMS) {
1708 u32 smr, sid, mask;
1709
1710 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1711 smmu->num_mapping_groups = (id >> ID0_NUMSMRG_SHIFT) &
1712 ID0_NUMSMRG_MASK;
1713 if (smmu->num_mapping_groups == 0) {
1714 dev_err(smmu->dev,
1715 "stream-matching supported, but no SMRs present!\n");
1716 return -ENODEV;
1717 }
1718
1719 smr = SMR_MASK_MASK << SMR_MASK_SHIFT;
1720 smr |= (SMR_ID_MASK << SMR_ID_SHIFT);
1721 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1722 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1723
1724 mask = (smr >> SMR_MASK_SHIFT) & SMR_MASK_MASK;
1725 sid = (smr >> SMR_ID_SHIFT) & SMR_ID_MASK;
1726 if ((mask & sid) != sid) {
1727 dev_err(smmu->dev,
1728 "SMR mask bits (0x%x) insufficient for ID field (0x%x)\n",
1729 mask, sid);
1730 return -ENODEV;
1731 }
1732
1733 dev_notice(smmu->dev,
1734 "\tstream matching with %u register groups, mask 0x%x",
1735 smmu->num_mapping_groups, mask);
1736 } else {
1737 smmu->num_mapping_groups = (id >> ID0_NUMSIDB_SHIFT) &
1738 ID0_NUMSIDB_MASK;
1739 }
1740
1741 if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1742 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1743 if (!(id & ID0_PTFS_NO_AARCH32S))
1744 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1745 }
1746
1747 /* ID1 */
1748 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1749 smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1750
1751 /* Check for size mismatch of SMMU address space from mapped region */
1752 size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1753 size *= 2 << smmu->pgshift;
1754 if (smmu->size != size)
1755 dev_warn(smmu->dev,
1756 "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1757 size, smmu->size);
1758
1759 smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1760 smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1761 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1762 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1763 return -ENODEV;
1764 }
1765 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1766 smmu->num_context_banks, smmu->num_s2_context_banks);
1767 /*
1768 * Cavium CN88xx erratum #27704.
1769 * Ensure ASID and VMID allocation is unique across all SMMUs in
1770 * the system.
1771 */
1772 if (smmu->model == CAVIUM_SMMUV2) {
1773 smmu->cavium_id_base =
1774 atomic_add_return(smmu->num_context_banks,
1775 &cavium_smmu_context_count);
1776 smmu->cavium_id_base -= smmu->num_context_banks;
1777 }
1778
1779 /* ID2 */
1780 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1781 size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1782 smmu->ipa_size = size;
1783
1784 /* The output mask is also applied for bypass */
1785 size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1786 smmu->pa_size = size;
1787
1788 if (id & ID2_VMID16)
1789 smmu->features |= ARM_SMMU_FEAT_VMID16;
1790
1791 /*
1792 * What the page table walker can address actually depends on which
1793 * descriptor format is in use, but since a) we don't know that yet,
1794 * and b) it can vary per context bank, this will have to do...
1795 */
1796 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1797 dev_warn(smmu->dev,
1798 "failed to set DMA mask for table walker\n");
1799
1800 if (smmu->version < ARM_SMMU_V2) {
1801 smmu->va_size = smmu->ipa_size;
1802 if (smmu->version == ARM_SMMU_V1_64K)
1803 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1804 } else {
1805 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1806 smmu->va_size = arm_smmu_id_size_to_bits(size);
1807 if (id & ID2_PTFS_4K)
1808 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1809 if (id & ID2_PTFS_16K)
1810 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1811 if (id & ID2_PTFS_64K)
1812 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1813 }
1814
1815 /* Now we've corralled the various formats, what'll it do? */
1816 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1817 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1818 if (smmu->features &
1819 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1820 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1821 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1822 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1823 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1824 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1825
1826 if (arm_smmu_ops.pgsize_bitmap == -1UL)
1827 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1828 else
1829 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1830 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1831 smmu->pgsize_bitmap);
1832
1833
1834 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1835 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1836 smmu->va_size, smmu->ipa_size);
1837
1838 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1839 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1840 smmu->ipa_size, smmu->pa_size);
1841
1842 return 0;
1843 }
1844
1845 struct arm_smmu_match_data {
1846 enum arm_smmu_arch_version version;
1847 enum arm_smmu_implementation model;
1848 };
1849
1850 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1851 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1852
1853 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1854 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1855 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1856 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1857 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1858
1859 static const struct of_device_id arm_smmu_of_match[] = {
1860 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1861 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1862 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1863 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1864 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1865 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1866 { },
1867 };
1868 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1869
1870 static int arm_smmu_device_dt_probe(struct platform_device *pdev)
1871 {
1872 const struct of_device_id *of_id;
1873 const struct arm_smmu_match_data *data;
1874 struct resource *res;
1875 struct arm_smmu_device *smmu;
1876 struct device *dev = &pdev->dev;
1877 struct rb_node *node;
1878 struct of_phandle_args masterspec;
1879 int num_irqs, i, err;
1880
1881 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
1882 if (!smmu) {
1883 dev_err(dev, "failed to allocate arm_smmu_device\n");
1884 return -ENOMEM;
1885 }
1886 smmu->dev = dev;
1887
1888 of_id = of_match_node(arm_smmu_of_match, dev->of_node);
1889 data = of_id->data;
1890 smmu->version = data->version;
1891 smmu->model = data->model;
1892
1893 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1894 smmu->base = devm_ioremap_resource(dev, res);
1895 if (IS_ERR(smmu->base))
1896 return PTR_ERR(smmu->base);
1897 smmu->size = resource_size(res);
1898
1899 if (of_property_read_u32(dev->of_node, "#global-interrupts",
1900 &smmu->num_global_irqs)) {
1901 dev_err(dev, "missing #global-interrupts property\n");
1902 return -ENODEV;
1903 }
1904
1905 num_irqs = 0;
1906 while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
1907 num_irqs++;
1908 if (num_irqs > smmu->num_global_irqs)
1909 smmu->num_context_irqs++;
1910 }
1911
1912 if (!smmu->num_context_irqs) {
1913 dev_err(dev, "found %d interrupts but expected at least %d\n",
1914 num_irqs, smmu->num_global_irqs + 1);
1915 return -ENODEV;
1916 }
1917
1918 smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
1919 GFP_KERNEL);
1920 if (!smmu->irqs) {
1921 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
1922 return -ENOMEM;
1923 }
1924
1925 for (i = 0; i < num_irqs; ++i) {
1926 int irq = platform_get_irq(pdev, i);
1927
1928 if (irq < 0) {
1929 dev_err(dev, "failed to get irq index %d\n", i);
1930 return -ENODEV;
1931 }
1932 smmu->irqs[i] = irq;
1933 }
1934
1935 err = arm_smmu_device_cfg_probe(smmu);
1936 if (err)
1937 return err;
1938
1939 i = 0;
1940 smmu->masters = RB_ROOT;
1941 while (!of_parse_phandle_with_args(dev->of_node, "mmu-masters",
1942 "#stream-id-cells", i,
1943 &masterspec)) {
1944 err = register_smmu_master(smmu, dev, &masterspec);
1945 if (err) {
1946 dev_err(dev, "failed to add master %s\n",
1947 masterspec.np->name);
1948 goto out_put_masters;
1949 }
1950
1951 i++;
1952 }
1953 dev_notice(dev, "registered %d master devices\n", i);
1954
1955 parse_driver_options(smmu);
1956
1957 if (smmu->version == ARM_SMMU_V2 &&
1958 smmu->num_context_banks != smmu->num_context_irqs) {
1959 dev_err(dev,
1960 "found only %d context interrupt(s) but %d required\n",
1961 smmu->num_context_irqs, smmu->num_context_banks);
1962 err = -ENODEV;
1963 goto out_put_masters;
1964 }
1965
1966 for (i = 0; i < smmu->num_global_irqs; ++i) {
1967 err = request_irq(smmu->irqs[i],
1968 arm_smmu_global_fault,
1969 IRQF_SHARED,
1970 "arm-smmu global fault",
1971 smmu);
1972 if (err) {
1973 dev_err(dev, "failed to request global IRQ %d (%u)\n",
1974 i, smmu->irqs[i]);
1975 goto out_free_irqs;
1976 }
1977 }
1978
1979 INIT_LIST_HEAD(&smmu->list);
1980 spin_lock(&arm_smmu_devices_lock);
1981 list_add(&smmu->list, &arm_smmu_devices);
1982 spin_unlock(&arm_smmu_devices_lock);
1983
1984 arm_smmu_device_reset(smmu);
1985 return 0;
1986
1987 out_free_irqs:
1988 while (i--)
1989 free_irq(smmu->irqs[i], smmu);
1990
1991 out_put_masters:
1992 for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {
1993 struct arm_smmu_master *master
1994 = container_of(node, struct arm_smmu_master, node);
1995 of_node_put(master->of_node);
1996 }
1997
1998 return err;
1999 }
2000
2001 static int arm_smmu_device_remove(struct platform_device *pdev)
2002 {
2003 int i;
2004 struct device *dev = &pdev->dev;
2005 struct arm_smmu_device *curr, *smmu = NULL;
2006 struct rb_node *node;
2007
2008 spin_lock(&arm_smmu_devices_lock);
2009 list_for_each_entry(curr, &arm_smmu_devices, list) {
2010 if (curr->dev == dev) {
2011 smmu = curr;
2012 list_del(&smmu->list);
2013 break;
2014 }
2015 }
2016 spin_unlock(&arm_smmu_devices_lock);
2017
2018 if (!smmu)
2019 return -ENODEV;
2020
2021 for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {
2022 struct arm_smmu_master *master
2023 = container_of(node, struct arm_smmu_master, node);
2024 of_node_put(master->of_node);
2025 }
2026
2027 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2028 dev_err(dev, "removing device with active domains!\n");
2029
2030 for (i = 0; i < smmu->num_global_irqs; ++i)
2031 free_irq(smmu->irqs[i], smmu);
2032
2033 /* Turn the thing off */
2034 writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2035 return 0;
2036 }
2037
2038 static struct platform_driver arm_smmu_driver = {
2039 .driver = {
2040 .name = "arm-smmu",
2041 .of_match_table = of_match_ptr(arm_smmu_of_match),
2042 },
2043 .probe = arm_smmu_device_dt_probe,
2044 .remove = arm_smmu_device_remove,
2045 };
2046
2047 static int __init arm_smmu_init(void)
2048 {
2049 struct device_node *np;
2050 int ret;
2051
2052 /*
2053 * Play nice with systems that don't have an ARM SMMU by checking that
2054 * an ARM SMMU exists in the system before proceeding with the driver
2055 * and IOMMU bus operation registration.
2056 */
2057 np = of_find_matching_node(NULL, arm_smmu_of_match);
2058 if (!np)
2059 return 0;
2060
2061 of_node_put(np);
2062
2063 ret = platform_driver_register(&arm_smmu_driver);
2064 if (ret)
2065 return ret;
2066
2067 /* Oh, for a proper bus abstraction */
2068 if (!iommu_present(&platform_bus_type))
2069 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2070
2071 #ifdef CONFIG_ARM_AMBA
2072 if (!iommu_present(&amba_bustype))
2073 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2074 #endif
2075
2076 #ifdef CONFIG_PCI
2077 if (!iommu_present(&pci_bus_type))
2078 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2079 #endif
2080
2081 return 0;
2082 }
2083
2084 static void __exit arm_smmu_exit(void)
2085 {
2086 return platform_driver_unregister(&arm_smmu_driver);
2087 }
2088
2089 subsys_initcall(arm_smmu_init);
2090 module_exit(arm_smmu_exit);
2091
2092 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2093 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2094 MODULE_LICENSE("GPL v2");
Ubuntu Focal Linux kernel mirror