2 * IOMMU API for ARM architected SMMU implementations.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 * Copyright (C) 2013 ARM Limited
19 * Author: Will Deacon <will.deacon@arm.com>
21 * This driver currently supports:
22 * - SMMUv1 and v2 implementations
23 * - Stream-matching and stream-indexing
24 * - v7/v8 long-descriptor format
25 * - Non-secure access to the SMMU
26 * - Context fault reporting
29 #define pr_fmt(fmt) "arm-smmu: " fmt
31 #include <linux/delay.h>
32 #include <linux/dma-iommu.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/err.h>
35 #include <linux/interrupt.h>
37 #include <linux/iommu.h>
38 #include <linux/iopoll.h>
39 #include <linux/module.h>
41 #include <linux/of_address.h>
42 #include <linux/pci.h>
43 #include <linux/platform_device.h>
44 #include <linux/slab.h>
45 #include <linux/spinlock.h>
47 #include <linux/amba/bus.h>
49 #include "io-pgtable.h"
51 /* Maximum number of stream IDs assigned to a single device */
52 #define MAX_MASTER_STREAMIDS MAX_PHANDLE_ARGS
54 /* Maximum number of context banks per SMMU */
55 #define ARM_SMMU_MAX_CBS 128
57 /* Maximum number of mapping groups per SMMU */
58 #define ARM_SMMU_MAX_SMRS 128
60 /* SMMU global address space */
61 #define ARM_SMMU_GR0(smmu) ((smmu)->base)
62 #define ARM_SMMU_GR1(smmu) ((smmu)->base + (1 << (smmu)->pgshift))
65 * SMMU global address space with conditional offset to access secure
66 * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
69 #define ARM_SMMU_GR0_NS(smmu) \
71 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \
75 #define smmu_writeq writeq_relaxed
77 #define smmu_writeq(reg64, addr) \
79 u64 __val = (reg64); \
80 void __iomem *__addr = (addr); \
81 writel_relaxed(__val >> 32, __addr + 4); \
82 writel_relaxed(__val, __addr); \
86 /* Configuration registers */
87 #define ARM_SMMU_GR0_sCR0 0x0
88 #define sCR0_CLIENTPD (1 << 0)
89 #define sCR0_GFRE (1 << 1)
90 #define sCR0_GFIE (1 << 2)
91 #define sCR0_GCFGFRE (1 << 4)
92 #define sCR0_GCFGFIE (1 << 5)
93 #define sCR0_USFCFG (1 << 10)
94 #define sCR0_VMIDPNE (1 << 11)
95 #define sCR0_PTM (1 << 12)
96 #define sCR0_FB (1 << 13)
97 #define sCR0_VMID16EN (1 << 31)
98 #define sCR0_BSU_SHIFT 14
99 #define sCR0_BSU_MASK 0x3
101 /* Identification registers */
102 #define ARM_SMMU_GR0_ID0 0x20
103 #define ARM_SMMU_GR0_ID1 0x24
104 #define ARM_SMMU_GR0_ID2 0x28
105 #define ARM_SMMU_GR0_ID3 0x2c
106 #define ARM_SMMU_GR0_ID4 0x30
107 #define ARM_SMMU_GR0_ID5 0x34
108 #define ARM_SMMU_GR0_ID6 0x38
109 #define ARM_SMMU_GR0_ID7 0x3c
110 #define ARM_SMMU_GR0_sGFSR 0x48
111 #define ARM_SMMU_GR0_sGFSYNR0 0x50
112 #define ARM_SMMU_GR0_sGFSYNR1 0x54
113 #define ARM_SMMU_GR0_sGFSYNR2 0x58
115 #define ID0_S1TS (1 << 30)
116 #define ID0_S2TS (1 << 29)
117 #define ID0_NTS (1 << 28)
118 #define ID0_SMS (1 << 27)
119 #define ID0_ATOSNS (1 << 26)
120 #define ID0_CTTW (1 << 14)
121 #define ID0_NUMIRPT_SHIFT 16
122 #define ID0_NUMIRPT_MASK 0xff
123 #define ID0_NUMSIDB_SHIFT 9
124 #define ID0_NUMSIDB_MASK 0xf
125 #define ID0_NUMSMRG_SHIFT 0
126 #define ID0_NUMSMRG_MASK 0xff
128 #define ID1_PAGESIZE (1 << 31)
129 #define ID1_NUMPAGENDXB_SHIFT 28
130 #define ID1_NUMPAGENDXB_MASK 7
131 #define ID1_NUMS2CB_SHIFT 16
132 #define ID1_NUMS2CB_MASK 0xff
133 #define ID1_NUMCB_SHIFT 0
134 #define ID1_NUMCB_MASK 0xff
136 #define ID2_OAS_SHIFT 4
137 #define ID2_OAS_MASK 0xf
138 #define ID2_IAS_SHIFT 0
139 #define ID2_IAS_MASK 0xf
140 #define ID2_UBS_SHIFT 8
141 #define ID2_UBS_MASK 0xf
142 #define ID2_PTFS_4K (1 << 12)
143 #define ID2_PTFS_16K (1 << 13)
144 #define ID2_PTFS_64K (1 << 14)
145 #define ID2_VMID16 (1 << 15)
147 /* Global TLB invalidation */
148 #define ARM_SMMU_GR0_TLBIVMID 0x64
149 #define ARM_SMMU_GR0_TLBIALLNSNH 0x68
150 #define ARM_SMMU_GR0_TLBIALLH 0x6c
151 #define ARM_SMMU_GR0_sTLBGSYNC 0x70
152 #define ARM_SMMU_GR0_sTLBGSTATUS 0x74
153 #define sTLBGSTATUS_GSACTIVE (1 << 0)
154 #define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
156 /* Stream mapping registers */
157 #define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
158 #define SMR_VALID (1 << 31)
159 #define SMR_MASK_SHIFT 16
160 #define SMR_MASK_MASK 0x7fff
161 #define SMR_ID_SHIFT 0
162 #define SMR_ID_MASK 0x7fff
164 #define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
165 #define S2CR_CBNDX_SHIFT 0
166 #define S2CR_CBNDX_MASK 0xff
167 #define S2CR_TYPE_SHIFT 16
168 #define S2CR_TYPE_MASK 0x3
169 #define S2CR_TYPE_TRANS (0 << S2CR_TYPE_SHIFT)
170 #define S2CR_TYPE_BYPASS (1 << S2CR_TYPE_SHIFT)
171 #define S2CR_TYPE_FAULT (2 << S2CR_TYPE_SHIFT)
173 #define S2CR_PRIVCFG_SHIFT 24
174 #define S2CR_PRIVCFG_UNPRIV (2 << S2CR_PRIVCFG_SHIFT)
176 /* Context bank attribute registers */
177 #define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
178 #define CBAR_VMID_SHIFT 0
179 #define CBAR_VMID_MASK 0xff
180 #define CBAR_S1_BPSHCFG_SHIFT 8
181 #define CBAR_S1_BPSHCFG_MASK 3
182 #define CBAR_S1_BPSHCFG_NSH 3
183 #define CBAR_S1_MEMATTR_SHIFT 12
184 #define CBAR_S1_MEMATTR_MASK 0xf
185 #define CBAR_S1_MEMATTR_WB 0xf
186 #define CBAR_TYPE_SHIFT 16
187 #define CBAR_TYPE_MASK 0x3
188 #define CBAR_TYPE_S2_TRANS (0 << CBAR_TYPE_SHIFT)
189 #define CBAR_TYPE_S1_TRANS_S2_BYPASS (1 << CBAR_TYPE_SHIFT)
190 #define CBAR_TYPE_S1_TRANS_S2_FAULT (2 << CBAR_TYPE_SHIFT)
191 #define CBAR_TYPE_S1_TRANS_S2_TRANS (3 << CBAR_TYPE_SHIFT)
192 #define CBAR_IRPTNDX_SHIFT 24
193 #define CBAR_IRPTNDX_MASK 0xff
195 #define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
196 #define CBA2R_RW64_32BIT (0 << 0)
197 #define CBA2R_RW64_64BIT (1 << 0)
198 #define CBA2R_VMID_SHIFT 16
199 #define CBA2R_VMID_MASK 0xffff
201 /* Translation context bank */
202 #define ARM_SMMU_CB_BASE(smmu) ((smmu)->base + ((smmu)->size >> 1))
203 #define ARM_SMMU_CB(smmu, n) ((n) * (1 << (smmu)->pgshift))
205 #define ARM_SMMU_CB_SCTLR 0x0
206 #define ARM_SMMU_CB_RESUME 0x8
207 #define ARM_SMMU_CB_TTBCR2 0x10
208 #define ARM_SMMU_CB_TTBR0 0x20
209 #define ARM_SMMU_CB_TTBR1 0x28
210 #define ARM_SMMU_CB_TTBCR 0x30
211 #define ARM_SMMU_CB_S1_MAIR0 0x38
212 #define ARM_SMMU_CB_S1_MAIR1 0x3c
213 #define ARM_SMMU_CB_PAR_LO 0x50
214 #define ARM_SMMU_CB_PAR_HI 0x54
215 #define ARM_SMMU_CB_FSR 0x58
216 #define ARM_SMMU_CB_FAR_LO 0x60
217 #define ARM_SMMU_CB_FAR_HI 0x64
218 #define ARM_SMMU_CB_FSYNR0 0x68
219 #define ARM_SMMU_CB_S1_TLBIVA 0x600
220 #define ARM_SMMU_CB_S1_TLBIASID 0x610
221 #define ARM_SMMU_CB_S1_TLBIVAL 0x620
222 #define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
223 #define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
224 #define ARM_SMMU_CB_ATS1PR 0x800
225 #define ARM_SMMU_CB_ATSR 0x8f0
227 #define SCTLR_S1_ASIDPNE (1 << 12)
228 #define SCTLR_CFCFG (1 << 7)
229 #define SCTLR_CFIE (1 << 6)
230 #define SCTLR_CFRE (1 << 5)
231 #define SCTLR_E (1 << 4)
232 #define SCTLR_AFE (1 << 2)
233 #define SCTLR_TRE (1 << 1)
234 #define SCTLR_M (1 << 0)
235 #define SCTLR_EAE_SBOP (SCTLR_AFE | SCTLR_TRE)
237 #define CB_PAR_F (1 << 0)
239 #define ATSR_ACTIVE (1 << 0)
241 #define RESUME_RETRY (0 << 0)
242 #define RESUME_TERMINATE (1 << 0)
244 #define TTBCR2_SEP_SHIFT 15
245 #define TTBCR2_SEP_UPSTREAM (0x7 << TTBCR2_SEP_SHIFT)
247 #define TTBRn_ASID_SHIFT 48
249 #define FSR_MULTI (1 << 31)
250 #define FSR_SS (1 << 30)
251 #define FSR_UUT (1 << 8)
252 #define FSR_ASF (1 << 7)
253 #define FSR_TLBLKF (1 << 6)
254 #define FSR_TLBMCF (1 << 5)
255 #define FSR_EF (1 << 4)
256 #define FSR_PF (1 << 3)
257 #define FSR_AFF (1 << 2)
258 #define FSR_TF (1 << 1)
260 #define FSR_IGN (FSR_AFF | FSR_ASF | \
261 FSR_TLBMCF | FSR_TLBLKF)
262 #define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \
263 FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
265 #define FSYNR0_WNR (1 << 4)
267 static int force_stage
;
268 module_param(force_stage
, int, S_IRUGO
);
269 MODULE_PARM_DESC(force_stage
,
270 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
271 static bool disable_bypass
;
272 module_param(disable_bypass
, bool, S_IRUGO
);
273 MODULE_PARM_DESC(disable_bypass
,
274 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
276 enum arm_smmu_arch_version
{
281 enum arm_smmu_implementation
{
285 struct arm_smmu_smr
{
291 struct arm_smmu_master_cfg
{
293 u16 streamids
[MAX_MASTER_STREAMIDS
];
294 struct arm_smmu_smr
*smrs
;
297 struct arm_smmu_master
{
298 struct device_node
*of_node
;
300 struct arm_smmu_master_cfg cfg
;
303 struct arm_smmu_device
{
308 unsigned long pgshift
;
310 #define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0)
311 #define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1)
312 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 2)
313 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 3)
314 #define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4)
315 #define ARM_SMMU_FEAT_TRANS_OPS (1 << 5)
316 #define ARM_SMMU_FEAT_VMID16 (1 << 6)
319 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
321 enum arm_smmu_arch_version version
;
322 enum arm_smmu_implementation model
;
324 u32 num_context_banks
;
325 u32 num_s2_context_banks
;
326 DECLARE_BITMAP(context_map
, ARM_SMMU_MAX_CBS
);
329 u32 num_mapping_groups
;
330 DECLARE_BITMAP(smr_map
, ARM_SMMU_MAX_SMRS
);
332 unsigned long va_size
;
333 unsigned long ipa_size
;
334 unsigned long pa_size
;
337 u32 num_context_irqs
;
340 struct list_head list
;
341 struct rb_root masters
;
343 u32 cavium_id_base
; /* Specific to Cavium */
346 struct arm_smmu_cfg
{
351 #define INVALID_IRPTNDX 0xff
353 #define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
354 #define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
356 enum arm_smmu_domain_stage
{
357 ARM_SMMU_DOMAIN_S1
= 0,
359 ARM_SMMU_DOMAIN_NESTED
,
362 struct arm_smmu_domain
{
363 struct arm_smmu_device
*smmu
;
364 struct io_pgtable_ops
*pgtbl_ops
;
365 spinlock_t pgtbl_lock
;
366 struct arm_smmu_cfg cfg
;
367 enum arm_smmu_domain_stage stage
;
368 struct mutex init_mutex
; /* Protects smmu pointer */
369 struct iommu_domain domain
;
372 static struct iommu_ops arm_smmu_ops
;
374 static DEFINE_SPINLOCK(arm_smmu_devices_lock
);
375 static LIST_HEAD(arm_smmu_devices
);
377 struct arm_smmu_option_prop
{
382 static atomic_t cavium_smmu_context_count
= ATOMIC_INIT(0);
384 static struct arm_smmu_option_prop arm_smmu_options
[] = {
385 { ARM_SMMU_OPT_SECURE_CFG_ACCESS
, "calxeda,smmu-secure-config-access" },
389 static struct arm_smmu_domain
*to_smmu_domain(struct iommu_domain
*dom
)
391 return container_of(dom
, struct arm_smmu_domain
, domain
);
394 static void parse_driver_options(struct arm_smmu_device
*smmu
)
399 if (of_property_read_bool(smmu
->dev
->of_node
,
400 arm_smmu_options
[i
].prop
)) {
401 smmu
->options
|= arm_smmu_options
[i
].opt
;
402 dev_notice(smmu
->dev
, "option %s\n",
403 arm_smmu_options
[i
].prop
);
405 } while (arm_smmu_options
[++i
].opt
);
408 static struct device_node
*dev_get_dev_node(struct device
*dev
)
410 if (dev_is_pci(dev
)) {
411 struct pci_bus
*bus
= to_pci_dev(dev
)->bus
;
413 while (!pci_is_root_bus(bus
))
415 return bus
->bridge
->parent
->of_node
;
421 static struct arm_smmu_master
*find_smmu_master(struct arm_smmu_device
*smmu
,
422 struct device_node
*dev_node
)
424 struct rb_node
*node
= smmu
->masters
.rb_node
;
427 struct arm_smmu_master
*master
;
429 master
= container_of(node
, struct arm_smmu_master
, node
);
431 if (dev_node
< master
->of_node
)
432 node
= node
->rb_left
;
433 else if (dev_node
> master
->of_node
)
434 node
= node
->rb_right
;
442 static struct arm_smmu_master_cfg
*
443 find_smmu_master_cfg(struct device
*dev
)
445 struct arm_smmu_master_cfg
*cfg
= NULL
;
446 struct iommu_group
*group
= iommu_group_get(dev
);
449 cfg
= iommu_group_get_iommudata(group
);
450 iommu_group_put(group
);
456 static int insert_smmu_master(struct arm_smmu_device
*smmu
,
457 struct arm_smmu_master
*master
)
459 struct rb_node
**new, *parent
;
461 new = &smmu
->masters
.rb_node
;
464 struct arm_smmu_master
*this
465 = container_of(*new, struct arm_smmu_master
, node
);
468 if (master
->of_node
< this->of_node
)
469 new = &((*new)->rb_left
);
470 else if (master
->of_node
> this->of_node
)
471 new = &((*new)->rb_right
);
476 rb_link_node(&master
->node
, parent
, new);
477 rb_insert_color(&master
->node
, &smmu
->masters
);
481 static int register_smmu_master(struct arm_smmu_device
*smmu
,
483 struct of_phandle_args
*masterspec
)
486 struct arm_smmu_master
*master
;
488 master
= find_smmu_master(smmu
, masterspec
->np
);
491 "rejecting multiple registrations for master device %s\n",
492 masterspec
->np
->name
);
496 if (masterspec
->args_count
> MAX_MASTER_STREAMIDS
) {
498 "reached maximum number (%d) of stream IDs for master device %s\n",
499 MAX_MASTER_STREAMIDS
, masterspec
->np
->name
);
503 master
= devm_kzalloc(dev
, sizeof(*master
), GFP_KERNEL
);
507 master
->of_node
= masterspec
->np
;
508 master
->cfg
.num_streamids
= masterspec
->args_count
;
510 for (i
= 0; i
< master
->cfg
.num_streamids
; ++i
) {
511 u16 streamid
= masterspec
->args
[i
];
513 if (!(smmu
->features
& ARM_SMMU_FEAT_STREAM_MATCH
) &&
514 (streamid
>= smmu
->num_mapping_groups
)) {
516 "stream ID for master device %s greater than maximum allowed (%d)\n",
517 masterspec
->np
->name
, smmu
->num_mapping_groups
);
520 master
->cfg
.streamids
[i
] = streamid
;
522 return insert_smmu_master(smmu
, master
);
525 static struct arm_smmu_device
*find_smmu_for_device(struct device
*dev
)
527 struct arm_smmu_device
*smmu
;
528 struct arm_smmu_master
*master
= NULL
;
529 struct device_node
*dev_node
= dev_get_dev_node(dev
);
531 spin_lock(&arm_smmu_devices_lock
);
532 list_for_each_entry(smmu
, &arm_smmu_devices
, list
) {
533 master
= find_smmu_master(smmu
, dev_node
);
537 spin_unlock(&arm_smmu_devices_lock
);
539 return master
? smmu
: NULL
;
542 static int __arm_smmu_alloc_bitmap(unsigned long *map
, int start
, int end
)
547 idx
= find_next_zero_bit(map
, end
, start
);
550 } while (test_and_set_bit(idx
, map
));
555 static void __arm_smmu_free_bitmap(unsigned long *map
, int idx
)
560 /* Wait for any pending TLB invalidations to complete */
561 static void __arm_smmu_tlb_sync(struct arm_smmu_device
*smmu
)
564 void __iomem
*gr0_base
= ARM_SMMU_GR0(smmu
);
566 writel_relaxed(0, gr0_base
+ ARM_SMMU_GR0_sTLBGSYNC
);
567 while (readl_relaxed(gr0_base
+ ARM_SMMU_GR0_sTLBGSTATUS
)
568 & sTLBGSTATUS_GSACTIVE
) {
570 if (++count
== TLB_LOOP_TIMEOUT
) {
571 dev_err_ratelimited(smmu
->dev
,
572 "TLB sync timed out -- SMMU may be deadlocked\n");
579 static void arm_smmu_tlb_sync(void *cookie
)
581 struct arm_smmu_domain
*smmu_domain
= cookie
;
582 __arm_smmu_tlb_sync(smmu_domain
->smmu
);
585 static void arm_smmu_tlb_inv_context(void *cookie
)
587 struct arm_smmu_domain
*smmu_domain
= cookie
;
588 struct arm_smmu_cfg
*cfg
= &smmu_domain
->cfg
;
589 struct arm_smmu_device
*smmu
= smmu_domain
->smmu
;
590 bool stage1
= cfg
->cbar
!= CBAR_TYPE_S2_TRANS
;
594 base
= ARM_SMMU_CB_BASE(smmu
) + ARM_SMMU_CB(smmu
, cfg
->cbndx
);
595 writel_relaxed(ARM_SMMU_CB_ASID(smmu
, cfg
),
596 base
+ ARM_SMMU_CB_S1_TLBIASID
);
598 base
= ARM_SMMU_GR0(smmu
);
599 writel_relaxed(ARM_SMMU_CB_VMID(smmu
, cfg
),
600 base
+ ARM_SMMU_GR0_TLBIVMID
);
603 __arm_smmu_tlb_sync(smmu
);
606 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova
, size_t size
,
607 size_t granule
, bool leaf
, void *cookie
)
609 struct arm_smmu_domain
*smmu_domain
= cookie
;
610 struct arm_smmu_cfg
*cfg
= &smmu_domain
->cfg
;
611 struct arm_smmu_device
*smmu
= smmu_domain
->smmu
;
612 bool stage1
= cfg
->cbar
!= CBAR_TYPE_S2_TRANS
;
616 reg
= ARM_SMMU_CB_BASE(smmu
) + ARM_SMMU_CB(smmu
, cfg
->cbndx
);
617 reg
+= leaf
? ARM_SMMU_CB_S1_TLBIVAL
: ARM_SMMU_CB_S1_TLBIVA
;
619 if (!IS_ENABLED(CONFIG_64BIT
) || smmu
->version
== ARM_SMMU_V1
) {
621 iova
|= ARM_SMMU_CB_ASID(smmu
, cfg
);
623 writel_relaxed(iova
, reg
);
625 } while (size
-= granule
);
629 iova
|= (u64
)ARM_SMMU_CB_ASID(smmu
, cfg
) << 48;
631 writeq_relaxed(iova
, reg
);
632 iova
+= granule
>> 12;
633 } while (size
-= granule
);
637 } else if (smmu
->version
== ARM_SMMU_V2
) {
638 reg
= ARM_SMMU_CB_BASE(smmu
) + ARM_SMMU_CB(smmu
, cfg
->cbndx
);
639 reg
+= leaf
? ARM_SMMU_CB_S2_TLBIIPAS2L
:
640 ARM_SMMU_CB_S2_TLBIIPAS2
;
643 writeq_relaxed(iova
, reg
);
644 iova
+= granule
>> 12;
645 } while (size
-= granule
);
648 reg
= ARM_SMMU_GR0(smmu
) + ARM_SMMU_GR0_TLBIVMID
;
649 writel_relaxed(ARM_SMMU_CB_VMID(smmu
, cfg
), reg
);
653 static struct iommu_gather_ops arm_smmu_gather_ops
= {
654 .tlb_flush_all
= arm_smmu_tlb_inv_context
,
655 .tlb_add_flush
= arm_smmu_tlb_inv_range_nosync
,
656 .tlb_sync
= arm_smmu_tlb_sync
,
659 static irqreturn_t
arm_smmu_context_fault(int irq
, void *dev
)
662 u32 fsr
, far
, fsynr
, resume
;
664 struct iommu_domain
*domain
= dev
;
665 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
666 struct arm_smmu_cfg
*cfg
= &smmu_domain
->cfg
;
667 struct arm_smmu_device
*smmu
= smmu_domain
->smmu
;
668 void __iomem
*cb_base
;
670 cb_base
= ARM_SMMU_CB_BASE(smmu
) + ARM_SMMU_CB(smmu
, cfg
->cbndx
);
671 fsr
= readl_relaxed(cb_base
+ ARM_SMMU_CB_FSR
);
673 if (!(fsr
& FSR_FAULT
))
677 dev_err_ratelimited(smmu
->dev
,
678 "Unexpected context fault (fsr 0x%x)\n",
681 fsynr
= readl_relaxed(cb_base
+ ARM_SMMU_CB_FSYNR0
);
682 flags
= fsynr
& FSYNR0_WNR
? IOMMU_FAULT_WRITE
: IOMMU_FAULT_READ
;
684 far
= readl_relaxed(cb_base
+ ARM_SMMU_CB_FAR_LO
);
687 far
= readl_relaxed(cb_base
+ ARM_SMMU_CB_FAR_HI
);
688 iova
|= ((unsigned long)far
<< 32);
691 if (!report_iommu_fault(domain
, smmu
->dev
, iova
, flags
)) {
693 resume
= RESUME_RETRY
;
695 dev_err_ratelimited(smmu
->dev
,
696 "Unhandled context fault: iova=0x%08lx, fsynr=0x%x, cb=%d\n",
697 iova
, fsynr
, cfg
->cbndx
);
699 resume
= RESUME_TERMINATE
;
702 /* Clear the faulting FSR */
703 writel(fsr
, cb_base
+ ARM_SMMU_CB_FSR
);
705 /* Retry or terminate any stalled transactions */
707 writel_relaxed(resume
, cb_base
+ ARM_SMMU_CB_RESUME
);
712 static irqreturn_t
arm_smmu_global_fault(int irq
, void *dev
)
714 u32 gfsr
, gfsynr0
, gfsynr1
, gfsynr2
;
715 struct arm_smmu_device
*smmu
= dev
;
716 void __iomem
*gr0_base
= ARM_SMMU_GR0_NS(smmu
);
718 gfsr
= readl_relaxed(gr0_base
+ ARM_SMMU_GR0_sGFSR
);
719 gfsynr0
= readl_relaxed(gr0_base
+ ARM_SMMU_GR0_sGFSYNR0
);
720 gfsynr1
= readl_relaxed(gr0_base
+ ARM_SMMU_GR0_sGFSYNR1
);
721 gfsynr2
= readl_relaxed(gr0_base
+ ARM_SMMU_GR0_sGFSYNR2
);
726 dev_err_ratelimited(smmu
->dev
,
727 "Unexpected global fault, this could be serious\n");
728 dev_err_ratelimited(smmu
->dev
,
729 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
730 gfsr
, gfsynr0
, gfsynr1
, gfsynr2
);
732 writel(gfsr
, gr0_base
+ ARM_SMMU_GR0_sGFSR
);
736 static void arm_smmu_init_context_bank(struct arm_smmu_domain
*smmu_domain
,
737 struct io_pgtable_cfg
*pgtbl_cfg
)
742 struct arm_smmu_cfg
*cfg
= &smmu_domain
->cfg
;
743 struct arm_smmu_device
*smmu
= smmu_domain
->smmu
;
744 void __iomem
*cb_base
, *gr1_base
;
746 gr1_base
= ARM_SMMU_GR1(smmu
);
747 stage1
= cfg
->cbar
!= CBAR_TYPE_S2_TRANS
;
748 cb_base
= ARM_SMMU_CB_BASE(smmu
) + ARM_SMMU_CB(smmu
, cfg
->cbndx
);
750 if (smmu
->version
> ARM_SMMU_V1
) {
752 reg
= CBA2R_RW64_64BIT
;
754 reg
= CBA2R_RW64_32BIT
;
756 /* 16-bit VMIDs live in CBA2R */
757 if (smmu
->features
& ARM_SMMU_FEAT_VMID16
)
758 reg
|= ARM_SMMU_CB_VMID(smmu
, cfg
) << CBA2R_VMID_SHIFT
;
760 writel_relaxed(reg
, gr1_base
+ ARM_SMMU_GR1_CBA2R(cfg
->cbndx
));
765 if (smmu
->version
== ARM_SMMU_V1
)
766 reg
|= cfg
->irptndx
<< CBAR_IRPTNDX_SHIFT
;
769 * Use the weakest shareability/memory types, so they are
770 * overridden by the ttbcr/pte.
773 reg
|= (CBAR_S1_BPSHCFG_NSH
<< CBAR_S1_BPSHCFG_SHIFT
) |
774 (CBAR_S1_MEMATTR_WB
<< CBAR_S1_MEMATTR_SHIFT
);
775 } else if (!(smmu
->features
& ARM_SMMU_FEAT_VMID16
)) {
776 /* 8-bit VMIDs live in CBAR */
777 reg
|= ARM_SMMU_CB_VMID(smmu
, cfg
) << CBAR_VMID_SHIFT
;
779 writel_relaxed(reg
, gr1_base
+ ARM_SMMU_GR1_CBAR(cfg
->cbndx
));
783 reg64
= pgtbl_cfg
->arm_lpae_s1_cfg
.ttbr
[0];
785 reg64
|= ((u64
)ARM_SMMU_CB_ASID(smmu
, cfg
)) << TTBRn_ASID_SHIFT
;
786 smmu_writeq(reg64
, cb_base
+ ARM_SMMU_CB_TTBR0
);
788 reg64
= pgtbl_cfg
->arm_lpae_s1_cfg
.ttbr
[1];
789 reg64
|= ((u64
)ARM_SMMU_CB_ASID(smmu
, cfg
)) << TTBRn_ASID_SHIFT
;
790 smmu_writeq(reg64
, cb_base
+ ARM_SMMU_CB_TTBR1
);
792 reg64
= pgtbl_cfg
->arm_lpae_s2_cfg
.vttbr
;
793 smmu_writeq(reg64
, cb_base
+ ARM_SMMU_CB_TTBR0
);
798 reg
= pgtbl_cfg
->arm_lpae_s1_cfg
.tcr
;
799 writel_relaxed(reg
, cb_base
+ ARM_SMMU_CB_TTBCR
);
800 if (smmu
->version
> ARM_SMMU_V1
) {
801 reg
= pgtbl_cfg
->arm_lpae_s1_cfg
.tcr
>> 32;
802 reg
|= TTBCR2_SEP_UPSTREAM
;
803 writel_relaxed(reg
, cb_base
+ ARM_SMMU_CB_TTBCR2
);
806 reg
= pgtbl_cfg
->arm_lpae_s2_cfg
.vtcr
;
807 writel_relaxed(reg
, cb_base
+ ARM_SMMU_CB_TTBCR
);
810 /* MAIRs (stage-1 only) */
812 reg
= pgtbl_cfg
->arm_lpae_s1_cfg
.mair
[0];
813 writel_relaxed(reg
, cb_base
+ ARM_SMMU_CB_S1_MAIR0
);
814 reg
= pgtbl_cfg
->arm_lpae_s1_cfg
.mair
[1];
815 writel_relaxed(reg
, cb_base
+ ARM_SMMU_CB_S1_MAIR1
);
819 reg
= SCTLR_CFCFG
| SCTLR_CFIE
| SCTLR_CFRE
| SCTLR_M
| SCTLR_EAE_SBOP
;
821 reg
|= SCTLR_S1_ASIDPNE
;
825 writel_relaxed(reg
, cb_base
+ ARM_SMMU_CB_SCTLR
);
828 static int arm_smmu_init_domain_context(struct iommu_domain
*domain
,
829 struct arm_smmu_device
*smmu
)
831 int irq
, start
, ret
= 0;
832 unsigned long ias
, oas
;
833 struct io_pgtable_ops
*pgtbl_ops
;
834 struct io_pgtable_cfg pgtbl_cfg
;
835 enum io_pgtable_fmt fmt
;
836 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
837 struct arm_smmu_cfg
*cfg
= &smmu_domain
->cfg
;
839 mutex_lock(&smmu_domain
->init_mutex
);
840 if (smmu_domain
->smmu
)
844 * Mapping the requested stage onto what we support is surprisingly
845 * complicated, mainly because the spec allows S1+S2 SMMUs without
846 * support for nested translation. That means we end up with the
849 * Requested Supported Actual
859 * Note that you can't actually request stage-2 mappings.
861 if (!(smmu
->features
& ARM_SMMU_FEAT_TRANS_S1
))
862 smmu_domain
->stage
= ARM_SMMU_DOMAIN_S2
;
863 if (!(smmu
->features
& ARM_SMMU_FEAT_TRANS_S2
))
864 smmu_domain
->stage
= ARM_SMMU_DOMAIN_S1
;
866 switch (smmu_domain
->stage
) {
867 case ARM_SMMU_DOMAIN_S1
:
868 cfg
->cbar
= CBAR_TYPE_S1_TRANS_S2_BYPASS
;
869 start
= smmu
->num_s2_context_banks
;
871 oas
= smmu
->ipa_size
;
872 if (IS_ENABLED(CONFIG_64BIT
))
873 fmt
= ARM_64_LPAE_S1
;
875 fmt
= ARM_32_LPAE_S1
;
877 case ARM_SMMU_DOMAIN_NESTED
:
879 * We will likely want to change this if/when KVM gets
882 case ARM_SMMU_DOMAIN_S2
:
883 cfg
->cbar
= CBAR_TYPE_S2_TRANS
;
885 ias
= smmu
->ipa_size
;
887 if (IS_ENABLED(CONFIG_64BIT
))
888 fmt
= ARM_64_LPAE_S2
;
890 fmt
= ARM_32_LPAE_S2
;
897 ret
= __arm_smmu_alloc_bitmap(smmu
->context_map
, start
,
898 smmu
->num_context_banks
);
899 if (IS_ERR_VALUE(ret
))
903 if (smmu
->version
== ARM_SMMU_V1
) {
904 cfg
->irptndx
= atomic_inc_return(&smmu
->irptndx
);
905 cfg
->irptndx
%= smmu
->num_context_irqs
;
907 cfg
->irptndx
= cfg
->cbndx
;
910 pgtbl_cfg
= (struct io_pgtable_cfg
) {
911 .pgsize_bitmap
= arm_smmu_ops
.pgsize_bitmap
,
914 .tlb
= &arm_smmu_gather_ops
,
915 .iommu_dev
= smmu
->dev
,
918 smmu_domain
->smmu
= smmu
;
919 pgtbl_ops
= alloc_io_pgtable_ops(fmt
, &pgtbl_cfg
, smmu_domain
);
925 /* Update our support page sizes to reflect the page table format */
926 arm_smmu_ops
.pgsize_bitmap
= pgtbl_cfg
.pgsize_bitmap
;
928 /* Initialise the context bank with our page table cfg */
929 arm_smmu_init_context_bank(smmu_domain
, &pgtbl_cfg
);
932 * Request context fault interrupt. Do this last to avoid the
933 * handler seeing a half-initialised domain state.
935 irq
= smmu
->irqs
[smmu
->num_global_irqs
+ cfg
->irptndx
];
936 ret
= request_irq(irq
, arm_smmu_context_fault
, IRQF_SHARED
,
937 "arm-smmu-context-fault", domain
);
938 if (IS_ERR_VALUE(ret
)) {
939 dev_err(smmu
->dev
, "failed to request context IRQ %d (%u)\n",
941 cfg
->irptndx
= INVALID_IRPTNDX
;
944 mutex_unlock(&smmu_domain
->init_mutex
);
946 /* Publish page table ops for map/unmap */
947 smmu_domain
->pgtbl_ops
= pgtbl_ops
;
951 smmu_domain
->smmu
= NULL
;
953 mutex_unlock(&smmu_domain
->init_mutex
);
957 static void arm_smmu_destroy_domain_context(struct iommu_domain
*domain
)
959 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
960 struct arm_smmu_device
*smmu
= smmu_domain
->smmu
;
961 struct arm_smmu_cfg
*cfg
= &smmu_domain
->cfg
;
962 void __iomem
*cb_base
;
969 * Disable the context bank and free the page tables before freeing
972 cb_base
= ARM_SMMU_CB_BASE(smmu
) + ARM_SMMU_CB(smmu
, cfg
->cbndx
);
973 writel_relaxed(0, cb_base
+ ARM_SMMU_CB_SCTLR
);
975 if (cfg
->irptndx
!= INVALID_IRPTNDX
) {
976 irq
= smmu
->irqs
[smmu
->num_global_irqs
+ cfg
->irptndx
];
977 free_irq(irq
, domain
);
980 free_io_pgtable_ops(smmu_domain
->pgtbl_ops
);
981 __arm_smmu_free_bitmap(smmu
->context_map
, cfg
->cbndx
);
984 static struct iommu_domain
*arm_smmu_domain_alloc(unsigned type
)
986 struct arm_smmu_domain
*smmu_domain
;
988 if (type
!= IOMMU_DOMAIN_UNMANAGED
&& type
!= IOMMU_DOMAIN_DMA
)
991 * Allocate the domain and initialise some of its data structures.
992 * We can't really do anything meaningful until we've added a
995 smmu_domain
= kzalloc(sizeof(*smmu_domain
), GFP_KERNEL
);
999 if (type
== IOMMU_DOMAIN_DMA
&&
1000 iommu_get_dma_cookie(&smmu_domain
->domain
)) {
1005 mutex_init(&smmu_domain
->init_mutex
);
1006 spin_lock_init(&smmu_domain
->pgtbl_lock
);
1008 return &smmu_domain
->domain
;
1011 static void arm_smmu_domain_free(struct iommu_domain
*domain
)
1013 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1016 * Free the domain resources. We assume that all devices have
1017 * already been detached.
1019 iommu_put_dma_cookie(domain
);
1020 arm_smmu_destroy_domain_context(domain
);
1024 static int arm_smmu_master_configure_smrs(struct arm_smmu_device
*smmu
,
1025 struct arm_smmu_master_cfg
*cfg
)
1028 struct arm_smmu_smr
*smrs
;
1029 void __iomem
*gr0_base
= ARM_SMMU_GR0(smmu
);
1031 if (!(smmu
->features
& ARM_SMMU_FEAT_STREAM_MATCH
))
1037 smrs
= kmalloc_array(cfg
->num_streamids
, sizeof(*smrs
), GFP_KERNEL
);
1039 dev_err(smmu
->dev
, "failed to allocate %d SMRs\n",
1040 cfg
->num_streamids
);
1044 /* Allocate the SMRs on the SMMU */
1045 for (i
= 0; i
< cfg
->num_streamids
; ++i
) {
1046 int idx
= __arm_smmu_alloc_bitmap(smmu
->smr_map
, 0,
1047 smmu
->num_mapping_groups
);
1048 if (IS_ERR_VALUE(idx
)) {
1049 dev_err(smmu
->dev
, "failed to allocate free SMR\n");
1053 smrs
[i
] = (struct arm_smmu_smr
) {
1055 .mask
= 0, /* We don't currently share SMRs */
1056 .id
= cfg
->streamids
[i
],
1060 /* It worked! Now, poke the actual hardware */
1061 for (i
= 0; i
< cfg
->num_streamids
; ++i
) {
1062 u32 reg
= SMR_VALID
| smrs
[i
].id
<< SMR_ID_SHIFT
|
1063 smrs
[i
].mask
<< SMR_MASK_SHIFT
;
1064 writel_relaxed(reg
, gr0_base
+ ARM_SMMU_GR0_SMR(smrs
[i
].idx
));
1072 __arm_smmu_free_bitmap(smmu
->smr_map
, smrs
[i
].idx
);
1077 static void arm_smmu_master_free_smrs(struct arm_smmu_device
*smmu
,
1078 struct arm_smmu_master_cfg
*cfg
)
1081 void __iomem
*gr0_base
= ARM_SMMU_GR0(smmu
);
1082 struct arm_smmu_smr
*smrs
= cfg
->smrs
;
1087 /* Invalidate the SMRs before freeing back to the allocator */
1088 for (i
= 0; i
< cfg
->num_streamids
; ++i
) {
1089 u8 idx
= smrs
[i
].idx
;
1091 writel_relaxed(~SMR_VALID
, gr0_base
+ ARM_SMMU_GR0_SMR(idx
));
1092 __arm_smmu_free_bitmap(smmu
->smr_map
, idx
);
1099 static int arm_smmu_domain_add_master(struct arm_smmu_domain
*smmu_domain
,
1100 struct arm_smmu_master_cfg
*cfg
)
1103 struct arm_smmu_device
*smmu
= smmu_domain
->smmu
;
1104 void __iomem
*gr0_base
= ARM_SMMU_GR0(smmu
);
1106 /* Devices in an IOMMU group may already be configured */
1107 ret
= arm_smmu_master_configure_smrs(smmu
, cfg
);
1109 return ret
== -EEXIST
? 0 : ret
;
1112 * FIXME: This won't be needed once we have IOMMU-backed DMA ops
1113 * for all devices behind the SMMU.
1115 if (smmu_domain
->domain
.type
== IOMMU_DOMAIN_DMA
)
1118 for (i
= 0; i
< cfg
->num_streamids
; ++i
) {
1121 idx
= cfg
->smrs
? cfg
->smrs
[i
].idx
: cfg
->streamids
[i
];
1122 s2cr
= S2CR_TYPE_TRANS
| S2CR_PRIVCFG_UNPRIV
|
1123 (smmu_domain
->cfg
.cbndx
<< S2CR_CBNDX_SHIFT
);
1124 writel_relaxed(s2cr
, gr0_base
+ ARM_SMMU_GR0_S2CR(idx
));
1130 static void arm_smmu_domain_remove_master(struct arm_smmu_domain
*smmu_domain
,
1131 struct arm_smmu_master_cfg
*cfg
)
1134 struct arm_smmu_device
*smmu
= smmu_domain
->smmu
;
1135 void __iomem
*gr0_base
= ARM_SMMU_GR0(smmu
);
1137 /* An IOMMU group is torn down by the first device to be removed */
1138 if ((smmu
->features
& ARM_SMMU_FEAT_STREAM_MATCH
) && !cfg
->smrs
)
1142 * We *must* clear the S2CR first, because freeing the SMR means
1143 * that it can be re-allocated immediately.
1145 for (i
= 0; i
< cfg
->num_streamids
; ++i
) {
1146 u32 idx
= cfg
->smrs
? cfg
->smrs
[i
].idx
: cfg
->streamids
[i
];
1147 u32 reg
= disable_bypass
? S2CR_TYPE_FAULT
: S2CR_TYPE_BYPASS
;
1149 writel_relaxed(reg
, gr0_base
+ ARM_SMMU_GR0_S2CR(idx
));
1152 arm_smmu_master_free_smrs(smmu
, cfg
);
1155 static void arm_smmu_detach_dev(struct device
*dev
,
1156 struct arm_smmu_master_cfg
*cfg
)
1158 struct iommu_domain
*domain
= dev
->archdata
.iommu
;
1159 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1161 dev
->archdata
.iommu
= NULL
;
1162 arm_smmu_domain_remove_master(smmu_domain
, cfg
);
1165 static int arm_smmu_attach_dev(struct iommu_domain
*domain
, struct device
*dev
)
1168 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1169 struct arm_smmu_device
*smmu
;
1170 struct arm_smmu_master_cfg
*cfg
;
1172 smmu
= find_smmu_for_device(dev
);
1174 dev_err(dev
, "cannot attach to SMMU, is it on the same bus?\n");
1178 /* Ensure that the domain is finalised */
1179 ret
= arm_smmu_init_domain_context(domain
, smmu
);
1180 if (IS_ERR_VALUE(ret
))
1184 * Sanity check the domain. We don't support domains across
1187 if (smmu_domain
->smmu
!= smmu
) {
1189 "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1190 dev_name(smmu_domain
->smmu
->dev
), dev_name(smmu
->dev
));
1194 /* Looks ok, so add the device to the domain */
1195 cfg
= find_smmu_master_cfg(dev
);
1199 /* Detach the dev from its current domain */
1200 if (dev
->archdata
.iommu
)
1201 arm_smmu_detach_dev(dev
, cfg
);
1203 ret
= arm_smmu_domain_add_master(smmu_domain
, cfg
);
1205 dev
->archdata
.iommu
= domain
;
1209 static int arm_smmu_map(struct iommu_domain
*domain
, unsigned long iova
,
1210 phys_addr_t paddr
, size_t size
, int prot
)
1213 unsigned long flags
;
1214 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1215 struct io_pgtable_ops
*ops
= smmu_domain
->pgtbl_ops
;
1220 spin_lock_irqsave(&smmu_domain
->pgtbl_lock
, flags
);
1221 ret
= ops
->map(ops
, iova
, paddr
, size
, prot
);
1222 spin_unlock_irqrestore(&smmu_domain
->pgtbl_lock
, flags
);
1226 static size_t arm_smmu_unmap(struct iommu_domain
*domain
, unsigned long iova
,
1230 unsigned long flags
;
1231 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1232 struct io_pgtable_ops
*ops
= smmu_domain
->pgtbl_ops
;
1237 spin_lock_irqsave(&smmu_domain
->pgtbl_lock
, flags
);
1238 ret
= ops
->unmap(ops
, iova
, size
);
1239 spin_unlock_irqrestore(&smmu_domain
->pgtbl_lock
, flags
);
1243 static phys_addr_t
arm_smmu_iova_to_phys_hard(struct iommu_domain
*domain
,
1246 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1247 struct arm_smmu_device
*smmu
= smmu_domain
->smmu
;
1248 struct arm_smmu_cfg
*cfg
= &smmu_domain
->cfg
;
1249 struct io_pgtable_ops
*ops
= smmu_domain
->pgtbl_ops
;
1250 struct device
*dev
= smmu
->dev
;
1251 void __iomem
*cb_base
;
1256 cb_base
= ARM_SMMU_CB_BASE(smmu
) + ARM_SMMU_CB(smmu
, cfg
->cbndx
);
1258 /* ATS1 registers can only be written atomically */
1259 va
= iova
& ~0xfffUL
;
1260 if (smmu
->version
== ARM_SMMU_V2
)
1261 smmu_writeq(va
, cb_base
+ ARM_SMMU_CB_ATS1PR
);
1263 writel_relaxed(va
, cb_base
+ ARM_SMMU_CB_ATS1PR
);
1265 if (readl_poll_timeout_atomic(cb_base
+ ARM_SMMU_CB_ATSR
, tmp
,
1266 !(tmp
& ATSR_ACTIVE
), 5, 50)) {
1268 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1270 return ops
->iova_to_phys(ops
, iova
);
1273 phys
= readl_relaxed(cb_base
+ ARM_SMMU_CB_PAR_LO
);
1274 phys
|= ((u64
)readl_relaxed(cb_base
+ ARM_SMMU_CB_PAR_HI
)) << 32;
1276 if (phys
& CB_PAR_F
) {
1277 dev_err(dev
, "translation fault!\n");
1278 dev_err(dev
, "PAR = 0x%llx\n", phys
);
1282 return (phys
& GENMASK_ULL(39, 12)) | (iova
& 0xfff);
1285 static phys_addr_t
arm_smmu_iova_to_phys(struct iommu_domain
*domain
,
1289 unsigned long flags
;
1290 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1291 struct io_pgtable_ops
*ops
= smmu_domain
->pgtbl_ops
;
1296 spin_lock_irqsave(&smmu_domain
->pgtbl_lock
, flags
);
1297 if (smmu_domain
->smmu
->features
& ARM_SMMU_FEAT_TRANS_OPS
&&
1298 smmu_domain
->stage
== ARM_SMMU_DOMAIN_S1
) {
1299 ret
= arm_smmu_iova_to_phys_hard(domain
, iova
);
1301 ret
= ops
->iova_to_phys(ops
, iova
);
1304 spin_unlock_irqrestore(&smmu_domain
->pgtbl_lock
, flags
);
1309 static bool arm_smmu_capable(enum iommu_cap cap
)
1312 case IOMMU_CAP_CACHE_COHERENCY
:
1314 * Return true here as the SMMU can always send out coherent
1318 case IOMMU_CAP_INTR_REMAP
:
1319 return true; /* MSIs are just memory writes */
1320 case IOMMU_CAP_NOEXEC
:
1327 static int __arm_smmu_get_pci_sid(struct pci_dev
*pdev
, u16 alias
, void *data
)
1329 *((u16
*)data
) = alias
;
1330 return 0; /* Continue walking */
1333 static void __arm_smmu_release_pci_iommudata(void *data
)
1338 static int arm_smmu_init_pci_device(struct pci_dev
*pdev
,
1339 struct iommu_group
*group
)
1341 struct arm_smmu_master_cfg
*cfg
;
1345 cfg
= iommu_group_get_iommudata(group
);
1347 cfg
= kzalloc(sizeof(*cfg
), GFP_KERNEL
);
1351 iommu_group_set_iommudata(group
, cfg
,
1352 __arm_smmu_release_pci_iommudata
);
1355 if (cfg
->num_streamids
>= MAX_MASTER_STREAMIDS
)
1359 * Assume Stream ID == Requester ID for now.
1360 * We need a way to describe the ID mappings in FDT.
1362 pci_for_each_dma_alias(pdev
, __arm_smmu_get_pci_sid
, &sid
);
1363 for (i
= 0; i
< cfg
->num_streamids
; ++i
)
1364 if (cfg
->streamids
[i
] == sid
)
1367 /* Avoid duplicate SIDs, as this can lead to SMR conflicts */
1368 if (i
== cfg
->num_streamids
)
1369 cfg
->streamids
[cfg
->num_streamids
++] = sid
;
1374 static int arm_smmu_init_platform_device(struct device
*dev
,
1375 struct iommu_group
*group
)
1377 struct arm_smmu_device
*smmu
= find_smmu_for_device(dev
);
1378 struct arm_smmu_master
*master
;
1383 master
= find_smmu_master(smmu
, dev
->of_node
);
1387 iommu_group_set_iommudata(group
, &master
->cfg
, NULL
);
1392 static int arm_smmu_add_device(struct device
*dev
)
1394 struct iommu_group
*group
;
1396 group
= iommu_group_get_for_dev(dev
);
1398 return PTR_ERR(group
);
1400 iommu_group_put(group
);
1404 static void arm_smmu_remove_device(struct device
*dev
)
1406 iommu_group_remove_device(dev
);
1409 static struct iommu_group
*arm_smmu_device_group(struct device
*dev
)
1411 struct iommu_group
*group
;
1414 if (dev_is_pci(dev
))
1415 group
= pci_device_group(dev
);
1417 group
= generic_device_group(dev
);
1422 if (dev_is_pci(dev
))
1423 ret
= arm_smmu_init_pci_device(to_pci_dev(dev
), group
);
1425 ret
= arm_smmu_init_platform_device(dev
, group
);
1428 iommu_group_put(group
);
1429 group
= ERR_PTR(ret
);
1435 static int arm_smmu_domain_get_attr(struct iommu_domain
*domain
,
1436 enum iommu_attr attr
, void *data
)
1438 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1441 case DOMAIN_ATTR_NESTING
:
1442 *(int *)data
= (smmu_domain
->stage
== ARM_SMMU_DOMAIN_NESTED
);
1449 static int arm_smmu_domain_set_attr(struct iommu_domain
*domain
,
1450 enum iommu_attr attr
, void *data
)
1453 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1455 mutex_lock(&smmu_domain
->init_mutex
);
1458 case DOMAIN_ATTR_NESTING
:
1459 if (smmu_domain
->smmu
) {
1465 smmu_domain
->stage
= ARM_SMMU_DOMAIN_NESTED
;
1467 smmu_domain
->stage
= ARM_SMMU_DOMAIN_S1
;
1475 mutex_unlock(&smmu_domain
->init_mutex
);
1479 static struct iommu_ops arm_smmu_ops
= {
1480 .capable
= arm_smmu_capable
,
1481 .domain_alloc
= arm_smmu_domain_alloc
,
1482 .domain_free
= arm_smmu_domain_free
,
1483 .attach_dev
= arm_smmu_attach_dev
,
1484 .map
= arm_smmu_map
,
1485 .unmap
= arm_smmu_unmap
,
1486 .map_sg
= default_iommu_map_sg
,
1487 .iova_to_phys
= arm_smmu_iova_to_phys
,
1488 .add_device
= arm_smmu_add_device
,
1489 .remove_device
= arm_smmu_remove_device
,
1490 .device_group
= arm_smmu_device_group
,
1491 .domain_get_attr
= arm_smmu_domain_get_attr
,
1492 .domain_set_attr
= arm_smmu_domain_set_attr
,
1493 .pgsize_bitmap
= -1UL, /* Restricted during device attach */
1496 static void arm_smmu_device_reset(struct arm_smmu_device
*smmu
)
1498 void __iomem
*gr0_base
= ARM_SMMU_GR0(smmu
);
1499 void __iomem
*cb_base
;
1503 /* clear global FSR */
1504 reg
= readl_relaxed(ARM_SMMU_GR0_NS(smmu
) + ARM_SMMU_GR0_sGFSR
);
1505 writel(reg
, ARM_SMMU_GR0_NS(smmu
) + ARM_SMMU_GR0_sGFSR
);
1507 /* Mark all SMRn as invalid and all S2CRn as bypass unless overridden */
1508 reg
= disable_bypass
? S2CR_TYPE_FAULT
: S2CR_TYPE_BYPASS
;
1509 for (i
= 0; i
< smmu
->num_mapping_groups
; ++i
) {
1510 writel_relaxed(0, gr0_base
+ ARM_SMMU_GR0_SMR(i
));
1511 writel_relaxed(reg
, gr0_base
+ ARM_SMMU_GR0_S2CR(i
));
1514 /* Make sure all context banks are disabled and clear CB_FSR */
1515 for (i
= 0; i
< smmu
->num_context_banks
; ++i
) {
1516 cb_base
= ARM_SMMU_CB_BASE(smmu
) + ARM_SMMU_CB(smmu
, i
);
1517 writel_relaxed(0, cb_base
+ ARM_SMMU_CB_SCTLR
);
1518 writel_relaxed(FSR_FAULT
, cb_base
+ ARM_SMMU_CB_FSR
);
1521 /* Invalidate the TLB, just in case */
1522 writel_relaxed(0, gr0_base
+ ARM_SMMU_GR0_TLBIALLH
);
1523 writel_relaxed(0, gr0_base
+ ARM_SMMU_GR0_TLBIALLNSNH
);
1525 reg
= readl_relaxed(ARM_SMMU_GR0_NS(smmu
) + ARM_SMMU_GR0_sCR0
);
1527 /* Enable fault reporting */
1528 reg
|= (sCR0_GFRE
| sCR0_GFIE
| sCR0_GCFGFRE
| sCR0_GCFGFIE
);
1530 /* Disable TLB broadcasting. */
1531 reg
|= (sCR0_VMIDPNE
| sCR0_PTM
);
1533 /* Enable client access, handling unmatched streams as appropriate */
1534 reg
&= ~sCR0_CLIENTPD
;
1538 reg
&= ~sCR0_USFCFG
;
1540 /* Disable forced broadcasting */
1543 /* Don't upgrade barriers */
1544 reg
&= ~(sCR0_BSU_MASK
<< sCR0_BSU_SHIFT
);
1546 if (smmu
->features
& ARM_SMMU_FEAT_VMID16
)
1547 reg
|= sCR0_VMID16EN
;
1549 /* Push the button */
1550 __arm_smmu_tlb_sync(smmu
);
1551 writel(reg
, ARM_SMMU_GR0_NS(smmu
) + ARM_SMMU_GR0_sCR0
);
1554 static int arm_smmu_id_size_to_bits(int size
)
1573 static int arm_smmu_device_cfg_probe(struct arm_smmu_device
*smmu
)
1576 void __iomem
*gr0_base
= ARM_SMMU_GR0(smmu
);
1578 bool cttw_dt
, cttw_reg
;
1580 dev_notice(smmu
->dev
, "probing hardware configuration...\n");
1581 dev_notice(smmu
->dev
, "SMMUv%d with:\n", smmu
->version
);
1584 id
= readl_relaxed(gr0_base
+ ARM_SMMU_GR0_ID0
);
1586 /* Restrict available stages based on module parameter */
1587 if (force_stage
== 1)
1588 id
&= ~(ID0_S2TS
| ID0_NTS
);
1589 else if (force_stage
== 2)
1590 id
&= ~(ID0_S1TS
| ID0_NTS
);
1592 if (id
& ID0_S1TS
) {
1593 smmu
->features
|= ARM_SMMU_FEAT_TRANS_S1
;
1594 dev_notice(smmu
->dev
, "\tstage 1 translation\n");
1597 if (id
& ID0_S2TS
) {
1598 smmu
->features
|= ARM_SMMU_FEAT_TRANS_S2
;
1599 dev_notice(smmu
->dev
, "\tstage 2 translation\n");
1603 smmu
->features
|= ARM_SMMU_FEAT_TRANS_NESTED
;
1604 dev_notice(smmu
->dev
, "\tnested translation\n");
1607 if (!(smmu
->features
&
1608 (ARM_SMMU_FEAT_TRANS_S1
| ARM_SMMU_FEAT_TRANS_S2
))) {
1609 dev_err(smmu
->dev
, "\tno translation support!\n");
1613 if ((id
& ID0_S1TS
) && ((smmu
->version
== 1) || !(id
& ID0_ATOSNS
))) {
1614 smmu
->features
|= ARM_SMMU_FEAT_TRANS_OPS
;
1615 dev_notice(smmu
->dev
, "\taddress translation ops\n");
1619 * In order for DMA API calls to work properly, we must defer to what
1620 * the DT says about coherency, regardless of what the hardware claims.
1621 * Fortunately, this also opens up a workaround for systems where the
1622 * ID register value has ended up configured incorrectly.
1624 cttw_dt
= of_dma_is_coherent(smmu
->dev
->of_node
);
1625 cttw_reg
= !!(id
& ID0_CTTW
);
1627 smmu
->features
|= ARM_SMMU_FEAT_COHERENT_WALK
;
1628 if (cttw_dt
|| cttw_reg
)
1629 dev_notice(smmu
->dev
, "\t%scoherent table walk\n",
1630 cttw_dt
? "" : "non-");
1631 if (cttw_dt
!= cttw_reg
)
1632 dev_notice(smmu
->dev
,
1633 "\t(IDR0.CTTW overridden by dma-coherent property)\n");
1638 smmu
->features
|= ARM_SMMU_FEAT_STREAM_MATCH
;
1639 smmu
->num_mapping_groups
= (id
>> ID0_NUMSMRG_SHIFT
) &
1641 if (smmu
->num_mapping_groups
== 0) {
1643 "stream-matching supported, but no SMRs present!\n");
1647 smr
= SMR_MASK_MASK
<< SMR_MASK_SHIFT
;
1648 smr
|= (SMR_ID_MASK
<< SMR_ID_SHIFT
);
1649 writel_relaxed(smr
, gr0_base
+ ARM_SMMU_GR0_SMR(0));
1650 smr
= readl_relaxed(gr0_base
+ ARM_SMMU_GR0_SMR(0));
1652 mask
= (smr
>> SMR_MASK_SHIFT
) & SMR_MASK_MASK
;
1653 sid
= (smr
>> SMR_ID_SHIFT
) & SMR_ID_MASK
;
1654 if ((mask
& sid
) != sid
) {
1656 "SMR mask bits (0x%x) insufficient for ID field (0x%x)\n",
1661 dev_notice(smmu
->dev
,
1662 "\tstream matching with %u register groups, mask 0x%x",
1663 smmu
->num_mapping_groups
, mask
);
1665 smmu
->num_mapping_groups
= (id
>> ID0_NUMSIDB_SHIFT
) &
1670 id
= readl_relaxed(gr0_base
+ ARM_SMMU_GR0_ID1
);
1671 smmu
->pgshift
= (id
& ID1_PAGESIZE
) ? 16 : 12;
1673 /* Check for size mismatch of SMMU address space from mapped region */
1674 size
= 1 << (((id
>> ID1_NUMPAGENDXB_SHIFT
) & ID1_NUMPAGENDXB_MASK
) + 1);
1675 size
*= 2 << smmu
->pgshift
;
1676 if (smmu
->size
!= size
)
1678 "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1681 smmu
->num_s2_context_banks
= (id
>> ID1_NUMS2CB_SHIFT
) & ID1_NUMS2CB_MASK
;
1682 smmu
->num_context_banks
= (id
>> ID1_NUMCB_SHIFT
) & ID1_NUMCB_MASK
;
1683 if (smmu
->num_s2_context_banks
> smmu
->num_context_banks
) {
1684 dev_err(smmu
->dev
, "impossible number of S2 context banks!\n");
1687 dev_notice(smmu
->dev
, "\t%u context banks (%u stage-2 only)\n",
1688 smmu
->num_context_banks
, smmu
->num_s2_context_banks
);
1691 id
= readl_relaxed(gr0_base
+ ARM_SMMU_GR0_ID2
);
1692 size
= arm_smmu_id_size_to_bits((id
>> ID2_IAS_SHIFT
) & ID2_IAS_MASK
);
1693 smmu
->ipa_size
= size
;
1695 /* The output mask is also applied for bypass */
1696 size
= arm_smmu_id_size_to_bits((id
>> ID2_OAS_SHIFT
) & ID2_OAS_MASK
);
1697 smmu
->pa_size
= size
;
1699 if (id
& ID2_VMID16
)
1700 smmu
->features
|= ARM_SMMU_FEAT_VMID16
;
1703 * What the page table walker can address actually depends on which
1704 * descriptor format is in use, but since a) we don't know that yet,
1705 * and b) it can vary per context bank, this will have to do...
1707 if (dma_set_mask_and_coherent(smmu
->dev
, DMA_BIT_MASK(size
)))
1709 "failed to set DMA mask for table walker\n");
1711 if (smmu
->version
== ARM_SMMU_V1
) {
1712 smmu
->va_size
= smmu
->ipa_size
;
1713 size
= SZ_4K
| SZ_2M
| SZ_1G
;
1715 size
= (id
>> ID2_UBS_SHIFT
) & ID2_UBS_MASK
;
1716 smmu
->va_size
= arm_smmu_id_size_to_bits(size
);
1717 #ifndef CONFIG_64BIT
1718 smmu
->va_size
= min(32UL, smmu
->va_size
);
1721 if (id
& ID2_PTFS_4K
)
1722 size
|= SZ_4K
| SZ_2M
| SZ_1G
;
1723 if (id
& ID2_PTFS_16K
)
1724 size
|= SZ_16K
| SZ_32M
;
1725 if (id
& ID2_PTFS_64K
)
1726 size
|= SZ_64K
| SZ_512M
;
1729 arm_smmu_ops
.pgsize_bitmap
&= size
;
1730 dev_notice(smmu
->dev
, "\tSupported page sizes: 0x%08lx\n", size
);
1732 if (smmu
->features
& ARM_SMMU_FEAT_TRANS_S1
)
1733 dev_notice(smmu
->dev
, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1734 smmu
->va_size
, smmu
->ipa_size
);
1736 if (smmu
->features
& ARM_SMMU_FEAT_TRANS_S2
)
1737 dev_notice(smmu
->dev
, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1738 smmu
->ipa_size
, smmu
->pa_size
);
1743 struct arm_smmu_match_data
{
1744 enum arm_smmu_arch_version version
;
1745 enum arm_smmu_implementation model
;
1748 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1749 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1751 ARM_SMMU_MATCH_DATA(smmu_generic_v1
, ARM_SMMU_V1
, GENERIC_SMMU
);
1752 ARM_SMMU_MATCH_DATA(smmu_generic_v2
, ARM_SMMU_V2
, GENERIC_SMMU
);
1754 static const struct of_device_id arm_smmu_of_match
[] = {
1755 { .compatible
= "arm,smmu-v1", .data
= &smmu_generic_v1
},
1756 { .compatible
= "arm,smmu-v2", .data
= &smmu_generic_v2
},
1757 { .compatible
= "arm,mmu-400", .data
= &smmu_generic_v1
},
1758 { .compatible
= "arm,mmu-401", .data
= &smmu_generic_v1
},
1759 { .compatible
= "arm,mmu-500", .data
= &smmu_generic_v2
},
1760 { .compatible
= "cavium,smmu-v2", .data
= &smmu_generic_v2
},
1763 MODULE_DEVICE_TABLE(of
, arm_smmu_of_match
);
1765 static int arm_smmu_device_dt_probe(struct platform_device
*pdev
)
1767 const struct of_device_id
*of_id
;
1768 const struct arm_smmu_match_data
*data
;
1769 struct resource
*res
;
1770 struct arm_smmu_device
*smmu
;
1771 struct device
*dev
= &pdev
->dev
;
1772 struct rb_node
*node
;
1773 struct of_phandle_args masterspec
;
1774 int num_irqs
, i
, err
;
1776 smmu
= devm_kzalloc(dev
, sizeof(*smmu
), GFP_KERNEL
);
1778 dev_err(dev
, "failed to allocate arm_smmu_device\n");
1783 of_id
= of_match_node(arm_smmu_of_match
, dev
->of_node
);
1785 smmu
->version
= data
->version
;
1786 smmu
->model
= data
->model
;
1788 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1789 smmu
->base
= devm_ioremap_resource(dev
, res
);
1790 if (IS_ERR(smmu
->base
))
1791 return PTR_ERR(smmu
->base
);
1792 smmu
->size
= resource_size(res
);
1794 if (of_property_read_u32(dev
->of_node
, "#global-interrupts",
1795 &smmu
->num_global_irqs
)) {
1796 dev_err(dev
, "missing #global-interrupts property\n");
1801 while ((res
= platform_get_resource(pdev
, IORESOURCE_IRQ
, num_irqs
))) {
1803 if (num_irqs
> smmu
->num_global_irqs
)
1804 smmu
->num_context_irqs
++;
1807 if (!smmu
->num_context_irqs
) {
1808 dev_err(dev
, "found %d interrupts but expected at least %d\n",
1809 num_irqs
, smmu
->num_global_irqs
+ 1);
1813 smmu
->irqs
= devm_kzalloc(dev
, sizeof(*smmu
->irqs
) * num_irqs
,
1816 dev_err(dev
, "failed to allocate %d irqs\n", num_irqs
);
1820 for (i
= 0; i
< num_irqs
; ++i
) {
1821 int irq
= platform_get_irq(pdev
, i
);
1824 dev_err(dev
, "failed to get irq index %d\n", i
);
1827 smmu
->irqs
[i
] = irq
;
1830 err
= arm_smmu_device_cfg_probe(smmu
);
1835 smmu
->masters
= RB_ROOT
;
1836 while (!of_parse_phandle_with_args(dev
->of_node
, "mmu-masters",
1837 "#stream-id-cells", i
,
1839 err
= register_smmu_master(smmu
, dev
, &masterspec
);
1841 dev_err(dev
, "failed to add master %s\n",
1842 masterspec
.np
->name
);
1843 goto out_put_masters
;
1848 dev_notice(dev
, "registered %d master devices\n", i
);
1850 parse_driver_options(smmu
);
1852 if (smmu
->version
> ARM_SMMU_V1
&&
1853 smmu
->num_context_banks
!= smmu
->num_context_irqs
) {
1855 "found only %d context interrupt(s) but %d required\n",
1856 smmu
->num_context_irqs
, smmu
->num_context_banks
);
1858 goto out_put_masters
;
1861 for (i
= 0; i
< smmu
->num_global_irqs
; ++i
) {
1862 err
= request_irq(smmu
->irqs
[i
],
1863 arm_smmu_global_fault
,
1865 "arm-smmu global fault",
1868 dev_err(dev
, "failed to request global IRQ %d (%u)\n",
1875 * Cavium CN88xx erratum #27704.
1876 * Ensure ASID and VMID allocation is unique across all SMMUs in
1879 if (of_device_is_compatible(dev
->of_node
, "cavium,smmu-v2")) {
1880 smmu
->cavium_id_base
=
1881 atomic_add_return(smmu
->num_context_banks
,
1882 &cavium_smmu_context_count
);
1883 smmu
->cavium_id_base
-= smmu
->num_context_banks
;
1886 INIT_LIST_HEAD(&smmu
->list
);
1887 spin_lock(&arm_smmu_devices_lock
);
1888 list_add(&smmu
->list
, &arm_smmu_devices
);
1889 spin_unlock(&arm_smmu_devices_lock
);
1891 arm_smmu_device_reset(smmu
);
1896 free_irq(smmu
->irqs
[i
], smmu
);
1899 for (node
= rb_first(&smmu
->masters
); node
; node
= rb_next(node
)) {
1900 struct arm_smmu_master
*master
1901 = container_of(node
, struct arm_smmu_master
, node
);
1902 of_node_put(master
->of_node
);
1908 static int arm_smmu_device_remove(struct platform_device
*pdev
)
1911 struct device
*dev
= &pdev
->dev
;
1912 struct arm_smmu_device
*curr
, *smmu
= NULL
;
1913 struct rb_node
*node
;
1915 spin_lock(&arm_smmu_devices_lock
);
1916 list_for_each_entry(curr
, &arm_smmu_devices
, list
) {
1917 if (curr
->dev
== dev
) {
1919 list_del(&smmu
->list
);
1923 spin_unlock(&arm_smmu_devices_lock
);
1928 for (node
= rb_first(&smmu
->masters
); node
; node
= rb_next(node
)) {
1929 struct arm_smmu_master
*master
1930 = container_of(node
, struct arm_smmu_master
, node
);
1931 of_node_put(master
->of_node
);
1934 if (!bitmap_empty(smmu
->context_map
, ARM_SMMU_MAX_CBS
))
1935 dev_err(dev
, "removing device with active domains!\n");
1937 for (i
= 0; i
< smmu
->num_global_irqs
; ++i
)
1938 free_irq(smmu
->irqs
[i
], smmu
);
1940 /* Turn the thing off */
1941 writel(sCR0_CLIENTPD
, ARM_SMMU_GR0_NS(smmu
) + ARM_SMMU_GR0_sCR0
);
1945 static struct platform_driver arm_smmu_driver
= {
1948 .of_match_table
= of_match_ptr(arm_smmu_of_match
),
1950 .probe
= arm_smmu_device_dt_probe
,
1951 .remove
= arm_smmu_device_remove
,
1954 static int __init
arm_smmu_init(void)
1956 struct device_node
*np
;
1960 * Play nice with systems that don't have an ARM SMMU by checking that
1961 * an ARM SMMU exists in the system before proceeding with the driver
1962 * and IOMMU bus operation registration.
1964 np
= of_find_matching_node(NULL
, arm_smmu_of_match
);
1970 ret
= platform_driver_register(&arm_smmu_driver
);
1974 /* Oh, for a proper bus abstraction */
1975 if (!iommu_present(&platform_bus_type
))
1976 bus_set_iommu(&platform_bus_type
, &arm_smmu_ops
);
1978 #ifdef CONFIG_ARM_AMBA
1979 if (!iommu_present(&amba_bustype
))
1980 bus_set_iommu(&amba_bustype
, &arm_smmu_ops
);
1984 if (!iommu_present(&pci_bus_type
))
1985 bus_set_iommu(&pci_bus_type
, &arm_smmu_ops
);
1991 static void __exit
arm_smmu_exit(void)
1993 return platform_driver_unregister(&arm_smmu_driver
);
1996 subsys_initcall(arm_smmu_init
);
1997 module_exit(arm_smmu_exit
);
1999 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2000 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2001 MODULE_LICENSE("GPL v2");