2 * IOMMU API for ARM architected SMMU implementations.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 * Copyright (C) 2013 ARM Limited
19 * Author: Will Deacon <will.deacon@arm.com>
21 * This driver currently supports:
22 * - SMMUv1 and v2 implementations
23 * - Stream-matching and stream-indexing
24 * - v7/v8 long-descriptor format
25 * - Non-secure access to the SMMU
26 * - Context fault reporting
29 #define pr_fmt(fmt) "arm-smmu: " fmt
31 #include <linux/atomic.h>
32 #include <linux/delay.h>
33 #include <linux/dma-iommu.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/err.h>
36 #include <linux/interrupt.h>
38 #include <linux/io-64-nonatomic-hi-lo.h>
39 #include <linux/iommu.h>
40 #include <linux/iopoll.h>
41 #include <linux/module.h>
43 #include <linux/of_address.h>
44 #include <linux/pci.h>
45 #include <linux/platform_device.h>
46 #include <linux/slab.h>
47 #include <linux/spinlock.h>
49 #include <linux/amba/bus.h>
51 #include "io-pgtable.h"
53 /* Maximum number of stream IDs assigned to a single device */
54 #define MAX_MASTER_STREAMIDS 128
56 /* Maximum number of context banks per SMMU */
57 #define ARM_SMMU_MAX_CBS 128
59 /* SMMU global address space */
60 #define ARM_SMMU_GR0(smmu) ((smmu)->base)
61 #define ARM_SMMU_GR1(smmu) ((smmu)->base + (1 << (smmu)->pgshift))
64 * SMMU global address space with conditional offset to access secure
65 * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
68 #define ARM_SMMU_GR0_NS(smmu) \
70 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \
74 * Some 64-bit registers only make sense to write atomically, but in such
75 * cases all the data relevant to AArch32 formats lies within the lower word,
76 * therefore this actually makes more sense than it might first appear.
79 #define smmu_write_atomic_lq writeq_relaxed
81 #define smmu_write_atomic_lq writel_relaxed
84 /* Configuration registers */
85 #define ARM_SMMU_GR0_sCR0 0x0
86 #define sCR0_CLIENTPD (1 << 0)
87 #define sCR0_GFRE (1 << 1)
88 #define sCR0_GFIE (1 << 2)
89 #define sCR0_GCFGFRE (1 << 4)
90 #define sCR0_GCFGFIE (1 << 5)
91 #define sCR0_USFCFG (1 << 10)
92 #define sCR0_VMIDPNE (1 << 11)
93 #define sCR0_PTM (1 << 12)
94 #define sCR0_FB (1 << 13)
95 #define sCR0_VMID16EN (1 << 31)
96 #define sCR0_BSU_SHIFT 14
97 #define sCR0_BSU_MASK 0x3
99 /* Auxiliary Configuration register */
100 #define ARM_SMMU_GR0_sACR 0x10
102 /* Identification registers */
103 #define ARM_SMMU_GR0_ID0 0x20
104 #define ARM_SMMU_GR0_ID1 0x24
105 #define ARM_SMMU_GR0_ID2 0x28
106 #define ARM_SMMU_GR0_ID3 0x2c
107 #define ARM_SMMU_GR0_ID4 0x30
108 #define ARM_SMMU_GR0_ID5 0x34
109 #define ARM_SMMU_GR0_ID6 0x38
110 #define ARM_SMMU_GR0_ID7 0x3c
111 #define ARM_SMMU_GR0_sGFSR 0x48
112 #define ARM_SMMU_GR0_sGFSYNR0 0x50
113 #define ARM_SMMU_GR0_sGFSYNR1 0x54
114 #define ARM_SMMU_GR0_sGFSYNR2 0x58
116 #define ID0_S1TS (1 << 30)
117 #define ID0_S2TS (1 << 29)
118 #define ID0_NTS (1 << 28)
119 #define ID0_SMS (1 << 27)
120 #define ID0_ATOSNS (1 << 26)
121 #define ID0_PTFS_NO_AARCH32 (1 << 25)
122 #define ID0_PTFS_NO_AARCH32S (1 << 24)
123 #define ID0_CTTW (1 << 14)
124 #define ID0_NUMIRPT_SHIFT 16
125 #define ID0_NUMIRPT_MASK 0xff
126 #define ID0_NUMSIDB_SHIFT 9
127 #define ID0_NUMSIDB_MASK 0xf
128 #define ID0_NUMSMRG_SHIFT 0
129 #define ID0_NUMSMRG_MASK 0xff
131 #define ID1_PAGESIZE (1 << 31)
132 #define ID1_NUMPAGENDXB_SHIFT 28
133 #define ID1_NUMPAGENDXB_MASK 7
134 #define ID1_NUMS2CB_SHIFT 16
135 #define ID1_NUMS2CB_MASK 0xff
136 #define ID1_NUMCB_SHIFT 0
137 #define ID1_NUMCB_MASK 0xff
139 #define ID2_OAS_SHIFT 4
140 #define ID2_OAS_MASK 0xf
141 #define ID2_IAS_SHIFT 0
142 #define ID2_IAS_MASK 0xf
143 #define ID2_UBS_SHIFT 8
144 #define ID2_UBS_MASK 0xf
145 #define ID2_PTFS_4K (1 << 12)
146 #define ID2_PTFS_16K (1 << 13)
147 #define ID2_PTFS_64K (1 << 14)
148 #define ID2_VMID16 (1 << 15)
150 #define ID7_MAJOR_SHIFT 4
151 #define ID7_MAJOR_MASK 0xf
153 /* Global TLB invalidation */
154 #define ARM_SMMU_GR0_TLBIVMID 0x64
155 #define ARM_SMMU_GR0_TLBIALLNSNH 0x68
156 #define ARM_SMMU_GR0_TLBIALLH 0x6c
157 #define ARM_SMMU_GR0_sTLBGSYNC 0x70
158 #define ARM_SMMU_GR0_sTLBGSTATUS 0x74
159 #define sTLBGSTATUS_GSACTIVE (1 << 0)
160 #define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
162 /* Stream mapping registers */
163 #define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
164 #define SMR_VALID (1 << 31)
165 #define SMR_MASK_SHIFT 16
166 #define SMR_ID_SHIFT 0
168 #define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
169 #define S2CR_CBNDX_SHIFT 0
170 #define S2CR_CBNDX_MASK 0xff
171 #define S2CR_TYPE_SHIFT 16
172 #define S2CR_TYPE_MASK 0x3
173 enum arm_smmu_s2cr_type
{
179 #define S2CR_PRIVCFG_SHIFT 24
180 #define S2CR_PRIVCFG_MASK 0x3
181 enum arm_smmu_s2cr_privcfg
{
182 S2CR_PRIVCFG_DEFAULT
,
188 /* Context bank attribute registers */
189 #define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
190 #define CBAR_VMID_SHIFT 0
191 #define CBAR_VMID_MASK 0xff
192 #define CBAR_S1_BPSHCFG_SHIFT 8
193 #define CBAR_S1_BPSHCFG_MASK 3
194 #define CBAR_S1_BPSHCFG_NSH 3
195 #define CBAR_S1_MEMATTR_SHIFT 12
196 #define CBAR_S1_MEMATTR_MASK 0xf
197 #define CBAR_S1_MEMATTR_WB 0xf
198 #define CBAR_TYPE_SHIFT 16
199 #define CBAR_TYPE_MASK 0x3
200 #define CBAR_TYPE_S2_TRANS (0 << CBAR_TYPE_SHIFT)
201 #define CBAR_TYPE_S1_TRANS_S2_BYPASS (1 << CBAR_TYPE_SHIFT)
202 #define CBAR_TYPE_S1_TRANS_S2_FAULT (2 << CBAR_TYPE_SHIFT)
203 #define CBAR_TYPE_S1_TRANS_S2_TRANS (3 << CBAR_TYPE_SHIFT)
204 #define CBAR_IRPTNDX_SHIFT 24
205 #define CBAR_IRPTNDX_MASK 0xff
207 #define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
208 #define CBA2R_RW64_32BIT (0 << 0)
209 #define CBA2R_RW64_64BIT (1 << 0)
210 #define CBA2R_VMID_SHIFT 16
211 #define CBA2R_VMID_MASK 0xffff
213 /* Translation context bank */
214 #define ARM_SMMU_CB_BASE(smmu) ((smmu)->base + ((smmu)->size >> 1))
215 #define ARM_SMMU_CB(smmu, n) ((n) * (1 << (smmu)->pgshift))
217 #define ARM_SMMU_CB_SCTLR 0x0
218 #define ARM_SMMU_CB_ACTLR 0x4
219 #define ARM_SMMU_CB_RESUME 0x8
220 #define ARM_SMMU_CB_TTBCR2 0x10
221 #define ARM_SMMU_CB_TTBR0 0x20
222 #define ARM_SMMU_CB_TTBR1 0x28
223 #define ARM_SMMU_CB_TTBCR 0x30
224 #define ARM_SMMU_CB_CONTEXTIDR 0x34
225 #define ARM_SMMU_CB_S1_MAIR0 0x38
226 #define ARM_SMMU_CB_S1_MAIR1 0x3c
227 #define ARM_SMMU_CB_PAR 0x50
228 #define ARM_SMMU_CB_FSR 0x58
229 #define ARM_SMMU_CB_FAR 0x60
230 #define ARM_SMMU_CB_FSYNR0 0x68
231 #define ARM_SMMU_CB_S1_TLBIVA 0x600
232 #define ARM_SMMU_CB_S1_TLBIASID 0x610
233 #define ARM_SMMU_CB_S1_TLBIVAL 0x620
234 #define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
235 #define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
236 #define ARM_SMMU_CB_ATS1PR 0x800
237 #define ARM_SMMU_CB_ATSR 0x8f0
239 #define SCTLR_S1_ASIDPNE (1 << 12)
240 #define SCTLR_CFCFG (1 << 7)
241 #define SCTLR_CFIE (1 << 6)
242 #define SCTLR_CFRE (1 << 5)
243 #define SCTLR_E (1 << 4)
244 #define SCTLR_AFE (1 << 2)
245 #define SCTLR_TRE (1 << 1)
246 #define SCTLR_M (1 << 0)
248 #define ARM_MMU500_ACTLR_CPRE (1 << 1)
250 #define ARM_MMU500_ACR_CACHE_LOCK (1 << 26)
252 #define CB_PAR_F (1 << 0)
254 #define ATSR_ACTIVE (1 << 0)
256 #define RESUME_RETRY (0 << 0)
257 #define RESUME_TERMINATE (1 << 0)
259 #define TTBCR2_SEP_SHIFT 15
260 #define TTBCR2_SEP_UPSTREAM (0x7 << TTBCR2_SEP_SHIFT)
262 #define TTBRn_ASID_SHIFT 48
264 #define FSR_MULTI (1 << 31)
265 #define FSR_SS (1 << 30)
266 #define FSR_UUT (1 << 8)
267 #define FSR_ASF (1 << 7)
268 #define FSR_TLBLKF (1 << 6)
269 #define FSR_TLBMCF (1 << 5)
270 #define FSR_EF (1 << 4)
271 #define FSR_PF (1 << 3)
272 #define FSR_AFF (1 << 2)
273 #define FSR_TF (1 << 1)
275 #define FSR_IGN (FSR_AFF | FSR_ASF | \
276 FSR_TLBMCF | FSR_TLBLKF)
277 #define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \
278 FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
280 #define FSYNR0_WNR (1 << 4)
282 static int force_stage
;
283 module_param(force_stage
, int, S_IRUGO
);
284 MODULE_PARM_DESC(force_stage
,
285 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
286 static bool disable_bypass
;
287 module_param(disable_bypass
, bool, S_IRUGO
);
288 MODULE_PARM_DESC(disable_bypass
,
289 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
291 enum arm_smmu_arch_version
{
297 enum arm_smmu_implementation
{
303 struct arm_smmu_s2cr
{
304 enum arm_smmu_s2cr_type type
;
305 enum arm_smmu_s2cr_privcfg privcfg
;
309 #define s2cr_init_val (struct arm_smmu_s2cr){ \
310 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
313 struct arm_smmu_smr
{
319 struct arm_smmu_master_cfg
{
320 struct arm_smmu_device
*smmu
;
322 u16 streamids
[MAX_MASTER_STREAMIDS
];
323 s16 smendx
[MAX_MASTER_STREAMIDS
];
325 #define INVALID_SMENDX -1
327 struct arm_smmu_device
{
332 unsigned long pgshift
;
334 #define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0)
335 #define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1)
336 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 2)
337 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 3)
338 #define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4)
339 #define ARM_SMMU_FEAT_TRANS_OPS (1 << 5)
340 #define ARM_SMMU_FEAT_VMID16 (1 << 6)
341 #define ARM_SMMU_FEAT_FMT_AARCH64_4K (1 << 7)
342 #define ARM_SMMU_FEAT_FMT_AARCH64_16K (1 << 8)
343 #define ARM_SMMU_FEAT_FMT_AARCH64_64K (1 << 9)
344 #define ARM_SMMU_FEAT_FMT_AARCH32_L (1 << 10)
345 #define ARM_SMMU_FEAT_FMT_AARCH32_S (1 << 11)
348 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
350 enum arm_smmu_arch_version version
;
351 enum arm_smmu_implementation model
;
353 u32 num_context_banks
;
354 u32 num_s2_context_banks
;
355 DECLARE_BITMAP(context_map
, ARM_SMMU_MAX_CBS
);
358 u32 num_mapping_groups
;
361 struct arm_smmu_smr
*smrs
;
362 struct arm_smmu_s2cr
*s2crs
;
364 unsigned long va_size
;
365 unsigned long ipa_size
;
366 unsigned long pa_size
;
367 unsigned long pgsize_bitmap
;
370 u32 num_context_irqs
;
373 struct list_head list
;
375 u32 cavium_id_base
; /* Specific to Cavium */
378 enum arm_smmu_context_fmt
{
379 ARM_SMMU_CTX_FMT_NONE
,
380 ARM_SMMU_CTX_FMT_AARCH64
,
381 ARM_SMMU_CTX_FMT_AARCH32_L
,
382 ARM_SMMU_CTX_FMT_AARCH32_S
,
385 struct arm_smmu_cfg
{
389 enum arm_smmu_context_fmt fmt
;
391 #define INVALID_IRPTNDX 0xff
393 #define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
394 #define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
396 enum arm_smmu_domain_stage
{
397 ARM_SMMU_DOMAIN_S1
= 0,
399 ARM_SMMU_DOMAIN_NESTED
,
402 struct arm_smmu_domain
{
403 struct arm_smmu_device
*smmu
;
404 struct io_pgtable_ops
*pgtbl_ops
;
405 spinlock_t pgtbl_lock
;
406 struct arm_smmu_cfg cfg
;
407 enum arm_smmu_domain_stage stage
;
408 struct mutex init_mutex
; /* Protects smmu pointer */
409 struct iommu_domain domain
;
412 static DEFINE_SPINLOCK(arm_smmu_devices_lock
);
413 static LIST_HEAD(arm_smmu_devices
);
415 struct arm_smmu_option_prop
{
420 static atomic_t cavium_smmu_context_count
= ATOMIC_INIT(0);
422 static struct arm_smmu_option_prop arm_smmu_options
[] = {
423 { ARM_SMMU_OPT_SECURE_CFG_ACCESS
, "calxeda,smmu-secure-config-access" },
427 static struct arm_smmu_domain
*to_smmu_domain(struct iommu_domain
*dom
)
429 return container_of(dom
, struct arm_smmu_domain
, domain
);
432 static void parse_driver_options(struct arm_smmu_device
*smmu
)
437 if (of_property_read_bool(smmu
->dev
->of_node
,
438 arm_smmu_options
[i
].prop
)) {
439 smmu
->options
|= arm_smmu_options
[i
].opt
;
440 dev_notice(smmu
->dev
, "option %s\n",
441 arm_smmu_options
[i
].prop
);
443 } while (arm_smmu_options
[++i
].opt
);
446 static struct device_node
*dev_get_dev_node(struct device
*dev
)
448 if (dev_is_pci(dev
)) {
449 struct pci_bus
*bus
= to_pci_dev(dev
)->bus
;
451 while (!pci_is_root_bus(bus
))
453 return of_node_get(bus
->bridge
->parent
->of_node
);
456 return of_node_get(dev
->of_node
);
459 static int __arm_smmu_get_pci_sid(struct pci_dev
*pdev
, u16 alias
, void *data
)
461 *((__be32
*)data
) = cpu_to_be32(alias
);
462 return 0; /* Continue walking */
465 static int __find_legacy_master_phandle(struct device
*dev
, void *data
)
467 struct of_phandle_iterator
*it
= *(void **)data
;
468 struct device_node
*np
= it
->node
;
471 of_for_each_phandle(it
, err
, dev
->of_node
, "mmu-masters",
472 "#stream-id-cells", 0)
473 if (it
->node
== np
) {
474 *(void **)data
= dev
;
478 return err
== -ENOENT
? 0 : err
;
481 static int arm_smmu_register_legacy_master(struct device
*dev
)
483 struct arm_smmu_device
*smmu
;
484 struct arm_smmu_master_cfg
*cfg
;
485 struct device_node
*np
;
486 struct of_phandle_iterator it
;
491 np
= dev_get_dev_node(dev
);
492 if (!np
|| !of_find_property(np
, "#stream-id-cells", NULL
)) {
498 spin_lock(&arm_smmu_devices_lock
);
499 list_for_each_entry(smmu
, &arm_smmu_devices
, list
) {
500 err
= __find_legacy_master_phandle(smmu
->dev
, &data
);
504 spin_unlock(&arm_smmu_devices_lock
);
511 if (it
.cur_count
> MAX_MASTER_STREAMIDS
) {
513 "reached maximum number (%d) of stream IDs for master device %s\n",
514 MAX_MASTER_STREAMIDS
, dev_name(dev
));
517 if (dev_is_pci(dev
)) {
518 /* "mmu-masters" assumes Stream ID == Requester ID */
519 pci_for_each_dma_alias(to_pci_dev(dev
), __arm_smmu_get_pci_sid
,
525 cfg
= kzalloc(sizeof(*cfg
), GFP_KERNEL
);
530 dev
->archdata
.iommu
= cfg
;
532 while (it
.cur_count
--)
533 cfg
->streamids
[cfg
->num_streamids
++] = be32_to_cpup(it
.cur
++);
538 static int __arm_smmu_alloc_bitmap(unsigned long *map
, int start
, int end
)
543 idx
= find_next_zero_bit(map
, end
, start
);
546 } while (test_and_set_bit(idx
, map
));
551 static void __arm_smmu_free_bitmap(unsigned long *map
, int idx
)
556 /* Wait for any pending TLB invalidations to complete */
557 static void __arm_smmu_tlb_sync(struct arm_smmu_device
*smmu
)
560 void __iomem
*gr0_base
= ARM_SMMU_GR0(smmu
);
562 writel_relaxed(0, gr0_base
+ ARM_SMMU_GR0_sTLBGSYNC
);
563 while (readl_relaxed(gr0_base
+ ARM_SMMU_GR0_sTLBGSTATUS
)
564 & sTLBGSTATUS_GSACTIVE
) {
566 if (++count
== TLB_LOOP_TIMEOUT
) {
567 dev_err_ratelimited(smmu
->dev
,
568 "TLB sync timed out -- SMMU may be deadlocked\n");
575 static void arm_smmu_tlb_sync(void *cookie
)
577 struct arm_smmu_domain
*smmu_domain
= cookie
;
578 __arm_smmu_tlb_sync(smmu_domain
->smmu
);
581 static void arm_smmu_tlb_inv_context(void *cookie
)
583 struct arm_smmu_domain
*smmu_domain
= cookie
;
584 struct arm_smmu_cfg
*cfg
= &smmu_domain
->cfg
;
585 struct arm_smmu_device
*smmu
= smmu_domain
->smmu
;
586 bool stage1
= cfg
->cbar
!= CBAR_TYPE_S2_TRANS
;
590 base
= ARM_SMMU_CB_BASE(smmu
) + ARM_SMMU_CB(smmu
, cfg
->cbndx
);
591 writel_relaxed(ARM_SMMU_CB_ASID(smmu
, cfg
),
592 base
+ ARM_SMMU_CB_S1_TLBIASID
);
594 base
= ARM_SMMU_GR0(smmu
);
595 writel_relaxed(ARM_SMMU_CB_VMID(smmu
, cfg
),
596 base
+ ARM_SMMU_GR0_TLBIVMID
);
599 __arm_smmu_tlb_sync(smmu
);
602 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova
, size_t size
,
603 size_t granule
, bool leaf
, void *cookie
)
605 struct arm_smmu_domain
*smmu_domain
= cookie
;
606 struct arm_smmu_cfg
*cfg
= &smmu_domain
->cfg
;
607 struct arm_smmu_device
*smmu
= smmu_domain
->smmu
;
608 bool stage1
= cfg
->cbar
!= CBAR_TYPE_S2_TRANS
;
612 reg
= ARM_SMMU_CB_BASE(smmu
) + ARM_SMMU_CB(smmu
, cfg
->cbndx
);
613 reg
+= leaf
? ARM_SMMU_CB_S1_TLBIVAL
: ARM_SMMU_CB_S1_TLBIVA
;
615 if (cfg
->fmt
!= ARM_SMMU_CTX_FMT_AARCH64
) {
617 iova
|= ARM_SMMU_CB_ASID(smmu
, cfg
);
619 writel_relaxed(iova
, reg
);
621 } while (size
-= granule
);
624 iova
|= (u64
)ARM_SMMU_CB_ASID(smmu
, cfg
) << 48;
626 writeq_relaxed(iova
, reg
);
627 iova
+= granule
>> 12;
628 } while (size
-= granule
);
630 } else if (smmu
->version
== ARM_SMMU_V2
) {
631 reg
= ARM_SMMU_CB_BASE(smmu
) + ARM_SMMU_CB(smmu
, cfg
->cbndx
);
632 reg
+= leaf
? ARM_SMMU_CB_S2_TLBIIPAS2L
:
633 ARM_SMMU_CB_S2_TLBIIPAS2
;
636 smmu_write_atomic_lq(iova
, reg
);
637 iova
+= granule
>> 12;
638 } while (size
-= granule
);
640 reg
= ARM_SMMU_GR0(smmu
) + ARM_SMMU_GR0_TLBIVMID
;
641 writel_relaxed(ARM_SMMU_CB_VMID(smmu
, cfg
), reg
);
645 static struct iommu_gather_ops arm_smmu_gather_ops
= {
646 .tlb_flush_all
= arm_smmu_tlb_inv_context
,
647 .tlb_add_flush
= arm_smmu_tlb_inv_range_nosync
,
648 .tlb_sync
= arm_smmu_tlb_sync
,
651 static irqreturn_t
arm_smmu_context_fault(int irq
, void *dev
)
655 struct iommu_domain
*domain
= dev
;
656 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
657 struct arm_smmu_cfg
*cfg
= &smmu_domain
->cfg
;
658 struct arm_smmu_device
*smmu
= smmu_domain
->smmu
;
659 void __iomem
*cb_base
;
661 cb_base
= ARM_SMMU_CB_BASE(smmu
) + ARM_SMMU_CB(smmu
, cfg
->cbndx
);
662 fsr
= readl_relaxed(cb_base
+ ARM_SMMU_CB_FSR
);
664 if (!(fsr
& FSR_FAULT
))
667 fsynr
= readl_relaxed(cb_base
+ ARM_SMMU_CB_FSYNR0
);
668 iova
= readq_relaxed(cb_base
+ ARM_SMMU_CB_FAR
);
670 dev_err_ratelimited(smmu
->dev
,
671 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
672 fsr
, iova
, fsynr
, cfg
->cbndx
);
674 writel(fsr
, cb_base
+ ARM_SMMU_CB_FSR
);
678 static irqreturn_t
arm_smmu_global_fault(int irq
, void *dev
)
680 u32 gfsr
, gfsynr0
, gfsynr1
, gfsynr2
;
681 struct arm_smmu_device
*smmu
= dev
;
682 void __iomem
*gr0_base
= ARM_SMMU_GR0_NS(smmu
);
684 gfsr
= readl_relaxed(gr0_base
+ ARM_SMMU_GR0_sGFSR
);
685 gfsynr0
= readl_relaxed(gr0_base
+ ARM_SMMU_GR0_sGFSYNR0
);
686 gfsynr1
= readl_relaxed(gr0_base
+ ARM_SMMU_GR0_sGFSYNR1
);
687 gfsynr2
= readl_relaxed(gr0_base
+ ARM_SMMU_GR0_sGFSYNR2
);
692 dev_err_ratelimited(smmu
->dev
,
693 "Unexpected global fault, this could be serious\n");
694 dev_err_ratelimited(smmu
->dev
,
695 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
696 gfsr
, gfsynr0
, gfsynr1
, gfsynr2
);
698 writel(gfsr
, gr0_base
+ ARM_SMMU_GR0_sGFSR
);
702 static void arm_smmu_init_context_bank(struct arm_smmu_domain
*smmu_domain
,
703 struct io_pgtable_cfg
*pgtbl_cfg
)
708 struct arm_smmu_cfg
*cfg
= &smmu_domain
->cfg
;
709 struct arm_smmu_device
*smmu
= smmu_domain
->smmu
;
710 void __iomem
*cb_base
, *gr1_base
;
712 gr1_base
= ARM_SMMU_GR1(smmu
);
713 stage1
= cfg
->cbar
!= CBAR_TYPE_S2_TRANS
;
714 cb_base
= ARM_SMMU_CB_BASE(smmu
) + ARM_SMMU_CB(smmu
, cfg
->cbndx
);
716 if (smmu
->version
> ARM_SMMU_V1
) {
717 if (cfg
->fmt
== ARM_SMMU_CTX_FMT_AARCH64
)
718 reg
= CBA2R_RW64_64BIT
;
720 reg
= CBA2R_RW64_32BIT
;
721 /* 16-bit VMIDs live in CBA2R */
722 if (smmu
->features
& ARM_SMMU_FEAT_VMID16
)
723 reg
|= ARM_SMMU_CB_VMID(smmu
, cfg
) << CBA2R_VMID_SHIFT
;
725 writel_relaxed(reg
, gr1_base
+ ARM_SMMU_GR1_CBA2R(cfg
->cbndx
));
730 if (smmu
->version
< ARM_SMMU_V2
)
731 reg
|= cfg
->irptndx
<< CBAR_IRPTNDX_SHIFT
;
734 * Use the weakest shareability/memory types, so they are
735 * overridden by the ttbcr/pte.
738 reg
|= (CBAR_S1_BPSHCFG_NSH
<< CBAR_S1_BPSHCFG_SHIFT
) |
739 (CBAR_S1_MEMATTR_WB
<< CBAR_S1_MEMATTR_SHIFT
);
740 } else if (!(smmu
->features
& ARM_SMMU_FEAT_VMID16
)) {
741 /* 8-bit VMIDs live in CBAR */
742 reg
|= ARM_SMMU_CB_VMID(smmu
, cfg
) << CBAR_VMID_SHIFT
;
744 writel_relaxed(reg
, gr1_base
+ ARM_SMMU_GR1_CBAR(cfg
->cbndx
));
748 u16 asid
= ARM_SMMU_CB_ASID(smmu
, cfg
);
750 if (cfg
->fmt
== ARM_SMMU_CTX_FMT_AARCH32_S
) {
751 reg
= pgtbl_cfg
->arm_v7s_cfg
.ttbr
[0];
752 writel_relaxed(reg
, cb_base
+ ARM_SMMU_CB_TTBR0
);
753 reg
= pgtbl_cfg
->arm_v7s_cfg
.ttbr
[1];
754 writel_relaxed(reg
, cb_base
+ ARM_SMMU_CB_TTBR1
);
755 writel_relaxed(asid
, cb_base
+ ARM_SMMU_CB_CONTEXTIDR
);
757 reg64
= pgtbl_cfg
->arm_lpae_s1_cfg
.ttbr
[0];
758 reg64
|= (u64
)asid
<< TTBRn_ASID_SHIFT
;
759 writeq_relaxed(reg64
, cb_base
+ ARM_SMMU_CB_TTBR0
);
760 reg64
= pgtbl_cfg
->arm_lpae_s1_cfg
.ttbr
[1];
761 reg64
|= (u64
)asid
<< TTBRn_ASID_SHIFT
;
762 writeq_relaxed(reg64
, cb_base
+ ARM_SMMU_CB_TTBR1
);
765 reg64
= pgtbl_cfg
->arm_lpae_s2_cfg
.vttbr
;
766 writeq_relaxed(reg64
, cb_base
+ ARM_SMMU_CB_TTBR0
);
771 if (cfg
->fmt
== ARM_SMMU_CTX_FMT_AARCH32_S
) {
772 reg
= pgtbl_cfg
->arm_v7s_cfg
.tcr
;
775 reg
= pgtbl_cfg
->arm_lpae_s1_cfg
.tcr
;
776 reg2
= pgtbl_cfg
->arm_lpae_s1_cfg
.tcr
>> 32;
777 reg2
|= TTBCR2_SEP_UPSTREAM
;
779 if (smmu
->version
> ARM_SMMU_V1
)
780 writel_relaxed(reg2
, cb_base
+ ARM_SMMU_CB_TTBCR2
);
782 reg
= pgtbl_cfg
->arm_lpae_s2_cfg
.vtcr
;
784 writel_relaxed(reg
, cb_base
+ ARM_SMMU_CB_TTBCR
);
786 /* MAIRs (stage-1 only) */
788 if (cfg
->fmt
== ARM_SMMU_CTX_FMT_AARCH32_S
) {
789 reg
= pgtbl_cfg
->arm_v7s_cfg
.prrr
;
790 reg2
= pgtbl_cfg
->arm_v7s_cfg
.nmrr
;
792 reg
= pgtbl_cfg
->arm_lpae_s1_cfg
.mair
[0];
793 reg2
= pgtbl_cfg
->arm_lpae_s1_cfg
.mair
[1];
795 writel_relaxed(reg
, cb_base
+ ARM_SMMU_CB_S1_MAIR0
);
796 writel_relaxed(reg2
, cb_base
+ ARM_SMMU_CB_S1_MAIR1
);
800 reg
= SCTLR_CFIE
| SCTLR_CFRE
| SCTLR_AFE
| SCTLR_TRE
| SCTLR_M
;
802 reg
|= SCTLR_S1_ASIDPNE
;
806 writel_relaxed(reg
, cb_base
+ ARM_SMMU_CB_SCTLR
);
809 static int arm_smmu_init_domain_context(struct iommu_domain
*domain
,
810 struct arm_smmu_device
*smmu
)
812 int irq
, start
, ret
= 0;
813 unsigned long ias
, oas
;
814 struct io_pgtable_ops
*pgtbl_ops
;
815 struct io_pgtable_cfg pgtbl_cfg
;
816 enum io_pgtable_fmt fmt
;
817 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
818 struct arm_smmu_cfg
*cfg
= &smmu_domain
->cfg
;
820 mutex_lock(&smmu_domain
->init_mutex
);
821 if (smmu_domain
->smmu
)
824 /* We're bypassing these SIDs, so don't allocate an actual context */
825 if (domain
->type
== IOMMU_DOMAIN_DMA
) {
826 smmu_domain
->smmu
= smmu
;
831 * Mapping the requested stage onto what we support is surprisingly
832 * complicated, mainly because the spec allows S1+S2 SMMUs without
833 * support for nested translation. That means we end up with the
836 * Requested Supported Actual
846 * Note that you can't actually request stage-2 mappings.
848 if (!(smmu
->features
& ARM_SMMU_FEAT_TRANS_S1
))
849 smmu_domain
->stage
= ARM_SMMU_DOMAIN_S2
;
850 if (!(smmu
->features
& ARM_SMMU_FEAT_TRANS_S2
))
851 smmu_domain
->stage
= ARM_SMMU_DOMAIN_S1
;
854 * Choosing a suitable context format is even more fiddly. Until we
855 * grow some way for the caller to express a preference, and/or move
856 * the decision into the io-pgtable code where it arguably belongs,
857 * just aim for the closest thing to the rest of the system, and hope
858 * that the hardware isn't esoteric enough that we can't assume AArch64
859 * support to be a superset of AArch32 support...
861 if (smmu
->features
& ARM_SMMU_FEAT_FMT_AARCH32_L
)
862 cfg
->fmt
= ARM_SMMU_CTX_FMT_AARCH32_L
;
863 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S
) &&
864 !IS_ENABLED(CONFIG_64BIT
) && !IS_ENABLED(CONFIG_ARM_LPAE
) &&
865 (smmu
->features
& ARM_SMMU_FEAT_FMT_AARCH32_S
) &&
866 (smmu_domain
->stage
== ARM_SMMU_DOMAIN_S1
))
867 cfg
->fmt
= ARM_SMMU_CTX_FMT_AARCH32_S
;
868 if ((IS_ENABLED(CONFIG_64BIT
) || cfg
->fmt
== ARM_SMMU_CTX_FMT_NONE
) &&
869 (smmu
->features
& (ARM_SMMU_FEAT_FMT_AARCH64_64K
|
870 ARM_SMMU_FEAT_FMT_AARCH64_16K
|
871 ARM_SMMU_FEAT_FMT_AARCH64_4K
)))
872 cfg
->fmt
= ARM_SMMU_CTX_FMT_AARCH64
;
874 if (cfg
->fmt
== ARM_SMMU_CTX_FMT_NONE
) {
879 switch (smmu_domain
->stage
) {
880 case ARM_SMMU_DOMAIN_S1
:
881 cfg
->cbar
= CBAR_TYPE_S1_TRANS_S2_BYPASS
;
882 start
= smmu
->num_s2_context_banks
;
884 oas
= smmu
->ipa_size
;
885 if (cfg
->fmt
== ARM_SMMU_CTX_FMT_AARCH64
) {
886 fmt
= ARM_64_LPAE_S1
;
887 } else if (cfg
->fmt
== ARM_SMMU_CTX_FMT_AARCH32_L
) {
888 fmt
= ARM_32_LPAE_S1
;
889 ias
= min(ias
, 32UL);
890 oas
= min(oas
, 40UL);
893 ias
= min(ias
, 32UL);
894 oas
= min(oas
, 32UL);
897 case ARM_SMMU_DOMAIN_NESTED
:
899 * We will likely want to change this if/when KVM gets
902 case ARM_SMMU_DOMAIN_S2
:
903 cfg
->cbar
= CBAR_TYPE_S2_TRANS
;
905 ias
= smmu
->ipa_size
;
907 if (cfg
->fmt
== ARM_SMMU_CTX_FMT_AARCH64
) {
908 fmt
= ARM_64_LPAE_S2
;
910 fmt
= ARM_32_LPAE_S2
;
911 ias
= min(ias
, 40UL);
912 oas
= min(oas
, 40UL);
920 ret
= __arm_smmu_alloc_bitmap(smmu
->context_map
, start
,
921 smmu
->num_context_banks
);
926 if (smmu
->version
< ARM_SMMU_V2
) {
927 cfg
->irptndx
= atomic_inc_return(&smmu
->irptndx
);
928 cfg
->irptndx
%= smmu
->num_context_irqs
;
930 cfg
->irptndx
= cfg
->cbndx
;
933 pgtbl_cfg
= (struct io_pgtable_cfg
) {
934 .pgsize_bitmap
= smmu
->pgsize_bitmap
,
937 .tlb
= &arm_smmu_gather_ops
,
938 .iommu_dev
= smmu
->dev
,
941 smmu_domain
->smmu
= smmu
;
942 pgtbl_ops
= alloc_io_pgtable_ops(fmt
, &pgtbl_cfg
, smmu_domain
);
948 /* Update the domain's page sizes to reflect the page table format */
949 domain
->pgsize_bitmap
= pgtbl_cfg
.pgsize_bitmap
;
951 /* Initialise the context bank with our page table cfg */
952 arm_smmu_init_context_bank(smmu_domain
, &pgtbl_cfg
);
955 * Request context fault interrupt. Do this last to avoid the
956 * handler seeing a half-initialised domain state.
958 irq
= smmu
->irqs
[smmu
->num_global_irqs
+ cfg
->irptndx
];
959 ret
= devm_request_irq(smmu
->dev
, irq
, arm_smmu_context_fault
,
960 IRQF_SHARED
, "arm-smmu-context-fault", domain
);
962 dev_err(smmu
->dev
, "failed to request context IRQ %d (%u)\n",
964 cfg
->irptndx
= INVALID_IRPTNDX
;
967 mutex_unlock(&smmu_domain
->init_mutex
);
969 /* Publish page table ops for map/unmap */
970 smmu_domain
->pgtbl_ops
= pgtbl_ops
;
974 smmu_domain
->smmu
= NULL
;
976 mutex_unlock(&smmu_domain
->init_mutex
);
980 static void arm_smmu_destroy_domain_context(struct iommu_domain
*domain
)
982 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
983 struct arm_smmu_device
*smmu
= smmu_domain
->smmu
;
984 struct arm_smmu_cfg
*cfg
= &smmu_domain
->cfg
;
985 void __iomem
*cb_base
;
988 if (!smmu
|| domain
->type
== IOMMU_DOMAIN_DMA
)
992 * Disable the context bank and free the page tables before freeing
995 cb_base
= ARM_SMMU_CB_BASE(smmu
) + ARM_SMMU_CB(smmu
, cfg
->cbndx
);
996 writel_relaxed(0, cb_base
+ ARM_SMMU_CB_SCTLR
);
998 if (cfg
->irptndx
!= INVALID_IRPTNDX
) {
999 irq
= smmu
->irqs
[smmu
->num_global_irqs
+ cfg
->irptndx
];
1000 devm_free_irq(smmu
->dev
, irq
, domain
);
1003 free_io_pgtable_ops(smmu_domain
->pgtbl_ops
);
1004 __arm_smmu_free_bitmap(smmu
->context_map
, cfg
->cbndx
);
1007 static struct iommu_domain
*arm_smmu_domain_alloc(unsigned type
)
1009 struct arm_smmu_domain
*smmu_domain
;
1011 if (type
!= IOMMU_DOMAIN_UNMANAGED
&& type
!= IOMMU_DOMAIN_DMA
)
1014 * Allocate the domain and initialise some of its data structures.
1015 * We can't really do anything meaningful until we've added a
1018 smmu_domain
= kzalloc(sizeof(*smmu_domain
), GFP_KERNEL
);
1022 if (type
== IOMMU_DOMAIN_DMA
&&
1023 iommu_get_dma_cookie(&smmu_domain
->domain
)) {
1028 mutex_init(&smmu_domain
->init_mutex
);
1029 spin_lock_init(&smmu_domain
->pgtbl_lock
);
1031 return &smmu_domain
->domain
;
1034 static void arm_smmu_domain_free(struct iommu_domain
*domain
)
1036 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1039 * Free the domain resources. We assume that all devices have
1040 * already been detached.
1042 iommu_put_dma_cookie(domain
);
1043 arm_smmu_destroy_domain_context(domain
);
1047 static int arm_smmu_alloc_smr(struct arm_smmu_device
*smmu
)
1051 for (i
= 0; i
< smmu
->num_mapping_groups
; i
++)
1052 if (!cmpxchg(&smmu
->smrs
[i
].valid
, false, true))
1055 return INVALID_SMENDX
;
1058 static void arm_smmu_free_smr(struct arm_smmu_device
*smmu
, int idx
)
1060 writel_relaxed(~SMR_VALID
, ARM_SMMU_GR0(smmu
) + ARM_SMMU_GR0_SMR(idx
));
1061 WRITE_ONCE(smmu
->smrs
[idx
].valid
, false);
1064 static void arm_smmu_write_smr(struct arm_smmu_device
*smmu
, int idx
)
1066 struct arm_smmu_smr
*smr
= smmu
->smrs
+ idx
;
1067 u32 reg
= smr
->id
<< SMR_ID_SHIFT
| smr
->mask
<< SMR_MASK_SHIFT
;
1071 writel_relaxed(reg
, ARM_SMMU_GR0(smmu
) + ARM_SMMU_GR0_SMR(idx
));
1074 static void arm_smmu_write_s2cr(struct arm_smmu_device
*smmu
, int idx
)
1076 struct arm_smmu_s2cr
*s2cr
= smmu
->s2crs
+ idx
;
1077 u32 reg
= (s2cr
->type
& S2CR_TYPE_MASK
) << S2CR_TYPE_SHIFT
|
1078 (s2cr
->cbndx
& S2CR_CBNDX_MASK
) << S2CR_CBNDX_SHIFT
|
1079 (s2cr
->privcfg
& S2CR_PRIVCFG_MASK
) << S2CR_PRIVCFG_SHIFT
;
1081 writel_relaxed(reg
, ARM_SMMU_GR0(smmu
) + ARM_SMMU_GR0_S2CR(idx
));
1084 static void arm_smmu_write_sme(struct arm_smmu_device
*smmu
, int idx
)
1086 arm_smmu_write_s2cr(smmu
, idx
);
1088 arm_smmu_write_smr(smmu
, idx
);
1091 static int arm_smmu_master_alloc_smes(struct arm_smmu_device
*smmu
,
1092 struct arm_smmu_master_cfg
*cfg
)
1094 struct arm_smmu_smr
*smrs
= smmu
->smrs
;
1097 /* Allocate the SMRs on the SMMU */
1098 for (i
= 0; i
< cfg
->num_streamids
; ++i
) {
1099 if (cfg
->smendx
[i
] != INVALID_SMENDX
)
1102 /* ...except on stream indexing hardware, of course */
1104 cfg
->smendx
[i
] = cfg
->streamids
[i
];
1108 idx
= arm_smmu_alloc_smr(smmu
);
1110 dev_err(smmu
->dev
, "failed to allocate free SMR\n");
1113 cfg
->smendx
[i
] = idx
;
1115 smrs
[idx
].id
= cfg
->streamids
[i
];
1116 smrs
[idx
].mask
= 0; /* We don't currently share SMRs */
1122 /* It worked! Now, poke the actual hardware */
1123 for (i
= 0; i
< cfg
->num_streamids
; ++i
)
1124 arm_smmu_write_smr(smmu
, cfg
->smendx
[i
]);
1130 arm_smmu_free_smr(smmu
, cfg
->smendx
[i
]);
1131 cfg
->smendx
[i
] = INVALID_SMENDX
;
1136 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg
*cfg
)
1138 struct arm_smmu_device
*smmu
= cfg
->smmu
;
1142 * We *must* clear the S2CR first, because freeing the SMR means
1143 * that it can be re-allocated immediately.
1145 for (i
= 0; i
< cfg
->num_streamids
; ++i
) {
1146 int idx
= cfg
->smendx
[i
];
1148 /* An IOMMU group is torn down by the first device to be removed */
1149 if (idx
== INVALID_SMENDX
)
1152 smmu
->s2crs
[idx
] = s2cr_init_val
;
1153 arm_smmu_write_s2cr(smmu
, idx
);
1155 /* Sync S2CR updates before touching anything else */
1158 /* Invalidate the SMRs before freeing back to the allocator */
1159 for (i
= 0; i
< cfg
->num_streamids
; ++i
) {
1161 arm_smmu_free_smr(smmu
, cfg
->smendx
[i
]);
1163 cfg
->smendx
[i
] = INVALID_SMENDX
;
1167 static int arm_smmu_domain_add_master(struct arm_smmu_domain
*smmu_domain
,
1168 struct arm_smmu_master_cfg
*cfg
)
1171 struct arm_smmu_device
*smmu
= smmu_domain
->smmu
;
1172 struct arm_smmu_s2cr
*s2cr
= smmu
->s2crs
;
1173 enum arm_smmu_s2cr_type type
= S2CR_TYPE_TRANS
;
1174 u8 cbndx
= smmu_domain
->cfg
.cbndx
;
1176 if (cfg
->smendx
[0] == INVALID_SMENDX
)
1177 ret
= arm_smmu_master_alloc_smes(smmu
, cfg
);
1182 * FIXME: This won't be needed once we have IOMMU-backed DMA ops
1183 * for all devices behind the SMMU. Note that we need to take
1184 * care configuring SMRs for devices both a platform_device and
1185 * and a PCI device (i.e. a PCI host controller)
1187 if (smmu_domain
->domain
.type
== IOMMU_DOMAIN_DMA
)
1188 type
= S2CR_TYPE_BYPASS
;
1190 for (i
= 0; i
< cfg
->num_streamids
; ++i
) {
1191 int idx
= cfg
->smendx
[i
];
1193 /* Devices in an IOMMU group may already be configured */
1194 if (type
== s2cr
[idx
].type
&& cbndx
== s2cr
[idx
].cbndx
)
1197 s2cr
[idx
].type
= type
;
1198 s2cr
[idx
].privcfg
= S2CR_PRIVCFG_UNPRIV
;
1199 s2cr
[idx
].cbndx
= cbndx
;
1200 arm_smmu_write_s2cr(smmu
, idx
);
1205 static int arm_smmu_attach_dev(struct iommu_domain
*domain
, struct device
*dev
)
1208 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1209 struct arm_smmu_master_cfg
*cfg
= dev
->archdata
.iommu
;
1212 dev_err(dev
, "cannot attach to SMMU, is it on the same bus?\n");
1216 /* Ensure that the domain is finalised */
1217 ret
= arm_smmu_init_domain_context(domain
, cfg
->smmu
);
1222 * Sanity check the domain. We don't support domains across
1225 if (smmu_domain
->smmu
!= cfg
->smmu
) {
1227 "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1228 dev_name(smmu_domain
->smmu
->dev
), dev_name(cfg
->smmu
->dev
));
1232 /* Looks ok, so add the device to the domain */
1233 return arm_smmu_domain_add_master(smmu_domain
, cfg
);
1236 static int arm_smmu_map(struct iommu_domain
*domain
, unsigned long iova
,
1237 phys_addr_t paddr
, size_t size
, int prot
)
1240 unsigned long flags
;
1241 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1242 struct io_pgtable_ops
*ops
= smmu_domain
->pgtbl_ops
;
1247 spin_lock_irqsave(&smmu_domain
->pgtbl_lock
, flags
);
1248 ret
= ops
->map(ops
, iova
, paddr
, size
, prot
);
1249 spin_unlock_irqrestore(&smmu_domain
->pgtbl_lock
, flags
);
1253 static size_t arm_smmu_unmap(struct iommu_domain
*domain
, unsigned long iova
,
1257 unsigned long flags
;
1258 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1259 struct io_pgtable_ops
*ops
= smmu_domain
->pgtbl_ops
;
1264 spin_lock_irqsave(&smmu_domain
->pgtbl_lock
, flags
);
1265 ret
= ops
->unmap(ops
, iova
, size
);
1266 spin_unlock_irqrestore(&smmu_domain
->pgtbl_lock
, flags
);
1270 static phys_addr_t
arm_smmu_iova_to_phys_hard(struct iommu_domain
*domain
,
1273 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1274 struct arm_smmu_device
*smmu
= smmu_domain
->smmu
;
1275 struct arm_smmu_cfg
*cfg
= &smmu_domain
->cfg
;
1276 struct io_pgtable_ops
*ops
= smmu_domain
->pgtbl_ops
;
1277 struct device
*dev
= smmu
->dev
;
1278 void __iomem
*cb_base
;
1283 cb_base
= ARM_SMMU_CB_BASE(smmu
) + ARM_SMMU_CB(smmu
, cfg
->cbndx
);
1285 /* ATS1 registers can only be written atomically */
1286 va
= iova
& ~0xfffUL
;
1287 if (smmu
->version
== ARM_SMMU_V2
)
1288 smmu_write_atomic_lq(va
, cb_base
+ ARM_SMMU_CB_ATS1PR
);
1289 else /* Register is only 32-bit in v1 */
1290 writel_relaxed(va
, cb_base
+ ARM_SMMU_CB_ATS1PR
);
1292 if (readl_poll_timeout_atomic(cb_base
+ ARM_SMMU_CB_ATSR
, tmp
,
1293 !(tmp
& ATSR_ACTIVE
), 5, 50)) {
1295 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1297 return ops
->iova_to_phys(ops
, iova
);
1300 phys
= readq_relaxed(cb_base
+ ARM_SMMU_CB_PAR
);
1301 if (phys
& CB_PAR_F
) {
1302 dev_err(dev
, "translation fault!\n");
1303 dev_err(dev
, "PAR = 0x%llx\n", phys
);
1307 return (phys
& GENMASK_ULL(39, 12)) | (iova
& 0xfff);
1310 static phys_addr_t
arm_smmu_iova_to_phys(struct iommu_domain
*domain
,
1314 unsigned long flags
;
1315 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1316 struct io_pgtable_ops
*ops
= smmu_domain
->pgtbl_ops
;
1321 spin_lock_irqsave(&smmu_domain
->pgtbl_lock
, flags
);
1322 if (smmu_domain
->smmu
->features
& ARM_SMMU_FEAT_TRANS_OPS
&&
1323 smmu_domain
->stage
== ARM_SMMU_DOMAIN_S1
) {
1324 ret
= arm_smmu_iova_to_phys_hard(domain
, iova
);
1326 ret
= ops
->iova_to_phys(ops
, iova
);
1329 spin_unlock_irqrestore(&smmu_domain
->pgtbl_lock
, flags
);
1334 static bool arm_smmu_capable(enum iommu_cap cap
)
1337 case IOMMU_CAP_CACHE_COHERENCY
:
1339 * Return true here as the SMMU can always send out coherent
1343 case IOMMU_CAP_INTR_REMAP
:
1344 return true; /* MSIs are just memory writes */
1345 case IOMMU_CAP_NOEXEC
:
1352 static int arm_smmu_add_device(struct device
*dev
)
1354 struct arm_smmu_master_cfg
*cfg
;
1355 struct iommu_group
*group
;
1358 ret
= arm_smmu_register_legacy_master(dev
);
1359 cfg
= dev
->archdata
.iommu
;
1364 for (i
= 0; i
< cfg
->num_streamids
; i
++) {
1365 u16 sid
= cfg
->streamids
[i
];
1367 if (sid
& ~cfg
->smmu
->streamid_mask
) {
1368 dev_err(dev
, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1369 sid
, cfg
->smmu
->streamid_mask
);
1372 cfg
->smendx
[i
] = INVALID_SMENDX
;
1375 group
= iommu_group_get_for_dev(dev
);
1376 if (IS_ERR(group
)) {
1377 ret
= PTR_ERR(group
);
1380 iommu_group_put(group
);
1385 dev
->archdata
.iommu
= NULL
;
1389 static void arm_smmu_remove_device(struct device
*dev
)
1391 struct arm_smmu_master_cfg
*cfg
= dev
->archdata
.iommu
;
1396 arm_smmu_master_free_smes(cfg
);
1397 iommu_group_remove_device(dev
);
1399 dev
->archdata
.iommu
= NULL
;
1402 static struct iommu_group
*arm_smmu_device_group(struct device
*dev
)
1404 struct iommu_group
*group
;
1406 if (dev_is_pci(dev
))
1407 group
= pci_device_group(dev
);
1409 group
= generic_device_group(dev
);
1414 static int arm_smmu_domain_get_attr(struct iommu_domain
*domain
,
1415 enum iommu_attr attr
, void *data
)
1417 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1420 case DOMAIN_ATTR_NESTING
:
1421 *(int *)data
= (smmu_domain
->stage
== ARM_SMMU_DOMAIN_NESTED
);
1428 static int arm_smmu_domain_set_attr(struct iommu_domain
*domain
,
1429 enum iommu_attr attr
, void *data
)
1432 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1434 mutex_lock(&smmu_domain
->init_mutex
);
1437 case DOMAIN_ATTR_NESTING
:
1438 if (smmu_domain
->smmu
) {
1444 smmu_domain
->stage
= ARM_SMMU_DOMAIN_NESTED
;
1446 smmu_domain
->stage
= ARM_SMMU_DOMAIN_S1
;
1454 mutex_unlock(&smmu_domain
->init_mutex
);
1458 static struct iommu_ops arm_smmu_ops
= {
1459 .capable
= arm_smmu_capable
,
1460 .domain_alloc
= arm_smmu_domain_alloc
,
1461 .domain_free
= arm_smmu_domain_free
,
1462 .attach_dev
= arm_smmu_attach_dev
,
1463 .map
= arm_smmu_map
,
1464 .unmap
= arm_smmu_unmap
,
1465 .map_sg
= default_iommu_map_sg
,
1466 .iova_to_phys
= arm_smmu_iova_to_phys
,
1467 .add_device
= arm_smmu_add_device
,
1468 .remove_device
= arm_smmu_remove_device
,
1469 .device_group
= arm_smmu_device_group
,
1470 .domain_get_attr
= arm_smmu_domain_get_attr
,
1471 .domain_set_attr
= arm_smmu_domain_set_attr
,
1472 .pgsize_bitmap
= -1UL, /* Restricted during device attach */
1475 static void arm_smmu_device_reset(struct arm_smmu_device
*smmu
)
1477 void __iomem
*gr0_base
= ARM_SMMU_GR0(smmu
);
1478 void __iomem
*cb_base
;
1482 /* clear global FSR */
1483 reg
= readl_relaxed(ARM_SMMU_GR0_NS(smmu
) + ARM_SMMU_GR0_sGFSR
);
1484 writel(reg
, ARM_SMMU_GR0_NS(smmu
) + ARM_SMMU_GR0_sGFSR
);
1487 * Reset stream mapping groups: Initial values mark all SMRn as
1488 * invalid and all S2CRn as bypass unless overridden.
1490 for (i
= 0; i
< smmu
->num_mapping_groups
; ++i
)
1491 arm_smmu_write_sme(smmu
, i
);
1494 * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1495 * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1496 * bit is only present in MMU-500r2 onwards.
1498 reg
= readl_relaxed(gr0_base
+ ARM_SMMU_GR0_ID7
);
1499 major
= (reg
>> ID7_MAJOR_SHIFT
) & ID7_MAJOR_MASK
;
1500 if ((smmu
->model
== ARM_MMU500
) && (major
>= 2)) {
1501 reg
= readl_relaxed(gr0_base
+ ARM_SMMU_GR0_sACR
);
1502 reg
&= ~ARM_MMU500_ACR_CACHE_LOCK
;
1503 writel_relaxed(reg
, gr0_base
+ ARM_SMMU_GR0_sACR
);
1506 /* Make sure all context banks are disabled and clear CB_FSR */
1507 for (i
= 0; i
< smmu
->num_context_banks
; ++i
) {
1508 cb_base
= ARM_SMMU_CB_BASE(smmu
) + ARM_SMMU_CB(smmu
, i
);
1509 writel_relaxed(0, cb_base
+ ARM_SMMU_CB_SCTLR
);
1510 writel_relaxed(FSR_FAULT
, cb_base
+ ARM_SMMU_CB_FSR
);
1512 * Disable MMU-500's not-particularly-beneficial next-page
1513 * prefetcher for the sake of errata #841119 and #826419.
1515 if (smmu
->model
== ARM_MMU500
) {
1516 reg
= readl_relaxed(cb_base
+ ARM_SMMU_CB_ACTLR
);
1517 reg
&= ~ARM_MMU500_ACTLR_CPRE
;
1518 writel_relaxed(reg
, cb_base
+ ARM_SMMU_CB_ACTLR
);
1522 /* Invalidate the TLB, just in case */
1523 writel_relaxed(0, gr0_base
+ ARM_SMMU_GR0_TLBIALLH
);
1524 writel_relaxed(0, gr0_base
+ ARM_SMMU_GR0_TLBIALLNSNH
);
1526 reg
= readl_relaxed(ARM_SMMU_GR0_NS(smmu
) + ARM_SMMU_GR0_sCR0
);
1528 /* Enable fault reporting */
1529 reg
|= (sCR0_GFRE
| sCR0_GFIE
| sCR0_GCFGFRE
| sCR0_GCFGFIE
);
1531 /* Disable TLB broadcasting. */
1532 reg
|= (sCR0_VMIDPNE
| sCR0_PTM
);
1534 /* Enable client access, handling unmatched streams as appropriate */
1535 reg
&= ~sCR0_CLIENTPD
;
1539 reg
&= ~sCR0_USFCFG
;
1541 /* Disable forced broadcasting */
1544 /* Don't upgrade barriers */
1545 reg
&= ~(sCR0_BSU_MASK
<< sCR0_BSU_SHIFT
);
1547 if (smmu
->features
& ARM_SMMU_FEAT_VMID16
)
1548 reg
|= sCR0_VMID16EN
;
1550 /* Push the button */
1551 __arm_smmu_tlb_sync(smmu
);
1552 writel(reg
, ARM_SMMU_GR0_NS(smmu
) + ARM_SMMU_GR0_sCR0
);
1555 static int arm_smmu_id_size_to_bits(int size
)
1574 static int arm_smmu_device_cfg_probe(struct arm_smmu_device
*smmu
)
1577 void __iomem
*gr0_base
= ARM_SMMU_GR0(smmu
);
1579 bool cttw_dt
, cttw_reg
;
1582 dev_notice(smmu
->dev
, "probing hardware configuration...\n");
1583 dev_notice(smmu
->dev
, "SMMUv%d with:\n",
1584 smmu
->version
== ARM_SMMU_V2
? 2 : 1);
1587 id
= readl_relaxed(gr0_base
+ ARM_SMMU_GR0_ID0
);
1589 /* Restrict available stages based on module parameter */
1590 if (force_stage
== 1)
1591 id
&= ~(ID0_S2TS
| ID0_NTS
);
1592 else if (force_stage
== 2)
1593 id
&= ~(ID0_S1TS
| ID0_NTS
);
1595 if (id
& ID0_S1TS
) {
1596 smmu
->features
|= ARM_SMMU_FEAT_TRANS_S1
;
1597 dev_notice(smmu
->dev
, "\tstage 1 translation\n");
1600 if (id
& ID0_S2TS
) {
1601 smmu
->features
|= ARM_SMMU_FEAT_TRANS_S2
;
1602 dev_notice(smmu
->dev
, "\tstage 2 translation\n");
1606 smmu
->features
|= ARM_SMMU_FEAT_TRANS_NESTED
;
1607 dev_notice(smmu
->dev
, "\tnested translation\n");
1610 if (!(smmu
->features
&
1611 (ARM_SMMU_FEAT_TRANS_S1
| ARM_SMMU_FEAT_TRANS_S2
))) {
1612 dev_err(smmu
->dev
, "\tno translation support!\n");
1616 if ((id
& ID0_S1TS
) &&
1617 ((smmu
->version
< ARM_SMMU_V2
) || !(id
& ID0_ATOSNS
))) {
1618 smmu
->features
|= ARM_SMMU_FEAT_TRANS_OPS
;
1619 dev_notice(smmu
->dev
, "\taddress translation ops\n");
1623 * In order for DMA API calls to work properly, we must defer to what
1624 * the DT says about coherency, regardless of what the hardware claims.
1625 * Fortunately, this also opens up a workaround for systems where the
1626 * ID register value has ended up configured incorrectly.
1628 cttw_dt
= of_dma_is_coherent(smmu
->dev
->of_node
);
1629 cttw_reg
= !!(id
& ID0_CTTW
);
1631 smmu
->features
|= ARM_SMMU_FEAT_COHERENT_WALK
;
1632 if (cttw_dt
|| cttw_reg
)
1633 dev_notice(smmu
->dev
, "\t%scoherent table walk\n",
1634 cttw_dt
? "" : "non-");
1635 if (cttw_dt
!= cttw_reg
)
1636 dev_notice(smmu
->dev
,
1637 "\t(IDR0.CTTW overridden by dma-coherent property)\n");
1639 /* Max. number of entries we have for stream matching/indexing */
1640 size
= 1 << ((id
>> ID0_NUMSIDB_SHIFT
) & ID0_NUMSIDB_MASK
);
1641 smmu
->streamid_mask
= size
- 1;
1645 smmu
->features
|= ARM_SMMU_FEAT_STREAM_MATCH
;
1646 size
= (id
>> ID0_NUMSMRG_SHIFT
) & ID0_NUMSMRG_MASK
;
1649 "stream-matching supported, but no SMRs present!\n");
1654 * SMR.ID bits may not be preserved if the corresponding MASK
1655 * bits are set, so check each one separately. We can reject
1656 * masters later if they try to claim IDs outside these masks.
1658 smr
= smmu
->streamid_mask
<< SMR_ID_SHIFT
;
1659 writel_relaxed(smr
, gr0_base
+ ARM_SMMU_GR0_SMR(0));
1660 smr
= readl_relaxed(gr0_base
+ ARM_SMMU_GR0_SMR(0));
1661 smmu
->streamid_mask
= smr
>> SMR_ID_SHIFT
;
1663 smr
= smmu
->streamid_mask
<< SMR_MASK_SHIFT
;
1664 writel_relaxed(smr
, gr0_base
+ ARM_SMMU_GR0_SMR(0));
1665 smr
= readl_relaxed(gr0_base
+ ARM_SMMU_GR0_SMR(0));
1666 smmu
->smr_mask_mask
= smr
>> SMR_MASK_SHIFT
;
1668 /* Zero-initialised to mark as invalid */
1669 smmu
->smrs
= devm_kcalloc(smmu
->dev
, size
, sizeof(*smmu
->smrs
),
1674 dev_notice(smmu
->dev
,
1675 "\tstream matching with %lu register groups, mask 0x%x",
1676 size
, smmu
->smr_mask_mask
);
1678 /* s2cr->type == 0 means translation, so initialise explicitly */
1679 smmu
->s2crs
= devm_kmalloc_array(smmu
->dev
, size
, sizeof(*smmu
->s2crs
),
1683 for (i
= 0; i
< size
; i
++)
1684 smmu
->s2crs
[i
] = s2cr_init_val
;
1686 smmu
->num_mapping_groups
= size
;
1688 if (smmu
->version
< ARM_SMMU_V2
|| !(id
& ID0_PTFS_NO_AARCH32
)) {
1689 smmu
->features
|= ARM_SMMU_FEAT_FMT_AARCH32_L
;
1690 if (!(id
& ID0_PTFS_NO_AARCH32S
))
1691 smmu
->features
|= ARM_SMMU_FEAT_FMT_AARCH32_S
;
1695 id
= readl_relaxed(gr0_base
+ ARM_SMMU_GR0_ID1
);
1696 smmu
->pgshift
= (id
& ID1_PAGESIZE
) ? 16 : 12;
1698 /* Check for size mismatch of SMMU address space from mapped region */
1699 size
= 1 << (((id
>> ID1_NUMPAGENDXB_SHIFT
) & ID1_NUMPAGENDXB_MASK
) + 1);
1700 size
*= 2 << smmu
->pgshift
;
1701 if (smmu
->size
!= size
)
1703 "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1706 smmu
->num_s2_context_banks
= (id
>> ID1_NUMS2CB_SHIFT
) & ID1_NUMS2CB_MASK
;
1707 smmu
->num_context_banks
= (id
>> ID1_NUMCB_SHIFT
) & ID1_NUMCB_MASK
;
1708 if (smmu
->num_s2_context_banks
> smmu
->num_context_banks
) {
1709 dev_err(smmu
->dev
, "impossible number of S2 context banks!\n");
1712 dev_notice(smmu
->dev
, "\t%u context banks (%u stage-2 only)\n",
1713 smmu
->num_context_banks
, smmu
->num_s2_context_banks
);
1715 * Cavium CN88xx erratum #27704.
1716 * Ensure ASID and VMID allocation is unique across all SMMUs in
1719 if (smmu
->model
== CAVIUM_SMMUV2
) {
1720 smmu
->cavium_id_base
=
1721 atomic_add_return(smmu
->num_context_banks
,
1722 &cavium_smmu_context_count
);
1723 smmu
->cavium_id_base
-= smmu
->num_context_banks
;
1727 id
= readl_relaxed(gr0_base
+ ARM_SMMU_GR0_ID2
);
1728 size
= arm_smmu_id_size_to_bits((id
>> ID2_IAS_SHIFT
) & ID2_IAS_MASK
);
1729 smmu
->ipa_size
= size
;
1731 /* The output mask is also applied for bypass */
1732 size
= arm_smmu_id_size_to_bits((id
>> ID2_OAS_SHIFT
) & ID2_OAS_MASK
);
1733 smmu
->pa_size
= size
;
1735 if (id
& ID2_VMID16
)
1736 smmu
->features
|= ARM_SMMU_FEAT_VMID16
;
1739 * What the page table walker can address actually depends on which
1740 * descriptor format is in use, but since a) we don't know that yet,
1741 * and b) it can vary per context bank, this will have to do...
1743 if (dma_set_mask_and_coherent(smmu
->dev
, DMA_BIT_MASK(size
)))
1745 "failed to set DMA mask for table walker\n");
1747 if (smmu
->version
< ARM_SMMU_V2
) {
1748 smmu
->va_size
= smmu
->ipa_size
;
1749 if (smmu
->version
== ARM_SMMU_V1_64K
)
1750 smmu
->features
|= ARM_SMMU_FEAT_FMT_AARCH64_64K
;
1752 size
= (id
>> ID2_UBS_SHIFT
) & ID2_UBS_MASK
;
1753 smmu
->va_size
= arm_smmu_id_size_to_bits(size
);
1754 if (id
& ID2_PTFS_4K
)
1755 smmu
->features
|= ARM_SMMU_FEAT_FMT_AARCH64_4K
;
1756 if (id
& ID2_PTFS_16K
)
1757 smmu
->features
|= ARM_SMMU_FEAT_FMT_AARCH64_16K
;
1758 if (id
& ID2_PTFS_64K
)
1759 smmu
->features
|= ARM_SMMU_FEAT_FMT_AARCH64_64K
;
1762 /* Now we've corralled the various formats, what'll it do? */
1763 if (smmu
->features
& ARM_SMMU_FEAT_FMT_AARCH32_S
)
1764 smmu
->pgsize_bitmap
|= SZ_4K
| SZ_64K
| SZ_1M
| SZ_16M
;
1765 if (smmu
->features
&
1766 (ARM_SMMU_FEAT_FMT_AARCH32_L
| ARM_SMMU_FEAT_FMT_AARCH64_4K
))
1767 smmu
->pgsize_bitmap
|= SZ_4K
| SZ_2M
| SZ_1G
;
1768 if (smmu
->features
& ARM_SMMU_FEAT_FMT_AARCH64_16K
)
1769 smmu
->pgsize_bitmap
|= SZ_16K
| SZ_32M
;
1770 if (smmu
->features
& ARM_SMMU_FEAT_FMT_AARCH64_64K
)
1771 smmu
->pgsize_bitmap
|= SZ_64K
| SZ_512M
;
1773 if (arm_smmu_ops
.pgsize_bitmap
== -1UL)
1774 arm_smmu_ops
.pgsize_bitmap
= smmu
->pgsize_bitmap
;
1776 arm_smmu_ops
.pgsize_bitmap
|= smmu
->pgsize_bitmap
;
1777 dev_notice(smmu
->dev
, "\tSupported page sizes: 0x%08lx\n",
1778 smmu
->pgsize_bitmap
);
1781 if (smmu
->features
& ARM_SMMU_FEAT_TRANS_S1
)
1782 dev_notice(smmu
->dev
, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1783 smmu
->va_size
, smmu
->ipa_size
);
1785 if (smmu
->features
& ARM_SMMU_FEAT_TRANS_S2
)
1786 dev_notice(smmu
->dev
, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1787 smmu
->ipa_size
, smmu
->pa_size
);
1792 struct arm_smmu_match_data
{
1793 enum arm_smmu_arch_version version
;
1794 enum arm_smmu_implementation model
;
1797 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1798 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1800 ARM_SMMU_MATCH_DATA(smmu_generic_v1
, ARM_SMMU_V1
, GENERIC_SMMU
);
1801 ARM_SMMU_MATCH_DATA(smmu_generic_v2
, ARM_SMMU_V2
, GENERIC_SMMU
);
1802 ARM_SMMU_MATCH_DATA(arm_mmu401
, ARM_SMMU_V1_64K
, GENERIC_SMMU
);
1803 ARM_SMMU_MATCH_DATA(arm_mmu500
, ARM_SMMU_V2
, ARM_MMU500
);
1804 ARM_SMMU_MATCH_DATA(cavium_smmuv2
, ARM_SMMU_V2
, CAVIUM_SMMUV2
);
1806 static const struct of_device_id arm_smmu_of_match
[] = {
1807 { .compatible
= "arm,smmu-v1", .data
= &smmu_generic_v1
},
1808 { .compatible
= "arm,smmu-v2", .data
= &smmu_generic_v2
},
1809 { .compatible
= "arm,mmu-400", .data
= &smmu_generic_v1
},
1810 { .compatible
= "arm,mmu-401", .data
= &arm_mmu401
},
1811 { .compatible
= "arm,mmu-500", .data
= &arm_mmu500
},
1812 { .compatible
= "cavium,smmu-v2", .data
= &cavium_smmuv2
},
1815 MODULE_DEVICE_TABLE(of
, arm_smmu_of_match
);
1817 static int arm_smmu_device_dt_probe(struct platform_device
*pdev
)
1819 const struct of_device_id
*of_id
;
1820 const struct arm_smmu_match_data
*data
;
1821 struct resource
*res
;
1822 struct arm_smmu_device
*smmu
;
1823 struct device
*dev
= &pdev
->dev
;
1824 int num_irqs
, i
, err
;
1826 smmu
= devm_kzalloc(dev
, sizeof(*smmu
), GFP_KERNEL
);
1828 dev_err(dev
, "failed to allocate arm_smmu_device\n");
1833 of_id
= of_match_node(arm_smmu_of_match
, dev
->of_node
);
1835 smmu
->version
= data
->version
;
1836 smmu
->model
= data
->model
;
1838 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1839 smmu
->base
= devm_ioremap_resource(dev
, res
);
1840 if (IS_ERR(smmu
->base
))
1841 return PTR_ERR(smmu
->base
);
1842 smmu
->size
= resource_size(res
);
1844 if (of_property_read_u32(dev
->of_node
, "#global-interrupts",
1845 &smmu
->num_global_irqs
)) {
1846 dev_err(dev
, "missing #global-interrupts property\n");
1851 while ((res
= platform_get_resource(pdev
, IORESOURCE_IRQ
, num_irqs
))) {
1853 if (num_irqs
> smmu
->num_global_irqs
)
1854 smmu
->num_context_irqs
++;
1857 if (!smmu
->num_context_irqs
) {
1858 dev_err(dev
, "found %d interrupts but expected at least %d\n",
1859 num_irqs
, smmu
->num_global_irqs
+ 1);
1863 smmu
->irqs
= devm_kzalloc(dev
, sizeof(*smmu
->irqs
) * num_irqs
,
1866 dev_err(dev
, "failed to allocate %d irqs\n", num_irqs
);
1870 for (i
= 0; i
< num_irqs
; ++i
) {
1871 int irq
= platform_get_irq(pdev
, i
);
1874 dev_err(dev
, "failed to get irq index %d\n", i
);
1877 smmu
->irqs
[i
] = irq
;
1880 err
= arm_smmu_device_cfg_probe(smmu
);
1884 parse_driver_options(smmu
);
1886 if (smmu
->version
== ARM_SMMU_V2
&&
1887 smmu
->num_context_banks
!= smmu
->num_context_irqs
) {
1889 "found only %d context interrupt(s) but %d required\n",
1890 smmu
->num_context_irqs
, smmu
->num_context_banks
);
1894 for (i
= 0; i
< smmu
->num_global_irqs
; ++i
) {
1895 err
= devm_request_irq(smmu
->dev
, smmu
->irqs
[i
],
1896 arm_smmu_global_fault
,
1898 "arm-smmu global fault",
1901 dev_err(dev
, "failed to request global IRQ %d (%u)\n",
1907 INIT_LIST_HEAD(&smmu
->list
);
1908 spin_lock(&arm_smmu_devices_lock
);
1909 list_add(&smmu
->list
, &arm_smmu_devices
);
1910 spin_unlock(&arm_smmu_devices_lock
);
1912 arm_smmu_device_reset(smmu
);
1916 static int arm_smmu_device_remove(struct platform_device
*pdev
)
1918 struct device
*dev
= &pdev
->dev
;
1919 struct arm_smmu_device
*curr
, *smmu
= NULL
;
1921 spin_lock(&arm_smmu_devices_lock
);
1922 list_for_each_entry(curr
, &arm_smmu_devices
, list
) {
1923 if (curr
->dev
== dev
) {
1925 list_del(&smmu
->list
);
1929 spin_unlock(&arm_smmu_devices_lock
);
1934 if (!bitmap_empty(smmu
->context_map
, ARM_SMMU_MAX_CBS
))
1935 dev_err(dev
, "removing device with active domains!\n");
1937 /* Turn the thing off */
1938 writel(sCR0_CLIENTPD
, ARM_SMMU_GR0_NS(smmu
) + ARM_SMMU_GR0_sCR0
);
1942 static struct platform_driver arm_smmu_driver
= {
1945 .of_match_table
= of_match_ptr(arm_smmu_of_match
),
1947 .probe
= arm_smmu_device_dt_probe
,
1948 .remove
= arm_smmu_device_remove
,
1951 static int __init
arm_smmu_init(void)
1953 struct device_node
*np
;
1957 * Play nice with systems that don't have an ARM SMMU by checking that
1958 * an ARM SMMU exists in the system before proceeding with the driver
1959 * and IOMMU bus operation registration.
1961 np
= of_find_matching_node(NULL
, arm_smmu_of_match
);
1967 ret
= platform_driver_register(&arm_smmu_driver
);
1971 /* Oh, for a proper bus abstraction */
1972 if (!iommu_present(&platform_bus_type
))
1973 bus_set_iommu(&platform_bus_type
, &arm_smmu_ops
);
1975 #ifdef CONFIG_ARM_AMBA
1976 if (!iommu_present(&amba_bustype
))
1977 bus_set_iommu(&amba_bustype
, &arm_smmu_ops
);
1981 if (!iommu_present(&pci_bus_type
)) {
1983 bus_set_iommu(&pci_bus_type
, &arm_smmu_ops
);
1990 static void __exit
arm_smmu_exit(void)
1992 return platform_driver_unregister(&arm_smmu_driver
);
1995 subsys_initcall(arm_smmu_init
);
1996 module_exit(arm_smmu_exit
);
1998 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
1999 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2000 MODULE_LICENSE("GPL v2");