2 * IOMMU API for ARM architected SMMUv3 implementations.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 * Copyright (C) 2015 ARM Limited
18 * Author: Will Deacon <will.deacon@arm.com>
20 * This driver is powered by bad coffee and bombay mix.
23 #include <linux/acpi.h>
24 #include <linux/acpi_iort.h>
25 #include <linux/delay.h>
26 #include <linux/dma-iommu.h>
27 #include <linux/err.h>
28 #include <linux/interrupt.h>
29 #include <linux/iommu.h>
30 #include <linux/iopoll.h>
31 #include <linux/module.h>
32 #include <linux/msi.h>
34 #include <linux/of_address.h>
35 #include <linux/of_iommu.h>
36 #include <linux/of_platform.h>
37 #include <linux/pci.h>
38 #include <linux/platform_device.h>
40 #include <linux/amba/bus.h>
42 #include "io-pgtable.h"
45 #define ARM_SMMU_IDR0 0x0
46 #define IDR0_ST_LVL_SHIFT 27
47 #define IDR0_ST_LVL_MASK 0x3
48 #define IDR0_ST_LVL_2LVL (1 << IDR0_ST_LVL_SHIFT)
49 #define IDR0_STALL_MODEL_SHIFT 24
50 #define IDR0_STALL_MODEL_MASK 0x3
51 #define IDR0_STALL_MODEL_STALL (0 << IDR0_STALL_MODEL_SHIFT)
52 #define IDR0_STALL_MODEL_FORCE (2 << IDR0_STALL_MODEL_SHIFT)
53 #define IDR0_TTENDIAN_SHIFT 21
54 #define IDR0_TTENDIAN_MASK 0x3
55 #define IDR0_TTENDIAN_LE (2 << IDR0_TTENDIAN_SHIFT)
56 #define IDR0_TTENDIAN_BE (3 << IDR0_TTENDIAN_SHIFT)
57 #define IDR0_TTENDIAN_MIXED (0 << IDR0_TTENDIAN_SHIFT)
58 #define IDR0_CD2L (1 << 19)
59 #define IDR0_VMID16 (1 << 18)
60 #define IDR0_PRI (1 << 16)
61 #define IDR0_SEV (1 << 14)
62 #define IDR0_MSI (1 << 13)
63 #define IDR0_ASID16 (1 << 12)
64 #define IDR0_ATS (1 << 10)
65 #define IDR0_HYP (1 << 9)
66 #define IDR0_COHACC (1 << 4)
67 #define IDR0_TTF_SHIFT 2
68 #define IDR0_TTF_MASK 0x3
69 #define IDR0_TTF_AARCH64 (2 << IDR0_TTF_SHIFT)
70 #define IDR0_TTF_AARCH32_64 (3 << IDR0_TTF_SHIFT)
71 #define IDR0_S1P (1 << 1)
72 #define IDR0_S2P (1 << 0)
74 #define ARM_SMMU_IDR1 0x4
75 #define IDR1_TABLES_PRESET (1 << 30)
76 #define IDR1_QUEUES_PRESET (1 << 29)
77 #define IDR1_REL (1 << 28)
78 #define IDR1_CMDQ_SHIFT 21
79 #define IDR1_CMDQ_MASK 0x1f
80 #define IDR1_EVTQ_SHIFT 16
81 #define IDR1_EVTQ_MASK 0x1f
82 #define IDR1_PRIQ_SHIFT 11
83 #define IDR1_PRIQ_MASK 0x1f
84 #define IDR1_SSID_SHIFT 6
85 #define IDR1_SSID_MASK 0x1f
86 #define IDR1_SID_SHIFT 0
87 #define IDR1_SID_MASK 0x3f
89 #define ARM_SMMU_IDR5 0x14
90 #define IDR5_STALL_MAX_SHIFT 16
91 #define IDR5_STALL_MAX_MASK 0xffff
92 #define IDR5_GRAN64K (1 << 6)
93 #define IDR5_GRAN16K (1 << 5)
94 #define IDR5_GRAN4K (1 << 4)
95 #define IDR5_OAS_SHIFT 0
96 #define IDR5_OAS_MASK 0x7
97 #define IDR5_OAS_32_BIT (0 << IDR5_OAS_SHIFT)
98 #define IDR5_OAS_36_BIT (1 << IDR5_OAS_SHIFT)
99 #define IDR5_OAS_40_BIT (2 << IDR5_OAS_SHIFT)
100 #define IDR5_OAS_42_BIT (3 << IDR5_OAS_SHIFT)
101 #define IDR5_OAS_44_BIT (4 << IDR5_OAS_SHIFT)
102 #define IDR5_OAS_48_BIT (5 << IDR5_OAS_SHIFT)
104 #define ARM_SMMU_CR0 0x20
105 #define CR0_CMDQEN (1 << 3)
106 #define CR0_EVTQEN (1 << 2)
107 #define CR0_PRIQEN (1 << 1)
108 #define CR0_SMMUEN (1 << 0)
110 #define ARM_SMMU_CR0ACK 0x24
112 #define ARM_SMMU_CR1 0x28
116 #define CR1_CACHE_NC 0
117 #define CR1_CACHE_WB 1
118 #define CR1_CACHE_WT 2
119 #define CR1_TABLE_SH_SHIFT 10
120 #define CR1_TABLE_OC_SHIFT 8
121 #define CR1_TABLE_IC_SHIFT 6
122 #define CR1_QUEUE_SH_SHIFT 4
123 #define CR1_QUEUE_OC_SHIFT 2
124 #define CR1_QUEUE_IC_SHIFT 0
126 #define ARM_SMMU_CR2 0x2c
127 #define CR2_PTM (1 << 2)
128 #define CR2_RECINVSID (1 << 1)
129 #define CR2_E2H (1 << 0)
131 #define ARM_SMMU_GBPA 0x44
132 #define GBPA_ABORT (1 << 20)
133 #define GBPA_UPDATE (1 << 31)
135 #define ARM_SMMU_IRQ_CTRL 0x50
136 #define IRQ_CTRL_EVTQ_IRQEN (1 << 2)
137 #define IRQ_CTRL_PRIQ_IRQEN (1 << 1)
138 #define IRQ_CTRL_GERROR_IRQEN (1 << 0)
140 #define ARM_SMMU_IRQ_CTRLACK 0x54
142 #define ARM_SMMU_GERROR 0x60
143 #define GERROR_SFM_ERR (1 << 8)
144 #define GERROR_MSI_GERROR_ABT_ERR (1 << 7)
145 #define GERROR_MSI_PRIQ_ABT_ERR (1 << 6)
146 #define GERROR_MSI_EVTQ_ABT_ERR (1 << 5)
147 #define GERROR_MSI_CMDQ_ABT_ERR (1 << 4)
148 #define GERROR_PRIQ_ABT_ERR (1 << 3)
149 #define GERROR_EVTQ_ABT_ERR (1 << 2)
150 #define GERROR_CMDQ_ERR (1 << 0)
151 #define GERROR_ERR_MASK 0xfd
153 #define ARM_SMMU_GERRORN 0x64
155 #define ARM_SMMU_GERROR_IRQ_CFG0 0x68
156 #define ARM_SMMU_GERROR_IRQ_CFG1 0x70
157 #define ARM_SMMU_GERROR_IRQ_CFG2 0x74
159 #define ARM_SMMU_STRTAB_BASE 0x80
160 #define STRTAB_BASE_RA (1UL << 62)
161 #define STRTAB_BASE_ADDR_SHIFT 6
162 #define STRTAB_BASE_ADDR_MASK 0x3ffffffffffUL
164 #define ARM_SMMU_STRTAB_BASE_CFG 0x88
165 #define STRTAB_BASE_CFG_LOG2SIZE_SHIFT 0
166 #define STRTAB_BASE_CFG_LOG2SIZE_MASK 0x3f
167 #define STRTAB_BASE_CFG_SPLIT_SHIFT 6
168 #define STRTAB_BASE_CFG_SPLIT_MASK 0x1f
169 #define STRTAB_BASE_CFG_FMT_SHIFT 16
170 #define STRTAB_BASE_CFG_FMT_MASK 0x3
171 #define STRTAB_BASE_CFG_FMT_LINEAR (0 << STRTAB_BASE_CFG_FMT_SHIFT)
172 #define STRTAB_BASE_CFG_FMT_2LVL (1 << STRTAB_BASE_CFG_FMT_SHIFT)
174 #define ARM_SMMU_CMDQ_BASE 0x90
175 #define ARM_SMMU_CMDQ_PROD 0x98
176 #define ARM_SMMU_CMDQ_CONS 0x9c
178 #define ARM_SMMU_EVTQ_BASE 0xa0
179 #define ARM_SMMU_EVTQ_PROD 0x100a8
180 #define ARM_SMMU_EVTQ_CONS 0x100ac
181 #define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0
182 #define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8
183 #define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc
185 #define ARM_SMMU_PRIQ_BASE 0xc0
186 #define ARM_SMMU_PRIQ_PROD 0x100c8
187 #define ARM_SMMU_PRIQ_CONS 0x100cc
188 #define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0
189 #define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
190 #define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
192 /* Common MSI config fields */
193 #define MSI_CFG0_ADDR_SHIFT 2
194 #define MSI_CFG0_ADDR_MASK 0x3fffffffffffUL
195 #define MSI_CFG2_SH_SHIFT 4
196 #define MSI_CFG2_SH_NSH (0UL << MSI_CFG2_SH_SHIFT)
197 #define MSI_CFG2_SH_OSH (2UL << MSI_CFG2_SH_SHIFT)
198 #define MSI_CFG2_SH_ISH (3UL << MSI_CFG2_SH_SHIFT)
199 #define MSI_CFG2_MEMATTR_SHIFT 0
200 #define MSI_CFG2_MEMATTR_DEVICE_nGnRE (0x1 << MSI_CFG2_MEMATTR_SHIFT)
202 #define Q_IDX(q, p) ((p) & ((1 << (q)->max_n_shift) - 1))
203 #define Q_WRP(q, p) ((p) & (1 << (q)->max_n_shift))
204 #define Q_OVERFLOW_FLAG (1 << 31)
205 #define Q_OVF(q, p) ((p) & Q_OVERFLOW_FLAG)
206 #define Q_ENT(q, p) ((q)->base + \
207 Q_IDX(q, p) * (q)->ent_dwords)
209 #define Q_BASE_RWA (1UL << 62)
210 #define Q_BASE_ADDR_SHIFT 5
211 #define Q_BASE_ADDR_MASK 0xfffffffffffUL
212 #define Q_BASE_LOG2SIZE_SHIFT 0
213 #define Q_BASE_LOG2SIZE_MASK 0x1fUL
218 * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
219 * 2lvl: 128k L1 entries,
220 * 256 lazy entries per table (each table covers a PCI bus)
222 #define STRTAB_L1_SZ_SHIFT 20
223 #define STRTAB_SPLIT 8
225 #define STRTAB_L1_DESC_DWORDS 1
226 #define STRTAB_L1_DESC_SPAN_SHIFT 0
227 #define STRTAB_L1_DESC_SPAN_MASK 0x1fUL
228 #define STRTAB_L1_DESC_L2PTR_SHIFT 6
229 #define STRTAB_L1_DESC_L2PTR_MASK 0x3ffffffffffUL
231 #define STRTAB_STE_DWORDS 8
232 #define STRTAB_STE_0_V (1UL << 0)
233 #define STRTAB_STE_0_CFG_SHIFT 1
234 #define STRTAB_STE_0_CFG_MASK 0x7UL
235 #define STRTAB_STE_0_CFG_ABORT (0UL << STRTAB_STE_0_CFG_SHIFT)
236 #define STRTAB_STE_0_CFG_BYPASS (4UL << STRTAB_STE_0_CFG_SHIFT)
237 #define STRTAB_STE_0_CFG_S1_TRANS (5UL << STRTAB_STE_0_CFG_SHIFT)
238 #define STRTAB_STE_0_CFG_S2_TRANS (6UL << STRTAB_STE_0_CFG_SHIFT)
240 #define STRTAB_STE_0_S1FMT_SHIFT 4
241 #define STRTAB_STE_0_S1FMT_LINEAR (0UL << STRTAB_STE_0_S1FMT_SHIFT)
242 #define STRTAB_STE_0_S1CTXPTR_SHIFT 6
243 #define STRTAB_STE_0_S1CTXPTR_MASK 0x3ffffffffffUL
244 #define STRTAB_STE_0_S1CDMAX_SHIFT 59
245 #define STRTAB_STE_0_S1CDMAX_MASK 0x1fUL
247 #define STRTAB_STE_1_S1C_CACHE_NC 0UL
248 #define STRTAB_STE_1_S1C_CACHE_WBRA 1UL
249 #define STRTAB_STE_1_S1C_CACHE_WT 2UL
250 #define STRTAB_STE_1_S1C_CACHE_WB 3UL
251 #define STRTAB_STE_1_S1C_SH_NSH 0UL
252 #define STRTAB_STE_1_S1C_SH_OSH 2UL
253 #define STRTAB_STE_1_S1C_SH_ISH 3UL
254 #define STRTAB_STE_1_S1CIR_SHIFT 2
255 #define STRTAB_STE_1_S1COR_SHIFT 4
256 #define STRTAB_STE_1_S1CSH_SHIFT 6
258 #define STRTAB_STE_1_S1STALLD (1UL << 27)
260 #define STRTAB_STE_1_EATS_ABT 0UL
261 #define STRTAB_STE_1_EATS_TRANS 1UL
262 #define STRTAB_STE_1_EATS_S1CHK 2UL
263 #define STRTAB_STE_1_EATS_SHIFT 28
265 #define STRTAB_STE_1_STRW_NSEL1 0UL
266 #define STRTAB_STE_1_STRW_EL2 2UL
267 #define STRTAB_STE_1_STRW_SHIFT 30
269 #define STRTAB_STE_1_SHCFG_INCOMING 1UL
270 #define STRTAB_STE_1_SHCFG_SHIFT 44
272 #define STRTAB_STE_2_S2VMID_SHIFT 0
273 #define STRTAB_STE_2_S2VMID_MASK 0xffffUL
274 #define STRTAB_STE_2_VTCR_SHIFT 32
275 #define STRTAB_STE_2_VTCR_MASK 0x7ffffUL
276 #define STRTAB_STE_2_S2AA64 (1UL << 51)
277 #define STRTAB_STE_2_S2ENDI (1UL << 52)
278 #define STRTAB_STE_2_S2PTW (1UL << 54)
279 #define STRTAB_STE_2_S2R (1UL << 58)
281 #define STRTAB_STE_3_S2TTB_SHIFT 4
282 #define STRTAB_STE_3_S2TTB_MASK 0xfffffffffffUL
284 /* Context descriptor (stage-1 only) */
285 #define CTXDESC_CD_DWORDS 8
286 #define CTXDESC_CD_0_TCR_T0SZ_SHIFT 0
287 #define ARM64_TCR_T0SZ_SHIFT 0
288 #define ARM64_TCR_T0SZ_MASK 0x1fUL
289 #define CTXDESC_CD_0_TCR_TG0_SHIFT 6
290 #define ARM64_TCR_TG0_SHIFT 14
291 #define ARM64_TCR_TG0_MASK 0x3UL
292 #define CTXDESC_CD_0_TCR_IRGN0_SHIFT 8
293 #define ARM64_TCR_IRGN0_SHIFT 8
294 #define ARM64_TCR_IRGN0_MASK 0x3UL
295 #define CTXDESC_CD_0_TCR_ORGN0_SHIFT 10
296 #define ARM64_TCR_ORGN0_SHIFT 10
297 #define ARM64_TCR_ORGN0_MASK 0x3UL
298 #define CTXDESC_CD_0_TCR_SH0_SHIFT 12
299 #define ARM64_TCR_SH0_SHIFT 12
300 #define ARM64_TCR_SH0_MASK 0x3UL
301 #define CTXDESC_CD_0_TCR_EPD0_SHIFT 14
302 #define ARM64_TCR_EPD0_SHIFT 7
303 #define ARM64_TCR_EPD0_MASK 0x1UL
304 #define CTXDESC_CD_0_TCR_EPD1_SHIFT 30
305 #define ARM64_TCR_EPD1_SHIFT 23
306 #define ARM64_TCR_EPD1_MASK 0x1UL
308 #define CTXDESC_CD_0_ENDI (1UL << 15)
309 #define CTXDESC_CD_0_V (1UL << 31)
311 #define CTXDESC_CD_0_TCR_IPS_SHIFT 32
312 #define ARM64_TCR_IPS_SHIFT 32
313 #define ARM64_TCR_IPS_MASK 0x7UL
314 #define CTXDESC_CD_0_TCR_TBI0_SHIFT 38
315 #define ARM64_TCR_TBI0_SHIFT 37
316 #define ARM64_TCR_TBI0_MASK 0x1UL
318 #define CTXDESC_CD_0_AA64 (1UL << 41)
319 #define CTXDESC_CD_0_S (1UL << 44)
320 #define CTXDESC_CD_0_R (1UL << 45)
321 #define CTXDESC_CD_0_A (1UL << 46)
322 #define CTXDESC_CD_0_ASET_SHIFT 47
323 #define CTXDESC_CD_0_ASET_SHARED (0UL << CTXDESC_CD_0_ASET_SHIFT)
324 #define CTXDESC_CD_0_ASET_PRIVATE (1UL << CTXDESC_CD_0_ASET_SHIFT)
325 #define CTXDESC_CD_0_ASID_SHIFT 48
326 #define CTXDESC_CD_0_ASID_MASK 0xffffUL
328 #define CTXDESC_CD_1_TTB0_SHIFT 4
329 #define CTXDESC_CD_1_TTB0_MASK 0xfffffffffffUL
331 #define CTXDESC_CD_3_MAIR_SHIFT 0
333 /* Convert between AArch64 (CPU) TCR format and SMMU CD format */
334 #define ARM_SMMU_TCR2CD(tcr, fld) \
335 (((tcr) >> ARM64_TCR_##fld##_SHIFT & ARM64_TCR_##fld##_MASK) \
336 << CTXDESC_CD_0_TCR_##fld##_SHIFT)
339 #define CMDQ_ENT_DWORDS 2
340 #define CMDQ_MAX_SZ_SHIFT 8
342 #define CMDQ_ERR_SHIFT 24
343 #define CMDQ_ERR_MASK 0x7f
344 #define CMDQ_ERR_CERROR_NONE_IDX 0
345 #define CMDQ_ERR_CERROR_ILL_IDX 1
346 #define CMDQ_ERR_CERROR_ABT_IDX 2
348 #define CMDQ_0_OP_SHIFT 0
349 #define CMDQ_0_OP_MASK 0xffUL
350 #define CMDQ_0_SSV (1UL << 11)
352 #define CMDQ_PREFETCH_0_SID_SHIFT 32
353 #define CMDQ_PREFETCH_1_SIZE_SHIFT 0
354 #define CMDQ_PREFETCH_1_ADDR_MASK ~0xfffUL
356 #define CMDQ_CFGI_0_SID_SHIFT 32
357 #define CMDQ_CFGI_0_SID_MASK 0xffffffffUL
358 #define CMDQ_CFGI_1_LEAF (1UL << 0)
359 #define CMDQ_CFGI_1_RANGE_SHIFT 0
360 #define CMDQ_CFGI_1_RANGE_MASK 0x1fUL
362 #define CMDQ_TLBI_0_VMID_SHIFT 32
363 #define CMDQ_TLBI_0_ASID_SHIFT 48
364 #define CMDQ_TLBI_1_LEAF (1UL << 0)
365 #define CMDQ_TLBI_1_VA_MASK ~0xfffUL
366 #define CMDQ_TLBI_1_IPA_MASK 0xfffffffff000UL
368 #define CMDQ_PRI_0_SSID_SHIFT 12
369 #define CMDQ_PRI_0_SSID_MASK 0xfffffUL
370 #define CMDQ_PRI_0_SID_SHIFT 32
371 #define CMDQ_PRI_0_SID_MASK 0xffffffffUL
372 #define CMDQ_PRI_1_GRPID_SHIFT 0
373 #define CMDQ_PRI_1_GRPID_MASK 0x1ffUL
374 #define CMDQ_PRI_1_RESP_SHIFT 12
375 #define CMDQ_PRI_1_RESP_DENY (0UL << CMDQ_PRI_1_RESP_SHIFT)
376 #define CMDQ_PRI_1_RESP_FAIL (1UL << CMDQ_PRI_1_RESP_SHIFT)
377 #define CMDQ_PRI_1_RESP_SUCC (2UL << CMDQ_PRI_1_RESP_SHIFT)
379 #define CMDQ_SYNC_0_CS_SHIFT 12
380 #define CMDQ_SYNC_0_CS_NONE (0UL << CMDQ_SYNC_0_CS_SHIFT)
381 #define CMDQ_SYNC_0_CS_IRQ (1UL << CMDQ_SYNC_0_CS_SHIFT)
382 #define CMDQ_SYNC_0_CS_SEV (2UL << CMDQ_SYNC_0_CS_SHIFT)
383 #define CMDQ_SYNC_0_MSH_SHIFT 22
384 #define CMDQ_SYNC_0_MSH_ISH (3UL << CMDQ_SYNC_0_MSH_SHIFT)
385 #define CMDQ_SYNC_0_MSIATTR_SHIFT 24
386 #define CMDQ_SYNC_0_MSIATTR_OIWB (0xfUL << CMDQ_SYNC_0_MSIATTR_SHIFT)
387 #define CMDQ_SYNC_0_MSIDATA_SHIFT 32
388 #define CMDQ_SYNC_0_MSIDATA_MASK 0xffffffffUL
389 #define CMDQ_SYNC_1_MSIADDR_SHIFT 0
390 #define CMDQ_SYNC_1_MSIADDR_MASK 0xffffffffffffcUL
393 #define EVTQ_ENT_DWORDS 4
394 #define EVTQ_MAX_SZ_SHIFT 7
396 #define EVTQ_0_ID_SHIFT 0
397 #define EVTQ_0_ID_MASK 0xffUL
400 #define PRIQ_ENT_DWORDS 2
401 #define PRIQ_MAX_SZ_SHIFT 8
403 #define PRIQ_0_SID_SHIFT 0
404 #define PRIQ_0_SID_MASK 0xffffffffUL
405 #define PRIQ_0_SSID_SHIFT 32
406 #define PRIQ_0_SSID_MASK 0xfffffUL
407 #define PRIQ_0_PERM_PRIV (1UL << 58)
408 #define PRIQ_0_PERM_EXEC (1UL << 59)
409 #define PRIQ_0_PERM_READ (1UL << 60)
410 #define PRIQ_0_PERM_WRITE (1UL << 61)
411 #define PRIQ_0_PRG_LAST (1UL << 62)
412 #define PRIQ_0_SSID_V (1UL << 63)
414 #define PRIQ_1_PRG_IDX_SHIFT 0
415 #define PRIQ_1_PRG_IDX_MASK 0x1ffUL
416 #define PRIQ_1_ADDR_SHIFT 12
417 #define PRIQ_1_ADDR_MASK 0xfffffffffffffUL
419 /* High-level queue structures */
420 #define ARM_SMMU_POLL_TIMEOUT_US 100
421 #define ARM_SMMU_CMDQ_SYNC_TIMEOUT_US 1000000 /* 1s! */
422 #define ARM_SMMU_CMDQ_SYNC_SPIN_COUNT 10
424 #define MSI_IOVA_BASE 0x8000000
425 #define MSI_IOVA_LENGTH 0x100000
427 static bool disable_bypass
;
428 module_param_named(disable_bypass
, disable_bypass
, bool, S_IRUGO
);
429 MODULE_PARM_DESC(disable_bypass
,
430 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
438 enum arm_smmu_msi_index
{
445 static phys_addr_t arm_smmu_msi_cfg
[ARM_SMMU_MAX_MSIS
][3] = {
447 ARM_SMMU_EVTQ_IRQ_CFG0
,
448 ARM_SMMU_EVTQ_IRQ_CFG1
,
449 ARM_SMMU_EVTQ_IRQ_CFG2
,
451 [GERROR_MSI_INDEX
] = {
452 ARM_SMMU_GERROR_IRQ_CFG0
,
453 ARM_SMMU_GERROR_IRQ_CFG1
,
454 ARM_SMMU_GERROR_IRQ_CFG2
,
457 ARM_SMMU_PRIQ_IRQ_CFG0
,
458 ARM_SMMU_PRIQ_IRQ_CFG1
,
459 ARM_SMMU_PRIQ_IRQ_CFG2
,
463 struct arm_smmu_cmdq_ent
{
466 bool substream_valid
;
468 /* Command-specific fields */
470 #define CMDQ_OP_PREFETCH_CFG 0x1
477 #define CMDQ_OP_CFGI_STE 0x3
478 #define CMDQ_OP_CFGI_ALL 0x4
487 #define CMDQ_OP_TLBI_NH_ASID 0x11
488 #define CMDQ_OP_TLBI_NH_VA 0x12
489 #define CMDQ_OP_TLBI_EL2_ALL 0x20
490 #define CMDQ_OP_TLBI_S12_VMALL 0x28
491 #define CMDQ_OP_TLBI_S2_IPA 0x2a
492 #define CMDQ_OP_TLBI_NSNH_ALL 0x30
500 #define CMDQ_OP_PRI_RESP 0x41
508 #define CMDQ_OP_CMD_SYNC 0x46
516 struct arm_smmu_queue
{
517 int irq
; /* Wired interrupt */
528 u32 __iomem
*prod_reg
;
529 u32 __iomem
*cons_reg
;
532 struct arm_smmu_cmdq
{
533 struct arm_smmu_queue q
;
537 struct arm_smmu_evtq
{
538 struct arm_smmu_queue q
;
542 struct arm_smmu_priq
{
543 struct arm_smmu_queue q
;
546 /* High-level stream table and context descriptor structures */
547 struct arm_smmu_strtab_l1_desc
{
551 dma_addr_t l2ptr_dma
;
554 struct arm_smmu_s1_cfg
{
556 dma_addr_t cdptr_dma
;
558 struct arm_smmu_ctx_desc
{
566 struct arm_smmu_s2_cfg
{
572 struct arm_smmu_strtab_ent
{
574 * An STE is "assigned" if the master emitting the corresponding SID
575 * is attached to a domain. The behaviour of an unassigned STE is
576 * determined by the disable_bypass parameter, whereas an assigned
577 * STE behaves according to s1_cfg/s2_cfg, which themselves are
578 * configured according to the domain type.
581 struct arm_smmu_s1_cfg
*s1_cfg
;
582 struct arm_smmu_s2_cfg
*s2_cfg
;
585 struct arm_smmu_strtab_cfg
{
587 dma_addr_t strtab_dma
;
588 struct arm_smmu_strtab_l1_desc
*l1_desc
;
589 unsigned int num_l1_ents
;
595 /* An SMMUv3 instance */
596 struct arm_smmu_device
{
600 #define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0)
601 #define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1)
602 #define ARM_SMMU_FEAT_TT_LE (1 << 2)
603 #define ARM_SMMU_FEAT_TT_BE (1 << 3)
604 #define ARM_SMMU_FEAT_PRI (1 << 4)
605 #define ARM_SMMU_FEAT_ATS (1 << 5)
606 #define ARM_SMMU_FEAT_SEV (1 << 6)
607 #define ARM_SMMU_FEAT_MSI (1 << 7)
608 #define ARM_SMMU_FEAT_COHERENCY (1 << 8)
609 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 9)
610 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 10)
611 #define ARM_SMMU_FEAT_STALLS (1 << 11)
612 #define ARM_SMMU_FEAT_HYP (1 << 12)
613 #define ARM_SMMU_FEAT_STALL_FORCE (1 << 13)
616 #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0)
617 #define ARM_SMMU_OPT_PAGE0_REGS_ONLY (1 << 1)
620 struct arm_smmu_cmdq cmdq
;
621 struct arm_smmu_evtq evtq
;
622 struct arm_smmu_priq priq
;
628 unsigned long ias
; /* IPA */
629 unsigned long oas
; /* PA */
630 unsigned long pgsize_bitmap
;
632 #define ARM_SMMU_MAX_ASIDS (1 << 16)
633 unsigned int asid_bits
;
634 DECLARE_BITMAP(asid_map
, ARM_SMMU_MAX_ASIDS
);
636 #define ARM_SMMU_MAX_VMIDS (1 << 16)
637 unsigned int vmid_bits
;
638 DECLARE_BITMAP(vmid_map
, ARM_SMMU_MAX_VMIDS
);
640 unsigned int ssid_bits
;
641 unsigned int sid_bits
;
643 struct arm_smmu_strtab_cfg strtab_cfg
;
647 /* IOMMU core code handle */
648 struct iommu_device iommu
;
651 /* SMMU private data for each master */
652 struct arm_smmu_master_data
{
653 struct arm_smmu_device
*smmu
;
654 struct arm_smmu_strtab_ent ste
;
657 /* SMMU private data for an IOMMU domain */
658 enum arm_smmu_domain_stage
{
659 ARM_SMMU_DOMAIN_S1
= 0,
661 ARM_SMMU_DOMAIN_NESTED
,
662 ARM_SMMU_DOMAIN_BYPASS
,
665 struct arm_smmu_domain
{
666 struct arm_smmu_device
*smmu
;
667 struct mutex init_mutex
; /* Protects smmu pointer */
669 struct io_pgtable_ops
*pgtbl_ops
;
671 enum arm_smmu_domain_stage stage
;
673 struct arm_smmu_s1_cfg s1_cfg
;
674 struct arm_smmu_s2_cfg s2_cfg
;
677 struct iommu_domain domain
;
680 struct arm_smmu_option_prop
{
685 static struct arm_smmu_option_prop arm_smmu_options
[] = {
686 { ARM_SMMU_OPT_SKIP_PREFETCH
, "hisilicon,broken-prefetch-cmd" },
687 { ARM_SMMU_OPT_PAGE0_REGS_ONLY
, "cavium,cn9900-broken-page1-regspace"},
691 static inline void __iomem
*arm_smmu_page1_fixup(unsigned long offset
,
692 struct arm_smmu_device
*smmu
)
694 if ((offset
> SZ_64K
) &&
695 (smmu
->options
& ARM_SMMU_OPT_PAGE0_REGS_ONLY
))
698 return smmu
->base
+ offset
;
701 static struct arm_smmu_domain
*to_smmu_domain(struct iommu_domain
*dom
)
703 return container_of(dom
, struct arm_smmu_domain
, domain
);
706 static void parse_driver_options(struct arm_smmu_device
*smmu
)
711 if (of_property_read_bool(smmu
->dev
->of_node
,
712 arm_smmu_options
[i
].prop
)) {
713 smmu
->options
|= arm_smmu_options
[i
].opt
;
714 dev_notice(smmu
->dev
, "option %s\n",
715 arm_smmu_options
[i
].prop
);
717 } while (arm_smmu_options
[++i
].opt
);
720 /* Low-level queue manipulation functions */
721 static bool queue_full(struct arm_smmu_queue
*q
)
723 return Q_IDX(q
, q
->prod
) == Q_IDX(q
, q
->cons
) &&
724 Q_WRP(q
, q
->prod
) != Q_WRP(q
, q
->cons
);
727 static bool queue_empty(struct arm_smmu_queue
*q
)
729 return Q_IDX(q
, q
->prod
) == Q_IDX(q
, q
->cons
) &&
730 Q_WRP(q
, q
->prod
) == Q_WRP(q
, q
->cons
);
733 static void queue_sync_cons(struct arm_smmu_queue
*q
)
735 q
->cons
= readl_relaxed(q
->cons_reg
);
738 static void queue_inc_cons(struct arm_smmu_queue
*q
)
740 u32 cons
= (Q_WRP(q
, q
->cons
) | Q_IDX(q
, q
->cons
)) + 1;
742 q
->cons
= Q_OVF(q
, q
->cons
) | Q_WRP(q
, cons
) | Q_IDX(q
, cons
);
743 writel(q
->cons
, q
->cons_reg
);
746 static int queue_sync_prod(struct arm_smmu_queue
*q
)
749 u32 prod
= readl_relaxed(q
->prod_reg
);
751 if (Q_OVF(q
, prod
) != Q_OVF(q
, q
->prod
))
758 static void queue_inc_prod(struct arm_smmu_queue
*q
)
760 u32 prod
= (Q_WRP(q
, q
->prod
) | Q_IDX(q
, q
->prod
)) + 1;
762 q
->prod
= Q_OVF(q
, q
->prod
) | Q_WRP(q
, prod
) | Q_IDX(q
, prod
);
763 writel(q
->prod
, q
->prod_reg
);
767 * Wait for the SMMU to consume items. If drain is true, wait until the queue
768 * is empty. Otherwise, wait until there is at least one free slot.
770 static int queue_poll_cons(struct arm_smmu_queue
*q
, bool sync
, bool wfe
)
773 unsigned int delay
= 1, spin_cnt
= 0;
775 /* Wait longer if it's a CMD_SYNC */
776 timeout
= ktime_add_us(ktime_get(), sync
?
777 ARM_SMMU_CMDQ_SYNC_TIMEOUT_US
:
778 ARM_SMMU_POLL_TIMEOUT_US
);
780 while (queue_sync_cons(q
), (sync
? !queue_empty(q
) : queue_full(q
))) {
781 if (ktime_compare(ktime_get(), timeout
) > 0)
786 } else if (++spin_cnt
< ARM_SMMU_CMDQ_SYNC_SPIN_COUNT
) {
799 static void queue_write(__le64
*dst
, u64
*src
, size_t n_dwords
)
803 for (i
= 0; i
< n_dwords
; ++i
)
804 *dst
++ = cpu_to_le64(*src
++);
807 static int queue_insert_raw(struct arm_smmu_queue
*q
, u64
*ent
)
812 queue_write(Q_ENT(q
, q
->prod
), ent
, q
->ent_dwords
);
817 static void queue_read(__le64
*dst
, u64
*src
, size_t n_dwords
)
821 for (i
= 0; i
< n_dwords
; ++i
)
822 *dst
++ = le64_to_cpu(*src
++);
825 static int queue_remove_raw(struct arm_smmu_queue
*q
, u64
*ent
)
830 queue_read(ent
, Q_ENT(q
, q
->cons
), q
->ent_dwords
);
835 /* High-level queue accessors */
836 static int arm_smmu_cmdq_build_cmd(u64
*cmd
, struct arm_smmu_cmdq_ent
*ent
)
838 memset(cmd
, 0, CMDQ_ENT_DWORDS
<< 3);
839 cmd
[0] |= (ent
->opcode
& CMDQ_0_OP_MASK
) << CMDQ_0_OP_SHIFT
;
841 switch (ent
->opcode
) {
842 case CMDQ_OP_TLBI_EL2_ALL
:
843 case CMDQ_OP_TLBI_NSNH_ALL
:
845 case CMDQ_OP_PREFETCH_CFG
:
846 cmd
[0] |= (u64
)ent
->prefetch
.sid
<< CMDQ_PREFETCH_0_SID_SHIFT
;
847 cmd
[1] |= ent
->prefetch
.size
<< CMDQ_PREFETCH_1_SIZE_SHIFT
;
848 cmd
[1] |= ent
->prefetch
.addr
& CMDQ_PREFETCH_1_ADDR_MASK
;
850 case CMDQ_OP_CFGI_STE
:
851 cmd
[0] |= (u64
)ent
->cfgi
.sid
<< CMDQ_CFGI_0_SID_SHIFT
;
852 cmd
[1] |= ent
->cfgi
.leaf
? CMDQ_CFGI_1_LEAF
: 0;
854 case CMDQ_OP_CFGI_ALL
:
855 /* Cover the entire SID range */
856 cmd
[1] |= CMDQ_CFGI_1_RANGE_MASK
<< CMDQ_CFGI_1_RANGE_SHIFT
;
858 case CMDQ_OP_TLBI_NH_VA
:
859 cmd
[0] |= (u64
)ent
->tlbi
.asid
<< CMDQ_TLBI_0_ASID_SHIFT
;
860 cmd
[1] |= ent
->tlbi
.leaf
? CMDQ_TLBI_1_LEAF
: 0;
861 cmd
[1] |= ent
->tlbi
.addr
& CMDQ_TLBI_1_VA_MASK
;
863 case CMDQ_OP_TLBI_S2_IPA
:
864 cmd
[0] |= (u64
)ent
->tlbi
.vmid
<< CMDQ_TLBI_0_VMID_SHIFT
;
865 cmd
[1] |= ent
->tlbi
.leaf
? CMDQ_TLBI_1_LEAF
: 0;
866 cmd
[1] |= ent
->tlbi
.addr
& CMDQ_TLBI_1_IPA_MASK
;
868 case CMDQ_OP_TLBI_NH_ASID
:
869 cmd
[0] |= (u64
)ent
->tlbi
.asid
<< CMDQ_TLBI_0_ASID_SHIFT
;
871 case CMDQ_OP_TLBI_S12_VMALL
:
872 cmd
[0] |= (u64
)ent
->tlbi
.vmid
<< CMDQ_TLBI_0_VMID_SHIFT
;
874 case CMDQ_OP_PRI_RESP
:
875 cmd
[0] |= ent
->substream_valid
? CMDQ_0_SSV
: 0;
876 cmd
[0] |= ent
->pri
.ssid
<< CMDQ_PRI_0_SSID_SHIFT
;
877 cmd
[0] |= (u64
)ent
->pri
.sid
<< CMDQ_PRI_0_SID_SHIFT
;
878 cmd
[1] |= ent
->pri
.grpid
<< CMDQ_PRI_1_GRPID_SHIFT
;
879 switch (ent
->pri
.resp
) {
881 cmd
[1] |= CMDQ_PRI_1_RESP_DENY
;
884 cmd
[1] |= CMDQ_PRI_1_RESP_FAIL
;
887 cmd
[1] |= CMDQ_PRI_1_RESP_SUCC
;
893 case CMDQ_OP_CMD_SYNC
:
894 if (ent
->sync
.msiaddr
)
895 cmd
[0] |= CMDQ_SYNC_0_CS_IRQ
;
897 cmd
[0] |= CMDQ_SYNC_0_CS_SEV
;
898 cmd
[0] |= CMDQ_SYNC_0_MSH_ISH
| CMDQ_SYNC_0_MSIATTR_OIWB
;
899 cmd
[0] |= (u64
)ent
->sync
.msidata
<< CMDQ_SYNC_0_MSIDATA_SHIFT
;
900 cmd
[1] |= ent
->sync
.msiaddr
& CMDQ_SYNC_1_MSIADDR_MASK
;
909 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device
*smmu
)
911 static const char *cerror_str
[] = {
912 [CMDQ_ERR_CERROR_NONE_IDX
] = "No error",
913 [CMDQ_ERR_CERROR_ILL_IDX
] = "Illegal command",
914 [CMDQ_ERR_CERROR_ABT_IDX
] = "Abort on command fetch",
918 u64 cmd
[CMDQ_ENT_DWORDS
];
919 struct arm_smmu_queue
*q
= &smmu
->cmdq
.q
;
920 u32 cons
= readl_relaxed(q
->cons_reg
);
921 u32 idx
= cons
>> CMDQ_ERR_SHIFT
& CMDQ_ERR_MASK
;
922 struct arm_smmu_cmdq_ent cmd_sync
= {
923 .opcode
= CMDQ_OP_CMD_SYNC
,
926 dev_err(smmu
->dev
, "CMDQ error (cons 0x%08x): %s\n", cons
,
927 idx
< ARRAY_SIZE(cerror_str
) ? cerror_str
[idx
] : "Unknown");
930 case CMDQ_ERR_CERROR_ABT_IDX
:
931 dev_err(smmu
->dev
, "retrying command fetch\n");
932 case CMDQ_ERR_CERROR_NONE_IDX
:
934 case CMDQ_ERR_CERROR_ILL_IDX
:
941 * We may have concurrent producers, so we need to be careful
942 * not to touch any of the shadow cmdq state.
944 queue_read(cmd
, Q_ENT(q
, cons
), q
->ent_dwords
);
945 dev_err(smmu
->dev
, "skipping command in error state:\n");
946 for (i
= 0; i
< ARRAY_SIZE(cmd
); ++i
)
947 dev_err(smmu
->dev
, "\t0x%016llx\n", (unsigned long long)cmd
[i
]);
949 /* Convert the erroneous command into a CMD_SYNC */
950 if (arm_smmu_cmdq_build_cmd(cmd
, &cmd_sync
)) {
951 dev_err(smmu
->dev
, "failed to convert to CMD_SYNC\n");
955 queue_write(Q_ENT(q
, cons
), cmd
, q
->ent_dwords
);
958 static void arm_smmu_cmdq_insert_cmd(struct arm_smmu_device
*smmu
, u64
*cmd
)
960 struct arm_smmu_queue
*q
= &smmu
->cmdq
.q
;
961 bool wfe
= !!(smmu
->features
& ARM_SMMU_FEAT_SEV
);
963 while (queue_insert_raw(q
, cmd
) == -ENOSPC
) {
964 if (queue_poll_cons(q
, false, wfe
))
965 dev_err_ratelimited(smmu
->dev
, "CMDQ timeout\n");
969 static void arm_smmu_cmdq_issue_cmd(struct arm_smmu_device
*smmu
,
970 struct arm_smmu_cmdq_ent
*ent
)
972 u64 cmd
[CMDQ_ENT_DWORDS
];
975 if (arm_smmu_cmdq_build_cmd(cmd
, ent
)) {
976 dev_warn(smmu
->dev
, "ignoring unknown CMDQ opcode 0x%x\n",
981 spin_lock_irqsave(&smmu
->cmdq
.lock
, flags
);
982 arm_smmu_cmdq_insert_cmd(smmu
, cmd
);
983 spin_unlock_irqrestore(&smmu
->cmdq
.lock
, flags
);
987 * The difference between val and sync_idx is bounded by the maximum size of
988 * a queue at 2^20 entries, so 32 bits is plenty for wrap-safe arithmetic.
990 static int __arm_smmu_sync_poll_msi(struct arm_smmu_device
*smmu
, u32 sync_idx
)
995 timeout
= ktime_add_us(ktime_get(), ARM_SMMU_CMDQ_SYNC_TIMEOUT_US
);
996 val
= smp_cond_load_acquire(&smmu
->sync_count
,
997 (int)(VAL
- sync_idx
) >= 0 ||
998 !ktime_before(ktime_get(), timeout
));
1000 return (int)(val
- sync_idx
) < 0 ? -ETIMEDOUT
: 0;
1003 static int __arm_smmu_cmdq_issue_sync_msi(struct arm_smmu_device
*smmu
)
1005 u64 cmd
[CMDQ_ENT_DWORDS
];
1006 unsigned long flags
;
1007 struct arm_smmu_cmdq_ent ent
= {
1008 .opcode
= CMDQ_OP_CMD_SYNC
,
1010 .msidata
= atomic_inc_return_relaxed(&smmu
->sync_nr
),
1011 .msiaddr
= virt_to_phys(&smmu
->sync_count
),
1015 arm_smmu_cmdq_build_cmd(cmd
, &ent
);
1017 spin_lock_irqsave(&smmu
->cmdq
.lock
, flags
);
1018 arm_smmu_cmdq_insert_cmd(smmu
, cmd
);
1019 spin_unlock_irqrestore(&smmu
->cmdq
.lock
, flags
);
1021 return __arm_smmu_sync_poll_msi(smmu
, ent
.sync
.msidata
);
1024 static int __arm_smmu_cmdq_issue_sync(struct arm_smmu_device
*smmu
)
1026 u64 cmd
[CMDQ_ENT_DWORDS
];
1027 unsigned long flags
;
1028 bool wfe
= !!(smmu
->features
& ARM_SMMU_FEAT_SEV
);
1029 struct arm_smmu_cmdq_ent ent
= { .opcode
= CMDQ_OP_CMD_SYNC
};
1032 arm_smmu_cmdq_build_cmd(cmd
, &ent
);
1034 spin_lock_irqsave(&smmu
->cmdq
.lock
, flags
);
1035 arm_smmu_cmdq_insert_cmd(smmu
, cmd
);
1036 ret
= queue_poll_cons(&smmu
->cmdq
.q
, true, wfe
);
1037 spin_unlock_irqrestore(&smmu
->cmdq
.lock
, flags
);
1042 static void arm_smmu_cmdq_issue_sync(struct arm_smmu_device
*smmu
)
1045 bool msi
= (smmu
->features
& ARM_SMMU_FEAT_MSI
) &&
1046 (smmu
->features
& ARM_SMMU_FEAT_COHERENCY
);
1048 ret
= msi
? __arm_smmu_cmdq_issue_sync_msi(smmu
)
1049 : __arm_smmu_cmdq_issue_sync(smmu
);
1051 dev_err_ratelimited(smmu
->dev
, "CMD_SYNC timeout\n");
1054 /* Context descriptor manipulation functions */
1055 static u64
arm_smmu_cpu_tcr_to_cd(u64 tcr
)
1059 /* Repack the TCR. Just care about TTBR0 for now */
1060 val
|= ARM_SMMU_TCR2CD(tcr
, T0SZ
);
1061 val
|= ARM_SMMU_TCR2CD(tcr
, TG0
);
1062 val
|= ARM_SMMU_TCR2CD(tcr
, IRGN0
);
1063 val
|= ARM_SMMU_TCR2CD(tcr
, ORGN0
);
1064 val
|= ARM_SMMU_TCR2CD(tcr
, SH0
);
1065 val
|= ARM_SMMU_TCR2CD(tcr
, EPD0
);
1066 val
|= ARM_SMMU_TCR2CD(tcr
, EPD1
);
1067 val
|= ARM_SMMU_TCR2CD(tcr
, IPS
);
1068 val
|= ARM_SMMU_TCR2CD(tcr
, TBI0
);
1073 static void arm_smmu_write_ctx_desc(struct arm_smmu_device
*smmu
,
1074 struct arm_smmu_s1_cfg
*cfg
)
1079 * We don't need to issue any invalidation here, as we'll invalidate
1080 * the STE when installing the new entry anyway.
1082 val
= arm_smmu_cpu_tcr_to_cd(cfg
->cd
.tcr
) |
1086 CTXDESC_CD_0_R
| CTXDESC_CD_0_A
| CTXDESC_CD_0_ASET_PRIVATE
|
1087 CTXDESC_CD_0_AA64
| (u64
)cfg
->cd
.asid
<< CTXDESC_CD_0_ASID_SHIFT
|
1090 /* STALL_MODEL==0b10 && CD.S==0 is ILLEGAL */
1091 if (smmu
->features
& ARM_SMMU_FEAT_STALL_FORCE
)
1092 val
|= CTXDESC_CD_0_S
;
1094 cfg
->cdptr
[0] = cpu_to_le64(val
);
1096 val
= cfg
->cd
.ttbr
& CTXDESC_CD_1_TTB0_MASK
<< CTXDESC_CD_1_TTB0_SHIFT
;
1097 cfg
->cdptr
[1] = cpu_to_le64(val
);
1099 cfg
->cdptr
[3] = cpu_to_le64(cfg
->cd
.mair
<< CTXDESC_CD_3_MAIR_SHIFT
);
1102 /* Stream table manipulation functions */
1104 arm_smmu_write_strtab_l1_desc(__le64
*dst
, struct arm_smmu_strtab_l1_desc
*desc
)
1108 val
|= (desc
->span
& STRTAB_L1_DESC_SPAN_MASK
)
1109 << STRTAB_L1_DESC_SPAN_SHIFT
;
1110 val
|= desc
->l2ptr_dma
&
1111 STRTAB_L1_DESC_L2PTR_MASK
<< STRTAB_L1_DESC_L2PTR_SHIFT
;
1113 *dst
= cpu_to_le64(val
);
1116 static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device
*smmu
, u32 sid
)
1118 struct arm_smmu_cmdq_ent cmd
= {
1119 .opcode
= CMDQ_OP_CFGI_STE
,
1126 arm_smmu_cmdq_issue_cmd(smmu
, &cmd
);
1127 arm_smmu_cmdq_issue_sync(smmu
);
1130 static void arm_smmu_write_strtab_ent(struct arm_smmu_device
*smmu
, u32 sid
,
1131 __le64
*dst
, struct arm_smmu_strtab_ent
*ste
)
1134 * This is hideously complicated, but we only really care about
1135 * three cases at the moment:
1137 * 1. Invalid (all zero) -> bypass/fault (init)
1138 * 2. Bypass/fault -> translation/bypass (attach)
1139 * 3. Translation/bypass -> bypass/fault (detach)
1141 * Given that we can't update the STE atomically and the SMMU
1142 * doesn't read the thing in a defined order, that leaves us
1143 * with the following maintenance requirements:
1145 * 1. Update Config, return (init time STEs aren't live)
1146 * 2. Write everything apart from dword 0, sync, write dword 0, sync
1147 * 3. Update Config, sync
1149 u64 val
= le64_to_cpu(dst
[0]);
1150 bool ste_live
= false;
1151 struct arm_smmu_cmdq_ent prefetch_cmd
= {
1152 .opcode
= CMDQ_OP_PREFETCH_CFG
,
1158 if (val
& STRTAB_STE_0_V
) {
1161 cfg
= val
& STRTAB_STE_0_CFG_MASK
<< STRTAB_STE_0_CFG_SHIFT
;
1163 case STRTAB_STE_0_CFG_BYPASS
:
1165 case STRTAB_STE_0_CFG_S1_TRANS
:
1166 case STRTAB_STE_0_CFG_S2_TRANS
:
1169 case STRTAB_STE_0_CFG_ABORT
:
1173 BUG(); /* STE corruption */
1177 /* Nuke the existing STE_0 value, as we're going to rewrite it */
1178 val
= STRTAB_STE_0_V
;
1181 if (!ste
->assigned
|| !(ste
->s1_cfg
|| ste
->s2_cfg
)) {
1182 if (!ste
->assigned
&& disable_bypass
)
1183 val
|= STRTAB_STE_0_CFG_ABORT
;
1185 val
|= STRTAB_STE_0_CFG_BYPASS
;
1187 dst
[0] = cpu_to_le64(val
);
1188 dst
[1] = cpu_to_le64(STRTAB_STE_1_SHCFG_INCOMING
1189 << STRTAB_STE_1_SHCFG_SHIFT
);
1190 dst
[2] = 0; /* Nuke the VMID */
1192 * The SMMU can perform negative caching, so we must sync
1193 * the STE regardless of whether the old value was live.
1196 arm_smmu_sync_ste_for_sid(smmu
, sid
);
1202 dst
[1] = cpu_to_le64(
1203 STRTAB_STE_1_S1C_CACHE_WBRA
1204 << STRTAB_STE_1_S1CIR_SHIFT
|
1205 STRTAB_STE_1_S1C_CACHE_WBRA
1206 << STRTAB_STE_1_S1COR_SHIFT
|
1207 STRTAB_STE_1_S1C_SH_ISH
<< STRTAB_STE_1_S1CSH_SHIFT
|
1208 #ifdef CONFIG_PCI_ATS
1209 STRTAB_STE_1_EATS_TRANS
<< STRTAB_STE_1_EATS_SHIFT
|
1211 STRTAB_STE_1_STRW_NSEL1
<< STRTAB_STE_1_STRW_SHIFT
);
1213 if (smmu
->features
& ARM_SMMU_FEAT_STALLS
&&
1214 !(smmu
->features
& ARM_SMMU_FEAT_STALL_FORCE
))
1215 dst
[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD
);
1217 val
|= (ste
->s1_cfg
->cdptr_dma
& STRTAB_STE_0_S1CTXPTR_MASK
1218 << STRTAB_STE_0_S1CTXPTR_SHIFT
) |
1219 STRTAB_STE_0_CFG_S1_TRANS
;
1224 dst
[2] = cpu_to_le64(
1225 ste
->s2_cfg
->vmid
<< STRTAB_STE_2_S2VMID_SHIFT
|
1226 (ste
->s2_cfg
->vtcr
& STRTAB_STE_2_VTCR_MASK
)
1227 << STRTAB_STE_2_VTCR_SHIFT
|
1229 STRTAB_STE_2_S2ENDI
|
1231 STRTAB_STE_2_S2PTW
| STRTAB_STE_2_S2AA64
|
1234 dst
[3] = cpu_to_le64(ste
->s2_cfg
->vttbr
&
1235 STRTAB_STE_3_S2TTB_MASK
<< STRTAB_STE_3_S2TTB_SHIFT
);
1237 val
|= STRTAB_STE_0_CFG_S2_TRANS
;
1240 arm_smmu_sync_ste_for_sid(smmu
, sid
);
1241 dst
[0] = cpu_to_le64(val
);
1242 arm_smmu_sync_ste_for_sid(smmu
, sid
);
1244 /* It's likely that we'll want to use the new STE soon */
1245 if (!(smmu
->options
& ARM_SMMU_OPT_SKIP_PREFETCH
))
1246 arm_smmu_cmdq_issue_cmd(smmu
, &prefetch_cmd
);
1249 static void arm_smmu_init_bypass_stes(u64
*strtab
, unsigned int nent
)
1252 struct arm_smmu_strtab_ent ste
= { .assigned
= false };
1254 for (i
= 0; i
< nent
; ++i
) {
1255 arm_smmu_write_strtab_ent(NULL
, -1, strtab
, &ste
);
1256 strtab
+= STRTAB_STE_DWORDS
;
1260 static int arm_smmu_init_l2_strtab(struct arm_smmu_device
*smmu
, u32 sid
)
1264 struct arm_smmu_strtab_cfg
*cfg
= &smmu
->strtab_cfg
;
1265 struct arm_smmu_strtab_l1_desc
*desc
= &cfg
->l1_desc
[sid
>> STRTAB_SPLIT
];
1270 size
= 1 << (STRTAB_SPLIT
+ ilog2(STRTAB_STE_DWORDS
) + 3);
1271 strtab
= &cfg
->strtab
[(sid
>> STRTAB_SPLIT
) * STRTAB_L1_DESC_DWORDS
];
1273 desc
->span
= STRTAB_SPLIT
+ 1;
1274 desc
->l2ptr
= dmam_alloc_coherent(smmu
->dev
, size
, &desc
->l2ptr_dma
,
1275 GFP_KERNEL
| __GFP_ZERO
);
1278 "failed to allocate l2 stream table for SID %u\n",
1283 arm_smmu_init_bypass_stes(desc
->l2ptr
, 1 << STRTAB_SPLIT
);
1284 arm_smmu_write_strtab_l1_desc(strtab
, desc
);
1288 /* IRQ and event handlers */
1289 static irqreturn_t
arm_smmu_evtq_thread(int irq
, void *dev
)
1292 struct arm_smmu_device
*smmu
= dev
;
1293 struct arm_smmu_queue
*q
= &smmu
->evtq
.q
;
1294 u64 evt
[EVTQ_ENT_DWORDS
];
1297 while (!queue_remove_raw(q
, evt
)) {
1298 u8 id
= evt
[0] >> EVTQ_0_ID_SHIFT
& EVTQ_0_ID_MASK
;
1300 dev_info(smmu
->dev
, "event 0x%02x received:\n", id
);
1301 for (i
= 0; i
< ARRAY_SIZE(evt
); ++i
)
1302 dev_info(smmu
->dev
, "\t0x%016llx\n",
1303 (unsigned long long)evt
[i
]);
1308 * Not much we can do on overflow, so scream and pretend we're
1311 if (queue_sync_prod(q
) == -EOVERFLOW
)
1312 dev_err(smmu
->dev
, "EVTQ overflow detected -- events lost\n");
1313 } while (!queue_empty(q
));
1315 /* Sync our overflow flag, as we believe we're up to speed */
1316 q
->cons
= Q_OVF(q
, q
->prod
) | Q_WRP(q
, q
->cons
) | Q_IDX(q
, q
->cons
);
1320 static void arm_smmu_handle_ppr(struct arm_smmu_device
*smmu
, u64
*evt
)
1326 sid
= evt
[0] >> PRIQ_0_SID_SHIFT
& PRIQ_0_SID_MASK
;
1327 ssv
= evt
[0] & PRIQ_0_SSID_V
;
1328 ssid
= ssv
? evt
[0] >> PRIQ_0_SSID_SHIFT
& PRIQ_0_SSID_MASK
: 0;
1329 last
= evt
[0] & PRIQ_0_PRG_LAST
;
1330 grpid
= evt
[1] >> PRIQ_1_PRG_IDX_SHIFT
& PRIQ_1_PRG_IDX_MASK
;
1332 dev_info(smmu
->dev
, "unexpected PRI request received:\n");
1334 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
1335 sid
, ssid
, grpid
, last
? "L" : "",
1336 evt
[0] & PRIQ_0_PERM_PRIV
? "" : "un",
1337 evt
[0] & PRIQ_0_PERM_READ
? "R" : "",
1338 evt
[0] & PRIQ_0_PERM_WRITE
? "W" : "",
1339 evt
[0] & PRIQ_0_PERM_EXEC
? "X" : "",
1340 evt
[1] & PRIQ_1_ADDR_MASK
<< PRIQ_1_ADDR_SHIFT
);
1343 struct arm_smmu_cmdq_ent cmd
= {
1344 .opcode
= CMDQ_OP_PRI_RESP
,
1345 .substream_valid
= ssv
,
1350 .resp
= PRI_RESP_DENY
,
1354 arm_smmu_cmdq_issue_cmd(smmu
, &cmd
);
1358 static irqreturn_t
arm_smmu_priq_thread(int irq
, void *dev
)
1360 struct arm_smmu_device
*smmu
= dev
;
1361 struct arm_smmu_queue
*q
= &smmu
->priq
.q
;
1362 u64 evt
[PRIQ_ENT_DWORDS
];
1365 while (!queue_remove_raw(q
, evt
))
1366 arm_smmu_handle_ppr(smmu
, evt
);
1368 if (queue_sync_prod(q
) == -EOVERFLOW
)
1369 dev_err(smmu
->dev
, "PRIQ overflow detected -- requests lost\n");
1370 } while (!queue_empty(q
));
1372 /* Sync our overflow flag, as we believe we're up to speed */
1373 q
->cons
= Q_OVF(q
, q
->prod
) | Q_WRP(q
, q
->cons
) | Q_IDX(q
, q
->cons
);
1377 static int arm_smmu_device_disable(struct arm_smmu_device
*smmu
);
1379 static irqreturn_t
arm_smmu_gerror_handler(int irq
, void *dev
)
1381 u32 gerror
, gerrorn
, active
;
1382 struct arm_smmu_device
*smmu
= dev
;
1384 gerror
= readl_relaxed(smmu
->base
+ ARM_SMMU_GERROR
);
1385 gerrorn
= readl_relaxed(smmu
->base
+ ARM_SMMU_GERRORN
);
1387 active
= gerror
^ gerrorn
;
1388 if (!(active
& GERROR_ERR_MASK
))
1389 return IRQ_NONE
; /* No errors pending */
1392 "unexpected global error reported (0x%08x), this could be serious\n",
1395 if (active
& GERROR_SFM_ERR
) {
1396 dev_err(smmu
->dev
, "device has entered Service Failure Mode!\n");
1397 arm_smmu_device_disable(smmu
);
1400 if (active
& GERROR_MSI_GERROR_ABT_ERR
)
1401 dev_warn(smmu
->dev
, "GERROR MSI write aborted\n");
1403 if (active
& GERROR_MSI_PRIQ_ABT_ERR
)
1404 dev_warn(smmu
->dev
, "PRIQ MSI write aborted\n");
1406 if (active
& GERROR_MSI_EVTQ_ABT_ERR
)
1407 dev_warn(smmu
->dev
, "EVTQ MSI write aborted\n");
1409 if (active
& GERROR_MSI_CMDQ_ABT_ERR
)
1410 dev_warn(smmu
->dev
, "CMDQ MSI write aborted\n");
1412 if (active
& GERROR_PRIQ_ABT_ERR
)
1413 dev_err(smmu
->dev
, "PRIQ write aborted -- events may have been lost\n");
1415 if (active
& GERROR_EVTQ_ABT_ERR
)
1416 dev_err(smmu
->dev
, "EVTQ write aborted -- events may have been lost\n");
1418 if (active
& GERROR_CMDQ_ERR
)
1419 arm_smmu_cmdq_skip_err(smmu
);
1421 writel(gerror
, smmu
->base
+ ARM_SMMU_GERRORN
);
1425 static irqreturn_t
arm_smmu_combined_irq_thread(int irq
, void *dev
)
1427 struct arm_smmu_device
*smmu
= dev
;
1429 arm_smmu_evtq_thread(irq
, dev
);
1430 if (smmu
->features
& ARM_SMMU_FEAT_PRI
)
1431 arm_smmu_priq_thread(irq
, dev
);
1436 static irqreturn_t
arm_smmu_combined_irq_handler(int irq
, void *dev
)
1438 arm_smmu_gerror_handler(irq
, dev
);
1439 return IRQ_WAKE_THREAD
;
1442 /* IO_PGTABLE API */
1443 static void __arm_smmu_tlb_sync(struct arm_smmu_device
*smmu
)
1445 arm_smmu_cmdq_issue_sync(smmu
);
1448 static void arm_smmu_tlb_sync(void *cookie
)
1450 struct arm_smmu_domain
*smmu_domain
= cookie
;
1451 __arm_smmu_tlb_sync(smmu_domain
->smmu
);
1454 static void arm_smmu_tlb_inv_context(void *cookie
)
1456 struct arm_smmu_domain
*smmu_domain
= cookie
;
1457 struct arm_smmu_device
*smmu
= smmu_domain
->smmu
;
1458 struct arm_smmu_cmdq_ent cmd
;
1460 if (smmu_domain
->stage
== ARM_SMMU_DOMAIN_S1
) {
1461 cmd
.opcode
= CMDQ_OP_TLBI_NH_ASID
;
1462 cmd
.tlbi
.asid
= smmu_domain
->s1_cfg
.cd
.asid
;
1465 cmd
.opcode
= CMDQ_OP_TLBI_S12_VMALL
;
1466 cmd
.tlbi
.vmid
= smmu_domain
->s2_cfg
.vmid
;
1469 arm_smmu_cmdq_issue_cmd(smmu
, &cmd
);
1470 __arm_smmu_tlb_sync(smmu
);
1473 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova
, size_t size
,
1474 size_t granule
, bool leaf
, void *cookie
)
1476 struct arm_smmu_domain
*smmu_domain
= cookie
;
1477 struct arm_smmu_device
*smmu
= smmu_domain
->smmu
;
1478 struct arm_smmu_cmdq_ent cmd
= {
1485 if (smmu_domain
->stage
== ARM_SMMU_DOMAIN_S1
) {
1486 cmd
.opcode
= CMDQ_OP_TLBI_NH_VA
;
1487 cmd
.tlbi
.asid
= smmu_domain
->s1_cfg
.cd
.asid
;
1489 cmd
.opcode
= CMDQ_OP_TLBI_S2_IPA
;
1490 cmd
.tlbi
.vmid
= smmu_domain
->s2_cfg
.vmid
;
1494 arm_smmu_cmdq_issue_cmd(smmu
, &cmd
);
1495 cmd
.tlbi
.addr
+= granule
;
1496 } while (size
-= granule
);
1499 static const struct iommu_gather_ops arm_smmu_gather_ops
= {
1500 .tlb_flush_all
= arm_smmu_tlb_inv_context
,
1501 .tlb_add_flush
= arm_smmu_tlb_inv_range_nosync
,
1502 .tlb_sync
= arm_smmu_tlb_sync
,
1506 static bool arm_smmu_capable(enum iommu_cap cap
)
1509 case IOMMU_CAP_CACHE_COHERENCY
:
1511 case IOMMU_CAP_NOEXEC
:
1518 static struct iommu_domain
*arm_smmu_domain_alloc(unsigned type
)
1520 struct arm_smmu_domain
*smmu_domain
;
1522 if (type
!= IOMMU_DOMAIN_UNMANAGED
&&
1523 type
!= IOMMU_DOMAIN_DMA
&&
1524 type
!= IOMMU_DOMAIN_IDENTITY
)
1528 * Allocate the domain and initialise some of its data structures.
1529 * We can't really do anything meaningful until we've added a
1532 smmu_domain
= kzalloc(sizeof(*smmu_domain
), GFP_KERNEL
);
1536 if (type
== IOMMU_DOMAIN_DMA
&&
1537 iommu_get_dma_cookie(&smmu_domain
->domain
)) {
1542 mutex_init(&smmu_domain
->init_mutex
);
1543 return &smmu_domain
->domain
;
1546 static int arm_smmu_bitmap_alloc(unsigned long *map
, int span
)
1548 int idx
, size
= 1 << span
;
1551 idx
= find_first_zero_bit(map
, size
);
1554 } while (test_and_set_bit(idx
, map
));
1559 static void arm_smmu_bitmap_free(unsigned long *map
, int idx
)
1561 clear_bit(idx
, map
);
1564 static void arm_smmu_domain_free(struct iommu_domain
*domain
)
1566 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1567 struct arm_smmu_device
*smmu
= smmu_domain
->smmu
;
1569 iommu_put_dma_cookie(domain
);
1570 free_io_pgtable_ops(smmu_domain
->pgtbl_ops
);
1572 /* Free the CD and ASID, if we allocated them */
1573 if (smmu_domain
->stage
== ARM_SMMU_DOMAIN_S1
) {
1574 struct arm_smmu_s1_cfg
*cfg
= &smmu_domain
->s1_cfg
;
1577 dmam_free_coherent(smmu_domain
->smmu
->dev
,
1578 CTXDESC_CD_DWORDS
<< 3,
1582 arm_smmu_bitmap_free(smmu
->asid_map
, cfg
->cd
.asid
);
1585 struct arm_smmu_s2_cfg
*cfg
= &smmu_domain
->s2_cfg
;
1587 arm_smmu_bitmap_free(smmu
->vmid_map
, cfg
->vmid
);
1593 static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain
*smmu_domain
,
1594 struct io_pgtable_cfg
*pgtbl_cfg
)
1598 struct arm_smmu_device
*smmu
= smmu_domain
->smmu
;
1599 struct arm_smmu_s1_cfg
*cfg
= &smmu_domain
->s1_cfg
;
1601 asid
= arm_smmu_bitmap_alloc(smmu
->asid_map
, smmu
->asid_bits
);
1605 cfg
->cdptr
= dmam_alloc_coherent(smmu
->dev
, CTXDESC_CD_DWORDS
<< 3,
1607 GFP_KERNEL
| __GFP_ZERO
);
1609 dev_warn(smmu
->dev
, "failed to allocate context descriptor\n");
1614 cfg
->cd
.asid
= (u16
)asid
;
1615 cfg
->cd
.ttbr
= pgtbl_cfg
->arm_lpae_s1_cfg
.ttbr
[0];
1616 cfg
->cd
.tcr
= pgtbl_cfg
->arm_lpae_s1_cfg
.tcr
;
1617 cfg
->cd
.mair
= pgtbl_cfg
->arm_lpae_s1_cfg
.mair
[0];
1621 arm_smmu_bitmap_free(smmu
->asid_map
, asid
);
1625 static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain
*smmu_domain
,
1626 struct io_pgtable_cfg
*pgtbl_cfg
)
1629 struct arm_smmu_device
*smmu
= smmu_domain
->smmu
;
1630 struct arm_smmu_s2_cfg
*cfg
= &smmu_domain
->s2_cfg
;
1632 vmid
= arm_smmu_bitmap_alloc(smmu
->vmid_map
, smmu
->vmid_bits
);
1636 cfg
->vmid
= (u16
)vmid
;
1637 cfg
->vttbr
= pgtbl_cfg
->arm_lpae_s2_cfg
.vttbr
;
1638 cfg
->vtcr
= pgtbl_cfg
->arm_lpae_s2_cfg
.vtcr
;
1642 static int arm_smmu_domain_finalise(struct iommu_domain
*domain
)
1645 unsigned long ias
, oas
;
1646 enum io_pgtable_fmt fmt
;
1647 struct io_pgtable_cfg pgtbl_cfg
;
1648 struct io_pgtable_ops
*pgtbl_ops
;
1649 int (*finalise_stage_fn
)(struct arm_smmu_domain
*,
1650 struct io_pgtable_cfg
*);
1651 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1652 struct arm_smmu_device
*smmu
= smmu_domain
->smmu
;
1654 if (domain
->type
== IOMMU_DOMAIN_IDENTITY
) {
1655 smmu_domain
->stage
= ARM_SMMU_DOMAIN_BYPASS
;
1659 /* Restrict the stage to what we can actually support */
1660 if (!(smmu
->features
& ARM_SMMU_FEAT_TRANS_S1
))
1661 smmu_domain
->stage
= ARM_SMMU_DOMAIN_S2
;
1662 if (!(smmu
->features
& ARM_SMMU_FEAT_TRANS_S2
))
1663 smmu_domain
->stage
= ARM_SMMU_DOMAIN_S1
;
1665 switch (smmu_domain
->stage
) {
1666 case ARM_SMMU_DOMAIN_S1
:
1669 fmt
= ARM_64_LPAE_S1
;
1670 finalise_stage_fn
= arm_smmu_domain_finalise_s1
;
1672 case ARM_SMMU_DOMAIN_NESTED
:
1673 case ARM_SMMU_DOMAIN_S2
:
1676 fmt
= ARM_64_LPAE_S2
;
1677 finalise_stage_fn
= arm_smmu_domain_finalise_s2
;
1683 pgtbl_cfg
= (struct io_pgtable_cfg
) {
1684 .pgsize_bitmap
= smmu
->pgsize_bitmap
,
1687 .tlb
= &arm_smmu_gather_ops
,
1688 .iommu_dev
= smmu
->dev
,
1691 if (smmu
->features
& ARM_SMMU_FEAT_COHERENCY
)
1692 pgtbl_cfg
.quirks
= IO_PGTABLE_QUIRK_NO_DMA
;
1694 pgtbl_ops
= alloc_io_pgtable_ops(fmt
, &pgtbl_cfg
, smmu_domain
);
1698 domain
->pgsize_bitmap
= pgtbl_cfg
.pgsize_bitmap
;
1699 domain
->geometry
.aperture_end
= (1UL << ias
) - 1;
1700 domain
->geometry
.force_aperture
= true;
1702 ret
= finalise_stage_fn(smmu_domain
, &pgtbl_cfg
);
1704 free_io_pgtable_ops(pgtbl_ops
);
1708 smmu_domain
->pgtbl_ops
= pgtbl_ops
;
1712 static __le64
*arm_smmu_get_step_for_sid(struct arm_smmu_device
*smmu
, u32 sid
)
1715 struct arm_smmu_strtab_cfg
*cfg
= &smmu
->strtab_cfg
;
1717 if (smmu
->features
& ARM_SMMU_FEAT_2_LVL_STRTAB
) {
1718 struct arm_smmu_strtab_l1_desc
*l1_desc
;
1721 /* Two-level walk */
1722 idx
= (sid
>> STRTAB_SPLIT
) * STRTAB_L1_DESC_DWORDS
;
1723 l1_desc
= &cfg
->l1_desc
[idx
];
1724 idx
= (sid
& ((1 << STRTAB_SPLIT
) - 1)) * STRTAB_STE_DWORDS
;
1725 step
= &l1_desc
->l2ptr
[idx
];
1727 /* Simple linear lookup */
1728 step
= &cfg
->strtab
[sid
* STRTAB_STE_DWORDS
];
1734 static void arm_smmu_install_ste_for_dev(struct iommu_fwspec
*fwspec
)
1737 struct arm_smmu_master_data
*master
= fwspec
->iommu_priv
;
1738 struct arm_smmu_device
*smmu
= master
->smmu
;
1740 for (i
= 0; i
< fwspec
->num_ids
; ++i
) {
1741 u32 sid
= fwspec
->ids
[i
];
1742 __le64
*step
= arm_smmu_get_step_for_sid(smmu
, sid
);
1744 arm_smmu_write_strtab_ent(smmu
, sid
, step
, &master
->ste
);
1748 static void arm_smmu_detach_dev(struct device
*dev
)
1750 struct arm_smmu_master_data
*master
= dev
->iommu_fwspec
->iommu_priv
;
1752 master
->ste
.assigned
= false;
1753 arm_smmu_install_ste_for_dev(dev
->iommu_fwspec
);
1756 static int arm_smmu_attach_dev(struct iommu_domain
*domain
, struct device
*dev
)
1759 struct arm_smmu_device
*smmu
;
1760 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1761 struct arm_smmu_master_data
*master
;
1762 struct arm_smmu_strtab_ent
*ste
;
1764 if (!dev
->iommu_fwspec
)
1767 master
= dev
->iommu_fwspec
->iommu_priv
;
1768 smmu
= master
->smmu
;
1771 /* Already attached to a different domain? */
1773 arm_smmu_detach_dev(dev
);
1775 mutex_lock(&smmu_domain
->init_mutex
);
1777 if (!smmu_domain
->smmu
) {
1778 smmu_domain
->smmu
= smmu
;
1779 ret
= arm_smmu_domain_finalise(domain
);
1781 smmu_domain
->smmu
= NULL
;
1784 } else if (smmu_domain
->smmu
!= smmu
) {
1786 "cannot attach to SMMU %s (upstream of %s)\n",
1787 dev_name(smmu_domain
->smmu
->dev
),
1788 dev_name(smmu
->dev
));
1793 ste
->assigned
= true;
1795 if (smmu_domain
->stage
== ARM_SMMU_DOMAIN_BYPASS
) {
1798 } else if (smmu_domain
->stage
== ARM_SMMU_DOMAIN_S1
) {
1799 ste
->s1_cfg
= &smmu_domain
->s1_cfg
;
1801 arm_smmu_write_ctx_desc(smmu
, ste
->s1_cfg
);
1804 ste
->s2_cfg
= &smmu_domain
->s2_cfg
;
1807 arm_smmu_install_ste_for_dev(dev
->iommu_fwspec
);
1809 mutex_unlock(&smmu_domain
->init_mutex
);
1813 static int arm_smmu_map(struct iommu_domain
*domain
, unsigned long iova
,
1814 phys_addr_t paddr
, size_t size
, int prot
)
1816 struct io_pgtable_ops
*ops
= to_smmu_domain(domain
)->pgtbl_ops
;
1821 return ops
->map(ops
, iova
, paddr
, size
, prot
);
1825 arm_smmu_unmap(struct iommu_domain
*domain
, unsigned long iova
, size_t size
)
1827 struct io_pgtable_ops
*ops
= to_smmu_domain(domain
)->pgtbl_ops
;
1832 return ops
->unmap(ops
, iova
, size
);
1835 static void arm_smmu_iotlb_sync(struct iommu_domain
*domain
)
1837 struct arm_smmu_device
*smmu
= to_smmu_domain(domain
)->smmu
;
1840 __arm_smmu_tlb_sync(smmu
);
1844 arm_smmu_iova_to_phys(struct iommu_domain
*domain
, dma_addr_t iova
)
1846 struct io_pgtable_ops
*ops
= to_smmu_domain(domain
)->pgtbl_ops
;
1848 if (domain
->type
== IOMMU_DOMAIN_IDENTITY
)
1854 return ops
->iova_to_phys(ops
, iova
);
1857 static struct platform_driver arm_smmu_driver
;
1859 static int arm_smmu_match_node(struct device
*dev
, void *data
)
1861 return dev
->fwnode
== data
;
1865 struct arm_smmu_device
*arm_smmu_get_by_fwnode(struct fwnode_handle
*fwnode
)
1867 struct device
*dev
= driver_find_device(&arm_smmu_driver
.driver
, NULL
,
1868 fwnode
, arm_smmu_match_node
);
1870 return dev
? dev_get_drvdata(dev
) : NULL
;
1873 static bool arm_smmu_sid_in_range(struct arm_smmu_device
*smmu
, u32 sid
)
1875 unsigned long limit
= smmu
->strtab_cfg
.num_l1_ents
;
1877 if (smmu
->features
& ARM_SMMU_FEAT_2_LVL_STRTAB
)
1878 limit
*= 1UL << STRTAB_SPLIT
;
1883 static struct iommu_ops arm_smmu_ops
;
1885 static int arm_smmu_add_device(struct device
*dev
)
1888 struct arm_smmu_device
*smmu
;
1889 struct arm_smmu_master_data
*master
;
1890 struct iommu_fwspec
*fwspec
= dev
->iommu_fwspec
;
1891 struct iommu_group
*group
;
1893 if (!fwspec
|| fwspec
->ops
!= &arm_smmu_ops
)
1896 * We _can_ actually withstand dodgy bus code re-calling add_device()
1897 * without an intervening remove_device()/of_xlate() sequence, but
1898 * we're not going to do so quietly...
1900 if (WARN_ON_ONCE(fwspec
->iommu_priv
)) {
1901 master
= fwspec
->iommu_priv
;
1902 smmu
= master
->smmu
;
1904 smmu
= arm_smmu_get_by_fwnode(fwspec
->iommu_fwnode
);
1907 master
= kzalloc(sizeof(*master
), GFP_KERNEL
);
1911 master
->smmu
= smmu
;
1912 fwspec
->iommu_priv
= master
;
1915 /* Check the SIDs are in range of the SMMU and our stream table */
1916 for (i
= 0; i
< fwspec
->num_ids
; i
++) {
1917 u32 sid
= fwspec
->ids
[i
];
1919 if (!arm_smmu_sid_in_range(smmu
, sid
))
1922 /* Ensure l2 strtab is initialised */
1923 if (smmu
->features
& ARM_SMMU_FEAT_2_LVL_STRTAB
) {
1924 ret
= arm_smmu_init_l2_strtab(smmu
, sid
);
1930 group
= iommu_group_get_for_dev(dev
);
1931 if (!IS_ERR(group
)) {
1932 iommu_group_put(group
);
1933 iommu_device_link(&smmu
->iommu
, dev
);
1936 return PTR_ERR_OR_ZERO(group
);
1939 static void arm_smmu_remove_device(struct device
*dev
)
1941 struct iommu_fwspec
*fwspec
= dev
->iommu_fwspec
;
1942 struct arm_smmu_master_data
*master
;
1943 struct arm_smmu_device
*smmu
;
1945 if (!fwspec
|| fwspec
->ops
!= &arm_smmu_ops
)
1948 master
= fwspec
->iommu_priv
;
1949 smmu
= master
->smmu
;
1950 if (master
&& master
->ste
.assigned
)
1951 arm_smmu_detach_dev(dev
);
1952 iommu_group_remove_device(dev
);
1953 iommu_device_unlink(&smmu
->iommu
, dev
);
1955 iommu_fwspec_free(dev
);
1958 static struct iommu_group
*arm_smmu_device_group(struct device
*dev
)
1960 struct iommu_group
*group
;
1963 * We don't support devices sharing stream IDs other than PCI RID
1964 * aliases, since the necessary ID-to-device lookup becomes rather
1965 * impractical given a potential sparse 32-bit stream ID space.
1967 if (dev_is_pci(dev
))
1968 group
= pci_device_group(dev
);
1970 group
= generic_device_group(dev
);
1975 static int arm_smmu_domain_get_attr(struct iommu_domain
*domain
,
1976 enum iommu_attr attr
, void *data
)
1978 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1980 if (domain
->type
!= IOMMU_DOMAIN_UNMANAGED
)
1984 case DOMAIN_ATTR_NESTING
:
1985 *(int *)data
= (smmu_domain
->stage
== ARM_SMMU_DOMAIN_NESTED
);
1992 static int arm_smmu_domain_set_attr(struct iommu_domain
*domain
,
1993 enum iommu_attr attr
, void *data
)
1996 struct arm_smmu_domain
*smmu_domain
= to_smmu_domain(domain
);
1998 if (domain
->type
!= IOMMU_DOMAIN_UNMANAGED
)
2001 mutex_lock(&smmu_domain
->init_mutex
);
2004 case DOMAIN_ATTR_NESTING
:
2005 if (smmu_domain
->smmu
) {
2011 smmu_domain
->stage
= ARM_SMMU_DOMAIN_NESTED
;
2013 smmu_domain
->stage
= ARM_SMMU_DOMAIN_S1
;
2021 mutex_unlock(&smmu_domain
->init_mutex
);
2025 static int arm_smmu_of_xlate(struct device
*dev
, struct of_phandle_args
*args
)
2027 return iommu_fwspec_add_ids(dev
, args
->args
, 1);
2030 static void arm_smmu_get_resv_regions(struct device
*dev
,
2031 struct list_head
*head
)
2033 struct iommu_resv_region
*region
;
2034 int prot
= IOMMU_WRITE
| IOMMU_NOEXEC
| IOMMU_MMIO
;
2036 region
= iommu_alloc_resv_region(MSI_IOVA_BASE
, MSI_IOVA_LENGTH
,
2037 prot
, IOMMU_RESV_SW_MSI
);
2041 list_add_tail(®ion
->list
, head
);
2043 iommu_dma_get_resv_regions(dev
, head
);
2046 static void arm_smmu_put_resv_regions(struct device
*dev
,
2047 struct list_head
*head
)
2049 struct iommu_resv_region
*entry
, *next
;
2051 list_for_each_entry_safe(entry
, next
, head
, list
)
2055 static struct iommu_ops arm_smmu_ops
= {
2056 .capable
= arm_smmu_capable
,
2057 .domain_alloc
= arm_smmu_domain_alloc
,
2058 .domain_free
= arm_smmu_domain_free
,
2059 .attach_dev
= arm_smmu_attach_dev
,
2060 .map
= arm_smmu_map
,
2061 .unmap
= arm_smmu_unmap
,
2062 .map_sg
= default_iommu_map_sg
,
2063 .flush_iotlb_all
= arm_smmu_iotlb_sync
,
2064 .iotlb_sync
= arm_smmu_iotlb_sync
,
2065 .iova_to_phys
= arm_smmu_iova_to_phys
,
2066 .add_device
= arm_smmu_add_device
,
2067 .remove_device
= arm_smmu_remove_device
,
2068 .device_group
= arm_smmu_device_group
,
2069 .domain_get_attr
= arm_smmu_domain_get_attr
,
2070 .domain_set_attr
= arm_smmu_domain_set_attr
,
2071 .of_xlate
= arm_smmu_of_xlate
,
2072 .get_resv_regions
= arm_smmu_get_resv_regions
,
2073 .put_resv_regions
= arm_smmu_put_resv_regions
,
2074 .pgsize_bitmap
= -1UL, /* Restricted during device attach */
2077 /* Probing and initialisation functions */
2078 static int arm_smmu_init_one_queue(struct arm_smmu_device
*smmu
,
2079 struct arm_smmu_queue
*q
,
2080 unsigned long prod_off
,
2081 unsigned long cons_off
,
2084 size_t qsz
= ((1 << q
->max_n_shift
) * dwords
) << 3;
2086 q
->base
= dmam_alloc_coherent(smmu
->dev
, qsz
, &q
->base_dma
, GFP_KERNEL
);
2088 dev_err(smmu
->dev
, "failed to allocate queue (0x%zx bytes)\n",
2093 q
->prod_reg
= arm_smmu_page1_fixup(prod_off
, smmu
);
2094 q
->cons_reg
= arm_smmu_page1_fixup(cons_off
, smmu
);
2095 q
->ent_dwords
= dwords
;
2097 q
->q_base
= Q_BASE_RWA
;
2098 q
->q_base
|= q
->base_dma
& Q_BASE_ADDR_MASK
<< Q_BASE_ADDR_SHIFT
;
2099 q
->q_base
|= (q
->max_n_shift
& Q_BASE_LOG2SIZE_MASK
)
2100 << Q_BASE_LOG2SIZE_SHIFT
;
2102 q
->prod
= q
->cons
= 0;
2106 static int arm_smmu_init_queues(struct arm_smmu_device
*smmu
)
2111 spin_lock_init(&smmu
->cmdq
.lock
);
2112 ret
= arm_smmu_init_one_queue(smmu
, &smmu
->cmdq
.q
, ARM_SMMU_CMDQ_PROD
,
2113 ARM_SMMU_CMDQ_CONS
, CMDQ_ENT_DWORDS
);
2118 ret
= arm_smmu_init_one_queue(smmu
, &smmu
->evtq
.q
, ARM_SMMU_EVTQ_PROD
,
2119 ARM_SMMU_EVTQ_CONS
, EVTQ_ENT_DWORDS
);
2124 if (!(smmu
->features
& ARM_SMMU_FEAT_PRI
))
2127 return arm_smmu_init_one_queue(smmu
, &smmu
->priq
.q
, ARM_SMMU_PRIQ_PROD
,
2128 ARM_SMMU_PRIQ_CONS
, PRIQ_ENT_DWORDS
);
2131 static int arm_smmu_init_l1_strtab(struct arm_smmu_device
*smmu
)
2134 struct arm_smmu_strtab_cfg
*cfg
= &smmu
->strtab_cfg
;
2135 size_t size
= sizeof(*cfg
->l1_desc
) * cfg
->num_l1_ents
;
2136 void *strtab
= smmu
->strtab_cfg
.strtab
;
2138 cfg
->l1_desc
= devm_kzalloc(smmu
->dev
, size
, GFP_KERNEL
);
2139 if (!cfg
->l1_desc
) {
2140 dev_err(smmu
->dev
, "failed to allocate l1 stream table desc\n");
2144 for (i
= 0; i
< cfg
->num_l1_ents
; ++i
) {
2145 arm_smmu_write_strtab_l1_desc(strtab
, &cfg
->l1_desc
[i
]);
2146 strtab
+= STRTAB_L1_DESC_DWORDS
<< 3;
2152 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device
*smmu
)
2157 struct arm_smmu_strtab_cfg
*cfg
= &smmu
->strtab_cfg
;
2159 /* Calculate the L1 size, capped to the SIDSIZE. */
2160 size
= STRTAB_L1_SZ_SHIFT
- (ilog2(STRTAB_L1_DESC_DWORDS
) + 3);
2161 size
= min(size
, smmu
->sid_bits
- STRTAB_SPLIT
);
2162 cfg
->num_l1_ents
= 1 << size
;
2164 size
+= STRTAB_SPLIT
;
2165 if (size
< smmu
->sid_bits
)
2167 "2-level strtab only covers %u/%u bits of SID\n",
2168 size
, smmu
->sid_bits
);
2170 l1size
= cfg
->num_l1_ents
* (STRTAB_L1_DESC_DWORDS
<< 3);
2171 strtab
= dmam_alloc_coherent(smmu
->dev
, l1size
, &cfg
->strtab_dma
,
2172 GFP_KERNEL
| __GFP_ZERO
);
2175 "failed to allocate l1 stream table (%u bytes)\n",
2179 cfg
->strtab
= strtab
;
2181 /* Configure strtab_base_cfg for 2 levels */
2182 reg
= STRTAB_BASE_CFG_FMT_2LVL
;
2183 reg
|= (size
& STRTAB_BASE_CFG_LOG2SIZE_MASK
)
2184 << STRTAB_BASE_CFG_LOG2SIZE_SHIFT
;
2185 reg
|= (STRTAB_SPLIT
& STRTAB_BASE_CFG_SPLIT_MASK
)
2186 << STRTAB_BASE_CFG_SPLIT_SHIFT
;
2187 cfg
->strtab_base_cfg
= reg
;
2189 return arm_smmu_init_l1_strtab(smmu
);
2192 static int arm_smmu_init_strtab_linear(struct arm_smmu_device
*smmu
)
2197 struct arm_smmu_strtab_cfg
*cfg
= &smmu
->strtab_cfg
;
2199 size
= (1 << smmu
->sid_bits
) * (STRTAB_STE_DWORDS
<< 3);
2200 strtab
= dmam_alloc_coherent(smmu
->dev
, size
, &cfg
->strtab_dma
,
2201 GFP_KERNEL
| __GFP_ZERO
);
2204 "failed to allocate linear stream table (%u bytes)\n",
2208 cfg
->strtab
= strtab
;
2209 cfg
->num_l1_ents
= 1 << smmu
->sid_bits
;
2211 /* Configure strtab_base_cfg for a linear table covering all SIDs */
2212 reg
= STRTAB_BASE_CFG_FMT_LINEAR
;
2213 reg
|= (smmu
->sid_bits
& STRTAB_BASE_CFG_LOG2SIZE_MASK
)
2214 << STRTAB_BASE_CFG_LOG2SIZE_SHIFT
;
2215 cfg
->strtab_base_cfg
= reg
;
2217 arm_smmu_init_bypass_stes(strtab
, cfg
->num_l1_ents
);
2221 static int arm_smmu_init_strtab(struct arm_smmu_device
*smmu
)
2226 if (smmu
->features
& ARM_SMMU_FEAT_2_LVL_STRTAB
)
2227 ret
= arm_smmu_init_strtab_2lvl(smmu
);
2229 ret
= arm_smmu_init_strtab_linear(smmu
);
2234 /* Set the strtab base address */
2235 reg
= smmu
->strtab_cfg
.strtab_dma
&
2236 STRTAB_BASE_ADDR_MASK
<< STRTAB_BASE_ADDR_SHIFT
;
2237 reg
|= STRTAB_BASE_RA
;
2238 smmu
->strtab_cfg
.strtab_base
= reg
;
2240 /* Allocate the first VMID for stage-2 bypass STEs */
2241 set_bit(0, smmu
->vmid_map
);
2245 static int arm_smmu_init_structures(struct arm_smmu_device
*smmu
)
2249 atomic_set(&smmu
->sync_nr
, 0);
2250 ret
= arm_smmu_init_queues(smmu
);
2254 return arm_smmu_init_strtab(smmu
);
2257 static int arm_smmu_write_reg_sync(struct arm_smmu_device
*smmu
, u32 val
,
2258 unsigned int reg_off
, unsigned int ack_off
)
2262 writel_relaxed(val
, smmu
->base
+ reg_off
);
2263 return readl_relaxed_poll_timeout(smmu
->base
+ ack_off
, reg
, reg
== val
,
2264 1, ARM_SMMU_POLL_TIMEOUT_US
);
2267 /* GBPA is "special" */
2268 static int arm_smmu_update_gbpa(struct arm_smmu_device
*smmu
, u32 set
, u32 clr
)
2271 u32 reg
, __iomem
*gbpa
= smmu
->base
+ ARM_SMMU_GBPA
;
2273 ret
= readl_relaxed_poll_timeout(gbpa
, reg
, !(reg
& GBPA_UPDATE
),
2274 1, ARM_SMMU_POLL_TIMEOUT_US
);
2280 writel_relaxed(reg
| GBPA_UPDATE
, gbpa
);
2281 return readl_relaxed_poll_timeout(gbpa
, reg
, !(reg
& GBPA_UPDATE
),
2282 1, ARM_SMMU_POLL_TIMEOUT_US
);
2285 static void arm_smmu_free_msis(void *data
)
2287 struct device
*dev
= data
;
2288 platform_msi_domain_free_irqs(dev
);
2291 static void arm_smmu_write_msi_msg(struct msi_desc
*desc
, struct msi_msg
*msg
)
2293 phys_addr_t doorbell
;
2294 struct device
*dev
= msi_desc_to_dev(desc
);
2295 struct arm_smmu_device
*smmu
= dev_get_drvdata(dev
);
2296 phys_addr_t
*cfg
= arm_smmu_msi_cfg
[desc
->platform
.msi_index
];
2298 doorbell
= (((u64
)msg
->address_hi
) << 32) | msg
->address_lo
;
2299 doorbell
&= MSI_CFG0_ADDR_MASK
<< MSI_CFG0_ADDR_SHIFT
;
2301 writeq_relaxed(doorbell
, smmu
->base
+ cfg
[0]);
2302 writel_relaxed(msg
->data
, smmu
->base
+ cfg
[1]);
2303 writel_relaxed(MSI_CFG2_MEMATTR_DEVICE_nGnRE
, smmu
->base
+ cfg
[2]);
2306 static void arm_smmu_setup_msis(struct arm_smmu_device
*smmu
)
2308 struct msi_desc
*desc
;
2309 int ret
, nvec
= ARM_SMMU_MAX_MSIS
;
2310 struct device
*dev
= smmu
->dev
;
2312 /* Clear the MSI address regs */
2313 writeq_relaxed(0, smmu
->base
+ ARM_SMMU_GERROR_IRQ_CFG0
);
2314 writeq_relaxed(0, smmu
->base
+ ARM_SMMU_EVTQ_IRQ_CFG0
);
2316 if (smmu
->features
& ARM_SMMU_FEAT_PRI
)
2317 writeq_relaxed(0, smmu
->base
+ ARM_SMMU_PRIQ_IRQ_CFG0
);
2321 if (!(smmu
->features
& ARM_SMMU_FEAT_MSI
))
2324 /* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */
2325 ret
= platform_msi_domain_alloc_irqs(dev
, nvec
, arm_smmu_write_msi_msg
);
2327 dev_warn(dev
, "failed to allocate MSIs\n");
2331 for_each_msi_entry(desc
, dev
) {
2332 switch (desc
->platform
.msi_index
) {
2333 case EVTQ_MSI_INDEX
:
2334 smmu
->evtq
.q
.irq
= desc
->irq
;
2336 case GERROR_MSI_INDEX
:
2337 smmu
->gerr_irq
= desc
->irq
;
2339 case PRIQ_MSI_INDEX
:
2340 smmu
->priq
.q
.irq
= desc
->irq
;
2342 default: /* Unknown */
2347 /* Add callback to free MSIs on teardown */
2348 devm_add_action(dev
, arm_smmu_free_msis
, dev
);
2351 static void arm_smmu_setup_unique_irqs(struct arm_smmu_device
*smmu
)
2355 arm_smmu_setup_msis(smmu
);
2357 /* Request interrupt lines */
2358 irq
= smmu
->evtq
.q
.irq
;
2360 ret
= devm_request_threaded_irq(smmu
->dev
, irq
, NULL
,
2361 arm_smmu_evtq_thread
,
2363 "arm-smmu-v3-evtq", smmu
);
2365 dev_warn(smmu
->dev
, "failed to enable evtq irq\n");
2368 irq
= smmu
->gerr_irq
;
2370 ret
= devm_request_irq(smmu
->dev
, irq
, arm_smmu_gerror_handler
,
2371 0, "arm-smmu-v3-gerror", smmu
);
2373 dev_warn(smmu
->dev
, "failed to enable gerror irq\n");
2376 if (smmu
->features
& ARM_SMMU_FEAT_PRI
) {
2377 irq
= smmu
->priq
.q
.irq
;
2379 ret
= devm_request_threaded_irq(smmu
->dev
, irq
, NULL
,
2380 arm_smmu_priq_thread
,
2386 "failed to enable priq irq\n");
2391 static int arm_smmu_setup_irqs(struct arm_smmu_device
*smmu
)
2394 u32 irqen_flags
= IRQ_CTRL_EVTQ_IRQEN
| IRQ_CTRL_GERROR_IRQEN
;
2396 /* Disable IRQs first */
2397 ret
= arm_smmu_write_reg_sync(smmu
, 0, ARM_SMMU_IRQ_CTRL
,
2398 ARM_SMMU_IRQ_CTRLACK
);
2400 dev_err(smmu
->dev
, "failed to disable irqs\n");
2404 irq
= smmu
->combined_irq
;
2407 * Cavium ThunderX2 implementation doesn't not support unique
2408 * irq lines. Use single irq line for all the SMMUv3 interrupts.
2410 ret
= devm_request_threaded_irq(smmu
->dev
, irq
,
2411 arm_smmu_combined_irq_handler
,
2412 arm_smmu_combined_irq_thread
,
2414 "arm-smmu-v3-combined-irq", smmu
);
2416 dev_warn(smmu
->dev
, "failed to enable combined irq\n");
2418 arm_smmu_setup_unique_irqs(smmu
);
2420 if (smmu
->features
& ARM_SMMU_FEAT_PRI
)
2421 irqen_flags
|= IRQ_CTRL_PRIQ_IRQEN
;
2423 /* Enable interrupt generation on the SMMU */
2424 ret
= arm_smmu_write_reg_sync(smmu
, irqen_flags
,
2425 ARM_SMMU_IRQ_CTRL
, ARM_SMMU_IRQ_CTRLACK
);
2427 dev_warn(smmu
->dev
, "failed to enable irqs\n");
2432 static int arm_smmu_device_disable(struct arm_smmu_device
*smmu
)
2436 ret
= arm_smmu_write_reg_sync(smmu
, 0, ARM_SMMU_CR0
, ARM_SMMU_CR0ACK
);
2438 dev_err(smmu
->dev
, "failed to clear cr0\n");
2443 static int arm_smmu_device_reset(struct arm_smmu_device
*smmu
, bool bypass
)
2447 struct arm_smmu_cmdq_ent cmd
;
2449 /* Clear CR0 and sync (disables SMMU and queue processing) */
2450 reg
= readl_relaxed(smmu
->base
+ ARM_SMMU_CR0
);
2451 if (reg
& CR0_SMMUEN
)
2452 dev_warn(smmu
->dev
, "SMMU currently enabled! Resetting...\n");
2454 ret
= arm_smmu_device_disable(smmu
);
2458 /* CR1 (table and queue memory attributes) */
2459 reg
= (CR1_SH_ISH
<< CR1_TABLE_SH_SHIFT
) |
2460 (CR1_CACHE_WB
<< CR1_TABLE_OC_SHIFT
) |
2461 (CR1_CACHE_WB
<< CR1_TABLE_IC_SHIFT
) |
2462 (CR1_SH_ISH
<< CR1_QUEUE_SH_SHIFT
) |
2463 (CR1_CACHE_WB
<< CR1_QUEUE_OC_SHIFT
) |
2464 (CR1_CACHE_WB
<< CR1_QUEUE_IC_SHIFT
);
2465 writel_relaxed(reg
, smmu
->base
+ ARM_SMMU_CR1
);
2467 /* CR2 (random crap) */
2468 reg
= CR2_PTM
| CR2_RECINVSID
| CR2_E2H
;
2469 writel_relaxed(reg
, smmu
->base
+ ARM_SMMU_CR2
);
2472 writeq_relaxed(smmu
->strtab_cfg
.strtab_base
,
2473 smmu
->base
+ ARM_SMMU_STRTAB_BASE
);
2474 writel_relaxed(smmu
->strtab_cfg
.strtab_base_cfg
,
2475 smmu
->base
+ ARM_SMMU_STRTAB_BASE_CFG
);
2478 writeq_relaxed(smmu
->cmdq
.q
.q_base
, smmu
->base
+ ARM_SMMU_CMDQ_BASE
);
2479 writel_relaxed(smmu
->cmdq
.q
.prod
, smmu
->base
+ ARM_SMMU_CMDQ_PROD
);
2480 writel_relaxed(smmu
->cmdq
.q
.cons
, smmu
->base
+ ARM_SMMU_CMDQ_CONS
);
2482 enables
= CR0_CMDQEN
;
2483 ret
= arm_smmu_write_reg_sync(smmu
, enables
, ARM_SMMU_CR0
,
2486 dev_err(smmu
->dev
, "failed to enable command queue\n");
2490 /* Invalidate any cached configuration */
2491 cmd
.opcode
= CMDQ_OP_CFGI_ALL
;
2492 arm_smmu_cmdq_issue_cmd(smmu
, &cmd
);
2493 arm_smmu_cmdq_issue_sync(smmu
);
2495 /* Invalidate any stale TLB entries */
2496 if (smmu
->features
& ARM_SMMU_FEAT_HYP
) {
2497 cmd
.opcode
= CMDQ_OP_TLBI_EL2_ALL
;
2498 arm_smmu_cmdq_issue_cmd(smmu
, &cmd
);
2501 cmd
.opcode
= CMDQ_OP_TLBI_NSNH_ALL
;
2502 arm_smmu_cmdq_issue_cmd(smmu
, &cmd
);
2503 arm_smmu_cmdq_issue_sync(smmu
);
2506 writeq_relaxed(smmu
->evtq
.q
.q_base
, smmu
->base
+ ARM_SMMU_EVTQ_BASE
);
2507 writel_relaxed(smmu
->evtq
.q
.prod
,
2508 arm_smmu_page1_fixup(ARM_SMMU_EVTQ_PROD
, smmu
));
2509 writel_relaxed(smmu
->evtq
.q
.cons
,
2510 arm_smmu_page1_fixup(ARM_SMMU_EVTQ_CONS
, smmu
));
2512 enables
|= CR0_EVTQEN
;
2513 ret
= arm_smmu_write_reg_sync(smmu
, enables
, ARM_SMMU_CR0
,
2516 dev_err(smmu
->dev
, "failed to enable event queue\n");
2521 if (smmu
->features
& ARM_SMMU_FEAT_PRI
) {
2522 writeq_relaxed(smmu
->priq
.q
.q_base
,
2523 smmu
->base
+ ARM_SMMU_PRIQ_BASE
);
2524 writel_relaxed(smmu
->priq
.q
.prod
,
2525 arm_smmu_page1_fixup(ARM_SMMU_PRIQ_PROD
, smmu
));
2526 writel_relaxed(smmu
->priq
.q
.cons
,
2527 arm_smmu_page1_fixup(ARM_SMMU_PRIQ_CONS
, smmu
));
2529 enables
|= CR0_PRIQEN
;
2530 ret
= arm_smmu_write_reg_sync(smmu
, enables
, ARM_SMMU_CR0
,
2533 dev_err(smmu
->dev
, "failed to enable PRI queue\n");
2538 ret
= arm_smmu_setup_irqs(smmu
);
2540 dev_err(smmu
->dev
, "failed to setup irqs\n");
2545 /* Enable the SMMU interface, or ensure bypass */
2546 if (!bypass
|| disable_bypass
) {
2547 enables
|= CR0_SMMUEN
;
2549 ret
= arm_smmu_update_gbpa(smmu
, 0, GBPA_ABORT
);
2551 dev_err(smmu
->dev
, "GBPA not responding to update\n");
2555 ret
= arm_smmu_write_reg_sync(smmu
, enables
, ARM_SMMU_CR0
,
2558 dev_err(smmu
->dev
, "failed to enable SMMU interface\n");
2565 static int arm_smmu_device_hw_probe(struct arm_smmu_device
*smmu
)
2568 bool coherent
= smmu
->features
& ARM_SMMU_FEAT_COHERENCY
;
2571 reg
= readl_relaxed(smmu
->base
+ ARM_SMMU_IDR0
);
2573 /* 2-level structures */
2574 if ((reg
& IDR0_ST_LVL_MASK
<< IDR0_ST_LVL_SHIFT
) == IDR0_ST_LVL_2LVL
)
2575 smmu
->features
|= ARM_SMMU_FEAT_2_LVL_STRTAB
;
2577 if (reg
& IDR0_CD2L
)
2578 smmu
->features
|= ARM_SMMU_FEAT_2_LVL_CDTAB
;
2581 * Translation table endianness.
2582 * We currently require the same endianness as the CPU, but this
2583 * could be changed later by adding a new IO_PGTABLE_QUIRK.
2585 switch (reg
& IDR0_TTENDIAN_MASK
<< IDR0_TTENDIAN_SHIFT
) {
2586 case IDR0_TTENDIAN_MIXED
:
2587 smmu
->features
|= ARM_SMMU_FEAT_TT_LE
| ARM_SMMU_FEAT_TT_BE
;
2590 case IDR0_TTENDIAN_BE
:
2591 smmu
->features
|= ARM_SMMU_FEAT_TT_BE
;
2594 case IDR0_TTENDIAN_LE
:
2595 smmu
->features
|= ARM_SMMU_FEAT_TT_LE
;
2599 dev_err(smmu
->dev
, "unknown/unsupported TT endianness!\n");
2603 /* Boolean feature flags */
2604 if (IS_ENABLED(CONFIG_PCI_PRI
) && reg
& IDR0_PRI
)
2605 smmu
->features
|= ARM_SMMU_FEAT_PRI
;
2607 if (IS_ENABLED(CONFIG_PCI_ATS
) && reg
& IDR0_ATS
)
2608 smmu
->features
|= ARM_SMMU_FEAT_ATS
;
2611 smmu
->features
|= ARM_SMMU_FEAT_SEV
;
2614 smmu
->features
|= ARM_SMMU_FEAT_MSI
;
2617 smmu
->features
|= ARM_SMMU_FEAT_HYP
;
2620 * The coherency feature as set by FW is used in preference to the ID
2621 * register, but warn on mismatch.
2623 if (!!(reg
& IDR0_COHACC
) != coherent
)
2624 dev_warn(smmu
->dev
, "IDR0.COHACC overridden by FW configuration (%s)\n",
2625 coherent
? "true" : "false");
2627 switch (reg
& IDR0_STALL_MODEL_MASK
<< IDR0_STALL_MODEL_SHIFT
) {
2628 case IDR0_STALL_MODEL_FORCE
:
2629 smmu
->features
|= ARM_SMMU_FEAT_STALL_FORCE
;
2631 case IDR0_STALL_MODEL_STALL
:
2632 smmu
->features
|= ARM_SMMU_FEAT_STALLS
;
2636 smmu
->features
|= ARM_SMMU_FEAT_TRANS_S1
;
2639 smmu
->features
|= ARM_SMMU_FEAT_TRANS_S2
;
2641 if (!(reg
& (IDR0_S1P
| IDR0_S2P
))) {
2642 dev_err(smmu
->dev
, "no translation support!\n");
2646 /* We only support the AArch64 table format at present */
2647 switch (reg
& IDR0_TTF_MASK
<< IDR0_TTF_SHIFT
) {
2648 case IDR0_TTF_AARCH32_64
:
2651 case IDR0_TTF_AARCH64
:
2654 dev_err(smmu
->dev
, "AArch64 table format not supported!\n");
2658 /* ASID/VMID sizes */
2659 smmu
->asid_bits
= reg
& IDR0_ASID16
? 16 : 8;
2660 smmu
->vmid_bits
= reg
& IDR0_VMID16
? 16 : 8;
2663 reg
= readl_relaxed(smmu
->base
+ ARM_SMMU_IDR1
);
2664 if (reg
& (IDR1_TABLES_PRESET
| IDR1_QUEUES_PRESET
| IDR1_REL
)) {
2665 dev_err(smmu
->dev
, "embedded implementation not supported\n");
2669 /* Queue sizes, capped at 4k */
2670 smmu
->cmdq
.q
.max_n_shift
= min((u32
)CMDQ_MAX_SZ_SHIFT
,
2671 reg
>> IDR1_CMDQ_SHIFT
& IDR1_CMDQ_MASK
);
2672 if (!smmu
->cmdq
.q
.max_n_shift
) {
2673 /* Odd alignment restrictions on the base, so ignore for now */
2674 dev_err(smmu
->dev
, "unit-length command queue not supported\n");
2678 smmu
->evtq
.q
.max_n_shift
= min((u32
)EVTQ_MAX_SZ_SHIFT
,
2679 reg
>> IDR1_EVTQ_SHIFT
& IDR1_EVTQ_MASK
);
2680 smmu
->priq
.q
.max_n_shift
= min((u32
)PRIQ_MAX_SZ_SHIFT
,
2681 reg
>> IDR1_PRIQ_SHIFT
& IDR1_PRIQ_MASK
);
2683 /* SID/SSID sizes */
2684 smmu
->ssid_bits
= reg
>> IDR1_SSID_SHIFT
& IDR1_SSID_MASK
;
2685 smmu
->sid_bits
= reg
>> IDR1_SID_SHIFT
& IDR1_SID_MASK
;
2688 * If the SMMU supports fewer bits than would fill a single L2 stream
2689 * table, use a linear table instead.
2691 if (smmu
->sid_bits
<= STRTAB_SPLIT
)
2692 smmu
->features
&= ~ARM_SMMU_FEAT_2_LVL_STRTAB
;
2695 reg
= readl_relaxed(smmu
->base
+ ARM_SMMU_IDR5
);
2697 /* Maximum number of outstanding stalls */
2698 smmu
->evtq
.max_stalls
= reg
>> IDR5_STALL_MAX_SHIFT
2699 & IDR5_STALL_MAX_MASK
;
2702 if (reg
& IDR5_GRAN64K
)
2703 smmu
->pgsize_bitmap
|= SZ_64K
| SZ_512M
;
2704 if (reg
& IDR5_GRAN16K
)
2705 smmu
->pgsize_bitmap
|= SZ_16K
| SZ_32M
;
2706 if (reg
& IDR5_GRAN4K
)
2707 smmu
->pgsize_bitmap
|= SZ_4K
| SZ_2M
| SZ_1G
;
2709 if (arm_smmu_ops
.pgsize_bitmap
== -1UL)
2710 arm_smmu_ops
.pgsize_bitmap
= smmu
->pgsize_bitmap
;
2712 arm_smmu_ops
.pgsize_bitmap
|= smmu
->pgsize_bitmap
;
2714 /* Output address size */
2715 switch (reg
& IDR5_OAS_MASK
<< IDR5_OAS_SHIFT
) {
2716 case IDR5_OAS_32_BIT
:
2719 case IDR5_OAS_36_BIT
:
2722 case IDR5_OAS_40_BIT
:
2725 case IDR5_OAS_42_BIT
:
2728 case IDR5_OAS_44_BIT
:
2733 "unknown output address size. Truncating to 48-bit\n");
2735 case IDR5_OAS_48_BIT
:
2739 /* Set the DMA mask for our table walker */
2740 if (dma_set_mask_and_coherent(smmu
->dev
, DMA_BIT_MASK(smmu
->oas
)))
2742 "failed to set DMA mask for table walker\n");
2744 smmu
->ias
= max(smmu
->ias
, smmu
->oas
);
2746 dev_info(smmu
->dev
, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
2747 smmu
->ias
, smmu
->oas
, smmu
->features
);
2752 static void acpi_smmu_get_options(u32 model
, struct arm_smmu_device
*smmu
)
2755 case ACPI_IORT_SMMU_V3_CAVIUM_CN99XX
:
2756 smmu
->options
|= ARM_SMMU_OPT_PAGE0_REGS_ONLY
;
2758 case ACPI_IORT_SMMU_V3_HISILICON_HI161X
:
2759 smmu
->options
|= ARM_SMMU_OPT_SKIP_PREFETCH
;
2763 dev_notice(smmu
->dev
, "option mask 0x%x\n", smmu
->options
);
2766 static int arm_smmu_device_acpi_probe(struct platform_device
*pdev
,
2767 struct arm_smmu_device
*smmu
)
2769 struct acpi_iort_smmu_v3
*iort_smmu
;
2770 struct device
*dev
= smmu
->dev
;
2771 struct acpi_iort_node
*node
;
2773 node
= *(struct acpi_iort_node
**)dev_get_platdata(dev
);
2775 /* Retrieve SMMUv3 specific data */
2776 iort_smmu
= (struct acpi_iort_smmu_v3
*)node
->node_data
;
2778 acpi_smmu_get_options(iort_smmu
->model
, smmu
);
2780 if (iort_smmu
->flags
& ACPI_IORT_SMMU_V3_COHACC_OVERRIDE
)
2781 smmu
->features
|= ARM_SMMU_FEAT_COHERENCY
;
2786 static inline int arm_smmu_device_acpi_probe(struct platform_device
*pdev
,
2787 struct arm_smmu_device
*smmu
)
2793 static int arm_smmu_device_dt_probe(struct platform_device
*pdev
,
2794 struct arm_smmu_device
*smmu
)
2796 struct device
*dev
= &pdev
->dev
;
2800 if (of_property_read_u32(dev
->of_node
, "#iommu-cells", &cells
))
2801 dev_err(dev
, "missing #iommu-cells property\n");
2802 else if (cells
!= 1)
2803 dev_err(dev
, "invalid #iommu-cells value (%d)\n", cells
);
2807 parse_driver_options(smmu
);
2809 if (of_dma_is_coherent(dev
->of_node
))
2810 smmu
->features
|= ARM_SMMU_FEAT_COHERENCY
;
2815 static unsigned long arm_smmu_resource_size(struct arm_smmu_device
*smmu
)
2817 if (smmu
->options
& ARM_SMMU_OPT_PAGE0_REGS_ONLY
)
2823 static int arm_smmu_device_probe(struct platform_device
*pdev
)
2826 struct resource
*res
;
2827 resource_size_t ioaddr
;
2828 struct arm_smmu_device
*smmu
;
2829 struct device
*dev
= &pdev
->dev
;
2832 smmu
= devm_kzalloc(dev
, sizeof(*smmu
), GFP_KERNEL
);
2834 dev_err(dev
, "failed to allocate arm_smmu_device\n");
2840 ret
= arm_smmu_device_dt_probe(pdev
, smmu
);
2842 ret
= arm_smmu_device_acpi_probe(pdev
, smmu
);
2847 /* Set bypass mode according to firmware probing result */
2851 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
2852 if (resource_size(res
) + 1 < arm_smmu_resource_size(smmu
)) {
2853 dev_err(dev
, "MMIO region too small (%pr)\n", res
);
2856 ioaddr
= res
->start
;
2858 smmu
->base
= devm_ioremap_resource(dev
, res
);
2859 if (IS_ERR(smmu
->base
))
2860 return PTR_ERR(smmu
->base
);
2862 /* Interrupt lines */
2864 irq
= platform_get_irq_byname(pdev
, "combined");
2866 smmu
->combined_irq
= irq
;
2868 irq
= platform_get_irq_byname(pdev
, "eventq");
2870 smmu
->evtq
.q
.irq
= irq
;
2872 irq
= platform_get_irq_byname(pdev
, "priq");
2874 smmu
->priq
.q
.irq
= irq
;
2876 irq
= platform_get_irq_byname(pdev
, "gerror");
2878 smmu
->gerr_irq
= irq
;
2881 ret
= arm_smmu_device_hw_probe(smmu
);
2885 /* Initialise in-memory data structures */
2886 ret
= arm_smmu_init_structures(smmu
);
2890 /* Record our private device structure */
2891 platform_set_drvdata(pdev
, smmu
);
2893 /* Reset the device */
2894 ret
= arm_smmu_device_reset(smmu
, bypass
);
2898 /* And we're up. Go go go! */
2899 ret
= iommu_device_sysfs_add(&smmu
->iommu
, dev
, NULL
,
2900 "smmu3.%pa", &ioaddr
);
2904 iommu_device_set_ops(&smmu
->iommu
, &arm_smmu_ops
);
2905 iommu_device_set_fwnode(&smmu
->iommu
, dev
->fwnode
);
2907 ret
= iommu_device_register(&smmu
->iommu
);
2909 dev_err(dev
, "Failed to register iommu\n");
2914 if (pci_bus_type
.iommu_ops
!= &arm_smmu_ops
) {
2916 ret
= bus_set_iommu(&pci_bus_type
, &arm_smmu_ops
);
2921 #ifdef CONFIG_ARM_AMBA
2922 if (amba_bustype
.iommu_ops
!= &arm_smmu_ops
) {
2923 ret
= bus_set_iommu(&amba_bustype
, &arm_smmu_ops
);
2928 if (platform_bus_type
.iommu_ops
!= &arm_smmu_ops
) {
2929 ret
= bus_set_iommu(&platform_bus_type
, &arm_smmu_ops
);
2936 static int arm_smmu_device_remove(struct platform_device
*pdev
)
2938 struct arm_smmu_device
*smmu
= platform_get_drvdata(pdev
);
2940 arm_smmu_device_disable(smmu
);
2945 static void arm_smmu_device_shutdown(struct platform_device
*pdev
)
2947 arm_smmu_device_remove(pdev
);
2950 static const struct of_device_id arm_smmu_of_match
[] = {
2951 { .compatible
= "arm,smmu-v3", },
2954 MODULE_DEVICE_TABLE(of
, arm_smmu_of_match
);
2956 static struct platform_driver arm_smmu_driver
= {
2958 .name
= "arm-smmu-v3",
2959 .of_match_table
= of_match_ptr(arm_smmu_of_match
),
2961 .probe
= arm_smmu_device_probe
,
2962 .remove
= arm_smmu_device_remove
,
2963 .shutdown
= arm_smmu_device_shutdown
,
2965 module_platform_driver(arm_smmu_driver
);
2967 IOMMU_OF_DECLARE(arm_smmuv3
, "arm,smmu-v3", NULL
);
2969 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
2970 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2971 MODULE_LICENSE("GPL v2");