]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/iommu/arm-smmu-v3.c
Merge branch 'for-joerg/arm-smmu/updates' of git://git.kernel.org/pub/scm/linux/kerne...
[mirror_ubuntu-bionic-kernel.git] / drivers / iommu / arm-smmu-v3.c
1 /*
2 * IOMMU API for ARM architected SMMUv3 implementations.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 *
16 * Copyright (C) 2015 ARM Limited
17 *
18 * Author: Will Deacon <will.deacon@arm.com>
19 *
20 * This driver is powered by bad coffee and bombay mix.
21 */
22
23 #include <linux/acpi.h>
24 #include <linux/acpi_iort.h>
25 #include <linux/delay.h>
26 #include <linux/dma-iommu.h>
27 #include <linux/err.h>
28 #include <linux/interrupt.h>
29 #include <linux/iommu.h>
30 #include <linux/iopoll.h>
31 #include <linux/module.h>
32 #include <linux/msi.h>
33 #include <linux/of.h>
34 #include <linux/of_address.h>
35 #include <linux/of_iommu.h>
36 #include <linux/of_platform.h>
37 #include <linux/pci.h>
38 #include <linux/platform_device.h>
39
40 #include <linux/amba/bus.h>
41
42 #include "io-pgtable.h"
43
44 /* MMIO registers */
45 #define ARM_SMMU_IDR0 0x0
46 #define IDR0_ST_LVL_SHIFT 27
47 #define IDR0_ST_LVL_MASK 0x3
48 #define IDR0_ST_LVL_2LVL (1 << IDR0_ST_LVL_SHIFT)
49 #define IDR0_STALL_MODEL_SHIFT 24
50 #define IDR0_STALL_MODEL_MASK 0x3
51 #define IDR0_STALL_MODEL_STALL (0 << IDR0_STALL_MODEL_SHIFT)
52 #define IDR0_STALL_MODEL_FORCE (2 << IDR0_STALL_MODEL_SHIFT)
53 #define IDR0_TTENDIAN_SHIFT 21
54 #define IDR0_TTENDIAN_MASK 0x3
55 #define IDR0_TTENDIAN_LE (2 << IDR0_TTENDIAN_SHIFT)
56 #define IDR0_TTENDIAN_BE (3 << IDR0_TTENDIAN_SHIFT)
57 #define IDR0_TTENDIAN_MIXED (0 << IDR0_TTENDIAN_SHIFT)
58 #define IDR0_CD2L (1 << 19)
59 #define IDR0_VMID16 (1 << 18)
60 #define IDR0_PRI (1 << 16)
61 #define IDR0_SEV (1 << 14)
62 #define IDR0_MSI (1 << 13)
63 #define IDR0_ASID16 (1 << 12)
64 #define IDR0_ATS (1 << 10)
65 #define IDR0_HYP (1 << 9)
66 #define IDR0_COHACC (1 << 4)
67 #define IDR0_TTF_SHIFT 2
68 #define IDR0_TTF_MASK 0x3
69 #define IDR0_TTF_AARCH64 (2 << IDR0_TTF_SHIFT)
70 #define IDR0_TTF_AARCH32_64 (3 << IDR0_TTF_SHIFT)
71 #define IDR0_S1P (1 << 1)
72 #define IDR0_S2P (1 << 0)
73
74 #define ARM_SMMU_IDR1 0x4
75 #define IDR1_TABLES_PRESET (1 << 30)
76 #define IDR1_QUEUES_PRESET (1 << 29)
77 #define IDR1_REL (1 << 28)
78 #define IDR1_CMDQ_SHIFT 21
79 #define IDR1_CMDQ_MASK 0x1f
80 #define IDR1_EVTQ_SHIFT 16
81 #define IDR1_EVTQ_MASK 0x1f
82 #define IDR1_PRIQ_SHIFT 11
83 #define IDR1_PRIQ_MASK 0x1f
84 #define IDR1_SSID_SHIFT 6
85 #define IDR1_SSID_MASK 0x1f
86 #define IDR1_SID_SHIFT 0
87 #define IDR1_SID_MASK 0x3f
88
89 #define ARM_SMMU_IDR5 0x14
90 #define IDR5_STALL_MAX_SHIFT 16
91 #define IDR5_STALL_MAX_MASK 0xffff
92 #define IDR5_GRAN64K (1 << 6)
93 #define IDR5_GRAN16K (1 << 5)
94 #define IDR5_GRAN4K (1 << 4)
95 #define IDR5_OAS_SHIFT 0
96 #define IDR5_OAS_MASK 0x7
97 #define IDR5_OAS_32_BIT (0 << IDR5_OAS_SHIFT)
98 #define IDR5_OAS_36_BIT (1 << IDR5_OAS_SHIFT)
99 #define IDR5_OAS_40_BIT (2 << IDR5_OAS_SHIFT)
100 #define IDR5_OAS_42_BIT (3 << IDR5_OAS_SHIFT)
101 #define IDR5_OAS_44_BIT (4 << IDR5_OAS_SHIFT)
102 #define IDR5_OAS_48_BIT (5 << IDR5_OAS_SHIFT)
103
104 #define ARM_SMMU_CR0 0x20
105 #define CR0_CMDQEN (1 << 3)
106 #define CR0_EVTQEN (1 << 2)
107 #define CR0_PRIQEN (1 << 1)
108 #define CR0_SMMUEN (1 << 0)
109
110 #define ARM_SMMU_CR0ACK 0x24
111
112 #define ARM_SMMU_CR1 0x28
113 #define CR1_SH_NSH 0
114 #define CR1_SH_OSH 2
115 #define CR1_SH_ISH 3
116 #define CR1_CACHE_NC 0
117 #define CR1_CACHE_WB 1
118 #define CR1_CACHE_WT 2
119 #define CR1_TABLE_SH_SHIFT 10
120 #define CR1_TABLE_OC_SHIFT 8
121 #define CR1_TABLE_IC_SHIFT 6
122 #define CR1_QUEUE_SH_SHIFT 4
123 #define CR1_QUEUE_OC_SHIFT 2
124 #define CR1_QUEUE_IC_SHIFT 0
125
126 #define ARM_SMMU_CR2 0x2c
127 #define CR2_PTM (1 << 2)
128 #define CR2_RECINVSID (1 << 1)
129 #define CR2_E2H (1 << 0)
130
131 #define ARM_SMMU_GBPA 0x44
132 #define GBPA_ABORT (1 << 20)
133 #define GBPA_UPDATE (1 << 31)
134
135 #define ARM_SMMU_IRQ_CTRL 0x50
136 #define IRQ_CTRL_EVTQ_IRQEN (1 << 2)
137 #define IRQ_CTRL_PRIQ_IRQEN (1 << 1)
138 #define IRQ_CTRL_GERROR_IRQEN (1 << 0)
139
140 #define ARM_SMMU_IRQ_CTRLACK 0x54
141
142 #define ARM_SMMU_GERROR 0x60
143 #define GERROR_SFM_ERR (1 << 8)
144 #define GERROR_MSI_GERROR_ABT_ERR (1 << 7)
145 #define GERROR_MSI_PRIQ_ABT_ERR (1 << 6)
146 #define GERROR_MSI_EVTQ_ABT_ERR (1 << 5)
147 #define GERROR_MSI_CMDQ_ABT_ERR (1 << 4)
148 #define GERROR_PRIQ_ABT_ERR (1 << 3)
149 #define GERROR_EVTQ_ABT_ERR (1 << 2)
150 #define GERROR_CMDQ_ERR (1 << 0)
151 #define GERROR_ERR_MASK 0xfd
152
153 #define ARM_SMMU_GERRORN 0x64
154
155 #define ARM_SMMU_GERROR_IRQ_CFG0 0x68
156 #define ARM_SMMU_GERROR_IRQ_CFG1 0x70
157 #define ARM_SMMU_GERROR_IRQ_CFG2 0x74
158
159 #define ARM_SMMU_STRTAB_BASE 0x80
160 #define STRTAB_BASE_RA (1UL << 62)
161 #define STRTAB_BASE_ADDR_SHIFT 6
162 #define STRTAB_BASE_ADDR_MASK 0x3ffffffffffUL
163
164 #define ARM_SMMU_STRTAB_BASE_CFG 0x88
165 #define STRTAB_BASE_CFG_LOG2SIZE_SHIFT 0
166 #define STRTAB_BASE_CFG_LOG2SIZE_MASK 0x3f
167 #define STRTAB_BASE_CFG_SPLIT_SHIFT 6
168 #define STRTAB_BASE_CFG_SPLIT_MASK 0x1f
169 #define STRTAB_BASE_CFG_FMT_SHIFT 16
170 #define STRTAB_BASE_CFG_FMT_MASK 0x3
171 #define STRTAB_BASE_CFG_FMT_LINEAR (0 << STRTAB_BASE_CFG_FMT_SHIFT)
172 #define STRTAB_BASE_CFG_FMT_2LVL (1 << STRTAB_BASE_CFG_FMT_SHIFT)
173
174 #define ARM_SMMU_CMDQ_BASE 0x90
175 #define ARM_SMMU_CMDQ_PROD 0x98
176 #define ARM_SMMU_CMDQ_CONS 0x9c
177
178 #define ARM_SMMU_EVTQ_BASE 0xa0
179 #define ARM_SMMU_EVTQ_PROD 0x100a8
180 #define ARM_SMMU_EVTQ_CONS 0x100ac
181 #define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0
182 #define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8
183 #define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc
184
185 #define ARM_SMMU_PRIQ_BASE 0xc0
186 #define ARM_SMMU_PRIQ_PROD 0x100c8
187 #define ARM_SMMU_PRIQ_CONS 0x100cc
188 #define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0
189 #define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
190 #define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
191
192 /* Common MSI config fields */
193 #define MSI_CFG0_ADDR_SHIFT 2
194 #define MSI_CFG0_ADDR_MASK 0x3fffffffffffUL
195 #define MSI_CFG2_SH_SHIFT 4
196 #define MSI_CFG2_SH_NSH (0UL << MSI_CFG2_SH_SHIFT)
197 #define MSI_CFG2_SH_OSH (2UL << MSI_CFG2_SH_SHIFT)
198 #define MSI_CFG2_SH_ISH (3UL << MSI_CFG2_SH_SHIFT)
199 #define MSI_CFG2_MEMATTR_SHIFT 0
200 #define MSI_CFG2_MEMATTR_DEVICE_nGnRE (0x1 << MSI_CFG2_MEMATTR_SHIFT)
201
202 #define Q_IDX(q, p) ((p) & ((1 << (q)->max_n_shift) - 1))
203 #define Q_WRP(q, p) ((p) & (1 << (q)->max_n_shift))
204 #define Q_OVERFLOW_FLAG (1 << 31)
205 #define Q_OVF(q, p) ((p) & Q_OVERFLOW_FLAG)
206 #define Q_ENT(q, p) ((q)->base + \
207 Q_IDX(q, p) * (q)->ent_dwords)
208
209 #define Q_BASE_RWA (1UL << 62)
210 #define Q_BASE_ADDR_SHIFT 5
211 #define Q_BASE_ADDR_MASK 0xfffffffffffUL
212 #define Q_BASE_LOG2SIZE_SHIFT 0
213 #define Q_BASE_LOG2SIZE_MASK 0x1fUL
214
215 /*
216 * Stream table.
217 *
218 * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
219 * 2lvl: 128k L1 entries,
220 * 256 lazy entries per table (each table covers a PCI bus)
221 */
222 #define STRTAB_L1_SZ_SHIFT 20
223 #define STRTAB_SPLIT 8
224
225 #define STRTAB_L1_DESC_DWORDS 1
226 #define STRTAB_L1_DESC_SPAN_SHIFT 0
227 #define STRTAB_L1_DESC_SPAN_MASK 0x1fUL
228 #define STRTAB_L1_DESC_L2PTR_SHIFT 6
229 #define STRTAB_L1_DESC_L2PTR_MASK 0x3ffffffffffUL
230
231 #define STRTAB_STE_DWORDS 8
232 #define STRTAB_STE_0_V (1UL << 0)
233 #define STRTAB_STE_0_CFG_SHIFT 1
234 #define STRTAB_STE_0_CFG_MASK 0x7UL
235 #define STRTAB_STE_0_CFG_ABORT (0UL << STRTAB_STE_0_CFG_SHIFT)
236 #define STRTAB_STE_0_CFG_BYPASS (4UL << STRTAB_STE_0_CFG_SHIFT)
237 #define STRTAB_STE_0_CFG_S1_TRANS (5UL << STRTAB_STE_0_CFG_SHIFT)
238 #define STRTAB_STE_0_CFG_S2_TRANS (6UL << STRTAB_STE_0_CFG_SHIFT)
239
240 #define STRTAB_STE_0_S1FMT_SHIFT 4
241 #define STRTAB_STE_0_S1FMT_LINEAR (0UL << STRTAB_STE_0_S1FMT_SHIFT)
242 #define STRTAB_STE_0_S1CTXPTR_SHIFT 6
243 #define STRTAB_STE_0_S1CTXPTR_MASK 0x3ffffffffffUL
244 #define STRTAB_STE_0_S1CDMAX_SHIFT 59
245 #define STRTAB_STE_0_S1CDMAX_MASK 0x1fUL
246
247 #define STRTAB_STE_1_S1C_CACHE_NC 0UL
248 #define STRTAB_STE_1_S1C_CACHE_WBRA 1UL
249 #define STRTAB_STE_1_S1C_CACHE_WT 2UL
250 #define STRTAB_STE_1_S1C_CACHE_WB 3UL
251 #define STRTAB_STE_1_S1C_SH_NSH 0UL
252 #define STRTAB_STE_1_S1C_SH_OSH 2UL
253 #define STRTAB_STE_1_S1C_SH_ISH 3UL
254 #define STRTAB_STE_1_S1CIR_SHIFT 2
255 #define STRTAB_STE_1_S1COR_SHIFT 4
256 #define STRTAB_STE_1_S1CSH_SHIFT 6
257
258 #define STRTAB_STE_1_S1STALLD (1UL << 27)
259
260 #define STRTAB_STE_1_EATS_ABT 0UL
261 #define STRTAB_STE_1_EATS_TRANS 1UL
262 #define STRTAB_STE_1_EATS_S1CHK 2UL
263 #define STRTAB_STE_1_EATS_SHIFT 28
264
265 #define STRTAB_STE_1_STRW_NSEL1 0UL
266 #define STRTAB_STE_1_STRW_EL2 2UL
267 #define STRTAB_STE_1_STRW_SHIFT 30
268
269 #define STRTAB_STE_1_SHCFG_INCOMING 1UL
270 #define STRTAB_STE_1_SHCFG_SHIFT 44
271
272 #define STRTAB_STE_1_PRIVCFG_UNPRIV 2UL
273 #define STRTAB_STE_1_PRIVCFG_SHIFT 48
274
275 #define STRTAB_STE_2_S2VMID_SHIFT 0
276 #define STRTAB_STE_2_S2VMID_MASK 0xffffUL
277 #define STRTAB_STE_2_VTCR_SHIFT 32
278 #define STRTAB_STE_2_VTCR_MASK 0x7ffffUL
279 #define STRTAB_STE_2_S2AA64 (1UL << 51)
280 #define STRTAB_STE_2_S2ENDI (1UL << 52)
281 #define STRTAB_STE_2_S2PTW (1UL << 54)
282 #define STRTAB_STE_2_S2R (1UL << 58)
283
284 #define STRTAB_STE_3_S2TTB_SHIFT 4
285 #define STRTAB_STE_3_S2TTB_MASK 0xfffffffffffUL
286
287 /* Context descriptor (stage-1 only) */
288 #define CTXDESC_CD_DWORDS 8
289 #define CTXDESC_CD_0_TCR_T0SZ_SHIFT 0
290 #define ARM64_TCR_T0SZ_SHIFT 0
291 #define ARM64_TCR_T0SZ_MASK 0x1fUL
292 #define CTXDESC_CD_0_TCR_TG0_SHIFT 6
293 #define ARM64_TCR_TG0_SHIFT 14
294 #define ARM64_TCR_TG0_MASK 0x3UL
295 #define CTXDESC_CD_0_TCR_IRGN0_SHIFT 8
296 #define ARM64_TCR_IRGN0_SHIFT 8
297 #define ARM64_TCR_IRGN0_MASK 0x3UL
298 #define CTXDESC_CD_0_TCR_ORGN0_SHIFT 10
299 #define ARM64_TCR_ORGN0_SHIFT 10
300 #define ARM64_TCR_ORGN0_MASK 0x3UL
301 #define CTXDESC_CD_0_TCR_SH0_SHIFT 12
302 #define ARM64_TCR_SH0_SHIFT 12
303 #define ARM64_TCR_SH0_MASK 0x3UL
304 #define CTXDESC_CD_0_TCR_EPD0_SHIFT 14
305 #define ARM64_TCR_EPD0_SHIFT 7
306 #define ARM64_TCR_EPD0_MASK 0x1UL
307 #define CTXDESC_CD_0_TCR_EPD1_SHIFT 30
308 #define ARM64_TCR_EPD1_SHIFT 23
309 #define ARM64_TCR_EPD1_MASK 0x1UL
310
311 #define CTXDESC_CD_0_ENDI (1UL << 15)
312 #define CTXDESC_CD_0_V (1UL << 31)
313
314 #define CTXDESC_CD_0_TCR_IPS_SHIFT 32
315 #define ARM64_TCR_IPS_SHIFT 32
316 #define ARM64_TCR_IPS_MASK 0x7UL
317 #define CTXDESC_CD_0_TCR_TBI0_SHIFT 38
318 #define ARM64_TCR_TBI0_SHIFT 37
319 #define ARM64_TCR_TBI0_MASK 0x1UL
320
321 #define CTXDESC_CD_0_AA64 (1UL << 41)
322 #define CTXDESC_CD_0_R (1UL << 45)
323 #define CTXDESC_CD_0_A (1UL << 46)
324 #define CTXDESC_CD_0_ASET_SHIFT 47
325 #define CTXDESC_CD_0_ASET_SHARED (0UL << CTXDESC_CD_0_ASET_SHIFT)
326 #define CTXDESC_CD_0_ASET_PRIVATE (1UL << CTXDESC_CD_0_ASET_SHIFT)
327 #define CTXDESC_CD_0_ASID_SHIFT 48
328 #define CTXDESC_CD_0_ASID_MASK 0xffffUL
329
330 #define CTXDESC_CD_1_TTB0_SHIFT 4
331 #define CTXDESC_CD_1_TTB0_MASK 0xfffffffffffUL
332
333 #define CTXDESC_CD_3_MAIR_SHIFT 0
334
335 /* Convert between AArch64 (CPU) TCR format and SMMU CD format */
336 #define ARM_SMMU_TCR2CD(tcr, fld) \
337 (((tcr) >> ARM64_TCR_##fld##_SHIFT & ARM64_TCR_##fld##_MASK) \
338 << CTXDESC_CD_0_TCR_##fld##_SHIFT)
339
340 /* Command queue */
341 #define CMDQ_ENT_DWORDS 2
342 #define CMDQ_MAX_SZ_SHIFT 8
343
344 #define CMDQ_ERR_SHIFT 24
345 #define CMDQ_ERR_MASK 0x7f
346 #define CMDQ_ERR_CERROR_NONE_IDX 0
347 #define CMDQ_ERR_CERROR_ILL_IDX 1
348 #define CMDQ_ERR_CERROR_ABT_IDX 2
349
350 #define CMDQ_0_OP_SHIFT 0
351 #define CMDQ_0_OP_MASK 0xffUL
352 #define CMDQ_0_SSV (1UL << 11)
353
354 #define CMDQ_PREFETCH_0_SID_SHIFT 32
355 #define CMDQ_PREFETCH_1_SIZE_SHIFT 0
356 #define CMDQ_PREFETCH_1_ADDR_MASK ~0xfffUL
357
358 #define CMDQ_CFGI_0_SID_SHIFT 32
359 #define CMDQ_CFGI_0_SID_MASK 0xffffffffUL
360 #define CMDQ_CFGI_1_LEAF (1UL << 0)
361 #define CMDQ_CFGI_1_RANGE_SHIFT 0
362 #define CMDQ_CFGI_1_RANGE_MASK 0x1fUL
363
364 #define CMDQ_TLBI_0_VMID_SHIFT 32
365 #define CMDQ_TLBI_0_ASID_SHIFT 48
366 #define CMDQ_TLBI_1_LEAF (1UL << 0)
367 #define CMDQ_TLBI_1_VA_MASK ~0xfffUL
368 #define CMDQ_TLBI_1_IPA_MASK 0xfffffffff000UL
369
370 #define CMDQ_PRI_0_SSID_SHIFT 12
371 #define CMDQ_PRI_0_SSID_MASK 0xfffffUL
372 #define CMDQ_PRI_0_SID_SHIFT 32
373 #define CMDQ_PRI_0_SID_MASK 0xffffffffUL
374 #define CMDQ_PRI_1_GRPID_SHIFT 0
375 #define CMDQ_PRI_1_GRPID_MASK 0x1ffUL
376 #define CMDQ_PRI_1_RESP_SHIFT 12
377 #define CMDQ_PRI_1_RESP_DENY (0UL << CMDQ_PRI_1_RESP_SHIFT)
378 #define CMDQ_PRI_1_RESP_FAIL (1UL << CMDQ_PRI_1_RESP_SHIFT)
379 #define CMDQ_PRI_1_RESP_SUCC (2UL << CMDQ_PRI_1_RESP_SHIFT)
380
381 #define CMDQ_SYNC_0_CS_SHIFT 12
382 #define CMDQ_SYNC_0_CS_NONE (0UL << CMDQ_SYNC_0_CS_SHIFT)
383 #define CMDQ_SYNC_0_CS_SEV (2UL << CMDQ_SYNC_0_CS_SHIFT)
384
385 /* Event queue */
386 #define EVTQ_ENT_DWORDS 4
387 #define EVTQ_MAX_SZ_SHIFT 7
388
389 #define EVTQ_0_ID_SHIFT 0
390 #define EVTQ_0_ID_MASK 0xffUL
391
392 /* PRI queue */
393 #define PRIQ_ENT_DWORDS 2
394 #define PRIQ_MAX_SZ_SHIFT 8
395
396 #define PRIQ_0_SID_SHIFT 0
397 #define PRIQ_0_SID_MASK 0xffffffffUL
398 #define PRIQ_0_SSID_SHIFT 32
399 #define PRIQ_0_SSID_MASK 0xfffffUL
400 #define PRIQ_0_PERM_PRIV (1UL << 58)
401 #define PRIQ_0_PERM_EXEC (1UL << 59)
402 #define PRIQ_0_PERM_READ (1UL << 60)
403 #define PRIQ_0_PERM_WRITE (1UL << 61)
404 #define PRIQ_0_PRG_LAST (1UL << 62)
405 #define PRIQ_0_SSID_V (1UL << 63)
406
407 #define PRIQ_1_PRG_IDX_SHIFT 0
408 #define PRIQ_1_PRG_IDX_MASK 0x1ffUL
409 #define PRIQ_1_ADDR_SHIFT 12
410 #define PRIQ_1_ADDR_MASK 0xfffffffffffffUL
411
412 /* High-level queue structures */
413 #define ARM_SMMU_POLL_TIMEOUT_US 100
414
415 static bool disable_bypass;
416 module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO);
417 MODULE_PARM_DESC(disable_bypass,
418 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
419
420 enum pri_resp {
421 PRI_RESP_DENY,
422 PRI_RESP_FAIL,
423 PRI_RESP_SUCC,
424 };
425
426 enum arm_smmu_msi_index {
427 EVTQ_MSI_INDEX,
428 GERROR_MSI_INDEX,
429 PRIQ_MSI_INDEX,
430 ARM_SMMU_MAX_MSIS,
431 };
432
433 static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
434 [EVTQ_MSI_INDEX] = {
435 ARM_SMMU_EVTQ_IRQ_CFG0,
436 ARM_SMMU_EVTQ_IRQ_CFG1,
437 ARM_SMMU_EVTQ_IRQ_CFG2,
438 },
439 [GERROR_MSI_INDEX] = {
440 ARM_SMMU_GERROR_IRQ_CFG0,
441 ARM_SMMU_GERROR_IRQ_CFG1,
442 ARM_SMMU_GERROR_IRQ_CFG2,
443 },
444 [PRIQ_MSI_INDEX] = {
445 ARM_SMMU_PRIQ_IRQ_CFG0,
446 ARM_SMMU_PRIQ_IRQ_CFG1,
447 ARM_SMMU_PRIQ_IRQ_CFG2,
448 },
449 };
450
451 struct arm_smmu_cmdq_ent {
452 /* Common fields */
453 u8 opcode;
454 bool substream_valid;
455
456 /* Command-specific fields */
457 union {
458 #define CMDQ_OP_PREFETCH_CFG 0x1
459 struct {
460 u32 sid;
461 u8 size;
462 u64 addr;
463 } prefetch;
464
465 #define CMDQ_OP_CFGI_STE 0x3
466 #define CMDQ_OP_CFGI_ALL 0x4
467 struct {
468 u32 sid;
469 union {
470 bool leaf;
471 u8 span;
472 };
473 } cfgi;
474
475 #define CMDQ_OP_TLBI_NH_ASID 0x11
476 #define CMDQ_OP_TLBI_NH_VA 0x12
477 #define CMDQ_OP_TLBI_EL2_ALL 0x20
478 #define CMDQ_OP_TLBI_S12_VMALL 0x28
479 #define CMDQ_OP_TLBI_S2_IPA 0x2a
480 #define CMDQ_OP_TLBI_NSNH_ALL 0x30
481 struct {
482 u16 asid;
483 u16 vmid;
484 bool leaf;
485 u64 addr;
486 } tlbi;
487
488 #define CMDQ_OP_PRI_RESP 0x41
489 struct {
490 u32 sid;
491 u32 ssid;
492 u16 grpid;
493 enum pri_resp resp;
494 } pri;
495
496 #define CMDQ_OP_CMD_SYNC 0x46
497 };
498 };
499
500 struct arm_smmu_queue {
501 int irq; /* Wired interrupt */
502
503 __le64 *base;
504 dma_addr_t base_dma;
505 u64 q_base;
506
507 size_t ent_dwords;
508 u32 max_n_shift;
509 u32 prod;
510 u32 cons;
511
512 u32 __iomem *prod_reg;
513 u32 __iomem *cons_reg;
514 };
515
516 struct arm_smmu_cmdq {
517 struct arm_smmu_queue q;
518 spinlock_t lock;
519 };
520
521 struct arm_smmu_evtq {
522 struct arm_smmu_queue q;
523 u32 max_stalls;
524 };
525
526 struct arm_smmu_priq {
527 struct arm_smmu_queue q;
528 };
529
530 /* High-level stream table and context descriptor structures */
531 struct arm_smmu_strtab_l1_desc {
532 u8 span;
533
534 __le64 *l2ptr;
535 dma_addr_t l2ptr_dma;
536 };
537
538 struct arm_smmu_s1_cfg {
539 __le64 *cdptr;
540 dma_addr_t cdptr_dma;
541
542 struct arm_smmu_ctx_desc {
543 u16 asid;
544 u64 ttbr;
545 u64 tcr;
546 u64 mair;
547 } cd;
548 };
549
550 struct arm_smmu_s2_cfg {
551 u16 vmid;
552 u64 vttbr;
553 u64 vtcr;
554 };
555
556 struct arm_smmu_strtab_ent {
557 bool valid;
558
559 bool bypass; /* Overrides s1/s2 config */
560 struct arm_smmu_s1_cfg *s1_cfg;
561 struct arm_smmu_s2_cfg *s2_cfg;
562 };
563
564 struct arm_smmu_strtab_cfg {
565 __le64 *strtab;
566 dma_addr_t strtab_dma;
567 struct arm_smmu_strtab_l1_desc *l1_desc;
568 unsigned int num_l1_ents;
569
570 u64 strtab_base;
571 u32 strtab_base_cfg;
572 };
573
574 /* An SMMUv3 instance */
575 struct arm_smmu_device {
576 struct device *dev;
577 void __iomem *base;
578
579 #define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0)
580 #define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1)
581 #define ARM_SMMU_FEAT_TT_LE (1 << 2)
582 #define ARM_SMMU_FEAT_TT_BE (1 << 3)
583 #define ARM_SMMU_FEAT_PRI (1 << 4)
584 #define ARM_SMMU_FEAT_ATS (1 << 5)
585 #define ARM_SMMU_FEAT_SEV (1 << 6)
586 #define ARM_SMMU_FEAT_MSI (1 << 7)
587 #define ARM_SMMU_FEAT_COHERENCY (1 << 8)
588 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 9)
589 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 10)
590 #define ARM_SMMU_FEAT_STALLS (1 << 11)
591 #define ARM_SMMU_FEAT_HYP (1 << 12)
592 u32 features;
593
594 #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0)
595 u32 options;
596
597 struct arm_smmu_cmdq cmdq;
598 struct arm_smmu_evtq evtq;
599 struct arm_smmu_priq priq;
600
601 int gerr_irq;
602
603 unsigned long ias; /* IPA */
604 unsigned long oas; /* PA */
605 unsigned long pgsize_bitmap;
606
607 #define ARM_SMMU_MAX_ASIDS (1 << 16)
608 unsigned int asid_bits;
609 DECLARE_BITMAP(asid_map, ARM_SMMU_MAX_ASIDS);
610
611 #define ARM_SMMU_MAX_VMIDS (1 << 16)
612 unsigned int vmid_bits;
613 DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
614
615 unsigned int ssid_bits;
616 unsigned int sid_bits;
617
618 struct arm_smmu_strtab_cfg strtab_cfg;
619 };
620
621 /* SMMU private data for each master */
622 struct arm_smmu_master_data {
623 struct arm_smmu_device *smmu;
624 struct arm_smmu_strtab_ent ste;
625 };
626
627 /* SMMU private data for an IOMMU domain */
628 enum arm_smmu_domain_stage {
629 ARM_SMMU_DOMAIN_S1 = 0,
630 ARM_SMMU_DOMAIN_S2,
631 ARM_SMMU_DOMAIN_NESTED,
632 };
633
634 struct arm_smmu_domain {
635 struct arm_smmu_device *smmu;
636 struct mutex init_mutex; /* Protects smmu pointer */
637
638 struct io_pgtable_ops *pgtbl_ops;
639 spinlock_t pgtbl_lock;
640
641 enum arm_smmu_domain_stage stage;
642 union {
643 struct arm_smmu_s1_cfg s1_cfg;
644 struct arm_smmu_s2_cfg s2_cfg;
645 };
646
647 struct iommu_domain domain;
648 };
649
650 struct arm_smmu_option_prop {
651 u32 opt;
652 const char *prop;
653 };
654
655 static struct arm_smmu_option_prop arm_smmu_options[] = {
656 { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
657 { 0, NULL},
658 };
659
660 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
661 {
662 return container_of(dom, struct arm_smmu_domain, domain);
663 }
664
665 static void parse_driver_options(struct arm_smmu_device *smmu)
666 {
667 int i = 0;
668
669 do {
670 if (of_property_read_bool(smmu->dev->of_node,
671 arm_smmu_options[i].prop)) {
672 smmu->options |= arm_smmu_options[i].opt;
673 dev_notice(smmu->dev, "option %s\n",
674 arm_smmu_options[i].prop);
675 }
676 } while (arm_smmu_options[++i].opt);
677 }
678
679 /* Low-level queue manipulation functions */
680 static bool queue_full(struct arm_smmu_queue *q)
681 {
682 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
683 Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
684 }
685
686 static bool queue_empty(struct arm_smmu_queue *q)
687 {
688 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
689 Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
690 }
691
692 static void queue_sync_cons(struct arm_smmu_queue *q)
693 {
694 q->cons = readl_relaxed(q->cons_reg);
695 }
696
697 static void queue_inc_cons(struct arm_smmu_queue *q)
698 {
699 u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
700
701 q->cons = Q_OVF(q, q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
702 writel(q->cons, q->cons_reg);
703 }
704
705 static int queue_sync_prod(struct arm_smmu_queue *q)
706 {
707 int ret = 0;
708 u32 prod = readl_relaxed(q->prod_reg);
709
710 if (Q_OVF(q, prod) != Q_OVF(q, q->prod))
711 ret = -EOVERFLOW;
712
713 q->prod = prod;
714 return ret;
715 }
716
717 static void queue_inc_prod(struct arm_smmu_queue *q)
718 {
719 u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + 1;
720
721 q->prod = Q_OVF(q, q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
722 writel(q->prod, q->prod_reg);
723 }
724
725 /*
726 * Wait for the SMMU to consume items. If drain is true, wait until the queue
727 * is empty. Otherwise, wait until there is at least one free slot.
728 */
729 static int queue_poll_cons(struct arm_smmu_queue *q, bool drain, bool wfe)
730 {
731 ktime_t timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
732
733 while (queue_sync_cons(q), (drain ? !queue_empty(q) : queue_full(q))) {
734 if (ktime_compare(ktime_get(), timeout) > 0)
735 return -ETIMEDOUT;
736
737 if (wfe) {
738 wfe();
739 } else {
740 cpu_relax();
741 udelay(1);
742 }
743 }
744
745 return 0;
746 }
747
748 static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
749 {
750 int i;
751
752 for (i = 0; i < n_dwords; ++i)
753 *dst++ = cpu_to_le64(*src++);
754 }
755
756 static int queue_insert_raw(struct arm_smmu_queue *q, u64 *ent)
757 {
758 if (queue_full(q))
759 return -ENOSPC;
760
761 queue_write(Q_ENT(q, q->prod), ent, q->ent_dwords);
762 queue_inc_prod(q);
763 return 0;
764 }
765
766 static void queue_read(__le64 *dst, u64 *src, size_t n_dwords)
767 {
768 int i;
769
770 for (i = 0; i < n_dwords; ++i)
771 *dst++ = le64_to_cpu(*src++);
772 }
773
774 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
775 {
776 if (queue_empty(q))
777 return -EAGAIN;
778
779 queue_read(ent, Q_ENT(q, q->cons), q->ent_dwords);
780 queue_inc_cons(q);
781 return 0;
782 }
783
784 /* High-level queue accessors */
785 static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
786 {
787 memset(cmd, 0, CMDQ_ENT_DWORDS << 3);
788 cmd[0] |= (ent->opcode & CMDQ_0_OP_MASK) << CMDQ_0_OP_SHIFT;
789
790 switch (ent->opcode) {
791 case CMDQ_OP_TLBI_EL2_ALL:
792 case CMDQ_OP_TLBI_NSNH_ALL:
793 break;
794 case CMDQ_OP_PREFETCH_CFG:
795 cmd[0] |= (u64)ent->prefetch.sid << CMDQ_PREFETCH_0_SID_SHIFT;
796 cmd[1] |= ent->prefetch.size << CMDQ_PREFETCH_1_SIZE_SHIFT;
797 cmd[1] |= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK;
798 break;
799 case CMDQ_OP_CFGI_STE:
800 cmd[0] |= (u64)ent->cfgi.sid << CMDQ_CFGI_0_SID_SHIFT;
801 cmd[1] |= ent->cfgi.leaf ? CMDQ_CFGI_1_LEAF : 0;
802 break;
803 case CMDQ_OP_CFGI_ALL:
804 /* Cover the entire SID range */
805 cmd[1] |= CMDQ_CFGI_1_RANGE_MASK << CMDQ_CFGI_1_RANGE_SHIFT;
806 break;
807 case CMDQ_OP_TLBI_NH_VA:
808 cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT;
809 cmd[1] |= ent->tlbi.leaf ? CMDQ_TLBI_1_LEAF : 0;
810 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
811 break;
812 case CMDQ_OP_TLBI_S2_IPA:
813 cmd[0] |= (u64)ent->tlbi.vmid << CMDQ_TLBI_0_VMID_SHIFT;
814 cmd[1] |= ent->tlbi.leaf ? CMDQ_TLBI_1_LEAF : 0;
815 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
816 break;
817 case CMDQ_OP_TLBI_NH_ASID:
818 cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT;
819 /* Fallthrough */
820 case CMDQ_OP_TLBI_S12_VMALL:
821 cmd[0] |= (u64)ent->tlbi.vmid << CMDQ_TLBI_0_VMID_SHIFT;
822 break;
823 case CMDQ_OP_PRI_RESP:
824 cmd[0] |= ent->substream_valid ? CMDQ_0_SSV : 0;
825 cmd[0] |= ent->pri.ssid << CMDQ_PRI_0_SSID_SHIFT;
826 cmd[0] |= (u64)ent->pri.sid << CMDQ_PRI_0_SID_SHIFT;
827 cmd[1] |= ent->pri.grpid << CMDQ_PRI_1_GRPID_SHIFT;
828 switch (ent->pri.resp) {
829 case PRI_RESP_DENY:
830 cmd[1] |= CMDQ_PRI_1_RESP_DENY;
831 break;
832 case PRI_RESP_FAIL:
833 cmd[1] |= CMDQ_PRI_1_RESP_FAIL;
834 break;
835 case PRI_RESP_SUCC:
836 cmd[1] |= CMDQ_PRI_1_RESP_SUCC;
837 break;
838 default:
839 return -EINVAL;
840 }
841 break;
842 case CMDQ_OP_CMD_SYNC:
843 cmd[0] |= CMDQ_SYNC_0_CS_SEV;
844 break;
845 default:
846 return -ENOENT;
847 }
848
849 return 0;
850 }
851
852 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
853 {
854 static const char *cerror_str[] = {
855 [CMDQ_ERR_CERROR_NONE_IDX] = "No error",
856 [CMDQ_ERR_CERROR_ILL_IDX] = "Illegal command",
857 [CMDQ_ERR_CERROR_ABT_IDX] = "Abort on command fetch",
858 };
859
860 int i;
861 u64 cmd[CMDQ_ENT_DWORDS];
862 struct arm_smmu_queue *q = &smmu->cmdq.q;
863 u32 cons = readl_relaxed(q->cons_reg);
864 u32 idx = cons >> CMDQ_ERR_SHIFT & CMDQ_ERR_MASK;
865 struct arm_smmu_cmdq_ent cmd_sync = {
866 .opcode = CMDQ_OP_CMD_SYNC,
867 };
868
869 dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
870 idx < ARRAY_SIZE(cerror_str) ? cerror_str[idx] : "Unknown");
871
872 switch (idx) {
873 case CMDQ_ERR_CERROR_ABT_IDX:
874 dev_err(smmu->dev, "retrying command fetch\n");
875 case CMDQ_ERR_CERROR_NONE_IDX:
876 return;
877 case CMDQ_ERR_CERROR_ILL_IDX:
878 /* Fallthrough */
879 default:
880 break;
881 }
882
883 /*
884 * We may have concurrent producers, so we need to be careful
885 * not to touch any of the shadow cmdq state.
886 */
887 queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
888 dev_err(smmu->dev, "skipping command in error state:\n");
889 for (i = 0; i < ARRAY_SIZE(cmd); ++i)
890 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
891
892 /* Convert the erroneous command into a CMD_SYNC */
893 if (arm_smmu_cmdq_build_cmd(cmd, &cmd_sync)) {
894 dev_err(smmu->dev, "failed to convert to CMD_SYNC\n");
895 return;
896 }
897
898 queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
899 }
900
901 static void arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
902 struct arm_smmu_cmdq_ent *ent)
903 {
904 u64 cmd[CMDQ_ENT_DWORDS];
905 unsigned long flags;
906 bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
907 struct arm_smmu_queue *q = &smmu->cmdq.q;
908
909 if (arm_smmu_cmdq_build_cmd(cmd, ent)) {
910 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
911 ent->opcode);
912 return;
913 }
914
915 spin_lock_irqsave(&smmu->cmdq.lock, flags);
916 while (queue_insert_raw(q, cmd) == -ENOSPC) {
917 if (queue_poll_cons(q, false, wfe))
918 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
919 }
920
921 if (ent->opcode == CMDQ_OP_CMD_SYNC && queue_poll_cons(q, true, wfe))
922 dev_err_ratelimited(smmu->dev, "CMD_SYNC timeout\n");
923 spin_unlock_irqrestore(&smmu->cmdq.lock, flags);
924 }
925
926 /* Context descriptor manipulation functions */
927 static u64 arm_smmu_cpu_tcr_to_cd(u64 tcr)
928 {
929 u64 val = 0;
930
931 /* Repack the TCR. Just care about TTBR0 for now */
932 val |= ARM_SMMU_TCR2CD(tcr, T0SZ);
933 val |= ARM_SMMU_TCR2CD(tcr, TG0);
934 val |= ARM_SMMU_TCR2CD(tcr, IRGN0);
935 val |= ARM_SMMU_TCR2CD(tcr, ORGN0);
936 val |= ARM_SMMU_TCR2CD(tcr, SH0);
937 val |= ARM_SMMU_TCR2CD(tcr, EPD0);
938 val |= ARM_SMMU_TCR2CD(tcr, EPD1);
939 val |= ARM_SMMU_TCR2CD(tcr, IPS);
940 val |= ARM_SMMU_TCR2CD(tcr, TBI0);
941
942 return val;
943 }
944
945 static void arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu,
946 struct arm_smmu_s1_cfg *cfg)
947 {
948 u64 val;
949
950 /*
951 * We don't need to issue any invalidation here, as we'll invalidate
952 * the STE when installing the new entry anyway.
953 */
954 val = arm_smmu_cpu_tcr_to_cd(cfg->cd.tcr) |
955 #ifdef __BIG_ENDIAN
956 CTXDESC_CD_0_ENDI |
957 #endif
958 CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET_PRIVATE |
959 CTXDESC_CD_0_AA64 | (u64)cfg->cd.asid << CTXDESC_CD_0_ASID_SHIFT |
960 CTXDESC_CD_0_V;
961 cfg->cdptr[0] = cpu_to_le64(val);
962
963 val = cfg->cd.ttbr & CTXDESC_CD_1_TTB0_MASK << CTXDESC_CD_1_TTB0_SHIFT;
964 cfg->cdptr[1] = cpu_to_le64(val);
965
966 cfg->cdptr[3] = cpu_to_le64(cfg->cd.mair << CTXDESC_CD_3_MAIR_SHIFT);
967 }
968
969 /* Stream table manipulation functions */
970 static void
971 arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc)
972 {
973 u64 val = 0;
974
975 val |= (desc->span & STRTAB_L1_DESC_SPAN_MASK)
976 << STRTAB_L1_DESC_SPAN_SHIFT;
977 val |= desc->l2ptr_dma &
978 STRTAB_L1_DESC_L2PTR_MASK << STRTAB_L1_DESC_L2PTR_SHIFT;
979
980 *dst = cpu_to_le64(val);
981 }
982
983 static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid)
984 {
985 struct arm_smmu_cmdq_ent cmd = {
986 .opcode = CMDQ_OP_CFGI_STE,
987 .cfgi = {
988 .sid = sid,
989 .leaf = true,
990 },
991 };
992
993 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
994 cmd.opcode = CMDQ_OP_CMD_SYNC;
995 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
996 }
997
998 static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid,
999 __le64 *dst, struct arm_smmu_strtab_ent *ste)
1000 {
1001 /*
1002 * This is hideously complicated, but we only really care about
1003 * three cases at the moment:
1004 *
1005 * 1. Invalid (all zero) -> bypass (init)
1006 * 2. Bypass -> translation (attach)
1007 * 3. Translation -> bypass (detach)
1008 *
1009 * Given that we can't update the STE atomically and the SMMU
1010 * doesn't read the thing in a defined order, that leaves us
1011 * with the following maintenance requirements:
1012 *
1013 * 1. Update Config, return (init time STEs aren't live)
1014 * 2. Write everything apart from dword 0, sync, write dword 0, sync
1015 * 3. Update Config, sync
1016 */
1017 u64 val = le64_to_cpu(dst[0]);
1018 bool ste_live = false;
1019 struct arm_smmu_cmdq_ent prefetch_cmd = {
1020 .opcode = CMDQ_OP_PREFETCH_CFG,
1021 .prefetch = {
1022 .sid = sid,
1023 },
1024 };
1025
1026 if (val & STRTAB_STE_0_V) {
1027 u64 cfg;
1028
1029 cfg = val & STRTAB_STE_0_CFG_MASK << STRTAB_STE_0_CFG_SHIFT;
1030 switch (cfg) {
1031 case STRTAB_STE_0_CFG_BYPASS:
1032 break;
1033 case STRTAB_STE_0_CFG_S1_TRANS:
1034 case STRTAB_STE_0_CFG_S2_TRANS:
1035 ste_live = true;
1036 break;
1037 case STRTAB_STE_0_CFG_ABORT:
1038 if (disable_bypass)
1039 break;
1040 default:
1041 BUG(); /* STE corruption */
1042 }
1043 }
1044
1045 /* Nuke the existing STE_0 value, as we're going to rewrite it */
1046 val = ste->valid ? STRTAB_STE_0_V : 0;
1047
1048 if (ste->bypass) {
1049 val |= disable_bypass ? STRTAB_STE_0_CFG_ABORT
1050 : STRTAB_STE_0_CFG_BYPASS;
1051 dst[0] = cpu_to_le64(val);
1052 dst[1] = cpu_to_le64(STRTAB_STE_1_SHCFG_INCOMING
1053 << STRTAB_STE_1_SHCFG_SHIFT);
1054 dst[2] = 0; /* Nuke the VMID */
1055 if (ste_live)
1056 arm_smmu_sync_ste_for_sid(smmu, sid);
1057 return;
1058 }
1059
1060 if (ste->s1_cfg) {
1061 BUG_ON(ste_live);
1062 dst[1] = cpu_to_le64(
1063 STRTAB_STE_1_S1C_CACHE_WBRA
1064 << STRTAB_STE_1_S1CIR_SHIFT |
1065 STRTAB_STE_1_S1C_CACHE_WBRA
1066 << STRTAB_STE_1_S1COR_SHIFT |
1067 STRTAB_STE_1_S1C_SH_ISH << STRTAB_STE_1_S1CSH_SHIFT |
1068 #ifdef CONFIG_PCI_ATS
1069 STRTAB_STE_1_EATS_TRANS << STRTAB_STE_1_EATS_SHIFT |
1070 #endif
1071 STRTAB_STE_1_STRW_NSEL1 << STRTAB_STE_1_STRW_SHIFT |
1072 STRTAB_STE_1_PRIVCFG_UNPRIV <<
1073 STRTAB_STE_1_PRIVCFG_SHIFT);
1074
1075 if (smmu->features & ARM_SMMU_FEAT_STALLS)
1076 dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
1077
1078 val |= (ste->s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK
1079 << STRTAB_STE_0_S1CTXPTR_SHIFT) |
1080 STRTAB_STE_0_CFG_S1_TRANS;
1081 }
1082
1083 if (ste->s2_cfg) {
1084 BUG_ON(ste_live);
1085 dst[2] = cpu_to_le64(
1086 ste->s2_cfg->vmid << STRTAB_STE_2_S2VMID_SHIFT |
1087 (ste->s2_cfg->vtcr & STRTAB_STE_2_VTCR_MASK)
1088 << STRTAB_STE_2_VTCR_SHIFT |
1089 #ifdef __BIG_ENDIAN
1090 STRTAB_STE_2_S2ENDI |
1091 #endif
1092 STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 |
1093 STRTAB_STE_2_S2R);
1094
1095 dst[3] = cpu_to_le64(ste->s2_cfg->vttbr &
1096 STRTAB_STE_3_S2TTB_MASK << STRTAB_STE_3_S2TTB_SHIFT);
1097
1098 val |= STRTAB_STE_0_CFG_S2_TRANS;
1099 }
1100
1101 arm_smmu_sync_ste_for_sid(smmu, sid);
1102 dst[0] = cpu_to_le64(val);
1103 arm_smmu_sync_ste_for_sid(smmu, sid);
1104
1105 /* It's likely that we'll want to use the new STE soon */
1106 if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))
1107 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
1108 }
1109
1110 static void arm_smmu_init_bypass_stes(u64 *strtab, unsigned int nent)
1111 {
1112 unsigned int i;
1113 struct arm_smmu_strtab_ent ste = {
1114 .valid = true,
1115 .bypass = true,
1116 };
1117
1118 for (i = 0; i < nent; ++i) {
1119 arm_smmu_write_strtab_ent(NULL, -1, strtab, &ste);
1120 strtab += STRTAB_STE_DWORDS;
1121 }
1122 }
1123
1124 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1125 {
1126 size_t size;
1127 void *strtab;
1128 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1129 struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT];
1130
1131 if (desc->l2ptr)
1132 return 0;
1133
1134 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
1135 strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];
1136
1137 desc->span = STRTAB_SPLIT + 1;
1138 desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
1139 GFP_KERNEL | __GFP_ZERO);
1140 if (!desc->l2ptr) {
1141 dev_err(smmu->dev,
1142 "failed to allocate l2 stream table for SID %u\n",
1143 sid);
1144 return -ENOMEM;
1145 }
1146
1147 arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT);
1148 arm_smmu_write_strtab_l1_desc(strtab, desc);
1149 return 0;
1150 }
1151
1152 /* IRQ and event handlers */
1153 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
1154 {
1155 int i;
1156 struct arm_smmu_device *smmu = dev;
1157 struct arm_smmu_queue *q = &smmu->evtq.q;
1158 u64 evt[EVTQ_ENT_DWORDS];
1159
1160 do {
1161 while (!queue_remove_raw(q, evt)) {
1162 u8 id = evt[0] >> EVTQ_0_ID_SHIFT & EVTQ_0_ID_MASK;
1163
1164 dev_info(smmu->dev, "event 0x%02x received:\n", id);
1165 for (i = 0; i < ARRAY_SIZE(evt); ++i)
1166 dev_info(smmu->dev, "\t0x%016llx\n",
1167 (unsigned long long)evt[i]);
1168
1169 }
1170
1171 /*
1172 * Not much we can do on overflow, so scream and pretend we're
1173 * trying harder.
1174 */
1175 if (queue_sync_prod(q) == -EOVERFLOW)
1176 dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
1177 } while (!queue_empty(q));
1178
1179 /* Sync our overflow flag, as we believe we're up to speed */
1180 q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
1181 return IRQ_HANDLED;
1182 }
1183
1184 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
1185 {
1186 u32 sid, ssid;
1187 u16 grpid;
1188 bool ssv, last;
1189
1190 sid = evt[0] >> PRIQ_0_SID_SHIFT & PRIQ_0_SID_MASK;
1191 ssv = evt[0] & PRIQ_0_SSID_V;
1192 ssid = ssv ? evt[0] >> PRIQ_0_SSID_SHIFT & PRIQ_0_SSID_MASK : 0;
1193 last = evt[0] & PRIQ_0_PRG_LAST;
1194 grpid = evt[1] >> PRIQ_1_PRG_IDX_SHIFT & PRIQ_1_PRG_IDX_MASK;
1195
1196 dev_info(smmu->dev, "unexpected PRI request received:\n");
1197 dev_info(smmu->dev,
1198 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
1199 sid, ssid, grpid, last ? "L" : "",
1200 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
1201 evt[0] & PRIQ_0_PERM_READ ? "R" : "",
1202 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
1203 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
1204 evt[1] & PRIQ_1_ADDR_MASK << PRIQ_1_ADDR_SHIFT);
1205
1206 if (last) {
1207 struct arm_smmu_cmdq_ent cmd = {
1208 .opcode = CMDQ_OP_PRI_RESP,
1209 .substream_valid = ssv,
1210 .pri = {
1211 .sid = sid,
1212 .ssid = ssid,
1213 .grpid = grpid,
1214 .resp = PRI_RESP_DENY,
1215 },
1216 };
1217
1218 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1219 }
1220 }
1221
1222 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
1223 {
1224 struct arm_smmu_device *smmu = dev;
1225 struct arm_smmu_queue *q = &smmu->priq.q;
1226 u64 evt[PRIQ_ENT_DWORDS];
1227
1228 do {
1229 while (!queue_remove_raw(q, evt))
1230 arm_smmu_handle_ppr(smmu, evt);
1231
1232 if (queue_sync_prod(q) == -EOVERFLOW)
1233 dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
1234 } while (!queue_empty(q));
1235
1236 /* Sync our overflow flag, as we believe we're up to speed */
1237 q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
1238 return IRQ_HANDLED;
1239 }
1240
1241 static irqreturn_t arm_smmu_cmdq_sync_handler(int irq, void *dev)
1242 {
1243 /* We don't actually use CMD_SYNC interrupts for anything */
1244 return IRQ_HANDLED;
1245 }
1246
1247 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
1248
1249 static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
1250 {
1251 u32 gerror, gerrorn, active;
1252 struct arm_smmu_device *smmu = dev;
1253
1254 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
1255 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
1256
1257 active = gerror ^ gerrorn;
1258 if (!(active & GERROR_ERR_MASK))
1259 return IRQ_NONE; /* No errors pending */
1260
1261 dev_warn(smmu->dev,
1262 "unexpected global error reported (0x%08x), this could be serious\n",
1263 active);
1264
1265 if (active & GERROR_SFM_ERR) {
1266 dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
1267 arm_smmu_device_disable(smmu);
1268 }
1269
1270 if (active & GERROR_MSI_GERROR_ABT_ERR)
1271 dev_warn(smmu->dev, "GERROR MSI write aborted\n");
1272
1273 if (active & GERROR_MSI_PRIQ_ABT_ERR)
1274 dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
1275
1276 if (active & GERROR_MSI_EVTQ_ABT_ERR)
1277 dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
1278
1279 if (active & GERROR_MSI_CMDQ_ABT_ERR) {
1280 dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
1281 arm_smmu_cmdq_sync_handler(irq, smmu->dev);
1282 }
1283
1284 if (active & GERROR_PRIQ_ABT_ERR)
1285 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
1286
1287 if (active & GERROR_EVTQ_ABT_ERR)
1288 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
1289
1290 if (active & GERROR_CMDQ_ERR)
1291 arm_smmu_cmdq_skip_err(smmu);
1292
1293 writel(gerror, smmu->base + ARM_SMMU_GERRORN);
1294 return IRQ_HANDLED;
1295 }
1296
1297 /* IO_PGTABLE API */
1298 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
1299 {
1300 struct arm_smmu_cmdq_ent cmd;
1301
1302 cmd.opcode = CMDQ_OP_CMD_SYNC;
1303 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1304 }
1305
1306 static void arm_smmu_tlb_sync(void *cookie)
1307 {
1308 struct arm_smmu_domain *smmu_domain = cookie;
1309 __arm_smmu_tlb_sync(smmu_domain->smmu);
1310 }
1311
1312 static void arm_smmu_tlb_inv_context(void *cookie)
1313 {
1314 struct arm_smmu_domain *smmu_domain = cookie;
1315 struct arm_smmu_device *smmu = smmu_domain->smmu;
1316 struct arm_smmu_cmdq_ent cmd;
1317
1318 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1319 cmd.opcode = CMDQ_OP_TLBI_NH_ASID;
1320 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
1321 cmd.tlbi.vmid = 0;
1322 } else {
1323 cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
1324 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1325 }
1326
1327 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1328 __arm_smmu_tlb_sync(smmu);
1329 }
1330
1331 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
1332 size_t granule, bool leaf, void *cookie)
1333 {
1334 struct arm_smmu_domain *smmu_domain = cookie;
1335 struct arm_smmu_device *smmu = smmu_domain->smmu;
1336 struct arm_smmu_cmdq_ent cmd = {
1337 .tlbi = {
1338 .leaf = leaf,
1339 .addr = iova,
1340 },
1341 };
1342
1343 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1344 cmd.opcode = CMDQ_OP_TLBI_NH_VA;
1345 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
1346 } else {
1347 cmd.opcode = CMDQ_OP_TLBI_S2_IPA;
1348 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1349 }
1350
1351 do {
1352 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1353 cmd.tlbi.addr += granule;
1354 } while (size -= granule);
1355 }
1356
1357 static const struct iommu_gather_ops arm_smmu_gather_ops = {
1358 .tlb_flush_all = arm_smmu_tlb_inv_context,
1359 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
1360 .tlb_sync = arm_smmu_tlb_sync,
1361 };
1362
1363 /* IOMMU API */
1364 static bool arm_smmu_capable(enum iommu_cap cap)
1365 {
1366 switch (cap) {
1367 case IOMMU_CAP_CACHE_COHERENCY:
1368 return true;
1369 case IOMMU_CAP_INTR_REMAP:
1370 return true; /* MSIs are just memory writes */
1371 case IOMMU_CAP_NOEXEC:
1372 return true;
1373 default:
1374 return false;
1375 }
1376 }
1377
1378 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1379 {
1380 struct arm_smmu_domain *smmu_domain;
1381
1382 if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
1383 return NULL;
1384
1385 /*
1386 * Allocate the domain and initialise some of its data structures.
1387 * We can't really do anything meaningful until we've added a
1388 * master.
1389 */
1390 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1391 if (!smmu_domain)
1392 return NULL;
1393
1394 if (type == IOMMU_DOMAIN_DMA &&
1395 iommu_get_dma_cookie(&smmu_domain->domain)) {
1396 kfree(smmu_domain);
1397 return NULL;
1398 }
1399
1400 mutex_init(&smmu_domain->init_mutex);
1401 spin_lock_init(&smmu_domain->pgtbl_lock);
1402 return &smmu_domain->domain;
1403 }
1404
1405 static int arm_smmu_bitmap_alloc(unsigned long *map, int span)
1406 {
1407 int idx, size = 1 << span;
1408
1409 do {
1410 idx = find_first_zero_bit(map, size);
1411 if (idx == size)
1412 return -ENOSPC;
1413 } while (test_and_set_bit(idx, map));
1414
1415 return idx;
1416 }
1417
1418 static void arm_smmu_bitmap_free(unsigned long *map, int idx)
1419 {
1420 clear_bit(idx, map);
1421 }
1422
1423 static void arm_smmu_domain_free(struct iommu_domain *domain)
1424 {
1425 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1426 struct arm_smmu_device *smmu = smmu_domain->smmu;
1427
1428 iommu_put_dma_cookie(domain);
1429 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1430
1431 /* Free the CD and ASID, if we allocated them */
1432 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1433 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1434
1435 if (cfg->cdptr) {
1436 dmam_free_coherent(smmu_domain->smmu->dev,
1437 CTXDESC_CD_DWORDS << 3,
1438 cfg->cdptr,
1439 cfg->cdptr_dma);
1440
1441 arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid);
1442 }
1443 } else {
1444 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
1445 if (cfg->vmid)
1446 arm_smmu_bitmap_free(smmu->vmid_map, cfg->vmid);
1447 }
1448
1449 kfree(smmu_domain);
1450 }
1451
1452 static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
1453 struct io_pgtable_cfg *pgtbl_cfg)
1454 {
1455 int ret;
1456 int asid;
1457 struct arm_smmu_device *smmu = smmu_domain->smmu;
1458 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1459
1460 asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits);
1461 if (asid < 0)
1462 return asid;
1463
1464 cfg->cdptr = dmam_alloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3,
1465 &cfg->cdptr_dma,
1466 GFP_KERNEL | __GFP_ZERO);
1467 if (!cfg->cdptr) {
1468 dev_warn(smmu->dev, "failed to allocate context descriptor\n");
1469 ret = -ENOMEM;
1470 goto out_free_asid;
1471 }
1472
1473 cfg->cd.asid = (u16)asid;
1474 cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
1475 cfg->cd.tcr = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
1476 cfg->cd.mair = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
1477 return 0;
1478
1479 out_free_asid:
1480 arm_smmu_bitmap_free(smmu->asid_map, asid);
1481 return ret;
1482 }
1483
1484 static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
1485 struct io_pgtable_cfg *pgtbl_cfg)
1486 {
1487 int vmid;
1488 struct arm_smmu_device *smmu = smmu_domain->smmu;
1489 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
1490
1491 vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits);
1492 if (vmid < 0)
1493 return vmid;
1494
1495 cfg->vmid = (u16)vmid;
1496 cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
1497 cfg->vtcr = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
1498 return 0;
1499 }
1500
1501 static int arm_smmu_domain_finalise(struct iommu_domain *domain)
1502 {
1503 int ret;
1504 unsigned long ias, oas;
1505 enum io_pgtable_fmt fmt;
1506 struct io_pgtable_cfg pgtbl_cfg;
1507 struct io_pgtable_ops *pgtbl_ops;
1508 int (*finalise_stage_fn)(struct arm_smmu_domain *,
1509 struct io_pgtable_cfg *);
1510 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1511 struct arm_smmu_device *smmu = smmu_domain->smmu;
1512
1513 /* Restrict the stage to what we can actually support */
1514 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
1515 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
1516 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
1517 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1518
1519 switch (smmu_domain->stage) {
1520 case ARM_SMMU_DOMAIN_S1:
1521 ias = VA_BITS;
1522 oas = smmu->ias;
1523 fmt = ARM_64_LPAE_S1;
1524 finalise_stage_fn = arm_smmu_domain_finalise_s1;
1525 break;
1526 case ARM_SMMU_DOMAIN_NESTED:
1527 case ARM_SMMU_DOMAIN_S2:
1528 ias = smmu->ias;
1529 oas = smmu->oas;
1530 fmt = ARM_64_LPAE_S2;
1531 finalise_stage_fn = arm_smmu_domain_finalise_s2;
1532 break;
1533 default:
1534 return -EINVAL;
1535 }
1536
1537 pgtbl_cfg = (struct io_pgtable_cfg) {
1538 .pgsize_bitmap = smmu->pgsize_bitmap,
1539 .ias = ias,
1540 .oas = oas,
1541 .tlb = &arm_smmu_gather_ops,
1542 .iommu_dev = smmu->dev,
1543 };
1544
1545 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
1546 if (!pgtbl_ops)
1547 return -ENOMEM;
1548
1549 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
1550 domain->geometry.aperture_end = (1UL << ias) - 1;
1551 domain->geometry.force_aperture = true;
1552 smmu_domain->pgtbl_ops = pgtbl_ops;
1553
1554 ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg);
1555 if (ret < 0)
1556 free_io_pgtable_ops(pgtbl_ops);
1557
1558 return ret;
1559 }
1560
1561 static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
1562 {
1563 __le64 *step;
1564 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1565
1566 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
1567 struct arm_smmu_strtab_l1_desc *l1_desc;
1568 int idx;
1569
1570 /* Two-level walk */
1571 idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS;
1572 l1_desc = &cfg->l1_desc[idx];
1573 idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
1574 step = &l1_desc->l2ptr[idx];
1575 } else {
1576 /* Simple linear lookup */
1577 step = &cfg->strtab[sid * STRTAB_STE_DWORDS];
1578 }
1579
1580 return step;
1581 }
1582
1583 static int arm_smmu_install_ste_for_dev(struct iommu_fwspec *fwspec)
1584 {
1585 int i;
1586 struct arm_smmu_master_data *master = fwspec->iommu_priv;
1587 struct arm_smmu_device *smmu = master->smmu;
1588
1589 for (i = 0; i < fwspec->num_ids; ++i) {
1590 u32 sid = fwspec->ids[i];
1591 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid);
1592
1593 arm_smmu_write_strtab_ent(smmu, sid, step, &master->ste);
1594 }
1595
1596 return 0;
1597 }
1598
1599 static void arm_smmu_detach_dev(struct device *dev)
1600 {
1601 struct arm_smmu_master_data *master = dev->iommu_fwspec->iommu_priv;
1602
1603 master->ste.bypass = true;
1604 if (arm_smmu_install_ste_for_dev(dev->iommu_fwspec) < 0)
1605 dev_warn(dev, "failed to install bypass STE\n");
1606 }
1607
1608 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1609 {
1610 int ret = 0;
1611 struct arm_smmu_device *smmu;
1612 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1613 struct arm_smmu_master_data *master;
1614 struct arm_smmu_strtab_ent *ste;
1615
1616 if (!dev->iommu_fwspec)
1617 return -ENOENT;
1618
1619 master = dev->iommu_fwspec->iommu_priv;
1620 smmu = master->smmu;
1621 ste = &master->ste;
1622
1623 /* Already attached to a different domain? */
1624 if (!ste->bypass)
1625 arm_smmu_detach_dev(dev);
1626
1627 mutex_lock(&smmu_domain->init_mutex);
1628
1629 if (!smmu_domain->smmu) {
1630 smmu_domain->smmu = smmu;
1631 ret = arm_smmu_domain_finalise(domain);
1632 if (ret) {
1633 smmu_domain->smmu = NULL;
1634 goto out_unlock;
1635 }
1636 } else if (smmu_domain->smmu != smmu) {
1637 dev_err(dev,
1638 "cannot attach to SMMU %s (upstream of %s)\n",
1639 dev_name(smmu_domain->smmu->dev),
1640 dev_name(smmu->dev));
1641 ret = -ENXIO;
1642 goto out_unlock;
1643 }
1644
1645 ste->bypass = false;
1646 ste->valid = true;
1647
1648 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1649 ste->s1_cfg = &smmu_domain->s1_cfg;
1650 ste->s2_cfg = NULL;
1651 arm_smmu_write_ctx_desc(smmu, ste->s1_cfg);
1652 } else {
1653 ste->s1_cfg = NULL;
1654 ste->s2_cfg = &smmu_domain->s2_cfg;
1655 }
1656
1657 ret = arm_smmu_install_ste_for_dev(dev->iommu_fwspec);
1658 if (ret < 0)
1659 ste->valid = false;
1660
1661 out_unlock:
1662 mutex_unlock(&smmu_domain->init_mutex);
1663 return ret;
1664 }
1665
1666 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1667 phys_addr_t paddr, size_t size, int prot)
1668 {
1669 int ret;
1670 unsigned long flags;
1671 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1672 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1673
1674 if (!ops)
1675 return -ENODEV;
1676
1677 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1678 ret = ops->map(ops, iova, paddr, size, prot);
1679 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1680 return ret;
1681 }
1682
1683 static size_t
1684 arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
1685 {
1686 size_t ret;
1687 unsigned long flags;
1688 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1689 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1690
1691 if (!ops)
1692 return 0;
1693
1694 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1695 ret = ops->unmap(ops, iova, size);
1696 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1697 return ret;
1698 }
1699
1700 static phys_addr_t
1701 arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
1702 {
1703 phys_addr_t ret;
1704 unsigned long flags;
1705 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1706 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1707
1708 if (!ops)
1709 return 0;
1710
1711 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1712 ret = ops->iova_to_phys(ops, iova);
1713 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1714
1715 return ret;
1716 }
1717
1718 static struct platform_driver arm_smmu_driver;
1719
1720 static int arm_smmu_match_node(struct device *dev, void *data)
1721 {
1722 return dev->fwnode == data;
1723 }
1724
1725 static
1726 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1727 {
1728 struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1729 fwnode, arm_smmu_match_node);
1730 put_device(dev);
1731 return dev ? dev_get_drvdata(dev) : NULL;
1732 }
1733
1734 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
1735 {
1736 unsigned long limit = smmu->strtab_cfg.num_l1_ents;
1737
1738 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
1739 limit *= 1UL << STRTAB_SPLIT;
1740
1741 return sid < limit;
1742 }
1743
1744 static struct iommu_ops arm_smmu_ops;
1745
1746 static int arm_smmu_add_device(struct device *dev)
1747 {
1748 int i, ret;
1749 struct arm_smmu_device *smmu;
1750 struct arm_smmu_master_data *master;
1751 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1752 struct iommu_group *group;
1753
1754 if (!fwspec || fwspec->ops != &arm_smmu_ops)
1755 return -ENODEV;
1756 /*
1757 * We _can_ actually withstand dodgy bus code re-calling add_device()
1758 * without an intervening remove_device()/of_xlate() sequence, but
1759 * we're not going to do so quietly...
1760 */
1761 if (WARN_ON_ONCE(fwspec->iommu_priv)) {
1762 master = fwspec->iommu_priv;
1763 smmu = master->smmu;
1764 } else {
1765 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1766 if (!smmu)
1767 return -ENODEV;
1768 master = kzalloc(sizeof(*master), GFP_KERNEL);
1769 if (!master)
1770 return -ENOMEM;
1771
1772 master->smmu = smmu;
1773 fwspec->iommu_priv = master;
1774 }
1775
1776 /* Check the SIDs are in range of the SMMU and our stream table */
1777 for (i = 0; i < fwspec->num_ids; i++) {
1778 u32 sid = fwspec->ids[i];
1779
1780 if (!arm_smmu_sid_in_range(smmu, sid))
1781 return -ERANGE;
1782
1783 /* Ensure l2 strtab is initialised */
1784 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
1785 ret = arm_smmu_init_l2_strtab(smmu, sid);
1786 if (ret)
1787 return ret;
1788 }
1789 }
1790
1791 group = iommu_group_get_for_dev(dev);
1792 if (!IS_ERR(group))
1793 iommu_group_put(group);
1794
1795 return PTR_ERR_OR_ZERO(group);
1796 }
1797
1798 static void arm_smmu_remove_device(struct device *dev)
1799 {
1800 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1801 struct arm_smmu_master_data *master;
1802
1803 if (!fwspec || fwspec->ops != &arm_smmu_ops)
1804 return;
1805
1806 master = fwspec->iommu_priv;
1807 if (master && master->ste.valid)
1808 arm_smmu_detach_dev(dev);
1809 iommu_group_remove_device(dev);
1810 kfree(master);
1811 iommu_fwspec_free(dev);
1812 }
1813
1814 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1815 {
1816 struct iommu_group *group;
1817
1818 /*
1819 * We don't support devices sharing stream IDs other than PCI RID
1820 * aliases, since the necessary ID-to-device lookup becomes rather
1821 * impractical given a potential sparse 32-bit stream ID space.
1822 */
1823 if (dev_is_pci(dev))
1824 group = pci_device_group(dev);
1825 else
1826 group = generic_device_group(dev);
1827
1828 return group;
1829 }
1830
1831 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1832 enum iommu_attr attr, void *data)
1833 {
1834 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1835
1836 switch (attr) {
1837 case DOMAIN_ATTR_NESTING:
1838 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1839 return 0;
1840 default:
1841 return -ENODEV;
1842 }
1843 }
1844
1845 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1846 enum iommu_attr attr, void *data)
1847 {
1848 int ret = 0;
1849 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1850
1851 mutex_lock(&smmu_domain->init_mutex);
1852
1853 switch (attr) {
1854 case DOMAIN_ATTR_NESTING:
1855 if (smmu_domain->smmu) {
1856 ret = -EPERM;
1857 goto out_unlock;
1858 }
1859
1860 if (*(int *)data)
1861 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1862 else
1863 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1864
1865 break;
1866 default:
1867 ret = -ENODEV;
1868 }
1869
1870 out_unlock:
1871 mutex_unlock(&smmu_domain->init_mutex);
1872 return ret;
1873 }
1874
1875 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1876 {
1877 return iommu_fwspec_add_ids(dev, args->args, 1);
1878 }
1879
1880 static struct iommu_ops arm_smmu_ops = {
1881 .capable = arm_smmu_capable,
1882 .domain_alloc = arm_smmu_domain_alloc,
1883 .domain_free = arm_smmu_domain_free,
1884 .attach_dev = arm_smmu_attach_dev,
1885 .map = arm_smmu_map,
1886 .unmap = arm_smmu_unmap,
1887 .map_sg = default_iommu_map_sg,
1888 .iova_to_phys = arm_smmu_iova_to_phys,
1889 .add_device = arm_smmu_add_device,
1890 .remove_device = arm_smmu_remove_device,
1891 .device_group = arm_smmu_device_group,
1892 .domain_get_attr = arm_smmu_domain_get_attr,
1893 .domain_set_attr = arm_smmu_domain_set_attr,
1894 .of_xlate = arm_smmu_of_xlate,
1895 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1896 };
1897
1898 /* Probing and initialisation functions */
1899 static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
1900 struct arm_smmu_queue *q,
1901 unsigned long prod_off,
1902 unsigned long cons_off,
1903 size_t dwords)
1904 {
1905 size_t qsz = ((1 << q->max_n_shift) * dwords) << 3;
1906
1907 q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma, GFP_KERNEL);
1908 if (!q->base) {
1909 dev_err(smmu->dev, "failed to allocate queue (0x%zx bytes)\n",
1910 qsz);
1911 return -ENOMEM;
1912 }
1913
1914 q->prod_reg = smmu->base + prod_off;
1915 q->cons_reg = smmu->base + cons_off;
1916 q->ent_dwords = dwords;
1917
1918 q->q_base = Q_BASE_RWA;
1919 q->q_base |= q->base_dma & Q_BASE_ADDR_MASK << Q_BASE_ADDR_SHIFT;
1920 q->q_base |= (q->max_n_shift & Q_BASE_LOG2SIZE_MASK)
1921 << Q_BASE_LOG2SIZE_SHIFT;
1922
1923 q->prod = q->cons = 0;
1924 return 0;
1925 }
1926
1927 static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
1928 {
1929 int ret;
1930
1931 /* cmdq */
1932 spin_lock_init(&smmu->cmdq.lock);
1933 ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD,
1934 ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS);
1935 if (ret)
1936 return ret;
1937
1938 /* evtq */
1939 ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD,
1940 ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS);
1941 if (ret)
1942 return ret;
1943
1944 /* priq */
1945 if (!(smmu->features & ARM_SMMU_FEAT_PRI))
1946 return 0;
1947
1948 return arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD,
1949 ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS);
1950 }
1951
1952 static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu)
1953 {
1954 unsigned int i;
1955 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1956 size_t size = sizeof(*cfg->l1_desc) * cfg->num_l1_ents;
1957 void *strtab = smmu->strtab_cfg.strtab;
1958
1959 cfg->l1_desc = devm_kzalloc(smmu->dev, size, GFP_KERNEL);
1960 if (!cfg->l1_desc) {
1961 dev_err(smmu->dev, "failed to allocate l1 stream table desc\n");
1962 return -ENOMEM;
1963 }
1964
1965 for (i = 0; i < cfg->num_l1_ents; ++i) {
1966 arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]);
1967 strtab += STRTAB_L1_DESC_DWORDS << 3;
1968 }
1969
1970 return 0;
1971 }
1972
1973 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
1974 {
1975 void *strtab;
1976 u64 reg;
1977 u32 size, l1size;
1978 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1979
1980 /* Calculate the L1 size, capped to the SIDSIZE. */
1981 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
1982 size = min(size, smmu->sid_bits - STRTAB_SPLIT);
1983 cfg->num_l1_ents = 1 << size;
1984
1985 size += STRTAB_SPLIT;
1986 if (size < smmu->sid_bits)
1987 dev_warn(smmu->dev,
1988 "2-level strtab only covers %u/%u bits of SID\n",
1989 size, smmu->sid_bits);
1990
1991 l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
1992 strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
1993 GFP_KERNEL | __GFP_ZERO);
1994 if (!strtab) {
1995 dev_err(smmu->dev,
1996 "failed to allocate l1 stream table (%u bytes)\n",
1997 size);
1998 return -ENOMEM;
1999 }
2000 cfg->strtab = strtab;
2001
2002 /* Configure strtab_base_cfg for 2 levels */
2003 reg = STRTAB_BASE_CFG_FMT_2LVL;
2004 reg |= (size & STRTAB_BASE_CFG_LOG2SIZE_MASK)
2005 << STRTAB_BASE_CFG_LOG2SIZE_SHIFT;
2006 reg |= (STRTAB_SPLIT & STRTAB_BASE_CFG_SPLIT_MASK)
2007 << STRTAB_BASE_CFG_SPLIT_SHIFT;
2008 cfg->strtab_base_cfg = reg;
2009
2010 return arm_smmu_init_l1_strtab(smmu);
2011 }
2012
2013 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
2014 {
2015 void *strtab;
2016 u64 reg;
2017 u32 size;
2018 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2019
2020 size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
2021 strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma,
2022 GFP_KERNEL | __GFP_ZERO);
2023 if (!strtab) {
2024 dev_err(smmu->dev,
2025 "failed to allocate linear stream table (%u bytes)\n",
2026 size);
2027 return -ENOMEM;
2028 }
2029 cfg->strtab = strtab;
2030 cfg->num_l1_ents = 1 << smmu->sid_bits;
2031
2032 /* Configure strtab_base_cfg for a linear table covering all SIDs */
2033 reg = STRTAB_BASE_CFG_FMT_LINEAR;
2034 reg |= (smmu->sid_bits & STRTAB_BASE_CFG_LOG2SIZE_MASK)
2035 << STRTAB_BASE_CFG_LOG2SIZE_SHIFT;
2036 cfg->strtab_base_cfg = reg;
2037
2038 arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents);
2039 return 0;
2040 }
2041
2042 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
2043 {
2044 u64 reg;
2045 int ret;
2046
2047 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2048 ret = arm_smmu_init_strtab_2lvl(smmu);
2049 else
2050 ret = arm_smmu_init_strtab_linear(smmu);
2051
2052 if (ret)
2053 return ret;
2054
2055 /* Set the strtab base address */
2056 reg = smmu->strtab_cfg.strtab_dma &
2057 STRTAB_BASE_ADDR_MASK << STRTAB_BASE_ADDR_SHIFT;
2058 reg |= STRTAB_BASE_RA;
2059 smmu->strtab_cfg.strtab_base = reg;
2060
2061 /* Allocate the first VMID for stage-2 bypass STEs */
2062 set_bit(0, smmu->vmid_map);
2063 return 0;
2064 }
2065
2066 static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
2067 {
2068 int ret;
2069
2070 ret = arm_smmu_init_queues(smmu);
2071 if (ret)
2072 return ret;
2073
2074 return arm_smmu_init_strtab(smmu);
2075 }
2076
2077 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
2078 unsigned int reg_off, unsigned int ack_off)
2079 {
2080 u32 reg;
2081
2082 writel_relaxed(val, smmu->base + reg_off);
2083 return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
2084 1, ARM_SMMU_POLL_TIMEOUT_US);
2085 }
2086
2087 /* GBPA is "special" */
2088 static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
2089 {
2090 int ret;
2091 u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
2092
2093 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
2094 1, ARM_SMMU_POLL_TIMEOUT_US);
2095 if (ret)
2096 return ret;
2097
2098 reg &= ~clr;
2099 reg |= set;
2100 writel_relaxed(reg | GBPA_UPDATE, gbpa);
2101 return readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
2102 1, ARM_SMMU_POLL_TIMEOUT_US);
2103 }
2104
2105 static void arm_smmu_free_msis(void *data)
2106 {
2107 struct device *dev = data;
2108 platform_msi_domain_free_irqs(dev);
2109 }
2110
2111 static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
2112 {
2113 phys_addr_t doorbell;
2114 struct device *dev = msi_desc_to_dev(desc);
2115 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2116 phys_addr_t *cfg = arm_smmu_msi_cfg[desc->platform.msi_index];
2117
2118 doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
2119 doorbell &= MSI_CFG0_ADDR_MASK << MSI_CFG0_ADDR_SHIFT;
2120
2121 writeq_relaxed(doorbell, smmu->base + cfg[0]);
2122 writel_relaxed(msg->data, smmu->base + cfg[1]);
2123 writel_relaxed(MSI_CFG2_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
2124 }
2125
2126 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)
2127 {
2128 struct msi_desc *desc;
2129 int ret, nvec = ARM_SMMU_MAX_MSIS;
2130 struct device *dev = smmu->dev;
2131
2132 /* Clear the MSI address regs */
2133 writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
2134 writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
2135
2136 if (smmu->features & ARM_SMMU_FEAT_PRI)
2137 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
2138 else
2139 nvec--;
2140
2141 if (!(smmu->features & ARM_SMMU_FEAT_MSI))
2142 return;
2143
2144 /* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */
2145 ret = platform_msi_domain_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg);
2146 if (ret) {
2147 dev_warn(dev, "failed to allocate MSIs\n");
2148 return;
2149 }
2150
2151 for_each_msi_entry(desc, dev) {
2152 switch (desc->platform.msi_index) {
2153 case EVTQ_MSI_INDEX:
2154 smmu->evtq.q.irq = desc->irq;
2155 break;
2156 case GERROR_MSI_INDEX:
2157 smmu->gerr_irq = desc->irq;
2158 break;
2159 case PRIQ_MSI_INDEX:
2160 smmu->priq.q.irq = desc->irq;
2161 break;
2162 default: /* Unknown */
2163 continue;
2164 }
2165 }
2166
2167 /* Add callback to free MSIs on teardown */
2168 devm_add_action(dev, arm_smmu_free_msis, dev);
2169 }
2170
2171 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
2172 {
2173 int ret, irq;
2174 u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
2175
2176 /* Disable IRQs first */
2177 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
2178 ARM_SMMU_IRQ_CTRLACK);
2179 if (ret) {
2180 dev_err(smmu->dev, "failed to disable irqs\n");
2181 return ret;
2182 }
2183
2184 arm_smmu_setup_msis(smmu);
2185
2186 /* Request interrupt lines */
2187 irq = smmu->evtq.q.irq;
2188 if (irq) {
2189 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
2190 arm_smmu_evtq_thread,
2191 IRQF_ONESHOT,
2192 "arm-smmu-v3-evtq", smmu);
2193 if (ret < 0)
2194 dev_warn(smmu->dev, "failed to enable evtq irq\n");
2195 }
2196
2197 irq = smmu->cmdq.q.irq;
2198 if (irq) {
2199 ret = devm_request_irq(smmu->dev, irq,
2200 arm_smmu_cmdq_sync_handler, 0,
2201 "arm-smmu-v3-cmdq-sync", smmu);
2202 if (ret < 0)
2203 dev_warn(smmu->dev, "failed to enable cmdq-sync irq\n");
2204 }
2205
2206 irq = smmu->gerr_irq;
2207 if (irq) {
2208 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
2209 0, "arm-smmu-v3-gerror", smmu);
2210 if (ret < 0)
2211 dev_warn(smmu->dev, "failed to enable gerror irq\n");
2212 }
2213
2214 if (smmu->features & ARM_SMMU_FEAT_PRI) {
2215 irq = smmu->priq.q.irq;
2216 if (irq) {
2217 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
2218 arm_smmu_priq_thread,
2219 IRQF_ONESHOT,
2220 "arm-smmu-v3-priq",
2221 smmu);
2222 if (ret < 0)
2223 dev_warn(smmu->dev,
2224 "failed to enable priq irq\n");
2225 else
2226 irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
2227 }
2228 }
2229
2230 /* Enable interrupt generation on the SMMU */
2231 ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
2232 ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
2233 if (ret)
2234 dev_warn(smmu->dev, "failed to enable irqs\n");
2235
2236 return 0;
2237 }
2238
2239 static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
2240 {
2241 int ret;
2242
2243 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
2244 if (ret)
2245 dev_err(smmu->dev, "failed to clear cr0\n");
2246
2247 return ret;
2248 }
2249
2250 static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)
2251 {
2252 int ret;
2253 u32 reg, enables;
2254 struct arm_smmu_cmdq_ent cmd;
2255
2256 /* Clear CR0 and sync (disables SMMU and queue processing) */
2257 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
2258 if (reg & CR0_SMMUEN)
2259 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
2260
2261 ret = arm_smmu_device_disable(smmu);
2262 if (ret)
2263 return ret;
2264
2265 /* CR1 (table and queue memory attributes) */
2266 reg = (CR1_SH_ISH << CR1_TABLE_SH_SHIFT) |
2267 (CR1_CACHE_WB << CR1_TABLE_OC_SHIFT) |
2268 (CR1_CACHE_WB << CR1_TABLE_IC_SHIFT) |
2269 (CR1_SH_ISH << CR1_QUEUE_SH_SHIFT) |
2270 (CR1_CACHE_WB << CR1_QUEUE_OC_SHIFT) |
2271 (CR1_CACHE_WB << CR1_QUEUE_IC_SHIFT);
2272 writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
2273
2274 /* CR2 (random crap) */
2275 reg = CR2_PTM | CR2_RECINVSID | CR2_E2H;
2276 writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
2277
2278 /* Stream table */
2279 writeq_relaxed(smmu->strtab_cfg.strtab_base,
2280 smmu->base + ARM_SMMU_STRTAB_BASE);
2281 writel_relaxed(smmu->strtab_cfg.strtab_base_cfg,
2282 smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
2283
2284 /* Command queue */
2285 writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
2286 writel_relaxed(smmu->cmdq.q.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
2287 writel_relaxed(smmu->cmdq.q.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
2288
2289 enables = CR0_CMDQEN;
2290 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2291 ARM_SMMU_CR0ACK);
2292 if (ret) {
2293 dev_err(smmu->dev, "failed to enable command queue\n");
2294 return ret;
2295 }
2296
2297 /* Invalidate any cached configuration */
2298 cmd.opcode = CMDQ_OP_CFGI_ALL;
2299 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2300 cmd.opcode = CMDQ_OP_CMD_SYNC;
2301 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2302
2303 /* Invalidate any stale TLB entries */
2304 if (smmu->features & ARM_SMMU_FEAT_HYP) {
2305 cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
2306 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2307 }
2308
2309 cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
2310 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2311 cmd.opcode = CMDQ_OP_CMD_SYNC;
2312 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2313
2314 /* Event queue */
2315 writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
2316 writel_relaxed(smmu->evtq.q.prod, smmu->base + ARM_SMMU_EVTQ_PROD);
2317 writel_relaxed(smmu->evtq.q.cons, smmu->base + ARM_SMMU_EVTQ_CONS);
2318
2319 enables |= CR0_EVTQEN;
2320 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2321 ARM_SMMU_CR0ACK);
2322 if (ret) {
2323 dev_err(smmu->dev, "failed to enable event queue\n");
2324 return ret;
2325 }
2326
2327 /* PRI queue */
2328 if (smmu->features & ARM_SMMU_FEAT_PRI) {
2329 writeq_relaxed(smmu->priq.q.q_base,
2330 smmu->base + ARM_SMMU_PRIQ_BASE);
2331 writel_relaxed(smmu->priq.q.prod,
2332 smmu->base + ARM_SMMU_PRIQ_PROD);
2333 writel_relaxed(smmu->priq.q.cons,
2334 smmu->base + ARM_SMMU_PRIQ_CONS);
2335
2336 enables |= CR0_PRIQEN;
2337 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2338 ARM_SMMU_CR0ACK);
2339 if (ret) {
2340 dev_err(smmu->dev, "failed to enable PRI queue\n");
2341 return ret;
2342 }
2343 }
2344
2345 ret = arm_smmu_setup_irqs(smmu);
2346 if (ret) {
2347 dev_err(smmu->dev, "failed to setup irqs\n");
2348 return ret;
2349 }
2350
2351
2352 /* Enable the SMMU interface, or ensure bypass */
2353 if (!bypass || disable_bypass) {
2354 enables |= CR0_SMMUEN;
2355 } else {
2356 ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT);
2357 if (ret) {
2358 dev_err(smmu->dev, "GBPA not responding to update\n");
2359 return ret;
2360 }
2361 }
2362 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2363 ARM_SMMU_CR0ACK);
2364 if (ret) {
2365 dev_err(smmu->dev, "failed to enable SMMU interface\n");
2366 return ret;
2367 }
2368
2369 return 0;
2370 }
2371
2372 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
2373 {
2374 u32 reg;
2375 bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;
2376
2377 /* IDR0 */
2378 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
2379
2380 /* 2-level structures */
2381 if ((reg & IDR0_ST_LVL_MASK << IDR0_ST_LVL_SHIFT) == IDR0_ST_LVL_2LVL)
2382 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
2383
2384 if (reg & IDR0_CD2L)
2385 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
2386
2387 /*
2388 * Translation table endianness.
2389 * We currently require the same endianness as the CPU, but this
2390 * could be changed later by adding a new IO_PGTABLE_QUIRK.
2391 */
2392 switch (reg & IDR0_TTENDIAN_MASK << IDR0_TTENDIAN_SHIFT) {
2393 case IDR0_TTENDIAN_MIXED:
2394 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
2395 break;
2396 #ifdef __BIG_ENDIAN
2397 case IDR0_TTENDIAN_BE:
2398 smmu->features |= ARM_SMMU_FEAT_TT_BE;
2399 break;
2400 #else
2401 case IDR0_TTENDIAN_LE:
2402 smmu->features |= ARM_SMMU_FEAT_TT_LE;
2403 break;
2404 #endif
2405 default:
2406 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
2407 return -ENXIO;
2408 }
2409
2410 /* Boolean feature flags */
2411 if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
2412 smmu->features |= ARM_SMMU_FEAT_PRI;
2413
2414 if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
2415 smmu->features |= ARM_SMMU_FEAT_ATS;
2416
2417 if (reg & IDR0_SEV)
2418 smmu->features |= ARM_SMMU_FEAT_SEV;
2419
2420 if (reg & IDR0_MSI)
2421 smmu->features |= ARM_SMMU_FEAT_MSI;
2422
2423 if (reg & IDR0_HYP)
2424 smmu->features |= ARM_SMMU_FEAT_HYP;
2425
2426 /*
2427 * The coherency feature as set by FW is used in preference to the ID
2428 * register, but warn on mismatch.
2429 */
2430 if (!!(reg & IDR0_COHACC) != coherent)
2431 dev_warn(smmu->dev, "IDR0.COHACC overridden by dma-coherent property (%s)\n",
2432 coherent ? "true" : "false");
2433
2434 switch (reg & IDR0_STALL_MODEL_MASK << IDR0_STALL_MODEL_SHIFT) {
2435 case IDR0_STALL_MODEL_STALL:
2436 /* Fallthrough */
2437 case IDR0_STALL_MODEL_FORCE:
2438 smmu->features |= ARM_SMMU_FEAT_STALLS;
2439 }
2440
2441 if (reg & IDR0_S1P)
2442 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
2443
2444 if (reg & IDR0_S2P)
2445 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
2446
2447 if (!(reg & (IDR0_S1P | IDR0_S2P))) {
2448 dev_err(smmu->dev, "no translation support!\n");
2449 return -ENXIO;
2450 }
2451
2452 /* We only support the AArch64 table format at present */
2453 switch (reg & IDR0_TTF_MASK << IDR0_TTF_SHIFT) {
2454 case IDR0_TTF_AARCH32_64:
2455 smmu->ias = 40;
2456 /* Fallthrough */
2457 case IDR0_TTF_AARCH64:
2458 break;
2459 default:
2460 dev_err(smmu->dev, "AArch64 table format not supported!\n");
2461 return -ENXIO;
2462 }
2463
2464 /* ASID/VMID sizes */
2465 smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
2466 smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
2467
2468 /* IDR1 */
2469 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
2470 if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
2471 dev_err(smmu->dev, "embedded implementation not supported\n");
2472 return -ENXIO;
2473 }
2474
2475 /* Queue sizes, capped at 4k */
2476 smmu->cmdq.q.max_n_shift = min((u32)CMDQ_MAX_SZ_SHIFT,
2477 reg >> IDR1_CMDQ_SHIFT & IDR1_CMDQ_MASK);
2478 if (!smmu->cmdq.q.max_n_shift) {
2479 /* Odd alignment restrictions on the base, so ignore for now */
2480 dev_err(smmu->dev, "unit-length command queue not supported\n");
2481 return -ENXIO;
2482 }
2483
2484 smmu->evtq.q.max_n_shift = min((u32)EVTQ_MAX_SZ_SHIFT,
2485 reg >> IDR1_EVTQ_SHIFT & IDR1_EVTQ_MASK);
2486 smmu->priq.q.max_n_shift = min((u32)PRIQ_MAX_SZ_SHIFT,
2487 reg >> IDR1_PRIQ_SHIFT & IDR1_PRIQ_MASK);
2488
2489 /* SID/SSID sizes */
2490 smmu->ssid_bits = reg >> IDR1_SSID_SHIFT & IDR1_SSID_MASK;
2491 smmu->sid_bits = reg >> IDR1_SID_SHIFT & IDR1_SID_MASK;
2492
2493 /*
2494 * If the SMMU supports fewer bits than would fill a single L2 stream
2495 * table, use a linear table instead.
2496 */
2497 if (smmu->sid_bits <= STRTAB_SPLIT)
2498 smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;
2499
2500 /* IDR5 */
2501 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
2502
2503 /* Maximum number of outstanding stalls */
2504 smmu->evtq.max_stalls = reg >> IDR5_STALL_MAX_SHIFT
2505 & IDR5_STALL_MAX_MASK;
2506
2507 /* Page sizes */
2508 if (reg & IDR5_GRAN64K)
2509 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
2510 if (reg & IDR5_GRAN16K)
2511 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
2512 if (reg & IDR5_GRAN4K)
2513 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
2514
2515 if (arm_smmu_ops.pgsize_bitmap == -1UL)
2516 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
2517 else
2518 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
2519
2520 /* Output address size */
2521 switch (reg & IDR5_OAS_MASK << IDR5_OAS_SHIFT) {
2522 case IDR5_OAS_32_BIT:
2523 smmu->oas = 32;
2524 break;
2525 case IDR5_OAS_36_BIT:
2526 smmu->oas = 36;
2527 break;
2528 case IDR5_OAS_40_BIT:
2529 smmu->oas = 40;
2530 break;
2531 case IDR5_OAS_42_BIT:
2532 smmu->oas = 42;
2533 break;
2534 case IDR5_OAS_44_BIT:
2535 smmu->oas = 44;
2536 break;
2537 default:
2538 dev_info(smmu->dev,
2539 "unknown output address size. Truncating to 48-bit\n");
2540 /* Fallthrough */
2541 case IDR5_OAS_48_BIT:
2542 smmu->oas = 48;
2543 }
2544
2545 /* Set the DMA mask for our table walker */
2546 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
2547 dev_warn(smmu->dev,
2548 "failed to set DMA mask for table walker\n");
2549
2550 smmu->ias = max(smmu->ias, smmu->oas);
2551
2552 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
2553 smmu->ias, smmu->oas, smmu->features);
2554 return 0;
2555 }
2556
2557 #ifdef CONFIG_ACPI
2558 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2559 struct arm_smmu_device *smmu)
2560 {
2561 struct acpi_iort_smmu_v3 *iort_smmu;
2562 struct device *dev = smmu->dev;
2563 struct acpi_iort_node *node;
2564
2565 node = *(struct acpi_iort_node **)dev_get_platdata(dev);
2566
2567 /* Retrieve SMMUv3 specific data */
2568 iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
2569
2570 if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
2571 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
2572
2573 return 0;
2574 }
2575 #else
2576 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2577 struct arm_smmu_device *smmu)
2578 {
2579 return -ENODEV;
2580 }
2581 #endif
2582
2583 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2584 struct arm_smmu_device *smmu)
2585 {
2586 struct device *dev = &pdev->dev;
2587 u32 cells;
2588 int ret = -EINVAL;
2589
2590 if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
2591 dev_err(dev, "missing #iommu-cells property\n");
2592 else if (cells != 1)
2593 dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
2594 else
2595 ret = 0;
2596
2597 parse_driver_options(smmu);
2598
2599 if (of_dma_is_coherent(dev->of_node))
2600 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
2601
2602 return ret;
2603 }
2604
2605 static int arm_smmu_device_probe(struct platform_device *pdev)
2606 {
2607 int irq, ret;
2608 struct resource *res;
2609 struct arm_smmu_device *smmu;
2610 struct device *dev = &pdev->dev;
2611 bool bypass;
2612
2613 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2614 if (!smmu) {
2615 dev_err(dev, "failed to allocate arm_smmu_device\n");
2616 return -ENOMEM;
2617 }
2618 smmu->dev = dev;
2619
2620 /* Base address */
2621 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2622 if (resource_size(res) + 1 < SZ_128K) {
2623 dev_err(dev, "MMIO region too small (%pr)\n", res);
2624 return -EINVAL;
2625 }
2626
2627 smmu->base = devm_ioremap_resource(dev, res);
2628 if (IS_ERR(smmu->base))
2629 return PTR_ERR(smmu->base);
2630
2631 /* Interrupt lines */
2632 irq = platform_get_irq_byname(pdev, "eventq");
2633 if (irq > 0)
2634 smmu->evtq.q.irq = irq;
2635
2636 irq = platform_get_irq_byname(pdev, "priq");
2637 if (irq > 0)
2638 smmu->priq.q.irq = irq;
2639
2640 irq = platform_get_irq_byname(pdev, "cmdq-sync");
2641 if (irq > 0)
2642 smmu->cmdq.q.irq = irq;
2643
2644 irq = platform_get_irq_byname(pdev, "gerror");
2645 if (irq > 0)
2646 smmu->gerr_irq = irq;
2647
2648 if (dev->of_node) {
2649 ret = arm_smmu_device_dt_probe(pdev, smmu);
2650 } else {
2651 ret = arm_smmu_device_acpi_probe(pdev, smmu);
2652 if (ret == -ENODEV)
2653 return ret;
2654 }
2655
2656 /* Set bypass mode according to firmware probing result */
2657 bypass = !!ret;
2658
2659 /* Probe the h/w */
2660 ret = arm_smmu_device_hw_probe(smmu);
2661 if (ret)
2662 return ret;
2663
2664 /* Initialise in-memory data structures */
2665 ret = arm_smmu_init_structures(smmu);
2666 if (ret)
2667 return ret;
2668
2669 /* Record our private device structure */
2670 platform_set_drvdata(pdev, smmu);
2671
2672 /* Reset the device */
2673 ret = arm_smmu_device_reset(smmu, bypass);
2674 if (ret)
2675 return ret;
2676
2677 /* And we're up. Go go go! */
2678 iommu_register_instance(dev->fwnode, &arm_smmu_ops);
2679
2680 #ifdef CONFIG_PCI
2681 if (pci_bus_type.iommu_ops != &arm_smmu_ops) {
2682 pci_request_acs();
2683 ret = bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2684 if (ret)
2685 return ret;
2686 }
2687 #endif
2688 #ifdef CONFIG_ARM_AMBA
2689 if (amba_bustype.iommu_ops != &arm_smmu_ops) {
2690 ret = bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2691 if (ret)
2692 return ret;
2693 }
2694 #endif
2695 if (platform_bus_type.iommu_ops != &arm_smmu_ops) {
2696 ret = bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2697 if (ret)
2698 return ret;
2699 }
2700 return 0;
2701 }
2702
2703 static int arm_smmu_device_remove(struct platform_device *pdev)
2704 {
2705 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2706
2707 arm_smmu_device_disable(smmu);
2708 return 0;
2709 }
2710
2711 static struct of_device_id arm_smmu_of_match[] = {
2712 { .compatible = "arm,smmu-v3", },
2713 { },
2714 };
2715 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
2716
2717 static struct platform_driver arm_smmu_driver = {
2718 .driver = {
2719 .name = "arm-smmu-v3",
2720 .of_match_table = of_match_ptr(arm_smmu_of_match),
2721 },
2722 .probe = arm_smmu_device_probe,
2723 .remove = arm_smmu_device_remove,
2724 };
2725
2726 static int __init arm_smmu_init(void)
2727 {
2728 static bool registered;
2729 int ret = 0;
2730
2731 if (!registered) {
2732 ret = platform_driver_register(&arm_smmu_driver);
2733 registered = !ret;
2734 }
2735 return ret;
2736 }
2737
2738 static void __exit arm_smmu_exit(void)
2739 {
2740 return platform_driver_unregister(&arm_smmu_driver);
2741 }
2742
2743 subsys_initcall(arm_smmu_init);
2744 module_exit(arm_smmu_exit);
2745
2746 static int __init arm_smmu_of_init(struct device_node *np)
2747 {
2748 int ret = arm_smmu_init();
2749
2750 if (ret)
2751 return ret;
2752
2753 if (!of_platform_device_create(np, NULL, platform_bus_type.dev_root))
2754 return -ENODEV;
2755
2756 return 0;
2757 }
2758 IOMMU_OF_DECLARE(arm_smmuv3, "arm,smmu-v3", arm_smmu_of_init);
2759
2760 #ifdef CONFIG_ACPI
2761 static int __init acpi_smmu_v3_init(struct acpi_table_header *table)
2762 {
2763 if (iort_node_match(ACPI_IORT_NODE_SMMU_V3))
2764 return arm_smmu_init();
2765
2766 return 0;
2767 }
2768 IORT_ACPI_DECLARE(arm_smmu_v3, ACPI_SIG_IORT, acpi_smmu_v3_init);
2769 #endif
2770
2771 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
2772 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2773 MODULE_LICENSE("GPL v2");