1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (c) 2015-2016 MediaTek Inc.
4 * Author: Yong Wu <yong.wu@mediatek.com>
6 #include <linux/bitfield.h>
9 #include <linux/component.h>
10 #include <linux/device.h>
11 #include <linux/dma-direct.h>
12 #include <linux/err.h>
13 #include <linux/interrupt.h>
15 #include <linux/iommu.h>
16 #include <linux/iopoll.h>
17 #include <linux/list.h>
18 #include <linux/mfd/syscon.h>
19 #include <linux/module.h>
20 #include <linux/of_address.h>
21 #include <linux/of_irq.h>
22 #include <linux/of_platform.h>
23 #include <linux/platform_device.h>
24 #include <linux/pm_runtime.h>
25 #include <linux/regmap.h>
26 #include <linux/slab.h>
27 #include <linux/spinlock.h>
28 #include <linux/soc/mediatek/infracfg.h>
29 #include <asm/barrier.h>
30 #include <soc/mediatek/smi.h>
32 #include "mtk_iommu.h"
34 #define REG_MMU_PT_BASE_ADDR 0x000
35 #define MMU_PT_ADDR_MASK GENMASK(31, 7)
37 #define REG_MMU_INVALIDATE 0x020
38 #define F_ALL_INVLD 0x2
39 #define F_MMU_INV_RANGE 0x1
41 #define REG_MMU_INVLD_START_A 0x024
42 #define REG_MMU_INVLD_END_A 0x028
44 #define REG_MMU_INV_SEL_GEN2 0x02c
45 #define REG_MMU_INV_SEL_GEN1 0x038
46 #define F_INVLD_EN0 BIT(0)
47 #define F_INVLD_EN1 BIT(1)
49 #define REG_MMU_MISC_CTRL 0x048
50 #define F_MMU_IN_ORDER_WR_EN_MASK (BIT(1) | BIT(17))
51 #define F_MMU_STANDARD_AXI_MODE_MASK (BIT(3) | BIT(19))
53 #define REG_MMU_DCM_DIS 0x050
54 #define REG_MMU_WR_LEN_CTRL 0x054
55 #define F_MMU_WR_THROT_DIS_MASK (BIT(5) | BIT(21))
57 #define REG_MMU_CTRL_REG 0x110
58 #define F_MMU_TF_PROT_TO_PROGRAM_ADDR (2 << 4)
59 #define F_MMU_PREFETCH_RT_REPLACE_MOD BIT(4)
60 #define F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173 (2 << 5)
62 #define REG_MMU_IVRP_PADDR 0x114
64 #define REG_MMU_VLD_PA_RNG 0x118
65 #define F_MMU_VLD_PA_RNG(EA, SA) (((EA) << 8) | (SA))
67 #define REG_MMU_INT_CONTROL0 0x120
68 #define F_L2_MULIT_HIT_EN BIT(0)
69 #define F_TABLE_WALK_FAULT_INT_EN BIT(1)
70 #define F_PREETCH_FIFO_OVERFLOW_INT_EN BIT(2)
71 #define F_MISS_FIFO_OVERFLOW_INT_EN BIT(3)
72 #define F_PREFETCH_FIFO_ERR_INT_EN BIT(5)
73 #define F_MISS_FIFO_ERR_INT_EN BIT(6)
74 #define F_INT_CLR_BIT BIT(12)
76 #define REG_MMU_INT_MAIN_CONTROL 0x124
78 #define F_INT_TRANSLATION_FAULT (BIT(0) | BIT(7))
79 #define F_INT_MAIN_MULTI_HIT_FAULT (BIT(1) | BIT(8))
80 #define F_INT_INVALID_PA_FAULT (BIT(2) | BIT(9))
81 #define F_INT_ENTRY_REPLACEMENT_FAULT (BIT(3) | BIT(10))
82 #define F_INT_TLB_MISS_FAULT (BIT(4) | BIT(11))
83 #define F_INT_MISS_TRANSACTION_FIFO_FAULT (BIT(5) | BIT(12))
84 #define F_INT_PRETETCH_TRANSATION_FIFO_FAULT (BIT(6) | BIT(13))
86 #define REG_MMU_CPE_DONE 0x12C
88 #define REG_MMU_FAULT_ST1 0x134
89 #define F_REG_MMU0_FAULT_MASK GENMASK(6, 0)
90 #define F_REG_MMU1_FAULT_MASK GENMASK(13, 7)
92 #define REG_MMU0_FAULT_VA 0x13c
93 #define F_MMU_INVAL_VA_31_12_MASK GENMASK(31, 12)
94 #define F_MMU_INVAL_VA_34_32_MASK GENMASK(11, 9)
95 #define F_MMU_INVAL_PA_34_32_MASK GENMASK(8, 6)
96 #define F_MMU_FAULT_VA_WRITE_BIT BIT(1)
97 #define F_MMU_FAULT_VA_LAYER_BIT BIT(0)
99 #define REG_MMU0_INVLD_PA 0x140
100 #define REG_MMU1_FAULT_VA 0x144
101 #define REG_MMU1_INVLD_PA 0x148
102 #define REG_MMU0_INT_ID 0x150
103 #define REG_MMU1_INT_ID 0x154
104 #define F_MMU_INT_ID_COMM_ID(a) (((a) >> 9) & 0x7)
105 #define F_MMU_INT_ID_SUB_COMM_ID(a) (((a) >> 7) & 0x3)
106 #define F_MMU_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7)
107 #define F_MMU_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f)
109 #define MTK_PROTECT_PA_ALIGN 256
111 #define HAS_4GB_MODE BIT(0)
112 /* HW will use the EMI clock if there isn't the "bclk". */
113 #define HAS_BCLK BIT(1)
114 #define HAS_VLD_PA_RNG BIT(2)
115 #define RESET_AXI BIT(3)
116 #define OUT_ORDER_WR_EN BIT(4)
117 #define HAS_SUB_COMM BIT(5)
118 #define WR_THROT_EN BIT(6)
119 #define HAS_LEGACY_IVRP_PADDR BIT(7)
120 #define IOVA_34_EN BIT(8)
122 #define MTK_IOMMU_HAS_FLAG(pdata, _x) \
123 ((((pdata)->flags) & (_x)) == (_x))
125 struct mtk_iommu_domain
{
126 struct io_pgtable_cfg cfg
;
127 struct io_pgtable_ops
*iop
;
129 struct mtk_iommu_data
*data
;
130 struct iommu_domain domain
;
133 static const struct iommu_ops mtk_iommu_ops
;
135 static int mtk_iommu_hw_init(const struct mtk_iommu_data
*data
);
137 #define MTK_IOMMU_TLB_ADDR(iova) ({ \
138 dma_addr_t _addr = iova; \
139 ((lower_32_bits(_addr) & GENMASK(31, 12)) | upper_32_bits(_addr));\
143 * In M4U 4GB mode, the physical address is remapped as below:
145 * CPU Physical address:
146 * ====================
149 * |---A---|---B---|---C---|---D---|---E---|
150 * +--I/O--+------------Memory-------------+
152 * IOMMU output physical address:
153 * =============================
156 * |---E---|---B---|---C---|---D---|
157 * +------------Memory-------------+
159 * The Region 'A'(I/O) can NOT be mapped by M4U; For Region 'B'/'C'/'D', the
160 * bit32 of the CPU physical address always is needed to set, and for Region
161 * 'E', the CPU physical address keep as is.
162 * Additionally, The iommu consumers always use the CPU phyiscal address.
164 #define MTK_IOMMU_4GB_MODE_REMAP_BASE 0x140000000UL
166 static LIST_HEAD(m4ulist
); /* List all the M4U HWs */
168 #define for_each_m4u(data) list_for_each_entry(data, &m4ulist, list)
170 struct mtk_iommu_iova_region
{
171 dma_addr_t iova_base
;
172 unsigned long long size
;
175 static const struct mtk_iommu_iova_region single_domain
[] = {
176 {.iova_base
= 0, .size
= SZ_4G
},
179 static const struct mtk_iommu_iova_region mt8192_multi_dom
[] = {
180 { .iova_base
= 0x0, .size
= SZ_4G
}, /* disp: 0 ~ 4G */
181 #if IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT)
182 { .iova_base
= SZ_4G
, .size
= SZ_4G
}, /* vdec: 4G ~ 8G */
183 { .iova_base
= SZ_4G
* 2, .size
= SZ_4G
}, /* CAM/MDP: 8G ~ 12G */
184 { .iova_base
= 0x240000000ULL
, .size
= 0x4000000}, /* CCU0 */
185 { .iova_base
= 0x244000000ULL
, .size
= 0x4000000}, /* CCU1 */
190 * There may be 1 or 2 M4U HWs, But we always expect they are in the same domain
191 * for the performance.
193 * Here always return the mtk_iommu_data of the first probed M4U where the
194 * iommu domain information is recorded.
196 static struct mtk_iommu_data
*mtk_iommu_get_m4u_data(void)
198 struct mtk_iommu_data
*data
;
206 static struct mtk_iommu_domain
*to_mtk_domain(struct iommu_domain
*dom
)
208 return container_of(dom
, struct mtk_iommu_domain
, domain
);
211 static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data
*data
)
214 if (pm_runtime_get_if_in_use(data
->dev
) <= 0)
217 writel_relaxed(F_INVLD_EN1
| F_INVLD_EN0
,
218 data
->base
+ data
->plat_data
->inv_sel_reg
);
219 writel_relaxed(F_ALL_INVLD
, data
->base
+ REG_MMU_INVALIDATE
);
220 wmb(); /* Make sure the tlb flush all done */
222 pm_runtime_put(data
->dev
);
226 static void mtk_iommu_tlb_flush_range_sync(unsigned long iova
, size_t size
,
228 struct mtk_iommu_data
*data
)
230 bool has_pm
= !!data
->dev
->pm_domain
;
237 if (pm_runtime_get_if_in_use(data
->dev
) <= 0)
241 spin_lock_irqsave(&data
->tlb_lock
, flags
);
242 writel_relaxed(F_INVLD_EN1
| F_INVLD_EN0
,
243 data
->base
+ data
->plat_data
->inv_sel_reg
);
245 writel_relaxed(MTK_IOMMU_TLB_ADDR(iova
),
246 data
->base
+ REG_MMU_INVLD_START_A
);
247 writel_relaxed(MTK_IOMMU_TLB_ADDR(iova
+ size
- 1),
248 data
->base
+ REG_MMU_INVLD_END_A
);
249 writel_relaxed(F_MMU_INV_RANGE
,
250 data
->base
+ REG_MMU_INVALIDATE
);
253 ret
= readl_poll_timeout_atomic(data
->base
+ REG_MMU_CPE_DONE
,
254 tmp
, tmp
!= 0, 10, 1000);
257 "Partial TLB flush timed out, falling back to full flush\n");
258 mtk_iommu_tlb_flush_all(data
);
260 /* Clear the CPE status */
261 writel_relaxed(0, data
->base
+ REG_MMU_CPE_DONE
);
262 spin_unlock_irqrestore(&data
->tlb_lock
, flags
);
265 pm_runtime_put(data
->dev
);
269 static irqreturn_t
mtk_iommu_isr(int irq
, void *dev_id
)
271 struct mtk_iommu_data
*data
= dev_id
;
272 struct mtk_iommu_domain
*dom
= data
->m4u_dom
;
273 unsigned int fault_larb
, fault_port
, sub_comm
= 0;
274 u32 int_state
, regval
, va34_32
, pa34_32
;
275 u64 fault_iova
, fault_pa
;
278 /* Read error info from registers */
279 int_state
= readl_relaxed(data
->base
+ REG_MMU_FAULT_ST1
);
280 if (int_state
& F_REG_MMU0_FAULT_MASK
) {
281 regval
= readl_relaxed(data
->base
+ REG_MMU0_INT_ID
);
282 fault_iova
= readl_relaxed(data
->base
+ REG_MMU0_FAULT_VA
);
283 fault_pa
= readl_relaxed(data
->base
+ REG_MMU0_INVLD_PA
);
285 regval
= readl_relaxed(data
->base
+ REG_MMU1_INT_ID
);
286 fault_iova
= readl_relaxed(data
->base
+ REG_MMU1_FAULT_VA
);
287 fault_pa
= readl_relaxed(data
->base
+ REG_MMU1_INVLD_PA
);
289 layer
= fault_iova
& F_MMU_FAULT_VA_LAYER_BIT
;
290 write
= fault_iova
& F_MMU_FAULT_VA_WRITE_BIT
;
291 if (MTK_IOMMU_HAS_FLAG(data
->plat_data
, IOVA_34_EN
)) {
292 va34_32
= FIELD_GET(F_MMU_INVAL_VA_34_32_MASK
, fault_iova
);
293 pa34_32
= FIELD_GET(F_MMU_INVAL_PA_34_32_MASK
, fault_iova
);
294 fault_iova
= fault_iova
& F_MMU_INVAL_VA_31_12_MASK
;
295 fault_iova
|= (u64
)va34_32
<< 32;
296 fault_pa
|= (u64
)pa34_32
<< 32;
299 fault_port
= F_MMU_INT_ID_PORT_ID(regval
);
300 if (MTK_IOMMU_HAS_FLAG(data
->plat_data
, HAS_SUB_COMM
)) {
301 fault_larb
= F_MMU_INT_ID_COMM_ID(regval
);
302 sub_comm
= F_MMU_INT_ID_SUB_COMM_ID(regval
);
304 fault_larb
= F_MMU_INT_ID_LARB_ID(regval
);
306 fault_larb
= data
->plat_data
->larbid_remap
[fault_larb
][sub_comm
];
308 if (report_iommu_fault(&dom
->domain
, data
->dev
, fault_iova
,
309 write
? IOMMU_FAULT_WRITE
: IOMMU_FAULT_READ
)) {
312 "fault type=0x%x iova=0x%llx pa=0x%llx larb=%d port=%d layer=%d %s\n",
313 int_state
, fault_iova
, fault_pa
, fault_larb
, fault_port
,
314 layer
, write
? "write" : "read");
317 /* Interrupt clear */
318 regval
= readl_relaxed(data
->base
+ REG_MMU_INT_CONTROL0
);
319 regval
|= F_INT_CLR_BIT
;
320 writel_relaxed(regval
, data
->base
+ REG_MMU_INT_CONTROL0
);
322 mtk_iommu_tlb_flush_all(data
);
327 static int mtk_iommu_get_domain_id(struct device
*dev
,
328 const struct mtk_iommu_plat_data
*plat_data
)
330 const struct mtk_iommu_iova_region
*rgn
= plat_data
->iova_region
;
331 const struct bus_dma_region
*dma_rgn
= dev
->dma_range_map
;
332 int i
, candidate
= -1;
335 if (!dma_rgn
|| plat_data
->iova_region_nr
== 1)
338 dma_end
= dma_rgn
->dma_start
+ dma_rgn
->size
- 1;
339 for (i
= 0; i
< plat_data
->iova_region_nr
; i
++, rgn
++) {
341 if (dma_rgn
->dma_start
== rgn
->iova_base
&&
342 dma_end
== rgn
->iova_base
+ rgn
->size
- 1)
344 /* ok if it is inside this region. */
345 if (dma_rgn
->dma_start
>= rgn
->iova_base
&&
346 dma_end
< rgn
->iova_base
+ rgn
->size
)
352 dev_err(dev
, "Can NOT find the iommu domain id(%pad 0x%llx).\n",
353 &dma_rgn
->dma_start
, dma_rgn
->size
);
357 static void mtk_iommu_config(struct mtk_iommu_data
*data
, struct device
*dev
,
358 bool enable
, unsigned int domid
)
360 struct mtk_smi_larb_iommu
*larb_mmu
;
361 unsigned int larbid
, portid
;
362 struct iommu_fwspec
*fwspec
= dev_iommu_fwspec_get(dev
);
363 const struct mtk_iommu_iova_region
*region
;
366 for (i
= 0; i
< fwspec
->num_ids
; ++i
) {
367 larbid
= MTK_M4U_TO_LARB(fwspec
->ids
[i
]);
368 portid
= MTK_M4U_TO_PORT(fwspec
->ids
[i
]);
370 larb_mmu
= &data
->larb_imu
[larbid
];
372 region
= data
->plat_data
->iova_region
+ domid
;
373 larb_mmu
->bank
[portid
] = upper_32_bits(region
->iova_base
);
375 dev_dbg(dev
, "%s iommu for larb(%s) port %d dom %d bank %d.\n",
376 enable
? "enable" : "disable", dev_name(larb_mmu
->dev
),
377 portid
, domid
, larb_mmu
->bank
[portid
]);
380 larb_mmu
->mmu
|= MTK_SMI_MMU_EN(portid
);
382 larb_mmu
->mmu
&= ~MTK_SMI_MMU_EN(portid
);
386 static int mtk_iommu_domain_finalise(struct mtk_iommu_domain
*dom
,
387 struct mtk_iommu_data
*data
,
390 const struct mtk_iommu_iova_region
*region
;
392 /* Use the exist domain as there is only one pgtable here. */
394 dom
->iop
= data
->m4u_dom
->iop
;
395 dom
->cfg
= data
->m4u_dom
->cfg
;
396 dom
->domain
.pgsize_bitmap
= data
->m4u_dom
->cfg
.pgsize_bitmap
;
397 goto update_iova_region
;
400 dom
->cfg
= (struct io_pgtable_cfg
) {
401 .quirks
= IO_PGTABLE_QUIRK_ARM_NS
|
402 IO_PGTABLE_QUIRK_NO_PERMS
|
403 IO_PGTABLE_QUIRK_ARM_MTK_EXT
,
404 .pgsize_bitmap
= mtk_iommu_ops
.pgsize_bitmap
,
405 .ias
= MTK_IOMMU_HAS_FLAG(data
->plat_data
, IOVA_34_EN
) ? 34 : 32,
406 .iommu_dev
= data
->dev
,
409 if (MTK_IOMMU_HAS_FLAG(data
->plat_data
, HAS_4GB_MODE
))
410 dom
->cfg
.oas
= data
->enable_4GB
? 33 : 32;
414 dom
->iop
= alloc_io_pgtable_ops(ARM_V7S
, &dom
->cfg
, data
);
416 dev_err(data
->dev
, "Failed to alloc io pgtable\n");
420 /* Update our support page sizes bitmap */
421 dom
->domain
.pgsize_bitmap
= dom
->cfg
.pgsize_bitmap
;
424 /* Update the iova region for this domain */
425 region
= data
->plat_data
->iova_region
+ domid
;
426 dom
->domain
.geometry
.aperture_start
= region
->iova_base
;
427 dom
->domain
.geometry
.aperture_end
= region
->iova_base
+ region
->size
- 1;
428 dom
->domain
.geometry
.force_aperture
= true;
432 static struct iommu_domain
*mtk_iommu_domain_alloc(unsigned type
)
434 struct mtk_iommu_domain
*dom
;
436 if (type
!= IOMMU_DOMAIN_DMA
)
439 dom
= kzalloc(sizeof(*dom
), GFP_KERNEL
);
446 static void mtk_iommu_domain_free(struct iommu_domain
*domain
)
448 kfree(to_mtk_domain(domain
));
451 static int mtk_iommu_attach_device(struct iommu_domain
*domain
,
454 struct mtk_iommu_data
*data
= dev_iommu_priv_get(dev
);
455 struct mtk_iommu_domain
*dom
= to_mtk_domain(domain
);
456 struct device
*m4udev
= data
->dev
;
459 domid
= mtk_iommu_get_domain_id(dev
, data
->plat_data
);
464 if (mtk_iommu_domain_finalise(dom
, data
, domid
))
469 if (!data
->m4u_dom
) { /* Initialize the M4U HW */
470 ret
= pm_runtime_resume_and_get(m4udev
);
474 ret
= mtk_iommu_hw_init(data
);
476 pm_runtime_put(m4udev
);
480 writel(dom
->cfg
.arm_v7s_cfg
.ttbr
& MMU_PT_ADDR_MASK
,
481 data
->base
+ REG_MMU_PT_BASE_ADDR
);
483 pm_runtime_put(m4udev
);
486 mtk_iommu_config(data
, dev
, true, domid
);
490 static void mtk_iommu_detach_device(struct iommu_domain
*domain
,
493 struct mtk_iommu_data
*data
= dev_iommu_priv_get(dev
);
495 mtk_iommu_config(data
, dev
, false, 0);
498 static int mtk_iommu_map(struct iommu_domain
*domain
, unsigned long iova
,
499 phys_addr_t paddr
, size_t size
, int prot
, gfp_t gfp
)
501 struct mtk_iommu_domain
*dom
= to_mtk_domain(domain
);
503 /* The "4GB mode" M4U physically can not use the lower remap of Dram. */
504 if (dom
->data
->enable_4GB
)
505 paddr
|= BIT_ULL(32);
507 /* Synchronize with the tlb_lock */
508 return dom
->iop
->map(dom
->iop
, iova
, paddr
, size
, prot
, gfp
);
511 static size_t mtk_iommu_unmap(struct iommu_domain
*domain
,
512 unsigned long iova
, size_t size
,
513 struct iommu_iotlb_gather
*gather
)
515 struct mtk_iommu_domain
*dom
= to_mtk_domain(domain
);
517 iommu_iotlb_gather_add_range(gather
, iova
, size
);
518 return dom
->iop
->unmap(dom
->iop
, iova
, size
, gather
);
521 static void mtk_iommu_flush_iotlb_all(struct iommu_domain
*domain
)
523 struct mtk_iommu_domain
*dom
= to_mtk_domain(domain
);
525 mtk_iommu_tlb_flush_all(dom
->data
);
528 static void mtk_iommu_iotlb_sync(struct iommu_domain
*domain
,
529 struct iommu_iotlb_gather
*gather
)
531 struct mtk_iommu_domain
*dom
= to_mtk_domain(domain
);
532 size_t length
= gather
->end
- gather
->start
+ 1;
534 mtk_iommu_tlb_flush_range_sync(gather
->start
, length
, gather
->pgsize
,
538 static void mtk_iommu_sync_map(struct iommu_domain
*domain
, unsigned long iova
,
541 struct mtk_iommu_domain
*dom
= to_mtk_domain(domain
);
543 mtk_iommu_tlb_flush_range_sync(iova
, size
, size
, dom
->data
);
546 static phys_addr_t
mtk_iommu_iova_to_phys(struct iommu_domain
*domain
,
549 struct mtk_iommu_domain
*dom
= to_mtk_domain(domain
);
552 pa
= dom
->iop
->iova_to_phys(dom
->iop
, iova
);
553 if (IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT
) &&
554 dom
->data
->enable_4GB
&&
555 pa
>= MTK_IOMMU_4GB_MODE_REMAP_BASE
)
561 static struct iommu_device
*mtk_iommu_probe_device(struct device
*dev
)
563 struct iommu_fwspec
*fwspec
= dev_iommu_fwspec_get(dev
);
564 struct mtk_iommu_data
*data
;
565 struct device_link
*link
;
566 struct device
*larbdev
;
567 unsigned int larbid
, larbidx
, i
;
569 if (!fwspec
|| fwspec
->ops
!= &mtk_iommu_ops
)
570 return ERR_PTR(-ENODEV
); /* Not a iommu client device */
572 data
= dev_iommu_priv_get(dev
);
575 * Link the consumer device with the smi-larb device(supplier).
576 * The device that connects with each a larb is a independent HW.
577 * All the ports in each a device should be in the same larbs.
579 larbid
= MTK_M4U_TO_LARB(fwspec
->ids
[0]);
580 for (i
= 1; i
< fwspec
->num_ids
; i
++) {
581 larbidx
= MTK_M4U_TO_LARB(fwspec
->ids
[i
]);
582 if (larbid
!= larbidx
) {
583 dev_err(dev
, "Can only use one larb. Fail@larb%d-%d.\n",
585 return ERR_PTR(-EINVAL
);
588 larbdev
= data
->larb_imu
[larbid
].dev
;
589 link
= device_link_add(dev
, larbdev
,
590 DL_FLAG_PM_RUNTIME
| DL_FLAG_STATELESS
);
592 dev_err(dev
, "Unable to link %s\n", dev_name(larbdev
));
596 static void mtk_iommu_release_device(struct device
*dev
)
598 struct iommu_fwspec
*fwspec
= dev_iommu_fwspec_get(dev
);
599 struct mtk_iommu_data
*data
;
600 struct device
*larbdev
;
603 if (!fwspec
|| fwspec
->ops
!= &mtk_iommu_ops
)
606 data
= dev_iommu_priv_get(dev
);
607 larbid
= MTK_M4U_TO_LARB(fwspec
->ids
[0]);
608 larbdev
= data
->larb_imu
[larbid
].dev
;
609 device_link_remove(dev
, larbdev
);
611 iommu_fwspec_free(dev
);
614 static struct iommu_group
*mtk_iommu_device_group(struct device
*dev
)
616 struct mtk_iommu_data
*data
= mtk_iommu_get_m4u_data();
617 struct iommu_group
*group
;
621 return ERR_PTR(-ENODEV
);
623 domid
= mtk_iommu_get_domain_id(dev
, data
->plat_data
);
625 return ERR_PTR(domid
);
627 group
= data
->m4u_group
[domid
];
629 group
= iommu_group_alloc();
631 data
->m4u_group
[domid
] = group
;
633 iommu_group_ref_get(group
);
638 static int mtk_iommu_of_xlate(struct device
*dev
, struct of_phandle_args
*args
)
640 struct platform_device
*m4updev
;
642 if (args
->args_count
!= 1) {
643 dev_err(dev
, "invalid #iommu-cells(%d) property for IOMMU\n",
648 if (!dev_iommu_priv_get(dev
)) {
649 /* Get the m4u device */
650 m4updev
= of_find_device_by_node(args
->np
);
651 if (WARN_ON(!m4updev
))
654 dev_iommu_priv_set(dev
, platform_get_drvdata(m4updev
));
657 return iommu_fwspec_add_ids(dev
, args
->args
, 1);
660 static void mtk_iommu_get_resv_regions(struct device
*dev
,
661 struct list_head
*head
)
663 struct mtk_iommu_data
*data
= dev_iommu_priv_get(dev
);
664 unsigned int domid
= mtk_iommu_get_domain_id(dev
, data
->plat_data
), i
;
665 const struct mtk_iommu_iova_region
*resv
, *curdom
;
666 struct iommu_resv_region
*region
;
667 int prot
= IOMMU_WRITE
| IOMMU_READ
;
671 curdom
= data
->plat_data
->iova_region
+ domid
;
672 for (i
= 0; i
< data
->plat_data
->iova_region_nr
; i
++) {
673 resv
= data
->plat_data
->iova_region
+ i
;
675 /* Only reserve when the region is inside the current domain */
676 if (resv
->iova_base
<= curdom
->iova_base
||
677 resv
->iova_base
+ resv
->size
>= curdom
->iova_base
+ curdom
->size
)
680 region
= iommu_alloc_resv_region(resv
->iova_base
, resv
->size
,
681 prot
, IOMMU_RESV_RESERVED
);
685 list_add_tail(®ion
->list
, head
);
689 static const struct iommu_ops mtk_iommu_ops
= {
690 .domain_alloc
= mtk_iommu_domain_alloc
,
691 .domain_free
= mtk_iommu_domain_free
,
692 .attach_dev
= mtk_iommu_attach_device
,
693 .detach_dev
= mtk_iommu_detach_device
,
694 .map
= mtk_iommu_map
,
695 .unmap
= mtk_iommu_unmap
,
696 .flush_iotlb_all
= mtk_iommu_flush_iotlb_all
,
697 .iotlb_sync
= mtk_iommu_iotlb_sync
,
698 .iotlb_sync_map
= mtk_iommu_sync_map
,
699 .iova_to_phys
= mtk_iommu_iova_to_phys
,
700 .probe_device
= mtk_iommu_probe_device
,
701 .release_device
= mtk_iommu_release_device
,
702 .device_group
= mtk_iommu_device_group
,
703 .of_xlate
= mtk_iommu_of_xlate
,
704 .get_resv_regions
= mtk_iommu_get_resv_regions
,
705 .put_resv_regions
= generic_iommu_put_resv_regions
,
706 .pgsize_bitmap
= SZ_4K
| SZ_64K
| SZ_1M
| SZ_16M
,
707 .owner
= THIS_MODULE
,
710 static int mtk_iommu_hw_init(const struct mtk_iommu_data
*data
)
714 if (data
->plat_data
->m4u_plat
== M4U_MT8173
) {
715 regval
= F_MMU_PREFETCH_RT_REPLACE_MOD
|
716 F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173
;
718 regval
= readl_relaxed(data
->base
+ REG_MMU_CTRL_REG
);
719 regval
|= F_MMU_TF_PROT_TO_PROGRAM_ADDR
;
721 writel_relaxed(regval
, data
->base
+ REG_MMU_CTRL_REG
);
723 regval
= F_L2_MULIT_HIT_EN
|
724 F_TABLE_WALK_FAULT_INT_EN
|
725 F_PREETCH_FIFO_OVERFLOW_INT_EN
|
726 F_MISS_FIFO_OVERFLOW_INT_EN
|
727 F_PREFETCH_FIFO_ERR_INT_EN
|
728 F_MISS_FIFO_ERR_INT_EN
;
729 writel_relaxed(regval
, data
->base
+ REG_MMU_INT_CONTROL0
);
731 regval
= F_INT_TRANSLATION_FAULT
|
732 F_INT_MAIN_MULTI_HIT_FAULT
|
733 F_INT_INVALID_PA_FAULT
|
734 F_INT_ENTRY_REPLACEMENT_FAULT
|
735 F_INT_TLB_MISS_FAULT
|
736 F_INT_MISS_TRANSACTION_FIFO_FAULT
|
737 F_INT_PRETETCH_TRANSATION_FIFO_FAULT
;
738 writel_relaxed(regval
, data
->base
+ REG_MMU_INT_MAIN_CONTROL
);
740 if (MTK_IOMMU_HAS_FLAG(data
->plat_data
, HAS_LEGACY_IVRP_PADDR
))
741 regval
= (data
->protect_base
>> 1) | (data
->enable_4GB
<< 31);
743 regval
= lower_32_bits(data
->protect_base
) |
744 upper_32_bits(data
->protect_base
);
745 writel_relaxed(regval
, data
->base
+ REG_MMU_IVRP_PADDR
);
747 if (data
->enable_4GB
&&
748 MTK_IOMMU_HAS_FLAG(data
->plat_data
, HAS_VLD_PA_RNG
)) {
750 * If 4GB mode is enabled, the validate PA range is from
751 * 0x1_0000_0000 to 0x1_ffff_ffff. here record bit[32:30].
753 regval
= F_MMU_VLD_PA_RNG(7, 4);
754 writel_relaxed(regval
, data
->base
+ REG_MMU_VLD_PA_RNG
);
756 writel_relaxed(0, data
->base
+ REG_MMU_DCM_DIS
);
757 if (MTK_IOMMU_HAS_FLAG(data
->plat_data
, WR_THROT_EN
)) {
758 /* write command throttling mode */
759 regval
= readl_relaxed(data
->base
+ REG_MMU_WR_LEN_CTRL
);
760 regval
&= ~F_MMU_WR_THROT_DIS_MASK
;
761 writel_relaxed(regval
, data
->base
+ REG_MMU_WR_LEN_CTRL
);
764 if (MTK_IOMMU_HAS_FLAG(data
->plat_data
, RESET_AXI
)) {
765 /* The register is called STANDARD_AXI_MODE in this case */
768 regval
= readl_relaxed(data
->base
+ REG_MMU_MISC_CTRL
);
769 regval
&= ~F_MMU_STANDARD_AXI_MODE_MASK
;
770 if (MTK_IOMMU_HAS_FLAG(data
->plat_data
, OUT_ORDER_WR_EN
))
771 regval
&= ~F_MMU_IN_ORDER_WR_EN_MASK
;
773 writel_relaxed(regval
, data
->base
+ REG_MMU_MISC_CTRL
);
775 if (devm_request_irq(data
->dev
, data
->irq
, mtk_iommu_isr
, 0,
776 dev_name(data
->dev
), (void *)data
)) {
777 writel_relaxed(0, data
->base
+ REG_MMU_PT_BASE_ADDR
);
778 dev_err(data
->dev
, "Failed @ IRQ-%d Request\n", data
->irq
);
785 static const struct component_master_ops mtk_iommu_com_ops
= {
786 .bind
= mtk_iommu_bind
,
787 .unbind
= mtk_iommu_unbind
,
790 static int mtk_iommu_probe(struct platform_device
*pdev
)
792 struct mtk_iommu_data
*data
;
793 struct device
*dev
= &pdev
->dev
;
794 struct device_node
*larbnode
, *smicomm_node
;
795 struct platform_device
*plarbdev
;
796 struct device_link
*link
;
797 struct resource
*res
;
798 resource_size_t ioaddr
;
799 struct component_match
*match
= NULL
;
800 struct regmap
*infracfg
;
806 data
= devm_kzalloc(dev
, sizeof(*data
), GFP_KERNEL
);
810 data
->plat_data
= of_device_get_match_data(dev
);
812 /* Protect memory. HW will access here while translation fault.*/
813 protect
= devm_kzalloc(dev
, MTK_PROTECT_PA_ALIGN
* 2, GFP_KERNEL
);
816 data
->protect_base
= ALIGN(virt_to_phys(protect
), MTK_PROTECT_PA_ALIGN
);
818 if (MTK_IOMMU_HAS_FLAG(data
->plat_data
, HAS_4GB_MODE
)) {
819 switch (data
->plat_data
->m4u_plat
) {
821 p
= "mediatek,mt2712-infracfg";
824 p
= "mediatek,mt8173-infracfg";
830 infracfg
= syscon_regmap_lookup_by_compatible(p
);
832 if (IS_ERR(infracfg
))
833 return PTR_ERR(infracfg
);
835 ret
= regmap_read(infracfg
, REG_INFRA_MISC
, &val
);
838 data
->enable_4GB
= !!(val
& F_DDR_4GB_SUPPORT_EN
);
841 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
842 data
->base
= devm_ioremap_resource(dev
, res
);
843 if (IS_ERR(data
->base
))
844 return PTR_ERR(data
->base
);
847 data
->irq
= platform_get_irq(pdev
, 0);
851 if (MTK_IOMMU_HAS_FLAG(data
->plat_data
, HAS_BCLK
)) {
852 data
->bclk
= devm_clk_get(dev
, "bclk");
853 if (IS_ERR(data
->bclk
))
854 return PTR_ERR(data
->bclk
);
857 larb_nr
= of_count_phandle_with_args(dev
->of_node
,
858 "mediatek,larbs", NULL
);
862 for (i
= 0; i
< larb_nr
; i
++) {
865 larbnode
= of_parse_phandle(dev
->of_node
, "mediatek,larbs", i
);
869 if (!of_device_is_available(larbnode
)) {
870 of_node_put(larbnode
);
874 ret
= of_property_read_u32(larbnode
, "mediatek,larb-id", &id
);
875 if (ret
)/* The id is consecutive if there is no this property */
878 plarbdev
= of_find_device_by_node(larbnode
);
880 of_node_put(larbnode
);
883 data
->larb_imu
[id
].dev
= &plarbdev
->dev
;
885 component_match_add_release(dev
, &match
, release_of
,
886 compare_of
, larbnode
);
889 /* Get smi-common dev from the last larb. */
890 smicomm_node
= of_parse_phandle(larbnode
, "mediatek,smi", 0);
894 plarbdev
= of_find_device_by_node(smicomm_node
);
895 of_node_put(smicomm_node
);
896 data
->smicomm_dev
= &plarbdev
->dev
;
898 pm_runtime_enable(dev
);
900 link
= device_link_add(data
->smicomm_dev
, dev
,
901 DL_FLAG_STATELESS
| DL_FLAG_PM_RUNTIME
);
903 dev_err(dev
, "Unable to link %s.\n", dev_name(data
->smicomm_dev
));
905 goto out_runtime_disable
;
908 platform_set_drvdata(pdev
, data
);
910 ret
= iommu_device_sysfs_add(&data
->iommu
, dev
, NULL
,
911 "mtk-iommu.%pa", &ioaddr
);
913 goto out_link_remove
;
915 ret
= iommu_device_register(&data
->iommu
, &mtk_iommu_ops
, dev
);
917 goto out_sysfs_remove
;
919 spin_lock_init(&data
->tlb_lock
);
920 list_add_tail(&data
->list
, &m4ulist
);
922 if (!iommu_present(&platform_bus_type
)) {
923 ret
= bus_set_iommu(&platform_bus_type
, &mtk_iommu_ops
);
928 ret
= component_master_add_with_match(dev
, &mtk_iommu_com_ops
, match
);
930 goto out_bus_set_null
;
934 bus_set_iommu(&platform_bus_type
, NULL
);
936 list_del(&data
->list
);
937 iommu_device_unregister(&data
->iommu
);
939 iommu_device_sysfs_remove(&data
->iommu
);
941 device_link_remove(data
->smicomm_dev
, dev
);
943 pm_runtime_disable(dev
);
947 static int mtk_iommu_remove(struct platform_device
*pdev
)
949 struct mtk_iommu_data
*data
= platform_get_drvdata(pdev
);
951 iommu_device_sysfs_remove(&data
->iommu
);
952 iommu_device_unregister(&data
->iommu
);
954 if (iommu_present(&platform_bus_type
))
955 bus_set_iommu(&platform_bus_type
, NULL
);
957 clk_disable_unprepare(data
->bclk
);
958 device_link_remove(data
->smicomm_dev
, &pdev
->dev
);
959 pm_runtime_disable(&pdev
->dev
);
960 devm_free_irq(&pdev
->dev
, data
->irq
, data
);
961 component_master_del(&pdev
->dev
, &mtk_iommu_com_ops
);
965 static int __maybe_unused
mtk_iommu_runtime_suspend(struct device
*dev
)
967 struct mtk_iommu_data
*data
= dev_get_drvdata(dev
);
968 struct mtk_iommu_suspend_reg
*reg
= &data
->reg
;
969 void __iomem
*base
= data
->base
;
971 reg
->wr_len_ctrl
= readl_relaxed(base
+ REG_MMU_WR_LEN_CTRL
);
972 reg
->misc_ctrl
= readl_relaxed(base
+ REG_MMU_MISC_CTRL
);
973 reg
->dcm_dis
= readl_relaxed(base
+ REG_MMU_DCM_DIS
);
974 reg
->ctrl_reg
= readl_relaxed(base
+ REG_MMU_CTRL_REG
);
975 reg
->int_control0
= readl_relaxed(base
+ REG_MMU_INT_CONTROL0
);
976 reg
->int_main_control
= readl_relaxed(base
+ REG_MMU_INT_MAIN_CONTROL
);
977 reg
->ivrp_paddr
= readl_relaxed(base
+ REG_MMU_IVRP_PADDR
);
978 reg
->vld_pa_rng
= readl_relaxed(base
+ REG_MMU_VLD_PA_RNG
);
979 clk_disable_unprepare(data
->bclk
);
983 static int __maybe_unused
mtk_iommu_runtime_resume(struct device
*dev
)
985 struct mtk_iommu_data
*data
= dev_get_drvdata(dev
);
986 struct mtk_iommu_suspend_reg
*reg
= &data
->reg
;
987 struct mtk_iommu_domain
*m4u_dom
= data
->m4u_dom
;
988 void __iomem
*base
= data
->base
;
991 ret
= clk_prepare_enable(data
->bclk
);
993 dev_err(data
->dev
, "Failed to enable clk(%d) in resume\n", ret
);
998 * Uppon first resume, only enable the clk and return, since the values of the
999 * registers are not yet set.
1004 writel_relaxed(reg
->wr_len_ctrl
, base
+ REG_MMU_WR_LEN_CTRL
);
1005 writel_relaxed(reg
->misc_ctrl
, base
+ REG_MMU_MISC_CTRL
);
1006 writel_relaxed(reg
->dcm_dis
, base
+ REG_MMU_DCM_DIS
);
1007 writel_relaxed(reg
->ctrl_reg
, base
+ REG_MMU_CTRL_REG
);
1008 writel_relaxed(reg
->int_control0
, base
+ REG_MMU_INT_CONTROL0
);
1009 writel_relaxed(reg
->int_main_control
, base
+ REG_MMU_INT_MAIN_CONTROL
);
1010 writel_relaxed(reg
->ivrp_paddr
, base
+ REG_MMU_IVRP_PADDR
);
1011 writel_relaxed(reg
->vld_pa_rng
, base
+ REG_MMU_VLD_PA_RNG
);
1012 writel(m4u_dom
->cfg
.arm_v7s_cfg
.ttbr
& MMU_PT_ADDR_MASK
, base
+ REG_MMU_PT_BASE_ADDR
);
1016 static const struct dev_pm_ops mtk_iommu_pm_ops
= {
1017 SET_RUNTIME_PM_OPS(mtk_iommu_runtime_suspend
, mtk_iommu_runtime_resume
, NULL
)
1018 SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend
,
1019 pm_runtime_force_resume
)
1022 static const struct mtk_iommu_plat_data mt2712_data
= {
1023 .m4u_plat
= M4U_MT2712
,
1024 .flags
= HAS_4GB_MODE
| HAS_BCLK
| HAS_VLD_PA_RNG
,
1025 .inv_sel_reg
= REG_MMU_INV_SEL_GEN1
,
1026 .iova_region
= single_domain
,
1027 .iova_region_nr
= ARRAY_SIZE(single_domain
),
1028 .larbid_remap
= {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}},
1031 static const struct mtk_iommu_plat_data mt6779_data
= {
1032 .m4u_plat
= M4U_MT6779
,
1033 .flags
= HAS_SUB_COMM
| OUT_ORDER_WR_EN
| WR_THROT_EN
,
1034 .inv_sel_reg
= REG_MMU_INV_SEL_GEN2
,
1035 .iova_region
= single_domain
,
1036 .iova_region_nr
= ARRAY_SIZE(single_domain
),
1037 .larbid_remap
= {{0}, {1}, {2}, {3}, {5}, {7, 8}, {10}, {9}},
1040 static const struct mtk_iommu_plat_data mt8167_data
= {
1041 .m4u_plat
= M4U_MT8167
,
1042 .flags
= RESET_AXI
| HAS_LEGACY_IVRP_PADDR
,
1043 .inv_sel_reg
= REG_MMU_INV_SEL_GEN1
,
1044 .iova_region
= single_domain
,
1045 .iova_region_nr
= ARRAY_SIZE(single_domain
),
1046 .larbid_remap
= {{0}, {1}, {2}}, /* Linear mapping. */
1049 static const struct mtk_iommu_plat_data mt8173_data
= {
1050 .m4u_plat
= M4U_MT8173
,
1051 .flags
= HAS_4GB_MODE
| HAS_BCLK
| RESET_AXI
|
1052 HAS_LEGACY_IVRP_PADDR
,
1053 .inv_sel_reg
= REG_MMU_INV_SEL_GEN1
,
1054 .iova_region
= single_domain
,
1055 .iova_region_nr
= ARRAY_SIZE(single_domain
),
1056 .larbid_remap
= {{0}, {1}, {2}, {3}, {4}, {5}}, /* Linear mapping. */
1059 static const struct mtk_iommu_plat_data mt8183_data
= {
1060 .m4u_plat
= M4U_MT8183
,
1062 .inv_sel_reg
= REG_MMU_INV_SEL_GEN1
,
1063 .iova_region
= single_domain
,
1064 .iova_region_nr
= ARRAY_SIZE(single_domain
),
1065 .larbid_remap
= {{0}, {4}, {5}, {6}, {7}, {2}, {3}, {1}},
1068 static const struct mtk_iommu_plat_data mt8192_data
= {
1069 .m4u_plat
= M4U_MT8192
,
1070 .flags
= HAS_BCLK
| HAS_SUB_COMM
| OUT_ORDER_WR_EN
|
1071 WR_THROT_EN
| IOVA_34_EN
,
1072 .inv_sel_reg
= REG_MMU_INV_SEL_GEN2
,
1073 .iova_region
= mt8192_multi_dom
,
1074 .iova_region_nr
= ARRAY_SIZE(mt8192_multi_dom
),
1075 .larbid_remap
= {{0}, {1}, {4, 5}, {7}, {2}, {9, 11, 19, 20},
1076 {0, 14, 16}, {0, 13, 18, 17}},
1079 static const struct of_device_id mtk_iommu_of_ids
[] = {
1080 { .compatible
= "mediatek,mt2712-m4u", .data
= &mt2712_data
},
1081 { .compatible
= "mediatek,mt6779-m4u", .data
= &mt6779_data
},
1082 { .compatible
= "mediatek,mt8167-m4u", .data
= &mt8167_data
},
1083 { .compatible
= "mediatek,mt8173-m4u", .data
= &mt8173_data
},
1084 { .compatible
= "mediatek,mt8183-m4u", .data
= &mt8183_data
},
1085 { .compatible
= "mediatek,mt8192-m4u", .data
= &mt8192_data
},
1089 static struct platform_driver mtk_iommu_driver
= {
1090 .probe
= mtk_iommu_probe
,
1091 .remove
= mtk_iommu_remove
,
1093 .name
= "mtk-iommu",
1094 .of_match_table
= mtk_iommu_of_ids
,
1095 .pm
= &mtk_iommu_pm_ops
,
1098 module_platform_driver(mtk_iommu_driver
);
1100 MODULE_DESCRIPTION("IOMMU API for MediaTek M4U implementations");
1101 MODULE_LICENSE("GPL v2");