1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright (C) 2020 Unisoc, Inc.
6 * Author: Chunyan Zhang <chunyan.zhang@unisoc.com>
10 #include <linux/device.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/errno.h>
13 #include <linux/iommu.h>
14 #include <linux/mfd/syscon.h>
15 #include <linux/module.h>
16 #include <linux/of_platform.h>
17 #include <linux/regmap.h>
18 #include <linux/slab.h>
20 #define SPRD_IOMMU_PAGE_SHIFT 12
21 #define SPRD_IOMMU_PAGE_SIZE SZ_4K
23 #define SPRD_EX_CFG 0x0
24 #define SPRD_IOMMU_VAOR_BYPASS BIT(4)
25 #define SPRD_IOMMU_GATE_EN BIT(1)
26 #define SPRD_IOMMU_EN BIT(0)
27 #define SPRD_EX_UPDATE 0x4
28 #define SPRD_EX_FIRST_VPN 0x8
29 #define SPRD_EX_VPN_RANGE 0xc
30 #define SPRD_EX_FIRST_PPN 0x10
31 #define SPRD_EX_DEFAULT_PPN 0x14
33 #define SPRD_IOMMU_VERSION 0x0
34 #define SPRD_VERSION_MASK GENMASK(15, 8)
35 #define SPRD_VERSION_SHIFT 0x8
36 #define SPRD_VAU_CFG 0x4
37 #define SPRD_VAU_UPDATE 0x8
38 #define SPRD_VAU_AUTH_CFG 0xc
39 #define SPRD_VAU_FIRST_PPN 0x10
40 #define SPRD_VAU_DEFAULT_PPN_RD 0x14
41 #define SPRD_VAU_DEFAULT_PPN_WR 0x18
42 #define SPRD_VAU_FIRST_VPN 0x1c
43 #define SPRD_VAU_VPN_RANGE 0x20
45 enum sprd_iommu_version
{
51 * struct sprd_iommu_device - high-level sprd IOMMU device representation,
52 * including hardware information and configuration, also driver data, etc
54 * @ver: sprd IOMMU IP version
55 * @prot_page_va: protect page base virtual address
56 * @prot_page_pa: protect page base physical address, data would be
57 * written to here while translation fault
58 * @base: mapped base address for accessing registers
59 * @dev: pointer to basic device structure
60 * @iommu: IOMMU core representation
62 * @eb: gate clock which controls IOMMU access
64 struct sprd_iommu_device
{
65 enum sprd_iommu_version ver
;
67 dma_addr_t prot_page_pa
;
70 struct iommu_device iommu
;
71 struct iommu_group
*group
;
75 struct sprd_iommu_domain
{
76 spinlock_t pgtlock
; /* lock for page table */
77 struct iommu_domain domain
;
78 u32
*pgt_va
; /* page table virtual address base */
79 dma_addr_t pgt_pa
; /* page table physical address base */
80 struct sprd_iommu_device
*sdev
;
83 static const struct iommu_ops sprd_iommu_ops
;
85 static struct sprd_iommu_domain
*to_sprd_domain(struct iommu_domain
*dom
)
87 return container_of(dom
, struct sprd_iommu_domain
, domain
);
91 sprd_iommu_write(struct sprd_iommu_device
*sdev
, unsigned int reg
, u32 val
)
93 writel_relaxed(val
, sdev
->base
+ reg
);
97 sprd_iommu_read(struct sprd_iommu_device
*sdev
, unsigned int reg
)
99 return readl_relaxed(sdev
->base
+ reg
);
103 sprd_iommu_update_bits(struct sprd_iommu_device
*sdev
, unsigned int reg
,
104 u32 mask
, u32 shift
, u32 val
)
106 u32 t
= sprd_iommu_read(sdev
, reg
);
108 t
= (t
& (~(mask
<< shift
))) | ((val
& mask
) << shift
);
109 sprd_iommu_write(sdev
, reg
, t
);
113 sprd_iommu_get_version(struct sprd_iommu_device
*sdev
)
115 int ver
= (sprd_iommu_read(sdev
, SPRD_IOMMU_VERSION
) &
116 SPRD_VERSION_MASK
) >> SPRD_VERSION_SHIFT
;
128 sprd_iommu_pgt_size(struct iommu_domain
*domain
)
130 return ((domain
->geometry
.aperture_end
-
131 domain
->geometry
.aperture_start
+ 1) >>
132 SPRD_IOMMU_PAGE_SHIFT
) * sizeof(u32
);
135 static struct iommu_domain
*sprd_iommu_domain_alloc(unsigned int domain_type
)
137 struct sprd_iommu_domain
*dom
;
139 if (domain_type
!= IOMMU_DOMAIN_DMA
&& domain_type
!= IOMMU_DOMAIN_UNMANAGED
)
142 dom
= kzalloc(sizeof(*dom
), GFP_KERNEL
);
146 spin_lock_init(&dom
->pgtlock
);
148 dom
->domain
.geometry
.aperture_start
= 0;
149 dom
->domain
.geometry
.aperture_end
= SZ_256M
- 1;
154 static void sprd_iommu_domain_free(struct iommu_domain
*domain
)
156 struct sprd_iommu_domain
*dom
= to_sprd_domain(domain
);
161 static void sprd_iommu_first_vpn(struct sprd_iommu_domain
*dom
)
163 struct sprd_iommu_device
*sdev
= dom
->sdev
;
167 if (sdev
->ver
== SPRD_IOMMU_EX
)
168 reg
= SPRD_EX_FIRST_VPN
;
170 reg
= SPRD_VAU_FIRST_VPN
;
172 val
= dom
->domain
.geometry
.aperture_start
>> SPRD_IOMMU_PAGE_SHIFT
;
173 sprd_iommu_write(sdev
, reg
, val
);
176 static void sprd_iommu_vpn_range(struct sprd_iommu_domain
*dom
)
178 struct sprd_iommu_device
*sdev
= dom
->sdev
;
182 if (sdev
->ver
== SPRD_IOMMU_EX
)
183 reg
= SPRD_EX_VPN_RANGE
;
185 reg
= SPRD_VAU_VPN_RANGE
;
187 val
= (dom
->domain
.geometry
.aperture_end
-
188 dom
->domain
.geometry
.aperture_start
) >> SPRD_IOMMU_PAGE_SHIFT
;
189 sprd_iommu_write(sdev
, reg
, val
);
192 static void sprd_iommu_first_ppn(struct sprd_iommu_domain
*dom
)
194 u32 val
= dom
->pgt_pa
>> SPRD_IOMMU_PAGE_SHIFT
;
195 struct sprd_iommu_device
*sdev
= dom
->sdev
;
198 if (sdev
->ver
== SPRD_IOMMU_EX
)
199 reg
= SPRD_EX_FIRST_PPN
;
201 reg
= SPRD_VAU_FIRST_PPN
;
203 sprd_iommu_write(sdev
, reg
, val
);
206 static void sprd_iommu_default_ppn(struct sprd_iommu_device
*sdev
)
208 u32 val
= sdev
->prot_page_pa
>> SPRD_IOMMU_PAGE_SHIFT
;
210 if (sdev
->ver
== SPRD_IOMMU_EX
) {
211 sprd_iommu_write(sdev
, SPRD_EX_DEFAULT_PPN
, val
);
212 } else if (sdev
->ver
== SPRD_IOMMU_VAU
) {
213 sprd_iommu_write(sdev
, SPRD_VAU_DEFAULT_PPN_RD
, val
);
214 sprd_iommu_write(sdev
, SPRD_VAU_DEFAULT_PPN_WR
, val
);
218 static void sprd_iommu_hw_en(struct sprd_iommu_device
*sdev
, bool en
)
220 unsigned int reg_cfg
;
223 if (sdev
->ver
== SPRD_IOMMU_EX
)
224 reg_cfg
= SPRD_EX_CFG
;
226 reg_cfg
= SPRD_VAU_CFG
;
228 mask
= SPRD_IOMMU_EN
| SPRD_IOMMU_GATE_EN
;
230 sprd_iommu_update_bits(sdev
, reg_cfg
, mask
, 0, val
);
233 static int sprd_iommu_attach_device(struct iommu_domain
*domain
,
236 struct sprd_iommu_device
*sdev
= dev_iommu_priv_get(dev
);
237 struct sprd_iommu_domain
*dom
= to_sprd_domain(domain
);
238 size_t pgt_size
= sprd_iommu_pgt_size(domain
);
241 pr_err("There's already a device attached to this domain.\n");
245 dom
->pgt_va
= dma_alloc_coherent(sdev
->dev
, pgt_size
, &dom
->pgt_pa
, GFP_KERNEL
);
251 sprd_iommu_first_ppn(dom
);
252 sprd_iommu_first_vpn(dom
);
253 sprd_iommu_vpn_range(dom
);
254 sprd_iommu_default_ppn(sdev
);
255 sprd_iommu_hw_en(sdev
, true);
260 static void sprd_iommu_detach_device(struct iommu_domain
*domain
,
263 struct sprd_iommu_domain
*dom
= to_sprd_domain(domain
);
264 struct sprd_iommu_device
*sdev
= dom
->sdev
;
265 size_t pgt_size
= sprd_iommu_pgt_size(domain
);
270 dma_free_coherent(sdev
->dev
, pgt_size
, dom
->pgt_va
, dom
->pgt_pa
);
271 sprd_iommu_hw_en(sdev
, false);
275 static int sprd_iommu_map(struct iommu_domain
*domain
, unsigned long iova
,
276 phys_addr_t paddr
, size_t size
, int prot
, gfp_t gfp
)
278 struct sprd_iommu_domain
*dom
= to_sprd_domain(domain
);
279 unsigned int page_num
= size
>> SPRD_IOMMU_PAGE_SHIFT
;
283 u32 pabase
= (u32
)paddr
;
284 unsigned long start
= domain
->geometry
.aperture_start
;
285 unsigned long end
= domain
->geometry
.aperture_end
;
288 pr_err("No sprd_iommu_device attached to the domain\n");
292 if (iova
< start
|| (iova
+ size
) > (end
+ 1)) {
293 dev_err(dom
->sdev
->dev
, "(iova(0x%lx) + size(%zx)) are not in the range!\n",
298 pgt_base_iova
= dom
->pgt_va
+ ((iova
- start
) >> SPRD_IOMMU_PAGE_SHIFT
);
300 spin_lock_irqsave(&dom
->pgtlock
, flags
);
301 for (i
= 0; i
< page_num
; i
++) {
302 pgt_base_iova
[i
] = pabase
>> SPRD_IOMMU_PAGE_SHIFT
;
303 pabase
+= SPRD_IOMMU_PAGE_SIZE
;
305 spin_unlock_irqrestore(&dom
->pgtlock
, flags
);
310 static size_t sprd_iommu_unmap(struct iommu_domain
*domain
, unsigned long iova
,
311 size_t size
, struct iommu_iotlb_gather
*iotlb_gather
)
313 struct sprd_iommu_domain
*dom
= to_sprd_domain(domain
);
316 unsigned int page_num
= size
>> SPRD_IOMMU_PAGE_SHIFT
;
317 unsigned long start
= domain
->geometry
.aperture_start
;
318 unsigned long end
= domain
->geometry
.aperture_end
;
320 if (iova
< start
|| (iova
+ size
) > (end
+ 1))
323 pgt_base_iova
= dom
->pgt_va
+ ((iova
- start
) >> SPRD_IOMMU_PAGE_SHIFT
);
325 spin_lock_irqsave(&dom
->pgtlock
, flags
);
326 memset(pgt_base_iova
, 0, page_num
* sizeof(u32
));
327 spin_unlock_irqrestore(&dom
->pgtlock
, flags
);
332 static void sprd_iommu_sync_map(struct iommu_domain
*domain
,
333 unsigned long iova
, size_t size
)
335 struct sprd_iommu_domain
*dom
= to_sprd_domain(domain
);
338 if (dom
->sdev
->ver
== SPRD_IOMMU_EX
)
339 reg
= SPRD_EX_UPDATE
;
341 reg
= SPRD_VAU_UPDATE
;
343 /* clear IOMMU TLB buffer after page table updated */
344 sprd_iommu_write(dom
->sdev
, reg
, 0xffffffff);
347 static void sprd_iommu_sync(struct iommu_domain
*domain
,
348 struct iommu_iotlb_gather
*iotlb_gather
)
350 sprd_iommu_sync_map(domain
, 0, 0);
353 static phys_addr_t
sprd_iommu_iova_to_phys(struct iommu_domain
*domain
,
356 struct sprd_iommu_domain
*dom
= to_sprd_domain(domain
);
359 unsigned long start
= domain
->geometry
.aperture_start
;
360 unsigned long end
= domain
->geometry
.aperture_end
;
362 if (WARN_ON(iova
< start
|| iova
> end
))
365 spin_lock_irqsave(&dom
->pgtlock
, flags
);
366 pa
= *(dom
->pgt_va
+ ((iova
- start
) >> SPRD_IOMMU_PAGE_SHIFT
));
367 pa
= (pa
<< SPRD_IOMMU_PAGE_SHIFT
) + ((iova
- start
) & (SPRD_IOMMU_PAGE_SIZE
- 1));
368 spin_unlock_irqrestore(&dom
->pgtlock
, flags
);
373 static struct iommu_device
*sprd_iommu_probe_device(struct device
*dev
)
375 struct iommu_fwspec
*fwspec
= dev_iommu_fwspec_get(dev
);
376 struct sprd_iommu_device
*sdev
;
378 if (!fwspec
|| fwspec
->ops
!= &sprd_iommu_ops
)
379 return ERR_PTR(-ENODEV
);
381 sdev
= dev_iommu_priv_get(dev
);
386 static void sprd_iommu_release_device(struct device
*dev
)
388 struct iommu_fwspec
*fwspec
= dev_iommu_fwspec_get(dev
);
390 if (!fwspec
|| fwspec
->ops
!= &sprd_iommu_ops
)
393 iommu_fwspec_free(dev
);
396 static struct iommu_group
*sprd_iommu_device_group(struct device
*dev
)
398 struct sprd_iommu_device
*sdev
= dev_iommu_priv_get(dev
);
400 return iommu_group_ref_get(sdev
->group
);
403 static int sprd_iommu_of_xlate(struct device
*dev
, struct of_phandle_args
*args
)
405 struct platform_device
*pdev
;
407 if (!dev_iommu_priv_get(dev
)) {
408 pdev
= of_find_device_by_node(args
->np
);
409 dev_iommu_priv_set(dev
, platform_get_drvdata(pdev
));
410 platform_device_put(pdev
);
417 static const struct iommu_ops sprd_iommu_ops
= {
418 .domain_alloc
= sprd_iommu_domain_alloc
,
419 .domain_free
= sprd_iommu_domain_free
,
420 .attach_dev
= sprd_iommu_attach_device
,
421 .detach_dev
= sprd_iommu_detach_device
,
422 .map
= sprd_iommu_map
,
423 .unmap
= sprd_iommu_unmap
,
424 .iotlb_sync_map
= sprd_iommu_sync_map
,
425 .iotlb_sync
= sprd_iommu_sync
,
426 .iova_to_phys
= sprd_iommu_iova_to_phys
,
427 .probe_device
= sprd_iommu_probe_device
,
428 .release_device
= sprd_iommu_release_device
,
429 .device_group
= sprd_iommu_device_group
,
430 .of_xlate
= sprd_iommu_of_xlate
,
431 .pgsize_bitmap
= ~0UL << SPRD_IOMMU_PAGE_SHIFT
,
432 .owner
= THIS_MODULE
,
435 static const struct of_device_id sprd_iommu_of_match
[] = {
436 { .compatible
= "sprd,iommu-v1" },
439 MODULE_DEVICE_TABLE(of
, sprd_iommu_of_match
);
442 * Clock is not required, access to some of IOMMUs is controlled by gate
443 * clk, enabled clocks for that kind of IOMMUs before accessing.
444 * Return 0 for success or no clocks found.
446 static int sprd_iommu_clk_enable(struct sprd_iommu_device
*sdev
)
450 eb
= devm_clk_get_optional(sdev
->dev
, NULL
);
458 return clk_prepare_enable(eb
);
461 static void sprd_iommu_clk_disable(struct sprd_iommu_device
*sdev
)
464 clk_disable_unprepare(sdev
->eb
);
467 static int sprd_iommu_probe(struct platform_device
*pdev
)
469 struct sprd_iommu_device
*sdev
;
470 struct device
*dev
= &pdev
->dev
;
474 sdev
= devm_kzalloc(dev
, sizeof(*sdev
), GFP_KERNEL
);
478 base
= devm_platform_ioremap_resource(pdev
, 0);
480 dev_err(dev
, "Failed to get ioremap resource.\n");
481 return PTR_ERR(base
);
485 sdev
->prot_page_va
= dma_alloc_coherent(dev
, SPRD_IOMMU_PAGE_SIZE
,
486 &sdev
->prot_page_pa
, GFP_KERNEL
);
487 if (!sdev
->prot_page_va
)
490 platform_set_drvdata(pdev
, sdev
);
493 /* All the client devices are in the same iommu-group */
494 sdev
->group
= iommu_group_alloc();
495 if (IS_ERR(sdev
->group
)) {
496 ret
= PTR_ERR(sdev
->group
);
500 ret
= iommu_device_sysfs_add(&sdev
->iommu
, dev
, NULL
, dev_name(dev
));
504 ret
= iommu_device_register(&sdev
->iommu
, &sprd_iommu_ops
, dev
);
508 if (!iommu_present(&platform_bus_type
))
509 bus_set_iommu(&platform_bus_type
, &sprd_iommu_ops
);
511 ret
= sprd_iommu_clk_enable(sdev
);
513 goto unregister_iommu
;
515 ret
= sprd_iommu_get_version(sdev
);
517 dev_err(dev
, "IOMMU version(%d) is invalid.\n", ret
);
525 sprd_iommu_clk_disable(sdev
);
527 iommu_device_unregister(&sdev
->iommu
);
529 iommu_device_sysfs_remove(&sdev
->iommu
);
531 iommu_group_put(sdev
->group
);
533 dma_free_coherent(sdev
->dev
, SPRD_IOMMU_PAGE_SIZE
, sdev
->prot_page_va
, sdev
->prot_page_pa
);
537 static int sprd_iommu_remove(struct platform_device
*pdev
)
539 struct sprd_iommu_device
*sdev
= platform_get_drvdata(pdev
);
541 dma_free_coherent(sdev
->dev
, SPRD_IOMMU_PAGE_SIZE
, sdev
->prot_page_va
, sdev
->prot_page_pa
);
543 iommu_group_put(sdev
->group
);
546 bus_set_iommu(&platform_bus_type
, NULL
);
548 platform_set_drvdata(pdev
, NULL
);
549 iommu_device_sysfs_remove(&sdev
->iommu
);
550 iommu_device_unregister(&sdev
->iommu
);
555 static struct platform_driver sprd_iommu_driver
= {
557 .name
= "sprd-iommu",
558 .of_match_table
= sprd_iommu_of_match
,
559 .suppress_bind_attrs
= true,
561 .probe
= sprd_iommu_probe
,
562 .remove
= sprd_iommu_remove
,
564 module_platform_driver(sprd_iommu_driver
);
566 MODULE_DESCRIPTION("IOMMU driver for Unisoc SoCs");
567 MODULE_ALIAS("platform:sprd-iommu");
568 MODULE_LICENSE("GPL");