2 * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd.
3 * http://www.samsung.com
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
10 #ifdef CONFIG_EXYNOS_IOMMU_DEBUG
14 #include <linux/clk.h>
15 #include <linux/dma-mapping.h>
16 #include <linux/err.h>
18 #include <linux/iommu.h>
19 #include <linux/interrupt.h>
20 #include <linux/list.h>
22 #include <linux/of_iommu.h>
23 #include <linux/of_platform.h>
24 #include <linux/platform_device.h>
25 #include <linux/pm_runtime.h>
26 #include <linux/slab.h>
27 #include <linux/dma-iommu.h>
29 typedef u32 sysmmu_iova_t
;
30 typedef u32 sysmmu_pte_t
;
32 /* We do not consider super section mapping (16MB) */
34 #define LPAGE_ORDER 16
35 #define SPAGE_ORDER 12
37 #define SECT_SIZE (1 << SECT_ORDER)
38 #define LPAGE_SIZE (1 << LPAGE_ORDER)
39 #define SPAGE_SIZE (1 << SPAGE_ORDER)
41 #define SECT_MASK (~(SECT_SIZE - 1))
42 #define LPAGE_MASK (~(LPAGE_SIZE - 1))
43 #define SPAGE_MASK (~(SPAGE_SIZE - 1))
45 #define lv1ent_fault(sent) ((*(sent) == ZERO_LV2LINK) || \
46 ((*(sent) & 3) == 0) || ((*(sent) & 3) == 3))
47 #define lv1ent_zero(sent) (*(sent) == ZERO_LV2LINK)
48 #define lv1ent_page_zero(sent) ((*(sent) & 3) == 1)
49 #define lv1ent_page(sent) ((*(sent) != ZERO_LV2LINK) && \
51 #define lv1ent_section(sent) ((*(sent) & 3) == 2)
53 #define lv2ent_fault(pent) ((*(pent) & 3) == 0)
54 #define lv2ent_small(pent) ((*(pent) & 2) == 2)
55 #define lv2ent_large(pent) ((*(pent) & 3) == 1)
57 #ifdef CONFIG_BIG_ENDIAN
58 #warning "revisit driver if we can enable big-endian ptes"
62 * v1.x - v3.x SYSMMU supports 32bit physical and 32bit virtual address spaces
63 * v5.0 introduced support for 36bit physical address space by shifting
64 * all page entry values by 4 bits.
65 * All SYSMMU controllers in the system support the address spaces of the same
66 * size, so PG_ENT_SHIFT can be initialized on first SYSMMU probe to proper
69 static short PG_ENT_SHIFT
= -1;
70 #define SYSMMU_PG_ENT_SHIFT 0
71 #define SYSMMU_V5_PG_ENT_SHIFT 4
73 static const sysmmu_pte_t
*LV1_PROT
;
74 static const sysmmu_pte_t SYSMMU_LV1_PROT
[] = {
75 ((0 << 15) | (0 << 10)), /* no access */
76 ((1 << 15) | (1 << 10)), /* IOMMU_READ only */
77 ((0 << 15) | (1 << 10)), /* IOMMU_WRITE not supported, use read/write */
78 ((0 << 15) | (1 << 10)), /* IOMMU_READ | IOMMU_WRITE */
80 static const sysmmu_pte_t SYSMMU_V5_LV1_PROT
[] = {
81 (0 << 4), /* no access */
82 (1 << 4), /* IOMMU_READ only */
83 (2 << 4), /* IOMMU_WRITE only */
84 (3 << 4), /* IOMMU_READ | IOMMU_WRITE */
87 static const sysmmu_pte_t
*LV2_PROT
;
88 static const sysmmu_pte_t SYSMMU_LV2_PROT
[] = {
89 ((0 << 9) | (0 << 4)), /* no access */
90 ((1 << 9) | (1 << 4)), /* IOMMU_READ only */
91 ((0 << 9) | (1 << 4)), /* IOMMU_WRITE not supported, use read/write */
92 ((0 << 9) | (1 << 4)), /* IOMMU_READ | IOMMU_WRITE */
94 static const sysmmu_pte_t SYSMMU_V5_LV2_PROT
[] = {
95 (0 << 2), /* no access */
96 (1 << 2), /* IOMMU_READ only */
97 (2 << 2), /* IOMMU_WRITE only */
98 (3 << 2), /* IOMMU_READ | IOMMU_WRITE */
101 #define SYSMMU_SUPPORTED_PROT_BITS (IOMMU_READ | IOMMU_WRITE)
103 #define sect_to_phys(ent) (((phys_addr_t) ent) << PG_ENT_SHIFT)
104 #define section_phys(sent) (sect_to_phys(*(sent)) & SECT_MASK)
105 #define section_offs(iova) (iova & (SECT_SIZE - 1))
106 #define lpage_phys(pent) (sect_to_phys(*(pent)) & LPAGE_MASK)
107 #define lpage_offs(iova) (iova & (LPAGE_SIZE - 1))
108 #define spage_phys(pent) (sect_to_phys(*(pent)) & SPAGE_MASK)
109 #define spage_offs(iova) (iova & (SPAGE_SIZE - 1))
111 #define NUM_LV1ENTRIES 4096
112 #define NUM_LV2ENTRIES (SECT_SIZE / SPAGE_SIZE)
114 static u32
lv1ent_offset(sysmmu_iova_t iova
)
116 return iova
>> SECT_ORDER
;
119 static u32
lv2ent_offset(sysmmu_iova_t iova
)
121 return (iova
>> SPAGE_ORDER
) & (NUM_LV2ENTRIES
- 1);
124 #define LV1TABLE_SIZE (NUM_LV1ENTRIES * sizeof(sysmmu_pte_t))
125 #define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(sysmmu_pte_t))
127 #define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE)
128 #define lv2table_base(sent) (sect_to_phys(*(sent) & 0xFFFFFFC0))
130 #define mk_lv1ent_sect(pa, prot) ((pa >> PG_ENT_SHIFT) | LV1_PROT[prot] | 2)
131 #define mk_lv1ent_page(pa) ((pa >> PG_ENT_SHIFT) | 1)
132 #define mk_lv2ent_lpage(pa, prot) ((pa >> PG_ENT_SHIFT) | LV2_PROT[prot] | 1)
133 #define mk_lv2ent_spage(pa, prot) ((pa >> PG_ENT_SHIFT) | LV2_PROT[prot] | 2)
135 #define CTRL_ENABLE 0x5
136 #define CTRL_BLOCK 0x7
137 #define CTRL_DISABLE 0x0
140 #define CFG_EAP (1 << 2)
141 #define CFG_QOS(n) ((n & 0xF) << 7)
142 #define CFG_ACGEN (1 << 24) /* System MMU 3.3 only */
143 #define CFG_SYSSEL (1 << 22) /* System MMU 3.2 only */
144 #define CFG_FLPDCACHE (1 << 20) /* System MMU 3.2+ only */
146 /* common registers */
147 #define REG_MMU_CTRL 0x000
148 #define REG_MMU_CFG 0x004
149 #define REG_MMU_STATUS 0x008
150 #define REG_MMU_VERSION 0x034
152 #define MMU_MAJ_VER(val) ((val) >> 7)
153 #define MMU_MIN_VER(val) ((val) & 0x7F)
154 #define MMU_RAW_VER(reg) (((reg) >> 21) & ((1 << 11) - 1)) /* 11 bits */
156 #define MAKE_MMU_VER(maj, min) ((((maj) & 0xF) << 7) | ((min) & 0x7F))
158 /* v1.x - v3.x registers */
159 #define REG_MMU_FLUSH 0x00C
160 #define REG_MMU_FLUSH_ENTRY 0x010
161 #define REG_PT_BASE_ADDR 0x014
162 #define REG_INT_STATUS 0x018
163 #define REG_INT_CLEAR 0x01C
165 #define REG_PAGE_FAULT_ADDR 0x024
166 #define REG_AW_FAULT_ADDR 0x028
167 #define REG_AR_FAULT_ADDR 0x02C
168 #define REG_DEFAULT_SLAVE_ADDR 0x030
171 #define REG_V5_PT_BASE_PFN 0x00C
172 #define REG_V5_MMU_FLUSH_ALL 0x010
173 #define REG_V5_MMU_FLUSH_ENTRY 0x014
174 #define REG_V5_MMU_FLUSH_RANGE 0x018
175 #define REG_V5_MMU_FLUSH_START 0x020
176 #define REG_V5_MMU_FLUSH_END 0x024
177 #define REG_V5_INT_STATUS 0x060
178 #define REG_V5_INT_CLEAR 0x064
179 #define REG_V5_FAULT_AR_VA 0x070
180 #define REG_V5_FAULT_AW_VA 0x080
182 #define has_sysmmu(dev) (dev->archdata.iommu != NULL)
184 static struct device
*dma_dev
;
185 static struct kmem_cache
*lv2table_kmem_cache
;
186 static sysmmu_pte_t
*zero_lv2_table
;
187 #define ZERO_LV2LINK mk_lv1ent_page(virt_to_phys(zero_lv2_table))
189 static sysmmu_pte_t
*section_entry(sysmmu_pte_t
*pgtable
, sysmmu_iova_t iova
)
191 return pgtable
+ lv1ent_offset(iova
);
194 static sysmmu_pte_t
*page_entry(sysmmu_pte_t
*sent
, sysmmu_iova_t iova
)
196 return (sysmmu_pte_t
*)phys_to_virt(
197 lv2table_base(sent
)) + lv2ent_offset(iova
);
201 * IOMMU fault information register
203 struct sysmmu_fault_info
{
204 unsigned int bit
; /* bit number in STATUS register */
205 unsigned short addr_reg
; /* register to read VA fault address */
206 const char *name
; /* human readable fault name */
207 unsigned int type
; /* fault type for report_iommu_fault */
210 static const struct sysmmu_fault_info sysmmu_faults
[] = {
211 { 0, REG_PAGE_FAULT_ADDR
, "PAGE", IOMMU_FAULT_READ
},
212 { 1, REG_AR_FAULT_ADDR
, "AR MULTI-HIT", IOMMU_FAULT_READ
},
213 { 2, REG_AW_FAULT_ADDR
, "AW MULTI-HIT", IOMMU_FAULT_WRITE
},
214 { 3, REG_DEFAULT_SLAVE_ADDR
, "BUS ERROR", IOMMU_FAULT_READ
},
215 { 4, REG_AR_FAULT_ADDR
, "AR SECURITY PROTECTION", IOMMU_FAULT_READ
},
216 { 5, REG_AR_FAULT_ADDR
, "AR ACCESS PROTECTION", IOMMU_FAULT_READ
},
217 { 6, REG_AW_FAULT_ADDR
, "AW SECURITY PROTECTION", IOMMU_FAULT_WRITE
},
218 { 7, REG_AW_FAULT_ADDR
, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE
},
221 static const struct sysmmu_fault_info sysmmu_v5_faults
[] = {
222 { 0, REG_V5_FAULT_AR_VA
, "AR PTW", IOMMU_FAULT_READ
},
223 { 1, REG_V5_FAULT_AR_VA
, "AR PAGE", IOMMU_FAULT_READ
},
224 { 2, REG_V5_FAULT_AR_VA
, "AR MULTI-HIT", IOMMU_FAULT_READ
},
225 { 3, REG_V5_FAULT_AR_VA
, "AR ACCESS PROTECTION", IOMMU_FAULT_READ
},
226 { 4, REG_V5_FAULT_AR_VA
, "AR SECURITY PROTECTION", IOMMU_FAULT_READ
},
227 { 16, REG_V5_FAULT_AW_VA
, "AW PTW", IOMMU_FAULT_WRITE
},
228 { 17, REG_V5_FAULT_AW_VA
, "AW PAGE", IOMMU_FAULT_WRITE
},
229 { 18, REG_V5_FAULT_AW_VA
, "AW MULTI-HIT", IOMMU_FAULT_WRITE
},
230 { 19, REG_V5_FAULT_AW_VA
, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE
},
231 { 20, REG_V5_FAULT_AW_VA
, "AW SECURITY PROTECTION", IOMMU_FAULT_WRITE
},
235 * This structure is attached to dev.archdata.iommu of the master device
236 * on device add, contains a list of SYSMMU controllers defined by device tree,
237 * which are bound to given master device. It is usually referenced by 'owner'
240 struct exynos_iommu_owner
{
241 struct list_head controllers
; /* list of sysmmu_drvdata.owner_node */
242 struct iommu_domain
*domain
; /* domain this device is attached */
243 struct mutex rpm_lock
; /* for runtime pm of all sysmmus */
247 * This structure exynos specific generalization of struct iommu_domain.
248 * It contains list of SYSMMU controllers from all master devices, which has
249 * been attached to this domain and page tables of IO address space defined by
250 * it. It is usually referenced by 'domain' pointer.
252 struct exynos_iommu_domain
{
253 struct list_head clients
; /* list of sysmmu_drvdata.domain_node */
254 sysmmu_pte_t
*pgtable
; /* lv1 page table, 16KB */
255 short *lv2entcnt
; /* free lv2 entry counter for each section */
256 spinlock_t lock
; /* lock for modyfying list of clients */
257 spinlock_t pgtablelock
; /* lock for modifying page table @ pgtable */
258 struct iommu_domain domain
; /* generic domain data structure */
262 * This structure hold all data of a single SYSMMU controller, this includes
263 * hw resources like registers and clocks, pointers and list nodes to connect
264 * it to all other structures, internal state and parameters read from device
265 * tree. It is usually referenced by 'data' pointer.
267 struct sysmmu_drvdata
{
268 struct device
*sysmmu
; /* SYSMMU controller device */
269 struct device
*master
; /* master device (owner) */
270 void __iomem
*sfrbase
; /* our registers */
271 struct clk
*clk
; /* SYSMMU's clock */
272 struct clk
*aclk
; /* SYSMMU's aclk clock */
273 struct clk
*pclk
; /* SYSMMU's pclk clock */
274 struct clk
*clk_master
; /* master's device clock */
275 spinlock_t lock
; /* lock for modyfying state */
276 bool active
; /* current status */
277 struct exynos_iommu_domain
*domain
; /* domain we belong to */
278 struct list_head domain_node
; /* node for domain clients list */
279 struct list_head owner_node
; /* node for owner controllers list */
280 phys_addr_t pgtable
; /* assigned page table structure */
281 unsigned int version
; /* our version */
283 struct iommu_device iommu
; /* IOMMU core handle */
286 static struct exynos_iommu_domain
*to_exynos_domain(struct iommu_domain
*dom
)
288 return container_of(dom
, struct exynos_iommu_domain
, domain
);
291 static void sysmmu_unblock(struct sysmmu_drvdata
*data
)
293 writel(CTRL_ENABLE
, data
->sfrbase
+ REG_MMU_CTRL
);
296 static bool sysmmu_block(struct sysmmu_drvdata
*data
)
300 writel(CTRL_BLOCK
, data
->sfrbase
+ REG_MMU_CTRL
);
301 while ((i
> 0) && !(readl(data
->sfrbase
+ REG_MMU_STATUS
) & 1))
304 if (!(readl(data
->sfrbase
+ REG_MMU_STATUS
) & 1)) {
305 sysmmu_unblock(data
);
312 static void __sysmmu_tlb_invalidate(struct sysmmu_drvdata
*data
)
314 if (MMU_MAJ_VER(data
->version
) < 5)
315 writel(0x1, data
->sfrbase
+ REG_MMU_FLUSH
);
317 writel(0x1, data
->sfrbase
+ REG_V5_MMU_FLUSH_ALL
);
320 static void __sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata
*data
,
321 sysmmu_iova_t iova
, unsigned int num_inv
)
325 if (MMU_MAJ_VER(data
->version
) < 5) {
326 for (i
= 0; i
< num_inv
; i
++) {
327 writel((iova
& SPAGE_MASK
) | 1,
328 data
->sfrbase
+ REG_MMU_FLUSH_ENTRY
);
333 writel((iova
& SPAGE_MASK
) | 1,
334 data
->sfrbase
+ REG_V5_MMU_FLUSH_ENTRY
);
336 writel((iova
& SPAGE_MASK
),
337 data
->sfrbase
+ REG_V5_MMU_FLUSH_START
);
338 writel((iova
& SPAGE_MASK
) + (num_inv
- 1) * SPAGE_SIZE
,
339 data
->sfrbase
+ REG_V5_MMU_FLUSH_END
);
340 writel(1, data
->sfrbase
+ REG_V5_MMU_FLUSH_RANGE
);
345 static void __sysmmu_set_ptbase(struct sysmmu_drvdata
*data
, phys_addr_t pgd
)
347 if (MMU_MAJ_VER(data
->version
) < 5)
348 writel(pgd
, data
->sfrbase
+ REG_PT_BASE_ADDR
);
350 writel(pgd
>> PAGE_SHIFT
,
351 data
->sfrbase
+ REG_V5_PT_BASE_PFN
);
353 __sysmmu_tlb_invalidate(data
);
356 static void __sysmmu_enable_clocks(struct sysmmu_drvdata
*data
)
358 BUG_ON(clk_prepare_enable(data
->clk_master
));
359 BUG_ON(clk_prepare_enable(data
->clk
));
360 BUG_ON(clk_prepare_enable(data
->pclk
));
361 BUG_ON(clk_prepare_enable(data
->aclk
));
364 static void __sysmmu_disable_clocks(struct sysmmu_drvdata
*data
)
366 clk_disable_unprepare(data
->aclk
);
367 clk_disable_unprepare(data
->pclk
);
368 clk_disable_unprepare(data
->clk
);
369 clk_disable_unprepare(data
->clk_master
);
372 static void __sysmmu_get_version(struct sysmmu_drvdata
*data
)
376 __sysmmu_enable_clocks(data
);
378 ver
= readl(data
->sfrbase
+ REG_MMU_VERSION
);
380 /* controllers on some SoCs don't report proper version */
381 if (ver
== 0x80000001u
)
382 data
->version
= MAKE_MMU_VER(1, 0);
384 data
->version
= MMU_RAW_VER(ver
);
386 dev_dbg(data
->sysmmu
, "hardware version: %d.%d\n",
387 MMU_MAJ_VER(data
->version
), MMU_MIN_VER(data
->version
));
389 __sysmmu_disable_clocks(data
);
392 static void show_fault_information(struct sysmmu_drvdata
*data
,
393 const struct sysmmu_fault_info
*finfo
,
394 sysmmu_iova_t fault_addr
)
398 dev_err(data
->sysmmu
, "%s: %s FAULT occurred at %#x\n",
399 dev_name(data
->master
), finfo
->name
, fault_addr
);
400 dev_dbg(data
->sysmmu
, "Page table base: %pa\n", &data
->pgtable
);
401 ent
= section_entry(phys_to_virt(data
->pgtable
), fault_addr
);
402 dev_dbg(data
->sysmmu
, "\tLv1 entry: %#x\n", *ent
);
403 if (lv1ent_page(ent
)) {
404 ent
= page_entry(ent
, fault_addr
);
405 dev_dbg(data
->sysmmu
, "\t Lv2 entry: %#x\n", *ent
);
409 static irqreturn_t
exynos_sysmmu_irq(int irq
, void *dev_id
)
411 /* SYSMMU is in blocked state when interrupt occurred. */
412 struct sysmmu_drvdata
*data
= dev_id
;
413 const struct sysmmu_fault_info
*finfo
;
414 unsigned int i
, n
, itype
;
415 sysmmu_iova_t fault_addr
= -1;
416 unsigned short reg_status
, reg_clear
;
419 WARN_ON(!data
->active
);
421 if (MMU_MAJ_VER(data
->version
) < 5) {
422 reg_status
= REG_INT_STATUS
;
423 reg_clear
= REG_INT_CLEAR
;
424 finfo
= sysmmu_faults
;
425 n
= ARRAY_SIZE(sysmmu_faults
);
427 reg_status
= REG_V5_INT_STATUS
;
428 reg_clear
= REG_V5_INT_CLEAR
;
429 finfo
= sysmmu_v5_faults
;
430 n
= ARRAY_SIZE(sysmmu_v5_faults
);
433 spin_lock(&data
->lock
);
435 clk_enable(data
->clk_master
);
437 itype
= __ffs(readl(data
->sfrbase
+ reg_status
));
438 for (i
= 0; i
< n
; i
++, finfo
++)
439 if (finfo
->bit
== itype
)
441 /* unknown/unsupported fault */
444 /* print debug message */
445 fault_addr
= readl(data
->sfrbase
+ finfo
->addr_reg
);
446 show_fault_information(data
, finfo
, fault_addr
);
449 ret
= report_iommu_fault(&data
->domain
->domain
,
450 data
->master
, fault_addr
, finfo
->type
);
451 /* fault is not recovered by fault handler */
454 writel(1 << itype
, data
->sfrbase
+ reg_clear
);
456 sysmmu_unblock(data
);
458 clk_disable(data
->clk_master
);
460 spin_unlock(&data
->lock
);
465 static void __sysmmu_disable(struct sysmmu_drvdata
*data
)
469 clk_enable(data
->clk_master
);
471 spin_lock_irqsave(&data
->lock
, flags
);
472 writel(CTRL_DISABLE
, data
->sfrbase
+ REG_MMU_CTRL
);
473 writel(0, data
->sfrbase
+ REG_MMU_CFG
);
474 data
->active
= false;
475 spin_unlock_irqrestore(&data
->lock
, flags
);
477 __sysmmu_disable_clocks(data
);
480 static void __sysmmu_init_config(struct sysmmu_drvdata
*data
)
484 if (data
->version
<= MAKE_MMU_VER(3, 1))
485 cfg
= CFG_LRU
| CFG_QOS(15);
486 else if (data
->version
<= MAKE_MMU_VER(3, 2))
487 cfg
= CFG_LRU
| CFG_QOS(15) | CFG_FLPDCACHE
| CFG_SYSSEL
;
489 cfg
= CFG_QOS(15) | CFG_FLPDCACHE
| CFG_ACGEN
;
491 cfg
|= CFG_EAP
; /* enable access protection bits check */
493 writel(cfg
, data
->sfrbase
+ REG_MMU_CFG
);
496 static void __sysmmu_enable(struct sysmmu_drvdata
*data
)
500 __sysmmu_enable_clocks(data
);
502 spin_lock_irqsave(&data
->lock
, flags
);
503 writel(CTRL_BLOCK
, data
->sfrbase
+ REG_MMU_CTRL
);
504 __sysmmu_init_config(data
);
505 __sysmmu_set_ptbase(data
, data
->pgtable
);
506 writel(CTRL_ENABLE
, data
->sfrbase
+ REG_MMU_CTRL
);
508 spin_unlock_irqrestore(&data
->lock
, flags
);
511 * SYSMMU driver keeps master's clock enabled only for the short
512 * time, while accessing the registers. For performing address
513 * translation during DMA transaction it relies on the client
514 * driver to enable it.
516 clk_disable(data
->clk_master
);
519 static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata
*data
,
524 spin_lock_irqsave(&data
->lock
, flags
);
525 if (data
->active
&& data
->version
>= MAKE_MMU_VER(3, 3)) {
526 clk_enable(data
->clk_master
);
527 if (sysmmu_block(data
)) {
528 if (data
->version
>= MAKE_MMU_VER(5, 0))
529 __sysmmu_tlb_invalidate(data
);
531 __sysmmu_tlb_invalidate_entry(data
, iova
, 1);
532 sysmmu_unblock(data
);
534 clk_disable(data
->clk_master
);
536 spin_unlock_irqrestore(&data
->lock
, flags
);
539 static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata
*data
,
540 sysmmu_iova_t iova
, size_t size
)
544 spin_lock_irqsave(&data
->lock
, flags
);
546 unsigned int num_inv
= 1;
548 clk_enable(data
->clk_master
);
551 * L2TLB invalidation required
552 * 4KB page: 1 invalidation
553 * 64KB page: 16 invalidations
554 * 1MB page: 64 invalidations
555 * because it is set-associative TLB
556 * with 8-way and 64 sets.
557 * 1MB page can be cached in one of all sets.
558 * 64KB page can be one of 16 consecutive sets.
560 if (MMU_MAJ_VER(data
->version
) == 2)
561 num_inv
= min_t(unsigned int, size
/ PAGE_SIZE
, 64);
563 if (sysmmu_block(data
)) {
564 __sysmmu_tlb_invalidate_entry(data
, iova
, num_inv
);
565 sysmmu_unblock(data
);
567 clk_disable(data
->clk_master
);
569 spin_unlock_irqrestore(&data
->lock
, flags
);
572 static struct iommu_ops exynos_iommu_ops
;
574 static int __init
exynos_sysmmu_probe(struct platform_device
*pdev
)
577 struct device
*dev
= &pdev
->dev
;
578 struct sysmmu_drvdata
*data
;
579 struct resource
*res
;
581 data
= devm_kzalloc(dev
, sizeof(*data
), GFP_KERNEL
);
585 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
586 data
->sfrbase
= devm_ioremap_resource(dev
, res
);
587 if (IS_ERR(data
->sfrbase
))
588 return PTR_ERR(data
->sfrbase
);
590 irq
= platform_get_irq(pdev
, 0);
592 dev_err(dev
, "Unable to find IRQ resource\n");
596 ret
= devm_request_irq(dev
, irq
, exynos_sysmmu_irq
, 0,
597 dev_name(dev
), data
);
599 dev_err(dev
, "Unabled to register handler of irq %d\n", irq
);
603 data
->clk
= devm_clk_get(dev
, "sysmmu");
604 if (PTR_ERR(data
->clk
) == -ENOENT
)
606 else if (IS_ERR(data
->clk
))
607 return PTR_ERR(data
->clk
);
609 data
->aclk
= devm_clk_get(dev
, "aclk");
610 if (PTR_ERR(data
->aclk
) == -ENOENT
)
612 else if (IS_ERR(data
->aclk
))
613 return PTR_ERR(data
->aclk
);
615 data
->pclk
= devm_clk_get(dev
, "pclk");
616 if (PTR_ERR(data
->pclk
) == -ENOENT
)
618 else if (IS_ERR(data
->pclk
))
619 return PTR_ERR(data
->pclk
);
621 if (!data
->clk
&& (!data
->aclk
|| !data
->pclk
)) {
622 dev_err(dev
, "Failed to get device clock(s)!\n");
626 data
->clk_master
= devm_clk_get(dev
, "master");
627 if (PTR_ERR(data
->clk_master
) == -ENOENT
)
628 data
->clk_master
= NULL
;
629 else if (IS_ERR(data
->clk_master
))
630 return PTR_ERR(data
->clk_master
);
633 spin_lock_init(&data
->lock
);
635 ret
= iommu_device_sysfs_add(&data
->iommu
, &pdev
->dev
, NULL
,
636 dev_name(data
->sysmmu
));
640 iommu_device_set_ops(&data
->iommu
, &exynos_iommu_ops
);
641 iommu_device_set_fwnode(&data
->iommu
, &dev
->of_node
->fwnode
);
643 ret
= iommu_device_register(&data
->iommu
);
647 platform_set_drvdata(pdev
, data
);
649 __sysmmu_get_version(data
);
650 if (PG_ENT_SHIFT
< 0) {
651 if (MMU_MAJ_VER(data
->version
) < 5) {
652 PG_ENT_SHIFT
= SYSMMU_PG_ENT_SHIFT
;
653 LV1_PROT
= SYSMMU_LV1_PROT
;
654 LV2_PROT
= SYSMMU_LV2_PROT
;
656 PG_ENT_SHIFT
= SYSMMU_V5_PG_ENT_SHIFT
;
657 LV1_PROT
= SYSMMU_V5_LV1_PROT
;
658 LV2_PROT
= SYSMMU_V5_LV2_PROT
;
662 pm_runtime_enable(dev
);
667 static int __maybe_unused
exynos_sysmmu_suspend(struct device
*dev
)
669 struct sysmmu_drvdata
*data
= dev_get_drvdata(dev
);
670 struct device
*master
= data
->master
;
673 struct exynos_iommu_owner
*owner
= master
->archdata
.iommu
;
675 mutex_lock(&owner
->rpm_lock
);
677 dev_dbg(data
->sysmmu
, "saving state\n");
678 __sysmmu_disable(data
);
680 mutex_unlock(&owner
->rpm_lock
);
685 static int __maybe_unused
exynos_sysmmu_resume(struct device
*dev
)
687 struct sysmmu_drvdata
*data
= dev_get_drvdata(dev
);
688 struct device
*master
= data
->master
;
691 struct exynos_iommu_owner
*owner
= master
->archdata
.iommu
;
693 mutex_lock(&owner
->rpm_lock
);
695 dev_dbg(data
->sysmmu
, "restoring state\n");
696 __sysmmu_enable(data
);
698 mutex_unlock(&owner
->rpm_lock
);
703 static const struct dev_pm_ops sysmmu_pm_ops
= {
704 SET_RUNTIME_PM_OPS(exynos_sysmmu_suspend
, exynos_sysmmu_resume
, NULL
)
705 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend
,
706 pm_runtime_force_resume
)
709 static const struct of_device_id sysmmu_of_match
[] __initconst
= {
710 { .compatible
= "samsung,exynos-sysmmu", },
714 static struct platform_driver exynos_sysmmu_driver __refdata
= {
715 .probe
= exynos_sysmmu_probe
,
717 .name
= "exynos-sysmmu",
718 .of_match_table
= sysmmu_of_match
,
719 .pm
= &sysmmu_pm_ops
,
720 .suppress_bind_attrs
= true,
724 static inline void update_pte(sysmmu_pte_t
*ent
, sysmmu_pte_t val
)
726 dma_sync_single_for_cpu(dma_dev
, virt_to_phys(ent
), sizeof(*ent
),
728 *ent
= cpu_to_le32(val
);
729 dma_sync_single_for_device(dma_dev
, virt_to_phys(ent
), sizeof(*ent
),
733 static struct iommu_domain
*exynos_iommu_domain_alloc(unsigned type
)
735 struct exynos_iommu_domain
*domain
;
739 /* Check if correct PTE offsets are initialized */
740 BUG_ON(PG_ENT_SHIFT
< 0 || !dma_dev
);
742 domain
= kzalloc(sizeof(*domain
), GFP_KERNEL
);
746 if (type
== IOMMU_DOMAIN_DMA
) {
747 if (iommu_get_dma_cookie(&domain
->domain
) != 0)
749 } else if (type
!= IOMMU_DOMAIN_UNMANAGED
) {
753 domain
->pgtable
= (sysmmu_pte_t
*)__get_free_pages(GFP_KERNEL
, 2);
754 if (!domain
->pgtable
)
757 domain
->lv2entcnt
= (short *)__get_free_pages(GFP_KERNEL
| __GFP_ZERO
, 1);
758 if (!domain
->lv2entcnt
)
761 /* Workaround for System MMU v3.3 to prevent caching 1MiB mapping */
762 for (i
= 0; i
< NUM_LV1ENTRIES
; i
++)
763 domain
->pgtable
[i
] = ZERO_LV2LINK
;
765 handle
= dma_map_single(dma_dev
, domain
->pgtable
, LV1TABLE_SIZE
,
767 /* For mapping page table entries we rely on dma == phys */
768 BUG_ON(handle
!= virt_to_phys(domain
->pgtable
));
769 if (dma_mapping_error(dma_dev
, handle
))
772 spin_lock_init(&domain
->lock
);
773 spin_lock_init(&domain
->pgtablelock
);
774 INIT_LIST_HEAD(&domain
->clients
);
776 domain
->domain
.geometry
.aperture_start
= 0;
777 domain
->domain
.geometry
.aperture_end
= ~0UL;
778 domain
->domain
.geometry
.force_aperture
= true;
780 return &domain
->domain
;
783 free_pages((unsigned long)domain
->lv2entcnt
, 1);
785 free_pages((unsigned long)domain
->pgtable
, 2);
787 if (type
== IOMMU_DOMAIN_DMA
)
788 iommu_put_dma_cookie(&domain
->domain
);
794 static void exynos_iommu_domain_free(struct iommu_domain
*iommu_domain
)
796 struct exynos_iommu_domain
*domain
= to_exynos_domain(iommu_domain
);
797 struct sysmmu_drvdata
*data
, *next
;
801 WARN_ON(!list_empty(&domain
->clients
));
803 spin_lock_irqsave(&domain
->lock
, flags
);
805 list_for_each_entry_safe(data
, next
, &domain
->clients
, domain_node
) {
806 spin_lock(&data
->lock
);
807 __sysmmu_disable(data
);
810 list_del_init(&data
->domain_node
);
811 spin_unlock(&data
->lock
);
814 spin_unlock_irqrestore(&domain
->lock
, flags
);
816 if (iommu_domain
->type
== IOMMU_DOMAIN_DMA
)
817 iommu_put_dma_cookie(iommu_domain
);
819 dma_unmap_single(dma_dev
, virt_to_phys(domain
->pgtable
), LV1TABLE_SIZE
,
822 for (i
= 0; i
< NUM_LV1ENTRIES
; i
++)
823 if (lv1ent_page(domain
->pgtable
+ i
)) {
824 phys_addr_t base
= lv2table_base(domain
->pgtable
+ i
);
826 dma_unmap_single(dma_dev
, base
, LV2TABLE_SIZE
,
828 kmem_cache_free(lv2table_kmem_cache
,
832 free_pages((unsigned long)domain
->pgtable
, 2);
833 free_pages((unsigned long)domain
->lv2entcnt
, 1);
837 static void exynos_iommu_detach_device(struct iommu_domain
*iommu_domain
,
840 struct exynos_iommu_owner
*owner
= dev
->archdata
.iommu
;
841 struct exynos_iommu_domain
*domain
= to_exynos_domain(iommu_domain
);
842 phys_addr_t pagetable
= virt_to_phys(domain
->pgtable
);
843 struct sysmmu_drvdata
*data
, *next
;
846 if (!has_sysmmu(dev
) || owner
->domain
!= iommu_domain
)
849 mutex_lock(&owner
->rpm_lock
);
851 list_for_each_entry(data
, &owner
->controllers
, owner_node
) {
852 pm_runtime_get_noresume(data
->sysmmu
);
853 if (pm_runtime_active(data
->sysmmu
))
854 __sysmmu_disable(data
);
855 pm_runtime_put(data
->sysmmu
);
858 spin_lock_irqsave(&domain
->lock
, flags
);
859 list_for_each_entry_safe(data
, next
, &domain
->clients
, domain_node
) {
860 spin_lock(&data
->lock
);
863 list_del_init(&data
->domain_node
);
864 spin_unlock(&data
->lock
);
866 owner
->domain
= NULL
;
867 spin_unlock_irqrestore(&domain
->lock
, flags
);
869 mutex_unlock(&owner
->rpm_lock
);
871 dev_dbg(dev
, "%s: Detached IOMMU with pgtable %pa\n", __func__
,
875 static int exynos_iommu_attach_device(struct iommu_domain
*iommu_domain
,
878 struct exynos_iommu_owner
*owner
= dev
->archdata
.iommu
;
879 struct exynos_iommu_domain
*domain
= to_exynos_domain(iommu_domain
);
880 struct sysmmu_drvdata
*data
;
881 phys_addr_t pagetable
= virt_to_phys(domain
->pgtable
);
884 if (!has_sysmmu(dev
))
888 exynos_iommu_detach_device(owner
->domain
, dev
);
890 mutex_lock(&owner
->rpm_lock
);
892 spin_lock_irqsave(&domain
->lock
, flags
);
893 list_for_each_entry(data
, &owner
->controllers
, owner_node
) {
894 spin_lock(&data
->lock
);
895 data
->pgtable
= pagetable
;
896 data
->domain
= domain
;
897 list_add_tail(&data
->domain_node
, &domain
->clients
);
898 spin_unlock(&data
->lock
);
900 owner
->domain
= iommu_domain
;
901 spin_unlock_irqrestore(&domain
->lock
, flags
);
903 list_for_each_entry(data
, &owner
->controllers
, owner_node
) {
904 pm_runtime_get_noresume(data
->sysmmu
);
905 if (pm_runtime_active(data
->sysmmu
))
906 __sysmmu_enable(data
);
907 pm_runtime_put(data
->sysmmu
);
910 mutex_unlock(&owner
->rpm_lock
);
912 dev_dbg(dev
, "%s: Attached IOMMU with pgtable %pa\n", __func__
,
918 static sysmmu_pte_t
*alloc_lv2entry(struct exynos_iommu_domain
*domain
,
919 sysmmu_pte_t
*sent
, sysmmu_iova_t iova
, short *pgcounter
)
921 if (lv1ent_section(sent
)) {
922 WARN(1, "Trying mapping on %#08x mapped with 1MiB page", iova
);
923 return ERR_PTR(-EADDRINUSE
);
926 if (lv1ent_fault(sent
)) {
929 bool need_flush_flpd_cache
= lv1ent_zero(sent
);
931 pent
= kmem_cache_zalloc(lv2table_kmem_cache
, GFP_ATOMIC
);
932 BUG_ON((uintptr_t)pent
& (LV2TABLE_SIZE
- 1));
934 return ERR_PTR(-ENOMEM
);
936 update_pte(sent
, mk_lv1ent_page(virt_to_phys(pent
)));
937 kmemleak_ignore(pent
);
938 *pgcounter
= NUM_LV2ENTRIES
;
939 handle
= dma_map_single(dma_dev
, pent
, LV2TABLE_SIZE
,
941 if (dma_mapping_error(dma_dev
, handle
)) {
942 kmem_cache_free(lv2table_kmem_cache
, pent
);
943 return ERR_PTR(-EADDRINUSE
);
947 * If pre-fetched SLPD is a faulty SLPD in zero_l2_table,
948 * FLPD cache may cache the address of zero_l2_table. This
949 * function replaces the zero_l2_table with new L2 page table
950 * to write valid mappings.
951 * Accessing the valid area may cause page fault since FLPD
952 * cache may still cache zero_l2_table for the valid area
953 * instead of new L2 page table that has the mapping
954 * information of the valid area.
955 * Thus any replacement of zero_l2_table with other valid L2
956 * page table must involve FLPD cache invalidation for System
958 * FLPD cache invalidation is performed with TLB invalidation
959 * by VPN without blocking. It is safe to invalidate TLB without
960 * blocking because the target address of TLB invalidation is
961 * not currently mapped.
963 if (need_flush_flpd_cache
) {
964 struct sysmmu_drvdata
*data
;
966 spin_lock(&domain
->lock
);
967 list_for_each_entry(data
, &domain
->clients
, domain_node
)
968 sysmmu_tlb_invalidate_flpdcache(data
, iova
);
969 spin_unlock(&domain
->lock
);
973 return page_entry(sent
, iova
);
976 static int lv1set_section(struct exynos_iommu_domain
*domain
,
977 sysmmu_pte_t
*sent
, sysmmu_iova_t iova
,
978 phys_addr_t paddr
, int prot
, short *pgcnt
)
980 if (lv1ent_section(sent
)) {
981 WARN(1, "Trying mapping on 1MiB@%#08x that is mapped",
986 if (lv1ent_page(sent
)) {
987 if (*pgcnt
!= NUM_LV2ENTRIES
) {
988 WARN(1, "Trying mapping on 1MiB@%#08x that is mapped",
993 kmem_cache_free(lv2table_kmem_cache
, page_entry(sent
, 0));
997 update_pte(sent
, mk_lv1ent_sect(paddr
, prot
));
999 spin_lock(&domain
->lock
);
1000 if (lv1ent_page_zero(sent
)) {
1001 struct sysmmu_drvdata
*data
;
1003 * Flushing FLPD cache in System MMU v3.3 that may cache a FLPD
1004 * entry by speculative prefetch of SLPD which has no mapping.
1006 list_for_each_entry(data
, &domain
->clients
, domain_node
)
1007 sysmmu_tlb_invalidate_flpdcache(data
, iova
);
1009 spin_unlock(&domain
->lock
);
1014 static int lv2set_page(sysmmu_pte_t
*pent
, phys_addr_t paddr
, size_t size
,
1015 int prot
, short *pgcnt
)
1017 if (size
== SPAGE_SIZE
) {
1018 if (WARN_ON(!lv2ent_fault(pent
)))
1021 update_pte(pent
, mk_lv2ent_spage(paddr
, prot
));
1023 } else { /* size == LPAGE_SIZE */
1025 dma_addr_t pent_base
= virt_to_phys(pent
);
1027 dma_sync_single_for_cpu(dma_dev
, pent_base
,
1028 sizeof(*pent
) * SPAGES_PER_LPAGE
,
1030 for (i
= 0; i
< SPAGES_PER_LPAGE
; i
++, pent
++) {
1031 if (WARN_ON(!lv2ent_fault(pent
))) {
1033 memset(pent
- i
, 0, sizeof(*pent
) * i
);
1037 *pent
= mk_lv2ent_lpage(paddr
, prot
);
1039 dma_sync_single_for_device(dma_dev
, pent_base
,
1040 sizeof(*pent
) * SPAGES_PER_LPAGE
,
1042 *pgcnt
-= SPAGES_PER_LPAGE
;
1049 * *CAUTION* to the I/O virtual memory managers that support exynos-iommu:
1051 * System MMU v3.x has advanced logic to improve address translation
1052 * performance with caching more page table entries by a page table walk.
1053 * However, the logic has a bug that while caching faulty page table entries,
1054 * System MMU reports page fault if the cached fault entry is hit even though
1055 * the fault entry is updated to a valid entry after the entry is cached.
1056 * To prevent caching faulty page table entries which may be updated to valid
1057 * entries later, the virtual memory manager should care about the workaround
1058 * for the problem. The following describes the workaround.
1060 * Any two consecutive I/O virtual address regions must have a hole of 128KiB
1061 * at maximum to prevent misbehavior of System MMU 3.x (workaround for h/w bug).
1063 * Precisely, any start address of I/O virtual region must be aligned with
1064 * the following sizes for System MMU v3.1 and v3.2.
1065 * System MMU v3.1: 128KiB
1066 * System MMU v3.2: 256KiB
1068 * Because System MMU v3.3 caches page table entries more aggressively, it needs
1070 * - Any two consecutive I/O virtual regions must have a hole of size larger
1071 * than or equal to 128KiB.
1072 * - Start address of an I/O virtual region must be aligned by 128KiB.
1074 static int exynos_iommu_map(struct iommu_domain
*iommu_domain
,
1075 unsigned long l_iova
, phys_addr_t paddr
, size_t size
,
1078 struct exynos_iommu_domain
*domain
= to_exynos_domain(iommu_domain
);
1079 sysmmu_pte_t
*entry
;
1080 sysmmu_iova_t iova
= (sysmmu_iova_t
)l_iova
;
1081 unsigned long flags
;
1084 BUG_ON(domain
->pgtable
== NULL
);
1085 prot
&= SYSMMU_SUPPORTED_PROT_BITS
;
1087 spin_lock_irqsave(&domain
->pgtablelock
, flags
);
1089 entry
= section_entry(domain
->pgtable
, iova
);
1091 if (size
== SECT_SIZE
) {
1092 ret
= lv1set_section(domain
, entry
, iova
, paddr
, prot
,
1093 &domain
->lv2entcnt
[lv1ent_offset(iova
)]);
1097 pent
= alloc_lv2entry(domain
, entry
, iova
,
1098 &domain
->lv2entcnt
[lv1ent_offset(iova
)]);
1101 ret
= PTR_ERR(pent
);
1103 ret
= lv2set_page(pent
, paddr
, size
, prot
,
1104 &domain
->lv2entcnt
[lv1ent_offset(iova
)]);
1108 pr_err("%s: Failed(%d) to map %#zx bytes @ %#x\n",
1109 __func__
, ret
, size
, iova
);
1111 spin_unlock_irqrestore(&domain
->pgtablelock
, flags
);
1116 static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain
*domain
,
1117 sysmmu_iova_t iova
, size_t size
)
1119 struct sysmmu_drvdata
*data
;
1120 unsigned long flags
;
1122 spin_lock_irqsave(&domain
->lock
, flags
);
1124 list_for_each_entry(data
, &domain
->clients
, domain_node
)
1125 sysmmu_tlb_invalidate_entry(data
, iova
, size
);
1127 spin_unlock_irqrestore(&domain
->lock
, flags
);
1130 static size_t exynos_iommu_unmap(struct iommu_domain
*iommu_domain
,
1131 unsigned long l_iova
, size_t size
)
1133 struct exynos_iommu_domain
*domain
= to_exynos_domain(iommu_domain
);
1134 sysmmu_iova_t iova
= (sysmmu_iova_t
)l_iova
;
1137 unsigned long flags
;
1139 BUG_ON(domain
->pgtable
== NULL
);
1141 spin_lock_irqsave(&domain
->pgtablelock
, flags
);
1143 ent
= section_entry(domain
->pgtable
, iova
);
1145 if (lv1ent_section(ent
)) {
1146 if (WARN_ON(size
< SECT_SIZE
)) {
1147 err_pgsize
= SECT_SIZE
;
1151 /* workaround for h/w bug in System MMU v3.3 */
1152 update_pte(ent
, ZERO_LV2LINK
);
1157 if (unlikely(lv1ent_fault(ent
))) {
1158 if (size
> SECT_SIZE
)
1163 /* lv1ent_page(sent) == true here */
1165 ent
= page_entry(ent
, iova
);
1167 if (unlikely(lv2ent_fault(ent
))) {
1172 if (lv2ent_small(ent
)) {
1175 domain
->lv2entcnt
[lv1ent_offset(iova
)] += 1;
1179 /* lv1ent_large(ent) == true here */
1180 if (WARN_ON(size
< LPAGE_SIZE
)) {
1181 err_pgsize
= LPAGE_SIZE
;
1185 dma_sync_single_for_cpu(dma_dev
, virt_to_phys(ent
),
1186 sizeof(*ent
) * SPAGES_PER_LPAGE
,
1188 memset(ent
, 0, sizeof(*ent
) * SPAGES_PER_LPAGE
);
1189 dma_sync_single_for_device(dma_dev
, virt_to_phys(ent
),
1190 sizeof(*ent
) * SPAGES_PER_LPAGE
,
1193 domain
->lv2entcnt
[lv1ent_offset(iova
)] += SPAGES_PER_LPAGE
;
1195 spin_unlock_irqrestore(&domain
->pgtablelock
, flags
);
1197 exynos_iommu_tlb_invalidate_entry(domain
, iova
, size
);
1201 spin_unlock_irqrestore(&domain
->pgtablelock
, flags
);
1203 pr_err("%s: Failed: size(%#zx) @ %#x is smaller than page size %#zx\n",
1204 __func__
, size
, iova
, err_pgsize
);
1209 static phys_addr_t
exynos_iommu_iova_to_phys(struct iommu_domain
*iommu_domain
,
1212 struct exynos_iommu_domain
*domain
= to_exynos_domain(iommu_domain
);
1213 sysmmu_pte_t
*entry
;
1214 unsigned long flags
;
1215 phys_addr_t phys
= 0;
1217 spin_lock_irqsave(&domain
->pgtablelock
, flags
);
1219 entry
= section_entry(domain
->pgtable
, iova
);
1221 if (lv1ent_section(entry
)) {
1222 phys
= section_phys(entry
) + section_offs(iova
);
1223 } else if (lv1ent_page(entry
)) {
1224 entry
= page_entry(entry
, iova
);
1226 if (lv2ent_large(entry
))
1227 phys
= lpage_phys(entry
) + lpage_offs(iova
);
1228 else if (lv2ent_small(entry
))
1229 phys
= spage_phys(entry
) + spage_offs(iova
);
1232 spin_unlock_irqrestore(&domain
->pgtablelock
, flags
);
1237 static struct iommu_group
*get_device_iommu_group(struct device
*dev
)
1239 struct iommu_group
*group
;
1241 group
= iommu_group_get(dev
);
1243 group
= iommu_group_alloc();
1248 static int exynos_iommu_add_device(struct device
*dev
)
1250 struct iommu_group
*group
;
1252 if (!has_sysmmu(dev
))
1255 group
= iommu_group_get_for_dev(dev
);
1258 return PTR_ERR(group
);
1260 iommu_group_put(group
);
1265 static void exynos_iommu_remove_device(struct device
*dev
)
1267 struct exynos_iommu_owner
*owner
= dev
->archdata
.iommu
;
1269 if (!has_sysmmu(dev
))
1272 if (owner
->domain
) {
1273 struct iommu_group
*group
= iommu_group_get(dev
);
1276 WARN_ON(owner
->domain
!=
1277 iommu_group_default_domain(group
));
1278 exynos_iommu_detach_device(owner
->domain
, dev
);
1279 iommu_group_put(group
);
1282 iommu_group_remove_device(dev
);
1285 static int exynos_iommu_of_xlate(struct device
*dev
,
1286 struct of_phandle_args
*spec
)
1288 struct exynos_iommu_owner
*owner
= dev
->archdata
.iommu
;
1289 struct platform_device
*sysmmu
= of_find_device_by_node(spec
->np
);
1290 struct sysmmu_drvdata
*data
, *entry
;
1295 data
= platform_get_drvdata(sysmmu
);
1300 owner
= kzalloc(sizeof(*owner
), GFP_KERNEL
);
1304 INIT_LIST_HEAD(&owner
->controllers
);
1305 mutex_init(&owner
->rpm_lock
);
1306 dev
->archdata
.iommu
= owner
;
1309 list_for_each_entry(entry
, &owner
->controllers
, owner_node
)
1313 list_add_tail(&data
->owner_node
, &owner
->controllers
);
1317 * SYSMMU will be runtime activated via device link (dependency) to its
1318 * master device, so there are no direct calls to pm_runtime_get/put
1321 device_link_add(dev
, data
->sysmmu
, DL_FLAG_PM_RUNTIME
);
1326 static struct iommu_ops exynos_iommu_ops
= {
1327 .domain_alloc
= exynos_iommu_domain_alloc
,
1328 .domain_free
= exynos_iommu_domain_free
,
1329 .attach_dev
= exynos_iommu_attach_device
,
1330 .detach_dev
= exynos_iommu_detach_device
,
1331 .map
= exynos_iommu_map
,
1332 .unmap
= exynos_iommu_unmap
,
1333 .map_sg
= default_iommu_map_sg
,
1334 .iova_to_phys
= exynos_iommu_iova_to_phys
,
1335 .device_group
= get_device_iommu_group
,
1336 .add_device
= exynos_iommu_add_device
,
1337 .remove_device
= exynos_iommu_remove_device
,
1338 .pgsize_bitmap
= SECT_SIZE
| LPAGE_SIZE
| SPAGE_SIZE
,
1339 .of_xlate
= exynos_iommu_of_xlate
,
1342 static bool init_done
;
1344 static int __init
exynos_iommu_init(void)
1348 lv2table_kmem_cache
= kmem_cache_create("exynos-iommu-lv2table",
1349 LV2TABLE_SIZE
, LV2TABLE_SIZE
, 0, NULL
);
1350 if (!lv2table_kmem_cache
) {
1351 pr_err("%s: Failed to create kmem cache\n", __func__
);
1355 ret
= platform_driver_register(&exynos_sysmmu_driver
);
1357 pr_err("%s: Failed to register driver\n", __func__
);
1358 goto err_reg_driver
;
1361 zero_lv2_table
= kmem_cache_zalloc(lv2table_kmem_cache
, GFP_KERNEL
);
1362 if (zero_lv2_table
== NULL
) {
1363 pr_err("%s: Failed to allocate zero level2 page table\n",
1369 ret
= bus_set_iommu(&platform_bus_type
, &exynos_iommu_ops
);
1371 pr_err("%s: Failed to register exynos-iommu driver.\n",
1380 kmem_cache_free(lv2table_kmem_cache
, zero_lv2_table
);
1382 platform_driver_unregister(&exynos_sysmmu_driver
);
1384 kmem_cache_destroy(lv2table_kmem_cache
);
1388 static int __init
exynos_iommu_of_setup(struct device_node
*np
)
1390 struct platform_device
*pdev
;
1393 exynos_iommu_init();
1395 pdev
= of_platform_device_create(np
, NULL
, platform_bus_type
.dev_root
);
1400 * use the first registered sysmmu device for performing
1401 * dma mapping operations on iommu page tables (cpu cache flush)
1404 dma_dev
= &pdev
->dev
;
1409 IOMMU_OF_DECLARE(exynos_iommu_of
, "samsung,exynos-sysmmu",
1410 exynos_iommu_of_setup
);