]>
Commit | Line | Data |
---|---|---|
41f3f513 | 1 | /* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved. |
a007dd51 PG |
2 | * |
3 | * Author: Stepan Moskovchenko <stepanm@codeaurora.org> | |
0720d1f0 SM |
4 | * |
5 | * This program is free software; you can redistribute it and/or modify | |
6 | * it under the terms of the GNU General Public License version 2 and | |
7 | * only version 2 as published by the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | * GNU General Public License for more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License | |
15 | * along with this program; if not, write to the Free Software | |
16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA | |
17 | * 02110-1301, USA. | |
18 | */ | |
19 | ||
20 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
21 | #include <linux/kernel.h> | |
a007dd51 | 22 | #include <linux/init.h> |
0720d1f0 SM |
23 | #include <linux/platform_device.h> |
24 | #include <linux/errno.h> | |
25 | #include <linux/io.h> | |
b77cf11f | 26 | #include <linux/io-pgtable.h> |
0720d1f0 SM |
27 | #include <linux/interrupt.h> |
28 | #include <linux/list.h> | |
29 | #include <linux/spinlock.h> | |
30 | #include <linux/slab.h> | |
31 | #include <linux/iommu.h> | |
41f3f513 | 32 | #include <linux/clk.h> |
f7f125ef | 33 | #include <linux/err.h> |
f78ebca8 | 34 | #include <linux/of_iommu.h> |
0720d1f0 SM |
35 | |
36 | #include <asm/cacheflush.h> | |
37 | #include <asm/sizes.h> | |
38 | ||
0b559df5 SB |
39 | #include "msm_iommu_hw-8xxx.h" |
40 | #include "msm_iommu.h" | |
0720d1f0 | 41 | |
100832c9 SM |
42 | #define MRC(reg, processor, op1, crn, crm, op2) \ |
43 | __asm__ __volatile__ ( \ | |
44 | " mrc " #processor "," #op1 ", %0," #crn "," #crm "," #op2 "\n" \ | |
45 | : "=r" (reg)) | |
46 | ||
83427275 OBC |
47 | /* bitmap of the page sizes currently supported */ |
48 | #define MSM_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M) | |
49 | ||
0720d1f0 | 50 | DEFINE_SPINLOCK(msm_iommu_lock); |
109bd48e | 51 | static LIST_HEAD(qcom_iommu_devices); |
c9220fbd | 52 | static struct iommu_ops msm_iommu_ops; |
0720d1f0 SM |
53 | |
54 | struct msm_priv { | |
0720d1f0 | 55 | struct list_head list_attached; |
3e116c3c | 56 | struct iommu_domain domain; |
c9220fbd S |
57 | struct io_pgtable_cfg cfg; |
58 | struct io_pgtable_ops *iop; | |
59 | struct device *dev; | |
60 | spinlock_t pgtlock; /* pagetable lock */ | |
0720d1f0 SM |
61 | }; |
62 | ||
3e116c3c JR |
63 | static struct msm_priv *to_msm_priv(struct iommu_domain *dom) |
64 | { | |
65 | return container_of(dom, struct msm_priv, domain); | |
66 | } | |
67 | ||
109bd48e | 68 | static int __enable_clocks(struct msm_iommu_dev *iommu) |
41f3f513 SM |
69 | { |
70 | int ret; | |
71 | ||
109bd48e | 72 | ret = clk_enable(iommu->pclk); |
41f3f513 SM |
73 | if (ret) |
74 | goto fail; | |
75 | ||
109bd48e S |
76 | if (iommu->clk) { |
77 | ret = clk_enable(iommu->clk); | |
41f3f513 | 78 | if (ret) |
109bd48e | 79 | clk_disable(iommu->pclk); |
41f3f513 SM |
80 | } |
81 | fail: | |
82 | return ret; | |
83 | } | |
84 | ||
109bd48e | 85 | static void __disable_clocks(struct msm_iommu_dev *iommu) |
41f3f513 | 86 | { |
109bd48e S |
87 | if (iommu->clk) |
88 | clk_disable(iommu->clk); | |
89 | clk_disable(iommu->pclk); | |
41f3f513 SM |
90 | } |
91 | ||
f7f125ef S |
92 | static void msm_iommu_reset(void __iomem *base, int ncb) |
93 | { | |
94 | int ctx; | |
95 | ||
96 | SET_RPUE(base, 0); | |
97 | SET_RPUEIE(base, 0); | |
98 | SET_ESRRESTORE(base, 0); | |
99 | SET_TBE(base, 0); | |
100 | SET_CR(base, 0); | |
101 | SET_SPDMBE(base, 0); | |
102 | SET_TESTBUSCR(base, 0); | |
103 | SET_TLBRSW(base, 0); | |
104 | SET_GLOBAL_TLBIALL(base, 0); | |
105 | SET_RPU_ACR(base, 0); | |
106 | SET_TLBLKCRWE(base, 1); | |
107 | ||
108 | for (ctx = 0; ctx < ncb; ctx++) { | |
109 | SET_BPRCOSH(base, ctx, 0); | |
110 | SET_BPRCISH(base, ctx, 0); | |
111 | SET_BPRCNSH(base, ctx, 0); | |
112 | SET_BPSHCFG(base, ctx, 0); | |
113 | SET_BPMTCFG(base, ctx, 0); | |
114 | SET_ACTLR(base, ctx, 0); | |
115 | SET_SCTLR(base, ctx, 0); | |
116 | SET_FSRRESTORE(base, ctx, 0); | |
117 | SET_TTBR0(base, ctx, 0); | |
118 | SET_TTBR1(base, ctx, 0); | |
119 | SET_TTBCR(base, ctx, 0); | |
120 | SET_BFBCR(base, ctx, 0); | |
121 | SET_PAR(base, ctx, 0); | |
122 | SET_FAR(base, ctx, 0); | |
123 | SET_CTX_TLBIALL(base, ctx, 0); | |
124 | SET_TLBFLPTER(base, ctx, 0); | |
125 | SET_TLBSLPTER(base, ctx, 0); | |
126 | SET_TLBLKCR(base, ctx, 0); | |
f7f125ef S |
127 | SET_CONTEXTIDR(base, ctx, 0); |
128 | } | |
129 | } | |
130 | ||
c9220fbd | 131 | static void __flush_iotlb(void *cookie) |
0720d1f0 | 132 | { |
c9220fbd | 133 | struct msm_priv *priv = cookie; |
109bd48e S |
134 | struct msm_iommu_dev *iommu = NULL; |
135 | struct msm_iommu_ctx_dev *master; | |
33069739 | 136 | int ret = 0; |
109bd48e | 137 | |
c9220fbd S |
138 | list_for_each_entry(iommu, &priv->list_attached, dom_node) { |
139 | ret = __enable_clocks(iommu); | |
140 | if (ret) | |
141 | goto fail; | |
0720d1f0 | 142 | |
c9220fbd S |
143 | list_for_each_entry(master, &iommu->ctx_list, list) |
144 | SET_CTX_TLBIALL(iommu->base, master->num, 0); | |
0720d1f0 | 145 | |
c9220fbd | 146 | __disable_clocks(iommu); |
f6f41eb9 | 147 | } |
c9220fbd S |
148 | fail: |
149 | return; | |
150 | } | |
151 | ||
152 | static void __flush_iotlb_range(unsigned long iova, size_t size, | |
153 | size_t granule, bool leaf, void *cookie) | |
154 | { | |
155 | struct msm_priv *priv = cookie; | |
156 | struct msm_iommu_dev *iommu = NULL; | |
157 | struct msm_iommu_ctx_dev *master; | |
158 | int ret = 0; | |
159 | int temp_size; | |
0720d1f0 | 160 | |
109bd48e S |
161 | list_for_each_entry(iommu, &priv->list_attached, dom_node) { |
162 | ret = __enable_clocks(iommu); | |
41f3f513 SM |
163 | if (ret) |
164 | goto fail; | |
165 | ||
c9220fbd S |
166 | list_for_each_entry(master, &iommu->ctx_list, list) { |
167 | temp_size = size; | |
168 | do { | |
169 | iova &= TLBIVA_VA; | |
170 | iova |= GET_CONTEXTIDR_ASID(iommu->base, | |
171 | master->num); | |
172 | SET_TLBIVA(iommu->base, master->num, iova); | |
173 | iova += granule; | |
174 | } while (temp_size -= granule); | |
175 | } | |
109bd48e S |
176 | |
177 | __disable_clocks(iommu); | |
0720d1f0 | 178 | } |
c9220fbd | 179 | |
41f3f513 | 180 | fail: |
c9220fbd | 181 | return; |
0720d1f0 SM |
182 | } |
183 | ||
c9220fbd S |
184 | static void __flush_iotlb_sync(void *cookie) |
185 | { | |
186 | /* | |
187 | * Nothing is needed here, the barrier to guarantee | |
188 | * completion of the tlb sync operation is implicitly | |
189 | * taken care when the iommu client does a writel before | |
190 | * kick starting the other master. | |
191 | */ | |
192 | } | |
193 | ||
194 | static const struct iommu_gather_ops msm_iommu_gather_ops = { | |
195 | .tlb_flush_all = __flush_iotlb, | |
196 | .tlb_add_flush = __flush_iotlb_range, | |
197 | .tlb_sync = __flush_iotlb_sync, | |
198 | }; | |
199 | ||
109bd48e S |
200 | static int msm_iommu_alloc_ctx(unsigned long *map, int start, int end) |
201 | { | |
202 | int idx; | |
203 | ||
204 | do { | |
205 | idx = find_next_zero_bit(map, end, start); | |
206 | if (idx == end) | |
207 | return -ENOSPC; | |
208 | } while (test_and_set_bit(idx, map)); | |
209 | ||
210 | return idx; | |
211 | } | |
212 | ||
213 | static void msm_iommu_free_ctx(unsigned long *map, int idx) | |
214 | { | |
215 | clear_bit(idx, map); | |
216 | } | |
217 | ||
218 | static void config_mids(struct msm_iommu_dev *iommu, | |
219 | struct msm_iommu_ctx_dev *master) | |
220 | { | |
221 | int mid, ctx, i; | |
222 | ||
223 | for (i = 0; i < master->num_mids; i++) { | |
224 | mid = master->mids[i]; | |
225 | ctx = master->num; | |
226 | ||
227 | SET_M2VCBR_N(iommu->base, mid, 0); | |
228 | SET_CBACR_N(iommu->base, ctx, 0); | |
229 | ||
230 | /* Set VMID = 0 */ | |
231 | SET_VMID(iommu->base, mid, 0); | |
232 | ||
233 | /* Set the context number for that MID to this context */ | |
234 | SET_CBNDX(iommu->base, mid, ctx); | |
235 | ||
236 | /* Set MID associated with this context bank to 0*/ | |
237 | SET_CBVMID(iommu->base, ctx, 0); | |
238 | ||
239 | /* Set the ASID for TLB tagging for this context */ | |
240 | SET_CONTEXTIDR_ASID(iommu->base, ctx, ctx); | |
241 | ||
242 | /* Set security bit override to be Non-secure */ | |
243 | SET_NSCFG(iommu->base, mid, 3); | |
244 | } | |
245 | } | |
246 | ||
0720d1f0 SM |
247 | static void __reset_context(void __iomem *base, int ctx) |
248 | { | |
249 | SET_BPRCOSH(base, ctx, 0); | |
250 | SET_BPRCISH(base, ctx, 0); | |
251 | SET_BPRCNSH(base, ctx, 0); | |
252 | SET_BPSHCFG(base, ctx, 0); | |
253 | SET_BPMTCFG(base, ctx, 0); | |
254 | SET_ACTLR(base, ctx, 0); | |
255 | SET_SCTLR(base, ctx, 0); | |
256 | SET_FSRRESTORE(base, ctx, 0); | |
257 | SET_TTBR0(base, ctx, 0); | |
258 | SET_TTBR1(base, ctx, 0); | |
259 | SET_TTBCR(base, ctx, 0); | |
260 | SET_BFBCR(base, ctx, 0); | |
261 | SET_PAR(base, ctx, 0); | |
262 | SET_FAR(base, ctx, 0); | |
263 | SET_CTX_TLBIALL(base, ctx, 0); | |
264 | SET_TLBFLPTER(base, ctx, 0); | |
265 | SET_TLBSLPTER(base, ctx, 0); | |
266 | SET_TLBLKCR(base, ctx, 0); | |
0720d1f0 SM |
267 | } |
268 | ||
c9220fbd S |
269 | static void __program_context(void __iomem *base, int ctx, |
270 | struct msm_priv *priv) | |
0720d1f0 SM |
271 | { |
272 | __reset_context(base, ctx); | |
273 | ||
c9220fbd S |
274 | /* Turn on TEX Remap */ |
275 | SET_TRE(base, ctx, 1); | |
276 | SET_AFE(base, ctx, 1); | |
277 | ||
0720d1f0 SM |
278 | /* Set up HTW mode */ |
279 | /* TLB miss configuration: perform HTW on miss */ | |
280 | SET_TLBMCFG(base, ctx, 0x3); | |
281 | ||
282 | /* V2P configuration: HTW for access */ | |
283 | SET_V2PCFG(base, ctx, 0x3); | |
284 | ||
c9220fbd S |
285 | SET_TTBCR(base, ctx, priv->cfg.arm_v7s_cfg.tcr); |
286 | SET_TTBR0(base, ctx, priv->cfg.arm_v7s_cfg.ttbr[0]); | |
287 | SET_TTBR1(base, ctx, priv->cfg.arm_v7s_cfg.ttbr[1]); | |
288 | ||
289 | /* Set prrr and nmrr */ | |
290 | SET_PRRR(base, ctx, priv->cfg.arm_v7s_cfg.prrr); | |
291 | SET_NMRR(base, ctx, priv->cfg.arm_v7s_cfg.nmrr); | |
0720d1f0 SM |
292 | |
293 | /* Invalidate the TLB for this context */ | |
294 | SET_CTX_TLBIALL(base, ctx, 0); | |
295 | ||
296 | /* Set interrupt number to "secure" interrupt */ | |
297 | SET_IRPTNDX(base, ctx, 0); | |
298 | ||
299 | /* Enable context fault interrupt */ | |
300 | SET_CFEIE(base, ctx, 1); | |
301 | ||
302 | /* Stall access on a context fault and let the handler deal with it */ | |
303 | SET_CFCFG(base, ctx, 1); | |
304 | ||
305 | /* Redirect all cacheable requests to L2 slave port. */ | |
306 | SET_RCISH(base, ctx, 1); | |
307 | SET_RCOSH(base, ctx, 1); | |
308 | SET_RCNSH(base, ctx, 1); | |
309 | ||
0720d1f0 SM |
310 | /* Turn on BFB prefetch */ |
311 | SET_BFBDFE(base, ctx, 1); | |
312 | ||
0720d1f0 SM |
313 | /* Enable the MMU */ |
314 | SET_M(base, ctx, 1); | |
315 | } | |
316 | ||
3e116c3c | 317 | static struct iommu_domain *msm_iommu_domain_alloc(unsigned type) |
0720d1f0 | 318 | { |
3e116c3c | 319 | struct msm_priv *priv; |
0720d1f0 | 320 | |
3e116c3c JR |
321 | if (type != IOMMU_DOMAIN_UNMANAGED) |
322 | return NULL; | |
323 | ||
324 | priv = kzalloc(sizeof(*priv), GFP_KERNEL); | |
0720d1f0 SM |
325 | if (!priv) |
326 | goto fail_nomem; | |
327 | ||
328 | INIT_LIST_HEAD(&priv->list_attached); | |
4be6a290 | 329 | |
3e116c3c JR |
330 | priv->domain.geometry.aperture_start = 0; |
331 | priv->domain.geometry.aperture_end = (1ULL << 32) - 1; | |
332 | priv->domain.geometry.force_aperture = true; | |
4be6a290 | 333 | |
3e116c3c | 334 | return &priv->domain; |
0720d1f0 SM |
335 | |
336 | fail_nomem: | |
337 | kfree(priv); | |
3e116c3c | 338 | return NULL; |
0720d1f0 SM |
339 | } |
340 | ||
3e116c3c | 341 | static void msm_iommu_domain_free(struct iommu_domain *domain) |
0720d1f0 SM |
342 | { |
343 | struct msm_priv *priv; | |
344 | unsigned long flags; | |
0720d1f0 SM |
345 | |
346 | spin_lock_irqsave(&msm_iommu_lock, flags); | |
3e116c3c | 347 | priv = to_msm_priv(domain); |
c9220fbd S |
348 | kfree(priv); |
349 | spin_unlock_irqrestore(&msm_iommu_lock, flags); | |
350 | } | |
0720d1f0 | 351 | |
c9220fbd S |
352 | static int msm_iommu_domain_config(struct msm_priv *priv) |
353 | { | |
354 | spin_lock_init(&priv->pgtlock); | |
0720d1f0 | 355 | |
c9220fbd S |
356 | priv->cfg = (struct io_pgtable_cfg) { |
357 | .quirks = IO_PGTABLE_QUIRK_TLBI_ON_MAP, | |
358 | .pgsize_bitmap = msm_iommu_ops.pgsize_bitmap, | |
359 | .ias = 32, | |
360 | .oas = 32, | |
361 | .tlb = &msm_iommu_gather_ops, | |
362 | .iommu_dev = priv->dev, | |
363 | }; | |
0720d1f0 | 364 | |
c9220fbd S |
365 | priv->iop = alloc_io_pgtable_ops(ARM_V7S, &priv->cfg, priv); |
366 | if (!priv->iop) { | |
367 | dev_err(priv->dev, "Failed to allocate pgtable\n"); | |
368 | return -EINVAL; | |
369 | } | |
0720d1f0 | 370 | |
c9220fbd S |
371 | msm_iommu_ops.pgsize_bitmap = priv->cfg.pgsize_bitmap; |
372 | ||
373 | return 0; | |
0720d1f0 SM |
374 | } |
375 | ||
42df43b3 JR |
376 | /* Must be called under msm_iommu_lock */ |
377 | static struct msm_iommu_dev *find_iommu_for_dev(struct device *dev) | |
378 | { | |
379 | struct msm_iommu_dev *iommu, *ret = NULL; | |
380 | struct msm_iommu_ctx_dev *master; | |
381 | ||
382 | list_for_each_entry(iommu, &qcom_iommu_devices, dev_node) { | |
383 | master = list_first_entry(&iommu->ctx_list, | |
384 | struct msm_iommu_ctx_dev, | |
385 | list); | |
386 | if (master->of_node == dev->of_node) { | |
387 | ret = iommu; | |
388 | break; | |
389 | } | |
390 | } | |
391 | ||
392 | return ret; | |
393 | } | |
394 | ||
395 | static int msm_iommu_add_device(struct device *dev) | |
396 | { | |
397 | struct msm_iommu_dev *iommu; | |
ce2eb8f4 | 398 | struct iommu_group *group; |
42df43b3 | 399 | unsigned long flags; |
42df43b3 JR |
400 | |
401 | spin_lock_irqsave(&msm_iommu_lock, flags); | |
42df43b3 | 402 | iommu = find_iommu_for_dev(dev); |
37952146 NC |
403 | spin_unlock_irqrestore(&msm_iommu_lock, flags); |
404 | ||
42df43b3 JR |
405 | if (iommu) |
406 | iommu_device_link(&iommu->iommu, dev); | |
407 | else | |
37952146 | 408 | return -ENODEV; |
ce2eb8f4 RM |
409 | |
410 | group = iommu_group_get_for_dev(dev); | |
411 | if (IS_ERR(group)) | |
412 | return PTR_ERR(group); | |
413 | ||
414 | iommu_group_put(group); | |
415 | ||
416 | return 0; | |
42df43b3 JR |
417 | } |
418 | ||
419 | static void msm_iommu_remove_device(struct device *dev) | |
420 | { | |
421 | struct msm_iommu_dev *iommu; | |
422 | unsigned long flags; | |
423 | ||
424 | spin_lock_irqsave(&msm_iommu_lock, flags); | |
42df43b3 | 425 | iommu = find_iommu_for_dev(dev); |
37952146 NC |
426 | spin_unlock_irqrestore(&msm_iommu_lock, flags); |
427 | ||
42df43b3 JR |
428 | if (iommu) |
429 | iommu_device_unlink(&iommu->iommu, dev); | |
430 | ||
ce2eb8f4 | 431 | iommu_group_remove_device(dev); |
42df43b3 JR |
432 | } |
433 | ||
0720d1f0 SM |
434 | static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) |
435 | { | |
0720d1f0 SM |
436 | int ret = 0; |
437 | unsigned long flags; | |
109bd48e S |
438 | struct msm_iommu_dev *iommu; |
439 | struct msm_priv *priv = to_msm_priv(domain); | |
440 | struct msm_iommu_ctx_dev *master; | |
0720d1f0 | 441 | |
c9220fbd S |
442 | priv->dev = dev; |
443 | msm_iommu_domain_config(priv); | |
444 | ||
0720d1f0 | 445 | spin_lock_irqsave(&msm_iommu_lock, flags); |
109bd48e S |
446 | list_for_each_entry(iommu, &qcom_iommu_devices, dev_node) { |
447 | master = list_first_entry(&iommu->ctx_list, | |
448 | struct msm_iommu_ctx_dev, | |
449 | list); | |
450 | if (master->of_node == dev->of_node) { | |
451 | ret = __enable_clocks(iommu); | |
452 | if (ret) | |
453 | goto fail; | |
454 | ||
455 | list_for_each_entry(master, &iommu->ctx_list, list) { | |
456 | if (master->num) { | |
457 | dev_err(dev, "domain already attached"); | |
458 | ret = -EEXIST; | |
459 | goto fail; | |
460 | } | |
461 | master->num = | |
462 | msm_iommu_alloc_ctx(iommu->context_map, | |
463 | 0, iommu->ncb); | |
ba93c357 JL |
464 | if (IS_ERR_VALUE(master->num)) { |
465 | ret = -ENODEV; | |
466 | goto fail; | |
467 | } | |
109bd48e S |
468 | config_mids(iommu, master); |
469 | __program_context(iommu->base, master->num, | |
c9220fbd | 470 | priv); |
109bd48e S |
471 | } |
472 | __disable_clocks(iommu); | |
473 | list_add(&iommu->dom_node, &priv->list_attached); | |
0720d1f0 | 474 | } |
109bd48e | 475 | } |
0720d1f0 | 476 | |
0720d1f0 SM |
477 | fail: |
478 | spin_unlock_irqrestore(&msm_iommu_lock, flags); | |
109bd48e | 479 | |
0720d1f0 SM |
480 | return ret; |
481 | } | |
482 | ||
483 | static void msm_iommu_detach_dev(struct iommu_domain *domain, | |
484 | struct device *dev) | |
485 | { | |
109bd48e | 486 | struct msm_priv *priv = to_msm_priv(domain); |
0720d1f0 | 487 | unsigned long flags; |
109bd48e S |
488 | struct msm_iommu_dev *iommu; |
489 | struct msm_iommu_ctx_dev *master; | |
33069739 | 490 | int ret; |
0720d1f0 | 491 | |
c9220fbd | 492 | free_io_pgtable_ops(priv->iop); |
33069739 | 493 | |
c9220fbd | 494 | spin_lock_irqsave(&msm_iommu_lock, flags); |
109bd48e S |
495 | list_for_each_entry(iommu, &priv->list_attached, dom_node) { |
496 | ret = __enable_clocks(iommu); | |
497 | if (ret) | |
498 | goto fail; | |
0720d1f0 | 499 | |
109bd48e S |
500 | list_for_each_entry(master, &iommu->ctx_list, list) { |
501 | msm_iommu_free_ctx(iommu->context_map, master->num); | |
502 | __reset_context(iommu->base, master->num); | |
503 | } | |
504 | __disable_clocks(iommu); | |
505 | } | |
0720d1f0 SM |
506 | fail: |
507 | spin_unlock_irqrestore(&msm_iommu_lock, flags); | |
508 | } | |
509 | ||
c9220fbd | 510 | static int msm_iommu_map(struct iommu_domain *domain, unsigned long iova, |
5009065d | 511 | phys_addr_t pa, size_t len, int prot) |
0720d1f0 | 512 | { |
c9220fbd | 513 | struct msm_priv *priv = to_msm_priv(domain); |
0720d1f0 | 514 | unsigned long flags; |
c9220fbd | 515 | int ret; |
0720d1f0 | 516 | |
c9220fbd S |
517 | spin_lock_irqsave(&priv->pgtlock, flags); |
518 | ret = priv->iop->map(priv->iop, iova, pa, len, prot); | |
519 | spin_unlock_irqrestore(&priv->pgtlock, flags); | |
0720d1f0 | 520 | |
0720d1f0 SM |
521 | return ret; |
522 | } | |
523 | ||
c9220fbd S |
524 | static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long iova, |
525 | size_t len) | |
0720d1f0 | 526 | { |
c9220fbd | 527 | struct msm_priv *priv = to_msm_priv(domain); |
0720d1f0 | 528 | unsigned long flags; |
0720d1f0 | 529 | |
c9220fbd S |
530 | spin_lock_irqsave(&priv->pgtlock, flags); |
531 | len = priv->iop->unmap(priv->iop, iova, len); | |
532 | spin_unlock_irqrestore(&priv->pgtlock, flags); | |
0720d1f0 | 533 | |
5009065d | 534 | return len; |
0720d1f0 SM |
535 | } |
536 | ||
537 | static phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain, | |
bb5547ac | 538 | dma_addr_t va) |
0720d1f0 SM |
539 | { |
540 | struct msm_priv *priv; | |
109bd48e S |
541 | struct msm_iommu_dev *iommu; |
542 | struct msm_iommu_ctx_dev *master; | |
0720d1f0 SM |
543 | unsigned int par; |
544 | unsigned long flags; | |
0720d1f0 | 545 | phys_addr_t ret = 0; |
0720d1f0 SM |
546 | |
547 | spin_lock_irqsave(&msm_iommu_lock, flags); | |
548 | ||
3e116c3c | 549 | priv = to_msm_priv(domain); |
109bd48e S |
550 | iommu = list_first_entry(&priv->list_attached, |
551 | struct msm_iommu_dev, dom_node); | |
0720d1f0 | 552 | |
109bd48e S |
553 | if (list_empty(&iommu->ctx_list)) |
554 | goto fail; | |
0720d1f0 | 555 | |
109bd48e S |
556 | master = list_first_entry(&iommu->ctx_list, |
557 | struct msm_iommu_ctx_dev, list); | |
558 | if (!master) | |
559 | goto fail; | |
0720d1f0 | 560 | |
109bd48e | 561 | ret = __enable_clocks(iommu); |
41f3f513 SM |
562 | if (ret) |
563 | goto fail; | |
564 | ||
0720d1f0 | 565 | /* Invalidate context TLB */ |
109bd48e S |
566 | SET_CTX_TLBIALL(iommu->base, master->num, 0); |
567 | SET_V2PPR(iommu->base, master->num, va & V2Pxx_VA); | |
0720d1f0 | 568 | |
109bd48e | 569 | par = GET_PAR(iommu->base, master->num); |
0720d1f0 SM |
570 | |
571 | /* We are dealing with a supersection */ | |
109bd48e | 572 | if (GET_NOFAULT_SS(iommu->base, master->num)) |
0720d1f0 SM |
573 | ret = (par & 0xFF000000) | (va & 0x00FFFFFF); |
574 | else /* Upper 20 bits from PAR, lower 12 from VA */ | |
575 | ret = (par & 0xFFFFF000) | (va & 0x00000FFF); | |
576 | ||
109bd48e | 577 | if (GET_FAULT(iommu->base, master->num)) |
33069739 SM |
578 | ret = 0; |
579 | ||
109bd48e | 580 | __disable_clocks(iommu); |
0720d1f0 SM |
581 | fail: |
582 | spin_unlock_irqrestore(&msm_iommu_lock, flags); | |
583 | return ret; | |
584 | } | |
585 | ||
4480845e | 586 | static bool msm_iommu_capable(enum iommu_cap cap) |
0720d1f0 | 587 | { |
4480845e | 588 | return false; |
0720d1f0 SM |
589 | } |
590 | ||
591 | static void print_ctx_regs(void __iomem *base, int ctx) | |
592 | { | |
593 | unsigned int fsr = GET_FSR(base, ctx); | |
594 | pr_err("FAR = %08x PAR = %08x\n", | |
595 | GET_FAR(base, ctx), GET_PAR(base, ctx)); | |
596 | pr_err("FSR = %08x [%s%s%s%s%s%s%s%s%s%s]\n", fsr, | |
597 | (fsr & 0x02) ? "TF " : "", | |
598 | (fsr & 0x04) ? "AFF " : "", | |
599 | (fsr & 0x08) ? "APF " : "", | |
600 | (fsr & 0x10) ? "TLBMF " : "", | |
601 | (fsr & 0x20) ? "HTWDEEF " : "", | |
602 | (fsr & 0x40) ? "HTWSEEF " : "", | |
603 | (fsr & 0x80) ? "MHF " : "", | |
604 | (fsr & 0x10000) ? "SL " : "", | |
605 | (fsr & 0x40000000) ? "SS " : "", | |
606 | (fsr & 0x80000000) ? "MULTI " : ""); | |
607 | ||
608 | pr_err("FSYNR0 = %08x FSYNR1 = %08x\n", | |
609 | GET_FSYNR0(base, ctx), GET_FSYNR1(base, ctx)); | |
610 | pr_err("TTBR0 = %08x TTBR1 = %08x\n", | |
611 | GET_TTBR0(base, ctx), GET_TTBR1(base, ctx)); | |
612 | pr_err("SCTLR = %08x ACTLR = %08x\n", | |
613 | GET_SCTLR(base, ctx), GET_ACTLR(base, ctx)); | |
0720d1f0 SM |
614 | } |
615 | ||
f78ebca8 S |
616 | static void insert_iommu_master(struct device *dev, |
617 | struct msm_iommu_dev **iommu, | |
618 | struct of_phandle_args *spec) | |
619 | { | |
620 | struct msm_iommu_ctx_dev *master = dev->archdata.iommu; | |
621 | int sid; | |
622 | ||
623 | if (list_empty(&(*iommu)->ctx_list)) { | |
624 | master = kzalloc(sizeof(*master), GFP_ATOMIC); | |
625 | master->of_node = dev->of_node; | |
626 | list_add(&master->list, &(*iommu)->ctx_list); | |
627 | dev->archdata.iommu = master; | |
628 | } | |
629 | ||
630 | for (sid = 0; sid < master->num_mids; sid++) | |
631 | if (master->mids[sid] == spec->args[0]) { | |
632 | dev_warn(dev, "Stream ID 0x%hx repeated; ignoring\n", | |
633 | sid); | |
634 | return; | |
635 | } | |
636 | ||
637 | master->mids[master->num_mids++] = spec->args[0]; | |
638 | } | |
639 | ||
640 | static int qcom_iommu_of_xlate(struct device *dev, | |
641 | struct of_phandle_args *spec) | |
642 | { | |
643 | struct msm_iommu_dev *iommu; | |
644 | unsigned long flags; | |
645 | int ret = 0; | |
646 | ||
647 | spin_lock_irqsave(&msm_iommu_lock, flags); | |
648 | list_for_each_entry(iommu, &qcom_iommu_devices, dev_node) | |
649 | if (iommu->dev->of_node == spec->np) | |
650 | break; | |
651 | ||
652 | if (!iommu || iommu->dev->of_node != spec->np) { | |
653 | ret = -ENODEV; | |
654 | goto fail; | |
655 | } | |
656 | ||
657 | insert_iommu_master(dev, &iommu, spec); | |
658 | fail: | |
659 | spin_unlock_irqrestore(&msm_iommu_lock, flags); | |
660 | ||
661 | return ret; | |
662 | } | |
663 | ||
0720d1f0 SM |
664 | irqreturn_t msm_iommu_fault_handler(int irq, void *dev_id) |
665 | { | |
109bd48e | 666 | struct msm_iommu_dev *iommu = dev_id; |
33069739 | 667 | unsigned int fsr; |
a43d8c10 | 668 | int i, ret; |
0720d1f0 SM |
669 | |
670 | spin_lock(&msm_iommu_lock); | |
671 | ||
109bd48e | 672 | if (!iommu) { |
0720d1f0 SM |
673 | pr_err("Invalid device ID in context interrupt handler\n"); |
674 | goto fail; | |
675 | } | |
676 | ||
0720d1f0 | 677 | pr_err("Unexpected IOMMU page fault!\n"); |
109bd48e | 678 | pr_err("base = %08x\n", (unsigned int)iommu->base); |
0720d1f0 | 679 | |
109bd48e | 680 | ret = __enable_clocks(iommu); |
41f3f513 SM |
681 | if (ret) |
682 | goto fail; | |
683 | ||
109bd48e S |
684 | for (i = 0; i < iommu->ncb; i++) { |
685 | fsr = GET_FSR(iommu->base, i); | |
0720d1f0 SM |
686 | if (fsr) { |
687 | pr_err("Fault occurred in context %d.\n", i); | |
688 | pr_err("Interesting registers:\n"); | |
109bd48e S |
689 | print_ctx_regs(iommu->base, i); |
690 | SET_FSR(iommu->base, i, 0x4000000F); | |
0720d1f0 SM |
691 | } |
692 | } | |
109bd48e | 693 | __disable_clocks(iommu); |
0720d1f0 SM |
694 | fail: |
695 | spin_unlock(&msm_iommu_lock); | |
696 | return 0; | |
697 | } | |
698 | ||
f78ebca8 | 699 | static struct iommu_ops msm_iommu_ops = { |
4480845e | 700 | .capable = msm_iommu_capable, |
3e116c3c JR |
701 | .domain_alloc = msm_iommu_domain_alloc, |
702 | .domain_free = msm_iommu_domain_free, | |
0720d1f0 SM |
703 | .attach_dev = msm_iommu_attach_dev, |
704 | .detach_dev = msm_iommu_detach_dev, | |
705 | .map = msm_iommu_map, | |
706 | .unmap = msm_iommu_unmap, | |
707 | .iova_to_phys = msm_iommu_iova_to_phys, | |
42df43b3 JR |
708 | .add_device = msm_iommu_add_device, |
709 | .remove_device = msm_iommu_remove_device, | |
ce2eb8f4 | 710 | .device_group = generic_device_group, |
83427275 | 711 | .pgsize_bitmap = MSM_IOMMU_PGSIZES, |
f78ebca8 | 712 | .of_xlate = qcom_iommu_of_xlate, |
0720d1f0 SM |
713 | }; |
714 | ||
f7f125ef S |
715 | static int msm_iommu_probe(struct platform_device *pdev) |
716 | { | |
717 | struct resource *r; | |
42df43b3 | 718 | resource_size_t ioaddr; |
f7f125ef S |
719 | struct msm_iommu_dev *iommu; |
720 | int ret, par, val; | |
721 | ||
722 | iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL); | |
723 | if (!iommu) | |
724 | return -ENODEV; | |
725 | ||
726 | iommu->dev = &pdev->dev; | |
727 | INIT_LIST_HEAD(&iommu->ctx_list); | |
728 | ||
729 | iommu->pclk = devm_clk_get(iommu->dev, "smmu_pclk"); | |
730 | if (IS_ERR(iommu->pclk)) { | |
731 | dev_err(iommu->dev, "could not get smmu_pclk\n"); | |
732 | return PTR_ERR(iommu->pclk); | |
733 | } | |
734 | ||
735 | ret = clk_prepare(iommu->pclk); | |
736 | if (ret) { | |
737 | dev_err(iommu->dev, "could not prepare smmu_pclk\n"); | |
738 | return ret; | |
739 | } | |
740 | ||
741 | iommu->clk = devm_clk_get(iommu->dev, "iommu_clk"); | |
742 | if (IS_ERR(iommu->clk)) { | |
743 | dev_err(iommu->dev, "could not get iommu_clk\n"); | |
744 | clk_unprepare(iommu->pclk); | |
745 | return PTR_ERR(iommu->clk); | |
746 | } | |
747 | ||
748 | ret = clk_prepare(iommu->clk); | |
749 | if (ret) { | |
750 | dev_err(iommu->dev, "could not prepare iommu_clk\n"); | |
751 | clk_unprepare(iommu->pclk); | |
752 | return ret; | |
753 | } | |
754 | ||
755 | r = platform_get_resource(pdev, IORESOURCE_MEM, 0); | |
756 | iommu->base = devm_ioremap_resource(iommu->dev, r); | |
757 | if (IS_ERR(iommu->base)) { | |
758 | dev_err(iommu->dev, "could not get iommu base\n"); | |
759 | ret = PTR_ERR(iommu->base); | |
760 | goto fail; | |
761 | } | |
42df43b3 | 762 | ioaddr = r->start; |
f7f125ef S |
763 | |
764 | iommu->irq = platform_get_irq(pdev, 0); | |
765 | if (iommu->irq < 0) { | |
766 | dev_err(iommu->dev, "could not get iommu irq\n"); | |
767 | ret = -ENODEV; | |
768 | goto fail; | |
769 | } | |
770 | ||
771 | ret = of_property_read_u32(iommu->dev->of_node, "qcom,ncb", &val); | |
772 | if (ret) { | |
773 | dev_err(iommu->dev, "could not get ncb\n"); | |
774 | goto fail; | |
775 | } | |
776 | iommu->ncb = val; | |
777 | ||
778 | msm_iommu_reset(iommu->base, iommu->ncb); | |
779 | SET_M(iommu->base, 0, 1); | |
780 | SET_PAR(iommu->base, 0, 0); | |
781 | SET_V2PCFG(iommu->base, 0, 1); | |
782 | SET_V2PPR(iommu->base, 0, 0); | |
783 | par = GET_PAR(iommu->base, 0); | |
784 | SET_V2PCFG(iommu->base, 0, 0); | |
785 | SET_M(iommu->base, 0, 0); | |
786 | ||
787 | if (!par) { | |
788 | pr_err("Invalid PAR value detected\n"); | |
789 | ret = -ENODEV; | |
790 | goto fail; | |
791 | } | |
792 | ||
793 | ret = devm_request_threaded_irq(iommu->dev, iommu->irq, NULL, | |
794 | msm_iommu_fault_handler, | |
795 | IRQF_ONESHOT | IRQF_SHARED, | |
796 | "msm_iommu_secure_irpt_handler", | |
797 | iommu); | |
798 | if (ret) { | |
799 | pr_err("Request IRQ %d failed with ret=%d\n", iommu->irq, ret); | |
800 | goto fail; | |
801 | } | |
802 | ||
803 | list_add(&iommu->dev_node, &qcom_iommu_devices); | |
42df43b3 JR |
804 | |
805 | ret = iommu_device_sysfs_add(&iommu->iommu, iommu->dev, NULL, | |
806 | "msm-smmu.%pa", &ioaddr); | |
807 | if (ret) { | |
808 | pr_err("Could not add msm-smmu at %pa to sysfs\n", &ioaddr); | |
809 | goto fail; | |
810 | } | |
811 | ||
812 | iommu_device_set_ops(&iommu->iommu, &msm_iommu_ops); | |
813 | iommu_device_set_fwnode(&iommu->iommu, &pdev->dev.of_node->fwnode); | |
814 | ||
815 | ret = iommu_device_register(&iommu->iommu); | |
816 | if (ret) { | |
817 | pr_err("Could not register msm-smmu at %pa\n", &ioaddr); | |
818 | goto fail; | |
819 | } | |
820 | ||
892d7aad RM |
821 | bus_set_iommu(&platform_bus_type, &msm_iommu_ops); |
822 | ||
f7f125ef S |
823 | pr_info("device mapped at %p, irq %d with %d ctx banks\n", |
824 | iommu->base, iommu->irq, iommu->ncb); | |
825 | ||
826 | return ret; | |
827 | fail: | |
828 | clk_unprepare(iommu->clk); | |
829 | clk_unprepare(iommu->pclk); | |
830 | return ret; | |
831 | } | |
832 | ||
833 | static const struct of_device_id msm_iommu_dt_match[] = { | |
834 | { .compatible = "qcom,apq8064-iommu" }, | |
835 | {} | |
836 | }; | |
837 | ||
838 | static int msm_iommu_remove(struct platform_device *pdev) | |
839 | { | |
840 | struct msm_iommu_dev *iommu = platform_get_drvdata(pdev); | |
841 | ||
842 | clk_unprepare(iommu->clk); | |
843 | clk_unprepare(iommu->pclk); | |
844 | return 0; | |
845 | } | |
846 | ||
847 | static struct platform_driver msm_iommu_driver = { | |
848 | .driver = { | |
849 | .name = "msm_iommu", | |
850 | .of_match_table = msm_iommu_dt_match, | |
851 | }, | |
852 | .probe = msm_iommu_probe, | |
853 | .remove = msm_iommu_remove, | |
854 | }; | |
855 | ||
856 | static int __init msm_iommu_driver_init(void) | |
857 | { | |
858 | int ret; | |
859 | ||
860 | ret = platform_driver_register(&msm_iommu_driver); | |
861 | if (ret != 0) | |
862 | pr_err("Failed to register IOMMU driver\n"); | |
863 | ||
864 | return ret; | |
865 | } | |
f7f125ef | 866 | subsys_initcall(msm_iommu_driver_init); |
f7f125ef | 867 |