]>
Commit | Line | Data |
---|---|---|
f2efa4ee | 1 | // SPDX-License-Identifier: GPL-2.0 |
fad9119b | 2 | // Copyright (C) 2013 - 2021 Intel Corporation |
f2efa4ee WY |
3 | |
4 | #include <asm/cacheflush.h> | |
5 | ||
6 | #include <linux/device.h> | |
7 | #include <linux/iova.h> | |
8 | #include <linux/module.h> | |
9 | #include <linux/sizes.h> | |
10 | ||
11 | #include "ipu.h" | |
12 | #include "ipu-platform.h" | |
13 | #include "ipu-dma.h" | |
14 | #include "ipu-mmu.h" | |
15 | #include "ipu-platform-regs.h" | |
16 | ||
17 | #define ISP_PAGE_SHIFT 12 | |
18 | #define ISP_PAGE_SIZE BIT(ISP_PAGE_SHIFT) | |
19 | #define ISP_PAGE_MASK (~(ISP_PAGE_SIZE - 1)) | |
20 | ||
21 | #define ISP_L1PT_SHIFT 22 | |
22 | #define ISP_L1PT_MASK (~((1U << ISP_L1PT_SHIFT) - 1)) | |
23 | ||
24 | #define ISP_L2PT_SHIFT 12 | |
25 | #define ISP_L2PT_MASK (~(ISP_L1PT_MASK | (~(ISP_PAGE_MASK)))) | |
26 | ||
27 | #define ISP_L1PT_PTES 1024 | |
28 | #define ISP_L2PT_PTES 1024 | |
29 | ||
30 | #define ISP_PADDR_SHIFT 12 | |
31 | ||
32 | #define REG_TLB_INVALIDATE 0x0000 | |
33 | ||
34 | #define REG_L1_PHYS 0x0004 /* 27-bit pfn */ | |
35 | #define REG_INFO 0x0008 | |
36 | ||
37 | /* The range of stream ID i in L1 cache is from 0 to 15 */ | |
38 | #define MMUV2_REG_L1_STREAMID(i) (0x0c + ((i) * 4)) | |
39 | ||
40 | /* The range of stream ID i in L2 cache is from 0 to 15 */ | |
41 | #define MMUV2_REG_L2_STREAMID(i) (0x4c + ((i) * 4)) | |
42 | ||
f2efa4ee | 43 | #define TBL_PHYS_ADDR(a) ((phys_addr_t)(a) << ISP_PADDR_SHIFT) |
f2efa4ee WY |
44 | |
45 | static void tlb_invalidate(struct ipu_mmu *mmu) | |
46 | { | |
47 | unsigned int i; | |
48 | unsigned long flags; | |
49 | ||
50 | spin_lock_irqsave(&mmu->ready_lock, flags); | |
51 | if (!mmu->ready) { | |
52 | spin_unlock_irqrestore(&mmu->ready_lock, flags); | |
53 | return; | |
54 | } | |
55 | ||
56 | for (i = 0; i < mmu->nr_mmus; i++) { | |
57 | /* | |
eaffc3a7 | 58 | * To avoid the HW bug induced dead lock in some of the IPU |
f2efa4ee WY |
59 | * MMUs on successive invalidate calls, we need to first do a |
60 | * read to the page table base before writing the invalidate | |
61 | * register. MMUs which need to implement this WA, will have | |
fad9119b | 62 | * the insert_read_before_invalidate flags set as true. |
f2efa4ee WY |
63 | * Disregard the return value of the read. |
64 | */ | |
65 | if (mmu->mmu_hw[i].insert_read_before_invalidate) | |
66 | readl(mmu->mmu_hw[i].base + REG_L1_PHYS); | |
67 | ||
fad9119b WY |
68 | writel(0xffffffff, mmu->mmu_hw[i].base + |
69 | REG_TLB_INVALIDATE); | |
70 | /* | |
71 | * The TLB invalidation is a "single cycle" (IOMMU clock cycles) | |
72 | * When the actual MMIO write reaches the IPU TLB Invalidate | |
73 | * register, wmb() will force the TLB invalidate out if the CPU | |
74 | * attempts to update the IOMMU page table (or sooner). | |
75 | */ | |
76 | wmb(); | |
f2efa4ee WY |
77 | } |
78 | spin_unlock_irqrestore(&mmu->ready_lock, flags); | |
79 | } | |
80 | ||
81 | #ifdef DEBUG | |
82 | static void page_table_dump(struct ipu_mmu_info *mmu_info) | |
83 | { | |
84 | u32 l1_idx; | |
85 | ||
fad9119b | 86 | dev_dbg(mmu_info->dev, "begin IOMMU page table dump\n"); |
f2efa4ee WY |
87 | |
88 | for (l1_idx = 0; l1_idx < ISP_L1PT_PTES; l1_idx++) { | |
89 | u32 l2_idx; | |
90 | u32 iova = (phys_addr_t)l1_idx << ISP_L1PT_SHIFT; | |
91 | ||
fad9119b | 92 | if (mmu_info->l1_pt[l1_idx] == mmu_info->dummy_l2_pteval) |
f2efa4ee | 93 | continue; |
fad9119b WY |
94 | dev_dbg(mmu_info->dev, |
95 | "l1 entry %u; iovas 0x%8.8x-0x%8.8x, at %p\n", | |
96 | l1_idx, iova, iova + ISP_PAGE_SIZE, | |
97 | (void *)TBL_PHYS_ADDR(mmu_info->l1_pt[l1_idx])); | |
f2efa4ee WY |
98 | |
99 | for (l2_idx = 0; l2_idx < ISP_L2PT_PTES; l2_idx++) { | |
fad9119b | 100 | u32 *l2_pt = mmu_info->l2_pts[l1_idx]; |
f2efa4ee WY |
101 | u32 iova2 = iova + (l2_idx << ISP_L2PT_SHIFT); |
102 | ||
fad9119b | 103 | if (l2_pt[l2_idx] == mmu_info->dummy_page_pteval) |
f2efa4ee WY |
104 | continue; |
105 | ||
fad9119b WY |
106 | dev_dbg(mmu_info->dev, |
107 | "\tl2 entry %u; iova 0x%8.8x, phys %p\n", | |
108 | l2_idx, iova2, | |
109 | (void *)TBL_PHYS_ADDR(l2_pt[l2_idx])); | |
f2efa4ee WY |
110 | } |
111 | } | |
112 | ||
fad9119b | 113 | dev_dbg(mmu_info->dev, "end IOMMU page table dump\n"); |
f2efa4ee WY |
114 | } |
115 | #endif /* DEBUG */ | |
116 | ||
fad9119b WY |
117 | static dma_addr_t map_single(struct ipu_mmu_info *mmu_info, void *ptr) |
118 | { | |
119 | dma_addr_t dma; | |
120 | ||
121 | dma = dma_map_single(mmu_info->dev, ptr, PAGE_SIZE, DMA_BIDIRECTIONAL); | |
122 | if (dma_mapping_error(mmu_info->dev, dma)) | |
123 | return 0; | |
124 | ||
125 | return dma; | |
126 | } | |
127 | ||
128 | static int get_dummy_page(struct ipu_mmu_info *mmu_info) | |
129 | { | |
130 | dma_addr_t dma; | |
131 | void *pt = (void *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32); | |
132 | ||
133 | if (!pt) | |
134 | return -ENOMEM; | |
135 | ||
136 | dev_dbg(mmu_info->dev, "%s get_zeroed_page() == %p\n", __func__, pt); | |
137 | ||
138 | dma = map_single(mmu_info, pt); | |
139 | if (!dma) { | |
140 | dev_err(mmu_info->dev, "Failed to map dummy page\n"); | |
141 | goto err_free_page; | |
142 | } | |
143 | ||
144 | mmu_info->dummy_page = pt; | |
145 | mmu_info->dummy_page_pteval = dma >> ISP_PAGE_SHIFT; | |
146 | ||
147 | return 0; | |
148 | ||
149 | err_free_page: | |
150 | free_page((unsigned long)pt); | |
151 | return -ENOMEM; | |
152 | } | |
153 | ||
154 | static void free_dummy_page(struct ipu_mmu_info *mmu_info) | |
155 | { | |
156 | dma_unmap_single(mmu_info->dev, | |
157 | TBL_PHYS_ADDR(mmu_info->dummy_page_pteval), | |
158 | PAGE_SIZE, DMA_BIDIRECTIONAL); | |
159 | free_page((unsigned long)mmu_info->dummy_page); | |
160 | } | |
161 | ||
162 | static int alloc_dummy_l2_pt(struct ipu_mmu_info *mmu_info) | |
163 | { | |
164 | dma_addr_t dma; | |
165 | u32 *pt = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32); | |
166 | int i; | |
167 | ||
168 | if (!pt) | |
169 | return -ENOMEM; | |
170 | ||
171 | dev_dbg(mmu_info->dev, "%s get_zeroed_page() == %p\n", __func__, pt); | |
172 | ||
173 | dma = map_single(mmu_info, pt); | |
174 | if (!dma) { | |
175 | dev_err(mmu_info->dev, "Failed to map l2pt page\n"); | |
176 | goto err_free_page; | |
177 | } | |
178 | ||
179 | for (i = 0; i < ISP_L2PT_PTES; i++) | |
180 | pt[i] = mmu_info->dummy_page_pteval; | |
181 | ||
182 | mmu_info->dummy_l2_pt = pt; | |
183 | mmu_info->dummy_l2_pteval = dma >> ISP_PAGE_SHIFT; | |
184 | ||
185 | return 0; | |
186 | ||
187 | err_free_page: | |
188 | free_page((unsigned long)pt); | |
189 | return -ENOMEM; | |
190 | } | |
191 | ||
192 | static void free_dummy_l2_pt(struct ipu_mmu_info *mmu_info) | |
193 | { | |
194 | dma_unmap_single(mmu_info->dev, | |
195 | TBL_PHYS_ADDR(mmu_info->dummy_l2_pteval), | |
196 | PAGE_SIZE, DMA_BIDIRECTIONAL); | |
197 | free_page((unsigned long)mmu_info->dummy_l2_pt); | |
198 | } | |
199 | ||
200 | static u32 *alloc_l1_pt(struct ipu_mmu_info *mmu_info) | |
f2efa4ee | 201 | { |
fad9119b | 202 | dma_addr_t dma; |
f2efa4ee WY |
203 | u32 *pt = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32); |
204 | int i; | |
205 | ||
206 | if (!pt) | |
207 | return NULL; | |
208 | ||
fad9119b | 209 | dev_dbg(mmu_info->dev, "%s get_zeroed_page() == %p\n", __func__, pt); |
f2efa4ee WY |
210 | |
211 | for (i = 0; i < ISP_L1PT_PTES; i++) | |
fad9119b WY |
212 | pt[i] = mmu_info->dummy_l2_pteval; |
213 | ||
214 | dma = map_single(mmu_info, pt); | |
215 | if (!dma) { | |
216 | dev_err(mmu_info->dev, "Failed to map l1pt page\n"); | |
217 | goto err_free_page; | |
218 | } | |
219 | ||
220 | mmu_info->l1_pt_dma = dma >> ISP_PADDR_SHIFT; | |
221 | dev_dbg(mmu_info->dev, "l1 pt %p mapped at %llx\n", pt, dma); | |
222 | ||
223 | return pt; | |
224 | ||
225 | err_free_page: | |
226 | free_page((unsigned long)pt); | |
227 | return NULL; | |
228 | } | |
229 | ||
230 | static u32 *alloc_l2_pt(struct ipu_mmu_info *mmu_info) | |
231 | { | |
232 | u32 *pt = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32); | |
233 | int i; | |
234 | ||
235 | if (!pt) | |
236 | return NULL; | |
237 | ||
238 | dev_dbg(mmu_info->dev, "%s get_zeroed_page() == %p\n", __func__, pt); | |
239 | ||
240 | for (i = 0; i < ISP_L1PT_PTES; i++) | |
241 | pt[i] = mmu_info->dummy_page_pteval; | |
f2efa4ee WY |
242 | |
243 | return pt; | |
244 | } | |
245 | ||
246 | static int l2_map(struct ipu_mmu_info *mmu_info, unsigned long iova, | |
247 | phys_addr_t paddr, size_t size) | |
248 | { | |
249 | u32 l1_idx = iova >> ISP_L1PT_SHIFT; | |
fad9119b WY |
250 | u32 l1_entry; |
251 | u32 *l2_pt, *l2_virt; | |
f2efa4ee WY |
252 | u32 iova_start = iova; |
253 | unsigned int l2_idx; | |
254 | unsigned long flags; | |
fad9119b | 255 | dma_addr_t dma; |
f2efa4ee | 256 | |
fad9119b WY |
257 | dev_dbg(mmu_info->dev, |
258 | "mapping l2 page table for l1 index %u (iova %8.8x)\n", | |
259 | l1_idx, (u32)iova); | |
f2efa4ee WY |
260 | |
261 | spin_lock_irqsave(&mmu_info->lock, flags); | |
fad9119b WY |
262 | l1_entry = mmu_info->l1_pt[l1_idx]; |
263 | if (l1_entry == mmu_info->dummy_l2_pteval) { | |
264 | l2_virt = mmu_info->l2_pts[l1_idx]; | |
265 | if (likely(!l2_virt)) { | |
266 | l2_virt = alloc_l2_pt(mmu_info); | |
267 | if (!l2_virt) { | |
268 | spin_unlock_irqrestore(&mmu_info->lock, flags); | |
269 | return -ENOMEM; | |
270 | } | |
271 | } | |
f2efa4ee | 272 | |
fad9119b WY |
273 | dma = map_single(mmu_info, l2_virt); |
274 | if (!dma) { | |
275 | dev_err(mmu_info->dev, "Failed to map l2pt page\n"); | |
276 | free_page((unsigned long)l2_virt); | |
f2efa4ee | 277 | spin_unlock_irqrestore(&mmu_info->lock, flags); |
fad9119b | 278 | return -EINVAL; |
f2efa4ee WY |
279 | } |
280 | ||
fad9119b | 281 | l1_entry = dma >> ISP_PADDR_SHIFT; |
f2efa4ee | 282 | |
fad9119b WY |
283 | dev_dbg(mmu_info->dev, "page for l1_idx %u %p allocated\n", |
284 | l1_idx, l2_virt); | |
285 | mmu_info->l1_pt[l1_idx] = l1_entry; | |
286 | mmu_info->l2_pts[l1_idx] = l2_virt; | |
287 | clflush_cache_range(&mmu_info->l1_pt[l1_idx], | |
288 | sizeof(mmu_info->l1_pt[l1_idx])); | |
f2efa4ee WY |
289 | } |
290 | ||
fad9119b | 291 | l2_pt = mmu_info->l2_pts[l1_idx]; |
f2efa4ee | 292 | |
fad9119b | 293 | dev_dbg(mmu_info->dev, "l2_pt at %p with dma 0x%x\n", l2_pt, l1_entry); |
f2efa4ee WY |
294 | |
295 | paddr = ALIGN(paddr, ISP_PAGE_SIZE); | |
296 | ||
297 | l2_idx = (iova_start & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT; | |
298 | ||
fad9119b WY |
299 | dev_dbg(mmu_info->dev, "l2_idx %u, phys 0x%8.8x\n", l2_idx, |
300 | l2_pt[l2_idx]); | |
301 | if (l2_pt[l2_idx] != mmu_info->dummy_page_pteval) { | |
f2efa4ee | 302 | spin_unlock_irqrestore(&mmu_info->lock, flags); |
fad9119b | 303 | return -EINVAL; |
f2efa4ee WY |
304 | } |
305 | ||
306 | l2_pt[l2_idx] = paddr >> ISP_PADDR_SHIFT; | |
307 | ||
f2efa4ee | 308 | clflush_cache_range(&l2_pt[l2_idx], sizeof(l2_pt[l2_idx])); |
f2efa4ee WY |
309 | spin_unlock_irqrestore(&mmu_info->lock, flags); |
310 | ||
fad9119b WY |
311 | dev_dbg(mmu_info->dev, "l2 index %u mapped as 0x%8.8x\n", l2_idx, |
312 | l2_pt[l2_idx]); | |
f2efa4ee WY |
313 | |
314 | return 0; | |
315 | } | |
316 | ||
317 | static int __ipu_mmu_map(struct ipu_mmu_info *mmu_info, unsigned long iova, | |
318 | phys_addr_t paddr, size_t size) | |
319 | { | |
320 | u32 iova_start = round_down(iova, ISP_PAGE_SIZE); | |
321 | u32 iova_end = ALIGN(iova + size, ISP_PAGE_SIZE); | |
322 | ||
fad9119b WY |
323 | dev_dbg(mmu_info->dev, |
324 | "mapping iova 0x%8.8x--0x%8.8x, size %zu at paddr 0x%10.10llx\n", | |
325 | iova_start, iova_end, size, paddr); | |
f2efa4ee WY |
326 | |
327 | return l2_map(mmu_info, iova_start, paddr, size); | |
328 | } | |
329 | ||
330 | static size_t l2_unmap(struct ipu_mmu_info *mmu_info, unsigned long iova, | |
331 | phys_addr_t dummy, size_t size) | |
332 | { | |
333 | u32 l1_idx = iova >> ISP_L1PT_SHIFT; | |
fad9119b | 334 | u32 *l2_pt; |
f2efa4ee WY |
335 | u32 iova_start = iova; |
336 | unsigned int l2_idx; | |
337 | size_t unmapped = 0; | |
fad9119b | 338 | unsigned long flags; |
f2efa4ee | 339 | |
fad9119b WY |
340 | dev_dbg(mmu_info->dev, "unmapping l2 page table for l1 index %u (iova 0x%8.8lx)\n", |
341 | l1_idx, iova); | |
f2efa4ee | 342 | |
fad9119b WY |
343 | spin_lock_irqsave(&mmu_info->lock, flags); |
344 | if (mmu_info->l1_pt[l1_idx] == mmu_info->dummy_l2_pteval) { | |
345 | spin_unlock_irqrestore(&mmu_info->lock, flags); | |
346 | dev_err(mmu_info->dev, | |
347 | "unmap iova 0x%8.8lx l1 idx %u which was not mapped\n", | |
348 | iova, l1_idx); | |
349 | return 0; | |
350 | } | |
f2efa4ee WY |
351 | |
352 | for (l2_idx = (iova_start & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT; | |
353 | (iova_start & ISP_L1PT_MASK) + (l2_idx << ISP_PAGE_SHIFT) | |
354 | < iova_start + size && l2_idx < ISP_L2PT_PTES; l2_idx++) { | |
fad9119b WY |
355 | l2_pt = mmu_info->l2_pts[l1_idx]; |
356 | dev_dbg(mmu_info->dev, | |
357 | "unmap l2 index %u with pteval 0x%10.10llx\n", | |
358 | l2_idx, TBL_PHYS_ADDR(l2_pt[l2_idx])); | |
359 | l2_pt[l2_idx] = mmu_info->dummy_page_pteval; | |
f2efa4ee | 360 | |
f2efa4ee | 361 | clflush_cache_range(&l2_pt[l2_idx], sizeof(l2_pt[l2_idx])); |
f2efa4ee WY |
362 | unmapped++; |
363 | } | |
fad9119b | 364 | spin_unlock_irqrestore(&mmu_info->lock, flags); |
f2efa4ee WY |
365 | |
366 | return unmapped << ISP_PAGE_SHIFT; | |
367 | } | |
368 | ||
369 | static size_t __ipu_mmu_unmap(struct ipu_mmu_info *mmu_info, | |
370 | unsigned long iova, size_t size) | |
371 | { | |
372 | return l2_unmap(mmu_info, iova, 0, size); | |
373 | } | |
374 | ||
375 | static int allocate_trash_buffer(struct ipu_mmu *mmu) | |
376 | { | |
377 | unsigned int n_pages = PAGE_ALIGN(IPU_MMUV2_TRASH_RANGE) >> PAGE_SHIFT; | |
378 | struct iova *iova; | |
379 | u32 iova_addr; | |
380 | unsigned int i; | |
fad9119b | 381 | dma_addr_t dma; |
f2efa4ee WY |
382 | int ret; |
383 | ||
384 | /* Allocate 8MB in iova range */ | |
385 | iova = alloc_iova(&mmu->dmap->iovad, n_pages, | |
386 | mmu->dmap->mmu_info->aperture_end >> PAGE_SHIFT, 0); | |
387 | if (!iova) { | |
388 | dev_err(mmu->dev, "cannot allocate iova range for trash\n"); | |
389 | return -ENOMEM; | |
390 | } | |
391 | ||
fad9119b WY |
392 | dma = dma_map_page(mmu->dmap->mmu_info->dev, mmu->trash_page, 0, |
393 | PAGE_SIZE, DMA_BIDIRECTIONAL); | |
394 | if (dma_mapping_error(mmu->dmap->mmu_info->dev, dma)) { | |
395 | dev_err(mmu->dmap->mmu_info->dev, "Failed to map trash page\n"); | |
396 | ret = -ENOMEM; | |
397 | goto out_free_iova; | |
398 | } | |
399 | ||
400 | mmu->pci_trash_page = dma; | |
401 | ||
f2efa4ee WY |
402 | /* |
403 | * Map the 8MB iova address range to the same physical trash page | |
404 | * mmu->trash_page which is already reserved at the probe | |
405 | */ | |
406 | iova_addr = iova->pfn_lo; | |
407 | for (i = 0; i < n_pages; i++) { | |
408 | ret = ipu_mmu_map(mmu->dmap->mmu_info, iova_addr << PAGE_SHIFT, | |
fad9119b | 409 | mmu->pci_trash_page, PAGE_SIZE); |
f2efa4ee WY |
410 | if (ret) { |
411 | dev_err(mmu->dev, | |
412 | "mapping trash buffer range failed\n"); | |
413 | goto out_unmap; | |
414 | } | |
415 | ||
416 | iova_addr++; | |
417 | } | |
418 | ||
fad9119b | 419 | mmu->iova_trash_page = iova->pfn_lo << PAGE_SHIFT; |
f2efa4ee | 420 | dev_dbg(mmu->dev, "iova trash buffer for MMUID: %d is %u\n", |
fad9119b | 421 | mmu->mmid, (unsigned int)mmu->iova_trash_page); |
f2efa4ee WY |
422 | return 0; |
423 | ||
424 | out_unmap: | |
425 | ipu_mmu_unmap(mmu->dmap->mmu_info, iova->pfn_lo << PAGE_SHIFT, | |
426 | (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT); | |
fad9119b WY |
427 | dma_unmap_page(mmu->dmap->mmu_info->dev, mmu->pci_trash_page, |
428 | PAGE_SIZE, DMA_BIDIRECTIONAL); | |
429 | out_free_iova: | |
f2efa4ee WY |
430 | __free_iova(&mmu->dmap->iovad, iova); |
431 | return ret; | |
432 | } | |
433 | ||
434 | int ipu_mmu_hw_init(struct ipu_mmu *mmu) | |
435 | { | |
436 | unsigned int i; | |
437 | unsigned long flags; | |
438 | struct ipu_mmu_info *mmu_info; | |
439 | ||
440 | dev_dbg(mmu->dev, "mmu hw init\n"); | |
441 | ||
442 | mmu_info = mmu->dmap->mmu_info; | |
443 | ||
444 | /* Initialise the each MMU HW block */ | |
445 | for (i = 0; i < mmu->nr_mmus; i++) { | |
446 | struct ipu_mmu_hw *mmu_hw = &mmu->mmu_hw[i]; | |
447 | unsigned int j; | |
448 | u16 block_addr; | |
449 | ||
450 | /* Write page table address per MMU */ | |
fad9119b WY |
451 | writel((phys_addr_t)mmu_info->l1_pt_dma, |
452 | mmu->mmu_hw[i].base + REG_L1_PHYS); | |
f2efa4ee WY |
453 | |
454 | /* Set info bits per MMU */ | |
455 | writel(mmu->mmu_hw[i].info_bits, | |
456 | mmu->mmu_hw[i].base + REG_INFO); | |
457 | ||
458 | /* Configure MMU TLB stream configuration for L1 */ | |
459 | for (j = 0, block_addr = 0; j < mmu_hw->nr_l1streams; | |
460 | block_addr += mmu->mmu_hw[i].l1_block_sz[j], j++) { | |
461 | if (block_addr > IPU_MAX_LI_BLOCK_ADDR) { | |
462 | dev_err(mmu->dev, "invalid L1 configuration\n"); | |
463 | return -EINVAL; | |
464 | } | |
465 | ||
466 | /* Write block start address for each streams */ | |
467 | writel(block_addr, mmu_hw->base + | |
468 | mmu_hw->l1_stream_id_reg_offset + 4 * j); | |
469 | } | |
470 | ||
471 | /* Configure MMU TLB stream configuration for L2 */ | |
472 | for (j = 0, block_addr = 0; j < mmu_hw->nr_l2streams; | |
473 | block_addr += mmu->mmu_hw[i].l2_block_sz[j], j++) { | |
474 | if (block_addr > IPU_MAX_L2_BLOCK_ADDR) { | |
475 | dev_err(mmu->dev, "invalid L2 configuration\n"); | |
476 | return -EINVAL; | |
477 | } | |
478 | ||
479 | writel(block_addr, mmu_hw->base + | |
480 | mmu_hw->l2_stream_id_reg_offset + 4 * j); | |
481 | } | |
482 | } | |
483 | ||
f2efa4ee | 484 | if (!mmu->trash_page) { |
fad9119b WY |
485 | int ret; |
486 | ||
f2efa4ee WY |
487 | mmu->trash_page = alloc_page(GFP_KERNEL); |
488 | if (!mmu->trash_page) { | |
489 | dev_err(mmu->dev, "insufficient memory for trash buffer\n"); | |
490 | return -ENOMEM; | |
491 | } | |
f2efa4ee WY |
492 | |
493 | ret = allocate_trash_buffer(mmu); | |
494 | if (ret) { | |
495 | __free_page(mmu->trash_page); | |
496 | mmu->trash_page = NULL; | |
497 | dev_err(mmu->dev, "trash buffer allocation failed\n"); | |
498 | return ret; | |
499 | } | |
500 | } | |
501 | ||
502 | spin_lock_irqsave(&mmu->ready_lock, flags); | |
503 | mmu->ready = true; | |
504 | spin_unlock_irqrestore(&mmu->ready_lock, flags); | |
505 | ||
506 | return 0; | |
507 | } | |
508 | EXPORT_SYMBOL(ipu_mmu_hw_init); | |
509 | ||
3ebd4441 | 510 | static struct ipu_mmu_info *ipu_mmu_alloc(struct ipu_device *isp) |
f2efa4ee WY |
511 | { |
512 | struct ipu_mmu_info *mmu_info; | |
fad9119b | 513 | int ret; |
f2efa4ee WY |
514 | |
515 | mmu_info = kzalloc(sizeof(*mmu_info), GFP_KERNEL); | |
516 | if (!mmu_info) | |
517 | return NULL; | |
518 | ||
519 | mmu_info->aperture_start = 0; | |
520 | mmu_info->aperture_end = DMA_BIT_MASK(isp->secure_mode ? | |
fad9119b WY |
521 | IPU_MMU_ADDRESS_BITS : |
522 | IPU_MMU_ADDRESS_BITS_NON_SECURE); | |
f2efa4ee | 523 | mmu_info->pgsize_bitmap = SZ_4K; |
fad9119b | 524 | mmu_info->dev = &isp->pdev->dev; |
f2efa4ee | 525 | |
fad9119b WY |
526 | ret = get_dummy_page(mmu_info); |
527 | if (ret) | |
528 | goto err_free_info; | |
f2efa4ee | 529 | |
fad9119b WY |
530 | ret = alloc_dummy_l2_pt(mmu_info); |
531 | if (ret) | |
532 | goto err_free_dummy_page; | |
f2efa4ee | 533 | |
fad9119b WY |
534 | mmu_info->l2_pts = vzalloc(ISP_L2PT_PTES * sizeof(*mmu_info->l2_pts)); |
535 | if (!mmu_info->l2_pts) | |
536 | goto err_free_dummy_l2_pt; | |
f2efa4ee WY |
537 | |
538 | /* | |
539 | * We always map the L1 page table (a single page as well as | |
540 | * the L2 page tables). | |
541 | */ | |
fad9119b WY |
542 | mmu_info->l1_pt = alloc_l1_pt(mmu_info); |
543 | if (!mmu_info->l1_pt) | |
544 | goto err_free_l2_pts; | |
f2efa4ee WY |
545 | |
546 | spin_lock_init(&mmu_info->lock); | |
547 | ||
fad9119b | 548 | dev_dbg(mmu_info->dev, "domain initialised\n"); |
f2efa4ee WY |
549 | |
550 | return mmu_info; | |
551 | ||
fad9119b WY |
552 | err_free_l2_pts: |
553 | vfree(mmu_info->l2_pts); | |
554 | err_free_dummy_l2_pt: | |
555 | free_dummy_l2_pt(mmu_info); | |
556 | err_free_dummy_page: | |
557 | free_dummy_page(mmu_info); | |
558 | err_free_info: | |
f2efa4ee WY |
559 | kfree(mmu_info); |
560 | ||
561 | return NULL; | |
562 | } | |
563 | ||
564 | int ipu_mmu_hw_cleanup(struct ipu_mmu *mmu) | |
565 | { | |
566 | unsigned long flags; | |
567 | ||
568 | spin_lock_irqsave(&mmu->ready_lock, flags); | |
569 | mmu->ready = false; | |
570 | spin_unlock_irqrestore(&mmu->ready_lock, flags); | |
571 | ||
572 | return 0; | |
573 | } | |
574 | EXPORT_SYMBOL(ipu_mmu_hw_cleanup); | |
575 | ||
576 | static struct ipu_dma_mapping *alloc_dma_mapping(struct ipu_device *isp) | |
577 | { | |
578 | struct ipu_dma_mapping *dmap; | |
579 | ||
580 | dmap = kzalloc(sizeof(*dmap), GFP_KERNEL); | |
581 | if (!dmap) | |
582 | return NULL; | |
583 | ||
584 | dmap->mmu_info = ipu_mmu_alloc(isp); | |
585 | if (!dmap->mmu_info) { | |
586 | kfree(dmap); | |
587 | return NULL; | |
588 | } | |
589 | init_iova_domain(&dmap->iovad, SZ_4K, 1); | |
590 | dmap->mmu_info->dmap = dmap; | |
591 | ||
592 | kref_init(&dmap->ref); | |
593 | ||
fad9119b | 594 | dev_dbg(&isp->pdev->dev, "alloc mapping\n"); |
f2efa4ee WY |
595 | |
596 | iova_cache_get(); | |
597 | ||
598 | return dmap; | |
599 | } | |
600 | ||
601 | phys_addr_t ipu_mmu_iova_to_phys(struct ipu_mmu_info *mmu_info, | |
602 | dma_addr_t iova) | |
603 | { | |
fad9119b WY |
604 | unsigned long flags; |
605 | u32 *l2_pt; | |
606 | phys_addr_t phy_addr; | |
f2efa4ee | 607 | |
fad9119b WY |
608 | spin_lock_irqsave(&mmu_info->lock, flags); |
609 | l2_pt = mmu_info->l2_pts[iova >> ISP_L1PT_SHIFT]; | |
610 | phy_addr = (phys_addr_t)l2_pt[(iova & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT]; | |
611 | phy_addr <<= ISP_PAGE_SHIFT; | |
612 | spin_unlock_irqrestore(&mmu_info->lock, flags); | |
613 | ||
614 | return phy_addr; | |
f2efa4ee WY |
615 | } |
616 | ||
3ebd4441 | 617 | /* |
f2efa4ee | 618 | * The following four functions are implemented based on iommu.c |
fad9119b | 619 | * drivers/iommu/iommu.c:iommu_pgsize(). |
f2efa4ee WY |
620 | */ |
621 | static size_t ipu_mmu_pgsize(unsigned long pgsize_bitmap, | |
622 | unsigned long addr_merge, size_t size) | |
623 | { | |
624 | unsigned int pgsize_idx; | |
625 | size_t pgsize; | |
626 | ||
627 | /* Max page size that still fits into 'size' */ | |
628 | pgsize_idx = __fls(size); | |
629 | ||
630 | /* need to consider alignment requirements ? */ | |
631 | if (likely(addr_merge)) { | |
632 | /* Max page size allowed by address */ | |
633 | unsigned int align_pgsize_idx = __ffs(addr_merge); | |
634 | ||
635 | pgsize_idx = min(pgsize_idx, align_pgsize_idx); | |
636 | } | |
637 | ||
638 | /* build a mask of acceptable page sizes */ | |
639 | pgsize = (1UL << (pgsize_idx + 1)) - 1; | |
640 | ||
641 | /* throw away page sizes not supported by the hardware */ | |
642 | pgsize &= pgsize_bitmap; | |
643 | ||
644 | /* make sure we're still sane */ | |
645 | WARN_ON(!pgsize); | |
646 | ||
647 | /* pick the biggest page */ | |
648 | pgsize_idx = __fls(pgsize); | |
649 | pgsize = 1UL << pgsize_idx; | |
650 | ||
651 | return pgsize; | |
652 | } | |
653 | ||
fad9119b | 654 | /* drivers/iommu/iommu.c:iommu_unmap() */ |
f2efa4ee WY |
655 | size_t ipu_mmu_unmap(struct ipu_mmu_info *mmu_info, unsigned long iova, |
656 | size_t size) | |
657 | { | |
658 | size_t unmapped_page, unmapped = 0; | |
659 | unsigned int min_pagesz; | |
660 | ||
661 | /* find out the minimum page size supported */ | |
662 | min_pagesz = 1 << __ffs(mmu_info->pgsize_bitmap); | |
663 | ||
664 | /* | |
665 | * The virtual address, as well as the size of the mapping, must be | |
666 | * aligned (at least) to the size of the smallest page supported | |
667 | * by the hardware | |
668 | */ | |
669 | if (!IS_ALIGNED(iova | size, min_pagesz)) { | |
670 | dev_err(NULL, "unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n", | |
671 | iova, size, min_pagesz); | |
672 | return -EINVAL; | |
673 | } | |
674 | ||
675 | /* | |
676 | * Keep iterating until we either unmap 'size' bytes (or more) | |
677 | * or we hit an area that isn't mapped. | |
678 | */ | |
679 | while (unmapped < size) { | |
680 | size_t pgsize = ipu_mmu_pgsize(mmu_info->pgsize_bitmap, | |
681 | iova, size - unmapped); | |
682 | ||
683 | unmapped_page = __ipu_mmu_unmap(mmu_info, iova, pgsize); | |
684 | if (!unmapped_page) | |
685 | break; | |
686 | ||
fad9119b | 687 | dev_dbg(mmu_info->dev, "unmapped: iova 0x%lx size 0x%zx\n", |
f2efa4ee WY |
688 | iova, unmapped_page); |
689 | ||
690 | iova += unmapped_page; | |
691 | unmapped += unmapped_page; | |
692 | } | |
693 | ||
694 | return unmapped; | |
695 | } | |
696 | ||
fad9119b | 697 | /* drivers/iommu/iommu.c:iommu_map() */ |
f2efa4ee WY |
698 | int ipu_mmu_map(struct ipu_mmu_info *mmu_info, unsigned long iova, |
699 | phys_addr_t paddr, size_t size) | |
700 | { | |
701 | unsigned long orig_iova = iova; | |
702 | unsigned int min_pagesz; | |
703 | size_t orig_size = size; | |
704 | int ret = 0; | |
705 | ||
706 | if (mmu_info->pgsize_bitmap == 0UL) | |
707 | return -ENODEV; | |
708 | ||
709 | /* find out the minimum page size supported */ | |
710 | min_pagesz = 1 << __ffs(mmu_info->pgsize_bitmap); | |
711 | ||
712 | /* | |
713 | * both the virtual address and the physical one, as well as | |
714 | * the size of the mapping, must be aligned (at least) to the | |
715 | * size of the smallest page supported by the hardware | |
716 | */ | |
717 | if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) { | |
fad9119b WY |
718 | dev_err(mmu_info->dev, |
719 | "unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n", | |
720 | iova, &paddr, size, min_pagesz); | |
f2efa4ee WY |
721 | return -EINVAL; |
722 | } | |
723 | ||
fad9119b WY |
724 | dev_dbg(mmu_info->dev, "map: iova 0x%lx pa %pa size 0x%zx\n", |
725 | iova, &paddr, size); | |
f2efa4ee WY |
726 | |
727 | while (size) { | |
728 | size_t pgsize = ipu_mmu_pgsize(mmu_info->pgsize_bitmap, | |
729 | iova | paddr, size); | |
730 | ||
fad9119b WY |
731 | dev_dbg(mmu_info->dev, |
732 | "mapping: iova 0x%lx pa %pa pgsize 0x%zx\n", | |
733 | iova, &paddr, pgsize); | |
f2efa4ee WY |
734 | |
735 | ret = __ipu_mmu_map(mmu_info, iova, paddr, pgsize); | |
736 | if (ret) | |
737 | break; | |
738 | ||
739 | iova += pgsize; | |
740 | paddr += pgsize; | |
741 | size -= pgsize; | |
742 | } | |
743 | ||
744 | /* unroll mapping in case something went wrong */ | |
745 | if (ret) | |
746 | ipu_mmu_unmap(mmu_info, orig_iova, orig_size - size); | |
747 | ||
748 | return ret; | |
749 | } | |
750 | ||
751 | static void ipu_mmu_destroy(struct ipu_mmu *mmu) | |
752 | { | |
753 | struct ipu_dma_mapping *dmap = mmu->dmap; | |
754 | struct ipu_mmu_info *mmu_info = dmap->mmu_info; | |
755 | struct iova *iova; | |
756 | u32 l1_idx; | |
757 | ||
fad9119b | 758 | if (mmu->iova_trash_page) { |
f2efa4ee | 759 | iova = find_iova(&dmap->iovad, |
fad9119b | 760 | mmu->iova_trash_page >> PAGE_SHIFT); |
f2efa4ee WY |
761 | if (iova) { |
762 | /* unmap and free the trash buffer iova */ | |
763 | ipu_mmu_unmap(mmu_info, iova->pfn_lo << PAGE_SHIFT, | |
764 | (iova->pfn_hi - iova->pfn_lo + 1) << | |
765 | PAGE_SHIFT); | |
766 | __free_iova(&dmap->iovad, iova); | |
767 | } else { | |
768 | dev_err(mmu->dev, "trash buffer iova not found.\n"); | |
769 | } | |
770 | ||
fad9119b WY |
771 | mmu->iova_trash_page = 0; |
772 | dma_unmap_page(mmu_info->dev, mmu->pci_trash_page, | |
773 | PAGE_SIZE, DMA_BIDIRECTIONAL); | |
774 | mmu->pci_trash_page = 0; | |
f2efa4ee | 775 | __free_page(mmu->trash_page); |
fad9119b | 776 | } |
f2efa4ee | 777 | |
fad9119b WY |
778 | for (l1_idx = 0; l1_idx < ISP_L1PT_PTES; l1_idx++) { |
779 | if (mmu_info->l1_pt[l1_idx] != mmu_info->dummy_l2_pteval) { | |
780 | dma_unmap_single(mmu_info->dev, | |
781 | TBL_PHYS_ADDR(mmu_info->l1_pt[l1_idx]), | |
782 | PAGE_SIZE, DMA_BIDIRECTIONAL); | |
783 | free_page((unsigned long)mmu_info->l2_pts[l1_idx]); | |
784 | } | |
785 | } | |
f2efa4ee | 786 | |
fad9119b WY |
787 | free_dummy_page(mmu_info); |
788 | dma_unmap_single(mmu_info->dev, mmu_info->l1_pt_dma, | |
789 | PAGE_SIZE, DMA_BIDIRECTIONAL); | |
790 | free_page((unsigned long)mmu_info->dummy_l2_pt); | |
791 | free_page((unsigned long)mmu_info->l1_pt); | |
f2efa4ee WY |
792 | kfree(mmu_info); |
793 | } | |
794 | ||
795 | struct ipu_mmu *ipu_mmu_init(struct device *dev, | |
796 | void __iomem *base, int mmid, | |
797 | const struct ipu_hw_variants *hw) | |
798 | { | |
799 | struct ipu_mmu *mmu; | |
800 | struct ipu_mmu_pdata *pdata; | |
801 | struct ipu_device *isp = pci_get_drvdata(to_pci_dev(dev)); | |
802 | unsigned int i; | |
803 | ||
804 | if (hw->nr_mmus > IPU_MMU_MAX_DEVICES) | |
805 | return ERR_PTR(-EINVAL); | |
806 | ||
807 | pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL); | |
808 | if (!pdata) | |
809 | return ERR_PTR(-ENOMEM); | |
810 | ||
811 | for (i = 0; i < hw->nr_mmus; i++) { | |
812 | struct ipu_mmu_hw *pdata_mmu = &pdata->mmu_hw[i]; | |
813 | const struct ipu_mmu_hw *src_mmu = &hw->mmu_hw[i]; | |
814 | ||
815 | if (src_mmu->nr_l1streams > IPU_MMU_MAX_TLB_L1_STREAMS || | |
816 | src_mmu->nr_l2streams > IPU_MMU_MAX_TLB_L2_STREAMS) | |
817 | return ERR_PTR(-EINVAL); | |
818 | ||
819 | *pdata_mmu = *src_mmu; | |
820 | pdata_mmu->base = base + src_mmu->offset; | |
821 | } | |
822 | ||
823 | mmu = devm_kzalloc(dev, sizeof(*mmu), GFP_KERNEL); | |
824 | if (!mmu) | |
825 | return ERR_PTR(-ENOMEM); | |
826 | ||
827 | mmu->mmid = mmid; | |
828 | mmu->mmu_hw = pdata->mmu_hw; | |
829 | mmu->nr_mmus = hw->nr_mmus; | |
830 | mmu->tlb_invalidate = tlb_invalidate; | |
831 | mmu->ready = false; | |
832 | INIT_LIST_HEAD(&mmu->vma_list); | |
833 | spin_lock_init(&mmu->ready_lock); | |
834 | ||
835 | mmu->dmap = alloc_dma_mapping(isp); | |
836 | if (!mmu->dmap) { | |
837 | dev_err(dev, "can't alloc dma mapping\n"); | |
838 | return ERR_PTR(-ENOMEM); | |
839 | } | |
840 | ||
841 | return mmu; | |
842 | } | |
843 | EXPORT_SYMBOL(ipu_mmu_init); | |
844 | ||
845 | void ipu_mmu_cleanup(struct ipu_mmu *mmu) | |
846 | { | |
847 | struct ipu_dma_mapping *dmap = mmu->dmap; | |
848 | ||
849 | ipu_mmu_destroy(mmu); | |
850 | mmu->dmap = NULL; | |
851 | iova_cache_put(); | |
852 | put_iova_domain(&dmap->iovad); | |
853 | kfree(dmap); | |
854 | } | |
855 | EXPORT_SYMBOL(ipu_mmu_cleanup); | |
856 | ||
857 | MODULE_AUTHOR("Sakari Ailus <sakari.ailus@linux.intel.com>"); | |
858 | MODULE_AUTHOR("Samu Onkalo <samu.onkalo@intel.com>"); | |
859 | MODULE_LICENSE("GPL"); | |
860 | MODULE_DESCRIPTION("Intel ipu mmu driver"); |