]>
Commit | Line | Data |
---|---|---|
de2ba664 AM |
1 | /* |
2 | * NVIDIA Tegra DRM GEM helper functions | |
3 | * | |
4 | * Copyright (C) 2012 Sascha Hauer, Pengutronix | |
7ecada3c | 5 | * Copyright (C) 2013-2015 NVIDIA CORPORATION, All rights reserved. |
de2ba664 AM |
6 | * |
7 | * Based on the GEM/CMA helpers | |
8 | * | |
9 | * Copyright (c) 2011 Samsung Electronics Co., Ltd. | |
10 | * | |
9a2ac2dc TR |
11 | * This program is free software; you can redistribute it and/or modify |
12 | * it under the terms of the GNU General Public License version 2 as | |
13 | * published by the Free Software Foundation. | |
de2ba664 AM |
14 | */ |
15 | ||
3800391d | 16 | #include <linux/dma-buf.h> |
df06b759 | 17 | #include <linux/iommu.h> |
773af77f TR |
18 | #include <drm/tegra_drm.h> |
19 | ||
d1f3e1e0 | 20 | #include "drm.h" |
de2ba664 AM |
21 | #include "gem.h" |
22 | ||
de2ba664 AM |
23 | static void tegra_bo_put(struct host1x_bo *bo) |
24 | { | |
3be82743 | 25 | struct tegra_bo *obj = host1x_to_tegra_bo(bo); |
de2ba664 | 26 | |
7664b2fa | 27 | drm_gem_object_put_unlocked(&obj->gem); |
de2ba664 AM |
28 | } |
29 | ||
30 | static dma_addr_t tegra_bo_pin(struct host1x_bo *bo, struct sg_table **sgt) | |
31 | { | |
3be82743 | 32 | struct tegra_bo *obj = host1x_to_tegra_bo(bo); |
de2ba664 | 33 | |
585ee0f2 MP |
34 | *sgt = obj->sgt; |
35 | ||
de2ba664 AM |
36 | return obj->paddr; |
37 | } | |
38 | ||
39 | static void tegra_bo_unpin(struct host1x_bo *bo, struct sg_table *sgt) | |
40 | { | |
41 | } | |
42 | ||
43 | static void *tegra_bo_mmap(struct host1x_bo *bo) | |
44 | { | |
3be82743 | 45 | struct tegra_bo *obj = host1x_to_tegra_bo(bo); |
de2ba664 | 46 | |
7ecada3c AM |
47 | if (obj->vaddr) |
48 | return obj->vaddr; | |
49 | else if (obj->gem.import_attach) | |
50 | return dma_buf_vmap(obj->gem.import_attach->dmabuf); | |
51 | else | |
52 | return vmap(obj->pages, obj->num_pages, VM_MAP, | |
53 | pgprot_writecombine(PAGE_KERNEL)); | |
de2ba664 AM |
54 | } |
55 | ||
56 | static void tegra_bo_munmap(struct host1x_bo *bo, void *addr) | |
57 | { | |
7ecada3c AM |
58 | struct tegra_bo *obj = host1x_to_tegra_bo(bo); |
59 | ||
60 | if (obj->vaddr) | |
61 | return; | |
62 | else if (obj->gem.import_attach) | |
63 | dma_buf_vunmap(obj->gem.import_attach->dmabuf, addr); | |
64 | else | |
65 | vunmap(addr); | |
de2ba664 AM |
66 | } |
67 | ||
68 | static void *tegra_bo_kmap(struct host1x_bo *bo, unsigned int page) | |
69 | { | |
3be82743 | 70 | struct tegra_bo *obj = host1x_to_tegra_bo(bo); |
de2ba664 | 71 | |
7ecada3c AM |
72 | if (obj->vaddr) |
73 | return obj->vaddr + page * PAGE_SIZE; | |
74 | else if (obj->gem.import_attach) | |
75 | return dma_buf_kmap(obj->gem.import_attach->dmabuf, page); | |
76 | else | |
77 | return vmap(obj->pages + page, 1, VM_MAP, | |
78 | pgprot_writecombine(PAGE_KERNEL)); | |
de2ba664 AM |
79 | } |
80 | ||
81 | static void tegra_bo_kunmap(struct host1x_bo *bo, unsigned int page, | |
82 | void *addr) | |
83 | { | |
7ecada3c AM |
84 | struct tegra_bo *obj = host1x_to_tegra_bo(bo); |
85 | ||
86 | if (obj->vaddr) | |
87 | return; | |
88 | else if (obj->gem.import_attach) | |
89 | dma_buf_kunmap(obj->gem.import_attach->dmabuf, page, addr); | |
90 | else | |
91 | vunmap(addr); | |
de2ba664 AM |
92 | } |
93 | ||
94 | static struct host1x_bo *tegra_bo_get(struct host1x_bo *bo) | |
95 | { | |
3be82743 | 96 | struct tegra_bo *obj = host1x_to_tegra_bo(bo); |
de2ba664 | 97 | |
7664b2fa | 98 | drm_gem_object_get(&obj->gem); |
de2ba664 AM |
99 | |
100 | return bo; | |
101 | } | |
102 | ||
425c0fdc | 103 | static const struct host1x_bo_ops tegra_bo_ops = { |
de2ba664 AM |
104 | .get = tegra_bo_get, |
105 | .put = tegra_bo_put, | |
106 | .pin = tegra_bo_pin, | |
107 | .unpin = tegra_bo_unpin, | |
108 | .mmap = tegra_bo_mmap, | |
109 | .munmap = tegra_bo_munmap, | |
110 | .kmap = tegra_bo_kmap, | |
111 | .kunmap = tegra_bo_kunmap, | |
112 | }; | |
113 | ||
df06b759 TR |
114 | static int tegra_bo_iommu_map(struct tegra_drm *tegra, struct tegra_bo *bo) |
115 | { | |
116 | int prot = IOMMU_READ | IOMMU_WRITE; | |
04184b1f | 117 | int err; |
df06b759 TR |
118 | |
119 | if (bo->mm) | |
120 | return -EBUSY; | |
121 | ||
122 | bo->mm = kzalloc(sizeof(*bo->mm), GFP_KERNEL); | |
123 | if (!bo->mm) | |
124 | return -ENOMEM; | |
125 | ||
347ad49d TR |
126 | mutex_lock(&tegra->mm_lock); |
127 | ||
4e64e553 CW |
128 | err = drm_mm_insert_node_generic(&tegra->mm, |
129 | bo->mm, bo->gem.size, PAGE_SIZE, 0, 0); | |
df06b759 | 130 | if (err < 0) { |
04184b1f | 131 | dev_err(tegra->drm->dev, "out of I/O virtual memory: %d\n", |
df06b759 | 132 | err); |
347ad49d | 133 | goto unlock; |
df06b759 TR |
134 | } |
135 | ||
136 | bo->paddr = bo->mm->start; | |
137 | ||
04184b1f DO |
138 | bo->size = iommu_map_sg(tegra->domain, bo->paddr, bo->sgt->sgl, |
139 | bo->sgt->nents, prot); | |
140 | if (!bo->size) { | |
141 | dev_err(tegra->drm->dev, "failed to map buffer\n"); | |
142 | err = -ENOMEM; | |
df06b759 TR |
143 | goto remove; |
144 | } | |
145 | ||
347ad49d TR |
146 | mutex_unlock(&tegra->mm_lock); |
147 | ||
df06b759 TR |
148 | return 0; |
149 | ||
150 | remove: | |
151 | drm_mm_remove_node(bo->mm); | |
347ad49d TR |
152 | unlock: |
153 | mutex_unlock(&tegra->mm_lock); | |
df06b759 TR |
154 | kfree(bo->mm); |
155 | return err; | |
156 | } | |
157 | ||
158 | static int tegra_bo_iommu_unmap(struct tegra_drm *tegra, struct tegra_bo *bo) | |
159 | { | |
160 | if (!bo->mm) | |
161 | return 0; | |
162 | ||
347ad49d | 163 | mutex_lock(&tegra->mm_lock); |
df06b759 TR |
164 | iommu_unmap(tegra->domain, bo->paddr, bo->size); |
165 | drm_mm_remove_node(bo->mm); | |
347ad49d TR |
166 | mutex_unlock(&tegra->mm_lock); |
167 | ||
df06b759 TR |
168 | kfree(bo->mm); |
169 | ||
170 | return 0; | |
171 | } | |
172 | ||
c28d4a31 TR |
173 | static struct tegra_bo *tegra_bo_alloc_object(struct drm_device *drm, |
174 | size_t size) | |
175 | { | |
176 | struct tegra_bo *bo; | |
177 | int err; | |
178 | ||
179 | bo = kzalloc(sizeof(*bo), GFP_KERNEL); | |
180 | if (!bo) | |
181 | return ERR_PTR(-ENOMEM); | |
182 | ||
183 | host1x_bo_init(&bo->base, &tegra_bo_ops); | |
184 | size = round_up(size, PAGE_SIZE); | |
185 | ||
186 | err = drm_gem_object_init(drm, &bo->gem, size); | |
187 | if (err < 0) | |
188 | goto free; | |
189 | ||
190 | err = drm_gem_create_mmap_offset(&bo->gem); | |
191 | if (err < 0) | |
192 | goto release; | |
193 | ||
194 | return bo; | |
195 | ||
196 | release: | |
197 | drm_gem_object_release(&bo->gem); | |
198 | free: | |
199 | kfree(bo); | |
200 | return ERR_PTR(err); | |
201 | } | |
202 | ||
df06b759 | 203 | static void tegra_bo_free(struct drm_device *drm, struct tegra_bo *bo) |
de2ba664 | 204 | { |
df06b759 | 205 | if (bo->pages) { |
bd43c9f0 TR |
206 | dma_unmap_sg(drm->dev, bo->sgt->sgl, bo->sgt->nents, |
207 | DMA_BIDIRECTIONAL); | |
df06b759 TR |
208 | drm_gem_put_pages(&bo->gem, bo->pages, true, true); |
209 | sg_free_table(bo->sgt); | |
210 | kfree(bo->sgt); | |
7e0180e3 | 211 | } else if (bo->vaddr) { |
f6e45661 | 212 | dma_free_wc(drm->dev, bo->gem.size, bo->vaddr, bo->paddr); |
df06b759 TR |
213 | } |
214 | } | |
215 | ||
73c42c79 | 216 | static int tegra_bo_get_pages(struct drm_device *drm, struct tegra_bo *bo) |
df06b759 | 217 | { |
bd43c9f0 | 218 | int err; |
a04251fc | 219 | |
df06b759 TR |
220 | bo->pages = drm_gem_get_pages(&bo->gem); |
221 | if (IS_ERR(bo->pages)) | |
222 | return PTR_ERR(bo->pages); | |
223 | ||
73c42c79 | 224 | bo->num_pages = bo->gem.size >> PAGE_SHIFT; |
df06b759 | 225 | |
fd73caa5 | 226 | bo->sgt = drm_prime_pages_to_sg(bo->pages, bo->num_pages); |
bd43c9f0 TR |
227 | if (IS_ERR(bo->sgt)) { |
228 | err = PTR_ERR(bo->sgt); | |
a04251fc | 229 | goto put_pages; |
bd43c9f0 | 230 | } |
a04251fc | 231 | |
bd43c9f0 TR |
232 | err = dma_map_sg(drm->dev, bo->sgt->sgl, bo->sgt->nents, |
233 | DMA_BIDIRECTIONAL); | |
234 | if (err == 0) { | |
235 | err = -EFAULT; | |
236 | goto free_sgt; | |
237 | } | |
a04251fc | 238 | |
df06b759 | 239 | return 0; |
a04251fc | 240 | |
bd43c9f0 TR |
241 | free_sgt: |
242 | sg_free_table(bo->sgt); | |
243 | kfree(bo->sgt); | |
a04251fc TR |
244 | put_pages: |
245 | drm_gem_put_pages(&bo->gem, bo->pages, false, false); | |
bd43c9f0 | 246 | return err; |
df06b759 TR |
247 | } |
248 | ||
73c42c79 | 249 | static int tegra_bo_alloc(struct drm_device *drm, struct tegra_bo *bo) |
df06b759 TR |
250 | { |
251 | struct tegra_drm *tegra = drm->dev_private; | |
252 | int err; | |
253 | ||
254 | if (tegra->domain) { | |
73c42c79 | 255 | err = tegra_bo_get_pages(drm, bo); |
df06b759 TR |
256 | if (err < 0) |
257 | return err; | |
258 | ||
259 | err = tegra_bo_iommu_map(tegra, bo); | |
260 | if (err < 0) { | |
261 | tegra_bo_free(drm, bo); | |
262 | return err; | |
263 | } | |
264 | } else { | |
73c42c79 TR |
265 | size_t size = bo->gem.size; |
266 | ||
f6e45661 LR |
267 | bo->vaddr = dma_alloc_wc(drm->dev, size, &bo->paddr, |
268 | GFP_KERNEL | __GFP_NOWARN); | |
df06b759 TR |
269 | if (!bo->vaddr) { |
270 | dev_err(drm->dev, | |
271 | "failed to allocate buffer of size %zu\n", | |
272 | size); | |
273 | return -ENOMEM; | |
274 | } | |
275 | } | |
276 | ||
277 | return 0; | |
de2ba664 AM |
278 | } |
279 | ||
71c38629 | 280 | struct tegra_bo *tegra_bo_create(struct drm_device *drm, size_t size, |
773af77f | 281 | unsigned long flags) |
de2ba664 AM |
282 | { |
283 | struct tegra_bo *bo; | |
284 | int err; | |
285 | ||
c28d4a31 TR |
286 | bo = tegra_bo_alloc_object(drm, size); |
287 | if (IS_ERR(bo)) | |
288 | return bo; | |
de2ba664 | 289 | |
73c42c79 | 290 | err = tegra_bo_alloc(drm, bo); |
df06b759 TR |
291 | if (err < 0) |
292 | goto release; | |
de2ba664 | 293 | |
773af77f | 294 | if (flags & DRM_TEGRA_GEM_CREATE_TILED) |
c134f019 | 295 | bo->tiling.mode = TEGRA_BO_TILING_MODE_TILED; |
773af77f | 296 | |
db7fbdfd TR |
297 | if (flags & DRM_TEGRA_GEM_CREATE_BOTTOM_UP) |
298 | bo->flags |= TEGRA_BO_BOTTOM_UP; | |
299 | ||
de2ba664 AM |
300 | return bo; |
301 | ||
df06b759 TR |
302 | release: |
303 | drm_gem_object_release(&bo->gem); | |
de2ba664 | 304 | kfree(bo); |
de2ba664 | 305 | return ERR_PTR(err); |
de2ba664 AM |
306 | } |
307 | ||
308 | struct tegra_bo *tegra_bo_create_with_handle(struct drm_file *file, | |
3be82743 | 309 | struct drm_device *drm, |
71c38629 | 310 | size_t size, |
773af77f | 311 | unsigned long flags, |
71c38629 | 312 | u32 *handle) |
de2ba664 AM |
313 | { |
314 | struct tegra_bo *bo; | |
a8b48df5 | 315 | int err; |
de2ba664 | 316 | |
773af77f | 317 | bo = tegra_bo_create(drm, size, flags); |
de2ba664 AM |
318 | if (IS_ERR(bo)) |
319 | return bo; | |
320 | ||
a8b48df5 TR |
321 | err = drm_gem_handle_create(file, &bo->gem, handle); |
322 | if (err) { | |
323 | tegra_bo_free_object(&bo->gem); | |
324 | return ERR_PTR(err); | |
325 | } | |
de2ba664 | 326 | |
7664b2fa | 327 | drm_gem_object_put_unlocked(&bo->gem); |
de2ba664 AM |
328 | |
329 | return bo; | |
de2ba664 AM |
330 | } |
331 | ||
540457cc TR |
332 | static struct tegra_bo *tegra_bo_import(struct drm_device *drm, |
333 | struct dma_buf *buf) | |
3800391d | 334 | { |
df06b759 | 335 | struct tegra_drm *tegra = drm->dev_private; |
3800391d TR |
336 | struct dma_buf_attachment *attach; |
337 | struct tegra_bo *bo; | |
3800391d TR |
338 | int err; |
339 | ||
c28d4a31 TR |
340 | bo = tegra_bo_alloc_object(drm, buf->size); |
341 | if (IS_ERR(bo)) | |
342 | return bo; | |
3800391d TR |
343 | |
344 | attach = dma_buf_attach(buf, drm->dev); | |
345 | if (IS_ERR(attach)) { | |
346 | err = PTR_ERR(attach); | |
c28d4a31 | 347 | goto free; |
3800391d TR |
348 | } |
349 | ||
350 | get_dma_buf(buf); | |
351 | ||
352 | bo->sgt = dma_buf_map_attachment(attach, DMA_TO_DEVICE); | |
3800391d TR |
353 | if (IS_ERR(bo->sgt)) { |
354 | err = PTR_ERR(bo->sgt); | |
355 | goto detach; | |
356 | } | |
357 | ||
df06b759 TR |
358 | if (tegra->domain) { |
359 | err = tegra_bo_iommu_map(tegra, bo); | |
360 | if (err < 0) | |
361 | goto detach; | |
362 | } else { | |
363 | if (bo->sgt->nents > 1) { | |
364 | err = -EINVAL; | |
365 | goto detach; | |
366 | } | |
367 | ||
368 | bo->paddr = sg_dma_address(bo->sgt->sgl); | |
3800391d TR |
369 | } |
370 | ||
3800391d TR |
371 | bo->gem.import_attach = attach; |
372 | ||
373 | return bo; | |
374 | ||
375 | detach: | |
376 | if (!IS_ERR_OR_NULL(bo->sgt)) | |
377 | dma_buf_unmap_attachment(attach, bo->sgt, DMA_TO_DEVICE); | |
378 | ||
379 | dma_buf_detach(buf, attach); | |
380 | dma_buf_put(buf); | |
3800391d | 381 | free: |
c28d4a31 | 382 | drm_gem_object_release(&bo->gem); |
3800391d | 383 | kfree(bo); |
3800391d TR |
384 | return ERR_PTR(err); |
385 | } | |
386 | ||
de2ba664 AM |
387 | void tegra_bo_free_object(struct drm_gem_object *gem) |
388 | { | |
df06b759 | 389 | struct tegra_drm *tegra = gem->dev->dev_private; |
de2ba664 AM |
390 | struct tegra_bo *bo = to_tegra_bo(gem); |
391 | ||
df06b759 TR |
392 | if (tegra->domain) |
393 | tegra_bo_iommu_unmap(tegra, bo); | |
394 | ||
3800391d TR |
395 | if (gem->import_attach) { |
396 | dma_buf_unmap_attachment(gem->import_attach, bo->sgt, | |
397 | DMA_TO_DEVICE); | |
398 | drm_prime_gem_destroy(gem, NULL); | |
399 | } else { | |
df06b759 | 400 | tegra_bo_free(gem->dev, bo); |
3800391d TR |
401 | } |
402 | ||
de2ba664 | 403 | drm_gem_object_release(gem); |
de2ba664 AM |
404 | kfree(bo); |
405 | } | |
406 | ||
407 | int tegra_bo_dumb_create(struct drm_file *file, struct drm_device *drm, | |
408 | struct drm_mode_create_dumb *args) | |
409 | { | |
dc6057ec | 410 | unsigned int min_pitch = DIV_ROUND_UP(args->width * args->bpp, 8); |
d1f3e1e0 | 411 | struct tegra_drm *tegra = drm->dev_private; |
de2ba664 AM |
412 | struct tegra_bo *bo; |
413 | ||
dc6057ec TR |
414 | args->pitch = round_up(min_pitch, tegra->pitch_align); |
415 | args->size = args->pitch * args->height; | |
de2ba664 | 416 | |
773af77f | 417 | bo = tegra_bo_create_with_handle(file, drm, args->size, 0, |
3be82743 | 418 | &args->handle); |
de2ba664 AM |
419 | if (IS_ERR(bo)) |
420 | return PTR_ERR(bo); | |
421 | ||
422 | return 0; | |
423 | } | |
424 | ||
cc7add70 | 425 | static vm_fault_t tegra_bo_fault(struct vm_fault *vmf) |
df06b759 | 426 | { |
11bac800 | 427 | struct vm_area_struct *vma = vmf->vma; |
df06b759 TR |
428 | struct drm_gem_object *gem = vma->vm_private_data; |
429 | struct tegra_bo *bo = to_tegra_bo(gem); | |
430 | struct page *page; | |
431 | pgoff_t offset; | |
df06b759 TR |
432 | |
433 | if (!bo->pages) | |
434 | return VM_FAULT_SIGBUS; | |
435 | ||
1a29d85e | 436 | offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT; |
df06b759 TR |
437 | page = bo->pages[offset]; |
438 | ||
cc7add70 | 439 | return vmf_insert_page(vma, vmf->address, page); |
df06b759 TR |
440 | } |
441 | ||
de2ba664 | 442 | const struct vm_operations_struct tegra_bo_vm_ops = { |
df06b759 | 443 | .fault = tegra_bo_fault, |
de2ba664 AM |
444 | .open = drm_gem_vm_open, |
445 | .close = drm_gem_vm_close, | |
446 | }; | |
447 | ||
04c07466 | 448 | int __tegra_gem_mmap(struct drm_gem_object *gem, struct vm_area_struct *vma) |
de2ba664 | 449 | { |
a8bc8c65 | 450 | struct tegra_bo *bo = to_tegra_bo(gem); |
de2ba664 | 451 | |
df06b759 TR |
452 | if (!bo->pages) { |
453 | unsigned long vm_pgoff = vma->vm_pgoff; | |
a8bc8c65 | 454 | int err; |
53ea7213 | 455 | |
a8bc8c65 TR |
456 | /* |
457 | * Clear the VM_PFNMAP flag that was set by drm_gem_mmap(), | |
458 | * and set the vm_pgoff (used as a fake buffer offset by DRM) | |
459 | * to 0 as we want to map the whole buffer. | |
460 | */ | |
df06b759 TR |
461 | vma->vm_flags &= ~VM_PFNMAP; |
462 | vma->vm_pgoff = 0; | |
463 | ||
a8bc8c65 | 464 | err = dma_mmap_wc(gem->dev->dev, vma, bo->vaddr, bo->paddr, |
f6e45661 | 465 | gem->size); |
a8bc8c65 | 466 | if (err < 0) { |
df06b759 | 467 | drm_gem_vm_close(vma); |
a8bc8c65 | 468 | return err; |
df06b759 TR |
469 | } |
470 | ||
471 | vma->vm_pgoff = vm_pgoff; | |
472 | } else { | |
473 | pgprot_t prot = vm_get_page_prot(vma->vm_flags); | |
53ea7213 | 474 | |
df06b759 TR |
475 | vma->vm_flags |= VM_MIXEDMAP; |
476 | vma->vm_flags &= ~VM_PFNMAP; | |
477 | ||
478 | vma->vm_page_prot = pgprot_writecombine(prot); | |
479 | } | |
de2ba664 | 480 | |
53ea7213 | 481 | return 0; |
de2ba664 | 482 | } |
3800391d | 483 | |
a8bc8c65 TR |
484 | int tegra_drm_mmap(struct file *file, struct vm_area_struct *vma) |
485 | { | |
486 | struct drm_gem_object *gem; | |
487 | int err; | |
488 | ||
489 | err = drm_gem_mmap(file, vma); | |
490 | if (err < 0) | |
491 | return err; | |
492 | ||
493 | gem = vma->vm_private_data; | |
494 | ||
04c07466 | 495 | return __tegra_gem_mmap(gem, vma); |
a8bc8c65 TR |
496 | } |
497 | ||
3800391d TR |
498 | static struct sg_table * |
499 | tegra_gem_prime_map_dma_buf(struct dma_buf_attachment *attach, | |
500 | enum dma_data_direction dir) | |
501 | { | |
502 | struct drm_gem_object *gem = attach->dmabuf->priv; | |
503 | struct tegra_bo *bo = to_tegra_bo(gem); | |
504 | struct sg_table *sgt; | |
505 | ||
506 | sgt = kmalloc(sizeof(*sgt), GFP_KERNEL); | |
507 | if (!sgt) | |
508 | return NULL; | |
509 | ||
df06b759 TR |
510 | if (bo->pages) { |
511 | struct scatterlist *sg; | |
512 | unsigned int i; | |
513 | ||
514 | if (sg_alloc_table(sgt, bo->num_pages, GFP_KERNEL)) | |
515 | goto free; | |
3800391d | 516 | |
df06b759 TR |
517 | for_each_sg(sgt->sgl, sg, bo->num_pages, i) |
518 | sg_set_page(sg, bo->pages[i], PAGE_SIZE, 0); | |
519 | ||
520 | if (dma_map_sg(attach->dev, sgt->sgl, sgt->nents, dir) == 0) | |
521 | goto free; | |
522 | } else { | |
523 | if (sg_alloc_table(sgt, 1, GFP_KERNEL)) | |
524 | goto free; | |
525 | ||
526 | sg_dma_address(sgt->sgl) = bo->paddr; | |
527 | sg_dma_len(sgt->sgl) = gem->size; | |
528 | } | |
3800391d TR |
529 | |
530 | return sgt; | |
df06b759 TR |
531 | |
532 | free: | |
533 | sg_free_table(sgt); | |
534 | kfree(sgt); | |
535 | return NULL; | |
3800391d TR |
536 | } |
537 | ||
538 | static void tegra_gem_prime_unmap_dma_buf(struct dma_buf_attachment *attach, | |
539 | struct sg_table *sgt, | |
540 | enum dma_data_direction dir) | |
541 | { | |
df06b759 TR |
542 | struct drm_gem_object *gem = attach->dmabuf->priv; |
543 | struct tegra_bo *bo = to_tegra_bo(gem); | |
544 | ||
545 | if (bo->pages) | |
546 | dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents, dir); | |
547 | ||
3800391d TR |
548 | sg_free_table(sgt); |
549 | kfree(sgt); | |
550 | } | |
551 | ||
552 | static void tegra_gem_prime_release(struct dma_buf *buf) | |
553 | { | |
554 | drm_gem_dmabuf_release(buf); | |
555 | } | |
556 | ||
27e92f1f TR |
557 | static int tegra_gem_prime_begin_cpu_access(struct dma_buf *buf, |
558 | enum dma_data_direction direction) | |
559 | { | |
560 | struct drm_gem_object *gem = buf->priv; | |
561 | struct tegra_bo *bo = to_tegra_bo(gem); | |
562 | struct drm_device *drm = gem->dev; | |
563 | ||
564 | if (bo->pages) | |
565 | dma_sync_sg_for_cpu(drm->dev, bo->sgt->sgl, bo->sgt->nents, | |
566 | DMA_FROM_DEVICE); | |
567 | ||
568 | return 0; | |
569 | } | |
570 | ||
571 | static int tegra_gem_prime_end_cpu_access(struct dma_buf *buf, | |
572 | enum dma_data_direction direction) | |
573 | { | |
574 | struct drm_gem_object *gem = buf->priv; | |
575 | struct tegra_bo *bo = to_tegra_bo(gem); | |
576 | struct drm_device *drm = gem->dev; | |
577 | ||
578 | if (bo->pages) | |
579 | dma_sync_sg_for_device(drm->dev, bo->sgt->sgl, bo->sgt->nents, | |
580 | DMA_TO_DEVICE); | |
581 | ||
582 | return 0; | |
583 | } | |
584 | ||
3800391d TR |
585 | static void *tegra_gem_prime_kmap(struct dma_buf *buf, unsigned long page) |
586 | { | |
587 | return NULL; | |
588 | } | |
589 | ||
590 | static void tegra_gem_prime_kunmap(struct dma_buf *buf, unsigned long page, | |
591 | void *addr) | |
592 | { | |
593 | } | |
594 | ||
595 | static int tegra_gem_prime_mmap(struct dma_buf *buf, struct vm_area_struct *vma) | |
596 | { | |
a8bc8c65 TR |
597 | struct drm_gem_object *gem = buf->priv; |
598 | int err; | |
599 | ||
600 | err = drm_gem_mmap_obj(gem, gem->size, vma); | |
601 | if (err < 0) | |
602 | return err; | |
603 | ||
04c07466 | 604 | return __tegra_gem_mmap(gem, vma); |
3800391d TR |
605 | } |
606 | ||
d40326f4 TR |
607 | static void *tegra_gem_prime_vmap(struct dma_buf *buf) |
608 | { | |
609 | struct drm_gem_object *gem = buf->priv; | |
610 | struct tegra_bo *bo = to_tegra_bo(gem); | |
611 | ||
612 | return bo->vaddr; | |
613 | } | |
614 | ||
615 | static void tegra_gem_prime_vunmap(struct dma_buf *buf, void *vaddr) | |
616 | { | |
617 | } | |
618 | ||
3800391d TR |
619 | static const struct dma_buf_ops tegra_gem_prime_dmabuf_ops = { |
620 | .map_dma_buf = tegra_gem_prime_map_dma_buf, | |
621 | .unmap_dma_buf = tegra_gem_prime_unmap_dma_buf, | |
622 | .release = tegra_gem_prime_release, | |
27e92f1f TR |
623 | .begin_cpu_access = tegra_gem_prime_begin_cpu_access, |
624 | .end_cpu_access = tegra_gem_prime_end_cpu_access, | |
f9b67f00 LG |
625 | .map = tegra_gem_prime_kmap, |
626 | .unmap = tegra_gem_prime_kunmap, | |
3800391d | 627 | .mmap = tegra_gem_prime_mmap, |
d40326f4 TR |
628 | .vmap = tegra_gem_prime_vmap, |
629 | .vunmap = tegra_gem_prime_vunmap, | |
3800391d TR |
630 | }; |
631 | ||
632 | struct dma_buf *tegra_gem_prime_export(struct drm_device *drm, | |
633 | struct drm_gem_object *gem, | |
634 | int flags) | |
635 | { | |
d8fbe341 SS |
636 | DEFINE_DMA_BUF_EXPORT_INFO(exp_info); |
637 | ||
4bd91a5b TR |
638 | exp_info.exp_name = KBUILD_MODNAME; |
639 | exp_info.owner = drm->driver->fops->owner; | |
d8fbe341 SS |
640 | exp_info.ops = &tegra_gem_prime_dmabuf_ops; |
641 | exp_info.size = gem->size; | |
642 | exp_info.flags = flags; | |
643 | exp_info.priv = gem; | |
644 | ||
a4fce9cb | 645 | return drm_gem_dmabuf_export(drm, &exp_info); |
3800391d TR |
646 | } |
647 | ||
648 | struct drm_gem_object *tegra_gem_prime_import(struct drm_device *drm, | |
649 | struct dma_buf *buf) | |
650 | { | |
651 | struct tegra_bo *bo; | |
652 | ||
653 | if (buf->ops == &tegra_gem_prime_dmabuf_ops) { | |
654 | struct drm_gem_object *gem = buf->priv; | |
655 | ||
656 | if (gem->dev == drm) { | |
7664b2fa | 657 | drm_gem_object_get(gem); |
3800391d TR |
658 | return gem; |
659 | } | |
660 | } | |
661 | ||
662 | bo = tegra_bo_import(drm, buf); | |
663 | if (IS_ERR(bo)) | |
664 | return ERR_CAST(bo); | |
665 | ||
666 | return &bo->gem; | |
667 | } |