]>
Commit | Line | Data |
---|---|---|
d38ceaf9 AD |
1 | /* |
2 | * Copyright 2012 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | * based on nouveau_prime.c | |
23 | * | |
24 | * Authors: Alex Deucher | |
25 | */ | |
baca30fa MD |
26 | |
27 | /** | |
28 | * DOC: PRIME Buffer Sharing | |
29 | * | |
30 | * The following callback implementations are used for :ref:`sharing GEM buffer | |
31 | * objects between different devices via PRIME <prime_buffer_sharing>`. | |
32 | */ | |
33 | ||
d38ceaf9 AD |
34 | #include <drm/drmP.h> |
35 | ||
36 | #include "amdgpu.h" | |
09052fc3 | 37 | #include "amdgpu_display.h" |
2cddc50e | 38 | #include "amdgpu_gem.h" |
d38ceaf9 AD |
39 | #include <drm/amdgpu_drm.h> |
40 | #include <linux/dma-buf.h> | |
41 | ||
9021d2ed CK |
42 | static const struct dma_buf_ops amdgpu_dmabuf_ops; |
43 | ||
baca30fa MD |
44 | /** |
45 | * amdgpu_gem_prime_get_sg_table - &drm_driver.gem_prime_get_sg_table | |
46 | * implementation | |
56ea0976 | 47 | * @obj: GEM buffer object (BO) |
baca30fa MD |
48 | * |
49 | * Returns: | |
56ea0976 | 50 | * A scatter/gather table for the pinned pages of the BO's memory. |
baca30fa | 51 | */ |
d38ceaf9 AD |
52 | struct sg_table *amdgpu_gem_prime_get_sg_table(struct drm_gem_object *obj) |
53 | { | |
54 | struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); | |
55 | int npages = bo->tbo.num_pages; | |
56 | ||
57 | return drm_prime_pages_to_sg(bo->tbo.ttm->pages, npages); | |
58 | } | |
59 | ||
baca30fa MD |
60 | /** |
61 | * amdgpu_gem_prime_vmap - &dma_buf_ops.vmap implementation | |
56ea0976 | 62 | * @obj: GEM BO |
baca30fa | 63 | * |
56ea0976 | 64 | * Sets up an in-kernel virtual mapping of the BO's memory. |
baca30fa MD |
65 | * |
66 | * Returns: | |
67 | * The virtual address of the mapping or an error pointer. | |
68 | */ | |
d38ceaf9 AD |
69 | void *amdgpu_gem_prime_vmap(struct drm_gem_object *obj) |
70 | { | |
71 | struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); | |
72 | int ret; | |
73 | ||
74 | ret = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, | |
75 | &bo->dma_buf_vmap); | |
76 | if (ret) | |
77 | return ERR_PTR(ret); | |
78 | ||
79 | return bo->dma_buf_vmap.virtual; | |
80 | } | |
81 | ||
baca30fa MD |
82 | /** |
83 | * amdgpu_gem_prime_vunmap - &dma_buf_ops.vunmap implementation | |
56ea0976 VM |
84 | * @obj: GEM BO |
85 | * @vaddr: Virtual address (unused) | |
baca30fa | 86 | * |
56ea0976 | 87 | * Tears down the in-kernel virtual mapping of the BO's memory. |
baca30fa | 88 | */ |
d38ceaf9 AD |
89 | void amdgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr) |
90 | { | |
91 | struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); | |
92 | ||
93 | ttm_bo_kunmap(&bo->dma_buf_vmap); | |
94 | } | |
95 | ||
baca30fa MD |
96 | /** |
97 | * amdgpu_gem_prime_mmap - &drm_driver.gem_prime_mmap implementation | |
56ea0976 VM |
98 | * @obj: GEM BO |
99 | * @vma: Virtual memory area | |
baca30fa | 100 | * |
56ea0976 | 101 | * Sets up a userspace mapping of the BO's memory in the given |
baca30fa MD |
102 | * virtual memory area. |
103 | * | |
104 | * Returns: | |
56ea0976 | 105 | * 0 on success or a negative error code on failure. |
baca30fa | 106 | */ |
dfced2e4 SL |
107 | int amdgpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) |
108 | { | |
109 | struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); | |
110 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); | |
111 | unsigned asize = amdgpu_bo_size(bo); | |
112 | int ret; | |
113 | ||
114 | if (!vma->vm_file) | |
115 | return -ENODEV; | |
116 | ||
117 | if (adev == NULL) | |
118 | return -ENODEV; | |
119 | ||
120 | /* Check for valid size. */ | |
121 | if (asize < vma->vm_end - vma->vm_start) | |
122 | return -EINVAL; | |
123 | ||
124 | if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) || | |
125 | (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) { | |
126 | return -EPERM; | |
127 | } | |
128 | vma->vm_pgoff += amdgpu_bo_mmap_offset(bo) >> PAGE_SHIFT; | |
129 | ||
130 | /* prime mmap does not need to check access, so allow here */ | |
131 | ret = drm_vma_node_allow(&obj->vma_node, vma->vm_file->private_data); | |
132 | if (ret) | |
133 | return ret; | |
134 | ||
135 | ret = ttm_bo_mmap(vma->vm_file, vma, &adev->mman.bdev); | |
136 | drm_vma_node_revoke(&obj->vma_node, vma->vm_file->private_data); | |
137 | ||
138 | return ret; | |
139 | } | |
140 | ||
baca30fa MD |
141 | /** |
142 | * amdgpu_gem_prime_import_sg_table - &drm_driver.gem_prime_import_sg_table | |
143 | * implementation | |
144 | * @dev: DRM device | |
145 | * @attach: DMA-buf attachment | |
146 | * @sg: Scatter/gather table | |
147 | * | |
56ea0976 | 148 | * Imports shared DMA buffer memory exported by another device. |
baca30fa MD |
149 | * |
150 | * Returns: | |
56ea0976 | 151 | * A new GEM BO of the given DRM device, representing the memory |
baca30fa MD |
152 | * described by the given DMA-buf attachment and scatter/gather table. |
153 | */ | |
4d9c514d CK |
154 | struct drm_gem_object * |
155 | amdgpu_gem_prime_import_sg_table(struct drm_device *dev, | |
156 | struct dma_buf_attachment *attach, | |
157 | struct sg_table *sg) | |
d38ceaf9 | 158 | { |
72d7668b | 159 | struct reservation_object *resv = attach->dmabuf->resv; |
d38ceaf9 AD |
160 | struct amdgpu_device *adev = dev->dev_private; |
161 | struct amdgpu_bo *bo; | |
3216c6b7 | 162 | struct amdgpu_bo_param bp; |
d38ceaf9 AD |
163 | int ret; |
164 | ||
3216c6b7 CZ |
165 | memset(&bp, 0, sizeof(bp)); |
166 | bp.size = attach->dmabuf->size; | |
167 | bp.byte_align = PAGE_SIZE; | |
168 | bp.domain = AMDGPU_GEM_DOMAIN_CPU; | |
169 | bp.flags = 0; | |
170 | bp.type = ttm_bo_type_sg; | |
171 | bp.resv = resv; | |
72d7668b | 172 | ww_mutex_lock(&resv->lock, NULL); |
3216c6b7 | 173 | ret = amdgpu_bo_create(adev, &bp, &bo); |
d38ceaf9 | 174 | if (ret) |
59dd4772 CK |
175 | goto error; |
176 | ||
eab3de23 CK |
177 | bo->tbo.sg = sg; |
178 | bo->tbo.ttm->sg = sg; | |
e3364dfc CK |
179 | bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT; |
180 | bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT; | |
59dd4772 CK |
181 | if (attach->dmabuf->ops != &amdgpu_dmabuf_ops) |
182 | bo->prime_shared_count = 1; | |
d38ceaf9 | 183 | |
59dd4772 | 184 | ww_mutex_unlock(&resv->lock); |
d38ceaf9 | 185 | return &bo->gem_base; |
59dd4772 CK |
186 | |
187 | error: | |
188 | ww_mutex_unlock(&resv->lock); | |
189 | return ERR_PTR(ret); | |
d38ceaf9 AD |
190 | } |
191 | ||
baca30fa MD |
192 | /** |
193 | * amdgpu_gem_map_attach - &dma_buf_ops.attach implementation | |
56ea0976 | 194 | * @dma_buf: Shared DMA buffer |
baca30fa MD |
195 | * @attach: DMA-buf attachment |
196 | * | |
197 | * Makes sure that the shared DMA buffer can be accessed by the target device. | |
198 | * For now, simply pins it to the GTT domain, where it should be accessible by | |
199 | * all DMA devices. | |
200 | * | |
201 | * Returns: | |
56ea0976 | 202 | * 0 on success or a negative error code on failure. |
baca30fa | 203 | */ |
5a13761f | 204 | static int amdgpu_gem_map_attach(struct dma_buf *dma_buf, |
5a13761f | 205 | struct dma_buf_attachment *attach) |
d38ceaf9 | 206 | { |
5a13761f | 207 | struct drm_gem_object *obj = dma_buf->priv; |
d38ceaf9 | 208 | struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); |
2333bf9a | 209 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
5a13761f | 210 | long r; |
d38ceaf9 | 211 | |
a19741e5 | 212 | r = drm_gem_map_attach(dma_buf, attach); |
5a13761f CK |
213 | if (r) |
214 | return r; | |
215 | ||
216 | r = amdgpu_bo_reserve(bo, false); | |
217 | if (unlikely(r != 0)) | |
218 | goto error_detach; | |
d38ceaf9 | 219 | |
9021d2ed | 220 | |
2333bf9a | 221 | if (attach->dev->driver != adev->dev->driver) { |
9021d2ed CK |
222 | /* |
223 | * Wait for all shared fences to complete before we switch to future | |
224 | * use of exclusive fence on this prime shared bo. | |
225 | */ | |
226 | r = reservation_object_wait_timeout_rcu(bo->tbo.resv, | |
227 | true, false, | |
228 | MAX_SCHEDULE_TIMEOUT); | |
229 | if (unlikely(r < 0)) { | |
230 | DRM_DEBUG_PRIME("Fence wait failed: %li\n", r); | |
231 | goto error_unreserve; | |
232 | } | |
8e94a46c MK |
233 | } |
234 | ||
d38ceaf9 | 235 | /* pin buffer into GTT */ |
7b7c6c81 | 236 | r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT); |
9021d2ed CK |
237 | if (r) |
238 | goto error_unreserve; | |
239 | ||
2333bf9a | 240 | if (attach->dev->driver != adev->dev->driver) |
8e94a46c MK |
241 | bo->prime_shared_count++; |
242 | ||
5a13761f | 243 | error_unreserve: |
d38ceaf9 | 244 | amdgpu_bo_unreserve(bo); |
5a13761f CK |
245 | |
246 | error_detach: | |
247 | if (r) | |
248 | drm_gem_map_detach(dma_buf, attach); | |
249 | return r; | |
d38ceaf9 AD |
250 | } |
251 | ||
baca30fa MD |
252 | /** |
253 | * amdgpu_gem_map_detach - &dma_buf_ops.detach implementation | |
56ea0976 | 254 | * @dma_buf: Shared DMA buffer |
baca30fa MD |
255 | * @attach: DMA-buf attachment |
256 | * | |
257 | * This is called when a shared DMA buffer no longer needs to be accessible by | |
56ea0976 | 258 | * another device. For now, simply unpins the buffer from GTT. |
baca30fa | 259 | */ |
5a13761f CK |
260 | static void amdgpu_gem_map_detach(struct dma_buf *dma_buf, |
261 | struct dma_buf_attachment *attach) | |
d38ceaf9 | 262 | { |
5a13761f | 263 | struct drm_gem_object *obj = dma_buf->priv; |
d38ceaf9 | 264 | struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); |
2333bf9a | 265 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
d38ceaf9 AD |
266 | int ret = 0; |
267 | ||
c81a1a74 | 268 | ret = amdgpu_bo_reserve(bo, true); |
d38ceaf9 | 269 | if (unlikely(ret != 0)) |
5a13761f | 270 | goto error; |
d38ceaf9 AD |
271 | |
272 | amdgpu_bo_unpin(bo); | |
2333bf9a | 273 | if (attach->dev->driver != adev->dev->driver && bo->prime_shared_count) |
8e94a46c | 274 | bo->prime_shared_count--; |
d38ceaf9 | 275 | amdgpu_bo_unreserve(bo); |
5a13761f CK |
276 | |
277 | error: | |
278 | drm_gem_map_detach(dma_buf, attach); | |
d38ceaf9 AD |
279 | } |
280 | ||
baca30fa MD |
281 | /** |
282 | * amdgpu_gem_prime_res_obj - &drm_driver.gem_prime_res_obj implementation | |
56ea0976 | 283 | * @obj: GEM BO |
baca30fa MD |
284 | * |
285 | * Returns: | |
56ea0976 | 286 | * The BO's reservation object. |
baca30fa | 287 | */ |
d38ceaf9 AD |
288 | struct reservation_object *amdgpu_gem_prime_res_obj(struct drm_gem_object *obj) |
289 | { | |
290 | struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); | |
291 | ||
292 | return bo->tbo.resv; | |
293 | } | |
294 | ||
baca30fa MD |
295 | /** |
296 | * amdgpu_gem_begin_cpu_access - &dma_buf_ops.begin_cpu_access implementation | |
56ea0976 VM |
297 | * @dma_buf: Shared DMA buffer |
298 | * @direction: Direction of DMA transfer | |
baca30fa MD |
299 | * |
300 | * This is called before CPU access to the shared DMA buffer's memory. If it's | |
301 | * a read access, the buffer is moved to the GTT domain if possible, for optimal | |
302 | * CPU read performance. | |
303 | * | |
304 | * Returns: | |
56ea0976 | 305 | * 0 on success or a negative error code on failure. |
baca30fa | 306 | */ |
09052fc3 SL |
307 | static int amdgpu_gem_begin_cpu_access(struct dma_buf *dma_buf, |
308 | enum dma_data_direction direction) | |
309 | { | |
310 | struct amdgpu_bo *bo = gem_to_amdgpu_bo(dma_buf->priv); | |
311 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); | |
312 | struct ttm_operation_ctx ctx = { true, false }; | |
1d2361e5 | 313 | u32 domain = amdgpu_display_supported_domains(adev); |
09052fc3 SL |
314 | int ret; |
315 | bool reads = (direction == DMA_BIDIRECTIONAL || | |
316 | direction == DMA_FROM_DEVICE); | |
317 | ||
318 | if (!reads || !(domain & AMDGPU_GEM_DOMAIN_GTT)) | |
319 | return 0; | |
320 | ||
321 | /* move to gtt */ | |
322 | ret = amdgpu_bo_reserve(bo, false); | |
323 | if (unlikely(ret != 0)) | |
324 | return ret; | |
325 | ||
326 | if (!bo->pin_count && (bo->allowed_domains & AMDGPU_GEM_DOMAIN_GTT)) { | |
c704ab18 | 327 | amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); |
09052fc3 SL |
328 | ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); |
329 | } | |
330 | ||
331 | amdgpu_bo_unreserve(bo); | |
332 | return ret; | |
333 | } | |
334 | ||
335 | static const struct dma_buf_ops amdgpu_dmabuf_ops = { | |
5a13761f CK |
336 | .attach = amdgpu_gem_map_attach, |
337 | .detach = amdgpu_gem_map_detach, | |
09052fc3 SL |
338 | .map_dma_buf = drm_gem_map_dma_buf, |
339 | .unmap_dma_buf = drm_gem_unmap_dma_buf, | |
340 | .release = drm_gem_dmabuf_release, | |
341 | .begin_cpu_access = amdgpu_gem_begin_cpu_access, | |
342 | .map = drm_gem_dmabuf_kmap, | |
09052fc3 | 343 | .unmap = drm_gem_dmabuf_kunmap, |
09052fc3 SL |
344 | .mmap = drm_gem_dmabuf_mmap, |
345 | .vmap = drm_gem_dmabuf_vmap, | |
346 | .vunmap = drm_gem_dmabuf_vunmap, | |
347 | }; | |
348 | ||
baca30fa MD |
349 | /** |
350 | * amdgpu_gem_prime_export - &drm_driver.gem_prime_export implementation | |
351 | * @dev: DRM device | |
56ea0976 VM |
352 | * @gobj: GEM BO |
353 | * @flags: Flags such as DRM_CLOEXEC and DRM_RDWR. | |
baca30fa MD |
354 | * |
355 | * The main work is done by the &drm_gem_prime_export helper, which in turn | |
356 | * uses &amdgpu_gem_prime_res_obj. | |
357 | * | |
358 | * Returns: | |
56ea0976 | 359 | * Shared DMA buffer representing the GEM BO from the given device. |
baca30fa | 360 | */ |
d38ceaf9 AD |
361 | struct dma_buf *amdgpu_gem_prime_export(struct drm_device *dev, |
362 | struct drm_gem_object *gobj, | |
363 | int flags) | |
364 | { | |
365 | struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj); | |
4b277247 | 366 | struct dma_buf *buf; |
d38ceaf9 | 367 | |
e1eb899b CK |
368 | if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) || |
369 | bo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) | |
d38ceaf9 AD |
370 | return ERR_PTR(-EPERM); |
371 | ||
4b277247 | 372 | buf = drm_gem_prime_export(dev, gobj, flags); |
09052fc3 | 373 | if (!IS_ERR(buf)) { |
4b277247 | 374 | buf->file->f_mapping = dev->anon_inode->i_mapping; |
09052fc3 SL |
375 | buf->ops = &amdgpu_dmabuf_ops; |
376 | } | |
377 | ||
4b277247 | 378 | return buf; |
d38ceaf9 | 379 | } |
09052fc3 | 380 | |
baca30fa MD |
381 | /** |
382 | * amdgpu_gem_prime_import - &drm_driver.gem_prime_import implementation | |
383 | * @dev: DRM device | |
384 | * @dma_buf: Shared DMA buffer | |
385 | * | |
386 | * The main work is done by the &drm_gem_prime_import helper, which in turn | |
387 | * uses &amdgpu_gem_prime_import_sg_table. | |
388 | * | |
389 | * Returns: | |
56ea0976 | 390 | * GEM BO representing the shared DMA buffer for the given device. |
baca30fa | 391 | */ |
09052fc3 SL |
392 | struct drm_gem_object *amdgpu_gem_prime_import(struct drm_device *dev, |
393 | struct dma_buf *dma_buf) | |
394 | { | |
395 | struct drm_gem_object *obj; | |
396 | ||
397 | if (dma_buf->ops == &amdgpu_dmabuf_ops) { | |
398 | obj = dma_buf->priv; | |
399 | if (obj->dev == dev) { | |
400 | /* | |
401 | * Importing dmabuf exported from out own gem increases | |
402 | * refcount on gem itself instead of f_count of dmabuf. | |
403 | */ | |
404 | drm_gem_object_get(obj); | |
405 | return obj; | |
406 | } | |
407 | } | |
408 | ||
409 | return drm_gem_prime_import(dev, dma_buf); | |
410 | } |