1 // SPDX-License-Identifier: GPL-2.0-or-later
3 #include <drm/drm_gem_vram_helper.h>
4 #include <drm/drm_device.h>
5 #include <drm/drm_mode.h>
6 #include <drm/drm_prime.h>
7 #include <drm/drm_vram_mm_helper.h>
8 #include <drm/ttm/ttm_page_alloc.h>
10 static const struct drm_gem_object_funcs drm_gem_vram_object_funcs
;
15 * This library provides a GEM buffer object that is backed by video RAM
16 * (VRAM). It can be used for framebuffer devices with dedicated memory.
20 * Buffer-objects helpers
23 static void drm_gem_vram_cleanup(struct drm_gem_vram_object
*gbo
)
25 /* We got here via ttm_bo_put(), which means that the
26 * TTM buffer object in 'bo' has already been cleaned
27 * up; only release the GEM object.
29 drm_gem_object_release(&gbo
->bo
.base
);
32 static void drm_gem_vram_destroy(struct drm_gem_vram_object
*gbo
)
34 drm_gem_vram_cleanup(gbo
);
38 static void ttm_buffer_object_destroy(struct ttm_buffer_object
*bo
)
40 struct drm_gem_vram_object
*gbo
= drm_gem_vram_of_bo(bo
);
42 drm_gem_vram_destroy(gbo
);
45 static void drm_gem_vram_placement(struct drm_gem_vram_object
*gbo
,
46 unsigned long pl_flag
)
51 gbo
->placement
.placement
= gbo
->placements
;
52 gbo
->placement
.busy_placement
= gbo
->placements
;
54 if (pl_flag
& TTM_PL_FLAG_VRAM
)
55 gbo
->placements
[c
++].flags
= TTM_PL_FLAG_WC
|
56 TTM_PL_FLAG_UNCACHED
|
59 if (pl_flag
& TTM_PL_FLAG_SYSTEM
)
60 gbo
->placements
[c
++].flags
= TTM_PL_MASK_CACHING
|
64 gbo
->placements
[c
++].flags
= TTM_PL_MASK_CACHING
|
67 gbo
->placement
.num_placement
= c
;
68 gbo
->placement
.num_busy_placement
= c
;
70 for (i
= 0; i
< c
; ++i
) {
71 gbo
->placements
[i
].fpfn
= 0;
72 gbo
->placements
[i
].lpfn
= 0;
76 static int drm_gem_vram_init(struct drm_device
*dev
,
77 struct ttm_bo_device
*bdev
,
78 struct drm_gem_vram_object
*gbo
,
79 size_t size
, unsigned long pg_align
,
85 if (!gbo
->bo
.base
.funcs
)
86 gbo
->bo
.base
.funcs
= &drm_gem_vram_object_funcs
;
88 ret
= drm_gem_object_init(dev
, &gbo
->bo
.base
, size
);
92 acc_size
= ttm_bo_dma_acc_size(bdev
, size
, sizeof(*gbo
));
95 drm_gem_vram_placement(gbo
, TTM_PL_FLAG_VRAM
| TTM_PL_FLAG_SYSTEM
);
97 ret
= ttm_bo_init(bdev
, &gbo
->bo
, size
, ttm_bo_type_device
,
98 &gbo
->placement
, pg_align
, interruptible
, acc_size
,
99 NULL
, NULL
, ttm_buffer_object_destroy
);
101 goto err_drm_gem_object_release
;
105 err_drm_gem_object_release
:
106 drm_gem_object_release(&gbo
->bo
.base
);
111 * drm_gem_vram_create() - Creates a VRAM-backed GEM object
112 * @dev: the DRM device
113 * @bdev: the TTM BO device backing the object
114 * @size: the buffer size in bytes
115 * @pg_align: the buffer's alignment in multiples of the page size
116 * @interruptible: sleep interruptible if waiting for memory
119 * A new instance of &struct drm_gem_vram_object on success, or
120 * an ERR_PTR()-encoded error code otherwise.
122 struct drm_gem_vram_object
*drm_gem_vram_create(struct drm_device
*dev
,
123 struct ttm_bo_device
*bdev
,
125 unsigned long pg_align
,
128 struct drm_gem_vram_object
*gbo
;
131 gbo
= kzalloc(sizeof(*gbo
), GFP_KERNEL
);
133 return ERR_PTR(-ENOMEM
);
135 ret
= drm_gem_vram_init(dev
, bdev
, gbo
, size
, pg_align
, interruptible
);
145 EXPORT_SYMBOL(drm_gem_vram_create
);
148 * drm_gem_vram_put() - Releases a reference to a VRAM-backed GEM object
149 * @gbo: the GEM VRAM object
151 * See ttm_bo_put() for more information.
153 void drm_gem_vram_put(struct drm_gem_vram_object
*gbo
)
155 ttm_bo_put(&gbo
->bo
);
157 EXPORT_SYMBOL(drm_gem_vram_put
);
160 * drm_gem_vram_mmap_offset() - Returns a GEM VRAM object's mmap offset
161 * @gbo: the GEM VRAM object
163 * See drm_vma_node_offset_addr() for more information.
166 * The buffer object's offset for userspace mappings on success, or
167 * 0 if no offset is allocated.
169 u64
drm_gem_vram_mmap_offset(struct drm_gem_vram_object
*gbo
)
171 return drm_vma_node_offset_addr(&gbo
->bo
.base
.vma_node
);
173 EXPORT_SYMBOL(drm_gem_vram_mmap_offset
);
176 * drm_gem_vram_offset() - \
177 Returns a GEM VRAM object's offset in video memory
178 * @gbo: the GEM VRAM object
180 * This function returns the buffer object's offset in the device's video
181 * memory. The buffer object has to be pinned to %TTM_PL_VRAM.
184 * The buffer object's offset in video memory on success, or
185 * a negative errno code otherwise.
187 s64
drm_gem_vram_offset(struct drm_gem_vram_object
*gbo
)
189 if (WARN_ON_ONCE(!gbo
->pin_count
))
191 return gbo
->bo
.offset
;
193 EXPORT_SYMBOL(drm_gem_vram_offset
);
196 * drm_gem_vram_pin() - Pins a GEM VRAM object in a region.
197 * @gbo: the GEM VRAM object
198 * @pl_flag: a bitmask of possible memory regions
200 * Pinning a buffer object ensures that it is not evicted from
201 * a memory region. A pinned buffer object has to be unpinned before
202 * it can be pinned to another region. If the pl_flag argument is 0,
203 * the buffer is pinned at its current location (video RAM or system
208 * a negative error code otherwise.
210 int drm_gem_vram_pin(struct drm_gem_vram_object
*gbo
, unsigned long pl_flag
)
213 struct ttm_operation_ctx ctx
= { false, false };
215 ret
= ttm_bo_reserve(&gbo
->bo
, true, false, NULL
);
223 drm_gem_vram_placement(gbo
, pl_flag
);
225 for (i
= 0; i
< gbo
->placement
.num_placement
; ++i
)
226 gbo
->placements
[i
].flags
|= TTM_PL_FLAG_NO_EVICT
;
228 ret
= ttm_bo_validate(&gbo
->bo
, &gbo
->placement
, &ctx
);
230 goto err_ttm_bo_unreserve
;
234 ttm_bo_unreserve(&gbo
->bo
);
238 err_ttm_bo_unreserve
:
239 ttm_bo_unreserve(&gbo
->bo
);
242 EXPORT_SYMBOL(drm_gem_vram_pin
);
245 * drm_gem_vram_unpin() - Unpins a GEM VRAM object
246 * @gbo: the GEM VRAM object
250 * a negative error code otherwise.
252 int drm_gem_vram_unpin(struct drm_gem_vram_object
*gbo
)
255 struct ttm_operation_ctx ctx
= { false, false };
257 ret
= ttm_bo_reserve(&gbo
->bo
, true, false, NULL
);
261 if (WARN_ON_ONCE(!gbo
->pin_count
))
268 for (i
= 0; i
< gbo
->placement
.num_placement
; ++i
)
269 gbo
->placements
[i
].flags
&= ~TTM_PL_FLAG_NO_EVICT
;
271 ret
= ttm_bo_validate(&gbo
->bo
, &gbo
->placement
, &ctx
);
273 goto err_ttm_bo_unreserve
;
276 ttm_bo_unreserve(&gbo
->bo
);
280 err_ttm_bo_unreserve
:
281 ttm_bo_unreserve(&gbo
->bo
);
284 EXPORT_SYMBOL(drm_gem_vram_unpin
);
287 * drm_gem_vram_kmap() - Maps a GEM VRAM object into kernel address space
288 * @gbo: the GEM VRAM object
289 * @map: establish a mapping if necessary
290 * @is_iomem: returns true if the mapped memory is I/O memory, or false \
291 otherwise; can be NULL
293 * This function maps the buffer object into the kernel's address space
294 * or returns the current mapping. If the parameter map is false, the
295 * function only queries the current mapping, but does not establish a
299 * The buffers virtual address if mapped, or
300 * NULL if not mapped, or
301 * an ERR_PTR()-encoded error code otherwise.
303 void *drm_gem_vram_kmap(struct drm_gem_vram_object
*gbo
, bool map
,
307 struct ttm_bo_kmap_obj
*kmap
= &gbo
->kmap
;
309 if (kmap
->virtual || !map
)
312 ret
= ttm_bo_kmap(&gbo
->bo
, 0, gbo
->bo
.num_pages
, kmap
);
318 return kmap
->virtual;
319 if (!kmap
->virtual) {
323 return ttm_kmap_obj_virtual(kmap
, is_iomem
);
325 EXPORT_SYMBOL(drm_gem_vram_kmap
);
328 * drm_gem_vram_kunmap() - Unmaps a GEM VRAM object
329 * @gbo: the GEM VRAM object
331 void drm_gem_vram_kunmap(struct drm_gem_vram_object
*gbo
)
333 struct ttm_bo_kmap_obj
*kmap
= &gbo
->kmap
;
339 kmap
->virtual = NULL
;
341 EXPORT_SYMBOL(drm_gem_vram_kunmap
);
344 * drm_gem_vram_fill_create_dumb() - \
345 Helper for implementing &struct drm_driver.dumb_create
346 * @file: the DRM file
347 * @dev: the DRM device
348 * @bdev: the TTM BO device managing the buffer object
349 * @pg_align: the buffer's alignment in multiples of the page size
350 * @interruptible: sleep interruptible if waiting for memory
351 * @args: the arguments as provided to \
352 &struct drm_driver.dumb_create
354 * This helper function fills &struct drm_mode_create_dumb, which is used
355 * by &struct drm_driver.dumb_create. Implementations of this interface
356 * should forwards their arguments to this helper, plus the driver-specific
361 * a negative error code otherwise.
363 int drm_gem_vram_fill_create_dumb(struct drm_file
*file
,
364 struct drm_device
*dev
,
365 struct ttm_bo_device
*bdev
,
366 unsigned long pg_align
,
368 struct drm_mode_create_dumb
*args
)
371 struct drm_gem_vram_object
*gbo
;
375 pitch
= args
->width
* ((args
->bpp
+ 7) / 8);
376 size
= pitch
* args
->height
;
378 size
= roundup(size
, PAGE_SIZE
);
382 gbo
= drm_gem_vram_create(dev
, bdev
, size
, pg_align
, interruptible
);
386 ret
= drm_gem_handle_create(file
, &gbo
->bo
.base
, &handle
);
388 goto err_drm_gem_object_put_unlocked
;
390 drm_gem_object_put_unlocked(&gbo
->bo
.base
);
394 args
->handle
= handle
;
398 err_drm_gem_object_put_unlocked
:
399 drm_gem_object_put_unlocked(&gbo
->bo
.base
);
402 EXPORT_SYMBOL(drm_gem_vram_fill_create_dumb
);
405 * Helpers for struct ttm_bo_driver
408 static bool drm_is_gem_vram(struct ttm_buffer_object
*bo
)
410 return (bo
->destroy
== ttm_buffer_object_destroy
);
414 * drm_gem_vram_bo_driver_evict_flags() - \
415 Implements &struct ttm_bo_driver.evict_flags
416 * @bo: TTM buffer object. Refers to &struct drm_gem_vram_object.bo
417 * @pl: TTM placement information.
419 void drm_gem_vram_bo_driver_evict_flags(struct ttm_buffer_object
*bo
,
420 struct ttm_placement
*pl
)
422 struct drm_gem_vram_object
*gbo
;
424 /* TTM may pass BOs that are not GEM VRAM BOs. */
425 if (!drm_is_gem_vram(bo
))
428 gbo
= drm_gem_vram_of_bo(bo
);
429 drm_gem_vram_placement(gbo
, TTM_PL_FLAG_SYSTEM
);
430 *pl
= gbo
->placement
;
432 EXPORT_SYMBOL(drm_gem_vram_bo_driver_evict_flags
);
435 * drm_gem_vram_bo_driver_verify_access() - \
436 Implements &struct ttm_bo_driver.verify_access
437 * @bo: TTM buffer object. Refers to &struct drm_gem_vram_object.bo
438 * @filp: File pointer.
442 * a negative errno code otherwise.
444 int drm_gem_vram_bo_driver_verify_access(struct ttm_buffer_object
*bo
,
447 struct drm_gem_vram_object
*gbo
= drm_gem_vram_of_bo(bo
);
449 return drm_vma_node_verify_access(&gbo
->bo
.base
.vma_node
,
452 EXPORT_SYMBOL(drm_gem_vram_bo_driver_verify_access
);
455 * drm_gem_vram_mm_funcs - Functions for &struct drm_vram_mm
457 * Most users of @struct drm_gem_vram_object will also use
458 * @struct drm_vram_mm. This instance of &struct drm_vram_mm_funcs
459 * can be used to connect both.
461 const struct drm_vram_mm_funcs drm_gem_vram_mm_funcs
= {
462 .evict_flags
= drm_gem_vram_bo_driver_evict_flags
,
463 .verify_access
= drm_gem_vram_bo_driver_verify_access
465 EXPORT_SYMBOL(drm_gem_vram_mm_funcs
);
468 * Helpers for struct drm_gem_object_funcs
472 * drm_gem_vram_object_free() - \
473 Implements &struct drm_gem_object_funcs.free
474 * @gem: GEM object. Refers to &struct drm_gem_vram_object.gem
476 static void drm_gem_vram_object_free(struct drm_gem_object
*gem
)
478 struct drm_gem_vram_object
*gbo
= drm_gem_vram_of_gem(gem
);
480 drm_gem_vram_put(gbo
);
484 * Helpers for dump buffers
488 * drm_gem_vram_driver_create_dumb() - \
489 Implements &struct drm_driver.dumb_create
490 * @file: the DRM file
491 * @dev: the DRM device
492 * @args: the arguments as provided to \
493 &struct drm_driver.dumb_create
495 * This function requires the driver to use @drm_device.vram_mm for its
496 * instance of VRAM MM.
500 * a negative error code otherwise.
502 int drm_gem_vram_driver_dumb_create(struct drm_file
*file
,
503 struct drm_device
*dev
,
504 struct drm_mode_create_dumb
*args
)
506 if (WARN_ONCE(!dev
->vram_mm
, "VRAM MM not initialized"))
509 return drm_gem_vram_fill_create_dumb(file
, dev
, &dev
->vram_mm
->bdev
, 0,
512 EXPORT_SYMBOL(drm_gem_vram_driver_dumb_create
);
515 * drm_gem_vram_driver_dumb_mmap_offset() - \
516 Implements &struct drm_driver.dumb_mmap_offset
517 * @file: DRM file pointer.
519 * @handle: GEM handle
520 * @offset: Returns the mapping's memory offset on success
524 * a negative errno code otherwise.
526 int drm_gem_vram_driver_dumb_mmap_offset(struct drm_file
*file
,
527 struct drm_device
*dev
,
528 uint32_t handle
, uint64_t *offset
)
530 struct drm_gem_object
*gem
;
531 struct drm_gem_vram_object
*gbo
;
533 gem
= drm_gem_object_lookup(file
, handle
);
537 gbo
= drm_gem_vram_of_gem(gem
);
538 *offset
= drm_gem_vram_mmap_offset(gbo
);
540 drm_gem_object_put_unlocked(gem
);
544 EXPORT_SYMBOL(drm_gem_vram_driver_dumb_mmap_offset
);
551 * drm_gem_vram_object_pin() - \
552 Implements &struct drm_gem_object_funcs.pin
553 * @gem: The GEM object to pin
557 * a negative errno code otherwise.
559 static int drm_gem_vram_object_pin(struct drm_gem_object
*gem
)
561 struct drm_gem_vram_object
*gbo
= drm_gem_vram_of_gem(gem
);
563 /* Fbdev console emulation is the use case of these PRIME
564 * helpers. This may involve updating a hardware buffer from
565 * a shadow FB. We pin the buffer to it's current location
566 * (either video RAM or system memory) to prevent it from
567 * being relocated during the update operation. If you require
568 * the buffer to be pinned to VRAM, implement a callback that
569 * sets the flags accordingly.
571 return drm_gem_vram_pin(gbo
, 0);
575 * drm_gem_vram_object_unpin() - \
576 Implements &struct drm_gem_object_funcs.unpin
577 * @gem: The GEM object to unpin
579 static void drm_gem_vram_object_unpin(struct drm_gem_object
*gem
)
581 struct drm_gem_vram_object
*gbo
= drm_gem_vram_of_gem(gem
);
583 drm_gem_vram_unpin(gbo
);
587 * drm_gem_vram_object_vmap() - \
588 Implements &struct drm_gem_object_funcs.vmap
589 * @gem: The GEM object to map
592 * The buffers virtual address on success, or
595 static void *drm_gem_vram_object_vmap(struct drm_gem_object
*gem
)
597 struct drm_gem_vram_object
*gbo
= drm_gem_vram_of_gem(gem
);
601 ret
= drm_gem_vram_pin(gbo
, 0);
604 base
= drm_gem_vram_kmap(gbo
, true, NULL
);
606 drm_gem_vram_unpin(gbo
);
613 * drm_gem_vram_object_vunmap() - \
614 Implements &struct drm_gem_object_funcs.vunmap
615 * @gem: The GEM object to unmap
616 * @vaddr: The mapping's base address
618 static void drm_gem_vram_object_vunmap(struct drm_gem_object
*gem
,
621 struct drm_gem_vram_object
*gbo
= drm_gem_vram_of_gem(gem
);
623 drm_gem_vram_kunmap(gbo
);
624 drm_gem_vram_unpin(gbo
);
631 static const struct drm_gem_object_funcs drm_gem_vram_object_funcs
= {
632 .free
= drm_gem_vram_object_free
,
633 .pin
= drm_gem_vram_object_pin
,
634 .unpin
= drm_gem_vram_object_unpin
,
635 .vmap
= drm_gem_vram_object_vmap
,
636 .vunmap
= drm_gem_vram_object_vunmap