1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /**************************************************************************
4 * Copyright 2009-2015 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 #include "vmwgfx_drv.h"
29 #include <drm/ttm/ttm_bo_driver.h>
30 #include <drm/ttm/ttm_placement.h>
32 static const struct ttm_place vram_placement_flags
= {
35 .mem_type
= TTM_PL_VRAM
,
39 static const struct ttm_place sys_placement_flags
= {
42 .mem_type
= TTM_PL_SYSTEM
,
46 static const struct ttm_place gmr_placement_flags
= {
49 .mem_type
= VMW_PL_GMR
,
53 static const struct ttm_place mob_placement_flags
= {
56 .mem_type
= VMW_PL_MOB
,
60 struct ttm_placement vmw_vram_placement
= {
62 .placement
= &vram_placement_flags
,
63 .num_busy_placement
= 1,
64 .busy_placement
= &vram_placement_flags
67 static const struct ttm_place vram_gmr_placement_flags
[] = {
71 .mem_type
= TTM_PL_VRAM
,
76 .mem_type
= VMW_PL_GMR
,
81 static const struct ttm_place gmr_vram_placement_flags
[] = {
85 .mem_type
= VMW_PL_GMR
,
90 .mem_type
= TTM_PL_VRAM
,
95 struct ttm_placement vmw_vram_gmr_placement
= {
97 .placement
= vram_gmr_placement_flags
,
98 .num_busy_placement
= 1,
99 .busy_placement
= &gmr_placement_flags
102 struct ttm_placement vmw_vram_sys_placement
= {
104 .placement
= &vram_placement_flags
,
105 .num_busy_placement
= 1,
106 .busy_placement
= &sys_placement_flags
109 struct ttm_placement vmw_sys_placement
= {
111 .placement
= &sys_placement_flags
,
112 .num_busy_placement
= 1,
113 .busy_placement
= &sys_placement_flags
116 static const struct ttm_place evictable_placement_flags
[] = {
120 .mem_type
= TTM_PL_SYSTEM
,
125 .mem_type
= TTM_PL_VRAM
,
130 .mem_type
= VMW_PL_GMR
,
135 .mem_type
= VMW_PL_MOB
,
140 static const struct ttm_place nonfixed_placement_flags
[] = {
144 .mem_type
= TTM_PL_SYSTEM
,
149 .mem_type
= VMW_PL_GMR
,
154 .mem_type
= VMW_PL_MOB
,
159 struct ttm_placement vmw_evictable_placement
= {
161 .placement
= evictable_placement_flags
,
162 .num_busy_placement
= 1,
163 .busy_placement
= &sys_placement_flags
166 struct ttm_placement vmw_srf_placement
= {
168 .num_busy_placement
= 2,
169 .placement
= &gmr_placement_flags
,
170 .busy_placement
= gmr_vram_placement_flags
173 struct ttm_placement vmw_mob_placement
= {
175 .num_busy_placement
= 1,
176 .placement
= &mob_placement_flags
,
177 .busy_placement
= &mob_placement_flags
180 struct ttm_placement vmw_nonfixed_placement
= {
182 .placement
= nonfixed_placement_flags
,
183 .num_busy_placement
= 1,
184 .busy_placement
= &sys_placement_flags
188 struct ttm_tt dma_ttm
;
189 struct vmw_private
*dev_priv
;
194 struct vmw_sg_table vsgt
;
195 uint64_t sg_alloc_size
;
200 const size_t vmw_tt_size
= sizeof(struct vmw_ttm_tt
);
203 * __vmw_piter_non_sg_next: Helper functions to advance
204 * a struct vmw_piter iterator.
206 * @viter: Pointer to the iterator.
208 * These functions return false if past the end of the list,
209 * true otherwise. Functions are selected depending on the current
212 static bool __vmw_piter_non_sg_next(struct vmw_piter
*viter
)
214 return ++(viter
->i
) < viter
->num_pages
;
217 static bool __vmw_piter_sg_next(struct vmw_piter
*viter
)
219 bool ret
= __vmw_piter_non_sg_next(viter
);
221 return __sg_page_iter_dma_next(&viter
->iter
) && ret
;
226 * __vmw_piter_non_sg_page: Helper functions to return a pointer
227 * to the current page.
229 * @viter: Pointer to the iterator
231 * These functions return a pointer to the page currently
232 * pointed to by @viter. Functions are selected depending on the
233 * current mapping mode.
235 static struct page
*__vmw_piter_non_sg_page(struct vmw_piter
*viter
)
237 return viter
->pages
[viter
->i
];
241 * __vmw_piter_phys_addr: Helper functions to return the DMA
242 * address of the current page.
244 * @viter: Pointer to the iterator
246 * These functions return the DMA address of the page currently
247 * pointed to by @viter. Functions are selected depending on the
248 * current mapping mode.
250 static dma_addr_t
__vmw_piter_phys_addr(struct vmw_piter
*viter
)
252 return page_to_phys(viter
->pages
[viter
->i
]);
255 static dma_addr_t
__vmw_piter_dma_addr(struct vmw_piter
*viter
)
257 return viter
->addrs
[viter
->i
];
260 static dma_addr_t
__vmw_piter_sg_addr(struct vmw_piter
*viter
)
262 return sg_page_iter_dma_address(&viter
->iter
);
267 * vmw_piter_start - Initialize a struct vmw_piter.
269 * @viter: Pointer to the iterator to initialize
270 * @vsgt: Pointer to a struct vmw_sg_table to initialize from
271 * @p_offset: Pointer offset used to update current array position
273 * Note that we're following the convention of __sg_page_iter_start, so that
274 * the iterator doesn't point to a valid page after initialization; it has
275 * to be advanced one step first.
277 void vmw_piter_start(struct vmw_piter
*viter
, const struct vmw_sg_table
*vsgt
,
278 unsigned long p_offset
)
280 viter
->i
= p_offset
- 1;
281 viter
->num_pages
= vsgt
->num_pages
;
282 viter
->page
= &__vmw_piter_non_sg_page
;
283 viter
->pages
= vsgt
->pages
;
284 switch (vsgt
->mode
) {
286 viter
->next
= &__vmw_piter_non_sg_next
;
287 viter
->dma_address
= &__vmw_piter_phys_addr
;
289 case vmw_dma_alloc_coherent
:
290 viter
->next
= &__vmw_piter_non_sg_next
;
291 viter
->dma_address
= &__vmw_piter_dma_addr
;
292 viter
->addrs
= vsgt
->addrs
;
294 case vmw_dma_map_populate
:
295 case vmw_dma_map_bind
:
296 viter
->next
= &__vmw_piter_sg_next
;
297 viter
->dma_address
= &__vmw_piter_sg_addr
;
298 __sg_page_iter_start(&viter
->iter
.base
, vsgt
->sgt
->sgl
,
299 vsgt
->sgt
->orig_nents
, p_offset
);
307 * vmw_ttm_unmap_from_dma - unmap device addresses previsouly mapped for
310 * @vmw_tt: Pointer to a struct vmw_ttm_backend
312 * Used to free dma mappings previously mapped by vmw_ttm_map_for_dma.
314 static void vmw_ttm_unmap_from_dma(struct vmw_ttm_tt
*vmw_tt
)
316 struct device
*dev
= vmw_tt
->dev_priv
->drm
.dev
;
318 dma_unmap_sgtable(dev
, &vmw_tt
->sgt
, DMA_BIDIRECTIONAL
, 0);
319 vmw_tt
->sgt
.nents
= vmw_tt
->sgt
.orig_nents
;
323 * vmw_ttm_map_for_dma - map TTM pages to get device addresses
325 * @vmw_tt: Pointer to a struct vmw_ttm_backend
327 * This function is used to get device addresses from the kernel DMA layer.
328 * However, it's violating the DMA API in that when this operation has been
329 * performed, it's illegal for the CPU to write to the pages without first
330 * unmapping the DMA mappings, or calling dma_sync_sg_for_cpu(). It is
331 * therefore only legal to call this function if we know that the function
332 * dma_sync_sg_for_cpu() is a NOP, and dma_sync_sg_for_device() is at most
333 * a CPU write buffer flush.
335 static int vmw_ttm_map_for_dma(struct vmw_ttm_tt
*vmw_tt
)
337 struct device
*dev
= vmw_tt
->dev_priv
->drm
.dev
;
339 return dma_map_sgtable(dev
, &vmw_tt
->sgt
, DMA_BIDIRECTIONAL
, 0);
343 * vmw_ttm_map_dma - Make sure TTM pages are visible to the device
345 * @vmw_tt: Pointer to a struct vmw_ttm_tt
347 * Select the correct function for and make sure the TTM pages are
348 * visible to the device. Allocate storage for the device mappings.
349 * If a mapping has already been performed, indicated by the storage
350 * pointer being non NULL, the function returns success.
352 static int vmw_ttm_map_dma(struct vmw_ttm_tt
*vmw_tt
)
354 struct vmw_private
*dev_priv
= vmw_tt
->dev_priv
;
355 struct ttm_mem_global
*glob
= vmw_mem_glob(dev_priv
);
356 struct vmw_sg_table
*vsgt
= &vmw_tt
->vsgt
;
357 struct ttm_operation_ctx ctx
= {
358 .interruptible
= true,
361 struct vmw_piter iter
;
364 static size_t sgl_size
;
365 static size_t sgt_size
;
366 struct scatterlist
*sg
;
371 vsgt
->mode
= dev_priv
->map_mode
;
372 vsgt
->pages
= vmw_tt
->dma_ttm
.pages
;
373 vsgt
->num_pages
= vmw_tt
->dma_ttm
.num_pages
;
374 vsgt
->addrs
= vmw_tt
->dma_ttm
.dma_address
;
375 vsgt
->sgt
= &vmw_tt
->sgt
;
377 switch (dev_priv
->map_mode
) {
378 case vmw_dma_map_bind
:
379 case vmw_dma_map_populate
:
380 if (unlikely(!sgl_size
)) {
381 sgl_size
= ttm_round_pot(sizeof(struct scatterlist
));
382 sgt_size
= ttm_round_pot(sizeof(struct sg_table
));
384 vmw_tt
->sg_alloc_size
= sgt_size
+ sgl_size
* vsgt
->num_pages
;
385 ret
= ttm_mem_global_alloc(glob
, vmw_tt
->sg_alloc_size
, &ctx
);
386 if (unlikely(ret
!= 0))
389 sg
= __sg_alloc_table_from_pages(&vmw_tt
->sgt
, vsgt
->pages
,
391 (unsigned long) vsgt
->num_pages
<< PAGE_SHIFT
,
392 dma_get_max_seg_size(dev_priv
->drm
.dev
),
393 NULL
, 0, GFP_KERNEL
);
396 goto out_sg_alloc_fail
;
399 if (vsgt
->num_pages
> vmw_tt
->sgt
.orig_nents
) {
400 uint64_t over_alloc
=
401 sgl_size
* (vsgt
->num_pages
-
402 vmw_tt
->sgt
.orig_nents
);
404 ttm_mem_global_free(glob
, over_alloc
);
405 vmw_tt
->sg_alloc_size
-= over_alloc
;
408 ret
= vmw_ttm_map_for_dma(vmw_tt
);
409 if (unlikely(ret
!= 0))
417 old
= ~((dma_addr_t
) 0);
418 vmw_tt
->vsgt
.num_regions
= 0;
419 for (vmw_piter_start(&iter
, vsgt
, 0); vmw_piter_next(&iter
);) {
420 dma_addr_t cur
= vmw_piter_dma_addr(&iter
);
422 if (cur
!= old
+ PAGE_SIZE
)
423 vmw_tt
->vsgt
.num_regions
++;
427 vmw_tt
->mapped
= true;
431 sg_free_table(vmw_tt
->vsgt
.sgt
);
432 vmw_tt
->vsgt
.sgt
= NULL
;
434 ttm_mem_global_free(glob
, vmw_tt
->sg_alloc_size
);
439 * vmw_ttm_unmap_dma - Tear down any TTM page device mappings
441 * @vmw_tt: Pointer to a struct vmw_ttm_tt
443 * Tear down any previously set up device DMA mappings and free
444 * any storage space allocated for them. If there are no mappings set up,
445 * this function is a NOP.
447 static void vmw_ttm_unmap_dma(struct vmw_ttm_tt
*vmw_tt
)
449 struct vmw_private
*dev_priv
= vmw_tt
->dev_priv
;
451 if (!vmw_tt
->vsgt
.sgt
)
454 switch (dev_priv
->map_mode
) {
455 case vmw_dma_map_bind
:
456 case vmw_dma_map_populate
:
457 vmw_ttm_unmap_from_dma(vmw_tt
);
458 sg_free_table(vmw_tt
->vsgt
.sgt
);
459 vmw_tt
->vsgt
.sgt
= NULL
;
460 ttm_mem_global_free(vmw_mem_glob(dev_priv
),
461 vmw_tt
->sg_alloc_size
);
466 vmw_tt
->mapped
= false;
470 * vmw_bo_sg_table - Return a struct vmw_sg_table object for a
473 * @bo: Pointer to a struct ttm_buffer_object
475 * Returns a pointer to a struct vmw_sg_table object. The object should
476 * not be freed after use.
477 * Note that for the device addresses to be valid, the buffer object must
478 * either be reserved or pinned.
480 const struct vmw_sg_table
*vmw_bo_sg_table(struct ttm_buffer_object
*bo
)
482 struct vmw_ttm_tt
*vmw_tt
=
483 container_of(bo
->ttm
, struct vmw_ttm_tt
, dma_ttm
);
485 return &vmw_tt
->vsgt
;
489 static int vmw_ttm_bind(struct ttm_device
*bdev
,
490 struct ttm_tt
*ttm
, struct ttm_resource
*bo_mem
)
492 struct vmw_ttm_tt
*vmw_be
=
493 container_of(ttm
, struct vmw_ttm_tt
, dma_ttm
);
502 ret
= vmw_ttm_map_dma(vmw_be
);
503 if (unlikely(ret
!= 0))
506 vmw_be
->gmr_id
= bo_mem
->start
;
507 vmw_be
->mem_type
= bo_mem
->mem_type
;
509 switch (bo_mem
->mem_type
) {
511 ret
= vmw_gmr_bind(vmw_be
->dev_priv
, &vmw_be
->vsgt
,
512 ttm
->num_pages
, vmw_be
->gmr_id
);
515 if (unlikely(vmw_be
->mob
== NULL
)) {
517 vmw_mob_create(ttm
->num_pages
);
518 if (unlikely(vmw_be
->mob
== NULL
))
522 ret
= vmw_mob_bind(vmw_be
->dev_priv
, vmw_be
->mob
,
523 &vmw_be
->vsgt
, ttm
->num_pages
,
529 vmw_be
->bound
= true;
533 static void vmw_ttm_unbind(struct ttm_device
*bdev
,
536 struct vmw_ttm_tt
*vmw_be
=
537 container_of(ttm
, struct vmw_ttm_tt
, dma_ttm
);
542 switch (vmw_be
->mem_type
) {
544 vmw_gmr_unbind(vmw_be
->dev_priv
, vmw_be
->gmr_id
);
547 vmw_mob_unbind(vmw_be
->dev_priv
, vmw_be
->mob
);
553 if (vmw_be
->dev_priv
->map_mode
== vmw_dma_map_bind
)
554 vmw_ttm_unmap_dma(vmw_be
);
555 vmw_be
->bound
= false;
559 static void vmw_ttm_destroy(struct ttm_device
*bdev
, struct ttm_tt
*ttm
)
561 struct vmw_ttm_tt
*vmw_be
=
562 container_of(ttm
, struct vmw_ttm_tt
, dma_ttm
);
564 vmw_ttm_unbind(bdev
, ttm
);
565 ttm_tt_destroy_common(bdev
, ttm
);
566 vmw_ttm_unmap_dma(vmw_be
);
567 if (vmw_be
->dev_priv
->map_mode
== vmw_dma_alloc_coherent
)
568 ttm_tt_fini(&vmw_be
->dma_ttm
);
573 vmw_mob_destroy(vmw_be
->mob
);
579 static int vmw_ttm_populate(struct ttm_device
*bdev
,
580 struct ttm_tt
*ttm
, struct ttm_operation_ctx
*ctx
)
585 /* TODO: maybe completely drop this ? */
586 if (ttm_tt_is_populated(ttm
))
589 ret
= ttm_pool_alloc(&bdev
->pool
, ttm
, ctx
);
593 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
594 ret
= ttm_mem_global_alloc_page(&ttm_mem_glob
, ttm
->pages
[i
],
603 ttm_mem_global_free_page(&ttm_mem_glob
, ttm
->pages
[i
],
605 ttm_pool_free(&bdev
->pool
, ttm
);
609 static void vmw_ttm_unpopulate(struct ttm_device
*bdev
,
612 struct vmw_ttm_tt
*vmw_tt
= container_of(ttm
, struct vmw_ttm_tt
,
617 vmw_mob_destroy(vmw_tt
->mob
);
621 vmw_ttm_unmap_dma(vmw_tt
);
623 for (i
= 0; i
< ttm
->num_pages
; ++i
)
624 ttm_mem_global_free_page(&ttm_mem_glob
, ttm
->pages
[i
],
627 ttm_pool_free(&bdev
->pool
, ttm
);
630 static struct ttm_tt
*vmw_ttm_tt_create(struct ttm_buffer_object
*bo
,
633 struct vmw_ttm_tt
*vmw_be
;
636 vmw_be
= kzalloc(sizeof(*vmw_be
), GFP_KERNEL
);
640 vmw_be
->dev_priv
= container_of(bo
->bdev
, struct vmw_private
, bdev
);
643 if (vmw_be
->dev_priv
->map_mode
== vmw_dma_alloc_coherent
)
644 ret
= ttm_sg_tt_init(&vmw_be
->dma_ttm
, bo
, page_flags
,
647 ret
= ttm_tt_init(&vmw_be
->dma_ttm
, bo
, page_flags
,
649 if (unlikely(ret
!= 0))
652 return &vmw_be
->dma_ttm
;
658 static void vmw_evict_flags(struct ttm_buffer_object
*bo
,
659 struct ttm_placement
*placement
)
661 *placement
= vmw_sys_placement
;
664 static int vmw_ttm_io_mem_reserve(struct ttm_device
*bdev
, struct ttm_resource
*mem
)
666 struct vmw_private
*dev_priv
= container_of(bdev
, struct vmw_private
, bdev
);
668 switch (mem
->mem_type
) {
674 mem
->bus
.offset
= (mem
->start
<< PAGE_SHIFT
) +
675 dev_priv
->vram_start
;
676 mem
->bus
.is_iomem
= true;
677 mem
->bus
.caching
= ttm_cached
;
686 * vmw_move_notify - TTM move_notify_callback
688 * @bo: The TTM buffer object about to move.
689 * @old_mem: The old memory where we move from
690 * @new_mem: The struct ttm_resource indicating to what memory
691 * region the move is taking place.
693 * Calls move_notify for all subsystems needing it.
694 * (currently only resources).
696 static void vmw_move_notify(struct ttm_buffer_object
*bo
,
697 struct ttm_resource
*old_mem
,
698 struct ttm_resource
*new_mem
)
700 vmw_bo_move_notify(bo
, new_mem
);
701 vmw_query_move_notify(bo
, old_mem
, new_mem
);
706 * vmw_swap_notify - TTM move_notify_callback
708 * @bo: The TTM buffer object about to be swapped out.
710 static void vmw_swap_notify(struct ttm_buffer_object
*bo
)
712 vmw_bo_swap_notify(bo
);
713 (void) ttm_bo_wait(bo
, false, false);
716 static int vmw_move(struct ttm_buffer_object
*bo
,
718 struct ttm_operation_ctx
*ctx
,
719 struct ttm_resource
*new_mem
,
720 struct ttm_place
*hop
)
722 struct ttm_resource_manager
*old_man
= ttm_manager_type(bo
->bdev
, bo
->resource
->mem_type
);
723 struct ttm_resource_manager
*new_man
= ttm_manager_type(bo
->bdev
, new_mem
->mem_type
);
726 if (new_man
->use_tt
&& new_mem
->mem_type
!= TTM_PL_SYSTEM
) {
727 ret
= vmw_ttm_bind(bo
->bdev
, bo
->ttm
, new_mem
);
732 vmw_move_notify(bo
, bo
->resource
, new_mem
);
734 if (old_man
->use_tt
&& new_man
->use_tt
) {
735 if (bo
->resource
->mem_type
== TTM_PL_SYSTEM
) {
736 ttm_bo_move_null(bo
, new_mem
);
739 ret
= ttm_bo_wait_ctx(bo
, ctx
);
743 vmw_ttm_unbind(bo
->bdev
, bo
->ttm
);
744 ttm_resource_free(bo
, &bo
->resource
);
745 ttm_bo_assign_mem(bo
, new_mem
);
748 ret
= ttm_bo_move_memcpy(bo
, ctx
, new_mem
);
754 vmw_move_notify(bo
, new_mem
, bo
->resource
);
758 struct ttm_device_funcs vmw_bo_driver
= {
759 .ttm_tt_create
= &vmw_ttm_tt_create
,
760 .ttm_tt_populate
= &vmw_ttm_populate
,
761 .ttm_tt_unpopulate
= &vmw_ttm_unpopulate
,
762 .ttm_tt_destroy
= &vmw_ttm_destroy
,
763 .eviction_valuable
= ttm_bo_eviction_valuable
,
764 .evict_flags
= vmw_evict_flags
,
766 .swap_notify
= vmw_swap_notify
,
767 .io_mem_reserve
= &vmw_ttm_io_mem_reserve
,
770 int vmw_bo_create_and_populate(struct vmw_private
*dev_priv
,
771 unsigned long bo_size
,
772 struct ttm_buffer_object
**bo_p
)
774 struct ttm_operation_ctx ctx
= {
775 .interruptible
= false,
778 struct ttm_buffer_object
*bo
;
781 ret
= vmw_bo_create_kernel(dev_priv
, bo_size
,
784 if (unlikely(ret
!= 0))
787 ret
= ttm_bo_reserve(bo
, false, true, NULL
);
789 ret
= vmw_ttm_populate(bo
->bdev
, bo
->ttm
, &ctx
);
790 if (likely(ret
== 0)) {
791 struct vmw_ttm_tt
*vmw_tt
=
792 container_of(bo
->ttm
, struct vmw_ttm_tt
, dma_ttm
);
793 ret
= vmw_ttm_map_dma(vmw_tt
);
796 ttm_bo_unreserve(bo
);
798 if (likely(ret
== 0))