2 * Copyright 2009 Jerome Glisse.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
28 * Jerome Glisse <glisse@freedesktop.org>
29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
33 #include <linux/dma-mapping.h>
34 #include <linux/pagemap.h>
35 #include <linux/pci.h>
36 #include <linux/seq_file.h>
37 #include <linux/slab.h>
38 #include <linux/swap.h>
39 #include <linux/swiotlb.h>
41 #include <drm/drm_agpsupport.h>
42 #include <drm/drm_debugfs.h>
43 #include <drm/drm_device.h>
44 #include <drm/drm_file.h>
45 #include <drm/drm_prime.h>
46 #include <drm/radeon_drm.h>
47 #include <drm/ttm/ttm_bo_api.h>
48 #include <drm/ttm/ttm_bo_driver.h>
49 #include <drm/ttm/ttm_module.h>
50 #include <drm/ttm/ttm_page_alloc.h>
51 #include <drm/ttm/ttm_placement.h>
53 #include "radeon_reg.h"
56 static int radeon_ttm_debugfs_init(struct radeon_device
*rdev
);
57 static void radeon_ttm_debugfs_fini(struct radeon_device
*rdev
);
59 struct radeon_device
*radeon_get_rdev(struct ttm_bo_device
*bdev
)
61 struct radeon_mman
*mman
;
62 struct radeon_device
*rdev
;
64 mman
= container_of(bdev
, struct radeon_mman
, bdev
);
65 rdev
= container_of(mman
, struct radeon_device
, mman
);
69 static int radeon_init_mem_type(struct ttm_bo_device
*bdev
, uint32_t type
,
70 struct ttm_mem_type_manager
*man
)
72 struct radeon_device
*rdev
;
74 rdev
= radeon_get_rdev(bdev
);
79 man
->flags
= TTM_MEMTYPE_FLAG_MAPPABLE
;
80 man
->available_caching
= TTM_PL_MASK_CACHING
;
81 man
->default_caching
= TTM_PL_FLAG_CACHED
;
84 man
->func
= &ttm_bo_manager_func
;
85 man
->available_caching
= TTM_PL_MASK_CACHING
;
86 man
->default_caching
= TTM_PL_FLAG_CACHED
;
87 man
->flags
= TTM_MEMTYPE_FLAG_MAPPABLE
;
88 #if IS_ENABLED(CONFIG_AGP)
89 if (rdev
->flags
& RADEON_IS_AGP
) {
90 if (!rdev
->ddev
->agp
) {
91 DRM_ERROR("AGP is not enabled for memory type %u\n",
95 if (!rdev
->ddev
->agp
->cant_use_aperture
)
96 man
->flags
= TTM_MEMTYPE_FLAG_MAPPABLE
;
97 man
->available_caching
= TTM_PL_FLAG_UNCACHED
|
99 man
->default_caching
= TTM_PL_FLAG_WC
;
104 /* "On-card" video ram */
105 man
->func
= &ttm_bo_manager_func
;
106 man
->flags
= TTM_MEMTYPE_FLAG_FIXED
|
107 TTM_MEMTYPE_FLAG_MAPPABLE
;
108 man
->available_caching
= TTM_PL_FLAG_UNCACHED
| TTM_PL_FLAG_WC
;
109 man
->default_caching
= TTM_PL_FLAG_WC
;
112 DRM_ERROR("Unsupported memory type %u\n", (unsigned)type
);
118 static void radeon_evict_flags(struct ttm_buffer_object
*bo
,
119 struct ttm_placement
*placement
)
121 static const struct ttm_place placements
= {
124 .flags
= TTM_PL_MASK_CACHING
| TTM_PL_FLAG_SYSTEM
127 struct radeon_bo
*rbo
;
129 if (!radeon_ttm_bo_is_radeon_bo(bo
)) {
130 placement
->placement
= &placements
;
131 placement
->busy_placement
= &placements
;
132 placement
->num_placement
= 1;
133 placement
->num_busy_placement
= 1;
136 rbo
= container_of(bo
, struct radeon_bo
, tbo
);
137 switch (bo
->mem
.mem_type
) {
139 if (rbo
->rdev
->ring
[radeon_copy_ring_index(rbo
->rdev
)].ready
== false)
140 radeon_ttm_placement_from_domain(rbo
, RADEON_GEM_DOMAIN_CPU
);
141 else if (rbo
->rdev
->mc
.visible_vram_size
< rbo
->rdev
->mc
.real_vram_size
&&
142 bo
->mem
.start
< (rbo
->rdev
->mc
.visible_vram_size
>> PAGE_SHIFT
)) {
143 unsigned fpfn
= rbo
->rdev
->mc
.visible_vram_size
>> PAGE_SHIFT
;
146 /* Try evicting to the CPU inaccessible part of VRAM
147 * first, but only set GTT as busy placement, so this
148 * BO will be evicted to GTT rather than causing other
149 * BOs to be evicted from VRAM
151 radeon_ttm_placement_from_domain(rbo
, RADEON_GEM_DOMAIN_VRAM
|
152 RADEON_GEM_DOMAIN_GTT
);
153 rbo
->placement
.num_busy_placement
= 0;
154 for (i
= 0; i
< rbo
->placement
.num_placement
; i
++) {
155 if (rbo
->placements
[i
].flags
& TTM_PL_FLAG_VRAM
) {
156 if (rbo
->placements
[i
].fpfn
< fpfn
)
157 rbo
->placements
[i
].fpfn
= fpfn
;
159 rbo
->placement
.busy_placement
=
161 rbo
->placement
.num_busy_placement
= 1;
165 radeon_ttm_placement_from_domain(rbo
, RADEON_GEM_DOMAIN_GTT
);
169 radeon_ttm_placement_from_domain(rbo
, RADEON_GEM_DOMAIN_CPU
);
171 *placement
= rbo
->placement
;
174 static int radeon_verify_access(struct ttm_buffer_object
*bo
, struct file
*filp
)
176 struct radeon_bo
*rbo
= container_of(bo
, struct radeon_bo
, tbo
);
178 if (radeon_ttm_tt_has_userptr(bo
->ttm
))
180 return drm_vma_node_verify_access(&rbo
->tbo
.base
.vma_node
,
184 static void radeon_move_null(struct ttm_buffer_object
*bo
,
185 struct ttm_mem_reg
*new_mem
)
187 struct ttm_mem_reg
*old_mem
= &bo
->mem
;
189 BUG_ON(old_mem
->mm_node
!= NULL
);
191 new_mem
->mm_node
= NULL
;
194 static int radeon_move_blit(struct ttm_buffer_object
*bo
,
195 bool evict
, bool no_wait_gpu
,
196 struct ttm_mem_reg
*new_mem
,
197 struct ttm_mem_reg
*old_mem
)
199 struct radeon_device
*rdev
;
200 uint64_t old_start
, new_start
;
201 struct radeon_fence
*fence
;
205 rdev
= radeon_get_rdev(bo
->bdev
);
206 ridx
= radeon_copy_ring_index(rdev
);
207 old_start
= (u64
)old_mem
->start
<< PAGE_SHIFT
;
208 new_start
= (u64
)new_mem
->start
<< PAGE_SHIFT
;
210 switch (old_mem
->mem_type
) {
212 old_start
+= rdev
->mc
.vram_start
;
215 old_start
+= rdev
->mc
.gtt_start
;
218 DRM_ERROR("Unknown placement %d\n", old_mem
->mem_type
);
221 switch (new_mem
->mem_type
) {
223 new_start
+= rdev
->mc
.vram_start
;
226 new_start
+= rdev
->mc
.gtt_start
;
229 DRM_ERROR("Unknown placement %d\n", old_mem
->mem_type
);
232 if (!rdev
->ring
[ridx
].ready
) {
233 DRM_ERROR("Trying to move memory with ring turned off.\n");
237 BUILD_BUG_ON((PAGE_SIZE
% RADEON_GPU_PAGE_SIZE
) != 0);
239 num_pages
= new_mem
->num_pages
* (PAGE_SIZE
/ RADEON_GPU_PAGE_SIZE
);
240 fence
= radeon_copy(rdev
, old_start
, new_start
, num_pages
, bo
->base
.resv
);
242 return PTR_ERR(fence
);
244 r
= ttm_bo_move_accel_cleanup(bo
, &fence
->base
, evict
, new_mem
);
245 radeon_fence_unref(&fence
);
249 static int radeon_move_vram_ram(struct ttm_buffer_object
*bo
,
250 bool evict
, bool interruptible
,
252 struct ttm_mem_reg
*new_mem
)
254 struct ttm_operation_ctx ctx
= { interruptible
, no_wait_gpu
};
255 struct ttm_mem_reg
*old_mem
= &bo
->mem
;
256 struct ttm_mem_reg tmp_mem
;
257 struct ttm_place placements
;
258 struct ttm_placement placement
;
262 tmp_mem
.mm_node
= NULL
;
263 placement
.num_placement
= 1;
264 placement
.placement
= &placements
;
265 placement
.num_busy_placement
= 1;
266 placement
.busy_placement
= &placements
;
269 placements
.flags
= TTM_PL_MASK_CACHING
| TTM_PL_FLAG_TT
;
270 r
= ttm_bo_mem_space(bo
, &placement
, &tmp_mem
, &ctx
);
275 r
= ttm_tt_set_placement_caching(bo
->ttm
, tmp_mem
.placement
);
280 r
= ttm_tt_bind(bo
->ttm
, &tmp_mem
, &ctx
);
284 r
= radeon_move_blit(bo
, true, no_wait_gpu
, &tmp_mem
, old_mem
);
288 r
= ttm_bo_move_ttm(bo
, &ctx
, new_mem
);
290 ttm_bo_mem_put(bo
, &tmp_mem
);
294 static int radeon_move_ram_vram(struct ttm_buffer_object
*bo
,
295 bool evict
, bool interruptible
,
297 struct ttm_mem_reg
*new_mem
)
299 struct ttm_operation_ctx ctx
= { interruptible
, no_wait_gpu
};
300 struct ttm_mem_reg
*old_mem
= &bo
->mem
;
301 struct ttm_mem_reg tmp_mem
;
302 struct ttm_placement placement
;
303 struct ttm_place placements
;
307 tmp_mem
.mm_node
= NULL
;
308 placement
.num_placement
= 1;
309 placement
.placement
= &placements
;
310 placement
.num_busy_placement
= 1;
311 placement
.busy_placement
= &placements
;
314 placements
.flags
= TTM_PL_MASK_CACHING
| TTM_PL_FLAG_TT
;
315 r
= ttm_bo_mem_space(bo
, &placement
, &tmp_mem
, &ctx
);
319 r
= ttm_bo_move_ttm(bo
, &ctx
, &tmp_mem
);
323 r
= radeon_move_blit(bo
, true, no_wait_gpu
, new_mem
, old_mem
);
328 ttm_bo_mem_put(bo
, &tmp_mem
);
332 static int radeon_bo_move(struct ttm_buffer_object
*bo
, bool evict
,
333 struct ttm_operation_ctx
*ctx
,
334 struct ttm_mem_reg
*new_mem
)
336 struct radeon_device
*rdev
;
337 struct radeon_bo
*rbo
;
338 struct ttm_mem_reg
*old_mem
= &bo
->mem
;
341 r
= ttm_bo_wait(bo
, ctx
->interruptible
, ctx
->no_wait_gpu
);
345 /* Can't move a pinned BO */
346 rbo
= container_of(bo
, struct radeon_bo
, tbo
);
347 if (WARN_ON_ONCE(rbo
->pin_count
> 0))
350 rdev
= radeon_get_rdev(bo
->bdev
);
351 if (old_mem
->mem_type
== TTM_PL_SYSTEM
&& bo
->ttm
== NULL
) {
352 radeon_move_null(bo
, new_mem
);
355 if ((old_mem
->mem_type
== TTM_PL_TT
&&
356 new_mem
->mem_type
== TTM_PL_SYSTEM
) ||
357 (old_mem
->mem_type
== TTM_PL_SYSTEM
&&
358 new_mem
->mem_type
== TTM_PL_TT
)) {
360 radeon_move_null(bo
, new_mem
);
363 if (!rdev
->ring
[radeon_copy_ring_index(rdev
)].ready
||
364 rdev
->asic
->copy
.copy
== NULL
) {
369 if (old_mem
->mem_type
== TTM_PL_VRAM
&&
370 new_mem
->mem_type
== TTM_PL_SYSTEM
) {
371 r
= radeon_move_vram_ram(bo
, evict
, ctx
->interruptible
,
372 ctx
->no_wait_gpu
, new_mem
);
373 } else if (old_mem
->mem_type
== TTM_PL_SYSTEM
&&
374 new_mem
->mem_type
== TTM_PL_VRAM
) {
375 r
= radeon_move_ram_vram(bo
, evict
, ctx
->interruptible
,
376 ctx
->no_wait_gpu
, new_mem
);
378 r
= radeon_move_blit(bo
, evict
, ctx
->no_wait_gpu
,
384 r
= ttm_bo_move_memcpy(bo
, ctx
, new_mem
);
390 /* update statistics */
391 atomic64_add((u64
)bo
->num_pages
<< PAGE_SHIFT
, &rdev
->num_bytes_moved
);
395 static int radeon_ttm_io_mem_reserve(struct ttm_bo_device
*bdev
, struct ttm_mem_reg
*mem
)
397 struct ttm_mem_type_manager
*man
= &bdev
->man
[mem
->mem_type
];
398 struct radeon_device
*rdev
= radeon_get_rdev(bdev
);
400 mem
->bus
.addr
= NULL
;
402 mem
->bus
.size
= mem
->num_pages
<< PAGE_SHIFT
;
404 mem
->bus
.is_iomem
= false;
405 if (!(man
->flags
& TTM_MEMTYPE_FLAG_MAPPABLE
))
407 switch (mem
->mem_type
) {
412 #if IS_ENABLED(CONFIG_AGP)
413 if (rdev
->flags
& RADEON_IS_AGP
) {
414 /* RADEON_IS_AGP is set only if AGP is active */
415 mem
->bus
.offset
= mem
->start
<< PAGE_SHIFT
;
416 mem
->bus
.base
= rdev
->mc
.agp_base
;
417 mem
->bus
.is_iomem
= !rdev
->ddev
->agp
->cant_use_aperture
;
422 mem
->bus
.offset
= mem
->start
<< PAGE_SHIFT
;
423 /* check if it's visible */
424 if ((mem
->bus
.offset
+ mem
->bus
.size
) > rdev
->mc
.visible_vram_size
)
426 mem
->bus
.base
= rdev
->mc
.aper_base
;
427 mem
->bus
.is_iomem
= true;
430 * Alpha: use bus.addr to hold the ioremap() return,
431 * so we can modify bus.base below.
433 if (mem
->placement
& TTM_PL_FLAG_WC
)
435 ioremap_wc(mem
->bus
.base
+ mem
->bus
.offset
,
439 ioremap(mem
->bus
.base
+ mem
->bus
.offset
,
445 * Alpha: Use just the bus offset plus
446 * the hose/domain memory base for bus.base.
447 * It then can be used to build PTEs for VRAM
448 * access, as done in ttm_bo_vm_fault().
450 mem
->bus
.base
= (mem
->bus
.base
& 0x0ffffffffUL
) +
451 rdev
->ddev
->hose
->dense_mem_base
;
461 * TTM backend functions.
463 struct radeon_ttm_tt
{
464 struct ttm_dma_tt ttm
;
465 struct radeon_device
*rdev
;
469 struct mm_struct
*usermm
;
473 /* prepare the sg table with the user pages */
474 static int radeon_ttm_tt_pin_userptr(struct ttm_tt
*ttm
)
476 struct radeon_device
*rdev
= radeon_get_rdev(ttm
->bdev
);
477 struct radeon_ttm_tt
*gtt
= (void *)ttm
;
481 int write
= !(gtt
->userflags
& RADEON_GEM_USERPTR_READONLY
);
482 enum dma_data_direction direction
= write
?
483 DMA_BIDIRECTIONAL
: DMA_TO_DEVICE
;
485 if (current
->mm
!= gtt
->usermm
)
488 if (gtt
->userflags
& RADEON_GEM_USERPTR_ANONONLY
) {
489 /* check that we only pin down anonymous memory
490 to prevent problems with writeback */
491 unsigned long end
= gtt
->userptr
+ ttm
->num_pages
* PAGE_SIZE
;
492 struct vm_area_struct
*vma
;
493 vma
= find_vma(gtt
->usermm
, gtt
->userptr
);
494 if (!vma
|| vma
->vm_file
|| vma
->vm_end
< end
)
499 unsigned num_pages
= ttm
->num_pages
- pinned
;
500 uint64_t userptr
= gtt
->userptr
+ pinned
* PAGE_SIZE
;
501 struct page
**pages
= ttm
->pages
+ pinned
;
503 r
= get_user_pages(userptr
, num_pages
, write
? FOLL_WRITE
: 0,
510 } while (pinned
< ttm
->num_pages
);
512 r
= sg_alloc_table_from_pages(ttm
->sg
, ttm
->pages
, ttm
->num_pages
, 0,
513 ttm
->num_pages
<< PAGE_SHIFT
,
518 r
= dma_map_sgtable(rdev
->dev
, ttm
->sg
, direction
, 0);
522 drm_prime_sg_to_page_addr_arrays(ttm
->sg
, ttm
->pages
,
523 gtt
->ttm
.dma_address
, ttm
->num_pages
);
531 release_pages(ttm
->pages
, pinned
);
535 static void radeon_ttm_tt_unpin_userptr(struct ttm_tt
*ttm
)
537 struct radeon_device
*rdev
= radeon_get_rdev(ttm
->bdev
);
538 struct radeon_ttm_tt
*gtt
= (void *)ttm
;
539 struct sg_page_iter sg_iter
;
541 int write
= !(gtt
->userflags
& RADEON_GEM_USERPTR_READONLY
);
542 enum dma_data_direction direction
= write
?
543 DMA_BIDIRECTIONAL
: DMA_TO_DEVICE
;
545 /* double check that we don't free the table twice */
549 /* free the sg table and pages again */
550 dma_unmap_sgtable(rdev
->dev
, ttm
->sg
, direction
, 0);
552 for_each_sgtable_page(ttm
->sg
, &sg_iter
, 0) {
553 struct page
*page
= sg_page_iter_page(&sg_iter
);
554 if (!(gtt
->userflags
& RADEON_GEM_USERPTR_READONLY
))
555 set_page_dirty(page
);
557 mark_page_accessed(page
);
561 sg_free_table(ttm
->sg
);
564 static int radeon_ttm_backend_bind(struct ttm_tt
*ttm
,
565 struct ttm_mem_reg
*bo_mem
)
567 struct radeon_ttm_tt
*gtt
= (void*)ttm
;
568 uint32_t flags
= RADEON_GART_PAGE_VALID
| RADEON_GART_PAGE_READ
|
569 RADEON_GART_PAGE_WRITE
;
573 radeon_ttm_tt_pin_userptr(ttm
);
574 flags
&= ~RADEON_GART_PAGE_WRITE
;
577 gtt
->offset
= (unsigned long)(bo_mem
->start
<< PAGE_SHIFT
);
578 if (!ttm
->num_pages
) {
579 WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n",
580 ttm
->num_pages
, bo_mem
, ttm
);
582 if (ttm
->caching_state
== tt_cached
)
583 flags
|= RADEON_GART_PAGE_SNOOP
;
584 r
= radeon_gart_bind(gtt
->rdev
, gtt
->offset
, ttm
->num_pages
,
585 ttm
->pages
, gtt
->ttm
.dma_address
, flags
);
587 DRM_ERROR("failed to bind %lu pages at 0x%08X\n",
588 ttm
->num_pages
, (unsigned)gtt
->offset
);
594 static void radeon_ttm_backend_unbind(struct ttm_tt
*ttm
)
596 struct radeon_ttm_tt
*gtt
= (void *)ttm
;
598 radeon_gart_unbind(gtt
->rdev
, gtt
->offset
, ttm
->num_pages
);
601 radeon_ttm_tt_unpin_userptr(ttm
);
604 static void radeon_ttm_backend_destroy(struct ttm_tt
*ttm
)
606 struct radeon_ttm_tt
*gtt
= (void *)ttm
;
608 ttm_dma_tt_fini(>t
->ttm
);
612 static struct ttm_backend_func radeon_backend_func
= {
613 .bind
= &radeon_ttm_backend_bind
,
614 .unbind
= &radeon_ttm_backend_unbind
,
615 .destroy
= &radeon_ttm_backend_destroy
,
618 static struct ttm_tt
*radeon_ttm_tt_create(struct ttm_buffer_object
*bo
,
621 struct radeon_device
*rdev
;
622 struct radeon_ttm_tt
*gtt
;
624 rdev
= radeon_get_rdev(bo
->bdev
);
625 #if IS_ENABLED(CONFIG_AGP)
626 if (rdev
->flags
& RADEON_IS_AGP
) {
627 return ttm_agp_tt_create(bo
, rdev
->ddev
->agp
->bridge
,
632 gtt
= kzalloc(sizeof(struct radeon_ttm_tt
), GFP_KERNEL
);
636 gtt
->ttm
.ttm
.func
= &radeon_backend_func
;
638 if (ttm_dma_tt_init(>t
->ttm
, bo
, page_flags
)) {
642 return >t
->ttm
.ttm
;
645 static struct radeon_ttm_tt
*radeon_ttm_tt_to_gtt(struct ttm_tt
*ttm
)
647 if (!ttm
|| ttm
->func
!= &radeon_backend_func
)
649 return (struct radeon_ttm_tt
*)ttm
;
652 static int radeon_ttm_tt_populate(struct ttm_tt
*ttm
,
653 struct ttm_operation_ctx
*ctx
)
655 struct radeon_ttm_tt
*gtt
= radeon_ttm_tt_to_gtt(ttm
);
656 struct radeon_device
*rdev
;
657 bool slave
= !!(ttm
->page_flags
& TTM_PAGE_FLAG_SG
);
659 if (gtt
&& gtt
->userptr
) {
660 ttm
->sg
= kzalloc(sizeof(struct sg_table
), GFP_KERNEL
);
664 ttm
->page_flags
|= TTM_PAGE_FLAG_SG
;
665 ttm
->state
= tt_unbound
;
669 if (slave
&& ttm
->sg
) {
670 drm_prime_sg_to_page_addr_arrays(ttm
->sg
, ttm
->pages
,
671 gtt
->ttm
.dma_address
, ttm
->num_pages
);
672 ttm
->state
= tt_unbound
;
676 rdev
= radeon_get_rdev(ttm
->bdev
);
677 #if IS_ENABLED(CONFIG_AGP)
678 if (rdev
->flags
& RADEON_IS_AGP
) {
679 return ttm_agp_tt_populate(ttm
, ctx
);
683 #ifdef CONFIG_SWIOTLB
684 if (rdev
->need_swiotlb
&& swiotlb_nr_tbl()) {
685 return ttm_dma_populate(>t
->ttm
, rdev
->dev
, ctx
);
689 return ttm_populate_and_map_pages(rdev
->dev
, >t
->ttm
, ctx
);
692 static void radeon_ttm_tt_unpopulate(struct ttm_tt
*ttm
)
694 struct radeon_device
*rdev
;
695 struct radeon_ttm_tt
*gtt
= radeon_ttm_tt_to_gtt(ttm
);
696 bool slave
= !!(ttm
->page_flags
& TTM_PAGE_FLAG_SG
);
698 if (gtt
&& gtt
->userptr
) {
700 ttm
->page_flags
&= ~TTM_PAGE_FLAG_SG
;
707 rdev
= radeon_get_rdev(ttm
->bdev
);
708 #if IS_ENABLED(CONFIG_AGP)
709 if (rdev
->flags
& RADEON_IS_AGP
) {
710 ttm_agp_tt_unpopulate(ttm
);
715 #ifdef CONFIG_SWIOTLB
716 if (rdev
->need_swiotlb
&& swiotlb_nr_tbl()) {
717 ttm_dma_unpopulate(>t
->ttm
, rdev
->dev
);
722 ttm_unmap_and_unpopulate_pages(rdev
->dev
, >t
->ttm
);
725 int radeon_ttm_tt_set_userptr(struct ttm_tt
*ttm
, uint64_t addr
,
728 struct radeon_ttm_tt
*gtt
= radeon_ttm_tt_to_gtt(ttm
);
734 gtt
->usermm
= current
->mm
;
735 gtt
->userflags
= flags
;
739 bool radeon_ttm_tt_has_userptr(struct ttm_tt
*ttm
)
741 struct radeon_ttm_tt
*gtt
= radeon_ttm_tt_to_gtt(ttm
);
746 return !!gtt
->userptr
;
749 bool radeon_ttm_tt_is_readonly(struct ttm_tt
*ttm
)
751 struct radeon_ttm_tt
*gtt
= radeon_ttm_tt_to_gtt(ttm
);
756 return !!(gtt
->userflags
& RADEON_GEM_USERPTR_READONLY
);
759 static struct ttm_bo_driver radeon_bo_driver
= {
760 .ttm_tt_create
= &radeon_ttm_tt_create
,
761 .ttm_tt_populate
= &radeon_ttm_tt_populate
,
762 .ttm_tt_unpopulate
= &radeon_ttm_tt_unpopulate
,
763 .init_mem_type
= &radeon_init_mem_type
,
764 .eviction_valuable
= ttm_bo_eviction_valuable
,
765 .evict_flags
= &radeon_evict_flags
,
766 .move
= &radeon_bo_move
,
767 .verify_access
= &radeon_verify_access
,
768 .move_notify
= &radeon_bo_move_notify
,
769 .fault_reserve_notify
= &radeon_bo_fault_reserve_notify
,
770 .io_mem_reserve
= &radeon_ttm_io_mem_reserve
,
773 int radeon_ttm_init(struct radeon_device
*rdev
)
777 /* No others user of address space so set it to 0 */
778 r
= ttm_bo_device_init(&rdev
->mman
.bdev
,
780 rdev
->ddev
->anon_inode
->i_mapping
,
781 rdev
->ddev
->vma_offset_manager
,
782 dma_addressing_limited(&rdev
->pdev
->dev
));
784 DRM_ERROR("failed initializing buffer object driver(%d).\n", r
);
787 rdev
->mman
.initialized
= true;
788 r
= ttm_bo_init_mm(&rdev
->mman
.bdev
, TTM_PL_VRAM
,
789 rdev
->mc
.real_vram_size
>> PAGE_SHIFT
);
791 DRM_ERROR("Failed initializing VRAM heap.\n");
794 /* Change the size here instead of the init above so only lpfn is affected */
795 radeon_ttm_set_active_vram_size(rdev
, rdev
->mc
.visible_vram_size
);
797 r
= radeon_bo_create(rdev
, 256 * 1024, PAGE_SIZE
, true,
798 RADEON_GEM_DOMAIN_VRAM
, 0, NULL
,
799 NULL
, &rdev
->stolen_vga_memory
);
803 r
= radeon_bo_reserve(rdev
->stolen_vga_memory
, false);
806 r
= radeon_bo_pin(rdev
->stolen_vga_memory
, RADEON_GEM_DOMAIN_VRAM
, NULL
);
807 radeon_bo_unreserve(rdev
->stolen_vga_memory
);
809 radeon_bo_unref(&rdev
->stolen_vga_memory
);
812 DRM_INFO("radeon: %uM of VRAM memory ready\n",
813 (unsigned) (rdev
->mc
.real_vram_size
/ (1024 * 1024)));
814 r
= ttm_bo_init_mm(&rdev
->mman
.bdev
, TTM_PL_TT
,
815 rdev
->mc
.gtt_size
>> PAGE_SHIFT
);
817 DRM_ERROR("Failed initializing GTT heap.\n");
820 DRM_INFO("radeon: %uM of GTT memory ready.\n",
821 (unsigned)(rdev
->mc
.gtt_size
/ (1024 * 1024)));
823 r
= radeon_ttm_debugfs_init(rdev
);
825 DRM_ERROR("Failed to init debugfs\n");
831 void radeon_ttm_fini(struct radeon_device
*rdev
)
835 if (!rdev
->mman
.initialized
)
837 radeon_ttm_debugfs_fini(rdev
);
838 if (rdev
->stolen_vga_memory
) {
839 r
= radeon_bo_reserve(rdev
->stolen_vga_memory
, false);
841 radeon_bo_unpin(rdev
->stolen_vga_memory
);
842 radeon_bo_unreserve(rdev
->stolen_vga_memory
);
844 radeon_bo_unref(&rdev
->stolen_vga_memory
);
846 ttm_bo_clean_mm(&rdev
->mman
.bdev
, TTM_PL_VRAM
);
847 ttm_bo_clean_mm(&rdev
->mman
.bdev
, TTM_PL_TT
);
848 ttm_bo_device_release(&rdev
->mman
.bdev
);
849 radeon_gart_fini(rdev
);
850 rdev
->mman
.initialized
= false;
851 DRM_INFO("radeon: ttm finalized\n");
854 /* this should only be called at bootup or when userspace
856 void radeon_ttm_set_active_vram_size(struct radeon_device
*rdev
, u64 size
)
858 struct ttm_mem_type_manager
*man
;
860 if (!rdev
->mman
.initialized
)
863 man
= &rdev
->mman
.bdev
.man
[TTM_PL_VRAM
];
864 /* this just adjusts TTM size idea, which sets lpfn to the correct value */
865 man
->size
= size
>> PAGE_SHIFT
;
868 static vm_fault_t
radeon_ttm_fault(struct vm_fault
*vmf
)
870 struct ttm_buffer_object
*bo
;
871 struct radeon_device
*rdev
;
874 bo
= (struct ttm_buffer_object
*)vmf
->vma
->vm_private_data
;
876 return VM_FAULT_NOPAGE
;
878 rdev
= radeon_get_rdev(bo
->bdev
);
879 down_read(&rdev
->pm
.mclk_lock
);
880 ret
= ttm_bo_vm_fault(vmf
);
881 up_read(&rdev
->pm
.mclk_lock
);
885 static struct vm_operations_struct radeon_ttm_vm_ops
= {
886 .fault
= radeon_ttm_fault
,
887 .open
= ttm_bo_vm_open
,
888 .close
= ttm_bo_vm_close
,
889 .access
= ttm_bo_vm_access
892 int radeon_mmap(struct file
*filp
, struct vm_area_struct
*vma
)
895 struct drm_file
*file_priv
= filp
->private_data
;
896 struct radeon_device
*rdev
= file_priv
->minor
->dev
->dev_private
;
901 r
= ttm_bo_mmap(filp
, vma
, &rdev
->mman
.bdev
);
902 if (unlikely(r
!= 0))
905 vma
->vm_ops
= &radeon_ttm_vm_ops
;
909 #if defined(CONFIG_DEBUG_FS)
911 static int radeon_mm_dump_table(struct seq_file
*m
, void *data
)
913 struct drm_info_node
*node
= (struct drm_info_node
*)m
->private;
914 unsigned ttm_pl
= *(int*)node
->info_ent
->data
;
915 struct drm_device
*dev
= node
->minor
->dev
;
916 struct radeon_device
*rdev
= dev
->dev_private
;
917 struct ttm_mem_type_manager
*man
= &rdev
->mman
.bdev
.man
[ttm_pl
];
918 struct drm_printer p
= drm_seq_file_printer(m
);
920 man
->func
->debug(man
, &p
);
925 static int ttm_pl_vram
= TTM_PL_VRAM
;
926 static int ttm_pl_tt
= TTM_PL_TT
;
928 static struct drm_info_list radeon_ttm_debugfs_list
[] = {
929 {"radeon_vram_mm", radeon_mm_dump_table
, 0, &ttm_pl_vram
},
930 {"radeon_gtt_mm", radeon_mm_dump_table
, 0, &ttm_pl_tt
},
931 {"ttm_page_pool", ttm_page_alloc_debugfs
, 0, NULL
},
932 #ifdef CONFIG_SWIOTLB
933 {"ttm_dma_page_pool", ttm_dma_page_alloc_debugfs
, 0, NULL
}
937 static int radeon_ttm_vram_open(struct inode
*inode
, struct file
*filep
)
939 struct radeon_device
*rdev
= inode
->i_private
;
940 i_size_write(inode
, rdev
->mc
.mc_vram_size
);
941 filep
->private_data
= inode
->i_private
;
945 static ssize_t
radeon_ttm_vram_read(struct file
*f
, char __user
*buf
,
946 size_t size
, loff_t
*pos
)
948 struct radeon_device
*rdev
= f
->private_data
;
952 if (size
& 0x3 || *pos
& 0x3)
959 if (*pos
>= rdev
->mc
.mc_vram_size
)
962 spin_lock_irqsave(&rdev
->mmio_idx_lock
, flags
);
963 WREG32(RADEON_MM_INDEX
, ((uint32_t)*pos
) | 0x80000000);
964 if (rdev
->family
>= CHIP_CEDAR
)
965 WREG32(EVERGREEN_MM_INDEX_HI
, *pos
>> 31);
966 value
= RREG32(RADEON_MM_DATA
);
967 spin_unlock_irqrestore(&rdev
->mmio_idx_lock
, flags
);
969 r
= put_user(value
, (uint32_t *)buf
);
982 static const struct file_operations radeon_ttm_vram_fops
= {
983 .owner
= THIS_MODULE
,
984 .open
= radeon_ttm_vram_open
,
985 .read
= radeon_ttm_vram_read
,
986 .llseek
= default_llseek
989 static int radeon_ttm_gtt_open(struct inode
*inode
, struct file
*filep
)
991 struct radeon_device
*rdev
= inode
->i_private
;
992 i_size_write(inode
, rdev
->mc
.gtt_size
);
993 filep
->private_data
= inode
->i_private
;
997 static ssize_t
radeon_ttm_gtt_read(struct file
*f
, char __user
*buf
,
998 size_t size
, loff_t
*pos
)
1000 struct radeon_device
*rdev
= f
->private_data
;
1005 loff_t p
= *pos
/ PAGE_SIZE
;
1006 unsigned off
= *pos
& ~PAGE_MASK
;
1007 size_t cur_size
= min_t(size_t, size
, PAGE_SIZE
- off
);
1011 if (p
>= rdev
->gart
.num_cpu_pages
)
1014 page
= rdev
->gart
.pages
[p
];
1019 r
= copy_to_user(buf
, ptr
, cur_size
);
1020 kunmap(rdev
->gart
.pages
[p
]);
1022 r
= clear_user(buf
, cur_size
);
1036 static const struct file_operations radeon_ttm_gtt_fops
= {
1037 .owner
= THIS_MODULE
,
1038 .open
= radeon_ttm_gtt_open
,
1039 .read
= radeon_ttm_gtt_read
,
1040 .llseek
= default_llseek
1045 static int radeon_ttm_debugfs_init(struct radeon_device
*rdev
)
1047 #if defined(CONFIG_DEBUG_FS)
1050 struct drm_minor
*minor
= rdev
->ddev
->primary
;
1051 struct dentry
*root
= minor
->debugfs_root
;
1053 rdev
->mman
.vram
= debugfs_create_file("radeon_vram", S_IFREG
| S_IRUGO
,
1055 &radeon_ttm_vram_fops
);
1057 rdev
->mman
.gtt
= debugfs_create_file("radeon_gtt", S_IFREG
| S_IRUGO
,
1058 root
, rdev
, &radeon_ttm_gtt_fops
);
1060 count
= ARRAY_SIZE(radeon_ttm_debugfs_list
);
1062 #ifdef CONFIG_SWIOTLB
1063 if (!(rdev
->need_swiotlb
&& swiotlb_nr_tbl()))
1067 return radeon_debugfs_add_files(rdev
, radeon_ttm_debugfs_list
, count
);
1074 static void radeon_ttm_debugfs_fini(struct radeon_device
*rdev
)
1076 #if defined(CONFIG_DEBUG_FS)
1078 debugfs_remove(rdev
->mman
.vram
);
1079 rdev
->mman
.vram
= NULL
;
1081 debugfs_remove(rdev
->mman
.gtt
);
1082 rdev
->mman
.gtt
= NULL
;