1 /**************************************************************************
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
31 #define pr_fmt(fmt) "[TTM] " fmt
33 #include <linux/sched.h>
34 #include <linux/highmem.h>
35 #include <linux/pagemap.h>
36 #include <linux/shmem_fs.h>
37 #include <linux/file.h>
38 #include <linux/swap.h>
39 #include <linux/slab.h>
40 #include <linux/export.h>
41 #include <drm/drm_cache.h>
42 #include <drm/drm_mem_util.h>
43 #include <drm/ttm/ttm_module.h>
44 #include <drm/ttm/ttm_bo_driver.h>
45 #include <drm/ttm/ttm_placement.h>
46 #include <drm/ttm/ttm_page_alloc.h>
49 * Allocates storage for pointers to the pages that back the ttm.
51 static void ttm_tt_alloc_page_directory(struct ttm_tt
*ttm
)
53 ttm
->pages
= drm_calloc_large(ttm
->num_pages
, sizeof(void*));
56 static void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt
*ttm
)
58 ttm
->ttm
.pages
= drm_calloc_large(ttm
->ttm
.num_pages
, sizeof(void*));
59 ttm
->dma_address
= drm_calloc_large(ttm
->ttm
.num_pages
,
60 sizeof(*ttm
->dma_address
));
64 static inline int ttm_tt_set_page_caching(struct page
*p
,
65 enum ttm_caching_state c_old
,
66 enum ttm_caching_state c_new
)
73 if (c_old
!= tt_cached
) {
74 /* p isn't in the default caching state, set it to
75 * writeback first to free its current memtype. */
77 ret
= set_pages_wb(p
, 1);
83 ret
= set_memory_wc((unsigned long) page_address(p
), 1);
84 else if (c_new
== tt_uncached
)
85 ret
= set_pages_uc(p
, 1);
89 #else /* CONFIG_X86 */
90 static inline int ttm_tt_set_page_caching(struct page
*p
,
91 enum ttm_caching_state c_old
,
92 enum ttm_caching_state c_new
)
96 #endif /* CONFIG_X86 */
99 * Change caching policy for the linear kernel map
100 * for range of pages in a ttm.
103 static int ttm_tt_set_caching(struct ttm_tt
*ttm
,
104 enum ttm_caching_state c_state
)
107 struct page
*cur_page
;
110 if (ttm
->caching_state
== c_state
)
113 if (ttm
->state
== tt_unpopulated
) {
114 /* Change caching but don't populate */
115 ttm
->caching_state
= c_state
;
119 if (ttm
->caching_state
== tt_cached
)
120 drm_clflush_pages(ttm
->pages
, ttm
->num_pages
);
122 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
123 cur_page
= ttm
->pages
[i
];
124 if (likely(cur_page
!= NULL
)) {
125 ret
= ttm_tt_set_page_caching(cur_page
,
128 if (unlikely(ret
!= 0))
133 ttm
->caching_state
= c_state
;
138 for (j
= 0; j
< i
; ++j
) {
139 cur_page
= ttm
->pages
[j
];
140 if (likely(cur_page
!= NULL
)) {
141 (void)ttm_tt_set_page_caching(cur_page
, c_state
,
149 int ttm_tt_set_placement_caching(struct ttm_tt
*ttm
, uint32_t placement
)
151 enum ttm_caching_state state
;
153 if (placement
& TTM_PL_FLAG_WC
)
155 else if (placement
& TTM_PL_FLAG_UNCACHED
)
160 return ttm_tt_set_caching(ttm
, state
);
162 EXPORT_SYMBOL(ttm_tt_set_placement_caching
);
164 void ttm_tt_destroy(struct ttm_tt
*ttm
)
166 if (unlikely(ttm
== NULL
))
169 if (ttm
->state
== tt_bound
) {
173 if (ttm
->state
== tt_unbound
)
174 ttm_tt_unpopulate(ttm
);
176 if (!(ttm
->page_flags
& TTM_PAGE_FLAG_PERSISTENT_SWAP
) &&
178 fput(ttm
->swap_storage
);
180 ttm
->swap_storage
= NULL
;
181 ttm
->func
->destroy(ttm
);
184 int ttm_tt_init(struct ttm_tt
*ttm
, struct ttm_bo_device
*bdev
,
185 unsigned long size
, uint32_t page_flags
,
186 struct page
*dummy_read_page
)
189 ttm
->glob
= bdev
->glob
;
190 ttm
->num_pages
= (size
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
191 ttm
->caching_state
= tt_cached
;
192 ttm
->page_flags
= page_flags
;
193 ttm
->dummy_read_page
= dummy_read_page
;
194 ttm
->state
= tt_unpopulated
;
195 ttm
->swap_storage
= NULL
;
197 ttm_tt_alloc_page_directory(ttm
);
200 pr_err("Failed allocating page table\n");
205 EXPORT_SYMBOL(ttm_tt_init
);
207 void ttm_tt_fini(struct ttm_tt
*ttm
)
209 drm_free_large(ttm
->pages
);
212 EXPORT_SYMBOL(ttm_tt_fini
);
214 int ttm_dma_tt_init(struct ttm_dma_tt
*ttm_dma
, struct ttm_bo_device
*bdev
,
215 unsigned long size
, uint32_t page_flags
,
216 struct page
*dummy_read_page
)
218 struct ttm_tt
*ttm
= &ttm_dma
->ttm
;
221 ttm
->glob
= bdev
->glob
;
222 ttm
->num_pages
= (size
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
223 ttm
->caching_state
= tt_cached
;
224 ttm
->page_flags
= page_flags
;
225 ttm
->dummy_read_page
= dummy_read_page
;
226 ttm
->state
= tt_unpopulated
;
227 ttm
->swap_storage
= NULL
;
229 INIT_LIST_HEAD(&ttm_dma
->pages_list
);
230 ttm_dma_tt_alloc_page_directory(ttm_dma
);
231 if (!ttm
->pages
|| !ttm_dma
->dma_address
) {
233 pr_err("Failed allocating page table\n");
238 EXPORT_SYMBOL(ttm_dma_tt_init
);
240 void ttm_dma_tt_fini(struct ttm_dma_tt
*ttm_dma
)
242 struct ttm_tt
*ttm
= &ttm_dma
->ttm
;
244 drm_free_large(ttm
->pages
);
246 drm_free_large(ttm_dma
->dma_address
);
247 ttm_dma
->dma_address
= NULL
;
249 EXPORT_SYMBOL(ttm_dma_tt_fini
);
251 void ttm_tt_unbind(struct ttm_tt
*ttm
)
255 if (ttm
->state
== tt_bound
) {
256 ret
= ttm
->func
->unbind(ttm
);
258 ttm
->state
= tt_unbound
;
262 int ttm_tt_bind(struct ttm_tt
*ttm
, struct ttm_mem_reg
*bo_mem
)
269 if (ttm
->state
== tt_bound
)
272 ret
= ttm
->bdev
->driver
->ttm_tt_populate(ttm
);
276 ret
= ttm
->func
->bind(ttm
, bo_mem
);
277 if (unlikely(ret
!= 0))
280 ttm
->state
= tt_bound
;
284 EXPORT_SYMBOL(ttm_tt_bind
);
286 int ttm_tt_swapin(struct ttm_tt
*ttm
)
288 struct address_space
*swap_space
;
289 struct file
*swap_storage
;
290 struct page
*from_page
;
291 struct page
*to_page
;
295 swap_storage
= ttm
->swap_storage
;
296 BUG_ON(swap_storage
== NULL
);
298 swap_space
= file_inode(swap_storage
)->i_mapping
;
300 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
301 from_page
= shmem_read_mapping_page(swap_space
, i
);
302 if (IS_ERR(from_page
)) {
303 ret
= PTR_ERR(from_page
);
306 to_page
= ttm
->pages
[i
];
307 if (unlikely(to_page
== NULL
))
310 copy_highpage(to_page
, from_page
);
311 page_cache_release(from_page
);
314 if (!(ttm
->page_flags
& TTM_PAGE_FLAG_PERSISTENT_SWAP
))
316 ttm
->swap_storage
= NULL
;
317 ttm
->page_flags
&= ~TTM_PAGE_FLAG_SWAPPED
;
324 int ttm_tt_swapout(struct ttm_tt
*ttm
, struct file
*persistent_swap_storage
)
326 struct address_space
*swap_space
;
327 struct file
*swap_storage
;
328 struct page
*from_page
;
329 struct page
*to_page
;
333 BUG_ON(ttm
->state
!= tt_unbound
&& ttm
->state
!= tt_unpopulated
);
334 BUG_ON(ttm
->caching_state
!= tt_cached
);
336 if (!persistent_swap_storage
) {
337 swap_storage
= shmem_file_setup("ttm swap",
338 ttm
->num_pages
<< PAGE_SHIFT
,
340 if (unlikely(IS_ERR(swap_storage
))) {
341 pr_err("Failed allocating swap storage\n");
342 return PTR_ERR(swap_storage
);
345 swap_storage
= persistent_swap_storage
;
347 swap_space
= file_inode(swap_storage
)->i_mapping
;
349 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
350 from_page
= ttm
->pages
[i
];
351 if (unlikely(from_page
== NULL
))
353 to_page
= shmem_read_mapping_page(swap_space
, i
);
354 if (unlikely(IS_ERR(to_page
))) {
355 ret
= PTR_ERR(to_page
);
358 copy_highpage(to_page
, from_page
);
359 set_page_dirty(to_page
);
360 mark_page_accessed(to_page
);
361 page_cache_release(to_page
);
364 ttm_tt_unpopulate(ttm
);
365 ttm
->swap_storage
= swap_storage
;
366 ttm
->page_flags
|= TTM_PAGE_FLAG_SWAPPED
;
367 if (persistent_swap_storage
)
368 ttm
->page_flags
|= TTM_PAGE_FLAG_PERSISTENT_SWAP
;
372 if (!persistent_swap_storage
)
378 static void ttm_tt_clear_mapping(struct ttm_tt
*ttm
)
381 struct page
**page
= ttm
->pages
;
383 if (ttm
->page_flags
& TTM_PAGE_FLAG_SG
)
386 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
387 (*page
)->mapping
= NULL
;
388 (*page
++)->index
= 0;
392 void ttm_tt_unpopulate(struct ttm_tt
*ttm
)
394 if (ttm
->state
== tt_unpopulated
)
397 ttm_tt_clear_mapping(ttm
);
398 ttm
->bdev
->driver
->ttm_tt_unpopulate(ttm
);