1 /**************************************************************************
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
31 #include <linux/vmalloc.h>
32 #include <linux/sched.h>
33 #include <linux/highmem.h>
34 #include <linux/pagemap.h>
35 #include <linux/file.h>
36 #include <linux/swap.h>
37 #include <linux/slab.h>
38 #include "drm_cache.h"
39 #include "ttm/ttm_module.h"
40 #include "ttm/ttm_bo_driver.h"
41 #include "ttm/ttm_placement.h"
43 static int ttm_tt_swapin(struct ttm_tt
*ttm
);
46 * Allocates storage for pointers to the pages that back the ttm.
48 * Uses kmalloc if possible. Otherwise falls back to vmalloc.
50 static void ttm_tt_alloc_page_directory(struct ttm_tt
*ttm
)
52 unsigned long size
= ttm
->num_pages
* sizeof(*ttm
->pages
);
55 if (size
<= PAGE_SIZE
)
56 ttm
->pages
= kzalloc(size
, GFP_KERNEL
);
59 ttm
->pages
= vmalloc_user(size
);
61 ttm
->page_flags
|= TTM_PAGE_FLAG_VMALLOC
;
65 static void ttm_tt_free_page_directory(struct ttm_tt
*ttm
)
67 if (ttm
->page_flags
& TTM_PAGE_FLAG_VMALLOC
) {
69 ttm
->page_flags
&= ~TTM_PAGE_FLAG_VMALLOC
;
76 static struct page
*ttm_tt_alloc_page(unsigned page_flags
)
78 gfp_t gfp_flags
= GFP_USER
;
80 if (page_flags
& TTM_PAGE_FLAG_ZERO_ALLOC
)
81 gfp_flags
|= __GFP_ZERO
;
83 if (page_flags
& TTM_PAGE_FLAG_DMA32
)
84 gfp_flags
|= __GFP_DMA32
;
86 gfp_flags
|= __GFP_HIGHMEM
;
88 return alloc_page(gfp_flags
);
91 static void ttm_tt_free_user_pages(struct ttm_tt
*ttm
)
97 struct ttm_backend
*be
= ttm
->be
;
99 BUG_ON(!(ttm
->page_flags
& TTM_PAGE_FLAG_USER
));
100 write
= ((ttm
->page_flags
& TTM_PAGE_FLAG_WRITE
) != 0);
101 dirty
= ((ttm
->page_flags
& TTM_PAGE_FLAG_USER_DIRTY
) != 0);
106 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
107 page
= ttm
->pages
[i
];
111 if (page
== ttm
->dummy_read_page
) {
116 if (write
&& dirty
&& !PageReserved(page
))
117 set_page_dirty_lock(page
);
119 ttm
->pages
[i
] = NULL
;
120 ttm_mem_global_free(ttm
->glob
->mem_glob
, PAGE_SIZE
);
123 ttm
->state
= tt_unpopulated
;
124 ttm
->first_himem_page
= ttm
->num_pages
;
125 ttm
->last_lomem_page
= -1;
128 static struct page
*__ttm_tt_get_page(struct ttm_tt
*ttm
, int index
)
131 struct ttm_mem_global
*mem_glob
= ttm
->glob
->mem_glob
;
134 while (NULL
== (p
= ttm
->pages
[index
])) {
135 p
= ttm_tt_alloc_page(ttm
->page_flags
);
140 ret
= ttm_mem_global_alloc_page(mem_glob
, p
, false, false);
141 if (unlikely(ret
!= 0))
145 ttm
->pages
[--ttm
->first_himem_page
] = p
;
147 ttm
->pages
[++ttm
->last_lomem_page
] = p
;
155 struct page
*ttm_tt_get_page(struct ttm_tt
*ttm
, int index
)
159 if (unlikely(ttm
->page_flags
& TTM_PAGE_FLAG_SWAPPED
)) {
160 ret
= ttm_tt_swapin(ttm
);
161 if (unlikely(ret
!= 0))
164 return __ttm_tt_get_page(ttm
, index
);
167 int ttm_tt_populate(struct ttm_tt
*ttm
)
171 struct ttm_backend
*be
;
174 if (ttm
->state
!= tt_unpopulated
)
177 if (unlikely(ttm
->page_flags
& TTM_PAGE_FLAG_SWAPPED
)) {
178 ret
= ttm_tt_swapin(ttm
);
179 if (unlikely(ret
!= 0))
185 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
186 page
= __ttm_tt_get_page(ttm
, i
);
191 be
->func
->populate(be
, ttm
->num_pages
, ttm
->pages
,
192 ttm
->dummy_read_page
);
193 ttm
->state
= tt_unbound
;
196 EXPORT_SYMBOL(ttm_tt_populate
);
199 static inline int ttm_tt_set_page_caching(struct page
*p
,
200 enum ttm_caching_state c_old
,
201 enum ttm_caching_state c_new
)
208 if (c_old
!= tt_cached
) {
209 /* p isn't in the default caching state, set it to
210 * writeback first to free its current memtype. */
212 ret
= set_pages_wb(p
, 1);
218 ret
= set_memory_wc((unsigned long) page_address(p
), 1);
219 else if (c_new
== tt_uncached
)
220 ret
= set_pages_uc(p
, 1);
224 #else /* CONFIG_X86 */
225 static inline int ttm_tt_set_page_caching(struct page
*p
,
226 enum ttm_caching_state c_old
,
227 enum ttm_caching_state c_new
)
231 #endif /* CONFIG_X86 */
234 * Change caching policy for the linear kernel map
235 * for range of pages in a ttm.
238 static int ttm_tt_set_caching(struct ttm_tt
*ttm
,
239 enum ttm_caching_state c_state
)
242 struct page
*cur_page
;
245 if (ttm
->caching_state
== c_state
)
248 if (c_state
!= tt_cached
) {
249 ret
= ttm_tt_populate(ttm
);
250 if (unlikely(ret
!= 0))
254 if (ttm
->caching_state
== tt_cached
)
255 drm_clflush_pages(ttm
->pages
, ttm
->num_pages
);
257 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
258 cur_page
= ttm
->pages
[i
];
259 if (likely(cur_page
!= NULL
)) {
260 ret
= ttm_tt_set_page_caching(cur_page
,
263 if (unlikely(ret
!= 0))
268 ttm
->caching_state
= c_state
;
273 for (j
= 0; j
< i
; ++j
) {
274 cur_page
= ttm
->pages
[j
];
275 if (likely(cur_page
!= NULL
)) {
276 (void)ttm_tt_set_page_caching(cur_page
, c_state
,
284 int ttm_tt_set_placement_caching(struct ttm_tt
*ttm
, uint32_t placement
)
286 enum ttm_caching_state state
;
288 if (placement
& TTM_PL_FLAG_WC
)
290 else if (placement
& TTM_PL_FLAG_UNCACHED
)
295 return ttm_tt_set_caching(ttm
, state
);
297 EXPORT_SYMBOL(ttm_tt_set_placement_caching
);
299 static void ttm_tt_free_alloced_pages(struct ttm_tt
*ttm
)
302 struct page
*cur_page
;
303 struct ttm_backend
*be
= ttm
->be
;
307 (void)ttm_tt_set_caching(ttm
, tt_cached
);
308 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
309 cur_page
= ttm
->pages
[i
];
310 ttm
->pages
[i
] = NULL
;
312 if (page_count(cur_page
) != 1)
313 printk(KERN_ERR TTM_PFX
314 "Erroneous page count. "
316 ttm_mem_global_free_page(ttm
->glob
->mem_glob
,
318 __free_page(cur_page
);
321 ttm
->state
= tt_unpopulated
;
322 ttm
->first_himem_page
= ttm
->num_pages
;
323 ttm
->last_lomem_page
= -1;
326 void ttm_tt_destroy(struct ttm_tt
*ttm
)
328 struct ttm_backend
*be
;
330 if (unlikely(ttm
== NULL
))
334 if (likely(be
!= NULL
)) {
335 be
->func
->destroy(be
);
339 if (likely(ttm
->pages
!= NULL
)) {
340 if (ttm
->page_flags
& TTM_PAGE_FLAG_USER
)
341 ttm_tt_free_user_pages(ttm
);
343 ttm_tt_free_alloced_pages(ttm
);
345 ttm_tt_free_page_directory(ttm
);
348 if (!(ttm
->page_flags
& TTM_PAGE_FLAG_PERSISTANT_SWAP
) &&
350 fput(ttm
->swap_storage
);
355 int ttm_tt_set_user(struct ttm_tt
*ttm
,
356 struct task_struct
*tsk
,
357 unsigned long start
, unsigned long num_pages
)
359 struct mm_struct
*mm
= tsk
->mm
;
361 int write
= (ttm
->page_flags
& TTM_PAGE_FLAG_WRITE
) != 0;
362 struct ttm_mem_global
*mem_glob
= ttm
->glob
->mem_glob
;
364 BUG_ON(num_pages
!= ttm
->num_pages
);
365 BUG_ON((ttm
->page_flags
& TTM_PAGE_FLAG_USER
) == 0);
368 * Account user pages as lowmem pages for now.
371 ret
= ttm_mem_global_alloc(mem_glob
, num_pages
* PAGE_SIZE
,
373 if (unlikely(ret
!= 0))
376 down_read(&mm
->mmap_sem
);
377 ret
= get_user_pages(tsk
, mm
, start
, num_pages
,
378 write
, 0, ttm
->pages
, NULL
);
379 up_read(&mm
->mmap_sem
);
381 if (ret
!= num_pages
&& write
) {
382 ttm_tt_free_user_pages(ttm
);
383 ttm_mem_global_free(mem_glob
, num_pages
* PAGE_SIZE
);
389 ttm
->state
= tt_unbound
;
394 struct ttm_tt
*ttm_tt_create(struct ttm_bo_device
*bdev
, unsigned long size
,
395 uint32_t page_flags
, struct page
*dummy_read_page
)
397 struct ttm_bo_driver
*bo_driver
= bdev
->driver
;
403 ttm
= kzalloc(sizeof(*ttm
), GFP_KERNEL
);
407 ttm
->glob
= bdev
->glob
;
408 ttm
->num_pages
= (size
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
409 ttm
->first_himem_page
= ttm
->num_pages
;
410 ttm
->last_lomem_page
= -1;
411 ttm
->caching_state
= tt_cached
;
412 ttm
->page_flags
= page_flags
;
414 ttm
->dummy_read_page
= dummy_read_page
;
416 ttm_tt_alloc_page_directory(ttm
);
419 printk(KERN_ERR TTM_PFX
"Failed allocating page table\n");
422 ttm
->be
= bo_driver
->create_ttm_backend_entry(bdev
);
425 printk(KERN_ERR TTM_PFX
"Failed creating ttm backend entry\n");
428 ttm
->state
= tt_unpopulated
;
432 void ttm_tt_unbind(struct ttm_tt
*ttm
)
435 struct ttm_backend
*be
= ttm
->be
;
437 if (ttm
->state
== tt_bound
) {
438 ret
= be
->func
->unbind(be
);
440 ttm
->state
= tt_unbound
;
444 int ttm_tt_bind(struct ttm_tt
*ttm
, struct ttm_mem_reg
*bo_mem
)
447 struct ttm_backend
*be
;
452 if (ttm
->state
== tt_bound
)
457 ret
= ttm_tt_populate(ttm
);
461 ret
= be
->func
->bind(be
, bo_mem
);
463 printk(KERN_ERR TTM_PFX
"Couldn't bind backend.\n");
467 ttm
->state
= tt_bound
;
469 if (ttm
->page_flags
& TTM_PAGE_FLAG_USER
)
470 ttm
->page_flags
|= TTM_PAGE_FLAG_USER_DIRTY
;
473 EXPORT_SYMBOL(ttm_tt_bind
);
475 static int ttm_tt_swapin(struct ttm_tt
*ttm
)
477 struct address_space
*swap_space
;
478 struct file
*swap_storage
;
479 struct page
*from_page
;
480 struct page
*to_page
;
486 if (ttm
->page_flags
& TTM_PAGE_FLAG_USER
) {
487 ret
= ttm_tt_set_user(ttm
, ttm
->tsk
, ttm
->start
,
489 if (unlikely(ret
!= 0))
492 ttm
->page_flags
&= ~TTM_PAGE_FLAG_SWAPPED
;
496 swap_storage
= ttm
->swap_storage
;
497 BUG_ON(swap_storage
== NULL
);
499 swap_space
= swap_storage
->f_path
.dentry
->d_inode
->i_mapping
;
501 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
502 from_page
= read_mapping_page(swap_space
, i
, NULL
);
503 if (IS_ERR(from_page
)) {
504 ret
= PTR_ERR(from_page
);
507 to_page
= __ttm_tt_get_page(ttm
, i
);
508 if (unlikely(to_page
== NULL
))
512 from_virtual
= kmap_atomic(from_page
, KM_USER0
);
513 to_virtual
= kmap_atomic(to_page
, KM_USER1
);
514 memcpy(to_virtual
, from_virtual
, PAGE_SIZE
);
515 kunmap_atomic(to_virtual
, KM_USER1
);
516 kunmap_atomic(from_virtual
, KM_USER0
);
518 page_cache_release(from_page
);
521 if (!(ttm
->page_flags
& TTM_PAGE_FLAG_PERSISTANT_SWAP
))
523 ttm
->swap_storage
= NULL
;
524 ttm
->page_flags
&= ~TTM_PAGE_FLAG_SWAPPED
;
528 ttm_tt_free_alloced_pages(ttm
);
532 int ttm_tt_swapout(struct ttm_tt
*ttm
, struct file
*persistant_swap_storage
)
534 struct address_space
*swap_space
;
535 struct file
*swap_storage
;
536 struct page
*from_page
;
537 struct page
*to_page
;
543 BUG_ON(ttm
->state
!= tt_unbound
&& ttm
->state
!= tt_unpopulated
);
544 BUG_ON(ttm
->caching_state
!= tt_cached
);
547 * For user buffers, just unpin the pages, as there should be
551 if (ttm
->page_flags
& TTM_PAGE_FLAG_USER
) {
552 ttm_tt_free_user_pages(ttm
);
553 ttm
->page_flags
|= TTM_PAGE_FLAG_SWAPPED
;
554 ttm
->swap_storage
= NULL
;
558 if (!persistant_swap_storage
) {
559 swap_storage
= shmem_file_setup("ttm swap",
560 ttm
->num_pages
<< PAGE_SHIFT
,
562 if (unlikely(IS_ERR(swap_storage
))) {
563 printk(KERN_ERR
"Failed allocating swap storage.\n");
564 return PTR_ERR(swap_storage
);
567 swap_storage
= persistant_swap_storage
;
569 swap_space
= swap_storage
->f_path
.dentry
->d_inode
->i_mapping
;
571 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
572 from_page
= ttm
->pages
[i
];
573 if (unlikely(from_page
== NULL
))
575 to_page
= read_mapping_page(swap_space
, i
, NULL
);
576 if (unlikely(IS_ERR(to_page
))) {
577 ret
= PTR_ERR(to_page
);
581 from_virtual
= kmap_atomic(from_page
, KM_USER0
);
582 to_virtual
= kmap_atomic(to_page
, KM_USER1
);
583 memcpy(to_virtual
, from_virtual
, PAGE_SIZE
);
584 kunmap_atomic(to_virtual
, KM_USER1
);
585 kunmap_atomic(from_virtual
, KM_USER0
);
587 set_page_dirty(to_page
);
588 mark_page_accessed(to_page
);
589 page_cache_release(to_page
);
592 ttm_tt_free_alloced_pages(ttm
);
593 ttm
->swap_storage
= swap_storage
;
594 ttm
->page_flags
|= TTM_PAGE_FLAG_SWAPPED
;
595 if (persistant_swap_storage
)
596 ttm
->page_flags
|= TTM_PAGE_FLAG_PERSISTANT_SWAP
;
600 if (!persistant_swap_storage
)