1 /**************************************************************************
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 #include "ttm/ttm_memory.h"
29 #include "ttm/ttm_module.h"
30 #include <linux/spinlock.h>
31 #include <linux/sched.h>
32 #include <linux/wait.h>
34 #include <linux/module.h>
35 #include <linux/slab.h>
37 #define TTM_MEMORY_ALLOC_RETRIES 4
41 struct ttm_mem_global
*glob
;
50 static struct attribute ttm_mem_sys
= {
51 .name
= "zone_memory",
54 static struct attribute ttm_mem_emer
= {
55 .name
= "emergency_memory",
56 .mode
= S_IRUGO
| S_IWUSR
58 static struct attribute ttm_mem_max
= {
59 .name
= "available_memory",
60 .mode
= S_IRUGO
| S_IWUSR
62 static struct attribute ttm_mem_swap
= {
64 .mode
= S_IRUGO
| S_IWUSR
66 static struct attribute ttm_mem_used
= {
67 .name
= "used_memory",
71 static void ttm_mem_zone_kobj_release(struct kobject
*kobj
)
73 struct ttm_mem_zone
*zone
=
74 container_of(kobj
, struct ttm_mem_zone
, kobj
);
76 printk(KERN_INFO TTM_PFX
77 "Zone %7s: Used memory at exit: %llu kiB.\n",
78 zone
->name
, (unsigned long long) zone
->used_mem
>> 10);
82 static ssize_t
ttm_mem_zone_show(struct kobject
*kobj
,
83 struct attribute
*attr
,
86 struct ttm_mem_zone
*zone
=
87 container_of(kobj
, struct ttm_mem_zone
, kobj
);
90 spin_lock(&zone
->glob
->lock
);
91 if (attr
== &ttm_mem_sys
)
93 else if (attr
== &ttm_mem_emer
)
95 else if (attr
== &ttm_mem_max
)
97 else if (attr
== &ttm_mem_swap
)
98 val
= zone
->swap_limit
;
99 else if (attr
== &ttm_mem_used
)
100 val
= zone
->used_mem
;
101 spin_unlock(&zone
->glob
->lock
);
103 return snprintf(buffer
, PAGE_SIZE
, "%llu\n",
104 (unsigned long long) val
>> 10);
107 static void ttm_check_swapping(struct ttm_mem_global
*glob
);
109 static ssize_t
ttm_mem_zone_store(struct kobject
*kobj
,
110 struct attribute
*attr
,
114 struct ttm_mem_zone
*zone
=
115 container_of(kobj
, struct ttm_mem_zone
, kobj
);
120 chars
= sscanf(buffer
, "%lu", &val
);
127 spin_lock(&zone
->glob
->lock
);
128 if (val64
> zone
->zone_mem
)
129 val64
= zone
->zone_mem
;
130 if (attr
== &ttm_mem_emer
) {
131 zone
->emer_mem
= val64
;
132 if (zone
->max_mem
> val64
)
133 zone
->max_mem
= val64
;
134 } else if (attr
== &ttm_mem_max
) {
135 zone
->max_mem
= val64
;
136 if (zone
->emer_mem
< val64
)
137 zone
->emer_mem
= val64
;
138 } else if (attr
== &ttm_mem_swap
)
139 zone
->swap_limit
= val64
;
140 spin_unlock(&zone
->glob
->lock
);
142 ttm_check_swapping(zone
->glob
);
147 static struct attribute
*ttm_mem_zone_attrs
[] = {
156 static const struct sysfs_ops ttm_mem_zone_ops
= {
157 .show
= &ttm_mem_zone_show
,
158 .store
= &ttm_mem_zone_store
161 static struct kobj_type ttm_mem_zone_kobj_type
= {
162 .release
= &ttm_mem_zone_kobj_release
,
163 .sysfs_ops
= &ttm_mem_zone_ops
,
164 .default_attrs
= ttm_mem_zone_attrs
,
167 static void ttm_mem_global_kobj_release(struct kobject
*kobj
)
169 struct ttm_mem_global
*glob
=
170 container_of(kobj
, struct ttm_mem_global
, kobj
);
175 static struct kobj_type ttm_mem_glob_kobj_type
= {
176 .release
= &ttm_mem_global_kobj_release
,
179 static bool ttm_zones_above_swap_target(struct ttm_mem_global
*glob
,
180 bool from_wq
, uint64_t extra
)
183 struct ttm_mem_zone
*zone
;
186 for (i
= 0; i
< glob
->num_zones
; ++i
) {
187 zone
= glob
->zones
[i
];
190 target
= zone
->swap_limit
;
191 else if (capable(CAP_SYS_ADMIN
))
192 target
= zone
->emer_mem
;
194 target
= zone
->max_mem
;
196 target
= (extra
> target
) ? 0ULL : target
;
198 if (zone
->used_mem
> target
)
205 * At this point we only support a single shrink callback.
206 * Extend this if needed, perhaps using a linked list of callbacks.
207 * Note that this function is reentrant:
208 * many threads may try to swap out at any given time.
211 static void ttm_shrink(struct ttm_mem_global
*glob
, bool from_wq
,
215 struct ttm_mem_shrink
*shrink
;
217 spin_lock(&glob
->lock
);
218 if (glob
->shrink
== NULL
)
221 while (ttm_zones_above_swap_target(glob
, from_wq
, extra
)) {
222 shrink
= glob
->shrink
;
223 spin_unlock(&glob
->lock
);
224 ret
= shrink
->do_shrink(shrink
);
225 spin_lock(&glob
->lock
);
226 if (unlikely(ret
!= 0))
230 spin_unlock(&glob
->lock
);
235 static void ttm_shrink_work(struct work_struct
*work
)
237 struct ttm_mem_global
*glob
=
238 container_of(work
, struct ttm_mem_global
, work
);
240 ttm_shrink(glob
, true, 0ULL);
243 static int ttm_mem_init_kernel_zone(struct ttm_mem_global
*glob
,
244 const struct sysinfo
*si
)
246 struct ttm_mem_zone
*zone
= kzalloc(sizeof(*zone
), GFP_KERNEL
);
253 mem
= si
->totalram
- si
->totalhigh
;
256 zone
->name
= "kernel";
257 zone
->zone_mem
= mem
;
258 zone
->max_mem
= mem
>> 1;
259 zone
->emer_mem
= (mem
>> 1) + (mem
>> 2);
260 zone
->swap_limit
= zone
->max_mem
- (mem
>> 3);
263 glob
->zone_kernel
= zone
;
264 kobject_init(&zone
->kobj
, &ttm_mem_zone_kobj_type
);
265 ret
= kobject_add(&zone
->kobj
, &glob
->kobj
, zone
->name
);
266 if (unlikely(ret
!= 0)) {
267 kobject_put(&zone
->kobj
);
270 glob
->zones
[glob
->num_zones
++] = zone
;
274 #ifdef CONFIG_HIGHMEM
275 static int ttm_mem_init_highmem_zone(struct ttm_mem_global
*glob
,
276 const struct sysinfo
*si
)
278 struct ttm_mem_zone
*zone
;
282 if (si
->totalhigh
== 0)
285 zone
= kzalloc(sizeof(*zone
), GFP_KERNEL
);
292 zone
->name
= "highmem";
293 zone
->zone_mem
= mem
;
294 zone
->max_mem
= mem
>> 1;
295 zone
->emer_mem
= (mem
>> 1) + (mem
>> 2);
296 zone
->swap_limit
= zone
->max_mem
- (mem
>> 3);
299 glob
->zone_highmem
= zone
;
300 kobject_init(&zone
->kobj
, &ttm_mem_zone_kobj_type
);
301 ret
= kobject_add(&zone
->kobj
, &glob
->kobj
, zone
->name
);
302 if (unlikely(ret
!= 0)) {
303 kobject_put(&zone
->kobj
);
306 glob
->zones
[glob
->num_zones
++] = zone
;
310 static int ttm_mem_init_dma32_zone(struct ttm_mem_global
*glob
,
311 const struct sysinfo
*si
)
313 struct ttm_mem_zone
*zone
= kzalloc(sizeof(*zone
), GFP_KERNEL
);
324 * No special dma32 zone needed.
327 if (mem
<= ((uint64_t) 1ULL << 32)) {
333 * Limit max dma32 memory to 4GB for now
334 * until we can figure out how big this
338 mem
= ((uint64_t) 1ULL << 32);
339 zone
->name
= "dma32";
340 zone
->zone_mem
= mem
;
341 zone
->max_mem
= mem
>> 1;
342 zone
->emer_mem
= (mem
>> 1) + (mem
>> 2);
343 zone
->swap_limit
= zone
->max_mem
- (mem
>> 3);
346 glob
->zone_dma32
= zone
;
347 kobject_init(&zone
->kobj
, &ttm_mem_zone_kobj_type
);
348 ret
= kobject_add(&zone
->kobj
, &glob
->kobj
, zone
->name
);
349 if (unlikely(ret
!= 0)) {
350 kobject_put(&zone
->kobj
);
353 glob
->zones
[glob
->num_zones
++] = zone
;
358 int ttm_mem_global_init(struct ttm_mem_global
*glob
)
363 struct ttm_mem_zone
*zone
;
365 spin_lock_init(&glob
->lock
);
366 glob
->swap_queue
= create_singlethread_workqueue("ttm_swap");
367 INIT_WORK(&glob
->work
, ttm_shrink_work
);
368 init_waitqueue_head(&glob
->queue
);
369 kobject_init(&glob
->kobj
, &ttm_mem_glob_kobj_type
);
370 ret
= kobject_add(&glob
->kobj
,
372 "memory_accounting");
373 if (unlikely(ret
!= 0)) {
374 kobject_put(&glob
->kobj
);
380 ret
= ttm_mem_init_kernel_zone(glob
, &si
);
381 if (unlikely(ret
!= 0))
383 #ifdef CONFIG_HIGHMEM
384 ret
= ttm_mem_init_highmem_zone(glob
, &si
);
385 if (unlikely(ret
!= 0))
388 ret
= ttm_mem_init_dma32_zone(glob
, &si
);
389 if (unlikely(ret
!= 0))
392 for (i
= 0; i
< glob
->num_zones
; ++i
) {
393 zone
= glob
->zones
[i
];
394 printk(KERN_INFO TTM_PFX
395 "Zone %7s: Available graphics memory: %llu kiB.\n",
396 zone
->name
, (unsigned long long) zone
->max_mem
>> 10);
400 ttm_mem_global_release(glob
);
403 EXPORT_SYMBOL(ttm_mem_global_init
);
405 void ttm_mem_global_release(struct ttm_mem_global
*glob
)
408 struct ttm_mem_zone
*zone
;
410 flush_workqueue(glob
->swap_queue
);
411 destroy_workqueue(glob
->swap_queue
);
412 glob
->swap_queue
= NULL
;
413 for (i
= 0; i
< glob
->num_zones
; ++i
) {
414 zone
= glob
->zones
[i
];
415 kobject_del(&zone
->kobj
);
416 kobject_put(&zone
->kobj
);
418 kobject_del(&glob
->kobj
);
419 kobject_put(&glob
->kobj
);
421 EXPORT_SYMBOL(ttm_mem_global_release
);
423 static void ttm_check_swapping(struct ttm_mem_global
*glob
)
425 bool needs_swapping
= false;
427 struct ttm_mem_zone
*zone
;
429 spin_lock(&glob
->lock
);
430 for (i
= 0; i
< glob
->num_zones
; ++i
) {
431 zone
= glob
->zones
[i
];
432 if (zone
->used_mem
> zone
->swap_limit
) {
433 needs_swapping
= true;
438 spin_unlock(&glob
->lock
);
440 if (unlikely(needs_swapping
))
441 (void)queue_work(glob
->swap_queue
, &glob
->work
);
445 static void ttm_mem_global_free_zone(struct ttm_mem_global
*glob
,
446 struct ttm_mem_zone
*single_zone
,
450 struct ttm_mem_zone
*zone
;
452 spin_lock(&glob
->lock
);
453 for (i
= 0; i
< glob
->num_zones
; ++i
) {
454 zone
= glob
->zones
[i
];
455 if (single_zone
&& zone
!= single_zone
)
457 zone
->used_mem
-= amount
;
459 spin_unlock(&glob
->lock
);
462 void ttm_mem_global_free(struct ttm_mem_global
*glob
,
465 return ttm_mem_global_free_zone(glob
, NULL
, amount
);
467 EXPORT_SYMBOL(ttm_mem_global_free
);
469 static int ttm_mem_global_reserve(struct ttm_mem_global
*glob
,
470 struct ttm_mem_zone
*single_zone
,
471 uint64_t amount
, bool reserve
)
476 struct ttm_mem_zone
*zone
;
478 spin_lock(&glob
->lock
);
479 for (i
= 0; i
< glob
->num_zones
; ++i
) {
480 zone
= glob
->zones
[i
];
481 if (single_zone
&& zone
!= single_zone
)
484 limit
= (capable(CAP_SYS_ADMIN
)) ?
485 zone
->emer_mem
: zone
->max_mem
;
487 if (zone
->used_mem
> limit
)
492 for (i
= 0; i
< glob
->num_zones
; ++i
) {
493 zone
= glob
->zones
[i
];
494 if (single_zone
&& zone
!= single_zone
)
496 zone
->used_mem
+= amount
;
502 spin_unlock(&glob
->lock
);
503 ttm_check_swapping(glob
);
509 static int ttm_mem_global_alloc_zone(struct ttm_mem_global
*glob
,
510 struct ttm_mem_zone
*single_zone
,
512 bool no_wait
, bool interruptible
)
514 int count
= TTM_MEMORY_ALLOC_RETRIES
;
516 while (unlikely(ttm_mem_global_reserve(glob
,
522 if (unlikely(count
-- == 0))
524 ttm_shrink(glob
, false, memory
+ (memory
>> 2) + 16);
530 int ttm_mem_global_alloc(struct ttm_mem_global
*glob
, uint64_t memory
,
531 bool no_wait
, bool interruptible
)
534 * Normal allocations of kernel memory are registered in
538 return ttm_mem_global_alloc_zone(glob
, NULL
, memory
, no_wait
,
541 EXPORT_SYMBOL(ttm_mem_global_alloc
);
543 int ttm_mem_global_alloc_page(struct ttm_mem_global
*glob
,
545 bool no_wait
, bool interruptible
)
548 struct ttm_mem_zone
*zone
= NULL
;
551 * Page allocations may be registed in a single zone
552 * only if highmem or !dma32.
555 #ifdef CONFIG_HIGHMEM
556 if (PageHighMem(page
) && glob
->zone_highmem
!= NULL
)
557 zone
= glob
->zone_highmem
;
559 if (glob
->zone_dma32
&& page_to_pfn(page
) > 0x00100000UL
)
560 zone
= glob
->zone_kernel
;
562 return ttm_mem_global_alloc_zone(glob
, zone
, PAGE_SIZE
, no_wait
,
566 void ttm_mem_global_free_page(struct ttm_mem_global
*glob
, struct page
*page
)
568 struct ttm_mem_zone
*zone
= NULL
;
570 #ifdef CONFIG_HIGHMEM
571 if (PageHighMem(page
) && glob
->zone_highmem
!= NULL
)
572 zone
= glob
->zone_highmem
;
574 if (glob
->zone_dma32
&& page_to_pfn(page
) > 0x00100000UL
)
575 zone
= glob
->zone_kernel
;
577 ttm_mem_global_free_zone(glob
, zone
, PAGE_SIZE
);
581 size_t ttm_round_pot(size_t size
)
583 if ((size
& (size
- 1)) == 0)
585 else if (size
> PAGE_SIZE
)
586 return PAGE_ALIGN(size
);
590 while (tmp_size
< size
)
597 EXPORT_SYMBOL(ttm_round_pot
);