]>
git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blob - kernel/power/snapshot.c
2 * linux/kernel/power/snapshot.c
4 * This file provides system snapshot/restore functionality for swsusp.
6 * Copyright (C) 1998-2005 Pavel Machek <pavel@ucw.cz>
7 * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
9 * This file is released under the GPLv2.
13 #include <linux/version.h>
14 #include <linux/module.h>
16 #include <linux/suspend.h>
17 #include <linux/delay.h>
18 #include <linux/bitops.h>
19 #include <linux/spinlock.h>
20 #include <linux/kernel.h>
22 #include <linux/device.h>
23 #include <linux/init.h>
24 #include <linux/bootmem.h>
25 #include <linux/syscalls.h>
26 #include <linux/console.h>
27 #include <linux/highmem.h>
28 #include <linux/list.h>
29 #include <linux/slab.h>
30 #include <linux/compiler.h>
32 #include <asm/uaccess.h>
33 #include <asm/mmu_context.h>
34 #include <asm/pgtable.h>
35 #include <asm/tlbflush.h>
40 static int swsusp_page_is_free(struct page
*);
41 static void swsusp_set_page_forbidden(struct page
*);
42 static void swsusp_unset_page_forbidden(struct page
*);
45 * Number of bytes to reserve for memory allocations made by device drivers
46 * from their ->freeze() and ->freeze_noirq() callbacks so that they don't
47 * cause image creation to fail (tunable via /sys/power/reserved_size).
49 unsigned long reserved_size
;
51 void __init
hibernate_reserved_size_init(void)
53 reserved_size
= SPARE_PAGES
* PAGE_SIZE
;
57 * Preferred image size in bytes (tunable via /sys/power/image_size).
58 * When it is set to N, swsusp will do its best to ensure the image
59 * size will not exceed N bytes, but if that is impossible, it will
60 * try to create the smallest image possible.
62 unsigned long image_size
;
64 void __init
hibernate_image_size_init(void)
66 image_size
= ((totalram_pages
* 2) / 5) * PAGE_SIZE
;
69 /* List of PBEs needed for restoring the pages that were allocated before
70 * the suspend and included in the suspend image, but have also been
71 * allocated by the "resume" kernel, so their contents cannot be written
72 * directly to their "original" page frames.
74 struct pbe
*restore_pblist
;
76 /* Pointer to an auxiliary buffer (1 page) */
80 * @safe_needed - on resume, for storing the PBE list and the image,
81 * we can only use memory pages that do not conflict with the pages
82 * used before suspend. The unsafe pages have PageNosaveFree set
83 * and we count them using unsafe_pages.
85 * Each allocated image page is marked as PageNosave and PageNosaveFree
86 * so that swsusp_free() can release it.
91 #define PG_UNSAFE_CLEAR 1
92 #define PG_UNSAFE_KEEP 0
94 static unsigned int allocated_unsafe_pages
;
96 static void *get_image_page(gfp_t gfp_mask
, int safe_needed
)
100 res
= (void *)get_zeroed_page(gfp_mask
);
102 while (res
&& swsusp_page_is_free(virt_to_page(res
))) {
103 /* The page is unsafe, mark it for swsusp_free() */
104 swsusp_set_page_forbidden(virt_to_page(res
));
105 allocated_unsafe_pages
++;
106 res
= (void *)get_zeroed_page(gfp_mask
);
109 swsusp_set_page_forbidden(virt_to_page(res
));
110 swsusp_set_page_free(virt_to_page(res
));
115 unsigned long get_safe_page(gfp_t gfp_mask
)
117 return (unsigned long)get_image_page(gfp_mask
, PG_SAFE
);
120 static struct page
*alloc_image_page(gfp_t gfp_mask
)
124 page
= alloc_page(gfp_mask
);
126 swsusp_set_page_forbidden(page
);
127 swsusp_set_page_free(page
);
133 * free_image_page - free page represented by @addr, allocated with
134 * get_image_page (page flags set by it must be cleared)
137 static inline void free_image_page(void *addr
, int clear_nosave_free
)
141 BUG_ON(!virt_addr_valid(addr
));
143 page
= virt_to_page(addr
);
145 swsusp_unset_page_forbidden(page
);
146 if (clear_nosave_free
)
147 swsusp_unset_page_free(page
);
152 /* struct linked_page is used to build chains of pages */
154 #define LINKED_PAGE_DATA_SIZE (PAGE_SIZE - sizeof(void *))
157 struct linked_page
*next
;
158 char data
[LINKED_PAGE_DATA_SIZE
];
162 free_list_of_pages(struct linked_page
*list
, int clear_page_nosave
)
165 struct linked_page
*lp
= list
->next
;
167 free_image_page(list
, clear_page_nosave
);
173 * struct chain_allocator is used for allocating small objects out of
174 * a linked list of pages called 'the chain'.
176 * The chain grows each time when there is no room for a new object in
177 * the current page. The allocated objects cannot be freed individually.
178 * It is only possible to free them all at once, by freeing the entire
181 * NOTE: The chain allocator may be inefficient if the allocated objects
182 * are not much smaller than PAGE_SIZE.
185 struct chain_allocator
{
186 struct linked_page
*chain
; /* the chain */
187 unsigned int used_space
; /* total size of objects allocated out
188 * of the current page
190 gfp_t gfp_mask
; /* mask for allocating pages */
191 int safe_needed
; /* if set, only "safe" pages are allocated */
195 chain_init(struct chain_allocator
*ca
, gfp_t gfp_mask
, int safe_needed
)
198 ca
->used_space
= LINKED_PAGE_DATA_SIZE
;
199 ca
->gfp_mask
= gfp_mask
;
200 ca
->safe_needed
= safe_needed
;
203 static void *chain_alloc(struct chain_allocator
*ca
, unsigned int size
)
207 if (LINKED_PAGE_DATA_SIZE
- ca
->used_space
< size
) {
208 struct linked_page
*lp
;
210 lp
= get_image_page(ca
->gfp_mask
, ca
->safe_needed
);
214 lp
->next
= ca
->chain
;
218 ret
= ca
->chain
->data
+ ca
->used_space
;
219 ca
->used_space
+= size
;
224 * Data types related to memory bitmaps.
226 * Memory bitmap is a structure consiting of many linked lists of
227 * objects. The main list's elements are of type struct zone_bitmap
228 * and each of them corresonds to one zone. For each zone bitmap
229 * object there is a list of objects of type struct bm_block that
230 * represent each blocks of bitmap in which information is stored.
232 * struct memory_bitmap contains a pointer to the main list of zone
233 * bitmap objects, a struct bm_position used for browsing the bitmap,
234 * and a pointer to the list of pages used for allocating all of the
235 * zone bitmap objects and bitmap block objects.
237 * NOTE: It has to be possible to lay out the bitmap in memory
238 * using only allocations of order 0. Additionally, the bitmap is
239 * designed to work with arbitrary number of zones (this is over the
240 * top for now, but let's avoid making unnecessary assumptions ;-).
242 * struct zone_bitmap contains a pointer to a list of bitmap block
243 * objects and a pointer to the bitmap block object that has been
244 * most recently used for setting bits. Additionally, it contains the
245 * pfns that correspond to the start and end of the represented zone.
247 * struct bm_block contains a pointer to the memory page in which
248 * information is stored (in the form of a block of bitmap)
249 * It also contains the pfns that correspond to the start and end of
250 * the represented memory area.
252 * The memory bitmap is organized as a radix tree to guarantee fast random
253 * access to the bits. There is one radix tree for each zone (as returned
254 * from create_mem_extents).
256 * One radix tree is represented by one struct mem_zone_bm_rtree. There are
257 * two linked lists for the nodes of the tree, one for the inner nodes and
258 * one for the leave nodes. The linked leave nodes are used for fast linear
259 * access of the memory bitmap.
261 * The struct rtree_node represents one node of the radix tree.
264 #define BM_END_OF_MAP (~0UL)
266 #define BM_BITS_PER_BLOCK (PAGE_SIZE * BITS_PER_BYTE)
267 #define BM_BLOCK_SHIFT (PAGE_SHIFT + 3)
268 #define BM_BLOCK_MASK ((1UL << BM_BLOCK_SHIFT) - 1)
271 struct list_head hook
; /* hook into a list of bitmap blocks */
272 unsigned long start_pfn
; /* pfn represented by the first bit */
273 unsigned long end_pfn
; /* pfn represented by the last bit plus 1 */
274 unsigned long *data
; /* bitmap representing pages */
277 static inline unsigned long bm_block_bits(struct bm_block
*bb
)
279 return bb
->end_pfn
- bb
->start_pfn
;
283 * struct rtree_node is a wrapper struct to link the nodes
284 * of the rtree together for easy linear iteration over
285 * bits and easy freeing
288 struct list_head list
;
293 * struct mem_zone_bm_rtree represents a bitmap used for one
294 * populated memory zone.
296 struct mem_zone_bm_rtree
{
297 struct list_head list
; /* Link Zones together */
298 struct list_head nodes
; /* Radix Tree inner nodes */
299 struct list_head leaves
; /* Radix Tree leaves */
300 unsigned long start_pfn
; /* Zone start page frame */
301 unsigned long end_pfn
; /* Zone end page frame + 1 */
302 struct rtree_node
*rtree
; /* Radix Tree Root */
303 int levels
; /* Number of Radix Tree Levels */
304 unsigned int blocks
; /* Number of Bitmap Blocks */
307 /* strcut bm_position is used for browsing memory bitmaps */
310 struct bm_block
*block
;
314 struct memory_bitmap
{
315 struct list_head zones
;
316 struct list_head blocks
; /* list of bitmap blocks */
317 struct linked_page
*p_list
; /* list of pages used to store zone
318 * bitmap objects and bitmap block
321 struct bm_position cur
; /* most recently used bit position */
324 /* Functions that operate on memory bitmaps */
326 #define BM_ENTRIES_PER_LEVEL (PAGE_SIZE / sizeof(unsigned long))
327 #if BITS_PER_LONG == 32
328 #define BM_RTREE_LEVEL_SHIFT (PAGE_SHIFT - 2)
330 #define BM_RTREE_LEVEL_SHIFT (PAGE_SHIFT - 3)
332 #define BM_RTREE_LEVEL_MASK ((1UL << BM_RTREE_LEVEL_SHIFT) - 1)
335 * alloc_rtree_node - Allocate a new node and add it to the radix tree.
337 * This function is used to allocate inner nodes as well as the
338 * leave nodes of the radix tree. It also adds the node to the
339 * corresponding linked list passed in by the *list parameter.
341 static struct rtree_node
*alloc_rtree_node(gfp_t gfp_mask
, int safe_needed
,
342 struct chain_allocator
*ca
,
343 struct list_head
*list
)
345 struct rtree_node
*node
;
347 node
= chain_alloc(ca
, sizeof(struct rtree_node
));
351 node
->data
= get_image_page(gfp_mask
, safe_needed
);
355 list_add_tail(&node
->list
, list
);
361 * add_rtree_block - Add a new leave node to the radix tree
363 * The leave nodes need to be allocated in order to keep the leaves
364 * linked list in order. This is guaranteed by the zone->blocks
367 static int add_rtree_block(struct mem_zone_bm_rtree
*zone
, gfp_t gfp_mask
,
368 int safe_needed
, struct chain_allocator
*ca
)
370 struct rtree_node
*node
, *block
, **dst
;
371 unsigned int levels_needed
, block_nr
;
374 block_nr
= zone
->blocks
;
377 /* How many levels do we need for this block nr? */
380 block_nr
>>= BM_RTREE_LEVEL_SHIFT
;
383 /* Make sure the rtree has enough levels */
384 for (i
= zone
->levels
; i
< levels_needed
; i
++) {
385 node
= alloc_rtree_node(gfp_mask
, safe_needed
, ca
,
390 node
->data
[0] = (unsigned long)zone
->rtree
;
395 /* Allocate new block */
396 block
= alloc_rtree_node(gfp_mask
, safe_needed
, ca
, &zone
->leaves
);
400 /* Now walk the rtree to insert the block */
403 block_nr
= zone
->blocks
;
404 for (i
= zone
->levels
; i
> 0; i
--) {
408 node
= alloc_rtree_node(gfp_mask
, safe_needed
, ca
,
415 index
= block_nr
>> ((i
- 1) * BM_RTREE_LEVEL_SHIFT
);
416 index
&= BM_RTREE_LEVEL_MASK
;
417 dst
= (struct rtree_node
**)&((*dst
)->data
[index
]);
427 static void free_zone_bm_rtree(struct mem_zone_bm_rtree
*zone
,
428 int clear_nosave_free
);
431 * create_zone_bm_rtree - create a radix tree for one zone
433 * Allocated the mem_zone_bm_rtree structure and initializes it.
434 * This function also allocated and builds the radix tree for the
437 static struct mem_zone_bm_rtree
*
438 create_zone_bm_rtree(gfp_t gfp_mask
, int safe_needed
,
439 struct chain_allocator
*ca
,
440 unsigned long start
, unsigned long end
)
442 struct mem_zone_bm_rtree
*zone
;
443 unsigned int i
, nr_blocks
;
447 zone
= chain_alloc(ca
, sizeof(struct mem_zone_bm_rtree
));
451 INIT_LIST_HEAD(&zone
->nodes
);
452 INIT_LIST_HEAD(&zone
->leaves
);
453 zone
->start_pfn
= start
;
455 nr_blocks
= DIV_ROUND_UP(pages
, BM_BITS_PER_BLOCK
);
457 for (i
= 0; i
< nr_blocks
; i
++) {
458 if (add_rtree_block(zone
, gfp_mask
, safe_needed
, ca
)) {
459 free_zone_bm_rtree(zone
, PG_UNSAFE_CLEAR
);
468 * free_zone_bm_rtree - Free the memory of the radix tree
470 * Free all node pages of the radix tree. The mem_zone_bm_rtree
471 * structure itself is not freed here nor are the rtree_node
474 static void free_zone_bm_rtree(struct mem_zone_bm_rtree
*zone
,
475 int clear_nosave_free
)
477 struct rtree_node
*node
;
479 list_for_each_entry(node
, &zone
->nodes
, list
)
480 free_image_page(node
->data
, clear_nosave_free
);
482 list_for_each_entry(node
, &zone
->leaves
, list
)
483 free_image_page(node
->data
, clear_nosave_free
);
486 static void memory_bm_position_reset(struct memory_bitmap
*bm
)
488 bm
->cur
.block
= list_entry(bm
->blocks
.next
, struct bm_block
, hook
);
492 static void memory_bm_free(struct memory_bitmap
*bm
, int clear_nosave_free
);
495 * create_bm_block_list - create a list of block bitmap objects
496 * @pages - number of pages to track
497 * @list - list to put the allocated blocks into
498 * @ca - chain allocator to be used for allocating memory
500 static int create_bm_block_list(unsigned long pages
,
501 struct list_head
*list
,
502 struct chain_allocator
*ca
)
504 unsigned int nr_blocks
= DIV_ROUND_UP(pages
, BM_BITS_PER_BLOCK
);
506 while (nr_blocks
-- > 0) {
509 bb
= chain_alloc(ca
, sizeof(struct bm_block
));
512 list_add(&bb
->hook
, list
);
519 struct list_head hook
;
525 * free_mem_extents - free a list of memory extents
526 * @list - list of extents to empty
528 static void free_mem_extents(struct list_head
*list
)
530 struct mem_extent
*ext
, *aux
;
532 list_for_each_entry_safe(ext
, aux
, list
, hook
) {
533 list_del(&ext
->hook
);
539 * create_mem_extents - create a list of memory extents representing
540 * contiguous ranges of PFNs
541 * @list - list to put the extents into
542 * @gfp_mask - mask to use for memory allocations
544 static int create_mem_extents(struct list_head
*list
, gfp_t gfp_mask
)
548 INIT_LIST_HEAD(list
);
550 for_each_populated_zone(zone
) {
551 unsigned long zone_start
, zone_end
;
552 struct mem_extent
*ext
, *cur
, *aux
;
554 zone_start
= zone
->zone_start_pfn
;
555 zone_end
= zone_end_pfn(zone
);
557 list_for_each_entry(ext
, list
, hook
)
558 if (zone_start
<= ext
->end
)
561 if (&ext
->hook
== list
|| zone_end
< ext
->start
) {
562 /* New extent is necessary */
563 struct mem_extent
*new_ext
;
565 new_ext
= kzalloc(sizeof(struct mem_extent
), gfp_mask
);
567 free_mem_extents(list
);
570 new_ext
->start
= zone_start
;
571 new_ext
->end
= zone_end
;
572 list_add_tail(&new_ext
->hook
, &ext
->hook
);
576 /* Merge this zone's range of PFNs with the existing one */
577 if (zone_start
< ext
->start
)
578 ext
->start
= zone_start
;
579 if (zone_end
> ext
->end
)
582 /* More merging may be possible */
584 list_for_each_entry_safe_continue(cur
, aux
, list
, hook
) {
585 if (zone_end
< cur
->start
)
587 if (zone_end
< cur
->end
)
589 list_del(&cur
->hook
);
598 * memory_bm_create - allocate memory for a memory bitmap
601 memory_bm_create(struct memory_bitmap
*bm
, gfp_t gfp_mask
, int safe_needed
)
603 struct chain_allocator ca
;
604 struct list_head mem_extents
;
605 struct mem_extent
*ext
;
608 chain_init(&ca
, gfp_mask
, safe_needed
);
609 INIT_LIST_HEAD(&bm
->blocks
);
610 INIT_LIST_HEAD(&bm
->zones
);
612 error
= create_mem_extents(&mem_extents
, gfp_mask
);
616 list_for_each_entry(ext
, &mem_extents
, hook
) {
617 struct mem_zone_bm_rtree
*zone
;
619 unsigned long pfn
= ext
->start
;
620 unsigned long pages
= ext
->end
- ext
->start
;
622 bb
= list_entry(bm
->blocks
.prev
, struct bm_block
, hook
);
624 error
= create_bm_block_list(pages
, bm
->blocks
.prev
, &ca
);
628 list_for_each_entry_continue(bb
, &bm
->blocks
, hook
) {
629 bb
->data
= get_image_page(gfp_mask
, safe_needed
);
636 if (pages
>= BM_BITS_PER_BLOCK
) {
637 pfn
+= BM_BITS_PER_BLOCK
;
638 pages
-= BM_BITS_PER_BLOCK
;
640 /* This is executed only once in the loop */
646 zone
= create_zone_bm_rtree(gfp_mask
, safe_needed
, &ca
,
647 ext
->start
, ext
->end
);
650 list_add_tail(&zone
->list
, &bm
->zones
);
653 bm
->p_list
= ca
.chain
;
654 memory_bm_position_reset(bm
);
656 free_mem_extents(&mem_extents
);
660 bm
->p_list
= ca
.chain
;
661 memory_bm_free(bm
, PG_UNSAFE_CLEAR
);
666 * memory_bm_free - free memory occupied by the memory bitmap @bm
668 static void memory_bm_free(struct memory_bitmap
*bm
, int clear_nosave_free
)
670 struct mem_zone_bm_rtree
*zone
;
673 list_for_each_entry(bb
, &bm
->blocks
, hook
)
675 free_image_page(bb
->data
, clear_nosave_free
);
677 list_for_each_entry(zone
, &bm
->zones
, list
)
678 free_zone_bm_rtree(zone
, clear_nosave_free
);
680 free_list_of_pages(bm
->p_list
, clear_nosave_free
);
682 INIT_LIST_HEAD(&bm
->zones
);
683 INIT_LIST_HEAD(&bm
->blocks
);
687 * memory_bm_find_bit - find the bit in the bitmap @bm that corresponds
688 * to given pfn. The cur_zone_bm member of @bm and the cur_block member
689 * of @bm->cur_zone_bm are updated.
691 static int memory_bm_find_bit(struct memory_bitmap
*bm
, unsigned long pfn
,
692 void **addr
, unsigned int *bit_nr
)
697 * Check if the pfn corresponds to the current bitmap block and find
698 * the block where it fits if this is not the case.
701 if (pfn
< bb
->start_pfn
)
702 list_for_each_entry_continue_reverse(bb
, &bm
->blocks
, hook
)
703 if (pfn
>= bb
->start_pfn
)
706 if (pfn
>= bb
->end_pfn
)
707 list_for_each_entry_continue(bb
, &bm
->blocks
, hook
)
708 if (pfn
>= bb
->start_pfn
&& pfn
< bb
->end_pfn
)
711 if (&bb
->hook
== &bm
->blocks
)
714 /* The block has been found */
716 pfn
-= bb
->start_pfn
;
717 bm
->cur
.bit
= pfn
+ 1;
724 * memory_rtree_find_bit - Find the bit for pfn in the memory
727 * Walks the radix tree to find the page which contains the bit for
728 * pfn and returns the bit position in **addr and *bit_nr.
730 static int memory_rtree_find_bit(struct memory_bitmap
*bm
, unsigned long pfn
,
731 void **addr
, unsigned int *bit_nr
)
733 struct mem_zone_bm_rtree
*curr
, *zone
;
734 struct rtree_node
*node
;
739 /* Find the right zone */
740 list_for_each_entry(curr
, &bm
->zones
, list
) {
741 if (pfn
>= curr
->start_pfn
&& pfn
< curr
->end_pfn
) {
751 * We have a zone. Now walk the radix tree to find the leave
755 block_nr
= (pfn
- zone
->start_pfn
) >> BM_BLOCK_SHIFT
;
757 for (i
= zone
->levels
; i
> 0; i
--) {
760 index
= block_nr
>> ((i
- 1) * BM_RTREE_LEVEL_SHIFT
);
761 index
&= BM_RTREE_LEVEL_MASK
;
762 BUG_ON(node
->data
[index
] == 0);
763 node
= (struct rtree_node
*)node
->data
[index
];
766 /* Set return values */
768 *bit_nr
= (pfn
- zone
->start_pfn
) & BM_BLOCK_MASK
;
773 static void memory_bm_set_bit(struct memory_bitmap
*bm
, unsigned long pfn
)
779 error
= memory_bm_find_bit(bm
, pfn
, &addr
, &bit
);
783 error
= memory_rtree_find_bit(bm
, pfn
, &addr
, &bit
);
788 static int mem_bm_set_bit_check(struct memory_bitmap
*bm
, unsigned long pfn
)
794 error
= memory_bm_find_bit(bm
, pfn
, &addr
, &bit
);
800 error
= memory_rtree_find_bit(bm
, pfn
, &addr
, &bit
);
807 static void memory_bm_clear_bit(struct memory_bitmap
*bm
, unsigned long pfn
)
813 error
= memory_bm_find_bit(bm
, pfn
, &addr
, &bit
);
815 clear_bit(bit
, addr
);
817 error
= memory_rtree_find_bit(bm
, pfn
, &addr
, &bit
);
819 clear_bit(bit
, addr
);
822 static int memory_bm_test_bit(struct memory_bitmap
*bm
, unsigned long pfn
)
829 error
= memory_bm_find_bit(bm
, pfn
, &addr
, &bit
);
831 v
= test_bit(bit
, addr
);
833 error2
= memory_rtree_find_bit(bm
, pfn
, &addr
, &bit
);
836 WARN_ON_ONCE(v
!= test_bit(bit
, addr
));
841 static bool memory_bm_pfn_present(struct memory_bitmap
*bm
, unsigned long pfn
)
847 present
= !memory_bm_find_bit(bm
, pfn
, &addr
, &bit
);
849 WARN_ON_ONCE(present
!= !memory_rtree_find_bit(bm
, pfn
, &addr
, &bit
));
855 * memory_bm_next_pfn - find the pfn that corresponds to the next set bit
856 * in the bitmap @bm. If the pfn cannot be found, BM_END_OF_MAP is
859 * It is required to run memory_bm_position_reset() before the first call to
863 static unsigned long memory_bm_next_pfn(struct memory_bitmap
*bm
)
871 bit
= find_next_bit(bb
->data
, bm_block_bits(bb
), bit
);
872 if (bit
< bm_block_bits(bb
))
875 bb
= list_entry(bb
->hook
.next
, struct bm_block
, hook
);
878 } while (&bb
->hook
!= &bm
->blocks
);
880 memory_bm_position_reset(bm
);
881 return BM_END_OF_MAP
;
884 bm
->cur
.bit
= bit
+ 1;
885 return bb
->start_pfn
+ bit
;
889 * This structure represents a range of page frames the contents of which
890 * should not be saved during the suspend.
893 struct nosave_region
{
894 struct list_head list
;
895 unsigned long start_pfn
;
896 unsigned long end_pfn
;
899 static LIST_HEAD(nosave_regions
);
902 * register_nosave_region - register a range of page frames the contents
903 * of which should not be saved during the suspend (to be used in the early
904 * initialization code)
908 __register_nosave_region(unsigned long start_pfn
, unsigned long end_pfn
,
911 struct nosave_region
*region
;
913 if (start_pfn
>= end_pfn
)
916 if (!list_empty(&nosave_regions
)) {
917 /* Try to extend the previous region (they should be sorted) */
918 region
= list_entry(nosave_regions
.prev
,
919 struct nosave_region
, list
);
920 if (region
->end_pfn
== start_pfn
) {
921 region
->end_pfn
= end_pfn
;
926 /* during init, this shouldn't fail */
927 region
= kmalloc(sizeof(struct nosave_region
), GFP_KERNEL
);
930 /* This allocation cannot fail */
931 region
= memblock_virt_alloc(sizeof(struct nosave_region
), 0);
932 region
->start_pfn
= start_pfn
;
933 region
->end_pfn
= end_pfn
;
934 list_add_tail(®ion
->list
, &nosave_regions
);
936 printk(KERN_INFO
"PM: Registered nosave memory: [mem %#010llx-%#010llx]\n",
937 (unsigned long long) start_pfn
<< PAGE_SHIFT
,
938 ((unsigned long long) end_pfn
<< PAGE_SHIFT
) - 1);
942 * Set bits in this map correspond to the page frames the contents of which
943 * should not be saved during the suspend.
945 static struct memory_bitmap
*forbidden_pages_map
;
947 /* Set bits in this map correspond to free page frames. */
948 static struct memory_bitmap
*free_pages_map
;
951 * Each page frame allocated for creating the image is marked by setting the
952 * corresponding bits in forbidden_pages_map and free_pages_map simultaneously
955 void swsusp_set_page_free(struct page
*page
)
958 memory_bm_set_bit(free_pages_map
, page_to_pfn(page
));
961 static int swsusp_page_is_free(struct page
*page
)
963 return free_pages_map
?
964 memory_bm_test_bit(free_pages_map
, page_to_pfn(page
)) : 0;
967 void swsusp_unset_page_free(struct page
*page
)
970 memory_bm_clear_bit(free_pages_map
, page_to_pfn(page
));
973 static void swsusp_set_page_forbidden(struct page
*page
)
975 if (forbidden_pages_map
)
976 memory_bm_set_bit(forbidden_pages_map
, page_to_pfn(page
));
979 int swsusp_page_is_forbidden(struct page
*page
)
981 return forbidden_pages_map
?
982 memory_bm_test_bit(forbidden_pages_map
, page_to_pfn(page
)) : 0;
985 static void swsusp_unset_page_forbidden(struct page
*page
)
987 if (forbidden_pages_map
)
988 memory_bm_clear_bit(forbidden_pages_map
, page_to_pfn(page
));
992 * mark_nosave_pages - set bits corresponding to the page frames the
993 * contents of which should not be saved in a given bitmap.
996 static void mark_nosave_pages(struct memory_bitmap
*bm
)
998 struct nosave_region
*region
;
1000 if (list_empty(&nosave_regions
))
1003 list_for_each_entry(region
, &nosave_regions
, list
) {
1006 pr_debug("PM: Marking nosave pages: [mem %#010llx-%#010llx]\n",
1007 (unsigned long long) region
->start_pfn
<< PAGE_SHIFT
,
1008 ((unsigned long long) region
->end_pfn
<< PAGE_SHIFT
)
1011 for (pfn
= region
->start_pfn
; pfn
< region
->end_pfn
; pfn
++)
1012 if (pfn_valid(pfn
)) {
1014 * It is safe to ignore the result of
1015 * mem_bm_set_bit_check() here, since we won't
1016 * touch the PFNs for which the error is
1019 mem_bm_set_bit_check(bm
, pfn
);
1025 * create_basic_memory_bitmaps - create bitmaps needed for marking page
1026 * frames that should not be saved and free page frames. The pointers
1027 * forbidden_pages_map and free_pages_map are only modified if everything
1028 * goes well, because we don't want the bits to be used before both bitmaps
1032 int create_basic_memory_bitmaps(void)
1034 struct memory_bitmap
*bm1
, *bm2
;
1037 if (forbidden_pages_map
&& free_pages_map
)
1040 BUG_ON(forbidden_pages_map
|| free_pages_map
);
1042 bm1
= kzalloc(sizeof(struct memory_bitmap
), GFP_KERNEL
);
1046 error
= memory_bm_create(bm1
, GFP_KERNEL
, PG_ANY
);
1048 goto Free_first_object
;
1050 bm2
= kzalloc(sizeof(struct memory_bitmap
), GFP_KERNEL
);
1052 goto Free_first_bitmap
;
1054 error
= memory_bm_create(bm2
, GFP_KERNEL
, PG_ANY
);
1056 goto Free_second_object
;
1058 forbidden_pages_map
= bm1
;
1059 free_pages_map
= bm2
;
1060 mark_nosave_pages(forbidden_pages_map
);
1062 pr_debug("PM: Basic memory bitmaps created\n");
1069 memory_bm_free(bm1
, PG_UNSAFE_CLEAR
);
1076 * free_basic_memory_bitmaps - free memory bitmaps allocated by
1077 * create_basic_memory_bitmaps(). The auxiliary pointers are necessary
1078 * so that the bitmaps themselves are not referred to while they are being
1082 void free_basic_memory_bitmaps(void)
1084 struct memory_bitmap
*bm1
, *bm2
;
1086 if (WARN_ON(!(forbidden_pages_map
&& free_pages_map
)))
1089 bm1
= forbidden_pages_map
;
1090 bm2
= free_pages_map
;
1091 forbidden_pages_map
= NULL
;
1092 free_pages_map
= NULL
;
1093 memory_bm_free(bm1
, PG_UNSAFE_CLEAR
);
1095 memory_bm_free(bm2
, PG_UNSAFE_CLEAR
);
1098 pr_debug("PM: Basic memory bitmaps freed\n");
1102 * snapshot_additional_pages - estimate the number of additional pages
1103 * be needed for setting up the suspend image data structures for given
1104 * zone (usually the returned value is greater than the exact number)
1107 unsigned int snapshot_additional_pages(struct zone
*zone
)
1109 unsigned int rtree
, nodes
;
1112 res
= DIV_ROUND_UP(zone
->spanned_pages
, BM_BITS_PER_BLOCK
);
1113 res
+= DIV_ROUND_UP(res
* sizeof(struct bm_block
),
1114 LINKED_PAGE_DATA_SIZE
);
1115 rtree
= nodes
= DIV_ROUND_UP(zone
->spanned_pages
, BM_BITS_PER_BLOCK
);
1116 rtree
+= DIV_ROUND_UP(rtree
* sizeof(struct rtree_node
),
1117 LINKED_PAGE_DATA_SIZE
);
1119 nodes
= DIV_ROUND_UP(nodes
, BM_ENTRIES_PER_LEVEL
);
1123 return 2 * (res
+ rtree
);
1126 #ifdef CONFIG_HIGHMEM
1128 * count_free_highmem_pages - compute the total number of free highmem
1129 * pages, system-wide.
1132 static unsigned int count_free_highmem_pages(void)
1135 unsigned int cnt
= 0;
1137 for_each_populated_zone(zone
)
1138 if (is_highmem(zone
))
1139 cnt
+= zone_page_state(zone
, NR_FREE_PAGES
);
1145 * saveable_highmem_page - Determine whether a highmem page should be
1146 * included in the suspend image.
1148 * We should save the page if it isn't Nosave or NosaveFree, or Reserved,
1149 * and it isn't a part of a free chunk of pages.
1151 static struct page
*saveable_highmem_page(struct zone
*zone
, unsigned long pfn
)
1155 if (!pfn_valid(pfn
))
1158 page
= pfn_to_page(pfn
);
1159 if (page_zone(page
) != zone
)
1162 BUG_ON(!PageHighMem(page
));
1164 if (swsusp_page_is_forbidden(page
) || swsusp_page_is_free(page
) ||
1168 if (page_is_guard(page
))
1175 * count_highmem_pages - compute the total number of saveable highmem
1179 static unsigned int count_highmem_pages(void)
1184 for_each_populated_zone(zone
) {
1185 unsigned long pfn
, max_zone_pfn
;
1187 if (!is_highmem(zone
))
1190 mark_free_pages(zone
);
1191 max_zone_pfn
= zone_end_pfn(zone
);
1192 for (pfn
= zone
->zone_start_pfn
; pfn
< max_zone_pfn
; pfn
++)
1193 if (saveable_highmem_page(zone
, pfn
))
1199 static inline void *saveable_highmem_page(struct zone
*z
, unsigned long p
)
1203 #endif /* CONFIG_HIGHMEM */
1206 * saveable_page - Determine whether a non-highmem page should be included
1207 * in the suspend image.
1209 * We should save the page if it isn't Nosave, and is not in the range
1210 * of pages statically defined as 'unsaveable', and it isn't a part of
1211 * a free chunk of pages.
1213 static struct page
*saveable_page(struct zone
*zone
, unsigned long pfn
)
1217 if (!pfn_valid(pfn
))
1220 page
= pfn_to_page(pfn
);
1221 if (page_zone(page
) != zone
)
1224 BUG_ON(PageHighMem(page
));
1226 if (swsusp_page_is_forbidden(page
) || swsusp_page_is_free(page
))
1229 if (PageReserved(page
)
1230 && (!kernel_page_present(page
) || pfn_is_nosave(pfn
)))
1233 if (page_is_guard(page
))
1240 * count_data_pages - compute the total number of saveable non-highmem
1244 static unsigned int count_data_pages(void)
1247 unsigned long pfn
, max_zone_pfn
;
1250 for_each_populated_zone(zone
) {
1251 if (is_highmem(zone
))
1254 mark_free_pages(zone
);
1255 max_zone_pfn
= zone_end_pfn(zone
);
1256 for (pfn
= zone
->zone_start_pfn
; pfn
< max_zone_pfn
; pfn
++)
1257 if (saveable_page(zone
, pfn
))
1263 /* This is needed, because copy_page and memcpy are not usable for copying
1266 static inline void do_copy_page(long *dst
, long *src
)
1270 for (n
= PAGE_SIZE
/ sizeof(long); n
; n
--)
1276 * safe_copy_page - check if the page we are going to copy is marked as
1277 * present in the kernel page tables (this always is the case if
1278 * CONFIG_DEBUG_PAGEALLOC is not set and in that case
1279 * kernel_page_present() always returns 'true').
1281 static void safe_copy_page(void *dst
, struct page
*s_page
)
1283 if (kernel_page_present(s_page
)) {
1284 do_copy_page(dst
, page_address(s_page
));
1286 kernel_map_pages(s_page
, 1, 1);
1287 do_copy_page(dst
, page_address(s_page
));
1288 kernel_map_pages(s_page
, 1, 0);
1293 #ifdef CONFIG_HIGHMEM
1294 static inline struct page
*
1295 page_is_saveable(struct zone
*zone
, unsigned long pfn
)
1297 return is_highmem(zone
) ?
1298 saveable_highmem_page(zone
, pfn
) : saveable_page(zone
, pfn
);
1301 static void copy_data_page(unsigned long dst_pfn
, unsigned long src_pfn
)
1303 struct page
*s_page
, *d_page
;
1306 s_page
= pfn_to_page(src_pfn
);
1307 d_page
= pfn_to_page(dst_pfn
);
1308 if (PageHighMem(s_page
)) {
1309 src
= kmap_atomic(s_page
);
1310 dst
= kmap_atomic(d_page
);
1311 do_copy_page(dst
, src
);
1315 if (PageHighMem(d_page
)) {
1316 /* Page pointed to by src may contain some kernel
1317 * data modified by kmap_atomic()
1319 safe_copy_page(buffer
, s_page
);
1320 dst
= kmap_atomic(d_page
);
1321 copy_page(dst
, buffer
);
1324 safe_copy_page(page_address(d_page
), s_page
);
1329 #define page_is_saveable(zone, pfn) saveable_page(zone, pfn)
1331 static inline void copy_data_page(unsigned long dst_pfn
, unsigned long src_pfn
)
1333 safe_copy_page(page_address(pfn_to_page(dst_pfn
)),
1334 pfn_to_page(src_pfn
));
1336 #endif /* CONFIG_HIGHMEM */
1339 copy_data_pages(struct memory_bitmap
*copy_bm
, struct memory_bitmap
*orig_bm
)
1344 for_each_populated_zone(zone
) {
1345 unsigned long max_zone_pfn
;
1347 mark_free_pages(zone
);
1348 max_zone_pfn
= zone_end_pfn(zone
);
1349 for (pfn
= zone
->zone_start_pfn
; pfn
< max_zone_pfn
; pfn
++)
1350 if (page_is_saveable(zone
, pfn
))
1351 memory_bm_set_bit(orig_bm
, pfn
);
1353 memory_bm_position_reset(orig_bm
);
1354 memory_bm_position_reset(copy_bm
);
1356 pfn
= memory_bm_next_pfn(orig_bm
);
1357 if (unlikely(pfn
== BM_END_OF_MAP
))
1359 copy_data_page(memory_bm_next_pfn(copy_bm
), pfn
);
1363 /* Total number of image pages */
1364 static unsigned int nr_copy_pages
;
1365 /* Number of pages needed for saving the original pfns of the image pages */
1366 static unsigned int nr_meta_pages
;
1368 * Numbers of normal and highmem page frames allocated for hibernation image
1369 * before suspending devices.
1371 unsigned int alloc_normal
, alloc_highmem
;
1373 * Memory bitmap used for marking saveable pages (during hibernation) or
1374 * hibernation image pages (during restore)
1376 static struct memory_bitmap orig_bm
;
1378 * Memory bitmap used during hibernation for marking allocated page frames that
1379 * will contain copies of saveable pages. During restore it is initially used
1380 * for marking hibernation image pages, but then the set bits from it are
1381 * duplicated in @orig_bm and it is released. On highmem systems it is next
1382 * used for marking "safe" highmem pages, but it has to be reinitialized for
1385 static struct memory_bitmap copy_bm
;
1388 * swsusp_free - free pages allocated for the suspend.
1390 * Suspend pages are alocated before the atomic copy is made, so we
1391 * need to release them after the resume.
1394 void swsusp_free(void)
1397 unsigned long pfn
, max_zone_pfn
;
1399 for_each_populated_zone(zone
) {
1400 max_zone_pfn
= zone_end_pfn(zone
);
1401 for (pfn
= zone
->zone_start_pfn
; pfn
< max_zone_pfn
; pfn
++)
1402 if (pfn_valid(pfn
)) {
1403 struct page
*page
= pfn_to_page(pfn
);
1405 if (swsusp_page_is_forbidden(page
) &&
1406 swsusp_page_is_free(page
)) {
1407 swsusp_unset_page_forbidden(page
);
1408 swsusp_unset_page_free(page
);
1415 restore_pblist
= NULL
;
1421 /* Helper functions used for the shrinking of memory. */
1423 #define GFP_IMAGE (GFP_KERNEL | __GFP_NOWARN)
1426 * preallocate_image_pages - Allocate a number of pages for hibernation image
1427 * @nr_pages: Number of page frames to allocate.
1428 * @mask: GFP flags to use for the allocation.
1430 * Return value: Number of page frames actually allocated
1432 static unsigned long preallocate_image_pages(unsigned long nr_pages
, gfp_t mask
)
1434 unsigned long nr_alloc
= 0;
1436 while (nr_pages
> 0) {
1439 page
= alloc_image_page(mask
);
1442 memory_bm_set_bit(©_bm
, page_to_pfn(page
));
1443 if (PageHighMem(page
))
1454 static unsigned long preallocate_image_memory(unsigned long nr_pages
,
1455 unsigned long avail_normal
)
1457 unsigned long alloc
;
1459 if (avail_normal
<= alloc_normal
)
1462 alloc
= avail_normal
- alloc_normal
;
1463 if (nr_pages
< alloc
)
1466 return preallocate_image_pages(alloc
, GFP_IMAGE
);
1469 #ifdef CONFIG_HIGHMEM
1470 static unsigned long preallocate_image_highmem(unsigned long nr_pages
)
1472 return preallocate_image_pages(nr_pages
, GFP_IMAGE
| __GFP_HIGHMEM
);
1476 * __fraction - Compute (an approximation of) x * (multiplier / base)
1478 static unsigned long __fraction(u64 x
, u64 multiplier
, u64 base
)
1482 return (unsigned long)x
;
1485 static unsigned long preallocate_highmem_fraction(unsigned long nr_pages
,
1486 unsigned long highmem
,
1487 unsigned long total
)
1489 unsigned long alloc
= __fraction(nr_pages
, highmem
, total
);
1491 return preallocate_image_pages(alloc
, GFP_IMAGE
| __GFP_HIGHMEM
);
1493 #else /* CONFIG_HIGHMEM */
1494 static inline unsigned long preallocate_image_highmem(unsigned long nr_pages
)
1499 static inline unsigned long preallocate_highmem_fraction(unsigned long nr_pages
,
1500 unsigned long highmem
,
1501 unsigned long total
)
1505 #endif /* CONFIG_HIGHMEM */
1508 * free_unnecessary_pages - Release preallocated pages not needed for the image
1510 static void free_unnecessary_pages(void)
1512 unsigned long save
, to_free_normal
, to_free_highmem
;
1514 save
= count_data_pages();
1515 if (alloc_normal
>= save
) {
1516 to_free_normal
= alloc_normal
- save
;
1520 save
-= alloc_normal
;
1522 save
+= count_highmem_pages();
1523 if (alloc_highmem
>= save
) {
1524 to_free_highmem
= alloc_highmem
- save
;
1526 to_free_highmem
= 0;
1527 save
-= alloc_highmem
;
1528 if (to_free_normal
> save
)
1529 to_free_normal
-= save
;
1534 memory_bm_position_reset(©_bm
);
1536 while (to_free_normal
> 0 || to_free_highmem
> 0) {
1537 unsigned long pfn
= memory_bm_next_pfn(©_bm
);
1538 struct page
*page
= pfn_to_page(pfn
);
1540 if (PageHighMem(page
)) {
1541 if (!to_free_highmem
)
1546 if (!to_free_normal
)
1551 memory_bm_clear_bit(©_bm
, pfn
);
1552 swsusp_unset_page_forbidden(page
);
1553 swsusp_unset_page_free(page
);
1559 * minimum_image_size - Estimate the minimum acceptable size of an image
1560 * @saveable: Number of saveable pages in the system.
1562 * We want to avoid attempting to free too much memory too hard, so estimate the
1563 * minimum acceptable size of a hibernation image to use as the lower limit for
1564 * preallocating memory.
1566 * We assume that the minimum image size should be proportional to
1568 * [number of saveable pages] - [number of pages that can be freed in theory]
1570 * where the second term is the sum of (1) reclaimable slab pages, (2) active
1571 * and (3) inactive anonymous pages, (4) active and (5) inactive file pages,
1572 * minus mapped file pages.
1574 static unsigned long minimum_image_size(unsigned long saveable
)
1578 size
= global_page_state(NR_SLAB_RECLAIMABLE
)
1579 + global_page_state(NR_ACTIVE_ANON
)
1580 + global_page_state(NR_INACTIVE_ANON
)
1581 + global_page_state(NR_ACTIVE_FILE
)
1582 + global_page_state(NR_INACTIVE_FILE
)
1583 - global_page_state(NR_FILE_MAPPED
);
1585 return saveable
<= size
? 0 : saveable
- size
;
1589 * hibernate_preallocate_memory - Preallocate memory for hibernation image
1591 * To create a hibernation image it is necessary to make a copy of every page
1592 * frame in use. We also need a number of page frames to be free during
1593 * hibernation for allocations made while saving the image and for device
1594 * drivers, in case they need to allocate memory from their hibernation
1595 * callbacks (these two numbers are given by PAGES_FOR_IO (which is a rough
1596 * estimate) and reserverd_size divided by PAGE_SIZE (which is tunable through
1597 * /sys/power/reserved_size, respectively). To make this happen, we compute the
1598 * total number of available page frames and allocate at least
1600 * ([page frames total] + PAGES_FOR_IO + [metadata pages]) / 2
1601 * + 2 * DIV_ROUND_UP(reserved_size, PAGE_SIZE)
1603 * of them, which corresponds to the maximum size of a hibernation image.
1605 * If image_size is set below the number following from the above formula,
1606 * the preallocation of memory is continued until the total number of saveable
1607 * pages in the system is below the requested image size or the minimum
1608 * acceptable image size returned by minimum_image_size(), whichever is greater.
1610 int hibernate_preallocate_memory(void)
1613 unsigned long saveable
, size
, max_size
, count
, highmem
, pages
= 0;
1614 unsigned long alloc
, save_highmem
, pages_highmem
, avail_normal
;
1615 struct timeval start
, stop
;
1618 printk(KERN_INFO
"PM: Preallocating image memory... ");
1619 do_gettimeofday(&start
);
1621 error
= memory_bm_create(&orig_bm
, GFP_IMAGE
, PG_ANY
);
1625 error
= memory_bm_create(©_bm
, GFP_IMAGE
, PG_ANY
);
1632 /* Count the number of saveable data pages. */
1633 save_highmem
= count_highmem_pages();
1634 saveable
= count_data_pages();
1637 * Compute the total number of page frames we can use (count) and the
1638 * number of pages needed for image metadata (size).
1641 saveable
+= save_highmem
;
1642 highmem
= save_highmem
;
1644 for_each_populated_zone(zone
) {
1645 size
+= snapshot_additional_pages(zone
);
1646 if (is_highmem(zone
))
1647 highmem
+= zone_page_state(zone
, NR_FREE_PAGES
);
1649 count
+= zone_page_state(zone
, NR_FREE_PAGES
);
1651 avail_normal
= count
;
1653 count
-= totalreserve_pages
;
1655 /* Add number of pages required for page keys (s390 only). */
1656 size
+= page_key_additional_pages(saveable
);
1658 /* Compute the maximum number of saveable pages to leave in memory. */
1659 max_size
= (count
- (size
+ PAGES_FOR_IO
)) / 2
1660 - 2 * DIV_ROUND_UP(reserved_size
, PAGE_SIZE
);
1661 /* Compute the desired number of image pages specified by image_size. */
1662 size
= DIV_ROUND_UP(image_size
, PAGE_SIZE
);
1663 if (size
> max_size
)
1666 * If the desired number of image pages is at least as large as the
1667 * current number of saveable pages in memory, allocate page frames for
1668 * the image and we're done.
1670 if (size
>= saveable
) {
1671 pages
= preallocate_image_highmem(save_highmem
);
1672 pages
+= preallocate_image_memory(saveable
- pages
, avail_normal
);
1676 /* Estimate the minimum size of the image. */
1677 pages
= minimum_image_size(saveable
);
1679 * To avoid excessive pressure on the normal zone, leave room in it to
1680 * accommodate an image of the minimum size (unless it's already too
1681 * small, in which case don't preallocate pages from it at all).
1683 if (avail_normal
> pages
)
1684 avail_normal
-= pages
;
1688 size
= min_t(unsigned long, pages
, max_size
);
1691 * Let the memory management subsystem know that we're going to need a
1692 * large number of page frames to allocate and make it free some memory.
1693 * NOTE: If this is not done, performance will be hurt badly in some
1696 shrink_all_memory(saveable
- size
);
1699 * The number of saveable pages in memory was too high, so apply some
1700 * pressure to decrease it. First, make room for the largest possible
1701 * image and fail if that doesn't work. Next, try to decrease the size
1702 * of the image as much as indicated by 'size' using allocations from
1703 * highmem and non-highmem zones separately.
1705 pages_highmem
= preallocate_image_highmem(highmem
/ 2);
1706 alloc
= count
- max_size
;
1707 if (alloc
> pages_highmem
)
1708 alloc
-= pages_highmem
;
1711 pages
= preallocate_image_memory(alloc
, avail_normal
);
1712 if (pages
< alloc
) {
1713 /* We have exhausted non-highmem pages, try highmem. */
1715 pages
+= pages_highmem
;
1716 pages_highmem
= preallocate_image_highmem(alloc
);
1717 if (pages_highmem
< alloc
)
1719 pages
+= pages_highmem
;
1721 * size is the desired number of saveable pages to leave in
1722 * memory, so try to preallocate (all memory - size) pages.
1724 alloc
= (count
- pages
) - size
;
1725 pages
+= preallocate_image_highmem(alloc
);
1728 * There are approximately max_size saveable pages at this point
1729 * and we want to reduce this number down to size.
1731 alloc
= max_size
- size
;
1732 size
= preallocate_highmem_fraction(alloc
, highmem
, count
);
1733 pages_highmem
+= size
;
1735 size
= preallocate_image_memory(alloc
, avail_normal
);
1736 pages_highmem
+= preallocate_image_highmem(alloc
- size
);
1737 pages
+= pages_highmem
+ size
;
1741 * We only need as many page frames for the image as there are saveable
1742 * pages in memory, but we have allocated more. Release the excessive
1745 free_unnecessary_pages();
1748 do_gettimeofday(&stop
);
1749 printk(KERN_CONT
"done (allocated %lu pages)\n", pages
);
1750 swsusp_show_speed(&start
, &stop
, pages
, "Allocated");
1755 printk(KERN_CONT
"\n");
1760 #ifdef CONFIG_HIGHMEM
1762 * count_pages_for_highmem - compute the number of non-highmem pages
1763 * that will be necessary for creating copies of highmem pages.
1766 static unsigned int count_pages_for_highmem(unsigned int nr_highmem
)
1768 unsigned int free_highmem
= count_free_highmem_pages() + alloc_highmem
;
1770 if (free_highmem
>= nr_highmem
)
1773 nr_highmem
-= free_highmem
;
1779 count_pages_for_highmem(unsigned int nr_highmem
) { return 0; }
1780 #endif /* CONFIG_HIGHMEM */
1783 * enough_free_mem - Make sure we have enough free memory for the
1787 static int enough_free_mem(unsigned int nr_pages
, unsigned int nr_highmem
)
1790 unsigned int free
= alloc_normal
;
1792 for_each_populated_zone(zone
)
1793 if (!is_highmem(zone
))
1794 free
+= zone_page_state(zone
, NR_FREE_PAGES
);
1796 nr_pages
+= count_pages_for_highmem(nr_highmem
);
1797 pr_debug("PM: Normal pages needed: %u + %u, available pages: %u\n",
1798 nr_pages
, PAGES_FOR_IO
, free
);
1800 return free
> nr_pages
+ PAGES_FOR_IO
;
1803 #ifdef CONFIG_HIGHMEM
1805 * get_highmem_buffer - if there are some highmem pages in the suspend
1806 * image, we may need the buffer to copy them and/or load their data.
1809 static inline int get_highmem_buffer(int safe_needed
)
1811 buffer
= get_image_page(GFP_ATOMIC
| __GFP_COLD
, safe_needed
);
1812 return buffer
? 0 : -ENOMEM
;
1816 * alloc_highmem_image_pages - allocate some highmem pages for the image.
1817 * Try to allocate as many pages as needed, but if the number of free
1818 * highmem pages is lesser than that, allocate them all.
1821 static inline unsigned int
1822 alloc_highmem_pages(struct memory_bitmap
*bm
, unsigned int nr_highmem
)
1824 unsigned int to_alloc
= count_free_highmem_pages();
1826 if (to_alloc
> nr_highmem
)
1827 to_alloc
= nr_highmem
;
1829 nr_highmem
-= to_alloc
;
1830 while (to_alloc
-- > 0) {
1833 page
= alloc_image_page(__GFP_HIGHMEM
);
1834 memory_bm_set_bit(bm
, page_to_pfn(page
));
1839 static inline int get_highmem_buffer(int safe_needed
) { return 0; }
1841 static inline unsigned int
1842 alloc_highmem_pages(struct memory_bitmap
*bm
, unsigned int n
) { return 0; }
1843 #endif /* CONFIG_HIGHMEM */
1846 * swsusp_alloc - allocate memory for the suspend image
1848 * We first try to allocate as many highmem pages as there are
1849 * saveable highmem pages in the system. If that fails, we allocate
1850 * non-highmem pages for the copies of the remaining highmem ones.
1852 * In this approach it is likely that the copies of highmem pages will
1853 * also be located in the high memory, because of the way in which
1854 * copy_data_pages() works.
1858 swsusp_alloc(struct memory_bitmap
*orig_bm
, struct memory_bitmap
*copy_bm
,
1859 unsigned int nr_pages
, unsigned int nr_highmem
)
1861 if (nr_highmem
> 0) {
1862 if (get_highmem_buffer(PG_ANY
))
1864 if (nr_highmem
> alloc_highmem
) {
1865 nr_highmem
-= alloc_highmem
;
1866 nr_pages
+= alloc_highmem_pages(copy_bm
, nr_highmem
);
1869 if (nr_pages
> alloc_normal
) {
1870 nr_pages
-= alloc_normal
;
1871 while (nr_pages
-- > 0) {
1874 page
= alloc_image_page(GFP_ATOMIC
| __GFP_COLD
);
1877 memory_bm_set_bit(copy_bm
, page_to_pfn(page
));
1888 asmlinkage __visible
int swsusp_save(void)
1890 unsigned int nr_pages
, nr_highmem
;
1892 printk(KERN_INFO
"PM: Creating hibernation image:\n");
1894 drain_local_pages(NULL
);
1895 nr_pages
= count_data_pages();
1896 nr_highmem
= count_highmem_pages();
1897 printk(KERN_INFO
"PM: Need to copy %u pages\n", nr_pages
+ nr_highmem
);
1899 if (!enough_free_mem(nr_pages
, nr_highmem
)) {
1900 printk(KERN_ERR
"PM: Not enough free memory\n");
1904 if (swsusp_alloc(&orig_bm
, ©_bm
, nr_pages
, nr_highmem
)) {
1905 printk(KERN_ERR
"PM: Memory allocation failed\n");
1909 /* During allocating of suspend pagedir, new cold pages may appear.
1912 drain_local_pages(NULL
);
1913 copy_data_pages(©_bm
, &orig_bm
);
1916 * End of critical section. From now on, we can write to memory,
1917 * but we should not touch disk. This specially means we must _not_
1918 * touch swap space! Except we must write out our image of course.
1921 nr_pages
+= nr_highmem
;
1922 nr_copy_pages
= nr_pages
;
1923 nr_meta_pages
= DIV_ROUND_UP(nr_pages
* sizeof(long), PAGE_SIZE
);
1925 printk(KERN_INFO
"PM: Hibernation image created (%d pages copied)\n",
1931 #ifndef CONFIG_ARCH_HIBERNATION_HEADER
1932 static int init_header_complete(struct swsusp_info
*info
)
1934 memcpy(&info
->uts
, init_utsname(), sizeof(struct new_utsname
));
1935 info
->version_code
= LINUX_VERSION_CODE
;
1939 static char *check_image_kernel(struct swsusp_info
*info
)
1941 if (info
->version_code
!= LINUX_VERSION_CODE
)
1942 return "kernel version";
1943 if (strcmp(info
->uts
.sysname
,init_utsname()->sysname
))
1944 return "system type";
1945 if (strcmp(info
->uts
.release
,init_utsname()->release
))
1946 return "kernel release";
1947 if (strcmp(info
->uts
.version
,init_utsname()->version
))
1949 if (strcmp(info
->uts
.machine
,init_utsname()->machine
))
1953 #endif /* CONFIG_ARCH_HIBERNATION_HEADER */
1955 unsigned long snapshot_get_image_size(void)
1957 return nr_copy_pages
+ nr_meta_pages
+ 1;
1960 static int init_header(struct swsusp_info
*info
)
1962 memset(info
, 0, sizeof(struct swsusp_info
));
1963 info
->num_physpages
= get_num_physpages();
1964 info
->image_pages
= nr_copy_pages
;
1965 info
->pages
= snapshot_get_image_size();
1966 info
->size
= info
->pages
;
1967 info
->size
<<= PAGE_SHIFT
;
1968 return init_header_complete(info
);
1972 * pack_pfns - pfns corresponding to the set bits found in the bitmap @bm
1973 * are stored in the array @buf[] (1 page at a time)
1977 pack_pfns(unsigned long *buf
, struct memory_bitmap
*bm
)
1981 for (j
= 0; j
< PAGE_SIZE
/ sizeof(long); j
++) {
1982 buf
[j
] = memory_bm_next_pfn(bm
);
1983 if (unlikely(buf
[j
] == BM_END_OF_MAP
))
1985 /* Save page key for data page (s390 only). */
1986 page_key_read(buf
+ j
);
1991 * snapshot_read_next - used for reading the system memory snapshot.
1993 * On the first call to it @handle should point to a zeroed
1994 * snapshot_handle structure. The structure gets updated and a pointer
1995 * to it should be passed to this function every next time.
1997 * On success the function returns a positive number. Then, the caller
1998 * is allowed to read up to the returned number of bytes from the memory
1999 * location computed by the data_of() macro.
2001 * The function returns 0 to indicate the end of data stream condition,
2002 * and a negative number is returned on error. In such cases the
2003 * structure pointed to by @handle is not updated and should not be used
2007 int snapshot_read_next(struct snapshot_handle
*handle
)
2009 if (handle
->cur
> nr_meta_pages
+ nr_copy_pages
)
2013 /* This makes the buffer be freed by swsusp_free() */
2014 buffer
= get_image_page(GFP_ATOMIC
, PG_ANY
);
2021 error
= init_header((struct swsusp_info
*)buffer
);
2024 handle
->buffer
= buffer
;
2025 memory_bm_position_reset(&orig_bm
);
2026 memory_bm_position_reset(©_bm
);
2027 } else if (handle
->cur
<= nr_meta_pages
) {
2029 pack_pfns(buffer
, &orig_bm
);
2033 page
= pfn_to_page(memory_bm_next_pfn(©_bm
));
2034 if (PageHighMem(page
)) {
2035 /* Highmem pages are copied to the buffer,
2036 * because we can't return with a kmapped
2037 * highmem page (we may not be called again).
2041 kaddr
= kmap_atomic(page
);
2042 copy_page(buffer
, kaddr
);
2043 kunmap_atomic(kaddr
);
2044 handle
->buffer
= buffer
;
2046 handle
->buffer
= page_address(page
);
2054 * mark_unsafe_pages - mark the pages that cannot be used for storing
2055 * the image during resume, because they conflict with the pages that
2056 * had been used before suspend
2059 static int mark_unsafe_pages(struct memory_bitmap
*bm
)
2062 unsigned long pfn
, max_zone_pfn
;
2064 /* Clear page flags */
2065 for_each_populated_zone(zone
) {
2066 max_zone_pfn
= zone_end_pfn(zone
);
2067 for (pfn
= zone
->zone_start_pfn
; pfn
< max_zone_pfn
; pfn
++)
2069 swsusp_unset_page_free(pfn_to_page(pfn
));
2072 /* Mark pages that correspond to the "original" pfns as "unsafe" */
2073 memory_bm_position_reset(bm
);
2075 pfn
= memory_bm_next_pfn(bm
);
2076 if (likely(pfn
!= BM_END_OF_MAP
)) {
2077 if (likely(pfn_valid(pfn
)))
2078 swsusp_set_page_free(pfn_to_page(pfn
));
2082 } while (pfn
!= BM_END_OF_MAP
);
2084 allocated_unsafe_pages
= 0;
2090 duplicate_memory_bitmap(struct memory_bitmap
*dst
, struct memory_bitmap
*src
)
2094 memory_bm_position_reset(src
);
2095 pfn
= memory_bm_next_pfn(src
);
2096 while (pfn
!= BM_END_OF_MAP
) {
2097 memory_bm_set_bit(dst
, pfn
);
2098 pfn
= memory_bm_next_pfn(src
);
2102 static int check_header(struct swsusp_info
*info
)
2106 reason
= check_image_kernel(info
);
2107 if (!reason
&& info
->num_physpages
!= get_num_physpages())
2108 reason
= "memory size";
2110 printk(KERN_ERR
"PM: Image mismatch: %s\n", reason
);
2117 * load header - check the image header and copy data from it
2121 load_header(struct swsusp_info
*info
)
2125 restore_pblist
= NULL
;
2126 error
= check_header(info
);
2128 nr_copy_pages
= info
->image_pages
;
2129 nr_meta_pages
= info
->pages
- info
->image_pages
- 1;
2135 * unpack_orig_pfns - for each element of @buf[] (1 page at a time) set
2136 * the corresponding bit in the memory bitmap @bm
2138 static int unpack_orig_pfns(unsigned long *buf
, struct memory_bitmap
*bm
)
2142 for (j
= 0; j
< PAGE_SIZE
/ sizeof(long); j
++) {
2143 if (unlikely(buf
[j
] == BM_END_OF_MAP
))
2146 /* Extract and buffer page key for data page (s390 only). */
2147 page_key_memorize(buf
+ j
);
2149 if (memory_bm_pfn_present(bm
, buf
[j
]))
2150 memory_bm_set_bit(bm
, buf
[j
]);
2158 /* List of "safe" pages that may be used to store data loaded from the suspend
2161 static struct linked_page
*safe_pages_list
;
2163 #ifdef CONFIG_HIGHMEM
2164 /* struct highmem_pbe is used for creating the list of highmem pages that
2165 * should be restored atomically during the resume from disk, because the page
2166 * frames they have occupied before the suspend are in use.
2168 struct highmem_pbe
{
2169 struct page
*copy_page
; /* data is here now */
2170 struct page
*orig_page
; /* data was here before the suspend */
2171 struct highmem_pbe
*next
;
2174 /* List of highmem PBEs needed for restoring the highmem pages that were
2175 * allocated before the suspend and included in the suspend image, but have
2176 * also been allocated by the "resume" kernel, so their contents cannot be
2177 * written directly to their "original" page frames.
2179 static struct highmem_pbe
*highmem_pblist
;
2182 * count_highmem_image_pages - compute the number of highmem pages in the
2183 * suspend image. The bits in the memory bitmap @bm that correspond to the
2184 * image pages are assumed to be set.
2187 static unsigned int count_highmem_image_pages(struct memory_bitmap
*bm
)
2190 unsigned int cnt
= 0;
2192 memory_bm_position_reset(bm
);
2193 pfn
= memory_bm_next_pfn(bm
);
2194 while (pfn
!= BM_END_OF_MAP
) {
2195 if (PageHighMem(pfn_to_page(pfn
)))
2198 pfn
= memory_bm_next_pfn(bm
);
2204 * prepare_highmem_image - try to allocate as many highmem pages as
2205 * there are highmem image pages (@nr_highmem_p points to the variable
2206 * containing the number of highmem image pages). The pages that are
2207 * "safe" (ie. will not be overwritten when the suspend image is
2208 * restored) have the corresponding bits set in @bm (it must be
2211 * NOTE: This function should not be called if there are no highmem
2215 static unsigned int safe_highmem_pages
;
2217 static struct memory_bitmap
*safe_highmem_bm
;
2220 prepare_highmem_image(struct memory_bitmap
*bm
, unsigned int *nr_highmem_p
)
2222 unsigned int to_alloc
;
2224 if (memory_bm_create(bm
, GFP_ATOMIC
, PG_SAFE
))
2227 if (get_highmem_buffer(PG_SAFE
))
2230 to_alloc
= count_free_highmem_pages();
2231 if (to_alloc
> *nr_highmem_p
)
2232 to_alloc
= *nr_highmem_p
;
2234 *nr_highmem_p
= to_alloc
;
2236 safe_highmem_pages
= 0;
2237 while (to_alloc
-- > 0) {
2240 page
= alloc_page(__GFP_HIGHMEM
);
2241 if (!swsusp_page_is_free(page
)) {
2242 /* The page is "safe", set its bit the bitmap */
2243 memory_bm_set_bit(bm
, page_to_pfn(page
));
2244 safe_highmem_pages
++;
2246 /* Mark the page as allocated */
2247 swsusp_set_page_forbidden(page
);
2248 swsusp_set_page_free(page
);
2250 memory_bm_position_reset(bm
);
2251 safe_highmem_bm
= bm
;
2256 * get_highmem_page_buffer - for given highmem image page find the buffer
2257 * that suspend_write_next() should set for its caller to write to.
2259 * If the page is to be saved to its "original" page frame or a copy of
2260 * the page is to be made in the highmem, @buffer is returned. Otherwise,
2261 * the copy of the page is to be made in normal memory, so the address of
2262 * the copy is returned.
2264 * If @buffer is returned, the caller of suspend_write_next() will write
2265 * the page's contents to @buffer, so they will have to be copied to the
2266 * right location on the next call to suspend_write_next() and it is done
2267 * with the help of copy_last_highmem_page(). For this purpose, if
2268 * @buffer is returned, @last_highmem page is set to the page to which
2269 * the data will have to be copied from @buffer.
2272 static struct page
*last_highmem_page
;
2275 get_highmem_page_buffer(struct page
*page
, struct chain_allocator
*ca
)
2277 struct highmem_pbe
*pbe
;
2280 if (swsusp_page_is_forbidden(page
) && swsusp_page_is_free(page
)) {
2281 /* We have allocated the "original" page frame and we can
2282 * use it directly to store the loaded page.
2284 last_highmem_page
= page
;
2287 /* The "original" page frame has not been allocated and we have to
2288 * use a "safe" page frame to store the loaded page.
2290 pbe
= chain_alloc(ca
, sizeof(struct highmem_pbe
));
2293 return ERR_PTR(-ENOMEM
);
2295 pbe
->orig_page
= page
;
2296 if (safe_highmem_pages
> 0) {
2299 /* Copy of the page will be stored in high memory */
2301 tmp
= pfn_to_page(memory_bm_next_pfn(safe_highmem_bm
));
2302 safe_highmem_pages
--;
2303 last_highmem_page
= tmp
;
2304 pbe
->copy_page
= tmp
;
2306 /* Copy of the page will be stored in normal memory */
2307 kaddr
= safe_pages_list
;
2308 safe_pages_list
= safe_pages_list
->next
;
2309 pbe
->copy_page
= virt_to_page(kaddr
);
2311 pbe
->next
= highmem_pblist
;
2312 highmem_pblist
= pbe
;
2317 * copy_last_highmem_page - copy the contents of a highmem image from
2318 * @buffer, where the caller of snapshot_write_next() has place them,
2319 * to the right location represented by @last_highmem_page .
2322 static void copy_last_highmem_page(void)
2324 if (last_highmem_page
) {
2327 dst
= kmap_atomic(last_highmem_page
);
2328 copy_page(dst
, buffer
);
2330 last_highmem_page
= NULL
;
2334 static inline int last_highmem_page_copied(void)
2336 return !last_highmem_page
;
2339 static inline void free_highmem_data(void)
2341 if (safe_highmem_bm
)
2342 memory_bm_free(safe_highmem_bm
, PG_UNSAFE_CLEAR
);
2345 free_image_page(buffer
, PG_UNSAFE_CLEAR
);
2348 static inline int get_safe_write_buffer(void) { return 0; }
2351 count_highmem_image_pages(struct memory_bitmap
*bm
) { return 0; }
2354 prepare_highmem_image(struct memory_bitmap
*bm
, unsigned int *nr_highmem_p
)
2359 static inline void *
2360 get_highmem_page_buffer(struct page
*page
, struct chain_allocator
*ca
)
2362 return ERR_PTR(-EINVAL
);
2365 static inline void copy_last_highmem_page(void) {}
2366 static inline int last_highmem_page_copied(void) { return 1; }
2367 static inline void free_highmem_data(void) {}
2368 #endif /* CONFIG_HIGHMEM */
2371 * prepare_image - use the memory bitmap @bm to mark the pages that will
2372 * be overwritten in the process of restoring the system memory state
2373 * from the suspend image ("unsafe" pages) and allocate memory for the
2376 * The idea is to allocate a new memory bitmap first and then allocate
2377 * as many pages as needed for the image data, but not to assign these
2378 * pages to specific tasks initially. Instead, we just mark them as
2379 * allocated and create a lists of "safe" pages that will be used
2380 * later. On systems with high memory a list of "safe" highmem pages is
2384 #define PBES_PER_LINKED_PAGE (LINKED_PAGE_DATA_SIZE / sizeof(struct pbe))
2387 prepare_image(struct memory_bitmap
*new_bm
, struct memory_bitmap
*bm
)
2389 unsigned int nr_pages
, nr_highmem
;
2390 struct linked_page
*sp_list
, *lp
;
2393 /* If there is no highmem, the buffer will not be necessary */
2394 free_image_page(buffer
, PG_UNSAFE_CLEAR
);
2397 nr_highmem
= count_highmem_image_pages(bm
);
2398 error
= mark_unsafe_pages(bm
);
2402 error
= memory_bm_create(new_bm
, GFP_ATOMIC
, PG_SAFE
);
2406 duplicate_memory_bitmap(new_bm
, bm
);
2407 memory_bm_free(bm
, PG_UNSAFE_KEEP
);
2408 if (nr_highmem
> 0) {
2409 error
= prepare_highmem_image(bm
, &nr_highmem
);
2413 /* Reserve some safe pages for potential later use.
2415 * NOTE: This way we make sure there will be enough safe pages for the
2416 * chain_alloc() in get_buffer(). It is a bit wasteful, but
2417 * nr_copy_pages cannot be greater than 50% of the memory anyway.
2420 /* nr_copy_pages cannot be lesser than allocated_unsafe_pages */
2421 nr_pages
= nr_copy_pages
- nr_highmem
- allocated_unsafe_pages
;
2422 nr_pages
= DIV_ROUND_UP(nr_pages
, PBES_PER_LINKED_PAGE
);
2423 while (nr_pages
> 0) {
2424 lp
= get_image_page(GFP_ATOMIC
, PG_SAFE
);
2433 /* Preallocate memory for the image */
2434 safe_pages_list
= NULL
;
2435 nr_pages
= nr_copy_pages
- nr_highmem
- allocated_unsafe_pages
;
2436 while (nr_pages
> 0) {
2437 lp
= (struct linked_page
*)get_zeroed_page(GFP_ATOMIC
);
2442 if (!swsusp_page_is_free(virt_to_page(lp
))) {
2443 /* The page is "safe", add it to the list */
2444 lp
->next
= safe_pages_list
;
2445 safe_pages_list
= lp
;
2447 /* Mark the page as allocated */
2448 swsusp_set_page_forbidden(virt_to_page(lp
));
2449 swsusp_set_page_free(virt_to_page(lp
));
2452 /* Free the reserved safe pages so that chain_alloc() can use them */
2455 free_image_page(sp_list
, PG_UNSAFE_CLEAR
);
2466 * get_buffer - compute the address that snapshot_write_next() should
2467 * set for its caller to write to.
2470 static void *get_buffer(struct memory_bitmap
*bm
, struct chain_allocator
*ca
)
2474 unsigned long pfn
= memory_bm_next_pfn(bm
);
2476 if (pfn
== BM_END_OF_MAP
)
2477 return ERR_PTR(-EFAULT
);
2479 page
= pfn_to_page(pfn
);
2480 if (PageHighMem(page
))
2481 return get_highmem_page_buffer(page
, ca
);
2483 if (swsusp_page_is_forbidden(page
) && swsusp_page_is_free(page
))
2484 /* We have allocated the "original" page frame and we can
2485 * use it directly to store the loaded page.
2487 return page_address(page
);
2489 /* The "original" page frame has not been allocated and we have to
2490 * use a "safe" page frame to store the loaded page.
2492 pbe
= chain_alloc(ca
, sizeof(struct pbe
));
2495 return ERR_PTR(-ENOMEM
);
2497 pbe
->orig_address
= page_address(page
);
2498 pbe
->address
= safe_pages_list
;
2499 safe_pages_list
= safe_pages_list
->next
;
2500 pbe
->next
= restore_pblist
;
2501 restore_pblist
= pbe
;
2502 return pbe
->address
;
2506 * snapshot_write_next - used for writing the system memory snapshot.
2508 * On the first call to it @handle should point to a zeroed
2509 * snapshot_handle structure. The structure gets updated and a pointer
2510 * to it should be passed to this function every next time.
2512 * On success the function returns a positive number. Then, the caller
2513 * is allowed to write up to the returned number of bytes to the memory
2514 * location computed by the data_of() macro.
2516 * The function returns 0 to indicate the "end of file" condition,
2517 * and a negative number is returned on error. In such cases the
2518 * structure pointed to by @handle is not updated and should not be used
2522 int snapshot_write_next(struct snapshot_handle
*handle
)
2524 static struct chain_allocator ca
;
2527 /* Check if we have already loaded the entire image */
2528 if (handle
->cur
> 1 && handle
->cur
> nr_meta_pages
+ nr_copy_pages
)
2531 handle
->sync_read
= 1;
2535 /* This makes the buffer be freed by swsusp_free() */
2536 buffer
= get_image_page(GFP_ATOMIC
, PG_ANY
);
2541 handle
->buffer
= buffer
;
2542 } else if (handle
->cur
== 1) {
2543 error
= load_header(buffer
);
2547 error
= memory_bm_create(©_bm
, GFP_ATOMIC
, PG_ANY
);
2551 /* Allocate buffer for page keys. */
2552 error
= page_key_alloc(nr_copy_pages
);
2556 } else if (handle
->cur
<= nr_meta_pages
+ 1) {
2557 error
= unpack_orig_pfns(buffer
, ©_bm
);
2561 if (handle
->cur
== nr_meta_pages
+ 1) {
2562 error
= prepare_image(&orig_bm
, ©_bm
);
2566 chain_init(&ca
, GFP_ATOMIC
, PG_SAFE
);
2567 memory_bm_position_reset(&orig_bm
);
2568 restore_pblist
= NULL
;
2569 handle
->buffer
= get_buffer(&orig_bm
, &ca
);
2570 handle
->sync_read
= 0;
2571 if (IS_ERR(handle
->buffer
))
2572 return PTR_ERR(handle
->buffer
);
2575 copy_last_highmem_page();
2576 /* Restore page key for data page (s390 only). */
2577 page_key_write(handle
->buffer
);
2578 handle
->buffer
= get_buffer(&orig_bm
, &ca
);
2579 if (IS_ERR(handle
->buffer
))
2580 return PTR_ERR(handle
->buffer
);
2581 if (handle
->buffer
!= buffer
)
2582 handle
->sync_read
= 0;
2589 * snapshot_write_finalize - must be called after the last call to
2590 * snapshot_write_next() in case the last page in the image happens
2591 * to be a highmem page and its contents should be stored in the
2592 * highmem. Additionally, it releases the memory that will not be
2596 void snapshot_write_finalize(struct snapshot_handle
*handle
)
2598 copy_last_highmem_page();
2599 /* Restore page key for data page (s390 only). */
2600 page_key_write(handle
->buffer
);
2602 /* Free only if we have loaded the image entirely */
2603 if (handle
->cur
> 1 && handle
->cur
> nr_meta_pages
+ nr_copy_pages
) {
2604 memory_bm_free(&orig_bm
, PG_UNSAFE_CLEAR
);
2605 free_highmem_data();
2609 int snapshot_image_loaded(struct snapshot_handle
*handle
)
2611 return !(!nr_copy_pages
|| !last_highmem_page_copied() ||
2612 handle
->cur
<= nr_meta_pages
+ nr_copy_pages
);
2615 #ifdef CONFIG_HIGHMEM
2616 /* Assumes that @buf is ready and points to a "safe" page */
2618 swap_two_pages_data(struct page
*p1
, struct page
*p2
, void *buf
)
2620 void *kaddr1
, *kaddr2
;
2622 kaddr1
= kmap_atomic(p1
);
2623 kaddr2
= kmap_atomic(p2
);
2624 copy_page(buf
, kaddr1
);
2625 copy_page(kaddr1
, kaddr2
);
2626 copy_page(kaddr2
, buf
);
2627 kunmap_atomic(kaddr2
);
2628 kunmap_atomic(kaddr1
);
2632 * restore_highmem - for each highmem page that was allocated before
2633 * the suspend and included in the suspend image, and also has been
2634 * allocated by the "resume" kernel swap its current (ie. "before
2635 * resume") contents with the previous (ie. "before suspend") one.
2637 * If the resume eventually fails, we can call this function once
2638 * again and restore the "before resume" highmem state.
2641 int restore_highmem(void)
2643 struct highmem_pbe
*pbe
= highmem_pblist
;
2649 buf
= get_image_page(GFP_ATOMIC
, PG_SAFE
);
2654 swap_two_pages_data(pbe
->copy_page
, pbe
->orig_page
, buf
);
2657 free_image_page(buf
, PG_UNSAFE_CLEAR
);
2660 #endif /* CONFIG_HIGHMEM */