[mirror_ubuntu-bionic-kernel.git] / arch / x86 / power / hibernate_64.c

/*
 * Hibernation support for x86-64
 *
 * Distribute under GPLv2
 *
 * Copyright (c) 2007 Rafael J. Wysocki <rjw@sisk.pl>
 * Copyright (c) 2002 Pavel Machek <pavel@ucw.cz>
 * Copyright (c) 2001 Patrick Mochel <mochel@osdl.org>
 */

#include <linux/gfp.h>
#include <linux/smp.h>
#include <linux/suspend.h>

#include <asm/init.h>
#include <asm/proto.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/mtrr.h>
#include <asm/sections.h>
#include <asm/suspend.h>
#include <asm/tlbflush.h>

/* Defined in hibernate_asm_64.S */
extern asmlinkage __visible int restore_image(void);

/*
 * Address to jump to in the last phase of restore in order to get to the image
 * kernel's text (this value is passed in the image header).
 */
unsigned long restore_jump_address __visible;
unsigned long jump_address_phys;

/*
 * Value of the cr3 register from before the hibernation (this value is passed
 * in the image header).
 */
unsigned long restore_cr3 __visible;

unsigned long temp_level4_pgt __visible;

unsigned long relocated_restore_code __visible;

static int set_up_temporary_text_mapping(pgd_t *pgd)
{
	pmd_t *pmd;
	pud_t *pud;

	/*
	 * The new mapping only has to cover the page containing the image
	 * kernel's entry point (jump_address_phys), because the switch over to
	 * it is carried out by relocated code running from a page allocated
	 * specifically for this purpose and covered by the identity mapping, so
	 * the temporary kernel text mapping is only needed for the final jump.
	 * Moreover, in that mapping the virtual address of the image kernel's
	 * entry point must be the same as its virtual address in the image
	 * kernel (restore_jump_address), so the image kernel's
	 * restore_registers() code doesn't find itself in a different area of
	 * the virtual address space after switching over to the original page
	 * tables used by the image kernel.
	 */
	pud = (pud_t *)get_safe_page(GFP_ATOMIC);
	if (!pud)
		return -ENOMEM;

	pmd = (pmd_t *)get_safe_page(GFP_ATOMIC);
	if (!pmd)
		return -ENOMEM;

	set_pmd(pmd + pmd_index(restore_jump_address),
		__pmd((jump_address_phys & PMD_MASK) | __PAGE_KERNEL_LARGE_EXEC));
	set_pud(pud + pud_index(restore_jump_address),
		__pud(__pa(pmd) | _KERNPG_TABLE));
	set_pgd(pgd + pgd_index(restore_jump_address),
		__pgd(__pa(pud) | _KERNPG_TABLE));

	return 0;
}

static void *alloc_pgt_page(void *context)
{
	return (void *)get_safe_page(GFP_ATOMIC);
}

static int set_up_temporary_mappings(void)
{
	struct x86_mapping_info info = {
		.alloc_pgt_page	= alloc_pgt_page,
		.pmd_flag	= __PAGE_KERNEL_LARGE_EXEC,
		.offset		= __PAGE_OFFSET,
	};
	unsigned long mstart, mend;
	pgd_t *pgd;
	int result;
	int i;

	pgd = (pgd_t *)get_safe_page(GFP_ATOMIC);
	if (!pgd)
		return -ENOMEM;

	/* Prepare a temporary mapping for the kernel text */
	result = set_up_temporary_text_mapping(pgd);
	if (result)
		return result;

	/* Set up the direct mapping from scratch */
	for (i = 0; i < nr_pfn_mapped; i++) {
		mstart = pfn_mapped[i].start << PAGE_SHIFT;
		mend   = pfn_mapped[i].end << PAGE_SHIFT;

		result = kernel_ident_mapping_init(&info, pgd, mstart, mend);
		if (result)
			return result;
	}

	temp_level4_pgt = __pa(pgd);
	return 0;
}

static int relocate_restore_code(void)
{
	pgd_t *pgd;
	pud_t *pud;

	relocated_restore_code = get_safe_page(GFP_ATOMIC);
	if (!relocated_restore_code)
		return -ENOMEM;

	memcpy((void *)relocated_restore_code, &core_restore_code, PAGE_SIZE);

	/* Make the page containing the relocated code executable */
	pgd = (pgd_t *)__va(read_cr3()) + pgd_index(relocated_restore_code);
	pud = pud_offset(pgd, relocated_restore_code);
	if (pud_large(*pud)) {
		set_pud(pud, __pud(pud_val(*pud) & ~_PAGE_NX));
	} else {
		pmd_t *pmd = pmd_offset(pud, relocated_restore_code);

		if (pmd_large(*pmd)) {
			set_pmd(pmd, __pmd(pmd_val(*pmd) & ~_PAGE_NX));
		} else {
			pte_t *pte = pte_offset_kernel(pmd, relocated_restore_code);

			set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_NX));
		}
	}
	__flush_tlb_all();

	return 0;
}

int swsusp_arch_resume(void)
{
	int error;

	/* We have got enough memory and from now on we cannot recover */
	error = set_up_temporary_mappings();
	if (error)
		return error;

	error = relocate_restore_code();
	if (error)
		return error;

	restore_image();
	return 0;
}

/*
 *	pfn_is_nosave - check if given pfn is in the 'nosave' section
 */

int pfn_is_nosave(unsigned long pfn)
{
	unsigned long nosave_begin_pfn = __pa_symbol(&__nosave_begin) >> PAGE_SHIFT;
	unsigned long nosave_end_pfn = PAGE_ALIGN(__pa_symbol(&__nosave_end)) >> PAGE_SHIFT;
	return (pfn >= nosave_begin_pfn) && (pfn < nosave_end_pfn);
}

struct restore_data_record {
	unsigned long jump_address;
	unsigned long jump_address_phys;
	unsigned long cr3;
	unsigned long magic;
};

#define RESTORE_MAGIC	0x123456789ABCDEF0UL

/**
 *	arch_hibernation_header_save - populate the architecture specific part
 *		of a hibernation image header
 *	@addr: address to save the data at
 */
int arch_hibernation_header_save(void *addr, unsigned int max_size)
{
	struct restore_data_record *rdr = addr;

	if (max_size < sizeof(struct restore_data_record))
		return -EOVERFLOW;
	rdr->jump_address = (unsigned long)&restore_registers;
	rdr->jump_address_phys = __pa_symbol(&restore_registers);
	rdr->cr3 = restore_cr3;
	rdr->magic = RESTORE_MAGIC;
	return 0;
}

/**
 *	arch_hibernation_header_restore - read the architecture specific data
 *		from the hibernation image header
 *	@addr: address to read the data from
 */
int arch_hibernation_header_restore(void *addr)
{
	struct restore_data_record *rdr = addr;

	restore_jump_address = rdr->jump_address;
	jump_address_phys = rdr->jump_address_phys;
	restore_cr3 = rdr->cr3;
	return (rdr->magic == RESTORE_MAGIC) ? 0 : -EINVAL;
}
Commit	Line	Data
ef8b03fa RW	1	/*
	2	* Hibernation support for x86-64
	3	*
	4	* Distribute under GPLv2
	5	*
	6	* Copyright (c) 2007 Rafael J. Wysocki <rjw@sisk.pl>
a2531293	7	* Copyright (c) 2002 Pavel Machek <pavel@ucw.cz>
ef8b03fa RW	8	* Copyright (c) 2001 Patrick Mochel <mochel@osdl.org>
	9	*/
	10
5a0e3ad6	11	#include <linux/gfp.h>
ef8b03fa RW	12	#include <linux/smp.h>
ef8b03fa RW	13	#include <linux/suspend.h>
8b78c21d YL	14
8b78c21d YL	15	#include <asm/init.h>
ef8b03fa RW	16	#include <asm/proto.h>
	17	#include <asm/page.h>
	18	#include <asm/pgtable.h>
	19	#include <asm/mtrr.h>
7f8998c7	20	#include <asm/sections.h>
a8af7898	21	#include <asm/suspend.h>
65c0554b	22	#include <asm/tlbflush.h>
ef8b03fa	23
261f0ce5	24	/* Defined in hibernate_asm_64.S */
2605fc21	25	extern asmlinkage __visible int restore_image(void);
ef8b03fa RW	26
	27	/*
	28	* Address to jump to in the last phase of restore in order to get to the image
	29	* kernel's text (this value is passed in the image header).
	30	*/
d6efc2f7	31	unsigned long restore_jump_address __visible;
65c0554b	32	unsigned long jump_address_phys;
ef8b03fa RW	33
	34	/*
	35	* Value of the cr3 register from before the hibernation (this value is passed
	36	* in the image header).
	37	*/
d6efc2f7	38	unsigned long restore_cr3 __visible;
ef8b03fa	39
c226fab4	40	unsigned long temp_level4_pgt __visible;
ef8b03fa	41
65c0554b RW	42	unsigned long relocated_restore_code __visible;
65c0554b RW	43
c226fab4	44	static int set_up_temporary_text_mapping(pgd_t *pgd)
65c0554b RW	45	{
	46	pmd_t *pmd;
	47	pud_t *pud;
	48
	49	/*
	50	* The new mapping only has to cover the page containing the image
	51	* kernel's entry point (jump_address_phys), because the switch over to
	52	* it is carried out by relocated code running from a page allocated
	53	* specifically for this purpose and covered by the identity mapping, so
	54	* the temporary kernel text mapping is only needed for the final jump.
	55	* Moreover, in that mapping the virtual address of the image kernel's
	56	* entry point must be the same as its virtual address in the image
	57	* kernel (restore_jump_address), so the image kernel's
	58	* restore_registers() code doesn't find itself in a different area of
	59	* the virtual address space after switching over to the original page
	60	* tables used by the image kernel.
	61	*/
	62	pud = (pud_t *)get_safe_page(GFP_ATOMIC);
	63	if (!pud)
	64	return -ENOMEM;
	65
	66	pmd = (pmd_t *)get_safe_page(GFP_ATOMIC);
	67	if (!pmd)
	68	return -ENOMEM;
	69
	70	set_pmd(pmd + pmd_index(restore_jump_address),
	71	__pmd((jump_address_phys & PMD_MASK) \| __PAGE_KERNEL_LARGE_EXEC));
	72	set_pud(pud + pud_index(restore_jump_address),
	73	__pud(__pa(pmd) \| _KERNPG_TABLE));
c226fab4	74	set_pgd(pgd + pgd_index(restore_jump_address),
65c0554b RW	75	__pgd(__pa(pud) \| _KERNPG_TABLE));
	76
	77	return 0;
	78	}
ef8b03fa	79
8b78c21d	80	static void alloc_pgt_page(void context)
ef8b03fa	81	{
8b78c21d	82	return (void *)get_safe_page(GFP_ATOMIC);
ef8b03fa RW	83	}
	84
	85	static int set_up_temporary_mappings(void)
	86	{
8b78c21d YL	87	struct x86_mapping_info info = {
	88	.alloc_pgt_page = alloc_pgt_page,
	89	.pmd_flag = __PAGE_KERNEL_LARGE_EXEC,
e4630fdd	90	.offset = __PAGE_OFFSET,
8b78c21d YL	91	};
8b78c21d YL	92	unsigned long mstart, mend;
c226fab4	93	pgd_t *pgd;
8b78c21d YL	94	int result;
8b78c21d YL	95	int i;
ef8b03fa	96
c226fab4 RW	97	pgd = (pgd_t *)get_safe_page(GFP_ATOMIC);
c226fab4 RW	98	if (!pgd)
ef8b03fa RW	99	return -ENOMEM;
ef8b03fa RW	100
65c0554b	101	/* Prepare a temporary mapping for the kernel text */
c226fab4	102	result = set_up_temporary_text_mapping(pgd);
65c0554b RW	103	if (result)
65c0554b RW	104	return result;
ef8b03fa RW	105
ef8b03fa RW	106	/* Set up the direct mapping from scratch */
8b78c21d YL	107	for (i = 0; i < nr_pfn_mapped; i++) {
	108	mstart = pfn_mapped[i].start << PAGE_SHIFT;
	109	mend = pfn_mapped[i].end << PAGE_SHIFT;
	110
c226fab4	111	result = kernel_ident_mapping_init(&info, pgd, mstart, mend);
8b78c21d YL	112	if (result)
8b78c21d YL	113	return result;
ef8b03fa	114	}
8b78c21d	115
5d87f493	116	temp_level4_pgt = __pa(pgd);
ef8b03fa RW	117	return 0;
	118	}
	119
65c0554b RW	120	static int relocate_restore_code(void)
	121	{
	122	pgd_t *pgd;
	123	pud_t *pud;
	124
	125	relocated_restore_code = get_safe_page(GFP_ATOMIC);
	126	if (!relocated_restore_code)
	127	return -ENOMEM;
	128
	129	memcpy((void *)relocated_restore_code, &core_restore_code, PAGE_SIZE);
	130
	131	/* Make the page containing the relocated code executable */
	132	pgd = (pgd_t *)__va(read_cr3()) + pgd_index(relocated_restore_code);
	133	pud = pud_offset(pgd, relocated_restore_code);
	134	if (pud_large(*pud)) {
	135	set_pud(pud, __pud(pud_val(*pud) & ~_PAGE_NX));
	136	} else {
	137	pmd_t *pmd = pmd_offset(pud, relocated_restore_code);
	138
	139	if (pmd_large(*pmd)) {
	140	set_pmd(pmd, __pmd(pmd_val(*pmd) & ~_PAGE_NX));
	141	} else {
	142	pte_t *pte = pte_offset_kernel(pmd, relocated_restore_code);
	143
	144	set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_NX));
	145	}
	146	}
	147	__flush_tlb_all();
	148
	149	return 0;
	150	}
	151
ef8b03fa RW	152	int swsusp_arch_resume(void)
	153	{
	154	int error;
	155
	156	/* We have got enough memory and from now on we cannot recover */
65c0554b RW	157	error = set_up_temporary_mappings();
65c0554b RW	158	if (error)
ef8b03fa RW	159	return error;
ef8b03fa RW	160
65c0554b RW	161	error = relocate_restore_code();
	162	if (error)
	163	return error;
ef8b03fa RW	164
	165	restore_image();
	166	return 0;
	167	}
	168
	169	/*
	170	* pfn_is_nosave - check if given pfn is in the 'nosave' section
	171	*/
	172
	173	int pfn_is_nosave(unsigned long pfn)
	174	{
	175	unsigned long nosave_begin_pfn = __pa_symbol(&__nosave_begin) >> PAGE_SHIFT;
	176	unsigned long nosave_end_pfn = PAGE_ALIGN(__pa_symbol(&__nosave_end)) >> PAGE_SHIFT;
	177	return (pfn >= nosave_begin_pfn) && (pfn < nosave_end_pfn);
	178	}
	179
	180	struct restore_data_record {
	181	unsigned long jump_address;
65c0554b	182	unsigned long jump_address_phys;
ef8b03fa RW	183	unsigned long cr3;
	184	unsigned long magic;
	185	};
	186
65c0554b	187	#define RESTORE_MAGIC 0x123456789ABCDEF0UL
ef8b03fa RW	188
	189	/**
	190	* arch_hibernation_header_save - populate the architecture specific part
	191	* of a hibernation image header
	192	* @addr: address to save the data at
	193	*/
	194	int arch_hibernation_header_save(void *addr, unsigned int max_size)
	195	{
	196	struct restore_data_record *rdr = addr;
	197
	198	if (max_size < sizeof(struct restore_data_record))
	199	return -EOVERFLOW;
65c0554b RW	200	rdr->jump_address = (unsigned long)&restore_registers;
65c0554b RW	201	rdr->jump_address_phys = __pa_symbol(&restore_registers);
ef8b03fa RW	202	rdr->cr3 = restore_cr3;
	203	rdr->magic = RESTORE_MAGIC;
	204	return 0;
	205	}
	206
	207	/**
	208	* arch_hibernation_header_restore - read the architecture specific data
	209	* from the hibernation image header
	210	* @addr: address to read the data from
	211	*/
	212	int arch_hibernation_header_restore(void *addr)
	213	{
	214	struct restore_data_record *rdr = addr;
	215
	216	restore_jump_address = rdr->jump_address;
65c0554b	217	jump_address_phys = rdr->jump_address_phys;
ef8b03fa RW	218	restore_cr3 = rdr->cr3;
	219	return (rdr->magic == RESTORE_MAGIC) ? 0 : -EINVAL;
	220	}