2 * Hibernate support specific for ARM64
4 * Derived from work on ARM hibernation support by:
6 * Ubuntu project, hibernation support for mach-dove
7 * Copyright (C) 2010 Nokia Corporation (Hiroshi Doyu)
8 * Copyright (C) 2010 Texas Instruments, Inc. (Teerth Reddy et al.)
9 * https://lkml.org/lkml/2010/6/18/4
10 * https://lists.linux-foundation.org/pipermail/linux-pm/2010-June/027422.html
11 * https://patchwork.kernel.org/patch/96442/
13 * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
15 * License terms: GNU General Public License (GPL) version 2
17 #define pr_fmt(x) "hibernate: " x
18 #include <linux/cpu.h>
19 #include <linux/kvm_host.h>
22 #include <linux/sched.h>
23 #include <linux/suspend.h>
24 #include <linux/utsname.h>
25 #include <linux/version.h>
27 #include <asm/barrier.h>
28 #include <asm/cacheflush.h>
29 #include <asm/cputype.h>
30 #include <asm/irqflags.h>
31 #include <asm/kexec.h>
32 #include <asm/memory.h>
33 #include <asm/mmu_context.h>
34 #include <asm/pgalloc.h>
35 #include <asm/pgtable.h>
36 #include <asm/pgtable-hwdef.h>
37 #include <asm/sections.h>
39 #include <asm/smp_plat.h>
40 #include <asm/suspend.h>
41 #include <asm/sysreg.h>
45 * Hibernate core relies on this value being 0 on resume, and marks it
46 * __nosavedata assuming it will keep the resume kernel's '0' value. This
47 * doesn't happen with either KASLR.
49 * defined as "__visible int in_suspend __nosavedata" in
50 * kernel/power/hibernate.c
52 extern int in_suspend
;
54 /* Do we need to reset el2? */
55 #define el2_reset_needed() (is_hyp_mode_available() && !is_kernel_in_hyp_mode())
57 /* temporary el2 vectors in the __hibernate_exit_text section. */
58 extern char hibernate_el2_vectors
[];
60 /* hyp-stub vectors, used to restore el2 during resume from hibernate. */
61 extern char __hyp_stub_vectors
[];
64 * The logical cpu number we should resume on, initialised to a non-cpu
67 static int sleep_cpu
= -EINVAL
;
70 * Values that may not change over hibernate/resume. We put the build number
71 * and date in here so that we guarantee not to resume with a different
74 struct arch_hibernate_hdr_invariants
{
75 char uts_version
[__NEW_UTS_LEN
+ 1];
78 /* These values need to be know across a hibernate/restore. */
79 static struct arch_hibernate_hdr
{
80 struct arch_hibernate_hdr_invariants invariants
;
82 /* These are needed to find the relocated kernel if built with kaslr */
83 phys_addr_t ttbr1_el1
;
84 void (*reenter_kernel
)(void);
87 * We need to know where the __hyp_stub_vectors are after restore to
90 phys_addr_t __hyp_stub_vectors
;
95 static inline void arch_hdr_invariants(struct arch_hibernate_hdr_invariants
*i
)
97 memset(i
, 0, sizeof(*i
));
98 memcpy(i
->uts_version
, init_utsname()->version
, sizeof(i
->uts_version
));
101 int pfn_is_nosave(unsigned long pfn
)
103 unsigned long nosave_begin_pfn
= sym_to_pfn(&__nosave_begin
);
104 unsigned long nosave_end_pfn
= sym_to_pfn(&__nosave_end
- 1);
106 return ((pfn
>= nosave_begin_pfn
) && (pfn
<= nosave_end_pfn
)) ||
107 crash_is_nosave(pfn
);
110 void notrace
save_processor_state(void)
112 WARN_ON(num_online_cpus() != 1);
115 void notrace
restore_processor_state(void)
119 int arch_hibernation_header_save(void *addr
, unsigned int max_size
)
121 struct arch_hibernate_hdr
*hdr
= addr
;
123 if (max_size
< sizeof(*hdr
))
126 arch_hdr_invariants(&hdr
->invariants
);
127 hdr
->ttbr1_el1
= __pa_symbol(swapper_pg_dir
);
128 hdr
->reenter_kernel
= _cpu_resume
;
130 /* We can't use __hyp_get_vectors() because kvm may still be loaded */
131 if (el2_reset_needed())
132 hdr
->__hyp_stub_vectors
= __pa_symbol(__hyp_stub_vectors
);
134 hdr
->__hyp_stub_vectors
= 0;
136 /* Save the mpidr of the cpu we called cpu_suspend() on... */
138 pr_err("Failing to hibernate on an unknown CPU.\n");
141 hdr
->sleep_cpu_mpidr
= cpu_logical_map(sleep_cpu
);
142 pr_info("Hibernating on CPU %d [mpidr:0x%llx]\n", sleep_cpu
,
143 hdr
->sleep_cpu_mpidr
);
147 EXPORT_SYMBOL(arch_hibernation_header_save
);
149 int arch_hibernation_header_restore(void *addr
)
152 struct arch_hibernate_hdr_invariants invariants
;
153 struct arch_hibernate_hdr
*hdr
= addr
;
155 arch_hdr_invariants(&invariants
);
156 if (memcmp(&hdr
->invariants
, &invariants
, sizeof(invariants
))) {
157 pr_crit("Hibernate image not generated by this kernel!\n");
161 sleep_cpu
= get_logical_index(hdr
->sleep_cpu_mpidr
);
162 pr_info("Hibernated on CPU %d [mpidr:0x%llx]\n", sleep_cpu
,
163 hdr
->sleep_cpu_mpidr
);
165 pr_crit("Hibernated on a CPU not known to this kernel!\n");
169 if (!cpu_online(sleep_cpu
)) {
170 pr_info("Hibernated on a CPU that is offline! Bringing CPU up.\n");
171 ret
= cpu_up(sleep_cpu
);
173 pr_err("Failed to bring hibernate-CPU up!\n");
183 EXPORT_SYMBOL(arch_hibernation_header_restore
);
186 * Copies length bytes, starting at src_start into an new page,
187 * perform cache maintentance, then maps it at the specified address low
188 * address as executable.
190 * This is used by hibernate to copy the code it needs to execute when
191 * overwriting the kernel text. This function generates a new set of page
192 * tables, which it loads into ttbr0.
194 * Length is provided as we probably only want 4K of data, even on a 64K
197 static int create_safe_exec_page(void *src_start
, size_t length
,
198 unsigned long dst_addr
,
199 phys_addr_t
*phys_dst_addr
,
200 void *(*allocator
)(gfp_t mask
),
208 unsigned long dst
= (unsigned long)allocator(mask
);
215 memcpy((void *)dst
, src_start
, length
);
216 flush_icache_range(dst
, dst
+ length
);
218 pgd
= pgd_offset_raw(allocator(mask
), dst_addr
);
219 if (pgd_none(*pgd
)) {
220 pud
= allocator(mask
);
225 pgd_populate(&init_mm
, pgd
, pud
);
228 pud
= pud_offset(pgd
, dst_addr
);
229 if (pud_none(*pud
)) {
230 pmd
= allocator(mask
);
235 pud_populate(&init_mm
, pud
, pmd
);
238 pmd
= pmd_offset(pud
, dst_addr
);
239 if (pmd_none(*pmd
)) {
240 pte
= allocator(mask
);
245 pmd_populate_kernel(&init_mm
, pmd
, pte
);
248 pte
= pte_offset_kernel(pmd
, dst_addr
);
249 set_pte(pte
, __pte(virt_to_phys((void *)dst
) |
250 pgprot_val(PAGE_KERNEL_EXEC
)));
253 * Load our new page tables. A strict BBM approach requires that we
254 * ensure that TLBs are free of any entries that may overlap with the
255 * global mappings we are about to install.
257 * For a real hibernate/resume cycle TTBR0 currently points to a zero
258 * page, but TLBs may contain stale ASID-tagged entries (e.g. for EFI
259 * runtime services), while for a userspace-driven test_resume cycle it
260 * points to userspace page tables (and we must point it at a zero page
261 * ourselves). Elsewhere we only (un)install the idmap with preemption
262 * disabled, so T0SZ should be as required regardless.
264 cpu_set_reserved_ttbr0();
265 local_flush_tlb_all();
266 write_sysreg(virt_to_phys(pgd
), ttbr0_el1
);
269 *phys_dst_addr
= virt_to_phys((void *)dst
);
275 #define dcache_clean_range(start, end) __flush_dcache_area(start, (end - start))
277 int swsusp_arch_suspend(void)
281 struct sleep_stack_data state
;
283 if (cpus_are_stuck_in_kernel()) {
284 pr_err("Can't hibernate: no mechanism to offline secondary CPUs.\n");
288 local_dbg_save(flags
);
290 if (__cpu_suspend_enter(&state
)) {
291 /* make the crash dump kernel image visible/saveable */
292 crash_prepare_suspend();
294 sleep_cpu
= smp_processor_id();
297 /* Clean kernel core startup/idle code to PoC*/
298 dcache_clean_range(__mmuoff_data_start
, __mmuoff_data_end
);
299 dcache_clean_range(__idmap_text_start
, __idmap_text_end
);
301 /* Clean kvm setup code to PoC? */
302 if (el2_reset_needed())
303 dcache_clean_range(__hyp_idmap_text_start
, __hyp_idmap_text_end
);
305 /* make the crash dump kernel image protected again */
309 * Tell the hibernation core that we've just restored
315 __cpu_suspend_exit();
318 local_dbg_restore(flags
);
323 static void _copy_pte(pte_t
*dst_pte
, pte_t
*src_pte
, unsigned long addr
)
325 pte_t pte
= *src_pte
;
327 if (pte_valid(pte
)) {
329 * Resume will overwrite areas that may be marked
330 * read only (code, rodata). Clear the RDONLY bit from
331 * the temporary mappings we use during restore.
333 set_pte(dst_pte
, pte_clear_rdonly(pte
));
334 } else if (debug_pagealloc_enabled() && !pte_none(pte
)) {
336 * debug_pagealloc will removed the PTE_VALID bit if
337 * the page isn't in use by the resume kernel. It may have
338 * been in use by the original kernel, in which case we need
339 * to put it back in our copy to do the restore.
341 * Before marking this entry valid, check the pfn should
344 BUG_ON(!pfn_valid(pte_pfn(pte
)));
346 set_pte(dst_pte
, pte_mkpresent(pte_clear_rdonly(pte
)));
350 static int copy_pte(pmd_t
*dst_pmd
, pmd_t
*src_pmd
, unsigned long start
,
355 unsigned long addr
= start
;
357 dst_pte
= (pte_t
*)get_safe_page(GFP_ATOMIC
);
360 pmd_populate_kernel(&init_mm
, dst_pmd
, dst_pte
);
361 dst_pte
= pte_offset_kernel(dst_pmd
, start
);
363 src_pte
= pte_offset_kernel(src_pmd
, start
);
365 _copy_pte(dst_pte
, src_pte
, addr
);
366 } while (dst_pte
++, src_pte
++, addr
+= PAGE_SIZE
, addr
!= end
);
371 static int copy_pmd(pud_t
*dst_pud
, pud_t
*src_pud
, unsigned long start
,
377 unsigned long addr
= start
;
379 if (pud_none(*dst_pud
)) {
380 dst_pmd
= (pmd_t
*)get_safe_page(GFP_ATOMIC
);
383 pud_populate(&init_mm
, dst_pud
, dst_pmd
);
385 dst_pmd
= pmd_offset(dst_pud
, start
);
387 src_pmd
= pmd_offset(src_pud
, start
);
389 next
= pmd_addr_end(addr
, end
);
390 if (pmd_none(*src_pmd
))
392 if (pmd_table(*src_pmd
)) {
393 if (copy_pte(dst_pmd
, src_pmd
, addr
, next
))
397 __pmd(pmd_val(*src_pmd
) & ~PMD_SECT_RDONLY
));
399 } while (dst_pmd
++, src_pmd
++, addr
= next
, addr
!= end
);
404 static int copy_pud(pgd_t
*dst_pgd
, pgd_t
*src_pgd
, unsigned long start
,
410 unsigned long addr
= start
;
412 if (pgd_none(*dst_pgd
)) {
413 dst_pud
= (pud_t
*)get_safe_page(GFP_ATOMIC
);
416 pgd_populate(&init_mm
, dst_pgd
, dst_pud
);
418 dst_pud
= pud_offset(dst_pgd
, start
);
420 src_pud
= pud_offset(src_pgd
, start
);
422 next
= pud_addr_end(addr
, end
);
423 if (pud_none(*src_pud
))
425 if (pud_table(*(src_pud
))) {
426 if (copy_pmd(dst_pud
, src_pud
, addr
, next
))
430 __pud(pud_val(*src_pud
) & ~PMD_SECT_RDONLY
));
432 } while (dst_pud
++, src_pud
++, addr
= next
, addr
!= end
);
437 static int copy_page_tables(pgd_t
*dst_pgd
, unsigned long start
,
441 unsigned long addr
= start
;
442 pgd_t
*src_pgd
= pgd_offset_k(start
);
444 dst_pgd
= pgd_offset_raw(dst_pgd
, start
);
446 next
= pgd_addr_end(addr
, end
);
447 if (pgd_none(*src_pgd
))
449 if (copy_pud(dst_pgd
, src_pgd
, addr
, next
))
451 } while (dst_pgd
++, src_pgd
++, addr
= next
, addr
!= end
);
457 * Setup then Resume from the hibernate image using swsusp_arch_suspend_exit().
459 * Memory allocated by get_safe_page() will be dealt with by the hibernate code,
460 * we don't need to free it here.
462 int swsusp_arch_resume(void)
468 phys_addr_t phys_hibernate_exit
;
469 void __noreturn (*hibernate_exit
)(phys_addr_t
, phys_addr_t
, void *,
470 void *, phys_addr_t
, phys_addr_t
);
473 * Restoring the memory image will overwrite the ttbr1 page tables.
474 * Create a second copy of just the linear map, and use this when
477 tmp_pg_dir
= (pgd_t
*)get_safe_page(GFP_ATOMIC
);
479 pr_err("Failed to allocate memory for temporary page tables.\n");
483 rc
= copy_page_tables(tmp_pg_dir
, PAGE_OFFSET
, 0);
488 * We need a zero page that is zero before & after resume in order to
489 * to break before make on the ttbr1 page tables.
491 zero_page
= (void *)get_safe_page(GFP_ATOMIC
);
493 pr_err("Failed to allocate zero page.\n");
499 * Locate the exit code in the bottom-but-one page, so that *NULL
500 * still has disastrous affects.
502 hibernate_exit
= (void *)PAGE_SIZE
;
503 exit_size
= __hibernate_exit_text_end
- __hibernate_exit_text_start
;
505 * Copy swsusp_arch_suspend_exit() to a safe page. This will generate
506 * a new set of ttbr0 page tables and load them.
508 rc
= create_safe_exec_page(__hibernate_exit_text_start
, exit_size
,
509 (unsigned long)hibernate_exit
,
510 &phys_hibernate_exit
,
511 (void *)get_safe_page
, GFP_ATOMIC
);
513 pr_err("Failed to create safe executable page for hibernate_exit code.\n");
518 * The hibernate exit text contains a set of el2 vectors, that will
519 * be executed at el2 with the mmu off in order to reload hyp-stub.
521 __flush_dcache_area(hibernate_exit
, exit_size
);
524 * KASLR will cause the el2 vectors to be in a different location in
525 * the resumed kernel. Load hibernate's temporary copy into el2.
527 * We can skip this step if we booted at EL1, or are running with VHE.
529 if (el2_reset_needed()) {
530 phys_addr_t el2_vectors
= phys_hibernate_exit
; /* base */
531 el2_vectors
+= hibernate_el2_vectors
-
532 __hibernate_exit_text_start
; /* offset */
534 __hyp_set_vectors(el2_vectors
);
537 hibernate_exit(virt_to_phys(tmp_pg_dir
), resume_hdr
.ttbr1_el1
,
538 resume_hdr
.reenter_kernel
, restore_pblist
,
539 resume_hdr
.__hyp_stub_vectors
, virt_to_phys(zero_page
));
545 int hibernate_resume_nonboot_cpu_disable(void)
548 pr_err("Failing to resume from hibernate on an unknown CPU.\n");
552 return freeze_secondary_cpus(sleep_cpu
);