Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64...

[mirror_ubuntu-focal-kernel.git] / arch / arm64 / kernel / head.S

diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 1fe5d8d2bdfd2f8c134cde57e968d026db854f0f..0fd56500077271402bb7af82f83526cd1dcef1ee 100644 (file)
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -26,6 +26,7 @@
 #include <asm/assembler.h>
 #include <asm/ptrace.h>
 #include <asm/asm-offsets.h>
+#include <asm/cache.h>
 #include <asm/cputype.h>
 #include <asm/memory.h>
 #include <asm/thread_info.h>
@@ -229,7 +230,11 @@ ENTRY(set_cpu_boot_mode_flag)
        cmp     w20, #BOOT_CPU_MODE_EL2
        b.ne    1f
        add     x1, x1, #4
-1:     str     w20, [x1]                       // This CPU has booted in EL1
+1:     dc      cvac, x1                        // Clean potentially dirty cache line
+       dsb     sy
+       str     w20, [x1]                       // This CPU has booted in EL1
+       dc      civac, x1                       // Clean&invalidate potentially stale cache line
+       dsb     sy
        ret
 ENDPROC(set_cpu_boot_mode_flag)
 
@@ -240,8 +245,9 @@ ENDPROC(set_cpu_boot_mode_flag)
  * This is not in .bss, because we set it sufficiently early that the boot-time
  * zeroing of .bss would clobber it.
  */
-       .pushsection    .data
+       .pushsection    .data..cacheline_aligned
 ENTRY(__boot_cpu_mode)
+       .align  L1_CACHE_SHIFT
        .long   BOOT_CPU_MODE_EL2
        .long   0
        .popsection
@@ -408,6 +414,15 @@ ENDPROC(__calc_phys_offset)
  */
 __create_page_tables:
        pgtbl   x25, x26, x24                   // idmap_pg_dir and swapper_pg_dir addresses
+       mov     x27, lr
+
+       /*
+        * Invalidate the idmap and swapper page tables to avoid potential
+        * dirty cache lines being evicted.
+        */
+       mov     x0, x25
+       add     x1, x26, #SWAPPER_DIR_SIZE
+       bl      __inval_cache_range
 
        /*
         * Clear the idmap and swapper page tables.
@@ -467,6 +482,17 @@ __create_page_tables:
        ldr     x5, =FIXADDR_TOP                // Fixed mapping virtual address
        add     x0, x26, #2 * PAGE_SIZE         // section table address
        create_pgd_entry x26, x0, x5, x6, x7
+
+       /*
+        * Since the page tables have been populated with non-cacheable
+        * accesses (MMU disabled), invalidate the idmap and swapper page
+        * tables again to remove any speculatively loaded cache lines.
+        */
+       mov     x0, x25
+       add     x1, x26, #SWAPPER_DIR_SIZE
+       bl      __inval_cache_range
+
+       mov     lr, x27
        ret
 ENDPROC(__create_page_tables)
        .ltorg