x86/mm: Provide general kernel support for memory encryption

[mirror_ubuntu-bionic-kernel.git] / arch / x86 / kernel / head64.c

diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index 1f0ddcc9675cb4910ed5c54ad35b9fdb2dc05966..5cd0b72a02834ca1f56efae4c4681220a56a0572 100644 (file)
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -102,7 +102,7 @@ unsigned long __head __startup_64(unsigned long physaddr)
 
        pud = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
        pmd = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
-       pgtable_flags = _KERNPG_TABLE + sme_get_me_mask();
+       pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();
 
        if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
                p4d = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
@@ -177,7 +177,7 @@ static void __init reset_early_page_tables(void)
 {
        memset(early_top_pgt, 0, sizeof(pgd_t)*(PTRS_PER_PGD-1));
        next_early_pgt = 0;
-       write_cr3(__pa_nodebug(early_top_pgt));
+       write_cr3(__sme_pa_nodebug(early_top_pgt));
 }
 
 /* Create a new PMD entry */
@@ -310,6 +310,13 @@ asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data)
 
        clear_page(init_top_pgt);
 
+       /*
+        * SME support may update early_pmd_flags to include the memory
+        * encryption mask, so it needs to be called before anything
+        * that may generate a page fault.
+        */
+       sme_early_init();
+
        kasan_early_init();
 
        for (i = 0; i < NUM_EXCEPTION_VECTORS; i++)