x86/mm/pti: Clear Global bit more aggressively

[mirror_ubuntu-bionic-kernel.git] / arch / x86 / mm / pti.c

diff --git a/arch/x86/mm/pti.c b/arch/x86/mm/pti.c
index 5e4b78ac6338606016beaf3bd5cbaef8ebda5a28..bd60762cbc14d5b27f238185a00b14d748da8ccf 100644 (file)
--- a/arch/x86/mm/pti.c
+++ b/arch/x86/mm/pti.c
@@ -470,6 +470,13 @@ static inline bool pti_kernel_image_global_ok(void)
        return true;
 }
 
+/*
+ * This is the only user for these and it is not arch-generic
+ * like the other set_memory.h functions.  Just extern them.
+ */
+extern int set_memory_nonglobal(unsigned long addr, int numpages);
+extern int set_memory_global(unsigned long addr, int numpages);
+
 /*
  * For some configurations, map all of kernel text into the user page
  * tables.  This reduces TLB misses, especially on non-PCID systems.
@@ -482,7 +489,8 @@ static void pti_clone_kernel_text(void)
         * clone the areas past rodata, they might contain secrets.
         */
        unsigned long start = PFN_ALIGN(_text);
-       unsigned long end = (unsigned long)__end_rodata_aligned;
+       unsigned long end_clone  = (unsigned long)__end_rodata_aligned;
+       unsigned long end_global = PFN_ALIGN((unsigned long)__stop___ex_table);
 
        if (!pti_kernel_image_global_ok())
                return;
@@ -494,14 +502,18 @@ static void pti_clone_kernel_text(void)
         * pti_set_kernel_image_nonglobal() did to clear the
         * global bit.
         */
-       pti_clone_pmds(start, end, 0);
+       pti_clone_pmds(start, end_clone, _PAGE_RW);
+
+       /*
+        * pti_clone_pmds() will set the global bit in any PMDs
+        * that it clones, but we also need to get any PTEs in
+        * the last level for areas that are not huge-page-aligned.
+        */
+
+       /* Set the global bit for normal non-__init kernel text: */
+       set_memory_global(start, (end_global - start) >> PAGE_SHIFT);
 }
 
-/*
- * This is the only user for it and it is not arch-generic like
- * the other set_memory.h functions.  Just extern it.
- */
-extern int set_memory_nonglobal(unsigned long addr, int numpages);
 void pti_set_kernel_image_nonglobal(void)
 {
        /*
@@ -513,9 +525,11 @@ void pti_set_kernel_image_nonglobal(void)
        unsigned long start = PFN_ALIGN(_text);
        unsigned long end = ALIGN((unsigned long)_end, PMD_PAGE_SIZE);
 
-       if (pti_kernel_image_global_ok())
-               return;
-
+       /*
+        * This clears _PAGE_GLOBAL from the entire kernel image.
+        * pti_clone_kernel_text() map put _PAGE_GLOBAL back for
+        * areas that are mapped to userspace.
+        */
        set_memory_nonglobal(start, (end - start) >> PAGE_SHIFT);
 }