x86/kasan: Extend KASAN to support 5-level paging

[mirror_ubuntu-bionic-kernel.git] / arch / x86 / mm / kasan_init_64.c

diff --git a/arch/x86/mm/kasan_init_64.c b/arch/x86/mm/kasan_init_64.c
index 4c90cfdc128b832c6065cdb8830f89d16bff63dd..47efdcfe711313c2d719e645d6dafbf0fdea3e7c 100644 (file)
--- a/arch/x86/mm/kasan_init_64.c
+++ b/arch/x86/mm/kasan_init_64.c
@@ -33,8 +33,19 @@ static int __init map_range(struct range *range)
 static void __init clear_pgds(unsigned long start,
                        unsigned long end)
 {
-       for (; start < end; start += PGDIR_SIZE)
-               pgd_clear(pgd_offset_k(start));
+       pgd_t *pgd;
+
+       for (; start < end; start += PGDIR_SIZE) {
+               pgd = pgd_offset_k(start);
+               /*
+                * With folded p4d, pgd_clear() is nop, use p4d_clear()
+                * instead.
+                */
+               if (CONFIG_PGTABLE_LEVELS < 5)
+                       p4d_clear(p4d_offset(pgd, start));
+               else
+                       pgd_clear(pgd);
+       }
 }
 
 static void __init kasan_map_early_shadow(pgd_t *pgd)
@@ -44,8 +55,18 @@ static void __init kasan_map_early_shadow(pgd_t *pgd)
        unsigned long end = KASAN_SHADOW_END;
 
        for (i = pgd_index(start); start < end; i++) {
-               pgd[i] = __pgd(__pa_nodebug(kasan_zero_pud)
-                               | _KERNPG_TABLE);
+               switch (CONFIG_PGTABLE_LEVELS) {
+               case 4:
+                       pgd[i] = __pgd(__pa_nodebug(kasan_zero_pud) |
+                                       _KERNPG_TABLE);
+                       break;
+               case 5:
+                       pgd[i] = __pgd(__pa_nodebug(kasan_zero_p4d) |
+                                       _KERNPG_TABLE);
+                       break;
+               default:
+                       BUILD_BUG();
+               }
                start += PGDIR_SIZE;
        }
 }
@@ -73,6 +94,7 @@ void __init kasan_early_init(void)
        pteval_t pte_val = __pa_nodebug(kasan_zero_page) | __PAGE_KERNEL;
        pmdval_t pmd_val = __pa_nodebug(kasan_zero_pte) | _KERNPG_TABLE;
        pudval_t pud_val = __pa_nodebug(kasan_zero_pmd) | _KERNPG_TABLE;
+       p4dval_t p4d_val = __pa_nodebug(kasan_zero_pud) | _KERNPG_TABLE;
 
        for (i = 0; i < PTRS_PER_PTE; i++)
                kasan_zero_pte[i] = __pte(pte_val);
@@ -83,6 +105,9 @@ void __init kasan_early_init(void)
        for (i = 0; i < PTRS_PER_PUD; i++)
                kasan_zero_pud[i] = __pud(pud_val);
 
+       for (i = 0; CONFIG_PGTABLE_LEVELS >= 5 && i < PTRS_PER_P4D; i++)
+               kasan_zero_p4d[i] = __p4d(p4d_val);
+
        kasan_map_early_shadow(early_level4_pgt);
        kasan_map_early_shadow(init_level4_pgt);
 }