arm64: proc: Set PTE_NG for table entries to avoid traversing them twice

When KASAN is enabled, the swapper page table contains many identical
mappings of the zero page, which can lead to a stall during boot whilst
the G -> nG code continually walks the same page table entries looking
for global mappings.

This patch sets the nG bit (bit 11, which is IGNORED) in table entries
after processing the subtree so we can easily skip them if we see them
a second time.

Tested-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index 71baed7e592a499196a1c7bc239dbbc3297d3bc2..c0af4761729986da832a8e844b256e64da8cb4c1 100644 (file)
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -205,7 +205,8 @@ ENDPROC(idmap_cpu_replace_ttbr1)
        dc      cvac, cur_\()\type\()p          // Ensure any existing dirty
        dmb     sy                              // lines are written back before
        ldr     \type, [cur_\()\type\()p]       // loading the entry
-       tbz     \type, #0, next_\()\type        // Skip invalid entries
+       tbz     \type, #0, skip_\()\type        // Skip invalid and
+       tbnz    \type, #11, skip_\()\type       // non-global entries
        .endm
 
        .macro __idmap_kpti_put_pgtable_ent_ng, type
@@ -265,8 +266,9 @@ ENTRY(idmap_kpti_install_ng_mappings)
        add     end_pgdp, cur_pgdp, #(PTRS_PER_PGD * 8)
 do_pgd:        __idmap_kpti_get_pgtable_ent    pgd
        tbnz    pgd, #1, walk_puds
-       __idmap_kpti_put_pgtable_ent_ng pgd
 next_pgd:
+       __idmap_kpti_put_pgtable_ent_ng pgd
+skip_pgd:
        add     cur_pgdp, cur_pgdp, #8
        cmp     cur_pgdp, end_pgdp
        b.ne    do_pgd
@@ -294,8 +296,9 @@ walk_puds:
        add     end_pudp, cur_pudp, #(PTRS_PER_PUD * 8)
 do_pud:        __idmap_kpti_get_pgtable_ent    pud
        tbnz    pud, #1, walk_pmds
-       __idmap_kpti_put_pgtable_ent_ng pud
 next_pud:
+       __idmap_kpti_put_pgtable_ent_ng pud
+skip_pud:
        add     cur_pudp, cur_pudp, 8
        cmp     cur_pudp, end_pudp
        b.ne    do_pud
@@ -314,8 +317,9 @@ walk_pmds:
        add     end_pmdp, cur_pmdp, #(PTRS_PER_PMD * 8)
 do_pmd:        __idmap_kpti_get_pgtable_ent    pmd
        tbnz    pmd, #1, walk_ptes
-       __idmap_kpti_put_pgtable_ent_ng pmd
 next_pmd:
+       __idmap_kpti_put_pgtable_ent_ng pmd
+skip_pmd:
        add     cur_pmdp, cur_pmdp, #8
        cmp     cur_pmdp, end_pmdp
        b.ne    do_pmd
@@ -333,7 +337,7 @@ walk_ptes:
        add     end_ptep, cur_ptep, #(PTRS_PER_PTE * 8)
 do_pte:        __idmap_kpti_get_pgtable_ent    pte
        __idmap_kpti_put_pgtable_ent_ng pte
-next_pte:
+skip_pte:
        add     cur_ptep, cur_ptep, #8
        cmp     cur_ptep, end_ptep
        b.ne    do_pte