arm64: mm: Flip kernel VA space

In order to allow for a KASAN shadow that changes size at boot time, one
must fix the KASAN_SHADOW_END for both 48 & 52-bit VAs and "grow" the
start address. Also, it is highly desirable to maintain the same
function addresses in the kernel .text between VA sizes. Both of these
requirements necessitate us to flip the kernel address space halves s.t.
the direct linear map occupies the lower addresses.

This patch puts the direct linear map in the lower addresses of the
kernel VA range and everything else in the higher ranges.

We need to adjust:
 *) KASAN shadow region placement logic,
 *) KASAN_SHADOW_OFFSET computation logic,
 *) virt_to_phys, phys_to_virt checks,
 *) page table dumper.

These are all small changes, that need to take place atomically, so they
are bundled into this commit.

As part of the re-arrangement, a guard region of 2MB (to preserve
alignment for fixed map) is added after the vmemmap. Otherwise the
vmemmap could intersect with IS_ERR pointers.

Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Steve Capper <steve.capper@arm.com>
Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile
index 61de992bbea3fad61895e4f1e796cea460fd9deb..61f7926fdeca23b471dfa4300228f023aadec522 100644 (file)
--- a/arch/arm64/Makefile
+++ b/arch/arm64/Makefile
@@ -130,7 +130,7 @@ KBUILD_AFLAGS += -DKASAN_SHADOW_SCALE_SHIFT=$(KASAN_SHADOW_SCALE_SHIFT)
 #                               - (1 << (64 - KASAN_SHADOW_SCALE_SHIFT))
 # in 32-bit arithmetic
 KASAN_SHADOW_OFFSET := $(shell printf "0x%08x00000000\n" $$(( \
-       (0xffffffff & (-1 << ($(CONFIG_ARM64_VA_BITS) - 32))) \
+       (0xffffffff & (-1 << ($(CONFIG_ARM64_VA_BITS) - 1 - 32))) \
        + (1 << ($(CONFIG_ARM64_VA_BITS) - 32 - $(KASAN_SHADOW_SCALE_SHIFT))) \
        - (1 << (64 - 32 - $(KASAN_SHADOW_SCALE_SHIFT))) )) )
 

diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h
index 98fda92a26127579529ba6b90503a7915528bb84..380594b1a0bad0d6b8e793d2c0c345e92bb47a35 100644 (file)
--- a/arch/arm64/include/asm/memory.h
+++ b/arch/arm64/include/asm/memory.h
@@ -38,9 +38,9 @@
  */
 #define VA_BITS                        (CONFIG_ARM64_VA_BITS)
 #define VA_START               (UL(0xffffffffffffffff) - \
-       (UL(1) << VA_BITS) + 1)
-#define PAGE_OFFSET            (UL(0xffffffffffffffff) - \
        (UL(1) << (VA_BITS - 1)) + 1)
+#define PAGE_OFFSET            (UL(0xffffffffffffffff) - \
+       (UL(1) << VA_BITS) + 1)
 #define KIMAGE_VADDR           (MODULES_END)
 #define BPF_JIT_REGION_START   (VA_START + KASAN_SHADOW_SIZE)
 #define BPF_JIT_REGION_SIZE    (SZ_128M)
@@ -48,7 +48,7 @@
 #define MODULES_END            (MODULES_VADDR + MODULES_VSIZE)
 #define MODULES_VADDR          (BPF_JIT_REGION_END)
 #define MODULES_VSIZE          (SZ_128M)
-#define VMEMMAP_START          (PAGE_OFFSET - VMEMMAP_SIZE)
+#define VMEMMAP_START          (-VMEMMAP_SIZE - SZ_2M)
 #define PCI_IO_END             (VMEMMAP_START - SZ_2M)
 #define PCI_IO_START           (PCI_IO_END - PCI_IO_SIZE)
 #define FIXADDR_TOP            (PCI_IO_START - SZ_2M)
@@ -231,7 +231,7 @@ static inline const void *__tag_set(const void *addr, u8 tag)
  * space. Testing the top bit for the start of the region is a
  * sufficient check.
  */
-#define __is_lm_address(addr)  (!!((addr) & BIT(VA_BITS - 1)))
+#define __is_lm_address(addr)  (!((addr) & BIT(VA_BITS - 1)))
 
 #define __lm_to_phys(addr)     (((addr) & ~PAGE_OFFSET) + PHYS_OFFSET)
 #define __kimg_to_phys(addr)   ((addr) - kimage_voffset)

diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 5fdcfe2373389ba630cf72ff94c062b61ca1b025..046b811309bb5670d321863d933a1ce027f8144d 100644 (file)
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -21,7 +21,7 @@
  *     and fixed mappings
  */
 #define VMALLOC_START          (MODULES_END)
-#define VMALLOC_END            (PAGE_OFFSET - PUD_SIZE - VMEMMAP_SIZE - SZ_64K)
+#define VMALLOC_END            (- PUD_SIZE - VMEMMAP_SIZE - SZ_64K)
 
 #define vmemmap                        ((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT))
 

diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c
index 9341fcc6e809bc678872fa4574666a85ada5ec6f..e130db05d9327767e50c17434a695c51530557d9 100644 (file)
--- a/arch/arm64/kernel/hibernate.c
+++ b/arch/arm64/kernel/hibernate.c
@@ -496,7 +496,7 @@ int swsusp_arch_resume(void)
                rc = -ENOMEM;
                goto out;
        }
-       rc = copy_page_tables(tmp_pg_dir, PAGE_OFFSET, 0);
+       rc = copy_page_tables(tmp_pg_dir, PAGE_OFFSET, VA_START);
        if (rc)
                goto out;
 

diff --git a/arch/arm64/mm/dump.c b/arch/arm64/mm/dump.c
index 82b3a7fdb4a68b83c1be6e5b65b7c8b2eb0bbe3c..beec87488e979c355961e0a7db26d2a4ae92567b 100644 (file)
--- a/arch/arm64/mm/dump.c
+++ b/arch/arm64/mm/dump.c
@@ -26,6 +26,8 @@
 #include <asm/ptdump.h>
 
 static const struct addr_marker address_markers[] = {
+       { PAGE_OFFSET,                  "Linear Mapping start" },
+       { VA_START,                     "Linear Mapping end" },
 #ifdef CONFIG_KASAN
        { KASAN_SHADOW_START,           "Kasan shadow start" },
        { KASAN_SHADOW_END,             "Kasan shadow end" },
@@ -42,7 +44,6 @@ static const struct addr_marker address_markers[] = {
        { VMEMMAP_START,                "vmemmap start" },
        { VMEMMAP_START + VMEMMAP_SIZE, "vmemmap end" },
 #endif
-       { PAGE_OFFSET,                  "Linear mapping" },
        { -1,                           NULL },
 };
 
@@ -376,7 +377,7 @@ static void ptdump_initialize(void)
 static struct ptdump_info kernel_ptdump_info = {
        .mm             = &init_mm,
        .markers        = address_markers,
-       .base_addr      = VA_START,
+       .base_addr      = PAGE_OFFSET,
 };
 
 void ptdump_check_wx(void)

diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index f3c795278def0ea01704c6b38fe3d2d140351224..62927ed02229210c3c6347edefa5b13068f99c26 100644 (file)
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -301,7 +301,7 @@ static void __init fdt_enforce_memory_region(void)
 
 void __init arm64_memblock_init(void)
 {
-       const s64 linear_region_size = -(s64)PAGE_OFFSET;
+       const s64 linear_region_size = BIT(VA_BITS - 1);
 
        /* Handle linux,usable-memory-range property */
        fdt_enforce_memory_region();
@@ -309,13 +309,6 @@ void __init arm64_memblock_init(void)
        /* Remove memory above our supported physical address size */
        memblock_remove(1ULL << PHYS_MASK_SHIFT, ULLONG_MAX);
 
-       /*
-        * Ensure that the linear region takes up exactly half of the kernel
-        * virtual address space. This way, we can distinguish a linear address
-        * from a kernel/module/vmalloc address by testing a single bit.
-        */
-       BUILD_BUG_ON(linear_region_size != BIT(VA_BITS - 1));
-
        /*
         * Select a suitable value for the base of physical memory.
         */

diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c
index 6cf97b904ebb4a29c8b7c18462d2799ad9b77872..05edfe9b02e4dbb8399e4f2ff73c937eb9d73e89 100644 (file)
--- a/arch/arm64/mm/kasan_init.c
+++ b/arch/arm64/mm/kasan_init.c
@@ -225,10 +225,10 @@ void __init kasan_init(void)
        kasan_map_populate(kimg_shadow_start, kimg_shadow_end,
                           early_pfn_to_nid(virt_to_pfn(lm_alias(_text))));
 
-       kasan_populate_early_shadow((void *)KASAN_SHADOW_START,
-                                   (void *)mod_shadow_start);
+       kasan_populate_early_shadow(kasan_mem_to_shadow((void *) VA_START),
+                                  (void *)mod_shadow_start);
        kasan_populate_early_shadow((void *)kimg_shadow_end,
-                                   kasan_mem_to_shadow((void *)PAGE_OFFSET));
+                                  (void *)KASAN_SHADOW_END);
 
        if (kimg_shadow_start > mod_shadow_end)
                kasan_populate_early_shadow((void *)mod_shadow_end,

diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 750a69dde39bdd76a425b4f7e4d59a093936c5b6..1d4247f9a4967c14e2a1c6cc09aee7fca79841af 100644 (file)
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -398,7 +398,7 @@ static phys_addr_t pgd_pgtable_alloc(int shift)
 static void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt,
                                  phys_addr_t size, pgprot_t prot)
 {
-       if (virt < VMALLOC_START) {
+       if ((virt >= VA_START) && (virt < VMALLOC_START)) {
                pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
                        &phys, virt);
                return;
@@ -425,7 +425,7 @@ void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
 static void update_mapping_prot(phys_addr_t phys, unsigned long virt,
                                phys_addr_t size, pgprot_t prot)
 {
-       if (virt < VMALLOC_START) {
+       if ((virt >= VA_START) && (virt < VMALLOC_START)) {
                pr_warn("BUG: not updating mapping for %pa at 0x%016lx - outside kernel range\n",
                        &phys, virt);
                return;