arm64: Add skeleton to harden the branch predictor against aliasing attacks

[mirror_ubuntu-artful-kernel.git] / arch / arm64 / mm / context.c

diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c
index ab9f5f0fb2c7fc6ada0605e31d8f73b0024ae277..abc56dc31ae06e0baa27e305b4a1f2077d746501 100644 (file)
--- a/arch/arm64/mm/context.c
+++ b/arch/arm64/mm/context.c
@@ -39,7 +39,16 @@ static cpumask_t tlb_flush_pending;
 
 #define ASID_MASK              (~GENMASK(asid_bits - 1, 0))
 #define ASID_FIRST_VERSION     (1UL << asid_bits)
-#define NUM_USER_ASIDS         ASID_FIRST_VERSION
+
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+#define NUM_USER_ASIDS         (ASID_FIRST_VERSION >> 1)
+#define asid2idx(asid)         (((asid) & ~ASID_MASK) >> 1)
+#define idx2asid(idx)          (((idx) << 1) & ~ASID_MASK)
+#else
+#define NUM_USER_ASIDS         (ASID_FIRST_VERSION)
+#define asid2idx(asid)         ((asid) & ~ASID_MASK)
+#define idx2asid(idx)          asid2idx(idx)
+#endif
 
 /* Get the ASIDBits supported by the current CPU */
 static u32 get_cpu_asid_bits(void)
@@ -79,13 +88,6 @@ void verify_cpu_asid_bits(void)
        }
 }
 
-static void set_reserved_asid_bits(void)
-{
-       if (IS_ENABLED(CONFIG_QCOM_FALKOR_ERRATUM_1003) &&
-           cpus_have_const_cap(ARM64_WORKAROUND_QCOM_FALKOR_E1003))
-               __set_bit(FALKOR_RESERVED_ASID, asid_map);
-}
-
 static void flush_context(unsigned int cpu)
 {
        int i;
@@ -94,8 +96,6 @@ static void flush_context(unsigned int cpu)
        /* Update the list of reserved ASIDs and the ASID bitmap. */
        bitmap_clear(asid_map, 0, NUM_USER_ASIDS);
 
-       set_reserved_asid_bits();
-
        /*
         * Ensure the generation bump is observed before we xchg the
         * active_asids.
@@ -113,7 +113,7 @@ static void flush_context(unsigned int cpu)
                 */
                if (asid == 0)
                        asid = per_cpu(reserved_asids, i);
-               __set_bit(asid & ~ASID_MASK, asid_map);
+               __set_bit(asid2idx(asid), asid_map);
                per_cpu(reserved_asids, i) = asid;
        }
 
@@ -165,16 +165,16 @@ static u64 new_context(struct mm_struct *mm, unsigned int cpu)
                 * We had a valid ASID in a previous life, so try to re-use
                 * it if possible.
                 */
-               asid &= ~ASID_MASK;
-               if (!__test_and_set_bit(asid, asid_map))
+               if (!__test_and_set_bit(asid2idx(asid), asid_map))
                        return newasid;
        }
 
        /*
         * Allocate a free ASID. If we can't find one, take a note of the
-        * currently active ASIDs and mark the TLBs as requiring flushes.
-        * We always count from ASID #1, as we use ASID #0 when setting a
-        * reserved TTBR0 for the init_mm.
+        * currently active ASIDs and mark the TLBs as requiring flushes.  We
+        * always count from ASID #2 (index 1), as we use ASID #0 when setting
+        * a reserved TTBR0 for the init_mm and we allocate ASIDs in even/odd
+        * pairs.
         */
        asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, cur_idx);
        if (asid != NUM_USER_ASIDS)
@@ -191,7 +191,7 @@ static u64 new_context(struct mm_struct *mm, unsigned int cpu)
 set_asid:
        __set_bit(asid, asid_map);
        cur_idx = asid;
-       return asid | generation;
+       return idx2asid(asid) | generation;
 }
 
 void check_and_switch_context(struct mm_struct *mm, unsigned int cpu)
@@ -235,6 +235,17 @@ switch_mm_fastpath:
                cpu_switch_mm(mm->pgd, mm);
 }
 
+/* Errata workaround post TTBRx_EL1 update. */
+asmlinkage void post_ttbr_update_workaround(void)
+{
+       asm(ALTERNATIVE("nop; nop; nop",
+                       "ic iallu; dsb nsh; isb",
+                       ARM64_WORKAROUND_CAVIUM_27456,
+                       CONFIG_CAVIUM_ERRATUM_27456));
+
+       arm64_apply_bp_hardening();
+}
+
 static int asids_init(void)
 {
        asid_bits = get_cpu_asid_bits();
@@ -250,8 +261,6 @@ static int asids_init(void)
                panic("Failed to allocate bitmap for %lu ASIDs\n",
                      NUM_USER_ASIDS);
 
-       set_reserved_asid_bits();
-
        pr_info("ASID allocator initialised with %lu entries\n", NUM_USER_ASIDS);
        return 0;
 }