patches/kernel/0179-x86-mm-Move-the-CR3-construction-functions-to-tlbflu.patch

   1 From f978416e1df8d655e6ac7ae848928441cf33d598 Mon Sep 17 00:00:00 2001
   2 From: Dave Hansen <dave.hansen@linux.intel.com>
   3 Date: Mon, 4 Dec 2017 15:07:54 +0100
   4 Subject: [PATCH 179/233] x86/mm: Move the CR3 construction functions to
   5  tlbflush.h
   6 MIME-Version: 1.0
   7 Content-Type: text/plain; charset=UTF-8
   8 Content-Transfer-Encoding: 8bit
   9
  10 CVE-2017-5754
  11
  12 For flushing the TLB, the ASID which has been programmed into the hardware
  13 must be known.  That differs from what is in 'cpu_tlbstate'.
  14
  15 Add functions to transform the 'cpu_tlbstate' values into to the one
  16 programmed into the hardware (CR3).
  17
  18 It's not easy to include mmu_context.h into tlbflush.h, so just move the
  19 CR3 building over to tlbflush.h.
  20
  21 Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
  22 Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
  23 Cc: Andy Lutomirski <luto@kernel.org>
  24 Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com>
  25 Cc: Borislav Petkov <bp@alien8.de>
  26 Cc: Brian Gerst <brgerst@gmail.com>
  27 Cc: David Laight <David.Laight@aculab.com>
  28 Cc: Denys Vlasenko <dvlasenk@redhat.com>
  29 Cc: Eduardo Valentin <eduval@amazon.com>
  30 Cc: Greg KH <gregkh@linuxfoundation.org>
  31 Cc: H. Peter Anvin <hpa@zytor.com>
  32 Cc: Josh Poimboeuf <jpoimboe@redhat.com>
  33 Cc: Juergen Gross <jgross@suse.com>
  34 Cc: Linus Torvalds <torvalds@linux-foundation.org>
  35 Cc: Peter Zijlstra <peterz@infradead.org>
  36 Cc: Will Deacon <will.deacon@arm.com>
  37 Cc: aliguori@amazon.com
  38 Cc: daniel.gruss@iaik.tugraz.at
  39 Cc: hughd@google.com
  40 Cc: keescook@google.com
  41 Cc: linux-mm@kvack.org
  42 Signed-off-by: Ingo Molnar <mingo@kernel.org>
  43 (cherry picked from commit 50fb83a62cf472dc53ba23bd3f7bd6c1b2b3b53e)
  44 Signed-off-by: Andy Whitcroft <apw@canonical.com>
  45 Signed-off-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com>
  46 (cherry picked from commit f741923acf51c1061c11b45a168f8864d37dc5cd)
  47 Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
  48 ---
  49  arch/x86/include/asm/mmu_context.h | 29 +----------------------------
  50  arch/x86/include/asm/tlbflush.h    | 26 ++++++++++++++++++++++++++
  51  arch/x86/mm/tlb.c                  |  8 ++++----
  52  3 files changed, 31 insertions(+), 32 deletions(-)
  53
  54 diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
  55 index 47ec51a821e8..89a01ad7e370 100644
  56 --- a/arch/x86/include/asm/mmu_context.h
  57 +++ b/arch/x86/include/asm/mmu_context.h
  58 @@ -289,33 +289,6 @@ static inline bool arch_vma_access_permitted(struct vm_area_struct *vma,
  59         return __pkru_allows_pkey(vma_pkey(vma), write);
  60  }
  61
  62 -/*
  63 - * If PCID is on, ASID-aware code paths put the ASID+1 into the PCID
  64 - * bits.  This serves two purposes.  It prevents a nasty situation in
  65 - * which PCID-unaware code saves CR3, loads some other value (with PCID
  66 - * == 0), and then restores CR3, thus corrupting the TLB for ASID 0 if
  67 - * the saved ASID was nonzero.  It also means that any bugs involving
  68 - * loading a PCID-enabled CR3 with CR4.PCIDE off will trigger
  69 - * deterministically.
  70 - */
  71 -
  72 -static inline unsigned long build_cr3(struct mm_struct *mm, u16 asid)
  73 -{
  74 -       if (static_cpu_has(X86_FEATURE_PCID)) {
  75 -               VM_WARN_ON_ONCE(asid > 4094);
  76 -               return __sme_pa(mm->pgd) | (asid + 1);
  77 -       } else {
  78 -               VM_WARN_ON_ONCE(asid != 0);
  79 -               return __sme_pa(mm->pgd);
  80 -       }
  81 -}
  82 -
  83 -static inline unsigned long build_cr3_noflush(struct mm_struct *mm, u16 asid)
  84 -{
  85 -       VM_WARN_ON_ONCE(asid > 4094);
  86 -       return __sme_pa(mm->pgd) | (asid + 1) | CR3_NOFLUSH;
  87 -}
  88 -
  89  /*
  90   * This can be used from process context to figure out what the value of
  91   * CR3 is without needing to do a (slow) __read_cr3().
  92 @@ -325,7 +298,7 @@ static inline unsigned long build_cr3_noflush(struct mm_struct *mm, u16 asid)
  93   */
  94  static inline unsigned long __get_current_cr3_fast(void)
  95  {
  96 -       unsigned long cr3 = build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm),
  97 +       unsigned long cr3 = build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd,
  98                 this_cpu_read(cpu_tlbstate.loaded_mm_asid));
  99
 100         /* For now, be very restrictive about when this can be called. */
 101 diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
 102 index ed5d483c4a1b..3a421b164868 100644
 103 --- a/arch/x86/include/asm/tlbflush.h
 104 +++ b/arch/x86/include/asm/tlbflush.h
 105 @@ -68,6 +68,32 @@ static inline u64 inc_mm_tlb_gen(struct mm_struct *mm)
 106         return atomic64_inc_return(&mm->context.tlb_gen);
 107  }
 108
 109 +/*
 110 + * If PCID is on, ASID-aware code paths put the ASID+1 into the PCID bits.
 111 + * This serves two purposes.  It prevents a nasty situation in which
 112 + * PCID-unaware code saves CR3, loads some other value (with PCID == 0),
 113 + * and then restores CR3, thus corrupting the TLB for ASID 0 if the saved
 114 + * ASID was nonzero.  It also means that any bugs involving loading a
 115 + * PCID-enabled CR3 with CR4.PCIDE off will trigger deterministically.
 116 + */
 117 +struct pgd_t;
 118 +static inline unsigned long build_cr3(pgd_t *pgd, u16 asid)
 119 +{
 120 +       if (static_cpu_has(X86_FEATURE_PCID)) {
 121 +               VM_WARN_ON_ONCE(asid > 4094);
 122 +               return __sme_pa(pgd) | (asid + 1);
 123 +       } else {
 124 +               VM_WARN_ON_ONCE(asid != 0);
 125 +               return __sme_pa(pgd);
 126 +       }
 127 +}
 128 +
 129 +static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid)
 130 +{
 131 +       VM_WARN_ON_ONCE(asid > 4094);
 132 +       return __sme_pa(pgd) | (asid + 1) | CR3_NOFLUSH;
 133 +}
 134 +
 135  #ifdef CONFIG_PARAVIRT
 136  #include <asm/paravirt.h>
 137  #else
 138 diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
 139 index 5b4342c5039c..87d4f961bcb4 100644
 140 --- a/arch/x86/mm/tlb.c
 141 +++ b/arch/x86/mm/tlb.c
 142 @@ -126,7 +126,7 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
 143          * does something like write_cr3(read_cr3_pa()).
 144          */
 145  #ifdef CONFIG_DEBUG_VM
 146 -       if (WARN_ON_ONCE(__read_cr3() != build_cr3(real_prev, prev_asid))) {
 147 +       if (WARN_ON_ONCE(__read_cr3() != build_cr3(real_prev->pgd, prev_asid))) {
 148                 /*
 149                  * If we were to BUG here, we'd be very likely to kill
 150                  * the system so hard that we don't see the call trace.
 151 @@ -193,7 +193,7 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
 152                 if (need_flush) {
 153                         this_cpu_write(cpu_tlbstate.ctxs[new_asid].ctx_id, next->context.ctx_id);
 154                         this_cpu_write(cpu_tlbstate.ctxs[new_asid].tlb_gen, next_tlb_gen);
 155 -                       write_cr3(build_cr3(next, new_asid));
 156 +                       write_cr3(build_cr3(next->pgd, new_asid));
 157
 158                         /*
 159                          * NB: This gets called via leave_mm() in the idle path
 160 @@ -206,7 +206,7 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
 161                         trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
 162                 } else {
 163                         /* The new ASID is already up to date. */
 164 -                       write_cr3(build_cr3_noflush(next, new_asid));
 165 +                       write_cr3(build_cr3_noflush(next->pgd, new_asid));
 166
 167                         /* See above wrt _rcuidle. */
 168                         trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH, 0);
 169 @@ -283,7 +283,7 @@ void initialize_tlbstate_and_flush(void)
 170                 !(cr4_read_shadow() & X86_CR4_PCIDE));
 171
 172         /* Force ASID 0 and force a TLB flush. */
 173 -       write_cr3(build_cr3(mm, 0));
 174 +       write_cr3(build_cr3(mm->pgd, 0));
 175
 176         /* Reinitialize tlbstate. */
 177         this_cpu_write(cpu_tlbstate.loaded_mm_asid, 0);
 178 --
 179 2.14.2
 180