patches/kernel/0210-x86-mm-Optimize-RESTORE_CR3.patch

   1 From eb4a670cb54266bfab7bb4d9fd9e5da7b296ecdf Mon Sep 17 00:00:00 2001
   2 From: Peter Zijlstra <peterz@infradead.org>
   3 Date: Mon, 4 Dec 2017 15:08:00 +0100
   4 Subject: [PATCH 210/241] x86/mm: Optimize RESTORE_CR3
   5 MIME-Version: 1.0
   6 Content-Type: text/plain; charset=UTF-8
   7 Content-Transfer-Encoding: 8bit
   8
   9 CVE-2017-5754
  10
  11 Most NMI/paranoid exceptions will not in fact change pagetables and would
  12 thus not require TLB flushing, however RESTORE_CR3 uses flushing CR3
  13 writes.
  14
  15 Restores to kernel PCIDs can be NOFLUSH, because we explicitly flush the
  16 kernel mappings and now that we track which user PCIDs need flushing we can
  17 avoid those too when possible.
  18
  19 This does mean RESTORE_CR3 needs an additional scratch_reg, luckily both
  20 sites have plenty available.
  21
  22 Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
  23 Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
  24 Cc: Andy Lutomirski <luto@kernel.org>
  25 Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com>
  26 Cc: Borislav Petkov <bp@alien8.de>
  27 Cc: Brian Gerst <brgerst@gmail.com>
  28 Cc: Dave Hansen <dave.hansen@linux.intel.com>
  29 Cc: David Laight <David.Laight@aculab.com>
  30 Cc: Denys Vlasenko <dvlasenk@redhat.com>
  31 Cc: Eduardo Valentin <eduval@amazon.com>
  32 Cc: Greg KH <gregkh@linuxfoundation.org>
  33 Cc: H. Peter Anvin <hpa@zytor.com>
  34 Cc: Josh Poimboeuf <jpoimboe@redhat.com>
  35 Cc: Juergen Gross <jgross@suse.com>
  36 Cc: Linus Torvalds <torvalds@linux-foundation.org>
  37 Cc: Peter Zijlstra <peterz@infradead.org>
  38 Cc: Will Deacon <will.deacon@arm.com>
  39 Cc: aliguori@amazon.com
  40 Cc: daniel.gruss@iaik.tugraz.at
  41 Cc: hughd@google.com
  42 Cc: keescook@google.com
  43 Signed-off-by: Ingo Molnar <mingo@kernel.org>
  44 (cherry picked from commit 21e94459110252d41b45c0c8ba50fd72a664d50c)
  45 Signed-off-by: Andy Whitcroft <apw@canonical.com>
  46 Signed-off-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com>
  47 (cherry picked from commit 6ebe6e2896841282357d43c09394b0ca47c41e4a)
  48 Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
  49 ---
  50  arch/x86/entry/calling.h  | 30 ++++++++++++++++++++++++++++--
  51  arch/x86/entry/entry_64.S |  4 ++--
  52  2 files changed, 30 insertions(+), 4 deletions(-)
  53
  54 diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h
  55 index ce5fb309926d..015e0a84bb99 100644
  56 --- a/arch/x86/entry/calling.h
  57 +++ b/arch/x86/entry/calling.h
  58 @@ -280,8 +280,34 @@ For 32-bit we have the following conventions - kernel is built with
  59  .Ldone_\@:
  60  .endm
  61
  62 -.macro RESTORE_CR3 save_reg:req
  63 +.macro RESTORE_CR3 scratch_reg:req save_reg:req
  64         ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
  65 +
  66 +       ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID
  67 +
  68 +       /*
  69 +        * KERNEL pages can always resume with NOFLUSH as we do
  70 +        * explicit flushes.
  71 +        */
  72 +       bt      $X86_CR3_PTI_SWITCH_BIT, \save_reg
  73 +       jnc     .Lnoflush_\@
  74 +
  75 +       /*
  76 +        * Check if there's a pending flush for the user ASID we're
  77 +        * about to set.
  78 +        */
  79 +       movq    \save_reg, \scratch_reg
  80 +       andq    $(0x7FF), \scratch_reg
  81 +       bt      \scratch_reg, THIS_CPU_user_pcid_flush_mask
  82 +       jnc     .Lnoflush_\@
  83 +
  84 +       btr     \scratch_reg, THIS_CPU_user_pcid_flush_mask
  85 +       jmp     .Lwrcr3_\@
  86 +
  87 +.Lnoflush_\@:
  88 +       SET_NOFLUSH_BIT \save_reg
  89 +
  90 +.Lwrcr3_\@:
  91         /*
  92          * The CR3 write could be avoided when not changing its value,
  93          * but would require a CR3 read *and* a scratch register.
  94 @@ -300,7 +326,7 @@ For 32-bit we have the following conventions - kernel is built with
  95  .endm
  96  .macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req
  97  .endm
  98 -.macro RESTORE_CR3 save_reg:req
  99 +.macro RESTORE_CR3 scratch_reg:req save_reg:req
 100  .endm
 101
 102  #endif
 103 diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
 104 index fb43f14ed299..b48f2c78a9bf 100644
 105 --- a/arch/x86/entry/entry_64.S
 106 +++ b/arch/x86/entry/entry_64.S
 107 @@ -1300,7 +1300,7 @@ ENTRY(paranoid_exit)
 108         testl   %ebx, %ebx                      /* swapgs needed? */
 109         jnz     .Lparanoid_exit_no_swapgs
 110         TRACE_IRQS_IRETQ
 111 -       RESTORE_CR3     save_reg=%r14
 112 +       RESTORE_CR3     scratch_reg=%rbx save_reg=%r14
 113         SWAPGS_UNSAFE_STACK
 114         jmp     .Lparanoid_exit_restore
 115  .Lparanoid_exit_no_swapgs:
 116 @@ -1742,7 +1742,7 @@ end_repeat_nmi:
 117         movq    $-1, %rsi
 118         call    do_nmi
 119
 120 -       RESTORE_CR3 save_reg=%r14
 121 +       RESTORE_CR3 scratch_reg=%r15 save_reg=%r14
 122
 123         testl   %ebx, %ebx                      /* swapgs needed? */
 124         jnz     nmi_restore
 125 --
 126 2.14.2
 127