patches/kernel/0156-x86-entry-64-Move-the-IST-stacks-into-struct-cpu_ent.patch

   1 From 548bfd4d539c4e13eb86236f8f09596e3663c38b Mon Sep 17 00:00:00 2001
   2 From: Andy Lutomirski <luto@kernel.org>
   3 Date: Mon, 4 Dec 2017 15:07:26 +0100
   4 Subject: [PATCH 156/241] x86/entry/64: Move the IST stacks into struct
   5  cpu_entry_area
   6 MIME-Version: 1.0
   7 Content-Type: text/plain; charset=UTF-8
   8 Content-Transfer-Encoding: 8bit
   9
  10 CVE-2017-5754
  11
  12 The IST stacks are needed when an IST exception occurs and are accessed
  13 before any kernel code at all runs.  Move them into struct cpu_entry_area.
  14
  15 The IST stacks are unlike the rest of cpu_entry_area: they're used even for
  16 entries from kernel mode.  This means that they should be set up before we
  17 load the final IDT.  Move cpu_entry_area setup to trap_init() for the boot
  18 CPU and set it up for all possible CPUs at once in native_smp_prepare_cpus().
  19
  20 Signed-off-by: Andy Lutomirski <luto@kernel.org>
  21 Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
  22 Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
  23 Reviewed-by: Borislav Petkov <bp@suse.de>
  24 Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com>
  25 Cc: Borislav Petkov <bp@alien8.de>
  26 Cc: Borislav Petkov <bpetkov@suse.de>
  27 Cc: Brian Gerst <brgerst@gmail.com>
  28 Cc: Dave Hansen <dave.hansen@intel.com>
  29 Cc: Dave Hansen <dave.hansen@linux.intel.com>
  30 Cc: David Laight <David.Laight@aculab.com>
  31 Cc: Denys Vlasenko <dvlasenk@redhat.com>
  32 Cc: Eduardo Valentin <eduval@amazon.com>
  33 Cc: Greg KH <gregkh@linuxfoundation.org>
  34 Cc: H. Peter Anvin <hpa@zytor.com>
  35 Cc: Josh Poimboeuf <jpoimboe@redhat.com>
  36 Cc: Juergen Gross <jgross@suse.com>
  37 Cc: Linus Torvalds <torvalds@linux-foundation.org>
  38 Cc: Peter Zijlstra <peterz@infradead.org>
  39 Cc: Rik van Riel <riel@redhat.com>
  40 Cc: Will Deacon <will.deacon@arm.com>
  41 Cc: aliguori@amazon.com
  42 Cc: daniel.gruss@iaik.tugraz.at
  43 Cc: hughd@google.com
  44 Cc: keescook@google.com
  45 Link: https://lkml.kernel.org/r/20171204150606.480598743@linutronix.de
  46 Signed-off-by: Ingo Molnar <mingo@kernel.org>
  47 (backported from commit 40e7f949e0d9a33968ebde5d67f7e3a47c97742a)
  48 Signed-off-by: Andy Whitcroft <apw@canonical.com>
  49 Signed-off-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com>
  50 (cherry picked from commit 88e7277709f2e7c023e66ff9ae158aeff4cf7c8f)
  51 Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
  52 ---
  53  arch/x86/include/asm/fixmap.h | 12 +++++++
  54  arch/x86/kernel/cpu/common.c  | 74 ++++++++++++++++++++++++-------------------
  55  arch/x86/kernel/traps.c       |  3 ++
  56  3 files changed, 57 insertions(+), 32 deletions(-)
  57
  58 diff --git a/arch/x86/include/asm/fixmap.h b/arch/x86/include/asm/fixmap.h
  59 index 189d12d8afe0..953aed54cb5e 100644
  60 --- a/arch/x86/include/asm/fixmap.h
  61 +++ b/arch/x86/include/asm/fixmap.h
  62 @@ -63,10 +63,22 @@ struct cpu_entry_area {
  63         struct tss_struct tss;
  64
  65         char entry_trampoline[PAGE_SIZE];
  66 +
  67 +#ifdef CONFIG_X86_64
  68 +       /*
  69 +        * Exception stacks used for IST entries.
  70 +        *
  71 +        * In the future, this should have a separate slot for each stack
  72 +        * with guard pages between them.
  73 +        */
  74 +       char exception_stacks[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ];
  75 +#endif
  76  };
  77
  78  #define CPU_ENTRY_AREA_PAGES (sizeof(struct cpu_entry_area) / PAGE_SIZE)
  79
  80 +extern void setup_cpu_entry_areas(void);
  81 +
  82  /*
  83   * Here we define all the compile-time 'special' virtual
  84   * addresses. The point is to have a constant address at
  85 diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
  86 index c2b2ee73b8a1..f487766855d3 100644
  87 --- a/arch/x86/kernel/cpu/common.c
  88 +++ b/arch/x86/kernel/cpu/common.c
  89 @@ -466,24 +466,36 @@ void load_percpu_segment(int cpu)
  90         load_stack_canary_segment();
  91  }
  92
  93 -static void set_percpu_fixmap_pages(int fixmap_index, void *ptr,
  94 -                                   int pages, pgprot_t prot)
  95 -{
  96 -       int i;
  97 -
  98 -       for (i = 0; i < pages; i++) {
  99 -               __set_fixmap(fixmap_index - i,
 100 -                            per_cpu_ptr_to_phys(ptr + i * PAGE_SIZE), prot);
 101 -       }
 102 -}
 103 -
 104  #ifdef CONFIG_X86_32
 105  /* The 32-bit entry code needs to find cpu_entry_area. */
 106  DEFINE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
 107  #endif
 108
 109 +#ifdef CONFIG_X86_64
 110 +/*
 111 + * Special IST stacks which the CPU switches to when it calls
 112 + * an IST-marked descriptor entry. Up to 7 stacks (hardware
 113 + * limit), all of them are 4K, except the debug stack which
 114 + * is 8K.
 115 + */
 116 +static const unsigned int exception_stack_sizes[N_EXCEPTION_STACKS] = {
 117 +         [0 ... N_EXCEPTION_STACKS - 1]        = EXCEPTION_STKSZ,
 118 +         [DEBUG_STACK - 1]                     = DEBUG_STKSZ
 119 +};
 120 +
 121 +static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks
 122 +       [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]);
 123 +#endif
 124 +
 125 +static void __init
 126 +set_percpu_fixmap_pages(int idx, void *ptr, int pages, pgprot_t prot)
 127 +{
 128 +       for ( ; pages; pages--, idx--, ptr += PAGE_SIZE)
 129 +               __set_fixmap(idx, per_cpu_ptr_to_phys(ptr), prot);
 130 +}
 131 +
 132  /* Setup the fixmap mappings only once per-processor */
 133 -static inline void setup_cpu_entry_area(int cpu)
 134 +static void __init setup_cpu_entry_area(int cpu)
 135  {
 136  #ifdef CONFIG_X86_64
 137         extern char _entry_trampoline[];
 138 @@ -532,15 +544,31 @@ static inline void setup_cpu_entry_area(int cpu)
 139                                 PAGE_KERNEL);
 140
 141  #ifdef CONFIG_X86_32
 142 -       this_cpu_write(cpu_entry_area, get_cpu_entry_area(cpu));
 143 +       per_cpu(cpu_entry_area, cpu) = get_cpu_entry_area(cpu);
 144  #endif
 145
 146  #ifdef CONFIG_X86_64
 147 +       BUILD_BUG_ON(sizeof(exception_stacks) % PAGE_SIZE != 0);
 148 +       BUILD_BUG_ON(sizeof(exception_stacks) !=
 149 +                    sizeof(((struct cpu_entry_area *)0)->exception_stacks));
 150 +       set_percpu_fixmap_pages(get_cpu_entry_area_index(cpu, exception_stacks),
 151 +                               &per_cpu(exception_stacks, cpu),
 152 +                               sizeof(exception_stacks) / PAGE_SIZE,
 153 +                               PAGE_KERNEL);
 154 +
 155         __set_fixmap(get_cpu_entry_area_index(cpu, entry_trampoline),
 156                      __pa_symbol(_entry_trampoline), PAGE_KERNEL_RX);
 157  #endif
 158  }
 159
 160 +void __init setup_cpu_entry_areas(void)
 161 +{
 162 +       unsigned int cpu;
 163 +
 164 +       for_each_possible_cpu(cpu)
 165 +               setup_cpu_entry_area(cpu);
 166 +}
 167 +
 168  /* Load the original GDT from the per-cpu structure */
 169  void load_direct_gdt(int cpu)
 170  {
 171 @@ -1386,20 +1414,6 @@ DEFINE_PER_CPU(unsigned int, irq_count) __visible = -1;
 172  DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
 173  EXPORT_PER_CPU_SYMBOL(__preempt_count);
 174
 175 -/*
 176 - * Special IST stacks which the CPU switches to when it calls
 177 - * an IST-marked descriptor entry. Up to 7 stacks (hardware
 178 - * limit), all of them are 4K, except the debug stack which
 179 - * is 8K.
 180 - */
 181 -static const unsigned int exception_stack_sizes[N_EXCEPTION_STACKS] = {
 182 -         [0 ... N_EXCEPTION_STACKS - 1]        = EXCEPTION_STKSZ,
 183 -         [DEBUG_STACK - 1]                     = DEBUG_STKSZ
 184 -};
 185 -
 186 -static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks
 187 -       [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]);
 188 -
 189  /* May not be marked __init: used by software suspend */
 190  void syscall_init(void)
 191  {
 192 @@ -1608,7 +1622,7 @@ void cpu_init(void)
 193          * set up and load the per-CPU TSS
 194          */
 195         if (!oist->ist[0]) {
 196 -               char *estacks = per_cpu(exception_stacks, cpu);
 197 +               char *estacks = get_cpu_entry_area(cpu)->exception_stacks;
 198
 199                 for (v = 0; v < N_EXCEPTION_STACKS; v++) {
 200                         estacks += exception_stack_sizes[v];
 201 @@ -1633,8 +1647,6 @@ void cpu_init(void)
 202         BUG_ON(me->mm);
 203         enter_lazy_tlb(&init_mm, me);
 204
 205 -       setup_cpu_entry_area(cpu);
 206 -
 207         /*
 208          * Initialize the TSS.  sp0 points to the entry trampoline stack
 209          * regardless of what task is running.
 210 @@ -1693,8 +1705,6 @@ void cpu_init(void)
 211         BUG_ON(curr->mm);
 212         enter_lazy_tlb(&init_mm, curr);
 213
 214 -       setup_cpu_entry_area(cpu);
 215 -
 216         /*
 217          * Initialize the TSS.  Don't bother initializing sp0, as the initial
 218          * task never enters user mode.
 219 diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
 220 index d9debdafe7a6..fd4d47e8672e 100644
 221 --- a/arch/x86/kernel/traps.c
 222 +++ b/arch/x86/kernel/traps.c
 223 @@ -992,6 +992,9 @@ void __init trap_init(void)
 224  {
 225         int i;
 226
 227 +       /* Init cpu_entry_area before IST entries are set up */
 228 +       setup_cpu_entry_areas();
 229 +
 230  #ifdef CONFIG_EISA
 231         void __iomem *p = early_ioremap(0x0FFFD9, 4);
 232
 233 --
 234 2.14.2
 235