arch/arm64/kernel/sdei.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 // Copyright (C) 2017 Arm Ltd.
   3 #define pr_fmt(fmt) "sdei: " fmt
   4
   5 #include <linux/arm_sdei.h>
   6 #include <linux/hardirq.h>
   7 #include <linux/irqflags.h>
   8 #include <linux/sched/task_stack.h>
   9 #include <linux/uaccess.h>
  10
  11 #include <asm/alternative.h>
  12 #include <asm/kprobes.h>
  13 #include <asm/ptrace.h>
  14 #include <asm/sysreg.h>
  15 #include <asm/vmap_stack.h>
  16
  17 unsigned long sdei_exit_mode;
  18
  19 /*
  20  * VMAP'd stacks checking for stack overflow on exception using sp as a scratch
  21  * register, meaning SDEI has to switch to its own stack. We need two stacks as
  22  * a critical event may interrupt a normal event that has just taken a
  23  * synchronous exception, and is using sp as scratch register. For a critical
  24  * event interrupting a normal event, we can't reliably tell if we were on the
  25  * sdei stack.
  26  * For now, we allocate stacks when the driver is probed.
  27  */
  28 DECLARE_PER_CPU(unsigned long *, sdei_stack_normal_ptr);
  29 DECLARE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);
  30
  31 #ifdef CONFIG_VMAP_STACK
  32 DEFINE_PER_CPU(unsigned long *, sdei_stack_normal_ptr);
  33 DEFINE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);
  34 #endif
  35
  36 static void _free_sdei_stack(unsigned long * __percpu *ptr, int cpu)
  37 {
  38         unsigned long *p;
  39
  40         p = per_cpu(*ptr, cpu);
  41         if (p) {
  42                 per_cpu(*ptr, cpu) = NULL;
  43                 vfree(p);
  44         }
  45 }
  46
  47 static void free_sdei_stacks(void)
  48 {
  49         int cpu;
  50
  51         for_each_possible_cpu(cpu) {
  52                 _free_sdei_stack(&sdei_stack_normal_ptr, cpu);
  53                 _free_sdei_stack(&sdei_stack_critical_ptr, cpu);
  54         }
  55 }
  56
  57 static int _init_sdei_stack(unsigned long * __percpu *ptr, int cpu)
  58 {
  59         unsigned long *p;
  60
  61         p = arch_alloc_vmap_stack(SDEI_STACK_SIZE, cpu_to_node(cpu));
  62         if (!p)
  63                 return -ENOMEM;
  64         per_cpu(*ptr, cpu) = p;
  65
  66         return 0;
  67 }
  68
  69 static int init_sdei_stacks(void)
  70 {
  71         int cpu;
  72         int err = 0;
  73
  74         for_each_possible_cpu(cpu) {
  75                 err = _init_sdei_stack(&sdei_stack_normal_ptr, cpu);
  76                 if (err)
  77                         break;
  78                 err = _init_sdei_stack(&sdei_stack_critical_ptr, cpu);
  79                 if (err)
  80                         break;
  81         }
  82
  83         if (err)
  84                 free_sdei_stacks();
  85
  86         return err;
  87 }
  88
  89 bool _on_sdei_stack(unsigned long sp)
  90 {
  91         unsigned long low, high;
  92
  93         if (!IS_ENABLED(CONFIG_VMAP_STACK))
  94                 return false;
  95
  96         low = (unsigned long)raw_cpu_read(sdei_stack_critical_ptr);
  97         high = low + SDEI_STACK_SIZE;
  98
  99         if (low <= sp && sp < high)
 100                 return true;
 101
 102         low = (unsigned long)raw_cpu_read(sdei_stack_normal_ptr);
 103         high = low + SDEI_STACK_SIZE;
 104
 105         return (low <= sp && sp < high);
 106 }
 107
 108 unsigned long sdei_arch_get_entry_point(int conduit)
 109 {
 110         /*
 111          * SDEI works between adjacent exception levels. If we booted at EL1 we
 112          * assume a hypervisor is marshalling events. If we booted at EL2 and
 113          * dropped to EL1 because we don't support VHE, then we can't support
 114          * SDEI.
 115          */
 116         if (is_hyp_mode_available() && !is_kernel_in_hyp_mode()) {
 117                 pr_err("Not supported on this hardware/boot configuration\n");
 118                 return 0;
 119         }
 120
 121         if (IS_ENABLED(CONFIG_VMAP_STACK)) {
 122                 if (init_sdei_stacks())
 123                         return 0;
 124         }
 125
 126         sdei_exit_mode = (conduit == CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;
 127         return (unsigned long)__sdei_asm_handler;
 128 }
 129
 130 /*
 131  * __sdei_handler() returns one of:
 132  *  SDEI_EV_HANDLED -  success, return to the interrupted context.
 133  *  SDEI_EV_FAILED  -  failure, return this error code to firmare.
 134  *  virtual-address -  success, return to this address.
 135  */
 136 static __kprobes unsigned long _sdei_handler(struct pt_regs *regs,
 137                                              struct sdei_registered_event *arg)
 138 {
 139         u32 mode;
 140         int i, err = 0;
 141         const int clobbered_registers = 4;
 142         u64 elr = read_sysreg(elr_el1);
 143         u32 kernel_mode = read_sysreg(CurrentEL) | 1;   /* +SPSel */
 144         unsigned long vbar = read_sysreg(vbar_el1);
 145
 146         /* Retrieve the missing registers values */
 147         for (i = 0; i < clobbered_registers; i++) {
 148                 /* from within the handler, this call always succeeds */
 149                 sdei_api_event_context(i, &regs->regs[i]);
 150         }
 151
 152         /*
 153          * We didn't take an exception to get here, set PAN. UAO will be cleared
 154          * by sdei_event_handler()s set_fs(USER_DS) call.
 155          */
 156         __uaccess_enable_hw_pan();
 157
 158         err = sdei_event_handler(regs, arg);
 159         if (err)
 160                 return SDEI_EV_FAILED;
 161
 162         if (elr != read_sysreg(elr_el1)) {
 163                 /*
 164                  * We took a synchronous exception from the SDEI handler.
 165                  * This could deadlock, and if you interrupt KVM it will
 166                  * hyp-panic instead.
 167                  */
 168                 pr_warn("unsafe: exception during handler\n");
 169         }
 170
 171         mode = regs->pstate & (PSR_MODE32_BIT | PSR_MODE_MASK);
 172
 173         /*
 174          * If we interrupted the kernel with interrupts masked, we always go
 175          * back to wherever we came from.
 176          */
 177         if (mode == kernel_mode && !interrupts_enabled(regs))
 178                 return SDEI_EV_HANDLED;
 179
 180         /*
 181          * Otherwise, we pretend this was an IRQ. This lets user space tasks
 182          * receive signals before we return to them, and KVM to invoke it's
 183          * world switch to do the same.
 184          *
 185          * See DDI0487B.a Table D1-7 'Vector offsets from vector table base
 186          * address'.
 187          */
 188         if (mode == kernel_mode)
 189                 return vbar + 0x280;
 190         else if (mode & PSR_MODE32_BIT)
 191                 return vbar + 0x680;
 192
 193         return vbar + 0x480;
 194 }
 195
 196
 197 asmlinkage __kprobes notrace unsigned long
 198 __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
 199 {
 200         unsigned long ret;
 201         bool do_nmi_exit = false;
 202
 203         /*
 204          * nmi_enter() deals with printk() re-entrance and use of RCU when
 205          * RCU believed this CPU was idle. Because critical events can
 206          * interrupt normal events, we may already be in_nmi().
 207          */
 208         if (!in_nmi()) {
 209                 nmi_enter();
 210                 do_nmi_exit = true;
 211         }
 212
 213         ret = _sdei_handler(regs, arg);
 214
 215         if (do_nmi_exit)
 216                 nmi_exit();
 217
 218         return ret;
 219 }