arch/x86/xen/xen-asm_64.S

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * Asm versions of Xen pv-ops, suitable for direct use.
   4  *
   5  * We only bother with direct forms (ie, vcpu in pda) of the
   6  * operations here; the indirect forms are better handled in C.
   7  */
   8
   9 #include <asm/errno.h>
  10 #include <asm/percpu.h>
  11 #include <asm/processor-flags.h>
  12 #include <asm/segment.h>
  13 #include <asm/asm-offsets.h>
  14 #include <asm/thread_info.h>
  15
  16 #include <xen/interface/xen.h>
  17
  18 #include <linux/init.h>
  19 #include <linux/linkage.h>
  20
  21 .macro xen_pv_trap name
  22 ENTRY(xen_\name)
  23         pop %rcx
  24         pop %r11
  25         jmp  \name
  26 END(xen_\name)
  27 .endm
  28
  29 xen_pv_trap divide_error
  30 xen_pv_trap debug
  31 xen_pv_trap xendebug
  32 xen_pv_trap int3
  33 xen_pv_trap xenint3
  34 xen_pv_trap xennmi
  35 xen_pv_trap overflow
  36 xen_pv_trap bounds
  37 xen_pv_trap invalid_op
  38 xen_pv_trap device_not_available
  39 xen_pv_trap double_fault
  40 xen_pv_trap coprocessor_segment_overrun
  41 xen_pv_trap invalid_TSS
  42 xen_pv_trap segment_not_present
  43 xen_pv_trap stack_segment
  44 xen_pv_trap general_protection
  45 xen_pv_trap page_fault
  46 xen_pv_trap spurious_interrupt_bug
  47 xen_pv_trap coprocessor_error
  48 xen_pv_trap alignment_check
  49 #ifdef CONFIG_X86_MCE
  50 xen_pv_trap machine_check
  51 #endif /* CONFIG_X86_MCE */
  52 xen_pv_trap simd_coprocessor_error
  53 #ifdef CONFIG_IA32_EMULATION
  54 xen_pv_trap entry_INT80_compat
  55 #endif
  56 xen_pv_trap hypervisor_callback
  57
  58         __INIT
  59 ENTRY(xen_early_idt_handler_array)
  60         i = 0
  61         .rept NUM_EXCEPTION_VECTORS
  62         pop %rcx
  63         pop %r11
  64         jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE
  65         i = i + 1
  66         .fill xen_early_idt_handler_array + i*XEN_EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
  67         .endr
  68 END(xen_early_idt_handler_array)
  69         __FINIT
  70
  71 hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32
  72 /*
  73  * Xen64 iret frame:
  74  *
  75  *      ss
  76  *      rsp
  77  *      rflags
  78  *      cs
  79  *      rip             <-- standard iret frame
  80  *
  81  *      flags
  82  *
  83  *      rcx             }
  84  *      r11             }<-- pushed by hypercall page
  85  * rsp->rax             }
  86  */
  87 ENTRY(xen_iret)
  88         pushq $0
  89         jmp hypercall_iret
  90
  91 ENTRY(xen_sysret64)
  92         /*
  93          * We're already on the usermode stack at this point, but
  94          * still with the kernel gs, so we can easily switch back
  95          */
  96         movq %rsp, PER_CPU_VAR(rsp_scratch)
  97         movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
  98
  99         pushq $__USER_DS
 100         pushq PER_CPU_VAR(rsp_scratch)
 101         pushq %r11
 102         pushq $__USER_CS
 103         pushq %rcx
 104
 105         pushq $VGCF_in_syscall
 106         jmp hypercall_iret
 107
 108 /*
 109  * Xen handles syscall callbacks much like ordinary exceptions, which
 110  * means we have:
 111  * - kernel gs
 112  * - kernel rsp
 113  * - an iret-like stack frame on the stack (including rcx and r11):
 114  *      ss
 115  *      rsp
 116  *      rflags
 117  *      cs
 118  *      rip
 119  *      r11
 120  * rsp->rcx
 121  */
 122
 123 /* Normal 64-bit system call target */
 124 ENTRY(xen_syscall_target)
 125         popq %rcx
 126         popq %r11
 127
 128         /*
 129          * Neither Xen nor the kernel really knows what the old SS and
 130          * CS were.  The kernel expects __USER_DS and __USER_CS, so
 131          * report those values even though Xen will guess its own values.
 132          */
 133         movq $__USER_DS, 4*8(%rsp)
 134         movq $__USER_CS, 1*8(%rsp)
 135
 136         jmp entry_SYSCALL_64_after_hwframe
 137 ENDPROC(xen_syscall_target)
 138
 139 #ifdef CONFIG_IA32_EMULATION
 140
 141 /* 32-bit compat syscall target */
 142 ENTRY(xen_syscall32_target)
 143         popq %rcx
 144         popq %r11
 145
 146         /*
 147          * Neither Xen nor the kernel really knows what the old SS and
 148          * CS were.  The kernel expects __USER32_DS and __USER32_CS, so
 149          * report those values even though Xen will guess its own values.
 150          */
 151         movq $__USER32_DS, 4*8(%rsp)
 152         movq $__USER32_CS, 1*8(%rsp)
 153
 154         jmp entry_SYSCALL_compat_after_hwframe
 155 ENDPROC(xen_syscall32_target)
 156
 157 /* 32-bit compat sysenter target */
 158 ENTRY(xen_sysenter_target)
 159         mov 0*8(%rsp), %rcx
 160         mov 1*8(%rsp), %r11
 161         mov 5*8(%rsp), %rsp
 162         jmp entry_SYSENTER_compat
 163 ENDPROC(xen_sysenter_target)
 164
 165 #else /* !CONFIG_IA32_EMULATION */
 166
 167 ENTRY(xen_syscall32_target)
 168 ENTRY(xen_sysenter_target)
 169         lea 16(%rsp), %rsp      /* strip %rcx, %r11 */
 170         mov $-ENOSYS, %rax
 171         pushq $0
 172         jmp hypercall_iret
 173 ENDPROC(xen_syscall32_target)
 174 ENDPROC(xen_sysenter_target)
 175
 176 #endif  /* CONFIG_IA32_EMULATION */