arch/x86/xen/xen-asm_64.S

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * Asm versions of Xen pv-ops, suitable for direct use.
   4  *
   5  * We only bother with direct forms (ie, vcpu in pda) of the
   6  * operations here; the indirect forms are better handled in C.
   7  */
   8
   9 #include <asm/errno.h>
  10 #include <asm/percpu.h>
  11 #include <asm/processor-flags.h>
  12 #include <asm/segment.h>
  13 #include <asm/asm-offsets.h>
  14 #include <asm/thread_info.h>
  15
  16 #include <xen/interface/xen.h>
  17
  18 #include <linux/linkage.h>
  19
  20 .macro xen_pv_trap name
  21 ENTRY(xen_\name)
  22         pop %rcx
  23         pop %r11
  24         jmp  \name
  25 END(xen_\name)
  26 .endm
  27
  28 xen_pv_trap divide_error
  29 xen_pv_trap debug
  30 xen_pv_trap xendebug
  31 xen_pv_trap int3
  32 xen_pv_trap xenint3
  33 xen_pv_trap nmi
  34 xen_pv_trap overflow
  35 xen_pv_trap bounds
  36 xen_pv_trap invalid_op
  37 xen_pv_trap device_not_available
  38 xen_pv_trap double_fault
  39 xen_pv_trap coprocessor_segment_overrun
  40 xen_pv_trap invalid_TSS
  41 xen_pv_trap segment_not_present
  42 xen_pv_trap stack_segment
  43 xen_pv_trap general_protection
  44 xen_pv_trap page_fault
  45 xen_pv_trap spurious_interrupt_bug
  46 xen_pv_trap coprocessor_error
  47 xen_pv_trap alignment_check
  48 #ifdef CONFIG_X86_MCE
  49 xen_pv_trap machine_check
  50 #endif /* CONFIG_X86_MCE */
  51 xen_pv_trap simd_coprocessor_error
  52 #ifdef CONFIG_IA32_EMULATION
  53 xen_pv_trap entry_INT80_compat
  54 #endif
  55 xen_pv_trap hypervisor_callback
  56
  57 hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32
  58 /*
  59  * Xen64 iret frame:
  60  *
  61  *      ss
  62  *      rsp
  63  *      rflags
  64  *      cs
  65  *      rip             <-- standard iret frame
  66  *
  67  *      flags
  68  *
  69  *      rcx             }
  70  *      r11             }<-- pushed by hypercall page
  71  * rsp->rax             }
  72  */
  73 ENTRY(xen_iret)
  74         pushq $0
  75         jmp hypercall_iret
  76
  77 ENTRY(xen_sysret64)
  78         /*
  79          * We're already on the usermode stack at this point, but
  80          * still with the kernel gs, so we can easily switch back
  81          */
  82         movq %rsp, PER_CPU_VAR(rsp_scratch)
  83         movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
  84
  85         pushq $__USER_DS
  86         pushq PER_CPU_VAR(rsp_scratch)
  87         pushq %r11
  88         pushq $__USER_CS
  89         pushq %rcx
  90
  91         pushq $VGCF_in_syscall
  92         jmp hypercall_iret
  93
  94 /*
  95  * Xen handles syscall callbacks much like ordinary exceptions, which
  96  * means we have:
  97  * - kernel gs
  98  * - kernel rsp
  99  * - an iret-like stack frame on the stack (including rcx and r11):
 100  *      ss
 101  *      rsp
 102  *      rflags
 103  *      cs
 104  *      rip
 105  *      r11
 106  * rsp->rcx
 107  */
 108
 109 /* Normal 64-bit system call target */
 110 ENTRY(xen_syscall_target)
 111         popq %rcx
 112         popq %r11
 113
 114         /*
 115          * Neither Xen nor the kernel really knows what the old SS and
 116          * CS were.  The kernel expects __USER_DS and __USER_CS, so
 117          * report those values even though Xen will guess its own values.
 118          */
 119         movq $__USER_DS, 4*8(%rsp)
 120         movq $__USER_CS, 1*8(%rsp)
 121
 122         jmp entry_SYSCALL_64_after_hwframe
 123 ENDPROC(xen_syscall_target)
 124
 125 #ifdef CONFIG_IA32_EMULATION
 126
 127 /* 32-bit compat syscall target */
 128 ENTRY(xen_syscall32_target)
 129         popq %rcx
 130         popq %r11
 131
 132         /*
 133          * Neither Xen nor the kernel really knows what the old SS and
 134          * CS were.  The kernel expects __USER32_DS and __USER32_CS, so
 135          * report those values even though Xen will guess its own values.
 136          */
 137         movq $__USER32_DS, 4*8(%rsp)
 138         movq $__USER32_CS, 1*8(%rsp)
 139
 140         jmp entry_SYSCALL_compat_after_hwframe
 141 ENDPROC(xen_syscall32_target)
 142
 143 /* 32-bit compat sysenter target */
 144 ENTRY(xen_sysenter_target)
 145         mov 0*8(%rsp), %rcx
 146         mov 1*8(%rsp), %r11
 147         mov 5*8(%rsp), %rsp
 148         jmp entry_SYSENTER_compat
 149 ENDPROC(xen_sysenter_target)
 150
 151 #else /* !CONFIG_IA32_EMULATION */
 152
 153 ENTRY(xen_syscall32_target)
 154 ENTRY(xen_sysenter_target)
 155         lea 16(%rsp), %rsp      /* strip %rcx, %r11 */
 156         mov $-ENOSYS, %rax
 157         pushq $0
 158         jmp hypercall_iret
 159 ENDPROC(xen_syscall32_target)
 160 ENDPROC(xen_sysenter_target)
 161
 162 #endif  /* CONFIG_IA32_EMULATION */