]>
Commit | Line | Data |
---|---|---|
1 | /* | |
2 | * Asm versions of Xen pv-ops, suitable for either direct use or | |
3 | * inlining. The inline versions are the same as the direct-use | |
4 | * versions, with the pre- and post-amble chopped off. | |
5 | * | |
6 | * This code is encoded for size rather than absolute efficiency, with | |
7 | * a view to being able to inline as much as possible. | |
8 | * | |
9 | * We only bother with direct forms (ie, vcpu in pda) of the | |
10 | * operations here; the indirect forms are better handled in C, since | |
11 | * they're generally too large to inline anyway. | |
12 | */ | |
13 | ||
14 | #include <asm/errno.h> | |
15 | #include <asm/percpu.h> | |
16 | #include <asm/processor-flags.h> | |
17 | #include <asm/segment.h> | |
18 | #include <asm/asm-offsets.h> | |
19 | #include <asm/thread_info.h> | |
20 | ||
21 | #include <xen/interface/xen.h> | |
22 | ||
23 | #include "xen-asm.h" | |
24 | ||
25 | ENTRY(xen_adjust_exception_frame) | |
26 | mov 8+0(%rsp), %rcx | |
27 | mov 8+8(%rsp), %r11 | |
28 | ret $16 | |
29 | ||
30 | hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32 | |
31 | /* | |
32 | * Xen64 iret frame: | |
33 | * | |
34 | * ss | |
35 | * rsp | |
36 | * rflags | |
37 | * cs | |
38 | * rip <-- standard iret frame | |
39 | * | |
40 | * flags | |
41 | * | |
42 | * rcx } | |
43 | * r11 }<-- pushed by hypercall page | |
44 | * rsp->rax } | |
45 | */ | |
46 | ENTRY(xen_iret) | |
47 | pushq $0 | |
48 | 1: jmp hypercall_iret | |
49 | ENDPATCH(xen_iret) | |
50 | RELOC(xen_iret, 1b+1) | |
51 | ||
52 | ENTRY(xen_sysret64) | |
53 | /* | |
54 | * We're already on the usermode stack at this point, but | |
55 | * still with the kernel gs, so we can easily switch back | |
56 | */ | |
57 | movq %rsp, PER_CPU_VAR(rsp_scratch) | |
58 | movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp | |
59 | ||
60 | pushq $__USER_DS | |
61 | pushq PER_CPU_VAR(rsp_scratch) | |
62 | pushq %r11 | |
63 | pushq $__USER_CS | |
64 | pushq %rcx | |
65 | ||
66 | pushq $VGCF_in_syscall | |
67 | 1: jmp hypercall_iret | |
68 | ENDPATCH(xen_sysret64) | |
69 | RELOC(xen_sysret64, 1b+1) | |
70 | ||
71 | ENTRY(xen_sysret32) | |
72 | /* | |
73 | * We're already on the usermode stack at this point, but | |
74 | * still with the kernel gs, so we can easily switch back | |
75 | */ | |
76 | movq %rsp, PER_CPU_VAR(rsp_scratch) | |
77 | movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp | |
78 | ||
79 | pushq $__USER32_DS | |
80 | pushq PER_CPU_VAR(rsp_scratch) | |
81 | pushq %r11 | |
82 | pushq $__USER32_CS | |
83 | pushq %rcx | |
84 | ||
85 | pushq $0 | |
86 | 1: jmp hypercall_iret | |
87 | ENDPATCH(xen_sysret32) | |
88 | RELOC(xen_sysret32, 1b+1) | |
89 | ||
90 | /* | |
91 | * Xen handles syscall callbacks much like ordinary exceptions, which | |
92 | * means we have: | |
93 | * - kernel gs | |
94 | * - kernel rsp | |
95 | * - an iret-like stack frame on the stack (including rcx and r11): | |
96 | * ss | |
97 | * rsp | |
98 | * rflags | |
99 | * cs | |
100 | * rip | |
101 | * r11 | |
102 | * rsp->rcx | |
103 | * | |
104 | * In all the entrypoints, we undo all that to make it look like a | |
105 | * CPU-generated syscall/sysenter and jump to the normal entrypoint. | |
106 | */ | |
107 | ||
108 | .macro undo_xen_syscall | |
109 | mov 0*8(%rsp), %rcx | |
110 | mov 1*8(%rsp), %r11 | |
111 | mov 5*8(%rsp), %rsp | |
112 | .endm | |
113 | ||
114 | /* Normal 64-bit system call target */ | |
115 | ENTRY(xen_syscall_target) | |
116 | undo_xen_syscall | |
117 | jmp system_call_after_swapgs | |
118 | ENDPROC(xen_syscall_target) | |
119 | ||
120 | #ifdef CONFIG_IA32_EMULATION | |
121 | ||
122 | /* 32-bit compat syscall target */ | |
123 | ENTRY(xen_syscall32_target) | |
124 | undo_xen_syscall | |
125 | jmp ia32_cstar_target | |
126 | ENDPROC(xen_syscall32_target) | |
127 | ||
128 | /* 32-bit compat sysenter target */ | |
129 | ENTRY(xen_sysenter_target) | |
130 | undo_xen_syscall | |
131 | jmp ia32_sysenter_target | |
132 | ENDPROC(xen_sysenter_target) | |
133 | ||
134 | #else /* !CONFIG_IA32_EMULATION */ | |
135 | ||
136 | ENTRY(xen_syscall32_target) | |
137 | ENTRY(xen_sysenter_target) | |
138 | lea 16(%rsp), %rsp /* strip %rcx, %r11 */ | |
139 | mov $-ENOSYS, %rax | |
140 | pushq $0 | |
141 | jmp hypercall_iret | |
142 | ENDPROC(xen_syscall32_target) | |
143 | ENDPROC(xen_sysenter_target) | |
144 | ||
145 | #endif /* CONFIG_IA32_EMULATION */ |