1 // SPDX-License-Identifier: GPL-2.0-only
3 * common.c - C code for kernel entry and exit
4 * Copyright (c) 2015 Andrew Lutomirski
6 * Based on asm and ptrace code by many authors. The code here originated
7 * in ptrace.c and signal.c.
10 #include <linux/kernel.h>
11 #include <linux/sched.h>
12 #include <linux/sched/task_stack.h>
13 #include <linux/entry-common.h>
15 #include <linux/smp.h>
16 #include <linux/errno.h>
17 #include <linux/ptrace.h>
18 #include <linux/export.h>
19 #include <linux/nospec.h>
20 #include <linux/syscalls.h>
21 #include <linux/uaccess.h>
24 #include <xen/xen-ops.h>
25 #include <xen/events.h>
29 #include <asm/traps.h>
31 #include <asm/cpufeature.h>
32 #include <asm/fpu/api.h>
33 #include <asm/nospec-branch.h>
34 #include <asm/io_bitmap.h>
35 #include <asm/syscall.h>
36 #include <asm/irq_stack.h>
39 __visible noinstr
void do_syscall_64(unsigned long nr
, struct pt_regs
*regs
)
41 nr
= syscall_enter_from_user_mode(regs
, nr
);
43 instrumentation_begin();
44 if (likely(nr
< NR_syscalls
)) {
45 nr
= array_index_nospec(nr
, NR_syscalls
);
46 regs
->ax
= sys_call_table
[nr
](regs
);
47 #ifdef CONFIG_X86_X32_ABI
48 } else if (likely((nr
& __X32_SYSCALL_BIT
) &&
49 (nr
& ~__X32_SYSCALL_BIT
) < X32_NR_syscalls
)) {
50 nr
= array_index_nospec(nr
& ~__X32_SYSCALL_BIT
,
52 regs
->ax
= x32_sys_call_table
[nr
](regs
);
55 instrumentation_end();
56 syscall_exit_to_user_mode(regs
);
60 #if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
61 static __always_inline
unsigned int syscall_32_enter(struct pt_regs
*regs
)
63 if (IS_ENABLED(CONFIG_IA32_EMULATION
))
64 current_thread_info()->status
|= TS_COMPAT
;
66 return (unsigned int)regs
->orig_ax
;
70 * Invoke a 32-bit syscall. Called with IRQs on in CONTEXT_KERNEL.
72 static __always_inline
void do_syscall_32_irqs_on(struct pt_regs
*regs
,
75 if (likely(nr
< IA32_NR_syscalls
)) {
76 nr
= array_index_nospec(nr
, IA32_NR_syscalls
);
77 regs
->ax
= ia32_sys_call_table
[nr
](regs
);
81 /* Handles int $0x80 */
82 __visible noinstr
void do_int80_syscall_32(struct pt_regs
*regs
)
84 unsigned int nr
= syscall_32_enter(regs
);
87 * Subtlety here: if ptrace pokes something larger than 2^32-1 into
88 * orig_ax, the unsigned int return value truncates it. This may
89 * or may not be necessary, but it matches the old asm behavior.
91 nr
= (unsigned int)syscall_enter_from_user_mode(regs
, nr
);
92 instrumentation_begin();
94 do_syscall_32_irqs_on(regs
, nr
);
96 instrumentation_end();
97 syscall_exit_to_user_mode(regs
);
100 static noinstr
bool __do_fast_syscall_32(struct pt_regs
*regs
)
102 unsigned int nr
= syscall_32_enter(regs
);
106 * This cannot use syscall_enter_from_user_mode() as it has to
107 * fetch EBP before invoking any of the syscall entry work
110 syscall_enter_from_user_mode_prepare(regs
);
112 instrumentation_begin();
113 /* Fetch EBP from where the vDSO stashed it. */
114 if (IS_ENABLED(CONFIG_X86_64
)) {
116 * Micro-optimization: the pointer we're following is
117 * explicitly 32 bits, so it can't be out of range.
119 res
= __get_user(*(u32
*)®s
->bp
,
120 (u32 __user __force
*)(unsigned long)(u32
)regs
->sp
);
122 res
= get_user(*(u32
*)®s
->bp
,
123 (u32 __user __force
*)(unsigned long)(u32
)regs
->sp
);
127 /* User code screwed up. */
130 instrumentation_end();
131 syscall_exit_to_user_mode(regs
);
135 /* The case truncates any ptrace induced syscall nr > 2^32 -1 */
136 nr
= (unsigned int)syscall_enter_from_user_mode_work(regs
, nr
);
138 /* Now this is just like a normal syscall. */
139 do_syscall_32_irqs_on(regs
, nr
);
141 instrumentation_end();
142 syscall_exit_to_user_mode(regs
);
146 /* Returns 0 to return using IRET or 1 to return using SYSEXIT/SYSRETL. */
147 __visible noinstr
long do_fast_syscall_32(struct pt_regs
*regs
)
150 * Called using the internal vDSO SYSENTER/SYSCALL32 calling
151 * convention. Adjust regs so it looks like we entered using int80.
153 unsigned long landing_pad
= (unsigned long)current
->mm
->context
.vdso
+
154 vdso_image_32
.sym_int80_landing_pad
;
157 * SYSENTER loses EIP, and even SYSCALL32 needs us to skip forward
158 * so that 'regs->ip -= 2' lands back on an int $0x80 instruction.
161 regs
->ip
= landing_pad
;
163 /* Invoke the syscall. If it failed, keep it simple: use IRET. */
164 if (!__do_fast_syscall_32(regs
))
169 * Opportunistic SYSRETL: if possible, try to return using SYSRETL.
170 * SYSRETL is available on all 64-bit CPUs, so we don't need to
171 * bother with SYSEXIT.
173 * Unlike 64-bit opportunistic SYSRET, we can't check that CX == IP,
174 * because the ECX fixup above will ensure that this is essentially
177 return regs
->cs
== __USER32_CS
&& regs
->ss
== __USER_DS
&&
178 regs
->ip
== landing_pad
&&
179 (regs
->flags
& (X86_EFLAGS_RF
| X86_EFLAGS_TF
)) == 0;
182 * Opportunistic SYSEXIT: if possible, try to return using SYSEXIT.
184 * Unlike 64-bit opportunistic SYSRET, we can't check that CX == IP,
185 * because the ECX fixup above will ensure that this is essentially
188 * We don't allow syscalls at all from VM86 mode, but we still
189 * need to check VM, because we might be returning from sys_vm86.
191 return static_cpu_has(X86_FEATURE_SEP
) &&
192 regs
->cs
== __USER_CS
&& regs
->ss
== __USER_DS
&&
193 regs
->ip
== landing_pad
&&
194 (regs
->flags
& (X86_EFLAGS_RF
| X86_EFLAGS_TF
| X86_EFLAGS_VM
)) == 0;
198 /* Returns 0 to return using IRET or 1 to return using SYSEXIT/SYSRETL. */
199 __visible noinstr
long do_SYSENTER_32(struct pt_regs
*regs
)
201 /* SYSENTER loses RSP, but the vDSO saved it in RBP. */
204 /* SYSENTER clobbers EFLAGS.IF. Assume it was set in usermode. */
205 regs
->flags
|= X86_EFLAGS_IF
;
207 return do_fast_syscall_32(regs
);
211 SYSCALL_DEFINE0(ni_syscall
)
217 #ifndef CONFIG_PREEMPTION
219 * Some hypercalls issued by the toolstack can take many 10s of
220 * seconds. Allow tasks running hypercalls via the privcmd driver to
221 * be voluntarily preempted even if full kernel preemption is
224 * Such preemptible hypercalls are bracketed by
225 * xen_preemptible_hcall_begin() and xen_preemptible_hcall_end()
228 DEFINE_PER_CPU(bool, xen_in_preemptible_hcall
);
229 EXPORT_SYMBOL_GPL(xen_in_preemptible_hcall
);
232 * In case of scheduling the flag must be cleared and restored after
233 * returning from schedule as the task might move to a different CPU.
235 static __always_inline
bool get_and_clear_inhcall(void)
237 bool inhcall
= __this_cpu_read(xen_in_preemptible_hcall
);
239 __this_cpu_write(xen_in_preemptible_hcall
, false);
243 static __always_inline
void restore_inhcall(bool inhcall
)
245 __this_cpu_write(xen_in_preemptible_hcall
, inhcall
);
248 static __always_inline
bool get_and_clear_inhcall(void) { return false; }
249 static __always_inline
void restore_inhcall(bool inhcall
) { }
252 static void __xen_pv_evtchn_do_upcall(void)
255 inc_irq_stat(irq_hv_callback_count
);
257 xen_hvm_evtchn_do_upcall();
262 __visible noinstr
void xen_pv_evtchn_do_upcall(struct pt_regs
*regs
)
264 struct pt_regs
*old_regs
;
266 irqentry_state_t state
;
268 state
= irqentry_enter(regs
);
269 old_regs
= set_irq_regs(regs
);
271 instrumentation_begin();
272 run_on_irqstack_cond(__xen_pv_evtchn_do_upcall
, regs
);
273 instrumentation_begin();
275 set_irq_regs(old_regs
);
277 inhcall
= get_and_clear_inhcall();
278 if (inhcall
&& !WARN_ON_ONCE(state
.exit_rcu
)) {
279 instrumentation_begin();
280 irqentry_exit_cond_resched();
281 instrumentation_end();
282 restore_inhcall(inhcall
);
284 irqentry_exit(regs
, state
);
287 #endif /* CONFIG_XEN_PV */