1 // SPDX-License-Identifier: GPL-2.0
3 #include <linux/context_tracking.h>
4 #include <linux/entry-common.h>
5 #include <linux/highmem.h>
6 #include <linux/livepatch.h>
7 #include <linux/audit.h>
8 #include <linux/tick.h>
12 #define CREATE_TRACE_POINTS
13 #include <trace/events/syscalls.h>
15 /* See comment for enter_from_user_mode() in entry-common.h */
16 static __always_inline
void __enter_from_user_mode(struct pt_regs
*regs
)
18 arch_check_user_regs(regs
);
19 lockdep_hardirqs_off(CALLER_ADDR0
);
21 CT_WARN_ON(ct_state() != CONTEXT_USER
);
24 instrumentation_begin();
25 trace_hardirqs_off_finish();
26 instrumentation_end();
29 void noinstr
enter_from_user_mode(struct pt_regs
*regs
)
31 __enter_from_user_mode(regs
);
34 static inline void syscall_enter_audit(struct pt_regs
*regs
, long syscall
)
36 if (unlikely(audit_context())) {
37 unsigned long args
[6];
39 syscall_get_arguments(current
, regs
, args
);
40 audit_syscall_entry(syscall
, args
[0], args
[1], args
[2], args
[3]);
44 static long syscall_trace_enter(struct pt_regs
*regs
, long syscall
,
50 * Handle Syscall User Dispatch. This must comes first, since
51 * the ABI here can be something that doesn't make sense for
52 * other syscall_work features.
54 if (work
& SYSCALL_WORK_SYSCALL_USER_DISPATCH
) {
55 if (syscall_user_dispatch(regs
))
60 if (work
& (SYSCALL_WORK_SYSCALL_TRACE
| SYSCALL_WORK_SYSCALL_EMU
)) {
61 ret
= arch_syscall_enter_tracehook(regs
);
62 if (ret
|| (work
& SYSCALL_WORK_SYSCALL_EMU
))
66 /* Do seccomp after ptrace, to catch any tracer changes. */
67 if (work
& SYSCALL_WORK_SECCOMP
) {
68 ret
= __secure_computing(NULL
);
73 /* Either of the above might have changed the syscall number */
74 syscall
= syscall_get_nr(current
, regs
);
76 if (unlikely(work
& SYSCALL_WORK_SYSCALL_TRACEPOINT
))
77 trace_sys_enter(regs
, syscall
);
79 syscall_enter_audit(regs
, syscall
);
81 return ret
? : syscall
;
84 static __always_inline
long
85 __syscall_enter_from_user_work(struct pt_regs
*regs
, long syscall
)
87 unsigned long work
= READ_ONCE(current_thread_info()->syscall_work
);
89 if (work
& SYSCALL_WORK_ENTER
)
90 syscall
= syscall_trace_enter(regs
, syscall
, work
);
95 long syscall_enter_from_user_mode_work(struct pt_regs
*regs
, long syscall
)
97 return __syscall_enter_from_user_work(regs
, syscall
);
100 noinstr
long syscall_enter_from_user_mode(struct pt_regs
*regs
, long syscall
)
104 __enter_from_user_mode(regs
);
106 instrumentation_begin();
108 ret
= __syscall_enter_from_user_work(regs
, syscall
);
109 instrumentation_end();
114 noinstr
void syscall_enter_from_user_mode_prepare(struct pt_regs
*regs
)
116 __enter_from_user_mode(regs
);
117 instrumentation_begin();
119 instrumentation_end();
122 /* See comment for exit_to_user_mode() in entry-common.h */
123 static __always_inline
void __exit_to_user_mode(void)
125 instrumentation_begin();
126 trace_hardirqs_on_prepare();
127 lockdep_hardirqs_on_prepare(CALLER_ADDR0
);
128 instrumentation_end();
131 arch_exit_to_user_mode();
132 lockdep_hardirqs_on(CALLER_ADDR0
);
135 void noinstr
exit_to_user_mode(void)
137 __exit_to_user_mode();
140 /* Workaround to allow gradual conversion of architecture code */
141 void __weak
arch_do_signal_or_restart(struct pt_regs
*regs
, bool has_signal
) { }
143 static void handle_signal_work(struct pt_regs
*regs
, unsigned long ti_work
)
145 if (ti_work
& _TIF_NOTIFY_SIGNAL
)
146 tracehook_notify_signal();
148 arch_do_signal_or_restart(regs
, ti_work
& _TIF_SIGPENDING
);
151 static unsigned long exit_to_user_mode_loop(struct pt_regs
*regs
,
152 unsigned long ti_work
)
155 * Before returning to user space ensure that all pending work
156 * items have been completed.
158 while (ti_work
& EXIT_TO_USER_MODE_WORK
) {
160 local_irq_enable_exit_to_user(ti_work
);
162 if (ti_work
& _TIF_NEED_RESCHED
)
165 if (ti_work
& _TIF_UPROBE
)
166 uprobe_notify_resume(regs
);
168 if (ti_work
& _TIF_PATCH_PENDING
)
169 klp_update_patch_state(current
);
171 if (ti_work
& (_TIF_SIGPENDING
| _TIF_NOTIFY_SIGNAL
))
172 handle_signal_work(regs
, ti_work
);
174 if (ti_work
& _TIF_NOTIFY_RESUME
)
175 tracehook_notify_resume(regs
);
177 /* Architecture specific TIF work */
178 arch_exit_to_user_mode_work(regs
, ti_work
);
181 * Disable interrupts and reevaluate the work flags as they
182 * might have changed while interrupts and preemption was
185 local_irq_disable_exit_to_user();
187 /* Check if any of the above work has queued a deferred wakeup */
188 tick_nohz_user_enter_prepare();
190 ti_work
= READ_ONCE(current_thread_info()->flags
);
193 /* Return the latest work state for arch_exit_to_user_mode() */
197 static void exit_to_user_mode_prepare(struct pt_regs
*regs
)
199 unsigned long ti_work
= READ_ONCE(current_thread_info()->flags
);
201 lockdep_assert_irqs_disabled();
203 /* Flush pending rcuog wakeup before the last need_resched() check */
204 tick_nohz_user_enter_prepare();
206 if (unlikely(ti_work
& EXIT_TO_USER_MODE_WORK
))
207 ti_work
= exit_to_user_mode_loop(regs
, ti_work
);
209 arch_exit_to_user_mode_prepare(regs
, ti_work
);
211 /* Ensure that the address limit is intact and no locks are held */
212 addr_limit_user_check();
214 lockdep_assert_irqs_disabled();
219 * If SYSCALL_EMU is set, then the only reason to report is when
220 * SINGLESTEP is set (i.e. PTRACE_SYSEMU_SINGLESTEP). This syscall
221 * instruction has been already reported in syscall_enter_from_user_mode().
223 static inline bool report_single_step(unsigned long work
)
225 if (work
& SYSCALL_WORK_SYSCALL_EMU
)
228 return work
& SYSCALL_WORK_SYSCALL_EXIT_TRAP
;
231 static void syscall_exit_work(struct pt_regs
*regs
, unsigned long work
)
236 * If the syscall was rolled back due to syscall user dispatching,
237 * then the tracers below are not invoked for the same reason as
238 * the entry side was not invoked in syscall_trace_enter(): The ABI
239 * of these syscalls is unknown.
241 if (work
& SYSCALL_WORK_SYSCALL_USER_DISPATCH
) {
242 if (unlikely(current
->syscall_dispatch
.on_dispatch
)) {
243 current
->syscall_dispatch
.on_dispatch
= false;
248 audit_syscall_exit(regs
);
250 if (work
& SYSCALL_WORK_SYSCALL_TRACEPOINT
)
251 trace_sys_exit(regs
, syscall_get_return_value(current
, regs
));
253 step
= report_single_step(work
);
254 if (step
|| work
& SYSCALL_WORK_SYSCALL_TRACE
)
255 arch_syscall_exit_tracehook(regs
, step
);
259 * Syscall specific exit to user mode preparation. Runs with interrupts
262 static void syscall_exit_to_user_mode_prepare(struct pt_regs
*regs
)
264 unsigned long work
= READ_ONCE(current_thread_info()->syscall_work
);
265 unsigned long nr
= syscall_get_nr(current
, regs
);
267 CT_WARN_ON(ct_state() != CONTEXT_KERNEL
);
269 if (IS_ENABLED(CONFIG_PROVE_LOCKING
)) {
270 if (WARN(irqs_disabled(), "syscall %lu left IRQs disabled", nr
))
277 * Do one-time syscall specific work. If these work items are
278 * enabled, we want to run them exactly once per syscall exit with
279 * interrupts enabled.
281 if (unlikely(work
& SYSCALL_WORK_EXIT
))
282 syscall_exit_work(regs
, work
);
285 static __always_inline
void __syscall_exit_to_user_mode_work(struct pt_regs
*regs
)
287 syscall_exit_to_user_mode_prepare(regs
);
288 local_irq_disable_exit_to_user();
289 exit_to_user_mode_prepare(regs
);
292 void syscall_exit_to_user_mode_work(struct pt_regs
*regs
)
294 __syscall_exit_to_user_mode_work(regs
);
297 __visible noinstr
void syscall_exit_to_user_mode(struct pt_regs
*regs
)
299 instrumentation_begin();
300 __syscall_exit_to_user_mode_work(regs
);
301 instrumentation_end();
302 __exit_to_user_mode();
305 noinstr
void irqentry_enter_from_user_mode(struct pt_regs
*regs
)
307 __enter_from_user_mode(regs
);
310 noinstr
void irqentry_exit_to_user_mode(struct pt_regs
*regs
)
312 instrumentation_begin();
313 exit_to_user_mode_prepare(regs
);
314 instrumentation_end();
315 __exit_to_user_mode();
318 noinstr irqentry_state_t
irqentry_enter(struct pt_regs
*regs
)
320 irqentry_state_t ret
= {
324 if (user_mode(regs
)) {
325 irqentry_enter_from_user_mode(regs
);
330 * If this entry hit the idle task invoke rcu_irq_enter() whether
331 * RCU is watching or not.
333 * Interrupts can nest when the first interrupt invokes softirq
334 * processing on return which enables interrupts.
336 * Scheduler ticks in the idle task can mark quiescent state and
337 * terminate a grace period, if and only if the timer interrupt is
338 * not nested into another interrupt.
340 * Checking for rcu_is_watching() here would prevent the nesting
341 * interrupt to invoke rcu_irq_enter(). If that nested interrupt is
342 * the tick then rcu_flavor_sched_clock_irq() would wrongfully
343 * assume that it is the first interrupt and eventually claim
344 * quiescent state and end grace periods prematurely.
346 * Unconditionally invoke rcu_irq_enter() so RCU state stays
349 * TINY_RCU does not support EQS, so let the compiler eliminate
350 * this part when enabled.
352 if (!IS_ENABLED(CONFIG_TINY_RCU
) && is_idle_task(current
)) {
354 * If RCU is not watching then the same careful
355 * sequence vs. lockdep and tracing is required
356 * as in irqentry_enter_from_user_mode().
358 lockdep_hardirqs_off(CALLER_ADDR0
);
360 instrumentation_begin();
361 trace_hardirqs_off_finish();
362 instrumentation_end();
369 * If RCU is watching then RCU only wants to check whether it needs
370 * to restart the tick in NOHZ mode. rcu_irq_enter_check_tick()
371 * already contains a warning when RCU is not watching, so no point
372 * in having another one here.
374 lockdep_hardirqs_off(CALLER_ADDR0
);
375 instrumentation_begin();
376 rcu_irq_enter_check_tick();
377 trace_hardirqs_off_finish();
378 instrumentation_end();
383 void irqentry_exit_cond_resched(void)
385 if (!preempt_count()) {
386 /* Sanity check RCU and thread stack */
387 rcu_irq_exit_check_preempt();
388 if (IS_ENABLED(CONFIG_DEBUG_ENTRY
))
389 WARN_ON_ONCE(!on_thread_stack());
391 preempt_schedule_irq();
394 #ifdef CONFIG_PREEMPT_DYNAMIC
395 DEFINE_STATIC_CALL(irqentry_exit_cond_resched
, irqentry_exit_cond_resched
);
398 noinstr
void irqentry_exit(struct pt_regs
*regs
, irqentry_state_t state
)
400 lockdep_assert_irqs_disabled();
402 /* Check whether this returns to user mode */
403 if (user_mode(regs
)) {
404 irqentry_exit_to_user_mode(regs
);
405 } else if (!regs_irqs_disabled(regs
)) {
407 * If RCU was not watching on entry this needs to be done
408 * carefully and needs the same ordering of lockdep/tracing
409 * and RCU as the return to user mode path.
411 if (state
.exit_rcu
) {
412 instrumentation_begin();
413 /* Tell the tracer that IRET will enable interrupts */
414 trace_hardirqs_on_prepare();
415 lockdep_hardirqs_on_prepare(CALLER_ADDR0
);
416 instrumentation_end();
418 lockdep_hardirqs_on(CALLER_ADDR0
);
422 instrumentation_begin();
423 if (IS_ENABLED(CONFIG_PREEMPTION
)) {
424 #ifdef CONFIG_PREEMPT_DYNAMIC
425 static_call(irqentry_exit_cond_resched
)();
427 irqentry_exit_cond_resched();
430 /* Covers both tracing and lockdep */
432 instrumentation_end();
435 * IRQ flags state is correct already. Just tell RCU if it
436 * was not watching on entry.
443 irqentry_state_t noinstr
irqentry_nmi_enter(struct pt_regs
*regs
)
445 irqentry_state_t irq_state
;
447 irq_state
.lockdep
= lockdep_hardirqs_enabled();
450 lockdep_hardirqs_off(CALLER_ADDR0
);
451 lockdep_hardirq_enter();
454 instrumentation_begin();
455 trace_hardirqs_off_finish();
457 instrumentation_end();
462 void noinstr
irqentry_nmi_exit(struct pt_regs
*regs
, irqentry_state_t irq_state
)
464 instrumentation_begin();
466 if (irq_state
.lockdep
) {
467 trace_hardirqs_on_prepare();
468 lockdep_hardirqs_on_prepare(CALLER_ADDR0
);
470 instrumentation_end();
473 lockdep_hardirq_exit();
474 if (irq_state
.lockdep
)
475 lockdep_hardirqs_on(CALLER_ADDR0
);