]> git.proxmox.com Git - mirror_ubuntu-eoan-kernel.git/blame - arch/x86_64/kernel/entry.S
projects / mirror_ubuntu-eoan-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[PATCH] x86_64: Minor microoptimization in syscall entry slow path
[mirror_ubuntu-eoan-kernel.git] / arch / x86_64 / kernel / entry.S
CommitLineData
1da177e4
LT		 1/*
2 * linux/arch/x86_64/entry.S
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
6 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
7 *
8 * $Id$
9 */
10
11/*
12 * entry.S contains the system-call and fault low-level handling routines.
13 *
14 * NOTE: This code handles signal-recognition, which happens every time
15 * after an interrupt and after each system call.
16 *
17 * Normal syscalls and interrupts don't save a full stack frame, this is
18 * only done for syscall tracing, signals or fork/exec et.al.
19 *
20 * A note on terminology:
21 * - top of stack: Architecture defined interrupt frame from SS to RIP
22 * at the top of the kernel process stack.
23 * - partial stack frame: partially saved registers upto R11.
24 * - full stack frame: Like partial stack frame, but all register saved.
25 *
26 * TODO:
27 * - schedule it carefully for the final hardware.
28 */
29
30#define ASSEMBLY 1
31#include <linux/config.h>
32#include <linux/linkage.h>
33#include <asm/segment.h>
34#include <asm/smp.h>
35#include <asm/cache.h>
36#include <asm/errno.h>
37#include <asm/dwarf2.h>
38#include <asm/calling.h>
39#include <asm/offset.h>
40#include <asm/msr.h>
41#include <asm/unistd.h>
42#include <asm/thread_info.h>
43#include <asm/hw_irq.h>
44
45 .code64
46
47#ifdef CONFIG_PREEMPT
48#define preempt_stop cli
49#else
50#define preempt_stop
51#define retint_kernel retint_restore_args
52#endif
53
54/*
55 * C code is not supposed to know about undefined top of stack. Every time
56 * a C function with an pt_regs argument is called from the SYSCALL based
57 * fast path FIXUP_TOP_OF_STACK is needed.
58 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
59 * manipulation.
60 */
61
62 /* %rsp:at FRAMEEND */
63 .macro FIXUP_TOP_OF_STACK tmp
64 movq %gs:pda_oldrsp,\tmp
65 movq \tmp,RSP(%rsp)
66 movq $__USER_DS,SS(%rsp)
67 movq $__USER_CS,CS(%rsp)
68 movq $-1,RCX(%rsp)
69 movq R11(%rsp),\tmp /* get eflags */
70 movq \tmp,EFLAGS(%rsp)
71 .endm
72
73 .macro RESTORE_TOP_OF_STACK tmp,offset=0
74 movq RSP-\offset(%rsp),\tmp
75 movq \tmp,%gs:pda_oldrsp
76 movq EFLAGS-\offset(%rsp),\tmp
77 movq \tmp,R11-\offset(%rsp)
78 .endm
79
80 .macro FAKE_STACK_FRAME child_rip
81 /* push in order ss, rsp, eflags, cs, rip */
82 xorq %rax, %rax
83 pushq %rax /* ss */
84 CFI_ADJUST_CFA_OFFSET 8
85 pushq %rax /* rsp */
86 CFI_ADJUST_CFA_OFFSET 8
87 CFI_OFFSET rip,0
88 pushq $(1<<9) /* eflags - interrupts on */
89 CFI_ADJUST_CFA_OFFSET 8
90 pushq $__KERNEL_CS /* cs */
91 CFI_ADJUST_CFA_OFFSET 8
92 pushq \child_rip /* rip */
93 CFI_ADJUST_CFA_OFFSET 8
94 CFI_OFFSET rip,0
95 pushq %rax /* orig rax */
96 CFI_ADJUST_CFA_OFFSET 8
97 .endm
98
99 .macro UNFAKE_STACK_FRAME
100 addq $8*6, %rsp
101 CFI_ADJUST_CFA_OFFSET -(6*8)
102 .endm
103
104 .macro CFI_DEFAULT_STACK
105 CFI_ADJUST_CFA_OFFSET (SS)
106 CFI_OFFSET r15,R15-SS
107 CFI_OFFSET r14,R14-SS
108 CFI_OFFSET r13,R13-SS
109 CFI_OFFSET r12,R12-SS
110 CFI_OFFSET rbp,RBP-SS
111 CFI_OFFSET rbx,RBX-SS
112 CFI_OFFSET r11,R11-SS
113 CFI_OFFSET r10,R10-SS
114 CFI_OFFSET r9,R9-SS
115 CFI_OFFSET r8,R8-SS
116 CFI_OFFSET rax,RAX-SS
117 CFI_OFFSET rcx,RCX-SS
118 CFI_OFFSET rdx,RDX-SS
119 CFI_OFFSET rsi,RSI-SS
120 CFI_OFFSET rdi,RDI-SS
121 CFI_OFFSET rsp,RSP-SS
122 CFI_OFFSET rip,RIP-SS
123 .endm
124/*
125 * A newly forked process directly context switches into this.
126 */
127/* rdi: prev */
128ENTRY(ret_from_fork)
129 CFI_STARTPROC
130 CFI_DEFAULT_STACK
131 call schedule_tail
132 GET_THREAD_INFO(%rcx)
133 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx)
134 jnz rff_trace
135rff_action:
136 RESTORE_REST
137 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
138 je int_ret_from_sys_call
139 testl $_TIF_IA32,threadinfo_flags(%rcx)
140 jnz int_ret_from_sys_call
141 RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
142 jmp ret_from_sys_call
143rff_trace:
144 movq %rsp,%rdi
145 call syscall_trace_leave
146 GET_THREAD_INFO(%rcx)
147 jmp rff_action
148 CFI_ENDPROC
149
150/*
151 * System call entry. Upto 6 arguments in registers are supported.
152 *
153 * SYSCALL does not save anything on the stack and does not change the
154 * stack pointer.
155 */
156
157/*
158 * Register setup:
159 * rax system call number
160 * rdi arg0
161 * rcx return address for syscall/sysret, C arg3
162 * rsi arg1
163 * rdx arg2
164 * r10 arg3 (--> moved to rcx for C)
165 * r8 arg4
166 * r9 arg5
167 * r11 eflags for syscall/sysret, temporary for C
168 * r12-r15,rbp,rbx saved by C code, not touched.
169 *
170 * Interrupts are off on entry.
171 * Only called from user space.
172 *
173 * XXX if we had a free scratch register we could save the RSP into the stack frame
174 * and report it properly in ps. Unfortunately we haven't.
175 */
176
177ENTRY(system_call)
178 CFI_STARTPROC
179 swapgs
180 movq %rsp,%gs:pda_oldrsp
181 movq %gs:pda_kernelstack,%rsp
182 sti
183 SAVE_ARGS 8,1
184 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
185 movq %rcx,RIP-ARGOFFSET(%rsp)
186 GET_THREAD_INFO(%rcx)
187 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
188 jnz tracesys
189 cmpq $__NR_syscall_max,%rax
190 ja badsys
191 movq %r10,%rcx
192 call *sys_call_table(,%rax,8) # XXX: rip relative
193 movq %rax,RAX-ARGOFFSET(%rsp)
194/*
195 * Syscall return path ending with SYSRET (fast path)
196 * Has incomplete stack frame and undefined top of stack.
197 */
198 .globl ret_from_sys_call
199ret_from_sys_call:
200 movl $_TIF_WORK_MASK,%edi
201 /* edi: flagmask */
202sysret_check:
203 GET_THREAD_INFO(%rcx)
204 cli
205 movl threadinfo_flags(%rcx),%edx
206 andl %edi,%edx
207 jnz sysret_careful
208 movq RIP-ARGOFFSET(%rsp),%rcx
209 RESTORE_ARGS 0,-ARG_SKIP,1
210 movq %gs:pda_oldrsp,%rsp
211 swapgs
212 sysretq
213
214 /* Handle reschedules */
215 /* edx: work, edi: workmask */
216sysret_careful:
217 bt $TIF_NEED_RESCHED,%edx
218 jnc sysret_signal
219 sti
220 pushq %rdi
221 call schedule
222 popq %rdi
223 jmp sysret_check
224
225 /* Handle a signal */
226sysret_signal:
227 sti
228 testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
229 jz 1f
230
231 /* Really a signal */
232 /* edx: work flags (arg3) */
233 leaq do_notify_resume(%rip),%rax
234 leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
235 xorl %esi,%esi # oldset -> arg2
236 call ptregscall_common
2371: movl $_TIF_NEED_RESCHED,%edi
238 jmp sysret_check
239
240 /* Do syscall tracing */
241tracesys:
242 SAVE_REST
243 movq $-ENOSYS,RAX(%rsp)
244 FIXUP_TOP_OF_STACK %rdi
245 movq %rsp,%rdi
246 call syscall_trace_enter
247 LOAD_ARGS ARGOFFSET /* reload args from stack in case ptrace changed it */
248 RESTORE_REST
249 cmpq $__NR_syscall_max,%rax
250 ja 1f
251 movq %r10,%rcx /* fixup for C */
252 call *sys_call_table(,%rax,8)
253 movq %rax,RAX-ARGOFFSET(%rsp)
2541: SAVE_REST
255 movq %rsp,%rdi
256 call syscall_trace_leave
257 RESTORE_TOP_OF_STACK %rbx
258 RESTORE_REST
259 jmp ret_from_sys_call
260
261badsys:
262 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
263 jmp ret_from_sys_call
264
265/*
266 * Syscall return path ending with IRET.
267 * Has correct top of stack, but partial stack frame.
268 */
269ENTRY(int_ret_from_sys_call)
270 cli
271 testl $3,CS-ARGOFFSET(%rsp)
272 je retint_restore_args
273 movl $_TIF_ALLWORK_MASK,%edi
274 /* edi: mask to check */
275int_with_check:
276 GET_THREAD_INFO(%rcx)
277 movl threadinfo_flags(%rcx),%edx
278 andl %edi,%edx
279 jnz int_careful
280 jmp retint_swapgs
281
282 /* Either reschedule or signal or syscall exit tracking needed. */
283 /* First do a reschedule test. */
284 /* edx: work, edi: workmask */
285int_careful:
286 bt $TIF_NEED_RESCHED,%edx
287 jnc int_very_careful
288 sti
289 pushq %rdi
290 call schedule
291 popq %rdi
292 jmp int_with_check
293
294 /* handle signals and tracing -- both require a full stack frame */
295int_very_careful:
296 sti
297 SAVE_REST
298 /* Check for syscall exit trace */
299 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
300 jz int_signal
301 pushq %rdi
302 leaq 8(%rsp),%rdi # &ptregs -> arg1
303 call syscall_trace_leave
304 popq %rdi
36c1104e 305 andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
1da177e4
LT		 306 jmp int_restore_rest
307
308int_signal:
309 testl $(_TIF_NOTIFY_RESUME|_TIF_SIGPENDING|_TIF_SINGLESTEP),%edx
310 jz 1f
311 movq %rsp,%rdi # &ptregs -> arg1
312 xorl %esi,%esi # oldset -> arg2
313 call do_notify_resume
3141: movl $_TIF_NEED_RESCHED,%edi
315int_restore_rest:
316 RESTORE_REST
317 jmp int_with_check
318 CFI_ENDPROC
319
320/*
321 * Certain special system calls that need to save a complete full stack frame.
322 */
323
324 .macro PTREGSCALL label,func,arg
325 .globl \label
326\label:
327 leaq \func(%rip),%rax
328 leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
329 jmp ptregscall_common
330 .endm
331
332 PTREGSCALL stub_clone, sys_clone, %r8
333 PTREGSCALL stub_fork, sys_fork, %rdi
334 PTREGSCALL stub_vfork, sys_vfork, %rdi
335 PTREGSCALL stub_rt_sigsuspend, sys_rt_sigsuspend, %rdx
336 PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
337 PTREGSCALL stub_iopl, sys_iopl, %rsi
338
339ENTRY(ptregscall_common)
340 CFI_STARTPROC
341 popq %r11
342 CFI_ADJUST_CFA_OFFSET -8
343 SAVE_REST
344 movq %r11, %r15
345 FIXUP_TOP_OF_STACK %r11
346 call *%rax
347 RESTORE_TOP_OF_STACK %r11
348 movq %r15, %r11
349 RESTORE_REST
350 pushq %r11
351 CFI_ADJUST_CFA_OFFSET 8
352 ret
353 CFI_ENDPROC
354
355ENTRY(stub_execve)
356 CFI_STARTPROC
357 popq %r11
358 CFI_ADJUST_CFA_OFFSET -8
359 SAVE_REST
360 movq %r11, %r15
361 FIXUP_TOP_OF_STACK %r11
362 call sys_execve
363 GET_THREAD_INFO(%rcx)
364 bt $TIF_IA32,threadinfo_flags(%rcx)
365 jc exec_32bit
366 RESTORE_TOP_OF_STACK %r11
367 movq %r15, %r11
368 RESTORE_REST
369 push %r11
370 ret
371
372exec_32bit:
373 CFI_ADJUST_CFA_OFFSET REST_SKIP
374 movq %rax,RAX(%rsp)
375 RESTORE_REST
376 jmp int_ret_from_sys_call
377 CFI_ENDPROC
378
379/*
380 * sigreturn is special because it needs to restore all registers on return.
381 * This cannot be done with SYSRET, so use the IRET return path instead.
382 */
383ENTRY(stub_rt_sigreturn)
384 CFI_STARTPROC
385 addq $8, %rsp
386 SAVE_REST
387 movq %rsp,%rdi
388 FIXUP_TOP_OF_STACK %r11
389 call sys_rt_sigreturn
390 movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
391 RESTORE_REST
392 jmp int_ret_from_sys_call
393 CFI_ENDPROC
394
395/*
396 * Interrupt entry/exit.
397 *
398 * Interrupt entry points save only callee clobbered registers in fast path.
399 *
400 * Entry runs with interrupts off.
401 */
402
403/* 0(%rsp): interrupt number */
404 .macro interrupt func
405 CFI_STARTPROC simple
406 CFI_DEF_CFA rsp,(SS-RDI)
407 CFI_REL_OFFSET rsp,(RSP-ORIG_RAX)
408 CFI_REL_OFFSET rip,(RIP-ORIG_RAX)
409 cld
410#ifdef CONFIG_DEBUG_INFO
411 SAVE_ALL
412 movq %rsp,%rdi
413 /*
414 * Setup a stack frame pointer. This allows gdb to trace
415 * back to the original stack.
416 */
417 movq %rsp,%rbp
418 CFI_DEF_CFA_REGISTER rbp
419#else
420 SAVE_ARGS
421 leaq -ARGOFFSET(%rsp),%rdi # arg1 for handler
422#endif
423 testl $3,CS(%rdi)
424 je 1f
425 swapgs
4261: addl $1,%gs:pda_irqcount # RED-PEN should check preempt count
427 movq %gs:pda_irqstackptr,%rax
428 cmoveq %rax,%rsp
429 pushq %rdi # save old stack
430 call \func
431 .endm
432
433ENTRY(common_interrupt)
434 interrupt do_IRQ
435 /* 0(%rsp): oldrsp-ARGOFFSET */
436ret_from_intr:
437 popq %rdi
438 cli
439 subl $1,%gs:pda_irqcount
440#ifdef CONFIG_DEBUG_INFO
441 movq RBP(%rdi),%rbp
442#endif
443 leaq ARGOFFSET(%rdi),%rsp
444exit_intr:
445 GET_THREAD_INFO(%rcx)
446 testl $3,CS-ARGOFFSET(%rsp)
447 je retint_kernel
448
449 /* Interrupt came from user space */
450 /*
451 * Has a correct top of stack, but a partial stack frame
452 * %rcx: thread info. Interrupts off.
453 */
454retint_with_reschedule:
455 movl $_TIF_WORK_MASK,%edi
456retint_check:
457 movl threadinfo_flags(%rcx),%edx
458 andl %edi,%edx
459 jnz retint_careful
460retint_swapgs:
461 cli
462 swapgs
463retint_restore_args:
464 cli
465 RESTORE_ARGS 0,8,0
466iret_label:
467 iretq
468
469 .section __ex_table,"a"
470 .quad iret_label,bad_iret
471 .previous
472 .section .fixup,"ax"
473 /* force a signal here? this matches i386 behaviour */
474 /* running with kernel gs */
475bad_iret:
476 movq $-9999,%rdi /* better code? */
477 jmp do_exit
478 .previous
479
480 /* edi: workmask, edx: work */
481retint_careful:
482 bt $TIF_NEED_RESCHED,%edx
483 jnc retint_signal
484 sti
485 pushq %rdi
486 call schedule
487 popq %rdi
488 GET_THREAD_INFO(%rcx)
489 cli
490 jmp retint_check
491
492retint_signal:
493 testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
494 jz retint_swapgs
495 sti
496 SAVE_REST
497 movq $-1,ORIG_RAX(%rsp)
498 xorq %rsi,%rsi # oldset
499 movq %rsp,%rdi # &pt_regs
500 call do_notify_resume
501 RESTORE_REST
502 cli
503 movl $_TIF_NEED_RESCHED,%edi
504 GET_THREAD_INFO(%rcx)
505 jmp retint_check
506
507#ifdef CONFIG_PREEMPT
508 /* Returning to kernel space. Check if we need preemption */
509 /* rcx: threadinfo. interrupts off. */
510 .p2align
511retint_kernel:
512 cmpl $0,threadinfo_preempt_count(%rcx)
513 jnz retint_restore_args
514 bt $TIF_NEED_RESCHED,threadinfo_flags(%rcx)
515 jnc retint_restore_args
516 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
517 jnc retint_restore_args
518 call preempt_schedule_irq
519 jmp exit_intr
520#endif
521 CFI_ENDPROC
522
523/*
524 * APIC interrupts.
525 */
526 .macro apicinterrupt num,func
527 pushq $\num-256
528 interrupt \func
529 jmp ret_from_intr
530 CFI_ENDPROC
531 .endm
532
533ENTRY(thermal_interrupt)
534 apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
535
536#ifdef CONFIG_SMP
537ENTRY(reschedule_interrupt)
538 apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
539
540ENTRY(invalidate_interrupt)
541 apicinterrupt INVALIDATE_TLB_VECTOR,smp_invalidate_interrupt
542
543ENTRY(call_function_interrupt)
544 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
545#endif
546
547#ifdef CONFIG_X86_LOCAL_APIC
548ENTRY(apic_timer_interrupt)
549 apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
550
551ENTRY(error_interrupt)
552 apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
553
554ENTRY(spurious_interrupt)
555 apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
556#endif
557
558/*
559 * Exception entry points.
560 */
561 .macro zeroentry sym
562 pushq $0 /* push error code/oldrax */
563 pushq %rax /* push real oldrax to the rdi slot */
564 leaq \sym(%rip),%rax
565 jmp error_entry
566 .endm
567
568 .macro errorentry sym
569 pushq %rax
570 leaq \sym(%rip),%rax
571 jmp error_entry
572 .endm
573
574 /* error code is on the stack already */
575 /* handle NMI like exceptions that can happen everywhere */
576 .macro paranoidentry sym
577 SAVE_ALL
578 cld
579 movl $1,%ebx
580 movl $MSR_GS_BASE,%ecx
581 rdmsr
582 testl %edx,%edx
583 js 1f
584 swapgs
585 xorl %ebx,%ebx
5861: movq %rsp,%rdi
587 movq ORIG_RAX(%rsp),%rsi
588 movq $-1,ORIG_RAX(%rsp)
589 call \sym
590 .endm
591
592/*
593 * Exception entry point. This expects an error code/orig_rax on the stack
594 * and the exception handler in %rax.
595 */
596ENTRY(error_entry)
597 CFI_STARTPROC simple
598 CFI_DEF_CFA rsp,(SS-RDI)
599 CFI_REL_OFFSET rsp,(RSP-RDI)
600 CFI_REL_OFFSET rip,(RIP-RDI)
601 /* rdi slot contains rax, oldrax contains error code */
602 cld
603 subq $14*8,%rsp
604 CFI_ADJUST_CFA_OFFSET (14*8)
605 movq %rsi,13*8(%rsp)
606 CFI_REL_OFFSET rsi,RSI
607 movq 14*8(%rsp),%rsi /* load rax from rdi slot */
608 movq %rdx,12*8(%rsp)
609 CFI_REL_OFFSET rdx,RDX
610 movq %rcx,11*8(%rsp)
611 CFI_REL_OFFSET rcx,RCX
612 movq %rsi,10*8(%rsp) /* store rax */
613 CFI_REL_OFFSET rax,RAX
614 movq %r8, 9*8(%rsp)
615 CFI_REL_OFFSET r8,R8
616 movq %r9, 8*8(%rsp)
617 CFI_REL_OFFSET r9,R9
618 movq %r10,7*8(%rsp)
619 CFI_REL_OFFSET r10,R10
620 movq %r11,6*8(%rsp)
621 CFI_REL_OFFSET r11,R11
622 movq %rbx,5*8(%rsp)
623 CFI_REL_OFFSET rbx,RBX
624 movq %rbp,4*8(%rsp)
625 CFI_REL_OFFSET rbp,RBP
626 movq %r12,3*8(%rsp)
627 CFI_REL_OFFSET r12,R12
628 movq %r13,2*8(%rsp)
629 CFI_REL_OFFSET r13,R13
630 movq %r14,1*8(%rsp)
631 CFI_REL_OFFSET r14,R14
632 movq %r15,(%rsp)
633 CFI_REL_OFFSET r15,R15
634 xorl %ebx,%ebx
635 testl $3,CS(%rsp)
636 je error_kernelspace
637error_swapgs:
638 swapgs
639error_sti:
640 movq %rdi,RDI(%rsp)
641 movq %rsp,%rdi
642 movq ORIG_RAX(%rsp),%rsi /* get error code */
643 movq $-1,ORIG_RAX(%rsp)
644 call *%rax
645 /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
646error_exit:
647 movl %ebx,%eax
648 RESTORE_REST
649 cli
650 GET_THREAD_INFO(%rcx)
651 testl %eax,%eax
652 jne retint_kernel
653 movl threadinfo_flags(%rcx),%edx
654 movl $_TIF_WORK_MASK,%edi
655 andl %edi,%edx
656 jnz retint_careful
657 swapgs
658 RESTORE_ARGS 0,8,0
659 iretq
660 CFI_ENDPROC
661
662error_kernelspace:
663 incl %ebx
664 /* There are two places in the kernel that can potentially fault with
665 usergs. Handle them here. The exception handlers after
666 iret run with kernel gs again, so don't set the user space flag.
667 B stepping K8s sometimes report an truncated RIP for IRET
668 exceptions returning to compat mode. Check for these here too. */
669 leaq iret_label(%rip),%rbp
670 cmpq %rbp,RIP(%rsp)
671 je error_swapgs
672 movl %ebp,%ebp /* zero extend */
673 cmpq %rbp,RIP(%rsp)
674 je error_swapgs
675 cmpq $gs_change,RIP(%rsp)
676 je error_swapgs
677 jmp error_sti
678
679 /* Reload gs selector with exception handling */
680 /* edi: new selector */
681ENTRY(load_gs_index)
682 pushf
683 cli
684 swapgs
685gs_change:
686 movl %edi,%gs
6872: mfence /* workaround */
688 swapgs
689 popf
690 ret
691
692 .section __ex_table,"a"
693 .align 8
694 .quad gs_change,bad_gs
695 .previous
696 .section .fixup,"ax"
697 /* running with kernelgs */
698bad_gs:
699 swapgs /* switch back to user gs */
700 xorl %eax,%eax
701 movl %eax,%gs
702 jmp 2b
703 .previous
704
705/*
706 * Create a kernel thread.
707 *
708 * C extern interface:
709 * extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
710 *
711 * asm input arguments:
712 * rdi: fn, rsi: arg, rdx: flags
713 */
714ENTRY(kernel_thread)
715 CFI_STARTPROC
716 FAKE_STACK_FRAME $child_rip
717 SAVE_ALL
718
719 # rdi: flags, rsi: usp, rdx: will be &pt_regs
720 movq %rdx,%rdi
721 orq kernel_thread_flags(%rip),%rdi
722 movq $-1, %rsi
723 movq %rsp, %rdx
724
725 xorl %r8d,%r8d
726 xorl %r9d,%r9d
727
728 # clone now
729 call do_fork
730 movq %rax,RAX(%rsp)
731 xorl %edi,%edi
732
733 /*
734 * It isn't worth to check for reschedule here,
735 * so internally to the x86_64 port you can rely on kernel_thread()
736 * not to reschedule the child before returning, this avoids the need
737 * of hacks for example to fork off the per-CPU idle tasks.
738 * [Hopefully no generic code relies on the reschedule -AK]
739 */
740 RESTORE_ALL
741 UNFAKE_STACK_FRAME
742 ret
743 CFI_ENDPROC
744
745
746child_rip:
747 /*
748 * Here we are in the child and the registers are set as they were
749 * at kernel_thread() invocation in the parent.
750 */
751 movq %rdi, %rax
752 movq %rsi, %rdi
753 call *%rax
754 # exit
755 xorq %rdi, %rdi
756 call do_exit
757
758/*
759 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
760 *
761 * C extern interface:
762 * extern long execve(char *name, char **argv, char **envp)
763 *
764 * asm input arguments:
765 * rdi: name, rsi: argv, rdx: envp
766 *
767 * We want to fallback into:
768 * extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs regs)
769 *
770 * do_sys_execve asm fallback arguments:
771 * rdi: name, rsi: argv, rdx: envp, fake frame on the stack
772 */
773ENTRY(execve)
774 CFI_STARTPROC
775 FAKE_STACK_FRAME $0
776 SAVE_ALL
777 call sys_execve
778 movq %rax, RAX(%rsp)
779 RESTORE_REST
780 testq %rax,%rax
781 je int_ret_from_sys_call
782 RESTORE_ARGS
783 UNFAKE_STACK_FRAME
784 ret
785 CFI_ENDPROC
786
787ENTRY(page_fault)
788 errorentry do_page_fault
789
790ENTRY(coprocessor_error)
791 zeroentry do_coprocessor_error
792
793ENTRY(simd_coprocessor_error)
794 zeroentry do_simd_coprocessor_error
795
796ENTRY(device_not_available)
797 zeroentry math_state_restore
798
799 /* runs on exception stack */
800ENTRY(debug)
801 CFI_STARTPROC
802 pushq $0
803 CFI_ADJUST_CFA_OFFSET 8
804 paranoidentry do_debug
805 /* switch back to process stack to restore the state ptrace touched */
806 movq %rax,%rsp
807 testl $3,CS(%rsp)
808 jnz paranoid_userspace
809 jmp paranoid_exit
810 CFI_ENDPROC
811
812 /* runs on exception stack */
813ENTRY(nmi)
814 CFI_STARTPROC
815 pushq $-1
816 CFI_ADJUST_CFA_OFFSET 8
817 paranoidentry do_nmi
818 /* ebx: no swapgs flag */
819paranoid_exit:
820 testl %ebx,%ebx /* swapgs needed? */
821 jnz paranoid_restore
822paranoid_swapgs:
823 cli
824 swapgs
825paranoid_restore:
826 RESTORE_ALL 8
827 iretq
828paranoid_userspace:
829 cli
830 GET_THREAD_INFO(%rcx)
831 movl threadinfo_flags(%rcx),%edx
832 testl $_TIF_NEED_RESCHED,%edx
833 jnz paranoid_resched
834 testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
835 jnz paranoid_signal
836 jmp paranoid_swapgs
837paranoid_resched:
838 sti
839 call schedule
840 jmp paranoid_exit
841paranoid_signal:
842 sti
843 xorl %esi,%esi /* oldset */
844 movq %rsp,%rdi /* &pt_regs */
845 call do_notify_resume
846 jmp paranoid_exit
847 CFI_ENDPROC
848
849ENTRY(int3)
850 zeroentry do_int3
851
852ENTRY(overflow)
853 zeroentry do_overflow
854
855ENTRY(bounds)
856 zeroentry do_bounds
857
858ENTRY(invalid_op)
859 zeroentry do_invalid_op
860
861ENTRY(coprocessor_segment_overrun)
862 zeroentry do_coprocessor_segment_overrun
863
864ENTRY(reserved)
865 zeroentry do_reserved
866
867 /* runs on exception stack */
868ENTRY(double_fault)
869 CFI_STARTPROC
870 paranoidentry do_double_fault
871 movq %rax,%rsp
872 testl $3,CS(%rsp)
873 jnz paranoid_userspace
874 jmp paranoid_exit
875 CFI_ENDPROC
876
877ENTRY(invalid_TSS)
878 errorentry do_invalid_TSS
879
880ENTRY(segment_not_present)
881 errorentry do_segment_not_present
882
883 /* runs on exception stack */
884ENTRY(stack_segment)
885 CFI_STARTPROC
886 paranoidentry do_stack_segment
887 movq %rax,%rsp
888 testl $3,CS(%rsp)
889 jnz paranoid_userspace
890 jmp paranoid_exit
891 CFI_ENDPROC
892
893ENTRY(general_protection)
894 errorentry do_general_protection
895
896ENTRY(alignment_check)
897 errorentry do_alignment_check
898
899ENTRY(divide_error)
900 zeroentry do_divide_error
901
902ENTRY(spurious_interrupt_bug)
903 zeroentry do_spurious_interrupt_bug
904
905#ifdef CONFIG_X86_MCE
906 /* runs on exception stack */
907ENTRY(machine_check)
908 CFI_STARTPROC
909 pushq $0
910 CFI_ADJUST_CFA_OFFSET 8
911 paranoidentry do_machine_check
912 jmp paranoid_exit
913 CFI_ENDPROC
914#endif
915
916ENTRY(call_debug)
917 zeroentry do_call_debug
918
Ubuntu Eoan Linux kernel mirror