2 * linux/arch/i386/entry.S
4 * Copyright (C) 1991, 1992 Linus Torvalds
8 * entry.S contains the system-call and fault low-level handling routines.
9 * This also contains the timer-interrupt handler, as well as all interrupts
10 * and faults that can result in a task-switch.
12 * NOTE: This code handles signal-recognition, which happens every time
13 * after a timer-interrupt and after each system call.
15 * I changed all the .align's to 4 (16 byte alignment), as that's faster
18 * Stack layout in 'ret_from_system_call':
19 * ptrace needs to have all regs on the stack.
20 * if the order here is changed, it needs to be
21 * updated in fork.c:copy_process, signal.c:do_signal,
22 * ptrace.c and ptrace.h
40 * "current" is in register %ebx during any slow entries.
43 #include <linux/config.h>
44 #include <linux/linkage.h>
45 #include <asm/thread_info.h>
46 #include <asm/errno.h>
47 #include <asm/segment.h>
51 #include "irq_vectors.h"
53 #define nr_syscalls ((syscall_table_size)/4)
79 #define preempt_stop cli
82 #define resume_kernel restore_nocheck
96 movl $(__USER_DS), %edx; \
100 #define RESTORE_INT_REGS \
109 #define RESTORE_REGS \
113 .section .fixup,"ax"; \
119 .section __ex_table,"a";\
129 GET_THREAD_INFO(%ebp)
134 * Return to user mode is not as complex as all this looks,
135 * but we want the default path for a system call return to
136 * go as quickly as possible which is why some of this is
137 * less clear than it otherwise should be.
140 # userspace resumption stub bypassing syscall exit tracing
145 GET_THREAD_INFO(%ebp)
146 movl EFLAGS(%esp), %eax # mix EFLAGS and CS
148 testl $(VM_MASK | 3), %eax
150 ENTRY(resume_userspace)
151 cli # make sure we don't miss an interrupt
152 # setting need_resched or sigpending
153 # between sampling and the iret
154 movl TI_flags(%ebp), %ecx
155 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on
156 # int/exception return?
160 #ifdef CONFIG_PREEMPT
163 cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ?
166 movl TI_flags(%ebp), %ecx # need_resched set ?
167 testb $_TIF_NEED_RESCHED, %cl
169 testl $IF_MASK,EFLAGS(%esp) # interrupts off (exception path) ?
171 call preempt_schedule_irq
175 /* SYSENTER_RETURN points to after the "sysenter" instruction in
176 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */
178 # sysenter call handler stub
179 ENTRY(sysenter_entry)
180 movl TSS_sysenter_esp0(%esp),%esp
187 pushl $SYSENTER_RETURN
190 * Load the potential sixth argument from user stack.
191 * Careful about security.
193 cmpl $__PAGE_OFFSET-3,%ebp
196 .section __ex_table,"a"
198 .long 1b,syscall_fault
203 GET_THREAD_INFO(%ebp)
205 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
206 testw $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),TI_flags(%ebp)
207 jnz syscall_trace_entry
208 cmpl $(nr_syscalls), %eax
210 call *sys_call_table(,%eax,4)
213 movl TI_flags(%ebp), %ecx
214 testw $_TIF_ALLWORK_MASK, %cx
215 jne syscall_exit_work
216 /* if something modifies registers it must also disable sysexit */
218 movl OLDESP(%esp), %ecx
224 # system call handler stub
226 pushl %eax # save orig_eax
228 GET_THREAD_INFO(%ebp)
229 # system call tracing in operation
230 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
231 testw $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),TI_flags(%ebp)
232 jnz syscall_trace_entry
233 cmpl $(nr_syscalls), %eax
236 call *sys_call_table(,%eax,4)
237 movl %eax,EAX(%esp) # store the return value
239 cli # make sure we don't miss an interrupt
240 # setting need_resched or sigpending
241 # between sampling and the iret
242 movl TI_flags(%ebp), %ecx
243 testw $_TIF_ALLWORK_MASK, %cx # current->work
244 jne syscall_exit_work
247 movl EFLAGS(%esp), %eax # mix EFLAGS, SS and CS
248 # Warning: OLDSS(%esp) contains the wrong/random values if we
249 # are returning to the kernel.
250 # See comments in process.c:copy_thread() for details.
251 movb OLDSS(%esp), %ah
253 andl $(VM_MASK | (4 << 8) | 3), %eax
254 cmpl $((4 << 8) | 3), %eax
255 je ldt_ss # returning to user-space with LDT SS
263 pushl $0 # no error code
267 .section __ex_table,"a"
273 larl OLDSS(%esp), %eax
275 testl $0x00400000, %eax # returning to 32bit stack?
276 jnz restore_nocheck # allright, normal return
277 /* If returning to userspace with 16bit stack,
278 * try to fix the higher word of ESP, as the CPU
280 * This is an "official" bug of all the x86-compatible
281 * CPUs, which we can try to work around to make
282 * dosemu and wine happy. */
283 subl $8, %esp # reserve space for switch16 pointer
286 /* Set up the 16bit stack frame with switch32 pointer on top,
287 * and a switch16 pointer on top of the current frame. */
288 call setup_x86_bogus_stack
290 lss 20+4(%esp), %esp # switch to 16bit stack
292 .section __ex_table,"a"
297 # perform work that needs to be done immediately before resumption
300 testb $_TIF_NEED_RESCHED, %cl
304 cli # make sure we don't miss an interrupt
305 # setting need_resched or sigpending
306 # between sampling and the iret
307 movl TI_flags(%ebp), %ecx
308 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other
309 # than syscall tracing?
311 testb $_TIF_NEED_RESCHED, %cl
314 work_notifysig: # deal with pending signals and
315 # notify-resume requests
316 testl $VM_MASK, EFLAGS(%esp)
318 jne work_notifysig_v86 # returning to kernel-space or
321 call do_notify_resume
326 pushl %ecx # save ti_flags for do_notify_resume
327 call save_v86_state # %eax contains pt_regs pointer
331 call do_notify_resume
334 # perform syscall exit tracing
337 movl $-ENOSYS,EAX(%esp)
340 call do_syscall_trace
341 movl ORIG_EAX(%esp), %eax
342 cmpl $(nr_syscalls), %eax
346 # perform syscall exit tracing
349 testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl
351 sti # could let do_syscall_trace() call
355 call do_syscall_trace
360 pushl %eax # save orig_eax
362 GET_THREAD_INFO(%ebp)
363 movl $-EFAULT,EAX(%esp)
368 movl $-ENOSYS,EAX(%esp)
371 #define FIXUP_ESPFIX_STACK \
373 /* switch to 32bit stack using the pointer on top of 16bit stack */ \
374 lss %ss:CPU_16BIT_STACK_SIZE-8, %esp; \
375 /* copy data from 16bit stack to 32bit stack */ \
376 call fixup_x86_bogus_stack; \
377 /* put ESP to the proper location */ \
379 #define UNWIND_ESPFIX_STACK \
382 /* see if on 16bit stack */ \
383 cmpw $__ESPFIX_SS, %ax; \
385 movl $__KERNEL_DS, %edx; \
388 /* switch to 32bit stack */ \
393 * Build the entry stubs and pointer table with
394 * some assembler magic.
401 ENTRY(irq_entries_start)
419 #define BUILD_INTERRUPT(name, nr) \
427 /* The include is where all of the SMP etc. interrupts come from */
428 #include "entry_arch.h"
431 pushl $0 # no error code
432 pushl $do_divide_error
449 movl ES(%esp), %edi # get the function address
450 movl ORIG_EAX(%esp), %edx # get the error code
451 movl %eax, ORIG_EAX(%esp)
453 movl $(__USER_DS), %ecx
456 movl %esp,%eax # pt_regs pointer
458 jmp ret_from_exception
460 ENTRY(coprocessor_error)
462 pushl $do_coprocessor_error
465 ENTRY(simd_coprocessor_error)
467 pushl $do_simd_coprocessor_error
470 ENTRY(device_not_available)
471 pushl $-1 # mark this as an int
474 testl $0x4, %eax # EM (math emulation bit)
475 jne device_not_available_emulate
477 call math_state_restore
478 jmp ret_from_exception
479 device_not_available_emulate:
480 pushl $0 # temporary storage for ORIG_EIP
483 jmp ret_from_exception
486 * Debug traps and NMI can happen at the one SYSENTER instruction
487 * that sets up the real kernel stack. Check here, since we can't
488 * allow the wrong stack to be used.
490 * "TSS_sysenter_esp0+12" is because the NMI/debug handler will have
491 * already pushed 3 words if it hits on the sysenter instruction:
492 * eflags, cs and eip.
494 * We just load the right stack, and push the three (known) values
495 * by hand onto the new stack - while updating the return eip past
496 * the instruction that would have done it for sysenter.
498 #define FIX_STACK(offset, ok, label) \
499 cmpw $__KERNEL_CS,4(%esp); \
502 movl TSS_sysenter_esp0+offset(%esp),%esp; \
504 pushl $__KERNEL_CS; \
505 pushl $sysenter_past_esp
508 cmpl $sysenter_entry,(%esp)
509 jne debug_stack_correct
510 FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
512 pushl $-1 # mark this as an int
514 xorl %edx,%edx # error code 0
515 movl %esp,%eax # pt_regs pointer
519 jmp ret_from_exception
522 * NMI is doubly nasty. It can happen _while_ we're handling
523 * a debug fault, and the debug fault hasn't yet been able to
524 * clear up the stack. So we first check whether we got an
525 * NMI on the sysenter entry path, but after that we need to
526 * check whether we got an NMI on the debug path where the debug
527 * fault happened on the sysenter path.
532 cmpw $__ESPFIX_SS, %ax
535 cmpl $sysenter_entry,(%esp)
539 /* Do not access memory above the end of our stack page,
540 * it might not exist.
542 andl $(THREAD_SIZE-1),%eax
543 cmpl $(THREAD_SIZE-20),%eax
545 jae nmi_stack_correct
546 cmpl $sysenter_entry,12(%esp)
547 je nmi_debug_stack_check
551 xorl %edx,%edx # zero error code
552 movl %esp,%eax # pt_regs pointer
557 FIX_STACK(12,nmi_stack_correct, 1)
558 jmp nmi_stack_correct
559 nmi_debug_stack_check:
560 cmpw $__KERNEL_CS,16(%esp)
561 jne nmi_stack_correct
562 cmpl $debug - 1,(%esp)
563 jle nmi_stack_correct
564 cmpl $debug_esp_fix_insn,(%esp)
565 jle nmi_debug_stack_fixup
566 nmi_debug_stack_fixup:
567 FIX_STACK(24,nmi_stack_correct, 1)
568 jmp nmi_stack_correct
571 /* create the pointer to lss back */
576 /* copy the iret frame of 12 bytes */
582 FIXUP_ESPFIX_STACK # %eax == %esp
583 xorl %edx,%edx # zero error code
586 lss 12+4(%esp), %esp # back to 16bit stack
588 .section __ex_table,"a"
594 pushl $-1 # mark this as an int
596 xorl %edx,%edx # zero error code
597 movl %esp,%eax # pt_regs pointer
601 jmp ret_from_exception
618 ENTRY(coprocessor_segment_overrun)
620 pushl $do_coprocessor_segment_overrun
624 pushl $do_invalid_TSS
627 ENTRY(segment_not_present)
628 pushl $do_segment_not_present
632 pushl $do_stack_segment
635 ENTRY(general_protection)
636 pushl $do_general_protection
639 ENTRY(alignment_check)
640 pushl $do_alignment_check
647 #ifdef CONFIG_X86_MCE
650 pushl machine_check_vector
654 ENTRY(spurious_interrupt_bug)
656 pushl $do_spurious_interrupt_bug
660 ENTRY(sys_call_table)
661 .long sys_restart_syscall /* 0 - old "setup()" system call, used for restarting */
666 .long sys_open /* 5 */
671 .long sys_unlink /* 10 */
676 .long sys_chmod /* 15 */
678 .long sys_ni_syscall /* old break syscall holder */
681 .long sys_getpid /* 20 */
686 .long sys_stime /* 25 */
691 .long sys_utime /* 30 */
692 .long sys_ni_syscall /* old stty syscall holder */
693 .long sys_ni_syscall /* old gtty syscall holder */
696 .long sys_ni_syscall /* 35 - old ftime syscall holder */
701 .long sys_rmdir /* 40 */
705 .long sys_ni_syscall /* old prof syscall holder */
706 .long sys_brk /* 45 */
711 .long sys_getegid16 /* 50 */
713 .long sys_umount /* recycled never used phys() */
714 .long sys_ni_syscall /* old lock syscall holder */
716 .long sys_fcntl /* 55 */
717 .long sys_ni_syscall /* old mpx syscall holder */
719 .long sys_ni_syscall /* old ulimit syscall holder */
721 .long sys_umask /* 60 */
726 .long sys_getpgrp /* 65 */
731 .long sys_setreuid16 /* 70 */
735 .long sys_sethostname
736 .long sys_setrlimit /* 75 */
737 .long sys_old_getrlimit
739 .long sys_gettimeofday
740 .long sys_settimeofday
741 .long sys_getgroups16 /* 80 */
742 .long sys_setgroups16
746 .long sys_readlink /* 85 */
751 .long old_mmap /* 90 */
756 .long sys_fchown16 /* 95 */
757 .long sys_getpriority
758 .long sys_setpriority
759 .long sys_ni_syscall /* old profil syscall holder */
761 .long sys_fstatfs /* 100 */
766 .long sys_getitimer /* 105 */
771 .long sys_iopl /* 110 */
773 .long sys_ni_syscall /* old "idle" system call */
776 .long sys_swapoff /* 115 */
781 .long sys_clone /* 120 */
782 .long sys_setdomainname
786 .long sys_mprotect /* 125 */
787 .long sys_sigprocmask
788 .long sys_ni_syscall /* old "create_module" */
789 .long sys_init_module
790 .long sys_delete_module
791 .long sys_ni_syscall /* 130: old "get_kernel_syms" */
796 .long sys_sysfs /* 135 */
797 .long sys_personality
798 .long sys_ni_syscall /* reserved for afs_syscall */
801 .long sys_llseek /* 140 */
806 .long sys_readv /* 145 */
811 .long sys_mlock /* 150 */
815 .long sys_sched_setparam
816 .long sys_sched_getparam /* 155 */
817 .long sys_sched_setscheduler
818 .long sys_sched_getscheduler
819 .long sys_sched_yield
820 .long sys_sched_get_priority_max
821 .long sys_sched_get_priority_min /* 160 */
822 .long sys_sched_rr_get_interval
825 .long sys_setresuid16
826 .long sys_getresuid16 /* 165 */
828 .long sys_ni_syscall /* Old sys_query_module */
831 .long sys_setresgid16 /* 170 */
832 .long sys_getresgid16
834 .long sys_rt_sigreturn
835 .long sys_rt_sigaction
836 .long sys_rt_sigprocmask /* 175 */
837 .long sys_rt_sigpending
838 .long sys_rt_sigtimedwait
839 .long sys_rt_sigqueueinfo
840 .long sys_rt_sigsuspend
841 .long sys_pread64 /* 180 */
846 .long sys_capset /* 185 */
847 .long sys_sigaltstack
849 .long sys_ni_syscall /* reserved for streams1 */
850 .long sys_ni_syscall /* reserved for streams2 */
851 .long sys_vfork /* 190 */
855 .long sys_ftruncate64
856 .long sys_stat64 /* 195 */
861 .long sys_getgid /* 200 */
866 .long sys_getgroups /* 205 */
871 .long sys_setresgid /* 210 */
876 .long sys_setfsuid /* 215 */
881 .long sys_getdents64 /* 220 */
883 .long sys_ni_syscall /* reserved for TUX */
886 .long sys_readahead /* 225 */
891 .long sys_lgetxattr /* 230 */
896 .long sys_removexattr /* 235 */
897 .long sys_lremovexattr
898 .long sys_fremovexattr
901 .long sys_futex /* 240 */
902 .long sys_sched_setaffinity
903 .long sys_sched_getaffinity
904 .long sys_set_thread_area
905 .long sys_get_thread_area
906 .long sys_io_setup /* 245 */
908 .long sys_io_getevents
911 .long sys_fadvise64 /* 250 */
914 .long sys_lookup_dcookie
915 .long sys_epoll_create
916 .long sys_epoll_ctl /* 255 */
918 .long sys_remap_file_pages
919 .long sys_set_tid_address
920 .long sys_timer_create
921 .long sys_timer_settime /* 260 */
922 .long sys_timer_gettime
923 .long sys_timer_getoverrun
924 .long sys_timer_delete
925 .long sys_clock_settime
926 .long sys_clock_gettime /* 265 */
927 .long sys_clock_getres
928 .long sys_clock_nanosleep
931 .long sys_tgkill /* 270 */
933 .long sys_fadvise64_64
934 .long sys_ni_syscall /* sys_vserver */
936 .long sys_get_mempolicy
937 .long sys_set_mempolicy
940 .long sys_mq_timedsend
941 .long sys_mq_timedreceive /* 280 */
943 .long sys_mq_getsetattr
944 .long sys_ni_syscall /* reserved for kexec */
946 .long sys_ni_syscall /* 285 */ /* available */
948 .long sys_request_key
951 syscall_table_size=(.-sys_call_table)