arch/arm/kernel/entry-common.S

   1 /*
   2  *  linux/arch/arm/kernel/entry-common.S
   3  *
   4  *  Copyright (C) 2000 Russell King
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10
  11 #include <asm/assembler.h>
  12 #include <asm/unistd.h>
  13 #include <asm/ftrace.h>
  14 #include <asm/unwind.h>
  15 #ifdef CONFIG_AEABI
  16 #include <asm/unistd-oabi.h>
  17 #endif
  18
  19         .equ    NR_syscalls, __NR_syscalls
  20
  21 #ifdef CONFIG_NEED_RET_TO_USER
  22 #include <mach/entry-macro.S>
  23 #else
  24         .macro  arch_ret_to_user, tmp1, tmp2
  25         .endm
  26 #endif
  27
  28 #include "entry-header.S"
  29
  30 saved_psr       .req    r8
  31 #if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING)
  32 saved_pc        .req    r9
  33 #define TRACE(x...) x
  34 #else
  35 saved_pc        .req    lr
  36 #define TRACE(x...)
  37 #endif
  38
  39         .align  5
  40 #if !(IS_ENABLED(CONFIG_TRACE_IRQFLAGS) || IS_ENABLED(CONFIG_CONTEXT_TRACKING))
  41 /*
  42  * This is the fast syscall return path.  We do as little as possible here,
  43  * such as avoiding writing r0 to the stack.  We only use this path if we
  44  * have tracing and context tracking disabled - the overheads from those
  45  * features make this path too inefficient.
  46  */
  47 ret_fast_syscall:
  48  UNWIND(.fnstart        )
  49  UNWIND(.cantunwind     )
  50         disable_irq_notrace                     @ disable interrupts
  51         ldr     r1, [tsk, #TI_FLAGS]            @ re-check for syscall tracing
  52         tst     r1, #_TIF_SYSCALL_WORK
  53         bne     fast_work_pending
  54         tst     r1, #_TIF_WORK_MASK
  55         bne     fast_work_pending
  56
  57         /* perform architecture specific actions before user return */
  58         arch_ret_to_user r1, lr
  59
  60         restore_user_regs fast = 1, offset = S_OFF
  61  UNWIND(.fnend          )
  62 ENDPROC(ret_fast_syscall)
  63
  64         /* Ok, we need to do extra processing, enter the slow path. */
  65 fast_work_pending:
  66         str     r0, [sp, #S_R0+S_OFF]!          @ returned r0
  67         /* fall through to work_pending */
  68 #else
  69 /*
  70  * The "replacement" ret_fast_syscall for when tracing or context tracking
  71  * is enabled.  As we will need to call out to some C functions, we save
  72  * r0 first to avoid needing to save registers around each C function call.
  73  */
  74 ret_fast_syscall:
  75  UNWIND(.fnstart        )
  76  UNWIND(.cantunwind     )
  77         str     r0, [sp, #S_R0 + S_OFF]!        @ save returned r0
  78         disable_irq_notrace                     @ disable interrupts
  79         ldr     r1, [tsk, #TI_FLAGS]            @ re-check for syscall tracing
  80         tst     r1, #_TIF_SYSCALL_WORK
  81         bne     fast_work_pending
  82         tst     r1, #_TIF_WORK_MASK
  83         beq     no_work_pending
  84  UNWIND(.fnend          )
  85 ENDPROC(ret_fast_syscall)
  86
  87         /* Slower path - fall through to work_pending */
  88 fast_work_pending:
  89 #endif
  90
  91         tst     r1, #_TIF_SYSCALL_WORK
  92         bne     __sys_trace_return_nosave
  93 slow_work_pending:
  94         mov     r0, sp                          @ 'regs'
  95         mov     r2, why                         @ 'syscall'
  96         bl      do_work_pending
  97         cmp     r0, #0
  98         beq     no_work_pending
  99         movlt   scno, #(__NR_restart_syscall - __NR_SYSCALL_BASE)
 100         ldmia   sp, {r0 - r6}                   @ have to reload r0 - r6
 101         b       local_restart                   @ ... and off we go
 102 ENDPROC(ret_fast_syscall)
 103
 104 /*
 105  * "slow" syscall return path.  "why" tells us if this was a real syscall.
 106  * IRQs may be enabled here, so always disable them.  Note that we use the
 107  * "notrace" version to avoid calling into the tracing code unnecessarily.
 108  * do_work_pending() will update this state if necessary.
 109  */
 110 ENTRY(ret_to_user)
 111 ret_slow_syscall:
 112         disable_irq_notrace                     @ disable interrupts
 113 ENTRY(ret_to_user_from_irq)
 114         ldr     r1, [tsk, #TI_FLAGS]
 115         tst     r1, #_TIF_WORK_MASK
 116         bne     slow_work_pending
 117 no_work_pending:
 118         asm_trace_hardirqs_on save = 0
 119
 120         /* perform architecture specific actions before user return */
 121         arch_ret_to_user r1, lr
 122         ct_user_enter save = 0
 123
 124         restore_user_regs fast = 0, offset = 0
 125 ENDPROC(ret_to_user_from_irq)
 126 ENDPROC(ret_to_user)
 127
 128 /*
 129  * This is how we return from a fork.
 130  */
 131 ENTRY(ret_from_fork)
 132         bl      schedule_tail
 133         cmp     r5, #0
 134         movne   r0, r4
 135         badrne  lr, 1f
 136         retne   r5
 137 1:      get_thread_info tsk
 138         b       ret_slow_syscall
 139 ENDPROC(ret_from_fork)
 140
 141 /*=============================================================================
 142  * SWI handler
 143  *-----------------------------------------------------------------------------
 144  */
 145
 146         .align  5
 147 ENTRY(vector_swi)
 148 #ifdef CONFIG_CPU_V7M
 149         v7m_exception_entry
 150 #else
 151         sub     sp, sp, #PT_REGS_SIZE
 152         stmia   sp, {r0 - r12}                  @ Calling r0 - r12
 153  ARM(   add     r8, sp, #S_PC           )
 154  ARM(   stmdb   r8, {sp, lr}^           )       @ Calling sp, lr
 155  THUMB( mov     r8, sp                  )
 156  THUMB( store_user_sp_lr r8, r10, S_SP  )       @ calling sp, lr
 157         mrs     saved_psr, spsr                 @ called from non-FIQ mode, so ok.
 158  TRACE( mov     saved_pc, lr            )
 159         str     saved_pc, [sp, #S_PC]           @ Save calling PC
 160         str     saved_psr, [sp, #S_PSR]         @ Save CPSR
 161         str     r0, [sp, #S_OLD_R0]             @ Save OLD_R0
 162 #endif
 163         zero_fp
 164         alignment_trap r10, ip, __cr_alignment
 165         asm_trace_hardirqs_on save=0
 166         enable_irq_notrace
 167         ct_user_exit save=0
 168
 169         /*
 170          * Get the system call number.
 171          */
 172
 173 #if defined(CONFIG_OABI_COMPAT)
 174
 175         /*
 176          * If we have CONFIG_OABI_COMPAT then we need to look at the swi
 177          * value to determine if it is an EABI or an old ABI call.
 178          */
 179 #ifdef CONFIG_ARM_THUMB
 180         tst     saved_psr, #PSR_T_BIT
 181         movne   r10, #0                         @ no thumb OABI emulation
 182  USER(  ldreq   r10, [saved_pc, #-4]    )       @ get SWI instruction
 183 #else
 184  USER(  ldr     r10, [saved_pc, #-4]    )       @ get SWI instruction
 185 #endif
 186  ARM_BE8(rev    r10, r10)                       @ little endian instruction
 187
 188 #elif defined(CONFIG_AEABI)
 189
 190         /*
 191          * Pure EABI user space always put syscall number into scno (r7).
 192          */
 193 #elif defined(CONFIG_ARM_THUMB)
 194         /* Legacy ABI only, possibly thumb mode. */
 195         tst     saved_psr, #PSR_T_BIT           @ this is SPSR from save_user_regs
 196         addne   scno, r7, #__NR_SYSCALL_BASE    @ put OS number in
 197  USER(  ldreq   scno, [saved_pc, #-4]   )
 198
 199 #else
 200         /* Legacy ABI only. */
 201  USER(  ldr     scno, [saved_pc, #-4]   )       @ get SWI instruction
 202 #endif
 203
 204         /* saved_psr and saved_pc are now dead */
 205
 206         uaccess_disable tbl
 207
 208         adr     tbl, sys_call_table             @ load syscall table pointer
 209
 210 #if defined(CONFIG_OABI_COMPAT)
 211         /*
 212          * If the swi argument is zero, this is an EABI call and we do nothing.
 213          *
 214          * If this is an old ABI call, get the syscall number into scno and
 215          * get the old ABI syscall table address.
 216          */
 217         bics    r10, r10, #0xff000000
 218         eorne   scno, r10, #__NR_OABI_SYSCALL_BASE
 219         ldrne   tbl, =sys_oabi_call_table
 220 #elif !defined(CONFIG_AEABI)
 221         bic     scno, scno, #0xff000000         @ mask off SWI op-code
 222         eor     scno, scno, #__NR_SYSCALL_BASE  @ check OS number
 223 #endif
 224         get_thread_info tsk
 225         /*
 226          * Reload the registers that may have been corrupted on entry to
 227          * the syscall assembly (by tracing or context tracking.)
 228          */
 229  TRACE( ldmia   sp, {r0 - r3}           )
 230
 231 local_restart:
 232         ldr     r10, [tsk, #TI_FLAGS]           @ check for syscall tracing
 233         stmdb   sp!, {r4, r5}                   @ push fifth and sixth args
 234
 235         tst     r10, #_TIF_SYSCALL_WORK         @ are we tracing syscalls?
 236         bne     __sys_trace
 237
 238         cmp     scno, #NR_syscalls              @ check upper syscall limit
 239         badr    lr, ret_fast_syscall            @ return address
 240         ldrcc   pc, [tbl, scno, lsl #2]         @ call sys_* routine
 241
 242         add     r1, sp, #S_OFF
 243 2:      cmp     scno, #(__ARM_NR_BASE - __NR_SYSCALL_BASE)
 244         eor     r0, scno, #__NR_SYSCALL_BASE    @ put OS number back
 245         bcs     arm_syscall
 246         mov     why, #0                         @ no longer a real syscall
 247         b       sys_ni_syscall                  @ not private func
 248
 249 #if defined(CONFIG_OABI_COMPAT) || !defined(CONFIG_AEABI)
 250         /*
 251          * We failed to handle a fault trying to access the page
 252          * containing the swi instruction, but we're not really in a
 253          * position to return -EFAULT. Instead, return back to the
 254          * instruction and re-enter the user fault handling path trying
 255          * to page it in. This will likely result in sending SEGV to the
 256          * current task.
 257          */
 258 9001:
 259         sub     lr, saved_pc, #4
 260         str     lr, [sp, #S_PC]
 261         get_thread_info tsk
 262         b       ret_fast_syscall
 263 #endif
 264 ENDPROC(vector_swi)
 265
 266         /*
 267          * This is the really slow path.  We're going to be doing
 268          * context switches, and waiting for our parent to respond.
 269          */
 270 __sys_trace:
 271         mov     r1, scno
 272         add     r0, sp, #S_OFF
 273         bl      syscall_trace_enter
 274
 275         badr    lr, __sys_trace_return          @ return address
 276         mov     scno, r0                        @ syscall number (possibly new)
 277         add     r1, sp, #S_R0 + S_OFF           @ pointer to regs
 278         cmp     scno, #NR_syscalls              @ check upper syscall limit
 279         ldmccia r1, {r0 - r6}                   @ have to reload r0 - r6
 280         stmccia sp, {r4, r5}                    @ and update the stack args
 281         ldrcc   pc, [tbl, scno, lsl #2]         @ call sys_* routine
 282         cmp     scno, #-1                       @ skip the syscall?
 283         bne     2b
 284         add     sp, sp, #S_OFF                  @ restore stack
 285         b       ret_slow_syscall
 286
 287 __sys_trace_return:
 288         str     r0, [sp, #S_R0 + S_OFF]!        @ save returned r0
 289         mov     r0, sp
 290         bl      syscall_trace_exit
 291         b       ret_slow_syscall
 292
 293 __sys_trace_return_nosave:
 294         enable_irq_notrace
 295         mov     r0, sp
 296         bl      syscall_trace_exit
 297         b       ret_slow_syscall
 298
 299         .align  5
 300 #ifdef CONFIG_ALIGNMENT_TRAP
 301         .type   __cr_alignment, #object
 302 __cr_alignment:
 303         .word   cr_alignment
 304 #endif
 305         .ltorg
 306
 307         .macro  syscall_table_start, sym
 308         .equ    __sys_nr, 0
 309         .type   \sym, #object
 310 ENTRY(\sym)
 311         .endm
 312
 313         .macro  syscall, nr, func
 314         .ifgt   __sys_nr - \nr
 315         .error  "Duplicated/unorded system call entry"
 316         .endif
 317         .rept   \nr - __sys_nr
 318         .long   sys_ni_syscall
 319         .endr
 320         .long   \func
 321         .equ    __sys_nr, \nr + 1
 322         .endm
 323
 324         .macro  syscall_table_end, sym
 325         .ifgt   __sys_nr - __NR_syscalls
 326         .error  "System call table too big"
 327         .endif
 328         .rept   __NR_syscalls - __sys_nr
 329         .long   sys_ni_syscall
 330         .endr
 331         .size   \sym, . - \sym
 332         .endm
 333
 334 #define NATIVE(nr, func) syscall nr, func
 335
 336 /*
 337  * This is the syscall table declaration for native ABI syscalls.
 338  * With EABI a couple syscalls are obsolete and defined as sys_ni_syscall.
 339  */
 340         syscall_table_start sys_call_table
 341 #define COMPAT(nr, native, compat) syscall nr, native
 342 #ifdef CONFIG_AEABI
 343 #include <calls-eabi.S>
 344 #else
 345 #include <calls-oabi.S>
 346 #endif
 347 #undef COMPAT
 348         syscall_table_end sys_call_table
 349
 350 /*============================================================================
 351  * Special system call wrappers
 352  */
 353 @ r0 = syscall number
 354 @ r8 = syscall table
 355 sys_syscall:
 356                 bic     scno, r0, #__NR_OABI_SYSCALL_BASE
 357                 cmp     scno, #__NR_syscall - __NR_SYSCALL_BASE
 358                 cmpne   scno, #NR_syscalls      @ check range
 359                 stmloia sp, {r5, r6}            @ shuffle args
 360                 movlo   r0, r1
 361                 movlo   r1, r2
 362                 movlo   r2, r3
 363                 movlo   r3, r4
 364                 ldrlo   pc, [tbl, scno, lsl #2]
 365                 b       sys_ni_syscall
 366 ENDPROC(sys_syscall)
 367
 368 sys_sigreturn_wrapper:
 369                 add     r0, sp, #S_OFF
 370                 mov     why, #0         @ prevent syscall restart handling
 371                 b       sys_sigreturn
 372 ENDPROC(sys_sigreturn_wrapper)
 373
 374 sys_rt_sigreturn_wrapper:
 375                 add     r0, sp, #S_OFF
 376                 mov     why, #0         @ prevent syscall restart handling
 377                 b       sys_rt_sigreturn
 378 ENDPROC(sys_rt_sigreturn_wrapper)
 379
 380 sys_statfs64_wrapper:
 381                 teq     r1, #88
 382                 moveq   r1, #84
 383                 b       sys_statfs64
 384 ENDPROC(sys_statfs64_wrapper)
 385
 386 sys_fstatfs64_wrapper:
 387                 teq     r1, #88
 388                 moveq   r1, #84
 389                 b       sys_fstatfs64
 390 ENDPROC(sys_fstatfs64_wrapper)
 391
 392 /*
 393  * Note: off_4k (r5) is always units of 4K.  If we can't do the requested
 394  * offset, we return EINVAL.
 395  */
 396 sys_mmap2:
 397 #if PAGE_SHIFT > 12
 398                 tst     r5, #PGOFF_MASK
 399                 moveq   r5, r5, lsr #PAGE_SHIFT - 12
 400                 streq   r5, [sp, #4]
 401                 beq     sys_mmap_pgoff
 402                 mov     r0, #-EINVAL
 403                 ret     lr
 404 #else
 405                 str     r5, [sp, #4]
 406                 b       sys_mmap_pgoff
 407 #endif
 408 ENDPROC(sys_mmap2)
 409
 410 #ifdef CONFIG_OABI_COMPAT
 411
 412 /*
 413  * These are syscalls with argument register differences
 414  */
 415
 416 sys_oabi_pread64:
 417                 stmia   sp, {r3, r4}
 418                 b       sys_pread64
 419 ENDPROC(sys_oabi_pread64)
 420
 421 sys_oabi_pwrite64:
 422                 stmia   sp, {r3, r4}
 423                 b       sys_pwrite64
 424 ENDPROC(sys_oabi_pwrite64)
 425
 426 sys_oabi_truncate64:
 427                 mov     r3, r2
 428                 mov     r2, r1
 429                 b       sys_truncate64
 430 ENDPROC(sys_oabi_truncate64)
 431
 432 sys_oabi_ftruncate64:
 433                 mov     r3, r2
 434                 mov     r2, r1
 435                 b       sys_ftruncate64
 436 ENDPROC(sys_oabi_ftruncate64)
 437
 438 sys_oabi_readahead:
 439                 str     r3, [sp]
 440                 mov     r3, r2
 441                 mov     r2, r1
 442                 b       sys_readahead
 443 ENDPROC(sys_oabi_readahead)
 444
 445 /*
 446  * Let's declare a second syscall table for old ABI binaries
 447  * using the compatibility syscall entries.
 448  */
 449         syscall_table_start sys_oabi_call_table
 450 #define COMPAT(nr, native, compat) syscall nr, compat
 451 #include <calls-oabi.S>
 452         syscall_table_end sys_oabi_call_table
 453
 454 #endif
 455