arch/s390/kernel/entry.S

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  *    S390 low-level entry points.
   4  *
   5  *    Copyright IBM Corp. 1999, 2012
   6  *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
   7  *               Hartmut Penner (hp@de.ibm.com),
   8  *               Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
   9  *               Heiko Carstens <heiko.carstens@de.ibm.com>
  10  */
  11
  12 #include <linux/init.h>
  13 #include <linux/linkage.h>
  14 #include <asm/alternative-asm.h>
  15 #include <asm/processor.h>
  16 #include <asm/cache.h>
  17 #include <asm/ctl_reg.h>
  18 #include <asm/dwarf.h>
  19 #include <asm/errno.h>
  20 #include <asm/ptrace.h>
  21 #include <asm/thread_info.h>
  22 #include <asm/asm-offsets.h>
  23 #include <asm/unistd.h>
  24 #include <asm/page.h>
  25 #include <asm/sigp.h>
  26 #include <asm/irq.h>
  27 #include <asm/vx-insn.h>
  28 #include <asm/setup.h>
  29 #include <asm/nmi.h>
  30 #include <asm/export.h>
  31 #include <asm/nospec-insn.h>
  32
  33 __PT_R0      =  __PT_GPRS
  34 __PT_R1      =  __PT_GPRS + 8
  35 __PT_R2      =  __PT_GPRS + 16
  36 __PT_R3      =  __PT_GPRS + 24
  37 __PT_R4      =  __PT_GPRS + 32
  38 __PT_R5      =  __PT_GPRS + 40
  39 __PT_R6      =  __PT_GPRS + 48
  40 __PT_R7      =  __PT_GPRS + 56
  41 __PT_R8      =  __PT_GPRS + 64
  42 __PT_R9      =  __PT_GPRS + 72
  43 __PT_R10     =  __PT_GPRS + 80
  44 __PT_R11     =  __PT_GPRS + 88
  45 __PT_R12     =  __PT_GPRS + 96
  46 __PT_R13     =  __PT_GPRS + 104
  47 __PT_R14     =  __PT_GPRS + 112
  48 __PT_R15     =  __PT_GPRS + 120
  49
  50 STACK_SHIFT = PAGE_SHIFT + THREAD_SIZE_ORDER
  51 STACK_SIZE  = 1 << STACK_SHIFT
  52 STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
  53
  54 _TIF_WORK       = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
  55                    _TIF_UPROBE | _TIF_GUARDED_STORAGE | _TIF_PATCH_PENDING | \
  56                    _TIF_NOTIFY_SIGNAL)
  57 _TIF_TRACE      = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
  58                    _TIF_SYSCALL_TRACEPOINT)
  59 _CIF_WORK       = (_CIF_ASCE_PRIMARY | _CIF_ASCE_SECONDARY | _CIF_FPU)
  60 _PIF_WORK       = (_PIF_PER_TRAP | _PIF_SYSCALL_RESTART)
  61
  62 _LPP_OFFSET     = __LC_LPP
  63
  64         .macro  TRACE_IRQS_ON
  65 #ifdef CONFIG_TRACE_IRQFLAGS
  66         basr    %r2,%r0
  67         brasl   %r14,trace_hardirqs_on_caller
  68 #endif
  69         .endm
  70
  71         .macro  TRACE_IRQS_OFF
  72 #ifdef CONFIG_TRACE_IRQFLAGS
  73         basr    %r2,%r0
  74         brasl   %r14,trace_hardirqs_off_caller
  75 #endif
  76         .endm
  77
  78         .macro  LOCKDEP_SYS_EXIT
  79 #ifdef CONFIG_LOCKDEP
  80         tm      __PT_PSW+1(%r11),0x01   # returning to user ?
  81         jz      .+10
  82         brasl   %r14,lockdep_sys_exit
  83 #endif
  84         .endm
  85
  86         .macro  CHECK_STACK savearea
  87 #ifdef CONFIG_CHECK_STACK
  88         tml     %r15,STACK_SIZE - CONFIG_STACK_GUARD
  89         lghi    %r14,\savearea
  90         jz      stack_overflow
  91 #endif
  92         .endm
  93
  94         .macro  CHECK_VMAP_STACK savearea,oklabel
  95 #ifdef CONFIG_VMAP_STACK
  96         lgr     %r14,%r15
  97         nill    %r14,0x10000 - STACK_SIZE
  98         oill    %r14,STACK_INIT
  99         clg     %r14,__LC_KERNEL_STACK
 100         je      \oklabel
 101         clg     %r14,__LC_ASYNC_STACK
 102         je      \oklabel
 103         clg     %r14,__LC_NODAT_STACK
 104         je      \oklabel
 105         clg     %r14,__LC_RESTART_STACK
 106         je      \oklabel
 107         lghi    %r14,\savearea
 108         j       stack_overflow
 109 #else
 110         j       \oklabel
 111 #endif
 112         .endm
 113
 114         .macro  SWITCH_ASYNC savearea,timer
 115         tmhh    %r8,0x0001              # interrupting from user ?
 116         jnz     2f
 117 #if IS_ENABLED(CONFIG_KVM)
 118         lgr     %r14,%r9
 119         larl    %r13,.Lsie_gmap
 120         slgr    %r14,%r13
 121         lghi    %r13,.Lsie_done - .Lsie_gmap
 122         clgr    %r14,%r13
 123         jhe     0f
 124         lghi    %r11,\savearea          # inside critical section, do cleanup
 125         brasl   %r14,.Lcleanup_sie
 126 #endif
 127 0:      larl    %r13,.Lpsw_idle_exit
 128         cgr     %r13,%r9
 129         jne     1f
 130
 131         mvc     __CLOCK_IDLE_EXIT(8,%r2), __LC_INT_CLOCK
 132         mvc     __TIMER_IDLE_EXIT(8,%r2), __LC_ASYNC_ENTER_TIMER
 133         # account system time going idle
 134         ni      __LC_CPU_FLAGS+7,255-_CIF_ENABLED_WAIT
 135
 136         lg      %r13,__LC_STEAL_TIMER
 137         alg     %r13,__CLOCK_IDLE_ENTER(%r2)
 138         slg     %r13,__LC_LAST_UPDATE_CLOCK
 139         stg     %r13,__LC_STEAL_TIMER
 140
 141         mvc     __LC_LAST_UPDATE_CLOCK(8),__CLOCK_IDLE_EXIT(%r2)
 142
 143         lg      %r13,__LC_SYSTEM_TIMER
 144         alg     %r13,__LC_LAST_UPDATE_TIMER
 145         slg     %r13,__TIMER_IDLE_ENTER(%r2)
 146         stg     %r13,__LC_SYSTEM_TIMER
 147         mvc     __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2)
 148
 149         nihh    %r8,0xfcfd              # clear wait state and irq bits
 150 1:      lg      %r14,__LC_ASYNC_STACK   # are we already on the target stack?
 151         slgr    %r14,%r15
 152         srag    %r14,%r14,STACK_SHIFT
 153         jnz     3f
 154         CHECK_STACK \savearea
 155         aghi    %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
 156         j       4f
 157 2:      UPDATE_VTIME %r14,%r15,\timer
 158         BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
 159 3:      lg      %r15,__LC_ASYNC_STACK   # load async stack
 160 4:      la      %r11,STACK_FRAME_OVERHEAD(%r15)
 161         .endm
 162
 163         .macro UPDATE_VTIME w1,w2,enter_timer
 164         lg      \w1,__LC_EXIT_TIMER
 165         lg      \w2,__LC_LAST_UPDATE_TIMER
 166         slg     \w1,\enter_timer
 167         slg     \w2,__LC_EXIT_TIMER
 168         alg     \w1,__LC_USER_TIMER
 169         alg     \w2,__LC_SYSTEM_TIMER
 170         stg     \w1,__LC_USER_TIMER
 171         stg     \w2,__LC_SYSTEM_TIMER
 172         mvc     __LC_LAST_UPDATE_TIMER(8),\enter_timer
 173         .endm
 174
 175         .macro RESTORE_SM_CLEAR_PER
 176         stg     %r8,__LC_RETURN_PSW
 177         ni      __LC_RETURN_PSW,0xbf
 178         ssm     __LC_RETURN_PSW
 179         .endm
 180
 181         .macro ENABLE_INTS
 182         stosm   __SF_EMPTY(%r15),3
 183         .endm
 184
 185         .macro ENABLE_INTS_TRACE
 186         TRACE_IRQS_ON
 187         ENABLE_INTS
 188         .endm
 189
 190         .macro DISABLE_INTS
 191         stnsm   __SF_EMPTY(%r15),0xfc
 192         .endm
 193
 194         .macro DISABLE_INTS_TRACE
 195         DISABLE_INTS
 196         TRACE_IRQS_OFF
 197         .endm
 198
 199         .macro STCK savearea
 200 #ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
 201         .insn   s,0xb27c0000,\savearea          # store clock fast
 202 #else
 203         .insn   s,0xb2050000,\savearea          # store clock
 204 #endif
 205         .endm
 206
 207         /*
 208          * The TSTMSK macro generates a test-under-mask instruction by
 209          * calculating the memory offset for the specified mask value.
 210          * Mask value can be any constant.  The macro shifts the mask
 211          * value to calculate the memory offset for the test-under-mask
 212          * instruction.
 213          */
 214         .macro TSTMSK addr, mask, size=8, bytepos=0
 215                 .if (\bytepos < \size) && (\mask >> 8)
 216                         .if (\mask & 0xff)
 217                                 .error "Mask exceeds byte boundary"
 218                         .endif
 219                         TSTMSK \addr, "(\mask >> 8)", \size, "(\bytepos + 1)"
 220                         .exitm
 221                 .endif
 222                 .ifeq \mask
 223                         .error "Mask must not be zero"
 224                 .endif
 225                 off = \size - \bytepos - 1
 226                 tm      off+\addr, \mask
 227         .endm
 228
 229         .macro BPOFF
 230         ALTERNATIVE "", ".long 0xb2e8c000", 82
 231         .endm
 232
 233         .macro BPON
 234         ALTERNATIVE "", ".long 0xb2e8d000", 82
 235         .endm
 236
 237         .macro BPENTER tif_ptr,tif_mask
 238         ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .long 0xb2e8d000", \
 239                     "", 82
 240         .endm
 241
 242         .macro BPEXIT tif_ptr,tif_mask
 243         TSTMSK  \tif_ptr,\tif_mask
 244         ALTERNATIVE "jz .+8;  .long 0xb2e8c000", \
 245                     "jnz .+8; .long 0xb2e8d000", 82
 246         .endm
 247
 248         GEN_BR_THUNK %r9
 249         GEN_BR_THUNK %r14
 250         GEN_BR_THUNK %r14,%r11
 251
 252         .section .kprobes.text, "ax"
 253 .Ldummy:
 254         /*
 255          * This nop exists only in order to avoid that __switch_to starts at
 256          * the beginning of the kprobes text section. In that case we would
 257          * have several symbols at the same address. E.g. objdump would take
 258          * an arbitrary symbol name when disassembling this code.
 259          * With the added nop in between the __switch_to symbol is unique
 260          * again.
 261          */
 262         nop     0
 263
 264 ENTRY(__bpon)
 265         .globl __bpon
 266         BPON
 267         BR_EX   %r14
 268 ENDPROC(__bpon)
 269
 270 /*
 271  * Scheduler resume function, called by switch_to
 272  *  gpr2 = (task_struct *) prev
 273  *  gpr3 = (task_struct *) next
 274  * Returns:
 275  *  gpr2 = prev
 276  */
 277 ENTRY(__switch_to)
 278         stmg    %r6,%r15,__SF_GPRS(%r15)        # store gprs of prev task
 279         lghi    %r4,__TASK_stack
 280         lghi    %r1,__TASK_thread
 281         llill   %r5,STACK_INIT
 282         stg     %r15,__THREAD_ksp(%r1,%r2)      # store kernel stack of prev
 283         lg      %r15,0(%r4,%r3)                 # start of kernel stack of next
 284         agr     %r15,%r5                        # end of kernel stack of next
 285         stg     %r3,__LC_CURRENT                # store task struct of next
 286         stg     %r15,__LC_KERNEL_STACK          # store end of kernel stack
 287         lg      %r15,__THREAD_ksp(%r1,%r3)      # load kernel stack of next
 288         aghi    %r3,__TASK_pid
 289         mvc     __LC_CURRENT_PID(4,%r0),0(%r3)  # store pid of next
 290         lmg     %r6,%r15,__SF_GPRS(%r15)        # load gprs of next task
 291         ALTERNATIVE "", ".insn s,0xb2800000,_LPP_OFFSET", 40
 292         BR_EX   %r14
 293 ENDPROC(__switch_to)
 294
 295 #if IS_ENABLED(CONFIG_KVM)
 296 /*
 297  * sie64a calling convention:
 298  * %r2 pointer to sie control block
 299  * %r3 guest register save area
 300  */
 301 ENTRY(sie64a)
 302         stmg    %r6,%r14,__SF_GPRS(%r15)        # save kernel registers
 303         lg      %r12,__LC_CURRENT
 304         stg     %r2,__SF_SIE_CONTROL(%r15)      # save control block pointer
 305         stg     %r3,__SF_SIE_SAVEAREA(%r15)     # save guest register save area
 306         xc      __SF_SIE_REASON(8,%r15),__SF_SIE_REASON(%r15) # reason code = 0
 307         mvc     __SF_SIE_FLAGS(8,%r15),__TI_flags(%r12) # copy thread flags
 308         TSTMSK  __LC_CPU_FLAGS,_CIF_FPU         # load guest fp/vx registers ?
 309         jno     .Lsie_load_guest_gprs
 310         brasl   %r14,load_fpu_regs              # load guest fp/vx regs
 311 .Lsie_load_guest_gprs:
 312         lmg     %r0,%r13,0(%r3)                 # load guest gprs 0-13
 313         lg      %r14,__LC_GMAP                  # get gmap pointer
 314         ltgr    %r14,%r14
 315         jz      .Lsie_gmap
 316         lctlg   %c1,%c1,__GMAP_ASCE(%r14)       # load primary asce
 317 .Lsie_gmap:
 318         lg      %r14,__SF_SIE_CONTROL(%r15)     # get control block pointer
 319         oi      __SIE_PROG0C+3(%r14),1          # we are going into SIE now
 320         tm      __SIE_PROG20+3(%r14),3          # last exit...
 321         jnz     .Lsie_skip
 322         TSTMSK  __LC_CPU_FLAGS,_CIF_FPU
 323         jo      .Lsie_skip                      # exit if fp/vx regs changed
 324         BPEXIT  __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
 325 .Lsie_entry:
 326         sie     0(%r14)
 327         BPOFF
 328         BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
 329 .Lsie_skip:
 330         ni      __SIE_PROG0C+3(%r14),0xfe       # no longer in SIE
 331         lctlg   %c1,%c1,__LC_USER_ASCE          # load primary asce
 332 .Lsie_done:
 333 # some program checks are suppressing. C code (e.g. do_protection_exception)
 334 # will rewind the PSW by the ILC, which is often 4 bytes in case of SIE. There
 335 # are some corner cases (e.g. runtime instrumentation) where ILC is unpredictable.
 336 # Other instructions between sie64a and .Lsie_done should not cause program
 337 # interrupts. So lets use 3 nops as a landing pad for all possible rewinds.
 338 # See also .Lcleanup_sie
 339 .Lrewind_pad6:
 340         nopr    7
 341 .Lrewind_pad4:
 342         nopr    7
 343 .Lrewind_pad2:
 344         nopr    7
 345         .globl sie_exit
 346 sie_exit:
 347         lg      %r14,__SF_SIE_SAVEAREA(%r15)    # load guest register save area
 348         stmg    %r0,%r13,0(%r14)                # save guest gprs 0-13
 349         xgr     %r0,%r0                         # clear guest registers to
 350         xgr     %r1,%r1                         # prevent speculative use
 351         xgr     %r2,%r2
 352         xgr     %r3,%r3
 353         xgr     %r4,%r4
 354         xgr     %r5,%r5
 355         lmg     %r6,%r14,__SF_GPRS(%r15)        # restore kernel registers
 356         lg      %r2,__SF_SIE_REASON(%r15)       # return exit reason code
 357         BR_EX   %r14
 358 .Lsie_fault:
 359         lghi    %r14,-EFAULT
 360         stg     %r14,__SF_SIE_REASON(%r15)      # set exit reason code
 361         j       sie_exit
 362
 363         EX_TABLE(.Lrewind_pad6,.Lsie_fault)
 364         EX_TABLE(.Lrewind_pad4,.Lsie_fault)
 365         EX_TABLE(.Lrewind_pad2,.Lsie_fault)
 366         EX_TABLE(sie_exit,.Lsie_fault)
 367 ENDPROC(sie64a)
 368 EXPORT_SYMBOL(sie64a)
 369 EXPORT_SYMBOL(sie_exit)
 370 #endif
 371
 372 /*
 373  * SVC interrupt handler routine. System calls are synchronous events and
 374  * are entered with interrupts disabled.
 375  */
 376
 377 ENTRY(system_call)
 378         stpt    __LC_SYNC_ENTER_TIMER
 379         stmg    %r8,%r15,__LC_SAVE_AREA_SYNC
 380         BPOFF
 381         lg      %r12,__LC_CURRENT
 382         lghi    %r14,_PIF_SYSCALL
 383 .Lsysc_per:
 384         lghi    %r13,__TASK_thread
 385         lg      %r15,__LC_KERNEL_STACK
 386         la      %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
 387         UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER
 388         BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
 389         stmg    %r0,%r7,__PT_R0(%r11)
 390         mvc     __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
 391         mvc     __PT_PSW(16,%r11),__LC_SVC_OLD_PSW
 392         mvc     __PT_INT_CODE(4,%r11),__LC_SVC_ILC
 393         stg     %r14,__PT_FLAGS(%r11)
 394         ENABLE_INTS
 395 .Lsysc_do_svc:
 396         # clear user controlled register to prevent speculative use
 397         xgr     %r0,%r0
 398         # load address of system call table
 399         lg      %r10,__THREAD_sysc_table(%r13,%r12)
 400         llgh    %r8,__PT_INT_CODE+2(%r11)
 401         slag    %r8,%r8,3                       # shift and test for svc 0
 402         jnz     .Lsysc_nr_ok
 403         # svc 0: system call number in %r1
 404         llgfr   %r1,%r1                         # clear high word in r1
 405         sth     %r1,__PT_INT_CODE+2(%r11)
 406         cghi    %r1,NR_syscalls
 407         jnl     .Lsysc_nr_ok
 408         slag    %r8,%r1,3
 409 .Lsysc_nr_ok:
 410         xc      __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
 411         stg     %r2,__PT_ORIG_GPR2(%r11)
 412         stg     %r7,STACK_FRAME_OVERHEAD(%r15)
 413         lg      %r9,0(%r8,%r10)                 # get system call add.
 414         TSTMSK  __TI_flags(%r12),_TIF_TRACE
 415         jnz     .Lsysc_tracesys
 416         BASR_EX %r14,%r9                        # call sys_xxxx
 417         stg     %r2,__PT_R2(%r11)               # store return value
 418
 419 .Lsysc_return:
 420 #ifdef CONFIG_DEBUG_RSEQ
 421         lgr     %r2,%r11
 422         brasl   %r14,rseq_syscall
 423 #endif
 424         LOCKDEP_SYS_EXIT
 425 .Lsysc_tif:
 426         TSTMSK  __PT_FLAGS(%r11),_PIF_WORK
 427         jnz     .Lsysc_work
 428         TSTMSK  __TI_flags(%r12),_TIF_WORK
 429         jnz     .Lsysc_work                     # check for work
 430         TSTMSK  __LC_CPU_FLAGS,(_CIF_WORK-_CIF_FPU)
 431         jnz     .Lsysc_work
 432         BPEXIT  __TI_flags(%r12),_TIF_ISOLATE_BP
 433 .Lsysc_restore:
 434         DISABLE_INTS
 435         TSTMSK  __LC_CPU_FLAGS, _CIF_FPU
 436         jz      .Lsysc_skip_fpu
 437         brasl   %r14,load_fpu_regs
 438 .Lsysc_skip_fpu:
 439         mvc     __LC_RETURN_PSW(16),__PT_PSW(%r11)
 440         stpt    __LC_EXIT_TIMER
 441         lmg     %r0,%r15,__PT_R0(%r11)
 442         b       __LC_RETURN_LPSWE
 443
 444 #
 445 # One of the work bits is on. Find out which one.
 446 #
 447 .Lsysc_work:
 448         TSTMSK  __TI_flags(%r12),_TIF_NEED_RESCHED
 449         jo      .Lsysc_reschedule
 450         TSTMSK  __PT_FLAGS(%r11),_PIF_SYSCALL_RESTART
 451         jo      .Lsysc_syscall_restart
 452 #ifdef CONFIG_UPROBES
 453         TSTMSK  __TI_flags(%r12),_TIF_UPROBE
 454         jo      .Lsysc_uprobe_notify
 455 #endif
 456         TSTMSK  __TI_flags(%r12),_TIF_GUARDED_STORAGE
 457         jo      .Lsysc_guarded_storage
 458         TSTMSK  __PT_FLAGS(%r11),_PIF_PER_TRAP
 459         jo      .Lsysc_singlestep
 460 #ifdef CONFIG_LIVEPATCH
 461         TSTMSK  __TI_flags(%r12),_TIF_PATCH_PENDING
 462         jo      .Lsysc_patch_pending    # handle live patching just before
 463                                         # signals and possible syscall restart
 464 #endif
 465         TSTMSK  __PT_FLAGS(%r11),_PIF_SYSCALL_RESTART
 466         jo      .Lsysc_syscall_restart
 467         TSTMSK  __TI_flags(%r12),(_TIF_SIGPENDING|_TIF_NOTIFY_SIGNAL)
 468         jnz     .Lsysc_sigpending
 469         TSTMSK  __TI_flags(%r12),_TIF_NOTIFY_RESUME
 470         jo      .Lsysc_notify_resume
 471         TSTMSK  __LC_CPU_FLAGS,(_CIF_ASCE_PRIMARY|_CIF_ASCE_SECONDARY)
 472         jnz     .Lsysc_asce
 473         j       .Lsysc_return
 474
 475 #
 476 # _TIF_NEED_RESCHED is set, call schedule
 477 #
 478 .Lsysc_reschedule:
 479         larl    %r14,.Lsysc_return
 480         jg      schedule
 481
 482 #
 483 # _CIF_ASCE_PRIMARY and/or _CIF_ASCE_SECONDARY set, load user space asce
 484 #
 485 .Lsysc_asce:
 486         ni      __LC_CPU_FLAGS+7,255-_CIF_ASCE_SECONDARY
 487         lctlg   %c7,%c7,__LC_VDSO_ASCE          # load secondary asce
 488         TSTMSK  __LC_CPU_FLAGS,_CIF_ASCE_PRIMARY
 489         jz      .Lsysc_return
 490 #ifndef CONFIG_HAVE_MARCH_Z10_FEATURES
 491         tm      __LC_STFLE_FAC_LIST+3,0x10      # has MVCOS ?
 492         jnz     .Lsysc_set_fs_fixup
 493         ni      __LC_CPU_FLAGS+7,255-_CIF_ASCE_PRIMARY
 494         lctlg   %c1,%c1,__LC_USER_ASCE          # load primary asce
 495         j       .Lsysc_return
 496 .Lsysc_set_fs_fixup:
 497 #endif
 498         larl    %r14,.Lsysc_return
 499         jg      set_fs_fixup
 500
 501
 502 #
 503 # _TIF_SIGPENDING is set, call do_signal
 504 #
 505 .Lsysc_sigpending:
 506         lgr     %r2,%r11                # pass pointer to pt_regs
 507         brasl   %r14,do_signal
 508         TSTMSK  __PT_FLAGS(%r11),_PIF_SYSCALL
 509         jno     .Lsysc_return
 510 .Lsysc_do_syscall:
 511         lghi    %r13,__TASK_thread
 512         lmg     %r2,%r7,__PT_R2(%r11)   # load svc arguments
 513         lghi    %r1,0                   # svc 0 returns -ENOSYS
 514         j       .Lsysc_do_svc
 515
 516 #
 517 # _TIF_NOTIFY_RESUME is set, call do_notify_resume
 518 #
 519 .Lsysc_notify_resume:
 520         lgr     %r2,%r11                # pass pointer to pt_regs
 521         larl    %r14,.Lsysc_return
 522         jg      do_notify_resume
 523
 524 #
 525 # _TIF_UPROBE is set, call uprobe_notify_resume
 526 #
 527 #ifdef CONFIG_UPROBES
 528 .Lsysc_uprobe_notify:
 529         lgr     %r2,%r11                # pass pointer to pt_regs
 530         larl    %r14,.Lsysc_return
 531         jg      uprobe_notify_resume
 532 #endif
 533
 534 #
 535 # _TIF_GUARDED_STORAGE is set, call guarded_storage_load
 536 #
 537 .Lsysc_guarded_storage:
 538         lgr     %r2,%r11                # pass pointer to pt_regs
 539         larl    %r14,.Lsysc_return
 540         jg      gs_load_bc_cb
 541 #
 542 # _TIF_PATCH_PENDING is set, call klp_update_patch_state
 543 #
 544 #ifdef CONFIG_LIVEPATCH
 545 .Lsysc_patch_pending:
 546         lg      %r2,__LC_CURRENT        # pass pointer to task struct
 547         larl    %r14,.Lsysc_return
 548         jg      klp_update_patch_state
 549 #endif
 550
 551 #
 552 # _PIF_PER_TRAP is set, call do_per_trap
 553 #
 554 .Lsysc_singlestep:
 555         ni      __PT_FLAGS+7(%r11),255-_PIF_PER_TRAP
 556         lgr     %r2,%r11                # pass pointer to pt_regs
 557         larl    %r14,.Lsysc_return
 558         jg      do_per_trap
 559
 560 #
 561 # _PIF_SYSCALL_RESTART is set, repeat the current system call
 562 #
 563 .Lsysc_syscall_restart:
 564         ni      __PT_FLAGS+7(%r11),255-_PIF_SYSCALL_RESTART
 565         lmg     %r1,%r7,__PT_R1(%r11)   # load svc arguments
 566         lg      %r2,__PT_ORIG_GPR2(%r11)
 567         j       .Lsysc_do_svc
 568
 569 #
 570 # call tracehook_report_syscall_entry/tracehook_report_syscall_exit before
 571 # and after the system call
 572 #
 573 .Lsysc_tracesys:
 574         lgr     %r2,%r11                # pass pointer to pt_regs
 575         la      %r3,0
 576         llgh    %r0,__PT_INT_CODE+2(%r11)
 577         stg     %r0,__PT_R2(%r11)
 578         brasl   %r14,do_syscall_trace_enter
 579         lghi    %r0,NR_syscalls
 580         clgr    %r0,%r2
 581         jnh     .Lsysc_tracenogo
 582         sllg    %r8,%r2,3
 583         lg      %r9,0(%r8,%r10)
 584         lmg     %r3,%r7,__PT_R3(%r11)
 585         stg     %r7,STACK_FRAME_OVERHEAD(%r15)
 586         lg      %r2,__PT_ORIG_GPR2(%r11)
 587         BASR_EX %r14,%r9                # call sys_xxx
 588         stg     %r2,__PT_R2(%r11)       # store return value
 589 .Lsysc_tracenogo:
 590         TSTMSK  __TI_flags(%r12),_TIF_TRACE
 591         jz      .Lsysc_return
 592         lgr     %r2,%r11                # pass pointer to pt_regs
 593         larl    %r14,.Lsysc_return
 594         jg      do_syscall_trace_exit
 595 ENDPROC(system_call)
 596
 597 #
 598 # a new process exits the kernel with ret_from_fork
 599 #
 600 ENTRY(ret_from_fork)
 601         la      %r11,STACK_FRAME_OVERHEAD(%r15)
 602         lg      %r12,__LC_CURRENT
 603         brasl   %r14,schedule_tail
 604         tm      __PT_PSW+1(%r11),0x01   # forking a kernel thread ?
 605         jne     .Lsysc_tracenogo
 606         # it's a kernel thread
 607         lmg     %r9,%r10,__PT_R9(%r11)  # load gprs
 608         la      %r2,0(%r10)
 609         BASR_EX %r14,%r9
 610         j       .Lsysc_tracenogo
 611 ENDPROC(ret_from_fork)
 612
 613 ENTRY(kernel_thread_starter)
 614         la      %r2,0(%r10)
 615         BASR_EX %r14,%r9
 616         j       .Lsysc_tracenogo
 617 ENDPROC(kernel_thread_starter)
 618
 619 /*
 620  * Program check handler routine
 621  */
 622
 623 ENTRY(pgm_check_handler)
 624         stpt    __LC_SYNC_ENTER_TIMER
 625         BPOFF
 626         stmg    %r8,%r15,__LC_SAVE_AREA_SYNC
 627         lg      %r10,__LC_LAST_BREAK
 628         srag    %r11,%r10,12
 629         jnz     0f
 630         /* if __LC_LAST_BREAK is < 4096, it contains one of
 631          * the lpswe addresses in lowcore. Set it to 1 (initial state)
 632          * to prevent leaking that address to userspace.
 633          */
 634         lghi    %r10,1
 635 0:      lg      %r12,__LC_CURRENT
 636         lghi    %r11,0
 637         lmg     %r8,%r9,__LC_PGM_OLD_PSW
 638         tmhh    %r8,0x0001              # test problem state bit
 639         jnz     3f                      # -> fault in user space
 640 #if IS_ENABLED(CONFIG_KVM)
 641         # cleanup critical section for program checks in sie64a
 642         lgr     %r14,%r9
 643         larl    %r13,.Lsie_gmap
 644         slgr    %r14,%r13
 645         lghi    %r13,.Lsie_done - .Lsie_gmap
 646         clgr    %r14,%r13
 647         jhe     1f
 648         lg      %r14,__SF_SIE_CONTROL(%r15)     # get control block pointer
 649         ni      __SIE_PROG0C+3(%r14),0xfe       # no longer in SIE
 650         lctlg   %c1,%c1,__LC_USER_ASCE          # load primary asce
 651         larl    %r9,sie_exit                    # skip forward to sie_exit
 652         lghi    %r11,_PIF_GUEST_FAULT
 653 #endif
 654 1:      tmhh    %r8,0x4000              # PER bit set in old PSW ?
 655         jnz     2f                      # -> enabled, can't be a double fault
 656         tm      __LC_PGM_ILC+3,0x80     # check for per exception
 657         jnz     .Lpgm_svcper            # -> single stepped svc
 658 2:      CHECK_STACK __LC_SAVE_AREA_SYNC
 659         aghi    %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
 660         # CHECK_VMAP_STACK branches to stack_overflow or 5f
 661         CHECK_VMAP_STACK __LC_SAVE_AREA_SYNC,5f
 662 3:      UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
 663         BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
 664         lg      %r15,__LC_KERNEL_STACK
 665         lgr     %r14,%r12
 666         aghi    %r14,__TASK_thread      # pointer to thread_struct
 667         lghi    %r13,__LC_PGM_TDB
 668         tm      __LC_PGM_ILC+2,0x02     # check for transaction abort
 669         jz      4f
 670         mvc     __THREAD_trap_tdb(256,%r14),0(%r13)
 671 4:      stg     %r10,__THREAD_last_break(%r14)
 672 5:      lgr     %r13,%r11
 673         la      %r11,STACK_FRAME_OVERHEAD(%r15)
 674         stmg    %r0,%r7,__PT_R0(%r11)
 675         # clear user controlled registers to prevent speculative use
 676         xgr     %r0,%r0
 677         xgr     %r1,%r1
 678         xgr     %r2,%r2
 679         xgr     %r3,%r3
 680         xgr     %r4,%r4
 681         xgr     %r5,%r5
 682         xgr     %r6,%r6
 683         xgr     %r7,%r7
 684         mvc     __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
 685         stmg    %r8,%r9,__PT_PSW(%r11)
 686         mvc     __PT_INT_CODE(4,%r11),__LC_PGM_ILC
 687         mvc     __PT_INT_PARM_LONG(8,%r11),__LC_TRANS_EXC_CODE
 688         stg     %r13,__PT_FLAGS(%r11)
 689         stg     %r10,__PT_ARGS(%r11)
 690         tm      __LC_PGM_ILC+3,0x80     # check for per exception
 691         jz      6f
 692         tmhh    %r8,0x0001              # kernel per event ?
 693         jz      .Lpgm_kprobe
 694         oi      __PT_FLAGS+7(%r11),_PIF_PER_TRAP
 695         mvc     __THREAD_per_address(8,%r14),__LC_PER_ADDRESS
 696         mvc     __THREAD_per_cause(2,%r14),__LC_PER_CODE
 697         mvc     __THREAD_per_paid(1,%r14),__LC_PER_ACCESS_ID
 698 6:      RESTORE_SM_CLEAR_PER
 699         xc      __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
 700         larl    %r1,pgm_check_table
 701         llgh    %r10,__PT_INT_CODE+2(%r11)
 702         nill    %r10,0x007f
 703         sll     %r10,3
 704         je      .Lpgm_return
 705         lg      %r9,0(%r10,%r1)         # load address of handler routine
 706         lgr     %r2,%r11                # pass pointer to pt_regs
 707         BASR_EX %r14,%r9                # branch to interrupt-handler
 708 .Lpgm_return:
 709         LOCKDEP_SYS_EXIT
 710         tm      __PT_PSW+1(%r11),0x01   # returning to user ?
 711         jno     .Lsysc_restore
 712         TSTMSK  __PT_FLAGS(%r11),_PIF_SYSCALL
 713         jo      .Lsysc_do_syscall
 714         j       .Lsysc_tif
 715
 716 #
 717 # PER event in supervisor state, must be kprobes
 718 #
 719 .Lpgm_kprobe:
 720         RESTORE_SM_CLEAR_PER
 721         xc      __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
 722         lgr     %r2,%r11                # pass pointer to pt_regs
 723         brasl   %r14,do_per_trap
 724         j       .Lpgm_return
 725
 726 #
 727 # single stepped system call
 728 #
 729 .Lpgm_svcper:
 730         mvc     __LC_RETURN_PSW(8),__LC_SVC_NEW_PSW
 731         larl    %r14,.Lsysc_per
 732         stg     %r14,__LC_RETURN_PSW+8
 733         lghi    %r14,_PIF_SYSCALL | _PIF_PER_TRAP
 734         lpswe   __LC_RETURN_PSW         # branch to .Lsysc_per
 735 ENDPROC(pgm_check_handler)
 736
 737 /*
 738  * IO interrupt handler routine
 739  */
 740 ENTRY(io_int_handler)
 741         STCK    __LC_INT_CLOCK
 742         stpt    __LC_ASYNC_ENTER_TIMER
 743         BPOFF
 744         stmg    %r8,%r15,__LC_SAVE_AREA_ASYNC
 745         lg      %r12,__LC_CURRENT
 746         lmg     %r8,%r9,__LC_IO_OLD_PSW
 747         SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER
 748         stmg    %r0,%r7,__PT_R0(%r11)
 749         # clear user controlled registers to prevent speculative use
 750         xgr     %r0,%r0
 751         xgr     %r1,%r1
 752         xgr     %r2,%r2
 753         xgr     %r3,%r3
 754         xgr     %r4,%r4
 755         xgr     %r5,%r5
 756         xgr     %r6,%r6
 757         xgr     %r7,%r7
 758         xgr     %r10,%r10
 759         mvc     __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
 760         stmg    %r8,%r9,__PT_PSW(%r11)
 761         mvc     __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
 762         xc      __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
 763         TSTMSK  __LC_CPU_FLAGS,_CIF_IGNORE_IRQ
 764         jo      .Lio_restore
 765 #if IS_ENABLED(CONFIG_TRACE_IRQFLAGS)
 766         tmhh    %r8,0x300
 767         jz      1f
 768         TRACE_IRQS_OFF
 769 1:
 770 #endif
 771         xc      __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
 772 .Lio_loop:
 773         lgr     %r2,%r11                # pass pointer to pt_regs
 774         lghi    %r3,IO_INTERRUPT
 775         tm      __PT_INT_CODE+8(%r11),0x80      # adapter interrupt ?
 776         jz      .Lio_call
 777         lghi    %r3,THIN_INTERRUPT
 778 .Lio_call:
 779         brasl   %r14,do_IRQ
 780         TSTMSK  __LC_MACHINE_FLAGS,MACHINE_FLAG_LPAR
 781         jz      .Lio_return
 782         tpi     0
 783         jz      .Lio_return
 784         mvc     __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
 785         j       .Lio_loop
 786 .Lio_return:
 787         LOCKDEP_SYS_EXIT
 788         TSTMSK  __TI_flags(%r12),_TIF_WORK
 789         jnz     .Lio_work               # there is work to do (signals etc.)
 790         TSTMSK  __LC_CPU_FLAGS,_CIF_WORK
 791         jnz     .Lio_work
 792 .Lio_restore:
 793 #if IS_ENABLED(CONFIG_TRACE_IRQFLAGS)
 794         tm      __PT_PSW(%r11),3
 795         jno     0f
 796         TRACE_IRQS_ON
 797 0:
 798 #endif
 799         mvc     __LC_RETURN_PSW(16),__PT_PSW(%r11)
 800         tm      __PT_PSW+1(%r11),0x01   # returning to user ?
 801         jno     .Lio_exit_kernel
 802         BPEXIT  __TI_flags(%r12),_TIF_ISOLATE_BP
 803         stpt    __LC_EXIT_TIMER
 804 .Lio_exit_kernel:
 805         lmg     %r0,%r15,__PT_R0(%r11)
 806         b       __LC_RETURN_LPSWE
 807 .Lio_done:
 808
 809 #
 810 # There is work todo, find out in which context we have been interrupted:
 811 # 1) if we return to user space we can do all _TIF_WORK work
 812 # 2) if we return to kernel code and kvm is enabled check if we need to
 813 #    modify the psw to leave SIE
 814 # 3) if we return to kernel code and preemptive scheduling is enabled check
 815 #    the preemption counter and if it is zero call preempt_schedule_irq
 816 # Before any work can be done, a switch to the kernel stack is required.
 817 #
 818 .Lio_work:
 819         tm      __PT_PSW+1(%r11),0x01   # returning to user ?
 820         jo      .Lio_work_user          # yes -> do resched & signal
 821 #ifdef CONFIG_PREEMPTION
 822         # check for preemptive scheduling
 823         icm     %r0,15,__LC_PREEMPT_COUNT
 824         jnz     .Lio_restore            # preemption is disabled
 825         TSTMSK  __TI_flags(%r12),_TIF_NEED_RESCHED
 826         jno     .Lio_restore
 827         # switch to kernel stack
 828         lg      %r1,__PT_R15(%r11)
 829         aghi    %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
 830         mvc     STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
 831         xc      __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
 832         la      %r11,STACK_FRAME_OVERHEAD(%r1)
 833         lgr     %r15,%r1
 834         brasl   %r14,preempt_schedule_irq
 835         j       .Lio_return
 836 #else
 837         j       .Lio_restore
 838 #endif
 839
 840 #
 841 # Need to do work before returning to userspace, switch to kernel stack
 842 #
 843 .Lio_work_user:
 844         lg      %r1,__LC_KERNEL_STACK
 845         mvc     STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
 846         xc      __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
 847         la      %r11,STACK_FRAME_OVERHEAD(%r1)
 848         lgr     %r15,%r1
 849
 850 #
 851 # One of the work bits is on. Find out which one.
 852 #
 853         TSTMSK  __TI_flags(%r12),_TIF_NEED_RESCHED
 854         jo      .Lio_reschedule
 855 #ifdef CONFIG_LIVEPATCH
 856         TSTMSK  __TI_flags(%r12),_TIF_PATCH_PENDING
 857         jo      .Lio_patch_pending
 858 #endif
 859         TSTMSK  __TI_flags(%r12),(_TIF_SIGPENDING|_TIF_NOTIFY_SIGNAL)
 860         jnz     .Lio_sigpending
 861         TSTMSK  __TI_flags(%r12),_TIF_NOTIFY_RESUME
 862         jo      .Lio_notify_resume
 863         TSTMSK  __TI_flags(%r12),_TIF_GUARDED_STORAGE
 864         jo      .Lio_guarded_storage
 865         TSTMSK  __LC_CPU_FLAGS,_CIF_FPU
 866         jo      .Lio_vxrs
 867         TSTMSK  __LC_CPU_FLAGS,(_CIF_ASCE_PRIMARY|_CIF_ASCE_SECONDARY)
 868         jnz     .Lio_asce
 869         j       .Lio_return
 870
 871 #
 872 # _CIF_ASCE_PRIMARY and/or CIF_ASCE_SECONDARY set, load user space asce
 873 #
 874 .Lio_asce:
 875         ni      __LC_CPU_FLAGS+7,255-_CIF_ASCE_SECONDARY
 876         lctlg   %c7,%c7,__LC_VDSO_ASCE          # load secondary asce
 877         TSTMSK  __LC_CPU_FLAGS,_CIF_ASCE_PRIMARY
 878         jz      .Lio_return
 879 #ifndef CONFIG_HAVE_MARCH_Z10_FEATURES
 880         tm      __LC_STFLE_FAC_LIST+3,0x10      # has MVCOS ?
 881         jnz     .Lio_set_fs_fixup
 882         ni      __LC_CPU_FLAGS+7,255-_CIF_ASCE_PRIMARY
 883         lctlg   %c1,%c1,__LC_USER_ASCE          # load primary asce
 884         j       .Lio_return
 885 .Lio_set_fs_fixup:
 886 #endif
 887         larl    %r14,.Lio_return
 888         jg      set_fs_fixup
 889
 890 #
 891 # CIF_FPU is set, restore floating-point controls and floating-point registers.
 892 #
 893 .Lio_vxrs:
 894         larl    %r14,.Lio_return
 895         jg      load_fpu_regs
 896
 897 #
 898 # _TIF_GUARDED_STORAGE is set, call guarded_storage_load
 899 #
 900 .Lio_guarded_storage:
 901         ENABLE_INTS_TRACE
 902         lgr     %r2,%r11                # pass pointer to pt_regs
 903         brasl   %r14,gs_load_bc_cb
 904         DISABLE_INTS_TRACE
 905         j       .Lio_return
 906
 907 #
 908 # _TIF_NEED_RESCHED is set, call schedule
 909 #
 910 .Lio_reschedule:
 911         ENABLE_INTS_TRACE
 912         brasl   %r14,schedule           # call scheduler
 913         DISABLE_INTS_TRACE
 914         j       .Lio_return
 915
 916 #
 917 # _TIF_PATCH_PENDING is set, call klp_update_patch_state
 918 #
 919 #ifdef CONFIG_LIVEPATCH
 920 .Lio_patch_pending:
 921         lg      %r2,__LC_CURRENT        # pass pointer to task struct
 922         larl    %r14,.Lio_return
 923         jg      klp_update_patch_state
 924 #endif
 925
 926 #
 927 # _TIF_SIGPENDING or is set, call do_signal
 928 #
 929 .Lio_sigpending:
 930         ENABLE_INTS_TRACE
 931         lgr     %r2,%r11                # pass pointer to pt_regs
 932         brasl   %r14,do_signal
 933         DISABLE_INTS_TRACE
 934         j       .Lio_return
 935
 936 #
 937 # _TIF_NOTIFY_RESUME or is set, call do_notify_resume
 938 #
 939 .Lio_notify_resume:
 940         ENABLE_INTS_TRACE
 941         lgr     %r2,%r11                # pass pointer to pt_regs
 942         brasl   %r14,do_notify_resume
 943         DISABLE_INTS_TRACE
 944         j       .Lio_return
 945 ENDPROC(io_int_handler)
 946
 947 /*
 948  * External interrupt handler routine
 949  */
 950 ENTRY(ext_int_handler)
 951         STCK    __LC_INT_CLOCK
 952         stpt    __LC_ASYNC_ENTER_TIMER
 953         BPOFF
 954         stmg    %r8,%r15,__LC_SAVE_AREA_ASYNC
 955         lg      %r12,__LC_CURRENT
 956         lmg     %r8,%r9,__LC_EXT_OLD_PSW
 957         SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER
 958         stmg    %r0,%r7,__PT_R0(%r11)
 959         # clear user controlled registers to prevent speculative use
 960         xgr     %r0,%r0
 961         xgr     %r1,%r1
 962         xgr     %r2,%r2
 963         xgr     %r3,%r3
 964         xgr     %r4,%r4
 965         xgr     %r5,%r5
 966         xgr     %r6,%r6
 967         xgr     %r7,%r7
 968         xgr     %r10,%r10
 969         mvc     __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
 970         stmg    %r8,%r9,__PT_PSW(%r11)
 971         lghi    %r1,__LC_EXT_PARAMS2
 972         mvc     __PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR
 973         mvc     __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS
 974         mvc     __PT_INT_PARM_LONG(8,%r11),0(%r1)
 975         xc      __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
 976         TSTMSK  __LC_CPU_FLAGS,_CIF_IGNORE_IRQ
 977         jo      .Lio_restore
 978 #if IS_ENABLED(CONFIG_TRACE_IRQFLAGS)
 979         tmhh    %r8,0x300
 980         jz      1f
 981         TRACE_IRQS_OFF
 982 1:
 983 #endif
 984         xc      __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
 985         lgr     %r2,%r11                # pass pointer to pt_regs
 986         lghi    %r3,EXT_INTERRUPT
 987         brasl   %r14,do_IRQ
 988         j       .Lio_return
 989 ENDPROC(ext_int_handler)
 990
 991 /*
 992  * Load idle PSW.
 993  */
 994 ENTRY(psw_idle)
 995         stg     %r3,__SF_EMPTY(%r15)
 996         larl    %r1,.Lpsw_idle_exit
 997         stg     %r1,__SF_EMPTY+8(%r15)
 998         larl    %r1,smp_cpu_mtid
 999         llgf    %r1,0(%r1)
1000         ltgr    %r1,%r1
1001         jz      .Lpsw_idle_stcctm
1002         .insn   rsy,0xeb0000000017,%r1,5,__SF_EMPTY+16(%r15)
1003 .Lpsw_idle_stcctm:
1004         oi      __LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT
1005         BPON
1006         STCK    __CLOCK_IDLE_ENTER(%r2)
1007         stpt    __TIMER_IDLE_ENTER(%r2)
1008         lpswe   __SF_EMPTY(%r15)
1009 .Lpsw_idle_exit:
1010         BR_EX   %r14
1011 ENDPROC(psw_idle)
1012
1013 /*
1014  * Store floating-point controls and floating-point or vector register
1015  * depending whether the vector facility is available.  A critical section
1016  * cleanup assures that the registers are stored even if interrupted for
1017  * some other work.  The CIF_FPU flag is set to trigger a lazy restore
1018  * of the register contents at return from io or a system call.
1019  */
1020 ENTRY(save_fpu_regs)
1021         stnsm   __SF_EMPTY(%r15),0xfc
1022         lg      %r2,__LC_CURRENT
1023         aghi    %r2,__TASK_thread
1024         TSTMSK  __LC_CPU_FLAGS,_CIF_FPU
1025         jo      .Lsave_fpu_regs_exit
1026         stfpc   __THREAD_FPU_fpc(%r2)
1027         lg      %r3,__THREAD_FPU_regs(%r2)
1028         TSTMSK  __LC_MACHINE_FLAGS,MACHINE_FLAG_VX
1029         jz      .Lsave_fpu_regs_fp        # no -> store FP regs
1030         VSTM    %v0,%v15,0,%r3            # vstm 0,15,0(3)
1031         VSTM    %v16,%v31,256,%r3         # vstm 16,31,256(3)
1032         j       .Lsave_fpu_regs_done      # -> set CIF_FPU flag
1033 .Lsave_fpu_regs_fp:
1034         std     0,0(%r3)
1035         std     1,8(%r3)
1036         std     2,16(%r3)
1037         std     3,24(%r3)
1038         std     4,32(%r3)
1039         std     5,40(%r3)
1040         std     6,48(%r3)
1041         std     7,56(%r3)
1042         std     8,64(%r3)
1043         std     9,72(%r3)
1044         std     10,80(%r3)
1045         std     11,88(%r3)
1046         std     12,96(%r3)
1047         std     13,104(%r3)
1048         std     14,112(%r3)
1049         std     15,120(%r3)
1050 .Lsave_fpu_regs_done:
1051         oi      __LC_CPU_FLAGS+7,_CIF_FPU
1052 .Lsave_fpu_regs_exit:
1053         ssm     __SF_EMPTY(%r15)
1054         BR_EX   %r14
1055 .Lsave_fpu_regs_end:
1056 ENDPROC(save_fpu_regs)
1057 EXPORT_SYMBOL(save_fpu_regs)
1058
1059 /*
1060  * Load floating-point controls and floating-point or vector registers.
1061  * A critical section cleanup assures that the register contents are
1062  * loaded even if interrupted for some other work.
1063  *
1064  * There are special calling conventions to fit into sysc and io return work:
1065  *      %r15:   <kernel stack>
1066  * The function requires:
1067  *      %r4
1068  */
1069 load_fpu_regs:
1070         lg      %r4,__LC_CURRENT
1071         aghi    %r4,__TASK_thread
1072         TSTMSK  __LC_CPU_FLAGS,_CIF_FPU
1073         jno     .Lload_fpu_regs_exit
1074         lfpc    __THREAD_FPU_fpc(%r4)
1075         TSTMSK  __LC_MACHINE_FLAGS,MACHINE_FLAG_VX
1076         lg      %r4,__THREAD_FPU_regs(%r4)      # %r4 <- reg save area
1077         jz      .Lload_fpu_regs_fp              # -> no VX, load FP regs
1078         VLM     %v0,%v15,0,%r4
1079         VLM     %v16,%v31,256,%r4
1080         j       .Lload_fpu_regs_done
1081 .Lload_fpu_regs_fp:
1082         ld      0,0(%r4)
1083         ld      1,8(%r4)
1084         ld      2,16(%r4)
1085         ld      3,24(%r4)
1086         ld      4,32(%r4)
1087         ld      5,40(%r4)
1088         ld      6,48(%r4)
1089         ld      7,56(%r4)
1090         ld      8,64(%r4)
1091         ld      9,72(%r4)
1092         ld      10,80(%r4)
1093         ld      11,88(%r4)
1094         ld      12,96(%r4)
1095         ld      13,104(%r4)
1096         ld      14,112(%r4)
1097         ld      15,120(%r4)
1098 .Lload_fpu_regs_done:
1099         ni      __LC_CPU_FLAGS+7,255-_CIF_FPU
1100 .Lload_fpu_regs_exit:
1101         BR_EX   %r14
1102 .Lload_fpu_regs_end:
1103 ENDPROC(load_fpu_regs)
1104
1105 /*
1106  * Machine check handler routines
1107  */
1108 ENTRY(mcck_int_handler)
1109         STCK    __LC_MCCK_CLOCK
1110         BPOFF
1111         la      %r1,4095                # validate r1
1112         spt     __LC_CPU_TIMER_SAVE_AREA-4095(%r1)      # validate cpu timer
1113         sckc    __LC_CLOCK_COMPARATOR                   # validate comparator
1114         lam     %a0,%a15,__LC_AREGS_SAVE_AREA-4095(%r1) # validate acrs
1115         lmg     %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# validate gprs
1116         lg      %r12,__LC_CURRENT
1117         lmg     %r8,%r9,__LC_MCK_OLD_PSW
1118         TSTMSK  __LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE
1119         jo      .Lmcck_panic            # yes -> rest of mcck code invalid
1120         TSTMSK  __LC_MCCK_CODE,MCCK_CODE_CR_VALID
1121         jno     .Lmcck_panic            # control registers invalid -> panic
1122         la      %r14,4095
1123         lctlg   %c0,%c15,__LC_CREGS_SAVE_AREA-4095(%r14) # validate ctl regs
1124         ptlb
1125         lg      %r11,__LC_MCESAD-4095(%r14) # extended machine check save area
1126         nill    %r11,0xfc00             # MCESA_ORIGIN_MASK
1127         TSTMSK  __LC_CREGS_SAVE_AREA+16-4095(%r14),CR2_GUARDED_STORAGE
1128         jno     0f
1129         TSTMSK  __LC_MCCK_CODE,MCCK_CODE_GS_VALID
1130         jno     0f
1131         .insn    rxy,0xe3000000004d,0,__MCESA_GS_SAVE_AREA(%r11) # LGSC
1132 0:      l       %r14,__LC_FP_CREG_SAVE_AREA-4095(%r14)
1133         TSTMSK  __LC_MCCK_CODE,MCCK_CODE_FC_VALID
1134         jo      0f
1135         sr      %r14,%r14
1136 0:      sfpc    %r14
1137         TSTMSK  __LC_MACHINE_FLAGS,MACHINE_FLAG_VX
1138         jo      0f
1139         lghi    %r14,__LC_FPREGS_SAVE_AREA
1140         ld      %f0,0(%r14)
1141         ld      %f1,8(%r14)
1142         ld      %f2,16(%r14)
1143         ld      %f3,24(%r14)
1144         ld      %f4,32(%r14)
1145         ld      %f5,40(%r14)
1146         ld      %f6,48(%r14)
1147         ld      %f7,56(%r14)
1148         ld      %f8,64(%r14)
1149         ld      %f9,72(%r14)
1150         ld      %f10,80(%r14)
1151         ld      %f11,88(%r14)
1152         ld      %f12,96(%r14)
1153         ld      %f13,104(%r14)
1154         ld      %f14,112(%r14)
1155         ld      %f15,120(%r14)
1156         j       1f
1157 0:      VLM     %v0,%v15,0,%r11
1158         VLM     %v16,%v31,256,%r11
1159 1:      lghi    %r14,__LC_CPU_TIMER_SAVE_AREA
1160         mvc     __LC_MCCK_ENTER_TIMER(8),0(%r14)
1161         TSTMSK  __LC_MCCK_CODE,MCCK_CODE_CPU_TIMER_VALID
1162         jo      3f
1163         la      %r14,__LC_SYNC_ENTER_TIMER
1164         clc     0(8,%r14),__LC_ASYNC_ENTER_TIMER
1165         jl      0f
1166         la      %r14,__LC_ASYNC_ENTER_TIMER
1167 0:      clc     0(8,%r14),__LC_EXIT_TIMER
1168         jl      1f
1169         la      %r14,__LC_EXIT_TIMER
1170 1:      clc     0(8,%r14),__LC_LAST_UPDATE_TIMER
1171         jl      2f
1172         la      %r14,__LC_LAST_UPDATE_TIMER
1173 2:      spt     0(%r14)
1174         mvc     __LC_MCCK_ENTER_TIMER(8),0(%r14)
1175 3:      TSTMSK  __LC_MCCK_CODE,MCCK_CODE_PSW_MWP_VALID
1176         jno     .Lmcck_panic
1177         tmhh    %r8,0x0001              # interrupting from user ?
1178         jnz     4f
1179         TSTMSK  __LC_MCCK_CODE,MCCK_CODE_PSW_IA_VALID
1180         jno     .Lmcck_panic
1181 4:      ssm     __LC_PGM_NEW_PSW        # turn dat on, keep irqs off
1182         SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+64,__LC_MCCK_ENTER_TIMER
1183 .Lmcck_skip:
1184         lghi    %r14,__LC_GPREGS_SAVE_AREA+64
1185         stmg    %r0,%r7,__PT_R0(%r11)
1186         # clear user controlled registers to prevent speculative use
1187         xgr     %r0,%r0
1188         xgr     %r1,%r1
1189         xgr     %r2,%r2
1190         xgr     %r3,%r3
1191         xgr     %r4,%r4
1192         xgr     %r5,%r5
1193         xgr     %r6,%r6
1194         xgr     %r7,%r7
1195         xgr     %r10,%r10
1196         mvc     __PT_R8(64,%r11),0(%r14)
1197         stmg    %r8,%r9,__PT_PSW(%r11)
1198         xc      __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
1199         xc      __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
1200         lgr     %r2,%r11                # pass pointer to pt_regs
1201         brasl   %r14,s390_do_machine_check
1202         cghi    %r2,0
1203         je      .Lmcck_return
1204         lg      %r1,__LC_KERNEL_STACK   # switch to kernel stack
1205         mvc     STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
1206         xc      __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
1207         la      %r11,STACK_FRAME_OVERHEAD(%r1)
1208         lgr     %r15,%r1
1209         TRACE_IRQS_OFF
1210         brasl   %r14,s390_handle_mcck
1211         TRACE_IRQS_ON
1212 .Lmcck_return:
1213         lmg     %r0,%r10,__PT_R0(%r11)
1214         mvc     __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
1215         tm      __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
1216         jno     0f
1217         BPEXIT  __TI_flags(%r12),_TIF_ISOLATE_BP
1218         stpt    __LC_EXIT_TIMER
1219 0:      lmg     %r11,%r15,__PT_R11(%r11)
1220         b       __LC_RETURN_MCCK_LPSWE
1221
1222 .Lmcck_panic:
1223         lg      %r15,__LC_NODAT_STACK
1224         la      %r11,STACK_FRAME_OVERHEAD(%r15)
1225         j       .Lmcck_skip
1226 ENDPROC(mcck_int_handler)
1227
1228 #
1229 # PSW restart interrupt handler
1230 #
1231 ENTRY(restart_int_handler)
1232         ALTERNATIVE "", ".insn s,0xb2800000,_LPP_OFFSET", 40
1233         stg     %r15,__LC_SAVE_AREA_RESTART
1234         lg      %r15,__LC_RESTART_STACK
1235         xc      STACK_FRAME_OVERHEAD(__PT_SIZE,%r15),STACK_FRAME_OVERHEAD(%r15)
1236         stmg    %r0,%r14,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
1237         mvc     STACK_FRAME_OVERHEAD+__PT_R15(8,%r15),__LC_SAVE_AREA_RESTART
1238         mvc     STACK_FRAME_OVERHEAD+__PT_PSW(16,%r15),__LC_RST_OLD_PSW
1239         xc      0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
1240         lg      %r1,__LC_RESTART_FN             # load fn, parm & source cpu
1241         lg      %r2,__LC_RESTART_DATA
1242         lg      %r3,__LC_RESTART_SOURCE
1243         ltgr    %r3,%r3                         # test source cpu address
1244         jm      1f                              # negative -> skip source stop
1245 0:      sigp    %r4,%r3,SIGP_SENSE              # sigp sense to source cpu
1246         brc     10,0b                           # wait for status stored
1247 1:      basr    %r14,%r1                        # call function
1248         stap    __SF_EMPTY(%r15)                # store cpu address
1249         llgh    %r3,__SF_EMPTY(%r15)
1250 2:      sigp    %r4,%r3,SIGP_STOP               # sigp stop to current cpu
1251         brc     2,2b
1252 3:      j       3b
1253 ENDPROC(restart_int_handler)
1254
1255         .section .kprobes.text, "ax"
1256
1257 #if defined(CONFIG_CHECK_STACK) || defined(CONFIG_VMAP_STACK)
1258 /*
1259  * The synchronous or the asynchronous stack overflowed. We are dead.
1260  * No need to properly save the registers, we are going to panic anyway.
1261  * Setup a pt_regs so that show_trace can provide a good call trace.
1262  */
1263 ENTRY(stack_overflow)
1264         lg      %r15,__LC_NODAT_STACK   # change to panic stack
1265         la      %r11,STACK_FRAME_OVERHEAD(%r15)
1266         stmg    %r0,%r7,__PT_R0(%r11)
1267         stmg    %r8,%r9,__PT_PSW(%r11)
1268         mvc     __PT_R8(64,%r11),0(%r14)
1269         stg     %r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2
1270         xc      __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
1271         lgr     %r2,%r11                # pass pointer to pt_regs
1272         jg      kernel_stack_overflow
1273 ENDPROC(stack_overflow)
1274 #endif
1275
1276 #if IS_ENABLED(CONFIG_KVM)
1277 .Lcleanup_sie:
1278         cghi    %r11,__LC_SAVE_AREA_ASYNC       #Is this in normal interrupt?
1279         je      1f
1280         larl    %r13,.Lsie_entry
1281         slgr    %r9,%r13
1282         larl    %r13,.Lsie_skip
1283         clgr    %r9,%r13
1284         jh      1f
1285         oi      __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST
1286 1:      BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
1287         lg      %r9,__SF_SIE_CONTROL(%r15)      # get control block pointer
1288         ni      __SIE_PROG0C+3(%r9),0xfe        # no longer in SIE
1289         lctlg   %c1,%c1,__LC_USER_ASCE          # load primary asce
1290         larl    %r9,sie_exit                    # skip forward to sie_exit
1291         BR_EX   %r14,%r11
1292
1293 #endif
1294         .section .rodata, "a"
1295 #define SYSCALL(esame,emu)      .quad __s390x_ ## esame
1296         .globl  sys_call_table
1297 sys_call_table:
1298 #include "asm/syscall_table.h"
1299 #undef SYSCALL
1300
1301 #ifdef CONFIG_COMPAT
1302
1303 #define SYSCALL(esame,emu)      .quad __s390_ ## emu
1304         .globl  sys_call_table_emu
1305 sys_call_table_emu:
1306 #include "asm/syscall_table.h"
1307 #undef SYSCALL
1308 #endif