]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - arch/s390/kernel/entry.S
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:


cc30d1fb000c25c8f74a8045b105762ccf32c9b2
[mirror_ubuntu-bionic-kernel.git] / arch / s390 / kernel / entry.S
1 /*
2 * S390 low-level entry points.
3 *
4 * Copyright IBM Corp. 1999, 2012
5 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
6 * Hartmut Penner (hp@de.ibm.com),
7 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
8 * Heiko Carstens <heiko.carstens@de.ibm.com>
9 */
10
11 #include <linux/init.h>
12 #include <linux/linkage.h>
13 #include <asm/cache.h>
14 #include <asm/errno.h>
15 #include <asm/ptrace.h>
16 #include <asm/thread_info.h>
17 #include <asm/asm-offsets.h>
18 #include <asm/unistd.h>
19 #include <asm/page.h>
20 #include <asm/sigp.h>
21 #include <asm/irq.h>
22
23 __PT_R0 = __PT_GPRS
24 __PT_R1 = __PT_GPRS + 4
25 __PT_R2 = __PT_GPRS + 8
26 __PT_R3 = __PT_GPRS + 12
27 __PT_R4 = __PT_GPRS + 16
28 __PT_R5 = __PT_GPRS + 20
29 __PT_R6 = __PT_GPRS + 24
30 __PT_R7 = __PT_GPRS + 28
31 __PT_R8 = __PT_GPRS + 32
32 __PT_R9 = __PT_GPRS + 36
33 __PT_R10 = __PT_GPRS + 40
34 __PT_R11 = __PT_GPRS + 44
35 __PT_R12 = __PT_GPRS + 48
36 __PT_R13 = __PT_GPRS + 524
37 __PT_R14 = __PT_GPRS + 56
38 __PT_R15 = __PT_GPRS + 60
39
40 _TIF_WORK_SVC = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
41 _TIF_MCCK_PENDING | _TIF_PER_TRAP )
42 _TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
43 _TIF_MCCK_PENDING)
44 _TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
45 _TIF_SYSCALL_TRACEPOINT)
46
47 STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER
48 STACK_SIZE = 1 << STACK_SHIFT
49 STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
50
51 #define BASED(name) name-system_call(%r13)
52
53 .macro TRACE_IRQS_ON
54 #ifdef CONFIG_TRACE_IRQFLAGS
55 basr %r2,%r0
56 l %r1,BASED(.Lhardirqs_on)
57 basr %r14,%r1 # call trace_hardirqs_on_caller
58 #endif
59 .endm
60
61 .macro TRACE_IRQS_OFF
62 #ifdef CONFIG_TRACE_IRQFLAGS
63 basr %r2,%r0
64 l %r1,BASED(.Lhardirqs_off)
65 basr %r14,%r1 # call trace_hardirqs_off_caller
66 #endif
67 .endm
68
69 .macro LOCKDEP_SYS_EXIT
70 #ifdef CONFIG_LOCKDEP
71 tm __PT_PSW+1(%r11),0x01 # returning to user ?
72 jz .+10
73 l %r1,BASED(.Llockdep_sys_exit)
74 basr %r14,%r1 # call lockdep_sys_exit
75 #endif
76 .endm
77
78 .macro CHECK_STACK stacksize,savearea
79 #ifdef CONFIG_CHECK_STACK
80 tml %r15,\stacksize - CONFIG_STACK_GUARD
81 la %r14,\savearea
82 jz stack_overflow
83 #endif
84 .endm
85
86 .macro SWITCH_ASYNC savearea,stack,shift
87 tmh %r8,0x0001 # interrupting from user ?
88 jnz 1f
89 lr %r14,%r9
90 sl %r14,BASED(.Lcritical_start)
91 cl %r14,BASED(.Lcritical_length)
92 jhe 0f
93 la %r11,\savearea # inside critical section, do cleanup
94 bras %r14,cleanup_critical
95 tmh %r8,0x0001 # retest problem state after cleanup
96 jnz 1f
97 0: l %r14,\stack # are we already on the target stack?
98 slr %r14,%r15
99 sra %r14,\shift
100 jnz 1f
101 CHECK_STACK 1<<\shift,\savearea
102 ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
103 j 2f
104 1: l %r15,\stack # load target stack
105 2: la %r11,STACK_FRAME_OVERHEAD(%r15)
106 .endm
107
108 .macro ADD64 high,low,timer
109 al \high,\timer
110 al \low,4+\timer
111 brc 12,.+8
112 ahi \high,1
113 .endm
114
115 .macro SUB64 high,low,timer
116 sl \high,\timer
117 sl \low,4+\timer
118 brc 3,.+8
119 ahi \high,-1
120 .endm
121
122 .macro UPDATE_VTIME high,low,enter_timer
123 lm \high,\low,__LC_EXIT_TIMER
124 SUB64 \high,\low,\enter_timer
125 ADD64 \high,\low,__LC_USER_TIMER
126 stm \high,\low,__LC_USER_TIMER
127 lm \high,\low,__LC_LAST_UPDATE_TIMER
128 SUB64 \high,\low,__LC_EXIT_TIMER
129 ADD64 \high,\low,__LC_SYSTEM_TIMER
130 stm \high,\low,__LC_SYSTEM_TIMER
131 mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer
132 .endm
133
134 .macro REENABLE_IRQS
135 st %r8,__LC_RETURN_PSW
136 ni __LC_RETURN_PSW,0xbf
137 ssm __LC_RETURN_PSW
138 .endm
139
140 .section .kprobes.text, "ax"
141
142 /*
143 * Scheduler resume function, called by switch_to
144 * gpr2 = (task_struct *) prev
145 * gpr3 = (task_struct *) next
146 * Returns:
147 * gpr2 = prev
148 */
149 ENTRY(__switch_to)
150 stm %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task
151 st %r15,__THREAD_ksp(%r2) # store kernel stack of prev
152 l %r4,__THREAD_info(%r2) # get thread_info of prev
153 l %r5,__THREAD_info(%r3) # get thread_info of next
154 lr %r15,%r5
155 ahi %r15,STACK_INIT # end of kernel stack of next
156 st %r3,__LC_CURRENT # store task struct of next
157 st %r5,__LC_THREAD_INFO # store thread info of next
158 st %r15,__LC_KERNEL_STACK # store end of kernel stack
159 lctl %c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4
160 mvc __LC_CURRENT_PID(4,%r0),__TASK_pid(%r3) # store pid of next
161 l %r15,__THREAD_ksp(%r3) # load kernel stack of next
162 tm __TI_flags+3(%r4),_TIF_MCCK_PENDING # machine check pending?
163 jz 0f
164 ni __TI_flags+3(%r4),255-_TIF_MCCK_PENDING # clear flag in prev
165 oi __TI_flags+3(%r5),_TIF_MCCK_PENDING # set it in next
166 0: lm %r6,%r15,__SF_GPRS(%r15) # load gprs of next task
167 br %r14
168
169 __critical_start:
170 /*
171 * SVC interrupt handler routine. System calls are synchronous events and
172 * are executed with interrupts enabled.
173 */
174
175 ENTRY(system_call)
176 stpt __LC_SYNC_ENTER_TIMER
177 sysc_stm:
178 stm %r8,%r15,__LC_SAVE_AREA_SYNC
179 l %r12,__LC_THREAD_INFO
180 l %r13,__LC_SVC_NEW_PSW+4
181 sysc_per:
182 l %r15,__LC_KERNEL_STACK
183 la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
184 sysc_vtime:
185 UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER
186 stm %r0,%r7,__PT_R0(%r11)
187 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC
188 mvc __PT_PSW(8,%r11),__LC_SVC_OLD_PSW
189 mvc __PT_INT_CODE(4,%r11),__LC_SVC_ILC
190 sysc_do_svc:
191 oi __TI_flags+3(%r12),_TIF_SYSCALL
192 l %r10,__TI_sysc_table(%r12) # 31 bit system call table
193 lh %r8,__PT_INT_CODE+2(%r11)
194 sla %r8,2 # shift and test for svc0
195 jnz sysc_nr_ok
196 # svc 0: system call number in %r1
197 cl %r1,BASED(.Lnr_syscalls)
198 jnl sysc_nr_ok
199 sth %r1,__PT_INT_CODE+2(%r11)
200 lr %r8,%r1
201 sla %r8,2
202 sysc_nr_ok:
203 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
204 st %r2,__PT_ORIG_GPR2(%r11)
205 st %r7,STACK_FRAME_OVERHEAD(%r15)
206 l %r9,0(%r8,%r10) # get system call addr.
207 tm __TI_flags+2(%r12),_TIF_TRACE >> 8
208 jnz sysc_tracesys
209 basr %r14,%r9 # call sys_xxxx
210 st %r2,__PT_R2(%r11) # store return value
211
212 sysc_return:
213 LOCKDEP_SYS_EXIT
214 sysc_tif:
215 tm __PT_PSW+1(%r11),0x01 # returning to user ?
216 jno sysc_restore
217 tm __TI_flags+3(%r12),_TIF_WORK_SVC
218 jnz sysc_work # check for work
219 ni __TI_flags+3(%r12),255-_TIF_SYSCALL
220 sysc_restore:
221 mvc __LC_RETURN_PSW(8),__PT_PSW(%r11)
222 stpt __LC_EXIT_TIMER
223 lm %r0,%r15,__PT_R0(%r11)
224 lpsw __LC_RETURN_PSW
225 sysc_done:
226
227 #
228 # One of the work bits is on. Find out which one.
229 #
230 sysc_work:
231 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING
232 jo sysc_mcck_pending
233 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
234 jo sysc_reschedule
235 tm __TI_flags+3(%r12),_TIF_PER_TRAP
236 jo sysc_singlestep
237 tm __TI_flags+3(%r12),_TIF_SIGPENDING
238 jo sysc_sigpending
239 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME
240 jo sysc_notify_resume
241 j sysc_return # beware of critical section cleanup
242
243 #
244 # _TIF_NEED_RESCHED is set, call schedule
245 #
246 sysc_reschedule:
247 l %r1,BASED(.Lschedule)
248 la %r14,BASED(sysc_return)
249 br %r1 # call schedule
250
251 #
252 # _TIF_MCCK_PENDING is set, call handler
253 #
254 sysc_mcck_pending:
255 l %r1,BASED(.Lhandle_mcck)
256 la %r14,BASED(sysc_return)
257 br %r1 # TIF bit will be cleared by handler
258
259 #
260 # _TIF_SIGPENDING is set, call do_signal
261 #
262 sysc_sigpending:
263 lr %r2,%r11 # pass pointer to pt_regs
264 l %r1,BASED(.Ldo_signal)
265 basr %r14,%r1 # call do_signal
266 tm __TI_flags+3(%r12),_TIF_SYSCALL
267 jno sysc_return
268 lm %r2,%r7,__PT_R2(%r11) # load svc arguments
269 xr %r8,%r8 # svc 0 returns -ENOSYS
270 clc __PT_INT_CODE+2(2,%r11),BASED(.Lnr_syscalls+2)
271 jnl sysc_nr_ok # invalid svc number -> do svc 0
272 lh %r8,__PT_INT_CODE+2(%r11) # load new svc number
273 sla %r8,2
274 j sysc_nr_ok # restart svc
275
276 #
277 # _TIF_NOTIFY_RESUME is set, call do_notify_resume
278 #
279 sysc_notify_resume:
280 lr %r2,%r11 # pass pointer to pt_regs
281 l %r1,BASED(.Ldo_notify_resume)
282 la %r14,BASED(sysc_return)
283 br %r1 # call do_notify_resume
284
285 #
286 # _TIF_PER_TRAP is set, call do_per_trap
287 #
288 sysc_singlestep:
289 ni __TI_flags+3(%r12),255-_TIF_PER_TRAP
290 lr %r2,%r11 # pass pointer to pt_regs
291 l %r1,BASED(.Ldo_per_trap)
292 la %r14,BASED(sysc_return)
293 br %r1 # call do_per_trap
294
295 #
296 # call tracehook_report_syscall_entry/tracehook_report_syscall_exit before
297 # and after the system call
298 #
299 sysc_tracesys:
300 l %r1,BASED(.Ltrace_enter)
301 lr %r2,%r11 # pass pointer to pt_regs
302 la %r3,0
303 xr %r0,%r0
304 icm %r0,3,__PT_INT_CODE+2(%r11)
305 st %r0,__PT_R2(%r11)
306 basr %r14,%r1 # call do_syscall_trace_enter
307 cl %r2,BASED(.Lnr_syscalls)
308 jnl sysc_tracenogo
309 lr %r8,%r2
310 sll %r8,2
311 l %r9,0(%r8,%r10)
312 sysc_tracego:
313 lm %r3,%r7,__PT_R3(%r11)
314 st %r7,STACK_FRAME_OVERHEAD(%r15)
315 l %r2,__PT_ORIG_GPR2(%r11)
316 basr %r14,%r9 # call sys_xxx
317 st %r2,__PT_R2(%r11) # store return value
318 sysc_tracenogo:
319 tm __TI_flags+2(%r12),_TIF_TRACE >> 8
320 jz sysc_return
321 l %r1,BASED(.Ltrace_exit)
322 lr %r2,%r11 # pass pointer to pt_regs
323 la %r14,BASED(sysc_return)
324 br %r1 # call do_syscall_trace_exit
325
326 #
327 # a new process exits the kernel with ret_from_fork
328 #
329 ENTRY(ret_from_fork)
330 la %r11,STACK_FRAME_OVERHEAD(%r15)
331 l %r12,__LC_THREAD_INFO
332 l %r13,__LC_SVC_NEW_PSW+4
333 l %r1,BASED(.Lschedule_tail)
334 basr %r14,%r1 # call schedule_tail
335 TRACE_IRQS_ON
336 ssm __LC_SVC_NEW_PSW # reenable interrupts
337 tm __PT_PSW+1(%r11),0x01 # forking a kernel thread ?
338 jne sysc_tracenogo
339 # it's a kernel thread
340 lm %r9,%r10,__PT_R9(%r11) # load gprs
341 ENTRY(kernel_thread_starter)
342 la %r2,0(%r10)
343 basr %r14,%r9
344 j sysc_tracenogo
345
346 /*
347 * Program check handler routine
348 */
349
350 ENTRY(pgm_check_handler)
351 stpt __LC_SYNC_ENTER_TIMER
352 stm %r8,%r15,__LC_SAVE_AREA_SYNC
353 l %r12,__LC_THREAD_INFO
354 l %r13,__LC_SVC_NEW_PSW+4
355 lm %r8,%r9,__LC_PGM_OLD_PSW
356 tmh %r8,0x0001 # test problem state bit
357 jnz 1f # -> fault in user space
358 tmh %r8,0x4000 # PER bit set in old PSW ?
359 jnz 0f # -> enabled, can't be a double fault
360 tm __LC_PGM_ILC+3,0x80 # check for per exception
361 jnz pgm_svcper # -> single stepped svc
362 0: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC
363 ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
364 j 2f
365 1: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
366 l %r15,__LC_KERNEL_STACK
367 2: la %r11,STACK_FRAME_OVERHEAD(%r15)
368 stm %r0,%r7,__PT_R0(%r11)
369 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC
370 stm %r8,%r9,__PT_PSW(%r11)
371 mvc __PT_INT_CODE(4,%r11),__LC_PGM_ILC
372 mvc __PT_INT_PARM_LONG(4,%r11),__LC_TRANS_EXC_CODE
373 tm __LC_PGM_ILC+3,0x80 # check for per exception
374 jz 0f
375 l %r1,__TI_task(%r12)
376 tmh %r8,0x0001 # kernel per event ?
377 jz pgm_kprobe
378 oi __TI_flags+3(%r12),_TIF_PER_TRAP
379 mvc __THREAD_per_address(4,%r1),__LC_PER_ADDRESS
380 mvc __THREAD_per_cause(2,%r1),__LC_PER_CAUSE
381 mvc __THREAD_per_paid(1,%r1),__LC_PER_PAID
382 0: REENABLE_IRQS
383 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
384 l %r1,BASED(.Ljump_table)
385 la %r10,0x7f
386 n %r10,__PT_INT_CODE(%r11)
387 je sysc_return
388 sll %r10,2
389 l %r1,0(%r10,%r1) # load address of handler routine
390 lr %r2,%r11 # pass pointer to pt_regs
391 basr %r14,%r1 # branch to interrupt-handler
392 j sysc_return
393
394 #
395 # PER event in supervisor state, must be kprobes
396 #
397 pgm_kprobe:
398 REENABLE_IRQS
399 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
400 l %r1,BASED(.Ldo_per_trap)
401 lr %r2,%r11 # pass pointer to pt_regs
402 basr %r14,%r1 # call do_per_trap
403 j sysc_return
404
405 #
406 # single stepped system call
407 #
408 pgm_svcper:
409 oi __TI_flags+3(%r12),_TIF_PER_TRAP
410 mvc __LC_RETURN_PSW(4),__LC_SVC_NEW_PSW
411 mvc __LC_RETURN_PSW+4(4),BASED(.Lsysc_per)
412 lpsw __LC_RETURN_PSW # branch to sysc_per and enable irqs
413
414 /*
415 * IO interrupt handler routine
416 */
417
418 ENTRY(io_int_handler)
419 stck __LC_INT_CLOCK
420 stpt __LC_ASYNC_ENTER_TIMER
421 stm %r8,%r15,__LC_SAVE_AREA_ASYNC
422 l %r12,__LC_THREAD_INFO
423 l %r13,__LC_SVC_NEW_PSW+4
424 lm %r8,%r9,__LC_IO_OLD_PSW
425 tmh %r8,0x0001 # interrupting from user ?
426 jz io_skip
427 UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER
428 io_skip:
429 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT
430 stm %r0,%r7,__PT_R0(%r11)
431 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC
432 stm %r8,%r9,__PT_PSW(%r11)
433 mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
434 TRACE_IRQS_OFF
435 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
436 io_loop:
437 l %r1,BASED(.Ldo_IRQ)
438 lr %r2,%r11 # pass pointer to pt_regs
439 lhi %r3,IO_INTERRUPT
440 tm __PT_INT_CODE+8(%r11),0x80 # adapter interrupt ?
441 jz io_call
442 lhi %r3,THIN_INTERRUPT
443 io_call:
444 basr %r14,%r1 # call do_IRQ
445 tm __LC_MACHINE_FLAGS+2,0x10 # MACHINE_FLAG_LPAR
446 jz io_return
447 tpi 0
448 jz io_return
449 mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
450 j io_loop
451 io_return:
452 LOCKDEP_SYS_EXIT
453 TRACE_IRQS_ON
454 io_tif:
455 tm __TI_flags+3(%r12),_TIF_WORK_INT
456 jnz io_work # there is work to do (signals etc.)
457 io_restore:
458 mvc __LC_RETURN_PSW(8),__PT_PSW(%r11)
459 stpt __LC_EXIT_TIMER
460 lm %r0,%r15,__PT_R0(%r11)
461 lpsw __LC_RETURN_PSW
462 io_done:
463
464 #
465 # There is work todo, find out in which context we have been interrupted:
466 # 1) if we return to user space we can do all _TIF_WORK_INT work
467 # 2) if we return to kernel code and preemptive scheduling is enabled check
468 # the preemption counter and if it is zero call preempt_schedule_irq
469 # Before any work can be done, a switch to the kernel stack is required.
470 #
471 io_work:
472 tm __PT_PSW+1(%r11),0x01 # returning to user ?
473 jo io_work_user # yes -> do resched & signal
474 #ifdef CONFIG_PREEMPT
475 # check for preemptive scheduling
476 icm %r0,15,__TI_precount(%r12)
477 jnz io_restore # preemption disabled
478 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
479 jno io_restore
480 # switch to kernel stack
481 l %r1,__PT_R15(%r11)
482 ahi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
483 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
484 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
485 la %r11,STACK_FRAME_OVERHEAD(%r1)
486 lr %r15,%r1
487 # TRACE_IRQS_ON already done at io_return, call
488 # TRACE_IRQS_OFF to keep things symmetrical
489 TRACE_IRQS_OFF
490 l %r1,BASED(.Lpreempt_irq)
491 basr %r14,%r1 # call preempt_schedule_irq
492 j io_return
493 #else
494 j io_restore
495 #endif
496
497 #
498 # Need to do work before returning to userspace, switch to kernel stack
499 #
500 io_work_user:
501 l %r1,__LC_KERNEL_STACK
502 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
503 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
504 la %r11,STACK_FRAME_OVERHEAD(%r1)
505 lr %r15,%r1
506
507 #
508 # One of the work bits is on. Find out which one.
509 # Checked are: _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_NEED_RESCHED
510 # and _TIF_MCCK_PENDING
511 #
512 io_work_tif:
513 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING
514 jo io_mcck_pending
515 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
516 jo io_reschedule
517 tm __TI_flags+3(%r12),_TIF_SIGPENDING
518 jo io_sigpending
519 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME
520 jo io_notify_resume
521 j io_return # beware of critical section cleanup
522
523 #
524 # _TIF_MCCK_PENDING is set, call handler
525 #
526 io_mcck_pending:
527 # TRACE_IRQS_ON already done at io_return
528 l %r1,BASED(.Lhandle_mcck)
529 basr %r14,%r1 # TIF bit will be cleared by handler
530 TRACE_IRQS_OFF
531 j io_return
532
533 #
534 # _TIF_NEED_RESCHED is set, call schedule
535 #
536 io_reschedule:
537 # TRACE_IRQS_ON already done at io_return
538 l %r1,BASED(.Lschedule)
539 ssm __LC_SVC_NEW_PSW # reenable interrupts
540 basr %r14,%r1 # call scheduler
541 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
542 TRACE_IRQS_OFF
543 j io_return
544
545 #
546 # _TIF_SIGPENDING is set, call do_signal
547 #
548 io_sigpending:
549 # TRACE_IRQS_ON already done at io_return
550 l %r1,BASED(.Ldo_signal)
551 ssm __LC_SVC_NEW_PSW # reenable interrupts
552 lr %r2,%r11 # pass pointer to pt_regs
553 basr %r14,%r1 # call do_signal
554 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
555 TRACE_IRQS_OFF
556 j io_return
557
558 #
559 # _TIF_SIGPENDING is set, call do_signal
560 #
561 io_notify_resume:
562 # TRACE_IRQS_ON already done at io_return
563 l %r1,BASED(.Ldo_notify_resume)
564 ssm __LC_SVC_NEW_PSW # reenable interrupts
565 lr %r2,%r11 # pass pointer to pt_regs
566 basr %r14,%r1 # call do_notify_resume
567 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
568 TRACE_IRQS_OFF
569 j io_return
570
571 /*
572 * External interrupt handler routine
573 */
574
575 ENTRY(ext_int_handler)
576 stck __LC_INT_CLOCK
577 stpt __LC_ASYNC_ENTER_TIMER
578 stm %r8,%r15,__LC_SAVE_AREA_ASYNC
579 l %r12,__LC_THREAD_INFO
580 l %r13,__LC_SVC_NEW_PSW+4
581 lm %r8,%r9,__LC_EXT_OLD_PSW
582 tmh %r8,0x0001 # interrupting from user ?
583 jz ext_skip
584 UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER
585 ext_skip:
586 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT
587 stm %r0,%r7,__PT_R0(%r11)
588 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC
589 stm %r8,%r9,__PT_PSW(%r11)
590 mvc __PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR
591 mvc __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS
592 TRACE_IRQS_OFF
593 l %r1,BASED(.Ldo_IRQ)
594 lr %r2,%r11 # pass pointer to pt_regs
595 lhi %r3,EXT_INTERRUPT
596 basr %r14,%r1 # call do_IRQ
597 j io_return
598
599 /*
600 * Load idle PSW. The second "half" of this function is in cleanup_idle.
601 */
602 ENTRY(psw_idle)
603 st %r3,__SF_EMPTY(%r15)
604 basr %r1,0
605 la %r1,psw_idle_lpsw+4-.(%r1)
606 st %r1,__SF_EMPTY+4(%r15)
607 oi __SF_EMPTY+4(%r15),0x80
608 stck __CLOCK_IDLE_ENTER(%r2)
609 stpt __TIMER_IDLE_ENTER(%r2)
610 psw_idle_lpsw:
611 lpsw __SF_EMPTY(%r15)
612 br %r14
613 psw_idle_end:
614
615 __critical_end:
616
617 /*
618 * Machine check handler routines
619 */
620
621 ENTRY(mcck_int_handler)
622 stck __LC_MCCK_CLOCK
623 spt __LC_CPU_TIMER_SAVE_AREA # revalidate cpu timer
624 lm %r0,%r15,__LC_GPREGS_SAVE_AREA # revalidate gprs
625 l %r12,__LC_THREAD_INFO
626 l %r13,__LC_SVC_NEW_PSW+4
627 lm %r8,%r9,__LC_MCK_OLD_PSW
628 tm __LC_MCCK_CODE,0x80 # system damage?
629 jo mcck_panic # yes -> rest of mcck code invalid
630 la %r14,__LC_CPU_TIMER_SAVE_AREA
631 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
632 tm __LC_MCCK_CODE+5,0x02 # stored cpu timer value valid?
633 jo 3f
634 la %r14,__LC_SYNC_ENTER_TIMER
635 clc 0(8,%r14),__LC_ASYNC_ENTER_TIMER
636 jl 0f
637 la %r14,__LC_ASYNC_ENTER_TIMER
638 0: clc 0(8,%r14),__LC_EXIT_TIMER
639 jl 1f
640 la %r14,__LC_EXIT_TIMER
641 1: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER
642 jl 2f
643 la %r14,__LC_LAST_UPDATE_TIMER
644 2: spt 0(%r14)
645 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
646 3: tm __LC_MCCK_CODE+2,0x09 # mwp + ia of old psw valid?
647 jno mcck_panic # no -> skip cleanup critical
648 tm %r8,0x0001 # interrupting from user ?
649 jz mcck_skip
650 UPDATE_VTIME %r14,%r15,__LC_MCCK_ENTER_TIMER
651 mcck_skip:
652 SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+32,__LC_PANIC_STACK,PAGE_SHIFT
653 stm %r0,%r7,__PT_R0(%r11)
654 mvc __PT_R8(32,%r11),__LC_GPREGS_SAVE_AREA+32
655 stm %r8,%r9,__PT_PSW(%r11)
656 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
657 l %r1,BASED(.Ldo_machine_check)
658 lr %r2,%r11 # pass pointer to pt_regs
659 basr %r14,%r1 # call s390_do_machine_check
660 tm __PT_PSW+1(%r11),0x01 # returning to user ?
661 jno mcck_return
662 l %r1,__LC_KERNEL_STACK # switch to kernel stack
663 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
664 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
665 la %r11,STACK_FRAME_OVERHEAD(%r15)
666 lr %r15,%r1
667 ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off
668 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING
669 jno mcck_return
670 TRACE_IRQS_OFF
671 l %r1,BASED(.Lhandle_mcck)
672 basr %r14,%r1 # call s390_handle_mcck
673 TRACE_IRQS_ON
674 mcck_return:
675 mvc __LC_RETURN_MCCK_PSW(8),__PT_PSW(%r11) # move return PSW
676 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
677 jno 0f
678 lm %r0,%r15,__PT_R0(%r11)
679 stpt __LC_EXIT_TIMER
680 lpsw __LC_RETURN_MCCK_PSW
681 0: lm %r0,%r15,__PT_R0(%r11)
682 lpsw __LC_RETURN_MCCK_PSW
683
684 mcck_panic:
685 l %r14,__LC_PANIC_STACK
686 slr %r14,%r15
687 sra %r14,PAGE_SHIFT
688 jz 0f
689 l %r15,__LC_PANIC_STACK
690 j mcck_skip
691 0: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
692 j mcck_skip
693
694 #
695 # PSW restart interrupt handler
696 #
697 ENTRY(restart_int_handler)
698 st %r15,__LC_SAVE_AREA_RESTART
699 l %r15,__LC_RESTART_STACK
700 ahi %r15,-__PT_SIZE # create pt_regs on stack
701 xc 0(__PT_SIZE,%r15),0(%r15)
702 stm %r0,%r14,__PT_R0(%r15)
703 mvc __PT_R15(4,%r15),__LC_SAVE_AREA_RESTART
704 mvc __PT_PSW(8,%r15),__LC_RST_OLD_PSW # store restart old psw
705 ahi %r15,-STACK_FRAME_OVERHEAD # create stack frame on stack
706 xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
707 l %r1,__LC_RESTART_FN # load fn, parm & source cpu
708 l %r2,__LC_RESTART_DATA
709 l %r3,__LC_RESTART_SOURCE
710 ltr %r3,%r3 # test source cpu address
711 jm 1f # negative -> skip source stop
712 0: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu
713 brc 10,0b # wait for status stored
714 1: basr %r14,%r1 # call function
715 stap __SF_EMPTY(%r15) # store cpu address
716 lh %r3,__SF_EMPTY(%r15)
717 2: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu
718 brc 2,2b
719 3: j 3b
720
721 .section .kprobes.text, "ax"
722
723 #ifdef CONFIG_CHECK_STACK
724 /*
725 * The synchronous or the asynchronous stack overflowed. We are dead.
726 * No need to properly save the registers, we are going to panic anyway.
727 * Setup a pt_regs so that show_trace can provide a good call trace.
728 */
729 stack_overflow:
730 l %r15,__LC_PANIC_STACK # change to panic stack
731 la %r11,STACK_FRAME_OVERHEAD(%r15)
732 stm %r0,%r7,__PT_R0(%r11)
733 stm %r8,%r9,__PT_PSW(%r11)
734 mvc __PT_R8(32,%r11),0(%r14)
735 l %r1,BASED(1f)
736 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
737 lr %r2,%r11 # pass pointer to pt_regs
738 br %r1 # branch to kernel_stack_overflow
739 1: .long kernel_stack_overflow
740 #endif
741
742 cleanup_table:
743 .long system_call + 0x80000000
744 .long sysc_do_svc + 0x80000000
745 .long sysc_tif + 0x80000000
746 .long sysc_restore + 0x80000000
747 .long sysc_done + 0x80000000
748 .long io_tif + 0x80000000
749 .long io_restore + 0x80000000
750 .long io_done + 0x80000000
751 .long psw_idle + 0x80000000
752 .long psw_idle_end + 0x80000000
753
754 cleanup_critical:
755 cl %r9,BASED(cleanup_table) # system_call
756 jl 0f
757 cl %r9,BASED(cleanup_table+4) # sysc_do_svc
758 jl cleanup_system_call
759 cl %r9,BASED(cleanup_table+8) # sysc_tif
760 jl 0f
761 cl %r9,BASED(cleanup_table+12) # sysc_restore
762 jl cleanup_sysc_tif
763 cl %r9,BASED(cleanup_table+16) # sysc_done
764 jl cleanup_sysc_restore
765 cl %r9,BASED(cleanup_table+20) # io_tif
766 jl 0f
767 cl %r9,BASED(cleanup_table+24) # io_restore
768 jl cleanup_io_tif
769 cl %r9,BASED(cleanup_table+28) # io_done
770 jl cleanup_io_restore
771 cl %r9,BASED(cleanup_table+32) # psw_idle
772 jl 0f
773 cl %r9,BASED(cleanup_table+36) # psw_idle_end
774 jl cleanup_idle
775 0: br %r14
776
777 cleanup_system_call:
778 # check if stpt has been executed
779 cl %r9,BASED(cleanup_system_call_insn)
780 jh 0f
781 mvc __LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER
782 chi %r11,__LC_SAVE_AREA_ASYNC
783 je 0f
784 mvc __LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER
785 0: # check if stm has been executed
786 cl %r9,BASED(cleanup_system_call_insn+4)
787 jh 0f
788 mvc __LC_SAVE_AREA_SYNC(32),0(%r11)
789 0: # set up saved registers r12, and r13
790 st %r12,16(%r11) # r12 thread-info pointer
791 st %r13,20(%r11) # r13 literal-pool pointer
792 # check if the user time calculation has been done
793 cl %r9,BASED(cleanup_system_call_insn+8)
794 jh 0f
795 l %r10,__LC_EXIT_TIMER
796 l %r15,__LC_EXIT_TIMER+4
797 SUB64 %r10,%r15,__LC_SYNC_ENTER_TIMER
798 ADD64 %r10,%r15,__LC_USER_TIMER
799 st %r10,__LC_USER_TIMER
800 st %r15,__LC_USER_TIMER+4
801 0: # check if the system time calculation has been done
802 cl %r9,BASED(cleanup_system_call_insn+12)
803 jh 0f
804 l %r10,__LC_LAST_UPDATE_TIMER
805 l %r15,__LC_LAST_UPDATE_TIMER+4
806 SUB64 %r10,%r15,__LC_EXIT_TIMER
807 ADD64 %r10,%r15,__LC_SYSTEM_TIMER
808 st %r10,__LC_SYSTEM_TIMER
809 st %r15,__LC_SYSTEM_TIMER+4
810 0: # update accounting time stamp
811 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
812 # set up saved register 11
813 l %r15,__LC_KERNEL_STACK
814 la %r9,STACK_FRAME_OVERHEAD(%r15)
815 st %r9,12(%r11) # r11 pt_regs pointer
816 # fill pt_regs
817 mvc __PT_R8(32,%r9),__LC_SAVE_AREA_SYNC
818 stm %r0,%r7,__PT_R0(%r9)
819 mvc __PT_PSW(8,%r9),__LC_SVC_OLD_PSW
820 mvc __PT_INT_CODE(4,%r9),__LC_SVC_ILC
821 # setup saved register 15
822 st %r15,28(%r11) # r15 stack pointer
823 # set new psw address and exit
824 l %r9,BASED(cleanup_table+4) # sysc_do_svc + 0x80000000
825 br %r14
826 cleanup_system_call_insn:
827 .long system_call + 0x80000000
828 .long sysc_stm + 0x80000000
829 .long sysc_vtime + 0x80000000 + 36
830 .long sysc_vtime + 0x80000000 + 76
831
832 cleanup_sysc_tif:
833 l %r9,BASED(cleanup_table+8) # sysc_tif + 0x80000000
834 br %r14
835
836 cleanup_sysc_restore:
837 cl %r9,BASED(cleanup_sysc_restore_insn)
838 jhe 0f
839 l %r9,12(%r11) # get saved pointer to pt_regs
840 mvc __LC_RETURN_PSW(8),__PT_PSW(%r9)
841 mvc 0(32,%r11),__PT_R8(%r9)
842 lm %r0,%r7,__PT_R0(%r9)
843 0: lm %r8,%r9,__LC_RETURN_PSW
844 br %r14
845 cleanup_sysc_restore_insn:
846 .long sysc_done - 4 + 0x80000000
847
848 cleanup_io_tif:
849 l %r9,BASED(cleanup_table+20) # io_tif + 0x80000000
850 br %r14
851
852 cleanup_io_restore:
853 cl %r9,BASED(cleanup_io_restore_insn)
854 jhe 0f
855 l %r9,12(%r11) # get saved r11 pointer to pt_regs
856 mvc __LC_RETURN_PSW(8),__PT_PSW(%r9)
857 mvc 0(32,%r11),__PT_R8(%r9)
858 lm %r0,%r7,__PT_R0(%r9)
859 0: lm %r8,%r9,__LC_RETURN_PSW
860 br %r14
861 cleanup_io_restore_insn:
862 .long io_done - 4 + 0x80000000
863
864 cleanup_idle:
865 # copy interrupt clock & cpu timer
866 mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_INT_CLOCK
867 mvc __TIMER_IDLE_EXIT(8,%r2),__LC_ASYNC_ENTER_TIMER
868 chi %r11,__LC_SAVE_AREA_ASYNC
869 je 0f
870 mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK
871 mvc __TIMER_IDLE_EXIT(8,%r2),__LC_MCCK_ENTER_TIMER
872 0: # check if stck has been executed
873 cl %r9,BASED(cleanup_idle_insn)
874 jhe 1f
875 mvc __CLOCK_IDLE_ENTER(8,%r2),__CLOCK_IDLE_EXIT(%r2)
876 mvc __TIMER_IDLE_ENTER(8,%r2),__TIMER_IDLE_EXIT(%r3)
877 1: # account system time going idle
878 lm %r9,%r10,__LC_STEAL_TIMER
879 ADD64 %r9,%r10,__CLOCK_IDLE_ENTER(%r2)
880 SUB64 %r9,%r10,__LC_LAST_UPDATE_CLOCK
881 stm %r9,%r10,__LC_STEAL_TIMER
882 mvc __LC_LAST_UPDATE_CLOCK(8),__CLOCK_IDLE_EXIT(%r2)
883 lm %r9,%r10,__LC_SYSTEM_TIMER
884 ADD64 %r9,%r10,__LC_LAST_UPDATE_TIMER
885 SUB64 %r9,%r10,__TIMER_IDLE_ENTER(%r2)
886 stm %r9,%r10,__LC_SYSTEM_TIMER
887 mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2)
888 # prepare return psw
889 n %r8,BASED(cleanup_idle_wait) # clear irq & wait state bits
890 l %r9,24(%r11) # return from psw_idle
891 br %r14
892 cleanup_idle_insn:
893 .long psw_idle_lpsw + 0x80000000
894 cleanup_idle_wait:
895 .long 0xfcfdffff
896
897 /*
898 * Integer constants
899 */
900 .align 4
901 .Lnr_syscalls:
902 .long NR_syscalls
903 .Lvtimer_max:
904 .quad 0x7fffffffffffffff
905
906 /*
907 * Symbol constants
908 */
909 .Ldo_machine_check: .long s390_do_machine_check
910 .Lhandle_mcck: .long s390_handle_mcck
911 .Ldo_IRQ: .long do_IRQ
912 .Ldo_signal: .long do_signal
913 .Ldo_notify_resume: .long do_notify_resume
914 .Ldo_per_trap: .long do_per_trap
915 .Ljump_table: .long pgm_check_table
916 .Lschedule: .long schedule
917 #ifdef CONFIG_PREEMPT
918 .Lpreempt_irq: .long preempt_schedule_irq
919 #endif
920 .Ltrace_enter: .long do_syscall_trace_enter
921 .Ltrace_exit: .long do_syscall_trace_exit
922 .Lschedule_tail: .long schedule_tail
923 .Lsysc_per: .long sysc_per + 0x80000000
924 #ifdef CONFIG_TRACE_IRQFLAGS
925 .Lhardirqs_on: .long trace_hardirqs_on_caller
926 .Lhardirqs_off: .long trace_hardirqs_off_caller
927 #endif
928 #ifdef CONFIG_LOCKDEP
929 .Llockdep_sys_exit: .long lockdep_sys_exit
930 #endif
931 .Lcritical_start: .long __critical_start + 0x80000000
932 .Lcritical_length: .long __critical_end - __critical_start
933
934 .section .rodata, "a"
935 #define SYSCALL(esa,esame,emu) .long esa
936 .globl sys_call_table
937 sys_call_table:
938 #include "syscalls.S"
939 #undef SYSCALL
ubuntu-bionic-kernel mirror