[mirror_ubuntu-bionic-kernel.git] / arch / cris / arch-v32 / mm / mmu.S

; WARNING : The refill handler has been modified, see below !!!

/*
 *  Copyright (C) 2003 Axis Communications AB
 *
 *  Authors:	Mikael Starvik (starvik@axis.com)
 *
 * Code for the fault low-level handling routines.
 *
 */

#include <asm/page.h>
#include <asm/pgtable.h>

; Save all register. Must save in same order as struct pt_regs.
.macro SAVE_ALL
	subq	12, $sp
	move	$erp, [$sp]
	subq	4, $sp
	move	$srp, [$sp]
	subq	4, $sp
	move	$ccs, [$sp]
	subq	4, $sp
	move	$spc, [$sp]
	subq	4, $sp
	move	$mof, [$sp]
	subq	4, $sp
	move	$srs, [$sp]
	subq	4, $sp
	move.d	$acr, [$sp]
	subq	14*4, $sp
	movem	$r13, [$sp]
	subq	4, $sp
	move.d	$r10, [$sp]
.endm

; Bus fault handler. Extracts relevant information and calls mm subsystem
; to handle the fault.
.macro	MMU_BUS_FAULT_HANDLER handler, mmu, we, ex
	.globl	\handler
\handler:
	SAVE_ALL
	move	\mmu, $srs	; Select MMU support register bank
	move.d  $sp, $r11	; regs
	moveq	1, $r12		; protection fault
	moveq   \we, $r13	; write exception?
	orq	\ex << 1, $r13	; execute?
	move    $s3, $r10	; rw_mm_cause
	and.d	~8191, $r10	; Get faulting page start address

	jsr	do_page_fault
	nop
	ba	ret_from_intr
	nop
.endm

; Refill handler. Three cases may occur:
;   1. PMD and PTE exists in mm subsystem but not in TLB
;   2. PMD exists but not PTE
;   3. PMD doesn't exist
; The code below handles case 1 and calls the mm subsystem for case 2 and 3.
; Do not touch this code without very good reasons and extensive testing.
; Note that the code is optimized to minimize stalls (makes the code harder
; to read).
;
; WARNING !!!
; Modified by Mikael Asker 060725: added a workaround for strange TLB
; behavior. If the same PTE is present in more than one set, the TLB
; doesn't recognize it and we get stuck in a loop of refill exceptions.
; The workaround detects such loops and exits them by flushing
; the TLB contents. The problem and workaround were verified
; in VCS by Mikael Starvik.
;
; Each page is 8 KB. Each PMD holds 8192/4 PTEs (each PTE is 4 bytes) so each
; PMD holds 16 MB of virtual memory.
;   Bits  0-12 : Offset within a page
;   Bits 13-23 : PTE offset within a PMD
;   Bits 24-31 : PMD offset within the PGD

.macro MMU_REFILL_HANDLER handler, mmu
	.data
1:	.dword	0		; refill_count
                                ;   == 0 <=> last_refill_cause is invalid
2:	.dword	0		; last_refill_cause
	.text
	.globl \handler
\handler:
	subq	4, $sp
; (The pipeline stalls for one cycle; $sp used as address in the next cycle.)
	move	$srs, [$sp]
	subq	4, $sp
	move	\mmu, $srs	; Select MMU support register bank
	move.d	$acr, [$sp]
	subq	12, $sp
	move.d	1b, $acr        ; Point to refill_count
	movem	$r2, [$sp]

	test.d	[$acr]	        ; refill_count == 0 ?
	beq	5f		;   yes, last_refill_cause is invalid
        move.d	$acr, $r1

	; last_refill_cause is valid, investigate cause
        addq    4, $r1          ; Point to last_refill_cause
	move	$s3, $r0	; Get rw_mm_cause
	move.d	[$r1], $r2	; Get last_refill_cause
	cmp.d	$r0, $r2	; rw_mm_cause == last_refill_cause ?
	beq	6f		;   yes, increment count
	moveq	1, $r2

        ; rw_mm_cause != last_refill_cause
	move.d	$r2, [$acr]	; refill_count = 1
	move.d	$r0, [$r1]	; last_refill_cause = rw_mm_cause

3:	; Probably not in a loop, continue normal processing
#ifdef CONFIG_SMP
	move    $s7, $acr	; PGD
#else
	move.d  current_pgd, $acr ; PGD
#endif
	; Look up PMD in PGD
	lsrq	24, $r0	; Get PMD index into PGD (bit 24-31)
	move.d  [$acr], $acr	; PGD for the current process
	addi	$r0.d, $acr, $acr
	move	$s3, $r0	; rw_mm_cause
	move.d  [$acr], $acr	; Get PMD
	beq	8f
	; Look up PTE in PMD
	lsrq	PAGE_SHIFT, $r0
	and.w	PAGE_MASK, $acr	; Remove PMD flags
	and.d	0x7ff, $r0	; Get PTE index into PMD (bit 13-23)
	addi    $r0.d, $acr, $acr
	move.d  [$acr], $acr	; Get PTE
	beq	9f
	movem	[$sp], $r2	; Restore r0-r2 in delay slot
	addq	12, $sp
	; Store in TLB
	move	$acr, $s5
4:	; Return
	move.d	[$sp+], $acr
	move	[$sp], $srs
	addq	4, $sp
	rete
	rfe

5:      ; last_refill_cause is invalid
	moveq	1, $r2
        addq    4, $r1          ; Point to last_refill_cause
	move.d	$r2, [$acr]	; refill_count = 1
	move	$s3, $r0	; Get rw_mm_cause
        ba      3b		; Continue normal processing
	move.d	$r0,[$r1]	; last_refill_cause = rw_mm_cause

6:      ; rw_mm_cause == last_refill_cause
        move.d  [$acr], $r2     ; Get refill_count
	cmpq	4, $r2		; refill_count > 4 ?
	bhi	7f		;   yes
	addq	1, $r2	        ; refill_count++
	ba	3b		; Continue normal processing
	move.d	$r2, [$acr]

7:	; refill_count > 4, error
	move.d	$acr, $r0       ; Save pointer to refill_count
	clear.d	[$r0]		; refill_count = 0

	;; rewind the short stack
	movem	[$sp], $r2	; Restore r0-r2
	addq	12, $sp
	move.d	[$sp+], $acr
	move	[$sp], $srs
	addq	4, $sp
	;; Keep it simple (slow), save all the regs.
	SAVE_ALL
	jsr	__flush_tlb_all
	nop
	ba	ret_from_intr	; Return
	nop

8:	; PMD missing, let the mm subsystem fix it up.
	movem	[$sp], $r2	; Restore r0-r2
9:      ; PTE missing, let the mm subsystem fix it up.
	addq	12, $sp
	move.d	[$sp+], $acr
	move	[$sp], $srs
	addq	4, $sp
	SAVE_ALL
	move    \mmu, $srs
	move.d	$sp, $r11	; regs
	clear.d	$r12		; Not a protection fault
	move.w  PAGE_MASK, $acr
	move    $s3, $r10	; rw_mm_cause
	btstq   9, $r10		; Check if write access
	smi     $r13
	and.w	PAGE_MASK, $r10	; Get VPN (virtual address)
	jsr	do_page_fault
	and.w   $acr, $r10
	; Return
	ba	ret_from_intr
	nop
.endm

	; This is the MMU bus fault handlers.

MMU_REFILL_HANDLER i_mmu_refill, 1
MMU_BUS_FAULT_HANDLER i_mmu_invalid, 1, 0, 0
MMU_BUS_FAULT_HANDLER i_mmu_access,  1, 0, 0
MMU_BUS_FAULT_HANDLER i_mmu_execute, 1, 0, 1
MMU_REFILL_HANDLER d_mmu_refill,  2
MMU_BUS_FAULT_HANDLER d_mmu_invalid, 2, 0, 0
MMU_BUS_FAULT_HANDLER d_mmu_access,  2, 0, 0
MMU_BUS_FAULT_HANDLER d_mmu_write,   2, 1, 0
Commit	Line	Data
3d44305a JN	1	; WARNING : The refill handler has been modified, see below !!!
3d44305a JN	2
51533b61 MS	3	/*
	4	* Copyright (C) 2003 Axis Communications AB
	5	*
	6	* Authors: Mikael Starvik (starvik@axis.com)
	7	*
	8	* Code for the fault low-level handling routines.
	9	*
	10	*/
	11
	12	#include <asm/page.h>
	13	#include <asm/pgtable.h>
	14
	15	; Save all register. Must save in same order as struct pt_regs.
	16	.macro SAVE_ALL
	17	subq 12, $sp
	18	move $erp, [$sp]
	19	subq 4, $sp
	20	move $srp, [$sp]
	21	subq 4, $sp
	22	move $ccs, [$sp]
	23	subq 4, $sp
	24	move $spc, [$sp]
	25	subq 4, $sp
	26	move $mof, [$sp]
	27	subq 4, $sp
	28	move $srs, [$sp]
	29	subq 4, $sp
	30	move.d $acr, [$sp]
	31	subq 14*4, $sp
	32	movem $r13, [$sp]
	33	subq 4, $sp
	34	move.d $r10, [$sp]
	35	.endm
	36
	37	; Bus fault handler. Extracts relevant information and calls mm subsystem
	38	; to handle the fault.
	39	.macro MMU_BUS_FAULT_HANDLER handler, mmu, we, ex
	40	.globl \handler
	41	\handler:
	42	SAVE_ALL
	43	move \mmu, $srs ; Select MMU support register bank
	44	move.d $sp, $r11 ; regs
	45	moveq 1, $r12 ; protection fault
	46	moveq \we, $r13 ; write exception?
	47	orq \ex << 1, $r13 ; execute?
	48	move $s3, $r10 ; rw_mm_cause
	49	and.d ~8191, $r10 ; Get faulting page start address
	50
	51	jsr do_page_fault
	52	nop
	53	ba ret_from_intr
	54	nop
	55	.endm
	56
	57	; Refill handler. Three cases may occur:
	58	; 1. PMD and PTE exists in mm subsystem but not in TLB
	59	; 2. PMD exists but not PTE
	60	; 3. PMD doesn't exist
	61	; The code below handles case 1 and calls the mm subsystem for case 2 and 3.
	62	; Do not touch this code without very good reasons and extensive testing.
	63	; Note that the code is optimized to minimize stalls (makes the code harder
	64	; to read).
	65	;
3d44305a JN	66	; WARNING !!!
	67	; Modified by Mikael Asker 060725: added a workaround for strange TLB
	68	; behavior. If the same PTE is present in more than one set, the TLB
	69	; doesn't recognize it and we get stuck in a loop of refill exceptions.
	70	; The workaround detects such loops and exits them by flushing
	71	; the TLB contents. The problem and workaround were verified
	72	; in VCS by Mikael Starvik.
	73	;
51533b61 MS	74	; Each page is 8 KB. Each PMD holds 8192/4 PTEs (each PTE is 4 bytes) so each
	75	; PMD holds 16 MB of virtual memory.
	76	; Bits 0-12 : Offset within a page
	77	; Bits 13-23 : PTE offset within a PMD
	78	; Bits 24-31 : PMD offset within the PGD
	79
	80	.macro MMU_REFILL_HANDLER handler, mmu
3d44305a JN	81	.data
	82	1: .dword 0 ; refill_count
	83	; == 0 <=> last_refill_cause is invalid
	84	2: .dword 0 ; last_refill_cause
	85	.text
51533b61 MS	86	.globl \handler
	87	\handler:
	88	subq 4, $sp
	89	; (The pipeline stalls for one cycle; $sp used as address in the next cycle.)
	90	move $srs, [$sp]
	91	subq 4, $sp
	92	move \mmu, $srs ; Select MMU support register bank
	93	move.d $acr, [$sp]
3d44305a JN	94	subq 12, $sp
	95	move.d 1b, $acr ; Point to refill_count
	96	movem $r2, [$sp]
	97
	98	test.d [$acr] ; refill_count == 0 ?
	99	beq 5f ; yes, last_refill_cause is invalid
	100	move.d $acr, $r1
	101
	102	; last_refill_cause is valid, investigate cause
	103	addq 4, $r1 ; Point to last_refill_cause
	104	move $s3, $r0 ; Get rw_mm_cause
	105	move.d [$r1], $r2 ; Get last_refill_cause
	106	cmp.d $r0, $r2 ; rw_mm_cause == last_refill_cause ?
	107	beq 6f ; yes, increment count
	108	moveq 1, $r2
	109
	110	; rw_mm_cause != last_refill_cause
	111	move.d $r2, [$acr] ; refill_count = 1
	112	move.d $r0, [$r1] ; last_refill_cause = rw_mm_cause
	113
	114	3: ; Probably not in a loop, continue normal processing
51533b61 MS	115	#ifdef CONFIG_SMP
	116	move $s7, $acr ; PGD
	117	#else
dd17c8f7	118	move.d current_pgd, $acr ; PGD
51533b61 MS	119	#endif
51533b61 MS	120	; Look up PMD in PGD
51533b61 MS	121	lsrq 24, $r0 ; Get PMD index into PGD (bit 24-31)
	122	move.d [$acr], $acr ; PGD for the current process
	123	addi $r0.d, $acr, $acr
	124	move $s3, $r0 ; rw_mm_cause
	125	move.d [$acr], $acr ; Get PMD
3d44305a	126	beq 8f
51533b61 MS	127	; Look up PTE in PMD
	128	lsrq PAGE_SHIFT, $r0
	129	and.w PAGE_MASK, $acr ; Remove PMD flags
	130	and.d 0x7ff, $r0 ; Get PTE index into PMD (bit 13-23)
	131	addi $r0.d, $acr, $acr
	132	move.d [$acr], $acr ; Get PTE
3d44305a JN	133	beq 9f
	134	movem [$sp], $r2 ; Restore r0-r2 in delay slot
	135	addq 12, $sp
51533b61 MS	136	; Store in TLB
51533b61 MS	137	move $acr, $s5
3d44305a	138	4: ; Return
51533b61	139	move.d [$sp+], $acr
3d44305a	140	move [$sp], $srs
51533b61 MS	141	addq 4, $sp
	142	rete
	143	rfe
3d44305a JN	144
	145	5: ; last_refill_cause is invalid
	146	moveq 1, $r2
	147	addq 4, $r1 ; Point to last_refill_cause
	148	move.d $r2, [$acr] ; refill_count = 1
	149	move $s3, $r0 ; Get rw_mm_cause
	150	ba 3b ; Continue normal processing
	151	move.d $r0,[$r1] ; last_refill_cause = rw_mm_cause
	152
	153	6: ; rw_mm_cause == last_refill_cause
	154	move.d [$acr], $r2 ; Get refill_count
	155	cmpq 4, $r2 ; refill_count > 4 ?
	156	bhi 7f ; yes
	157	addq 1, $r2 ; refill_count++
	158	ba 3b ; Continue normal processing
	159	move.d $r2, [$acr]
	160
	161	7: ; refill_count > 4, error
	162	move.d $acr, $r0 ; Save pointer to refill_count
	163	clear.d [$r0] ; refill_count = 0
	164
	165	;; rewind the short stack
	166	movem [$sp], $r2 ; Restore r0-r2
	167	addq 12, $sp
	168	move.d [$sp+], $acr
	169	move [$sp], $srs
	170	addq 4, $sp
	171	;; Keep it simple (slow), save all the regs.
	172	SAVE_ALL
	173	jsr __flush_tlb_all
	174	nop
	175	ba ret_from_intr ; Return
	176	nop
	177
	178	8: ; PMD missing, let the mm subsystem fix it up.
	179	movem [$sp], $r2 ; Restore r0-r2
	180	9: ; PTE missing, let the mm subsystem fix it up.
	181	addq 12, $sp
51533b61	182	move.d [$sp+], $acr
3d44305a	183	move [$sp], $srs
51533b61 MS	184	addq 4, $sp
	185	SAVE_ALL
	186	move \mmu, $srs
	187	move.d $sp, $r11 ; regs
	188	clear.d $r12 ; Not a protection fault
	189	move.w PAGE_MASK, $acr
	190	move $s3, $r10 ; rw_mm_cause
	191	btstq 9, $r10 ; Check if write access
	192	smi $r13
	193	and.w PAGE_MASK, $r10 ; Get VPN (virtual address)
	194	jsr do_page_fault
	195	and.w $acr, $r10
	196	; Return
	197	ba ret_from_intr
	198	nop
	199	.endm
	200
	201	; This is the MMU bus fault handlers.
	202
	203	MMU_REFILL_HANDLER i_mmu_refill, 1
	204	MMU_BUS_FAULT_HANDLER i_mmu_invalid, 1, 0, 0
	205	MMU_BUS_FAULT_HANDLER i_mmu_access, 1, 0, 0
	206	MMU_BUS_FAULT_HANDLER i_mmu_execute, 1, 0, 1
	207	MMU_REFILL_HANDLER d_mmu_refill, 2
	208	MMU_BUS_FAULT_HANDLER d_mmu_invalid, 2, 0, 0
	209	MMU_BUS_FAULT_HANDLER d_mmu_access, 2, 0, 0
	210	MMU_BUS_FAULT_HANDLER d_mmu_write, 2, 1, 0