[mirror_ubuntu-disco-kernel.git] / arch / cris / arch-v32 / mm / mmu.S

; SPDX-License-Identifier: GPL-2.0
; WARNING : The refill handler has been modified, see below !!!

/*
 *  Copyright (C) 2003 Axis Communications AB
 *
 *  Authors:	Mikael Starvik (starvik@axis.com)
 *
 * Code for the fault low-level handling routines.
 *
 */

#include <asm/page.h>
#include <asm/pgtable.h>

; Save all register. Must save in same order as struct pt_regs.
.macro SAVE_ALL
	subq	12, $sp
	move	$erp, [$sp]
	subq	4, $sp
	move	$srp, [$sp]
	subq	4, $sp
	move	$ccs, [$sp]
	subq	4, $sp
	move	$spc, [$sp]
	subq	4, $sp
	move	$mof, [$sp]
	subq	4, $sp
	move	$srs, [$sp]
	subq	4, $sp
	move.d	$acr, [$sp]
	subq	14*4, $sp
	movem	$r13, [$sp]
	subq	4, $sp
	move.d	$r10, [$sp]
.endm

; Bus fault handler. Extracts relevant information and calls mm subsystem
; to handle the fault.
.macro	MMU_BUS_FAULT_HANDLER handler, mmu, we, ex
	.globl	\handler
	.type   \handler,"function"
\handler:
	SAVE_ALL
	move	\mmu, $srs	; Select MMU support register bank
	move.d  $sp, $r11	; regs
	moveq	1, $r12		; protection fault
	moveq   \we, $r13	; write exception?
	orq	\ex << 1, $r13	; execute?
	move    $s3, $r10	; rw_mm_cause
	and.d	~8191, $r10	; Get faulting page start address

	jsr	do_page_fault
	nop
	ba	ret_from_intr
	nop
	.size   \handler, . - \handler
.endm

; Refill handler. Three cases may occur:
;   1. PMD and PTE exists in mm subsystem but not in TLB
;   2. PMD exists but not PTE
;   3. PMD doesn't exist
; The code below handles case 1 and calls the mm subsystem for case 2 and 3.
; Do not touch this code without very good reasons and extensive testing.
; Note that the code is optimized to minimize stalls (makes the code harder
; to read).
;
; WARNING !!!
; Modified by Mikael Asker 060725: added a workaround for strange TLB
; behavior. If the same PTE is present in more than one set, the TLB
; doesn't recognize it and we get stuck in a loop of refill exceptions.
; The workaround detects such loops and exits them by flushing
; the TLB contents. The problem and workaround were verified
; in VCS by Mikael Starvik.
;
; Each page is 8 KB. Each PMD holds 8192/4 PTEs (each PTE is 4 bytes) so each
; PMD holds 16 MB of virtual memory.
;   Bits  0-12 : Offset within a page
;   Bits 13-23 : PTE offset within a PMD
;   Bits 24-31 : PMD offset within the PGD

.macro MMU_REFILL_HANDLER handler, mmu
	.data
1:	.dword	0		; refill_count
                                ;   == 0 <=> last_refill_cause is invalid
2:	.dword	0		; last_refill_cause
	.text
	.globl \handler
	.type   \handler, "function"
\handler:
	subq	4, $sp
; (The pipeline stalls for one cycle; $sp used as address in the next cycle.)
	move	$srs, [$sp]
	subq	4, $sp
	move	\mmu, $srs	; Select MMU support register bank
	move.d	$acr, [$sp]
	subq	12, $sp
	move.d	1b, $acr        ; Point to refill_count
	movem	$r2, [$sp]

	test.d	[$acr]	        ; refill_count == 0 ?
	beq	5f		;   yes, last_refill_cause is invalid
        move.d	$acr, $r1

	; last_refill_cause is valid, investigate cause
        addq    4, $r1          ; Point to last_refill_cause
	move	$s3, $r0	; Get rw_mm_cause
	move.d	[$r1], $r2	; Get last_refill_cause
	cmp.d	$r0, $r2	; rw_mm_cause == last_refill_cause ?
	beq	6f		;   yes, increment count
	moveq	1, $r2

        ; rw_mm_cause != last_refill_cause
	move.d	$r2, [$acr]	; refill_count = 1
	move.d	$r0, [$r1]	; last_refill_cause = rw_mm_cause

3:	; Probably not in a loop, continue normal processing
	move.d  current_pgd, $acr ; PGD
	; Look up PMD in PGD
	lsrq	24, $r0	; Get PMD index into PGD (bit 24-31)
	move.d  [$acr], $acr	; PGD for the current process
	addi	$r0.d, $acr, $acr
	move	$s3, $r0	; rw_mm_cause
	move.d  [$acr], $acr	; Get PMD
	beq	8f
	; Look up PTE in PMD
	lsrq	PAGE_SHIFT, $r0
	and.w	PAGE_MASK, $acr	; Remove PMD flags
	and.d	0x7ff, $r0	; Get PTE index into PMD (bit 13-23)
	addi    $r0.d, $acr, $acr
	move.d  [$acr], $acr	; Get PTE
	beq	9f
	movem	[$sp], $r2	; Restore r0-r2 in delay slot
	addq	12, $sp
	; Store in TLB
	move	$acr, $s5
4:	; Return
	move.d	[$sp+], $acr
	move	[$sp], $srs
	addq	4, $sp
	rete
	rfe

5:      ; last_refill_cause is invalid
	moveq	1, $r2
        addq    4, $r1          ; Point to last_refill_cause
	move.d	$r2, [$acr]	; refill_count = 1
	move	$s3, $r0	; Get rw_mm_cause
        ba      3b		; Continue normal processing
	move.d	$r0,[$r1]	; last_refill_cause = rw_mm_cause

6:      ; rw_mm_cause == last_refill_cause
        move.d  [$acr], $r2     ; Get refill_count
	cmpq	4, $r2		; refill_count > 4 ?
	bhi	7f		;   yes
	addq	1, $r2	        ; refill_count++
	ba	3b		; Continue normal processing
	move.d	$r2, [$acr]

7:	; refill_count > 4, error
	move.d	$acr, $r0       ; Save pointer to refill_count
	clear.d	[$r0]		; refill_count = 0

	;; rewind the short stack
	movem	[$sp], $r2	; Restore r0-r2
	addq	12, $sp
	move.d	[$sp+], $acr
	move	[$sp], $srs
	addq	4, $sp
	;; Keep it simple (slow), save all the regs.
	SAVE_ALL
	jsr	__flush_tlb_all
	nop
	ba	ret_from_intr	; Return
	nop

8:	; PMD missing, let the mm subsystem fix it up.
	movem	[$sp], $r2	; Restore r0-r2
9:      ; PTE missing, let the mm subsystem fix it up.
	addq	12, $sp
	move.d	[$sp+], $acr
	move	[$sp], $srs
	addq	4, $sp
	SAVE_ALL
	move    \mmu, $srs
	move.d	$sp, $r11	; regs
	clear.d	$r12		; Not a protection fault
	move.w  PAGE_MASK, $acr
	move    $s3, $r10	; rw_mm_cause
	btstq   9, $r10		; Check if write access
	smi     $r13
	and.w	PAGE_MASK, $r10	; Get VPN (virtual address)
	jsr	do_page_fault
	and.w   $acr, $r10
	; Return
	ba	ret_from_intr
	nop
	.size   \handler, . - \handler
.endm

	; This is the MMU bus fault handlers.

MMU_REFILL_HANDLER i_mmu_refill, 1
MMU_BUS_FAULT_HANDLER i_mmu_invalid, 1, 0, 0
MMU_BUS_FAULT_HANDLER i_mmu_access,  1, 0, 0
MMU_BUS_FAULT_HANDLER i_mmu_execute, 1, 0, 1
MMU_REFILL_HANDLER d_mmu_refill,  2
MMU_BUS_FAULT_HANDLER d_mmu_invalid, 2, 0, 0
MMU_BUS_FAULT_HANDLER d_mmu_access,  2, 0, 0
MMU_BUS_FAULT_HANDLER d_mmu_write,   2, 1, 0
Commit	Line	Data
b2441318	1	; SPDX-License-Identifier: GPL-2.0
3d44305a JN	2	; WARNING : The refill handler has been modified, see below !!!
3d44305a JN	3
51533b61 MS	4	/*
	5	* Copyright (C) 2003 Axis Communications AB
	6	*
	7	* Authors: Mikael Starvik (starvik@axis.com)
	8	*
	9	* Code for the fault low-level handling routines.
	10	*
	11	*/
	12
	13	#include <asm/page.h>
	14	#include <asm/pgtable.h>
	15
	16	; Save all register. Must save in same order as struct pt_regs.
	17	.macro SAVE_ALL
	18	subq 12, $sp
	19	move $erp, [$sp]
	20	subq 4, $sp
	21	move $srp, [$sp]
	22	subq 4, $sp
	23	move $ccs, [$sp]
	24	subq 4, $sp
	25	move $spc, [$sp]
	26	subq 4, $sp
	27	move $mof, [$sp]
	28	subq 4, $sp
	29	move $srs, [$sp]
	30	subq 4, $sp
	31	move.d $acr, [$sp]
	32	subq 14*4, $sp
	33	movem $r13, [$sp]
	34	subq 4, $sp
	35	move.d $r10, [$sp]
	36	.endm
	37
	38	; Bus fault handler. Extracts relevant information and calls mm subsystem
	39	; to handle the fault.
	40	.macro MMU_BUS_FAULT_HANDLER handler, mmu, we, ex
	41	.globl \handler
ac93f621	42	.type \handler,"function"
51533b61 MS	43	\handler:
	44	SAVE_ALL
	45	move \mmu, $srs ; Select MMU support register bank
	46	move.d $sp, $r11 ; regs
	47	moveq 1, $r12 ; protection fault
	48	moveq \we, $r13 ; write exception?
	49	orq \ex << 1, $r13 ; execute?
	50	move $s3, $r10 ; rw_mm_cause
	51	and.d ~8191, $r10 ; Get faulting page start address
	52
	53	jsr do_page_fault
	54	nop
	55	ba ret_from_intr
	56	nop
ac93f621	57	.size \handler, . - \handler
51533b61 MS	58	.endm
	59
	60	; Refill handler. Three cases may occur:
	61	; 1. PMD and PTE exists in mm subsystem but not in TLB
	62	; 2. PMD exists but not PTE
	63	; 3. PMD doesn't exist
	64	; The code below handles case 1 and calls the mm subsystem for case 2 and 3.
	65	; Do not touch this code without very good reasons and extensive testing.
	66	; Note that the code is optimized to minimize stalls (makes the code harder
	67	; to read).
	68	;
3d44305a JN	69	; WARNING !!!
	70	; Modified by Mikael Asker 060725: added a workaround for strange TLB
	71	; behavior. If the same PTE is present in more than one set, the TLB
	72	; doesn't recognize it and we get stuck in a loop of refill exceptions.
	73	; The workaround detects such loops and exits them by flushing
	74	; the TLB contents. The problem and workaround were verified
	75	; in VCS by Mikael Starvik.
	76	;
51533b61 MS	77	; Each page is 8 KB. Each PMD holds 8192/4 PTEs (each PTE is 4 bytes) so each
	78	; PMD holds 16 MB of virtual memory.
	79	; Bits 0-12 : Offset within a page
	80	; Bits 13-23 : PTE offset within a PMD
	81	; Bits 24-31 : PMD offset within the PGD
	82
	83	.macro MMU_REFILL_HANDLER handler, mmu
3d44305a JN	84	.data
	85	1: .dword 0 ; refill_count
	86	; == 0 <=> last_refill_cause is invalid
	87	2: .dword 0 ; last_refill_cause
	88	.text
51533b61	89	.globl \handler
ac93f621	90	.type \handler, "function"
51533b61 MS	91	\handler:
	92	subq 4, $sp
	93	; (The pipeline stalls for one cycle; $sp used as address in the next cycle.)
	94	move $srs, [$sp]
	95	subq 4, $sp
	96	move \mmu, $srs ; Select MMU support register bank
	97	move.d $acr, [$sp]
3d44305a JN	98	subq 12, $sp
	99	move.d 1b, $acr ; Point to refill_count
	100	movem $r2, [$sp]
	101
	102	test.d [$acr] ; refill_count == 0 ?
	103	beq 5f ; yes, last_refill_cause is invalid
	104	move.d $acr, $r1
	105
	106	; last_refill_cause is valid, investigate cause
	107	addq 4, $r1 ; Point to last_refill_cause
	108	move $s3, $r0 ; Get rw_mm_cause
	109	move.d [$r1], $r2 ; Get last_refill_cause
	110	cmp.d $r0, $r2 ; rw_mm_cause == last_refill_cause ?
	111	beq 6f ; yes, increment count
	112	moveq 1, $r2
	113
	114	; rw_mm_cause != last_refill_cause
	115	move.d $r2, [$acr] ; refill_count = 1
	116	move.d $r0, [$r1] ; last_refill_cause = rw_mm_cause
	117
	118	3: ; Probably not in a loop, continue normal processing
dd17c8f7	119	move.d current_pgd, $acr ; PGD
51533b61	120	; Look up PMD in PGD
51533b61 MS	121	lsrq 24, $r0 ; Get PMD index into PGD (bit 24-31)
	122	move.d [$acr], $acr ; PGD for the current process
	123	addi $r0.d, $acr, $acr
	124	move $s3, $r0 ; rw_mm_cause
	125	move.d [$acr], $acr ; Get PMD
3d44305a	126	beq 8f
51533b61 MS	127	; Look up PTE in PMD
	128	lsrq PAGE_SHIFT, $r0
	129	and.w PAGE_MASK, $acr ; Remove PMD flags
	130	and.d 0x7ff, $r0 ; Get PTE index into PMD (bit 13-23)
	131	addi $r0.d, $acr, $acr
	132	move.d [$acr], $acr ; Get PTE
3d44305a JN	133	beq 9f
	134	movem [$sp], $r2 ; Restore r0-r2 in delay slot
	135	addq 12, $sp
51533b61 MS	136	; Store in TLB
51533b61 MS	137	move $acr, $s5
3d44305a	138	4: ; Return
51533b61	139	move.d [$sp+], $acr
3d44305a	140	move [$sp], $srs
51533b61 MS	141	addq 4, $sp
	142	rete
	143	rfe
3d44305a JN	144
	145	5: ; last_refill_cause is invalid
	146	moveq 1, $r2
	147	addq 4, $r1 ; Point to last_refill_cause
	148	move.d $r2, [$acr] ; refill_count = 1
	149	move $s3, $r0 ; Get rw_mm_cause
	150	ba 3b ; Continue normal processing
	151	move.d $r0,[$r1] ; last_refill_cause = rw_mm_cause
	152
	153	6: ; rw_mm_cause == last_refill_cause
	154	move.d [$acr], $r2 ; Get refill_count
	155	cmpq 4, $r2 ; refill_count > 4 ?
	156	bhi 7f ; yes
	157	addq 1, $r2 ; refill_count++
	158	ba 3b ; Continue normal processing
	159	move.d $r2, [$acr]
	160
	161	7: ; refill_count > 4, error
	162	move.d $acr, $r0 ; Save pointer to refill_count
	163	clear.d [$r0] ; refill_count = 0
	164
	165	;; rewind the short stack
	166	movem [$sp], $r2 ; Restore r0-r2
	167	addq 12, $sp
	168	move.d [$sp+], $acr
	169	move [$sp], $srs
	170	addq 4, $sp
	171	;; Keep it simple (slow), save all the regs.
	172	SAVE_ALL
	173	jsr __flush_tlb_all
	174	nop
	175	ba ret_from_intr ; Return
	176	nop
	177
	178	8: ; PMD missing, let the mm subsystem fix it up.
	179	movem [$sp], $r2 ; Restore r0-r2
	180	9: ; PTE missing, let the mm subsystem fix it up.
	181	addq 12, $sp
51533b61	182	move.d [$sp+], $acr
3d44305a	183	move [$sp], $srs
51533b61 MS	184	addq 4, $sp
	185	SAVE_ALL
	186	move \mmu, $srs
	187	move.d $sp, $r11 ; regs
	188	clear.d $r12 ; Not a protection fault
	189	move.w PAGE_MASK, $acr
	190	move $s3, $r10 ; rw_mm_cause
	191	btstq 9, $r10 ; Check if write access
	192	smi $r13
	193	and.w PAGE_MASK, $r10 ; Get VPN (virtual address)
	194	jsr do_page_fault
	195	and.w $acr, $r10
	196	; Return
	197	ba ret_from_intr
	198	nop
ac93f621	199	.size \handler, . - \handler
51533b61 MS	200	.endm
	201
	202	; This is the MMU bus fault handlers.
	203
	204	MMU_REFILL_HANDLER i_mmu_refill, 1
	205	MMU_BUS_FAULT_HANDLER i_mmu_invalid, 1, 0, 0
	206	MMU_BUS_FAULT_HANDLER i_mmu_access, 1, 0, 0
	207	MMU_BUS_FAULT_HANDLER i_mmu_execute, 1, 0, 1
	208	MMU_REFILL_HANDLER d_mmu_refill, 2
	209	MMU_BUS_FAULT_HANDLER d_mmu_invalid, 2, 0, 0
	210	MMU_BUS_FAULT_HANDLER d_mmu_access, 2, 0, 0
	211	MMU_BUS_FAULT_HANDLER d_mmu_write, 2, 1, 0