[mirror_ubuntu-artful-kernel.git] / arch / sparc64 / kernel / ktlb.S

/* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
 *
 * Copyright (C) 1995, 1997, 2005 David S. Miller <davem@davemloft.net>
 * Copyright (C) 1996 Eddie C. Dost        (ecd@brainaid.de)
 * Copyright (C) 1996 Miguel de Icaza      (miguel@nuclecu.unam.mx)
 * Copyright (C) 1996,98,99 Jakub Jelinek  (jj@sunsite.mff.cuni.cz)
 */

#include <asm/head.h>
#include <asm/asi.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/tsb.h>

	.text
	.align		32

kvmap_itlb:
	/* g6: TAG TARGET */
	mov		TLB_TAG_ACCESS, %g4
	ldxa		[%g4] ASI_IMMU, %g4

	/* sun4v_itlb_miss branches here with the missing virtual
	 * address already loaded into %g4
	 */
kvmap_itlb_4v:

kvmap_itlb_nonlinear:
	/* Catch kernel NULL pointer calls.  */
	sethi		%hi(PAGE_SIZE), %g5
	cmp		%g4, %g5
	bleu,pn		%xcc, kvmap_dtlb_longpath
	 nop

	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)

kvmap_itlb_tsb_miss:
	sethi		%hi(LOW_OBP_ADDRESS), %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_itlb_vmalloc_addr
	 mov		0x1, %g5
	sllx		%g5, 32, %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_itlb_obp
	 nop

kvmap_itlb_vmalloc_addr:
	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)

	KTSB_LOCK_TAG(%g1, %g2, %g7)

	/* Load and check PTE.  */
	ldxa		[%g5] ASI_PHYS_USE_EC, %g5
	mov		1, %g7
	sllx		%g7, TSB_TAG_INVALID_BIT, %g7
	brgez,a,pn	%g5, kvmap_itlb_longpath
	 KTSB_STORE(%g1, %g7)

	KTSB_WRITE(%g1, %g5, %g6)

	/* fallthrough to TLB load */

kvmap_itlb_load:

661:	stxa		%g5, [%g0] ASI_ITLB_DATA_IN
	retry
	.section	.sun4v_2insn_patch, "ax"
	.word		661b
	nop
	nop
	.previous

	/* For sun4v the ASI_ITLB_DATA_IN store and the retry
	 * instruction get nop'd out and we get here to branch
	 * to the sun4v tlb load code.  The registers are setup
	 * as follows:
	 *
	 * %g4: vaddr
	 * %g5: PTE
	 * %g6:	TAG
	 *
	 * The sun4v TLB load wants the PTE in %g3 so we fix that
	 * up here.
	 */
	ba,pt		%xcc, sun4v_itlb_load
	 mov		%g5, %g3

kvmap_itlb_longpath:

661:	rdpr	%pstate, %g5
	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
	.section .sun4v_2insn_patch, "ax"
	.word	661b
	SET_GL(1)
	nop
	.previous

	rdpr	%tpc, %g5
	ba,pt	%xcc, sparc64_realfault_common
	 mov	FAULT_CODE_ITLB, %g4

kvmap_itlb_obp:
	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)

	KTSB_LOCK_TAG(%g1, %g2, %g7)

	KTSB_WRITE(%g1, %g5, %g6)

	ba,pt		%xcc, kvmap_itlb_load
	 nop

kvmap_dtlb_obp:
	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)

	KTSB_LOCK_TAG(%g1, %g2, %g7)

	KTSB_WRITE(%g1, %g5, %g6)

	ba,pt		%xcc, kvmap_dtlb_load
	 nop

	.align		32
kvmap_dtlb_tsb4m_load:
	KTSB_LOCK_TAG(%g1, %g2, %g7)
	KTSB_WRITE(%g1, %g5, %g6)
	ba,pt		%xcc, kvmap_dtlb_load
	 nop

kvmap_dtlb:
	/* %g6: TAG TARGET */
	mov		TLB_TAG_ACCESS, %g4
	ldxa		[%g4] ASI_DMMU, %g4

	/* sun4v_dtlb_miss branches here with the missing virtual
	 * address already loaded into %g4
	 */
kvmap_dtlb_4v:
	brgez,pn	%g4, kvmap_dtlb_nonlinear
	 nop

#ifdef CONFIG_DEBUG_PAGEALLOC
	/* Index through the base page size TSB even for linear
	 * mappings when using page allocation debugging.
	 */
	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
#else
	/* Correct TAG_TARGET is already in %g6, check 4mb TSB.  */
	KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
#endif
	/* TSB entry address left in %g1, lookup linear PTE.
	 * Must preserve %g1 and %g6 (TAG).
	 */
kvmap_dtlb_tsb4m_miss:
	sethi		%hi(kpte_linear_bitmap), %g2
	or		%g2, %lo(kpte_linear_bitmap), %g2

	/* Clear the PAGE_OFFSET top virtual bits, then shift
	 * down to get a 256MB physical address index.
	 */
	sllx		%g4, 21, %g5
	mov		1, %g7
	srlx		%g5, 21 + 28, %g5

	/* Don't try this at home kids... this depends upon srlx
	 * only taking the low 6 bits of the shift count in %g5.
	 */
	sllx		%g7, %g5, %g7

	/* Divide by 64 to get the offset into the bitmask.  */
	srlx		%g5, 6, %g5
	sllx		%g5, 3, %g5

	/* kern_linear_pte_xor[((mask & bit) ? 1 : 0)] */
	ldx		[%g2 + %g5], %g2
	andcc		%g2, %g7, %g0
	sethi		%hi(kern_linear_pte_xor), %g5
	or		%g5, %lo(kern_linear_pte_xor), %g5
	bne,a,pt	%xcc, 1f
	 add		%g5, 8, %g5

1:	ldx		[%g5], %g2

	.globl		kvmap_linear_patch
kvmap_linear_patch:
	ba,pt		%xcc, kvmap_dtlb_tsb4m_load
	 xor		%g2, %g4, %g5

kvmap_dtlb_vmalloc_addr:
	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)

	KTSB_LOCK_TAG(%g1, %g2, %g7)

	/* Load and check PTE.  */
	ldxa		[%g5] ASI_PHYS_USE_EC, %g5
	mov		1, %g7
	sllx		%g7, TSB_TAG_INVALID_BIT, %g7
	brgez,a,pn	%g5, kvmap_dtlb_longpath
	 KTSB_STORE(%g1, %g7)

	KTSB_WRITE(%g1, %g5, %g6)

	/* fallthrough to TLB load */

kvmap_dtlb_load:

661:	stxa		%g5, [%g0] ASI_DTLB_DATA_IN	! Reload TLB
	retry
	.section	.sun4v_2insn_patch, "ax"
	.word		661b
	nop
	nop
	.previous

	/* For sun4v the ASI_DTLB_DATA_IN store and the retry
	 * instruction get nop'd out and we get here to branch
	 * to the sun4v tlb load code.  The registers are setup
	 * as follows:
	 *
	 * %g4: vaddr
	 * %g5: PTE
	 * %g6:	TAG
	 *
	 * The sun4v TLB load wants the PTE in %g3 so we fix that
	 * up here.
	 */
	ba,pt		%xcc, sun4v_dtlb_load
	 mov		%g5, %g3

kvmap_dtlb_nonlinear:
	/* Catch kernel NULL pointer derefs.  */
	sethi		%hi(PAGE_SIZE), %g5
	cmp		%g4, %g5
	bleu,pn		%xcc, kvmap_dtlb_longpath
	 nop

	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)

kvmap_dtlb_tsbmiss:
	sethi		%hi(MODULES_VADDR), %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_dtlb_longpath
	 mov		(VMALLOC_END >> 24), %g5
	sllx		%g5, 24, %g5
	cmp		%g4, %g5
	bgeu,pn		%xcc, kvmap_dtlb_longpath
	 nop

kvmap_check_obp:
	sethi		%hi(LOW_OBP_ADDRESS), %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_dtlb_vmalloc_addr
	 mov		0x1, %g5
	sllx		%g5, 32, %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_dtlb_obp
	 nop
	ba,pt		%xcc, kvmap_dtlb_vmalloc_addr
	 nop

kvmap_dtlb_longpath:

661:	rdpr	%pstate, %g5
	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
	.section .sun4v_2insn_patch, "ax"
	.word	661b
	SET_GL(1)
	ldxa		[%g0] ASI_SCRATCHPAD, %g5
	.previous

	rdpr	%tl, %g3
	cmp	%g3, 1

661:	mov	TLB_TAG_ACCESS, %g4
	ldxa	[%g4] ASI_DMMU, %g5
	.section .sun4v_2insn_patch, "ax"
	.word	661b
	ldx	[%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
	nop
	.previous

	be,pt	%xcc, sparc64_realfault_common
	 mov	FAULT_CODE_DTLB, %g4
	ba,pt	%xcc, winfix_trampoline
	 nop
Commit	Line	Data
2a7e2990 DM	1	/* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
	2	*
	3	* Copyright (C) 1995, 1997, 2005 David S. Miller <davem@davemloft.net>
	4	* Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de)
	5	* Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx)
	6	* Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
74bf4312	7	*/
2a7e2990	8
2a7e2990 DM	9	#include <asm/head.h>
	10	#include <asm/asi.h>
	11	#include <asm/page.h>
	12	#include <asm/pgtable.h>
74bf4312	13	#include <asm/tsb.h>
2a7e2990 DM	14
	15	.text
	16	.align 32
	17
74bf4312 DM	18	kvmap_itlb:
	19	/* g6: TAG TARGET */
	20	mov TLB_TAG_ACCESS, %g4
	21	ldxa [%g4] ASI_IMMU, %g4
	22
d257d5da DM	23	/* sun4v_itlb_miss branches here with the missing virtual
	24	* address already loaded into %g4
	25	*/
	26	kvmap_itlb_4v:
	27
74bf4312 DM	28	kvmap_itlb_nonlinear:
	29	/* Catch kernel NULL pointer calls. */
	30	sethi %hi(PAGE_SIZE), %g5
	31	cmp %g4, %g5
	32	bleu,pn %xcc, kvmap_dtlb_longpath
	33	nop
	34
	35	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)
	36
	37	kvmap_itlb_tsb_miss:
2a7e2990 DM	38	sethi %hi(LOW_OBP_ADDRESS), %g5
2a7e2990 DM	39	cmp %g4, %g5
74bf4312	40	blu,pn %xcc, kvmap_itlb_vmalloc_addr
2a7e2990 DM	41	mov 0x1, %g5
	42	sllx %g5, 32, %g5
	43	cmp %g4, %g5
74bf4312	44	blu,pn %xcc, kvmap_itlb_obp
2a7e2990 DM	45	nop
2a7e2990 DM	46
74bf4312 DM	47	kvmap_itlb_vmalloc_addr:
	48	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
	49
459b6e62	50	KTSB_LOCK_TAG(%g1, %g2, %g7)
74bf4312 DM	51
	52	/* Load and check PTE. */
	53	ldxa [%g5] ASI_PHYS_USE_EC, %g5
8b234274 DM	54	mov 1, %g7
8b234274 DM	55	sllx %g7, TSB_TAG_INVALID_BIT, %g7
74bf4312	56	brgez,a,pn %g5, kvmap_itlb_longpath
8b234274	57	KTSB_STORE(%g1, %g7)
2a7e2990	58
517af332	59	KTSB_WRITE(%g1, %g5, %g6)
74bf4312 DM	60
	61	/* fallthrough to TLB load */
	62
	63	kvmap_itlb_load:
459b6e62 DM	64
459b6e62 DM	65	661: stxa %g5, [%g0] ASI_ITLB_DATA_IN
2a7e2990	66	retry
459b6e62 DM	67	.section .sun4v_2insn_patch, "ax"
	68	.word 661b
	69	nop
	70	nop
	71	.previous
	72
	73	/* For sun4v the ASI_ITLB_DATA_IN store and the retry
	74	* instruction get nop'd out and we get here to branch
	75	* to the sun4v tlb load code. The registers are setup
	76	* as follows:
	77	*
	78	* %g4: vaddr
	79	* %g5: PTE
	80	* %g6: TAG
	81	*
	82	* The sun4v TLB load wants the PTE in %g3 so we fix that
	83	* up here.
	84	*/
	85	ba,pt %xcc, sun4v_itlb_load
	86	mov %g5, %g3
2a7e2990	87
74bf4312	88	kvmap_itlb_longpath:
45fec05f DM	89
45fec05f DM	90	661: rdpr %pstate, %g5
74bf4312	91	wrpr %g5, PSTATE_AG \| PSTATE_MG, %pstate
df7d6aec	92	.section .sun4v_2insn_patch, "ax"
45fec05f	93	.word 661b
6cc200db	94	SET_GL(1)
45fec05f DM	95	nop
	96	.previous
	97
74bf4312 DM	98	rdpr %tpc, %g5
	99	ba,pt %xcc, sparc64_realfault_common
	100	mov FAULT_CODE_ITLB, %g4
	101
	102	kvmap_itlb_obp:
	103	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)
	104
459b6e62	105	KTSB_LOCK_TAG(%g1, %g2, %g7)
74bf4312	106
517af332	107	KTSB_WRITE(%g1, %g5, %g6)
74bf4312 DM	108
	109	ba,pt %xcc, kvmap_itlb_load
	110	nop
	111
	112	kvmap_dtlb_obp:
	113	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)
	114
459b6e62	115	KTSB_LOCK_TAG(%g1, %g2, %g7)
74bf4312	116
517af332	117	KTSB_WRITE(%g1, %g5, %g6)
74bf4312 DM	118
	119	ba,pt %xcc, kvmap_dtlb_load
	120	nop
c9c10830	121
2a7e2990	122	.align 32
d7744a09 DM	123	kvmap_dtlb_tsb4m_load:
	124	KTSB_LOCK_TAG(%g1, %g2, %g7)
	125	KTSB_WRITE(%g1, %g5, %g6)
	126	ba,pt %xcc, kvmap_dtlb_load
	127	nop
	128
74bf4312 DM	129	kvmap_dtlb:
	130	/* %g6: TAG TARGET */
	131	mov TLB_TAG_ACCESS, %g4
	132	ldxa [%g4] ASI_DMMU, %g4
d257d5da DM	133
	134	/* sun4v_dtlb_miss branches here with the missing virtual
	135	* address already loaded into %g4
	136	*/
	137	kvmap_dtlb_4v:
74bf4312	138	brgez,pn %g4, kvmap_dtlb_nonlinear
56425306 DM	139	nop
56425306 DM	140
d1acb421 DM	141	#ifdef CONFIG_DEBUG_PAGEALLOC
	142	/* Index through the base page size TSB even for linear
	143	* mappings when using page allocation debugging.
	144	*/
	145	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
	146	#else
d7744a09 DM	147	/* Correct TAG_TARGET is already in %g6, check 4mb TSB. */
d7744a09 DM	148	KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
d1acb421	149	#endif
d7744a09 DM	150	/* TSB entry address left in %g1, lookup linear PTE.
	151	* Must preserve %g1 and %g6 (TAG).
	152	*/
	153	kvmap_dtlb_tsb4m_miss:
9cc3a1ac DM	154	sethi %hi(kpte_linear_bitmap), %g2
	155	or %g2, %lo(kpte_linear_bitmap), %g2
	156
	157	/* Clear the PAGE_OFFSET top virtual bits, then shift
	158	* down to get a 256MB physical address index.
	159	*/
	160	sllx %g4, 21, %g5
	161	mov 1, %g7
	162	srlx %g5, 21 + 28, %g5
	163
	164	/* Don't try this at home kids... this depends upon srlx
	165	* only taking the low 6 bits of the shift count in %g5.
	166	*/
	167	sllx %g7, %g5, %g7
	168
	169	/* Divide by 64 to get the offset into the bitmask. */
	170	srlx %g5, 6, %g5
6889331a	171	sllx %g5, 3, %g5
9cc3a1ac DM	172
	173	/* kern_linear_pte_xor[((mask & bit) ? 1 : 0)] */
	174	ldx [%g2 + %g5], %g2
	175	andcc %g2, %g7, %g0
	176	sethi %hi(kern_linear_pte_xor), %g5
	177	or %g5, %lo(kern_linear_pte_xor), %g5
	178	bne,a,pt %xcc, 1f
	179	add %g5, 8, %g5
	180
	181	1: ldx [%g5], %g2
74bf4312	182
56425306 DM	183	.globl kvmap_linear_patch
56425306 DM	184	kvmap_linear_patch:
d7744a09	185	ba,pt %xcc, kvmap_dtlb_tsb4m_load
2a7e2990 DM	186	xor %g2, %g4, %g5
2a7e2990 DM	187
74bf4312 DM	188	kvmap_dtlb_vmalloc_addr:
	189	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
	190
459b6e62	191	KTSB_LOCK_TAG(%g1, %g2, %g7)
74bf4312 DM	192
	193	/* Load and check PTE. */
	194	ldxa [%g5] ASI_PHYS_USE_EC, %g5
8b234274 DM	195	mov 1, %g7
8b234274 DM	196	sllx %g7, TSB_TAG_INVALID_BIT, %g7
74bf4312	197	brgez,a,pn %g5, kvmap_dtlb_longpath
8b234274	198	KTSB_STORE(%g1, %g7)
74bf4312	199
517af332	200	KTSB_WRITE(%g1, %g5, %g6)
74bf4312 DM	201
	202	/* fallthrough to TLB load */
	203
	204	kvmap_dtlb_load:
459b6e62 DM	205
459b6e62 DM	206	661: stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB
74bf4312	207	retry
459b6e62 DM	208	.section .sun4v_2insn_patch, "ax"
	209	.word 661b
	210	nop
	211	nop
	212	.previous
	213
	214	/* For sun4v the ASI_DTLB_DATA_IN store and the retry
	215	* instruction get nop'd out and we get here to branch
	216	* to the sun4v tlb load code. The registers are setup
	217	* as follows:
	218	*
	219	* %g4: vaddr
	220	* %g5: PTE
	221	* %g6: TAG
	222	*
	223	* The sun4v TLB load wants the PTE in %g3 so we fix that
	224	* up here.
	225	*/
	226	ba,pt %xcc, sun4v_dtlb_load
	227	mov %g5, %g3
74bf4312 DM	228
	229	kvmap_dtlb_nonlinear:
	230	/* Catch kernel NULL pointer derefs. */
	231	sethi %hi(PAGE_SIZE), %g5
	232	cmp %g4, %g5
	233	bleu,pn %xcc, kvmap_dtlb_longpath
56425306	234	nop
56425306	235
74bf4312 DM	236	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
	237
	238	kvmap_dtlb_tsbmiss:
2a7e2990 DM	239	sethi %hi(MODULES_VADDR), %g5
2a7e2990 DM	240	cmp %g4, %g5
74bf4312	241	blu,pn %xcc, kvmap_dtlb_longpath
2a7e2990 DM	242	mov (VMALLOC_END >> 24), %g5
	243	sllx %g5, 24, %g5
	244	cmp %g4, %g5
74bf4312	245	bgeu,pn %xcc, kvmap_dtlb_longpath
2a7e2990 DM	246	nop
	247
	248	kvmap_check_obp:
	249	sethi %hi(LOW_OBP_ADDRESS), %g5
	250	cmp %g4, %g5
74bf4312	251	blu,pn %xcc, kvmap_dtlb_vmalloc_addr
2a7e2990 DM	252	mov 0x1, %g5
	253	sllx %g5, 32, %g5
	254	cmp %g4, %g5
74bf4312	255	blu,pn %xcc, kvmap_dtlb_obp
2a7e2990	256	nop
74bf4312	257	ba,pt %xcc, kvmap_dtlb_vmalloc_addr
2a7e2990 DM	258	nop
2a7e2990 DM	259
74bf4312	260	kvmap_dtlb_longpath:
45fec05f DM	261
45fec05f DM	262	661: rdpr %pstate, %g5
74bf4312	263	wrpr %g5, PSTATE_AG \| PSTATE_MG, %pstate
df7d6aec	264	.section .sun4v_2insn_patch, "ax"
45fec05f	265	.word 661b
8b234274 DM	266	SET_GL(1)
8b234274 DM	267	ldxa [%g0] ASI_SCRATCHPAD, %g5
45fec05f DM	268	.previous
45fec05f DM	269
459b6e62 DM	270	rdpr %tl, %g3
	271	cmp %g3, 1
	272
	273	661: mov TLB_TAG_ACCESS, %g4
74bf4312	274	ldxa [%g4] ASI_DMMU, %g5
459b6e62 DM	275	.section .sun4v_2insn_patch, "ax"
459b6e62 DM	276	.word 661b
8b234274	277	ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
459b6e62 DM	278	nop
	279	.previous
	280
74bf4312 DM	281	be,pt %xcc, sparc64_realfault_common
	282	mov FAULT_CODE_DTLB, %g4
	283	ba,pt %xcc, winfix_trampoline
	284	nop