[mirror_ubuntu-artful-kernel.git] / arch / sparc / kernel / ktlb.S

/* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
 *
 * Copyright (C) 1995, 1997, 2005, 2008 David S. Miller <davem@davemloft.net>
 * Copyright (C) 1996 Eddie C. Dost        (ecd@brainaid.de)
 * Copyright (C) 1996 Miguel de Icaza      (miguel@nuclecu.unam.mx)
 * Copyright (C) 1996,98,99 Jakub Jelinek  (jj@sunsite.mff.cuni.cz)
 */

#include <asm/head.h>
#include <asm/asi.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/tsb.h>

	.text
	.align		32

kvmap_itlb:
	/* g6: TAG TARGET */
	mov		TLB_TAG_ACCESS, %g4
	ldxa		[%g4] ASI_IMMU, %g4

	/* The kernel executes in context zero, therefore we do not
	 * need to clear the context ID bits out of %g4 here.
	 */

	/* sun4v_itlb_miss branches here with the missing virtual
	 * address already loaded into %g4
	 */
kvmap_itlb_4v:

	/* Catch kernel NULL pointer calls.  */
	sethi		%hi(PAGE_SIZE), %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_itlb_longpath
	 nop

	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)

kvmap_itlb_tsb_miss:
	sethi		%hi(LOW_OBP_ADDRESS), %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_itlb_vmalloc_addr
	 mov		0x1, %g5
	sllx		%g5, 32, %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_itlb_obp
	 nop

kvmap_itlb_vmalloc_addr:
	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)

	TSB_LOCK_TAG(%g1, %g2, %g7)
	TSB_WRITE(%g1, %g5, %g6)

	/* fallthrough to TLB load */

kvmap_itlb_load:

661:	stxa		%g5, [%g0] ASI_ITLB_DATA_IN
	retry
	.section	.sun4v_2insn_patch, "ax"
	.word		661b
	nop
	nop
	.previous

	/* For sun4v the ASI_ITLB_DATA_IN store and the retry
	 * instruction get nop'd out and we get here to branch
	 * to the sun4v tlb load code.  The registers are setup
	 * as follows:
	 *
	 * %g4: vaddr
	 * %g5: PTE
	 * %g6:	TAG
	 *
	 * The sun4v TLB load wants the PTE in %g3 so we fix that
	 * up here.
	 */
	ba,pt		%xcc, sun4v_itlb_load
	 mov		%g5, %g3

kvmap_itlb_longpath:

661:	rdpr	%pstate, %g5
	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
	.section .sun4v_2insn_patch, "ax"
	.word	661b
	SET_GL(1)
	nop
	.previous

	rdpr	%tpc, %g5
	ba,pt	%xcc, sparc64_realfault_common
	 mov	FAULT_CODE_ITLB, %g4

kvmap_itlb_obp:
	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)

	TSB_LOCK_TAG(%g1, %g2, %g7)

	TSB_WRITE(%g1, %g5, %g6)

	ba,pt		%xcc, kvmap_itlb_load
	 nop

kvmap_dtlb_obp:
	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)

	TSB_LOCK_TAG(%g1, %g2, %g7)

	TSB_WRITE(%g1, %g5, %g6)

	ba,pt		%xcc, kvmap_dtlb_load
	 nop

kvmap_linear_early:
	sethi		%hi(kern_linear_pte_xor), %g7
	ldx		[%g7 + %lo(kern_linear_pte_xor)], %g2
	ba,pt		%xcc, kvmap_dtlb_tsb4m_load
	 xor		%g2, %g4, %g5

	.align		32
kvmap_dtlb_tsb4m_load:
	TSB_LOCK_TAG(%g1, %g2, %g7)
	TSB_WRITE(%g1, %g5, %g6)
	ba,pt		%xcc, kvmap_dtlb_load
	 nop

kvmap_dtlb:
	/* %g6: TAG TARGET */
	mov		TLB_TAG_ACCESS, %g4
	ldxa		[%g4] ASI_DMMU, %g4

	/* The kernel executes in context zero, therefore we do not
	 * need to clear the context ID bits out of %g4 here.
	 */

	/* sun4v_dtlb_miss branches here with the missing virtual
	 * address already loaded into %g4
	 */
kvmap_dtlb_4v:
	brgez,pn	%g4, kvmap_dtlb_nonlinear
	 nop

#ifdef CONFIG_DEBUG_PAGEALLOC
	/* Index through the base page size TSB even for linear
	 * mappings when using page allocation debugging.
	 */
	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
#else
	/* Correct TAG_TARGET is already in %g6, check 4mb TSB.  */
	KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
#endif
	/* Linear mapping TSB lookup failed.  Fallthrough to kernel
	 * page table based lookup.
	 */
	.globl		kvmap_linear_patch
kvmap_linear_patch:
	ba,a,pt		%xcc, kvmap_linear_early

kvmap_dtlb_vmalloc_addr:
	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)

	TSB_LOCK_TAG(%g1, %g2, %g7)
	TSB_WRITE(%g1, %g5, %g6)

	/* fallthrough to TLB load */

kvmap_dtlb_load:

661:	stxa		%g5, [%g0] ASI_DTLB_DATA_IN	! Reload TLB
	retry
	.section	.sun4v_2insn_patch, "ax"
	.word		661b
	nop
	nop
	.previous

	/* For sun4v the ASI_DTLB_DATA_IN store and the retry
	 * instruction get nop'd out and we get here to branch
	 * to the sun4v tlb load code.  The registers are setup
	 * as follows:
	 *
	 * %g4: vaddr
	 * %g5: PTE
	 * %g6:	TAG
	 *
	 * The sun4v TLB load wants the PTE in %g3 so we fix that
	 * up here.
	 */
	ba,pt		%xcc, sun4v_dtlb_load
	 mov		%g5, %g3

#ifdef CONFIG_SPARSEMEM_VMEMMAP
kvmap_vmemmap:
	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
	ba,a,pt		%xcc, kvmap_dtlb_load
#endif

kvmap_dtlb_nonlinear:
	/* Catch kernel NULL pointer derefs.  */
	sethi		%hi(PAGE_SIZE), %g5
	cmp		%g4, %g5
	bleu,pn		%xcc, kvmap_dtlb_longpath
	 nop

#ifdef CONFIG_SPARSEMEM_VMEMMAP
	/* Do not use the TSB for vmemmap.  */
	sethi		%hi(VMEMMAP_BASE), %g5
	ldx		[%g5 + %lo(VMEMMAP_BASE)], %g5
	cmp		%g4,%g5
	bgeu,pn		%xcc, kvmap_vmemmap
	 nop
#endif

	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)

kvmap_dtlb_tsbmiss:
	sethi		%hi(MODULES_VADDR), %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_dtlb_longpath
	 sethi		%hi(VMALLOC_END), %g5
	ldx		[%g5 + %lo(VMALLOC_END)], %g5
	cmp		%g4, %g5
	bgeu,pn		%xcc, kvmap_dtlb_longpath
	 nop

kvmap_check_obp:
	sethi		%hi(LOW_OBP_ADDRESS), %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_dtlb_vmalloc_addr
	 mov		0x1, %g5
	sllx		%g5, 32, %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_dtlb_obp
	 nop
	ba,pt		%xcc, kvmap_dtlb_vmalloc_addr
	 nop

kvmap_dtlb_longpath:

661:	rdpr	%pstate, %g5
	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
	.section .sun4v_2insn_patch, "ax"
	.word	661b
	SET_GL(1)
	ldxa		[%g0] ASI_SCRATCHPAD, %g5
	.previous

	rdpr	%tl, %g3
	cmp	%g3, 1

661:	mov	TLB_TAG_ACCESS, %g4
	ldxa	[%g4] ASI_DMMU, %g5
	.section .sun4v_2insn_patch, "ax"
	.word	661b
	ldx	[%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
	nop
	.previous

	/* The kernel executes in context zero, therefore we do not
	 * need to clear the context ID bits out of %g5 here.
	 */

	be,pt	%xcc, sparc64_realfault_common
	 mov	FAULT_CODE_DTLB, %g4
	ba,pt	%xcc, winfix_trampoline
	 nop
Commit	Line	Data
2a7e2990 DM	1	/* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
2a7e2990 DM	2	*
bf4a7972	3	* Copyright (C) 1995, 1997, 2005, 2008 David S. Miller <davem@davemloft.net>
2a7e2990 DM	4	* Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de)
	5	* Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx)
	6	* Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
74bf4312	7	*/
2a7e2990	8
2a7e2990 DM	9	#include <asm/head.h>
	10	#include <asm/asi.h>
	11	#include <asm/page.h>
	12	#include <asm/pgtable.h>
74bf4312	13	#include <asm/tsb.h>
2a7e2990 DM	14
	15	.text
	16	.align 32
	17
74bf4312 DM	18	kvmap_itlb:
	19	/* g6: TAG TARGET */
	20	mov TLB_TAG_ACCESS, %g4
	21	ldxa [%g4] ASI_IMMU, %g4
	22
4f6deb8c DM	23	/* The kernel executes in context zero, therefore we do not
	24	* need to clear the context ID bits out of %g4 here.
	25	*/
	26
d257d5da DM	27	/* sun4v_itlb_miss branches here with the missing virtual
	28	* address already loaded into %g4
	29	*/
	30	kvmap_itlb_4v:
	31
74bf4312 DM	32	/* Catch kernel NULL pointer calls. */
	33	sethi %hi(PAGE_SIZE), %g5
	34	cmp %g4, %g5
1c2696cd	35	blu,pn %xcc, kvmap_itlb_longpath
74bf4312 DM	36	nop
	37
	38	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)
	39
	40	kvmap_itlb_tsb_miss:
2a7e2990 DM	41	sethi %hi(LOW_OBP_ADDRESS), %g5
2a7e2990 DM	42	cmp %g4, %g5
74bf4312	43	blu,pn %xcc, kvmap_itlb_vmalloc_addr
2a7e2990 DM	44	mov 0x1, %g5
	45	sllx %g5, 32, %g5
	46	cmp %g4, %g5
74bf4312	47	blu,pn %xcc, kvmap_itlb_obp
2a7e2990 DM	48	nop
2a7e2990 DM	49
74bf4312 DM	50	kvmap_itlb_vmalloc_addr:
	51	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
	52
9076d0e7	53	TSB_LOCK_TAG(%g1, %g2, %g7)
9076d0e7	54	TSB_WRITE(%g1, %g5, %g6)
74bf4312 DM	55
	56	/* fallthrough to TLB load */
	57
	58	kvmap_itlb_load:
459b6e62 DM	59
459b6e62 DM	60	661: stxa %g5, [%g0] ASI_ITLB_DATA_IN
2a7e2990	61	retry
459b6e62 DM	62	.section .sun4v_2insn_patch, "ax"
	63	.word 661b
	64	nop
	65	nop
	66	.previous
	67
	68	/* For sun4v the ASI_ITLB_DATA_IN store and the retry
	69	* instruction get nop'd out and we get here to branch
	70	* to the sun4v tlb load code. The registers are setup
	71	* as follows:
	72	*
	73	* %g4: vaddr
	74	* %g5: PTE
	75	* %g6: TAG
	76	*
	77	* The sun4v TLB load wants the PTE in %g3 so we fix that
	78	* up here.
	79	*/
	80	ba,pt %xcc, sun4v_itlb_load
	81	mov %g5, %g3
2a7e2990	82
74bf4312	83	kvmap_itlb_longpath:
45fec05f DM	84
45fec05f DM	85	661: rdpr %pstate, %g5
74bf4312	86	wrpr %g5, PSTATE_AG \| PSTATE_MG, %pstate
df7d6aec	87	.section .sun4v_2insn_patch, "ax"
45fec05f	88	.word 661b
6cc200db	89	SET_GL(1)
45fec05f DM	90	nop
	91	.previous
	92
74bf4312 DM	93	rdpr %tpc, %g5
	94	ba,pt %xcc, sparc64_realfault_common
	95	mov FAULT_CODE_ITLB, %g4
	96
	97	kvmap_itlb_obp:
	98	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)
	99
9076d0e7	100	TSB_LOCK_TAG(%g1, %g2, %g7)
74bf4312	101
9076d0e7	102	TSB_WRITE(%g1, %g5, %g6)
74bf4312 DM	103
	104	ba,pt %xcc, kvmap_itlb_load
	105	nop
	106
	107	kvmap_dtlb_obp:
	108	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)
	109
9076d0e7	110	TSB_LOCK_TAG(%g1, %g2, %g7)
74bf4312	111
9076d0e7	112	TSB_WRITE(%g1, %g5, %g6)
74bf4312 DM	113
	114	ba,pt %xcc, kvmap_dtlb_load
	115	nop
c9c10830	116
0dd5b7b0 DM	117	kvmap_linear_early:
	118	sethi %hi(kern_linear_pte_xor), %g7
	119	ldx [%g7 + %lo(kern_linear_pte_xor)], %g2
	120	ba,pt %xcc, kvmap_dtlb_tsb4m_load
	121	xor %g2, %g4, %g5
	122
2a7e2990	123	.align 32
d7744a09	124	kvmap_dtlb_tsb4m_load:
9076d0e7 DM	125	TSB_LOCK_TAG(%g1, %g2, %g7)
9076d0e7 DM	126	TSB_WRITE(%g1, %g5, %g6)
d7744a09 DM	127	ba,pt %xcc, kvmap_dtlb_load
	128	nop
	129
74bf4312 DM	130	kvmap_dtlb:
	131	/* %g6: TAG TARGET */
	132	mov TLB_TAG_ACCESS, %g4
	133	ldxa [%g4] ASI_DMMU, %g4
d257d5da	134
4f6deb8c DM	135	/* The kernel executes in context zero, therefore we do not
	136	* need to clear the context ID bits out of %g4 here.
	137	*/
	138
d257d5da DM	139	/* sun4v_dtlb_miss branches here with the missing virtual
	140	* address already loaded into %g4
	141	*/
	142	kvmap_dtlb_4v:
74bf4312	143	brgez,pn %g4, kvmap_dtlb_nonlinear
56425306 DM	144	nop
56425306 DM	145
d1acb421 DM	146	#ifdef CONFIG_DEBUG_PAGEALLOC
	147	/* Index through the base page size TSB even for linear
	148	* mappings when using page allocation debugging.
	149	*/
	150	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
	151	#else
d7744a09 DM	152	/* Correct TAG_TARGET is already in %g6, check 4mb TSB. */
d7744a09 DM	153	KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
d1acb421	154	#endif
0dd5b7b0 DM	155	/* Linear mapping TSB lookup failed. Fallthrough to kernel
0dd5b7b0 DM	156	* page table based lookup.
9cc3a1ac	157	*/
56425306 DM	158	.globl kvmap_linear_patch
56425306 DM	159	kvmap_linear_patch:
0dd5b7b0	160	ba,a,pt %xcc, kvmap_linear_early
2a7e2990	161
74bf4312 DM	162	kvmap_dtlb_vmalloc_addr:
	163	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
	164
9076d0e7	165	TSB_LOCK_TAG(%g1, %g2, %g7)
9076d0e7	166	TSB_WRITE(%g1, %g5, %g6)
74bf4312 DM	167
	168	/* fallthrough to TLB load */
	169
	170	kvmap_dtlb_load:
459b6e62 DM	171
459b6e62 DM	172	661: stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB
74bf4312	173	retry
459b6e62 DM	174	.section .sun4v_2insn_patch, "ax"
	175	.word 661b
	176	nop
	177	nop
	178	.previous
	179
	180	/* For sun4v the ASI_DTLB_DATA_IN store and the retry
	181	* instruction get nop'd out and we get here to branch
	182	* to the sun4v tlb load code. The registers are setup
	183	* as follows:
	184	*
	185	* %g4: vaddr
	186	* %g5: PTE
	187	* %g6: TAG
	188	*
	189	* The sun4v TLB load wants the PTE in %g3 so we fix that
	190	* up here.
	191	*/
	192	ba,pt %xcc, sun4v_dtlb_load
	193	mov %g5, %g3
74bf4312	194
bf4a7972	195	#ifdef CONFIG_SPARSEMEM_VMEMMAP
46644c24	196	kvmap_vmemmap:
c06240c7 DM	197	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
c06240c7 DM	198	ba,a,pt %xcc, kvmap_dtlb_load
bf4a7972	199	#endif
46644c24	200
74bf4312 DM	201	kvmap_dtlb_nonlinear:
	202	/* Catch kernel NULL pointer derefs. */
	203	sethi %hi(PAGE_SIZE), %g5
	204	cmp %g4, %g5
	205	bleu,pn %xcc, kvmap_dtlb_longpath
56425306	206	nop
56425306	207
bf4a7972	208	#ifdef CONFIG_SPARSEMEM_VMEMMAP
46644c24	209	/* Do not use the TSB for vmemmap. */
bb4e6e85 DM	210	sethi %hi(VMEMMAP_BASE), %g5
bb4e6e85 DM	211	ldx [%g5 + %lo(VMEMMAP_BASE)], %g5
46644c24 DM	212	cmp %g4,%g5
	213	bgeu,pn %xcc, kvmap_vmemmap
	214	nop
bf4a7972	215	#endif
46644c24	216
74bf4312 DM	217	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
	218
	219	kvmap_dtlb_tsbmiss:
2a7e2990 DM	220	sethi %hi(MODULES_VADDR), %g5
2a7e2990 DM	221	cmp %g4, %g5
74bf4312	222	blu,pn %xcc, kvmap_dtlb_longpath
bb4e6e85 DM	223	sethi %hi(VMALLOC_END), %g5
bb4e6e85 DM	224	ldx [%g5 + %lo(VMALLOC_END)], %g5
2a7e2990	225	cmp %g4, %g5
74bf4312	226	bgeu,pn %xcc, kvmap_dtlb_longpath
2a7e2990 DM	227	nop
	228
	229	kvmap_check_obp:
	230	sethi %hi(LOW_OBP_ADDRESS), %g5
	231	cmp %g4, %g5
74bf4312	232	blu,pn %xcc, kvmap_dtlb_vmalloc_addr
2a7e2990 DM	233	mov 0x1, %g5
	234	sllx %g5, 32, %g5
	235	cmp %g4, %g5
74bf4312	236	blu,pn %xcc, kvmap_dtlb_obp
2a7e2990	237	nop
74bf4312	238	ba,pt %xcc, kvmap_dtlb_vmalloc_addr
2a7e2990 DM	239	nop
2a7e2990 DM	240
74bf4312	241	kvmap_dtlb_longpath:
45fec05f DM	242
45fec05f DM	243	661: rdpr %pstate, %g5
74bf4312	244	wrpr %g5, PSTATE_AG \| PSTATE_MG, %pstate
df7d6aec	245	.section .sun4v_2insn_patch, "ax"
45fec05f	246	.word 661b
8b234274 DM	247	SET_GL(1)
8b234274 DM	248	ldxa [%g0] ASI_SCRATCHPAD, %g5
45fec05f DM	249	.previous
45fec05f DM	250
459b6e62 DM	251	rdpr %tl, %g3
	252	cmp %g3, 1
	253
	254	661: mov TLB_TAG_ACCESS, %g4
74bf4312	255	ldxa [%g4] ASI_DMMU, %g5
459b6e62 DM	256	.section .sun4v_2insn_patch, "ax"
459b6e62 DM	257	.word 661b
8b234274	258	ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
459b6e62 DM	259	nop
	260	.previous
	261
4f6deb8c DM	262	/* The kernel executes in context zero, therefore we do not
	263	* need to clear the context ID bits out of %g5 here.
	264	*/
	265
74bf4312 DM	266	be,pt %xcc, sparc64_realfault_common
	267	mov FAULT_CODE_DTLB, %g4
	268	ba,pt %xcc, winfix_trampoline
	269	nop