[mirror_ubuntu-zesty-kernel.git] / drivers / gpu / drm / nouveau / nvkm / engine / gr / fuc / gpc.fuc

/* fuc microcode for gf100 PGRAPH/GPC
 *
 * Copyright 2011 Red Hat Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors: Ben Skeggs
 */

/* TODO
 * - bracket certain functions with scratch writes, useful for debugging
 * - watchdog timer around ctx operations
 */

#ifdef INCLUDE_DATA
gpc_mmio_list_head:	.b32 #mmio_list_base
gpc_mmio_list_tail:
tpc_mmio_list_head:	.b32 #mmio_list_base
tpc_mmio_list_tail:
unk_mmio_list_head:	.b32 #mmio_list_base
unk_mmio_list_tail:	.b32 #mmio_list_base

gpc_id:			.b32 0

tpc_count:		.b32 0
tpc_mask:		.b32 0

#if NV_PGRAPH_GPCX_UNK__SIZE > 0
unk_count:		.b32 0
unk_mask:		.b32 0
#endif

cmd_queue:		queue_init

mmio_list_base:
#endif

#ifdef INCLUDE_CODE
#define gpc_addr(reg,addr)                                                    /*
*/	imm32(reg,addr)                                                       /*
*/	or reg NV_PGRAPH_GPCX_GPCCS_MMIO_CTRL_BASE_ENABLE
#define gpc_wr32(addr,reg)                                                    /*
*/	gpc_addr($r14,addr)                                                   /*
*/	mov b32 $r15 reg                                                      /*
*/	call(nv_wr32)

// reports an exception to the host
//
// In: $r15 error code (see os.h)
//
error:
	push $r14
	nv_wr32(NV_PGRAPH_FECS_CC_SCRATCH_VAL(5), $r15)
	mov $r15 1
	nv_wr32(NV_PGRAPH_FECS_INTR_UP_SET, $r15)
	pop $r14
	ret

#if CHIPSET >= GM107
tpc_strand_wait:
	push $r9
	trace_set(T_STRTPC)
	tpc_strand_busy:
		nv_iord($r9, NV_PGRAPH_GPCX_GPCCS_TPC_STATUS, 0)
		bra b32 $r9 0x0 ne #tpc_strand_busy
	trace_clr(T_STRTPC)
	pop $r9
	ret

#define tpc_strand_wait() call(tpc_strand_wait)
#define tpc_strand_enable()                                                   /*
*/	mov $r15 NV_PGRAPH_GPC0_TPCX_STRAND_CMD_ENABLE                        /*
*/	gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_CMD, $r15)                        /*
*/	tpc_strand_wait()
#define tpc_strand_disable()                                                  /*
*/	mov $r15 NV_PGRAPH_GPC0_TPCX_STRAND_CMD_DISABLE                       /*
*/	gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_CMD, $r15)                        /*
*/	tpc_strand_wait()
#define tpc_strand_seek(p)                                                    /*
*/	mov $r15 NV_PGRAPH_GPC0_TPCX_STRAND_INDEX_ALL                         /*
*/	gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_INDEX, $r15)                      /*
*/	mov $r15 p                                                            /*
*/	gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_SELECT, $r15)                     /*
*/	mov $r15 NV_PGRAPH_GPC0_TPCX_STRAND_CMD_SEEK                          /*
*/	tpc_strand_wait()
#define tpc_strand_info(m)                                                    /*
*/	gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_CMD, $r15)                        /*
*/	mov $r15 m                                                            /*
*/	gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_DATA, $r15)                       /*
*/	mov $r15 NV_PGRAPH_GPC0_TPCX_STRAND_CMD_GET_INFO                      /*
*/	gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_CMD, $r15)                        /*
*/	tpc_strand_wait()
#endif


// GPC fuc initialisation, executed by triggering ucode start, will
// fall through to main loop after completion.
//
// Input:
//   CC_SCRATCH[1]: context base
//
// Output:
//   CC_SCRATCH[0]:
//	     31:31: set to signal completion
//   CC_SCRATCH[1]:
//	      31:0: GPC context size
//
init:
	clear b32 $r0

	// setup stack
	nv_iord($r1, NV_PGRAPH_GPCX_GPCCS_CAPS, 0)
	extr $r1 $r1 9:17
	shl b32 $r1 8
	mov $sp $r1

	// enable fifo access
	mov $r2 NV_PGRAPH_GPCX_GPCCS_ACCESS_FIFO
	nv_iowr(NV_PGRAPH_GPCX_GPCCS_ACCESS, 0, $r2)

	// setup i0 handler, and route all interrupts to it
	mov $r1 #ih
	mov $iv0 $r1
	nv_iowr(NV_PGRAPH_GPCX_GPCCS_INTR_ROUTE, 0, $r0)

	// enable fifo interrupt
	mov $r2 NV_PGRAPH_GPCX_GPCCS_INTR_EN_SET_FIFO
	nv_iowr(NV_PGRAPH_GPCX_GPCCS_INTR_EN_SET, 0, $r2)

	// enable interrupts
	bset $flags ie0

	// how many TPCs do we have?
	nv_iord($r2, NV_PGRAPH_GPCX_GPCCS_UNITS, 0)
	mov $r3 1
	and $r2 0x1f
	shl b32 $r3 $r2
	sub b32 $r3 1
	st b32 D[$r0 + #tpc_count] $r2
	st b32 D[$r0 + #tpc_mask] $r3

	// determine which GPC we are, setup (optional) mmio access offset
	nv_iord($r2, NV_PGRAPH_GPCX_GPCCS_MYINDEX, 0)
	st b32 D[$r0 + #gpc_id] $r2
	shl b32 $r2 15
	nv_iowr(NV_PGRAPH_GPCX_GPCCS_MMIO_BASE, 0, $r2)

#if NV_PGRAPH_GPCX_UNK__SIZE > 0
	// figure out which, and how many, UNKs are actually present
	gpc_addr($r14, 0x500c30)
	clear b32 $r2
	clear b32 $r3
	clear b32 $r4
	init_unk_loop:
		call(nv_rd32)
		cmp b32 $r15 0
		bra z #init_unk_next
			mov $r15 1
			shl b32 $r15 $r2
			or $r4 $r15
			add b32 $r3 1
		init_unk_next:
		add b32 $r2 1
		add b32 $r14 4
		cmp b32 $r2 NV_PGRAPH_GPCX_UNK__SIZE
		bra ne #init_unk_loop
	init_unk_done:
	st b32 D[$r0 + #unk_count] $r3
	st b32 D[$r0 + #unk_mask] $r4
#endif

	// initialise context base, and size tracking
	nv_iord($r2, NV_PGRAPH_GPCX_GPCCS_CC_SCRATCH_VAL(1), 0)
	clear b32 $r3		// track GPC context size here

	// set mmctx base addresses now so we don't have to do it later,
	// they don't currently ever change
	shr b32 $r5 $r2 8
	nv_iowr(NV_PGRAPH_GPCX_GPCCS_MMCTX_SAVE_SWBASE, 0, $r5)
	nv_iowr(NV_PGRAPH_GPCX_GPCCS_MMCTX_LOAD_SWBASE, 0, $r5)

	// calculate GPC mmio context size
	ld b32 $r14 D[$r0 + #gpc_mmio_list_head]
	ld b32 $r15 D[$r0 + #gpc_mmio_list_tail]
	call(mmctx_size)
	add b32 $r2 $r15
	add b32 $r3 $r15

	// calculate per-TPC mmio context size
	ld b32 $r14 D[$r0 + #tpc_mmio_list_head]
	ld b32 $r15 D[$r0 + #tpc_mmio_list_tail]
	call(mmctx_size)
	ld b32 $r14 D[$r0 + #tpc_count]
	mulu $r14 $r15
	add b32 $r2 $r14
	add b32 $r3 $r14

#if NV_PGRAPH_GPCX_UNK__SIZE > 0
	// calculate per-UNK mmio context size
	ld b32 $r14 D[$r0 + #unk_mmio_list_head]
	ld b32 $r15 D[$r0 + #unk_mmio_list_tail]
	call(mmctx_size)
	ld b32 $r14 D[$r0 + #unk_count]
	mulu $r14 $r15
	add b32 $r2 $r14
	add b32 $r3 $r14
#endif

	// round up base/size to 256 byte boundary (for strand SWBASE)
	shr b32 $r3 2
	nv_iowr(NV_PGRAPH_GPCX_GPCCS_MMCTX_LOAD_COUNT, 0, $r3) // wtf for?!
	shr b32 $r2 8
	shr b32 $r3 6
	add b32 $r2 1
	add b32 $r3 1
	shl b32 $r2 8
	shl b32 $r3 8

	// calculate size of strand context data
	mov b32 $r15 $r2
	call(strand_ctx_init)
	add b32 $r2 $r15
	add b32 $r3 $r15

#if CHIPSET >= GM107
	// calculate size of tpc strand context data
	mov $r15 NV_PGRAPH_GPC0_TPCX_STRAND_INDEX_ALL
	gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_INDEX, $r15)
	tpc_strand_enable();
	tpc_strand_seek(0);
	tpc_strand_info(-1);

	ld b32 $r4 D[$r0 + #tpc_count]
	mov $r5 NV_PGRAPH_GPC0_TPC0
	ld b32 $r6 D[$r0 + #gpc_id]
	shl b32 $r6 15
	add b32 $r5 $r6
	tpc_strand_init_tpc_loop:
		add b32 $r14 $r5 NV_TPC_STRAND_CNT
		call(nv_rd32)
		mov b32 $r6 $r15
		clear b32 $r7
		tpc_strand_init_idx_loop:
			add b32 $r14 $r5 NV_TPC_STRAND_INDEX
			mov b32 $r15 $r7
			call(nv_wr32)
			add b32 $r14 $r5 NV_TPC_STRAND_SAVE_SWBASE
			shr b32 $r15 $r2 8
			call(nv_wr32)
			add b32 $r14 $r5 NV_TPC_STRAND_LOAD_SWBASE
			shr b32 $r15 $r2 8
			call(nv_wr32)
			add b32 $r14 $r5 NV_TPC_STRAND_WORDS
			call(nv_rd32)
			shr b32 $r15 6
			add b32 $r15 1
			shl b32 $r15 8
			add b32 $r2 $r15
			add b32 $r3 $r15
			add b32 $r7 1
			sub b32 $r6 1
			bra nz #tpc_strand_init_idx_loop
		add b32 $r5 NV_PGRAPH_GPC0_TPC0__SIZE
		sub b32 $r4 1
		bra nz #tpc_strand_init_tpc_loop

	mov $r15 NV_PGRAPH_GPC0_TPCX_STRAND_INDEX_ALL
	gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_INDEX, $r15)
	tpc_strand_disable();
#endif

	// save context size, and tell HUB we're done
	nv_iowr(NV_PGRAPH_GPCX_GPCCS_CC_SCRATCH_VAL(1), 0, $r3)
	clear b32 $r2
	bset $r2 31
	nv_iowr(NV_PGRAPH_GPCX_GPCCS_CC_SCRATCH_SET(0), 0, $r2)

// Main program loop, very simple, sleeps until woken up by the interrupt
// handler, pulls a command from the queue and executes its handler
//
main:
	bset $flags $p0
	sleep $p0
	mov $r13 #cmd_queue
	call(queue_get)
	bra $p1 #main

	// 0x0000-0x0003 are all context transfers
	cmpu b32 $r14 0x04
	bra nc #main_not_ctx_xfer
		// fetch $flags and mask off $p1/$p2
		mov $r1 $flags
		mov $r2 0x0006
		not b32 $r2
		and $r1 $r2
		// set $p1/$p2 according to transfer type
		shl b32 $r14 1
		or $r1 $r14
		mov $flags $r1
		// transfer context data
		call(ctx_xfer)
		bra #main

	main_not_ctx_xfer:
	shl b32 $r15 $r14 16
	or $r15 E_BAD_COMMAND
	call(error)
	bra #main

// interrupt handler
ih:
	push $r8
	mov $r8 $flags
	push $r8
	push $r9
	push $r10
	push $r11
	push $r13
	push $r14
	push $r15
	clear b32 $r0

	// incoming fifo command?
	nv_iord($r10, NV_PGRAPH_GPCX_GPCCS_INTR, 0)
	and $r11 $r10 NV_PGRAPH_GPCX_GPCCS_INTR_FIFO
	bra e #ih_no_fifo
		// queue incoming fifo command for later processing
		mov $r13 #cmd_queue
		nv_iord($r14, NV_PGRAPH_GPCX_GPCCS_FIFO_CMD, 0)
		nv_iord($r15, NV_PGRAPH_GPCX_GPCCS_FIFO_DATA, 0)
		call(queue_put)
		mov $r14 1
		nv_iowr(NV_PGRAPH_GPCX_GPCCS_FIFO_ACK, 0, $r14)

	// ack, and wake up main()
	ih_no_fifo:
	nv_iowr(NV_PGRAPH_GPCX_GPCCS_INTR_ACK, 0, $r10)

	pop $r15
	pop $r14
	pop $r13
	pop $r11
	pop $r10
	pop $r9
	pop $r8
	mov $flags $r8
	pop $r8
	bclr $flags $p0
	iret

// Set this GPC's bit in HUB_BAR, used to signal completion of various
// activities to the HUB fuc
//
hub_barrier_done:
	mov $r15 1
	ld b32 $r14 D[$r0 + #gpc_id]
	shl b32 $r15 $r14
	nv_wr32(0x409418, $r15)	// 0x409418 - HUB_BAR_SET
	ret

// Disables various things, waits a bit, and re-enables them..
//
// Not sure how exactly this helps, perhaps "ENABLE" is not such a
// good description for the bits we turn off?  Anyways, without this,
// funny things happen.
//
ctx_redswitch:
	mov $r15 NV_PGRAPH_GPCX_GPCCS_RED_SWITCH_POWER
	nv_iowr(NV_PGRAPH_GPCX_GPCCS_RED_SWITCH, 0, $r15)
	mov $r14 8
	ctx_redswitch_delay:
		sub b32 $r14 1
		bra ne #ctx_redswitch_delay
	or $r15 NV_PGRAPH_GPCX_GPCCS_RED_SWITCH_UNK11
	or $r15 NV_PGRAPH_GPCX_GPCCS_RED_SWITCH_ENABLE
	nv_iowr(NV_PGRAPH_GPCX_GPCCS_RED_SWITCH, 0, $r15)
	ret

// Transfer GPC context data between GPU and storage area
//
// In: $r15 context base address
//     $p1 clear on save, set on load
//     $p2 set if opposite direction done/will be done, so:
//		on save it means: "a load will follow this save"
//		on load it means: "a save preceeded this load"
//
ctx_xfer:
	// set context base address
	nv_iowr(NV_PGRAPH_GPCX_GPCCS_MEM_BASE, 0, $r15)
#if CHIPSET >= GM107
	gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_MEM_BASE, $r15)
#endif
	bra not $p1 #ctx_xfer_not_load
		call(ctx_redswitch)
	ctx_xfer_not_load:

	// strands
	call(strand_pre)
	clear b32 $r2
	nv_iowr(NV_PGRAPH_GPCX_GPCCS_STRAND_SELECT, 0x3f, $r2)
	xbit $r2 $flags $p1	// SAVE/LOAD
	add b32 $r2 NV_PGRAPH_GPCX_GPCCS_STRAND_CMD_SAVE
	nv_iowr(NV_PGRAPH_GPCX_GPCCS_STRAND_CMD, 0x3f, $r2)

#if CHIPSET >= GM107
	tpc_strand_enable();
	tpc_strand_seek(0);
	xbit $r15 $flags $p1	// SAVE/LOAD
	add b32 $r15 NV_PGRAPH_GPC0_TPCX_STRAND_CMD_SAVE
	gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_CMD, $r15)
#endif

	// mmio context
	xbit $r10 $flags $p1	// direction
	or $r10 2		// first
	imm32($r11,0x500000)
	ld b32 $r12 D[$r0 + #gpc_id]
	shl b32 $r12 15
	add b32 $r11 $r12	// base = NV_PGRAPH_GPCn
	ld b32 $r12 D[$r0 + #gpc_mmio_list_head]
	ld b32 $r13 D[$r0 + #gpc_mmio_list_tail]
	mov $r14 0		// not multi
	call(mmctx_xfer)

	// per-TPC mmio context
	xbit $r10 $flags $p1	// direction
#if !NV_PGRAPH_GPCX_UNK__SIZE
	or $r10 4		// last
#endif
	imm32($r11, 0x504000)
	ld b32 $r12 D[$r0 + #gpc_id]
	shl b32 $r12 15
	add b32 $r11 $r12	// base = NV_PGRAPH_GPCn_TPC0
	ld b32 $r12 D[$r0 + #tpc_mmio_list_head]
	ld b32 $r13 D[$r0 + #tpc_mmio_list_tail]
	ld b32 $r15 D[$r0 + #tpc_mask]
	mov $r14 0x800		// stride = 0x800
	call(mmctx_xfer)

#if NV_PGRAPH_GPCX_UNK__SIZE > 0
	// per-UNK mmio context
	xbit $r10 $flags $p1	// direction
	or $r10 4		// last
	imm32($r11, 0x503000)
	ld b32 $r12 D[$r0 + #gpc_id]
	shl b32 $r12 15
	add b32 $r11 $r12	// base = NV_PGRAPH_GPCn_UNK0
	ld b32 $r12 D[$r0 + #unk_mmio_list_head]
	ld b32 $r13 D[$r0 + #unk_mmio_list_tail]
	ld b32 $r15 D[$r0 + #unk_mask]
	mov $r14 0x200		// stride = 0x200
	call(mmctx_xfer)
#endif

	// wait for strands to finish
	call(strand_wait)
#if CHIPSET >= GM107
	tpc_strand_wait()
#endif

	// if load, or a save without a load following, do some
	// unknown stuff that's done after finishing a block of
	// strand commands
	bra $p1 #ctx_xfer_post
	bra not $p2 #ctx_xfer_done
	ctx_xfer_post:
		call(strand_post)
#if CHIPSET >= GM107
		tpc_strand_disable()
#endif

	// mark completion in HUB's barrier
	ctx_xfer_done:
	call(hub_barrier_done)
	ret
#endif
Commit	Line	Data
e3c71eb2	1	/* fuc microcode for gf100 PGRAPH/GPC
e99716f1 BS	2	*
	3	* Copyright 2011 Red Hat Inc.
	4	*
	5	* Permission is hereby granted, free of charge, to any person obtaining a
	6	* copy of this software and associated documentation files (the "Software"),
	7	* to deal in the Software without restriction, including without limitation
	8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	9	* and/or sell copies of the Software, and to permit persons to whom the
	10	* Software is furnished to do so, subject to the following conditions:
	11	*
	12	* The above copyright notice and this permission notice shall be included in
	13	* all copies or substantial portions of the Software.
	14	*
	15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	18	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	19	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	20	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	21	* OTHER DEALINGS IN THE SOFTWARE.
	22	*
	23	* Authors: Ben Skeggs
	24	*/
	25
	26	/* TODO
	27	* - bracket certain functions with scratch writes, useful for debugging
	28	* - watchdog timer around ctx operations
	29	*/
	30
	31	#ifdef INCLUDE_DATA
a32b2ffb BS	32	gpc_mmio_list_head: .b32 #mmio_list_base
	33	gpc_mmio_list_tail:
	34	tpc_mmio_list_head: .b32 #mmio_list_base
	35	tpc_mmio_list_tail:
	36	unk_mmio_list_head: .b32 #mmio_list_base
	37	unk_mmio_list_tail: .b32 #mmio_list_base
	38
e99716f1	39	gpc_id: .b32 0
e99716f1 BS	40
	41	tpc_count: .b32 0
	42	tpc_mask: .b32 0
e99716f1	43
26410c67	44	#if NV_PGRAPH_GPCX_UNK__SIZE > 0
f8adeb82 BS	45	unk_count: .b32 0
f8adeb82 BS	46	unk_mask: .b32 0
5ee86c41 BS	47	#endif
5ee86c41 BS	48
e99716f1	49	cmd_queue: queue_init
a32b2ffb BS	50
a32b2ffb BS	51	mmio_list_base:
e99716f1 BS	52	#endif
	53
	54	#ifdef INCLUDE_CODE
7028156a BS	55	#define gpc_addr(reg,addr) /*
	56	/ imm32(reg,addr) /
	57	*/ or reg NV_PGRAPH_GPCX_GPCCS_MMIO_CTRL_BASE_ENABLE
2a19b3ed	58	#define gpc_wr32(addr,reg) /*
7028156a	59	/ gpc_addr($r14,addr) /
2a19b3ed	60	/ mov b32 $r15 reg /
2a19b3ed BS	61	*/ call(nv_wr32)
2a19b3ed BS	62
e99716f1 BS	63	// reports an exception to the host
e99716f1 BS	64	//
23f67841	65	// In: $r15 error code (see os.h)
e99716f1 BS	66	//
	67	error:
	68	push $r14
67af60f0	69	nv_wr32(NV_PGRAPH_FECS_CC_SCRATCH_VAL(5), $r15)
e99716f1	70	mov $r15 1
67af60f0	71	nv_wr32(NV_PGRAPH_FECS_INTR_UP_SET, $r15)
e99716f1 BS	72	pop $r14
	73	ret
	74
8fd4b7d4 BS	75	#if CHIPSET >= GM107
	76	tpc_strand_wait:
	77	push $r9
	78	trace_set(T_STRTPC)
	79	tpc_strand_busy:
	80	nv_iord($r9, NV_PGRAPH_GPCX_GPCCS_TPC_STATUS, 0)
	81	bra b32 $r9 0x0 ne #tpc_strand_busy
	82	trace_clr(T_STRTPC)
	83	pop $r9
	84	ret
	85
	86	#define tpc_strand_wait() call(tpc_strand_wait)
	87	#define tpc_strand_enable() /*
	88	/ mov $r15 NV_PGRAPH_GPC0_TPCX_STRAND_CMD_ENABLE /
	89	/ gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_CMD, $r15) /
	90	*/ tpc_strand_wait()
	91	#define tpc_strand_disable() /*
	92	/ mov $r15 NV_PGRAPH_GPC0_TPCX_STRAND_CMD_DISABLE /
	93	/ gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_CMD, $r15) /
	94	*/ tpc_strand_wait()
	95	#define tpc_strand_seek(p) /*
	96	/ mov $r15 NV_PGRAPH_GPC0_TPCX_STRAND_INDEX_ALL /
	97	/ gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_INDEX, $r15) /
	98	/ mov $r15 p /
	99	/ gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_SELECT, $r15) /
	100	/ mov $r15 NV_PGRAPH_GPC0_TPCX_STRAND_CMD_SEEK /
	101	*/ tpc_strand_wait()
	102	#define tpc_strand_info(m) /*
	103	/ gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_CMD, $r15) /
	104	/ mov $r15 m /
	105	/ gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_DATA, $r15) /
	106	/ mov $r15 NV_PGRAPH_GPC0_TPCX_STRAND_CMD_GET_INFO /
	107	/ gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_CMD, $r15) /
	108	*/ tpc_strand_wait()
	109	#endif
	110
	111
e99716f1 BS	112	// GPC fuc initialisation, executed by triggering ucode start, will
	113	// fall through to main loop after completion.
	114	//
	115	// Input:
e99716f1 BS	116	// CC_SCRATCH[1]: context base
	117	//
	118	// Output:
	119	// CC_SCRATCH[0]:
	120	// 31:31: set to signal completion
	121	// CC_SCRATCH[1]:
	122	// 31:0: GPC context size
	123	//
	124	init:
	125	clear b32 $r0
6acc09b9 BS	126
	127	// setup stack
	128	nv_iord($r1, NV_PGRAPH_GPCX_GPCCS_CAPS, 0)
	129	extr $r1 $r1 9:17
	130	shl b32 $r1 8
	131	mov $sp $r1
e99716f1 BS	132
e99716f1 BS	133	// enable fifo access
67af60f0 BS	134	mov $r2 NV_PGRAPH_GPCX_GPCCS_ACCESS_FIFO
67af60f0 BS	135	nv_iowr(NV_PGRAPH_GPCX_GPCCS_ACCESS, 0, $r2)
e99716f1 BS	136
	137	// setup i0 handler, and route all interrupts to it
	138	mov $r1 #ih
	139	mov $iv0 $r1
67af60f0	140	nv_iowr(NV_PGRAPH_GPCX_GPCCS_INTR_ROUTE, 0, $r0)
e99716f1 BS	141
e99716f1 BS	142	// enable fifo interrupt
67af60f0 BS	143	mov $r2 NV_PGRAPH_GPCX_GPCCS_INTR_EN_SET_FIFO
67af60f0 BS	144	nv_iowr(NV_PGRAPH_GPCX_GPCCS_INTR_EN_SET, 0, $r2)
e99716f1 BS	145
	146	// enable interrupts
	147	bset $flags ie0
	148
2a19b3ed	149	// how many TPCs do we have?
67af60f0	150	nv_iord($r2, NV_PGRAPH_GPCX_GPCCS_UNITS, 0)
e99716f1 BS	151	mov $r3 1
	152	and $r2 0x1f
	153	shl b32 $r3 $r2
	154	sub b32 $r3 1
	155	st b32 D[$r0 + #tpc_count] $r2
	156	st b32 D[$r0 + #tpc_mask] $r3
2a19b3ed BS	157
2a19b3ed BS	158	// determine which GPC we are, setup (optional) mmio access offset
67af60f0	159	nv_iord($r2, NV_PGRAPH_GPCX_GPCCS_MYINDEX, 0)
e99716f1	160	st b32 D[$r0 + #gpc_id] $r2
2a19b3ed BS	161	shl b32 $r2 15
2a19b3ed BS	162	nv_iowr(NV_PGRAPH_GPCX_GPCCS_MMIO_BASE, 0, $r2)
e99716f1	163
f8adeb82 BS	164	#if NV_PGRAPH_GPCX_UNK__SIZE > 0
f8adeb82 BS	165	// figure out which, and how many, UNKs are actually present
ccb7b6ba	166	gpc_addr($r14, 0x500c30)
f8adeb82 BS	167	clear b32 $r2
	168	clear b32 $r3
	169	clear b32 $r4
	170	init_unk_loop:
67af60f0	171	call(nv_rd32)
f8adeb82 BS	172	cmp b32 $r15 0
	173	bra z #init_unk_next
	174	mov $r15 1
	175	shl b32 $r15 $r2
	176	or $r4 $r15
	177	add b32 $r3 1
	178	init_unk_next:
	179	add b32 $r2 1
	180	add b32 $r14 4
	181	cmp b32 $r2 NV_PGRAPH_GPCX_UNK__SIZE
	182	bra ne #init_unk_loop
	183	init_unk_done:
	184	st b32 D[$r0 + #unk_count] $r3
	185	st b32 D[$r0 + #unk_mask] $r4
	186	#endif
	187
e99716f1	188	// initialise context base, and size tracking
60a4acd7	189	nv_iord($r2, NV_PGRAPH_GPCX_GPCCS_CC_SCRATCH_VAL(1), 0)
e99716f1 BS	190	clear b32 $r3 // track GPC context size here
	191
	192	// set mmctx base addresses now so we don't have to do it later,
	193	// they don't currently ever change
e99716f1	194	shr b32 $r5 $r2 8
67af60f0 BS	195	nv_iowr(NV_PGRAPH_GPCX_GPCCS_MMCTX_SAVE_SWBASE, 0, $r5)
67af60f0 BS	196	nv_iowr(NV_PGRAPH_GPCX_GPCCS_MMCTX_LOAD_SWBASE, 0, $r5)
e99716f1	197
a32b2ffb BS	198	// calculate GPC mmio context size
	199	ld b32 $r14 D[$r0 + #gpc_mmio_list_head]
	200	ld b32 $r15 D[$r0 + #gpc_mmio_list_tail]
67af60f0	201	call(mmctx_size)
e99716f1 BS	202	add b32 $r2 $r15
	203	add b32 $r3 $r15
	204
a32b2ffb BS	205	// calculate per-TPC mmio context size
	206	ld b32 $r14 D[$r0 + #tpc_mmio_list_head]
	207	ld b32 $r15 D[$r0 + #tpc_mmio_list_tail]
67af60f0	208	call(mmctx_size)
e99716f1 BS	209	ld b32 $r14 D[$r0 + #tpc_count]
	210	mulu $r14 $r15
	211	add b32 $r2 $r14
	212	add b32 $r3 $r14
	213
26410c67	214	#if NV_PGRAPH_GPCX_UNK__SIZE > 0
5ee86c41 BS	215	// calculate per-UNK mmio context size
	216	ld b32 $r14 D[$r0 + #unk_mmio_list_head]
	217	ld b32 $r15 D[$r0 + #unk_mmio_list_tail]
67af60f0	218	call(mmctx_size)
5ee86c41 BS	219	ld b32 $r14 D[$r0 + #unk_count]
	220	mulu $r14 $r15
	221	add b32 $r2 $r14
	222	add b32 $r3 $r14
	223	#endif
	224
e99716f1	225	// round up base/size to 256 byte boundary (for strand SWBASE)
e99716f1	226	shr b32 $r3 2
67af60f0	227	nv_iowr(NV_PGRAPH_GPCX_GPCCS_MMCTX_LOAD_COUNT, 0, $r3) // wtf for?!
e99716f1 BS	228	shr b32 $r2 8
	229	shr b32 $r3 6
	230	add b32 $r2 1
	231	add b32 $r3 1
	232	shl b32 $r2 8
	233	shl b32 $r3 8
	234
	235	// calculate size of strand context data
	236	mov b32 $r15 $r2
67af60f0	237	call(strand_ctx_init)
8fd4b7d4	238	add b32 $r2 $r15
e99716f1 BS	239	add b32 $r3 $r15
e99716f1 BS	240
8fd4b7d4 BS	241	#if CHIPSET >= GM107
	242	// calculate size of tpc strand context data
	243	mov $r15 NV_PGRAPH_GPC0_TPCX_STRAND_INDEX_ALL
	244	gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_INDEX, $r15)
	245	tpc_strand_enable();
	246	tpc_strand_seek(0);
	247	tpc_strand_info(-1);
	248
	249	ld b32 $r4 D[$r0 + #tpc_count]
	250	mov $r5 NV_PGRAPH_GPC0_TPC0
	251	ld b32 $r6 D[$r0 + #gpc_id]
	252	shl b32 $r6 15
	253	add b32 $r5 $r6
	254	tpc_strand_init_tpc_loop:
	255	add b32 $r14 $r5 NV_TPC_STRAND_CNT
	256	call(nv_rd32)
	257	mov b32 $r6 $r15
	258	clear b32 $r7
	259	tpc_strand_init_idx_loop:
	260	add b32 $r14 $r5 NV_TPC_STRAND_INDEX
	261	mov b32 $r15 $r7
	262	call(nv_wr32)
	263	add b32 $r14 $r5 NV_TPC_STRAND_SAVE_SWBASE
	264	shr b32 $r15 $r2 8
	265	call(nv_wr32)
	266	add b32 $r14 $r5 NV_TPC_STRAND_LOAD_SWBASE
	267	shr b32 $r15 $r2 8
	268	call(nv_wr32)
	269	add b32 $r14 $r5 NV_TPC_STRAND_WORDS
	270	call(nv_rd32)
	271	shr b32 $r15 6
	272	add b32 $r15 1
	273	shl b32 $r15 8
	274	add b32 $r2 $r15
	275	add b32 $r3 $r15
	276	add b32 $r7 1
	277	sub b32 $r6 1
	278	bra nz #tpc_strand_init_idx_loop
	279	add b32 $r5 NV_PGRAPH_GPC0_TPC0__SIZE
	280	sub b32 $r4 1
	281	bra nz #tpc_strand_init_tpc_loop
	282
	283	mov $r15 NV_PGRAPH_GPC0_TPCX_STRAND_INDEX_ALL
	284	gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_INDEX, $r15)
	285	tpc_strand_disable();
	286	#endif
	287
e99716f1	288	// save context size, and tell HUB we're done
60a4acd7	289	nv_iowr(NV_PGRAPH_GPCX_GPCCS_CC_SCRATCH_VAL(1), 0, $r3)
e99716f1 BS	290	clear b32 $r2
e99716f1 BS	291	bset $r2 31
60a4acd7	292	nv_iowr(NV_PGRAPH_GPCX_GPCCS_CC_SCRATCH_SET(0), 0, $r2)
e99716f1 BS	293
	294	// Main program loop, very simple, sleeps until woken up by the interrupt
	295	// handler, pulls a command from the queue and executes its handler
	296	//
	297	main:
	298	bset $flags $p0
	299	sleep $p0
	300	mov $r13 #cmd_queue
67af60f0	301	call(queue_get)
e99716f1 BS	302	bra $p1 #main
	303
	304	// 0x0000-0x0003 are all context transfers
	305	cmpu b32 $r14 0x04
	306	bra nc #main_not_ctx_xfer
	307	// fetch $flags and mask off $p1/$p2
	308	mov $r1 $flags
	309	mov $r2 0x0006
	310	not b32 $r2
	311	and $r1 $r2
	312	// set $p1/$p2 according to transfer type
	313	shl b32 $r14 1
	314	or $r1 $r14
	315	mov $flags $r1
	316	// transfer context data
67af60f0	317	call(ctx_xfer)
e99716f1 BS	318	bra #main
	319
	320	main_not_ctx_xfer:
	321	shl b32 $r15 $r14 16
	322	or $r15 E_BAD_COMMAND
67af60f0	323	call(error)
e99716f1 BS	324	bra #main
	325
	326	// interrupt handler
	327	ih:
	328	push $r8
	329	mov $r8 $flags
	330	push $r8
	331	push $r9
	332	push $r10
	333	push $r11
	334	push $r13
	335	push $r14
	336	push $r15
60a4acd7	337	clear b32 $r0
e99716f1 BS	338
e99716f1 BS	339	// incoming fifo command?
67af60f0 BS	340	nv_iord($r10, NV_PGRAPH_GPCX_GPCCS_INTR, 0)
67af60f0 BS	341	and $r11 $r10 NV_PGRAPH_GPCX_GPCCS_INTR_FIFO
e99716f1 BS	342	bra e #ih_no_fifo
e99716f1 BS	343	// queue incoming fifo command for later processing
e99716f1	344	mov $r13 #cmd_queue
67af60f0 BS	345	nv_iord($r14, NV_PGRAPH_GPCX_GPCCS_FIFO_CMD, 0)
	346	nv_iord($r15, NV_PGRAPH_GPCX_GPCCS_FIFO_DATA, 0)
	347	call(queue_put)
e99716f1	348	mov $r14 1
67af60f0	349	nv_iowr(NV_PGRAPH_GPCX_GPCCS_FIFO_ACK, 0, $r14)
e99716f1 BS	350
	351	// ack, and wake up main()
	352	ih_no_fifo:
67af60f0	353	nv_iowr(NV_PGRAPH_GPCX_GPCCS_INTR_ACK, 0, $r10)
e99716f1 BS	354
	355	pop $r15
	356	pop $r14
	357	pop $r13
	358	pop $r11
	359	pop $r10
	360	pop $r9
	361	pop $r8
	362	mov $flags $r8
	363	pop $r8
	364	bclr $flags $p0
	365	iret
	366
	367	// Set this GPC's bit in HUB_BAR, used to signal completion of various
	368	// activities to the HUB fuc
	369	//
	370	hub_barrier_done:
	371	mov $r15 1
	372	ld b32 $r14 D[$r0 + #gpc_id]
	373	shl b32 $r15 $r14
67af60f0	374	nv_wr32(0x409418, $r15) // 0x409418 - HUB_BAR_SET
e99716f1 BS	375	ret
	376
	377	// Disables various things, waits a bit, and re-enables them..
	378	//
	379	// Not sure how exactly this helps, perhaps "ENABLE" is not such a
	380	// good description for the bits we turn off? Anyways, without this,
	381	// funny things happen.
	382	//
	383	ctx_redswitch:
67af60f0 BS	384	mov $r15 NV_PGRAPH_GPCX_GPCCS_RED_SWITCH_POWER
	385	nv_iowr(NV_PGRAPH_GPCX_GPCCS_RED_SWITCH, 0, $r15)
	386	mov $r14 8
e99716f1	387	ctx_redswitch_delay:
67af60f0	388	sub b32 $r14 1
e99716f1	389	bra ne #ctx_redswitch_delay
67af60f0 BS	390	or $r15 NV_PGRAPH_GPCX_GPCCS_RED_SWITCH_UNK11
	391	or $r15 NV_PGRAPH_GPCX_GPCCS_RED_SWITCH_ENABLE
	392	nv_iowr(NV_PGRAPH_GPCX_GPCCS_RED_SWITCH, 0, $r15)
e99716f1 BS	393	ret
	394
	395	// Transfer GPC context data between GPU and storage area
	396	//
	397	// In: $r15 context base address
	398	// $p1 clear on save, set on load
	399	// $p2 set if opposite direction done/will be done, so:
	400	// on save it means: "a load will follow this save"
	401	// on load it means: "a save preceeded this load"
	402	//
	403	ctx_xfer:
	404	// set context base address
67af60f0	405	nv_iowr(NV_PGRAPH_GPCX_GPCCS_MEM_BASE, 0, $r15)
8fd4b7d4 BS	406	#if CHIPSET >= GM107
	407	gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_MEM_BASE, $r15)
	408	#endif
e99716f1	409	bra not $p1 #ctx_xfer_not_load
67af60f0	410	call(ctx_redswitch)
e99716f1 BS	411	ctx_xfer_not_load:
	412
	413	// strands
67af60f0 BS	414	call(strand_pre)
	415	clear b32 $r2
	416	nv_iowr(NV_PGRAPH_GPCX_GPCCS_STRAND_SELECT, 0x3f, $r2)
	417	xbit $r2 $flags $p1 // SAVE/LOAD
	418	add b32 $r2 NV_PGRAPH_GPCX_GPCCS_STRAND_CMD_SAVE
	419	nv_iowr(NV_PGRAPH_GPCX_GPCCS_STRAND_CMD, 0x3f, $r2)
e99716f1	420
8fd4b7d4 BS	421	#if CHIPSET >= GM107
	422	tpc_strand_enable();
	423	tpc_strand_seek(0);
	424	xbit $r15 $flags $p1 // SAVE/LOAD
	425	add b32 $r15 NV_PGRAPH_GPC0_TPCX_STRAND_CMD_SAVE
	426	gpc_wr32(NV_PGRAPH_GPC0_TPCX_STRAND_CMD, $r15)
	427	#endif
	428
e99716f1 BS	429	// mmio context
	430	xbit $r10 $flags $p1 // direction
	431	or $r10 2 // first
67af60f0	432	imm32($r11,0x500000)
e99716f1 BS	433	ld b32 $r12 D[$r0 + #gpc_id]
	434	shl b32 $r12 15
	435	add b32 $r11 $r12 // base = NV_PGRAPH_GPCn
	436	ld b32 $r12 D[$r0 + #gpc_mmio_list_head]
	437	ld b32 $r13 D[$r0 + #gpc_mmio_list_tail]
	438	mov $r14 0 // not multi
67af60f0	439	call(mmctx_xfer)
e99716f1 BS	440
	441	// per-TPC mmio context
	442	xbit $r10 $flags $p1 // direction
d2989b53 ML	443	#if !NV_PGRAPH_GPCX_UNK__SIZE
	444	or $r10 4 // last
	445	#endif
67af60f0	446	imm32($r11, 0x504000)
e99716f1 BS	447	ld b32 $r12 D[$r0 + #gpc_id]
	448	shl b32 $r12 15
	449	add b32 $r11 $r12 // base = NV_PGRAPH_GPCn_TPC0
	450	ld b32 $r12 D[$r0 + #tpc_mmio_list_head]
	451	ld b32 $r13 D[$r0 + #tpc_mmio_list_tail]
	452	ld b32 $r15 D[$r0 + #tpc_mask]
	453	mov $r14 0x800 // stride = 0x800
67af60f0	454	call(mmctx_xfer)
e99716f1	455
26410c67	456	#if NV_PGRAPH_GPCX_UNK__SIZE > 0
5ee86c41 BS	457	// per-UNK mmio context
	458	xbit $r10 $flags $p1 // direction
	459	or $r10 4 // last
67af60f0	460	imm32($r11, 0x503000)
5ee86c41 BS	461	ld b32 $r12 D[$r0 + #gpc_id]
	462	shl b32 $r12 15
	463	add b32 $r11 $r12 // base = NV_PGRAPH_GPCn_UNK0
	464	ld b32 $r12 D[$r0 + #unk_mmio_list_head]
	465	ld b32 $r13 D[$r0 + #unk_mmio_list_tail]
	466	ld b32 $r15 D[$r0 + #unk_mask]
	467	mov $r14 0x200 // stride = 0x200
67af60f0	468	call(mmctx_xfer)
5ee86c41 BS	469	#endif
5ee86c41 BS	470
e99716f1	471	// wait for strands to finish
67af60f0	472	call(strand_wait)
8fd4b7d4 BS	473	#if CHIPSET >= GM107
	474	tpc_strand_wait()
	475	#endif
e99716f1 BS	476
	477	// if load, or a save without a load following, do some
	478	// unknown stuff that's done after finishing a block of
	479	// strand commands
	480	bra $p1 #ctx_xfer_post
	481	bra not $p2 #ctx_xfer_done
	482	ctx_xfer_post:
67af60f0	483	call(strand_post)
8fd4b7d4 BS	484	#if CHIPSET >= GM107
	485	tpc_strand_disable()
	486	#endif
e99716f1 BS	487
	488	// mark completion in HUB's barrier
	489	ctx_xfer_done:
67af60f0	490	call(hub_barrier_done)
e99716f1 BS	491	ret
e99716f1 BS	492	#endif