[mirror_ubuntu-disco-kernel.git] / drivers / gpu / drm / amd / amdgpu / gfxhub_v1_0.c

/*
 * Copyright 2016 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 */
#include "amdgpu.h"
#include "gfxhub_v1_0.h"

#include "gc/gc_9_0_offset.h"
#include "gc/gc_9_0_sh_mask.h"
#include "gc/gc_9_0_default.h"
#include "vega10_enum.h"

#include "soc15_common.h"

u64 gfxhub_v1_0_get_mc_fb_offset(struct amdgpu_device *adev)
{
	return (u64)RREG32_SOC15(GC, 0, mmMC_VM_FB_OFFSET) << 24;
}

static void gfxhub_v1_0_init_gart_pt_regs(struct amdgpu_device *adev)
{
	uint64_t value;

	BUG_ON(adev->gart.table_addr & (~0x0000FFFFFFFFF000ULL));
	value = adev->gart.table_addr - adev->gmc.vram_start
		+ adev->vm_manager.vram_base_offset;
	value &= 0x0000FFFFFFFFF000ULL;
	value |= 0x1; /*valid bit*/

	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
		     lower_32_bits(value));

	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
		     upper_32_bits(value));
}

static void gfxhub_v1_0_init_gart_aperture_regs(struct amdgpu_device *adev)
{
	gfxhub_v1_0_init_gart_pt_regs(adev);

	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
		     (u32)(adev->gmc.gart_start >> 12));
	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
		     (u32)(adev->gmc.gart_start >> 44));

	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
		     (u32)(adev->gmc.gart_end >> 12));
	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
		     (u32)(adev->gmc.gart_end >> 44));
}

static void gfxhub_v1_0_init_system_aperture_regs(struct amdgpu_device *adev)
{
	uint64_t value;

	/* Disable AGP. */
	WREG32_SOC15(GC, 0, mmMC_VM_AGP_BASE, 0);
	WREG32_SOC15(GC, 0, mmMC_VM_AGP_TOP, 0);
	WREG32_SOC15(GC, 0, mmMC_VM_AGP_BOT, 0xFFFFFFFF);

	/* Program the system aperture low logical page number. */
	WREG32_SOC15(GC, 0, mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
		     adev->gmc.vram_start >> 18);
	WREG32_SOC15(GC, 0, mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		     adev->gmc.vram_end >> 18);

	/* Set default page address. */
	value = adev->vram_scratch.gpu_addr - adev->gmc.vram_start
		+ adev->vm_manager.vram_base_offset;
	WREG32_SOC15(GC, 0, mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
		     (u32)(value >> 12));
	WREG32_SOC15(GC, 0, mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
		     (u32)(value >> 44));

	/* Program "protection fault". */
	WREG32_SOC15(GC, 0, mmVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
		     (u32)(adev->dummy_page_addr >> 12));
	WREG32_SOC15(GC, 0, mmVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
		     (u32)((u64)adev->dummy_page_addr >> 44));

	WREG32_FIELD15(GC, 0, VM_L2_PROTECTION_FAULT_CNTL2,
		       ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
}

static void gfxhub_v1_0_init_tlb_regs(struct amdgpu_device *adev)
{
	uint32_t tmp;

	/* Setup TLB control */
	tmp = RREG32_SOC15(GC, 0, mmMC_VM_MX_L1_TLB_CNTL);

	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
			    ENABLE_ADVANCED_DRIVER_MODEL, 1);
	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
			    SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
			    MTYPE, MTYPE_UC);/* XXX for emulation. */
	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1);

	WREG32_SOC15(GC, 0, mmMC_VM_MX_L1_TLB_CNTL, tmp);
}

static void gfxhub_v1_0_init_cache_regs(struct amdgpu_device *adev)
{
	uint32_t tmp;

	/* Setup L2 cache */
	tmp = RREG32_SOC15(GC, 0, mmVM_L2_CNTL);
	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);
	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1);
	/* XXX for emulation, Refer to closed source code.*/
	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, L2_PDE0_CACHE_TAG_GENERATION_MODE,
			    0);
	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 1);
	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0);
	WREG32_SOC15(GC, 0, mmVM_L2_CNTL, tmp);

	tmp = RREG32_SOC15(GC, 0, mmVM_L2_CNTL2);
	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
	WREG32_SOC15(GC, 0, mmVM_L2_CNTL2, tmp);

	tmp = mmVM_L2_CNTL3_DEFAULT;
	if (adev->gmc.translate_further) {
		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 12);
		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3,
				    L2_CACHE_BIGK_FRAGMENT_SIZE, 9);
	} else {
		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9);
		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3,
				    L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
	}
	WREG32_SOC15(GC, 0, mmVM_L2_CNTL3, tmp);

	tmp = mmVM_L2_CNTL4_DEFAULT;
	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
	WREG32_SOC15(GC, 0, mmVM_L2_CNTL4, tmp);
}

static void gfxhub_v1_0_enable_system_domain(struct amdgpu_device *adev)
{
	uint32_t tmp;

	tmp = RREG32_SOC15(GC, 0, mmVM_CONTEXT0_CNTL);
	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_CNTL, tmp);
}

static void gfxhub_v1_0_disable_identity_aperture(struct amdgpu_device *adev)
{
	WREG32_SOC15(GC, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
		     0XFFFFFFFF);
	WREG32_SOC15(GC, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
		     0x0000000F);

	WREG32_SOC15(GC, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32,
		     0);
	WREG32_SOC15(GC, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32,
		     0);

	WREG32_SOC15(GC, 0, mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 0);
	WREG32_SOC15(GC, 0, mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 0);

}

static void gfxhub_v1_0_setup_vmid_config(struct amdgpu_device *adev)
{
	unsigned num_level, block_size;
	uint32_t tmp;
	int i;

	num_level = adev->vm_manager.num_level;
	block_size = adev->vm_manager.block_size;
	if (adev->gmc.translate_further)
		num_level -= 1;
	else
		block_size -= 9;

	for (i = 0; i <= 14; i++) {
		tmp = RREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_CNTL, i);
		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH,
				    num_level);
		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
				    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
				    DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT,
				    1);
		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
				    PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
				    VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
				    READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
				    WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
				    EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
				    PAGE_TABLE_BLOCK_SIZE,
				    block_size);
		/* Send no-retry XNACK on fault to suppress VM fault storm. */
		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
				    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
		WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_CNTL, i, tmp);
		WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, i*2, 0);
		WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, i*2, 0);
		WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32,  i*2,
			lower_32_bits(adev->vm_manager.max_pfn - 1));
		WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, i*2,
			upper_32_bits(adev->vm_manager.max_pfn - 1));
	}
}

static void gfxhub_v1_0_program_invalidation(struct amdgpu_device *adev)
{
	unsigned i;

	for (i = 0 ; i < 18; ++i) {
		WREG32_SOC15_OFFSET(GC, 0, mmVM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
				    2 * i, 0xffffffff);
		WREG32_SOC15_OFFSET(GC, 0, mmVM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
				    2 * i, 0x1f);
	}
}

int gfxhub_v1_0_gart_enable(struct amdgpu_device *adev)
{
	if (amdgpu_sriov_vf(adev)) {
		/*
		 * MC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase they are
		 * VF copy registers so vbios post doesn't program them, for
		 * SRIOV driver need to program them
		 */
		WREG32_SOC15(GC, 0, mmMC_VM_FB_LOCATION_BASE,
			     adev->gmc.vram_start >> 24);
		WREG32_SOC15(GC, 0, mmMC_VM_FB_LOCATION_TOP,
			     adev->gmc.vram_end >> 24);
	}

	/* GART Enable. */
	gfxhub_v1_0_init_gart_aperture_regs(adev);
	gfxhub_v1_0_init_system_aperture_regs(adev);
	gfxhub_v1_0_init_tlb_regs(adev);
	gfxhub_v1_0_init_cache_regs(adev);

	gfxhub_v1_0_enable_system_domain(adev);
	gfxhub_v1_0_disable_identity_aperture(adev);
	gfxhub_v1_0_setup_vmid_config(adev);
	gfxhub_v1_0_program_invalidation(adev);

	return 0;
}

void gfxhub_v1_0_gart_disable(struct amdgpu_device *adev)
{
	u32 tmp;
	u32 i;

	/* Disable all tables */
	for (i = 0; i < 16; i++)
		WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT0_CNTL, i, 0);

	/* Setup TLB control */
	tmp = RREG32_SOC15(GC, 0, mmMC_VM_MX_L1_TLB_CNTL);
	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
	tmp = REG_SET_FIELD(tmp,
				MC_VM_MX_L1_TLB_CNTL,
				ENABLE_ADVANCED_DRIVER_MODEL,
				0);
	WREG32_SOC15(GC, 0, mmMC_VM_MX_L1_TLB_CNTL, tmp);

	/* Setup L2 cache */
	WREG32_FIELD15(GC, 0, VM_L2_CNTL, ENABLE_L2_CACHE, 0);
	WREG32_SOC15(GC, 0, mmVM_L2_CNTL3, 0);
}

/**
 * gfxhub_v1_0_set_fault_enable_default - update GART/VM fault handling
 *
 * @adev: amdgpu_device pointer
 * @value: true redirects VM faults to the default page
 */
void gfxhub_v1_0_set_fault_enable_default(struct amdgpu_device *adev,
					  bool value)
{
	u32 tmp;
	tmp = RREG32_SOC15(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL);
	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
			RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
			PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
			PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value);
	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
			PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value);
	tmp = REG_SET_FIELD(tmp,
			VM_L2_PROTECTION_FAULT_CNTL,
			TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
			value);
	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
			NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value);
	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
			DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
			VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
			READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
			WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
			EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
	if (!value) {
		tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
				CRASH_ON_NO_RETRY_FAULT, 1);
		tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
				CRASH_ON_RETRY_FAULT, 1);
    }
	WREG32_SOC15(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL, tmp);
}

void gfxhub_v1_0_init(struct amdgpu_device *adev)
{
	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB];

	hub->ctx0_ptb_addr_lo32 =
		SOC15_REG_OFFSET(GC, 0,
				 mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32);
	hub->ctx0_ptb_addr_hi32 =
		SOC15_REG_OFFSET(GC, 0,
				 mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32);
	hub->vm_inv_eng0_req =
		SOC15_REG_OFFSET(GC, 0, mmVM_INVALIDATE_ENG0_REQ);
	hub->vm_inv_eng0_ack =
		SOC15_REG_OFFSET(GC, 0, mmVM_INVALIDATE_ENG0_ACK);
	hub->vm_context0_cntl =
		SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT0_CNTL);
	hub->vm_l2_pro_fault_status =
		SOC15_REG_OFFSET(GC, 0, mmVM_L2_PROTECTION_FAULT_STATUS);
	hub->vm_l2_pro_fault_cntl =
		SOC15_REG_OFFSET(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL);
}
Commit	Line	Data
e60f8db5 AX	1	/*
	2	* Copyright 2016 Advanced Micro Devices, Inc.
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice shall be included in
	12	* all copies or substantial portions of the Software.
	13	*
	14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	20	* OTHER DEALINGS IN THE SOFTWARE.
	21	*
	22	*/
	23	#include "amdgpu.h"
	24	#include "gfxhub_v1_0.h"
	25
cde5c34f FX	26	#include "gc/gc_9_0_offset.h"
	27	#include "gc/gc_9_0_sh_mask.h"
	28	#include "gc/gc_9_0_default.h"
fb960bd2	29	#include "vega10_enum.h"
e60f8db5 AX	30
	31	#include "soc15_common.h"
	32
2d8e898e CZ	33	u64 gfxhub_v1_0_get_mc_fb_offset(struct amdgpu_device *adev)
2d8e898e CZ	34	{
f7047402	35	return (u64)RREG32_SOC15(GC, 0, mmMC_VM_FB_OFFSET) << 24;
2d8e898e CZ	36	}
2d8e898e CZ	37
a51dca4f HR	38	static void gfxhub_v1_0_init_gart_pt_regs(struct amdgpu_device *adev)
	39	{
	40	uint64_t value;
	41
	42	BUG_ON(adev->gart.table_addr & (~0x0000FFFFFFFFF000ULL));
770d13b1	43	value = adev->gart.table_addr - adev->gmc.vram_start
a51dca4f HR	44	+ adev->vm_manager.vram_base_offset;
	45	value &= 0x0000FFFFFFFFF000ULL;
	46	value \|= 0x1; /valid bit/
	47
89f99ceb HR	48	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
89f99ceb HR	49	lower_32_bits(value));
a51dca4f	50
89f99ceb HR	51	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
89f99ceb HR	52	upper_32_bits(value));
a51dca4f HR	53	}
a51dca4f HR	54
9bbad6fd HR	55	static void gfxhub_v1_0_init_gart_aperture_regs(struct amdgpu_device *adev)
	56	{
	57	gfxhub_v1_0_init_gart_pt_regs(adev);
	58
89f99ceb	59	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
770d13b1	60	(u32)(adev->gmc.gart_start >> 12));
89f99ceb	61	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
770d13b1	62	(u32)(adev->gmc.gart_start >> 44));
89f99ceb HR	63
89f99ceb HR	64	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
770d13b1	65	(u32)(adev->gmc.gart_end >> 12));
89f99ceb	66	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
770d13b1	67	(u32)(adev->gmc.gart_end >> 44));
9bbad6fd HR	68	}
9bbad6fd HR	69
fc4b884b	70	static void gfxhub_v1_0_init_system_aperture_regs(struct amdgpu_device *adev)
e60f8db5	71	{
fc4b884b	72	uint64_t value;
e60f8db5	73
fc4b884b	74	/* Disable AGP. */
89f99ceb HR	75	WREG32_SOC15(GC, 0, mmMC_VM_AGP_BASE, 0);
	76	WREG32_SOC15(GC, 0, mmMC_VM_AGP_TOP, 0);
	77	WREG32_SOC15(GC, 0, mmMC_VM_AGP_BOT, 0xFFFFFFFF);
a51dca4f	78
fc4b884b	79	/* Program the system aperture low logical page number. */
89f99ceb	80	WREG32_SOC15(GC, 0, mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
770d13b1	81	adev->gmc.vram_start >> 18);
89f99ceb	82	WREG32_SOC15(GC, 0, mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
770d13b1	83	adev->gmc.vram_end >> 18);
e60f8db5	84
fc4b884b	85	/* Set default page address. */
770d13b1	86	value = adev->vram_scratch.gpu_addr - adev->gmc.vram_start
e60f8db5	87	+ adev->vm_manager.vram_base_offset;
89f99ceb HR	88	WREG32_SOC15(GC, 0, mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
	89	(u32)(value >> 12));
	90	WREG32_SOC15(GC, 0, mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
	91	(u32)(value >> 44));
fc4b884b HR	92
fc4b884b HR	93	/* Program "protection fault". */
89f99ceb	94	WREG32_SOC15(GC, 0, mmVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
92e71b06	95	(u32)(adev->dummy_page_addr >> 12));
89f99ceb	96	WREG32_SOC15(GC, 0, mmVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
92e71b06	97	(u32)((u64)adev->dummy_page_addr >> 44));
89f99ceb	98
805cb75c TSD	99	WREG32_FIELD15(GC, 0, VM_L2_PROTECTION_FAULT_CNTL2,
805cb75c TSD	100	ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
fc4b884b HR	101	}
fc4b884b HR	102
34269839 HR	103	static void gfxhub_v1_0_init_tlb_regs(struct amdgpu_device *adev)
	104	{
	105	uint32_t tmp;
	106
	107	/* Setup TLB control */
89f99ceb	108	tmp = RREG32_SOC15(GC, 0, mmMC_VM_MX_L1_TLB_CNTL);
34269839 HR	109
	110	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
	111	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
	112	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
	113	ENABLE_ADVANCED_DRIVER_MODEL, 1);
	114	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
	115	SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
	116	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
	117	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
	118	MTYPE, MTYPE_UC);/* XXX for emulation. */
	119	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1);
	120
89f99ceb	121	WREG32_SOC15(GC, 0, mmMC_VM_MX_L1_TLB_CNTL, tmp);
34269839 HR	122	}
34269839 HR	123
41f6f311 HR	124	static void gfxhub_v1_0_init_cache_regs(struct amdgpu_device *adev)
41f6f311 HR	125	{
a3ce3645	126	uint32_t tmp;
41f6f311 HR	127
41f6f311 HR	128	/* Setup L2 cache */
89f99ceb	129	tmp = RREG32_SOC15(GC, 0, mmVM_L2_CNTL);
41f6f311	130	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);
6be7adb3	131	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1);
41f6f311 HR	132	/* XXX for emulation, Refer to closed source code.*/
	133	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, L2_PDE0_CACHE_TAG_GENERATION_MODE,
	134	0);
	135	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 1);
	136	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
	137	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0);
89f99ceb	138	WREG32_SOC15(GC, 0, mmVM_L2_CNTL, tmp);
41f6f311	139
89f99ceb	140	tmp = RREG32_SOC15(GC, 0, mmVM_L2_CNTL2);
41f6f311 HR	141	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
41f6f311 HR	142	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
89f99ceb	143	WREG32_SOC15(GC, 0, mmVM_L2_CNTL2, tmp);
41f6f311 HR	144
41f6f311 HR	145	tmp = mmVM_L2_CNTL3_DEFAULT;
770d13b1	146	if (adev->gmc.translate_further) {
6a42fd6f CK	147	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 12);
	148	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3,
	149	L2_CACHE_BIGK_FRAGMENT_SIZE, 9);
	150	} else {
	151	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9);
	152	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3,
	153	L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
	154	}
89f99ceb	155	WREG32_SOC15(GC, 0, mmVM_L2_CNTL3, tmp);
41f6f311 HR	156
	157	tmp = mmVM_L2_CNTL4_DEFAULT;
	158	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
	159	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
89f99ceb	160	WREG32_SOC15(GC, 0, mmVM_L2_CNTL4, tmp);
41f6f311 HR	161	}
41f6f311 HR	162
02c4704b HR	163	static void gfxhub_v1_0_enable_system_domain(struct amdgpu_device *adev)
	164	{
	165	uint32_t tmp;
	166
89f99ceb	167	tmp = RREG32_SOC15(GC, 0, mmVM_CONTEXT0_CNTL);
02c4704b HR	168	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
02c4704b HR	169	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
89f99ceb	170	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_CNTL, tmp);
02c4704b HR	171	}
02c4704b HR	172
d5c87390 HR	173	static void gfxhub_v1_0_disable_identity_aperture(struct amdgpu_device *adev)
d5c87390 HR	174	{
89f99ceb HR	175	WREG32_SOC15(GC, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
	176	0XFFFFFFFF);
	177	WREG32_SOC15(GC, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
	178	0x0000000F);
	179
	180	WREG32_SOC15(GC, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32,
	181	0);
	182	WREG32_SOC15(GC, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32,
	183	0);
	184
	185	WREG32_SOC15(GC, 0, mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 0);
	186	WREG32_SOC15(GC, 0, mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 0);
d5c87390 HR	187
	188	}
	189
3dff4cc4	190	static void gfxhub_v1_0_setup_vmid_config(struct amdgpu_device *adev)
fc4b884b	191	{
6a42fd6f	192	unsigned num_level, block_size;
3dff4cc4	193	uint32_t tmp;
6a42fd6f CK	194	int i;
	195
	196	num_level = adev->vm_manager.num_level;
	197	block_size = adev->vm_manager.block_size;
770d13b1	198	if (adev->gmc.translate_further)
6a42fd6f CK	199	num_level -= 1;
	200	else
	201	block_size -= 9;
e60f8db5 AX	202
e60f8db5 AX	203	for (i = 0; i <= 14; i++) {
f7047402	204	tmp = RREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_CNTL, i);
e60f8db5	205	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
4fb1cf3a	206	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH,
6a42fd6f	207	num_level);
e60f8db5	208	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
6a42fd6f	209	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
e60f8db5	210	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
6a42fd6f CK	211	DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT,
6a42fd6f CK	212	1);
e60f8db5	213	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
6a42fd6f	214	PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
e60f8db5	215	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
6a42fd6f	216	VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
e60f8db5	217	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
6a42fd6f	218	READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
e60f8db5	219	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
6a42fd6f	220	WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
e60f8db5	221	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
6a42fd6f	222	EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
e60f8db5	223	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
6a42fd6f CK	224	PAGE_TABLE_BLOCK_SIZE,
6a42fd6f CK	225	block_size);
9f57f7b4 JC	226	/* Send no-retry XNACK on fault to suppress VM fault storm. */
	227	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
	228	RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
f7047402 TSD	229	WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_CNTL, i, tmp);
	230	WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, i*2, 0);
	231	WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, i*2, 0);
	232	WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, i*2,
22770e5a	233	lower_32_bits(adev->vm_manager.max_pfn - 1));
f7047402	234	WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, i*2,
22770e5a	235	upper_32_bits(adev->vm_manager.max_pfn - 1));
e60f8db5	236	}
3dff4cc4 HR	237	}
3dff4cc4 HR	238
1e4eccda HR	239	static void gfxhub_v1_0_program_invalidation(struct amdgpu_device *adev)
	240	{
	241	unsigned i;
	242
	243	for (i = 0 ; i < 18; ++i) {
f7047402 TSD	244	WREG32_SOC15_OFFSET(GC, 0, mmVM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
	245	2 * i, 0xffffffff);
	246	WREG32_SOC15_OFFSET(GC, 0, mmVM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
	247	2 * i, 0x1f);
1e4eccda HR	248	}
	249	}
	250
3dff4cc4 HR	251	int gfxhub_v1_0_gart_enable(struct amdgpu_device *adev)
	252	{
	253	if (amdgpu_sriov_vf(adev)) {
	254	/*
	255	* MC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase they are
	256	* VF copy registers so vbios post doesn't program them, for
	257	* SRIOV driver need to program them
	258	*/
89f99ceb	259	WREG32_SOC15(GC, 0, mmMC_VM_FB_LOCATION_BASE,
770d13b1	260	adev->gmc.vram_start >> 24);
89f99ceb	261	WREG32_SOC15(GC, 0, mmMC_VM_FB_LOCATION_TOP,
770d13b1	262	adev->gmc.vram_end >> 24);
3dff4cc4 HR	263	}
	264
	265	/* GART Enable. */
	266	gfxhub_v1_0_init_gart_aperture_regs(adev);
	267	gfxhub_v1_0_init_system_aperture_regs(adev);
	268	gfxhub_v1_0_init_tlb_regs(adev);
	269	gfxhub_v1_0_init_cache_regs(adev);
e60f8db5	270
3dff4cc4 HR	271	gfxhub_v1_0_enable_system_domain(adev);
	272	gfxhub_v1_0_disable_identity_aperture(adev);
	273	gfxhub_v1_0_setup_vmid_config(adev);
1e4eccda	274	gfxhub_v1_0_program_invalidation(adev);
e60f8db5 AX	275
	276	return 0;
	277	}
	278
	279	void gfxhub_v1_0_gart_disable(struct amdgpu_device *adev)
	280	{
	281	u32 tmp;
	282	u32 i;
	283
	284	/* Disable all tables */
	285	for (i = 0; i < 16; i++)
f7047402	286	WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT0_CNTL, i, 0);
e60f8db5 AX	287
e60f8db5 AX	288	/* Setup TLB control */
89f99ceb	289	tmp = RREG32_SOC15(GC, 0, mmMC_VM_MX_L1_TLB_CNTL);
e60f8db5 AX	290	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
	291	tmp = REG_SET_FIELD(tmp,
	292	MC_VM_MX_L1_TLB_CNTL,
	293	ENABLE_ADVANCED_DRIVER_MODEL,
	294	0);
89f99ceb	295	WREG32_SOC15(GC, 0, mmMC_VM_MX_L1_TLB_CNTL, tmp);
e60f8db5 AX	296
e60f8db5 AX	297	/* Setup L2 cache */
805cb75c	298	WREG32_FIELD15(GC, 0, VM_L2_CNTL, ENABLE_L2_CACHE, 0);
89f99ceb	299	WREG32_SOC15(GC, 0, mmVM_L2_CNTL3, 0);
e60f8db5 AX	300	}
	301
	302	/**
	303	* gfxhub_v1_0_set_fault_enable_default - update GART/VM fault handling
	304	*
	305	* @adev: amdgpu_device pointer
	306	* @value: true redirects VM faults to the default page
	307	*/
	308	void gfxhub_v1_0_set_fault_enable_default(struct amdgpu_device *adev,
	309	bool value)
	310	{
	311	u32 tmp;
89f99ceb	312	tmp = RREG32_SOC15(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL);
e60f8db5 AX	313	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
	314	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
	315	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
	316	PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
	317	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
	318	PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value);
	319	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
	320	PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value);
	321	tmp = REG_SET_FIELD(tmp,
	322	VM_L2_PROTECTION_FAULT_CNTL,
	323	TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
	324	value);
	325	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
	326	NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value);
	327	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
	328	DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
	329	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
	330	VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
	331	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
	332	READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
	333	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
	334	WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
	335	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
	336	EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
4bd9a67e ML	337	if (!value) {
	338	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
	339	CRASH_ON_NO_RETRY_FAULT, 1);
	340	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
	341	CRASH_ON_RETRY_FAULT, 1);
	342	}
89f99ceb	343	WREG32_SOC15(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL, tmp);
e60f8db5 AX	344	}
e60f8db5 AX	345
0c8c0847	346	void gfxhub_v1_0_init(struct amdgpu_device *adev)
e60f8db5	347	{
e60f8db5 AX	348	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB];
	349
	350	hub->ctx0_ptb_addr_lo32 =
	351	SOC15_REG_OFFSET(GC, 0,
	352	mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32);
	353	hub->ctx0_ptb_addr_hi32 =
	354	SOC15_REG_OFFSET(GC, 0,
	355	mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32);
	356	hub->vm_inv_eng0_req =
	357	SOC15_REG_OFFSET(GC, 0, mmVM_INVALIDATE_ENG0_REQ);
	358	hub->vm_inv_eng0_ack =
	359	SOC15_REG_OFFSET(GC, 0, mmVM_INVALIDATE_ENG0_ACK);
	360	hub->vm_context0_cntl =
	361	SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT0_CNTL);
	362	hub->vm_l2_pro_fault_status =
	363	SOC15_REG_OFFSET(GC, 0, mmVM_L2_PROTECTION_FAULT_STATUS);
	364	hub->vm_l2_pro_fault_cntl =
	365	SOC15_REG_OFFSET(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL);
0c8c0847	366	}