[mirror_ubuntu-bionic-kernel.git] / drivers / gpu / drm / amd / amdgpu / vega10_ih.c

/*
 * Copyright 2016 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 */
#include <drm/drmP.h>
#include "amdgpu.h"
#include "amdgpu_ih.h"
#include "soc15.h"


#include "vega10/soc15ip.h"
#include "vega10/OSSSYS/osssys_4_0_offset.h"
#include "vega10/OSSSYS/osssys_4_0_sh_mask.h"

#include "soc15_common.h"
#include "vega10_ih.h"


static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev);

/**
 * vega10_ih_enable_interrupts - Enable the interrupt ring buffer
 *
 * @adev: amdgpu_device pointer
 *
 * Enable the interrupt ring buffer (VEGA10).
 */
static void vega10_ih_enable_interrupts(struct amdgpu_device *adev)
{
	u32 ih_rb_cntl = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));

	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1);
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), ih_rb_cntl);
	adev->irq.ih.enabled = true;
}

/**
 * vega10_ih_disable_interrupts - Disable the interrupt ring buffer
 *
 * @adev: amdgpu_device pointer
 *
 * Disable the interrupt ring buffer (VEGA10).
 */
static void vega10_ih_disable_interrupts(struct amdgpu_device *adev)
{
	u32 ih_rb_cntl = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));

	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), ih_rb_cntl);
	/* set rptr, wptr to 0 */
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR), 0);
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR), 0);
	adev->irq.ih.enabled = false;
	adev->irq.ih.rptr = 0;
}

/**
 * vega10_ih_irq_init - init and enable the interrupt ring
 *
 * @adev: amdgpu_device pointer
 *
 * Allocate a ring buffer for the interrupt controller,
 * enable the RLC, disable interrupts, enable the IH
 * ring buffer and enable it (VI).
 * Called at device load and reume.
 * Returns 0 for success, errors for failure.
 */
static int vega10_ih_irq_init(struct amdgpu_device *adev)
{
	int ret = 0;
	int rb_bufsz;
	u32 ih_rb_cntl, ih_doorbell_rtpr;
	u32 tmp;
	u64 wptr_off;

	/* disable irqs */
	vega10_ih_disable_interrupts(adev);

	if (adev->flags & AMD_IS_APU)
		nbio_v7_0_ih_control(adev);
	else
		nbio_v6_1_ih_control(adev);

	ih_rb_cntl = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
	/* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
	if (adev->irq.ih.use_bus_addr) {
		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE), adev->irq.ih.rb_dma_addr >> 8);
		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI), ((u64)adev->irq.ih.rb_dma_addr >> 40) & 0xff);
		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 1);
	} else {
		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE), adev->irq.ih.gpu_addr >> 8);
		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI), (adev->irq.ih.gpu_addr >> 40) & 0xff);
		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 4);
	}
	rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 1);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
	/* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register value is written to memory */
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_WRITEBACK_ENABLE, 1);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0);
	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0);

	if (adev->irq.msi_enabled)
		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, 1);

	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), ih_rb_cntl);

	/* set the writeback address whether it's enabled or not */
	if (adev->irq.ih.use_bus_addr)
		wptr_off = adev->irq.ih.rb_dma_addr + (adev->irq.ih.wptr_offs * 4);
	else
		wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4);
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO), lower_32_bits(wptr_off));
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI), upper_32_bits(wptr_off) & 0xFF);

	/* set rptr, wptr to 0 */
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR), 0);
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR), 0);

	ih_doorbell_rtpr = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR));
	if (adev->irq.ih.use_doorbell) {
		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
						 OFFSET, adev->irq.ih.doorbell_index);
		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
						 ENABLE, 1);
	} else {
		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
						 ENABLE, 0);
	}
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR), ih_doorbell_rtpr);
	if (adev->flags & AMD_IS_APU)
		nbio_v7_0_ih_doorbell_range(adev, adev->irq.ih.use_doorbell, adev->irq.ih.doorbell_index);
	else
		nbio_v6_1_ih_doorbell_range(adev, adev->irq.ih.use_doorbell, adev->irq.ih.doorbell_index);

	tmp = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL));
	tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL,
			    CLIENT18_IS_STORM_CLIENT, 1);
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL), tmp);

	tmp = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_INT_FLOOD_CNTL));
	tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1);
	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_INT_FLOOD_CNTL), tmp);

	pci_set_master(adev->pdev);

	/* enable interrupts */
	vega10_ih_enable_interrupts(adev);

	return ret;
}

/**
 * vega10_ih_irq_disable - disable interrupts
 *
 * @adev: amdgpu_device pointer
 *
 * Disable interrupts on the hw (VEGA10).
 */
static void vega10_ih_irq_disable(struct amdgpu_device *adev)
{
	vega10_ih_disable_interrupts(adev);

	/* Wait and acknowledge irq */
	mdelay(1);
}

/**
 * vega10_ih_get_wptr - get the IH ring buffer wptr
 *
 * @adev: amdgpu_device pointer
 *
 * Get the IH ring buffer wptr from either the register
 * or the writeback memory buffer (VEGA10).  Also check for
 * ring buffer overflow and deal with it.
 * Returns the value of the wptr.
 */
static u32 vega10_ih_get_wptr(struct amdgpu_device *adev)
{
	u32 wptr, tmp;

	if (adev->irq.ih.use_bus_addr)
		wptr = le32_to_cpu(adev->irq.ih.ring[adev->irq.ih.wptr_offs]);
	else
		wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]);

	if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) {
		wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);

		/* When a ring buffer overflow happen start parsing interrupt
		 * from the last not overwritten vector (wptr + 32). Hopefully
		 * this should allow us to catchup.
		 */
		tmp = (wptr + 32) & adev->irq.ih.ptr_mask;
		dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
			wptr, adev->irq.ih.rptr, tmp);
		adev->irq.ih.rptr = tmp;

		tmp = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
		tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), tmp);
	}
	return (wptr & adev->irq.ih.ptr_mask);
}

/**
 * vega10_ih_prescreen_iv - prescreen an interrupt vector
 *
 * @adev: amdgpu_device pointer
 *
 * Returns true if the interrupt vector should be further processed.
 */
static bool vega10_ih_prescreen_iv(struct amdgpu_device *adev)
{
	/* TODO: Filter known pending page faults */
	return true;
}

/**
 * vega10_ih_decode_iv - decode an interrupt vector
 *
 * @adev: amdgpu_device pointer
 *
 * Decodes the interrupt vector at the current rptr
 * position and also advance the position.
 */
static void vega10_ih_decode_iv(struct amdgpu_device *adev,
				 struct amdgpu_iv_entry *entry)
{
	/* wptr/rptr are in bytes! */
	u32 ring_index = adev->irq.ih.rptr >> 2;
	uint32_t dw[8];

	dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
	dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]);
	dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]);
	dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
	dw[4] = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);
	dw[5] = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);
	dw[6] = le32_to_cpu(adev->irq.ih.ring[ring_index + 6]);
	dw[7] = le32_to_cpu(adev->irq.ih.ring[ring_index + 7]);

	entry->client_id = dw[0] & 0xff;
	entry->src_id = (dw[0] >> 8) & 0xff;
	entry->ring_id = (dw[0] >> 16) & 0xff;
	entry->vm_id = (dw[0] >> 24) & 0xf;
	entry->vm_id_src = (dw[0] >> 31);
	entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32);
	entry->timestamp_src = dw[2] >> 31;
	entry->pas_id = dw[3] & 0xffff;
	entry->pasid_src = dw[3] >> 31;
	entry->src_data[0] = dw[4];
	entry->src_data[1] = dw[5];
	entry->src_data[2] = dw[6];
	entry->src_data[3] = dw[7];


	/* wptr/rptr are in bytes! */
	adev->irq.ih.rptr += 32;
}

/**
 * vega10_ih_set_rptr - set the IH ring buffer rptr
 *
 * @adev: amdgpu_device pointer
 *
 * Set the IH ring buffer rptr.
 */
static void vega10_ih_set_rptr(struct amdgpu_device *adev)
{
	if (adev->irq.ih.use_doorbell) {
		/* XXX check if swapping is necessary on BE */
		if (adev->irq.ih.use_bus_addr)
			adev->irq.ih.ring[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
		else
			adev->wb.wb[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
		WDOORBELL32(adev->irq.ih.doorbell_index, adev->irq.ih.rptr);
	} else {
		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR), adev->irq.ih.rptr);
	}
}

static int vega10_ih_early_init(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	vega10_ih_set_interrupt_funcs(adev);
	return 0;
}

static int vega10_ih_sw_init(void *handle)
{
	int r;
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	r = amdgpu_ih_ring_init(adev, 256 * 1024, true);
	if (r)
		return r;

	adev->irq.ih.use_doorbell = true;
	adev->irq.ih.doorbell_index = AMDGPU_DOORBELL64_IH << 1;

	r = amdgpu_irq_init(adev);

	return r;
}

static int vega10_ih_sw_fini(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	amdgpu_irq_fini(adev);
	amdgpu_ih_ring_fini(adev);

	return 0;
}

static int vega10_ih_hw_init(void *handle)
{
	int r;
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	r = vega10_ih_irq_init(adev);
	if (r)
		return r;

	return 0;
}

static int vega10_ih_hw_fini(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	vega10_ih_irq_disable(adev);

	return 0;
}

static int vega10_ih_suspend(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	return vega10_ih_hw_fini(adev);
}

static int vega10_ih_resume(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	return vega10_ih_hw_init(adev);
}

static bool vega10_ih_is_idle(void *handle)
{
	/* todo */
	return true;
}

static int vega10_ih_wait_for_idle(void *handle)
{
	/* todo */
	return -ETIMEDOUT;
}

static int vega10_ih_soft_reset(void *handle)
{
	/* todo */

	return 0;
}

static int vega10_ih_set_clockgating_state(void *handle,
					  enum amd_clockgating_state state)
{
	return 0;
}

static int vega10_ih_set_powergating_state(void *handle,
					  enum amd_powergating_state state)
{
	return 0;
}

const struct amd_ip_funcs vega10_ih_ip_funcs = {
	.name = "vega10_ih",
	.early_init = vega10_ih_early_init,
	.late_init = NULL,
	.sw_init = vega10_ih_sw_init,
	.sw_fini = vega10_ih_sw_fini,
	.hw_init = vega10_ih_hw_init,
	.hw_fini = vega10_ih_hw_fini,
	.suspend = vega10_ih_suspend,
	.resume = vega10_ih_resume,
	.is_idle = vega10_ih_is_idle,
	.wait_for_idle = vega10_ih_wait_for_idle,
	.soft_reset = vega10_ih_soft_reset,
	.set_clockgating_state = vega10_ih_set_clockgating_state,
	.set_powergating_state = vega10_ih_set_powergating_state,
};

static const struct amdgpu_ih_funcs vega10_ih_funcs = {
	.get_wptr = vega10_ih_get_wptr,
	.prescreen_iv = vega10_ih_prescreen_iv,
	.decode_iv = vega10_ih_decode_iv,
	.set_rptr = vega10_ih_set_rptr
};

static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev)
{
	if (adev->irq.ih_funcs == NULL)
		adev->irq.ih_funcs = &vega10_ih_funcs;
}

const struct amdgpu_ip_block_version vega10_ih_ip_block =
{
	.type = AMD_IP_BLOCK_TYPE_IH,
	.major = 4,
	.minor = 0,
	.rev = 0,
	.funcs = &vega10_ih_ip_funcs,
};
Commit	Line	Data
282aae55 KW	1	/*
	2	* Copyright 2016 Advanced Micro Devices, Inc.
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice shall be included in
	12	* all copies or substantial portions of the Software.
	13	*
	14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	20	* OTHER DEALINGS IN THE SOFTWARE.
	21	*
	22	*/
248a1d6f	23	#include <drm/drmP.h>
282aae55 KW	24	#include "amdgpu.h"
	25	#include "amdgpu_ih.h"
	26	#include "soc15.h"
	27
	28
	29	#include "vega10/soc15ip.h"
	30	#include "vega10/OSSSYS/osssys_4_0_offset.h"
	31	#include "vega10/OSSSYS/osssys_4_0_sh_mask.h"
	32
	33	#include "soc15_common.h"
	34	#include "vega10_ih.h"
	35
	36
	37
	38	static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev);
	39
	40	/**
	41	* vega10_ih_enable_interrupts - Enable the interrupt ring buffer
	42	*
	43	* @adev: amdgpu_device pointer
	44	*
	45	* Enable the interrupt ring buffer (VEGA10).
	46	*/
	47	static void vega10_ih_enable_interrupts(struct amdgpu_device *adev)
	48	{
	49	u32 ih_rb_cntl = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
	50
	51	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
	52	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1);
	53	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), ih_rb_cntl);
	54	adev->irq.ih.enabled = true;
	55	}
	56
	57	/**
	58	* vega10_ih_disable_interrupts - Disable the interrupt ring buffer
	59	*
	60	* @adev: amdgpu_device pointer
	61	*
	62	* Disable the interrupt ring buffer (VEGA10).
	63	*/
	64	static void vega10_ih_disable_interrupts(struct amdgpu_device *adev)
	65	{
	66	u32 ih_rb_cntl = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
	67
	68	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
	69	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
	70	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), ih_rb_cntl);
	71	/* set rptr, wptr to 0 */
	72	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR), 0);
	73	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR), 0);
	74	adev->irq.ih.enabled = false;
	75	adev->irq.ih.rptr = 0;
	76	}
	77
	78	/**
	79	* vega10_ih_irq_init - init and enable the interrupt ring
	80	*
	81	* @adev: amdgpu_device pointer
	82	*
	83	* Allocate a ring buffer for the interrupt controller,
	84	* enable the RLC, disable interrupts, enable the IH
	85	* ring buffer and enable it (VI).
	86	* Called at device load and reume.
	87	* Returns 0 for success, errors for failure.
88	*/
89	static int vega10_ih_irq_init(struct amdgpu_device *adev)
90	{
91	int ret = 0;
92	int rb_bufsz;
93	u32 ih_rb_cntl, ih_doorbell_rtpr;
94	u32 tmp;
95	u64 wptr_off;
96
97	/* disable irqs */
98	vega10_ih_disable_interrupts(adev);
99
aecbe64f CZ	100	if (adev->flags & AMD_IS_APU)
	101	nbio_v7_0_ih_control(adev);
	102	else
	103	nbio_v6_1_ih_control(adev);
282aae55 KW	104
	105	ih_rb_cntl = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
	106	/* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
	107	if (adev->irq.ih.use_bus_addr) {
	108	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE), adev->irq.ih.rb_dma_addr >> 8);
60508d3d	109	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI), ((u64)adev->irq.ih.rb_dma_addr >> 40) & 0xff);
282aae55 KW	110	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 1);
	111	} else {
	112	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE), adev->irq.ih.gpu_addr >> 8);
	113	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI), (adev->irq.ih.gpu_addr >> 40) & 0xff);
	114	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 4);
	115	}
	116	rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);
	117	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
	118	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 1);
	119	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
	120	/* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register value is written to memory */
	121	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_WRITEBACK_ENABLE, 1);
	122	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1);
	123	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0);
	124	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0);
	125
	126	if (adev->irq.msi_enabled)
	127	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, 1);
	128
	129	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), ih_rb_cntl);
	130
	131	/* set the writeback address whether it's enabled or not */
	132	if (adev->irq.ih.use_bus_addr)
	133	wptr_off = adev->irq.ih.rb_dma_addr + (adev->irq.ih.wptr_offs * 4);
	134	else
	135	wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4);
	136	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO), lower_32_bits(wptr_off));
	137	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI), upper_32_bits(wptr_off) & 0xFF);
	138
	139	/* set rptr, wptr to 0 */
	140	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR), 0);
	141	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR), 0);
	142
	143	ih_doorbell_rtpr = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR));
	144	if (adev->irq.ih.use_doorbell) {
	145	ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
	146	OFFSET, adev->irq.ih.doorbell_index);
	147	ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
	148	ENABLE, 1);
	149	} else {
	150	ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
	151	ENABLE, 0);
	152	}
	153	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR), ih_doorbell_rtpr);
aecbe64f CZ	154	if (adev->flags & AMD_IS_APU)
	155	nbio_v7_0_ih_doorbell_range(adev, adev->irq.ih.use_doorbell, adev->irq.ih.doorbell_index);
	156	else
	157	nbio_v6_1_ih_doorbell_range(adev, adev->irq.ih.use_doorbell, adev->irq.ih.doorbell_index);
282aae55 KW	158
	159	tmp = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL));
	160	tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL,
	161	CLIENT18_IS_STORM_CLIENT, 1);
	162	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL), tmp);
	163
	164	tmp = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_INT_FLOOD_CNTL));
	165	tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1);
	166	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_INT_FLOOD_CNTL), tmp);
	167
	168	pci_set_master(adev->pdev);
	169
	170	/* enable interrupts */
	171	vega10_ih_enable_interrupts(adev);
	172
	173	return ret;
	174	}
	175
	176	/**
	177	* vega10_ih_irq_disable - disable interrupts
	178	*
	179	* @adev: amdgpu_device pointer
	180	*
	181	* Disable interrupts on the hw (VEGA10).
	182	*/
	183	static void vega10_ih_irq_disable(struct amdgpu_device *adev)
	184	{
	185	vega10_ih_disable_interrupts(adev);
	186
	187	/* Wait and acknowledge irq */
	188	mdelay(1);
	189	}
	190
	191	/**
	192	* vega10_ih_get_wptr - get the IH ring buffer wptr
	193	*
	194	* @adev: amdgpu_device pointer
	195	*
	196	* Get the IH ring buffer wptr from either the register
	197	* or the writeback memory buffer (VEGA10). Also check for
	198	* ring buffer overflow and deal with it.
	199	* Returns the value of the wptr.
	200	*/
	201	static u32 vega10_ih_get_wptr(struct amdgpu_device *adev)
	202	{
	203	u32 wptr, tmp;
	204
	205	if (adev->irq.ih.use_bus_addr)
	206	wptr = le32_to_cpu(adev->irq.ih.ring[adev->irq.ih.wptr_offs]);
	207	else
	208	wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]);
	209
	210	if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) {
	211	wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);
	212
	213	/* When a ring buffer overflow happen start parsing interrupt
	214	* from the last not overwritten vector (wptr + 32). Hopefully
	215	* this should allow us to catchup.
	216	*/
	217	tmp = (wptr + 32) & adev->irq.ih.ptr_mask;
	218	dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
	219	wptr, adev->irq.ih.rptr, tmp);
	220	adev->irq.ih.rptr = tmp;
	221
222	tmp = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
223	tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
224	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), tmp);
225	}
226	return (wptr & adev->irq.ih.ptr_mask);
227	}
228
00ecd8a2 FK	229	/**
	230	* vega10_ih_prescreen_iv - prescreen an interrupt vector
	231	*
	232	* @adev: amdgpu_device pointer
	233	*
	234	* Returns true if the interrupt vector should be further processed.
	235	*/
	236	static bool vega10_ih_prescreen_iv(struct amdgpu_device *adev)
	237	{
	238	/* TODO: Filter known pending page faults */
	239	return true;
	240	}
	241
282aae55 KW	242	/**
	243	* vega10_ih_decode_iv - decode an interrupt vector
	244	*
	245	* @adev: amdgpu_device pointer
	246	*
	247	* Decodes the interrupt vector at the current rptr
	248	* position and also advance the position.
	249	*/
	250	static void vega10_ih_decode_iv(struct amdgpu_device *adev,
	251	struct amdgpu_iv_entry *entry)
	252	{
	253	/* wptr/rptr are in bytes! */
	254	u32 ring_index = adev->irq.ih.rptr >> 2;
	255	uint32_t dw[8];
	256
	257	dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
	258	dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]);
	259	dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]);
	260	dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
	261	dw[4] = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);
	262	dw[5] = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);
	263	dw[6] = le32_to_cpu(adev->irq.ih.ring[ring_index + 6]);
	264	dw[7] = le32_to_cpu(adev->irq.ih.ring[ring_index + 7]);
	265
	266	entry->client_id = dw[0] & 0xff;
	267	entry->src_id = (dw[0] >> 8) & 0xff;
	268	entry->ring_id = (dw[0] >> 16) & 0xff;
	269	entry->vm_id = (dw[0] >> 24) & 0xf;
	270	entry->vm_id_src = (dw[0] >> 31);
	271	entry->timestamp = dw[1] \| ((u64)(dw[2] & 0xffff) << 32);
	272	entry->timestamp_src = dw[2] >> 31;
	273	entry->pas_id = dw[3] & 0xffff;
	274	entry->pasid_src = dw[3] >> 31;
	275	entry->src_data[0] = dw[4];
	276	entry->src_data[1] = dw[5];
	277	entry->src_data[2] = dw[6];
	278	entry->src_data[3] = dw[7];
	279
	280
	281	/* wptr/rptr are in bytes! */
	282	adev->irq.ih.rptr += 32;
	283	}
	284
	285	/**
	286	* vega10_ih_set_rptr - set the IH ring buffer rptr
	287	*
	288	* @adev: amdgpu_device pointer
	289	*
	290	* Set the IH ring buffer rptr.
	291	*/
	292	static void vega10_ih_set_rptr(struct amdgpu_device *adev)
	293	{
	294	if (adev->irq.ih.use_doorbell) {
	295	/* XXX check if swapping is necessary on BE */
	296	if (adev->irq.ih.use_bus_addr)
	297	adev->irq.ih.ring[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
	298	else
	299	adev->wb.wb[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
	300	WDOORBELL32(adev->irq.ih.doorbell_index, adev->irq.ih.rptr);
	301	} else {
	302	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR), adev->irq.ih.rptr);
	303	}
	304	}
	305
306	static int vega10_ih_early_init(void *handle)
307	{
308	struct amdgpu_device adev = (struct amdgpu_device )handle;
309
310	vega10_ih_set_interrupt_funcs(adev);
311	return 0;
312	}
313
314	static int vega10_ih_sw_init(void *handle)
315	{
316	int r;
317	struct amdgpu_device adev = (struct amdgpu_device )handle;
318
319	r = amdgpu_ih_ring_init(adev, 256 * 1024, true);
320	if (r)
321	return r;
322
323	adev->irq.ih.use_doorbell = true;
324	adev->irq.ih.doorbell_index = AMDGPU_DOORBELL64_IH << 1;
325
326	r = amdgpu_irq_init(adev);
327
328	return r;
329	}
330
331	static int vega10_ih_sw_fini(void *handle)
332	{
333	struct amdgpu_device adev = (struct amdgpu_device )handle;
334
335	amdgpu_irq_fini(adev);
336	amdgpu_ih_ring_fini(adev);
337
338	return 0;
339	}
340
341	static int vega10_ih_hw_init(void *handle)
342	{
343	int r;
344	struct amdgpu_device adev = (struct amdgpu_device )handle;
345
346	r = vega10_ih_irq_init(adev);
347	if (r)
348	return r;
349
350	return 0;
351	}
352
353	static int vega10_ih_hw_fini(void *handle)
354	{
355	struct amdgpu_device adev = (struct amdgpu_device )handle;
356
357	vega10_ih_irq_disable(adev);
358
359	return 0;
360	}
361
362	static int vega10_ih_suspend(void *handle)
363	{
364	struct amdgpu_device adev = (struct amdgpu_device )handle;
365
366	return vega10_ih_hw_fini(adev);
367	}
368
369	static int vega10_ih_resume(void *handle)
370	{
371	struct amdgpu_device adev = (struct amdgpu_device )handle;
372
373	return vega10_ih_hw_init(adev);
374	}
375
376	static bool vega10_ih_is_idle(void *handle)
377	{
378	/* todo */
379	return true;
380	}
381
382	static int vega10_ih_wait_for_idle(void *handle)
383	{
384	/* todo */
385	return -ETIMEDOUT;
386	}
387
388	static int vega10_ih_soft_reset(void *handle)
389	{
390	/* todo */
391
392	return 0;
393	}
394
395	static int vega10_ih_set_clockgating_state(void *handle,
396	enum amd_clockgating_state state)
397	{
398	return 0;
399	}
400
401	static int vega10_ih_set_powergating_state(void *handle,
402	enum amd_powergating_state state)
403	{
404	return 0;
405	}
406
407	const struct amd_ip_funcs vega10_ih_ip_funcs = {
408	.name = "vega10_ih",
409	.early_init = vega10_ih_early_init,
410	.late_init = NULL,
411	.sw_init = vega10_ih_sw_init,
412	.sw_fini = vega10_ih_sw_fini,
413	.hw_init = vega10_ih_hw_init,
414	.hw_fini = vega10_ih_hw_fini,
415	.suspend = vega10_ih_suspend,
416	.resume = vega10_ih_resume,
417	.is_idle = vega10_ih_is_idle,
418	.wait_for_idle = vega10_ih_wait_for_idle,
419	.soft_reset = vega10_ih_soft_reset,
420	.set_clockgating_state = vega10_ih_set_clockgating_state,
421	.set_powergating_state = vega10_ih_set_powergating_state,
422	};
423
424	static const struct amdgpu_ih_funcs vega10_ih_funcs = {
425	.get_wptr = vega10_ih_get_wptr,
00ecd8a2	426	.prescreen_iv = vega10_ih_prescreen_iv,
282aae55 KW	427	.decode_iv = vega10_ih_decode_iv,
	428	.set_rptr = vega10_ih_set_rptr
	429	};
	430
	431	static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev)
	432	{
	433	if (adev->irq.ih_funcs == NULL)
	434	adev->irq.ih_funcs = &vega10_ih_funcs;
	435	}
	436
	437	const struct amdgpu_ip_block_version vega10_ih_ip_block =
	438	{
	439	.type = AMD_IP_BLOCK_TYPE_IH,
	440	.major = 4,
	441	.minor = 0,
	442	.rev = 0,
	443	.funcs = &vega10_ih_ip_funcs,
	444	};