[mirror_ubuntu-kernels.git] / drivers / gpu / drm / amd / amdgpu / cik_ih.c

/*
 * Copyright 2012 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 */

#include <linux/pci.h>

#include "amdgpu.h"
#include "amdgpu_ih.h"
#include "cikd.h"

#include "bif/bif_4_1_d.h"
#include "bif/bif_4_1_sh_mask.h"

#include "oss/oss_2_0_d.h"
#include "oss/oss_2_0_sh_mask.h"

/*
 * Interrupts
 * Starting with r6xx, interrupts are handled via a ring buffer.
 * Ring buffers are areas of GPU accessible memory that the GPU
 * writes interrupt vectors into and the host reads vectors out of.
 * There is a rptr (read pointer) that determines where the
 * host is currently reading, and a wptr (write pointer)
 * which determines where the GPU has written.  When the
 * pointers are equal, the ring is idle.  When the GPU
 * writes vectors to the ring buffer, it increments the
 * wptr.  When there is an interrupt, the host then starts
 * fetching commands and processing them until the pointers are
 * equal again at which point it updates the rptr.
 */

static void cik_ih_set_interrupt_funcs(struct amdgpu_device *adev);

/**
 * cik_ih_enable_interrupts - Enable the interrupt ring buffer
 *
 * @adev: amdgpu_device pointer
 *
 * Enable the interrupt ring buffer (CIK).
 */
static void cik_ih_enable_interrupts(struct amdgpu_device *adev)
{
	u32 ih_cntl = RREG32(mmIH_CNTL);
	u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL);

	ih_cntl |= IH_CNTL__ENABLE_INTR_MASK;
	ih_rb_cntl |= IH_RB_CNTL__RB_ENABLE_MASK;
	WREG32(mmIH_CNTL, ih_cntl);
	WREG32(mmIH_RB_CNTL, ih_rb_cntl);
	adev->irq.ih.enabled = true;
}

/**
 * cik_ih_disable_interrupts - Disable the interrupt ring buffer
 *
 * @adev: amdgpu_device pointer
 *
 * Disable the interrupt ring buffer (CIK).
 */
static void cik_ih_disable_interrupts(struct amdgpu_device *adev)
{
	u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL);
	u32 ih_cntl = RREG32(mmIH_CNTL);

	ih_rb_cntl &= ~IH_RB_CNTL__RB_ENABLE_MASK;
	ih_cntl &= ~IH_CNTL__ENABLE_INTR_MASK;
	WREG32(mmIH_RB_CNTL, ih_rb_cntl);
	WREG32(mmIH_CNTL, ih_cntl);
	/* set rptr, wptr to 0 */
	WREG32(mmIH_RB_RPTR, 0);
	WREG32(mmIH_RB_WPTR, 0);
	adev->irq.ih.enabled = false;
	adev->irq.ih.rptr = 0;
}

/**
 * cik_ih_irq_init - init and enable the interrupt ring
 *
 * @adev: amdgpu_device pointer
 *
 * Allocate a ring buffer for the interrupt controller,
 * enable the RLC, disable interrupts, enable the IH
 * ring buffer and enable it (CIK).
 * Called at device load and reume.
 * Returns 0 for success, errors for failure.
 */
static int cik_ih_irq_init(struct amdgpu_device *adev)
{
	struct amdgpu_ih_ring *ih = &adev->irq.ih;
	int rb_bufsz;
	u32 interrupt_cntl, ih_cntl, ih_rb_cntl;

	/* disable irqs */
	cik_ih_disable_interrupts(adev);

	/* setup interrupt control */
	WREG32(mmINTERRUPT_CNTL2, adev->dummy_page_addr >> 8);
	interrupt_cntl = RREG32(mmINTERRUPT_CNTL);
	/* INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi
	 * INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN
	 */
	interrupt_cntl &= ~INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK;
	/* INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */
	interrupt_cntl &= ~INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK;
	WREG32(mmINTERRUPT_CNTL, interrupt_cntl);

	WREG32(mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8);
	rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);

	ih_rb_cntl = (IH_RB_CNTL__WPTR_OVERFLOW_ENABLE_MASK |
		      IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK |
		      (rb_bufsz << 1));

	ih_rb_cntl |= IH_RB_CNTL__WPTR_WRITEBACK_ENABLE_MASK;

	/* set the writeback address whether it's enabled or not */
	WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(ih->wptr_addr));
	WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(ih->wptr_addr) & 0xFF);

	WREG32(mmIH_RB_CNTL, ih_rb_cntl);

	/* set rptr, wptr to 0 */
	WREG32(mmIH_RB_RPTR, 0);
	WREG32(mmIH_RB_WPTR, 0);

	/* Default settings for IH_CNTL (disabled at first) */
	ih_cntl = (0x10 << IH_CNTL__MC_WRREQ_CREDIT__SHIFT) |
		(0x10 << IH_CNTL__MC_WR_CLEAN_CNT__SHIFT) |
		(0 << IH_CNTL__MC_VMID__SHIFT);
	/* IH_CNTL__RPTR_REARM_MASK only works if msi's are enabled */
	if (adev->irq.msi_enabled)
		ih_cntl |= IH_CNTL__RPTR_REARM_MASK;
	WREG32(mmIH_CNTL, ih_cntl);

	pci_set_master(adev->pdev);

	/* enable irqs */
	cik_ih_enable_interrupts(adev);

	return 0;
}

/**
 * cik_ih_irq_disable - disable interrupts
 *
 * @adev: amdgpu_device pointer
 *
 * Disable interrupts on the hw (CIK).
 */
static void cik_ih_irq_disable(struct amdgpu_device *adev)
{
	cik_ih_disable_interrupts(adev);
	/* Wait and acknowledge irq */
	mdelay(1);
}

/**
 * cik_ih_get_wptr - get the IH ring buffer wptr
 *
 * @adev: amdgpu_device pointer
 *
 * Get the IH ring buffer wptr from either the register
 * or the writeback memory buffer (CIK).  Also check for
 * ring buffer overflow and deal with it.
 * Used by cik_irq_process().
 * Returns the value of the wptr.
 */
static u32 cik_ih_get_wptr(struct amdgpu_device *adev,
			   struct amdgpu_ih_ring *ih)
{
	u32 wptr, tmp;

	wptr = le32_to_cpu(*ih->wptr_cpu);

	if (wptr & IH_RB_WPTR__RB_OVERFLOW_MASK) {
		wptr &= ~IH_RB_WPTR__RB_OVERFLOW_MASK;
		/* When a ring buffer overflow happen start parsing interrupt
		 * from the last not overwritten vector (wptr + 16). Hopefully
		 * this should allow us to catchup.
		 */
		dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
			 wptr, ih->rptr, (wptr + 16) & ih->ptr_mask);
		ih->rptr = (wptr + 16) & ih->ptr_mask;
		tmp = RREG32(mmIH_RB_CNTL);
		tmp |= IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK;
		WREG32(mmIH_RB_CNTL, tmp);
	}
	return (wptr & ih->ptr_mask);
}

/*        CIK IV Ring
 * Each IV ring entry is 128 bits:
 * [7:0]    - interrupt source id
 * [31:8]   - reserved
 * [59:32]  - interrupt source data
 * [63:60]  - reserved
 * [71:64]  - RINGID
 *            CP:
 *            ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
 *            QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
 *                     - for gfx, hw shader state (0=PS...5=LS, 6=CS)
 *            ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
 *            PIPE_ID - ME0 0=3D
 *                    - ME1&2 compute dispatcher (4 pipes each)
 *            SDMA:
 *            INSTANCE_ID [1:0], QUEUE_ID[1:0]
 *            INSTANCE_ID - 0 = sdma0, 1 = sdma1
 *            QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
 * [79:72]  - VMID
 * [95:80]  - PASID
 * [127:96] - reserved
 */

 /**
 * cik_ih_decode_iv - decode an interrupt vector
 *
 * @adev: amdgpu_device pointer
 *
 * Decodes the interrupt vector at the current rptr
 * position and also advance the position.
 */
static void cik_ih_decode_iv(struct amdgpu_device *adev,
			     struct amdgpu_ih_ring *ih,
			     struct amdgpu_iv_entry *entry)
{
	/* wptr/rptr are in bytes! */
	u32 ring_index = ih->rptr >> 2;
	uint32_t dw[4];

	dw[0] = le32_to_cpu(ih->ring[ring_index + 0]);
	dw[1] = le32_to_cpu(ih->ring[ring_index + 1]);
	dw[2] = le32_to_cpu(ih->ring[ring_index + 2]);
	dw[3] = le32_to_cpu(ih->ring[ring_index + 3]);

	entry->client_id = AMDGPU_IRQ_CLIENTID_LEGACY;
	entry->src_id = dw[0] & 0xff;
	entry->src_data[0] = dw[1] & 0xfffffff;
	entry->ring_id = dw[2] & 0xff;
	entry->vmid = (dw[2] >> 8) & 0xff;
	entry->pasid = (dw[2] >> 16) & 0xffff;

	/* wptr/rptr are in bytes! */
	ih->rptr += 16;
}

/**
 * cik_ih_set_rptr - set the IH ring buffer rptr
 *
 * @adev: amdgpu_device pointer
 *
 * Set the IH ring buffer rptr.
 */
static void cik_ih_set_rptr(struct amdgpu_device *adev,
			    struct amdgpu_ih_ring *ih)
{
	WREG32(mmIH_RB_RPTR, ih->rptr);
}

static int cik_ih_early_init(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
	int ret;

	ret = amdgpu_irq_add_domain(adev);
	if (ret)
		return ret;

	cik_ih_set_interrupt_funcs(adev);

	return 0;
}

static int cik_ih_sw_init(void *handle)
{
	int r;
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 64 * 1024, false);
	if (r)
		return r;

	r = amdgpu_irq_init(adev);

	return r;
}

static int cik_ih_sw_fini(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	amdgpu_irq_fini(adev);
	amdgpu_ih_ring_fini(adev, &adev->irq.ih);
	amdgpu_irq_remove_domain(adev);

	return 0;
}

static int cik_ih_hw_init(void *handle)
{
	int r;
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	r = cik_ih_irq_init(adev);
	if (r)
		return r;

	return 0;
}

static int cik_ih_hw_fini(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	cik_ih_irq_disable(adev);

	return 0;
}

static int cik_ih_suspend(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	return cik_ih_hw_fini(adev);
}

static int cik_ih_resume(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	return cik_ih_hw_init(adev);
}

static bool cik_ih_is_idle(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
	u32 tmp = RREG32(mmSRBM_STATUS);

	if (tmp & SRBM_STATUS__IH_BUSY_MASK)
		return false;

	return true;
}

static int cik_ih_wait_for_idle(void *handle)
{
	unsigned i;
	u32 tmp;
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	for (i = 0; i < adev->usec_timeout; i++) {
		/* read MC_STATUS */
		tmp = RREG32(mmSRBM_STATUS) & SRBM_STATUS__IH_BUSY_MASK;
		if (!tmp)
			return 0;
		udelay(1);
	}
	return -ETIMEDOUT;
}

static int cik_ih_soft_reset(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	u32 srbm_soft_reset = 0;
	u32 tmp = RREG32(mmSRBM_STATUS);

	if (tmp & SRBM_STATUS__IH_BUSY_MASK)
		srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_IH_MASK;

	if (srbm_soft_reset) {
		tmp = RREG32(mmSRBM_SOFT_RESET);
		tmp |= srbm_soft_reset;
		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
		WREG32(mmSRBM_SOFT_RESET, tmp);
		tmp = RREG32(mmSRBM_SOFT_RESET);

		udelay(50);

		tmp &= ~srbm_soft_reset;
		WREG32(mmSRBM_SOFT_RESET, tmp);
		tmp = RREG32(mmSRBM_SOFT_RESET);

		/* Wait a little for things to settle down */
		udelay(50);
	}

	return 0;
}

static int cik_ih_set_clockgating_state(void *handle,
					  enum amd_clockgating_state state)
{
	return 0;
}

static int cik_ih_set_powergating_state(void *handle,
					  enum amd_powergating_state state)
{
	return 0;
}

static const struct amd_ip_funcs cik_ih_ip_funcs = {
	.name = "cik_ih",
	.early_init = cik_ih_early_init,
	.late_init = NULL,
	.sw_init = cik_ih_sw_init,
	.sw_fini = cik_ih_sw_fini,
	.hw_init = cik_ih_hw_init,
	.hw_fini = cik_ih_hw_fini,
	.suspend = cik_ih_suspend,
	.resume = cik_ih_resume,
	.is_idle = cik_ih_is_idle,
	.wait_for_idle = cik_ih_wait_for_idle,
	.soft_reset = cik_ih_soft_reset,
	.set_clockgating_state = cik_ih_set_clockgating_state,
	.set_powergating_state = cik_ih_set_powergating_state,
};

static const struct amdgpu_ih_funcs cik_ih_funcs = {
	.get_wptr = cik_ih_get_wptr,
	.decode_iv = cik_ih_decode_iv,
	.set_rptr = cik_ih_set_rptr
};

static void cik_ih_set_interrupt_funcs(struct amdgpu_device *adev)
{
	adev->irq.ih_funcs = &cik_ih_funcs;
}

const struct amdgpu_ip_block_version cik_ih_ip_block =
{
	.type = AMD_IP_BLOCK_TYPE_IH,
	.major = 2,
	.minor = 0,
	.rev = 0,
	.funcs = &cik_ih_ip_funcs,
};
Commit	Line	Data
a2e73f56 AD	1	/*
	2	* Copyright 2012 Advanced Micro Devices, Inc.
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice shall be included in
	12	* all copies or substantial portions of the Software.
	13	*
	14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	20	* OTHER DEALINGS IN THE SOFTWARE.
	21	*
	22	*/
47b757fb SR	23
	24	#include <linux/pci.h>
	25
a2e73f56 AD	26	#include "amdgpu.h"
	27	#include "amdgpu_ih.h"
	28	#include "cikd.h"
	29
	30	#include "bif/bif_4_1_d.h"
	31	#include "bif/bif_4_1_sh_mask.h"
	32
	33	#include "oss/oss_2_0_d.h"
	34	#include "oss/oss_2_0_sh_mask.h"
	35
	36	/*
	37	* Interrupts
	38	* Starting with r6xx, interrupts are handled via a ring buffer.
	39	* Ring buffers are areas of GPU accessible memory that the GPU
	40	* writes interrupt vectors into and the host reads vectors out of.
	41	* There is a rptr (read pointer) that determines where the
	42	* host is currently reading, and a wptr (write pointer)
	43	* which determines where the GPU has written. When the
	44	* pointers are equal, the ring is idle. When the GPU
	45	* writes vectors to the ring buffer, it increments the
	46	* wptr. When there is an interrupt, the host then starts
	47	* fetching commands and processing them until the pointers are
	48	* equal again at which point it updates the rptr.
	49	*/
	50
	51	static void cik_ih_set_interrupt_funcs(struct amdgpu_device *adev);
	52
	53	/**
	54	* cik_ih_enable_interrupts - Enable the interrupt ring buffer
	55	*
	56	* @adev: amdgpu_device pointer
	57	*
	58	* Enable the interrupt ring buffer (CIK).
	59	*/
	60	static void cik_ih_enable_interrupts(struct amdgpu_device *adev)
	61	{
	62	u32 ih_cntl = RREG32(mmIH_CNTL);
	63	u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL);
	64
	65	ih_cntl \|= IH_CNTL__ENABLE_INTR_MASK;
	66	ih_rb_cntl \|= IH_RB_CNTL__RB_ENABLE_MASK;
	67	WREG32(mmIH_CNTL, ih_cntl);
	68	WREG32(mmIH_RB_CNTL, ih_rb_cntl);
	69	adev->irq.ih.enabled = true;
	70	}
	71
	72	/**
	73	* cik_ih_disable_interrupts - Disable the interrupt ring buffer
	74	*
	75	* @adev: amdgpu_device pointer
	76	*
	77	* Disable the interrupt ring buffer (CIK).
	78	*/
	79	static void cik_ih_disable_interrupts(struct amdgpu_device *adev)
	80	{
	81	u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL);
	82	u32 ih_cntl = RREG32(mmIH_CNTL);
	83
	84	ih_rb_cntl &= ~IH_RB_CNTL__RB_ENABLE_MASK;
	85	ih_cntl &= ~IH_CNTL__ENABLE_INTR_MASK;
	86	WREG32(mmIH_RB_CNTL, ih_rb_cntl);
	87	WREG32(mmIH_CNTL, ih_cntl);
	88	/* set rptr, wptr to 0 */
	89	WREG32(mmIH_RB_RPTR, 0);
90	WREG32(mmIH_RB_WPTR, 0);
91	adev->irq.ih.enabled = false;
92	adev->irq.ih.rptr = 0;
93	}
94
95	/**
96	* cik_ih_irq_init - init and enable the interrupt ring
97	*
98	* @adev: amdgpu_device pointer
99	*
100	* Allocate a ring buffer for the interrupt controller,
101	* enable the RLC, disable interrupts, enable the IH
102	* ring buffer and enable it (CIK).
103	* Called at device load and reume.
104	* Returns 0 for success, errors for failure.
105	*/
106	static int cik_ih_irq_init(struct amdgpu_device *adev)
107	{
d81f78b4	108	struct amdgpu_ih_ring *ih = &adev->irq.ih;
a2e73f56 AD	109	int rb_bufsz;
a2e73f56 AD	110	u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
a2e73f56 AD	111
	112	/* disable irqs */
	113	cik_ih_disable_interrupts(adev);
	114
	115	/* setup interrupt control */
92e71b06	116	WREG32(mmINTERRUPT_CNTL2, adev->dummy_page_addr >> 8);
a2e73f56 AD	117	interrupt_cntl = RREG32(mmINTERRUPT_CNTL);
	118	/* INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi
	119	* INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN
	120	*/
	121	interrupt_cntl &= ~INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK;
	122	/* INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */
	123	interrupt_cntl &= ~INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK;
	124	WREG32(mmINTERRUPT_CNTL, interrupt_cntl);
	125
	126	WREG32(mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8);
	127	rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);
	128
	129	ih_rb_cntl = (IH_RB_CNTL__WPTR_OVERFLOW_ENABLE_MASK \|
	130	IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK \|
	131	(rb_bufsz << 1));
	132
	133	ih_rb_cntl \|= IH_RB_CNTL__WPTR_WRITEBACK_ENABLE_MASK;
	134
	135	/* set the writeback address whether it's enabled or not */
d81f78b4 CK	136	WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(ih->wptr_addr));
d81f78b4 CK	137	WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(ih->wptr_addr) & 0xFF);
a2e73f56 AD	138
	139	WREG32(mmIH_RB_CNTL, ih_rb_cntl);
	140
	141	/* set rptr, wptr to 0 */
	142	WREG32(mmIH_RB_RPTR, 0);
	143	WREG32(mmIH_RB_WPTR, 0);
	144
	145	/* Default settings for IH_CNTL (disabled at first) */
	146	ih_cntl = (0x10 << IH_CNTL__MC_WRREQ_CREDIT__SHIFT) \|
	147	(0x10 << IH_CNTL__MC_WR_CLEAN_CNT__SHIFT) \|
	148	(0 << IH_CNTL__MC_VMID__SHIFT);
	149	/* IH_CNTL__RPTR_REARM_MASK only works if msi's are enabled */
	150	if (adev->irq.msi_enabled)
	151	ih_cntl \|= IH_CNTL__RPTR_REARM_MASK;
	152	WREG32(mmIH_CNTL, ih_cntl);
	153
	154	pci_set_master(adev->pdev);
	155
	156	/* enable irqs */
	157	cik_ih_enable_interrupts(adev);
	158
734711b7	159	return 0;
a2e73f56 AD	160	}
	161
	162	/**
	163	* cik_ih_irq_disable - disable interrupts
	164	*
	165	* @adev: amdgpu_device pointer
	166	*
	167	* Disable interrupts on the hw (CIK).
	168	*/
	169	static void cik_ih_irq_disable(struct amdgpu_device *adev)
	170	{
	171	cik_ih_disable_interrupts(adev);
	172	/* Wait and acknowledge irq */
	173	mdelay(1);
	174	}
	175
	176	/**
	177	* cik_ih_get_wptr - get the IH ring buffer wptr
	178	*
	179	* @adev: amdgpu_device pointer
	180	*
	181	* Get the IH ring buffer wptr from either the register
	182	* or the writeback memory buffer (CIK). Also check for
	183	* ring buffer overflow and deal with it.
	184	* Used by cik_irq_process().
	185	* Returns the value of the wptr.
	186	*/
8bb9eb48 CK	187	static u32 cik_ih_get_wptr(struct amdgpu_device *adev,
8bb9eb48 CK	188	struct amdgpu_ih_ring *ih)
a2e73f56 AD	189	{
	190	u32 wptr, tmp;
	191
d81f78b4	192	wptr = le32_to_cpu(*ih->wptr_cpu);
a2e73f56 AD	193
	194	if (wptr & IH_RB_WPTR__RB_OVERFLOW_MASK) {
	195	wptr &= ~IH_RB_WPTR__RB_OVERFLOW_MASK;
	196	/* When a ring buffer overflow happen start parsing interrupt
	197	* from the last not overwritten vector (wptr + 16). Hopefully
	198	* this should allow us to catchup.
	199	*/
	200	dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
8bb9eb48 CK	201	wptr, ih->rptr, (wptr + 16) & ih->ptr_mask);
8bb9eb48 CK	202	ih->rptr = (wptr + 16) & ih->ptr_mask;
a2e73f56 AD	203	tmp = RREG32(mmIH_RB_CNTL);
	204	tmp \|= IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK;
	205	WREG32(mmIH_RB_CNTL, tmp);
	206	}
8bb9eb48	207	return (wptr & ih->ptr_mask);
a2e73f56 AD	208	}
	209
	210	/* CIK IV Ring
	211	* Each IV ring entry is 128 bits:
	212	* [7:0] - interrupt source id
	213	* [31:8] - reserved
	214	* [59:32] - interrupt source data
	215	* [63:60] - reserved
	216	* [71:64] - RINGID
	217	* CP:
	218	* ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
	219	* QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
	220	* - for gfx, hw shader state (0=PS...5=LS, 6=CS)
	221	* ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
	222	* PIPE_ID - ME0 0=3D
	223	* - ME1&2 compute dispatcher (4 pipes each)
	224	* SDMA:
	225	* INSTANCE_ID [1:0], QUEUE_ID[1:0]
	226	* INSTANCE_ID - 0 = sdma0, 1 = sdma1
	227	* QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
	228	* [79:72] - VMID
	229	* [95:80] - PASID
	230	* [127:96] - reserved
	231	*/
	232
	233	/**
	234	* cik_ih_decode_iv - decode an interrupt vector
	235	*
	236	* @adev: amdgpu_device pointer
	237	*
	238	* Decodes the interrupt vector at the current rptr
	239	* position and also advance the position.
	240	*/
	241	static void cik_ih_decode_iv(struct amdgpu_device *adev,
8bb9eb48	242	struct amdgpu_ih_ring *ih,
a2e73f56 AD	243	struct amdgpu_iv_entry *entry)
	244	{
	245	/* wptr/rptr are in bytes! */
8bb9eb48	246	u32 ring_index = ih->rptr >> 2;
a2e73f56	247	uint32_t dw[4];
edf600da	248
8bb9eb48 CK	249	dw[0] = le32_to_cpu(ih->ring[ring_index + 0]);
	250	dw[1] = le32_to_cpu(ih->ring[ring_index + 1]);
	251	dw[2] = le32_to_cpu(ih->ring[ring_index + 2]);
	252	dw[3] = le32_to_cpu(ih->ring[ring_index + 3]);
a2e73f56	253
1ffdeca6	254	entry->client_id = AMDGPU_IRQ_CLIENTID_LEGACY;
a2e73f56	255	entry->src_id = dw[0] & 0xff;
7ccf5aa8	256	entry->src_data[0] = dw[1] & 0xfffffff;
a2e73f56	257	entry->ring_id = dw[2] & 0xff;
c4f46f22	258	entry->vmid = (dw[2] >> 8) & 0xff;
3816e42f	259	entry->pasid = (dw[2] >> 16) & 0xffff;
a2e73f56 AD	260
a2e73f56 AD	261	/* wptr/rptr are in bytes! */
8bb9eb48	262	ih->rptr += 16;
a2e73f56 AD	263	}
	264
	265	/**
	266	* cik_ih_set_rptr - set the IH ring buffer rptr
	267	*
	268	* @adev: amdgpu_device pointer
	269	*
	270	* Set the IH ring buffer rptr.
	271	*/
8bb9eb48 CK	272	static void cik_ih_set_rptr(struct amdgpu_device *adev,
8bb9eb48 CK	273	struct amdgpu_ih_ring *ih)
a2e73f56	274	{
8bb9eb48	275	WREG32(mmIH_RB_RPTR, ih->rptr);
a2e73f56 AD	276	}
a2e73f56 AD	277
5fc3aeeb	278	static int cik_ih_early_init(void *handle)
a2e73f56	279	{
5fc3aeeb	280	struct amdgpu_device adev = (struct amdgpu_device )handle;
5f232365 AD	281	int ret;
	282
	283	ret = amdgpu_irq_add_domain(adev);
	284	if (ret)
	285	return ret;
5fc3aeeb	286
a2e73f56 AD	287	cik_ih_set_interrupt_funcs(adev);
	288
	289	return 0;
	290	}
	291
5fc3aeeb	292	static int cik_ih_sw_init(void *handle)
a2e73f56 AD	293	{
a2e73f56 AD	294	int r;
5fc3aeeb	295	struct amdgpu_device adev = (struct amdgpu_device )handle;
a2e73f56	296
425c3143	297	r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 64 * 1024, false);
a2e73f56 AD	298	if (r)
	299	return r;
	300
	301	r = amdgpu_irq_init(adev);
	302
	303	return r;
	304	}
	305
5fc3aeeb	306	static int cik_ih_sw_fini(void *handle)
a2e73f56	307	{
5fc3aeeb	308	struct amdgpu_device adev = (struct amdgpu_device )handle;
5fc3aeeb	309
a2e73f56	310	amdgpu_irq_fini(adev);
425c3143	311	amdgpu_ih_ring_fini(adev, &adev->irq.ih);
5f232365	312	amdgpu_irq_remove_domain(adev);
a2e73f56 AD	313
	314	return 0;
	315	}
	316
5fc3aeeb	317	static int cik_ih_hw_init(void *handle)
a2e73f56 AD	318	{
a2e73f56 AD	319	int r;
5fc3aeeb	320	struct amdgpu_device adev = (struct amdgpu_device )handle;
a2e73f56 AD	321
	322	r = cik_ih_irq_init(adev);
	323	if (r)
	324	return r;
	325
	326	return 0;
	327	}
	328
5fc3aeeb	329	static int cik_ih_hw_fini(void *handle)
a2e73f56	330	{
5fc3aeeb	331	struct amdgpu_device adev = (struct amdgpu_device )handle;
5fc3aeeb	332
a2e73f56 AD	333	cik_ih_irq_disable(adev);
	334
	335	return 0;
	336	}
	337
5fc3aeeb	338	static int cik_ih_suspend(void *handle)
a2e73f56	339	{
5fc3aeeb	340	struct amdgpu_device adev = (struct amdgpu_device )handle;
5fc3aeeb	341
a2e73f56 AD	342	return cik_ih_hw_fini(adev);
	343	}
	344
5fc3aeeb	345	static int cik_ih_resume(void *handle)
a2e73f56	346	{
5fc3aeeb	347	struct amdgpu_device adev = (struct amdgpu_device )handle;
5fc3aeeb	348
a2e73f56 AD	349	return cik_ih_hw_init(adev);
	350	}
	351
5fc3aeeb	352	static bool cik_ih_is_idle(void *handle)
a2e73f56	353	{
5fc3aeeb	354	struct amdgpu_device adev = (struct amdgpu_device )handle;
a2e73f56 AD	355	u32 tmp = RREG32(mmSRBM_STATUS);
	356
	357	if (tmp & SRBM_STATUS__IH_BUSY_MASK)
	358	return false;
	359
	360	return true;
	361	}
	362
5fc3aeeb	363	static int cik_ih_wait_for_idle(void *handle)
a2e73f56 AD	364	{
	365	unsigned i;
	366	u32 tmp;
5fc3aeeb	367	struct amdgpu_device adev = (struct amdgpu_device )handle;
a2e73f56 AD	368
	369	for (i = 0; i < adev->usec_timeout; i++) {
	370	/* read MC_STATUS */
	371	tmp = RREG32(mmSRBM_STATUS) & SRBM_STATUS__IH_BUSY_MASK;
	372	if (!tmp)
	373	return 0;
	374	udelay(1);
	375	}
	376	return -ETIMEDOUT;
	377	}
	378
5fc3aeeb	379	static int cik_ih_soft_reset(void *handle)
a2e73f56	380	{
5fc3aeeb	381	struct amdgpu_device adev = (struct amdgpu_device )handle;
5fc3aeeb	382
a2e73f56 AD	383	u32 srbm_soft_reset = 0;
	384	u32 tmp = RREG32(mmSRBM_STATUS);
	385
	386	if (tmp & SRBM_STATUS__IH_BUSY_MASK)
	387	srbm_soft_reset \|= SRBM_SOFT_RESET__SOFT_RESET_IH_MASK;
	388
	389	if (srbm_soft_reset) {
a2e73f56 AD	390	tmp = RREG32(mmSRBM_SOFT_RESET);
	391	tmp \|= srbm_soft_reset;
	392	dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
	393	WREG32(mmSRBM_SOFT_RESET, tmp);
	394	tmp = RREG32(mmSRBM_SOFT_RESET);
	395
	396	udelay(50);
	397
	398	tmp &= ~srbm_soft_reset;
	399	WREG32(mmSRBM_SOFT_RESET, tmp);
	400	tmp = RREG32(mmSRBM_SOFT_RESET);
	401
	402	/* Wait a little for things to settle down */
	403	udelay(50);
a2e73f56 AD	404	}
	405
	406	return 0;
	407	}
	408
5fc3aeeb	409	static int cik_ih_set_clockgating_state(void *handle,
5fc3aeeb	410	enum amd_clockgating_state state)
a2e73f56 AD	411	{
	412	return 0;
	413	}
	414
5fc3aeeb	415	static int cik_ih_set_powergating_state(void *handle,
5fc3aeeb	416	enum amd_powergating_state state)
a2e73f56 AD	417	{
	418	return 0;
	419	}
	420
a1255107	421	static const struct amd_ip_funcs cik_ih_ip_funcs = {
88a907d6	422	.name = "cik_ih",
a2e73f56 AD	423	.early_init = cik_ih_early_init,
	424	.late_init = NULL,
	425	.sw_init = cik_ih_sw_init,
	426	.sw_fini = cik_ih_sw_fini,
	427	.hw_init = cik_ih_hw_init,
	428	.hw_fini = cik_ih_hw_fini,
	429	.suspend = cik_ih_suspend,
	430	.resume = cik_ih_resume,
	431	.is_idle = cik_ih_is_idle,
	432	.wait_for_idle = cik_ih_wait_for_idle,
	433	.soft_reset = cik_ih_soft_reset,
a2e73f56 AD	434	.set_clockgating_state = cik_ih_set_clockgating_state,
	435	.set_powergating_state = cik_ih_set_powergating_state,
	436	};
	437
	438	static const struct amdgpu_ih_funcs cik_ih_funcs = {
	439	.get_wptr = cik_ih_get_wptr,
	440	.decode_iv = cik_ih_decode_iv,
	441	.set_rptr = cik_ih_set_rptr
	442	};
	443
	444	static void cik_ih_set_interrupt_funcs(struct amdgpu_device *adev)
	445	{
f54b30d7	446	adev->irq.ih_funcs = &cik_ih_funcs;
a2e73f56	447	}
a1255107 AD	448
	449	const struct amdgpu_ip_block_version cik_ih_ip_block =
	450	{
	451	.type = AMD_IP_BLOCK_TYPE_IH,
	452	.major = 2,
	453	.minor = 0,
	454	.rev = 0,
	455	.funcs = &cik_ih_ip_funcs,
	456	};