2 * Copyright 2016 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
25 #include "amdgpu_ih.h"
29 #include "vega10/soc15ip.h"
30 #include "vega10/OSSSYS/osssys_4_0_offset.h"
31 #include "vega10/OSSSYS/osssys_4_0_sh_mask.h"
33 #include "soc15_common.h"
34 #include "vega10_ih.h"
38 static void vega10_ih_set_interrupt_funcs(struct amdgpu_device
*adev
);
41 * vega10_ih_enable_interrupts - Enable the interrupt ring buffer
43 * @adev: amdgpu_device pointer
45 * Enable the interrupt ring buffer (VEGA10).
47 static void vega10_ih_enable_interrupts(struct amdgpu_device
*adev
)
49 u32 ih_rb_cntl
= RREG32_SOC15(OSSSYS
, 0, mmIH_RB_CNTL
);
51 ih_rb_cntl
= REG_SET_FIELD(ih_rb_cntl
, IH_RB_CNTL
, RB_ENABLE
, 1);
52 ih_rb_cntl
= REG_SET_FIELD(ih_rb_cntl
, IH_RB_CNTL
, ENABLE_INTR
, 1);
53 WREG32_SOC15(OSSSYS
, 0, mmIH_RB_CNTL
, ih_rb_cntl
);
54 adev
->irq
.ih
.enabled
= true;
58 * vega10_ih_disable_interrupts - Disable the interrupt ring buffer
60 * @adev: amdgpu_device pointer
62 * Disable the interrupt ring buffer (VEGA10).
64 static void vega10_ih_disable_interrupts(struct amdgpu_device
*adev
)
66 u32 ih_rb_cntl
= RREG32_SOC15(OSSSYS
, 0, mmIH_RB_CNTL
);
68 ih_rb_cntl
= REG_SET_FIELD(ih_rb_cntl
, IH_RB_CNTL
, RB_ENABLE
, 0);
69 ih_rb_cntl
= REG_SET_FIELD(ih_rb_cntl
, IH_RB_CNTL
, ENABLE_INTR
, 0);
70 WREG32_SOC15(OSSSYS
, 0, mmIH_RB_CNTL
, ih_rb_cntl
);
71 /* set rptr, wptr to 0 */
72 WREG32_SOC15(OSSSYS
, 0, mmIH_RB_RPTR
, 0);
73 WREG32_SOC15(OSSSYS
, 0, mmIH_RB_WPTR
, 0);
74 adev
->irq
.ih
.enabled
= false;
75 adev
->irq
.ih
.rptr
= 0;
79 * vega10_ih_irq_init - init and enable the interrupt ring
81 * @adev: amdgpu_device pointer
83 * Allocate a ring buffer for the interrupt controller,
84 * enable the RLC, disable interrupts, enable the IH
85 * ring buffer and enable it (VI).
86 * Called at device load and reume.
87 * Returns 0 for success, errors for failure.
89 static int vega10_ih_irq_init(struct amdgpu_device
*adev
)
93 u32 ih_rb_cntl
, ih_doorbell_rtpr
;
98 vega10_ih_disable_interrupts(adev
);
100 if (adev
->flags
& AMD_IS_APU
)
101 nbio_v7_0_ih_control(adev
);
103 nbio_v6_1_ih_control(adev
);
105 ih_rb_cntl
= RREG32_SOC15(OSSSYS
, 0, mmIH_RB_CNTL
);
106 /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
107 if (adev
->irq
.ih
.use_bus_addr
) {
108 WREG32_SOC15(OSSSYS
, 0, mmIH_RB_BASE
, adev
->irq
.ih
.rb_dma_addr
>> 8);
109 WREG32_SOC15(OSSSYS
, 0, mmIH_RB_BASE_HI
, ((u64
)adev
->irq
.ih
.rb_dma_addr
>> 40) & 0xff);
110 ih_rb_cntl
= REG_SET_FIELD(ih_rb_cntl
, IH_RB_CNTL
, MC_SPACE
, 1);
112 WREG32_SOC15(OSSSYS
, 0, mmIH_RB_BASE
, adev
->irq
.ih
.gpu_addr
>> 8);
113 WREG32_SOC15(OSSSYS
, 0, mmIH_RB_BASE_HI
, (adev
->irq
.ih
.gpu_addr
>> 40) & 0xff);
114 ih_rb_cntl
= REG_SET_FIELD(ih_rb_cntl
, IH_RB_CNTL
, MC_SPACE
, 4);
116 rb_bufsz
= order_base_2(adev
->irq
.ih
.ring_size
/ 4);
117 ih_rb_cntl
= REG_SET_FIELD(ih_rb_cntl
, IH_RB_CNTL
, WPTR_OVERFLOW_CLEAR
, 1);
118 ih_rb_cntl
= REG_SET_FIELD(ih_rb_cntl
, IH_RB_CNTL
, WPTR_OVERFLOW_ENABLE
, 1);
119 ih_rb_cntl
= REG_SET_FIELD(ih_rb_cntl
, IH_RB_CNTL
, RB_SIZE
, rb_bufsz
);
120 /* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register value is written to memory */
121 ih_rb_cntl
= REG_SET_FIELD(ih_rb_cntl
, IH_RB_CNTL
, WPTR_WRITEBACK_ENABLE
, 1);
122 ih_rb_cntl
= REG_SET_FIELD(ih_rb_cntl
, IH_RB_CNTL
, MC_SNOOP
, 1);
123 ih_rb_cntl
= REG_SET_FIELD(ih_rb_cntl
, IH_RB_CNTL
, MC_RO
, 0);
124 ih_rb_cntl
= REG_SET_FIELD(ih_rb_cntl
, IH_RB_CNTL
, MC_VMID
, 0);
126 if (adev
->irq
.msi_enabled
)
127 ih_rb_cntl
= REG_SET_FIELD(ih_rb_cntl
, IH_RB_CNTL
, RPTR_REARM
, 1);
129 WREG32_SOC15(OSSSYS
, 0, mmIH_RB_CNTL
, ih_rb_cntl
);
131 /* set the writeback address whether it's enabled or not */
132 if (adev
->irq
.ih
.use_bus_addr
)
133 wptr_off
= adev
->irq
.ih
.rb_dma_addr
+ (adev
->irq
.ih
.wptr_offs
* 4);
135 wptr_off
= adev
->wb
.gpu_addr
+ (adev
->irq
.ih
.wptr_offs
* 4);
136 WREG32_SOC15(OSSSYS
, 0, mmIH_RB_WPTR_ADDR_LO
, lower_32_bits(wptr_off
));
137 WREG32_SOC15(OSSSYS
, 0, mmIH_RB_WPTR_ADDR_HI
, upper_32_bits(wptr_off
) & 0xFF);
139 /* set rptr, wptr to 0 */
140 WREG32_SOC15(OSSSYS
, 0, mmIH_RB_RPTR
, 0);
141 WREG32_SOC15(OSSSYS
, 0, mmIH_RB_WPTR
, 0);
143 ih_doorbell_rtpr
= RREG32_SOC15(OSSSYS
, 0, mmIH_DOORBELL_RPTR
);
144 if (adev
->irq
.ih
.use_doorbell
) {
145 ih_doorbell_rtpr
= REG_SET_FIELD(ih_doorbell_rtpr
, IH_DOORBELL_RPTR
,
146 OFFSET
, adev
->irq
.ih
.doorbell_index
);
147 ih_doorbell_rtpr
= REG_SET_FIELD(ih_doorbell_rtpr
, IH_DOORBELL_RPTR
,
150 ih_doorbell_rtpr
= REG_SET_FIELD(ih_doorbell_rtpr
, IH_DOORBELL_RPTR
,
153 WREG32_SOC15(OSSSYS
, 0, mmIH_DOORBELL_RPTR
, ih_doorbell_rtpr
);
154 if (adev
->flags
& AMD_IS_APU
)
155 nbio_v7_0_ih_doorbell_range(adev
, adev
->irq
.ih
.use_doorbell
, adev
->irq
.ih
.doorbell_index
);
157 nbio_v6_1_ih_doorbell_range(adev
, adev
->irq
.ih
.use_doorbell
, adev
->irq
.ih
.doorbell_index
);
159 tmp
= RREG32_SOC15(OSSSYS
, 0, mmIH_STORM_CLIENT_LIST_CNTL
);
160 tmp
= REG_SET_FIELD(tmp
, IH_STORM_CLIENT_LIST_CNTL
,
161 CLIENT18_IS_STORM_CLIENT
, 1);
162 WREG32_SOC15(OSSSYS
, 0, mmIH_STORM_CLIENT_LIST_CNTL
, tmp
);
164 tmp
= RREG32_SOC15(OSSSYS
, 0, mmIH_INT_FLOOD_CNTL
);
165 tmp
= REG_SET_FIELD(tmp
, IH_INT_FLOOD_CNTL
, FLOOD_CNTL_ENABLE
, 1);
166 WREG32_SOC15(OSSSYS
, 0, mmIH_INT_FLOOD_CNTL
, tmp
);
168 pci_set_master(adev
->pdev
);
170 /* enable interrupts */
171 vega10_ih_enable_interrupts(adev
);
177 * vega10_ih_irq_disable - disable interrupts
179 * @adev: amdgpu_device pointer
181 * Disable interrupts on the hw (VEGA10).
183 static void vega10_ih_irq_disable(struct amdgpu_device
*adev
)
185 vega10_ih_disable_interrupts(adev
);
187 /* Wait and acknowledge irq */
192 * vega10_ih_get_wptr - get the IH ring buffer wptr
194 * @adev: amdgpu_device pointer
196 * Get the IH ring buffer wptr from either the register
197 * or the writeback memory buffer (VEGA10). Also check for
198 * ring buffer overflow and deal with it.
199 * Returns the value of the wptr.
201 static u32
vega10_ih_get_wptr(struct amdgpu_device
*adev
)
205 if (adev
->irq
.ih
.use_bus_addr
)
206 wptr
= le32_to_cpu(adev
->irq
.ih
.ring
[adev
->irq
.ih
.wptr_offs
]);
208 wptr
= le32_to_cpu(adev
->wb
.wb
[adev
->irq
.ih
.wptr_offs
]);
210 if (REG_GET_FIELD(wptr
, IH_RB_WPTR
, RB_OVERFLOW
)) {
211 wptr
= REG_SET_FIELD(wptr
, IH_RB_WPTR
, RB_OVERFLOW
, 0);
213 /* When a ring buffer overflow happen start parsing interrupt
214 * from the last not overwritten vector (wptr + 32). Hopefully
215 * this should allow us to catchup.
217 tmp
= (wptr
+ 32) & adev
->irq
.ih
.ptr_mask
;
218 dev_warn(adev
->dev
, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
219 wptr
, adev
->irq
.ih
.rptr
, tmp
);
220 adev
->irq
.ih
.rptr
= tmp
;
222 tmp
= RREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS
, 0, mmIH_RB_CNTL
));
223 tmp
= REG_SET_FIELD(tmp
, IH_RB_CNTL
, WPTR_OVERFLOW_CLEAR
, 1);
224 WREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS
, 0, mmIH_RB_CNTL
), tmp
);
226 return (wptr
& adev
->irq
.ih
.ptr_mask
);
230 * vega10_ih_prescreen_iv - prescreen an interrupt vector
232 * @adev: amdgpu_device pointer
234 * Returns true if the interrupt vector should be further processed.
236 static bool vega10_ih_prescreen_iv(struct amdgpu_device
*adev
)
238 u32 ring_index
= adev
->irq
.ih
.rptr
>> 2;
239 u32 dw0
, dw3
, dw4
, dw5
;
242 struct amdgpu_vm
*vm
;
245 dw0
= le32_to_cpu(adev
->irq
.ih
.ring
[ring_index
+ 0]);
246 dw3
= le32_to_cpu(adev
->irq
.ih
.ring
[ring_index
+ 3]);
247 dw4
= le32_to_cpu(adev
->irq
.ih
.ring
[ring_index
+ 4]);
248 dw5
= le32_to_cpu(adev
->irq
.ih
.ring
[ring_index
+ 5]);
250 /* Filter retry page faults, let only the first one pass. If
251 * there are too many outstanding faults, ignore them until
252 * some faults get cleared.
254 switch (dw0
& 0xff) {
255 case AMDGPU_IH_CLIENTID_VMC
:
256 case AMDGPU_IH_CLIENTID_UTCL2
:
263 pasid
= dw3
& 0xffff;
264 /* No PASID, can't identify faulting process */
268 /* Not a retry fault, check fault credit */
270 if (!amdgpu_vm_pasid_fault_credit(adev
, pasid
))
275 addr
= ((u64
)(dw5
& 0xf) << 44) | ((u64
)dw4
<< 12);
276 key
= AMDGPU_VM_FAULT(pasid
, addr
);
277 r
= amdgpu_ih_add_fault(adev
, key
);
279 /* Hash table is full or the fault is already being processed,
280 * ignore further page faults
285 /* Track retry faults in per-VM fault FIFO. */
286 spin_lock(&adev
->vm_manager
.pasid_lock
);
287 vm
= idr_find(&adev
->vm_manager
.pasid_idr
, pasid
);
288 spin_unlock(&adev
->vm_manager
.pasid_lock
);
289 if (WARN_ON_ONCE(!vm
)) {
290 /* VM not found, process it normally */
291 amdgpu_ih_clear_fault(adev
, key
);
294 /* No locking required with single writer and single reader */
295 r
= kfifo_put(&vm
->faults
, key
);
297 /* FIFO is full. Ignore it until there is space */
298 amdgpu_ih_clear_fault(adev
, key
);
302 /* It's the first fault for this address, process it normally */
306 adev
->irq
.ih
.rptr
+= 32;
311 * vega10_ih_decode_iv - decode an interrupt vector
313 * @adev: amdgpu_device pointer
315 * Decodes the interrupt vector at the current rptr
316 * position and also advance the position.
318 static void vega10_ih_decode_iv(struct amdgpu_device
*adev
,
319 struct amdgpu_iv_entry
*entry
)
321 /* wptr/rptr are in bytes! */
322 u32 ring_index
= adev
->irq
.ih
.rptr
>> 2;
325 dw
[0] = le32_to_cpu(adev
->irq
.ih
.ring
[ring_index
+ 0]);
326 dw
[1] = le32_to_cpu(adev
->irq
.ih
.ring
[ring_index
+ 1]);
327 dw
[2] = le32_to_cpu(adev
->irq
.ih
.ring
[ring_index
+ 2]);
328 dw
[3] = le32_to_cpu(adev
->irq
.ih
.ring
[ring_index
+ 3]);
329 dw
[4] = le32_to_cpu(adev
->irq
.ih
.ring
[ring_index
+ 4]);
330 dw
[5] = le32_to_cpu(adev
->irq
.ih
.ring
[ring_index
+ 5]);
331 dw
[6] = le32_to_cpu(adev
->irq
.ih
.ring
[ring_index
+ 6]);
332 dw
[7] = le32_to_cpu(adev
->irq
.ih
.ring
[ring_index
+ 7]);
334 entry
->client_id
= dw
[0] & 0xff;
335 entry
->src_id
= (dw
[0] >> 8) & 0xff;
336 entry
->ring_id
= (dw
[0] >> 16) & 0xff;
337 entry
->vm_id
= (dw
[0] >> 24) & 0xf;
338 entry
->vm_id_src
= (dw
[0] >> 31);
339 entry
->timestamp
= dw
[1] | ((u64
)(dw
[2] & 0xffff) << 32);
340 entry
->timestamp_src
= dw
[2] >> 31;
341 entry
->pas_id
= dw
[3] & 0xffff;
342 entry
->pasid_src
= dw
[3] >> 31;
343 entry
->src_data
[0] = dw
[4];
344 entry
->src_data
[1] = dw
[5];
345 entry
->src_data
[2] = dw
[6];
346 entry
->src_data
[3] = dw
[7];
349 /* wptr/rptr are in bytes! */
350 adev
->irq
.ih
.rptr
+= 32;
354 * vega10_ih_set_rptr - set the IH ring buffer rptr
356 * @adev: amdgpu_device pointer
358 * Set the IH ring buffer rptr.
360 static void vega10_ih_set_rptr(struct amdgpu_device
*adev
)
362 if (adev
->irq
.ih
.use_doorbell
) {
363 /* XXX check if swapping is necessary on BE */
364 if (adev
->irq
.ih
.use_bus_addr
)
365 adev
->irq
.ih
.ring
[adev
->irq
.ih
.rptr_offs
] = adev
->irq
.ih
.rptr
;
367 adev
->wb
.wb
[adev
->irq
.ih
.rptr_offs
] = adev
->irq
.ih
.rptr
;
368 WDOORBELL32(adev
->irq
.ih
.doorbell_index
, adev
->irq
.ih
.rptr
);
370 WREG32_SOC15(OSSSYS
, 0, mmIH_RB_RPTR
, adev
->irq
.ih
.rptr
);
374 static int vega10_ih_early_init(void *handle
)
376 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
378 vega10_ih_set_interrupt_funcs(adev
);
382 static int vega10_ih_sw_init(void *handle
)
385 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
387 r
= amdgpu_ih_ring_init(adev
, 256 * 1024, true);
391 adev
->irq
.ih
.use_doorbell
= true;
392 adev
->irq
.ih
.doorbell_index
= AMDGPU_DOORBELL64_IH
<< 1;
394 adev
->irq
.ih
.faults
= kmalloc(sizeof(*adev
->irq
.ih
.faults
), GFP_KERNEL
);
395 if (!adev
->irq
.ih
.faults
)
397 INIT_CHASH_TABLE(adev
->irq
.ih
.faults
->hash
,
398 AMDGPU_PAGEFAULT_HASH_BITS
, 8, 0);
399 spin_lock_init(&adev
->irq
.ih
.faults
->lock
);
400 adev
->irq
.ih
.faults
->count
= 0;
402 r
= amdgpu_irq_init(adev
);
407 static int vega10_ih_sw_fini(void *handle
)
409 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
411 amdgpu_irq_fini(adev
);
412 amdgpu_ih_ring_fini(adev
);
414 kfree(adev
->irq
.ih
.faults
);
415 adev
->irq
.ih
.faults
= NULL
;
420 static int vega10_ih_hw_init(void *handle
)
423 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
425 r
= vega10_ih_irq_init(adev
);
432 static int vega10_ih_hw_fini(void *handle
)
434 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
436 vega10_ih_irq_disable(adev
);
441 static int vega10_ih_suspend(void *handle
)
443 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
445 return vega10_ih_hw_fini(adev
);
448 static int vega10_ih_resume(void *handle
)
450 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
452 return vega10_ih_hw_init(adev
);
455 static bool vega10_ih_is_idle(void *handle
)
461 static int vega10_ih_wait_for_idle(void *handle
)
467 static int vega10_ih_soft_reset(void *handle
)
474 static int vega10_ih_set_clockgating_state(void *handle
,
475 enum amd_clockgating_state state
)
480 static int vega10_ih_set_powergating_state(void *handle
,
481 enum amd_powergating_state state
)
486 const struct amd_ip_funcs vega10_ih_ip_funcs
= {
488 .early_init
= vega10_ih_early_init
,
490 .sw_init
= vega10_ih_sw_init
,
491 .sw_fini
= vega10_ih_sw_fini
,
492 .hw_init
= vega10_ih_hw_init
,
493 .hw_fini
= vega10_ih_hw_fini
,
494 .suspend
= vega10_ih_suspend
,
495 .resume
= vega10_ih_resume
,
496 .is_idle
= vega10_ih_is_idle
,
497 .wait_for_idle
= vega10_ih_wait_for_idle
,
498 .soft_reset
= vega10_ih_soft_reset
,
499 .set_clockgating_state
= vega10_ih_set_clockgating_state
,
500 .set_powergating_state
= vega10_ih_set_powergating_state
,
503 static const struct amdgpu_ih_funcs vega10_ih_funcs
= {
504 .get_wptr
= vega10_ih_get_wptr
,
505 .prescreen_iv
= vega10_ih_prescreen_iv
,
506 .decode_iv
= vega10_ih_decode_iv
,
507 .set_rptr
= vega10_ih_set_rptr
510 static void vega10_ih_set_interrupt_funcs(struct amdgpu_device
*adev
)
512 if (adev
->irq
.ih_funcs
== NULL
)
513 adev
->irq
.ih_funcs
= &vega10_ih_funcs
;
516 const struct amdgpu_ip_block_version vega10_ih_ip_block
=
518 .type
= AMD_IP_BLOCK_TYPE_IH
,
522 .funcs
= &vega10_ih_ip_funcs
,