]>
Commit | Line | Data |
---|---|---|
aaa36a97 AD |
1 | /* |
2 | * Copyright 2014 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | */ | |
248a1d6f | 23 | #include <drm/drmP.h> |
aaa36a97 AD |
24 | #include "amdgpu.h" |
25 | #include "amdgpu_ih.h" | |
26 | #include "vid.h" | |
27 | ||
28 | #include "oss/oss_2_4_d.h" | |
29 | #include "oss/oss_2_4_sh_mask.h" | |
30 | ||
31 | #include "bif/bif_5_1_d.h" | |
32 | #include "bif/bif_5_1_sh_mask.h" | |
33 | ||
34 | /* | |
35 | * Interrupts | |
36 | * Starting with r6xx, interrupts are handled via a ring buffer. | |
37 | * Ring buffers are areas of GPU accessible memory that the GPU | |
38 | * writes interrupt vectors into and the host reads vectors out of. | |
39 | * There is a rptr (read pointer) that determines where the | |
40 | * host is currently reading, and a wptr (write pointer) | |
41 | * which determines where the GPU has written. When the | |
42 | * pointers are equal, the ring is idle. When the GPU | |
43 | * writes vectors to the ring buffer, it increments the | |
44 | * wptr. When there is an interrupt, the host then starts | |
45 | * fetching commands and processing them until the pointers are | |
46 | * equal again at which point it updates the rptr. | |
47 | */ | |
48 | ||
49 | static void iceland_ih_set_interrupt_funcs(struct amdgpu_device *adev); | |
50 | ||
51 | /** | |
52 | * iceland_ih_enable_interrupts - Enable the interrupt ring buffer | |
53 | * | |
54 | * @adev: amdgpu_device pointer | |
55 | * | |
56 | * Enable the interrupt ring buffer (VI). | |
57 | */ | |
58 | static void iceland_ih_enable_interrupts(struct amdgpu_device *adev) | |
59 | { | |
60 | u32 ih_cntl = RREG32(mmIH_CNTL); | |
61 | u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL); | |
62 | ||
63 | ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, ENABLE_INTR, 1); | |
64 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1); | |
65 | WREG32(mmIH_CNTL, ih_cntl); | |
66 | WREG32(mmIH_RB_CNTL, ih_rb_cntl); | |
67 | adev->irq.ih.enabled = true; | |
68 | } | |
69 | ||
70 | /** | |
71 | * iceland_ih_disable_interrupts - Disable the interrupt ring buffer | |
72 | * | |
73 | * @adev: amdgpu_device pointer | |
74 | * | |
75 | * Disable the interrupt ring buffer (VI). | |
76 | */ | |
77 | static void iceland_ih_disable_interrupts(struct amdgpu_device *adev) | |
78 | { | |
79 | u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL); | |
80 | u32 ih_cntl = RREG32(mmIH_CNTL); | |
81 | ||
82 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0); | |
83 | ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, ENABLE_INTR, 0); | |
84 | WREG32(mmIH_RB_CNTL, ih_rb_cntl); | |
85 | WREG32(mmIH_CNTL, ih_cntl); | |
86 | /* set rptr, wptr to 0 */ | |
87 | WREG32(mmIH_RB_RPTR, 0); | |
88 | WREG32(mmIH_RB_WPTR, 0); | |
89 | adev->irq.ih.enabled = false; | |
90 | adev->irq.ih.rptr = 0; | |
91 | } | |
92 | ||
93 | /** | |
94 | * iceland_ih_irq_init - init and enable the interrupt ring | |
95 | * | |
96 | * @adev: amdgpu_device pointer | |
97 | * | |
98 | * Allocate a ring buffer for the interrupt controller, | |
99 | * enable the RLC, disable interrupts, enable the IH | |
100 | * ring buffer and enable it (VI). | |
101 | * Called at device load and reume. | |
102 | * Returns 0 for success, errors for failure. | |
103 | */ | |
104 | static int iceland_ih_irq_init(struct amdgpu_device *adev) | |
105 | { | |
aaa36a97 AD |
106 | int rb_bufsz; |
107 | u32 interrupt_cntl, ih_cntl, ih_rb_cntl; | |
108 | u64 wptr_off; | |
109 | ||
110 | /* disable irqs */ | |
111 | iceland_ih_disable_interrupts(adev); | |
112 | ||
113 | /* setup interrupt control */ | |
92e71b06 | 114 | WREG32(mmINTERRUPT_CNTL2, adev->dummy_page_addr >> 8); |
aaa36a97 AD |
115 | interrupt_cntl = RREG32(mmINTERRUPT_CNTL); |
116 | /* INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi | |
117 | * INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN | |
118 | */ | |
119 | interrupt_cntl = REG_SET_FIELD(interrupt_cntl, INTERRUPT_CNTL, IH_DUMMY_RD_OVERRIDE, 0); | |
120 | /* INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */ | |
121 | interrupt_cntl = REG_SET_FIELD(interrupt_cntl, INTERRUPT_CNTL, IH_REQ_NONSNOOP_EN, 0); | |
122 | WREG32(mmINTERRUPT_CNTL, interrupt_cntl); | |
123 | ||
124 | /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/ | |
125 | WREG32(mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8); | |
126 | ||
127 | rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4); | |
128 | ih_rb_cntl = REG_SET_FIELD(0, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 1); | |
129 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); | |
130 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz); | |
131 | ||
132 | /* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register value is written to memory */ | |
133 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_WRITEBACK_ENABLE, 1); | |
134 | ||
135 | /* set the writeback address whether it's enabled or not */ | |
136 | wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4); | |
137 | WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(wptr_off)); | |
138 | WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(wptr_off) & 0xFF); | |
139 | ||
140 | WREG32(mmIH_RB_CNTL, ih_rb_cntl); | |
141 | ||
142 | /* set rptr, wptr to 0 */ | |
143 | WREG32(mmIH_RB_RPTR, 0); | |
144 | WREG32(mmIH_RB_WPTR, 0); | |
145 | ||
146 | /* Default settings for IH_CNTL (disabled at first) */ | |
147 | ih_cntl = RREG32(mmIH_CNTL); | |
148 | ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, MC_VMID, 0); | |
149 | ||
150 | if (adev->irq.msi_enabled) | |
151 | ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, RPTR_REARM, 1); | |
152 | WREG32(mmIH_CNTL, ih_cntl); | |
153 | ||
154 | pci_set_master(adev->pdev); | |
155 | ||
156 | /* enable interrupts */ | |
157 | iceland_ih_enable_interrupts(adev); | |
158 | ||
a280a42a | 159 | return 0; |
aaa36a97 AD |
160 | } |
161 | ||
162 | /** | |
163 | * iceland_ih_irq_disable - disable interrupts | |
164 | * | |
165 | * @adev: amdgpu_device pointer | |
166 | * | |
167 | * Disable interrupts on the hw (VI). | |
168 | */ | |
169 | static void iceland_ih_irq_disable(struct amdgpu_device *adev) | |
170 | { | |
171 | iceland_ih_disable_interrupts(adev); | |
172 | ||
173 | /* Wait and acknowledge irq */ | |
174 | mdelay(1); | |
175 | } | |
176 | ||
177 | /** | |
178 | * iceland_ih_get_wptr - get the IH ring buffer wptr | |
179 | * | |
180 | * @adev: amdgpu_device pointer | |
181 | * | |
182 | * Get the IH ring buffer wptr from either the register | |
183 | * or the writeback memory buffer (VI). Also check for | |
184 | * ring buffer overflow and deal with it. | |
185 | * Used by cz_irq_process(VI). | |
186 | * Returns the value of the wptr. | |
187 | */ | |
188 | static u32 iceland_ih_get_wptr(struct amdgpu_device *adev) | |
189 | { | |
190 | u32 wptr, tmp; | |
191 | ||
192 | wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]); | |
193 | ||
194 | if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) { | |
195 | wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); | |
196 | /* When a ring buffer overflow happen start parsing interrupt | |
197 | * from the last not overwritten vector (wptr + 16). Hopefully | |
198 | * this should allow us to catchup. | |
199 | */ | |
200 | dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", | |
201 | wptr, adev->irq.ih.rptr, (wptr + 16) & adev->irq.ih.ptr_mask); | |
202 | adev->irq.ih.rptr = (wptr + 16) & adev->irq.ih.ptr_mask; | |
203 | tmp = RREG32(mmIH_RB_CNTL); | |
204 | tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); | |
205 | WREG32(mmIH_RB_CNTL, tmp); | |
206 | } | |
207 | return (wptr & adev->irq.ih.ptr_mask); | |
208 | } | |
209 | ||
00ecd8a2 FK |
210 | /** |
211 | * iceland_ih_prescreen_iv - prescreen an interrupt vector | |
212 | * | |
213 | * @adev: amdgpu_device pointer | |
214 | * | |
215 | * Returns true if the interrupt vector should be further processed. | |
216 | */ | |
217 | static bool iceland_ih_prescreen_iv(struct amdgpu_device *adev) | |
218 | { | |
c98171cc FK |
219 | u32 ring_index = adev->irq.ih.rptr >> 2; |
220 | u16 pasid; | |
221 | ||
222 | switch (le32_to_cpu(adev->irq.ih.ring[ring_index]) & 0xff) { | |
223 | case 146: | |
224 | case 147: | |
225 | pasid = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]) >> 16; | |
226 | if (!pasid || amdgpu_vm_pasid_fault_credit(adev, pasid)) | |
227 | return true; | |
228 | break; | |
229 | default: | |
230 | /* Not a VM fault */ | |
231 | return true; | |
232 | } | |
233 | ||
234 | adev->irq.ih.rptr += 16; | |
235 | return false; | |
00ecd8a2 FK |
236 | } |
237 | ||
aaa36a97 AD |
238 | /** |
239 | * iceland_ih_decode_iv - decode an interrupt vector | |
240 | * | |
241 | * @adev: amdgpu_device pointer | |
242 | * | |
243 | * Decodes the interrupt vector at the current rptr | |
244 | * position and also advance the position. | |
245 | */ | |
246 | static void iceland_ih_decode_iv(struct amdgpu_device *adev, | |
247 | struct amdgpu_iv_entry *entry) | |
248 | { | |
249 | /* wptr/rptr are in bytes! */ | |
250 | u32 ring_index = adev->irq.ih.rptr >> 2; | |
251 | uint32_t dw[4]; | |
252 | ||
253 | dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]); | |
254 | dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]); | |
255 | dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]); | |
256 | dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]); | |
257 | ||
d766e6a3 | 258 | entry->client_id = AMDGPU_IH_CLIENTID_LEGACY; |
aaa36a97 | 259 | entry->src_id = dw[0] & 0xff; |
7ccf5aa8 | 260 | entry->src_data[0] = dw[1] & 0xfffffff; |
aaa36a97 | 261 | entry->ring_id = dw[2] & 0xff; |
c4f46f22 | 262 | entry->vmid = (dw[2] >> 8) & 0xff; |
3816e42f | 263 | entry->pasid = (dw[2] >> 16) & 0xffff; |
aaa36a97 AD |
264 | |
265 | /* wptr/rptr are in bytes! */ | |
266 | adev->irq.ih.rptr += 16; | |
267 | } | |
268 | ||
269 | /** | |
270 | * iceland_ih_set_rptr - set the IH ring buffer rptr | |
271 | * | |
272 | * @adev: amdgpu_device pointer | |
273 | * | |
274 | * Set the IH ring buffer rptr. | |
275 | */ | |
276 | static void iceland_ih_set_rptr(struct amdgpu_device *adev) | |
277 | { | |
278 | WREG32(mmIH_RB_RPTR, adev->irq.ih.rptr); | |
279 | } | |
280 | ||
5fc3aeeb | 281 | static int iceland_ih_early_init(void *handle) |
aaa36a97 | 282 | { |
5fc3aeeb | 283 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
5f232365 AD |
284 | int ret; |
285 | ||
286 | ret = amdgpu_irq_add_domain(adev); | |
287 | if (ret) | |
288 | return ret; | |
5fc3aeeb | 289 | |
aaa36a97 | 290 | iceland_ih_set_interrupt_funcs(adev); |
5f232365 | 291 | |
aaa36a97 AD |
292 | return 0; |
293 | } | |
294 | ||
5fc3aeeb | 295 | static int iceland_ih_sw_init(void *handle) |
aaa36a97 AD |
296 | { |
297 | int r; | |
5fc3aeeb | 298 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
aaa36a97 AD |
299 | |
300 | r = amdgpu_ih_ring_init(adev, 64 * 1024, false); | |
301 | if (r) | |
302 | return r; | |
303 | ||
304 | r = amdgpu_irq_init(adev); | |
305 | ||
306 | return r; | |
307 | } | |
308 | ||
5fc3aeeb | 309 | static int iceland_ih_sw_fini(void *handle) |
aaa36a97 | 310 | { |
5fc3aeeb | 311 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
312 | ||
aaa36a97 AD |
313 | amdgpu_irq_fini(adev); |
314 | amdgpu_ih_ring_fini(adev); | |
5f232365 | 315 | amdgpu_irq_remove_domain(adev); |
aaa36a97 AD |
316 | |
317 | return 0; | |
318 | } | |
319 | ||
5fc3aeeb | 320 | static int iceland_ih_hw_init(void *handle) |
aaa36a97 AD |
321 | { |
322 | int r; | |
5fc3aeeb | 323 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
aaa36a97 AD |
324 | |
325 | r = iceland_ih_irq_init(adev); | |
326 | if (r) | |
327 | return r; | |
328 | ||
329 | return 0; | |
330 | } | |
331 | ||
5fc3aeeb | 332 | static int iceland_ih_hw_fini(void *handle) |
aaa36a97 | 333 | { |
5fc3aeeb | 334 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
335 | ||
aaa36a97 AD |
336 | iceland_ih_irq_disable(adev); |
337 | ||
338 | return 0; | |
339 | } | |
340 | ||
5fc3aeeb | 341 | static int iceland_ih_suspend(void *handle) |
aaa36a97 | 342 | { |
5fc3aeeb | 343 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
344 | ||
aaa36a97 AD |
345 | return iceland_ih_hw_fini(adev); |
346 | } | |
347 | ||
5fc3aeeb | 348 | static int iceland_ih_resume(void *handle) |
aaa36a97 | 349 | { |
5fc3aeeb | 350 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
351 | ||
aaa36a97 AD |
352 | return iceland_ih_hw_init(adev); |
353 | } | |
354 | ||
5fc3aeeb | 355 | static bool iceland_ih_is_idle(void *handle) |
aaa36a97 | 356 | { |
5fc3aeeb | 357 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
aaa36a97 AD |
358 | u32 tmp = RREG32(mmSRBM_STATUS); |
359 | ||
360 | if (REG_GET_FIELD(tmp, SRBM_STATUS, IH_BUSY)) | |
361 | return false; | |
362 | ||
363 | return true; | |
364 | } | |
365 | ||
5fc3aeeb | 366 | static int iceland_ih_wait_for_idle(void *handle) |
aaa36a97 AD |
367 | { |
368 | unsigned i; | |
369 | u32 tmp; | |
5fc3aeeb | 370 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
aaa36a97 AD |
371 | |
372 | for (i = 0; i < adev->usec_timeout; i++) { | |
373 | /* read MC_STATUS */ | |
374 | tmp = RREG32(mmSRBM_STATUS); | |
375 | if (!REG_GET_FIELD(tmp, SRBM_STATUS, IH_BUSY)) | |
376 | return 0; | |
377 | udelay(1); | |
378 | } | |
379 | return -ETIMEDOUT; | |
380 | } | |
381 | ||
5fc3aeeb | 382 | static int iceland_ih_soft_reset(void *handle) |
aaa36a97 AD |
383 | { |
384 | u32 srbm_soft_reset = 0; | |
5fc3aeeb | 385 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
aaa36a97 AD |
386 | u32 tmp = RREG32(mmSRBM_STATUS); |
387 | ||
388 | if (tmp & SRBM_STATUS__IH_BUSY_MASK) | |
389 | srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, | |
390 | SOFT_RESET_IH, 1); | |
391 | ||
392 | if (srbm_soft_reset) { | |
aaa36a97 AD |
393 | tmp = RREG32(mmSRBM_SOFT_RESET); |
394 | tmp |= srbm_soft_reset; | |
395 | dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); | |
396 | WREG32(mmSRBM_SOFT_RESET, tmp); | |
397 | tmp = RREG32(mmSRBM_SOFT_RESET); | |
398 | ||
399 | udelay(50); | |
400 | ||
401 | tmp &= ~srbm_soft_reset; | |
402 | WREG32(mmSRBM_SOFT_RESET, tmp); | |
403 | tmp = RREG32(mmSRBM_SOFT_RESET); | |
404 | ||
405 | /* Wait a little for things to settle down */ | |
406 | udelay(50); | |
aaa36a97 AD |
407 | } |
408 | ||
409 | return 0; | |
410 | } | |
411 | ||
5fc3aeeb | 412 | static int iceland_ih_set_clockgating_state(void *handle, |
413 | enum amd_clockgating_state state) | |
aaa36a97 | 414 | { |
aaa36a97 AD |
415 | return 0; |
416 | } | |
417 | ||
5fc3aeeb | 418 | static int iceland_ih_set_powergating_state(void *handle, |
419 | enum amd_powergating_state state) | |
aaa36a97 | 420 | { |
aaa36a97 AD |
421 | return 0; |
422 | } | |
423 | ||
a1255107 | 424 | static const struct amd_ip_funcs iceland_ih_ip_funcs = { |
88a907d6 | 425 | .name = "iceland_ih", |
aaa36a97 AD |
426 | .early_init = iceland_ih_early_init, |
427 | .late_init = NULL, | |
428 | .sw_init = iceland_ih_sw_init, | |
429 | .sw_fini = iceland_ih_sw_fini, | |
430 | .hw_init = iceland_ih_hw_init, | |
431 | .hw_fini = iceland_ih_hw_fini, | |
432 | .suspend = iceland_ih_suspend, | |
433 | .resume = iceland_ih_resume, | |
434 | .is_idle = iceland_ih_is_idle, | |
435 | .wait_for_idle = iceland_ih_wait_for_idle, | |
436 | .soft_reset = iceland_ih_soft_reset, | |
aaa36a97 AD |
437 | .set_clockgating_state = iceland_ih_set_clockgating_state, |
438 | .set_powergating_state = iceland_ih_set_powergating_state, | |
439 | }; | |
440 | ||
441 | static const struct amdgpu_ih_funcs iceland_ih_funcs = { | |
442 | .get_wptr = iceland_ih_get_wptr, | |
00ecd8a2 | 443 | .prescreen_iv = iceland_ih_prescreen_iv, |
aaa36a97 AD |
444 | .decode_iv = iceland_ih_decode_iv, |
445 | .set_rptr = iceland_ih_set_rptr | |
446 | }; | |
447 | ||
448 | static void iceland_ih_set_interrupt_funcs(struct amdgpu_device *adev) | |
449 | { | |
450 | if (adev->irq.ih_funcs == NULL) | |
451 | adev->irq.ih_funcs = &iceland_ih_funcs; | |
452 | } | |
453 | ||
a1255107 AD |
454 | const struct amdgpu_ip_block_version iceland_ih_ip_block = |
455 | { | |
456 | .type = AMD_IP_BLOCK_TYPE_IH, | |
457 | .major = 2, | |
458 | .minor = 4, | |
459 | .rev = 0, | |
460 | .funcs = &iceland_ih_ip_funcs, | |
461 | }; |