]>
Commit | Line | Data |
---|---|---|
edc61147 HZ |
1 | /* |
2 | * Copyright 2019 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | */ | |
23 | ||
b23b2e9e AD |
24 | #include <linux/pci.h> |
25 | ||
edc61147 HZ |
26 | #include "amdgpu.h" |
27 | #include "amdgpu_ih.h" | |
28 | ||
29 | #include "oss/osssys_5_0_0_offset.h" | |
30 | #include "oss/osssys_5_0_0_sh_mask.h" | |
31 | ||
32 | #include "soc15_common.h" | |
33 | #include "navi10_ih.h" | |
34 | ||
022b6518 | 35 | #define MAX_REARM_RETRY 10 |
edc61147 HZ |
36 | |
37 | static void navi10_ih_set_interrupt_funcs(struct amdgpu_device *adev); | |
38 | ||
39 | /** | |
40 | * navi10_ih_enable_interrupts - Enable the interrupt ring buffer | |
41 | * | |
42 | * @adev: amdgpu_device pointer | |
43 | * | |
44 | * Enable the interrupt ring buffer (NAVI10). | |
45 | */ | |
46 | static void navi10_ih_enable_interrupts(struct amdgpu_device *adev) | |
47 | { | |
48 | u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL); | |
49 | ||
50 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1); | |
51 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1); | |
193cce34 | 52 | if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { |
0ab176e6 AS |
53 | if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) { |
54 | DRM_ERROR("PSP program IH_RB_CNTL failed!\n"); | |
55 | return; | |
56 | } | |
57 | } else { | |
58 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl); | |
59 | } | |
60 | ||
edc61147 | 61 | adev->irq.ih.enabled = true; |
ab518012 AS |
62 | |
63 | if (adev->irq.ih1.ring_size) { | |
64 | ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); | |
65 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1, | |
66 | RB_ENABLE, 1); | |
193cce34 | 67 | if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { |
0ab176e6 AS |
68 | if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, |
69 | ih_rb_cntl)) { | |
70 | DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n"); | |
71 | return; | |
72 | } | |
73 | } else { | |
74 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); | |
75 | } | |
ab518012 AS |
76 | adev->irq.ih1.enabled = true; |
77 | } | |
78 | ||
79 | if (adev->irq.ih2.ring_size) { | |
80 | ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2); | |
81 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2, | |
82 | RB_ENABLE, 1); | |
193cce34 | 83 | if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { |
0ab176e6 AS |
84 | if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2, |
85 | ih_rb_cntl)) { | |
86 | DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n"); | |
87 | return; | |
88 | } | |
89 | } else { | |
90 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); | |
91 | } | |
ab518012 AS |
92 | adev->irq.ih2.enabled = true; |
93 | } | |
edc61147 HZ |
94 | } |
95 | ||
96 | /** | |
97 | * navi10_ih_disable_interrupts - Disable the interrupt ring buffer | |
98 | * | |
99 | * @adev: amdgpu_device pointer | |
100 | * | |
101 | * Disable the interrupt ring buffer (NAVI10). | |
102 | */ | |
103 | static void navi10_ih_disable_interrupts(struct amdgpu_device *adev) | |
104 | { | |
105 | u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL); | |
106 | ||
107 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0); | |
108 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0); | |
193cce34 | 109 | if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { |
0ab176e6 AS |
110 | if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) { |
111 | DRM_ERROR("PSP program IH_RB_CNTL failed!\n"); | |
112 | return; | |
113 | } | |
114 | } else { | |
115 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl); | |
116 | } | |
117 | ||
edc61147 HZ |
118 | /* set rptr, wptr to 0 */ |
119 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0); | |
120 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0); | |
121 | adev->irq.ih.enabled = false; | |
122 | adev->irq.ih.rptr = 0; | |
ab518012 AS |
123 | |
124 | if (adev->irq.ih1.ring_size) { | |
125 | ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); | |
126 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1, | |
127 | RB_ENABLE, 0); | |
193cce34 | 128 | if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { |
0ab176e6 AS |
129 | if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, |
130 | ih_rb_cntl)) { | |
131 | DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n"); | |
132 | return; | |
133 | } | |
134 | } else { | |
135 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); | |
136 | } | |
ab518012 AS |
137 | /* set rptr, wptr to 0 */ |
138 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0); | |
139 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0); | |
140 | adev->irq.ih1.enabled = false; | |
141 | adev->irq.ih1.rptr = 0; | |
142 | } | |
143 | ||
144 | if (adev->irq.ih2.ring_size) { | |
145 | ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2); | |
146 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2, | |
147 | RB_ENABLE, 0); | |
193cce34 | 148 | if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { |
0ab176e6 AS |
149 | if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2, |
150 | ih_rb_cntl)) { | |
151 | DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n"); | |
152 | return; | |
153 | } | |
154 | } else { | |
155 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); | |
156 | } | |
ab518012 AS |
157 | /* set rptr, wptr to 0 */ |
158 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0); | |
159 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0); | |
160 | adev->irq.ih2.enabled = false; | |
161 | adev->irq.ih2.rptr = 0; | |
162 | } | |
163 | ||
edc61147 HZ |
164 | } |
165 | ||
166 | static uint32_t navi10_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl) | |
167 | { | |
168 | int rb_bufsz = order_base_2(ih->ring_size / 4); | |
169 | ||
170 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, | |
171 | MC_SPACE, ih->use_bus_addr ? 1 : 4); | |
172 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, | |
173 | WPTR_OVERFLOW_CLEAR, 1); | |
174 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, | |
175 | WPTR_OVERFLOW_ENABLE, 1); | |
176 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz); | |
177 | /* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register | |
178 | * value is written to memory | |
179 | */ | |
180 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, | |
181 | WPTR_WRITEBACK_ENABLE, 1); | |
182 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1); | |
183 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0); | |
184 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0); | |
185 | ||
186 | return ih_rb_cntl; | |
187 | } | |
188 | ||
ab518012 AS |
189 | static uint32_t navi10_ih_doorbell_rptr(struct amdgpu_ih_ring *ih) |
190 | { | |
191 | u32 ih_doorbell_rtpr = 0; | |
192 | ||
193 | if (ih->use_doorbell) { | |
194 | ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, | |
195 | IH_DOORBELL_RPTR, OFFSET, | |
196 | ih->doorbell_index); | |
197 | ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, | |
198 | IH_DOORBELL_RPTR, | |
199 | ENABLE, 1); | |
200 | } else { | |
201 | ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, | |
202 | IH_DOORBELL_RPTR, | |
203 | ENABLE, 0); | |
204 | } | |
205 | return ih_doorbell_rtpr; | |
206 | } | |
207 | ||
9e94ff33 AS |
208 | static void navi10_ih_reroute_ih(struct amdgpu_device *adev) |
209 | { | |
210 | uint32_t tmp; | |
211 | ||
212 | /* Reroute to IH ring 1 for VMC */ | |
213 | WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_INDEX, 0x12); | |
214 | tmp = RREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA); | |
215 | tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, CLIENT_TYPE, 1); | |
216 | tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1); | |
217 | WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA, tmp); | |
218 | ||
219 | /* Reroute IH ring 1 for UMC */ | |
220 | WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_INDEX, 0x1B); | |
221 | tmp = RREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA); | |
222 | tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1); | |
223 | WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA, tmp); | |
224 | } | |
225 | ||
edc61147 HZ |
226 | /** |
227 | * navi10_ih_irq_init - init and enable the interrupt ring | |
228 | * | |
229 | * @adev: amdgpu_device pointer | |
230 | * | |
231 | * Allocate a ring buffer for the interrupt controller, | |
232 | * enable the RLC, disable interrupts, enable the IH | |
233 | * ring buffer and enable it (NAVI). | |
234 | * Called at device load and reume. | |
235 | * Returns 0 for success, errors for failure. | |
236 | */ | |
237 | static int navi10_ih_irq_init(struct amdgpu_device *adev) | |
238 | { | |
239 | struct amdgpu_ih_ring *ih = &adev->irq.ih; | |
ab518012 | 240 | u32 ih_rb_cntl, ih_chicken; |
edc61147 HZ |
241 | u32 tmp; |
242 | ||
243 | /* disable irqs */ | |
244 | navi10_ih_disable_interrupts(adev); | |
245 | ||
bebc0762 | 246 | adev->nbio.funcs->ih_control(adev); |
edc61147 HZ |
247 | |
248 | /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/ | |
249 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, ih->gpu_addr >> 8); | |
250 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (ih->gpu_addr >> 40) & 0xff); | |
251 | ||
252 | ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL); | |
253 | ih_rb_cntl = navi10_ih_rb_cntl(ih, ih_rb_cntl); | |
254 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, | |
255 | !!adev->irq.msi_enabled); | |
193cce34 | 256 | if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { |
0ab176e6 AS |
257 | if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) { |
258 | DRM_ERROR("PSP program IH_RB_CNTL failed!\n"); | |
259 | return -ETIMEDOUT; | |
260 | } | |
261 | } else { | |
262 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl); | |
263 | } | |
9e94ff33 | 264 | navi10_ih_reroute_ih(adev); |
edc61147 HZ |
265 | |
266 | if (unlikely(adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT)) { | |
267 | if (ih->use_bus_addr) { | |
268 | ih_chicken = RREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN); | |
269 | ih_chicken = REG_SET_FIELD(ih_chicken, | |
270 | IH_CHICKEN, MC_SPACE_GPA_ENABLE, 1); | |
271 | WREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN, ih_chicken); | |
272 | } | |
273 | } | |
274 | ||
edc61147 HZ |
275 | /* set the writeback address whether it's enabled or not */ |
276 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO, | |
277 | lower_32_bits(ih->wptr_addr)); | |
278 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI, | |
279 | upper_32_bits(ih->wptr_addr) & 0xFFFF); | |
280 | ||
281 | /* set rptr, wptr to 0 */ | |
282 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0); | |
283 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0); | |
284 | ||
ab518012 AS |
285 | WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR, |
286 | navi10_ih_doorbell_rptr(ih)); | |
edc61147 | 287 | |
bebc0762 | 288 | adev->nbio.funcs->ih_doorbell_range(adev, ih->use_doorbell, |
edc61147 HZ |
289 | ih->doorbell_index); |
290 | ||
ab518012 AS |
291 | ih = &adev->irq.ih1; |
292 | if (ih->ring_size) { | |
293 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING1, ih->gpu_addr >> 8); | |
294 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING1, | |
295 | (ih->gpu_addr >> 40) & 0xff); | |
296 | ||
297 | ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); | |
298 | ih_rb_cntl = navi10_ih_rb_cntl(ih, ih_rb_cntl); | |
299 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, | |
300 | WPTR_OVERFLOW_ENABLE, 0); | |
301 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, | |
302 | RB_FULL_DRAIN_ENABLE, 1); | |
193cce34 | 303 | if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { |
0ab176e6 AS |
304 | if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, |
305 | ih_rb_cntl)) { | |
306 | DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n"); | |
307 | return -ETIMEDOUT; | |
308 | } | |
309 | } else { | |
310 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); | |
311 | } | |
ab518012 AS |
312 | /* set rptr, wptr to 0 */ |
313 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0); | |
314 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0); | |
315 | ||
316 | WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING1, | |
317 | navi10_ih_doorbell_rptr(ih)); | |
318 | } | |
319 | ||
320 | ih = &adev->irq.ih2; | |
321 | if (ih->ring_size) { | |
322 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING2, ih->gpu_addr >> 8); | |
323 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING2, | |
324 | (ih->gpu_addr >> 40) & 0xff); | |
325 | ||
326 | ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2); | |
327 | ih_rb_cntl = navi10_ih_rb_cntl(ih, ih_rb_cntl); | |
328 | ||
193cce34 | 329 | if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { |
0ab176e6 AS |
330 | if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2, |
331 | ih_rb_cntl)) { | |
332 | DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n"); | |
333 | return -ETIMEDOUT; | |
334 | } | |
335 | } else { | |
336 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); | |
337 | } | |
ab518012 AS |
338 | /* set rptr, wptr to 0 */ |
339 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0); | |
340 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0); | |
341 | ||
342 | WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING2, | |
343 | navi10_ih_doorbell_rptr(ih)); | |
344 | } | |
345 | ||
346 | ||
edc61147 HZ |
347 | tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL); |
348 | tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL, | |
349 | CLIENT18_IS_STORM_CLIENT, 1); | |
350 | WREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL, tmp); | |
351 | ||
352 | tmp = RREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL); | |
353 | tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1); | |
354 | WREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL, tmp); | |
355 | ||
356 | pci_set_master(adev->pdev); | |
357 | ||
358 | /* enable interrupts */ | |
359 | navi10_ih_enable_interrupts(adev); | |
360 | ||
7eca4006 | 361 | return 0; |
edc61147 HZ |
362 | } |
363 | ||
364 | /** | |
365 | * navi10_ih_irq_disable - disable interrupts | |
366 | * | |
367 | * @adev: amdgpu_device pointer | |
368 | * | |
369 | * Disable interrupts on the hw (NAVI10). | |
370 | */ | |
371 | static void navi10_ih_irq_disable(struct amdgpu_device *adev) | |
372 | { | |
373 | navi10_ih_disable_interrupts(adev); | |
374 | ||
375 | /* Wait and acknowledge irq */ | |
376 | mdelay(1); | |
377 | } | |
378 | ||
379 | /** | |
380 | * navi10_ih_get_wptr - get the IH ring buffer wptr | |
381 | * | |
382 | * @adev: amdgpu_device pointer | |
383 | * | |
384 | * Get the IH ring buffer wptr from either the register | |
385 | * or the writeback memory buffer (NAVI10). Also check for | |
386 | * ring buffer overflow and deal with it. | |
387 | * Returns the value of the wptr. | |
388 | */ | |
389 | static u32 navi10_ih_get_wptr(struct amdgpu_device *adev, | |
390 | struct amdgpu_ih_ring *ih) | |
391 | { | |
392 | u32 wptr, reg, tmp; | |
393 | ||
394 | wptr = le32_to_cpu(*ih->wptr_cpu); | |
395 | ||
396 | if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) | |
397 | goto out; | |
398 | ||
ab518012 AS |
399 | if (ih == &adev->irq.ih) |
400 | reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR); | |
401 | else if (ih == &adev->irq.ih1) | |
402 | reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING1); | |
403 | else if (ih == &adev->irq.ih2) | |
404 | reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING2); | |
405 | else | |
406 | BUG(); | |
407 | ||
edc61147 HZ |
408 | wptr = RREG32_NO_KIQ(reg); |
409 | if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) | |
410 | goto out; | |
411 | wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); | |
412 | ||
413 | /* When a ring buffer overflow happen start parsing interrupt | |
414 | * from the last not overwritten vector (wptr + 32). Hopefully | |
415 | * this should allow us to catch up. | |
416 | */ | |
417 | tmp = (wptr + 32) & ih->ptr_mask; | |
418 | dev_warn(adev->dev, "IH ring buffer overflow " | |
419 | "(0x%08X, 0x%08X, 0x%08X)\n", | |
420 | wptr, ih->rptr, tmp); | |
421 | ih->rptr = tmp; | |
422 | ||
ab518012 AS |
423 | if (ih == &adev->irq.ih) |
424 | reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL); | |
425 | else if (ih == &adev->irq.ih1) | |
426 | reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING1); | |
427 | else if (ih == &adev->irq.ih2) | |
428 | reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING2); | |
429 | else | |
430 | BUG(); | |
431 | ||
edc61147 HZ |
432 | tmp = RREG32_NO_KIQ(reg); |
433 | tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); | |
434 | WREG32_NO_KIQ(reg, tmp); | |
435 | out: | |
436 | return (wptr & ih->ptr_mask); | |
437 | } | |
438 | ||
439 | /** | |
440 | * navi10_ih_decode_iv - decode an interrupt vector | |
441 | * | |
442 | * @adev: amdgpu_device pointer | |
443 | * | |
444 | * Decodes the interrupt vector at the current rptr | |
445 | * position and also advance the position. | |
446 | */ | |
447 | static void navi10_ih_decode_iv(struct amdgpu_device *adev, | |
448 | struct amdgpu_ih_ring *ih, | |
449 | struct amdgpu_iv_entry *entry) | |
450 | { | |
451 | /* wptr/rptr are in bytes! */ | |
452 | u32 ring_index = ih->rptr >> 2; | |
453 | uint32_t dw[8]; | |
454 | ||
455 | dw[0] = le32_to_cpu(ih->ring[ring_index + 0]); | |
456 | dw[1] = le32_to_cpu(ih->ring[ring_index + 1]); | |
457 | dw[2] = le32_to_cpu(ih->ring[ring_index + 2]); | |
458 | dw[3] = le32_to_cpu(ih->ring[ring_index + 3]); | |
459 | dw[4] = le32_to_cpu(ih->ring[ring_index + 4]); | |
460 | dw[5] = le32_to_cpu(ih->ring[ring_index + 5]); | |
461 | dw[6] = le32_to_cpu(ih->ring[ring_index + 6]); | |
462 | dw[7] = le32_to_cpu(ih->ring[ring_index + 7]); | |
463 | ||
464 | entry->client_id = dw[0] & 0xff; | |
465 | entry->src_id = (dw[0] >> 8) & 0xff; | |
466 | entry->ring_id = (dw[0] >> 16) & 0xff; | |
467 | entry->vmid = (dw[0] >> 24) & 0xf; | |
468 | entry->vmid_src = (dw[0] >> 31); | |
469 | entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32); | |
470 | entry->timestamp_src = dw[2] >> 31; | |
471 | entry->pasid = dw[3] & 0xffff; | |
472 | entry->pasid_src = dw[3] >> 31; | |
473 | entry->src_data[0] = dw[4]; | |
474 | entry->src_data[1] = dw[5]; | |
475 | entry->src_data[2] = dw[6]; | |
476 | entry->src_data[3] = dw[7]; | |
477 | ||
478 | /* wptr/rptr are in bytes! */ | |
479 | ih->rptr += 32; | |
480 | } | |
481 | ||
022b6518 SD |
482 | /** |
483 | * navi10_ih_irq_rearm - rearm IRQ if lost | |
484 | * | |
485 | * @adev: amdgpu_device pointer | |
486 | * | |
487 | */ | |
488 | static void navi10_ih_irq_rearm(struct amdgpu_device *adev, | |
489 | struct amdgpu_ih_ring *ih) | |
490 | { | |
491 | uint32_t reg_rptr = 0; | |
492 | uint32_t v = 0; | |
493 | uint32_t i = 0; | |
494 | ||
495 | if (ih == &adev->irq.ih) | |
496 | reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR); | |
497 | else if (ih == &adev->irq.ih1) | |
498 | reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING1); | |
499 | else if (ih == &adev->irq.ih2) | |
500 | reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING2); | |
501 | else | |
502 | return; | |
503 | ||
504 | /* Rearm IRQ / re-write doorbell if doorbell write is lost */ | |
505 | for (i = 0; i < MAX_REARM_RETRY; i++) { | |
506 | v = RREG32_NO_KIQ(reg_rptr); | |
507 | if ((v < ih->ring_size) && (v != ih->rptr)) | |
508 | WDOORBELL32(ih->doorbell_index, ih->rptr); | |
509 | else | |
510 | break; | |
511 | } | |
512 | } | |
513 | ||
edc61147 HZ |
514 | /** |
515 | * navi10_ih_set_rptr - set the IH ring buffer rptr | |
516 | * | |
517 | * @adev: amdgpu_device pointer | |
518 | * | |
519 | * Set the IH ring buffer rptr. | |
520 | */ | |
521 | static void navi10_ih_set_rptr(struct amdgpu_device *adev, | |
522 | struct amdgpu_ih_ring *ih) | |
523 | { | |
524 | if (ih->use_doorbell) { | |
525 | /* XXX check if swapping is necessary on BE */ | |
526 | *ih->rptr_cpu = ih->rptr; | |
527 | WDOORBELL32(ih->doorbell_index, ih->rptr); | |
022b6518 SD |
528 | |
529 | if (amdgpu_sriov_vf(adev)) | |
530 | navi10_ih_irq_rearm(adev, ih); | |
ab518012 | 531 | } else if (ih == &adev->irq.ih) { |
edc61147 | 532 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, ih->rptr); |
ab518012 AS |
533 | } else if (ih == &adev->irq.ih1) { |
534 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, ih->rptr); | |
535 | } else if (ih == &adev->irq.ih2) { | |
536 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, ih->rptr); | |
537 | } | |
538 | } | |
539 | ||
540 | /** | |
541 | * navi10_ih_self_irq - dispatch work for ring 1 and 2 | |
542 | * | |
543 | * @adev: amdgpu_device pointer | |
544 | * @source: irq source | |
545 | * @entry: IV with WPTR update | |
546 | * | |
547 | * Update the WPTR from the IV and schedule work to handle the entries. | |
548 | */ | |
549 | static int navi10_ih_self_irq(struct amdgpu_device *adev, | |
550 | struct amdgpu_irq_src *source, | |
551 | struct amdgpu_iv_entry *entry) | |
552 | { | |
553 | uint32_t wptr = cpu_to_le32(entry->src_data[0]); | |
554 | ||
555 | switch (entry->ring_id) { | |
556 | case 1: | |
557 | *adev->irq.ih1.wptr_cpu = wptr; | |
558 | schedule_work(&adev->irq.ih1_work); | |
559 | break; | |
560 | case 2: | |
561 | *adev->irq.ih2.wptr_cpu = wptr; | |
562 | schedule_work(&adev->irq.ih2_work); | |
563 | break; | |
564 | default: break; | |
565 | } | |
566 | return 0; | |
567 | } | |
568 | ||
569 | static const struct amdgpu_irq_src_funcs navi10_ih_self_irq_funcs = { | |
570 | .process = navi10_ih_self_irq, | |
571 | }; | |
572 | ||
573 | static void navi10_ih_set_self_irq_funcs(struct amdgpu_device *adev) | |
574 | { | |
575 | adev->irq.self_irq.num_types = 0; | |
576 | adev->irq.self_irq.funcs = &navi10_ih_self_irq_funcs; | |
edc61147 HZ |
577 | } |
578 | ||
579 | static int navi10_ih_early_init(void *handle) | |
580 | { | |
581 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
582 | ||
583 | navi10_ih_set_interrupt_funcs(adev); | |
ab518012 | 584 | navi10_ih_set_self_irq_funcs(adev); |
edc61147 HZ |
585 | return 0; |
586 | } | |
587 | ||
588 | static int navi10_ih_sw_init(void *handle) | |
589 | { | |
590 | int r; | |
591 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
592 | bool use_bus_addr; | |
593 | ||
ab518012 AS |
594 | r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_IH, 0, |
595 | &adev->irq.self_irq); | |
596 | ||
597 | if (r) | |
598 | return r; | |
599 | ||
edc61147 HZ |
600 | /* use gpu virtual address for ih ring |
601 | * until ih_checken is programmed to allow | |
602 | * use bus address for ih ring by psp bl */ | |
603 | use_bus_addr = | |
604 | (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) ? false : true; | |
605 | r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 256 * 1024, use_bus_addr); | |
606 | if (r) | |
607 | return r; | |
608 | ||
609 | adev->irq.ih.use_doorbell = true; | |
610 | adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1; | |
611 | ||
ab518012 AS |
612 | r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, PAGE_SIZE, true); |
613 | if (r) | |
614 | return r; | |
615 | ||
616 | adev->irq.ih1.use_doorbell = true; | |
617 | adev->irq.ih1.doorbell_index = (adev->doorbell_index.ih + 1) << 1; | |
618 | ||
619 | r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true); | |
620 | if (r) | |
621 | return r; | |
622 | ||
623 | adev->irq.ih2.use_doorbell = true; | |
624 | adev->irq.ih2.doorbell_index = (adev->doorbell_index.ih + 2) << 1; | |
625 | ||
edc61147 HZ |
626 | r = amdgpu_irq_init(adev); |
627 | ||
628 | return r; | |
629 | } | |
630 | ||
631 | static int navi10_ih_sw_fini(void *handle) | |
632 | { | |
633 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
634 | ||
635 | amdgpu_irq_fini(adev); | |
ab518012 AS |
636 | amdgpu_ih_ring_fini(adev, &adev->irq.ih2); |
637 | amdgpu_ih_ring_fini(adev, &adev->irq.ih1); | |
edc61147 HZ |
638 | amdgpu_ih_ring_fini(adev, &adev->irq.ih); |
639 | ||
640 | return 0; | |
641 | } | |
642 | ||
643 | static int navi10_ih_hw_init(void *handle) | |
644 | { | |
645 | int r; | |
646 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
647 | ||
648 | r = navi10_ih_irq_init(adev); | |
649 | if (r) | |
650 | return r; | |
651 | ||
652 | return 0; | |
653 | } | |
654 | ||
655 | static int navi10_ih_hw_fini(void *handle) | |
656 | { | |
657 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
658 | ||
659 | navi10_ih_irq_disable(adev); | |
660 | ||
661 | return 0; | |
662 | } | |
663 | ||
664 | static int navi10_ih_suspend(void *handle) | |
665 | { | |
666 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
667 | ||
668 | return navi10_ih_hw_fini(adev); | |
669 | } | |
670 | ||
671 | static int navi10_ih_resume(void *handle) | |
672 | { | |
673 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
674 | ||
675 | return navi10_ih_hw_init(adev); | |
676 | } | |
677 | ||
678 | static bool navi10_ih_is_idle(void *handle) | |
679 | { | |
680 | /* todo */ | |
681 | return true; | |
682 | } | |
683 | ||
684 | static int navi10_ih_wait_for_idle(void *handle) | |
685 | { | |
686 | /* todo */ | |
687 | return -ETIMEDOUT; | |
688 | } | |
689 | ||
690 | static int navi10_ih_soft_reset(void *handle) | |
691 | { | |
692 | /* todo */ | |
693 | return 0; | |
694 | } | |
695 | ||
696 | static void navi10_ih_update_clockgating_state(struct amdgpu_device *adev, | |
697 | bool enable) | |
698 | { | |
699 | uint32_t data, def, field_val; | |
700 | ||
701 | if (adev->cg_flags & AMD_CG_SUPPORT_IH_CG) { | |
702 | def = data = RREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL); | |
703 | field_val = enable ? 0 : 1; | |
704 | data = REG_SET_FIELD(data, IH_CLK_CTRL, | |
705 | DBUS_MUX_CLK_SOFT_OVERRIDE, field_val); | |
706 | data = REG_SET_FIELD(data, IH_CLK_CTRL, | |
707 | OSSSYS_SHARE_CLK_SOFT_OVERRIDE, field_val); | |
708 | data = REG_SET_FIELD(data, IH_CLK_CTRL, | |
709 | LIMIT_SMN_CLK_SOFT_OVERRIDE, field_val); | |
710 | data = REG_SET_FIELD(data, IH_CLK_CTRL, | |
711 | DYN_CLK_SOFT_OVERRIDE, field_val); | |
712 | data = REG_SET_FIELD(data, IH_CLK_CTRL, | |
713 | REG_CLK_SOFT_OVERRIDE, field_val); | |
714 | if (def != data) | |
715 | WREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL, data); | |
716 | } | |
717 | ||
718 | return; | |
719 | } | |
720 | ||
721 | static int navi10_ih_set_clockgating_state(void *handle, | |
722 | enum amd_clockgating_state state) | |
723 | { | |
724 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
725 | ||
726 | navi10_ih_update_clockgating_state(adev, | |
a9d4fe2f | 727 | state == AMD_CG_STATE_GATE); |
edc61147 HZ |
728 | return 0; |
729 | } | |
730 | ||
731 | static int navi10_ih_set_powergating_state(void *handle, | |
732 | enum amd_powergating_state state) | |
733 | { | |
734 | return 0; | |
735 | } | |
736 | ||
737 | static void navi10_ih_get_clockgating_state(void *handle, u32 *flags) | |
738 | { | |
739 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
740 | ||
741 | if (!RREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL)) | |
742 | *flags |= AMD_CG_SUPPORT_IH_CG; | |
743 | ||
744 | return; | |
745 | } | |
746 | ||
747 | static const struct amd_ip_funcs navi10_ih_ip_funcs = { | |
748 | .name = "navi10_ih", | |
749 | .early_init = navi10_ih_early_init, | |
750 | .late_init = NULL, | |
751 | .sw_init = navi10_ih_sw_init, | |
752 | .sw_fini = navi10_ih_sw_fini, | |
753 | .hw_init = navi10_ih_hw_init, | |
754 | .hw_fini = navi10_ih_hw_fini, | |
755 | .suspend = navi10_ih_suspend, | |
756 | .resume = navi10_ih_resume, | |
757 | .is_idle = navi10_ih_is_idle, | |
758 | .wait_for_idle = navi10_ih_wait_for_idle, | |
759 | .soft_reset = navi10_ih_soft_reset, | |
760 | .set_clockgating_state = navi10_ih_set_clockgating_state, | |
761 | .set_powergating_state = navi10_ih_set_powergating_state, | |
762 | .get_clockgating_state = navi10_ih_get_clockgating_state, | |
763 | }; | |
764 | ||
765 | static const struct amdgpu_ih_funcs navi10_ih_funcs = { | |
766 | .get_wptr = navi10_ih_get_wptr, | |
767 | .decode_iv = navi10_ih_decode_iv, | |
768 | .set_rptr = navi10_ih_set_rptr | |
769 | }; | |
770 | ||
771 | static void navi10_ih_set_interrupt_funcs(struct amdgpu_device *adev) | |
772 | { | |
773 | if (adev->irq.ih_funcs == NULL) | |
774 | adev->irq.ih_funcs = &navi10_ih_funcs; | |
775 | } | |
776 | ||
777 | const struct amdgpu_ip_block_version navi10_ih_ip_block = | |
778 | { | |
779 | .type = AMD_IP_BLOCK_TYPE_IH, | |
780 | .major = 5, | |
781 | .minor = 0, | |
782 | .rev = 0, | |
783 | .funcs = &navi10_ih_ip_funcs, | |
784 | }; |