]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/gpu/drm/radeon/cik.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'mmc-v4.15-2' of git://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc
[mirror_ubuntu-bionic-kernel.git] / drivers / gpu / drm / radeon / cik.c
1 /*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include <drm/drmP.h>
28 #include "radeon.h"
29 #include "radeon_asic.h"
30 #include "radeon_audio.h"
31 #include "cikd.h"
32 #include "atom.h"
33 #include "cik_blit_shaders.h"
34 #include "radeon_ucode.h"
35 #include "clearstate_ci.h"
36
37 #define SH_MEM_CONFIG_GFX_DEFAULT \
38 ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED)
39
40 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
44 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
45 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
46 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
47 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
48 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
49
50 MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
51 MODULE_FIRMWARE("radeon/bonaire_me.bin");
52 MODULE_FIRMWARE("radeon/bonaire_ce.bin");
53 MODULE_FIRMWARE("radeon/bonaire_mec.bin");
54 MODULE_FIRMWARE("radeon/bonaire_mc.bin");
55 MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
56 MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
57 MODULE_FIRMWARE("radeon/bonaire_smc.bin");
58 MODULE_FIRMWARE("radeon/bonaire_k_smc.bin");
59
60 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
61 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
62 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
63 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
64 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
65 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
66 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
67 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
68 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
69
70 MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
71 MODULE_FIRMWARE("radeon/hawaii_me.bin");
72 MODULE_FIRMWARE("radeon/hawaii_ce.bin");
73 MODULE_FIRMWARE("radeon/hawaii_mec.bin");
74 MODULE_FIRMWARE("radeon/hawaii_mc.bin");
75 MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
76 MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
77 MODULE_FIRMWARE("radeon/hawaii_smc.bin");
78 MODULE_FIRMWARE("radeon/hawaii_k_smc.bin");
79
80 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
81 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
82 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
83 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
84 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
85 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
86
87 MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
88 MODULE_FIRMWARE("radeon/kaveri_me.bin");
89 MODULE_FIRMWARE("radeon/kaveri_ce.bin");
90 MODULE_FIRMWARE("radeon/kaveri_mec.bin");
91 MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
92 MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
93 MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
94
95 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
96 MODULE_FIRMWARE("radeon/KABINI_me.bin");
97 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
98 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
99 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
100 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
101
102 MODULE_FIRMWARE("radeon/kabini_pfp.bin");
103 MODULE_FIRMWARE("radeon/kabini_me.bin");
104 MODULE_FIRMWARE("radeon/kabini_ce.bin");
105 MODULE_FIRMWARE("radeon/kabini_mec.bin");
106 MODULE_FIRMWARE("radeon/kabini_rlc.bin");
107 MODULE_FIRMWARE("radeon/kabini_sdma.bin");
108
109 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
110 MODULE_FIRMWARE("radeon/MULLINS_me.bin");
111 MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
112 MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
113 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
114 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
115
116 MODULE_FIRMWARE("radeon/mullins_pfp.bin");
117 MODULE_FIRMWARE("radeon/mullins_me.bin");
118 MODULE_FIRMWARE("radeon/mullins_ce.bin");
119 MODULE_FIRMWARE("radeon/mullins_mec.bin");
120 MODULE_FIRMWARE("radeon/mullins_rlc.bin");
121 MODULE_FIRMWARE("radeon/mullins_sdma.bin");
122
123 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
124 extern void r600_ih_ring_fini(struct radeon_device *rdev);
125 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
126 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
127 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
128 extern void sumo_rlc_fini(struct radeon_device *rdev);
129 extern int sumo_rlc_init(struct radeon_device *rdev);
130 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
131 extern void si_rlc_reset(struct radeon_device *rdev);
132 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
133 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
134 extern int cik_sdma_resume(struct radeon_device *rdev);
135 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
136 extern void cik_sdma_fini(struct radeon_device *rdev);
137 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
138 static void cik_rlc_stop(struct radeon_device *rdev);
139 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
140 static void cik_program_aspm(struct radeon_device *rdev);
141 static void cik_init_pg(struct radeon_device *rdev);
142 static void cik_init_cg(struct radeon_device *rdev);
143 static void cik_fini_pg(struct radeon_device *rdev);
144 static void cik_fini_cg(struct radeon_device *rdev);
145 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
146 bool enable);
147
148 /**
149 * cik_get_allowed_info_register - fetch the register for the info ioctl
150 *
151 * @rdev: radeon_device pointer
152 * @reg: register offset in bytes
153 * @val: register value
154 *
155 * Returns 0 for success or -EINVAL for an invalid register
156 *
157 */
158 int cik_get_allowed_info_register(struct radeon_device *rdev,
159 u32 reg, u32 *val)
160 {
161 switch (reg) {
162 case GRBM_STATUS:
163 case GRBM_STATUS2:
164 case GRBM_STATUS_SE0:
165 case GRBM_STATUS_SE1:
166 case GRBM_STATUS_SE2:
167 case GRBM_STATUS_SE3:
168 case SRBM_STATUS:
169 case SRBM_STATUS2:
170 case (SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET):
171 case (SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET):
172 case UVD_STATUS:
173 /* TODO VCE */
174 *val = RREG32(reg);
175 return 0;
176 default:
177 return -EINVAL;
178 }
179 }
180
181 /*
182 * Indirect registers accessor
183 */
184 u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg)
185 {
186 unsigned long flags;
187 u32 r;
188
189 spin_lock_irqsave(&rdev->didt_idx_lock, flags);
190 WREG32(CIK_DIDT_IND_INDEX, (reg));
191 r = RREG32(CIK_DIDT_IND_DATA);
192 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
193 return r;
194 }
195
196 void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v)
197 {
198 unsigned long flags;
199
200 spin_lock_irqsave(&rdev->didt_idx_lock, flags);
201 WREG32(CIK_DIDT_IND_INDEX, (reg));
202 WREG32(CIK_DIDT_IND_DATA, (v));
203 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
204 }
205
206 /* get temperature in millidegrees */
207 int ci_get_temp(struct radeon_device *rdev)
208 {
209 u32 temp;
210 int actual_temp = 0;
211
212 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
213 CTF_TEMP_SHIFT;
214
215 if (temp & 0x200)
216 actual_temp = 255;
217 else
218 actual_temp = temp & 0x1ff;
219
220 actual_temp = actual_temp * 1000;
221
222 return actual_temp;
223 }
224
225 /* get temperature in millidegrees */
226 int kv_get_temp(struct radeon_device *rdev)
227 {
228 u32 temp;
229 int actual_temp = 0;
230
231 temp = RREG32_SMC(0xC0300E0C);
232
233 if (temp)
234 actual_temp = (temp / 8) - 49;
235 else
236 actual_temp = 0;
237
238 actual_temp = actual_temp * 1000;
239
240 return actual_temp;
241 }
242
243 /*
244 * Indirect registers accessor
245 */
246 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
247 {
248 unsigned long flags;
249 u32 r;
250
251 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
252 WREG32(PCIE_INDEX, reg);
253 (void)RREG32(PCIE_INDEX);
254 r = RREG32(PCIE_DATA);
255 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
256 return r;
257 }
258
259 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
260 {
261 unsigned long flags;
262
263 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
264 WREG32(PCIE_INDEX, reg);
265 (void)RREG32(PCIE_INDEX);
266 WREG32(PCIE_DATA, v);
267 (void)RREG32(PCIE_DATA);
268 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
269 }
270
271 static const u32 spectre_rlc_save_restore_register_list[] =
272 {
273 (0x0e00 << 16) | (0xc12c >> 2),
274 0x00000000,
275 (0x0e00 << 16) | (0xc140 >> 2),
276 0x00000000,
277 (0x0e00 << 16) | (0xc150 >> 2),
278 0x00000000,
279 (0x0e00 << 16) | (0xc15c >> 2),
280 0x00000000,
281 (0x0e00 << 16) | (0xc168 >> 2),
282 0x00000000,
283 (0x0e00 << 16) | (0xc170 >> 2),
284 0x00000000,
285 (0x0e00 << 16) | (0xc178 >> 2),
286 0x00000000,
287 (0x0e00 << 16) | (0xc204 >> 2),
288 0x00000000,
289 (0x0e00 << 16) | (0xc2b4 >> 2),
290 0x00000000,
291 (0x0e00 << 16) | (0xc2b8 >> 2),
292 0x00000000,
293 (0x0e00 << 16) | (0xc2bc >> 2),
294 0x00000000,
295 (0x0e00 << 16) | (0xc2c0 >> 2),
296 0x00000000,
297 (0x0e00 << 16) | (0x8228 >> 2),
298 0x00000000,
299 (0x0e00 << 16) | (0x829c >> 2),
300 0x00000000,
301 (0x0e00 << 16) | (0x869c >> 2),
302 0x00000000,
303 (0x0600 << 16) | (0x98f4 >> 2),
304 0x00000000,
305 (0x0e00 << 16) | (0x98f8 >> 2),
306 0x00000000,
307 (0x0e00 << 16) | (0x9900 >> 2),
308 0x00000000,
309 (0x0e00 << 16) | (0xc260 >> 2),
310 0x00000000,
311 (0x0e00 << 16) | (0x90e8 >> 2),
312 0x00000000,
313 (0x0e00 << 16) | (0x3c000 >> 2),
314 0x00000000,
315 (0x0e00 << 16) | (0x3c00c >> 2),
316 0x00000000,
317 (0x0e00 << 16) | (0x8c1c >> 2),
318 0x00000000,
319 (0x0e00 << 16) | (0x9700 >> 2),
320 0x00000000,
321 (0x0e00 << 16) | (0xcd20 >> 2),
322 0x00000000,
323 (0x4e00 << 16) | (0xcd20 >> 2),
324 0x00000000,
325 (0x5e00 << 16) | (0xcd20 >> 2),
326 0x00000000,
327 (0x6e00 << 16) | (0xcd20 >> 2),
328 0x00000000,
329 (0x7e00 << 16) | (0xcd20 >> 2),
330 0x00000000,
331 (0x8e00 << 16) | (0xcd20 >> 2),
332 0x00000000,
333 (0x9e00 << 16) | (0xcd20 >> 2),
334 0x00000000,
335 (0xae00 << 16) | (0xcd20 >> 2),
336 0x00000000,
337 (0xbe00 << 16) | (0xcd20 >> 2),
338 0x00000000,
339 (0x0e00 << 16) | (0x89bc >> 2),
340 0x00000000,
341 (0x0e00 << 16) | (0x8900 >> 2),
342 0x00000000,
343 0x3,
344 (0x0e00 << 16) | (0xc130 >> 2),
345 0x00000000,
346 (0x0e00 << 16) | (0xc134 >> 2),
347 0x00000000,
348 (0x0e00 << 16) | (0xc1fc >> 2),
349 0x00000000,
350 (0x0e00 << 16) | (0xc208 >> 2),
351 0x00000000,
352 (0x0e00 << 16) | (0xc264 >> 2),
353 0x00000000,
354 (0x0e00 << 16) | (0xc268 >> 2),
355 0x00000000,
356 (0x0e00 << 16) | (0xc26c >> 2),
357 0x00000000,
358 (0x0e00 << 16) | (0xc270 >> 2),
359 0x00000000,
360 (0x0e00 << 16) | (0xc274 >> 2),
361 0x00000000,
362 (0x0e00 << 16) | (0xc278 >> 2),
363 0x00000000,
364 (0x0e00 << 16) | (0xc27c >> 2),
365 0x00000000,
366 (0x0e00 << 16) | (0xc280 >> 2),
367 0x00000000,
368 (0x0e00 << 16) | (0xc284 >> 2),
369 0x00000000,
370 (0x0e00 << 16) | (0xc288 >> 2),
371 0x00000000,
372 (0x0e00 << 16) | (0xc28c >> 2),
373 0x00000000,
374 (0x0e00 << 16) | (0xc290 >> 2),
375 0x00000000,
376 (0x0e00 << 16) | (0xc294 >> 2),
377 0x00000000,
378 (0x0e00 << 16) | (0xc298 >> 2),
379 0x00000000,
380 (0x0e00 << 16) | (0xc29c >> 2),
381 0x00000000,
382 (0x0e00 << 16) | (0xc2a0 >> 2),
383 0x00000000,
384 (0x0e00 << 16) | (0xc2a4 >> 2),
385 0x00000000,
386 (0x0e00 << 16) | (0xc2a8 >> 2),
387 0x00000000,
388 (0x0e00 << 16) | (0xc2ac >> 2),
389 0x00000000,
390 (0x0e00 << 16) | (0xc2b0 >> 2),
391 0x00000000,
392 (0x0e00 << 16) | (0x301d0 >> 2),
393 0x00000000,
394 (0x0e00 << 16) | (0x30238 >> 2),
395 0x00000000,
396 (0x0e00 << 16) | (0x30250 >> 2),
397 0x00000000,
398 (0x0e00 << 16) | (0x30254 >> 2),
399 0x00000000,
400 (0x0e00 << 16) | (0x30258 >> 2),
401 0x00000000,
402 (0x0e00 << 16) | (0x3025c >> 2),
403 0x00000000,
404 (0x4e00 << 16) | (0xc900 >> 2),
405 0x00000000,
406 (0x5e00 << 16) | (0xc900 >> 2),
407 0x00000000,
408 (0x6e00 << 16) | (0xc900 >> 2),
409 0x00000000,
410 (0x7e00 << 16) | (0xc900 >> 2),
411 0x00000000,
412 (0x8e00 << 16) | (0xc900 >> 2),
413 0x00000000,
414 (0x9e00 << 16) | (0xc900 >> 2),
415 0x00000000,
416 (0xae00 << 16) | (0xc900 >> 2),
417 0x00000000,
418 (0xbe00 << 16) | (0xc900 >> 2),
419 0x00000000,
420 (0x4e00 << 16) | (0xc904 >> 2),
421 0x00000000,
422 (0x5e00 << 16) | (0xc904 >> 2),
423 0x00000000,
424 (0x6e00 << 16) | (0xc904 >> 2),
425 0x00000000,
426 (0x7e00 << 16) | (0xc904 >> 2),
427 0x00000000,
428 (0x8e00 << 16) | (0xc904 >> 2),
429 0x00000000,
430 (0x9e00 << 16) | (0xc904 >> 2),
431 0x00000000,
432 (0xae00 << 16) | (0xc904 >> 2),
433 0x00000000,
434 (0xbe00 << 16) | (0xc904 >> 2),
435 0x00000000,
436 (0x4e00 << 16) | (0xc908 >> 2),
437 0x00000000,
438 (0x5e00 << 16) | (0xc908 >> 2),
439 0x00000000,
440 (0x6e00 << 16) | (0xc908 >> 2),
441 0x00000000,
442 (0x7e00 << 16) | (0xc908 >> 2),
443 0x00000000,
444 (0x8e00 << 16) | (0xc908 >> 2),
445 0x00000000,
446 (0x9e00 << 16) | (0xc908 >> 2),
447 0x00000000,
448 (0xae00 << 16) | (0xc908 >> 2),
449 0x00000000,
450 (0xbe00 << 16) | (0xc908 >> 2),
451 0x00000000,
452 (0x4e00 << 16) | (0xc90c >> 2),
453 0x00000000,
454 (0x5e00 << 16) | (0xc90c >> 2),
455 0x00000000,
456 (0x6e00 << 16) | (0xc90c >> 2),
457 0x00000000,
458 (0x7e00 << 16) | (0xc90c >> 2),
459 0x00000000,
460 (0x8e00 << 16) | (0xc90c >> 2),
461 0x00000000,
462 (0x9e00 << 16) | (0xc90c >> 2),
463 0x00000000,
464 (0xae00 << 16) | (0xc90c >> 2),
465 0x00000000,
466 (0xbe00 << 16) | (0xc90c >> 2),
467 0x00000000,
468 (0x4e00 << 16) | (0xc910 >> 2),
469 0x00000000,
470 (0x5e00 << 16) | (0xc910 >> 2),
471 0x00000000,
472 (0x6e00 << 16) | (0xc910 >> 2),
473 0x00000000,
474 (0x7e00 << 16) | (0xc910 >> 2),
475 0x00000000,
476 (0x8e00 << 16) | (0xc910 >> 2),
477 0x00000000,
478 (0x9e00 << 16) | (0xc910 >> 2),
479 0x00000000,
480 (0xae00 << 16) | (0xc910 >> 2),
481 0x00000000,
482 (0xbe00 << 16) | (0xc910 >> 2),
483 0x00000000,
484 (0x0e00 << 16) | (0xc99c >> 2),
485 0x00000000,
486 (0x0e00 << 16) | (0x9834 >> 2),
487 0x00000000,
488 (0x0000 << 16) | (0x30f00 >> 2),
489 0x00000000,
490 (0x0001 << 16) | (0x30f00 >> 2),
491 0x00000000,
492 (0x0000 << 16) | (0x30f04 >> 2),
493 0x00000000,
494 (0x0001 << 16) | (0x30f04 >> 2),
495 0x00000000,
496 (0x0000 << 16) | (0x30f08 >> 2),
497 0x00000000,
498 (0x0001 << 16) | (0x30f08 >> 2),
499 0x00000000,
500 (0x0000 << 16) | (0x30f0c >> 2),
501 0x00000000,
502 (0x0001 << 16) | (0x30f0c >> 2),
503 0x00000000,
504 (0x0600 << 16) | (0x9b7c >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0x8a14 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0x8a18 >> 2),
509 0x00000000,
510 (0x0600 << 16) | (0x30a00 >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0x8bf0 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0x8bcc >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0x8b24 >> 2),
517 0x00000000,
518 (0x0e00 << 16) | (0x30a04 >> 2),
519 0x00000000,
520 (0x0600 << 16) | (0x30a10 >> 2),
521 0x00000000,
522 (0x0600 << 16) | (0x30a14 >> 2),
523 0x00000000,
524 (0x0600 << 16) | (0x30a18 >> 2),
525 0x00000000,
526 (0x0600 << 16) | (0x30a2c >> 2),
527 0x00000000,
528 (0x0e00 << 16) | (0xc700 >> 2),
529 0x00000000,
530 (0x0e00 << 16) | (0xc704 >> 2),
531 0x00000000,
532 (0x0e00 << 16) | (0xc708 >> 2),
533 0x00000000,
534 (0x0e00 << 16) | (0xc768 >> 2),
535 0x00000000,
536 (0x0400 << 16) | (0xc770 >> 2),
537 0x00000000,
538 (0x0400 << 16) | (0xc774 >> 2),
539 0x00000000,
540 (0x0400 << 16) | (0xc778 >> 2),
541 0x00000000,
542 (0x0400 << 16) | (0xc77c >> 2),
543 0x00000000,
544 (0x0400 << 16) | (0xc780 >> 2),
545 0x00000000,
546 (0x0400 << 16) | (0xc784 >> 2),
547 0x00000000,
548 (0x0400 << 16) | (0xc788 >> 2),
549 0x00000000,
550 (0x0400 << 16) | (0xc78c >> 2),
551 0x00000000,
552 (0x0400 << 16) | (0xc798 >> 2),
553 0x00000000,
554 (0x0400 << 16) | (0xc79c >> 2),
555 0x00000000,
556 (0x0400 << 16) | (0xc7a0 >> 2),
557 0x00000000,
558 (0x0400 << 16) | (0xc7a4 >> 2),
559 0x00000000,
560 (0x0400 << 16) | (0xc7a8 >> 2),
561 0x00000000,
562 (0x0400 << 16) | (0xc7ac >> 2),
563 0x00000000,
564 (0x0400 << 16) | (0xc7b0 >> 2),
565 0x00000000,
566 (0x0400 << 16) | (0xc7b4 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0x9100 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0x3c010 >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x92a8 >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x92ac >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x92b4 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x92b8 >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x92bc >> 2),
581 0x00000000,
582 (0x0e00 << 16) | (0x92c0 >> 2),
583 0x00000000,
584 (0x0e00 << 16) | (0x92c4 >> 2),
585 0x00000000,
586 (0x0e00 << 16) | (0x92c8 >> 2),
587 0x00000000,
588 (0x0e00 << 16) | (0x92cc >> 2),
589 0x00000000,
590 (0x0e00 << 16) | (0x92d0 >> 2),
591 0x00000000,
592 (0x0e00 << 16) | (0x8c00 >> 2),
593 0x00000000,
594 (0x0e00 << 16) | (0x8c04 >> 2),
595 0x00000000,
596 (0x0e00 << 16) | (0x8c20 >> 2),
597 0x00000000,
598 (0x0e00 << 16) | (0x8c38 >> 2),
599 0x00000000,
600 (0x0e00 << 16) | (0x8c3c >> 2),
601 0x00000000,
602 (0x0e00 << 16) | (0xae00 >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0x9604 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0xac08 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0xac0c >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0xac10 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0xac14 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0xac58 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0xac68 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0xac6c >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0xac70 >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0xac74 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0xac78 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0xac7c >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0xac80 >> 2),
629 0x00000000,
630 (0x0e00 << 16) | (0xac84 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0xac88 >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0xac8c >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0x970c >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x9714 >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0x9718 >> 2),
641 0x00000000,
642 (0x0e00 << 16) | (0x971c >> 2),
643 0x00000000,
644 (0x0e00 << 16) | (0x31068 >> 2),
645 0x00000000,
646 (0x4e00 << 16) | (0x31068 >> 2),
647 0x00000000,
648 (0x5e00 << 16) | (0x31068 >> 2),
649 0x00000000,
650 (0x6e00 << 16) | (0x31068 >> 2),
651 0x00000000,
652 (0x7e00 << 16) | (0x31068 >> 2),
653 0x00000000,
654 (0x8e00 << 16) | (0x31068 >> 2),
655 0x00000000,
656 (0x9e00 << 16) | (0x31068 >> 2),
657 0x00000000,
658 (0xae00 << 16) | (0x31068 >> 2),
659 0x00000000,
660 (0xbe00 << 16) | (0x31068 >> 2),
661 0x00000000,
662 (0x0e00 << 16) | (0xcd10 >> 2),
663 0x00000000,
664 (0x0e00 << 16) | (0xcd14 >> 2),
665 0x00000000,
666 (0x0e00 << 16) | (0x88b0 >> 2),
667 0x00000000,
668 (0x0e00 << 16) | (0x88b4 >> 2),
669 0x00000000,
670 (0x0e00 << 16) | (0x88b8 >> 2),
671 0x00000000,
672 (0x0e00 << 16) | (0x88bc >> 2),
673 0x00000000,
674 (0x0400 << 16) | (0x89c0 >> 2),
675 0x00000000,
676 (0x0e00 << 16) | (0x88c4 >> 2),
677 0x00000000,
678 (0x0e00 << 16) | (0x88c8 >> 2),
679 0x00000000,
680 (0x0e00 << 16) | (0x88d0 >> 2),
681 0x00000000,
682 (0x0e00 << 16) | (0x88d4 >> 2),
683 0x00000000,
684 (0x0e00 << 16) | (0x88d8 >> 2),
685 0x00000000,
686 (0x0e00 << 16) | (0x8980 >> 2),
687 0x00000000,
688 (0x0e00 << 16) | (0x30938 >> 2),
689 0x00000000,
690 (0x0e00 << 16) | (0x3093c >> 2),
691 0x00000000,
692 (0x0e00 << 16) | (0x30940 >> 2),
693 0x00000000,
694 (0x0e00 << 16) | (0x89a0 >> 2),
695 0x00000000,
696 (0x0e00 << 16) | (0x30900 >> 2),
697 0x00000000,
698 (0x0e00 << 16) | (0x30904 >> 2),
699 0x00000000,
700 (0x0e00 << 16) | (0x89b4 >> 2),
701 0x00000000,
702 (0x0e00 << 16) | (0x3c210 >> 2),
703 0x00000000,
704 (0x0e00 << 16) | (0x3c214 >> 2),
705 0x00000000,
706 (0x0e00 << 16) | (0x3c218 >> 2),
707 0x00000000,
708 (0x0e00 << 16) | (0x8904 >> 2),
709 0x00000000,
710 0x5,
711 (0x0e00 << 16) | (0x8c28 >> 2),
712 (0x0e00 << 16) | (0x8c2c >> 2),
713 (0x0e00 << 16) | (0x8c30 >> 2),
714 (0x0e00 << 16) | (0x8c34 >> 2),
715 (0x0e00 << 16) | (0x9600 >> 2),
716 };
717
718 static const u32 kalindi_rlc_save_restore_register_list[] =
719 {
720 (0x0e00 << 16) | (0xc12c >> 2),
721 0x00000000,
722 (0x0e00 << 16) | (0xc140 >> 2),
723 0x00000000,
724 (0x0e00 << 16) | (0xc150 >> 2),
725 0x00000000,
726 (0x0e00 << 16) | (0xc15c >> 2),
727 0x00000000,
728 (0x0e00 << 16) | (0xc168 >> 2),
729 0x00000000,
730 (0x0e00 << 16) | (0xc170 >> 2),
731 0x00000000,
732 (0x0e00 << 16) | (0xc204 >> 2),
733 0x00000000,
734 (0x0e00 << 16) | (0xc2b4 >> 2),
735 0x00000000,
736 (0x0e00 << 16) | (0xc2b8 >> 2),
737 0x00000000,
738 (0x0e00 << 16) | (0xc2bc >> 2),
739 0x00000000,
740 (0x0e00 << 16) | (0xc2c0 >> 2),
741 0x00000000,
742 (0x0e00 << 16) | (0x8228 >> 2),
743 0x00000000,
744 (0x0e00 << 16) | (0x829c >> 2),
745 0x00000000,
746 (0x0e00 << 16) | (0x869c >> 2),
747 0x00000000,
748 (0x0600 << 16) | (0x98f4 >> 2),
749 0x00000000,
750 (0x0e00 << 16) | (0x98f8 >> 2),
751 0x00000000,
752 (0x0e00 << 16) | (0x9900 >> 2),
753 0x00000000,
754 (0x0e00 << 16) | (0xc260 >> 2),
755 0x00000000,
756 (0x0e00 << 16) | (0x90e8 >> 2),
757 0x00000000,
758 (0x0e00 << 16) | (0x3c000 >> 2),
759 0x00000000,
760 (0x0e00 << 16) | (0x3c00c >> 2),
761 0x00000000,
762 (0x0e00 << 16) | (0x8c1c >> 2),
763 0x00000000,
764 (0x0e00 << 16) | (0x9700 >> 2),
765 0x00000000,
766 (0x0e00 << 16) | (0xcd20 >> 2),
767 0x00000000,
768 (0x4e00 << 16) | (0xcd20 >> 2),
769 0x00000000,
770 (0x5e00 << 16) | (0xcd20 >> 2),
771 0x00000000,
772 (0x6e00 << 16) | (0xcd20 >> 2),
773 0x00000000,
774 (0x7e00 << 16) | (0xcd20 >> 2),
775 0x00000000,
776 (0x0e00 << 16) | (0x89bc >> 2),
777 0x00000000,
778 (0x0e00 << 16) | (0x8900 >> 2),
779 0x00000000,
780 0x3,
781 (0x0e00 << 16) | (0xc130 >> 2),
782 0x00000000,
783 (0x0e00 << 16) | (0xc134 >> 2),
784 0x00000000,
785 (0x0e00 << 16) | (0xc1fc >> 2),
786 0x00000000,
787 (0x0e00 << 16) | (0xc208 >> 2),
788 0x00000000,
789 (0x0e00 << 16) | (0xc264 >> 2),
790 0x00000000,
791 (0x0e00 << 16) | (0xc268 >> 2),
792 0x00000000,
793 (0x0e00 << 16) | (0xc26c >> 2),
794 0x00000000,
795 (0x0e00 << 16) | (0xc270 >> 2),
796 0x00000000,
797 (0x0e00 << 16) | (0xc274 >> 2),
798 0x00000000,
799 (0x0e00 << 16) | (0xc28c >> 2),
800 0x00000000,
801 (0x0e00 << 16) | (0xc290 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0xc294 >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0xc298 >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0xc2a0 >> 2),
808 0x00000000,
809 (0x0e00 << 16) | (0xc2a4 >> 2),
810 0x00000000,
811 (0x0e00 << 16) | (0xc2a8 >> 2),
812 0x00000000,
813 (0x0e00 << 16) | (0xc2ac >> 2),
814 0x00000000,
815 (0x0e00 << 16) | (0x301d0 >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0x30238 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0x30250 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0x30254 >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0x30258 >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0x3025c >> 2),
826 0x00000000,
827 (0x4e00 << 16) | (0xc900 >> 2),
828 0x00000000,
829 (0x5e00 << 16) | (0xc900 >> 2),
830 0x00000000,
831 (0x6e00 << 16) | (0xc900 >> 2),
832 0x00000000,
833 (0x7e00 << 16) | (0xc900 >> 2),
834 0x00000000,
835 (0x4e00 << 16) | (0xc904 >> 2),
836 0x00000000,
837 (0x5e00 << 16) | (0xc904 >> 2),
838 0x00000000,
839 (0x6e00 << 16) | (0xc904 >> 2),
840 0x00000000,
841 (0x7e00 << 16) | (0xc904 >> 2),
842 0x00000000,
843 (0x4e00 << 16) | (0xc908 >> 2),
844 0x00000000,
845 (0x5e00 << 16) | (0xc908 >> 2),
846 0x00000000,
847 (0x6e00 << 16) | (0xc908 >> 2),
848 0x00000000,
849 (0x7e00 << 16) | (0xc908 >> 2),
850 0x00000000,
851 (0x4e00 << 16) | (0xc90c >> 2),
852 0x00000000,
853 (0x5e00 << 16) | (0xc90c >> 2),
854 0x00000000,
855 (0x6e00 << 16) | (0xc90c >> 2),
856 0x00000000,
857 (0x7e00 << 16) | (0xc90c >> 2),
858 0x00000000,
859 (0x4e00 << 16) | (0xc910 >> 2),
860 0x00000000,
861 (0x5e00 << 16) | (0xc910 >> 2),
862 0x00000000,
863 (0x6e00 << 16) | (0xc910 >> 2),
864 0x00000000,
865 (0x7e00 << 16) | (0xc910 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0xc99c >> 2),
868 0x00000000,
869 (0x0e00 << 16) | (0x9834 >> 2),
870 0x00000000,
871 (0x0000 << 16) | (0x30f00 >> 2),
872 0x00000000,
873 (0x0000 << 16) | (0x30f04 >> 2),
874 0x00000000,
875 (0x0000 << 16) | (0x30f08 >> 2),
876 0x00000000,
877 (0x0000 << 16) | (0x30f0c >> 2),
878 0x00000000,
879 (0x0600 << 16) | (0x9b7c >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0x8a14 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0x8a18 >> 2),
884 0x00000000,
885 (0x0600 << 16) | (0x30a00 >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0x8bf0 >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0x8bcc >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0x8b24 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0x30a04 >> 2),
894 0x00000000,
895 (0x0600 << 16) | (0x30a10 >> 2),
896 0x00000000,
897 (0x0600 << 16) | (0x30a14 >> 2),
898 0x00000000,
899 (0x0600 << 16) | (0x30a18 >> 2),
900 0x00000000,
901 (0x0600 << 16) | (0x30a2c >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0xc700 >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0xc704 >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0xc708 >> 2),
908 0x00000000,
909 (0x0e00 << 16) | (0xc768 >> 2),
910 0x00000000,
911 (0x0400 << 16) | (0xc770 >> 2),
912 0x00000000,
913 (0x0400 << 16) | (0xc774 >> 2),
914 0x00000000,
915 (0x0400 << 16) | (0xc798 >> 2),
916 0x00000000,
917 (0x0400 << 16) | (0xc79c >> 2),
918 0x00000000,
919 (0x0e00 << 16) | (0x9100 >> 2),
920 0x00000000,
921 (0x0e00 << 16) | (0x3c010 >> 2),
922 0x00000000,
923 (0x0e00 << 16) | (0x8c00 >> 2),
924 0x00000000,
925 (0x0e00 << 16) | (0x8c04 >> 2),
926 0x00000000,
927 (0x0e00 << 16) | (0x8c20 >> 2),
928 0x00000000,
929 (0x0e00 << 16) | (0x8c38 >> 2),
930 0x00000000,
931 (0x0e00 << 16) | (0x8c3c >> 2),
932 0x00000000,
933 (0x0e00 << 16) | (0xae00 >> 2),
934 0x00000000,
935 (0x0e00 << 16) | (0x9604 >> 2),
936 0x00000000,
937 (0x0e00 << 16) | (0xac08 >> 2),
938 0x00000000,
939 (0x0e00 << 16) | (0xac0c >> 2),
940 0x00000000,
941 (0x0e00 << 16) | (0xac10 >> 2),
942 0x00000000,
943 (0x0e00 << 16) | (0xac14 >> 2),
944 0x00000000,
945 (0x0e00 << 16) | (0xac58 >> 2),
946 0x00000000,
947 (0x0e00 << 16) | (0xac68 >> 2),
948 0x00000000,
949 (0x0e00 << 16) | (0xac6c >> 2),
950 0x00000000,
951 (0x0e00 << 16) | (0xac70 >> 2),
952 0x00000000,
953 (0x0e00 << 16) | (0xac74 >> 2),
954 0x00000000,
955 (0x0e00 << 16) | (0xac78 >> 2),
956 0x00000000,
957 (0x0e00 << 16) | (0xac7c >> 2),
958 0x00000000,
959 (0x0e00 << 16) | (0xac80 >> 2),
960 0x00000000,
961 (0x0e00 << 16) | (0xac84 >> 2),
962 0x00000000,
963 (0x0e00 << 16) | (0xac88 >> 2),
964 0x00000000,
965 (0x0e00 << 16) | (0xac8c >> 2),
966 0x00000000,
967 (0x0e00 << 16) | (0x970c >> 2),
968 0x00000000,
969 (0x0e00 << 16) | (0x9714 >> 2),
970 0x00000000,
971 (0x0e00 << 16) | (0x9718 >> 2),
972 0x00000000,
973 (0x0e00 << 16) | (0x971c >> 2),
974 0x00000000,
975 (0x0e00 << 16) | (0x31068 >> 2),
976 0x00000000,
977 (0x4e00 << 16) | (0x31068 >> 2),
978 0x00000000,
979 (0x5e00 << 16) | (0x31068 >> 2),
980 0x00000000,
981 (0x6e00 << 16) | (0x31068 >> 2),
982 0x00000000,
983 (0x7e00 << 16) | (0x31068 >> 2),
984 0x00000000,
985 (0x0e00 << 16) | (0xcd10 >> 2),
986 0x00000000,
987 (0x0e00 << 16) | (0xcd14 >> 2),
988 0x00000000,
989 (0x0e00 << 16) | (0x88b0 >> 2),
990 0x00000000,
991 (0x0e00 << 16) | (0x88b4 >> 2),
992 0x00000000,
993 (0x0e00 << 16) | (0x88b8 >> 2),
994 0x00000000,
995 (0x0e00 << 16) | (0x88bc >> 2),
996 0x00000000,
997 (0x0400 << 16) | (0x89c0 >> 2),
998 0x00000000,
999 (0x0e00 << 16) | (0x88c4 >> 2),
1000 0x00000000,
1001 (0x0e00 << 16) | (0x88c8 >> 2),
1002 0x00000000,
1003 (0x0e00 << 16) | (0x88d0 >> 2),
1004 0x00000000,
1005 (0x0e00 << 16) | (0x88d4 >> 2),
1006 0x00000000,
1007 (0x0e00 << 16) | (0x88d8 >> 2),
1008 0x00000000,
1009 (0x0e00 << 16) | (0x8980 >> 2),
1010 0x00000000,
1011 (0x0e00 << 16) | (0x30938 >> 2),
1012 0x00000000,
1013 (0x0e00 << 16) | (0x3093c >> 2),
1014 0x00000000,
1015 (0x0e00 << 16) | (0x30940 >> 2),
1016 0x00000000,
1017 (0x0e00 << 16) | (0x89a0 >> 2),
1018 0x00000000,
1019 (0x0e00 << 16) | (0x30900 >> 2),
1020 0x00000000,
1021 (0x0e00 << 16) | (0x30904 >> 2),
1022 0x00000000,
1023 (0x0e00 << 16) | (0x89b4 >> 2),
1024 0x00000000,
1025 (0x0e00 << 16) | (0x3e1fc >> 2),
1026 0x00000000,
1027 (0x0e00 << 16) | (0x3c210 >> 2),
1028 0x00000000,
1029 (0x0e00 << 16) | (0x3c214 >> 2),
1030 0x00000000,
1031 (0x0e00 << 16) | (0x3c218 >> 2),
1032 0x00000000,
1033 (0x0e00 << 16) | (0x8904 >> 2),
1034 0x00000000,
1035 0x5,
1036 (0x0e00 << 16) | (0x8c28 >> 2),
1037 (0x0e00 << 16) | (0x8c2c >> 2),
1038 (0x0e00 << 16) | (0x8c30 >> 2),
1039 (0x0e00 << 16) | (0x8c34 >> 2),
1040 (0x0e00 << 16) | (0x9600 >> 2),
1041 };
1042
1043 static const u32 bonaire_golden_spm_registers[] =
1044 {
1045 0x30800, 0xe0ffffff, 0xe0000000
1046 };
1047
1048 static const u32 bonaire_golden_common_registers[] =
1049 {
1050 0xc770, 0xffffffff, 0x00000800,
1051 0xc774, 0xffffffff, 0x00000800,
1052 0xc798, 0xffffffff, 0x00007fbf,
1053 0xc79c, 0xffffffff, 0x00007faf
1054 };
1055
1056 static const u32 bonaire_golden_registers[] =
1057 {
1058 0x3354, 0x00000333, 0x00000333,
1059 0x3350, 0x000c0fc0, 0x00040200,
1060 0x9a10, 0x00010000, 0x00058208,
1061 0x3c000, 0xffff1fff, 0x00140000,
1062 0x3c200, 0xfdfc0fff, 0x00000100,
1063 0x3c234, 0x40000000, 0x40000200,
1064 0x9830, 0xffffffff, 0x00000000,
1065 0x9834, 0xf00fffff, 0x00000400,
1066 0x9838, 0x0002021c, 0x00020200,
1067 0xc78, 0x00000080, 0x00000000,
1068 0x5bb0, 0x000000f0, 0x00000070,
1069 0x5bc0, 0xf0311fff, 0x80300000,
1070 0x98f8, 0x73773777, 0x12010001,
1071 0x350c, 0x00810000, 0x408af000,
1072 0x7030, 0x31000111, 0x00000011,
1073 0x2f48, 0x73773777, 0x12010001,
1074 0x220c, 0x00007fb6, 0x0021a1b1,
1075 0x2210, 0x00007fb6, 0x002021b1,
1076 0x2180, 0x00007fb6, 0x00002191,
1077 0x2218, 0x00007fb6, 0x002121b1,
1078 0x221c, 0x00007fb6, 0x002021b1,
1079 0x21dc, 0x00007fb6, 0x00002191,
1080 0x21e0, 0x00007fb6, 0x00002191,
1081 0x3628, 0x0000003f, 0x0000000a,
1082 0x362c, 0x0000003f, 0x0000000a,
1083 0x2ae4, 0x00073ffe, 0x000022a2,
1084 0x240c, 0x000007ff, 0x00000000,
1085 0x8a14, 0xf000003f, 0x00000007,
1086 0x8bf0, 0x00002001, 0x00000001,
1087 0x8b24, 0xffffffff, 0x00ffffff,
1088 0x30a04, 0x0000ff0f, 0x00000000,
1089 0x28a4c, 0x07ffffff, 0x06000000,
1090 0x4d8, 0x00000fff, 0x00000100,
1091 0x3e78, 0x00000001, 0x00000002,
1092 0x9100, 0x03000000, 0x0362c688,
1093 0x8c00, 0x000000ff, 0x00000001,
1094 0xe40, 0x00001fff, 0x00001fff,
1095 0x9060, 0x0000007f, 0x00000020,
1096 0x9508, 0x00010000, 0x00010000,
1097 0xac14, 0x000003ff, 0x000000f3,
1098 0xac0c, 0xffffffff, 0x00001032
1099 };
1100
1101 static const u32 bonaire_mgcg_cgcg_init[] =
1102 {
1103 0xc420, 0xffffffff, 0xfffffffc,
1104 0x30800, 0xffffffff, 0xe0000000,
1105 0x3c2a0, 0xffffffff, 0x00000100,
1106 0x3c208, 0xffffffff, 0x00000100,
1107 0x3c2c0, 0xffffffff, 0xc0000100,
1108 0x3c2c8, 0xffffffff, 0xc0000100,
1109 0x3c2c4, 0xffffffff, 0xc0000100,
1110 0x55e4, 0xffffffff, 0x00600100,
1111 0x3c280, 0xffffffff, 0x00000100,
1112 0x3c214, 0xffffffff, 0x06000100,
1113 0x3c220, 0xffffffff, 0x00000100,
1114 0x3c218, 0xffffffff, 0x06000100,
1115 0x3c204, 0xffffffff, 0x00000100,
1116 0x3c2e0, 0xffffffff, 0x00000100,
1117 0x3c224, 0xffffffff, 0x00000100,
1118 0x3c200, 0xffffffff, 0x00000100,
1119 0x3c230, 0xffffffff, 0x00000100,
1120 0x3c234, 0xffffffff, 0x00000100,
1121 0x3c250, 0xffffffff, 0x00000100,
1122 0x3c254, 0xffffffff, 0x00000100,
1123 0x3c258, 0xffffffff, 0x00000100,
1124 0x3c25c, 0xffffffff, 0x00000100,
1125 0x3c260, 0xffffffff, 0x00000100,
1126 0x3c27c, 0xffffffff, 0x00000100,
1127 0x3c278, 0xffffffff, 0x00000100,
1128 0x3c210, 0xffffffff, 0x06000100,
1129 0x3c290, 0xffffffff, 0x00000100,
1130 0x3c274, 0xffffffff, 0x00000100,
1131 0x3c2b4, 0xffffffff, 0x00000100,
1132 0x3c2b0, 0xffffffff, 0x00000100,
1133 0x3c270, 0xffffffff, 0x00000100,
1134 0x30800, 0xffffffff, 0xe0000000,
1135 0x3c020, 0xffffffff, 0x00010000,
1136 0x3c024, 0xffffffff, 0x00030002,
1137 0x3c028, 0xffffffff, 0x00040007,
1138 0x3c02c, 0xffffffff, 0x00060005,
1139 0x3c030, 0xffffffff, 0x00090008,
1140 0x3c034, 0xffffffff, 0x00010000,
1141 0x3c038, 0xffffffff, 0x00030002,
1142 0x3c03c, 0xffffffff, 0x00040007,
1143 0x3c040, 0xffffffff, 0x00060005,
1144 0x3c044, 0xffffffff, 0x00090008,
1145 0x3c048, 0xffffffff, 0x00010000,
1146 0x3c04c, 0xffffffff, 0x00030002,
1147 0x3c050, 0xffffffff, 0x00040007,
1148 0x3c054, 0xffffffff, 0x00060005,
1149 0x3c058, 0xffffffff, 0x00090008,
1150 0x3c05c, 0xffffffff, 0x00010000,
1151 0x3c060, 0xffffffff, 0x00030002,
1152 0x3c064, 0xffffffff, 0x00040007,
1153 0x3c068, 0xffffffff, 0x00060005,
1154 0x3c06c, 0xffffffff, 0x00090008,
1155 0x3c070, 0xffffffff, 0x00010000,
1156 0x3c074, 0xffffffff, 0x00030002,
1157 0x3c078, 0xffffffff, 0x00040007,
1158 0x3c07c, 0xffffffff, 0x00060005,
1159 0x3c080, 0xffffffff, 0x00090008,
1160 0x3c084, 0xffffffff, 0x00010000,
1161 0x3c088, 0xffffffff, 0x00030002,
1162 0x3c08c, 0xffffffff, 0x00040007,
1163 0x3c090, 0xffffffff, 0x00060005,
1164 0x3c094, 0xffffffff, 0x00090008,
1165 0x3c098, 0xffffffff, 0x00010000,
1166 0x3c09c, 0xffffffff, 0x00030002,
1167 0x3c0a0, 0xffffffff, 0x00040007,
1168 0x3c0a4, 0xffffffff, 0x00060005,
1169 0x3c0a8, 0xffffffff, 0x00090008,
1170 0x3c000, 0xffffffff, 0x96e00200,
1171 0x8708, 0xffffffff, 0x00900100,
1172 0xc424, 0xffffffff, 0x0020003f,
1173 0x38, 0xffffffff, 0x0140001c,
1174 0x3c, 0x000f0000, 0x000f0000,
1175 0x220, 0xffffffff, 0xC060000C,
1176 0x224, 0xc0000fff, 0x00000100,
1177 0xf90, 0xffffffff, 0x00000100,
1178 0xf98, 0x00000101, 0x00000000,
1179 0x20a8, 0xffffffff, 0x00000104,
1180 0x55e4, 0xff000fff, 0x00000100,
1181 0x30cc, 0xc0000fff, 0x00000104,
1182 0xc1e4, 0x00000001, 0x00000001,
1183 0xd00c, 0xff000ff0, 0x00000100,
1184 0xd80c, 0xff000ff0, 0x00000100
1185 };
1186
1187 static const u32 spectre_golden_spm_registers[] =
1188 {
1189 0x30800, 0xe0ffffff, 0xe0000000
1190 };
1191
1192 static const u32 spectre_golden_common_registers[] =
1193 {
1194 0xc770, 0xffffffff, 0x00000800,
1195 0xc774, 0xffffffff, 0x00000800,
1196 0xc798, 0xffffffff, 0x00007fbf,
1197 0xc79c, 0xffffffff, 0x00007faf
1198 };
1199
1200 static const u32 spectre_golden_registers[] =
1201 {
1202 0x3c000, 0xffff1fff, 0x96940200,
1203 0x3c00c, 0xffff0001, 0xff000000,
1204 0x3c200, 0xfffc0fff, 0x00000100,
1205 0x6ed8, 0x00010101, 0x00010000,
1206 0x9834, 0xf00fffff, 0x00000400,
1207 0x9838, 0xfffffffc, 0x00020200,
1208 0x5bb0, 0x000000f0, 0x00000070,
1209 0x5bc0, 0xf0311fff, 0x80300000,
1210 0x98f8, 0x73773777, 0x12010001,
1211 0x9b7c, 0x00ff0000, 0x00fc0000,
1212 0x2f48, 0x73773777, 0x12010001,
1213 0x8a14, 0xf000003f, 0x00000007,
1214 0x8b24, 0xffffffff, 0x00ffffff,
1215 0x28350, 0x3f3f3fff, 0x00000082,
1216 0x28354, 0x0000003f, 0x00000000,
1217 0x3e78, 0x00000001, 0x00000002,
1218 0x913c, 0xffff03df, 0x00000004,
1219 0xc768, 0x00000008, 0x00000008,
1220 0x8c00, 0x000008ff, 0x00000800,
1221 0x9508, 0x00010000, 0x00010000,
1222 0xac0c, 0xffffffff, 0x54763210,
1223 0x214f8, 0x01ff01ff, 0x00000002,
1224 0x21498, 0x007ff800, 0x00200000,
1225 0x2015c, 0xffffffff, 0x00000f40,
1226 0x30934, 0xffffffff, 0x00000001
1227 };
1228
1229 static const u32 spectre_mgcg_cgcg_init[] =
1230 {
1231 0xc420, 0xffffffff, 0xfffffffc,
1232 0x30800, 0xffffffff, 0xe0000000,
1233 0x3c2a0, 0xffffffff, 0x00000100,
1234 0x3c208, 0xffffffff, 0x00000100,
1235 0x3c2c0, 0xffffffff, 0x00000100,
1236 0x3c2c8, 0xffffffff, 0x00000100,
1237 0x3c2c4, 0xffffffff, 0x00000100,
1238 0x55e4, 0xffffffff, 0x00600100,
1239 0x3c280, 0xffffffff, 0x00000100,
1240 0x3c214, 0xffffffff, 0x06000100,
1241 0x3c220, 0xffffffff, 0x00000100,
1242 0x3c218, 0xffffffff, 0x06000100,
1243 0x3c204, 0xffffffff, 0x00000100,
1244 0x3c2e0, 0xffffffff, 0x00000100,
1245 0x3c224, 0xffffffff, 0x00000100,
1246 0x3c200, 0xffffffff, 0x00000100,
1247 0x3c230, 0xffffffff, 0x00000100,
1248 0x3c234, 0xffffffff, 0x00000100,
1249 0x3c250, 0xffffffff, 0x00000100,
1250 0x3c254, 0xffffffff, 0x00000100,
1251 0x3c258, 0xffffffff, 0x00000100,
1252 0x3c25c, 0xffffffff, 0x00000100,
1253 0x3c260, 0xffffffff, 0x00000100,
1254 0x3c27c, 0xffffffff, 0x00000100,
1255 0x3c278, 0xffffffff, 0x00000100,
1256 0x3c210, 0xffffffff, 0x06000100,
1257 0x3c290, 0xffffffff, 0x00000100,
1258 0x3c274, 0xffffffff, 0x00000100,
1259 0x3c2b4, 0xffffffff, 0x00000100,
1260 0x3c2b0, 0xffffffff, 0x00000100,
1261 0x3c270, 0xffffffff, 0x00000100,
1262 0x30800, 0xffffffff, 0xe0000000,
1263 0x3c020, 0xffffffff, 0x00010000,
1264 0x3c024, 0xffffffff, 0x00030002,
1265 0x3c028, 0xffffffff, 0x00040007,
1266 0x3c02c, 0xffffffff, 0x00060005,
1267 0x3c030, 0xffffffff, 0x00090008,
1268 0x3c034, 0xffffffff, 0x00010000,
1269 0x3c038, 0xffffffff, 0x00030002,
1270 0x3c03c, 0xffffffff, 0x00040007,
1271 0x3c040, 0xffffffff, 0x00060005,
1272 0x3c044, 0xffffffff, 0x00090008,
1273 0x3c048, 0xffffffff, 0x00010000,
1274 0x3c04c, 0xffffffff, 0x00030002,
1275 0x3c050, 0xffffffff, 0x00040007,
1276 0x3c054, 0xffffffff, 0x00060005,
1277 0x3c058, 0xffffffff, 0x00090008,
1278 0x3c05c, 0xffffffff, 0x00010000,
1279 0x3c060, 0xffffffff, 0x00030002,
1280 0x3c064, 0xffffffff, 0x00040007,
1281 0x3c068, 0xffffffff, 0x00060005,
1282 0x3c06c, 0xffffffff, 0x00090008,
1283 0x3c070, 0xffffffff, 0x00010000,
1284 0x3c074, 0xffffffff, 0x00030002,
1285 0x3c078, 0xffffffff, 0x00040007,
1286 0x3c07c, 0xffffffff, 0x00060005,
1287 0x3c080, 0xffffffff, 0x00090008,
1288 0x3c084, 0xffffffff, 0x00010000,
1289 0x3c088, 0xffffffff, 0x00030002,
1290 0x3c08c, 0xffffffff, 0x00040007,
1291 0x3c090, 0xffffffff, 0x00060005,
1292 0x3c094, 0xffffffff, 0x00090008,
1293 0x3c098, 0xffffffff, 0x00010000,
1294 0x3c09c, 0xffffffff, 0x00030002,
1295 0x3c0a0, 0xffffffff, 0x00040007,
1296 0x3c0a4, 0xffffffff, 0x00060005,
1297 0x3c0a8, 0xffffffff, 0x00090008,
1298 0x3c0ac, 0xffffffff, 0x00010000,
1299 0x3c0b0, 0xffffffff, 0x00030002,
1300 0x3c0b4, 0xffffffff, 0x00040007,
1301 0x3c0b8, 0xffffffff, 0x00060005,
1302 0x3c0bc, 0xffffffff, 0x00090008,
1303 0x3c000, 0xffffffff, 0x96e00200,
1304 0x8708, 0xffffffff, 0x00900100,
1305 0xc424, 0xffffffff, 0x0020003f,
1306 0x38, 0xffffffff, 0x0140001c,
1307 0x3c, 0x000f0000, 0x000f0000,
1308 0x220, 0xffffffff, 0xC060000C,
1309 0x224, 0xc0000fff, 0x00000100,
1310 0xf90, 0xffffffff, 0x00000100,
1311 0xf98, 0x00000101, 0x00000000,
1312 0x20a8, 0xffffffff, 0x00000104,
1313 0x55e4, 0xff000fff, 0x00000100,
1314 0x30cc, 0xc0000fff, 0x00000104,
1315 0xc1e4, 0x00000001, 0x00000001,
1316 0xd00c, 0xff000ff0, 0x00000100,
1317 0xd80c, 0xff000ff0, 0x00000100
1318 };
1319
1320 static const u32 kalindi_golden_spm_registers[] =
1321 {
1322 0x30800, 0xe0ffffff, 0xe0000000
1323 };
1324
1325 static const u32 kalindi_golden_common_registers[] =
1326 {
1327 0xc770, 0xffffffff, 0x00000800,
1328 0xc774, 0xffffffff, 0x00000800,
1329 0xc798, 0xffffffff, 0x00007fbf,
1330 0xc79c, 0xffffffff, 0x00007faf
1331 };
1332
1333 static const u32 kalindi_golden_registers[] =
1334 {
1335 0x3c000, 0xffffdfff, 0x6e944040,
1336 0x55e4, 0xff607fff, 0xfc000100,
1337 0x3c220, 0xff000fff, 0x00000100,
1338 0x3c224, 0xff000fff, 0x00000100,
1339 0x3c200, 0xfffc0fff, 0x00000100,
1340 0x6ed8, 0x00010101, 0x00010000,
1341 0x9830, 0xffffffff, 0x00000000,
1342 0x9834, 0xf00fffff, 0x00000400,
1343 0x5bb0, 0x000000f0, 0x00000070,
1344 0x5bc0, 0xf0311fff, 0x80300000,
1345 0x98f8, 0x73773777, 0x12010001,
1346 0x98fc, 0xffffffff, 0x00000010,
1347 0x9b7c, 0x00ff0000, 0x00fc0000,
1348 0x8030, 0x00001f0f, 0x0000100a,
1349 0x2f48, 0x73773777, 0x12010001,
1350 0x2408, 0x000fffff, 0x000c007f,
1351 0x8a14, 0xf000003f, 0x00000007,
1352 0x8b24, 0x3fff3fff, 0x00ffcfff,
1353 0x30a04, 0x0000ff0f, 0x00000000,
1354 0x28a4c, 0x07ffffff, 0x06000000,
1355 0x4d8, 0x00000fff, 0x00000100,
1356 0x3e78, 0x00000001, 0x00000002,
1357 0xc768, 0x00000008, 0x00000008,
1358 0x8c00, 0x000000ff, 0x00000003,
1359 0x214f8, 0x01ff01ff, 0x00000002,
1360 0x21498, 0x007ff800, 0x00200000,
1361 0x2015c, 0xffffffff, 0x00000f40,
1362 0x88c4, 0x001f3ae3, 0x00000082,
1363 0x88d4, 0x0000001f, 0x00000010,
1364 0x30934, 0xffffffff, 0x00000000
1365 };
1366
1367 static const u32 kalindi_mgcg_cgcg_init[] =
1368 {
1369 0xc420, 0xffffffff, 0xfffffffc,
1370 0x30800, 0xffffffff, 0xe0000000,
1371 0x3c2a0, 0xffffffff, 0x00000100,
1372 0x3c208, 0xffffffff, 0x00000100,
1373 0x3c2c0, 0xffffffff, 0x00000100,
1374 0x3c2c8, 0xffffffff, 0x00000100,
1375 0x3c2c4, 0xffffffff, 0x00000100,
1376 0x55e4, 0xffffffff, 0x00600100,
1377 0x3c280, 0xffffffff, 0x00000100,
1378 0x3c214, 0xffffffff, 0x06000100,
1379 0x3c220, 0xffffffff, 0x00000100,
1380 0x3c218, 0xffffffff, 0x06000100,
1381 0x3c204, 0xffffffff, 0x00000100,
1382 0x3c2e0, 0xffffffff, 0x00000100,
1383 0x3c224, 0xffffffff, 0x00000100,
1384 0x3c200, 0xffffffff, 0x00000100,
1385 0x3c230, 0xffffffff, 0x00000100,
1386 0x3c234, 0xffffffff, 0x00000100,
1387 0x3c250, 0xffffffff, 0x00000100,
1388 0x3c254, 0xffffffff, 0x00000100,
1389 0x3c258, 0xffffffff, 0x00000100,
1390 0x3c25c, 0xffffffff, 0x00000100,
1391 0x3c260, 0xffffffff, 0x00000100,
1392 0x3c27c, 0xffffffff, 0x00000100,
1393 0x3c278, 0xffffffff, 0x00000100,
1394 0x3c210, 0xffffffff, 0x06000100,
1395 0x3c290, 0xffffffff, 0x00000100,
1396 0x3c274, 0xffffffff, 0x00000100,
1397 0x3c2b4, 0xffffffff, 0x00000100,
1398 0x3c2b0, 0xffffffff, 0x00000100,
1399 0x3c270, 0xffffffff, 0x00000100,
1400 0x30800, 0xffffffff, 0xe0000000,
1401 0x3c020, 0xffffffff, 0x00010000,
1402 0x3c024, 0xffffffff, 0x00030002,
1403 0x3c028, 0xffffffff, 0x00040007,
1404 0x3c02c, 0xffffffff, 0x00060005,
1405 0x3c030, 0xffffffff, 0x00090008,
1406 0x3c034, 0xffffffff, 0x00010000,
1407 0x3c038, 0xffffffff, 0x00030002,
1408 0x3c03c, 0xffffffff, 0x00040007,
1409 0x3c040, 0xffffffff, 0x00060005,
1410 0x3c044, 0xffffffff, 0x00090008,
1411 0x3c000, 0xffffffff, 0x96e00200,
1412 0x8708, 0xffffffff, 0x00900100,
1413 0xc424, 0xffffffff, 0x0020003f,
1414 0x38, 0xffffffff, 0x0140001c,
1415 0x3c, 0x000f0000, 0x000f0000,
1416 0x220, 0xffffffff, 0xC060000C,
1417 0x224, 0xc0000fff, 0x00000100,
1418 0x20a8, 0xffffffff, 0x00000104,
1419 0x55e4, 0xff000fff, 0x00000100,
1420 0x30cc, 0xc0000fff, 0x00000104,
1421 0xc1e4, 0x00000001, 0x00000001,
1422 0xd00c, 0xff000ff0, 0x00000100,
1423 0xd80c, 0xff000ff0, 0x00000100
1424 };
1425
1426 static const u32 hawaii_golden_spm_registers[] =
1427 {
1428 0x30800, 0xe0ffffff, 0xe0000000
1429 };
1430
1431 static const u32 hawaii_golden_common_registers[] =
1432 {
1433 0x30800, 0xffffffff, 0xe0000000,
1434 0x28350, 0xffffffff, 0x3a00161a,
1435 0x28354, 0xffffffff, 0x0000002e,
1436 0x9a10, 0xffffffff, 0x00018208,
1437 0x98f8, 0xffffffff, 0x12011003
1438 };
1439
1440 static const u32 hawaii_golden_registers[] =
1441 {
1442 0x3354, 0x00000333, 0x00000333,
1443 0x9a10, 0x00010000, 0x00058208,
1444 0x9830, 0xffffffff, 0x00000000,
1445 0x9834, 0xf00fffff, 0x00000400,
1446 0x9838, 0x0002021c, 0x00020200,
1447 0xc78, 0x00000080, 0x00000000,
1448 0x5bb0, 0x000000f0, 0x00000070,
1449 0x5bc0, 0xf0311fff, 0x80300000,
1450 0x350c, 0x00810000, 0x408af000,
1451 0x7030, 0x31000111, 0x00000011,
1452 0x2f48, 0x73773777, 0x12010001,
1453 0x2120, 0x0000007f, 0x0000001b,
1454 0x21dc, 0x00007fb6, 0x00002191,
1455 0x3628, 0x0000003f, 0x0000000a,
1456 0x362c, 0x0000003f, 0x0000000a,
1457 0x2ae4, 0x00073ffe, 0x000022a2,
1458 0x240c, 0x000007ff, 0x00000000,
1459 0x8bf0, 0x00002001, 0x00000001,
1460 0x8b24, 0xffffffff, 0x00ffffff,
1461 0x30a04, 0x0000ff0f, 0x00000000,
1462 0x28a4c, 0x07ffffff, 0x06000000,
1463 0x3e78, 0x00000001, 0x00000002,
1464 0xc768, 0x00000008, 0x00000008,
1465 0xc770, 0x00000f00, 0x00000800,
1466 0xc774, 0x00000f00, 0x00000800,
1467 0xc798, 0x00ffffff, 0x00ff7fbf,
1468 0xc79c, 0x00ffffff, 0x00ff7faf,
1469 0x8c00, 0x000000ff, 0x00000800,
1470 0xe40, 0x00001fff, 0x00001fff,
1471 0x9060, 0x0000007f, 0x00000020,
1472 0x9508, 0x00010000, 0x00010000,
1473 0xae00, 0x00100000, 0x000ff07c,
1474 0xac14, 0x000003ff, 0x0000000f,
1475 0xac10, 0xffffffff, 0x7564fdec,
1476 0xac0c, 0xffffffff, 0x3120b9a8,
1477 0xac08, 0x20000000, 0x0f9c0000
1478 };
1479
1480 static const u32 hawaii_mgcg_cgcg_init[] =
1481 {
1482 0xc420, 0xffffffff, 0xfffffffd,
1483 0x30800, 0xffffffff, 0xe0000000,
1484 0x3c2a0, 0xffffffff, 0x00000100,
1485 0x3c208, 0xffffffff, 0x00000100,
1486 0x3c2c0, 0xffffffff, 0x00000100,
1487 0x3c2c8, 0xffffffff, 0x00000100,
1488 0x3c2c4, 0xffffffff, 0x00000100,
1489 0x55e4, 0xffffffff, 0x00200100,
1490 0x3c280, 0xffffffff, 0x00000100,
1491 0x3c214, 0xffffffff, 0x06000100,
1492 0x3c220, 0xffffffff, 0x00000100,
1493 0x3c218, 0xffffffff, 0x06000100,
1494 0x3c204, 0xffffffff, 0x00000100,
1495 0x3c2e0, 0xffffffff, 0x00000100,
1496 0x3c224, 0xffffffff, 0x00000100,
1497 0x3c200, 0xffffffff, 0x00000100,
1498 0x3c230, 0xffffffff, 0x00000100,
1499 0x3c234, 0xffffffff, 0x00000100,
1500 0x3c250, 0xffffffff, 0x00000100,
1501 0x3c254, 0xffffffff, 0x00000100,
1502 0x3c258, 0xffffffff, 0x00000100,
1503 0x3c25c, 0xffffffff, 0x00000100,
1504 0x3c260, 0xffffffff, 0x00000100,
1505 0x3c27c, 0xffffffff, 0x00000100,
1506 0x3c278, 0xffffffff, 0x00000100,
1507 0x3c210, 0xffffffff, 0x06000100,
1508 0x3c290, 0xffffffff, 0x00000100,
1509 0x3c274, 0xffffffff, 0x00000100,
1510 0x3c2b4, 0xffffffff, 0x00000100,
1511 0x3c2b0, 0xffffffff, 0x00000100,
1512 0x3c270, 0xffffffff, 0x00000100,
1513 0x30800, 0xffffffff, 0xe0000000,
1514 0x3c020, 0xffffffff, 0x00010000,
1515 0x3c024, 0xffffffff, 0x00030002,
1516 0x3c028, 0xffffffff, 0x00040007,
1517 0x3c02c, 0xffffffff, 0x00060005,
1518 0x3c030, 0xffffffff, 0x00090008,
1519 0x3c034, 0xffffffff, 0x00010000,
1520 0x3c038, 0xffffffff, 0x00030002,
1521 0x3c03c, 0xffffffff, 0x00040007,
1522 0x3c040, 0xffffffff, 0x00060005,
1523 0x3c044, 0xffffffff, 0x00090008,
1524 0x3c048, 0xffffffff, 0x00010000,
1525 0x3c04c, 0xffffffff, 0x00030002,
1526 0x3c050, 0xffffffff, 0x00040007,
1527 0x3c054, 0xffffffff, 0x00060005,
1528 0x3c058, 0xffffffff, 0x00090008,
1529 0x3c05c, 0xffffffff, 0x00010000,
1530 0x3c060, 0xffffffff, 0x00030002,
1531 0x3c064, 0xffffffff, 0x00040007,
1532 0x3c068, 0xffffffff, 0x00060005,
1533 0x3c06c, 0xffffffff, 0x00090008,
1534 0x3c070, 0xffffffff, 0x00010000,
1535 0x3c074, 0xffffffff, 0x00030002,
1536 0x3c078, 0xffffffff, 0x00040007,
1537 0x3c07c, 0xffffffff, 0x00060005,
1538 0x3c080, 0xffffffff, 0x00090008,
1539 0x3c084, 0xffffffff, 0x00010000,
1540 0x3c088, 0xffffffff, 0x00030002,
1541 0x3c08c, 0xffffffff, 0x00040007,
1542 0x3c090, 0xffffffff, 0x00060005,
1543 0x3c094, 0xffffffff, 0x00090008,
1544 0x3c098, 0xffffffff, 0x00010000,
1545 0x3c09c, 0xffffffff, 0x00030002,
1546 0x3c0a0, 0xffffffff, 0x00040007,
1547 0x3c0a4, 0xffffffff, 0x00060005,
1548 0x3c0a8, 0xffffffff, 0x00090008,
1549 0x3c0ac, 0xffffffff, 0x00010000,
1550 0x3c0b0, 0xffffffff, 0x00030002,
1551 0x3c0b4, 0xffffffff, 0x00040007,
1552 0x3c0b8, 0xffffffff, 0x00060005,
1553 0x3c0bc, 0xffffffff, 0x00090008,
1554 0x3c0c0, 0xffffffff, 0x00010000,
1555 0x3c0c4, 0xffffffff, 0x00030002,
1556 0x3c0c8, 0xffffffff, 0x00040007,
1557 0x3c0cc, 0xffffffff, 0x00060005,
1558 0x3c0d0, 0xffffffff, 0x00090008,
1559 0x3c0d4, 0xffffffff, 0x00010000,
1560 0x3c0d8, 0xffffffff, 0x00030002,
1561 0x3c0dc, 0xffffffff, 0x00040007,
1562 0x3c0e0, 0xffffffff, 0x00060005,
1563 0x3c0e4, 0xffffffff, 0x00090008,
1564 0x3c0e8, 0xffffffff, 0x00010000,
1565 0x3c0ec, 0xffffffff, 0x00030002,
1566 0x3c0f0, 0xffffffff, 0x00040007,
1567 0x3c0f4, 0xffffffff, 0x00060005,
1568 0x3c0f8, 0xffffffff, 0x00090008,
1569 0xc318, 0xffffffff, 0x00020200,
1570 0x3350, 0xffffffff, 0x00000200,
1571 0x15c0, 0xffffffff, 0x00000400,
1572 0x55e8, 0xffffffff, 0x00000000,
1573 0x2f50, 0xffffffff, 0x00000902,
1574 0x3c000, 0xffffffff, 0x96940200,
1575 0x8708, 0xffffffff, 0x00900100,
1576 0xc424, 0xffffffff, 0x0020003f,
1577 0x38, 0xffffffff, 0x0140001c,
1578 0x3c, 0x000f0000, 0x000f0000,
1579 0x220, 0xffffffff, 0xc060000c,
1580 0x224, 0xc0000fff, 0x00000100,
1581 0xf90, 0xffffffff, 0x00000100,
1582 0xf98, 0x00000101, 0x00000000,
1583 0x20a8, 0xffffffff, 0x00000104,
1584 0x55e4, 0xff000fff, 0x00000100,
1585 0x30cc, 0xc0000fff, 0x00000104,
1586 0xc1e4, 0x00000001, 0x00000001,
1587 0xd00c, 0xff000ff0, 0x00000100,
1588 0xd80c, 0xff000ff0, 0x00000100
1589 };
1590
1591 static const u32 godavari_golden_registers[] =
1592 {
1593 0x55e4, 0xff607fff, 0xfc000100,
1594 0x6ed8, 0x00010101, 0x00010000,
1595 0x9830, 0xffffffff, 0x00000000,
1596 0x98302, 0xf00fffff, 0x00000400,
1597 0x6130, 0xffffffff, 0x00010000,
1598 0x5bb0, 0x000000f0, 0x00000070,
1599 0x5bc0, 0xf0311fff, 0x80300000,
1600 0x98f8, 0x73773777, 0x12010001,
1601 0x98fc, 0xffffffff, 0x00000010,
1602 0x8030, 0x00001f0f, 0x0000100a,
1603 0x2f48, 0x73773777, 0x12010001,
1604 0x2408, 0x000fffff, 0x000c007f,
1605 0x8a14, 0xf000003f, 0x00000007,
1606 0x8b24, 0xffffffff, 0x00ff0fff,
1607 0x30a04, 0x0000ff0f, 0x00000000,
1608 0x28a4c, 0x07ffffff, 0x06000000,
1609 0x4d8, 0x00000fff, 0x00000100,
1610 0xd014, 0x00010000, 0x00810001,
1611 0xd814, 0x00010000, 0x00810001,
1612 0x3e78, 0x00000001, 0x00000002,
1613 0xc768, 0x00000008, 0x00000008,
1614 0xc770, 0x00000f00, 0x00000800,
1615 0xc774, 0x00000f00, 0x00000800,
1616 0xc798, 0x00ffffff, 0x00ff7fbf,
1617 0xc79c, 0x00ffffff, 0x00ff7faf,
1618 0x8c00, 0x000000ff, 0x00000001,
1619 0x214f8, 0x01ff01ff, 0x00000002,
1620 0x21498, 0x007ff800, 0x00200000,
1621 0x2015c, 0xffffffff, 0x00000f40,
1622 0x88c4, 0x001f3ae3, 0x00000082,
1623 0x88d4, 0x0000001f, 0x00000010,
1624 0x30934, 0xffffffff, 0x00000000
1625 };
1626
1627
1628 static void cik_init_golden_registers(struct radeon_device *rdev)
1629 {
1630 /* Some of the registers might be dependent on GRBM_GFX_INDEX */
1631 mutex_lock(&rdev->grbm_idx_mutex);
1632 switch (rdev->family) {
1633 case CHIP_BONAIRE:
1634 radeon_program_register_sequence(rdev,
1635 bonaire_mgcg_cgcg_init,
1636 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1637 radeon_program_register_sequence(rdev,
1638 bonaire_golden_registers,
1639 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1640 radeon_program_register_sequence(rdev,
1641 bonaire_golden_common_registers,
1642 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1643 radeon_program_register_sequence(rdev,
1644 bonaire_golden_spm_registers,
1645 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1646 break;
1647 case CHIP_KABINI:
1648 radeon_program_register_sequence(rdev,
1649 kalindi_mgcg_cgcg_init,
1650 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1651 radeon_program_register_sequence(rdev,
1652 kalindi_golden_registers,
1653 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1654 radeon_program_register_sequence(rdev,
1655 kalindi_golden_common_registers,
1656 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1657 radeon_program_register_sequence(rdev,
1658 kalindi_golden_spm_registers,
1659 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1660 break;
1661 case CHIP_MULLINS:
1662 radeon_program_register_sequence(rdev,
1663 kalindi_mgcg_cgcg_init,
1664 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1665 radeon_program_register_sequence(rdev,
1666 godavari_golden_registers,
1667 (const u32)ARRAY_SIZE(godavari_golden_registers));
1668 radeon_program_register_sequence(rdev,
1669 kalindi_golden_common_registers,
1670 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1671 radeon_program_register_sequence(rdev,
1672 kalindi_golden_spm_registers,
1673 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1674 break;
1675 case CHIP_KAVERI:
1676 radeon_program_register_sequence(rdev,
1677 spectre_mgcg_cgcg_init,
1678 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1679 radeon_program_register_sequence(rdev,
1680 spectre_golden_registers,
1681 (const u32)ARRAY_SIZE(spectre_golden_registers));
1682 radeon_program_register_sequence(rdev,
1683 spectre_golden_common_registers,
1684 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1685 radeon_program_register_sequence(rdev,
1686 spectre_golden_spm_registers,
1687 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1688 break;
1689 case CHIP_HAWAII:
1690 radeon_program_register_sequence(rdev,
1691 hawaii_mgcg_cgcg_init,
1692 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1693 radeon_program_register_sequence(rdev,
1694 hawaii_golden_registers,
1695 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1696 radeon_program_register_sequence(rdev,
1697 hawaii_golden_common_registers,
1698 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1699 radeon_program_register_sequence(rdev,
1700 hawaii_golden_spm_registers,
1701 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1702 break;
1703 default:
1704 break;
1705 }
1706 mutex_unlock(&rdev->grbm_idx_mutex);
1707 }
1708
1709 /**
1710 * cik_get_xclk - get the xclk
1711 *
1712 * @rdev: radeon_device pointer
1713 *
1714 * Returns the reference clock used by the gfx engine
1715 * (CIK).
1716 */
1717 u32 cik_get_xclk(struct radeon_device *rdev)
1718 {
1719 u32 reference_clock = rdev->clock.spll.reference_freq;
1720
1721 if (rdev->flags & RADEON_IS_IGP) {
1722 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1723 return reference_clock / 2;
1724 } else {
1725 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1726 return reference_clock / 4;
1727 }
1728 return reference_clock;
1729 }
1730
1731 /**
1732 * cik_mm_rdoorbell - read a doorbell dword
1733 *
1734 * @rdev: radeon_device pointer
1735 * @index: doorbell index
1736 *
1737 * Returns the value in the doorbell aperture at the
1738 * requested doorbell index (CIK).
1739 */
1740 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1741 {
1742 if (index < rdev->doorbell.num_doorbells) {
1743 return readl(rdev->doorbell.ptr + index);
1744 } else {
1745 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1746 return 0;
1747 }
1748 }
1749
1750 /**
1751 * cik_mm_wdoorbell - write a doorbell dword
1752 *
1753 * @rdev: radeon_device pointer
1754 * @index: doorbell index
1755 * @v: value to write
1756 *
1757 * Writes @v to the doorbell aperture at the
1758 * requested doorbell index (CIK).
1759 */
1760 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1761 {
1762 if (index < rdev->doorbell.num_doorbells) {
1763 writel(v, rdev->doorbell.ptr + index);
1764 } else {
1765 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1766 }
1767 }
1768
1769 #define BONAIRE_IO_MC_REGS_SIZE 36
1770
1771 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1772 {
1773 {0x00000070, 0x04400000},
1774 {0x00000071, 0x80c01803},
1775 {0x00000072, 0x00004004},
1776 {0x00000073, 0x00000100},
1777 {0x00000074, 0x00ff0000},
1778 {0x00000075, 0x34000000},
1779 {0x00000076, 0x08000014},
1780 {0x00000077, 0x00cc08ec},
1781 {0x00000078, 0x00000400},
1782 {0x00000079, 0x00000000},
1783 {0x0000007a, 0x04090000},
1784 {0x0000007c, 0x00000000},
1785 {0x0000007e, 0x4408a8e8},
1786 {0x0000007f, 0x00000304},
1787 {0x00000080, 0x00000000},
1788 {0x00000082, 0x00000001},
1789 {0x00000083, 0x00000002},
1790 {0x00000084, 0xf3e4f400},
1791 {0x00000085, 0x052024e3},
1792 {0x00000087, 0x00000000},
1793 {0x00000088, 0x01000000},
1794 {0x0000008a, 0x1c0a0000},
1795 {0x0000008b, 0xff010000},
1796 {0x0000008d, 0xffffefff},
1797 {0x0000008e, 0xfff3efff},
1798 {0x0000008f, 0xfff3efbf},
1799 {0x00000092, 0xf7ffffff},
1800 {0x00000093, 0xffffff7f},
1801 {0x00000095, 0x00101101},
1802 {0x00000096, 0x00000fff},
1803 {0x00000097, 0x00116fff},
1804 {0x00000098, 0x60010000},
1805 {0x00000099, 0x10010000},
1806 {0x0000009a, 0x00006000},
1807 {0x0000009b, 0x00001000},
1808 {0x0000009f, 0x00b48000}
1809 };
1810
1811 #define HAWAII_IO_MC_REGS_SIZE 22
1812
1813 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1814 {
1815 {0x0000007d, 0x40000000},
1816 {0x0000007e, 0x40180304},
1817 {0x0000007f, 0x0000ff00},
1818 {0x00000081, 0x00000000},
1819 {0x00000083, 0x00000800},
1820 {0x00000086, 0x00000000},
1821 {0x00000087, 0x00000100},
1822 {0x00000088, 0x00020100},
1823 {0x00000089, 0x00000000},
1824 {0x0000008b, 0x00040000},
1825 {0x0000008c, 0x00000100},
1826 {0x0000008e, 0xff010000},
1827 {0x00000090, 0xffffefff},
1828 {0x00000091, 0xfff3efff},
1829 {0x00000092, 0xfff3efbf},
1830 {0x00000093, 0xf7ffffff},
1831 {0x00000094, 0xffffff7f},
1832 {0x00000095, 0x00000fff},
1833 {0x00000096, 0x00116fff},
1834 {0x00000097, 0x60010000},
1835 {0x00000098, 0x10010000},
1836 {0x0000009f, 0x00c79000}
1837 };
1838
1839
1840 /**
1841 * cik_srbm_select - select specific register instances
1842 *
1843 * @rdev: radeon_device pointer
1844 * @me: selected ME (micro engine)
1845 * @pipe: pipe
1846 * @queue: queue
1847 * @vmid: VMID
1848 *
1849 * Switches the currently active registers instances. Some
1850 * registers are instanced per VMID, others are instanced per
1851 * me/pipe/queue combination.
1852 */
1853 static void cik_srbm_select(struct radeon_device *rdev,
1854 u32 me, u32 pipe, u32 queue, u32 vmid)
1855 {
1856 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1857 MEID(me & 0x3) |
1858 VMID(vmid & 0xf) |
1859 QUEUEID(queue & 0x7));
1860 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1861 }
1862
1863 /* ucode loading */
1864 /**
1865 * ci_mc_load_microcode - load MC ucode into the hw
1866 *
1867 * @rdev: radeon_device pointer
1868 *
1869 * Load the GDDR MC ucode into the hw (CIK).
1870 * Returns 0 on success, error on failure.
1871 */
1872 int ci_mc_load_microcode(struct radeon_device *rdev)
1873 {
1874 const __be32 *fw_data = NULL;
1875 const __le32 *new_fw_data = NULL;
1876 u32 running, tmp;
1877 u32 *io_mc_regs = NULL;
1878 const __le32 *new_io_mc_regs = NULL;
1879 int i, regs_size, ucode_size;
1880
1881 if (!rdev->mc_fw)
1882 return -EINVAL;
1883
1884 if (rdev->new_fw) {
1885 const struct mc_firmware_header_v1_0 *hdr =
1886 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1887
1888 radeon_ucode_print_mc_hdr(&hdr->header);
1889
1890 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1891 new_io_mc_regs = (const __le32 *)
1892 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1893 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1894 new_fw_data = (const __le32 *)
1895 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1896 } else {
1897 ucode_size = rdev->mc_fw->size / 4;
1898
1899 switch (rdev->family) {
1900 case CHIP_BONAIRE:
1901 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1902 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1903 break;
1904 case CHIP_HAWAII:
1905 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1906 regs_size = HAWAII_IO_MC_REGS_SIZE;
1907 break;
1908 default:
1909 return -EINVAL;
1910 }
1911 fw_data = (const __be32 *)rdev->mc_fw->data;
1912 }
1913
1914 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1915
1916 if (running == 0) {
1917 /* reset the engine and set to writable */
1918 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1919 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1920
1921 /* load mc io regs */
1922 for (i = 0; i < regs_size; i++) {
1923 if (rdev->new_fw) {
1924 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1925 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1926 } else {
1927 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1928 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1929 }
1930 }
1931
1932 tmp = RREG32(MC_SEQ_MISC0);
1933 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {
1934 WREG32(MC_SEQ_IO_DEBUG_INDEX, 5);
1935 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023);
1936 WREG32(MC_SEQ_IO_DEBUG_INDEX, 9);
1937 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0);
1938 }
1939
1940 /* load the MC ucode */
1941 for (i = 0; i < ucode_size; i++) {
1942 if (rdev->new_fw)
1943 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1944 else
1945 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1946 }
1947
1948 /* put the engine back into the active state */
1949 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1950 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1951 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1952
1953 /* wait for training to complete */
1954 for (i = 0; i < rdev->usec_timeout; i++) {
1955 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1956 break;
1957 udelay(1);
1958 }
1959 for (i = 0; i < rdev->usec_timeout; i++) {
1960 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1961 break;
1962 udelay(1);
1963 }
1964 }
1965
1966 return 0;
1967 }
1968
1969 /**
1970 * cik_init_microcode - load ucode images from disk
1971 *
1972 * @rdev: radeon_device pointer
1973 *
1974 * Use the firmware interface to load the ucode images into
1975 * the driver (not loaded into hw).
1976 * Returns 0 on success, error on failure.
1977 */
1978 static int cik_init_microcode(struct radeon_device *rdev)
1979 {
1980 const char *chip_name;
1981 const char *new_chip_name;
1982 size_t pfp_req_size, me_req_size, ce_req_size,
1983 mec_req_size, rlc_req_size, mc_req_size = 0,
1984 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1985 char fw_name[30];
1986 int new_fw = 0;
1987 int err;
1988 int num_fw;
1989 bool new_smc = false;
1990
1991 DRM_DEBUG("\n");
1992
1993 switch (rdev->family) {
1994 case CHIP_BONAIRE:
1995 chip_name = "BONAIRE";
1996 if ((rdev->pdev->revision == 0x80) ||
1997 (rdev->pdev->revision == 0x81) ||
1998 (rdev->pdev->device == 0x665f))
1999 new_smc = true;
2000 new_chip_name = "bonaire";
2001 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2002 me_req_size = CIK_ME_UCODE_SIZE * 4;
2003 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2004 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2005 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
2006 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
2007 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
2008 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2009 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
2010 num_fw = 8;
2011 break;
2012 case CHIP_HAWAII:
2013 chip_name = "HAWAII";
2014 if (rdev->pdev->revision == 0x80)
2015 new_smc = true;
2016 new_chip_name = "hawaii";
2017 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2018 me_req_size = CIK_ME_UCODE_SIZE * 4;
2019 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2020 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2021 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
2022 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
2023 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
2024 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2025 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
2026 num_fw = 8;
2027 break;
2028 case CHIP_KAVERI:
2029 chip_name = "KAVERI";
2030 new_chip_name = "kaveri";
2031 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2032 me_req_size = CIK_ME_UCODE_SIZE * 4;
2033 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2034 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2035 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
2036 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2037 num_fw = 7;
2038 break;
2039 case CHIP_KABINI:
2040 chip_name = "KABINI";
2041 new_chip_name = "kabini";
2042 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2043 me_req_size = CIK_ME_UCODE_SIZE * 4;
2044 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2045 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2046 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
2047 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2048 num_fw = 6;
2049 break;
2050 case CHIP_MULLINS:
2051 chip_name = "MULLINS";
2052 new_chip_name = "mullins";
2053 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2054 me_req_size = CIK_ME_UCODE_SIZE * 4;
2055 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2056 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2057 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
2058 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2059 num_fw = 6;
2060 break;
2061 default: BUG();
2062 }
2063
2064 DRM_INFO("Loading %s Microcode\n", new_chip_name);
2065
2066 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
2067 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2068 if (err) {
2069 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2070 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2071 if (err)
2072 goto out;
2073 if (rdev->pfp_fw->size != pfp_req_size) {
2074 pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2075 rdev->pfp_fw->size, fw_name);
2076 err = -EINVAL;
2077 goto out;
2078 }
2079 } else {
2080 err = radeon_ucode_validate(rdev->pfp_fw);
2081 if (err) {
2082 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2083 fw_name);
2084 goto out;
2085 } else {
2086 new_fw++;
2087 }
2088 }
2089
2090 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
2091 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2092 if (err) {
2093 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2094 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2095 if (err)
2096 goto out;
2097 if (rdev->me_fw->size != me_req_size) {
2098 pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2099 rdev->me_fw->size, fw_name);
2100 err = -EINVAL;
2101 }
2102 } else {
2103 err = radeon_ucode_validate(rdev->me_fw);
2104 if (err) {
2105 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2106 fw_name);
2107 goto out;
2108 } else {
2109 new_fw++;
2110 }
2111 }
2112
2113 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
2114 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2115 if (err) {
2116 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2117 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2118 if (err)
2119 goto out;
2120 if (rdev->ce_fw->size != ce_req_size) {
2121 pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2122 rdev->ce_fw->size, fw_name);
2123 err = -EINVAL;
2124 }
2125 } else {
2126 err = radeon_ucode_validate(rdev->ce_fw);
2127 if (err) {
2128 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2129 fw_name);
2130 goto out;
2131 } else {
2132 new_fw++;
2133 }
2134 }
2135
2136 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
2137 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2138 if (err) {
2139 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2140 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2141 if (err)
2142 goto out;
2143 if (rdev->mec_fw->size != mec_req_size) {
2144 pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2145 rdev->mec_fw->size, fw_name);
2146 err = -EINVAL;
2147 }
2148 } else {
2149 err = radeon_ucode_validate(rdev->mec_fw);
2150 if (err) {
2151 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2152 fw_name);
2153 goto out;
2154 } else {
2155 new_fw++;
2156 }
2157 }
2158
2159 if (rdev->family == CHIP_KAVERI) {
2160 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2161 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2162 if (err) {
2163 goto out;
2164 } else {
2165 err = radeon_ucode_validate(rdev->mec2_fw);
2166 if (err) {
2167 goto out;
2168 } else {
2169 new_fw++;
2170 }
2171 }
2172 }
2173
2174 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
2175 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2176 if (err) {
2177 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2178 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2179 if (err)
2180 goto out;
2181 if (rdev->rlc_fw->size != rlc_req_size) {
2182 pr_err("cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2183 rdev->rlc_fw->size, fw_name);
2184 err = -EINVAL;
2185 }
2186 } else {
2187 err = radeon_ucode_validate(rdev->rlc_fw);
2188 if (err) {
2189 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2190 fw_name);
2191 goto out;
2192 } else {
2193 new_fw++;
2194 }
2195 }
2196
2197 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
2198 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2199 if (err) {
2200 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2201 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2202 if (err)
2203 goto out;
2204 if (rdev->sdma_fw->size != sdma_req_size) {
2205 pr_err("cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2206 rdev->sdma_fw->size, fw_name);
2207 err = -EINVAL;
2208 }
2209 } else {
2210 err = radeon_ucode_validate(rdev->sdma_fw);
2211 if (err) {
2212 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2213 fw_name);
2214 goto out;
2215 } else {
2216 new_fw++;
2217 }
2218 }
2219
2220 /* No SMC, MC ucode on APUs */
2221 if (!(rdev->flags & RADEON_IS_IGP)) {
2222 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
2223 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2224 if (err) {
2225 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
2226 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2227 if (err) {
2228 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2229 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2230 if (err)
2231 goto out;
2232 }
2233 if ((rdev->mc_fw->size != mc_req_size) &&
2234 (rdev->mc_fw->size != mc2_req_size)){
2235 pr_err("cik_mc: Bogus length %zu in firmware \"%s\"\n",
2236 rdev->mc_fw->size, fw_name);
2237 err = -EINVAL;
2238 }
2239 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2240 } else {
2241 err = radeon_ucode_validate(rdev->mc_fw);
2242 if (err) {
2243 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2244 fw_name);
2245 goto out;
2246 } else {
2247 new_fw++;
2248 }
2249 }
2250
2251 if (new_smc)
2252 snprintf(fw_name, sizeof(fw_name), "radeon/%s_k_smc.bin", new_chip_name);
2253 else
2254 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
2255 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2256 if (err) {
2257 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2258 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2259 if (err) {
2260 pr_err("smc: error loading firmware \"%s\"\n",
2261 fw_name);
2262 release_firmware(rdev->smc_fw);
2263 rdev->smc_fw = NULL;
2264 err = 0;
2265 } else if (rdev->smc_fw->size != smc_req_size) {
2266 pr_err("cik_smc: Bogus length %zu in firmware \"%s\"\n",
2267 rdev->smc_fw->size, fw_name);
2268 err = -EINVAL;
2269 }
2270 } else {
2271 err = radeon_ucode_validate(rdev->smc_fw);
2272 if (err) {
2273 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2274 fw_name);
2275 goto out;
2276 } else {
2277 new_fw++;
2278 }
2279 }
2280 }
2281
2282 if (new_fw == 0) {
2283 rdev->new_fw = false;
2284 } else if (new_fw < num_fw) {
2285 pr_err("ci_fw: mixing new and old firmware!\n");
2286 err = -EINVAL;
2287 } else {
2288 rdev->new_fw = true;
2289 }
2290
2291 out:
2292 if (err) {
2293 if (err != -EINVAL)
2294 pr_err("cik_cp: Failed to load firmware \"%s\"\n",
2295 fw_name);
2296 release_firmware(rdev->pfp_fw);
2297 rdev->pfp_fw = NULL;
2298 release_firmware(rdev->me_fw);
2299 rdev->me_fw = NULL;
2300 release_firmware(rdev->ce_fw);
2301 rdev->ce_fw = NULL;
2302 release_firmware(rdev->mec_fw);
2303 rdev->mec_fw = NULL;
2304 release_firmware(rdev->mec2_fw);
2305 rdev->mec2_fw = NULL;
2306 release_firmware(rdev->rlc_fw);
2307 rdev->rlc_fw = NULL;
2308 release_firmware(rdev->sdma_fw);
2309 rdev->sdma_fw = NULL;
2310 release_firmware(rdev->mc_fw);
2311 rdev->mc_fw = NULL;
2312 release_firmware(rdev->smc_fw);
2313 rdev->smc_fw = NULL;
2314 }
2315 return err;
2316 }
2317
2318 /*
2319 * Core functions
2320 */
2321 /**
2322 * cik_tiling_mode_table_init - init the hw tiling table
2323 *
2324 * @rdev: radeon_device pointer
2325 *
2326 * Starting with SI, the tiling setup is done globally in a
2327 * set of 32 tiling modes. Rather than selecting each set of
2328 * parameters per surface as on older asics, we just select
2329 * which index in the tiling table we want to use, and the
2330 * surface uses those parameters (CIK).
2331 */
2332 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2333 {
2334 u32 *tile = rdev->config.cik.tile_mode_array;
2335 u32 *macrotile = rdev->config.cik.macrotile_mode_array;
2336 const u32 num_tile_mode_states =
2337 ARRAY_SIZE(rdev->config.cik.tile_mode_array);
2338 const u32 num_secondary_tile_mode_states =
2339 ARRAY_SIZE(rdev->config.cik.macrotile_mode_array);
2340 u32 reg_offset, split_equal_to_row_size;
2341 u32 num_pipe_configs;
2342 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2343 rdev->config.cik.max_shader_engines;
2344
2345 switch (rdev->config.cik.mem_row_size_in_kb) {
2346 case 1:
2347 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2348 break;
2349 case 2:
2350 default:
2351 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2352 break;
2353 case 4:
2354 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2355 break;
2356 }
2357
2358 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2359 if (num_pipe_configs > 8)
2360 num_pipe_configs = 16;
2361
2362 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2363 tile[reg_offset] = 0;
2364 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2365 macrotile[reg_offset] = 0;
2366
2367 switch(num_pipe_configs) {
2368 case 16:
2369 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2370 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2371 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2372 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2373 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2374 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2375 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2376 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2377 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2378 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2379 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2380 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2381 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2382 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2383 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2384 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2385 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2386 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2387 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2388 TILE_SPLIT(split_equal_to_row_size));
2389 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2390 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2391 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2392 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2393 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2394 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2395 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2396 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2397 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2398 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2399 TILE_SPLIT(split_equal_to_row_size));
2400 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2401 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2402 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2403 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2404 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2405 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2406 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2407 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2408 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2409 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2410 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2411 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2412 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2413 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2414 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2415 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2416 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2417 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2418 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2419 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2420 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2421 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2422 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2423 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2424 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2425 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2426 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2427 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2428 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2429 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2430 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2431 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2432 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2433 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2434 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2435 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2436 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2437 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2438 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2439 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2440 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2441 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2442 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2443 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2444 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2445 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2446 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2447
2448 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2449 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2450 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2451 NUM_BANKS(ADDR_SURF_16_BANK));
2452 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2453 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2454 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2455 NUM_BANKS(ADDR_SURF_16_BANK));
2456 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2457 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2458 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2459 NUM_BANKS(ADDR_SURF_16_BANK));
2460 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2461 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2462 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2463 NUM_BANKS(ADDR_SURF_16_BANK));
2464 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2465 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2466 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2467 NUM_BANKS(ADDR_SURF_8_BANK));
2468 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2469 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2470 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2471 NUM_BANKS(ADDR_SURF_4_BANK));
2472 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2473 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2474 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2475 NUM_BANKS(ADDR_SURF_2_BANK));
2476 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2477 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2478 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2479 NUM_BANKS(ADDR_SURF_16_BANK));
2480 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2481 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2482 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2483 NUM_BANKS(ADDR_SURF_16_BANK));
2484 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2485 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2486 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2487 NUM_BANKS(ADDR_SURF_16_BANK));
2488 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2489 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2490 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2491 NUM_BANKS(ADDR_SURF_8_BANK));
2492 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2493 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2494 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2495 NUM_BANKS(ADDR_SURF_4_BANK));
2496 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2497 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2498 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2499 NUM_BANKS(ADDR_SURF_2_BANK));
2500 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2501 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2502 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2503 NUM_BANKS(ADDR_SURF_2_BANK));
2504
2505 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2506 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2507 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2508 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2509 break;
2510
2511 case 8:
2512 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2513 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2514 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2515 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2516 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2517 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2518 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2519 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2520 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2521 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2522 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2523 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2524 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2525 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2526 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2527 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2528 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2529 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2530 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2531 TILE_SPLIT(split_equal_to_row_size));
2532 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2533 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2534 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2535 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2536 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2537 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2538 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2539 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2540 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2541 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2542 TILE_SPLIT(split_equal_to_row_size));
2543 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2544 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2545 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2546 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2547 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2548 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2549 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2550 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2551 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2552 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2553 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2554 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2555 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2556 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2557 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2558 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2559 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2560 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2561 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2562 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2563 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2564 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2565 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2566 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2567 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2568 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2569 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2570 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2571 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2572 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2573 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2574 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2575 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2576 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2577 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2578 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2579 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2580 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2581 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2582 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2583 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2584 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2585 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2586 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2587 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2588 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2589 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2590
2591 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2592 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2593 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2594 NUM_BANKS(ADDR_SURF_16_BANK));
2595 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2596 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2597 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2598 NUM_BANKS(ADDR_SURF_16_BANK));
2599 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2600 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2601 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2602 NUM_BANKS(ADDR_SURF_16_BANK));
2603 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2604 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2605 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2606 NUM_BANKS(ADDR_SURF_16_BANK));
2607 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2608 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2609 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2610 NUM_BANKS(ADDR_SURF_8_BANK));
2611 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2612 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2613 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2614 NUM_BANKS(ADDR_SURF_4_BANK));
2615 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2616 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2617 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2618 NUM_BANKS(ADDR_SURF_2_BANK));
2619 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2620 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2621 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2622 NUM_BANKS(ADDR_SURF_16_BANK));
2623 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2624 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2625 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2626 NUM_BANKS(ADDR_SURF_16_BANK));
2627 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2628 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2629 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2630 NUM_BANKS(ADDR_SURF_16_BANK));
2631 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2632 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2633 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2634 NUM_BANKS(ADDR_SURF_16_BANK));
2635 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2636 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2637 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2638 NUM_BANKS(ADDR_SURF_8_BANK));
2639 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2640 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2641 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2642 NUM_BANKS(ADDR_SURF_4_BANK));
2643 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2644 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2645 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2646 NUM_BANKS(ADDR_SURF_2_BANK));
2647
2648 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2649 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2650 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2651 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2652 break;
2653
2654 case 4:
2655 if (num_rbs == 4) {
2656 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2657 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2658 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2659 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2660 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2661 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2662 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2663 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2664 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2665 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2666 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2667 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2668 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2669 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2670 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2671 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2672 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2673 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2674 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2675 TILE_SPLIT(split_equal_to_row_size));
2676 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2677 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2678 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2679 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2680 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2681 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2682 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2683 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2684 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2685 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2686 TILE_SPLIT(split_equal_to_row_size));
2687 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2688 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2689 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2690 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2691 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2692 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2693 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2694 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2695 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2696 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2697 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2698 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2699 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2700 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2701 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2702 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2703 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2704 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2705 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2706 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2707 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2708 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2709 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2710 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2711 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2712 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2713 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2714 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2715 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2716 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2717 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2718 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2719 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2720 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2721 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2722 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2723 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2724 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2725 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2726 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2727 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2728 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2729 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2730 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2731 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2732 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2733 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2734
2735 } else if (num_rbs < 4) {
2736 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2737 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2738 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2739 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2740 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2741 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2742 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2743 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2744 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2745 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2746 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2747 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2748 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2749 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2750 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2751 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2752 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2753 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2754 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2755 TILE_SPLIT(split_equal_to_row_size));
2756 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2757 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2758 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2759 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2760 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2761 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2762 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2763 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2764 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2765 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2766 TILE_SPLIT(split_equal_to_row_size));
2767 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2768 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2769 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2770 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2771 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2772 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2773 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2774 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2775 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2776 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2777 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2778 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2779 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2780 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2781 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2782 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2783 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2784 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2785 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2786 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2787 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2788 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2789 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2790 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2791 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2792 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2793 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2794 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2795 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2796 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2797 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2798 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2799 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2800 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2801 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2802 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2803 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2804 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2805 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2806 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2807 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2808 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2809 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2810 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2811 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2812 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2813 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2814 }
2815
2816 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2817 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2818 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2819 NUM_BANKS(ADDR_SURF_16_BANK));
2820 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2821 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2822 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2823 NUM_BANKS(ADDR_SURF_16_BANK));
2824 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2825 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2826 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2827 NUM_BANKS(ADDR_SURF_16_BANK));
2828 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2829 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2830 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2831 NUM_BANKS(ADDR_SURF_16_BANK));
2832 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2833 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2834 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2835 NUM_BANKS(ADDR_SURF_16_BANK));
2836 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2837 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2838 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2839 NUM_BANKS(ADDR_SURF_8_BANK));
2840 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2841 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2842 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2843 NUM_BANKS(ADDR_SURF_4_BANK));
2844 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2845 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2846 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2847 NUM_BANKS(ADDR_SURF_16_BANK));
2848 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2849 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2850 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2851 NUM_BANKS(ADDR_SURF_16_BANK));
2852 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2853 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2854 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2855 NUM_BANKS(ADDR_SURF_16_BANK));
2856 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2857 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2858 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2859 NUM_BANKS(ADDR_SURF_16_BANK));
2860 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2861 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2862 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2863 NUM_BANKS(ADDR_SURF_16_BANK));
2864 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2865 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2866 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2867 NUM_BANKS(ADDR_SURF_8_BANK));
2868 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2869 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2870 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2871 NUM_BANKS(ADDR_SURF_4_BANK));
2872
2873 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2874 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2875 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2876 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2877 break;
2878
2879 case 2:
2880 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2881 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2882 PIPE_CONFIG(ADDR_SURF_P2) |
2883 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2884 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2885 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2886 PIPE_CONFIG(ADDR_SURF_P2) |
2887 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2888 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2889 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2890 PIPE_CONFIG(ADDR_SURF_P2) |
2891 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2892 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2893 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2894 PIPE_CONFIG(ADDR_SURF_P2) |
2895 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2896 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2897 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2898 PIPE_CONFIG(ADDR_SURF_P2) |
2899 TILE_SPLIT(split_equal_to_row_size));
2900 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2901 PIPE_CONFIG(ADDR_SURF_P2) |
2902 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2903 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2904 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2905 PIPE_CONFIG(ADDR_SURF_P2) |
2906 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2907 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2908 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2909 PIPE_CONFIG(ADDR_SURF_P2) |
2910 TILE_SPLIT(split_equal_to_row_size));
2911 tile[8] = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2912 PIPE_CONFIG(ADDR_SURF_P2);
2913 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2914 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2915 PIPE_CONFIG(ADDR_SURF_P2));
2916 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2917 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2918 PIPE_CONFIG(ADDR_SURF_P2) |
2919 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2920 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2921 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2922 PIPE_CONFIG(ADDR_SURF_P2) |
2923 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2924 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2925 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2926 PIPE_CONFIG(ADDR_SURF_P2) |
2927 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2928 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2929 PIPE_CONFIG(ADDR_SURF_P2) |
2930 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2931 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2932 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2933 PIPE_CONFIG(ADDR_SURF_P2) |
2934 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2935 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2936 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2937 PIPE_CONFIG(ADDR_SURF_P2) |
2938 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2939 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2940 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2941 PIPE_CONFIG(ADDR_SURF_P2) |
2942 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2943 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2944 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2945 PIPE_CONFIG(ADDR_SURF_P2));
2946 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2947 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2948 PIPE_CONFIG(ADDR_SURF_P2) |
2949 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2950 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2951 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2952 PIPE_CONFIG(ADDR_SURF_P2) |
2953 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2954 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2955 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2956 PIPE_CONFIG(ADDR_SURF_P2) |
2957 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2958
2959 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2960 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2961 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2962 NUM_BANKS(ADDR_SURF_16_BANK));
2963 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2964 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2965 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2966 NUM_BANKS(ADDR_SURF_16_BANK));
2967 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2968 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2969 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2970 NUM_BANKS(ADDR_SURF_16_BANK));
2971 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2972 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2973 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2974 NUM_BANKS(ADDR_SURF_16_BANK));
2975 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2976 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2977 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2978 NUM_BANKS(ADDR_SURF_16_BANK));
2979 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2980 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2981 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2982 NUM_BANKS(ADDR_SURF_16_BANK));
2983 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2984 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2985 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2986 NUM_BANKS(ADDR_SURF_8_BANK));
2987 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2988 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2989 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2990 NUM_BANKS(ADDR_SURF_16_BANK));
2991 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2992 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2993 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2994 NUM_BANKS(ADDR_SURF_16_BANK));
2995 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2996 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2997 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2998 NUM_BANKS(ADDR_SURF_16_BANK));
2999 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3000 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3001 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3002 NUM_BANKS(ADDR_SURF_16_BANK));
3003 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3004 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3005 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3006 NUM_BANKS(ADDR_SURF_16_BANK));
3007 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3008 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3009 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3010 NUM_BANKS(ADDR_SURF_16_BANK));
3011 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3012 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3013 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3014 NUM_BANKS(ADDR_SURF_8_BANK));
3015
3016 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3017 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
3018 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3019 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
3020 break;
3021
3022 default:
3023 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3024 }
3025 }
3026
3027 /**
3028 * cik_select_se_sh - select which SE, SH to address
3029 *
3030 * @rdev: radeon_device pointer
3031 * @se_num: shader engine to address
3032 * @sh_num: sh block to address
3033 *
3034 * Select which SE, SH combinations to address. Certain
3035 * registers are instanced per SE or SH. 0xffffffff means
3036 * broadcast to all SEs or SHs (CIK).
3037 */
3038 static void cik_select_se_sh(struct radeon_device *rdev,
3039 u32 se_num, u32 sh_num)
3040 {
3041 u32 data = INSTANCE_BROADCAST_WRITES;
3042
3043 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3044 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3045 else if (se_num == 0xffffffff)
3046 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3047 else if (sh_num == 0xffffffff)
3048 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3049 else
3050 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3051 WREG32(GRBM_GFX_INDEX, data);
3052 }
3053
3054 /**
3055 * cik_create_bitmask - create a bitmask
3056 *
3057 * @bit_width: length of the mask
3058 *
3059 * create a variable length bit mask (CIK).
3060 * Returns the bitmask.
3061 */
3062 static u32 cik_create_bitmask(u32 bit_width)
3063 {
3064 u32 i, mask = 0;
3065
3066 for (i = 0; i < bit_width; i++) {
3067 mask <<= 1;
3068 mask |= 1;
3069 }
3070 return mask;
3071 }
3072
3073 /**
3074 * cik_get_rb_disabled - computes the mask of disabled RBs
3075 *
3076 * @rdev: radeon_device pointer
3077 * @max_rb_num: max RBs (render backends) for the asic
3078 * @se_num: number of SEs (shader engines) for the asic
3079 * @sh_per_se: number of SH blocks per SE for the asic
3080 *
3081 * Calculates the bitmask of disabled RBs (CIK).
3082 * Returns the disabled RB bitmask.
3083 */
3084 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3085 u32 max_rb_num_per_se,
3086 u32 sh_per_se)
3087 {
3088 u32 data, mask;
3089
3090 data = RREG32(CC_RB_BACKEND_DISABLE);
3091 if (data & 1)
3092 data &= BACKEND_DISABLE_MASK;
3093 else
3094 data = 0;
3095 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3096
3097 data >>= BACKEND_DISABLE_SHIFT;
3098
3099 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3100
3101 return data & mask;
3102 }
3103
3104 /**
3105 * cik_setup_rb - setup the RBs on the asic
3106 *
3107 * @rdev: radeon_device pointer
3108 * @se_num: number of SEs (shader engines) for the asic
3109 * @sh_per_se: number of SH blocks per SE for the asic
3110 * @max_rb_num: max RBs (render backends) for the asic
3111 *
3112 * Configures per-SE/SH RB registers (CIK).
3113 */
3114 static void cik_setup_rb(struct radeon_device *rdev,
3115 u32 se_num, u32 sh_per_se,
3116 u32 max_rb_num_per_se)
3117 {
3118 int i, j;
3119 u32 data, mask;
3120 u32 disabled_rbs = 0;
3121 u32 enabled_rbs = 0;
3122
3123 mutex_lock(&rdev->grbm_idx_mutex);
3124 for (i = 0; i < se_num; i++) {
3125 for (j = 0; j < sh_per_se; j++) {
3126 cik_select_se_sh(rdev, i, j);
3127 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3128 if (rdev->family == CHIP_HAWAII)
3129 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3130 else
3131 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3132 }
3133 }
3134 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3135 mutex_unlock(&rdev->grbm_idx_mutex);
3136
3137 mask = 1;
3138 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3139 if (!(disabled_rbs & mask))
3140 enabled_rbs |= mask;
3141 mask <<= 1;
3142 }
3143
3144 rdev->config.cik.backend_enable_mask = enabled_rbs;
3145
3146 mutex_lock(&rdev->grbm_idx_mutex);
3147 for (i = 0; i < se_num; i++) {
3148 cik_select_se_sh(rdev, i, 0xffffffff);
3149 data = 0;
3150 for (j = 0; j < sh_per_se; j++) {
3151 switch (enabled_rbs & 3) {
3152 case 0:
3153 if (j == 0)
3154 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3155 else
3156 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3157 break;
3158 case 1:
3159 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3160 break;
3161 case 2:
3162 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3163 break;
3164 case 3:
3165 default:
3166 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3167 break;
3168 }
3169 enabled_rbs >>= 2;
3170 }
3171 WREG32(PA_SC_RASTER_CONFIG, data);
3172 }
3173 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3174 mutex_unlock(&rdev->grbm_idx_mutex);
3175 }
3176
3177 /**
3178 * cik_gpu_init - setup the 3D engine
3179 *
3180 * @rdev: radeon_device pointer
3181 *
3182 * Configures the 3D engine and tiling configuration
3183 * registers so that the 3D engine is usable.
3184 */
3185 static void cik_gpu_init(struct radeon_device *rdev)
3186 {
3187 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3188 u32 mc_shared_chmap, mc_arb_ramcfg;
3189 u32 hdp_host_path_cntl;
3190 u32 tmp;
3191 int i, j;
3192
3193 switch (rdev->family) {
3194 case CHIP_BONAIRE:
3195 rdev->config.cik.max_shader_engines = 2;
3196 rdev->config.cik.max_tile_pipes = 4;
3197 rdev->config.cik.max_cu_per_sh = 7;
3198 rdev->config.cik.max_sh_per_se = 1;
3199 rdev->config.cik.max_backends_per_se = 2;
3200 rdev->config.cik.max_texture_channel_caches = 4;
3201 rdev->config.cik.max_gprs = 256;
3202 rdev->config.cik.max_gs_threads = 32;
3203 rdev->config.cik.max_hw_contexts = 8;
3204
3205 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3206 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3207 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3208 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3209 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3210 break;
3211 case CHIP_HAWAII:
3212 rdev->config.cik.max_shader_engines = 4;
3213 rdev->config.cik.max_tile_pipes = 16;
3214 rdev->config.cik.max_cu_per_sh = 11;
3215 rdev->config.cik.max_sh_per_se = 1;
3216 rdev->config.cik.max_backends_per_se = 4;
3217 rdev->config.cik.max_texture_channel_caches = 16;
3218 rdev->config.cik.max_gprs = 256;
3219 rdev->config.cik.max_gs_threads = 32;
3220 rdev->config.cik.max_hw_contexts = 8;
3221
3222 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3223 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3224 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3225 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3226 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3227 break;
3228 case CHIP_KAVERI:
3229 rdev->config.cik.max_shader_engines = 1;
3230 rdev->config.cik.max_tile_pipes = 4;
3231 if ((rdev->pdev->device == 0x1304) ||
3232 (rdev->pdev->device == 0x1305) ||
3233 (rdev->pdev->device == 0x130C) ||
3234 (rdev->pdev->device == 0x130F) ||
3235 (rdev->pdev->device == 0x1310) ||
3236 (rdev->pdev->device == 0x1311) ||
3237 (rdev->pdev->device == 0x131C)) {
3238 rdev->config.cik.max_cu_per_sh = 8;
3239 rdev->config.cik.max_backends_per_se = 2;
3240 } else if ((rdev->pdev->device == 0x1309) ||
3241 (rdev->pdev->device == 0x130A) ||
3242 (rdev->pdev->device == 0x130D) ||
3243 (rdev->pdev->device == 0x1313) ||
3244 (rdev->pdev->device == 0x131D)) {
3245 rdev->config.cik.max_cu_per_sh = 6;
3246 rdev->config.cik.max_backends_per_se = 2;
3247 } else if ((rdev->pdev->device == 0x1306) ||
3248 (rdev->pdev->device == 0x1307) ||
3249 (rdev->pdev->device == 0x130B) ||
3250 (rdev->pdev->device == 0x130E) ||
3251 (rdev->pdev->device == 0x1315) ||
3252 (rdev->pdev->device == 0x1318) ||
3253 (rdev->pdev->device == 0x131B)) {
3254 rdev->config.cik.max_cu_per_sh = 4;
3255 rdev->config.cik.max_backends_per_se = 1;
3256 } else {
3257 rdev->config.cik.max_cu_per_sh = 3;
3258 rdev->config.cik.max_backends_per_se = 1;
3259 }
3260 rdev->config.cik.max_sh_per_se = 1;
3261 rdev->config.cik.max_texture_channel_caches = 4;
3262 rdev->config.cik.max_gprs = 256;
3263 rdev->config.cik.max_gs_threads = 16;
3264 rdev->config.cik.max_hw_contexts = 8;
3265
3266 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3267 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3268 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3269 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3270 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3271 break;
3272 case CHIP_KABINI:
3273 case CHIP_MULLINS:
3274 default:
3275 rdev->config.cik.max_shader_engines = 1;
3276 rdev->config.cik.max_tile_pipes = 2;
3277 rdev->config.cik.max_cu_per_sh = 2;
3278 rdev->config.cik.max_sh_per_se = 1;
3279 rdev->config.cik.max_backends_per_se = 1;
3280 rdev->config.cik.max_texture_channel_caches = 2;
3281 rdev->config.cik.max_gprs = 256;
3282 rdev->config.cik.max_gs_threads = 16;
3283 rdev->config.cik.max_hw_contexts = 8;
3284
3285 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3286 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3287 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3288 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3289 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3290 break;
3291 }
3292
3293 /* Initialize HDP */
3294 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3295 WREG32((0x2c14 + j), 0x00000000);
3296 WREG32((0x2c18 + j), 0x00000000);
3297 WREG32((0x2c1c + j), 0x00000000);
3298 WREG32((0x2c20 + j), 0x00000000);
3299 WREG32((0x2c24 + j), 0x00000000);
3300 }
3301
3302 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3303 WREG32(SRBM_INT_CNTL, 0x1);
3304 WREG32(SRBM_INT_ACK, 0x1);
3305
3306 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3307
3308 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3309 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3310
3311 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3312 rdev->config.cik.mem_max_burst_length_bytes = 256;
3313 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3314 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3315 if (rdev->config.cik.mem_row_size_in_kb > 4)
3316 rdev->config.cik.mem_row_size_in_kb = 4;
3317 /* XXX use MC settings? */
3318 rdev->config.cik.shader_engine_tile_size = 32;
3319 rdev->config.cik.num_gpus = 1;
3320 rdev->config.cik.multi_gpu_tile_size = 64;
3321
3322 /* fix up row size */
3323 gb_addr_config &= ~ROW_SIZE_MASK;
3324 switch (rdev->config.cik.mem_row_size_in_kb) {
3325 case 1:
3326 default:
3327 gb_addr_config |= ROW_SIZE(0);
3328 break;
3329 case 2:
3330 gb_addr_config |= ROW_SIZE(1);
3331 break;
3332 case 4:
3333 gb_addr_config |= ROW_SIZE(2);
3334 break;
3335 }
3336
3337 /* setup tiling info dword. gb_addr_config is not adequate since it does
3338 * not have bank info, so create a custom tiling dword.
3339 * bits 3:0 num_pipes
3340 * bits 7:4 num_banks
3341 * bits 11:8 group_size
3342 * bits 15:12 row_size
3343 */
3344 rdev->config.cik.tile_config = 0;
3345 switch (rdev->config.cik.num_tile_pipes) {
3346 case 1:
3347 rdev->config.cik.tile_config |= (0 << 0);
3348 break;
3349 case 2:
3350 rdev->config.cik.tile_config |= (1 << 0);
3351 break;
3352 case 4:
3353 rdev->config.cik.tile_config |= (2 << 0);
3354 break;
3355 case 8:
3356 default:
3357 /* XXX what about 12? */
3358 rdev->config.cik.tile_config |= (3 << 0);
3359 break;
3360 }
3361 rdev->config.cik.tile_config |=
3362 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3363 rdev->config.cik.tile_config |=
3364 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3365 rdev->config.cik.tile_config |=
3366 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3367
3368 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3369 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3370 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3371 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3372 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3373 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3374 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3375 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3376
3377 cik_tiling_mode_table_init(rdev);
3378
3379 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3380 rdev->config.cik.max_sh_per_se,
3381 rdev->config.cik.max_backends_per_se);
3382
3383 rdev->config.cik.active_cus = 0;
3384 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3385 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3386 rdev->config.cik.active_cus +=
3387 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3388 }
3389 }
3390
3391 /* set HW defaults for 3D engine */
3392 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3393
3394 mutex_lock(&rdev->grbm_idx_mutex);
3395 /*
3396 * making sure that the following register writes will be broadcasted
3397 * to all the shaders
3398 */
3399 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3400 WREG32(SX_DEBUG_1, 0x20);
3401
3402 WREG32(TA_CNTL_AUX, 0x00010000);
3403
3404 tmp = RREG32(SPI_CONFIG_CNTL);
3405 tmp |= 0x03000000;
3406 WREG32(SPI_CONFIG_CNTL, tmp);
3407
3408 WREG32(SQ_CONFIG, 1);
3409
3410 WREG32(DB_DEBUG, 0);
3411
3412 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3413 tmp |= 0x00000400;
3414 WREG32(DB_DEBUG2, tmp);
3415
3416 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3417 tmp |= 0x00020200;
3418 WREG32(DB_DEBUG3, tmp);
3419
3420 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3421 tmp |= 0x00018208;
3422 WREG32(CB_HW_CONTROL, tmp);
3423
3424 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3425
3426 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3427 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3428 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3429 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3430
3431 WREG32(VGT_NUM_INSTANCES, 1);
3432
3433 WREG32(CP_PERFMON_CNTL, 0);
3434
3435 WREG32(SQ_CONFIG, 0);
3436
3437 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3438 FORCE_EOV_MAX_REZ_CNT(255)));
3439
3440 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3441 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3442
3443 WREG32(VGT_GS_VERTEX_REUSE, 16);
3444 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3445
3446 tmp = RREG32(HDP_MISC_CNTL);
3447 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3448 WREG32(HDP_MISC_CNTL, tmp);
3449
3450 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3451 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3452
3453 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3454 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3455 mutex_unlock(&rdev->grbm_idx_mutex);
3456
3457 udelay(50);
3458 }
3459
3460 /*
3461 * GPU scratch registers helpers function.
3462 */
3463 /**
3464 * cik_scratch_init - setup driver info for CP scratch regs
3465 *
3466 * @rdev: radeon_device pointer
3467 *
3468 * Set up the number and offset of the CP scratch registers.
3469 * NOTE: use of CP scratch registers is a legacy inferface and
3470 * is not used by default on newer asics (r6xx+). On newer asics,
3471 * memory buffers are used for fences rather than scratch regs.
3472 */
3473 static void cik_scratch_init(struct radeon_device *rdev)
3474 {
3475 int i;
3476
3477 rdev->scratch.num_reg = 7;
3478 rdev->scratch.reg_base = SCRATCH_REG0;
3479 for (i = 0; i < rdev->scratch.num_reg; i++) {
3480 rdev->scratch.free[i] = true;
3481 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3482 }
3483 }
3484
3485 /**
3486 * cik_ring_test - basic gfx ring test
3487 *
3488 * @rdev: radeon_device pointer
3489 * @ring: radeon_ring structure holding ring information
3490 *
3491 * Allocate a scratch register and write to it using the gfx ring (CIK).
3492 * Provides a basic gfx ring test to verify that the ring is working.
3493 * Used by cik_cp_gfx_resume();
3494 * Returns 0 on success, error on failure.
3495 */
3496 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3497 {
3498 uint32_t scratch;
3499 uint32_t tmp = 0;
3500 unsigned i;
3501 int r;
3502
3503 r = radeon_scratch_get(rdev, &scratch);
3504 if (r) {
3505 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3506 return r;
3507 }
3508 WREG32(scratch, 0xCAFEDEAD);
3509 r = radeon_ring_lock(rdev, ring, 3);
3510 if (r) {
3511 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3512 radeon_scratch_free(rdev, scratch);
3513 return r;
3514 }
3515 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3516 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3517 radeon_ring_write(ring, 0xDEADBEEF);
3518 radeon_ring_unlock_commit(rdev, ring, false);
3519
3520 for (i = 0; i < rdev->usec_timeout; i++) {
3521 tmp = RREG32(scratch);
3522 if (tmp == 0xDEADBEEF)
3523 break;
3524 DRM_UDELAY(1);
3525 }
3526 if (i < rdev->usec_timeout) {
3527 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3528 } else {
3529 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3530 ring->idx, scratch, tmp);
3531 r = -EINVAL;
3532 }
3533 radeon_scratch_free(rdev, scratch);
3534 return r;
3535 }
3536
3537 /**
3538 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3539 *
3540 * @rdev: radeon_device pointer
3541 * @ridx: radeon ring index
3542 *
3543 * Emits an hdp flush on the cp.
3544 */
3545 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3546 int ridx)
3547 {
3548 struct radeon_ring *ring = &rdev->ring[ridx];
3549 u32 ref_and_mask;
3550
3551 switch (ring->idx) {
3552 case CAYMAN_RING_TYPE_CP1_INDEX:
3553 case CAYMAN_RING_TYPE_CP2_INDEX:
3554 default:
3555 switch (ring->me) {
3556 case 0:
3557 ref_and_mask = CP2 << ring->pipe;
3558 break;
3559 case 1:
3560 ref_and_mask = CP6 << ring->pipe;
3561 break;
3562 default:
3563 return;
3564 }
3565 break;
3566 case RADEON_RING_TYPE_GFX_INDEX:
3567 ref_and_mask = CP0;
3568 break;
3569 }
3570
3571 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3572 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3573 WAIT_REG_MEM_FUNCTION(3) | /* == */
3574 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3575 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3576 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3577 radeon_ring_write(ring, ref_and_mask);
3578 radeon_ring_write(ring, ref_and_mask);
3579 radeon_ring_write(ring, 0x20); /* poll interval */
3580 }
3581
3582 /**
3583 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3584 *
3585 * @rdev: radeon_device pointer
3586 * @fence: radeon fence object
3587 *
3588 * Emits a fence sequnce number on the gfx ring and flushes
3589 * GPU caches.
3590 */
3591 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3592 struct radeon_fence *fence)
3593 {
3594 struct radeon_ring *ring = &rdev->ring[fence->ring];
3595 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3596
3597 /* Workaround for cache flush problems. First send a dummy EOP
3598 * event down the pipe with seq one below.
3599 */
3600 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3601 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3602 EOP_TC_ACTION_EN |
3603 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3604 EVENT_INDEX(5)));
3605 radeon_ring_write(ring, addr & 0xfffffffc);
3606 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
3607 DATA_SEL(1) | INT_SEL(0));
3608 radeon_ring_write(ring, fence->seq - 1);
3609 radeon_ring_write(ring, 0);
3610
3611 /* Then send the real EOP event down the pipe. */
3612 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3613 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3614 EOP_TC_ACTION_EN |
3615 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3616 EVENT_INDEX(5)));
3617 radeon_ring_write(ring, addr & 0xfffffffc);
3618 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3619 radeon_ring_write(ring, fence->seq);
3620 radeon_ring_write(ring, 0);
3621 }
3622
3623 /**
3624 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3625 *
3626 * @rdev: radeon_device pointer
3627 * @fence: radeon fence object
3628 *
3629 * Emits a fence sequnce number on the compute ring and flushes
3630 * GPU caches.
3631 */
3632 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3633 struct radeon_fence *fence)
3634 {
3635 struct radeon_ring *ring = &rdev->ring[fence->ring];
3636 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3637
3638 /* RELEASE_MEM - flush caches, send int */
3639 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3640 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3641 EOP_TC_ACTION_EN |
3642 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3643 EVENT_INDEX(5)));
3644 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3645 radeon_ring_write(ring, addr & 0xfffffffc);
3646 radeon_ring_write(ring, upper_32_bits(addr));
3647 radeon_ring_write(ring, fence->seq);
3648 radeon_ring_write(ring, 0);
3649 }
3650
3651 /**
3652 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3653 *
3654 * @rdev: radeon_device pointer
3655 * @ring: radeon ring buffer object
3656 * @semaphore: radeon semaphore object
3657 * @emit_wait: Is this a sempahore wait?
3658 *
3659 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3660 * from running ahead of semaphore waits.
3661 */
3662 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3663 struct radeon_ring *ring,
3664 struct radeon_semaphore *semaphore,
3665 bool emit_wait)
3666 {
3667 uint64_t addr = semaphore->gpu_addr;
3668 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3669
3670 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3671 radeon_ring_write(ring, lower_32_bits(addr));
3672 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3673
3674 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3675 /* Prevent the PFP from running ahead of the semaphore wait */
3676 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3677 radeon_ring_write(ring, 0x0);
3678 }
3679
3680 return true;
3681 }
3682
3683 /**
3684 * cik_copy_cpdma - copy pages using the CP DMA engine
3685 *
3686 * @rdev: radeon_device pointer
3687 * @src_offset: src GPU address
3688 * @dst_offset: dst GPU address
3689 * @num_gpu_pages: number of GPU pages to xfer
3690 * @resv: reservation object to sync to
3691 *
3692 * Copy GPU paging using the CP DMA engine (CIK+).
3693 * Used by the radeon ttm implementation to move pages if
3694 * registered as the asic copy callback.
3695 */
3696 struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
3697 uint64_t src_offset, uint64_t dst_offset,
3698 unsigned num_gpu_pages,
3699 struct reservation_object *resv)
3700 {
3701 struct radeon_fence *fence;
3702 struct radeon_sync sync;
3703 int ring_index = rdev->asic->copy.blit_ring_index;
3704 struct radeon_ring *ring = &rdev->ring[ring_index];
3705 u32 size_in_bytes, cur_size_in_bytes, control;
3706 int i, num_loops;
3707 int r = 0;
3708
3709 radeon_sync_create(&sync);
3710
3711 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3712 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3713 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3714 if (r) {
3715 DRM_ERROR("radeon: moving bo (%d).\n", r);
3716 radeon_sync_free(rdev, &sync, NULL);
3717 return ERR_PTR(r);
3718 }
3719
3720 radeon_sync_resv(rdev, &sync, resv, false);
3721 radeon_sync_rings(rdev, &sync, ring->idx);
3722
3723 for (i = 0; i < num_loops; i++) {
3724 cur_size_in_bytes = size_in_bytes;
3725 if (cur_size_in_bytes > 0x1fffff)
3726 cur_size_in_bytes = 0x1fffff;
3727 size_in_bytes -= cur_size_in_bytes;
3728 control = 0;
3729 if (size_in_bytes == 0)
3730 control |= PACKET3_DMA_DATA_CP_SYNC;
3731 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3732 radeon_ring_write(ring, control);
3733 radeon_ring_write(ring, lower_32_bits(src_offset));
3734 radeon_ring_write(ring, upper_32_bits(src_offset));
3735 radeon_ring_write(ring, lower_32_bits(dst_offset));
3736 radeon_ring_write(ring, upper_32_bits(dst_offset));
3737 radeon_ring_write(ring, cur_size_in_bytes);
3738 src_offset += cur_size_in_bytes;
3739 dst_offset += cur_size_in_bytes;
3740 }
3741
3742 r = radeon_fence_emit(rdev, &fence, ring->idx);
3743 if (r) {
3744 radeon_ring_unlock_undo(rdev, ring);
3745 radeon_sync_free(rdev, &sync, NULL);
3746 return ERR_PTR(r);
3747 }
3748
3749 radeon_ring_unlock_commit(rdev, ring, false);
3750 radeon_sync_free(rdev, &sync, fence);
3751
3752 return fence;
3753 }
3754
3755 /*
3756 * IB stuff
3757 */
3758 /**
3759 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3760 *
3761 * @rdev: radeon_device pointer
3762 * @ib: radeon indirect buffer object
3763 *
3764 * Emits a DE (drawing engine) or CE (constant engine) IB
3765 * on the gfx ring. IBs are usually generated by userspace
3766 * acceleration drivers and submitted to the kernel for
3767 * scheduling on the ring. This function schedules the IB
3768 * on the gfx ring for execution by the GPU.
3769 */
3770 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3771 {
3772 struct radeon_ring *ring = &rdev->ring[ib->ring];
3773 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
3774 u32 header, control = INDIRECT_BUFFER_VALID;
3775
3776 if (ib->is_const_ib) {
3777 /* set switch buffer packet before const IB */
3778 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3779 radeon_ring_write(ring, 0);
3780
3781 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3782 } else {
3783 u32 next_rptr;
3784 if (ring->rptr_save_reg) {
3785 next_rptr = ring->wptr + 3 + 4;
3786 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3787 radeon_ring_write(ring, ((ring->rptr_save_reg -
3788 PACKET3_SET_UCONFIG_REG_START) >> 2));
3789 radeon_ring_write(ring, next_rptr);
3790 } else if (rdev->wb.enabled) {
3791 next_rptr = ring->wptr + 5 + 4;
3792 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3793 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3794 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3795 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3796 radeon_ring_write(ring, next_rptr);
3797 }
3798
3799 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3800 }
3801
3802 control |= ib->length_dw | (vm_id << 24);
3803
3804 radeon_ring_write(ring, header);
3805 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFFC));
3806 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3807 radeon_ring_write(ring, control);
3808 }
3809
3810 /**
3811 * cik_ib_test - basic gfx ring IB test
3812 *
3813 * @rdev: radeon_device pointer
3814 * @ring: radeon_ring structure holding ring information
3815 *
3816 * Allocate an IB and execute it on the gfx ring (CIK).
3817 * Provides a basic gfx ring test to verify that IBs are working.
3818 * Returns 0 on success, error on failure.
3819 */
3820 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3821 {
3822 struct radeon_ib ib;
3823 uint32_t scratch;
3824 uint32_t tmp = 0;
3825 unsigned i;
3826 int r;
3827
3828 r = radeon_scratch_get(rdev, &scratch);
3829 if (r) {
3830 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3831 return r;
3832 }
3833 WREG32(scratch, 0xCAFEDEAD);
3834 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3835 if (r) {
3836 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3837 radeon_scratch_free(rdev, scratch);
3838 return r;
3839 }
3840 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3841 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3842 ib.ptr[2] = 0xDEADBEEF;
3843 ib.length_dw = 3;
3844 r = radeon_ib_schedule(rdev, &ib, NULL, false);
3845 if (r) {
3846 radeon_scratch_free(rdev, scratch);
3847 radeon_ib_free(rdev, &ib);
3848 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3849 return r;
3850 }
3851 r = radeon_fence_wait_timeout(ib.fence, false, usecs_to_jiffies(
3852 RADEON_USEC_IB_TEST_TIMEOUT));
3853 if (r < 0) {
3854 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3855 radeon_scratch_free(rdev, scratch);
3856 radeon_ib_free(rdev, &ib);
3857 return r;
3858 } else if (r == 0) {
3859 DRM_ERROR("radeon: fence wait timed out.\n");
3860 radeon_scratch_free(rdev, scratch);
3861 radeon_ib_free(rdev, &ib);
3862 return -ETIMEDOUT;
3863 }
3864 r = 0;
3865 for (i = 0; i < rdev->usec_timeout; i++) {
3866 tmp = RREG32(scratch);
3867 if (tmp == 0xDEADBEEF)
3868 break;
3869 DRM_UDELAY(1);
3870 }
3871 if (i < rdev->usec_timeout) {
3872 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3873 } else {
3874 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3875 scratch, tmp);
3876 r = -EINVAL;
3877 }
3878 radeon_scratch_free(rdev, scratch);
3879 radeon_ib_free(rdev, &ib);
3880 return r;
3881 }
3882
3883 /*
3884 * CP.
3885 * On CIK, gfx and compute now have independant command processors.
3886 *
3887 * GFX
3888 * Gfx consists of a single ring and can process both gfx jobs and
3889 * compute jobs. The gfx CP consists of three microengines (ME):
3890 * PFP - Pre-Fetch Parser
3891 * ME - Micro Engine
3892 * CE - Constant Engine
3893 * The PFP and ME make up what is considered the Drawing Engine (DE).
3894 * The CE is an asynchronous engine used for updating buffer desciptors
3895 * used by the DE so that they can be loaded into cache in parallel
3896 * while the DE is processing state update packets.
3897 *
3898 * Compute
3899 * The compute CP consists of two microengines (ME):
3900 * MEC1 - Compute MicroEngine 1
3901 * MEC2 - Compute MicroEngine 2
3902 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3903 * The queues are exposed to userspace and are programmed directly
3904 * by the compute runtime.
3905 */
3906 /**
3907 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3908 *
3909 * @rdev: radeon_device pointer
3910 * @enable: enable or disable the MEs
3911 *
3912 * Halts or unhalts the gfx MEs.
3913 */
3914 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3915 {
3916 if (enable)
3917 WREG32(CP_ME_CNTL, 0);
3918 else {
3919 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3920 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3921 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3922 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3923 }
3924 udelay(50);
3925 }
3926
3927 /**
3928 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3929 *
3930 * @rdev: radeon_device pointer
3931 *
3932 * Loads the gfx PFP, ME, and CE ucode.
3933 * Returns 0 for success, -EINVAL if the ucode is not available.
3934 */
3935 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3936 {
3937 int i;
3938
3939 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3940 return -EINVAL;
3941
3942 cik_cp_gfx_enable(rdev, false);
3943
3944 if (rdev->new_fw) {
3945 const struct gfx_firmware_header_v1_0 *pfp_hdr =
3946 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3947 const struct gfx_firmware_header_v1_0 *ce_hdr =
3948 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3949 const struct gfx_firmware_header_v1_0 *me_hdr =
3950 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3951 const __le32 *fw_data;
3952 u32 fw_size;
3953
3954 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3955 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3956 radeon_ucode_print_gfx_hdr(&me_hdr->header);
3957
3958 /* PFP */
3959 fw_data = (const __le32 *)
3960 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3961 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3962 WREG32(CP_PFP_UCODE_ADDR, 0);
3963 for (i = 0; i < fw_size; i++)
3964 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3965 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version));
3966
3967 /* CE */
3968 fw_data = (const __le32 *)
3969 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3970 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3971 WREG32(CP_CE_UCODE_ADDR, 0);
3972 for (i = 0; i < fw_size; i++)
3973 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3974 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version));
3975
3976 /* ME */
3977 fw_data = (const __be32 *)
3978 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3979 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3980 WREG32(CP_ME_RAM_WADDR, 0);
3981 for (i = 0; i < fw_size; i++)
3982 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3983 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version));
3984 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version));
3985 } else {
3986 const __be32 *fw_data;
3987
3988 /* PFP */
3989 fw_data = (const __be32 *)rdev->pfp_fw->data;
3990 WREG32(CP_PFP_UCODE_ADDR, 0);
3991 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3992 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3993 WREG32(CP_PFP_UCODE_ADDR, 0);
3994
3995 /* CE */
3996 fw_data = (const __be32 *)rdev->ce_fw->data;
3997 WREG32(CP_CE_UCODE_ADDR, 0);
3998 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3999 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
4000 WREG32(CP_CE_UCODE_ADDR, 0);
4001
4002 /* ME */
4003 fw_data = (const __be32 *)rdev->me_fw->data;
4004 WREG32(CP_ME_RAM_WADDR, 0);
4005 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
4006 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
4007 WREG32(CP_ME_RAM_WADDR, 0);
4008 }
4009
4010 return 0;
4011 }
4012
4013 /**
4014 * cik_cp_gfx_start - start the gfx ring
4015 *
4016 * @rdev: radeon_device pointer
4017 *
4018 * Enables the ring and loads the clear state context and other
4019 * packets required to init the ring.
4020 * Returns 0 for success, error for failure.
4021 */
4022 static int cik_cp_gfx_start(struct radeon_device *rdev)
4023 {
4024 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4025 int r, i;
4026
4027 /* init the CP */
4028 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4029 WREG32(CP_ENDIAN_SWAP, 0);
4030 WREG32(CP_DEVICE_ID, 1);
4031
4032 cik_cp_gfx_enable(rdev, true);
4033
4034 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4035 if (r) {
4036 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4037 return r;
4038 }
4039
4040 /* init the CE partitions. CE only used for gfx on CIK */
4041 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4042 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4043 radeon_ring_write(ring, 0x8000);
4044 radeon_ring_write(ring, 0x8000);
4045
4046 /* setup clear context state */
4047 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4048 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4049
4050 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4051 radeon_ring_write(ring, 0x80000000);
4052 radeon_ring_write(ring, 0x80000000);
4053
4054 for (i = 0; i < cik_default_size; i++)
4055 radeon_ring_write(ring, cik_default_state[i]);
4056
4057 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4058 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4059
4060 /* set clear context state */
4061 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4062 radeon_ring_write(ring, 0);
4063
4064 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4065 radeon_ring_write(ring, 0x00000316);
4066 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4067 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4068
4069 radeon_ring_unlock_commit(rdev, ring, false);
4070
4071 return 0;
4072 }
4073
4074 /**
4075 * cik_cp_gfx_fini - stop the gfx ring
4076 *
4077 * @rdev: radeon_device pointer
4078 *
4079 * Stop the gfx ring and tear down the driver ring
4080 * info.
4081 */
4082 static void cik_cp_gfx_fini(struct radeon_device *rdev)
4083 {
4084 cik_cp_gfx_enable(rdev, false);
4085 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4086 }
4087
4088 /**
4089 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4090 *
4091 * @rdev: radeon_device pointer
4092 *
4093 * Program the location and size of the gfx ring buffer
4094 * and test it to make sure it's working.
4095 * Returns 0 for success, error for failure.
4096 */
4097 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4098 {
4099 struct radeon_ring *ring;
4100 u32 tmp;
4101 u32 rb_bufsz;
4102 u64 rb_addr;
4103 int r;
4104
4105 WREG32(CP_SEM_WAIT_TIMER, 0x0);
4106 if (rdev->family != CHIP_HAWAII)
4107 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4108
4109 /* Set the write pointer delay */
4110 WREG32(CP_RB_WPTR_DELAY, 0);
4111
4112 /* set the RB to use vmid 0 */
4113 WREG32(CP_RB_VMID, 0);
4114
4115 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4116
4117 /* ring 0 - compute and gfx */
4118 /* Set ring buffer size */
4119 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4120 rb_bufsz = order_base_2(ring->ring_size / 8);
4121 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4122 #ifdef __BIG_ENDIAN
4123 tmp |= BUF_SWAP_32BIT;
4124 #endif
4125 WREG32(CP_RB0_CNTL, tmp);
4126
4127 /* Initialize the ring buffer's read and write pointers */
4128 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4129 ring->wptr = 0;
4130 WREG32(CP_RB0_WPTR, ring->wptr);
4131
4132 /* set the wb address wether it's enabled or not */
4133 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4134 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4135
4136 /* scratch register shadowing is no longer supported */
4137 WREG32(SCRATCH_UMSK, 0);
4138
4139 if (!rdev->wb.enabled)
4140 tmp |= RB_NO_UPDATE;
4141
4142 mdelay(1);
4143 WREG32(CP_RB0_CNTL, tmp);
4144
4145 rb_addr = ring->gpu_addr >> 8;
4146 WREG32(CP_RB0_BASE, rb_addr);
4147 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4148
4149 /* start the ring */
4150 cik_cp_gfx_start(rdev);
4151 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4152 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4153 if (r) {
4154 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4155 return r;
4156 }
4157
4158 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4159 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4160
4161 return 0;
4162 }
4163
4164 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4165 struct radeon_ring *ring)
4166 {
4167 u32 rptr;
4168
4169 if (rdev->wb.enabled)
4170 rptr = rdev->wb.wb[ring->rptr_offs/4];
4171 else
4172 rptr = RREG32(CP_RB0_RPTR);
4173
4174 return rptr;
4175 }
4176
4177 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4178 struct radeon_ring *ring)
4179 {
4180 return RREG32(CP_RB0_WPTR);
4181 }
4182
4183 void cik_gfx_set_wptr(struct radeon_device *rdev,
4184 struct radeon_ring *ring)
4185 {
4186 WREG32(CP_RB0_WPTR, ring->wptr);
4187 (void)RREG32(CP_RB0_WPTR);
4188 }
4189
4190 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4191 struct radeon_ring *ring)
4192 {
4193 u32 rptr;
4194
4195 if (rdev->wb.enabled) {
4196 rptr = rdev->wb.wb[ring->rptr_offs/4];
4197 } else {
4198 mutex_lock(&rdev->srbm_mutex);
4199 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4200 rptr = RREG32(CP_HQD_PQ_RPTR);
4201 cik_srbm_select(rdev, 0, 0, 0, 0);
4202 mutex_unlock(&rdev->srbm_mutex);
4203 }
4204
4205 return rptr;
4206 }
4207
4208 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4209 struct radeon_ring *ring)
4210 {
4211 u32 wptr;
4212
4213 if (rdev->wb.enabled) {
4214 /* XXX check if swapping is necessary on BE */
4215 wptr = rdev->wb.wb[ring->wptr_offs/4];
4216 } else {
4217 mutex_lock(&rdev->srbm_mutex);
4218 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4219 wptr = RREG32(CP_HQD_PQ_WPTR);
4220 cik_srbm_select(rdev, 0, 0, 0, 0);
4221 mutex_unlock(&rdev->srbm_mutex);
4222 }
4223
4224 return wptr;
4225 }
4226
4227 void cik_compute_set_wptr(struct radeon_device *rdev,
4228 struct radeon_ring *ring)
4229 {
4230 /* XXX check if swapping is necessary on BE */
4231 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4232 WDOORBELL32(ring->doorbell_index, ring->wptr);
4233 }
4234
4235 static void cik_compute_stop(struct radeon_device *rdev,
4236 struct radeon_ring *ring)
4237 {
4238 u32 j, tmp;
4239
4240 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4241 /* Disable wptr polling. */
4242 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4243 tmp &= ~WPTR_POLL_EN;
4244 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4245 /* Disable HQD. */
4246 if (RREG32(CP_HQD_ACTIVE) & 1) {
4247 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4248 for (j = 0; j < rdev->usec_timeout; j++) {
4249 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4250 break;
4251 udelay(1);
4252 }
4253 WREG32(CP_HQD_DEQUEUE_REQUEST, 0);
4254 WREG32(CP_HQD_PQ_RPTR, 0);
4255 WREG32(CP_HQD_PQ_WPTR, 0);
4256 }
4257 cik_srbm_select(rdev, 0, 0, 0, 0);
4258 }
4259
4260 /**
4261 * cik_cp_compute_enable - enable/disable the compute CP MEs
4262 *
4263 * @rdev: radeon_device pointer
4264 * @enable: enable or disable the MEs
4265 *
4266 * Halts or unhalts the compute MEs.
4267 */
4268 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4269 {
4270 if (enable)
4271 WREG32(CP_MEC_CNTL, 0);
4272 else {
4273 /*
4274 * To make hibernation reliable we need to clear compute ring
4275 * configuration before halting the compute ring.
4276 */
4277 mutex_lock(&rdev->srbm_mutex);
4278 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
4279 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
4280 mutex_unlock(&rdev->srbm_mutex);
4281
4282 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4283 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4284 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4285 }
4286 udelay(50);
4287 }
4288
4289 /**
4290 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4291 *
4292 * @rdev: radeon_device pointer
4293 *
4294 * Loads the compute MEC1&2 ucode.
4295 * Returns 0 for success, -EINVAL if the ucode is not available.
4296 */
4297 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4298 {
4299 int i;
4300
4301 if (!rdev->mec_fw)
4302 return -EINVAL;
4303
4304 cik_cp_compute_enable(rdev, false);
4305
4306 if (rdev->new_fw) {
4307 const struct gfx_firmware_header_v1_0 *mec_hdr =
4308 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4309 const __le32 *fw_data;
4310 u32 fw_size;
4311
4312 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4313
4314 /* MEC1 */
4315 fw_data = (const __le32 *)
4316 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4317 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4318 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4319 for (i = 0; i < fw_size; i++)
4320 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
4321 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version));
4322
4323 /* MEC2 */
4324 if (rdev->family == CHIP_KAVERI) {
4325 const struct gfx_firmware_header_v1_0 *mec2_hdr =
4326 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4327
4328 fw_data = (const __le32 *)
4329 (rdev->mec2_fw->data +
4330 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4331 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4332 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4333 for (i = 0; i < fw_size; i++)
4334 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
4335 WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version));
4336 }
4337 } else {
4338 const __be32 *fw_data;
4339
4340 /* MEC1 */
4341 fw_data = (const __be32 *)rdev->mec_fw->data;
4342 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4343 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4344 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4345 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4346
4347 if (rdev->family == CHIP_KAVERI) {
4348 /* MEC2 */
4349 fw_data = (const __be32 *)rdev->mec_fw->data;
4350 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4351 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4352 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4353 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4354 }
4355 }
4356
4357 return 0;
4358 }
4359
4360 /**
4361 * cik_cp_compute_start - start the compute queues
4362 *
4363 * @rdev: radeon_device pointer
4364 *
4365 * Enable the compute queues.
4366 * Returns 0 for success, error for failure.
4367 */
4368 static int cik_cp_compute_start(struct radeon_device *rdev)
4369 {
4370 cik_cp_compute_enable(rdev, true);
4371
4372 return 0;
4373 }
4374
4375 /**
4376 * cik_cp_compute_fini - stop the compute queues
4377 *
4378 * @rdev: radeon_device pointer
4379 *
4380 * Stop the compute queues and tear down the driver queue
4381 * info.
4382 */
4383 static void cik_cp_compute_fini(struct radeon_device *rdev)
4384 {
4385 int i, idx, r;
4386
4387 cik_cp_compute_enable(rdev, false);
4388
4389 for (i = 0; i < 2; i++) {
4390 if (i == 0)
4391 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4392 else
4393 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4394
4395 if (rdev->ring[idx].mqd_obj) {
4396 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4397 if (unlikely(r != 0))
4398 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4399
4400 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4401 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4402
4403 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4404 rdev->ring[idx].mqd_obj = NULL;
4405 }
4406 }
4407 }
4408
4409 static void cik_mec_fini(struct radeon_device *rdev)
4410 {
4411 int r;
4412
4413 if (rdev->mec.hpd_eop_obj) {
4414 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4415 if (unlikely(r != 0))
4416 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4417 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4418 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4419
4420 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4421 rdev->mec.hpd_eop_obj = NULL;
4422 }
4423 }
4424
4425 #define MEC_HPD_SIZE 2048
4426
4427 static int cik_mec_init(struct radeon_device *rdev)
4428 {
4429 int r;
4430 u32 *hpd;
4431
4432 /*
4433 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4434 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4435 * Nonetheless, we assign only 1 pipe because all other pipes will
4436 * be handled by KFD
4437 */
4438 rdev->mec.num_mec = 1;
4439 rdev->mec.num_pipe = 1;
4440 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4441
4442 if (rdev->mec.hpd_eop_obj == NULL) {
4443 r = radeon_bo_create(rdev,
4444 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4445 PAGE_SIZE, true,
4446 RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL,
4447 &rdev->mec.hpd_eop_obj);
4448 if (r) {
4449 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4450 return r;
4451 }
4452 }
4453
4454 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4455 if (unlikely(r != 0)) {
4456 cik_mec_fini(rdev);
4457 return r;
4458 }
4459 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4460 &rdev->mec.hpd_eop_gpu_addr);
4461 if (r) {
4462 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4463 cik_mec_fini(rdev);
4464 return r;
4465 }
4466 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4467 if (r) {
4468 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4469 cik_mec_fini(rdev);
4470 return r;
4471 }
4472
4473 /* clear memory. Not sure if this is required or not */
4474 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4475
4476 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4477 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4478
4479 return 0;
4480 }
4481
4482 struct hqd_registers
4483 {
4484 u32 cp_mqd_base_addr;
4485 u32 cp_mqd_base_addr_hi;
4486 u32 cp_hqd_active;
4487 u32 cp_hqd_vmid;
4488 u32 cp_hqd_persistent_state;
4489 u32 cp_hqd_pipe_priority;
4490 u32 cp_hqd_queue_priority;
4491 u32 cp_hqd_quantum;
4492 u32 cp_hqd_pq_base;
4493 u32 cp_hqd_pq_base_hi;
4494 u32 cp_hqd_pq_rptr;
4495 u32 cp_hqd_pq_rptr_report_addr;
4496 u32 cp_hqd_pq_rptr_report_addr_hi;
4497 u32 cp_hqd_pq_wptr_poll_addr;
4498 u32 cp_hqd_pq_wptr_poll_addr_hi;
4499 u32 cp_hqd_pq_doorbell_control;
4500 u32 cp_hqd_pq_wptr;
4501 u32 cp_hqd_pq_control;
4502 u32 cp_hqd_ib_base_addr;
4503 u32 cp_hqd_ib_base_addr_hi;
4504 u32 cp_hqd_ib_rptr;
4505 u32 cp_hqd_ib_control;
4506 u32 cp_hqd_iq_timer;
4507 u32 cp_hqd_iq_rptr;
4508 u32 cp_hqd_dequeue_request;
4509 u32 cp_hqd_dma_offload;
4510 u32 cp_hqd_sema_cmd;
4511 u32 cp_hqd_msg_type;
4512 u32 cp_hqd_atomic0_preop_lo;
4513 u32 cp_hqd_atomic0_preop_hi;
4514 u32 cp_hqd_atomic1_preop_lo;
4515 u32 cp_hqd_atomic1_preop_hi;
4516 u32 cp_hqd_hq_scheduler0;
4517 u32 cp_hqd_hq_scheduler1;
4518 u32 cp_mqd_control;
4519 };
4520
4521 struct bonaire_mqd
4522 {
4523 u32 header;
4524 u32 dispatch_initiator;
4525 u32 dimensions[3];
4526 u32 start_idx[3];
4527 u32 num_threads[3];
4528 u32 pipeline_stat_enable;
4529 u32 perf_counter_enable;
4530 u32 pgm[2];
4531 u32 tba[2];
4532 u32 tma[2];
4533 u32 pgm_rsrc[2];
4534 u32 vmid;
4535 u32 resource_limits;
4536 u32 static_thread_mgmt01[2];
4537 u32 tmp_ring_size;
4538 u32 static_thread_mgmt23[2];
4539 u32 restart[3];
4540 u32 thread_trace_enable;
4541 u32 reserved1;
4542 u32 user_data[16];
4543 u32 vgtcs_invoke_count[2];
4544 struct hqd_registers queue_state;
4545 u32 dequeue_cntr;
4546 u32 interrupt_queue[64];
4547 };
4548
4549 /**
4550 * cik_cp_compute_resume - setup the compute queue registers
4551 *
4552 * @rdev: radeon_device pointer
4553 *
4554 * Program the compute queues and test them to make sure they
4555 * are working.
4556 * Returns 0 for success, error for failure.
4557 */
4558 static int cik_cp_compute_resume(struct radeon_device *rdev)
4559 {
4560 int r, i, j, idx;
4561 u32 tmp;
4562 bool use_doorbell = true;
4563 u64 hqd_gpu_addr;
4564 u64 mqd_gpu_addr;
4565 u64 eop_gpu_addr;
4566 u64 wb_gpu_addr;
4567 u32 *buf;
4568 struct bonaire_mqd *mqd;
4569
4570 r = cik_cp_compute_start(rdev);
4571 if (r)
4572 return r;
4573
4574 /* fix up chicken bits */
4575 tmp = RREG32(CP_CPF_DEBUG);
4576 tmp |= (1 << 23);
4577 WREG32(CP_CPF_DEBUG, tmp);
4578
4579 /* init the pipes */
4580 mutex_lock(&rdev->srbm_mutex);
4581
4582 for (i = 0; i < rdev->mec.num_pipe; ++i) {
4583 cik_srbm_select(rdev, 0, i, 0, 0);
4584
4585 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2) ;
4586 /* write the EOP addr */
4587 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4588 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4589
4590 /* set the VMID assigned */
4591 WREG32(CP_HPD_EOP_VMID, 0);
4592
4593 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4594 tmp = RREG32(CP_HPD_EOP_CONTROL);
4595 tmp &= ~EOP_SIZE_MASK;
4596 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4597 WREG32(CP_HPD_EOP_CONTROL, tmp);
4598
4599 }
4600 mutex_unlock(&rdev->srbm_mutex);
4601
4602 /* init the queues. Just two for now. */
4603 for (i = 0; i < 2; i++) {
4604 if (i == 0)
4605 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4606 else
4607 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4608
4609 if (rdev->ring[idx].mqd_obj == NULL) {
4610 r = radeon_bo_create(rdev,
4611 sizeof(struct bonaire_mqd),
4612 PAGE_SIZE, true,
4613 RADEON_GEM_DOMAIN_GTT, 0, NULL,
4614 NULL, &rdev->ring[idx].mqd_obj);
4615 if (r) {
4616 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4617 return r;
4618 }
4619 }
4620
4621 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4622 if (unlikely(r != 0)) {
4623 cik_cp_compute_fini(rdev);
4624 return r;
4625 }
4626 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4627 &mqd_gpu_addr);
4628 if (r) {
4629 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4630 cik_cp_compute_fini(rdev);
4631 return r;
4632 }
4633 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4634 if (r) {
4635 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4636 cik_cp_compute_fini(rdev);
4637 return r;
4638 }
4639
4640 /* init the mqd struct */
4641 memset(buf, 0, sizeof(struct bonaire_mqd));
4642
4643 mqd = (struct bonaire_mqd *)buf;
4644 mqd->header = 0xC0310800;
4645 mqd->static_thread_mgmt01[0] = 0xffffffff;
4646 mqd->static_thread_mgmt01[1] = 0xffffffff;
4647 mqd->static_thread_mgmt23[0] = 0xffffffff;
4648 mqd->static_thread_mgmt23[1] = 0xffffffff;
4649
4650 mutex_lock(&rdev->srbm_mutex);
4651 cik_srbm_select(rdev, rdev->ring[idx].me,
4652 rdev->ring[idx].pipe,
4653 rdev->ring[idx].queue, 0);
4654
4655 /* disable wptr polling */
4656 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4657 tmp &= ~WPTR_POLL_EN;
4658 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4659
4660 /* enable doorbell? */
4661 mqd->queue_state.cp_hqd_pq_doorbell_control =
4662 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4663 if (use_doorbell)
4664 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4665 else
4666 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4667 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4668 mqd->queue_state.cp_hqd_pq_doorbell_control);
4669
4670 /* disable the queue if it's active */
4671 mqd->queue_state.cp_hqd_dequeue_request = 0;
4672 mqd->queue_state.cp_hqd_pq_rptr = 0;
4673 mqd->queue_state.cp_hqd_pq_wptr= 0;
4674 if (RREG32(CP_HQD_ACTIVE) & 1) {
4675 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4676 for (j = 0; j < rdev->usec_timeout; j++) {
4677 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4678 break;
4679 udelay(1);
4680 }
4681 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4682 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4683 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4684 }
4685
4686 /* set the pointer to the MQD */
4687 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4688 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4689 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4690 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4691 /* set MQD vmid to 0 */
4692 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4693 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4694 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4695
4696 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4697 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4698 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4699 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4700 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4701 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4702
4703 /* set up the HQD, this is similar to CP_RB0_CNTL */
4704 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4705 mqd->queue_state.cp_hqd_pq_control &=
4706 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4707
4708 mqd->queue_state.cp_hqd_pq_control |=
4709 order_base_2(rdev->ring[idx].ring_size / 8);
4710 mqd->queue_state.cp_hqd_pq_control |=
4711 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4712 #ifdef __BIG_ENDIAN
4713 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4714 #endif
4715 mqd->queue_state.cp_hqd_pq_control &=
4716 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4717 mqd->queue_state.cp_hqd_pq_control |=
4718 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4719 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4720
4721 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4722 if (i == 0)
4723 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4724 else
4725 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4726 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4727 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4728 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4729 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4730 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4731
4732 /* set the wb address wether it's enabled or not */
4733 if (i == 0)
4734 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4735 else
4736 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4737 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4738 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4739 upper_32_bits(wb_gpu_addr) & 0xffff;
4740 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4741 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4742 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4743 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4744
4745 /* enable the doorbell if requested */
4746 if (use_doorbell) {
4747 mqd->queue_state.cp_hqd_pq_doorbell_control =
4748 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4749 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4750 mqd->queue_state.cp_hqd_pq_doorbell_control |=
4751 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
4752 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4753 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4754 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4755
4756 } else {
4757 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4758 }
4759 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4760 mqd->queue_state.cp_hqd_pq_doorbell_control);
4761
4762 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4763 rdev->ring[idx].wptr = 0;
4764 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4765 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4766 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
4767
4768 /* set the vmid for the queue */
4769 mqd->queue_state.cp_hqd_vmid = 0;
4770 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4771
4772 /* activate the queue */
4773 mqd->queue_state.cp_hqd_active = 1;
4774 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4775
4776 cik_srbm_select(rdev, 0, 0, 0, 0);
4777 mutex_unlock(&rdev->srbm_mutex);
4778
4779 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4780 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4781
4782 rdev->ring[idx].ready = true;
4783 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4784 if (r)
4785 rdev->ring[idx].ready = false;
4786 }
4787
4788 return 0;
4789 }
4790
4791 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4792 {
4793 cik_cp_gfx_enable(rdev, enable);
4794 cik_cp_compute_enable(rdev, enable);
4795 }
4796
4797 static int cik_cp_load_microcode(struct radeon_device *rdev)
4798 {
4799 int r;
4800
4801 r = cik_cp_gfx_load_microcode(rdev);
4802 if (r)
4803 return r;
4804 r = cik_cp_compute_load_microcode(rdev);
4805 if (r)
4806 return r;
4807
4808 return 0;
4809 }
4810
4811 static void cik_cp_fini(struct radeon_device *rdev)
4812 {
4813 cik_cp_gfx_fini(rdev);
4814 cik_cp_compute_fini(rdev);
4815 }
4816
4817 static int cik_cp_resume(struct radeon_device *rdev)
4818 {
4819 int r;
4820
4821 cik_enable_gui_idle_interrupt(rdev, false);
4822
4823 r = cik_cp_load_microcode(rdev);
4824 if (r)
4825 return r;
4826
4827 r = cik_cp_gfx_resume(rdev);
4828 if (r)
4829 return r;
4830 r = cik_cp_compute_resume(rdev);
4831 if (r)
4832 return r;
4833
4834 cik_enable_gui_idle_interrupt(rdev, true);
4835
4836 return 0;
4837 }
4838
4839 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4840 {
4841 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4842 RREG32(GRBM_STATUS));
4843 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4844 RREG32(GRBM_STATUS2));
4845 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4846 RREG32(GRBM_STATUS_SE0));
4847 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4848 RREG32(GRBM_STATUS_SE1));
4849 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4850 RREG32(GRBM_STATUS_SE2));
4851 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4852 RREG32(GRBM_STATUS_SE3));
4853 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4854 RREG32(SRBM_STATUS));
4855 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4856 RREG32(SRBM_STATUS2));
4857 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4858 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4859 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4860 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4861 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4862 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4863 RREG32(CP_STALLED_STAT1));
4864 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4865 RREG32(CP_STALLED_STAT2));
4866 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4867 RREG32(CP_STALLED_STAT3));
4868 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4869 RREG32(CP_CPF_BUSY_STAT));
4870 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4871 RREG32(CP_CPF_STALLED_STAT1));
4872 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4873 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4874 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4875 RREG32(CP_CPC_STALLED_STAT1));
4876 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4877 }
4878
4879 /**
4880 * cik_gpu_check_soft_reset - check which blocks are busy
4881 *
4882 * @rdev: radeon_device pointer
4883 *
4884 * Check which blocks are busy and return the relevant reset
4885 * mask to be used by cik_gpu_soft_reset().
4886 * Returns a mask of the blocks to be reset.
4887 */
4888 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4889 {
4890 u32 reset_mask = 0;
4891 u32 tmp;
4892
4893 /* GRBM_STATUS */
4894 tmp = RREG32(GRBM_STATUS);
4895 if (tmp & (PA_BUSY | SC_BUSY |
4896 BCI_BUSY | SX_BUSY |
4897 TA_BUSY | VGT_BUSY |
4898 DB_BUSY | CB_BUSY |
4899 GDS_BUSY | SPI_BUSY |
4900 IA_BUSY | IA_BUSY_NO_DMA))
4901 reset_mask |= RADEON_RESET_GFX;
4902
4903 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4904 reset_mask |= RADEON_RESET_CP;
4905
4906 /* GRBM_STATUS2 */
4907 tmp = RREG32(GRBM_STATUS2);
4908 if (tmp & RLC_BUSY)
4909 reset_mask |= RADEON_RESET_RLC;
4910
4911 /* SDMA0_STATUS_REG */
4912 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4913 if (!(tmp & SDMA_IDLE))
4914 reset_mask |= RADEON_RESET_DMA;
4915
4916 /* SDMA1_STATUS_REG */
4917 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4918 if (!(tmp & SDMA_IDLE))
4919 reset_mask |= RADEON_RESET_DMA1;
4920
4921 /* SRBM_STATUS2 */
4922 tmp = RREG32(SRBM_STATUS2);
4923 if (tmp & SDMA_BUSY)
4924 reset_mask |= RADEON_RESET_DMA;
4925
4926 if (tmp & SDMA1_BUSY)
4927 reset_mask |= RADEON_RESET_DMA1;
4928
4929 /* SRBM_STATUS */
4930 tmp = RREG32(SRBM_STATUS);
4931
4932 if (tmp & IH_BUSY)
4933 reset_mask |= RADEON_RESET_IH;
4934
4935 if (tmp & SEM_BUSY)
4936 reset_mask |= RADEON_RESET_SEM;
4937
4938 if (tmp & GRBM_RQ_PENDING)
4939 reset_mask |= RADEON_RESET_GRBM;
4940
4941 if (tmp & VMC_BUSY)
4942 reset_mask |= RADEON_RESET_VMC;
4943
4944 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4945 MCC_BUSY | MCD_BUSY))
4946 reset_mask |= RADEON_RESET_MC;
4947
4948 if (evergreen_is_display_hung(rdev))
4949 reset_mask |= RADEON_RESET_DISPLAY;
4950
4951 /* Skip MC reset as it's mostly likely not hung, just busy */
4952 if (reset_mask & RADEON_RESET_MC) {
4953 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4954 reset_mask &= ~RADEON_RESET_MC;
4955 }
4956
4957 return reset_mask;
4958 }
4959
4960 /**
4961 * cik_gpu_soft_reset - soft reset GPU
4962 *
4963 * @rdev: radeon_device pointer
4964 * @reset_mask: mask of which blocks to reset
4965 *
4966 * Soft reset the blocks specified in @reset_mask.
4967 */
4968 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4969 {
4970 struct evergreen_mc_save save;
4971 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4972 u32 tmp;
4973
4974 if (reset_mask == 0)
4975 return;
4976
4977 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4978
4979 cik_print_gpu_status_regs(rdev);
4980 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4981 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4982 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4983 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4984
4985 /* disable CG/PG */
4986 cik_fini_pg(rdev);
4987 cik_fini_cg(rdev);
4988
4989 /* stop the rlc */
4990 cik_rlc_stop(rdev);
4991
4992 /* Disable GFX parsing/prefetching */
4993 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4994
4995 /* Disable MEC parsing/prefetching */
4996 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4997
4998 if (reset_mask & RADEON_RESET_DMA) {
4999 /* sdma0 */
5000 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5001 tmp |= SDMA_HALT;
5002 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5003 }
5004 if (reset_mask & RADEON_RESET_DMA1) {
5005 /* sdma1 */
5006 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5007 tmp |= SDMA_HALT;
5008 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5009 }
5010
5011 evergreen_mc_stop(rdev, &save);
5012 if (evergreen_mc_wait_for_idle(rdev)) {
5013 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5014 }
5015
5016 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5017 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
5018
5019 if (reset_mask & RADEON_RESET_CP) {
5020 grbm_soft_reset |= SOFT_RESET_CP;
5021
5022 srbm_soft_reset |= SOFT_RESET_GRBM;
5023 }
5024
5025 if (reset_mask & RADEON_RESET_DMA)
5026 srbm_soft_reset |= SOFT_RESET_SDMA;
5027
5028 if (reset_mask & RADEON_RESET_DMA1)
5029 srbm_soft_reset |= SOFT_RESET_SDMA1;
5030
5031 if (reset_mask & RADEON_RESET_DISPLAY)
5032 srbm_soft_reset |= SOFT_RESET_DC;
5033
5034 if (reset_mask & RADEON_RESET_RLC)
5035 grbm_soft_reset |= SOFT_RESET_RLC;
5036
5037 if (reset_mask & RADEON_RESET_SEM)
5038 srbm_soft_reset |= SOFT_RESET_SEM;
5039
5040 if (reset_mask & RADEON_RESET_IH)
5041 srbm_soft_reset |= SOFT_RESET_IH;
5042
5043 if (reset_mask & RADEON_RESET_GRBM)
5044 srbm_soft_reset |= SOFT_RESET_GRBM;
5045
5046 if (reset_mask & RADEON_RESET_VMC)
5047 srbm_soft_reset |= SOFT_RESET_VMC;
5048
5049 if (!(rdev->flags & RADEON_IS_IGP)) {
5050 if (reset_mask & RADEON_RESET_MC)
5051 srbm_soft_reset |= SOFT_RESET_MC;
5052 }
5053
5054 if (grbm_soft_reset) {
5055 tmp = RREG32(GRBM_SOFT_RESET);
5056 tmp |= grbm_soft_reset;
5057 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5058 WREG32(GRBM_SOFT_RESET, tmp);
5059 tmp = RREG32(GRBM_SOFT_RESET);
5060
5061 udelay(50);
5062
5063 tmp &= ~grbm_soft_reset;
5064 WREG32(GRBM_SOFT_RESET, tmp);
5065 tmp = RREG32(GRBM_SOFT_RESET);
5066 }
5067
5068 if (srbm_soft_reset) {
5069 tmp = RREG32(SRBM_SOFT_RESET);
5070 tmp |= srbm_soft_reset;
5071 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5072 WREG32(SRBM_SOFT_RESET, tmp);
5073 tmp = RREG32(SRBM_SOFT_RESET);
5074
5075 udelay(50);
5076
5077 tmp &= ~srbm_soft_reset;
5078 WREG32(SRBM_SOFT_RESET, tmp);
5079 tmp = RREG32(SRBM_SOFT_RESET);
5080 }
5081
5082 /* Wait a little for things to settle down */
5083 udelay(50);
5084
5085 evergreen_mc_resume(rdev, &save);
5086 udelay(50);
5087
5088 cik_print_gpu_status_regs(rdev);
5089 }
5090
5091 struct kv_reset_save_regs {
5092 u32 gmcon_reng_execute;
5093 u32 gmcon_misc;
5094 u32 gmcon_misc3;
5095 };
5096
5097 static void kv_save_regs_for_reset(struct radeon_device *rdev,
5098 struct kv_reset_save_regs *save)
5099 {
5100 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5101 save->gmcon_misc = RREG32(GMCON_MISC);
5102 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5103
5104 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5105 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5106 STCTRL_STUTTER_EN));
5107 }
5108
5109 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5110 struct kv_reset_save_regs *save)
5111 {
5112 int i;
5113
5114 WREG32(GMCON_PGFSM_WRITE, 0);
5115 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5116
5117 for (i = 0; i < 5; i++)
5118 WREG32(GMCON_PGFSM_WRITE, 0);
5119
5120 WREG32(GMCON_PGFSM_WRITE, 0);
5121 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5122
5123 for (i = 0; i < 5; i++)
5124 WREG32(GMCON_PGFSM_WRITE, 0);
5125
5126 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5127 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5128
5129 for (i = 0; i < 5; i++)
5130 WREG32(GMCON_PGFSM_WRITE, 0);
5131
5132 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5133 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5134
5135 for (i = 0; i < 5; i++)
5136 WREG32(GMCON_PGFSM_WRITE, 0);
5137
5138 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5139 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5140
5141 for (i = 0; i < 5; i++)
5142 WREG32(GMCON_PGFSM_WRITE, 0);
5143
5144 WREG32(GMCON_PGFSM_WRITE, 0);
5145 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5146
5147 for (i = 0; i < 5; i++)
5148 WREG32(GMCON_PGFSM_WRITE, 0);
5149
5150 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5151 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5152
5153 for (i = 0; i < 5; i++)
5154 WREG32(GMCON_PGFSM_WRITE, 0);
5155
5156 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5157 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5158
5159 for (i = 0; i < 5; i++)
5160 WREG32(GMCON_PGFSM_WRITE, 0);
5161
5162 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5163 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5164
5165 for (i = 0; i < 5; i++)
5166 WREG32(GMCON_PGFSM_WRITE, 0);
5167
5168 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5169 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5170
5171 for (i = 0; i < 5; i++)
5172 WREG32(GMCON_PGFSM_WRITE, 0);
5173
5174 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5175 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5176
5177 WREG32(GMCON_MISC3, save->gmcon_misc3);
5178 WREG32(GMCON_MISC, save->gmcon_misc);
5179 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5180 }
5181
5182 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5183 {
5184 struct evergreen_mc_save save;
5185 struct kv_reset_save_regs kv_save = { 0 };
5186 u32 tmp, i;
5187
5188 dev_info(rdev->dev, "GPU pci config reset\n");
5189
5190 /* disable dpm? */
5191
5192 /* disable cg/pg */
5193 cik_fini_pg(rdev);
5194 cik_fini_cg(rdev);
5195
5196 /* Disable GFX parsing/prefetching */
5197 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5198
5199 /* Disable MEC parsing/prefetching */
5200 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5201
5202 /* sdma0 */
5203 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5204 tmp |= SDMA_HALT;
5205 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5206 /* sdma1 */
5207 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5208 tmp |= SDMA_HALT;
5209 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5210 /* XXX other engines? */
5211
5212 /* halt the rlc, disable cp internal ints */
5213 cik_rlc_stop(rdev);
5214
5215 udelay(50);
5216
5217 /* disable mem access */
5218 evergreen_mc_stop(rdev, &save);
5219 if (evergreen_mc_wait_for_idle(rdev)) {
5220 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5221 }
5222
5223 if (rdev->flags & RADEON_IS_IGP)
5224 kv_save_regs_for_reset(rdev, &kv_save);
5225
5226 /* disable BM */
5227 pci_clear_master(rdev->pdev);
5228 /* reset */
5229 radeon_pci_config_reset(rdev);
5230
5231 udelay(100);
5232
5233 /* wait for asic to come out of reset */
5234 for (i = 0; i < rdev->usec_timeout; i++) {
5235 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5236 break;
5237 udelay(1);
5238 }
5239
5240 /* does asic init need to be run first??? */
5241 if (rdev->flags & RADEON_IS_IGP)
5242 kv_restore_regs_for_reset(rdev, &kv_save);
5243 }
5244
5245 /**
5246 * cik_asic_reset - soft reset GPU
5247 *
5248 * @rdev: radeon_device pointer
5249 * @hard: force hard reset
5250 *
5251 * Look up which blocks are hung and attempt
5252 * to reset them.
5253 * Returns 0 for success.
5254 */
5255 int cik_asic_reset(struct radeon_device *rdev, bool hard)
5256 {
5257 u32 reset_mask;
5258
5259 if (hard) {
5260 cik_gpu_pci_config_reset(rdev);
5261 return 0;
5262 }
5263
5264 reset_mask = cik_gpu_check_soft_reset(rdev);
5265
5266 if (reset_mask)
5267 r600_set_bios_scratch_engine_hung(rdev, true);
5268
5269 /* try soft reset */
5270 cik_gpu_soft_reset(rdev, reset_mask);
5271
5272 reset_mask = cik_gpu_check_soft_reset(rdev);
5273
5274 /* try pci config reset */
5275 if (reset_mask && radeon_hard_reset)
5276 cik_gpu_pci_config_reset(rdev);
5277
5278 reset_mask = cik_gpu_check_soft_reset(rdev);
5279
5280 if (!reset_mask)
5281 r600_set_bios_scratch_engine_hung(rdev, false);
5282
5283 return 0;
5284 }
5285
5286 /**
5287 * cik_gfx_is_lockup - check if the 3D engine is locked up
5288 *
5289 * @rdev: radeon_device pointer
5290 * @ring: radeon_ring structure holding ring information
5291 *
5292 * Check if the 3D engine is locked up (CIK).
5293 * Returns true if the engine is locked, false if not.
5294 */
5295 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5296 {
5297 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5298
5299 if (!(reset_mask & (RADEON_RESET_GFX |
5300 RADEON_RESET_COMPUTE |
5301 RADEON_RESET_CP))) {
5302 radeon_ring_lockup_update(rdev, ring);
5303 return false;
5304 }
5305 return radeon_ring_test_lockup(rdev, ring);
5306 }
5307
5308 /* MC */
5309 /**
5310 * cik_mc_program - program the GPU memory controller
5311 *
5312 * @rdev: radeon_device pointer
5313 *
5314 * Set the location of vram, gart, and AGP in the GPU's
5315 * physical address space (CIK).
5316 */
5317 static void cik_mc_program(struct radeon_device *rdev)
5318 {
5319 struct evergreen_mc_save save;
5320 u32 tmp;
5321 int i, j;
5322
5323 /* Initialize HDP */
5324 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5325 WREG32((0x2c14 + j), 0x00000000);
5326 WREG32((0x2c18 + j), 0x00000000);
5327 WREG32((0x2c1c + j), 0x00000000);
5328 WREG32((0x2c20 + j), 0x00000000);
5329 WREG32((0x2c24 + j), 0x00000000);
5330 }
5331 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5332
5333 evergreen_mc_stop(rdev, &save);
5334 if (radeon_mc_wait_for_idle(rdev)) {
5335 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5336 }
5337 /* Lockout access through VGA aperture*/
5338 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5339 /* Update configuration */
5340 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5341 rdev->mc.vram_start >> 12);
5342 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5343 rdev->mc.vram_end >> 12);
5344 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5345 rdev->vram_scratch.gpu_addr >> 12);
5346 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5347 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5348 WREG32(MC_VM_FB_LOCATION, tmp);
5349 /* XXX double check these! */
5350 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5351 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5352 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5353 WREG32(MC_VM_AGP_BASE, 0);
5354 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5355 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5356 if (radeon_mc_wait_for_idle(rdev)) {
5357 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5358 }
5359 evergreen_mc_resume(rdev, &save);
5360 /* we need to own VRAM, so turn off the VGA renderer here
5361 * to stop it overwriting our objects */
5362 rv515_vga_render_disable(rdev);
5363 }
5364
5365 /**
5366 * cik_mc_init - initialize the memory controller driver params
5367 *
5368 * @rdev: radeon_device pointer
5369 *
5370 * Look up the amount of vram, vram width, and decide how to place
5371 * vram and gart within the GPU's physical address space (CIK).
5372 * Returns 0 for success.
5373 */
5374 static int cik_mc_init(struct radeon_device *rdev)
5375 {
5376 u32 tmp;
5377 int chansize, numchan;
5378
5379 /* Get VRAM informations */
5380 rdev->mc.vram_is_ddr = true;
5381 tmp = RREG32(MC_ARB_RAMCFG);
5382 if (tmp & CHANSIZE_MASK) {
5383 chansize = 64;
5384 } else {
5385 chansize = 32;
5386 }
5387 tmp = RREG32(MC_SHARED_CHMAP);
5388 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5389 case 0:
5390 default:
5391 numchan = 1;
5392 break;
5393 case 1:
5394 numchan = 2;
5395 break;
5396 case 2:
5397 numchan = 4;
5398 break;
5399 case 3:
5400 numchan = 8;
5401 break;
5402 case 4:
5403 numchan = 3;
5404 break;
5405 case 5:
5406 numchan = 6;
5407 break;
5408 case 6:
5409 numchan = 10;
5410 break;
5411 case 7:
5412 numchan = 12;
5413 break;
5414 case 8:
5415 numchan = 16;
5416 break;
5417 }
5418 rdev->mc.vram_width = numchan * chansize;
5419 /* Could aper size report 0 ? */
5420 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5421 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5422 /* size in MB on si */
5423 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5424 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5425 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5426 si_vram_gtt_location(rdev, &rdev->mc);
5427 radeon_update_bandwidth_info(rdev);
5428
5429 return 0;
5430 }
5431
5432 /*
5433 * GART
5434 * VMID 0 is the physical GPU addresses as used by the kernel.
5435 * VMIDs 1-15 are used for userspace clients and are handled
5436 * by the radeon vm/hsa code.
5437 */
5438 /**
5439 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5440 *
5441 * @rdev: radeon_device pointer
5442 *
5443 * Flush the TLB for the VMID 0 page table (CIK).
5444 */
5445 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5446 {
5447 /* flush hdp cache */
5448 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5449
5450 /* bits 0-15 are the VM contexts0-15 */
5451 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5452 }
5453
5454 /**
5455 * cik_pcie_gart_enable - gart enable
5456 *
5457 * @rdev: radeon_device pointer
5458 *
5459 * This sets up the TLBs, programs the page tables for VMID0,
5460 * sets up the hw for VMIDs 1-15 which are allocated on
5461 * demand, and sets up the global locations for the LDS, GDS,
5462 * and GPUVM for FSA64 clients (CIK).
5463 * Returns 0 for success, errors for failure.
5464 */
5465 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5466 {
5467 int r, i;
5468
5469 if (rdev->gart.robj == NULL) {
5470 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5471 return -EINVAL;
5472 }
5473 r = radeon_gart_table_vram_pin(rdev);
5474 if (r)
5475 return r;
5476 /* Setup TLB control */
5477 WREG32(MC_VM_MX_L1_TLB_CNTL,
5478 (0xA << 7) |
5479 ENABLE_L1_TLB |
5480 ENABLE_L1_FRAGMENT_PROCESSING |
5481 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5482 ENABLE_ADVANCED_DRIVER_MODEL |
5483 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5484 /* Setup L2 cache */
5485 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5486 ENABLE_L2_FRAGMENT_PROCESSING |
5487 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5488 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5489 EFFECTIVE_L2_QUEUE_SIZE(7) |
5490 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5491 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5492 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5493 BANK_SELECT(4) |
5494 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5495 /* setup context0 */
5496 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5497 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5498 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5499 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5500 (u32)(rdev->dummy_page.addr >> 12));
5501 WREG32(VM_CONTEXT0_CNTL2, 0);
5502 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5503 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5504
5505 WREG32(0x15D4, 0);
5506 WREG32(0x15D8, 0);
5507 WREG32(0x15DC, 0);
5508
5509 /* restore context1-15 */
5510 /* set vm size, must be a multiple of 4 */
5511 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5512 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
5513 for (i = 1; i < 16; i++) {
5514 if (i < 8)
5515 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5516 rdev->vm_manager.saved_table_addr[i]);
5517 else
5518 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5519 rdev->vm_manager.saved_table_addr[i]);
5520 }
5521
5522 /* enable context1-15 */
5523 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5524 (u32)(rdev->dummy_page.addr >> 12));
5525 WREG32(VM_CONTEXT1_CNTL2, 4);
5526 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5527 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
5528 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5529 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5530 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5531 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5532 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5533 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5534 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5535 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5536 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5537 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5538 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5539 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5540
5541 if (rdev->family == CHIP_KAVERI) {
5542 u32 tmp = RREG32(CHUB_CONTROL);
5543 tmp &= ~BYPASS_VM;
5544 WREG32(CHUB_CONTROL, tmp);
5545 }
5546
5547 /* XXX SH_MEM regs */
5548 /* where to put LDS, scratch, GPUVM in FSA64 space */
5549 mutex_lock(&rdev->srbm_mutex);
5550 for (i = 0; i < 16; i++) {
5551 cik_srbm_select(rdev, 0, 0, 0, i);
5552 /* CP and shaders */
5553 WREG32(SH_MEM_CONFIG, SH_MEM_CONFIG_GFX_DEFAULT);
5554 WREG32(SH_MEM_APE1_BASE, 1);
5555 WREG32(SH_MEM_APE1_LIMIT, 0);
5556 WREG32(SH_MEM_BASES, 0);
5557 /* SDMA GFX */
5558 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5559 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5560 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5561 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5562 /* XXX SDMA RLC - todo */
5563 }
5564 cik_srbm_select(rdev, 0, 0, 0, 0);
5565 mutex_unlock(&rdev->srbm_mutex);
5566
5567 cik_pcie_gart_tlb_flush(rdev);
5568 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5569 (unsigned)(rdev->mc.gtt_size >> 20),
5570 (unsigned long long)rdev->gart.table_addr);
5571 rdev->gart.ready = true;
5572 return 0;
5573 }
5574
5575 /**
5576 * cik_pcie_gart_disable - gart disable
5577 *
5578 * @rdev: radeon_device pointer
5579 *
5580 * This disables all VM page table (CIK).
5581 */
5582 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5583 {
5584 unsigned i;
5585
5586 for (i = 1; i < 16; ++i) {
5587 uint32_t reg;
5588 if (i < 8)
5589 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
5590 else
5591 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
5592 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
5593 }
5594
5595 /* Disable all tables */
5596 WREG32(VM_CONTEXT0_CNTL, 0);
5597 WREG32(VM_CONTEXT1_CNTL, 0);
5598 /* Setup TLB control */
5599 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5600 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5601 /* Setup L2 cache */
5602 WREG32(VM_L2_CNTL,
5603 ENABLE_L2_FRAGMENT_PROCESSING |
5604 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5605 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5606 EFFECTIVE_L2_QUEUE_SIZE(7) |
5607 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5608 WREG32(VM_L2_CNTL2, 0);
5609 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5610 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5611 radeon_gart_table_vram_unpin(rdev);
5612 }
5613
5614 /**
5615 * cik_pcie_gart_fini - vm fini callback
5616 *
5617 * @rdev: radeon_device pointer
5618 *
5619 * Tears down the driver GART/VM setup (CIK).
5620 */
5621 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5622 {
5623 cik_pcie_gart_disable(rdev);
5624 radeon_gart_table_vram_free(rdev);
5625 radeon_gart_fini(rdev);
5626 }
5627
5628 /* vm parser */
5629 /**
5630 * cik_ib_parse - vm ib_parse callback
5631 *
5632 * @rdev: radeon_device pointer
5633 * @ib: indirect buffer pointer
5634 *
5635 * CIK uses hw IB checking so this is a nop (CIK).
5636 */
5637 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5638 {
5639 return 0;
5640 }
5641
5642 /*
5643 * vm
5644 * VMID 0 is the physical GPU addresses as used by the kernel.
5645 * VMIDs 1-15 are used for userspace clients and are handled
5646 * by the radeon vm/hsa code.
5647 */
5648 /**
5649 * cik_vm_init - cik vm init callback
5650 *
5651 * @rdev: radeon_device pointer
5652 *
5653 * Inits cik specific vm parameters (number of VMs, base of vram for
5654 * VMIDs 1-15) (CIK).
5655 * Returns 0 for success.
5656 */
5657 int cik_vm_init(struct radeon_device *rdev)
5658 {
5659 /*
5660 * number of VMs
5661 * VMID 0 is reserved for System
5662 * radeon graphics/compute will use VMIDs 1-15
5663 */
5664 rdev->vm_manager.nvm = 16;
5665 /* base offset of vram pages */
5666 if (rdev->flags & RADEON_IS_IGP) {
5667 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5668 tmp <<= 22;
5669 rdev->vm_manager.vram_base_offset = tmp;
5670 } else
5671 rdev->vm_manager.vram_base_offset = 0;
5672
5673 return 0;
5674 }
5675
5676 /**
5677 * cik_vm_fini - cik vm fini callback
5678 *
5679 * @rdev: radeon_device pointer
5680 *
5681 * Tear down any asic specific VM setup (CIK).
5682 */
5683 void cik_vm_fini(struct radeon_device *rdev)
5684 {
5685 }
5686
5687 /**
5688 * cik_vm_decode_fault - print human readable fault info
5689 *
5690 * @rdev: radeon_device pointer
5691 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5692 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5693 *
5694 * Print human readable fault information (CIK).
5695 */
5696 static void cik_vm_decode_fault(struct radeon_device *rdev,
5697 u32 status, u32 addr, u32 mc_client)
5698 {
5699 u32 mc_id;
5700 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5701 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5702 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5703 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5704
5705 if (rdev->family == CHIP_HAWAII)
5706 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5707 else
5708 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5709
5710 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5711 protections, vmid, addr,
5712 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5713 block, mc_client, mc_id);
5714 }
5715
5716 /**
5717 * cik_vm_flush - cik vm flush using the CP
5718 *
5719 * @rdev: radeon_device pointer
5720 *
5721 * Update the page table base and flush the VM TLB
5722 * using the CP (CIK).
5723 */
5724 void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
5725 unsigned vm_id, uint64_t pd_addr)
5726 {
5727 int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX);
5728
5729 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5730 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5731 WRITE_DATA_DST_SEL(0)));
5732 if (vm_id < 8) {
5733 radeon_ring_write(ring,
5734 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
5735 } else {
5736 radeon_ring_write(ring,
5737 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
5738 }
5739 radeon_ring_write(ring, 0);
5740 radeon_ring_write(ring, pd_addr >> 12);
5741
5742 /* update SH_MEM_* regs */
5743 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5744 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5745 WRITE_DATA_DST_SEL(0)));
5746 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5747 radeon_ring_write(ring, 0);
5748 radeon_ring_write(ring, VMID(vm_id));
5749
5750 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5751 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5752 WRITE_DATA_DST_SEL(0)));
5753 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5754 radeon_ring_write(ring, 0);
5755
5756 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5757 radeon_ring_write(ring, SH_MEM_CONFIG_GFX_DEFAULT); /* SH_MEM_CONFIG */
5758 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5759 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5760
5761 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5762 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5763 WRITE_DATA_DST_SEL(0)));
5764 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5765 radeon_ring_write(ring, 0);
5766 radeon_ring_write(ring, VMID(0));
5767
5768 /* HDP flush */
5769 cik_hdp_flush_cp_ring_emit(rdev, ring->idx);
5770
5771 /* bits 0-15 are the VM contexts0-15 */
5772 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5773 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5774 WRITE_DATA_DST_SEL(0)));
5775 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5776 radeon_ring_write(ring, 0);
5777 radeon_ring_write(ring, 1 << vm_id);
5778
5779 /* wait for the invalidate to complete */
5780 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5781 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
5782 WAIT_REG_MEM_FUNCTION(0) | /* always */
5783 WAIT_REG_MEM_ENGINE(0))); /* me */
5784 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5785 radeon_ring_write(ring, 0);
5786 radeon_ring_write(ring, 0); /* ref */
5787 radeon_ring_write(ring, 0); /* mask */
5788 radeon_ring_write(ring, 0x20); /* poll interval */
5789
5790 /* compute doesn't have PFP */
5791 if (usepfp) {
5792 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5793 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5794 radeon_ring_write(ring, 0x0);
5795 }
5796 }
5797
5798 /*
5799 * RLC
5800 * The RLC is a multi-purpose microengine that handles a
5801 * variety of functions, the most important of which is
5802 * the interrupt controller.
5803 */
5804 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5805 bool enable)
5806 {
5807 u32 tmp = RREG32(CP_INT_CNTL_RING0);
5808
5809 if (enable)
5810 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5811 else
5812 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5813 WREG32(CP_INT_CNTL_RING0, tmp);
5814 }
5815
5816 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5817 {
5818 u32 tmp;
5819
5820 tmp = RREG32(RLC_LB_CNTL);
5821 if (enable)
5822 tmp |= LOAD_BALANCE_ENABLE;
5823 else
5824 tmp &= ~LOAD_BALANCE_ENABLE;
5825 WREG32(RLC_LB_CNTL, tmp);
5826 }
5827
5828 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5829 {
5830 u32 i, j, k;
5831 u32 mask;
5832
5833 mutex_lock(&rdev->grbm_idx_mutex);
5834 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5835 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5836 cik_select_se_sh(rdev, i, j);
5837 for (k = 0; k < rdev->usec_timeout; k++) {
5838 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5839 break;
5840 udelay(1);
5841 }
5842 }
5843 }
5844 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5845 mutex_unlock(&rdev->grbm_idx_mutex);
5846
5847 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5848 for (k = 0; k < rdev->usec_timeout; k++) {
5849 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5850 break;
5851 udelay(1);
5852 }
5853 }
5854
5855 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5856 {
5857 u32 tmp;
5858
5859 tmp = RREG32(RLC_CNTL);
5860 if (tmp != rlc)
5861 WREG32(RLC_CNTL, rlc);
5862 }
5863
5864 static u32 cik_halt_rlc(struct radeon_device *rdev)
5865 {
5866 u32 data, orig;
5867
5868 orig = data = RREG32(RLC_CNTL);
5869
5870 if (data & RLC_ENABLE) {
5871 u32 i;
5872
5873 data &= ~RLC_ENABLE;
5874 WREG32(RLC_CNTL, data);
5875
5876 for (i = 0; i < rdev->usec_timeout; i++) {
5877 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5878 break;
5879 udelay(1);
5880 }
5881
5882 cik_wait_for_rlc_serdes(rdev);
5883 }
5884
5885 return orig;
5886 }
5887
5888 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5889 {
5890 u32 tmp, i, mask;
5891
5892 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5893 WREG32(RLC_GPR_REG2, tmp);
5894
5895 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5896 for (i = 0; i < rdev->usec_timeout; i++) {
5897 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5898 break;
5899 udelay(1);
5900 }
5901
5902 for (i = 0; i < rdev->usec_timeout; i++) {
5903 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5904 break;
5905 udelay(1);
5906 }
5907 }
5908
5909 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5910 {
5911 u32 tmp;
5912
5913 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5914 WREG32(RLC_GPR_REG2, tmp);
5915 }
5916
5917 /**
5918 * cik_rlc_stop - stop the RLC ME
5919 *
5920 * @rdev: radeon_device pointer
5921 *
5922 * Halt the RLC ME (MicroEngine) (CIK).
5923 */
5924 static void cik_rlc_stop(struct radeon_device *rdev)
5925 {
5926 WREG32(RLC_CNTL, 0);
5927
5928 cik_enable_gui_idle_interrupt(rdev, false);
5929
5930 cik_wait_for_rlc_serdes(rdev);
5931 }
5932
5933 /**
5934 * cik_rlc_start - start the RLC ME
5935 *
5936 * @rdev: radeon_device pointer
5937 *
5938 * Unhalt the RLC ME (MicroEngine) (CIK).
5939 */
5940 static void cik_rlc_start(struct radeon_device *rdev)
5941 {
5942 WREG32(RLC_CNTL, RLC_ENABLE);
5943
5944 cik_enable_gui_idle_interrupt(rdev, true);
5945
5946 udelay(50);
5947 }
5948
5949 /**
5950 * cik_rlc_resume - setup the RLC hw
5951 *
5952 * @rdev: radeon_device pointer
5953 *
5954 * Initialize the RLC registers, load the ucode,
5955 * and start the RLC (CIK).
5956 * Returns 0 for success, -EINVAL if the ucode is not available.
5957 */
5958 static int cik_rlc_resume(struct radeon_device *rdev)
5959 {
5960 u32 i, size, tmp;
5961
5962 if (!rdev->rlc_fw)
5963 return -EINVAL;
5964
5965 cik_rlc_stop(rdev);
5966
5967 /* disable CG */
5968 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5969 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5970
5971 si_rlc_reset(rdev);
5972
5973 cik_init_pg(rdev);
5974
5975 cik_init_cg(rdev);
5976
5977 WREG32(RLC_LB_CNTR_INIT, 0);
5978 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5979
5980 mutex_lock(&rdev->grbm_idx_mutex);
5981 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5982 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5983 WREG32(RLC_LB_PARAMS, 0x00600408);
5984 WREG32(RLC_LB_CNTL, 0x80000004);
5985 mutex_unlock(&rdev->grbm_idx_mutex);
5986
5987 WREG32(RLC_MC_CNTL, 0);
5988 WREG32(RLC_UCODE_CNTL, 0);
5989
5990 if (rdev->new_fw) {
5991 const struct rlc_firmware_header_v1_0 *hdr =
5992 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5993 const __le32 *fw_data = (const __le32 *)
5994 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5995
5996 radeon_ucode_print_rlc_hdr(&hdr->header);
5997
5998 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5999 WREG32(RLC_GPM_UCODE_ADDR, 0);
6000 for (i = 0; i < size; i++)
6001 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
6002 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version));
6003 } else {
6004 const __be32 *fw_data;
6005
6006 switch (rdev->family) {
6007 case CHIP_BONAIRE:
6008 case CHIP_HAWAII:
6009 default:
6010 size = BONAIRE_RLC_UCODE_SIZE;
6011 break;
6012 case CHIP_KAVERI:
6013 size = KV_RLC_UCODE_SIZE;
6014 break;
6015 case CHIP_KABINI:
6016 size = KB_RLC_UCODE_SIZE;
6017 break;
6018 case CHIP_MULLINS:
6019 size = ML_RLC_UCODE_SIZE;
6020 break;
6021 }
6022
6023 fw_data = (const __be32 *)rdev->rlc_fw->data;
6024 WREG32(RLC_GPM_UCODE_ADDR, 0);
6025 for (i = 0; i < size; i++)
6026 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6027 WREG32(RLC_GPM_UCODE_ADDR, 0);
6028 }
6029
6030 /* XXX - find out what chips support lbpw */
6031 cik_enable_lbpw(rdev, false);
6032
6033 if (rdev->family == CHIP_BONAIRE)
6034 WREG32(RLC_DRIVER_DMA_STATUS, 0);
6035
6036 cik_rlc_start(rdev);
6037
6038 return 0;
6039 }
6040
6041 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6042 {
6043 u32 data, orig, tmp, tmp2;
6044
6045 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
6046
6047 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
6048 cik_enable_gui_idle_interrupt(rdev, true);
6049
6050 tmp = cik_halt_rlc(rdev);
6051
6052 mutex_lock(&rdev->grbm_idx_mutex);
6053 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6054 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6055 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6056 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6057 WREG32(RLC_SERDES_WR_CTRL, tmp2);
6058 mutex_unlock(&rdev->grbm_idx_mutex);
6059
6060 cik_update_rlc(rdev, tmp);
6061
6062 data |= CGCG_EN | CGLS_EN;
6063 } else {
6064 cik_enable_gui_idle_interrupt(rdev, false);
6065
6066 RREG32(CB_CGTT_SCLK_CTRL);
6067 RREG32(CB_CGTT_SCLK_CTRL);
6068 RREG32(CB_CGTT_SCLK_CTRL);
6069 RREG32(CB_CGTT_SCLK_CTRL);
6070
6071 data &= ~(CGCG_EN | CGLS_EN);
6072 }
6073
6074 if (orig != data)
6075 WREG32(RLC_CGCG_CGLS_CTRL, data);
6076
6077 }
6078
6079 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6080 {
6081 u32 data, orig, tmp = 0;
6082
6083 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6084 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6085 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6086 orig = data = RREG32(CP_MEM_SLP_CNTL);
6087 data |= CP_MEM_LS_EN;
6088 if (orig != data)
6089 WREG32(CP_MEM_SLP_CNTL, data);
6090 }
6091 }
6092
6093 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6094 data |= 0x00000001;
6095 data &= 0xfffffffd;
6096 if (orig != data)
6097 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6098
6099 tmp = cik_halt_rlc(rdev);
6100
6101 mutex_lock(&rdev->grbm_idx_mutex);
6102 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6103 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6104 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6105 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6106 WREG32(RLC_SERDES_WR_CTRL, data);
6107 mutex_unlock(&rdev->grbm_idx_mutex);
6108
6109 cik_update_rlc(rdev, tmp);
6110
6111 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6112 orig = data = RREG32(CGTS_SM_CTRL_REG);
6113 data &= ~SM_MODE_MASK;
6114 data |= SM_MODE(0x2);
6115 data |= SM_MODE_ENABLE;
6116 data &= ~CGTS_OVERRIDE;
6117 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6118 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6119 data &= ~CGTS_LS_OVERRIDE;
6120 data &= ~ON_MONITOR_ADD_MASK;
6121 data |= ON_MONITOR_ADD_EN;
6122 data |= ON_MONITOR_ADD(0x96);
6123 if (orig != data)
6124 WREG32(CGTS_SM_CTRL_REG, data);
6125 }
6126 } else {
6127 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6128 data |= 0x00000003;
6129 if (orig != data)
6130 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6131
6132 data = RREG32(RLC_MEM_SLP_CNTL);
6133 if (data & RLC_MEM_LS_EN) {
6134 data &= ~RLC_MEM_LS_EN;
6135 WREG32(RLC_MEM_SLP_CNTL, data);
6136 }
6137
6138 data = RREG32(CP_MEM_SLP_CNTL);
6139 if (data & CP_MEM_LS_EN) {
6140 data &= ~CP_MEM_LS_EN;
6141 WREG32(CP_MEM_SLP_CNTL, data);
6142 }
6143
6144 orig = data = RREG32(CGTS_SM_CTRL_REG);
6145 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6146 if (orig != data)
6147 WREG32(CGTS_SM_CTRL_REG, data);
6148
6149 tmp = cik_halt_rlc(rdev);
6150
6151 mutex_lock(&rdev->grbm_idx_mutex);
6152 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6153 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6154 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6155 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6156 WREG32(RLC_SERDES_WR_CTRL, data);
6157 mutex_unlock(&rdev->grbm_idx_mutex);
6158
6159 cik_update_rlc(rdev, tmp);
6160 }
6161 }
6162
6163 static const u32 mc_cg_registers[] =
6164 {
6165 MC_HUB_MISC_HUB_CG,
6166 MC_HUB_MISC_SIP_CG,
6167 MC_HUB_MISC_VM_CG,
6168 MC_XPB_CLK_GAT,
6169 ATC_MISC_CG,
6170 MC_CITF_MISC_WR_CG,
6171 MC_CITF_MISC_RD_CG,
6172 MC_CITF_MISC_VM_CG,
6173 VM_L2_CG,
6174 };
6175
6176 static void cik_enable_mc_ls(struct radeon_device *rdev,
6177 bool enable)
6178 {
6179 int i;
6180 u32 orig, data;
6181
6182 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6183 orig = data = RREG32(mc_cg_registers[i]);
6184 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
6185 data |= MC_LS_ENABLE;
6186 else
6187 data &= ~MC_LS_ENABLE;
6188 if (data != orig)
6189 WREG32(mc_cg_registers[i], data);
6190 }
6191 }
6192
6193 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6194 bool enable)
6195 {
6196 int i;
6197 u32 orig, data;
6198
6199 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6200 orig = data = RREG32(mc_cg_registers[i]);
6201 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6202 data |= MC_CG_ENABLE;
6203 else
6204 data &= ~MC_CG_ENABLE;
6205 if (data != orig)
6206 WREG32(mc_cg_registers[i], data);
6207 }
6208 }
6209
6210 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6211 bool enable)
6212 {
6213 u32 orig, data;
6214
6215 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6216 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6217 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6218 } else {
6219 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6220 data |= 0xff000000;
6221 if (data != orig)
6222 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6223
6224 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6225 data |= 0xff000000;
6226 if (data != orig)
6227 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6228 }
6229 }
6230
6231 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6232 bool enable)
6233 {
6234 u32 orig, data;
6235
6236 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6237 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6238 data |= 0x100;
6239 if (orig != data)
6240 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6241
6242 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6243 data |= 0x100;
6244 if (orig != data)
6245 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6246 } else {
6247 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6248 data &= ~0x100;
6249 if (orig != data)
6250 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6251
6252 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6253 data &= ~0x100;
6254 if (orig != data)
6255 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6256 }
6257 }
6258
6259 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6260 bool enable)
6261 {
6262 u32 orig, data;
6263
6264 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6265 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6266 data = 0xfff;
6267 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6268
6269 orig = data = RREG32(UVD_CGC_CTRL);
6270 data |= DCM;
6271 if (orig != data)
6272 WREG32(UVD_CGC_CTRL, data);
6273 } else {
6274 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6275 data &= ~0xfff;
6276 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6277
6278 orig = data = RREG32(UVD_CGC_CTRL);
6279 data &= ~DCM;
6280 if (orig != data)
6281 WREG32(UVD_CGC_CTRL, data);
6282 }
6283 }
6284
6285 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6286 bool enable)
6287 {
6288 u32 orig, data;
6289
6290 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6291
6292 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6293 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6294 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6295 else
6296 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6297 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6298
6299 if (orig != data)
6300 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6301 }
6302
6303 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6304 bool enable)
6305 {
6306 u32 orig, data;
6307
6308 orig = data = RREG32(HDP_HOST_PATH_CNTL);
6309
6310 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6311 data &= ~CLOCK_GATING_DIS;
6312 else
6313 data |= CLOCK_GATING_DIS;
6314
6315 if (orig != data)
6316 WREG32(HDP_HOST_PATH_CNTL, data);
6317 }
6318
6319 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6320 bool enable)
6321 {
6322 u32 orig, data;
6323
6324 orig = data = RREG32(HDP_MEM_POWER_LS);
6325
6326 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6327 data |= HDP_LS_ENABLE;
6328 else
6329 data &= ~HDP_LS_ENABLE;
6330
6331 if (orig != data)
6332 WREG32(HDP_MEM_POWER_LS, data);
6333 }
6334
6335 void cik_update_cg(struct radeon_device *rdev,
6336 u32 block, bool enable)
6337 {
6338
6339 if (block & RADEON_CG_BLOCK_GFX) {
6340 cik_enable_gui_idle_interrupt(rdev, false);
6341 /* order matters! */
6342 if (enable) {
6343 cik_enable_mgcg(rdev, true);
6344 cik_enable_cgcg(rdev, true);
6345 } else {
6346 cik_enable_cgcg(rdev, false);
6347 cik_enable_mgcg(rdev, false);
6348 }
6349 cik_enable_gui_idle_interrupt(rdev, true);
6350 }
6351
6352 if (block & RADEON_CG_BLOCK_MC) {
6353 if (!(rdev->flags & RADEON_IS_IGP)) {
6354 cik_enable_mc_mgcg(rdev, enable);
6355 cik_enable_mc_ls(rdev, enable);
6356 }
6357 }
6358
6359 if (block & RADEON_CG_BLOCK_SDMA) {
6360 cik_enable_sdma_mgcg(rdev, enable);
6361 cik_enable_sdma_mgls(rdev, enable);
6362 }
6363
6364 if (block & RADEON_CG_BLOCK_BIF) {
6365 cik_enable_bif_mgls(rdev, enable);
6366 }
6367
6368 if (block & RADEON_CG_BLOCK_UVD) {
6369 if (rdev->has_uvd)
6370 cik_enable_uvd_mgcg(rdev, enable);
6371 }
6372
6373 if (block & RADEON_CG_BLOCK_HDP) {
6374 cik_enable_hdp_mgcg(rdev, enable);
6375 cik_enable_hdp_ls(rdev, enable);
6376 }
6377
6378 if (block & RADEON_CG_BLOCK_VCE) {
6379 vce_v2_0_enable_mgcg(rdev, enable);
6380 }
6381 }
6382
6383 static void cik_init_cg(struct radeon_device *rdev)
6384 {
6385
6386 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6387
6388 if (rdev->has_uvd)
6389 si_init_uvd_internal_cg(rdev);
6390
6391 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6392 RADEON_CG_BLOCK_SDMA |
6393 RADEON_CG_BLOCK_BIF |
6394 RADEON_CG_BLOCK_UVD |
6395 RADEON_CG_BLOCK_HDP), true);
6396 }
6397
6398 static void cik_fini_cg(struct radeon_device *rdev)
6399 {
6400 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6401 RADEON_CG_BLOCK_SDMA |
6402 RADEON_CG_BLOCK_BIF |
6403 RADEON_CG_BLOCK_UVD |
6404 RADEON_CG_BLOCK_HDP), false);
6405
6406 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6407 }
6408
6409 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6410 bool enable)
6411 {
6412 u32 data, orig;
6413
6414 orig = data = RREG32(RLC_PG_CNTL);
6415 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6416 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6417 else
6418 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6419 if (orig != data)
6420 WREG32(RLC_PG_CNTL, data);
6421 }
6422
6423 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6424 bool enable)
6425 {
6426 u32 data, orig;
6427
6428 orig = data = RREG32(RLC_PG_CNTL);
6429 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6430 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6431 else
6432 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6433 if (orig != data)
6434 WREG32(RLC_PG_CNTL, data);
6435 }
6436
6437 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6438 {
6439 u32 data, orig;
6440
6441 orig = data = RREG32(RLC_PG_CNTL);
6442 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6443 data &= ~DISABLE_CP_PG;
6444 else
6445 data |= DISABLE_CP_PG;
6446 if (orig != data)
6447 WREG32(RLC_PG_CNTL, data);
6448 }
6449
6450 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6451 {
6452 u32 data, orig;
6453
6454 orig = data = RREG32(RLC_PG_CNTL);
6455 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6456 data &= ~DISABLE_GDS_PG;
6457 else
6458 data |= DISABLE_GDS_PG;
6459 if (orig != data)
6460 WREG32(RLC_PG_CNTL, data);
6461 }
6462
6463 #define CP_ME_TABLE_SIZE 96
6464 #define CP_ME_TABLE_OFFSET 2048
6465 #define CP_MEC_TABLE_OFFSET 4096
6466
6467 void cik_init_cp_pg_table(struct radeon_device *rdev)
6468 {
6469 volatile u32 *dst_ptr;
6470 int me, i, max_me = 4;
6471 u32 bo_offset = 0;
6472 u32 table_offset, table_size;
6473
6474 if (rdev->family == CHIP_KAVERI)
6475 max_me = 5;
6476
6477 if (rdev->rlc.cp_table_ptr == NULL)
6478 return;
6479
6480 /* write the cp table buffer */
6481 dst_ptr = rdev->rlc.cp_table_ptr;
6482 for (me = 0; me < max_me; me++) {
6483 if (rdev->new_fw) {
6484 const __le32 *fw_data;
6485 const struct gfx_firmware_header_v1_0 *hdr;
6486
6487 if (me == 0) {
6488 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6489 fw_data = (const __le32 *)
6490 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6491 table_offset = le32_to_cpu(hdr->jt_offset);
6492 table_size = le32_to_cpu(hdr->jt_size);
6493 } else if (me == 1) {
6494 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6495 fw_data = (const __le32 *)
6496 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6497 table_offset = le32_to_cpu(hdr->jt_offset);
6498 table_size = le32_to_cpu(hdr->jt_size);
6499 } else if (me == 2) {
6500 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6501 fw_data = (const __le32 *)
6502 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6503 table_offset = le32_to_cpu(hdr->jt_offset);
6504 table_size = le32_to_cpu(hdr->jt_size);
6505 } else if (me == 3) {
6506 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6507 fw_data = (const __le32 *)
6508 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6509 table_offset = le32_to_cpu(hdr->jt_offset);
6510 table_size = le32_to_cpu(hdr->jt_size);
6511 } else {
6512 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6513 fw_data = (const __le32 *)
6514 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6515 table_offset = le32_to_cpu(hdr->jt_offset);
6516 table_size = le32_to_cpu(hdr->jt_size);
6517 }
6518
6519 for (i = 0; i < table_size; i ++) {
6520 dst_ptr[bo_offset + i] =
6521 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6522 }
6523 bo_offset += table_size;
6524 } else {
6525 const __be32 *fw_data;
6526 table_size = CP_ME_TABLE_SIZE;
6527
6528 if (me == 0) {
6529 fw_data = (const __be32 *)rdev->ce_fw->data;
6530 table_offset = CP_ME_TABLE_OFFSET;
6531 } else if (me == 1) {
6532 fw_data = (const __be32 *)rdev->pfp_fw->data;
6533 table_offset = CP_ME_TABLE_OFFSET;
6534 } else if (me == 2) {
6535 fw_data = (const __be32 *)rdev->me_fw->data;
6536 table_offset = CP_ME_TABLE_OFFSET;
6537 } else {
6538 fw_data = (const __be32 *)rdev->mec_fw->data;
6539 table_offset = CP_MEC_TABLE_OFFSET;
6540 }
6541
6542 for (i = 0; i < table_size; i ++) {
6543 dst_ptr[bo_offset + i] =
6544 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6545 }
6546 bo_offset += table_size;
6547 }
6548 }
6549 }
6550
6551 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6552 bool enable)
6553 {
6554 u32 data, orig;
6555
6556 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6557 orig = data = RREG32(RLC_PG_CNTL);
6558 data |= GFX_PG_ENABLE;
6559 if (orig != data)
6560 WREG32(RLC_PG_CNTL, data);
6561
6562 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6563 data |= AUTO_PG_EN;
6564 if (orig != data)
6565 WREG32(RLC_AUTO_PG_CTRL, data);
6566 } else {
6567 orig = data = RREG32(RLC_PG_CNTL);
6568 data &= ~GFX_PG_ENABLE;
6569 if (orig != data)
6570 WREG32(RLC_PG_CNTL, data);
6571
6572 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6573 data &= ~AUTO_PG_EN;
6574 if (orig != data)
6575 WREG32(RLC_AUTO_PG_CTRL, data);
6576
6577 data = RREG32(DB_RENDER_CONTROL);
6578 }
6579 }
6580
6581 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6582 {
6583 u32 mask = 0, tmp, tmp1;
6584 int i;
6585
6586 mutex_lock(&rdev->grbm_idx_mutex);
6587 cik_select_se_sh(rdev, se, sh);
6588 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6589 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6590 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6591 mutex_unlock(&rdev->grbm_idx_mutex);
6592
6593 tmp &= 0xffff0000;
6594
6595 tmp |= tmp1;
6596 tmp >>= 16;
6597
6598 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6599 mask <<= 1;
6600 mask |= 1;
6601 }
6602
6603 return (~tmp) & mask;
6604 }
6605
6606 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6607 {
6608 u32 i, j, k, active_cu_number = 0;
6609 u32 mask, counter, cu_bitmap;
6610 u32 tmp = 0;
6611
6612 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6613 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6614 mask = 1;
6615 cu_bitmap = 0;
6616 counter = 0;
6617 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6618 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6619 if (counter < 2)
6620 cu_bitmap |= mask;
6621 counter ++;
6622 }
6623 mask <<= 1;
6624 }
6625
6626 active_cu_number += counter;
6627 tmp |= (cu_bitmap << (i * 16 + j * 8));
6628 }
6629 }
6630
6631 WREG32(RLC_PG_AO_CU_MASK, tmp);
6632
6633 tmp = RREG32(RLC_MAX_PG_CU);
6634 tmp &= ~MAX_PU_CU_MASK;
6635 tmp |= MAX_PU_CU(active_cu_number);
6636 WREG32(RLC_MAX_PG_CU, tmp);
6637 }
6638
6639 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6640 bool enable)
6641 {
6642 u32 data, orig;
6643
6644 orig = data = RREG32(RLC_PG_CNTL);
6645 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6646 data |= STATIC_PER_CU_PG_ENABLE;
6647 else
6648 data &= ~STATIC_PER_CU_PG_ENABLE;
6649 if (orig != data)
6650 WREG32(RLC_PG_CNTL, data);
6651 }
6652
6653 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6654 bool enable)
6655 {
6656 u32 data, orig;
6657
6658 orig = data = RREG32(RLC_PG_CNTL);
6659 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6660 data |= DYN_PER_CU_PG_ENABLE;
6661 else
6662 data &= ~DYN_PER_CU_PG_ENABLE;
6663 if (orig != data)
6664 WREG32(RLC_PG_CNTL, data);
6665 }
6666
6667 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6668 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6669
6670 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6671 {
6672 u32 data, orig;
6673 u32 i;
6674
6675 if (rdev->rlc.cs_data) {
6676 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6677 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6678 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6679 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6680 } else {
6681 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6682 for (i = 0; i < 3; i++)
6683 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6684 }
6685 if (rdev->rlc.reg_list) {
6686 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6687 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6688 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6689 }
6690
6691 orig = data = RREG32(RLC_PG_CNTL);
6692 data |= GFX_PG_SRC;
6693 if (orig != data)
6694 WREG32(RLC_PG_CNTL, data);
6695
6696 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6697 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6698
6699 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6700 data &= ~IDLE_POLL_COUNT_MASK;
6701 data |= IDLE_POLL_COUNT(0x60);
6702 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6703
6704 data = 0x10101010;
6705 WREG32(RLC_PG_DELAY, data);
6706
6707 data = RREG32(RLC_PG_DELAY_2);
6708 data &= ~0xff;
6709 data |= 0x3;
6710 WREG32(RLC_PG_DELAY_2, data);
6711
6712 data = RREG32(RLC_AUTO_PG_CTRL);
6713 data &= ~GRBM_REG_SGIT_MASK;
6714 data |= GRBM_REG_SGIT(0x700);
6715 WREG32(RLC_AUTO_PG_CTRL, data);
6716
6717 }
6718
6719 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6720 {
6721 cik_enable_gfx_cgpg(rdev, enable);
6722 cik_enable_gfx_static_mgpg(rdev, enable);
6723 cik_enable_gfx_dynamic_mgpg(rdev, enable);
6724 }
6725
6726 u32 cik_get_csb_size(struct radeon_device *rdev)
6727 {
6728 u32 count = 0;
6729 const struct cs_section_def *sect = NULL;
6730 const struct cs_extent_def *ext = NULL;
6731
6732 if (rdev->rlc.cs_data == NULL)
6733 return 0;
6734
6735 /* begin clear state */
6736 count += 2;
6737 /* context control state */
6738 count += 3;
6739
6740 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6741 for (ext = sect->section; ext->extent != NULL; ++ext) {
6742 if (sect->id == SECT_CONTEXT)
6743 count += 2 + ext->reg_count;
6744 else
6745 return 0;
6746 }
6747 }
6748 /* pa_sc_raster_config/pa_sc_raster_config1 */
6749 count += 4;
6750 /* end clear state */
6751 count += 2;
6752 /* clear state */
6753 count += 2;
6754
6755 return count;
6756 }
6757
6758 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6759 {
6760 u32 count = 0, i;
6761 const struct cs_section_def *sect = NULL;
6762 const struct cs_extent_def *ext = NULL;
6763
6764 if (rdev->rlc.cs_data == NULL)
6765 return;
6766 if (buffer == NULL)
6767 return;
6768
6769 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6770 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6771
6772 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6773 buffer[count++] = cpu_to_le32(0x80000000);
6774 buffer[count++] = cpu_to_le32(0x80000000);
6775
6776 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6777 for (ext = sect->section; ext->extent != NULL; ++ext) {
6778 if (sect->id == SECT_CONTEXT) {
6779 buffer[count++] =
6780 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6781 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6782 for (i = 0; i < ext->reg_count; i++)
6783 buffer[count++] = cpu_to_le32(ext->extent[i]);
6784 } else {
6785 return;
6786 }
6787 }
6788 }
6789
6790 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6791 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
6792 switch (rdev->family) {
6793 case CHIP_BONAIRE:
6794 buffer[count++] = cpu_to_le32(0x16000012);
6795 buffer[count++] = cpu_to_le32(0x00000000);
6796 break;
6797 case CHIP_KAVERI:
6798 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6799 buffer[count++] = cpu_to_le32(0x00000000);
6800 break;
6801 case CHIP_KABINI:
6802 case CHIP_MULLINS:
6803 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6804 buffer[count++] = cpu_to_le32(0x00000000);
6805 break;
6806 case CHIP_HAWAII:
6807 buffer[count++] = cpu_to_le32(0x3a00161a);
6808 buffer[count++] = cpu_to_le32(0x0000002e);
6809 break;
6810 default:
6811 buffer[count++] = cpu_to_le32(0x00000000);
6812 buffer[count++] = cpu_to_le32(0x00000000);
6813 break;
6814 }
6815
6816 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6817 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
6818
6819 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6820 buffer[count++] = cpu_to_le32(0);
6821 }
6822
6823 static void cik_init_pg(struct radeon_device *rdev)
6824 {
6825 if (rdev->pg_flags) {
6826 cik_enable_sck_slowdown_on_pu(rdev, true);
6827 cik_enable_sck_slowdown_on_pd(rdev, true);
6828 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6829 cik_init_gfx_cgpg(rdev);
6830 cik_enable_cp_pg(rdev, true);
6831 cik_enable_gds_pg(rdev, true);
6832 }
6833 cik_init_ao_cu_mask(rdev);
6834 cik_update_gfx_pg(rdev, true);
6835 }
6836 }
6837
6838 static void cik_fini_pg(struct radeon_device *rdev)
6839 {
6840 if (rdev->pg_flags) {
6841 cik_update_gfx_pg(rdev, false);
6842 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6843 cik_enable_cp_pg(rdev, false);
6844 cik_enable_gds_pg(rdev, false);
6845 }
6846 }
6847 }
6848
6849 /*
6850 * Interrupts
6851 * Starting with r6xx, interrupts are handled via a ring buffer.
6852 * Ring buffers are areas of GPU accessible memory that the GPU
6853 * writes interrupt vectors into and the host reads vectors out of.
6854 * There is a rptr (read pointer) that determines where the
6855 * host is currently reading, and a wptr (write pointer)
6856 * which determines where the GPU has written. When the
6857 * pointers are equal, the ring is idle. When the GPU
6858 * writes vectors to the ring buffer, it increments the
6859 * wptr. When there is an interrupt, the host then starts
6860 * fetching commands and processing them until the pointers are
6861 * equal again at which point it updates the rptr.
6862 */
6863
6864 /**
6865 * cik_enable_interrupts - Enable the interrupt ring buffer
6866 *
6867 * @rdev: radeon_device pointer
6868 *
6869 * Enable the interrupt ring buffer (CIK).
6870 */
6871 static void cik_enable_interrupts(struct radeon_device *rdev)
6872 {
6873 u32 ih_cntl = RREG32(IH_CNTL);
6874 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6875
6876 ih_cntl |= ENABLE_INTR;
6877 ih_rb_cntl |= IH_RB_ENABLE;
6878 WREG32(IH_CNTL, ih_cntl);
6879 WREG32(IH_RB_CNTL, ih_rb_cntl);
6880 rdev->ih.enabled = true;
6881 }
6882
6883 /**
6884 * cik_disable_interrupts - Disable the interrupt ring buffer
6885 *
6886 * @rdev: radeon_device pointer
6887 *
6888 * Disable the interrupt ring buffer (CIK).
6889 */
6890 static void cik_disable_interrupts(struct radeon_device *rdev)
6891 {
6892 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6893 u32 ih_cntl = RREG32(IH_CNTL);
6894
6895 ih_rb_cntl &= ~IH_RB_ENABLE;
6896 ih_cntl &= ~ENABLE_INTR;
6897 WREG32(IH_RB_CNTL, ih_rb_cntl);
6898 WREG32(IH_CNTL, ih_cntl);
6899 /* set rptr, wptr to 0 */
6900 WREG32(IH_RB_RPTR, 0);
6901 WREG32(IH_RB_WPTR, 0);
6902 rdev->ih.enabled = false;
6903 rdev->ih.rptr = 0;
6904 }
6905
6906 /**
6907 * cik_disable_interrupt_state - Disable all interrupt sources
6908 *
6909 * @rdev: radeon_device pointer
6910 *
6911 * Clear all interrupt enable bits used by the driver (CIK).
6912 */
6913 static void cik_disable_interrupt_state(struct radeon_device *rdev)
6914 {
6915 u32 tmp;
6916
6917 /* gfx ring */
6918 tmp = RREG32(CP_INT_CNTL_RING0) &
6919 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6920 WREG32(CP_INT_CNTL_RING0, tmp);
6921 /* sdma */
6922 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6923 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6924 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6925 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
6926 /* compute queues */
6927 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6928 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6929 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6930 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6931 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6932 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6933 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6934 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6935 /* grbm */
6936 WREG32(GRBM_INT_CNTL, 0);
6937 /* SRBM */
6938 WREG32(SRBM_INT_CNTL, 0);
6939 /* vline/vblank, etc. */
6940 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6941 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6942 if (rdev->num_crtc >= 4) {
6943 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6944 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6945 }
6946 if (rdev->num_crtc >= 6) {
6947 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6948 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6949 }
6950 /* pflip */
6951 if (rdev->num_crtc >= 2) {
6952 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6953 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6954 }
6955 if (rdev->num_crtc >= 4) {
6956 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6957 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6958 }
6959 if (rdev->num_crtc >= 6) {
6960 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6961 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6962 }
6963
6964 /* dac hotplug */
6965 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6966
6967 /* digital hotplug */
6968 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6969 WREG32(DC_HPD1_INT_CONTROL, tmp);
6970 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6971 WREG32(DC_HPD2_INT_CONTROL, tmp);
6972 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6973 WREG32(DC_HPD3_INT_CONTROL, tmp);
6974 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6975 WREG32(DC_HPD4_INT_CONTROL, tmp);
6976 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6977 WREG32(DC_HPD5_INT_CONTROL, tmp);
6978 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6979 WREG32(DC_HPD6_INT_CONTROL, tmp);
6980
6981 }
6982
6983 /**
6984 * cik_irq_init - init and enable the interrupt ring
6985 *
6986 * @rdev: radeon_device pointer
6987 *
6988 * Allocate a ring buffer for the interrupt controller,
6989 * enable the RLC, disable interrupts, enable the IH
6990 * ring buffer and enable it (CIK).
6991 * Called at device load and reume.
6992 * Returns 0 for success, errors for failure.
6993 */
6994 static int cik_irq_init(struct radeon_device *rdev)
6995 {
6996 int ret = 0;
6997 int rb_bufsz;
6998 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6999
7000 /* allocate ring */
7001 ret = r600_ih_ring_alloc(rdev);
7002 if (ret)
7003 return ret;
7004
7005 /* disable irqs */
7006 cik_disable_interrupts(rdev);
7007
7008 /* init rlc */
7009 ret = cik_rlc_resume(rdev);
7010 if (ret) {
7011 r600_ih_ring_fini(rdev);
7012 return ret;
7013 }
7014
7015 /* setup interrupt control */
7016 /* XXX this should actually be a bus address, not an MC address. same on older asics */
7017 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7018 interrupt_cntl = RREG32(INTERRUPT_CNTL);
7019 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7020 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7021 */
7022 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7023 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7024 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7025 WREG32(INTERRUPT_CNTL, interrupt_cntl);
7026
7027 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
7028 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
7029
7030 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7031 IH_WPTR_OVERFLOW_CLEAR |
7032 (rb_bufsz << 1));
7033
7034 if (rdev->wb.enabled)
7035 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7036
7037 /* set the writeback address whether it's enabled or not */
7038 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7039 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7040
7041 WREG32(IH_RB_CNTL, ih_rb_cntl);
7042
7043 /* set rptr, wptr to 0 */
7044 WREG32(IH_RB_RPTR, 0);
7045 WREG32(IH_RB_WPTR, 0);
7046
7047 /* Default settings for IH_CNTL (disabled at first) */
7048 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7049 /* RPTR_REARM only works if msi's are enabled */
7050 if (rdev->msi_enabled)
7051 ih_cntl |= RPTR_REARM;
7052 WREG32(IH_CNTL, ih_cntl);
7053
7054 /* force the active interrupt state to all disabled */
7055 cik_disable_interrupt_state(rdev);
7056
7057 pci_set_master(rdev->pdev);
7058
7059 /* enable irqs */
7060 cik_enable_interrupts(rdev);
7061
7062 return ret;
7063 }
7064
7065 /**
7066 * cik_irq_set - enable/disable interrupt sources
7067 *
7068 * @rdev: radeon_device pointer
7069 *
7070 * Enable interrupt sources on the GPU (vblanks, hpd,
7071 * etc.) (CIK).
7072 * Returns 0 for success, errors for failure.
7073 */
7074 int cik_irq_set(struct radeon_device *rdev)
7075 {
7076 u32 cp_int_cntl;
7077 u32 cp_m1p0;
7078 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7079 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7080 u32 grbm_int_cntl = 0;
7081 u32 dma_cntl, dma_cntl1;
7082
7083 if (!rdev->irq.installed) {
7084 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7085 return -EINVAL;
7086 }
7087 /* don't enable anything if the ih is disabled */
7088 if (!rdev->ih.enabled) {
7089 cik_disable_interrupts(rdev);
7090 /* force the active interrupt state to all disabled */
7091 cik_disable_interrupt_state(rdev);
7092 return 0;
7093 }
7094
7095 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7096 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7097 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7098
7099 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7100 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7101 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7102 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7103 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7104 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7105
7106 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7107 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7108
7109 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7110
7111 /* enable CP interrupts on all rings */
7112 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7113 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7114 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7115 }
7116 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7117 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7118 DRM_DEBUG("si_irq_set: sw int cp1\n");
7119 if (ring->me == 1) {
7120 switch (ring->pipe) {
7121 case 0:
7122 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7123 break;
7124 default:
7125 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7126 break;
7127 }
7128 } else {
7129 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7130 }
7131 }
7132 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7133 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7134 DRM_DEBUG("si_irq_set: sw int cp2\n");
7135 if (ring->me == 1) {
7136 switch (ring->pipe) {
7137 case 0:
7138 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7139 break;
7140 default:
7141 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7142 break;
7143 }
7144 } else {
7145 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7146 }
7147 }
7148
7149 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7150 DRM_DEBUG("cik_irq_set: sw int dma\n");
7151 dma_cntl |= TRAP_ENABLE;
7152 }
7153
7154 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7155 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7156 dma_cntl1 |= TRAP_ENABLE;
7157 }
7158
7159 if (rdev->irq.crtc_vblank_int[0] ||
7160 atomic_read(&rdev->irq.pflip[0])) {
7161 DRM_DEBUG("cik_irq_set: vblank 0\n");
7162 crtc1 |= VBLANK_INTERRUPT_MASK;
7163 }
7164 if (rdev->irq.crtc_vblank_int[1] ||
7165 atomic_read(&rdev->irq.pflip[1])) {
7166 DRM_DEBUG("cik_irq_set: vblank 1\n");
7167 crtc2 |= VBLANK_INTERRUPT_MASK;
7168 }
7169 if (rdev->irq.crtc_vblank_int[2] ||
7170 atomic_read(&rdev->irq.pflip[2])) {
7171 DRM_DEBUG("cik_irq_set: vblank 2\n");
7172 crtc3 |= VBLANK_INTERRUPT_MASK;
7173 }
7174 if (rdev->irq.crtc_vblank_int[3] ||
7175 atomic_read(&rdev->irq.pflip[3])) {
7176 DRM_DEBUG("cik_irq_set: vblank 3\n");
7177 crtc4 |= VBLANK_INTERRUPT_MASK;
7178 }
7179 if (rdev->irq.crtc_vblank_int[4] ||
7180 atomic_read(&rdev->irq.pflip[4])) {
7181 DRM_DEBUG("cik_irq_set: vblank 4\n");
7182 crtc5 |= VBLANK_INTERRUPT_MASK;
7183 }
7184 if (rdev->irq.crtc_vblank_int[5] ||
7185 atomic_read(&rdev->irq.pflip[5])) {
7186 DRM_DEBUG("cik_irq_set: vblank 5\n");
7187 crtc6 |= VBLANK_INTERRUPT_MASK;
7188 }
7189 if (rdev->irq.hpd[0]) {
7190 DRM_DEBUG("cik_irq_set: hpd 1\n");
7191 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7192 }
7193 if (rdev->irq.hpd[1]) {
7194 DRM_DEBUG("cik_irq_set: hpd 2\n");
7195 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7196 }
7197 if (rdev->irq.hpd[2]) {
7198 DRM_DEBUG("cik_irq_set: hpd 3\n");
7199 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7200 }
7201 if (rdev->irq.hpd[3]) {
7202 DRM_DEBUG("cik_irq_set: hpd 4\n");
7203 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7204 }
7205 if (rdev->irq.hpd[4]) {
7206 DRM_DEBUG("cik_irq_set: hpd 5\n");
7207 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7208 }
7209 if (rdev->irq.hpd[5]) {
7210 DRM_DEBUG("cik_irq_set: hpd 6\n");
7211 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7212 }
7213
7214 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7215
7216 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7217 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7218
7219 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7220
7221 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7222
7223 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7224 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7225 if (rdev->num_crtc >= 4) {
7226 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7227 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7228 }
7229 if (rdev->num_crtc >= 6) {
7230 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7231 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7232 }
7233
7234 if (rdev->num_crtc >= 2) {
7235 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7236 GRPH_PFLIP_INT_MASK);
7237 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7238 GRPH_PFLIP_INT_MASK);
7239 }
7240 if (rdev->num_crtc >= 4) {
7241 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7242 GRPH_PFLIP_INT_MASK);
7243 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7244 GRPH_PFLIP_INT_MASK);
7245 }
7246 if (rdev->num_crtc >= 6) {
7247 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7248 GRPH_PFLIP_INT_MASK);
7249 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7250 GRPH_PFLIP_INT_MASK);
7251 }
7252
7253 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7254 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7255 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7256 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7257 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7258 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7259
7260 /* posting read */
7261 RREG32(SRBM_STATUS);
7262
7263 return 0;
7264 }
7265
7266 /**
7267 * cik_irq_ack - ack interrupt sources
7268 *
7269 * @rdev: radeon_device pointer
7270 *
7271 * Ack interrupt sources on the GPU (vblanks, hpd,
7272 * etc.) (CIK). Certain interrupts sources are sw
7273 * generated and do not require an explicit ack.
7274 */
7275 static inline void cik_irq_ack(struct radeon_device *rdev)
7276 {
7277 u32 tmp;
7278
7279 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7280 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7281 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7282 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7283 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7284 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7285 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7286
7287 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7288 EVERGREEN_CRTC0_REGISTER_OFFSET);
7289 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7290 EVERGREEN_CRTC1_REGISTER_OFFSET);
7291 if (rdev->num_crtc >= 4) {
7292 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7293 EVERGREEN_CRTC2_REGISTER_OFFSET);
7294 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7295 EVERGREEN_CRTC3_REGISTER_OFFSET);
7296 }
7297 if (rdev->num_crtc >= 6) {
7298 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7299 EVERGREEN_CRTC4_REGISTER_OFFSET);
7300 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7301 EVERGREEN_CRTC5_REGISTER_OFFSET);
7302 }
7303
7304 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7305 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7306 GRPH_PFLIP_INT_CLEAR);
7307 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7308 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7309 GRPH_PFLIP_INT_CLEAR);
7310 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7311 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7312 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7313 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7314 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7315 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7316 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7317 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7318
7319 if (rdev->num_crtc >= 4) {
7320 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7321 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7322 GRPH_PFLIP_INT_CLEAR);
7323 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7324 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7325 GRPH_PFLIP_INT_CLEAR);
7326 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7327 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7328 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7329 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7330 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7331 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7332 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7333 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7334 }
7335
7336 if (rdev->num_crtc >= 6) {
7337 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7338 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7339 GRPH_PFLIP_INT_CLEAR);
7340 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7341 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7342 GRPH_PFLIP_INT_CLEAR);
7343 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7344 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7345 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7346 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7347 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7348 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7349 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7350 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7351 }
7352
7353 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7354 tmp = RREG32(DC_HPD1_INT_CONTROL);
7355 tmp |= DC_HPDx_INT_ACK;
7356 WREG32(DC_HPD1_INT_CONTROL, tmp);
7357 }
7358 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7359 tmp = RREG32(DC_HPD2_INT_CONTROL);
7360 tmp |= DC_HPDx_INT_ACK;
7361 WREG32(DC_HPD2_INT_CONTROL, tmp);
7362 }
7363 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7364 tmp = RREG32(DC_HPD3_INT_CONTROL);
7365 tmp |= DC_HPDx_INT_ACK;
7366 WREG32(DC_HPD3_INT_CONTROL, tmp);
7367 }
7368 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7369 tmp = RREG32(DC_HPD4_INT_CONTROL);
7370 tmp |= DC_HPDx_INT_ACK;
7371 WREG32(DC_HPD4_INT_CONTROL, tmp);
7372 }
7373 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7374 tmp = RREG32(DC_HPD5_INT_CONTROL);
7375 tmp |= DC_HPDx_INT_ACK;
7376 WREG32(DC_HPD5_INT_CONTROL, tmp);
7377 }
7378 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7379 tmp = RREG32(DC_HPD6_INT_CONTROL);
7380 tmp |= DC_HPDx_INT_ACK;
7381 WREG32(DC_HPD6_INT_CONTROL, tmp);
7382 }
7383 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT) {
7384 tmp = RREG32(DC_HPD1_INT_CONTROL);
7385 tmp |= DC_HPDx_RX_INT_ACK;
7386 WREG32(DC_HPD1_INT_CONTROL, tmp);
7387 }
7388 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
7389 tmp = RREG32(DC_HPD2_INT_CONTROL);
7390 tmp |= DC_HPDx_RX_INT_ACK;
7391 WREG32(DC_HPD2_INT_CONTROL, tmp);
7392 }
7393 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
7394 tmp = RREG32(DC_HPD3_INT_CONTROL);
7395 tmp |= DC_HPDx_RX_INT_ACK;
7396 WREG32(DC_HPD3_INT_CONTROL, tmp);
7397 }
7398 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
7399 tmp = RREG32(DC_HPD4_INT_CONTROL);
7400 tmp |= DC_HPDx_RX_INT_ACK;
7401 WREG32(DC_HPD4_INT_CONTROL, tmp);
7402 }
7403 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
7404 tmp = RREG32(DC_HPD5_INT_CONTROL);
7405 tmp |= DC_HPDx_RX_INT_ACK;
7406 WREG32(DC_HPD5_INT_CONTROL, tmp);
7407 }
7408 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
7409 tmp = RREG32(DC_HPD6_INT_CONTROL);
7410 tmp |= DC_HPDx_RX_INT_ACK;
7411 WREG32(DC_HPD6_INT_CONTROL, tmp);
7412 }
7413 }
7414
7415 /**
7416 * cik_irq_disable - disable interrupts
7417 *
7418 * @rdev: radeon_device pointer
7419 *
7420 * Disable interrupts on the hw (CIK).
7421 */
7422 static void cik_irq_disable(struct radeon_device *rdev)
7423 {
7424 cik_disable_interrupts(rdev);
7425 /* Wait and acknowledge irq */
7426 mdelay(1);
7427 cik_irq_ack(rdev);
7428 cik_disable_interrupt_state(rdev);
7429 }
7430
7431 /**
7432 * cik_irq_disable - disable interrupts for suspend
7433 *
7434 * @rdev: radeon_device pointer
7435 *
7436 * Disable interrupts and stop the RLC (CIK).
7437 * Used for suspend.
7438 */
7439 static void cik_irq_suspend(struct radeon_device *rdev)
7440 {
7441 cik_irq_disable(rdev);
7442 cik_rlc_stop(rdev);
7443 }
7444
7445 /**
7446 * cik_irq_fini - tear down interrupt support
7447 *
7448 * @rdev: radeon_device pointer
7449 *
7450 * Disable interrupts on the hw and free the IH ring
7451 * buffer (CIK).
7452 * Used for driver unload.
7453 */
7454 static void cik_irq_fini(struct radeon_device *rdev)
7455 {
7456 cik_irq_suspend(rdev);
7457 r600_ih_ring_fini(rdev);
7458 }
7459
7460 /**
7461 * cik_get_ih_wptr - get the IH ring buffer wptr
7462 *
7463 * @rdev: radeon_device pointer
7464 *
7465 * Get the IH ring buffer wptr from either the register
7466 * or the writeback memory buffer (CIK). Also check for
7467 * ring buffer overflow and deal with it.
7468 * Used by cik_irq_process().
7469 * Returns the value of the wptr.
7470 */
7471 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7472 {
7473 u32 wptr, tmp;
7474
7475 if (rdev->wb.enabled)
7476 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7477 else
7478 wptr = RREG32(IH_RB_WPTR);
7479
7480 if (wptr & RB_OVERFLOW) {
7481 wptr &= ~RB_OVERFLOW;
7482 /* When a ring buffer overflow happen start parsing interrupt
7483 * from the last not overwritten vector (wptr + 16). Hopefully
7484 * this should allow us to catchup.
7485 */
7486 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
7487 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
7488 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7489 tmp = RREG32(IH_RB_CNTL);
7490 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7491 WREG32(IH_RB_CNTL, tmp);
7492 }
7493 return (wptr & rdev->ih.ptr_mask);
7494 }
7495
7496 /* CIK IV Ring
7497 * Each IV ring entry is 128 bits:
7498 * [7:0] - interrupt source id
7499 * [31:8] - reserved
7500 * [59:32] - interrupt source data
7501 * [63:60] - reserved
7502 * [71:64] - RINGID
7503 * CP:
7504 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7505 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7506 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7507 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7508 * PIPE_ID - ME0 0=3D
7509 * - ME1&2 compute dispatcher (4 pipes each)
7510 * SDMA:
7511 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7512 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7513 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7514 * [79:72] - VMID
7515 * [95:80] - PASID
7516 * [127:96] - reserved
7517 */
7518 /**
7519 * cik_irq_process - interrupt handler
7520 *
7521 * @rdev: radeon_device pointer
7522 *
7523 * Interrupt hander (CIK). Walk the IH ring,
7524 * ack interrupts and schedule work to handle
7525 * interrupt events.
7526 * Returns irq process return code.
7527 */
7528 int cik_irq_process(struct radeon_device *rdev)
7529 {
7530 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7531 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7532 u32 wptr;
7533 u32 rptr;
7534 u32 src_id, src_data, ring_id;
7535 u8 me_id, pipe_id, queue_id;
7536 u32 ring_index;
7537 bool queue_hotplug = false;
7538 bool queue_dp = false;
7539 bool queue_reset = false;
7540 u32 addr, status, mc_client;
7541 bool queue_thermal = false;
7542
7543 if (!rdev->ih.enabled || rdev->shutdown)
7544 return IRQ_NONE;
7545
7546 wptr = cik_get_ih_wptr(rdev);
7547
7548 restart_ih:
7549 /* is somebody else already processing irqs? */
7550 if (atomic_xchg(&rdev->ih.lock, 1))
7551 return IRQ_NONE;
7552
7553 rptr = rdev->ih.rptr;
7554 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7555
7556 /* Order reading of wptr vs. reading of IH ring data */
7557 rmb();
7558
7559 /* display interrupts */
7560 cik_irq_ack(rdev);
7561
7562 while (rptr != wptr) {
7563 /* wptr/rptr are in bytes! */
7564 ring_index = rptr / 4;
7565
7566 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7567 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7568 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7569
7570 switch (src_id) {
7571 case 1: /* D1 vblank/vline */
7572 switch (src_data) {
7573 case 0: /* D1 vblank */
7574 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT))
7575 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7576
7577 if (rdev->irq.crtc_vblank_int[0]) {
7578 drm_handle_vblank(rdev->ddev, 0);
7579 rdev->pm.vblank_sync = true;
7580 wake_up(&rdev->irq.vblank_queue);
7581 }
7582 if (atomic_read(&rdev->irq.pflip[0]))
7583 radeon_crtc_handle_vblank(rdev, 0);
7584 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7585 DRM_DEBUG("IH: D1 vblank\n");
7586
7587 break;
7588 case 1: /* D1 vline */
7589 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT))
7590 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7591
7592 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7593 DRM_DEBUG("IH: D1 vline\n");
7594
7595 break;
7596 default:
7597 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7598 break;
7599 }
7600 break;
7601 case 2: /* D2 vblank/vline */
7602 switch (src_data) {
7603 case 0: /* D2 vblank */
7604 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
7605 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7606
7607 if (rdev->irq.crtc_vblank_int[1]) {
7608 drm_handle_vblank(rdev->ddev, 1);
7609 rdev->pm.vblank_sync = true;
7610 wake_up(&rdev->irq.vblank_queue);
7611 }
7612 if (atomic_read(&rdev->irq.pflip[1]))
7613 radeon_crtc_handle_vblank(rdev, 1);
7614 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7615 DRM_DEBUG("IH: D2 vblank\n");
7616
7617 break;
7618 case 1: /* D2 vline */
7619 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT))
7620 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7621
7622 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7623 DRM_DEBUG("IH: D2 vline\n");
7624
7625 break;
7626 default:
7627 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7628 break;
7629 }
7630 break;
7631 case 3: /* D3 vblank/vline */
7632 switch (src_data) {
7633 case 0: /* D3 vblank */
7634 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
7635 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7636
7637 if (rdev->irq.crtc_vblank_int[2]) {
7638 drm_handle_vblank(rdev->ddev, 2);
7639 rdev->pm.vblank_sync = true;
7640 wake_up(&rdev->irq.vblank_queue);
7641 }
7642 if (atomic_read(&rdev->irq.pflip[2]))
7643 radeon_crtc_handle_vblank(rdev, 2);
7644 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7645 DRM_DEBUG("IH: D3 vblank\n");
7646
7647 break;
7648 case 1: /* D3 vline */
7649 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
7650 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7651
7652 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7653 DRM_DEBUG("IH: D3 vline\n");
7654
7655 break;
7656 default:
7657 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7658 break;
7659 }
7660 break;
7661 case 4: /* D4 vblank/vline */
7662 switch (src_data) {
7663 case 0: /* D4 vblank */
7664 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
7665 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7666
7667 if (rdev->irq.crtc_vblank_int[3]) {
7668 drm_handle_vblank(rdev->ddev, 3);
7669 rdev->pm.vblank_sync = true;
7670 wake_up(&rdev->irq.vblank_queue);
7671 }
7672 if (atomic_read(&rdev->irq.pflip[3]))
7673 radeon_crtc_handle_vblank(rdev, 3);
7674 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7675 DRM_DEBUG("IH: D4 vblank\n");
7676
7677 break;
7678 case 1: /* D4 vline */
7679 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
7680 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7681
7682 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7683 DRM_DEBUG("IH: D4 vline\n");
7684
7685 break;
7686 default:
7687 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7688 break;
7689 }
7690 break;
7691 case 5: /* D5 vblank/vline */
7692 switch (src_data) {
7693 case 0: /* D5 vblank */
7694 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
7695 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7696
7697 if (rdev->irq.crtc_vblank_int[4]) {
7698 drm_handle_vblank(rdev->ddev, 4);
7699 rdev->pm.vblank_sync = true;
7700 wake_up(&rdev->irq.vblank_queue);
7701 }
7702 if (atomic_read(&rdev->irq.pflip[4]))
7703 radeon_crtc_handle_vblank(rdev, 4);
7704 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7705 DRM_DEBUG("IH: D5 vblank\n");
7706
7707 break;
7708 case 1: /* D5 vline */
7709 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
7710 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7711
7712 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7713 DRM_DEBUG("IH: D5 vline\n");
7714
7715 break;
7716 default:
7717 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7718 break;
7719 }
7720 break;
7721 case 6: /* D6 vblank/vline */
7722 switch (src_data) {
7723 case 0: /* D6 vblank */
7724 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
7725 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7726
7727 if (rdev->irq.crtc_vblank_int[5]) {
7728 drm_handle_vblank(rdev->ddev, 5);
7729 rdev->pm.vblank_sync = true;
7730 wake_up(&rdev->irq.vblank_queue);
7731 }
7732 if (atomic_read(&rdev->irq.pflip[5]))
7733 radeon_crtc_handle_vblank(rdev, 5);
7734 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7735 DRM_DEBUG("IH: D6 vblank\n");
7736
7737 break;
7738 case 1: /* D6 vline */
7739 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
7740 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7741
7742 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7743 DRM_DEBUG("IH: D6 vline\n");
7744
7745 break;
7746 default:
7747 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7748 break;
7749 }
7750 break;
7751 case 8: /* D1 page flip */
7752 case 10: /* D2 page flip */
7753 case 12: /* D3 page flip */
7754 case 14: /* D4 page flip */
7755 case 16: /* D5 page flip */
7756 case 18: /* D6 page flip */
7757 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
7758 if (radeon_use_pflipirq > 0)
7759 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
7760 break;
7761 case 42: /* HPD hotplug */
7762 switch (src_data) {
7763 case 0:
7764 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT))
7765 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7766
7767 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7768 queue_hotplug = true;
7769 DRM_DEBUG("IH: HPD1\n");
7770
7771 break;
7772 case 1:
7773 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT))
7774 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7775
7776 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7777 queue_hotplug = true;
7778 DRM_DEBUG("IH: HPD2\n");
7779
7780 break;
7781 case 2:
7782 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT))
7783 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7784
7785 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7786 queue_hotplug = true;
7787 DRM_DEBUG("IH: HPD3\n");
7788
7789 break;
7790 case 3:
7791 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT))
7792 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7793
7794 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7795 queue_hotplug = true;
7796 DRM_DEBUG("IH: HPD4\n");
7797
7798 break;
7799 case 4:
7800 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT))
7801 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7802
7803 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7804 queue_hotplug = true;
7805 DRM_DEBUG("IH: HPD5\n");
7806
7807 break;
7808 case 5:
7809 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT))
7810 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7811
7812 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7813 queue_hotplug = true;
7814 DRM_DEBUG("IH: HPD6\n");
7815
7816 break;
7817 case 6:
7818 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT))
7819 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7820
7821 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_RX_INTERRUPT;
7822 queue_dp = true;
7823 DRM_DEBUG("IH: HPD_RX 1\n");
7824
7825 break;
7826 case 7:
7827 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT))
7828 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7829
7830 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
7831 queue_dp = true;
7832 DRM_DEBUG("IH: HPD_RX 2\n");
7833
7834 break;
7835 case 8:
7836 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
7837 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7838
7839 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
7840 queue_dp = true;
7841 DRM_DEBUG("IH: HPD_RX 3\n");
7842
7843 break;
7844 case 9:
7845 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
7846 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7847
7848 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
7849 queue_dp = true;
7850 DRM_DEBUG("IH: HPD_RX 4\n");
7851
7852 break;
7853 case 10:
7854 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
7855 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7856
7857 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
7858 queue_dp = true;
7859 DRM_DEBUG("IH: HPD_RX 5\n");
7860
7861 break;
7862 case 11:
7863 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
7864 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7865
7866 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
7867 queue_dp = true;
7868 DRM_DEBUG("IH: HPD_RX 6\n");
7869
7870 break;
7871 default:
7872 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7873 break;
7874 }
7875 break;
7876 case 96:
7877 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
7878 WREG32(SRBM_INT_ACK, 0x1);
7879 break;
7880 case 124: /* UVD */
7881 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7882 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7883 break;
7884 case 146:
7885 case 147:
7886 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7887 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7888 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
7889 /* reset addr and status */
7890 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7891 if (addr == 0x0 && status == 0x0)
7892 break;
7893 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7894 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
7895 addr);
7896 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
7897 status);
7898 cik_vm_decode_fault(rdev, status, addr, mc_client);
7899 break;
7900 case 167: /* VCE */
7901 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
7902 switch (src_data) {
7903 case 0:
7904 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
7905 break;
7906 case 1:
7907 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
7908 break;
7909 default:
7910 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
7911 break;
7912 }
7913 break;
7914 case 176: /* GFX RB CP_INT */
7915 case 177: /* GFX IB CP_INT */
7916 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7917 break;
7918 case 181: /* CP EOP event */
7919 DRM_DEBUG("IH: CP EOP\n");
7920 /* XXX check the bitfield order! */
7921 me_id = (ring_id & 0x60) >> 5;
7922 pipe_id = (ring_id & 0x18) >> 3;
7923 queue_id = (ring_id & 0x7) >> 0;
7924 switch (me_id) {
7925 case 0:
7926 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7927 break;
7928 case 1:
7929 case 2:
7930 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7931 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7932 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7933 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7934 break;
7935 }
7936 break;
7937 case 184: /* CP Privileged reg access */
7938 DRM_ERROR("Illegal register access in command stream\n");
7939 /* XXX check the bitfield order! */
7940 me_id = (ring_id & 0x60) >> 5;
7941 pipe_id = (ring_id & 0x18) >> 3;
7942 queue_id = (ring_id & 0x7) >> 0;
7943 switch (me_id) {
7944 case 0:
7945 /* This results in a full GPU reset, but all we need to do is soft
7946 * reset the CP for gfx
7947 */
7948 queue_reset = true;
7949 break;
7950 case 1:
7951 /* XXX compute */
7952 queue_reset = true;
7953 break;
7954 case 2:
7955 /* XXX compute */
7956 queue_reset = true;
7957 break;
7958 }
7959 break;
7960 case 185: /* CP Privileged inst */
7961 DRM_ERROR("Illegal instruction in command stream\n");
7962 /* XXX check the bitfield order! */
7963 me_id = (ring_id & 0x60) >> 5;
7964 pipe_id = (ring_id & 0x18) >> 3;
7965 queue_id = (ring_id & 0x7) >> 0;
7966 switch (me_id) {
7967 case 0:
7968 /* This results in a full GPU reset, but all we need to do is soft
7969 * reset the CP for gfx
7970 */
7971 queue_reset = true;
7972 break;
7973 case 1:
7974 /* XXX compute */
7975 queue_reset = true;
7976 break;
7977 case 2:
7978 /* XXX compute */
7979 queue_reset = true;
7980 break;
7981 }
7982 break;
7983 case 224: /* SDMA trap event */
7984 /* XXX check the bitfield order! */
7985 me_id = (ring_id & 0x3) >> 0;
7986 queue_id = (ring_id & 0xc) >> 2;
7987 DRM_DEBUG("IH: SDMA trap\n");
7988 switch (me_id) {
7989 case 0:
7990 switch (queue_id) {
7991 case 0:
7992 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7993 break;
7994 case 1:
7995 /* XXX compute */
7996 break;
7997 case 2:
7998 /* XXX compute */
7999 break;
8000 }
8001 break;
8002 case 1:
8003 switch (queue_id) {
8004 case 0:
8005 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8006 break;
8007 case 1:
8008 /* XXX compute */
8009 break;
8010 case 2:
8011 /* XXX compute */
8012 break;
8013 }
8014 break;
8015 }
8016 break;
8017 case 230: /* thermal low to high */
8018 DRM_DEBUG("IH: thermal low to high\n");
8019 rdev->pm.dpm.thermal.high_to_low = false;
8020 queue_thermal = true;
8021 break;
8022 case 231: /* thermal high to low */
8023 DRM_DEBUG("IH: thermal high to low\n");
8024 rdev->pm.dpm.thermal.high_to_low = true;
8025 queue_thermal = true;
8026 break;
8027 case 233: /* GUI IDLE */
8028 DRM_DEBUG("IH: GUI idle\n");
8029 break;
8030 case 241: /* SDMA Privileged inst */
8031 case 247: /* SDMA Privileged inst */
8032 DRM_ERROR("Illegal instruction in SDMA command stream\n");
8033 /* XXX check the bitfield order! */
8034 me_id = (ring_id & 0x3) >> 0;
8035 queue_id = (ring_id & 0xc) >> 2;
8036 switch (me_id) {
8037 case 0:
8038 switch (queue_id) {
8039 case 0:
8040 queue_reset = true;
8041 break;
8042 case 1:
8043 /* XXX compute */
8044 queue_reset = true;
8045 break;
8046 case 2:
8047 /* XXX compute */
8048 queue_reset = true;
8049 break;
8050 }
8051 break;
8052 case 1:
8053 switch (queue_id) {
8054 case 0:
8055 queue_reset = true;
8056 break;
8057 case 1:
8058 /* XXX compute */
8059 queue_reset = true;
8060 break;
8061 case 2:
8062 /* XXX compute */
8063 queue_reset = true;
8064 break;
8065 }
8066 break;
8067 }
8068 break;
8069 default:
8070 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8071 break;
8072 }
8073
8074 /* wptr/rptr are in bytes! */
8075 rptr += 16;
8076 rptr &= rdev->ih.ptr_mask;
8077 WREG32(IH_RB_RPTR, rptr);
8078 }
8079 if (queue_dp)
8080 schedule_work(&rdev->dp_work);
8081 if (queue_hotplug)
8082 schedule_delayed_work(&rdev->hotplug_work, 0);
8083 if (queue_reset) {
8084 rdev->needs_reset = true;
8085 wake_up_all(&rdev->fence_queue);
8086 }
8087 if (queue_thermal)
8088 schedule_work(&rdev->pm.dpm.thermal.work);
8089 rdev->ih.rptr = rptr;
8090 atomic_set(&rdev->ih.lock, 0);
8091
8092 /* make sure wptr hasn't changed while processing */
8093 wptr = cik_get_ih_wptr(rdev);
8094 if (wptr != rptr)
8095 goto restart_ih;
8096
8097 return IRQ_HANDLED;
8098 }
8099
8100 /*
8101 * startup/shutdown callbacks
8102 */
8103 static void cik_uvd_init(struct radeon_device *rdev)
8104 {
8105 int r;
8106
8107 if (!rdev->has_uvd)
8108 return;
8109
8110 r = radeon_uvd_init(rdev);
8111 if (r) {
8112 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
8113 /*
8114 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
8115 * to early fails cik_uvd_start() and thus nothing happens
8116 * there. So it is pointless to try to go through that code
8117 * hence why we disable uvd here.
8118 */
8119 rdev->has_uvd = 0;
8120 return;
8121 }
8122 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
8123 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
8124 }
8125
8126 static void cik_uvd_start(struct radeon_device *rdev)
8127 {
8128 int r;
8129
8130 if (!rdev->has_uvd)
8131 return;
8132
8133 r = radeon_uvd_resume(rdev);
8134 if (r) {
8135 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
8136 goto error;
8137 }
8138 r = uvd_v4_2_resume(rdev);
8139 if (r) {
8140 dev_err(rdev->dev, "failed UVD 4.2 resume (%d).\n", r);
8141 goto error;
8142 }
8143 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
8144 if (r) {
8145 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
8146 goto error;
8147 }
8148 return;
8149
8150 error:
8151 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8152 }
8153
8154 static void cik_uvd_resume(struct radeon_device *rdev)
8155 {
8156 struct radeon_ring *ring;
8157 int r;
8158
8159 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
8160 return;
8161
8162 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8163 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
8164 if (r) {
8165 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
8166 return;
8167 }
8168 r = uvd_v1_0_init(rdev);
8169 if (r) {
8170 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
8171 return;
8172 }
8173 }
8174
8175 static void cik_vce_init(struct radeon_device *rdev)
8176 {
8177 int r;
8178
8179 if (!rdev->has_vce)
8180 return;
8181
8182 r = radeon_vce_init(rdev);
8183 if (r) {
8184 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
8185 /*
8186 * At this point rdev->vce.vcpu_bo is NULL which trickles down
8187 * to early fails cik_vce_start() and thus nothing happens
8188 * there. So it is pointless to try to go through that code
8189 * hence why we disable vce here.
8190 */
8191 rdev->has_vce = 0;
8192 return;
8193 }
8194 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
8195 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
8196 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
8197 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
8198 }
8199
8200 static void cik_vce_start(struct radeon_device *rdev)
8201 {
8202 int r;
8203
8204 if (!rdev->has_vce)
8205 return;
8206
8207 r = radeon_vce_resume(rdev);
8208 if (r) {
8209 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
8210 goto error;
8211 }
8212 r = vce_v2_0_resume(rdev);
8213 if (r) {
8214 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
8215 goto error;
8216 }
8217 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
8218 if (r) {
8219 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
8220 goto error;
8221 }
8222 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
8223 if (r) {
8224 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
8225 goto error;
8226 }
8227 return;
8228
8229 error:
8230 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8231 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8232 }
8233
8234 static void cik_vce_resume(struct radeon_device *rdev)
8235 {
8236 struct radeon_ring *ring;
8237 int r;
8238
8239 if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
8240 return;
8241
8242 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8243 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
8244 if (r) {
8245 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
8246 return;
8247 }
8248 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8249 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
8250 if (r) {
8251 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
8252 return;
8253 }
8254 r = vce_v1_0_init(rdev);
8255 if (r) {
8256 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
8257 return;
8258 }
8259 }
8260
8261 /**
8262 * cik_startup - program the asic to a functional state
8263 *
8264 * @rdev: radeon_device pointer
8265 *
8266 * Programs the asic to a functional state (CIK).
8267 * Called by cik_init() and cik_resume().
8268 * Returns 0 for success, error for failure.
8269 */
8270 static int cik_startup(struct radeon_device *rdev)
8271 {
8272 struct radeon_ring *ring;
8273 u32 nop;
8274 int r;
8275
8276 /* enable pcie gen2/3 link */
8277 cik_pcie_gen3_enable(rdev);
8278 /* enable aspm */
8279 cik_program_aspm(rdev);
8280
8281 /* scratch needs to be initialized before MC */
8282 r = r600_vram_scratch_init(rdev);
8283 if (r)
8284 return r;
8285
8286 cik_mc_program(rdev);
8287
8288 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
8289 r = ci_mc_load_microcode(rdev);
8290 if (r) {
8291 DRM_ERROR("Failed to load MC firmware!\n");
8292 return r;
8293 }
8294 }
8295
8296 r = cik_pcie_gart_enable(rdev);
8297 if (r)
8298 return r;
8299 cik_gpu_init(rdev);
8300
8301 /* allocate rlc buffers */
8302 if (rdev->flags & RADEON_IS_IGP) {
8303 if (rdev->family == CHIP_KAVERI) {
8304 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8305 rdev->rlc.reg_list_size =
8306 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8307 } else {
8308 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8309 rdev->rlc.reg_list_size =
8310 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8311 }
8312 }
8313 rdev->rlc.cs_data = ci_cs_data;
8314 rdev->rlc.cp_table_size = ALIGN(CP_ME_TABLE_SIZE * 5 * 4, 2048); /* CP JT */
8315 rdev->rlc.cp_table_size += 64 * 1024; /* GDS */
8316 r = sumo_rlc_init(rdev);
8317 if (r) {
8318 DRM_ERROR("Failed to init rlc BOs!\n");
8319 return r;
8320 }
8321
8322 /* allocate wb buffer */
8323 r = radeon_wb_init(rdev);
8324 if (r)
8325 return r;
8326
8327 /* allocate mec buffers */
8328 r = cik_mec_init(rdev);
8329 if (r) {
8330 DRM_ERROR("Failed to init MEC BOs!\n");
8331 return r;
8332 }
8333
8334 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8335 if (r) {
8336 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8337 return r;
8338 }
8339
8340 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8341 if (r) {
8342 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8343 return r;
8344 }
8345
8346 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8347 if (r) {
8348 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8349 return r;
8350 }
8351
8352 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8353 if (r) {
8354 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8355 return r;
8356 }
8357
8358 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8359 if (r) {
8360 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8361 return r;
8362 }
8363
8364 cik_uvd_start(rdev);
8365 cik_vce_start(rdev);
8366
8367 /* Enable IRQ */
8368 if (!rdev->irq.installed) {
8369 r = radeon_irq_kms_init(rdev);
8370 if (r)
8371 return r;
8372 }
8373
8374 r = cik_irq_init(rdev);
8375 if (r) {
8376 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8377 radeon_irq_kms_fini(rdev);
8378 return r;
8379 }
8380 cik_irq_set(rdev);
8381
8382 if (rdev->family == CHIP_HAWAII) {
8383 if (rdev->new_fw)
8384 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8385 else
8386 nop = RADEON_CP_PACKET2;
8387 } else {
8388 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8389 }
8390
8391 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8392 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
8393 nop);
8394 if (r)
8395 return r;
8396
8397 /* set up the compute queues */
8398 /* type-2 packets are deprecated on MEC, use type-3 instead */
8399 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8400 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
8401 nop);
8402 if (r)
8403 return r;
8404 ring->me = 1; /* first MEC */
8405 ring->pipe = 0; /* first pipe */
8406 ring->queue = 0; /* first queue */
8407 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8408
8409 /* type-2 packets are deprecated on MEC, use type-3 instead */
8410 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8411 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
8412 nop);
8413 if (r)
8414 return r;
8415 /* dGPU only have 1 MEC */
8416 ring->me = 1; /* first MEC */
8417 ring->pipe = 0; /* first pipe */
8418 ring->queue = 1; /* second queue */
8419 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8420
8421 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8422 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
8423 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8424 if (r)
8425 return r;
8426
8427 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8428 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
8429 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8430 if (r)
8431 return r;
8432
8433 r = cik_cp_resume(rdev);
8434 if (r)
8435 return r;
8436
8437 r = cik_sdma_resume(rdev);
8438 if (r)
8439 return r;
8440
8441 cik_uvd_resume(rdev);
8442 cik_vce_resume(rdev);
8443
8444 r = radeon_ib_pool_init(rdev);
8445 if (r) {
8446 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8447 return r;
8448 }
8449
8450 r = radeon_vm_manager_init(rdev);
8451 if (r) {
8452 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8453 return r;
8454 }
8455
8456 r = radeon_audio_init(rdev);
8457 if (r)
8458 return r;
8459
8460 return 0;
8461 }
8462
8463 /**
8464 * cik_resume - resume the asic to a functional state
8465 *
8466 * @rdev: radeon_device pointer
8467 *
8468 * Programs the asic to a functional state (CIK).
8469 * Called at resume.
8470 * Returns 0 for success, error for failure.
8471 */
8472 int cik_resume(struct radeon_device *rdev)
8473 {
8474 int r;
8475
8476 /* post card */
8477 atom_asic_init(rdev->mode_info.atom_context);
8478
8479 /* init golden registers */
8480 cik_init_golden_registers(rdev);
8481
8482 if (rdev->pm.pm_method == PM_METHOD_DPM)
8483 radeon_pm_resume(rdev);
8484
8485 rdev->accel_working = true;
8486 r = cik_startup(rdev);
8487 if (r) {
8488 DRM_ERROR("cik startup failed on resume\n");
8489 rdev->accel_working = false;
8490 return r;
8491 }
8492
8493 return r;
8494
8495 }
8496
8497 /**
8498 * cik_suspend - suspend the asic
8499 *
8500 * @rdev: radeon_device pointer
8501 *
8502 * Bring the chip into a state suitable for suspend (CIK).
8503 * Called at suspend.
8504 * Returns 0 for success.
8505 */
8506 int cik_suspend(struct radeon_device *rdev)
8507 {
8508 radeon_pm_suspend(rdev);
8509 radeon_audio_fini(rdev);
8510 radeon_vm_manager_fini(rdev);
8511 cik_cp_enable(rdev, false);
8512 cik_sdma_enable(rdev, false);
8513 if (rdev->has_uvd) {
8514 uvd_v1_0_fini(rdev);
8515 radeon_uvd_suspend(rdev);
8516 }
8517 if (rdev->has_vce)
8518 radeon_vce_suspend(rdev);
8519 cik_fini_pg(rdev);
8520 cik_fini_cg(rdev);
8521 cik_irq_suspend(rdev);
8522 radeon_wb_disable(rdev);
8523 cik_pcie_gart_disable(rdev);
8524 return 0;
8525 }
8526
8527 /* Plan is to move initialization in that function and use
8528 * helper function so that radeon_device_init pretty much
8529 * do nothing more than calling asic specific function. This
8530 * should also allow to remove a bunch of callback function
8531 * like vram_info.
8532 */
8533 /**
8534 * cik_init - asic specific driver and hw init
8535 *
8536 * @rdev: radeon_device pointer
8537 *
8538 * Setup asic specific driver variables and program the hw
8539 * to a functional state (CIK).
8540 * Called at driver startup.
8541 * Returns 0 for success, errors for failure.
8542 */
8543 int cik_init(struct radeon_device *rdev)
8544 {
8545 struct radeon_ring *ring;
8546 int r;
8547
8548 /* Read BIOS */
8549 if (!radeon_get_bios(rdev)) {
8550 if (ASIC_IS_AVIVO(rdev))
8551 return -EINVAL;
8552 }
8553 /* Must be an ATOMBIOS */
8554 if (!rdev->is_atom_bios) {
8555 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8556 return -EINVAL;
8557 }
8558 r = radeon_atombios_init(rdev);
8559 if (r)
8560 return r;
8561
8562 /* Post card if necessary */
8563 if (!radeon_card_posted(rdev)) {
8564 if (!rdev->bios) {
8565 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8566 return -EINVAL;
8567 }
8568 DRM_INFO("GPU not posted. posting now...\n");
8569 atom_asic_init(rdev->mode_info.atom_context);
8570 }
8571 /* init golden registers */
8572 cik_init_golden_registers(rdev);
8573 /* Initialize scratch registers */
8574 cik_scratch_init(rdev);
8575 /* Initialize surface registers */
8576 radeon_surface_init(rdev);
8577 /* Initialize clocks */
8578 radeon_get_clock_info(rdev->ddev);
8579
8580 /* Fence driver */
8581 r = radeon_fence_driver_init(rdev);
8582 if (r)
8583 return r;
8584
8585 /* initialize memory controller */
8586 r = cik_mc_init(rdev);
8587 if (r)
8588 return r;
8589 /* Memory manager */
8590 r = radeon_bo_init(rdev);
8591 if (r)
8592 return r;
8593
8594 if (rdev->flags & RADEON_IS_IGP) {
8595 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8596 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8597 r = cik_init_microcode(rdev);
8598 if (r) {
8599 DRM_ERROR("Failed to load firmware!\n");
8600 return r;
8601 }
8602 }
8603 } else {
8604 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8605 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8606 !rdev->mc_fw) {
8607 r = cik_init_microcode(rdev);
8608 if (r) {
8609 DRM_ERROR("Failed to load firmware!\n");
8610 return r;
8611 }
8612 }
8613 }
8614
8615 /* Initialize power management */
8616 radeon_pm_init(rdev);
8617
8618 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8619 ring->ring_obj = NULL;
8620 r600_ring_init(rdev, ring, 1024 * 1024);
8621
8622 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8623 ring->ring_obj = NULL;
8624 r600_ring_init(rdev, ring, 1024 * 1024);
8625 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8626 if (r)
8627 return r;
8628
8629 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8630 ring->ring_obj = NULL;
8631 r600_ring_init(rdev, ring, 1024 * 1024);
8632 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8633 if (r)
8634 return r;
8635
8636 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8637 ring->ring_obj = NULL;
8638 r600_ring_init(rdev, ring, 256 * 1024);
8639
8640 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8641 ring->ring_obj = NULL;
8642 r600_ring_init(rdev, ring, 256 * 1024);
8643
8644 cik_uvd_init(rdev);
8645 cik_vce_init(rdev);
8646
8647 rdev->ih.ring_obj = NULL;
8648 r600_ih_ring_init(rdev, 64 * 1024);
8649
8650 r = r600_pcie_gart_init(rdev);
8651 if (r)
8652 return r;
8653
8654 rdev->accel_working = true;
8655 r = cik_startup(rdev);
8656 if (r) {
8657 dev_err(rdev->dev, "disabling GPU acceleration\n");
8658 cik_cp_fini(rdev);
8659 cik_sdma_fini(rdev);
8660 cik_irq_fini(rdev);
8661 sumo_rlc_fini(rdev);
8662 cik_mec_fini(rdev);
8663 radeon_wb_fini(rdev);
8664 radeon_ib_pool_fini(rdev);
8665 radeon_vm_manager_fini(rdev);
8666 radeon_irq_kms_fini(rdev);
8667 cik_pcie_gart_fini(rdev);
8668 rdev->accel_working = false;
8669 }
8670
8671 /* Don't start up if the MC ucode is missing.
8672 * The default clocks and voltages before the MC ucode
8673 * is loaded are not suffient for advanced operations.
8674 */
8675 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8676 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8677 return -EINVAL;
8678 }
8679
8680 return 0;
8681 }
8682
8683 /**
8684 * cik_fini - asic specific driver and hw fini
8685 *
8686 * @rdev: radeon_device pointer
8687 *
8688 * Tear down the asic specific driver variables and program the hw
8689 * to an idle state (CIK).
8690 * Called at driver unload.
8691 */
8692 void cik_fini(struct radeon_device *rdev)
8693 {
8694 radeon_pm_fini(rdev);
8695 cik_cp_fini(rdev);
8696 cik_sdma_fini(rdev);
8697 cik_fini_pg(rdev);
8698 cik_fini_cg(rdev);
8699 cik_irq_fini(rdev);
8700 sumo_rlc_fini(rdev);
8701 cik_mec_fini(rdev);
8702 radeon_wb_fini(rdev);
8703 radeon_vm_manager_fini(rdev);
8704 radeon_ib_pool_fini(rdev);
8705 radeon_irq_kms_fini(rdev);
8706 uvd_v1_0_fini(rdev);
8707 radeon_uvd_fini(rdev);
8708 radeon_vce_fini(rdev);
8709 cik_pcie_gart_fini(rdev);
8710 r600_vram_scratch_fini(rdev);
8711 radeon_gem_fini(rdev);
8712 radeon_fence_driver_fini(rdev);
8713 radeon_bo_fini(rdev);
8714 radeon_atombios_fini(rdev);
8715 kfree(rdev->bios);
8716 rdev->bios = NULL;
8717 }
8718
8719 void dce8_program_fmt(struct drm_encoder *encoder)
8720 {
8721 struct drm_device *dev = encoder->dev;
8722 struct radeon_device *rdev = dev->dev_private;
8723 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8724 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8725 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8726 int bpc = 0;
8727 u32 tmp = 0;
8728 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8729
8730 if (connector) {
8731 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8732 bpc = radeon_get_monitor_bpc(connector);
8733 dither = radeon_connector->dither;
8734 }
8735
8736 /* LVDS/eDP FMT is set up by atom */
8737 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8738 return;
8739
8740 /* not needed for analog */
8741 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8742 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8743 return;
8744
8745 if (bpc == 0)
8746 return;
8747
8748 switch (bpc) {
8749 case 6:
8750 if (dither == RADEON_FMT_DITHER_ENABLE)
8751 /* XXX sort out optimal dither settings */
8752 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8753 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8754 else
8755 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8756 break;
8757 case 8:
8758 if (dither == RADEON_FMT_DITHER_ENABLE)
8759 /* XXX sort out optimal dither settings */
8760 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8761 FMT_RGB_RANDOM_ENABLE |
8762 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8763 else
8764 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8765 break;
8766 case 10:
8767 if (dither == RADEON_FMT_DITHER_ENABLE)
8768 /* XXX sort out optimal dither settings */
8769 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8770 FMT_RGB_RANDOM_ENABLE |
8771 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8772 else
8773 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8774 break;
8775 default:
8776 /* not needed */
8777 break;
8778 }
8779
8780 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8781 }
8782
8783 /* display watermark setup */
8784 /**
8785 * dce8_line_buffer_adjust - Set up the line buffer
8786 *
8787 * @rdev: radeon_device pointer
8788 * @radeon_crtc: the selected display controller
8789 * @mode: the current display mode on the selected display
8790 * controller
8791 *
8792 * Setup up the line buffer allocation for
8793 * the selected display controller (CIK).
8794 * Returns the line buffer size in pixels.
8795 */
8796 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8797 struct radeon_crtc *radeon_crtc,
8798 struct drm_display_mode *mode)
8799 {
8800 u32 tmp, buffer_alloc, i;
8801 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8802 /*
8803 * Line Buffer Setup
8804 * There are 6 line buffers, one for each display controllers.
8805 * There are 3 partitions per LB. Select the number of partitions
8806 * to enable based on the display width. For display widths larger
8807 * than 4096, you need use to use 2 display controllers and combine
8808 * them using the stereo blender.
8809 */
8810 if (radeon_crtc->base.enabled && mode) {
8811 if (mode->crtc_hdisplay < 1920) {
8812 tmp = 1;
8813 buffer_alloc = 2;
8814 } else if (mode->crtc_hdisplay < 2560) {
8815 tmp = 2;
8816 buffer_alloc = 2;
8817 } else if (mode->crtc_hdisplay < 4096) {
8818 tmp = 0;
8819 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8820 } else {
8821 DRM_DEBUG_KMS("Mode too big for LB!\n");
8822 tmp = 0;
8823 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8824 }
8825 } else {
8826 tmp = 1;
8827 buffer_alloc = 0;
8828 }
8829
8830 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8831 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8832
8833 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8834 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8835 for (i = 0; i < rdev->usec_timeout; i++) {
8836 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8837 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8838 break;
8839 udelay(1);
8840 }
8841
8842 if (radeon_crtc->base.enabled && mode) {
8843 switch (tmp) {
8844 case 0:
8845 default:
8846 return 4096 * 2;
8847 case 1:
8848 return 1920 * 2;
8849 case 2:
8850 return 2560 * 2;
8851 }
8852 }
8853
8854 /* controller not enabled, so no lb used */
8855 return 0;
8856 }
8857
8858 /**
8859 * cik_get_number_of_dram_channels - get the number of dram channels
8860 *
8861 * @rdev: radeon_device pointer
8862 *
8863 * Look up the number of video ram channels (CIK).
8864 * Used for display watermark bandwidth calculations
8865 * Returns the number of dram channels
8866 */
8867 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8868 {
8869 u32 tmp = RREG32(MC_SHARED_CHMAP);
8870
8871 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8872 case 0:
8873 default:
8874 return 1;
8875 case 1:
8876 return 2;
8877 case 2:
8878 return 4;
8879 case 3:
8880 return 8;
8881 case 4:
8882 return 3;
8883 case 5:
8884 return 6;
8885 case 6:
8886 return 10;
8887 case 7:
8888 return 12;
8889 case 8:
8890 return 16;
8891 }
8892 }
8893
8894 struct dce8_wm_params {
8895 u32 dram_channels; /* number of dram channels */
8896 u32 yclk; /* bandwidth per dram data pin in kHz */
8897 u32 sclk; /* engine clock in kHz */
8898 u32 disp_clk; /* display clock in kHz */
8899 u32 src_width; /* viewport width */
8900 u32 active_time; /* active display time in ns */
8901 u32 blank_time; /* blank time in ns */
8902 bool interlaced; /* mode is interlaced */
8903 fixed20_12 vsc; /* vertical scale ratio */
8904 u32 num_heads; /* number of active crtcs */
8905 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8906 u32 lb_size; /* line buffer allocated to pipe */
8907 u32 vtaps; /* vertical scaler taps */
8908 };
8909
8910 /**
8911 * dce8_dram_bandwidth - get the dram bandwidth
8912 *
8913 * @wm: watermark calculation data
8914 *
8915 * Calculate the raw dram bandwidth (CIK).
8916 * Used for display watermark bandwidth calculations
8917 * Returns the dram bandwidth in MBytes/s
8918 */
8919 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8920 {
8921 /* Calculate raw DRAM Bandwidth */
8922 fixed20_12 dram_efficiency; /* 0.7 */
8923 fixed20_12 yclk, dram_channels, bandwidth;
8924 fixed20_12 a;
8925
8926 a.full = dfixed_const(1000);
8927 yclk.full = dfixed_const(wm->yclk);
8928 yclk.full = dfixed_div(yclk, a);
8929 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8930 a.full = dfixed_const(10);
8931 dram_efficiency.full = dfixed_const(7);
8932 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8933 bandwidth.full = dfixed_mul(dram_channels, yclk);
8934 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8935
8936 return dfixed_trunc(bandwidth);
8937 }
8938
8939 /**
8940 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8941 *
8942 * @wm: watermark calculation data
8943 *
8944 * Calculate the dram bandwidth used for display (CIK).
8945 * Used for display watermark bandwidth calculations
8946 * Returns the dram bandwidth for display in MBytes/s
8947 */
8948 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8949 {
8950 /* Calculate DRAM Bandwidth and the part allocated to display. */
8951 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8952 fixed20_12 yclk, dram_channels, bandwidth;
8953 fixed20_12 a;
8954
8955 a.full = dfixed_const(1000);
8956 yclk.full = dfixed_const(wm->yclk);
8957 yclk.full = dfixed_div(yclk, a);
8958 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8959 a.full = dfixed_const(10);
8960 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8961 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8962 bandwidth.full = dfixed_mul(dram_channels, yclk);
8963 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8964
8965 return dfixed_trunc(bandwidth);
8966 }
8967
8968 /**
8969 * dce8_data_return_bandwidth - get the data return bandwidth
8970 *
8971 * @wm: watermark calculation data
8972 *
8973 * Calculate the data return bandwidth used for display (CIK).
8974 * Used for display watermark bandwidth calculations
8975 * Returns the data return bandwidth in MBytes/s
8976 */
8977 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8978 {
8979 /* Calculate the display Data return Bandwidth */
8980 fixed20_12 return_efficiency; /* 0.8 */
8981 fixed20_12 sclk, bandwidth;
8982 fixed20_12 a;
8983
8984 a.full = dfixed_const(1000);
8985 sclk.full = dfixed_const(wm->sclk);
8986 sclk.full = dfixed_div(sclk, a);
8987 a.full = dfixed_const(10);
8988 return_efficiency.full = dfixed_const(8);
8989 return_efficiency.full = dfixed_div(return_efficiency, a);
8990 a.full = dfixed_const(32);
8991 bandwidth.full = dfixed_mul(a, sclk);
8992 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8993
8994 return dfixed_trunc(bandwidth);
8995 }
8996
8997 /**
8998 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8999 *
9000 * @wm: watermark calculation data
9001 *
9002 * Calculate the dmif bandwidth used for display (CIK).
9003 * Used for display watermark bandwidth calculations
9004 * Returns the dmif bandwidth in MBytes/s
9005 */
9006 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9007 {
9008 /* Calculate the DMIF Request Bandwidth */
9009 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9010 fixed20_12 disp_clk, bandwidth;
9011 fixed20_12 a, b;
9012
9013 a.full = dfixed_const(1000);
9014 disp_clk.full = dfixed_const(wm->disp_clk);
9015 disp_clk.full = dfixed_div(disp_clk, a);
9016 a.full = dfixed_const(32);
9017 b.full = dfixed_mul(a, disp_clk);
9018
9019 a.full = dfixed_const(10);
9020 disp_clk_request_efficiency.full = dfixed_const(8);
9021 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9022
9023 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9024
9025 return dfixed_trunc(bandwidth);
9026 }
9027
9028 /**
9029 * dce8_available_bandwidth - get the min available bandwidth
9030 *
9031 * @wm: watermark calculation data
9032 *
9033 * Calculate the min available bandwidth used for display (CIK).
9034 * Used for display watermark bandwidth calculations
9035 * Returns the min available bandwidth in MBytes/s
9036 */
9037 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9038 {
9039 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9040 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9041 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9042 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9043
9044 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9045 }
9046
9047 /**
9048 * dce8_average_bandwidth - get the average available bandwidth
9049 *
9050 * @wm: watermark calculation data
9051 *
9052 * Calculate the average available bandwidth used for display (CIK).
9053 * Used for display watermark bandwidth calculations
9054 * Returns the average available bandwidth in MBytes/s
9055 */
9056 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9057 {
9058 /* Calculate the display mode Average Bandwidth
9059 * DisplayMode should contain the source and destination dimensions,
9060 * timing, etc.
9061 */
9062 fixed20_12 bpp;
9063 fixed20_12 line_time;
9064 fixed20_12 src_width;
9065 fixed20_12 bandwidth;
9066 fixed20_12 a;
9067
9068 a.full = dfixed_const(1000);
9069 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9070 line_time.full = dfixed_div(line_time, a);
9071 bpp.full = dfixed_const(wm->bytes_per_pixel);
9072 src_width.full = dfixed_const(wm->src_width);
9073 bandwidth.full = dfixed_mul(src_width, bpp);
9074 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9075 bandwidth.full = dfixed_div(bandwidth, line_time);
9076
9077 return dfixed_trunc(bandwidth);
9078 }
9079
9080 /**
9081 * dce8_latency_watermark - get the latency watermark
9082 *
9083 * @wm: watermark calculation data
9084 *
9085 * Calculate the latency watermark (CIK).
9086 * Used for display watermark bandwidth calculations
9087 * Returns the latency watermark in ns
9088 */
9089 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9090 {
9091 /* First calculate the latency in ns */
9092 u32 mc_latency = 2000; /* 2000 ns. */
9093 u32 available_bandwidth = dce8_available_bandwidth(wm);
9094 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9095 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9096 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9097 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9098 (wm->num_heads * cursor_line_pair_return_time);
9099 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9100 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9101 u32 tmp, dmif_size = 12288;
9102 fixed20_12 a, b, c;
9103
9104 if (wm->num_heads == 0)
9105 return 0;
9106
9107 a.full = dfixed_const(2);
9108 b.full = dfixed_const(1);
9109 if ((wm->vsc.full > a.full) ||
9110 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9111 (wm->vtaps >= 5) ||
9112 ((wm->vsc.full >= a.full) && wm->interlaced))
9113 max_src_lines_per_dst_line = 4;
9114 else
9115 max_src_lines_per_dst_line = 2;
9116
9117 a.full = dfixed_const(available_bandwidth);
9118 b.full = dfixed_const(wm->num_heads);
9119 a.full = dfixed_div(a, b);
9120 tmp = div_u64((u64) dmif_size * (u64) wm->disp_clk, mc_latency + 512);
9121 tmp = min(dfixed_trunc(a), tmp);
9122
9123 lb_fill_bw = min(tmp, wm->disp_clk * wm->bytes_per_pixel / 1000);
9124
9125 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9126 b.full = dfixed_const(1000);
9127 c.full = dfixed_const(lb_fill_bw);
9128 b.full = dfixed_div(c, b);
9129 a.full = dfixed_div(a, b);
9130 line_fill_time = dfixed_trunc(a);
9131
9132 if (line_fill_time < wm->active_time)
9133 return latency;
9134 else
9135 return latency + (line_fill_time - wm->active_time);
9136
9137 }
9138
9139 /**
9140 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9141 * average and available dram bandwidth
9142 *
9143 * @wm: watermark calculation data
9144 *
9145 * Check if the display average bandwidth fits in the display
9146 * dram bandwidth (CIK).
9147 * Used for display watermark bandwidth calculations
9148 * Returns true if the display fits, false if not.
9149 */
9150 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9151 {
9152 if (dce8_average_bandwidth(wm) <=
9153 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9154 return true;
9155 else
9156 return false;
9157 }
9158
9159 /**
9160 * dce8_average_bandwidth_vs_available_bandwidth - check
9161 * average and available bandwidth
9162 *
9163 * @wm: watermark calculation data
9164 *
9165 * Check if the display average bandwidth fits in the display
9166 * available bandwidth (CIK).
9167 * Used for display watermark bandwidth calculations
9168 * Returns true if the display fits, false if not.
9169 */
9170 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9171 {
9172 if (dce8_average_bandwidth(wm) <=
9173 (dce8_available_bandwidth(wm) / wm->num_heads))
9174 return true;
9175 else
9176 return false;
9177 }
9178
9179 /**
9180 * dce8_check_latency_hiding - check latency hiding
9181 *
9182 * @wm: watermark calculation data
9183 *
9184 * Check latency hiding (CIK).
9185 * Used for display watermark bandwidth calculations
9186 * Returns true if the display fits, false if not.
9187 */
9188 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9189 {
9190 u32 lb_partitions = wm->lb_size / wm->src_width;
9191 u32 line_time = wm->active_time + wm->blank_time;
9192 u32 latency_tolerant_lines;
9193 u32 latency_hiding;
9194 fixed20_12 a;
9195
9196 a.full = dfixed_const(1);
9197 if (wm->vsc.full > a.full)
9198 latency_tolerant_lines = 1;
9199 else {
9200 if (lb_partitions <= (wm->vtaps + 1))
9201 latency_tolerant_lines = 1;
9202 else
9203 latency_tolerant_lines = 2;
9204 }
9205
9206 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9207
9208 if (dce8_latency_watermark(wm) <= latency_hiding)
9209 return true;
9210 else
9211 return false;
9212 }
9213
9214 /**
9215 * dce8_program_watermarks - program display watermarks
9216 *
9217 * @rdev: radeon_device pointer
9218 * @radeon_crtc: the selected display controller
9219 * @lb_size: line buffer size
9220 * @num_heads: number of display controllers in use
9221 *
9222 * Calculate and program the display watermarks for the
9223 * selected display controller (CIK).
9224 */
9225 static void dce8_program_watermarks(struct radeon_device *rdev,
9226 struct radeon_crtc *radeon_crtc,
9227 u32 lb_size, u32 num_heads)
9228 {
9229 struct drm_display_mode *mode = &radeon_crtc->base.mode;
9230 struct dce8_wm_params wm_low, wm_high;
9231 u32 active_time;
9232 u32 line_time = 0;
9233 u32 latency_watermark_a = 0, latency_watermark_b = 0;
9234 u32 tmp, wm_mask;
9235
9236 if (radeon_crtc->base.enabled && num_heads && mode) {
9237 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
9238 (u32)mode->clock);
9239 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
9240 (u32)mode->clock);
9241 line_time = min(line_time, (u32)65535);
9242
9243 /* watermark for high clocks */
9244 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9245 rdev->pm.dpm_enabled) {
9246 wm_high.yclk =
9247 radeon_dpm_get_mclk(rdev, false) * 10;
9248 wm_high.sclk =
9249 radeon_dpm_get_sclk(rdev, false) * 10;
9250 } else {
9251 wm_high.yclk = rdev->pm.current_mclk * 10;
9252 wm_high.sclk = rdev->pm.current_sclk * 10;
9253 }
9254
9255 wm_high.disp_clk = mode->clock;
9256 wm_high.src_width = mode->crtc_hdisplay;
9257 wm_high.active_time = active_time;
9258 wm_high.blank_time = line_time - wm_high.active_time;
9259 wm_high.interlaced = false;
9260 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9261 wm_high.interlaced = true;
9262 wm_high.vsc = radeon_crtc->vsc;
9263 wm_high.vtaps = 1;
9264 if (radeon_crtc->rmx_type != RMX_OFF)
9265 wm_high.vtaps = 2;
9266 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9267 wm_high.lb_size = lb_size;
9268 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9269 wm_high.num_heads = num_heads;
9270
9271 /* set for high clocks */
9272 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9273
9274 /* possibly force display priority to high */
9275 /* should really do this at mode validation time... */
9276 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9277 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9278 !dce8_check_latency_hiding(&wm_high) ||
9279 (rdev->disp_priority == 2)) {
9280 DRM_DEBUG_KMS("force priority to high\n");
9281 }
9282
9283 /* watermark for low clocks */
9284 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9285 rdev->pm.dpm_enabled) {
9286 wm_low.yclk =
9287 radeon_dpm_get_mclk(rdev, true) * 10;
9288 wm_low.sclk =
9289 radeon_dpm_get_sclk(rdev, true) * 10;
9290 } else {
9291 wm_low.yclk = rdev->pm.current_mclk * 10;
9292 wm_low.sclk = rdev->pm.current_sclk * 10;
9293 }
9294
9295 wm_low.disp_clk = mode->clock;
9296 wm_low.src_width = mode->crtc_hdisplay;
9297 wm_low.active_time = active_time;
9298 wm_low.blank_time = line_time - wm_low.active_time;
9299 wm_low.interlaced = false;
9300 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9301 wm_low.interlaced = true;
9302 wm_low.vsc = radeon_crtc->vsc;
9303 wm_low.vtaps = 1;
9304 if (radeon_crtc->rmx_type != RMX_OFF)
9305 wm_low.vtaps = 2;
9306 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9307 wm_low.lb_size = lb_size;
9308 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9309 wm_low.num_heads = num_heads;
9310
9311 /* set for low clocks */
9312 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
9313
9314 /* possibly force display priority to high */
9315 /* should really do this at mode validation time... */
9316 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9317 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9318 !dce8_check_latency_hiding(&wm_low) ||
9319 (rdev->disp_priority == 2)) {
9320 DRM_DEBUG_KMS("force priority to high\n");
9321 }
9322
9323 /* Save number of lines the linebuffer leads before the scanout */
9324 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
9325 }
9326
9327 /* select wm A */
9328 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9329 tmp = wm_mask;
9330 tmp &= ~LATENCY_WATERMARK_MASK(3);
9331 tmp |= LATENCY_WATERMARK_MASK(1);
9332 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9333 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9334 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9335 LATENCY_HIGH_WATERMARK(line_time)));
9336 /* select wm B */
9337 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9338 tmp &= ~LATENCY_WATERMARK_MASK(3);
9339 tmp |= LATENCY_WATERMARK_MASK(2);
9340 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9341 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9342 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9343 LATENCY_HIGH_WATERMARK(line_time)));
9344 /* restore original selection */
9345 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
9346
9347 /* save values for DPM */
9348 radeon_crtc->line_time = line_time;
9349 radeon_crtc->wm_high = latency_watermark_a;
9350 radeon_crtc->wm_low = latency_watermark_b;
9351 }
9352
9353 /**
9354 * dce8_bandwidth_update - program display watermarks
9355 *
9356 * @rdev: radeon_device pointer
9357 *
9358 * Calculate and program the display watermarks and line
9359 * buffer allocation (CIK).
9360 */
9361 void dce8_bandwidth_update(struct radeon_device *rdev)
9362 {
9363 struct drm_display_mode *mode = NULL;
9364 u32 num_heads = 0, lb_size;
9365 int i;
9366
9367 if (!rdev->mode_info.mode_config_initialized)
9368 return;
9369
9370 radeon_update_display_priority(rdev);
9371
9372 for (i = 0; i < rdev->num_crtc; i++) {
9373 if (rdev->mode_info.crtcs[i]->base.enabled)
9374 num_heads++;
9375 }
9376 for (i = 0; i < rdev->num_crtc; i++) {
9377 mode = &rdev->mode_info.crtcs[i]->base.mode;
9378 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9379 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9380 }
9381 }
9382
9383 /**
9384 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9385 *
9386 * @rdev: radeon_device pointer
9387 *
9388 * Fetches a GPU clock counter snapshot (SI).
9389 * Returns the 64 bit clock counter snapshot.
9390 */
9391 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9392 {
9393 uint64_t clock;
9394
9395 mutex_lock(&rdev->gpu_clock_mutex);
9396 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9397 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9398 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9399 mutex_unlock(&rdev->gpu_clock_mutex);
9400 return clock;
9401 }
9402
9403 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9404 u32 cntl_reg, u32 status_reg)
9405 {
9406 int r, i;
9407 struct atom_clock_dividers dividers;
9408 uint32_t tmp;
9409
9410 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9411 clock, false, &dividers);
9412 if (r)
9413 return r;
9414
9415 tmp = RREG32_SMC(cntl_reg);
9416 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9417 tmp |= dividers.post_divider;
9418 WREG32_SMC(cntl_reg, tmp);
9419
9420 for (i = 0; i < 100; i++) {
9421 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9422 break;
9423 mdelay(10);
9424 }
9425 if (i == 100)
9426 return -ETIMEDOUT;
9427
9428 return 0;
9429 }
9430
9431 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9432 {
9433 int r = 0;
9434
9435 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9436 if (r)
9437 return r;
9438
9439 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9440 return r;
9441 }
9442
9443 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9444 {
9445 int r, i;
9446 struct atom_clock_dividers dividers;
9447 u32 tmp;
9448
9449 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9450 ecclk, false, &dividers);
9451 if (r)
9452 return r;
9453
9454 for (i = 0; i < 100; i++) {
9455 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9456 break;
9457 mdelay(10);
9458 }
9459 if (i == 100)
9460 return -ETIMEDOUT;
9461
9462 tmp = RREG32_SMC(CG_ECLK_CNTL);
9463 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9464 tmp |= dividers.post_divider;
9465 WREG32_SMC(CG_ECLK_CNTL, tmp);
9466
9467 for (i = 0; i < 100; i++) {
9468 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9469 break;
9470 mdelay(10);
9471 }
9472 if (i == 100)
9473 return -ETIMEDOUT;
9474
9475 return 0;
9476 }
9477
9478 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9479 {
9480 struct pci_dev *root = rdev->pdev->bus->self;
9481 int bridge_pos, gpu_pos;
9482 u32 speed_cntl, mask, current_data_rate;
9483 int ret, i;
9484 u16 tmp16;
9485
9486 if (pci_is_root_bus(rdev->pdev->bus))
9487 return;
9488
9489 if (radeon_pcie_gen2 == 0)
9490 return;
9491
9492 if (rdev->flags & RADEON_IS_IGP)
9493 return;
9494
9495 if (!(rdev->flags & RADEON_IS_PCIE))
9496 return;
9497
9498 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9499 if (ret != 0)
9500 return;
9501
9502 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9503 return;
9504
9505 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9506 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9507 LC_CURRENT_DATA_RATE_SHIFT;
9508 if (mask & DRM_PCIE_SPEED_80) {
9509 if (current_data_rate == 2) {
9510 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9511 return;
9512 }
9513 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9514 } else if (mask & DRM_PCIE_SPEED_50) {
9515 if (current_data_rate == 1) {
9516 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9517 return;
9518 }
9519 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9520 }
9521
9522 bridge_pos = pci_pcie_cap(root);
9523 if (!bridge_pos)
9524 return;
9525
9526 gpu_pos = pci_pcie_cap(rdev->pdev);
9527 if (!gpu_pos)
9528 return;
9529
9530 if (mask & DRM_PCIE_SPEED_80) {
9531 /* re-try equalization if gen3 is not already enabled */
9532 if (current_data_rate != 2) {
9533 u16 bridge_cfg, gpu_cfg;
9534 u16 bridge_cfg2, gpu_cfg2;
9535 u32 max_lw, current_lw, tmp;
9536
9537 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9538 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9539
9540 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9541 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9542
9543 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9544 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9545
9546 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9547 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9548 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9549
9550 if (current_lw < max_lw) {
9551 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9552 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9553 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9554 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9555 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9556 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9557 }
9558 }
9559
9560 for (i = 0; i < 10; i++) {
9561 /* check status */
9562 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9563 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9564 break;
9565
9566 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9567 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9568
9569 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9570 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9571
9572 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9573 tmp |= LC_SET_QUIESCE;
9574 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9575
9576 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9577 tmp |= LC_REDO_EQ;
9578 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9579
9580 mdelay(100);
9581
9582 /* linkctl */
9583 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9584 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9585 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9586 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9587
9588 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9589 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9590 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9591 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9592
9593 /* linkctl2 */
9594 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9595 tmp16 &= ~((1 << 4) | (7 << 9));
9596 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9597 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9598
9599 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9600 tmp16 &= ~((1 << 4) | (7 << 9));
9601 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9602 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9603
9604 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9605 tmp &= ~LC_SET_QUIESCE;
9606 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9607 }
9608 }
9609 }
9610
9611 /* set the link speed */
9612 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9613 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9614 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9615
9616 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9617 tmp16 &= ~0xf;
9618 if (mask & DRM_PCIE_SPEED_80)
9619 tmp16 |= 3; /* gen3 */
9620 else if (mask & DRM_PCIE_SPEED_50)
9621 tmp16 |= 2; /* gen2 */
9622 else
9623 tmp16 |= 1; /* gen1 */
9624 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9625
9626 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9627 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9628 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9629
9630 for (i = 0; i < rdev->usec_timeout; i++) {
9631 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9632 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9633 break;
9634 udelay(1);
9635 }
9636 }
9637
9638 static void cik_program_aspm(struct radeon_device *rdev)
9639 {
9640 u32 data, orig;
9641 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9642 bool disable_clkreq = false;
9643
9644 if (radeon_aspm == 0)
9645 return;
9646
9647 /* XXX double check IGPs */
9648 if (rdev->flags & RADEON_IS_IGP)
9649 return;
9650
9651 if (!(rdev->flags & RADEON_IS_PCIE))
9652 return;
9653
9654 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9655 data &= ~LC_XMIT_N_FTS_MASK;
9656 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9657 if (orig != data)
9658 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9659
9660 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9661 data |= LC_GO_TO_RECOVERY;
9662 if (orig != data)
9663 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9664
9665 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9666 data |= P_IGNORE_EDB_ERR;
9667 if (orig != data)
9668 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9669
9670 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9671 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9672 data |= LC_PMI_TO_L1_DIS;
9673 if (!disable_l0s)
9674 data |= LC_L0S_INACTIVITY(7);
9675
9676 if (!disable_l1) {
9677 data |= LC_L1_INACTIVITY(7);
9678 data &= ~LC_PMI_TO_L1_DIS;
9679 if (orig != data)
9680 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9681
9682 if (!disable_plloff_in_l1) {
9683 bool clk_req_support;
9684
9685 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9686 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9687 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9688 if (orig != data)
9689 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9690
9691 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9692 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9693 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9694 if (orig != data)
9695 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9696
9697 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9698 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9699 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9700 if (orig != data)
9701 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9702
9703 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9704 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9705 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9706 if (orig != data)
9707 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9708
9709 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9710 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9711 data |= LC_DYN_LANES_PWR_STATE(3);
9712 if (orig != data)
9713 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9714
9715 if (!disable_clkreq &&
9716 !pci_is_root_bus(rdev->pdev->bus)) {
9717 struct pci_dev *root = rdev->pdev->bus->self;
9718 u32 lnkcap;
9719
9720 clk_req_support = false;
9721 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9722 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9723 clk_req_support = true;
9724 } else {
9725 clk_req_support = false;
9726 }
9727
9728 if (clk_req_support) {
9729 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9730 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9731 if (orig != data)
9732 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9733
9734 orig = data = RREG32_SMC(THM_CLK_CNTL);
9735 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9736 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9737 if (orig != data)
9738 WREG32_SMC(THM_CLK_CNTL, data);
9739
9740 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9741 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9742 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9743 if (orig != data)
9744 WREG32_SMC(MISC_CLK_CTRL, data);
9745
9746 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9747 data &= ~BCLK_AS_XCLK;
9748 if (orig != data)
9749 WREG32_SMC(CG_CLKPIN_CNTL, data);
9750
9751 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9752 data &= ~FORCE_BIF_REFCLK_EN;
9753 if (orig != data)
9754 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9755
9756 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9757 data &= ~MPLL_CLKOUT_SEL_MASK;
9758 data |= MPLL_CLKOUT_SEL(4);
9759 if (orig != data)
9760 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9761 }
9762 }
9763 } else {
9764 if (orig != data)
9765 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9766 }
9767
9768 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9769 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9770 if (orig != data)
9771 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9772
9773 if (!disable_l0s) {
9774 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9775 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9776 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9777 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9778 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9779 data &= ~LC_L0S_INACTIVITY_MASK;
9780 if (orig != data)
9781 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9782 }
9783 }
9784 }
9785 }
ubuntu-bionic-kernel mirror