]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - drivers/gpu/drm/radeon/cik.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: fix bad DMA from INTERRUPT_CNTL2
[mirror_ubuntu-bionic-kernel.git] / drivers / gpu / drm / radeon / cik.c
CommitLineData
8cc1a532
AD		 1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
8cc1a532
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
64a9dfc4 27#include <drm/drmP.h>
8cc1a532 28#include "radeon.h"
6f2043ce 29#include "radeon_asic.h"
bfc1f97d 30#include "radeon_audio.h"
8cc1a532
AD		 31#include "cikd.h"
32#include "atom.h"
841cf442 33#include "cik_blit_shaders.h"
8c68e393 34#include "radeon_ucode.h"
22c775ce 35#include "clearstate_ci.h"
02c81327 36
75cb00dc
MO		 37#define SH_MEM_CONFIG_GFX_DEFAULT \
38 ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED)
39
02c81327
AD		 40MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
41MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
42MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
43MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
44MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
277babc3 45MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
02c81327 46MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
21a93e13 47MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
cc8dbbb4 48MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
f2c6b0f4
AD		 49
50MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
51MODULE_FIRMWARE("radeon/bonaire_me.bin");
52MODULE_FIRMWARE("radeon/bonaire_ce.bin");
53MODULE_FIRMWARE("radeon/bonaire_mec.bin");
54MODULE_FIRMWARE("radeon/bonaire_mc.bin");
55MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
56MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
57MODULE_FIRMWARE("radeon/bonaire_smc.bin");
b2ea0dcd 58MODULE_FIRMWARE("radeon/bonaire_k_smc.bin");
f2c6b0f4 59
d4775655
AD		 60MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
61MODULE_FIRMWARE("radeon/HAWAII_me.bin");
62MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
63MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
64MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
277babc3 65MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
d4775655
AD		 66MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
67MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
68MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
f2c6b0f4
AD		 69
70MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
71MODULE_FIRMWARE("radeon/hawaii_me.bin");
72MODULE_FIRMWARE("radeon/hawaii_ce.bin");
73MODULE_FIRMWARE("radeon/hawaii_mec.bin");
74MODULE_FIRMWARE("radeon/hawaii_mc.bin");
75MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
76MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
77MODULE_FIRMWARE("radeon/hawaii_smc.bin");
b2ea0dcd 78MODULE_FIRMWARE("radeon/hawaii_k_smc.bin");
f2c6b0f4 79
02c81327
AD		 80MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
81MODULE_FIRMWARE("radeon/KAVERI_me.bin");
82MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
83MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
84MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
21a93e13 85MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
f2c6b0f4
AD		 86
87MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
88MODULE_FIRMWARE("radeon/kaveri_me.bin");
89MODULE_FIRMWARE("radeon/kaveri_ce.bin");
90MODULE_FIRMWARE("radeon/kaveri_mec.bin");
91MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
92MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
93MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
94
02c81327
AD		 95MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
96MODULE_FIRMWARE("radeon/KABINI_me.bin");
97MODULE_FIRMWARE("radeon/KABINI_ce.bin");
98MODULE_FIRMWARE("radeon/KABINI_mec.bin");
99MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
21a93e13 100MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
f2c6b0f4
AD		 101
102MODULE_FIRMWARE("radeon/kabini_pfp.bin");
103MODULE_FIRMWARE("radeon/kabini_me.bin");
104MODULE_FIRMWARE("radeon/kabini_ce.bin");
105MODULE_FIRMWARE("radeon/kabini_mec.bin");
106MODULE_FIRMWARE("radeon/kabini_rlc.bin");
107MODULE_FIRMWARE("radeon/kabini_sdma.bin");
108
f73a9e83
SL		 109MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
110MODULE_FIRMWARE("radeon/MULLINS_me.bin");
111MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
112MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
113MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
114MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
02c81327 115
f2c6b0f4
AD		 116MODULE_FIRMWARE("radeon/mullins_pfp.bin");
117MODULE_FIRMWARE("radeon/mullins_me.bin");
118MODULE_FIRMWARE("radeon/mullins_ce.bin");
119MODULE_FIRMWARE("radeon/mullins_mec.bin");
120MODULE_FIRMWARE("radeon/mullins_rlc.bin");
121MODULE_FIRMWARE("radeon/mullins_sdma.bin");
122
a59781bb
AD		 123extern int r600_ih_ring_alloc(struct radeon_device *rdev);
124extern void r600_ih_ring_fini(struct radeon_device *rdev);
6f2043ce
AD		 125extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
126extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
cc066715 127extern bool evergreen_is_display_hung(struct radeon_device *rdev);
1fd11777
AD		 128extern void sumo_rlc_fini(struct radeon_device *rdev);
129extern int sumo_rlc_init(struct radeon_device *rdev);
1c49165d 130extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
866d83de 131extern void si_rlc_reset(struct radeon_device *rdev);
22c775ce 132extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
65fcf668 133static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
2483b4ea
CK		 134extern int cik_sdma_resume(struct radeon_device *rdev);
135extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
136extern void cik_sdma_fini(struct radeon_device *rdev);
a1d6f97c 137extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
cc066715 138static void cik_rlc_stop(struct radeon_device *rdev);
8a7cd276 139static void cik_pcie_gen3_enable(struct radeon_device *rdev);
7235711a 140static void cik_program_aspm(struct radeon_device *rdev);
22c775ce
AD		 141static void cik_init_pg(struct radeon_device *rdev);
142static void cik_init_cg(struct radeon_device *rdev);
fb2c7f4d
AD		 143static void cik_fini_pg(struct radeon_device *rdev);
144static void cik_fini_cg(struct radeon_device *rdev);
4214faf6
AD		 145static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
146 bool enable);
6f2043ce 147
353eec2a
AD		 148/**
149 * cik_get_allowed_info_register - fetch the register for the info ioctl
150 *
151 * @rdev: radeon_device pointer
152 * @reg: register offset in bytes
153 * @val: register value
154 *
155 * Returns 0 for success or -EINVAL for an invalid register
156 *
157 */
158int cik_get_allowed_info_register(struct radeon_device *rdev,
159 u32 reg, u32 *val)
160{
161 switch (reg) {
162 case GRBM_STATUS:
163 case GRBM_STATUS2:
164 case GRBM_STATUS_SE0:
165 case GRBM_STATUS_SE1:
166 case GRBM_STATUS_SE2:
167 case GRBM_STATUS_SE3:
168 case SRBM_STATUS:
169 case SRBM_STATUS2:
170 case (SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET):
171 case (SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET):
172 case UVD_STATUS:
173 /* TODO VCE */
174 *val = RREG32(reg);
175 return 0;
176 default:
177 return -EINVAL;
178 }
179}
180
9e5acbc2
DV		 181/*
182 * Indirect registers accessor
183 */
184u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg)
185{
186 unsigned long flags;
187 u32 r;
188
189 spin_lock_irqsave(&rdev->didt_idx_lock, flags);
190 WREG32(CIK_DIDT_IND_INDEX, (reg));
191 r = RREG32(CIK_DIDT_IND_DATA);
192 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
193 return r;
194}
195
196void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v)
197{
198 unsigned long flags;
199
200 spin_lock_irqsave(&rdev->didt_idx_lock, flags);
201 WREG32(CIK_DIDT_IND_INDEX, (reg));
202 WREG32(CIK_DIDT_IND_DATA, (v));
203 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
204}
205
286d9cc6
AD		 206/* get temperature in millidegrees */
207int ci_get_temp(struct radeon_device *rdev)
208{
209 u32 temp;
210 int actual_temp = 0;
211
212 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
213 CTF_TEMP_SHIFT;
214
215 if (temp & 0x200)
216 actual_temp = 255;
217 else
218 actual_temp = temp & 0x1ff;
219
220 actual_temp = actual_temp * 1000;
221
222 return actual_temp;
223}
224
225/* get temperature in millidegrees */
226int kv_get_temp(struct radeon_device *rdev)
227{
228 u32 temp;
229 int actual_temp = 0;
230
231 temp = RREG32_SMC(0xC0300E0C);
232
233 if (temp)
234 actual_temp = (temp / 8) - 49;
235 else
236 actual_temp = 0;
237
238 actual_temp = actual_temp * 1000;
239
240 return actual_temp;
241}
6f2043ce 242
6e2c3c0a
AD		 243/*
244 * Indirect registers accessor
245 */
246u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
247{
0a5b7b0b 248 unsigned long flags;
6e2c3c0a
AD		 249 u32 r;
250
0a5b7b0b 251 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 252 WREG32(PCIE_INDEX, reg);
253 (void)RREG32(PCIE_INDEX);
254 r = RREG32(PCIE_DATA);
0a5b7b0b 255 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 256 return r;
257}
258
259void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
260{
0a5b7b0b
AD		 261 unsigned long flags;
262
263 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 264 WREG32(PCIE_INDEX, reg);
265 (void)RREG32(PCIE_INDEX);
266 WREG32(PCIE_DATA, v);
267 (void)RREG32(PCIE_DATA);
0a5b7b0b 268 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 269}
270
22c775ce
AD		 271static const u32 spectre_rlc_save_restore_register_list[] =
272{
273 (0x0e00 << 16) | (0xc12c >> 2),
274 0x00000000,
275 (0x0e00 << 16) | (0xc140 >> 2),
276 0x00000000,
277 (0x0e00 << 16) | (0xc150 >> 2),
278 0x00000000,
279 (0x0e00 << 16) | (0xc15c >> 2),
280 0x00000000,
281 (0x0e00 << 16) | (0xc168 >> 2),
282 0x00000000,
283 (0x0e00 << 16) | (0xc170 >> 2),
284 0x00000000,
285 (0x0e00 << 16) | (0xc178 >> 2),
286 0x00000000,
287 (0x0e00 << 16) | (0xc204 >> 2),
288 0x00000000,
289 (0x0e00 << 16) | (0xc2b4 >> 2),
290 0x00000000,
291 (0x0e00 << 16) | (0xc2b8 >> 2),
292 0x00000000,
293 (0x0e00 << 16) | (0xc2bc >> 2),
294 0x00000000,
295 (0x0e00 << 16) | (0xc2c0 >> 2),
296 0x00000000,
297 (0x0e00 << 16) | (0x8228 >> 2),
298 0x00000000,
299 (0x0e00 << 16) | (0x829c >> 2),
300 0x00000000,
301 (0x0e00 << 16) | (0x869c >> 2),
302 0x00000000,
303 (0x0600 << 16) | (0x98f4 >> 2),
304 0x00000000,
305 (0x0e00 << 16) | (0x98f8 >> 2),
306 0x00000000,
307 (0x0e00 << 16) | (0x9900 >> 2),
308 0x00000000,
309 (0x0e00 << 16) | (0xc260 >> 2),
310 0x00000000,
311 (0x0e00 << 16) | (0x90e8 >> 2),
312 0x00000000,
313 (0x0e00 << 16) | (0x3c000 >> 2),
314 0x00000000,
315 (0x0e00 << 16) | (0x3c00c >> 2),
316 0x00000000,
317 (0x0e00 << 16) | (0x8c1c >> 2),
318 0x00000000,
319 (0x0e00 << 16) | (0x9700 >> 2),
320 0x00000000,
321 (0x0e00 << 16) | (0xcd20 >> 2),
322 0x00000000,
323 (0x4e00 << 16) | (0xcd20 >> 2),
324 0x00000000,
325 (0x5e00 << 16) | (0xcd20 >> 2),
326 0x00000000,
327 (0x6e00 << 16) | (0xcd20 >> 2),
328 0x00000000,
329 (0x7e00 << 16) | (0xcd20 >> 2),
330 0x00000000,
331 (0x8e00 << 16) | (0xcd20 >> 2),
332 0x00000000,
333 (0x9e00 << 16) | (0xcd20 >> 2),
334 0x00000000,
335 (0xae00 << 16) | (0xcd20 >> 2),
336 0x00000000,
337 (0xbe00 << 16) | (0xcd20 >> 2),
338 0x00000000,
339 (0x0e00 << 16) | (0x89bc >> 2),
340 0x00000000,
341 (0x0e00 << 16) | (0x8900 >> 2),
342 0x00000000,
343 0x3,
344 (0x0e00 << 16) | (0xc130 >> 2),
345 0x00000000,
346 (0x0e00 << 16) | (0xc134 >> 2),
347 0x00000000,
348 (0x0e00 << 16) | (0xc1fc >> 2),
349 0x00000000,
350 (0x0e00 << 16) | (0xc208 >> 2),
351 0x00000000,
352 (0x0e00 << 16) | (0xc264 >> 2),
353 0x00000000,
354 (0x0e00 << 16) | (0xc268 >> 2),
355 0x00000000,
356 (0x0e00 << 16) | (0xc26c >> 2),
357 0x00000000,
358 (0x0e00 << 16) | (0xc270 >> 2),
359 0x00000000,
360 (0x0e00 << 16) | (0xc274 >> 2),
361 0x00000000,
362 (0x0e00 << 16) | (0xc278 >> 2),
363 0x00000000,
364 (0x0e00 << 16) | (0xc27c >> 2),
365 0x00000000,
366 (0x0e00 << 16) | (0xc280 >> 2),
367 0x00000000,
368 (0x0e00 << 16) | (0xc284 >> 2),
369 0x00000000,
370 (0x0e00 << 16) | (0xc288 >> 2),
371 0x00000000,
372 (0x0e00 << 16) | (0xc28c >> 2),
373 0x00000000,
374 (0x0e00 << 16) | (0xc290 >> 2),
375 0x00000000,
376 (0x0e00 << 16) | (0xc294 >> 2),
377 0x00000000,
378 (0x0e00 << 16) | (0xc298 >> 2),
379 0x00000000,
380 (0x0e00 << 16) | (0xc29c >> 2),
381 0x00000000,
382 (0x0e00 << 16) | (0xc2a0 >> 2),
383 0x00000000,
384 (0x0e00 << 16) | (0xc2a4 >> 2),
385 0x00000000,
386 (0x0e00 << 16) | (0xc2a8 >> 2),
387 0x00000000,
388 (0x0e00 << 16) | (0xc2ac >> 2),
389 0x00000000,
390 (0x0e00 << 16) | (0xc2b0 >> 2),
391 0x00000000,
392 (0x0e00 << 16) | (0x301d0 >> 2),
393 0x00000000,
394 (0x0e00 << 16) | (0x30238 >> 2),
395 0x00000000,
396 (0x0e00 << 16) | (0x30250 >> 2),
397 0x00000000,
398 (0x0e00 << 16) | (0x30254 >> 2),
399 0x00000000,
400 (0x0e00 << 16) | (0x30258 >> 2),
401 0x00000000,
402 (0x0e00 << 16) | (0x3025c >> 2),
403 0x00000000,
404 (0x4e00 << 16) | (0xc900 >> 2),
405 0x00000000,
406 (0x5e00 << 16) | (0xc900 >> 2),
407 0x00000000,
408 (0x6e00 << 16) | (0xc900 >> 2),
409 0x00000000,
410 (0x7e00 << 16) | (0xc900 >> 2),
411 0x00000000,
412 (0x8e00 << 16) | (0xc900 >> 2),
413 0x00000000,
414 (0x9e00 << 16) | (0xc900 >> 2),
415 0x00000000,
416 (0xae00 << 16) | (0xc900 >> 2),
417 0x00000000,
418 (0xbe00 << 16) | (0xc900 >> 2),
419 0x00000000,
420 (0x4e00 << 16) | (0xc904 >> 2),
421 0x00000000,
422 (0x5e00 << 16) | (0xc904 >> 2),
423 0x00000000,
424 (0x6e00 << 16) | (0xc904 >> 2),
425 0x00000000,
426 (0x7e00 << 16) | (0xc904 >> 2),
427 0x00000000,
428 (0x8e00 << 16) | (0xc904 >> 2),
429 0x00000000,
430 (0x9e00 << 16) | (0xc904 >> 2),
431 0x00000000,
432 (0xae00 << 16) | (0xc904 >> 2),
433 0x00000000,
434 (0xbe00 << 16) | (0xc904 >> 2),
435 0x00000000,
436 (0x4e00 << 16) | (0xc908 >> 2),
437 0x00000000,
438 (0x5e00 << 16) | (0xc908 >> 2),
439 0x00000000,
440 (0x6e00 << 16) | (0xc908 >> 2),
441 0x00000000,
442 (0x7e00 << 16) | (0xc908 >> 2),
443 0x00000000,
444 (0x8e00 << 16) | (0xc908 >> 2),
445 0x00000000,
446 (0x9e00 << 16) | (0xc908 >> 2),
447 0x00000000,
448 (0xae00 << 16) | (0xc908 >> 2),
449 0x00000000,
450 (0xbe00 << 16) | (0xc908 >> 2),
451 0x00000000,
452 (0x4e00 << 16) | (0xc90c >> 2),
453 0x00000000,
454 (0x5e00 << 16) | (0xc90c >> 2),
455 0x00000000,
456 (0x6e00 << 16) | (0xc90c >> 2),
457 0x00000000,
458 (0x7e00 << 16) | (0xc90c >> 2),
459 0x00000000,
460 (0x8e00 << 16) | (0xc90c >> 2),
461 0x00000000,
462 (0x9e00 << 16) | (0xc90c >> 2),
463 0x00000000,
464 (0xae00 << 16) | (0xc90c >> 2),
465 0x00000000,
466 (0xbe00 << 16) | (0xc90c >> 2),
467 0x00000000,
468 (0x4e00 << 16) | (0xc910 >> 2),
469 0x00000000,
470 (0x5e00 << 16) | (0xc910 >> 2),
471 0x00000000,
472 (0x6e00 << 16) | (0xc910 >> 2),
473 0x00000000,
474 (0x7e00 << 16) | (0xc910 >> 2),
475 0x00000000,
476 (0x8e00 << 16) | (0xc910 >> 2),
477 0x00000000,
478 (0x9e00 << 16) | (0xc910 >> 2),
479 0x00000000,
480 (0xae00 << 16) | (0xc910 >> 2),
481 0x00000000,
482 (0xbe00 << 16) | (0xc910 >> 2),
483 0x00000000,
484 (0x0e00 << 16) | (0xc99c >> 2),
485 0x00000000,
486 (0x0e00 << 16) | (0x9834 >> 2),
487 0x00000000,
488 (0x0000 << 16) | (0x30f00 >> 2),
489 0x00000000,
490 (0x0001 << 16) | (0x30f00 >> 2),
491 0x00000000,
492 (0x0000 << 16) | (0x30f04 >> 2),
493 0x00000000,
494 (0x0001 << 16) | (0x30f04 >> 2),
495 0x00000000,
496 (0x0000 << 16) | (0x30f08 >> 2),
497 0x00000000,
498 (0x0001 << 16) | (0x30f08 >> 2),
499 0x00000000,
500 (0x0000 << 16) | (0x30f0c >> 2),
501 0x00000000,
502 (0x0001 << 16) | (0x30f0c >> 2),
503 0x00000000,
504 (0x0600 << 16) | (0x9b7c >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0x8a14 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0x8a18 >> 2),
509 0x00000000,
510 (0x0600 << 16) | (0x30a00 >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0x8bf0 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0x8bcc >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0x8b24 >> 2),
517 0x00000000,
518 (0x0e00 << 16) | (0x30a04 >> 2),
519 0x00000000,
520 (0x0600 << 16) | (0x30a10 >> 2),
521 0x00000000,
522 (0x0600 << 16) | (0x30a14 >> 2),
523 0x00000000,
524 (0x0600 << 16) | (0x30a18 >> 2),
525 0x00000000,
526 (0x0600 << 16) | (0x30a2c >> 2),
527 0x00000000,
528 (0x0e00 << 16) | (0xc700 >> 2),
529 0x00000000,
530 (0x0e00 << 16) | (0xc704 >> 2),
531 0x00000000,
532 (0x0e00 << 16) | (0xc708 >> 2),
533 0x00000000,
534 (0x0e00 << 16) | (0xc768 >> 2),
535 0x00000000,
536 (0x0400 << 16) | (0xc770 >> 2),
537 0x00000000,
538 (0x0400 << 16) | (0xc774 >> 2),
539 0x00000000,
540 (0x0400 << 16) | (0xc778 >> 2),
541 0x00000000,
542 (0x0400 << 16) | (0xc77c >> 2),
543 0x00000000,
544 (0x0400 << 16) | (0xc780 >> 2),
545 0x00000000,
546 (0x0400 << 16) | (0xc784 >> 2),
547 0x00000000,
548 (0x0400 << 16) | (0xc788 >> 2),
549 0x00000000,
550 (0x0400 << 16) | (0xc78c >> 2),
551 0x00000000,
552 (0x0400 << 16) | (0xc798 >> 2),
553 0x00000000,
554 (0x0400 << 16) | (0xc79c >> 2),
555 0x00000000,
556 (0x0400 << 16) | (0xc7a0 >> 2),
557 0x00000000,
558 (0x0400 << 16) | (0xc7a4 >> 2),
559 0x00000000,
560 (0x0400 << 16) | (0xc7a8 >> 2),
561 0x00000000,
562 (0x0400 << 16) | (0xc7ac >> 2),
563 0x00000000,
564 (0x0400 << 16) | (0xc7b0 >> 2),
565 0x00000000,
566 (0x0400 << 16) | (0xc7b4 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0x9100 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0x3c010 >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x92a8 >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x92ac >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x92b4 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x92b8 >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x92bc >> 2),
581 0x00000000,
582 (0x0e00 << 16) | (0x92c0 >> 2),
583 0x00000000,
584 (0x0e00 << 16) | (0x92c4 >> 2),
585 0x00000000,
586 (0x0e00 << 16) | (0x92c8 >> 2),
587 0x00000000,
588 (0x0e00 << 16) | (0x92cc >> 2),
589 0x00000000,
590 (0x0e00 << 16) | (0x92d0 >> 2),
591 0x00000000,
592 (0x0e00 << 16) | (0x8c00 >> 2),
593 0x00000000,
594 (0x0e00 << 16) | (0x8c04 >> 2),
595 0x00000000,
596 (0x0e00 << 16) | (0x8c20 >> 2),
597 0x00000000,
598 (0x0e00 << 16) | (0x8c38 >> 2),
599 0x00000000,
600 (0x0e00 << 16) | (0x8c3c >> 2),
601 0x00000000,
602 (0x0e00 << 16) | (0xae00 >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0x9604 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0xac08 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0xac0c >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0xac10 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0xac14 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0xac58 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0xac68 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0xac6c >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0xac70 >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0xac74 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0xac78 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0xac7c >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0xac80 >> 2),
629 0x00000000,
630 (0x0e00 << 16) | (0xac84 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0xac88 >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0xac8c >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0x970c >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x9714 >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0x9718 >> 2),
641 0x00000000,
642 (0x0e00 << 16) | (0x971c >> 2),
643 0x00000000,
644 (0x0e00 << 16) | (0x31068 >> 2),
645 0x00000000,
646 (0x4e00 << 16) | (0x31068 >> 2),
647 0x00000000,
648 (0x5e00 << 16) | (0x31068 >> 2),
649 0x00000000,
650 (0x6e00 << 16) | (0x31068 >> 2),
651 0x00000000,
652 (0x7e00 << 16) | (0x31068 >> 2),
653 0x00000000,
654 (0x8e00 << 16) | (0x31068 >> 2),
655 0x00000000,
656 (0x9e00 << 16) | (0x31068 >> 2),
657 0x00000000,
658 (0xae00 << 16) | (0x31068 >> 2),
659 0x00000000,
660 (0xbe00 << 16) | (0x31068 >> 2),
661 0x00000000,
662 (0x0e00 << 16) | (0xcd10 >> 2),
663 0x00000000,
664 (0x0e00 << 16) | (0xcd14 >> 2),
665 0x00000000,
666 (0x0e00 << 16) | (0x88b0 >> 2),
667 0x00000000,
668 (0x0e00 << 16) | (0x88b4 >> 2),
669 0x00000000,
670 (0x0e00 << 16) | (0x88b8 >> 2),
671 0x00000000,
672 (0x0e00 << 16) | (0x88bc >> 2),
673 0x00000000,
674 (0x0400 << 16) | (0x89c0 >> 2),
675 0x00000000,
676 (0x0e00 << 16) | (0x88c4 >> 2),
677 0x00000000,
678 (0x0e00 << 16) | (0x88c8 >> 2),
679 0x00000000,
680 (0x0e00 << 16) | (0x88d0 >> 2),
681 0x00000000,
682 (0x0e00 << 16) | (0x88d4 >> 2),
683 0x00000000,
684 (0x0e00 << 16) | (0x88d8 >> 2),
685 0x00000000,
686 (0x0e00 << 16) | (0x8980 >> 2),
687 0x00000000,
688 (0x0e00 << 16) | (0x30938 >> 2),
689 0x00000000,
690 (0x0e00 << 16) | (0x3093c >> 2),
691 0x00000000,
692 (0x0e00 << 16) | (0x30940 >> 2),
693 0x00000000,
694 (0x0e00 << 16) | (0x89a0 >> 2),
695 0x00000000,
696 (0x0e00 << 16) | (0x30900 >> 2),
697 0x00000000,
698 (0x0e00 << 16) | (0x30904 >> 2),
699 0x00000000,
700 (0x0e00 << 16) | (0x89b4 >> 2),
701 0x00000000,
702 (0x0e00 << 16) | (0x3c210 >> 2),
703 0x00000000,
704 (0x0e00 << 16) | (0x3c214 >> 2),
705 0x00000000,
706 (0x0e00 << 16) | (0x3c218 >> 2),
707 0x00000000,
708 (0x0e00 << 16) | (0x8904 >> 2),
709 0x00000000,
710 0x5,
711 (0x0e00 << 16) | (0x8c28 >> 2),
712 (0x0e00 << 16) | (0x8c2c >> 2),
713 (0x0e00 << 16) | (0x8c30 >> 2),
714 (0x0e00 << 16) | (0x8c34 >> 2),
715 (0x0e00 << 16) | (0x9600 >> 2),
716};
717
718static const u32 kalindi_rlc_save_restore_register_list[] =
719{
720 (0x0e00 << 16) | (0xc12c >> 2),
721 0x00000000,
722 (0x0e00 << 16) | (0xc140 >> 2),
723 0x00000000,
724 (0x0e00 << 16) | (0xc150 >> 2),
725 0x00000000,
726 (0x0e00 << 16) | (0xc15c >> 2),
727 0x00000000,
728 (0x0e00 << 16) | (0xc168 >> 2),
729 0x00000000,
730 (0x0e00 << 16) | (0xc170 >> 2),
731 0x00000000,
732 (0x0e00 << 16) | (0xc204 >> 2),
733 0x00000000,
734 (0x0e00 << 16) | (0xc2b4 >> 2),
735 0x00000000,
736 (0x0e00 << 16) | (0xc2b8 >> 2),
737 0x00000000,
738 (0x0e00 << 16) | (0xc2bc >> 2),
739 0x00000000,
740 (0x0e00 << 16) | (0xc2c0 >> 2),
741 0x00000000,
742 (0x0e00 << 16) | (0x8228 >> 2),
743 0x00000000,
744 (0x0e00 << 16) | (0x829c >> 2),
745 0x00000000,
746 (0x0e00 << 16) | (0x869c >> 2),
747 0x00000000,
748 (0x0600 << 16) | (0x98f4 >> 2),
749 0x00000000,
750 (0x0e00 << 16) | (0x98f8 >> 2),
751 0x00000000,
752 (0x0e00 << 16) | (0x9900 >> 2),
753 0x00000000,
754 (0x0e00 << 16) | (0xc260 >> 2),
755 0x00000000,
756 (0x0e00 << 16) | (0x90e8 >> 2),
757 0x00000000,
758 (0x0e00 << 16) | (0x3c000 >> 2),
759 0x00000000,
760 (0x0e00 << 16) | (0x3c00c >> 2),
761 0x00000000,
762 (0x0e00 << 16) | (0x8c1c >> 2),
763 0x00000000,
764 (0x0e00 << 16) | (0x9700 >> 2),
765 0x00000000,
766 (0x0e00 << 16) | (0xcd20 >> 2),
767 0x00000000,
768 (0x4e00 << 16) | (0xcd20 >> 2),
769 0x00000000,
770 (0x5e00 << 16) | (0xcd20 >> 2),
771 0x00000000,
772 (0x6e00 << 16) | (0xcd20 >> 2),
773 0x00000000,
774 (0x7e00 << 16) | (0xcd20 >> 2),
775 0x00000000,
776 (0x0e00 << 16) | (0x89bc >> 2),
777 0x00000000,
778 (0x0e00 << 16) | (0x8900 >> 2),
779 0x00000000,
780 0x3,
781 (0x0e00 << 16) | (0xc130 >> 2),
782 0x00000000,
783 (0x0e00 << 16) | (0xc134 >> 2),
784 0x00000000,
785 (0x0e00 << 16) | (0xc1fc >> 2),
786 0x00000000,
787 (0x0e00 << 16) | (0xc208 >> 2),
788 0x00000000,
789 (0x0e00 << 16) | (0xc264 >> 2),
790 0x00000000,
791 (0x0e00 << 16) | (0xc268 >> 2),
792 0x00000000,
793 (0x0e00 << 16) | (0xc26c >> 2),
794 0x00000000,
795 (0x0e00 << 16) | (0xc270 >> 2),
796 0x00000000,
797 (0x0e00 << 16) | (0xc274 >> 2),
798 0x00000000,
799 (0x0e00 << 16) | (0xc28c >> 2),
800 0x00000000,
801 (0x0e00 << 16) | (0xc290 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0xc294 >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0xc298 >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0xc2a0 >> 2),
808 0x00000000,
809 (0x0e00 << 16) | (0xc2a4 >> 2),
810 0x00000000,
811 (0x0e00 << 16) | (0xc2a8 >> 2),
812 0x00000000,
813 (0x0e00 << 16) | (0xc2ac >> 2),
814 0x00000000,
815 (0x0e00 << 16) | (0x301d0 >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0x30238 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0x30250 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0x30254 >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0x30258 >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0x3025c >> 2),
826 0x00000000,
827 (0x4e00 << 16) | (0xc900 >> 2),
828 0x00000000,
829 (0x5e00 << 16) | (0xc900 >> 2),
830 0x00000000,
831 (0x6e00 << 16) | (0xc900 >> 2),
832 0x00000000,
833 (0x7e00 << 16) | (0xc900 >> 2),
834 0x00000000,
835 (0x4e00 << 16) | (0xc904 >> 2),
836 0x00000000,
837 (0x5e00 << 16) | (0xc904 >> 2),
838 0x00000000,
839 (0x6e00 << 16) | (0xc904 >> 2),
840 0x00000000,
841 (0x7e00 << 16) | (0xc904 >> 2),
842 0x00000000,
843 (0x4e00 << 16) | (0xc908 >> 2),
844 0x00000000,
845 (0x5e00 << 16) | (0xc908 >> 2),
846 0x00000000,
847 (0x6e00 << 16) | (0xc908 >> 2),
848 0x00000000,
849 (0x7e00 << 16) | (0xc908 >> 2),
850 0x00000000,
851 (0x4e00 << 16) | (0xc90c >> 2),
852 0x00000000,
853 (0x5e00 << 16) | (0xc90c >> 2),
854 0x00000000,
855 (0x6e00 << 16) | (0xc90c >> 2),
856 0x00000000,
857 (0x7e00 << 16) | (0xc90c >> 2),
858 0x00000000,
859 (0x4e00 << 16) | (0xc910 >> 2),
860 0x00000000,
861 (0x5e00 << 16) | (0xc910 >> 2),
862 0x00000000,
863 (0x6e00 << 16) | (0xc910 >> 2),
864 0x00000000,
865 (0x7e00 << 16) | (0xc910 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0xc99c >> 2),
868 0x00000000,
869 (0x0e00 << 16) | (0x9834 >> 2),
870 0x00000000,
871 (0x0000 << 16) | (0x30f00 >> 2),
872 0x00000000,
873 (0x0000 << 16) | (0x30f04 >> 2),
874 0x00000000,
875 (0x0000 << 16) | (0x30f08 >> 2),
876 0x00000000,
877 (0x0000 << 16) | (0x30f0c >> 2),
878 0x00000000,
879 (0x0600 << 16) | (0x9b7c >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0x8a14 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0x8a18 >> 2),
884 0x00000000,
885 (0x0600 << 16) | (0x30a00 >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0x8bf0 >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0x8bcc >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0x8b24 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0x30a04 >> 2),
894 0x00000000,
895 (0x0600 << 16) | (0x30a10 >> 2),
896 0x00000000,
897 (0x0600 << 16) | (0x30a14 >> 2),
898 0x00000000,
899 (0x0600 << 16) | (0x30a18 >> 2),
900 0x00000000,
901 (0x0600 << 16) | (0x30a2c >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0xc700 >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0xc704 >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0xc708 >> 2),
908 0x00000000,
909 (0x0e00 << 16) | (0xc768 >> 2),
910 0x00000000,
911 (0x0400 << 16) | (0xc770 >> 2),
912 0x00000000,
913 (0x0400 << 16) | (0xc774 >> 2),
914 0x00000000,
915 (0x0400 << 16) | (0xc798 >> 2),
916 0x00000000,
917 (0x0400 << 16) | (0xc79c >> 2),
918 0x00000000,
919 (0x0e00 << 16) | (0x9100 >> 2),
920 0x00000000,
921 (0x0e00 << 16) | (0x3c010 >> 2),
922 0x00000000,
923 (0x0e00 << 16) | (0x8c00 >> 2),
924 0x00000000,
925 (0x0e00 << 16) | (0x8c04 >> 2),
926 0x00000000,
927 (0x0e00 << 16) | (0x8c20 >> 2),
928 0x00000000,
929 (0x0e00 << 16) | (0x8c38 >> 2),
930 0x00000000,
931 (0x0e00 << 16) | (0x8c3c >> 2),
932 0x00000000,
933 (0x0e00 << 16) | (0xae00 >> 2),
934 0x00000000,
935 (0x0e00 << 16) | (0x9604 >> 2),
936 0x00000000,
937 (0x0e00 << 16) | (0xac08 >> 2),
938 0x00000000,
939 (0x0e00 << 16) | (0xac0c >> 2),
940 0x00000000,
941 (0x0e00 << 16) | (0xac10 >> 2),
942 0x00000000,
943 (0x0e00 << 16) | (0xac14 >> 2),
944 0x00000000,
945 (0x0e00 << 16) | (0xac58 >> 2),
946 0x00000000,
947 (0x0e00 << 16) | (0xac68 >> 2),
948 0x00000000,
949 (0x0e00 << 16) | (0xac6c >> 2),
950 0x00000000,
951 (0x0e00 << 16) | (0xac70 >> 2),
952 0x00000000,
953 (0x0e00 << 16) | (0xac74 >> 2),
954 0x00000000,
955 (0x0e00 << 16) | (0xac78 >> 2),
956 0x00000000,
957 (0x0e00 << 16) | (0xac7c >> 2),
958 0x00000000,
959 (0x0e00 << 16) | (0xac80 >> 2),
960 0x00000000,
961 (0x0e00 << 16) | (0xac84 >> 2),
962 0x00000000,
963 (0x0e00 << 16) | (0xac88 >> 2),
964 0x00000000,
965 (0x0e00 << 16) | (0xac8c >> 2),
966 0x00000000,
967 (0x0e00 << 16) | (0x970c >> 2),
968 0x00000000,
969 (0x0e00 << 16) | (0x9714 >> 2),
970 0x00000000,
971 (0x0e00 << 16) | (0x9718 >> 2),
972 0x00000000,
973 (0x0e00 << 16) | (0x971c >> 2),
974 0x00000000,
975 (0x0e00 << 16) | (0x31068 >> 2),
976 0x00000000,
977 (0x4e00 << 16) | (0x31068 >> 2),
978 0x00000000,
979 (0x5e00 << 16) | (0x31068 >> 2),
980 0x00000000,
981 (0x6e00 << 16) | (0x31068 >> 2),
982 0x00000000,
983 (0x7e00 << 16) | (0x31068 >> 2),
984 0x00000000,
985 (0x0e00 << 16) | (0xcd10 >> 2),
986 0x00000000,
987 (0x0e00 << 16) | (0xcd14 >> 2),
988 0x00000000,
989 (0x0e00 << 16) | (0x88b0 >> 2),
990 0x00000000,
991 (0x0e00 << 16) | (0x88b4 >> 2),
992 0x00000000,
993 (0x0e00 << 16) | (0x88b8 >> 2),
994 0x00000000,
995 (0x0e00 << 16) | (0x88bc >> 2),
996 0x00000000,
997 (0x0400 << 16) | (0x89c0 >> 2),
998 0x00000000,
999 (0x0e00 << 16) | (0x88c4 >> 2),
1000 0x00000000,
1001 (0x0e00 << 16) | (0x88c8 >> 2),
1002 0x00000000,
1003 (0x0e00 << 16) | (0x88d0 >> 2),
1004 0x00000000,
1005 (0x0e00 << 16) | (0x88d4 >> 2),
1006 0x00000000,
1007 (0x0e00 << 16) | (0x88d8 >> 2),
1008 0x00000000,
1009 (0x0e00 << 16) | (0x8980 >> 2),
1010 0x00000000,
1011 (0x0e00 << 16) | (0x30938 >> 2),
1012 0x00000000,
1013 (0x0e00 << 16) | (0x3093c >> 2),
1014 0x00000000,
1015 (0x0e00 << 16) | (0x30940 >> 2),
1016 0x00000000,
1017 (0x0e00 << 16) | (0x89a0 >> 2),
1018 0x00000000,
1019 (0x0e00 << 16) | (0x30900 >> 2),
1020 0x00000000,
1021 (0x0e00 << 16) | (0x30904 >> 2),
1022 0x00000000,
1023 (0x0e00 << 16) | (0x89b4 >> 2),
1024 0x00000000,
1025 (0x0e00 << 16) | (0x3e1fc >> 2),
1026 0x00000000,
1027 (0x0e00 << 16) | (0x3c210 >> 2),
1028 0x00000000,
1029 (0x0e00 << 16) | (0x3c214 >> 2),
1030 0x00000000,
1031 (0x0e00 << 16) | (0x3c218 >> 2),
1032 0x00000000,
1033 (0x0e00 << 16) | (0x8904 >> 2),
1034 0x00000000,
1035 0x5,
1036 (0x0e00 << 16) | (0x8c28 >> 2),
1037 (0x0e00 << 16) | (0x8c2c >> 2),
1038 (0x0e00 << 16) | (0x8c30 >> 2),
1039 (0x0e00 << 16) | (0x8c34 >> 2),
1040 (0x0e00 << 16) | (0x9600 >> 2),
1041};
1042
0aafd313
AD		 1043static const u32 bonaire_golden_spm_registers[] =
1044{
1045 0x30800, 0xe0ffffff, 0xe0000000
1046};
1047
1048static const u32 bonaire_golden_common_registers[] =
1049{
1050 0xc770, 0xffffffff, 0x00000800,
1051 0xc774, 0xffffffff, 0x00000800,
1052 0xc798, 0xffffffff, 0x00007fbf,
1053 0xc79c, 0xffffffff, 0x00007faf
1054};
1055
1056static const u32 bonaire_golden_registers[] =
1057{
1058 0x3354, 0x00000333, 0x00000333,
1059 0x3350, 0x000c0fc0, 0x00040200,
1060 0x9a10, 0x00010000, 0x00058208,
1061 0x3c000, 0xffff1fff, 0x00140000,
1062 0x3c200, 0xfdfc0fff, 0x00000100,
1063 0x3c234, 0x40000000, 0x40000200,
1064 0x9830, 0xffffffff, 0x00000000,
1065 0x9834, 0xf00fffff, 0x00000400,
1066 0x9838, 0x0002021c, 0x00020200,
1067 0xc78, 0x00000080, 0x00000000,
1068 0x5bb0, 0x000000f0, 0x00000070,
1069 0x5bc0, 0xf0311fff, 0x80300000,
1070 0x98f8, 0x73773777, 0x12010001,
1071 0x350c, 0x00810000, 0x408af000,
1072 0x7030, 0x31000111, 0x00000011,
1073 0x2f48, 0x73773777, 0x12010001,
1074 0x220c, 0x00007fb6, 0x0021a1b1,
1075 0x2210, 0x00007fb6, 0x002021b1,
1076 0x2180, 0x00007fb6, 0x00002191,
1077 0x2218, 0x00007fb6, 0x002121b1,
1078 0x221c, 0x00007fb6, 0x002021b1,
1079 0x21dc, 0x00007fb6, 0x00002191,
1080 0x21e0, 0x00007fb6, 0x00002191,
1081 0x3628, 0x0000003f, 0x0000000a,
1082 0x362c, 0x0000003f, 0x0000000a,
1083 0x2ae4, 0x00073ffe, 0x000022a2,
1084 0x240c, 0x000007ff, 0x00000000,
1085 0x8a14, 0xf000003f, 0x00000007,
1086 0x8bf0, 0x00002001, 0x00000001,
1087 0x8b24, 0xffffffff, 0x00ffffff,
1088 0x30a04, 0x0000ff0f, 0x00000000,
1089 0x28a4c, 0x07ffffff, 0x06000000,
1090 0x4d8, 0x00000fff, 0x00000100,
1091 0x3e78, 0x00000001, 0x00000002,
1092 0x9100, 0x03000000, 0x0362c688,
1093 0x8c00, 0x000000ff, 0x00000001,
1094 0xe40, 0x00001fff, 0x00001fff,
1095 0x9060, 0x0000007f, 0x00000020,
1096 0x9508, 0x00010000, 0x00010000,
1097 0xac14, 0x000003ff, 0x000000f3,
1098 0xac0c, 0xffffffff, 0x00001032
1099};
1100
1101static const u32 bonaire_mgcg_cgcg_init[] =
1102{
1103 0xc420, 0xffffffff, 0xfffffffc,
1104 0x30800, 0xffffffff, 0xe0000000,
1105 0x3c2a0, 0xffffffff, 0x00000100,
1106 0x3c208, 0xffffffff, 0x00000100,
1107 0x3c2c0, 0xffffffff, 0xc0000100,
1108 0x3c2c8, 0xffffffff, 0xc0000100,
1109 0x3c2c4, 0xffffffff, 0xc0000100,
1110 0x55e4, 0xffffffff, 0x00600100,
1111 0x3c280, 0xffffffff, 0x00000100,
1112 0x3c214, 0xffffffff, 0x06000100,
1113 0x3c220, 0xffffffff, 0x00000100,
1114 0x3c218, 0xffffffff, 0x06000100,
1115 0x3c204, 0xffffffff, 0x00000100,
1116 0x3c2e0, 0xffffffff, 0x00000100,
1117 0x3c224, 0xffffffff, 0x00000100,
1118 0x3c200, 0xffffffff, 0x00000100,
1119 0x3c230, 0xffffffff, 0x00000100,
1120 0x3c234, 0xffffffff, 0x00000100,
1121 0x3c250, 0xffffffff, 0x00000100,
1122 0x3c254, 0xffffffff, 0x00000100,
1123 0x3c258, 0xffffffff, 0x00000100,
1124 0x3c25c, 0xffffffff, 0x00000100,
1125 0x3c260, 0xffffffff, 0x00000100,
1126 0x3c27c, 0xffffffff, 0x00000100,
1127 0x3c278, 0xffffffff, 0x00000100,
1128 0x3c210, 0xffffffff, 0x06000100,
1129 0x3c290, 0xffffffff, 0x00000100,
1130 0x3c274, 0xffffffff, 0x00000100,
1131 0x3c2b4, 0xffffffff, 0x00000100,
1132 0x3c2b0, 0xffffffff, 0x00000100,
1133 0x3c270, 0xffffffff, 0x00000100,
1134 0x30800, 0xffffffff, 0xe0000000,
1135 0x3c020, 0xffffffff, 0x00010000,
1136 0x3c024, 0xffffffff, 0x00030002,
1137 0x3c028, 0xffffffff, 0x00040007,
1138 0x3c02c, 0xffffffff, 0x00060005,
1139 0x3c030, 0xffffffff, 0x00090008,
1140 0x3c034, 0xffffffff, 0x00010000,
1141 0x3c038, 0xffffffff, 0x00030002,
1142 0x3c03c, 0xffffffff, 0x00040007,
1143 0x3c040, 0xffffffff, 0x00060005,
1144 0x3c044, 0xffffffff, 0x00090008,
1145 0x3c048, 0xffffffff, 0x00010000,
1146 0x3c04c, 0xffffffff, 0x00030002,
1147 0x3c050, 0xffffffff, 0x00040007,
1148 0x3c054, 0xffffffff, 0x00060005,
1149 0x3c058, 0xffffffff, 0x00090008,
1150 0x3c05c, 0xffffffff, 0x00010000,
1151 0x3c060, 0xffffffff, 0x00030002,
1152 0x3c064, 0xffffffff, 0x00040007,
1153 0x3c068, 0xffffffff, 0x00060005,
1154 0x3c06c, 0xffffffff, 0x00090008,
1155 0x3c070, 0xffffffff, 0x00010000,
1156 0x3c074, 0xffffffff, 0x00030002,
1157 0x3c078, 0xffffffff, 0x00040007,
1158 0x3c07c, 0xffffffff, 0x00060005,
1159 0x3c080, 0xffffffff, 0x00090008,
1160 0x3c084, 0xffffffff, 0x00010000,
1161 0x3c088, 0xffffffff, 0x00030002,
1162 0x3c08c, 0xffffffff, 0x00040007,
1163 0x3c090, 0xffffffff, 0x00060005,
1164 0x3c094, 0xffffffff, 0x00090008,
1165 0x3c098, 0xffffffff, 0x00010000,
1166 0x3c09c, 0xffffffff, 0x00030002,
1167 0x3c0a0, 0xffffffff, 0x00040007,
1168 0x3c0a4, 0xffffffff, 0x00060005,
1169 0x3c0a8, 0xffffffff, 0x00090008,
1170 0x3c000, 0xffffffff, 0x96e00200,
1171 0x8708, 0xffffffff, 0x00900100,
1172 0xc424, 0xffffffff, 0x0020003f,
1173 0x38, 0xffffffff, 0x0140001c,
1174 0x3c, 0x000f0000, 0x000f0000,
1175 0x220, 0xffffffff, 0xC060000C,
1176 0x224, 0xc0000fff, 0x00000100,
1177 0xf90, 0xffffffff, 0x00000100,
1178 0xf98, 0x00000101, 0x00000000,
1179 0x20a8, 0xffffffff, 0x00000104,
1180 0x55e4, 0xff000fff, 0x00000100,
1181 0x30cc, 0xc0000fff, 0x00000104,
1182 0xc1e4, 0x00000001, 0x00000001,
1183 0xd00c, 0xff000ff0, 0x00000100,
1184 0xd80c, 0xff000ff0, 0x00000100
1185};
1186
1187static const u32 spectre_golden_spm_registers[] =
1188{
1189 0x30800, 0xe0ffffff, 0xe0000000
1190};
1191
1192static const u32 spectre_golden_common_registers[] =
1193{
1194 0xc770, 0xffffffff, 0x00000800,
1195 0xc774, 0xffffffff, 0x00000800,
1196 0xc798, 0xffffffff, 0x00007fbf,
1197 0xc79c, 0xffffffff, 0x00007faf
1198};
1199
1200static const u32 spectre_golden_registers[] =
1201{
1202 0x3c000, 0xffff1fff, 0x96940200,
1203 0x3c00c, 0xffff0001, 0xff000000,
1204 0x3c200, 0xfffc0fff, 0x00000100,
1205 0x6ed8, 0x00010101, 0x00010000,
1206 0x9834, 0xf00fffff, 0x00000400,
1207 0x9838, 0xfffffffc, 0x00020200,
1208 0x5bb0, 0x000000f0, 0x00000070,
1209 0x5bc0, 0xf0311fff, 0x80300000,
1210 0x98f8, 0x73773777, 0x12010001,
1211 0x9b7c, 0x00ff0000, 0x00fc0000,
1212 0x2f48, 0x73773777, 0x12010001,
1213 0x8a14, 0xf000003f, 0x00000007,
1214 0x8b24, 0xffffffff, 0x00ffffff,
1215 0x28350, 0x3f3f3fff, 0x00000082,
f1553174 1216 0x28354, 0x0000003f, 0x00000000,
0aafd313
AD		 1217 0x3e78, 0x00000001, 0x00000002,
1218 0x913c, 0xffff03df, 0x00000004,
1219 0xc768, 0x00000008, 0x00000008,
1220 0x8c00, 0x000008ff, 0x00000800,
1221 0x9508, 0x00010000, 0x00010000,
1222 0xac0c, 0xffffffff, 0x54763210,
1223 0x214f8, 0x01ff01ff, 0x00000002,
1224 0x21498, 0x007ff800, 0x00200000,
1225 0x2015c, 0xffffffff, 0x00000f40,
1226 0x30934, 0xffffffff, 0x00000001
1227};
1228
1229static const u32 spectre_mgcg_cgcg_init[] =
1230{
1231 0xc420, 0xffffffff, 0xfffffffc,
1232 0x30800, 0xffffffff, 0xe0000000,
1233 0x3c2a0, 0xffffffff, 0x00000100,
1234 0x3c208, 0xffffffff, 0x00000100,
1235 0x3c2c0, 0xffffffff, 0x00000100,
1236 0x3c2c8, 0xffffffff, 0x00000100,
1237 0x3c2c4, 0xffffffff, 0x00000100,
1238 0x55e4, 0xffffffff, 0x00600100,
1239 0x3c280, 0xffffffff, 0x00000100,
1240 0x3c214, 0xffffffff, 0x06000100,
1241 0x3c220, 0xffffffff, 0x00000100,
1242 0x3c218, 0xffffffff, 0x06000100,
1243 0x3c204, 0xffffffff, 0x00000100,
1244 0x3c2e0, 0xffffffff, 0x00000100,
1245 0x3c224, 0xffffffff, 0x00000100,
1246 0x3c200, 0xffffffff, 0x00000100,
1247 0x3c230, 0xffffffff, 0x00000100,
1248 0x3c234, 0xffffffff, 0x00000100,
1249 0x3c250, 0xffffffff, 0x00000100,
1250 0x3c254, 0xffffffff, 0x00000100,
1251 0x3c258, 0xffffffff, 0x00000100,
1252 0x3c25c, 0xffffffff, 0x00000100,
1253 0x3c260, 0xffffffff, 0x00000100,
1254 0x3c27c, 0xffffffff, 0x00000100,
1255 0x3c278, 0xffffffff, 0x00000100,
1256 0x3c210, 0xffffffff, 0x06000100,
1257 0x3c290, 0xffffffff, 0x00000100,
1258 0x3c274, 0xffffffff, 0x00000100,
1259 0x3c2b4, 0xffffffff, 0x00000100,
1260 0x3c2b0, 0xffffffff, 0x00000100,
1261 0x3c270, 0xffffffff, 0x00000100,
1262 0x30800, 0xffffffff, 0xe0000000,
1263 0x3c020, 0xffffffff, 0x00010000,
1264 0x3c024, 0xffffffff, 0x00030002,
1265 0x3c028, 0xffffffff, 0x00040007,
1266 0x3c02c, 0xffffffff, 0x00060005,
1267 0x3c030, 0xffffffff, 0x00090008,
1268 0x3c034, 0xffffffff, 0x00010000,
1269 0x3c038, 0xffffffff, 0x00030002,
1270 0x3c03c, 0xffffffff, 0x00040007,
1271 0x3c040, 0xffffffff, 0x00060005,
1272 0x3c044, 0xffffffff, 0x00090008,
1273 0x3c048, 0xffffffff, 0x00010000,
1274 0x3c04c, 0xffffffff, 0x00030002,
1275 0x3c050, 0xffffffff, 0x00040007,
1276 0x3c054, 0xffffffff, 0x00060005,
1277 0x3c058, 0xffffffff, 0x00090008,
1278 0x3c05c, 0xffffffff, 0x00010000,
1279 0x3c060, 0xffffffff, 0x00030002,
1280 0x3c064, 0xffffffff, 0x00040007,
1281 0x3c068, 0xffffffff, 0x00060005,
1282 0x3c06c, 0xffffffff, 0x00090008,
1283 0x3c070, 0xffffffff, 0x00010000,
1284 0x3c074, 0xffffffff, 0x00030002,
1285 0x3c078, 0xffffffff, 0x00040007,
1286 0x3c07c, 0xffffffff, 0x00060005,
1287 0x3c080, 0xffffffff, 0x00090008,
1288 0x3c084, 0xffffffff, 0x00010000,
1289 0x3c088, 0xffffffff, 0x00030002,
1290 0x3c08c, 0xffffffff, 0x00040007,
1291 0x3c090, 0xffffffff, 0x00060005,
1292 0x3c094, 0xffffffff, 0x00090008,
1293 0x3c098, 0xffffffff, 0x00010000,
1294 0x3c09c, 0xffffffff, 0x00030002,
1295 0x3c0a0, 0xffffffff, 0x00040007,
1296 0x3c0a4, 0xffffffff, 0x00060005,
1297 0x3c0a8, 0xffffffff, 0x00090008,
1298 0x3c0ac, 0xffffffff, 0x00010000,
1299 0x3c0b0, 0xffffffff, 0x00030002,
1300 0x3c0b4, 0xffffffff, 0x00040007,
1301 0x3c0b8, 0xffffffff, 0x00060005,
1302 0x3c0bc, 0xffffffff, 0x00090008,
1303 0x3c000, 0xffffffff, 0x96e00200,
1304 0x8708, 0xffffffff, 0x00900100,
1305 0xc424, 0xffffffff, 0x0020003f,
1306 0x38, 0xffffffff, 0x0140001c,
1307 0x3c, 0x000f0000, 0x000f0000,
1308 0x220, 0xffffffff, 0xC060000C,
1309 0x224, 0xc0000fff, 0x00000100,
1310 0xf90, 0xffffffff, 0x00000100,
1311 0xf98, 0x00000101, 0x00000000,
1312 0x20a8, 0xffffffff, 0x00000104,
1313 0x55e4, 0xff000fff, 0x00000100,
1314 0x30cc, 0xc0000fff, 0x00000104,
1315 0xc1e4, 0x00000001, 0x00000001,
1316 0xd00c, 0xff000ff0, 0x00000100,
1317 0xd80c, 0xff000ff0, 0x00000100
1318};
1319
1320static const u32 kalindi_golden_spm_registers[] =
1321{
1322 0x30800, 0xe0ffffff, 0xe0000000
1323};
1324
1325static const u32 kalindi_golden_common_registers[] =
1326{
1327 0xc770, 0xffffffff, 0x00000800,
1328 0xc774, 0xffffffff, 0x00000800,
1329 0xc798, 0xffffffff, 0x00007fbf,
1330 0xc79c, 0xffffffff, 0x00007faf
1331};
1332
1333static const u32 kalindi_golden_registers[] =
1334{
1335 0x3c000, 0xffffdfff, 0x6e944040,
1336 0x55e4, 0xff607fff, 0xfc000100,
1337 0x3c220, 0xff000fff, 0x00000100,
1338 0x3c224, 0xff000fff, 0x00000100,
1339 0x3c200, 0xfffc0fff, 0x00000100,
1340 0x6ed8, 0x00010101, 0x00010000,
1341 0x9830, 0xffffffff, 0x00000000,
1342 0x9834, 0xf00fffff, 0x00000400,
1343 0x5bb0, 0x000000f0, 0x00000070,
1344 0x5bc0, 0xf0311fff, 0x80300000,
1345 0x98f8, 0x73773777, 0x12010001,
1346 0x98fc, 0xffffffff, 0x00000010,
1347 0x9b7c, 0x00ff0000, 0x00fc0000,
1348 0x8030, 0x00001f0f, 0x0000100a,
1349 0x2f48, 0x73773777, 0x12010001,
1350 0x2408, 0x000fffff, 0x000c007f,
1351 0x8a14, 0xf000003f, 0x00000007,
1352 0x8b24, 0x3fff3fff, 0x00ffcfff,
1353 0x30a04, 0x0000ff0f, 0x00000000,
1354 0x28a4c, 0x07ffffff, 0x06000000,
1355 0x4d8, 0x00000fff, 0x00000100,
1356 0x3e78, 0x00000001, 0x00000002,
1357 0xc768, 0x00000008, 0x00000008,
1358 0x8c00, 0x000000ff, 0x00000003,
1359 0x214f8, 0x01ff01ff, 0x00000002,
1360 0x21498, 0x007ff800, 0x00200000,
1361 0x2015c, 0xffffffff, 0x00000f40,
1362 0x88c4, 0x001f3ae3, 0x00000082,
1363 0x88d4, 0x0000001f, 0x00000010,
1364 0x30934, 0xffffffff, 0x00000000
1365};
1366
1367static const u32 kalindi_mgcg_cgcg_init[] =
1368{
1369 0xc420, 0xffffffff, 0xfffffffc,
1370 0x30800, 0xffffffff, 0xe0000000,
1371 0x3c2a0, 0xffffffff, 0x00000100,
1372 0x3c208, 0xffffffff, 0x00000100,
1373 0x3c2c0, 0xffffffff, 0x00000100,
1374 0x3c2c8, 0xffffffff, 0x00000100,
1375 0x3c2c4, 0xffffffff, 0x00000100,
1376 0x55e4, 0xffffffff, 0x00600100,
1377 0x3c280, 0xffffffff, 0x00000100,
1378 0x3c214, 0xffffffff, 0x06000100,
1379 0x3c220, 0xffffffff, 0x00000100,
1380 0x3c218, 0xffffffff, 0x06000100,
1381 0x3c204, 0xffffffff, 0x00000100,
1382 0x3c2e0, 0xffffffff, 0x00000100,
1383 0x3c224, 0xffffffff, 0x00000100,
1384 0x3c200, 0xffffffff, 0x00000100,
1385 0x3c230, 0xffffffff, 0x00000100,
1386 0x3c234, 0xffffffff, 0x00000100,
1387 0x3c250, 0xffffffff, 0x00000100,
1388 0x3c254, 0xffffffff, 0x00000100,
1389 0x3c258, 0xffffffff, 0x00000100,
1390 0x3c25c, 0xffffffff, 0x00000100,
1391 0x3c260, 0xffffffff, 0x00000100,
1392 0x3c27c, 0xffffffff, 0x00000100,
1393 0x3c278, 0xffffffff, 0x00000100,
1394 0x3c210, 0xffffffff, 0x06000100,
1395 0x3c290, 0xffffffff, 0x00000100,
1396 0x3c274, 0xffffffff, 0x00000100,
1397 0x3c2b4, 0xffffffff, 0x00000100,
1398 0x3c2b0, 0xffffffff, 0x00000100,
1399 0x3c270, 0xffffffff, 0x00000100,
1400 0x30800, 0xffffffff, 0xe0000000,
1401 0x3c020, 0xffffffff, 0x00010000,
1402 0x3c024, 0xffffffff, 0x00030002,
1403 0x3c028, 0xffffffff, 0x00040007,
1404 0x3c02c, 0xffffffff, 0x00060005,
1405 0x3c030, 0xffffffff, 0x00090008,
1406 0x3c034, 0xffffffff, 0x00010000,
1407 0x3c038, 0xffffffff, 0x00030002,
1408 0x3c03c, 0xffffffff, 0x00040007,
1409 0x3c040, 0xffffffff, 0x00060005,
1410 0x3c044, 0xffffffff, 0x00090008,
1411 0x3c000, 0xffffffff, 0x96e00200,
1412 0x8708, 0xffffffff, 0x00900100,
1413 0xc424, 0xffffffff, 0x0020003f,
1414 0x38, 0xffffffff, 0x0140001c,
1415 0x3c, 0x000f0000, 0x000f0000,
1416 0x220, 0xffffffff, 0xC060000C,
1417 0x224, 0xc0000fff, 0x00000100,
1418 0x20a8, 0xffffffff, 0x00000104,
1419 0x55e4, 0xff000fff, 0x00000100,
1420 0x30cc, 0xc0000fff, 0x00000104,
1421 0xc1e4, 0x00000001, 0x00000001,
1422 0xd00c, 0xff000ff0, 0x00000100,
1423 0xd80c, 0xff000ff0, 0x00000100
1424};
1425
8efff337
AD		 1426static const u32 hawaii_golden_spm_registers[] =
1427{
1428 0x30800, 0xe0ffffff, 0xe0000000
1429};
1430
1431static const u32 hawaii_golden_common_registers[] =
1432{
1433 0x30800, 0xffffffff, 0xe0000000,
1434 0x28350, 0xffffffff, 0x3a00161a,
1435 0x28354, 0xffffffff, 0x0000002e,
1436 0x9a10, 0xffffffff, 0x00018208,
1437 0x98f8, 0xffffffff, 0x12011003
1438};
1439
1440static const u32 hawaii_golden_registers[] =
1441{
1442 0x3354, 0x00000333, 0x00000333,
1443 0x9a10, 0x00010000, 0x00058208,
1444 0x9830, 0xffffffff, 0x00000000,
1445 0x9834, 0xf00fffff, 0x00000400,
1446 0x9838, 0x0002021c, 0x00020200,
1447 0xc78, 0x00000080, 0x00000000,
1448 0x5bb0, 0x000000f0, 0x00000070,
1449 0x5bc0, 0xf0311fff, 0x80300000,
1450 0x350c, 0x00810000, 0x408af000,
1451 0x7030, 0x31000111, 0x00000011,
1452 0x2f48, 0x73773777, 0x12010001,
1453 0x2120, 0x0000007f, 0x0000001b,
1454 0x21dc, 0x00007fb6, 0x00002191,
1455 0x3628, 0x0000003f, 0x0000000a,
1456 0x362c, 0x0000003f, 0x0000000a,
1457 0x2ae4, 0x00073ffe, 0x000022a2,
1458 0x240c, 0x000007ff, 0x00000000,
1459 0x8bf0, 0x00002001, 0x00000001,
1460 0x8b24, 0xffffffff, 0x00ffffff,
1461 0x30a04, 0x0000ff0f, 0x00000000,
1462 0x28a4c, 0x07ffffff, 0x06000000,
1463 0x3e78, 0x00000001, 0x00000002,
1464 0xc768, 0x00000008, 0x00000008,
1465 0xc770, 0x00000f00, 0x00000800,
1466 0xc774, 0x00000f00, 0x00000800,
1467 0xc798, 0x00ffffff, 0x00ff7fbf,
1468 0xc79c, 0x00ffffff, 0x00ff7faf,
1469 0x8c00, 0x000000ff, 0x00000800,
1470 0xe40, 0x00001fff, 0x00001fff,
1471 0x9060, 0x0000007f, 0x00000020,
1472 0x9508, 0x00010000, 0x00010000,
1473 0xae00, 0x00100000, 0x000ff07c,
1474 0xac14, 0x000003ff, 0x0000000f,
1475 0xac10, 0xffffffff, 0x7564fdec,
1476 0xac0c, 0xffffffff, 0x3120b9a8,
1477 0xac08, 0x20000000, 0x0f9c0000
1478};
1479
1480static const u32 hawaii_mgcg_cgcg_init[] =
1481{
1482 0xc420, 0xffffffff, 0xfffffffd,
1483 0x30800, 0xffffffff, 0xe0000000,
1484 0x3c2a0, 0xffffffff, 0x00000100,
1485 0x3c208, 0xffffffff, 0x00000100,
1486 0x3c2c0, 0xffffffff, 0x00000100,
1487 0x3c2c8, 0xffffffff, 0x00000100,
1488 0x3c2c4, 0xffffffff, 0x00000100,
1489 0x55e4, 0xffffffff, 0x00200100,
1490 0x3c280, 0xffffffff, 0x00000100,
1491 0x3c214, 0xffffffff, 0x06000100,
1492 0x3c220, 0xffffffff, 0x00000100,
1493 0x3c218, 0xffffffff, 0x06000100,
1494 0x3c204, 0xffffffff, 0x00000100,
1495 0x3c2e0, 0xffffffff, 0x00000100,
1496 0x3c224, 0xffffffff, 0x00000100,
1497 0x3c200, 0xffffffff, 0x00000100,
1498 0x3c230, 0xffffffff, 0x00000100,
1499 0x3c234, 0xffffffff, 0x00000100,
1500 0x3c250, 0xffffffff, 0x00000100,
1501 0x3c254, 0xffffffff, 0x00000100,
1502 0x3c258, 0xffffffff, 0x00000100,
1503 0x3c25c, 0xffffffff, 0x00000100,
1504 0x3c260, 0xffffffff, 0x00000100,
1505 0x3c27c, 0xffffffff, 0x00000100,
1506 0x3c278, 0xffffffff, 0x00000100,
1507 0x3c210, 0xffffffff, 0x06000100,
1508 0x3c290, 0xffffffff, 0x00000100,
1509 0x3c274, 0xffffffff, 0x00000100,
1510 0x3c2b4, 0xffffffff, 0x00000100,
1511 0x3c2b0, 0xffffffff, 0x00000100,
1512 0x3c270, 0xffffffff, 0x00000100,
1513 0x30800, 0xffffffff, 0xe0000000,
1514 0x3c020, 0xffffffff, 0x00010000,
1515 0x3c024, 0xffffffff, 0x00030002,
1516 0x3c028, 0xffffffff, 0x00040007,
1517 0x3c02c, 0xffffffff, 0x00060005,
1518 0x3c030, 0xffffffff, 0x00090008,
1519 0x3c034, 0xffffffff, 0x00010000,
1520 0x3c038, 0xffffffff, 0x00030002,
1521 0x3c03c, 0xffffffff, 0x00040007,
1522 0x3c040, 0xffffffff, 0x00060005,
1523 0x3c044, 0xffffffff, 0x00090008,
1524 0x3c048, 0xffffffff, 0x00010000,
1525 0x3c04c, 0xffffffff, 0x00030002,
1526 0x3c050, 0xffffffff, 0x00040007,
1527 0x3c054, 0xffffffff, 0x00060005,
1528 0x3c058, 0xffffffff, 0x00090008,
1529 0x3c05c, 0xffffffff, 0x00010000,
1530 0x3c060, 0xffffffff, 0x00030002,
1531 0x3c064, 0xffffffff, 0x00040007,
1532 0x3c068, 0xffffffff, 0x00060005,
1533 0x3c06c, 0xffffffff, 0x00090008,
1534 0x3c070, 0xffffffff, 0x00010000,
1535 0x3c074, 0xffffffff, 0x00030002,
1536 0x3c078, 0xffffffff, 0x00040007,
1537 0x3c07c, 0xffffffff, 0x00060005,
1538 0x3c080, 0xffffffff, 0x00090008,
1539 0x3c084, 0xffffffff, 0x00010000,
1540 0x3c088, 0xffffffff, 0x00030002,
1541 0x3c08c, 0xffffffff, 0x00040007,
1542 0x3c090, 0xffffffff, 0x00060005,
1543 0x3c094, 0xffffffff, 0x00090008,
1544 0x3c098, 0xffffffff, 0x00010000,
1545 0x3c09c, 0xffffffff, 0x00030002,
1546 0x3c0a0, 0xffffffff, 0x00040007,
1547 0x3c0a4, 0xffffffff, 0x00060005,
1548 0x3c0a8, 0xffffffff, 0x00090008,
1549 0x3c0ac, 0xffffffff, 0x00010000,
1550 0x3c0b0, 0xffffffff, 0x00030002,
1551 0x3c0b4, 0xffffffff, 0x00040007,
1552 0x3c0b8, 0xffffffff, 0x00060005,
1553 0x3c0bc, 0xffffffff, 0x00090008,
1554 0x3c0c0, 0xffffffff, 0x00010000,
1555 0x3c0c4, 0xffffffff, 0x00030002,
1556 0x3c0c8, 0xffffffff, 0x00040007,
1557 0x3c0cc, 0xffffffff, 0x00060005,
1558 0x3c0d0, 0xffffffff, 0x00090008,
1559 0x3c0d4, 0xffffffff, 0x00010000,
1560 0x3c0d8, 0xffffffff, 0x00030002,
1561 0x3c0dc, 0xffffffff, 0x00040007,
1562 0x3c0e0, 0xffffffff, 0x00060005,
1563 0x3c0e4, 0xffffffff, 0x00090008,
1564 0x3c0e8, 0xffffffff, 0x00010000,
1565 0x3c0ec, 0xffffffff, 0x00030002,
1566 0x3c0f0, 0xffffffff, 0x00040007,
1567 0x3c0f4, 0xffffffff, 0x00060005,
1568 0x3c0f8, 0xffffffff, 0x00090008,
1569 0xc318, 0xffffffff, 0x00020200,
1570 0x3350, 0xffffffff, 0x00000200,
1571 0x15c0, 0xffffffff, 0x00000400,
1572 0x55e8, 0xffffffff, 0x00000000,
1573 0x2f50, 0xffffffff, 0x00000902,
1574 0x3c000, 0xffffffff, 0x96940200,
1575 0x8708, 0xffffffff, 0x00900100,
1576 0xc424, 0xffffffff, 0x0020003f,
1577 0x38, 0xffffffff, 0x0140001c,
1578 0x3c, 0x000f0000, 0x000f0000,
1579 0x220, 0xffffffff, 0xc060000c,
1580 0x224, 0xc0000fff, 0x00000100,
1581 0xf90, 0xffffffff, 0x00000100,
1582 0xf98, 0x00000101, 0x00000000,
1583 0x20a8, 0xffffffff, 0x00000104,
1584 0x55e4, 0xff000fff, 0x00000100,
1585 0x30cc, 0xc0000fff, 0x00000104,
1586 0xc1e4, 0x00000001, 0x00000001,
1587 0xd00c, 0xff000ff0, 0x00000100,
1588 0xd80c, 0xff000ff0, 0x00000100
1589};
1590
f73a9e83
SL		 1591static const u32 godavari_golden_registers[] =
1592{
1593 0x55e4, 0xff607fff, 0xfc000100,
1594 0x6ed8, 0x00010101, 0x00010000,
1595 0x9830, 0xffffffff, 0x00000000,
1596 0x98302, 0xf00fffff, 0x00000400,
1597 0x6130, 0xffffffff, 0x00010000,
1598 0x5bb0, 0x000000f0, 0x00000070,
1599 0x5bc0, 0xf0311fff, 0x80300000,
1600 0x98f8, 0x73773777, 0x12010001,
1601 0x98fc, 0xffffffff, 0x00000010,
1602 0x8030, 0x00001f0f, 0x0000100a,
1603 0x2f48, 0x73773777, 0x12010001,
1604 0x2408, 0x000fffff, 0x000c007f,
1605 0x8a14, 0xf000003f, 0x00000007,
1606 0x8b24, 0xffffffff, 0x00ff0fff,
1607 0x30a04, 0x0000ff0f, 0x00000000,
1608 0x28a4c, 0x07ffffff, 0x06000000,
1609 0x4d8, 0x00000fff, 0x00000100,
1610 0xd014, 0x00010000, 0x00810001,
1611 0xd814, 0x00010000, 0x00810001,
1612 0x3e78, 0x00000001, 0x00000002,
1613 0xc768, 0x00000008, 0x00000008,
1614 0xc770, 0x00000f00, 0x00000800,
1615 0xc774, 0x00000f00, 0x00000800,
1616 0xc798, 0x00ffffff, 0x00ff7fbf,
1617 0xc79c, 0x00ffffff, 0x00ff7faf,
1618 0x8c00, 0x000000ff, 0x00000001,
1619 0x214f8, 0x01ff01ff, 0x00000002,
1620 0x21498, 0x007ff800, 0x00200000,
1621 0x2015c, 0xffffffff, 0x00000f40,
1622 0x88c4, 0x001f3ae3, 0x00000082,
1623 0x88d4, 0x0000001f, 0x00000010,
1624 0x30934, 0xffffffff, 0x00000000
1625};
1626
1627
0aafd313
AD		 1628static void cik_init_golden_registers(struct radeon_device *rdev)
1629{
1c0a4625
OG		 1630 /* Some of the registers might be dependent on GRBM_GFX_INDEX */
1631 mutex_lock(&rdev->grbm_idx_mutex);
0aafd313
AD		 1632 switch (rdev->family) {
1633 case CHIP_BONAIRE:
1634 radeon_program_register_sequence(rdev,
1635 bonaire_mgcg_cgcg_init,
1636 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1637 radeon_program_register_sequence(rdev,
1638 bonaire_golden_registers,
1639 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1640 radeon_program_register_sequence(rdev,
1641 bonaire_golden_common_registers,
1642 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1643 radeon_program_register_sequence(rdev,
1644 bonaire_golden_spm_registers,
1645 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1646 break;
1647 case CHIP_KABINI:
1648 radeon_program_register_sequence(rdev,
1649 kalindi_mgcg_cgcg_init,
1650 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1651 radeon_program_register_sequence(rdev,
1652 kalindi_golden_registers,
1653 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1654 radeon_program_register_sequence(rdev,
1655 kalindi_golden_common_registers,
1656 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1657 radeon_program_register_sequence(rdev,
1658 kalindi_golden_spm_registers,
1659 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1660 break;
f73a9e83
SL		 1661 case CHIP_MULLINS:
1662 radeon_program_register_sequence(rdev,
1663 kalindi_mgcg_cgcg_init,
1664 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1665 radeon_program_register_sequence(rdev,
1666 godavari_golden_registers,
1667 (const u32)ARRAY_SIZE(godavari_golden_registers));
1668 radeon_program_register_sequence(rdev,
1669 kalindi_golden_common_registers,
1670 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1671 radeon_program_register_sequence(rdev,
1672 kalindi_golden_spm_registers,
1673 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1674 break;
0aafd313
AD		 1675 case CHIP_KAVERI:
1676 radeon_program_register_sequence(rdev,
1677 spectre_mgcg_cgcg_init,
1678 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1679 radeon_program_register_sequence(rdev,
1680 spectre_golden_registers,
1681 (const u32)ARRAY_SIZE(spectre_golden_registers));
1682 radeon_program_register_sequence(rdev,
1683 spectre_golden_common_registers,
1684 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1685 radeon_program_register_sequence(rdev,
1686 spectre_golden_spm_registers,
1687 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1688 break;
8efff337
AD		 1689 case CHIP_HAWAII:
1690 radeon_program_register_sequence(rdev,
1691 hawaii_mgcg_cgcg_init,
1692 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1693 radeon_program_register_sequence(rdev,
1694 hawaii_golden_registers,
1695 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1696 radeon_program_register_sequence(rdev,
1697 hawaii_golden_common_registers,
1698 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1699 radeon_program_register_sequence(rdev,
1700 hawaii_golden_spm_registers,
1701 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1702 break;
0aafd313
AD		 1703 default:
1704 break;
1705 }
1c0a4625 1706 mutex_unlock(&rdev->grbm_idx_mutex);
0aafd313
AD		 1707}
1708
2c67912c
AD		 1709/**
1710 * cik_get_xclk - get the xclk
1711 *
1712 * @rdev: radeon_device pointer
1713 *
1714 * Returns the reference clock used by the gfx engine
1715 * (CIK).
1716 */
1717u32 cik_get_xclk(struct radeon_device *rdev)
1718{
3cf8bb1a 1719 u32 reference_clock = rdev->clock.spll.reference_freq;
2c67912c
AD		 1720
1721 if (rdev->flags & RADEON_IS_IGP) {
1722 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1723 return reference_clock / 2;
1724 } else {
1725 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1726 return reference_clock / 4;
1727 }
1728 return reference_clock;
1729}
1730
75efdee1
AD		 1731/**
1732 * cik_mm_rdoorbell - read a doorbell dword
1733 *
1734 * @rdev: radeon_device pointer
d5754ab8 1735 * @index: doorbell index
75efdee1
AD		 1736 *
1737 * Returns the value in the doorbell aperture at the
d5754ab8 1738 * requested doorbell index (CIK).
75efdee1 1739 */
d5754ab8 1740u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
75efdee1 1741{
d5754ab8
AL		 1742 if (index < rdev->doorbell.num_doorbells) {
1743 return readl(rdev->doorbell.ptr + index);
75efdee1 1744 } else {
d5754ab8 1745 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1746 return 0;
1747 }
1748}
1749
1750/**
1751 * cik_mm_wdoorbell - write a doorbell dword
1752 *
1753 * @rdev: radeon_device pointer
d5754ab8 1754 * @index: doorbell index
75efdee1
AD		 1755 * @v: value to write
1756 *
1757 * Writes @v to the doorbell aperture at the
d5754ab8 1758 * requested doorbell index (CIK).
75efdee1 1759 */
d5754ab8 1760void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
75efdee1 1761{
d5754ab8
AL		 1762 if (index < rdev->doorbell.num_doorbells) {
1763 writel(v, rdev->doorbell.ptr + index);
75efdee1 1764 } else {
d5754ab8 1765 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1766 }
1767}
1768
bc8273fe
AD		 1769#define BONAIRE_IO_MC_REGS_SIZE 36
1770
1771static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1772{
1773 {0x00000070, 0x04400000},
1774 {0x00000071, 0x80c01803},
1775 {0x00000072, 0x00004004},
1776 {0x00000073, 0x00000100},
1777 {0x00000074, 0x00ff0000},
1778 {0x00000075, 0x34000000},
1779 {0x00000076, 0x08000014},
1780 {0x00000077, 0x00cc08ec},
1781 {0x00000078, 0x00000400},
1782 {0x00000079, 0x00000000},
1783 {0x0000007a, 0x04090000},
1784 {0x0000007c, 0x00000000},
1785 {0x0000007e, 0x4408a8e8},
1786 {0x0000007f, 0x00000304},
1787 {0x00000080, 0x00000000},
1788 {0x00000082, 0x00000001},
1789 {0x00000083, 0x00000002},
1790 {0x00000084, 0xf3e4f400},
1791 {0x00000085, 0x052024e3},
1792 {0x00000087, 0x00000000},
1793 {0x00000088, 0x01000000},
1794 {0x0000008a, 0x1c0a0000},
1795 {0x0000008b, 0xff010000},
1796 {0x0000008d, 0xffffefff},
1797 {0x0000008e, 0xfff3efff},
1798 {0x0000008f, 0xfff3efbf},
1799 {0x00000092, 0xf7ffffff},
1800 {0x00000093, 0xffffff7f},
1801 {0x00000095, 0x00101101},
1802 {0x00000096, 0x00000fff},
1803 {0x00000097, 0x00116fff},
1804 {0x00000098, 0x60010000},
1805 {0x00000099, 0x10010000},
1806 {0x0000009a, 0x00006000},
1807 {0x0000009b, 0x00001000},
1808 {0x0000009f, 0x00b48000}
1809};
1810
d4775655
AD		 1811#define HAWAII_IO_MC_REGS_SIZE 22
1812
1813static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1814{
1815 {0x0000007d, 0x40000000},
1816 {0x0000007e, 0x40180304},
1817 {0x0000007f, 0x0000ff00},
1818 {0x00000081, 0x00000000},
1819 {0x00000083, 0x00000800},
1820 {0x00000086, 0x00000000},
1821 {0x00000087, 0x00000100},
1822 {0x00000088, 0x00020100},
1823 {0x00000089, 0x00000000},
1824 {0x0000008b, 0x00040000},
1825 {0x0000008c, 0x00000100},
1826 {0x0000008e, 0xff010000},
1827 {0x00000090, 0xffffefff},
1828 {0x00000091, 0xfff3efff},
1829 {0x00000092, 0xfff3efbf},
1830 {0x00000093, 0xf7ffffff},
1831 {0x00000094, 0xffffff7f},
1832 {0x00000095, 0x00000fff},
1833 {0x00000096, 0x00116fff},
1834 {0x00000097, 0x60010000},
1835 {0x00000098, 0x10010000},
1836 {0x0000009f, 0x00c79000}
1837};
1838
1839
b556b12e
AD		 1840/**
1841 * cik_srbm_select - select specific register instances
1842 *
1843 * @rdev: radeon_device pointer
1844 * @me: selected ME (micro engine)
1845 * @pipe: pipe
1846 * @queue: queue
1847 * @vmid: VMID
1848 *
1849 * Switches the currently active registers instances. Some
1850 * registers are instanced per VMID, others are instanced per
1851 * me/pipe/queue combination.
1852 */
1853static void cik_srbm_select(struct radeon_device *rdev,
1854 u32 me, u32 pipe, u32 queue, u32 vmid)
1855{
1856 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1857 MEID(me & 0x3) |
1858 VMID(vmid & 0xf) |
1859 QUEUEID(queue & 0x7));
1860 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1861}
1862
bc8273fe
AD		 1863/* ucode loading */
1864/**
1865 * ci_mc_load_microcode - load MC ucode into the hw
1866 *
1867 * @rdev: radeon_device pointer
1868 *
1869 * Load the GDDR MC ucode into the hw (CIK).
1870 * Returns 0 on success, error on failure.
1871 */
6c7bccea 1872int ci_mc_load_microcode(struct radeon_device *rdev)
bc8273fe 1873{
f2c6b0f4
AD		 1874 const __be32 *fw_data = NULL;
1875 const __le32 *new_fw_data = NULL;
6e4b070e 1876 u32 running, tmp;
f2c6b0f4
AD		 1877 u32 *io_mc_regs = NULL;
1878 const __le32 *new_io_mc_regs = NULL;
bcddee29 1879 int i, regs_size, ucode_size;
bc8273fe
AD		 1880
1881 if (!rdev->mc_fw)
1882 return -EINVAL;
1883
f2c6b0f4
AD		 1884 if (rdev->new_fw) {
1885 const struct mc_firmware_header_v1_0 *hdr =
1886 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
bcddee29 1887
f2c6b0f4
AD		 1888 radeon_ucode_print_mc_hdr(&hdr->header);
1889
1890 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1891 new_io_mc_regs = (const __le32 *)
1892 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1893 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1894 new_fw_data = (const __le32 *)
1895 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1896 } else {
1897 ucode_size = rdev->mc_fw->size / 4;
1898
1899 switch (rdev->family) {
1900 case CHIP_BONAIRE:
1901 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1902 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1903 break;
1904 case CHIP_HAWAII:
1905 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1906 regs_size = HAWAII_IO_MC_REGS_SIZE;
1907 break;
1908 default:
1909 return -EINVAL;
1910 }
1911 fw_data = (const __be32 *)rdev->mc_fw->data;
bc8273fe
AD		 1912 }
1913
1914 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1915
1916 if (running == 0) {
bc8273fe
AD		 1917 /* reset the engine and set to writable */
1918 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1919 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1920
1921 /* load mc io regs */
1922 for (i = 0; i < regs_size; i++) {
f2c6b0f4
AD		 1923 if (rdev->new_fw) {
1924 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1925 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1926 } else {
1927 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1928 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1929 }
bc8273fe 1930 }
9feb3dda
AD		 1931
1932 tmp = RREG32(MC_SEQ_MISC0);
1933 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {
1934 WREG32(MC_SEQ_IO_DEBUG_INDEX, 5);
1935 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023);
1936 WREG32(MC_SEQ_IO_DEBUG_INDEX, 9);
1937 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0);
1938 }
1939
bc8273fe 1940 /* load the MC ucode */
f2c6b0f4
AD		 1941 for (i = 0; i < ucode_size; i++) {
1942 if (rdev->new_fw)
1943 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1944 else
1945 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1946 }
bc8273fe
AD		 1947
1948 /* put the engine back into the active state */
1949 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1950 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1951 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1952
1953 /* wait for training to complete */
1954 for (i = 0; i < rdev->usec_timeout; i++) {
1955 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1956 break;
1957 udelay(1);
1958 }
1959 for (i = 0; i < rdev->usec_timeout; i++) {
1960 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1961 break;
1962 udelay(1);
1963 }
bc8273fe
AD		 1964 }
1965
1966 return 0;
1967}
1968
02c81327
AD		 1969/**
1970 * cik_init_microcode - load ucode images from disk
1971 *
1972 * @rdev: radeon_device pointer
1973 *
1974 * Use the firmware interface to load the ucode images into
1975 * the driver (not loaded into hw).
1976 * Returns 0 on success, error on failure.
1977 */
1978static int cik_init_microcode(struct radeon_device *rdev)
1979{
02c81327 1980 const char *chip_name;
f2c6b0f4 1981 const char *new_chip_name;
02c81327 1982 size_t pfp_req_size, me_req_size, ce_req_size,
d4775655 1983 mec_req_size, rlc_req_size, mc_req_size = 0,
277babc3 1984 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
02c81327 1985 char fw_name[30];
f2c6b0f4 1986 int new_fw = 0;
02c81327 1987 int err;
f2c6b0f4 1988 int num_fw;
b2ea0dcd 1989 bool new_smc = false;
02c81327
AD		 1990
1991 DRM_DEBUG("\n");
1992
02c81327
AD		 1993 switch (rdev->family) {
1994 case CHIP_BONAIRE:
1995 chip_name = "BONAIRE";
b2ea0dcd
AD		 1996 if ((rdev->pdev->revision == 0x80) ||
1997 (rdev->pdev->revision == 0x81) ||
1998 (rdev->pdev->device == 0x665f))
1999 new_smc = true;
f2c6b0f4 2000 new_chip_name = "bonaire";
02c81327
AD		 2001 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2002 me_req_size = CIK_ME_UCODE_SIZE * 4;
2003 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2004 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2005 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
277babc3
AD		 2006 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
2007 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
21a93e13 2008 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
cc8dbbb4 2009 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
f2c6b0f4 2010 num_fw = 8;
02c81327 2011 break;
d4775655
AD		 2012 case CHIP_HAWAII:
2013 chip_name = "HAWAII";
b2ea0dcd
AD		 2014 if (rdev->pdev->revision == 0x80)
2015 new_smc = true;
f2c6b0f4 2016 new_chip_name = "hawaii";
d4775655
AD		 2017 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2018 me_req_size = CIK_ME_UCODE_SIZE * 4;
2019 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2020 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2021 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
2022 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
277babc3 2023 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
d4775655
AD		 2024 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2025 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
f2c6b0f4 2026 num_fw = 8;
d4775655 2027 break;
02c81327
AD		 2028 case CHIP_KAVERI:
2029 chip_name = "KAVERI";
f2c6b0f4 2030 new_chip_name = "kaveri";
02c81327
AD		 2031 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2032 me_req_size = CIK_ME_UCODE_SIZE * 4;
2033 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2034 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2035 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
21a93e13 2036 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 2037 num_fw = 7;
02c81327
AD		 2038 break;
2039 case CHIP_KABINI:
2040 chip_name = "KABINI";
f2c6b0f4 2041 new_chip_name = "kabini";
02c81327
AD		 2042 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2043 me_req_size = CIK_ME_UCODE_SIZE * 4;
2044 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2045 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2046 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
21a93e13 2047 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 2048 num_fw = 6;
02c81327 2049 break;
f73a9e83
SL		 2050 case CHIP_MULLINS:
2051 chip_name = "MULLINS";
f2c6b0f4 2052 new_chip_name = "mullins";
f73a9e83
SL		 2053 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2054 me_req_size = CIK_ME_UCODE_SIZE * 4;
2055 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2056 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2057 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
2058 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 2059 num_fw = 6;
f73a9e83 2060 break;
02c81327
AD		 2061 default: BUG();
2062 }
2063
f2c6b0f4 2064 DRM_INFO("Loading %s Microcode\n", new_chip_name);
02c81327 2065
f2c6b0f4 2066 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
0a168933 2067 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2068 if (err) {
2069 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2070 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2071 if (err)
2072 goto out;
2073 if (rdev->pfp_fw->size != pfp_req_size) {
7ca85295 2074 pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
f2c6b0f4
AD		 2075 rdev->pfp_fw->size, fw_name);
2076 err = -EINVAL;
2077 goto out;
2078 }
2079 } else {
2080 err = radeon_ucode_validate(rdev->pfp_fw);
2081 if (err) {
7ca85295 2082 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
f2c6b0f4
AD		 2083 fw_name);
2084 goto out;
2085 } else {
2086 new_fw++;
2087 }
02c81327
AD		 2088 }
2089
f2c6b0f4 2090 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
0a168933 2091 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2092 if (err) {
2093 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2094 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2095 if (err)
2096 goto out;
2097 if (rdev->me_fw->size != me_req_size) {
7ca85295 2098 pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
f2c6b0f4
AD		 2099 rdev->me_fw->size, fw_name);
2100 err = -EINVAL;
2101 }
2102 } else {
2103 err = radeon_ucode_validate(rdev->me_fw);
2104 if (err) {
7ca85295 2105 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
f2c6b0f4
AD		 2106 fw_name);
2107 goto out;
2108 } else {
2109 new_fw++;
2110 }
02c81327
AD		 2111 }
2112
f2c6b0f4 2113 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
0a168933 2114 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2115 if (err) {
2116 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2117 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2118 if (err)
2119 goto out;
2120 if (rdev->ce_fw->size != ce_req_size) {
7ca85295 2121 pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
f2c6b0f4
AD		 2122 rdev->ce_fw->size, fw_name);
2123 err = -EINVAL;
2124 }
2125 } else {
2126 err = radeon_ucode_validate(rdev->ce_fw);
2127 if (err) {
7ca85295 2128 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
f2c6b0f4
AD		 2129 fw_name);
2130 goto out;
2131 } else {
2132 new_fw++;
2133 }
02c81327
AD		 2134 }
2135
f2c6b0f4 2136 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
0a168933 2137 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2138 if (err) {
2139 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2140 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2141 if (err)
2142 goto out;
2143 if (rdev->mec_fw->size != mec_req_size) {
7ca85295 2144 pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
f2c6b0f4
AD		 2145 rdev->mec_fw->size, fw_name);
2146 err = -EINVAL;
2147 }
2148 } else {
2149 err = radeon_ucode_validate(rdev->mec_fw);
2150 if (err) {
7ca85295 2151 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
f2c6b0f4
AD		 2152 fw_name);
2153 goto out;
2154 } else {
2155 new_fw++;
2156 }
2157 }
2158
2159 if (rdev->family == CHIP_KAVERI) {
2160 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2161 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2162 if (err) {
2163 goto out;
2164 } else {
2165 err = radeon_ucode_validate(rdev->mec2_fw);
2166 if (err) {
2167 goto out;
2168 } else {
2169 new_fw++;
2170 }
2171 }
02c81327
AD		 2172 }
2173
f2c6b0f4 2174 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
0a168933 2175 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2176 if (err) {
2177 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2178 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2179 if (err)
2180 goto out;
2181 if (rdev->rlc_fw->size != rlc_req_size) {
7ca85295 2182 pr_err("cik_rlc: Bogus length %zu in firmware \"%s\"\n",
f2c6b0f4
AD		 2183 rdev->rlc_fw->size, fw_name);
2184 err = -EINVAL;
2185 }
2186 } else {
2187 err = radeon_ucode_validate(rdev->rlc_fw);
2188 if (err) {
7ca85295 2189 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
f2c6b0f4
AD		 2190 fw_name);
2191 goto out;
2192 } else {
2193 new_fw++;
2194 }
02c81327
AD		 2195 }
2196
f2c6b0f4 2197 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
0a168933 2198 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2199 if (err) {
2200 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2201 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2202 if (err)
2203 goto out;
2204 if (rdev->sdma_fw->size != sdma_req_size) {
7ca85295 2205 pr_err("cik_sdma: Bogus length %zu in firmware \"%s\"\n",
f2c6b0f4
AD		 2206 rdev->sdma_fw->size, fw_name);
2207 err = -EINVAL;
2208 }
2209 } else {
2210 err = radeon_ucode_validate(rdev->sdma_fw);
2211 if (err) {
7ca85295 2212 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
f2c6b0f4
AD		 2213 fw_name);
2214 goto out;
2215 } else {
2216 new_fw++;
2217 }
21a93e13
AD		 2218 }
2219
cc8dbbb4 2220 /* No SMC, MC ucode on APUs */
02c81327 2221 if (!(rdev->flags & RADEON_IS_IGP)) {
f2c6b0f4 2222 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
0a168933 2223 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
277babc3 2224 if (err) {
f2c6b0f4 2225 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
277babc3 2226 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2227 if (err) {
2228 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2229 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2230 if (err)
2231 goto out;
2232 }
2233 if ((rdev->mc_fw->size != mc_req_size) &&
2234 (rdev->mc_fw->size != mc2_req_size)){
7ca85295 2235 pr_err("cik_mc: Bogus length %zu in firmware \"%s\"\n",
f2c6b0f4
AD		 2236 rdev->mc_fw->size, fw_name);
2237 err = -EINVAL;
2238 }
2239 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2240 } else {
2241 err = radeon_ucode_validate(rdev->mc_fw);
2242 if (err) {
7ca85295 2243 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
f2c6b0f4 2244 fw_name);
277babc3 2245 goto out;
f2c6b0f4
AD		 2246 } else {
2247 new_fw++;
2248 }
277babc3 2249 }
cc8dbbb4 2250
b2ea0dcd
AD		 2251 if (new_smc)
2252 snprintf(fw_name, sizeof(fw_name), "radeon/%s_k_smc.bin", new_chip_name);
2253 else
2254 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
cc8dbbb4
AD		 2255 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2256 if (err) {
f2c6b0f4
AD		 2257 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2258 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2259 if (err) {
7ca85295 2260 pr_err("smc: error loading firmware \"%s\"\n",
f2c6b0f4
AD		 2261 fw_name);
2262 release_firmware(rdev->smc_fw);
2263 rdev->smc_fw = NULL;
2264 err = 0;
2265 } else if (rdev->smc_fw->size != smc_req_size) {
7ca85295 2266 pr_err("cik_smc: Bogus length %zu in firmware \"%s\"\n",
f2c6b0f4
AD		 2267 rdev->smc_fw->size, fw_name);
2268 err = -EINVAL;
2269 }
2270 } else {
2271 err = radeon_ucode_validate(rdev->smc_fw);
2272 if (err) {
7ca85295 2273 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
f2c6b0f4
AD		 2274 fw_name);
2275 goto out;
2276 } else {
2277 new_fw++;
2278 }
cc8dbbb4 2279 }
02c81327
AD		 2280 }
2281
f2c6b0f4
AD		 2282 if (new_fw == 0) {
2283 rdev->new_fw = false;
2284 } else if (new_fw < num_fw) {
7ca85295 2285 pr_err("ci_fw: mixing new and old firmware!\n");
f2c6b0f4
AD		 2286 err = -EINVAL;
2287 } else {
2288 rdev->new_fw = true;
2289 }
2290
02c81327 2291out:
02c81327
AD		 2292 if (err) {
2293 if (err != -EINVAL)
7ca85295 2294 pr_err("cik_cp: Failed to load firmware \"%s\"\n",
02c81327
AD		 2295 fw_name);
2296 release_firmware(rdev->pfp_fw);
2297 rdev->pfp_fw = NULL;
2298 release_firmware(rdev->me_fw);
2299 rdev->me_fw = NULL;
2300 release_firmware(rdev->ce_fw);
2301 rdev->ce_fw = NULL;
f2c6b0f4
AD		 2302 release_firmware(rdev->mec_fw);
2303 rdev->mec_fw = NULL;
2304 release_firmware(rdev->mec2_fw);
2305 rdev->mec2_fw = NULL;
02c81327
AD		 2306 release_firmware(rdev->rlc_fw);
2307 rdev->rlc_fw = NULL;
f2c6b0f4
AD		 2308 release_firmware(rdev->sdma_fw);
2309 rdev->sdma_fw = NULL;
02c81327
AD		 2310 release_firmware(rdev->mc_fw);
2311 rdev->mc_fw = NULL;
cc8dbbb4
AD		 2312 release_firmware(rdev->smc_fw);
2313 rdev->smc_fw = NULL;
02c81327
AD		 2314 }
2315 return err;
2316}
2317
8cc1a532
AD		 2318/*
2319 * Core functions
2320 */
2321/**
2322 * cik_tiling_mode_table_init - init the hw tiling table
2323 *
2324 * @rdev: radeon_device pointer
2325 *
2326 * Starting with SI, the tiling setup is done globally in a
2327 * set of 32 tiling modes. Rather than selecting each set of
2328 * parameters per surface as on older asics, we just select
2329 * which index in the tiling table we want to use, and the
2330 * surface uses those parameters (CIK).
2331 */
2332static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2333{
f0e201f2
JP		 2334 u32 *tile = rdev->config.cik.tile_mode_array;
2335 u32 *macrotile = rdev->config.cik.macrotile_mode_array;
2336 const u32 num_tile_mode_states =
2337 ARRAY_SIZE(rdev->config.cik.tile_mode_array);
2338 const u32 num_secondary_tile_mode_states =
2339 ARRAY_SIZE(rdev->config.cik.macrotile_mode_array);
2340 u32 reg_offset, split_equal_to_row_size;
8cc1a532
AD		 2341 u32 num_pipe_configs;
2342 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2343 rdev->config.cik.max_shader_engines;
2344
2345 switch (rdev->config.cik.mem_row_size_in_kb) {
2346 case 1:
2347 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2348 break;
2349 case 2:
2350 default:
2351 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2352 break;
2353 case 4:
2354 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2355 break;
2356 }
2357
2358 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2359 if (num_pipe_configs > 8)
21e438af 2360 num_pipe_configs = 16;
8cc1a532 2361
f0e201f2
JP		 2362 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2363 tile[reg_offset] = 0;
2364 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2365 macrotile[reg_offset] = 0;
2366
2367 switch(num_pipe_configs) {
2368 case 16:
2369 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2370 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2371 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2372 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2373 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2374 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2375 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2376 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2377 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2378 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2379 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2380 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2381 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2382 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2383 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2384 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2385 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2386 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2387 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2388 TILE_SPLIT(split_equal_to_row_size));
2389 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2390 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2391 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2392 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2393 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2394 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2395 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2396 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2397 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2398 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2399 TILE_SPLIT(split_equal_to_row_size));
2400 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2401 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2402 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2403 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2404 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2405 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2406 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2407 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2408 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2409 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2410 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2411 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2412 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2413 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2414 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2415 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2416 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2417 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2418 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2419 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2420 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2421 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2422 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2423 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2424 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2425 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2426 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2427 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2428 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2429 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2430 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2431 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2432 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2433 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2434 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2435 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2436 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2437 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2438 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2439 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2440 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2441 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2442 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2443 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2444 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2445 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2446 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2447
2448 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2449 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2450 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2451 NUM_BANKS(ADDR_SURF_16_BANK));
2452 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2453 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2454 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2455 NUM_BANKS(ADDR_SURF_16_BANK));
2456 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2457 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2458 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2459 NUM_BANKS(ADDR_SURF_16_BANK));
2460 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2461 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2462 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2463 NUM_BANKS(ADDR_SURF_16_BANK));
2464 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2465 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2466 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2467 NUM_BANKS(ADDR_SURF_8_BANK));
2468 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2469 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2470 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2471 NUM_BANKS(ADDR_SURF_4_BANK));
2472 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2473 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2474 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2475 NUM_BANKS(ADDR_SURF_2_BANK));
2476 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2477 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2478 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2479 NUM_BANKS(ADDR_SURF_16_BANK));
2480 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2481 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2482 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2483 NUM_BANKS(ADDR_SURF_16_BANK));
2484 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2485 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2486 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2487 NUM_BANKS(ADDR_SURF_16_BANK));
2488 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2489 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2490 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2491 NUM_BANKS(ADDR_SURF_8_BANK));
2492 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2493 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2494 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2495 NUM_BANKS(ADDR_SURF_4_BANK));
2496 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2497 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2498 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2499 NUM_BANKS(ADDR_SURF_2_BANK));
2500 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2501 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2502 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2503 NUM_BANKS(ADDR_SURF_2_BANK));
2504
2505 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2506 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2507 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2508 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2509 break;
2510
2511 case 8:
2512 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2513 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2514 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2515 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2516 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2517 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2518 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2519 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2520 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2521 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2522 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2523 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2524 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2525 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2526 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2527 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2528 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2529 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2530 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2531 TILE_SPLIT(split_equal_to_row_size));
2532 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2533 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2534 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2535 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2536 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2537 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2538 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2539 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2540 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2541 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2542 TILE_SPLIT(split_equal_to_row_size));
2543 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2544 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2545 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2546 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2547 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2548 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2549 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2550 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2551 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2552 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2553 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2554 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2555 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2556 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2557 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2558 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2559 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2560 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2561 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2562 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2563 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2564 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2565 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2566 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2567 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2568 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2569 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2570 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2571 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2572 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2573 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2574 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2575 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2576 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2577 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2578 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2579 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2580 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2581 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2582 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2583 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2584 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2585 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2586 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2587 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2588 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2589 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2590
2591 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2592 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2593 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2594 NUM_BANKS(ADDR_SURF_16_BANK));
2595 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2596 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2597 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2598 NUM_BANKS(ADDR_SURF_16_BANK));
2599 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2600 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2601 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2602 NUM_BANKS(ADDR_SURF_16_BANK));
2603 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2604 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2605 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2606 NUM_BANKS(ADDR_SURF_16_BANK));
2607 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2608 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2609 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2610 NUM_BANKS(ADDR_SURF_8_BANK));
2611 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2612 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2613 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2614 NUM_BANKS(ADDR_SURF_4_BANK));
2615 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2616 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2617 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2618 NUM_BANKS(ADDR_SURF_2_BANK));
2619 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2620 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2621 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2622 NUM_BANKS(ADDR_SURF_16_BANK));
2623 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2624 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2625 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2626 NUM_BANKS(ADDR_SURF_16_BANK));
2627 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2628 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2629 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2630 NUM_BANKS(ADDR_SURF_16_BANK));
2631 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2632 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2633 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2634 NUM_BANKS(ADDR_SURF_16_BANK));
2635 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2636 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2637 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2638 NUM_BANKS(ADDR_SURF_8_BANK));
2639 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2640 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2641 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2642 NUM_BANKS(ADDR_SURF_4_BANK));
2643 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2644 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2645 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2646 NUM_BANKS(ADDR_SURF_2_BANK));
2647
2648 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2649 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2650 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2651 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2652 break;
2653
2654 case 4:
8cc1a532 2655 if (num_rbs == 4) {
f0e201f2
JP		 2656 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2657 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2658 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2659 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2660 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2661 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2662 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2663 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2664 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2665 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2666 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2667 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2668 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2669 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2670 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2671 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2672 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2673 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2674 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2675 TILE_SPLIT(split_equal_to_row_size));
2676 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2677 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2678 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2679 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2680 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2681 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2682 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2683 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2684 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2685 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2686 TILE_SPLIT(split_equal_to_row_size));
2687 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2688 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2689 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2690 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2691 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2692 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2693 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2694 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2695 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2696 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2697 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2698 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2699 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2700 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2701 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2702 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2703 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2704 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2705 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2706 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2707 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2708 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2709 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2710 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2711 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2712 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2713 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2714 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2715 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2716 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2717 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2718 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2719 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2720 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2721 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2722 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2723 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2724 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2725 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2726 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2727 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2728 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2729 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2730 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2731 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2732 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2733 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2734
8cc1a532 2735 } else if (num_rbs < 4) {
f0e201f2
JP		 2736 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2737 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2738 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2739 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2740 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2741 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2742 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2743 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2744 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2745 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2746 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2747 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2748 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2749 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2750 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2751 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2752 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2753 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2754 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2755 TILE_SPLIT(split_equal_to_row_size));
2756 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2757 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2758 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2759 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2760 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2761 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2762 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2763 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2764 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2765 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2766 TILE_SPLIT(split_equal_to_row_size));
2767 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2768 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2769 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2770 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2771 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2772 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2773 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2774 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2775 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2776 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2777 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2778 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2779 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2780 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2781 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2782 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2783 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2784 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2785 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2786 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2787 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2788 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2789 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2790 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2791 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2792 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2793 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2794 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2795 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2796 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2797 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2798 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2799 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2800 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2801 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2802 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2803 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2804 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2805 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2806 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2807 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2808 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2809 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2810 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2811 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2812 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2813 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
8cc1a532 2814 }
f0e201f2
JP		 2815
2816 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2817 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2818 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2819 NUM_BANKS(ADDR_SURF_16_BANK));
2820 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2821 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2822 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2823 NUM_BANKS(ADDR_SURF_16_BANK));
2824 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2825 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2826 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2827 NUM_BANKS(ADDR_SURF_16_BANK));
2828 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2829 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2830 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2831 NUM_BANKS(ADDR_SURF_16_BANK));
2832 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2833 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2834 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2835 NUM_BANKS(ADDR_SURF_16_BANK));
2836 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2837 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2838 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2839 NUM_BANKS(ADDR_SURF_8_BANK));
2840 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2841 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2842 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2843 NUM_BANKS(ADDR_SURF_4_BANK));
2844 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2845 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2846 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2847 NUM_BANKS(ADDR_SURF_16_BANK));
2848 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2849 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2850 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2851 NUM_BANKS(ADDR_SURF_16_BANK));
2852 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2853 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2854 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2855 NUM_BANKS(ADDR_SURF_16_BANK));
2856 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2857 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2858 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2859 NUM_BANKS(ADDR_SURF_16_BANK));
2860 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2861 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2862 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2863 NUM_BANKS(ADDR_SURF_16_BANK));
2864 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2865 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2866 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2867 NUM_BANKS(ADDR_SURF_8_BANK));
2868 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2869 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2870 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2871 NUM_BANKS(ADDR_SURF_4_BANK));
2872
2873 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2874 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2875 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2876 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2877 break;
2878
2879 case 2:
2880 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2881 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2882 PIPE_CONFIG(ADDR_SURF_P2) |
2883 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2884 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2885 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2886 PIPE_CONFIG(ADDR_SURF_P2) |
2887 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2888 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2889 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2890 PIPE_CONFIG(ADDR_SURF_P2) |
2891 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2892 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2893 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2894 PIPE_CONFIG(ADDR_SURF_P2) |
2895 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2896 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2897 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2898 PIPE_CONFIG(ADDR_SURF_P2) |
2899 TILE_SPLIT(split_equal_to_row_size));
2900 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2901 PIPE_CONFIG(ADDR_SURF_P2) |
2902 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2903 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2904 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2905 PIPE_CONFIG(ADDR_SURF_P2) |
2906 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2907 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2908 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2909 PIPE_CONFIG(ADDR_SURF_P2) |
2910 TILE_SPLIT(split_equal_to_row_size));
2911 tile[8] = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2912 PIPE_CONFIG(ADDR_SURF_P2);
2913 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2914 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2915 PIPE_CONFIG(ADDR_SURF_P2));
2916 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2917 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2918 PIPE_CONFIG(ADDR_SURF_P2) |
2919 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2920 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2921 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2922 PIPE_CONFIG(ADDR_SURF_P2) |
2923 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2924 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2925 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2926 PIPE_CONFIG(ADDR_SURF_P2) |
2927 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2928 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2929 PIPE_CONFIG(ADDR_SURF_P2) |
2930 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2931 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2932 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2933 PIPE_CONFIG(ADDR_SURF_P2) |
2934 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2935 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2936 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2937 PIPE_CONFIG(ADDR_SURF_P2) |
2938 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2939 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2940 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2941 PIPE_CONFIG(ADDR_SURF_P2) |
2942 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2943 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2944 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2945 PIPE_CONFIG(ADDR_SURF_P2));
2946 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2947 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2948 PIPE_CONFIG(ADDR_SURF_P2) |
2949 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2950 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2951 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2952 PIPE_CONFIG(ADDR_SURF_P2) |
2953 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2954 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2955 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2956 PIPE_CONFIG(ADDR_SURF_P2) |
2957 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2958
2959 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2960 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2961 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2962 NUM_BANKS(ADDR_SURF_16_BANK));
2963 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2964 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2965 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2966 NUM_BANKS(ADDR_SURF_16_BANK));
2967 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2968 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2969 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2970 NUM_BANKS(ADDR_SURF_16_BANK));
2971 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2972 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2973 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2974 NUM_BANKS(ADDR_SURF_16_BANK));
2975 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2976 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2977 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2978 NUM_BANKS(ADDR_SURF_16_BANK));
2979 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2980 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2981 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2982 NUM_BANKS(ADDR_SURF_16_BANK));
2983 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2984 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2985 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2986 NUM_BANKS(ADDR_SURF_8_BANK));
2987 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2988 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2989 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2990 NUM_BANKS(ADDR_SURF_16_BANK));
2991 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2992 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2993 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2994 NUM_BANKS(ADDR_SURF_16_BANK));
2995 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2996 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2997 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2998 NUM_BANKS(ADDR_SURF_16_BANK));
2999 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3000 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3001 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3002 NUM_BANKS(ADDR_SURF_16_BANK));
3003 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3004 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3005 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3006 NUM_BANKS(ADDR_SURF_16_BANK));
3007 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3008 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3009 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3010 NUM_BANKS(ADDR_SURF_16_BANK));
3011 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3012 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3013 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3014 NUM_BANKS(ADDR_SURF_8_BANK));
3015
3016 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3017 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
3018 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3019 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
3020 break;
3021
3022 default:
8cc1a532 3023 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
f0e201f2 3024 }
8cc1a532
AD		 3025}
3026
3027/**
3028 * cik_select_se_sh - select which SE, SH to address
3029 *
3030 * @rdev: radeon_device pointer
3031 * @se_num: shader engine to address
3032 * @sh_num: sh block to address
3033 *
3034 * Select which SE, SH combinations to address. Certain
3035 * registers are instanced per SE or SH. 0xffffffff means
3036 * broadcast to all SEs or SHs (CIK).
3037 */
3038static void cik_select_se_sh(struct radeon_device *rdev,
3039 u32 se_num, u32 sh_num)
3040{
3041 u32 data = INSTANCE_BROADCAST_WRITES;
3042
3043 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
b0fe3d39 3044 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
8cc1a532
AD		 3045 else if (se_num == 0xffffffff)
3046 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3047 else if (sh_num == 0xffffffff)
3048 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3049 else
3050 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3051 WREG32(GRBM_GFX_INDEX, data);
3052}
3053
3054/**
3055 * cik_create_bitmask - create a bitmask
3056 *
3057 * @bit_width: length of the mask
3058 *
3059 * create a variable length bit mask (CIK).
3060 * Returns the bitmask.
3061 */
3062static u32 cik_create_bitmask(u32 bit_width)
3063{
3064 u32 i, mask = 0;
3065
3066 for (i = 0; i < bit_width; i++) {
3067 mask <<= 1;
3068 mask |= 1;
3069 }
3070 return mask;
3071}
3072
3073/**
972c5ddb 3074 * cik_get_rb_disabled - computes the mask of disabled RBs
8cc1a532
AD		 3075 *
3076 * @rdev: radeon_device pointer
3077 * @max_rb_num: max RBs (render backends) for the asic
3078 * @se_num: number of SEs (shader engines) for the asic
3079 * @sh_per_se: number of SH blocks per SE for the asic
3080 *
3081 * Calculates the bitmask of disabled RBs (CIK).
3082 * Returns the disabled RB bitmask.
3083 */
3084static u32 cik_get_rb_disabled(struct radeon_device *rdev,
9fadb352 3085 u32 max_rb_num_per_se,
8cc1a532
AD		 3086 u32 sh_per_se)
3087{
3088 u32 data, mask;
3089
3090 data = RREG32(CC_RB_BACKEND_DISABLE);
3091 if (data & 1)
3092 data &= BACKEND_DISABLE_MASK;
3093 else
3094 data = 0;
3095 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3096
3097 data >>= BACKEND_DISABLE_SHIFT;
3098
9fadb352 3099 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
8cc1a532
AD		 3100
3101 return data & mask;
3102}
3103
3104/**
3105 * cik_setup_rb - setup the RBs on the asic
3106 *
3107 * @rdev: radeon_device pointer
3108 * @se_num: number of SEs (shader engines) for the asic
3109 * @sh_per_se: number of SH blocks per SE for the asic
3110 * @max_rb_num: max RBs (render backends) for the asic
3111 *
3112 * Configures per-SE/SH RB registers (CIK).
3113 */
3114static void cik_setup_rb(struct radeon_device *rdev,
3115 u32 se_num, u32 sh_per_se,
9fadb352 3116 u32 max_rb_num_per_se)
8cc1a532
AD		 3117{
3118 int i, j;
3119 u32 data, mask;
3120 u32 disabled_rbs = 0;
3121 u32 enabled_rbs = 0;
3122
1c0a4625 3123 mutex_lock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3124 for (i = 0; i < se_num; i++) {
3125 for (j = 0; j < sh_per_se; j++) {
3126 cik_select_se_sh(rdev, i, j);
9fadb352 3127 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
fc821b70
AD		 3128 if (rdev->family == CHIP_HAWAII)
3129 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3130 else
3131 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
8cc1a532
AD		 3132 }
3133 }
3134 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 3135 mutex_unlock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3136
3137 mask = 1;
9fadb352 3138 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
8cc1a532
AD		 3139 if (!(disabled_rbs & mask))
3140 enabled_rbs |= mask;
3141 mask <<= 1;
3142 }
3143
439a1cff
MO		 3144 rdev->config.cik.backend_enable_mask = enabled_rbs;
3145
1c0a4625 3146 mutex_lock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3147 for (i = 0; i < se_num; i++) {
3148 cik_select_se_sh(rdev, i, 0xffffffff);
3149 data = 0;
3150 for (j = 0; j < sh_per_se; j++) {
3151 switch (enabled_rbs & 3) {
fc821b70
AD		 3152 case 0:
3153 if (j == 0)
3154 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3155 else
3156 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3157 break;
8cc1a532
AD		 3158 case 1:
3159 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3160 break;
3161 case 2:
3162 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3163 break;
3164 case 3:
3165 default:
3166 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3167 break;
3168 }
3169 enabled_rbs >>= 2;
3170 }
3171 WREG32(PA_SC_RASTER_CONFIG, data);
3172 }
3173 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 3174 mutex_unlock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3175}
3176
3177/**
3178 * cik_gpu_init - setup the 3D engine
3179 *
3180 * @rdev: radeon_device pointer
3181 *
3182 * Configures the 3D engine and tiling configuration
3183 * registers so that the 3D engine is usable.
3184 */
3185static void cik_gpu_init(struct radeon_device *rdev)
3186{
3187 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3188 u32 mc_shared_chmap, mc_arb_ramcfg;
3189 u32 hdp_host_path_cntl;
3190 u32 tmp;
6101b3ae 3191 int i, j;
8cc1a532
AD		 3192
3193 switch (rdev->family) {
3194 case CHIP_BONAIRE:
3195 rdev->config.cik.max_shader_engines = 2;
3196 rdev->config.cik.max_tile_pipes = 4;
3197 rdev->config.cik.max_cu_per_sh = 7;
3198 rdev->config.cik.max_sh_per_se = 1;
3199 rdev->config.cik.max_backends_per_se = 2;
3200 rdev->config.cik.max_texture_channel_caches = 4;
3201 rdev->config.cik.max_gprs = 256;
3202 rdev->config.cik.max_gs_threads = 32;
3203 rdev->config.cik.max_hw_contexts = 8;
3204
3205 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3206 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3207 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3208 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3209 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3210 break;
b496038b
AD		 3211 case CHIP_HAWAII:
3212 rdev->config.cik.max_shader_engines = 4;
3213 rdev->config.cik.max_tile_pipes = 16;
3214 rdev->config.cik.max_cu_per_sh = 11;
3215 rdev->config.cik.max_sh_per_se = 1;
3216 rdev->config.cik.max_backends_per_se = 4;
3217 rdev->config.cik.max_texture_channel_caches = 16;
3218 rdev->config.cik.max_gprs = 256;
3219 rdev->config.cik.max_gs_threads = 32;
3220 rdev->config.cik.max_hw_contexts = 8;
3221
3222 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3223 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3224 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3225 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3226 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3227 break;
8cc1a532 3228 case CHIP_KAVERI:
b2e4c70a
AD		 3229 rdev->config.cik.max_shader_engines = 1;
3230 rdev->config.cik.max_tile_pipes = 4;
7958247e
AD		 3231 rdev->config.cik.max_cu_per_sh = 8;
3232 rdev->config.cik.max_backends_per_se = 2;
b2e4c70a
AD		 3233 rdev->config.cik.max_sh_per_se = 1;
3234 rdev->config.cik.max_texture_channel_caches = 4;
3235 rdev->config.cik.max_gprs = 256;
3236 rdev->config.cik.max_gs_threads = 16;
3237 rdev->config.cik.max_hw_contexts = 8;
3238
3239 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3240 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3241 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3242 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3243 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
8cc1a532
AD		 3244 break;
3245 case CHIP_KABINI:
f73a9e83 3246 case CHIP_MULLINS:
8cc1a532
AD		 3247 default:
3248 rdev->config.cik.max_shader_engines = 1;
3249 rdev->config.cik.max_tile_pipes = 2;
3250 rdev->config.cik.max_cu_per_sh = 2;
3251 rdev->config.cik.max_sh_per_se = 1;
3252 rdev->config.cik.max_backends_per_se = 1;
3253 rdev->config.cik.max_texture_channel_caches = 2;
3254 rdev->config.cik.max_gprs = 256;
3255 rdev->config.cik.max_gs_threads = 16;
3256 rdev->config.cik.max_hw_contexts = 8;
3257
3258 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3259 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3260 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3261 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3262 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3263 break;
3264 }
3265
3266 /* Initialize HDP */
3267 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3268 WREG32((0x2c14 + j), 0x00000000);
3269 WREG32((0x2c18 + j), 0x00000000);
3270 WREG32((0x2c1c + j), 0x00000000);
3271 WREG32((0x2c20 + j), 0x00000000);
3272 WREG32((0x2c24 + j), 0x00000000);
3273 }
3274
3275 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
dc12a3ec
LL		 3276 WREG32(SRBM_INT_CNTL, 0x1);
3277 WREG32(SRBM_INT_ACK, 0x1);
8cc1a532
AD		 3278
3279 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3280
3281 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3282 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3283
3284 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3285 rdev->config.cik.mem_max_burst_length_bytes = 256;
3286 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3287 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3288 if (rdev->config.cik.mem_row_size_in_kb > 4)
3289 rdev->config.cik.mem_row_size_in_kb = 4;
3290 /* XXX use MC settings? */
3291 rdev->config.cik.shader_engine_tile_size = 32;
3292 rdev->config.cik.num_gpus = 1;
3293 rdev->config.cik.multi_gpu_tile_size = 64;
3294
3295 /* fix up row size */
3296 gb_addr_config &= ~ROW_SIZE_MASK;
3297 switch (rdev->config.cik.mem_row_size_in_kb) {
3298 case 1:
3299 default:
3300 gb_addr_config |= ROW_SIZE(0);
3301 break;
3302 case 2:
3303 gb_addr_config |= ROW_SIZE(1);
3304 break;
3305 case 4:
3306 gb_addr_config |= ROW_SIZE(2);
3307 break;
3308 }
3309
3310 /* setup tiling info dword. gb_addr_config is not adequate since it does
3311 * not have bank info, so create a custom tiling dword.
3312 * bits 3:0 num_pipes
3313 * bits 7:4 num_banks
3314 * bits 11:8 group_size
3315 * bits 15:12 row_size
3316 */
3317 rdev->config.cik.tile_config = 0;
3318 switch (rdev->config.cik.num_tile_pipes) {
3319 case 1:
3320 rdev->config.cik.tile_config |= (0 << 0);
3321 break;
3322 case 2:
3323 rdev->config.cik.tile_config |= (1 << 0);
3324 break;
3325 case 4:
3326 rdev->config.cik.tile_config |= (2 << 0);
3327 break;
3328 case 8:
3329 default:
3330 /* XXX what about 12? */
3331 rdev->config.cik.tile_config |= (3 << 0);
3332 break;
3333 }
a537314e
MD		 3334 rdev->config.cik.tile_config |=
3335 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
8cc1a532
AD		 3336 rdev->config.cik.tile_config |=
3337 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3338 rdev->config.cik.tile_config |=
3339 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3340
3341 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3342 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3343 WREG32(DMIF_ADDR_CALC, gb_addr_config);
21a93e13
AD		 3344 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3345 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
87167bb1
CK		 3346 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3347 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3348 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
8cc1a532
AD		 3349
3350 cik_tiling_mode_table_init(rdev);
3351
3352 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3353 rdev->config.cik.max_sh_per_se,
3354 rdev->config.cik.max_backends_per_se);
3355
52da51f0 3356 rdev->config.cik.active_cus = 0;
65fcf668
AD		 3357 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3358 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6101b3ae
AD		 3359 rdev->config.cik.active_cus +=
3360 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
65fcf668
AD		 3361 }
3362 }
3363
8cc1a532
AD		 3364 /* set HW defaults for 3D engine */
3365 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3366
1c0a4625
OG		 3367 mutex_lock(&rdev->grbm_idx_mutex);
3368 /*
3369 * making sure that the following register writes will be broadcasted
3370 * to all the shaders
3371 */
3372 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
8cc1a532
AD		 3373 WREG32(SX_DEBUG_1, 0x20);
3374
3375 WREG32(TA_CNTL_AUX, 0x00010000);
3376
3377 tmp = RREG32(SPI_CONFIG_CNTL);
3378 tmp |= 0x03000000;
3379 WREG32(SPI_CONFIG_CNTL, tmp);
3380
3381 WREG32(SQ_CONFIG, 1);
3382
3383 WREG32(DB_DEBUG, 0);
3384
3385 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3386 tmp |= 0x00000400;
3387 WREG32(DB_DEBUG2, tmp);
3388
3389 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3390 tmp |= 0x00020200;
3391 WREG32(DB_DEBUG3, tmp);
3392
3393 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3394 tmp |= 0x00018208;
3395 WREG32(CB_HW_CONTROL, tmp);
3396
3397 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3398
3399 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3400 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3401 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3402 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3403
3404 WREG32(VGT_NUM_INSTANCES, 1);
3405
3406 WREG32(CP_PERFMON_CNTL, 0);
3407
3408 WREG32(SQ_CONFIG, 0);
3409
3410 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3411 FORCE_EOV_MAX_REZ_CNT(255)));
3412
3413 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3414 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3415
3416 WREG32(VGT_GS_VERTEX_REUSE, 16);
3417 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3418
3419 tmp = RREG32(HDP_MISC_CNTL);
3420 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3421 WREG32(HDP_MISC_CNTL, tmp);
3422
3423 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3424 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3425
3426 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3427 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
1c0a4625 3428 mutex_unlock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3429
3430 udelay(50);
3431}
3432
2cae3bc3
AD		 3433/*
3434 * GPU scratch registers helpers function.
3435 */
3436/**
3437 * cik_scratch_init - setup driver info for CP scratch regs
3438 *
3439 * @rdev: radeon_device pointer
3440 *
3441 * Set up the number and offset of the CP scratch registers.
3442 * NOTE: use of CP scratch registers is a legacy inferface and
3443 * is not used by default on newer asics (r6xx+). On newer asics,
3444 * memory buffers are used for fences rather than scratch regs.
3445 */
3446static void cik_scratch_init(struct radeon_device *rdev)
3447{
3448 int i;
3449
3450 rdev->scratch.num_reg = 7;
3451 rdev->scratch.reg_base = SCRATCH_REG0;
3452 for (i = 0; i < rdev->scratch.num_reg; i++) {
3453 rdev->scratch.free[i] = true;
3454 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3455 }
3456}
3457
fbc832c7
AD		 3458/**
3459 * cik_ring_test - basic gfx ring test
3460 *
3461 * @rdev: radeon_device pointer
3462 * @ring: radeon_ring structure holding ring information
3463 *
3464 * Allocate a scratch register and write to it using the gfx ring (CIK).
3465 * Provides a basic gfx ring test to verify that the ring is working.
3466 * Used by cik_cp_gfx_resume();
3467 * Returns 0 on success, error on failure.
3468 */
3469int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3470{
3471 uint32_t scratch;
3472 uint32_t tmp = 0;
3473 unsigned i;
3474 int r;
3475
3476 r = radeon_scratch_get(rdev, &scratch);
3477 if (r) {
3478 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3479 return r;
3480 }
3481 WREG32(scratch, 0xCAFEDEAD);
3482 r = radeon_ring_lock(rdev, ring, 3);
3483 if (r) {
3484 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3485 radeon_scratch_free(rdev, scratch);
3486 return r;
3487 }
3488 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3489 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3490 radeon_ring_write(ring, 0xDEADBEEF);
1538a9e0 3491 radeon_ring_unlock_commit(rdev, ring, false);
963e81f9 3492
fbc832c7
AD		 3493 for (i = 0; i < rdev->usec_timeout; i++) {
3494 tmp = RREG32(scratch);
3495 if (tmp == 0xDEADBEEF)
3496 break;
3497 DRM_UDELAY(1);
3498 }
3499 if (i < rdev->usec_timeout) {
3500 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3501 } else {
3502 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3503 ring->idx, scratch, tmp);
3504 r = -EINVAL;
3505 }
3506 radeon_scratch_free(rdev, scratch);
3507 return r;
3508}
3509
780f5ddd
AD		 3510/**
3511 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3512 *
3513 * @rdev: radeon_device pointer
3514 * @ridx: radeon ring index
3515 *
3516 * Emits an hdp flush on the cp.
3517 */
3518static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3519 int ridx)
3520{
3521 struct radeon_ring *ring = &rdev->ring[ridx];
5d259067 3522 u32 ref_and_mask;
780f5ddd 3523
5d259067
AD		 3524 switch (ring->idx) {
3525 case CAYMAN_RING_TYPE_CP1_INDEX:
3526 case CAYMAN_RING_TYPE_CP2_INDEX:
3527 default:
3528 switch (ring->me) {
3529 case 0:
3530 ref_and_mask = CP2 << ring->pipe;
3531 break;
3532 case 1:
3533 ref_and_mask = CP6 << ring->pipe;
3534 break;
3535 default:
3536 return;
3537 }
3538 break;
3539 case RADEON_RING_TYPE_GFX_INDEX:
3540 ref_and_mask = CP0;
3541 break;
3542 }
3543
3544 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3545 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3546 WAIT_REG_MEM_FUNCTION(3) | /* == */
3547 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3548 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3549 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3550 radeon_ring_write(ring, ref_and_mask);
3551 radeon_ring_write(ring, ref_and_mask);
3552 radeon_ring_write(ring, 0x20); /* poll interval */
780f5ddd
AD		 3553}
3554
2cae3bc3 3555/**
b07fdd38 3556 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
2cae3bc3
AD		 3557 *
3558 * @rdev: radeon_device pointer
3559 * @fence: radeon fence object
3560 *
3561 * Emits a fence sequnce number on the gfx ring and flushes
3562 * GPU caches.
3563 */
b07fdd38
AD		 3564void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3565 struct radeon_fence *fence)
2cae3bc3
AD		 3566{
3567 struct radeon_ring *ring = &rdev->ring[fence->ring];
3568 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3569
a9c73a0e
CK		 3570 /* Workaround for cache flush problems. First send a dummy EOP
3571 * event down the pipe with seq one below.
3572 */
3573 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3574 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3575 EOP_TC_ACTION_EN |
3576 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3577 EVENT_INDEX(5)));
3578 radeon_ring_write(ring, addr & 0xfffffffc);
3579 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
3580 DATA_SEL(1) | INT_SEL(0));
3581 radeon_ring_write(ring, fence->seq - 1);
3582 radeon_ring_write(ring, 0);
3583
3584 /* Then send the real EOP event down the pipe. */
2cae3bc3
AD		 3585 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3586 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3587 EOP_TC_ACTION_EN |
3588 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3589 EVENT_INDEX(5)));
3590 radeon_ring_write(ring, addr & 0xfffffffc);
3591 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3592 radeon_ring_write(ring, fence->seq);
3593 radeon_ring_write(ring, 0);
2cae3bc3
AD		 3594}
3595
b07fdd38
AD		 3596/**
3597 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3598 *
3599 * @rdev: radeon_device pointer
3600 * @fence: radeon fence object
3601 *
3602 * Emits a fence sequnce number on the compute ring and flushes
3603 * GPU caches.
3604 */
3605void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3606 struct radeon_fence *fence)
3607{
3608 struct radeon_ring *ring = &rdev->ring[fence->ring];
3609 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3610
3611 /* RELEASE_MEM - flush caches, send int */
3612 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3613 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3614 EOP_TC_ACTION_EN |
3615 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3616 EVENT_INDEX(5)));
3617 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3618 radeon_ring_write(ring, addr & 0xfffffffc);
3619 radeon_ring_write(ring, upper_32_bits(addr));
3620 radeon_ring_write(ring, fence->seq);
3621 radeon_ring_write(ring, 0);
b07fdd38
AD		 3622}
3623
86302eea
CK		 3624/**
3625 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3626 *
3627 * @rdev: radeon_device pointer
3628 * @ring: radeon ring buffer object
3629 * @semaphore: radeon semaphore object
3630 * @emit_wait: Is this a sempahore wait?
3631 *
3632 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3633 * from running ahead of semaphore waits.
3634 */
1654b817 3635bool cik_semaphore_ring_emit(struct radeon_device *rdev,
2cae3bc3
AD		 3636 struct radeon_ring *ring,
3637 struct radeon_semaphore *semaphore,
3638 bool emit_wait)
3639{
3640 uint64_t addr = semaphore->gpu_addr;
3641 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3642
3643 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
5e167cdb 3644 radeon_ring_write(ring, lower_32_bits(addr));
2cae3bc3 3645 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
1654b817 3646
86302eea
CK		 3647 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3648 /* Prevent the PFP from running ahead of the semaphore wait */
3649 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3650 radeon_ring_write(ring, 0x0);
3651 }
3652
1654b817 3653 return true;
2cae3bc3
AD		 3654}
3655
c9dbd705
AD		 3656/**
3657 * cik_copy_cpdma - copy pages using the CP DMA engine
3658 *
3659 * @rdev: radeon_device pointer
3660 * @src_offset: src GPU address
3661 * @dst_offset: dst GPU address
3662 * @num_gpu_pages: number of GPU pages to xfer
57d20a43 3663 * @resv: reservation object to sync to
c9dbd705
AD		 3664 *
3665 * Copy GPU paging using the CP DMA engine (CIK+).
3666 * Used by the radeon ttm implementation to move pages if
3667 * registered as the asic copy callback.
3668 */
57d20a43
CK		 3669struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
3670 uint64_t src_offset, uint64_t dst_offset,
3671 unsigned num_gpu_pages,
3672 struct reservation_object *resv)
c9dbd705 3673{
57d20a43 3674 struct radeon_fence *fence;
975700d2 3675 struct radeon_sync sync;
c9dbd705
AD		 3676 int ring_index = rdev->asic->copy.blit_ring_index;
3677 struct radeon_ring *ring = &rdev->ring[ring_index];
3678 u32 size_in_bytes, cur_size_in_bytes, control;
3679 int i, num_loops;
3680 int r = 0;
3681
975700d2 3682 radeon_sync_create(&sync);
c9dbd705
AD		 3683
3684 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3685 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3686 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3687 if (r) {
3688 DRM_ERROR("radeon: moving bo (%d).\n", r);
975700d2 3689 radeon_sync_free(rdev, &sync, NULL);
57d20a43 3690 return ERR_PTR(r);
c9dbd705
AD		 3691 }
3692
975700d2
CK		 3693 radeon_sync_resv(rdev, &sync, resv, false);
3694 radeon_sync_rings(rdev, &sync, ring->idx);
c9dbd705
AD		 3695
3696 for (i = 0; i < num_loops; i++) {
3697 cur_size_in_bytes = size_in_bytes;
3698 if (cur_size_in_bytes > 0x1fffff)
3699 cur_size_in_bytes = 0x1fffff;
3700 size_in_bytes -= cur_size_in_bytes;
3701 control = 0;
3702 if (size_in_bytes == 0)
3703 control |= PACKET3_DMA_DATA_CP_SYNC;
3704 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3705 radeon_ring_write(ring, control);
3706 radeon_ring_write(ring, lower_32_bits(src_offset));
3707 radeon_ring_write(ring, upper_32_bits(src_offset));
3708 radeon_ring_write(ring, lower_32_bits(dst_offset));
3709 radeon_ring_write(ring, upper_32_bits(dst_offset));
3710 radeon_ring_write(ring, cur_size_in_bytes);
3711 src_offset += cur_size_in_bytes;
3712 dst_offset += cur_size_in_bytes;
3713 }
3714
57d20a43 3715 r = radeon_fence_emit(rdev, &fence, ring->idx);
c9dbd705
AD		 3716 if (r) {
3717 radeon_ring_unlock_undo(rdev, ring);
975700d2 3718 radeon_sync_free(rdev, &sync, NULL);
57d20a43 3719 return ERR_PTR(r);
c9dbd705
AD		 3720 }
3721
1538a9e0 3722 radeon_ring_unlock_commit(rdev, ring, false);
975700d2 3723 radeon_sync_free(rdev, &sync, fence);
c9dbd705 3724
57d20a43 3725 return fence;
c9dbd705
AD		 3726}
3727
2cae3bc3
AD		 3728/*
3729 * IB stuff
3730 */
3731/**
3732 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3733 *
3734 * @rdev: radeon_device pointer
3735 * @ib: radeon indirect buffer object
3736 *
5fc45397 3737 * Emits a DE (drawing engine) or CE (constant engine) IB
2cae3bc3
AD		 3738 * on the gfx ring. IBs are usually generated by userspace
3739 * acceleration drivers and submitted to the kernel for
5fc45397 3740 * scheduling on the ring. This function schedules the IB
2cae3bc3
AD		 3741 * on the gfx ring for execution by the GPU.
3742 */
3743void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3744{
3745 struct radeon_ring *ring = &rdev->ring[ib->ring];
7c42bc1a 3746 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
2cae3bc3
AD		 3747 u32 header, control = INDIRECT_BUFFER_VALID;
3748
3749 if (ib->is_const_ib) {
3750 /* set switch buffer packet before const IB */
3751 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3752 radeon_ring_write(ring, 0);
3753
3754 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3755 } else {
3756 u32 next_rptr;
3757 if (ring->rptr_save_reg) {
3758 next_rptr = ring->wptr + 3 + 4;
3759 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3760 radeon_ring_write(ring, ((ring->rptr_save_reg -
3761 PACKET3_SET_UCONFIG_REG_START) >> 2));
3762 radeon_ring_write(ring, next_rptr);
3763 } else if (rdev->wb.enabled) {
3764 next_rptr = ring->wptr + 5 + 4;
3765 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3766 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3767 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
5e167cdb 3768 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
2cae3bc3
AD		 3769 radeon_ring_write(ring, next_rptr);
3770 }
3771
3772 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3773 }
3774
7c42bc1a 3775 control |= ib->length_dw | (vm_id << 24);
2cae3bc3
AD		 3776
3777 radeon_ring_write(ring, header);
5f3e226f 3778 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFFC));
2cae3bc3
AD		 3779 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3780 radeon_ring_write(ring, control);
3781}
3782
fbc832c7
AD		 3783/**
3784 * cik_ib_test - basic gfx ring IB test
3785 *
3786 * @rdev: radeon_device pointer
3787 * @ring: radeon_ring structure holding ring information
3788 *
3789 * Allocate an IB and execute it on the gfx ring (CIK).
3790 * Provides a basic gfx ring test to verify that IBs are working.
3791 * Returns 0 on success, error on failure.
3792 */
3793int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3794{
3795 struct radeon_ib ib;
3796 uint32_t scratch;
3797 uint32_t tmp = 0;
3798 unsigned i;
3799 int r;
3800
3801 r = radeon_scratch_get(rdev, &scratch);
3802 if (r) {
3803 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3804 return r;
3805 }
3806 WREG32(scratch, 0xCAFEDEAD);
3807 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3808 if (r) {
3809 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
5510f124 3810 radeon_scratch_free(rdev, scratch);
fbc832c7
AD		 3811 return r;
3812 }
3813 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3814 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3815 ib.ptr[2] = 0xDEADBEEF;
3816 ib.length_dw = 3;
1538a9e0 3817 r = radeon_ib_schedule(rdev, &ib, NULL, false);
fbc832c7
AD		 3818 if (r) {
3819 radeon_scratch_free(rdev, scratch);
3820 radeon_ib_free(rdev, &ib);
3821 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3822 return r;
3823 }
04db4caf
MD		 3824 r = radeon_fence_wait_timeout(ib.fence, false, usecs_to_jiffies(
3825 RADEON_USEC_IB_TEST_TIMEOUT));
3826 if (r < 0) {
fbc832c7 3827 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
5510f124
CK		 3828 radeon_scratch_free(rdev, scratch);
3829 radeon_ib_free(rdev, &ib);
fbc832c7 3830 return r;
04db4caf
MD		 3831 } else if (r == 0) {
3832 DRM_ERROR("radeon: fence wait timed out.\n");
3833 radeon_scratch_free(rdev, scratch);
3834 radeon_ib_free(rdev, &ib);
3835 return -ETIMEDOUT;
fbc832c7 3836 }
04db4caf 3837 r = 0;
fbc832c7
AD		 3838 for (i = 0; i < rdev->usec_timeout; i++) {
3839 tmp = RREG32(scratch);
3840 if (tmp == 0xDEADBEEF)
3841 break;
3842 DRM_UDELAY(1);
3843 }
3844 if (i < rdev->usec_timeout) {
3845 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3846 } else {
3847 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3848 scratch, tmp);
3849 r = -EINVAL;
3850 }
3851 radeon_scratch_free(rdev, scratch);
3852 radeon_ib_free(rdev, &ib);
3853 return r;
3854}
3855
841cf442
AD		 3856/*
3857 * CP.
3858 * On CIK, gfx and compute now have independant command processors.
3859 *
3860 * GFX
3861 * Gfx consists of a single ring and can process both gfx jobs and
3862 * compute jobs. The gfx CP consists of three microengines (ME):
3863 * PFP - Pre-Fetch Parser
3864 * ME - Micro Engine
3865 * CE - Constant Engine
3866 * The PFP and ME make up what is considered the Drawing Engine (DE).
3867 * The CE is an asynchronous engine used for updating buffer desciptors
3868 * used by the DE so that they can be loaded into cache in parallel
3869 * while the DE is processing state update packets.
3870 *
3871 * Compute
3872 * The compute CP consists of two microengines (ME):
3873 * MEC1 - Compute MicroEngine 1
3874 * MEC2 - Compute MicroEngine 2
3875 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3876 * The queues are exposed to userspace and are programmed directly
3877 * by the compute runtime.
3878 */
3879/**
3880 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3881 *
3882 * @rdev: radeon_device pointer
3883 * @enable: enable or disable the MEs
3884 *
3885 * Halts or unhalts the gfx MEs.
3886 */
3887static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3888{
3889 if (enable)
3890 WREG32(CP_ME_CNTL, 0);
3891 else {
50efa51a
AD		 3892 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3893 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
841cf442
AD		 3894 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3895 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3896 }
3897 udelay(50);
3898}
3899
3900/**
3901 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3902 *
3903 * @rdev: radeon_device pointer
3904 *
3905 * Loads the gfx PFP, ME, and CE ucode.
3906 * Returns 0 for success, -EINVAL if the ucode is not available.
3907 */
3908static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3909{
841cf442
AD		 3910 int i;
3911
3912 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3913 return -EINVAL;
3914
3915 cik_cp_gfx_enable(rdev, false);
3916
f2c6b0f4
AD		 3917 if (rdev->new_fw) {
3918 const struct gfx_firmware_header_v1_0 *pfp_hdr =
3919 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3920 const struct gfx_firmware_header_v1_0 *ce_hdr =
3921 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3922 const struct gfx_firmware_header_v1_0 *me_hdr =
3923 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3924 const __le32 *fw_data;
3925 u32 fw_size;
3926
3927 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3928 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3929 radeon_ucode_print_gfx_hdr(&me_hdr->header);
3930
3931 /* PFP */
3932 fw_data = (const __le32 *)
3933 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3934 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3935 WREG32(CP_PFP_UCODE_ADDR, 0);
3936 for (i = 0; i < fw_size; i++)
3937 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 3938 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version));
f2c6b0f4
AD		 3939
3940 /* CE */
3941 fw_data = (const __le32 *)
3942 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3943 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3944 WREG32(CP_CE_UCODE_ADDR, 0);
3945 for (i = 0; i < fw_size; i++)
3946 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 3947 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version));
f2c6b0f4
AD		 3948
3949 /* ME */
3950 fw_data = (const __be32 *)
3951 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3952 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3953 WREG32(CP_ME_RAM_WADDR, 0);
3954 for (i = 0; i < fw_size; i++)
3955 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
38aea071
AD		 3956 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version));
3957 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version));
f2c6b0f4
AD		 3958 } else {
3959 const __be32 *fw_data;
3960
3961 /* PFP */
3962 fw_data = (const __be32 *)rdev->pfp_fw->data;
3963 WREG32(CP_PFP_UCODE_ADDR, 0);
3964 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3965 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3966 WREG32(CP_PFP_UCODE_ADDR, 0);
3967
3968 /* CE */
3969 fw_data = (const __be32 *)rdev->ce_fw->data;
3970 WREG32(CP_CE_UCODE_ADDR, 0);
3971 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3972 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3973 WREG32(CP_CE_UCODE_ADDR, 0);
3974
3975 /* ME */
3976 fw_data = (const __be32 *)rdev->me_fw->data;
3977 WREG32(CP_ME_RAM_WADDR, 0);
3978 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3979 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3980 WREG32(CP_ME_RAM_WADDR, 0);
3981 }
841cf442 3982
841cf442
AD		 3983 return 0;
3984}
3985
3986/**
3987 * cik_cp_gfx_start - start the gfx ring
3988 *
3989 * @rdev: radeon_device pointer
3990 *
3991 * Enables the ring and loads the clear state context and other
3992 * packets required to init the ring.
3993 * Returns 0 for success, error for failure.
3994 */
3995static int cik_cp_gfx_start(struct radeon_device *rdev)
3996{
3997 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3998 int r, i;
3999
4000 /* init the CP */
4001 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4002 WREG32(CP_ENDIAN_SWAP, 0);
4003 WREG32(CP_DEVICE_ID, 1);
4004
4005 cik_cp_gfx_enable(rdev, true);
4006
4007 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4008 if (r) {
4009 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4010 return r;
4011 }
4012
4013 /* init the CE partitions. CE only used for gfx on CIK */
4014 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4015 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
dc4edad6
JZ		 4016 radeon_ring_write(ring, 0x8000);
4017 radeon_ring_write(ring, 0x8000);
841cf442
AD		 4018
4019 /* setup clear context state */
4020 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4021 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4022
4023 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4024 radeon_ring_write(ring, 0x80000000);
4025 radeon_ring_write(ring, 0x80000000);
4026
4027 for (i = 0; i < cik_default_size; i++)
4028 radeon_ring_write(ring, cik_default_state[i]);
4029
4030 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4031 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4032
4033 /* set clear context state */
4034 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4035 radeon_ring_write(ring, 0);
4036
4037 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4038 radeon_ring_write(ring, 0x00000316);
4039 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4040 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4041
1538a9e0 4042 radeon_ring_unlock_commit(rdev, ring, false);
841cf442
AD		 4043
4044 return 0;
4045}
4046
4047/**
4048 * cik_cp_gfx_fini - stop the gfx ring
4049 *
4050 * @rdev: radeon_device pointer
4051 *
4052 * Stop the gfx ring and tear down the driver ring
4053 * info.
4054 */
4055static void cik_cp_gfx_fini(struct radeon_device *rdev)
4056{
4057 cik_cp_gfx_enable(rdev, false);
4058 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4059}
4060
4061/**
4062 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4063 *
4064 * @rdev: radeon_device pointer
4065 *
4066 * Program the location and size of the gfx ring buffer
4067 * and test it to make sure it's working.
4068 * Returns 0 for success, error for failure.
4069 */
4070static int cik_cp_gfx_resume(struct radeon_device *rdev)
4071{
4072 struct radeon_ring *ring;
4073 u32 tmp;
4074 u32 rb_bufsz;
4075 u64 rb_addr;
4076 int r;
4077
4078 WREG32(CP_SEM_WAIT_TIMER, 0x0);
939c0d3c
AD		 4079 if (rdev->family != CHIP_HAWAII)
4080 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
841cf442
AD		 4081
4082 /* Set the write pointer delay */
4083 WREG32(CP_RB_WPTR_DELAY, 0);
4084
4085 /* set the RB to use vmid 0 */
4086 WREG32(CP_RB_VMID, 0);
4087
4088 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4089
4090 /* ring 0 - compute and gfx */
4091 /* Set ring buffer size */
4092 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 4093 rb_bufsz = order_base_2(ring->ring_size / 8);
4094 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
841cf442
AD		 4095#ifdef __BIG_ENDIAN
4096 tmp |= BUF_SWAP_32BIT;
4097#endif
4098 WREG32(CP_RB0_CNTL, tmp);
4099
4100 /* Initialize the ring buffer's read and write pointers */
4101 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4102 ring->wptr = 0;
4103 WREG32(CP_RB0_WPTR, ring->wptr);
4104
4105 /* set the wb address wether it's enabled or not */
4106 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4107 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4108
4109 /* scratch register shadowing is no longer supported */
4110 WREG32(SCRATCH_UMSK, 0);
4111
4112 if (!rdev->wb.enabled)
4113 tmp |= RB_NO_UPDATE;
4114
4115 mdelay(1);
4116 WREG32(CP_RB0_CNTL, tmp);
4117
4118 rb_addr = ring->gpu_addr >> 8;
4119 WREG32(CP_RB0_BASE, rb_addr);
4120 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4121
841cf442
AD		 4122 /* start the ring */
4123 cik_cp_gfx_start(rdev);
4124 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4125 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4126 if (r) {
4127 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4128 return r;
4129 }
50efa51a
AD		 4130
4131 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4132 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4133
841cf442
AD		 4134 return 0;
4135}
4136
ea31bf69
AD		 4137u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4138 struct radeon_ring *ring)
963e81f9
AD		 4139{
4140 u32 rptr;
4141
ea31bf69
AD		 4142 if (rdev->wb.enabled)
4143 rptr = rdev->wb.wb[ring->rptr_offs/4];
4144 else
4145 rptr = RREG32(CP_RB0_RPTR);
4146
4147 return rptr;
4148}
4149
4150u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4151 struct radeon_ring *ring)
4152{
0003b8d2 4153 return RREG32(CP_RB0_WPTR);
ea31bf69
AD		 4154}
4155
4156void cik_gfx_set_wptr(struct radeon_device *rdev,
4157 struct radeon_ring *ring)
4158{
4159 WREG32(CP_RB0_WPTR, ring->wptr);
4160 (void)RREG32(CP_RB0_WPTR);
4161}
4162
4163u32 cik_compute_get_rptr(struct radeon_device *rdev,
4164 struct radeon_ring *ring)
4165{
4166 u32 rptr;
963e81f9
AD		 4167
4168 if (rdev->wb.enabled) {
ea31bf69 4169 rptr = rdev->wb.wb[ring->rptr_offs/4];
963e81f9 4170 } else {
f61d5b46 4171 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4172 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4173 rptr = RREG32(CP_HQD_PQ_RPTR);
4174 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4175 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4176 }
963e81f9
AD		 4177
4178 return rptr;
4179}
4180
ea31bf69
AD		 4181u32 cik_compute_get_wptr(struct radeon_device *rdev,
4182 struct radeon_ring *ring)
963e81f9
AD		 4183{
4184 u32 wptr;
4185
4186 if (rdev->wb.enabled) {
ea31bf69
AD		 4187 /* XXX check if swapping is necessary on BE */
4188 wptr = rdev->wb.wb[ring->wptr_offs/4];
963e81f9 4189 } else {
f61d5b46 4190 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4191 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4192 wptr = RREG32(CP_HQD_PQ_WPTR);
4193 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4194 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4195 }
963e81f9
AD		 4196
4197 return wptr;
4198}
4199
ea31bf69
AD		 4200void cik_compute_set_wptr(struct radeon_device *rdev,
4201 struct radeon_ring *ring)
963e81f9 4202{
ea31bf69
AD		 4203 /* XXX check if swapping is necessary on BE */
4204 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
d5754ab8 4205 WDOORBELL32(ring->doorbell_index, ring->wptr);
963e81f9
AD		 4206}
4207
161569de
JG		 4208static void cik_compute_stop(struct radeon_device *rdev,
4209 struct radeon_ring *ring)
4210{
4211 u32 j, tmp;
4212
4213 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4214 /* Disable wptr polling. */
4215 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4216 tmp &= ~WPTR_POLL_EN;
4217 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4218 /* Disable HQD. */
4219 if (RREG32(CP_HQD_ACTIVE) & 1) {
4220 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4221 for (j = 0; j < rdev->usec_timeout; j++) {
4222 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4223 break;
4224 udelay(1);
4225 }
4226 WREG32(CP_HQD_DEQUEUE_REQUEST, 0);
4227 WREG32(CP_HQD_PQ_RPTR, 0);
4228 WREG32(CP_HQD_PQ_WPTR, 0);
4229 }
4230 cik_srbm_select(rdev, 0, 0, 0, 0);
4231}
4232
841cf442
AD		 4233/**
4234 * cik_cp_compute_enable - enable/disable the compute CP MEs
4235 *
4236 * @rdev: radeon_device pointer
4237 * @enable: enable or disable the MEs
4238 *
4239 * Halts or unhalts the compute MEs.
4240 */
4241static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4242{
4243 if (enable)
4244 WREG32(CP_MEC_CNTL, 0);
b2b3d8d9 4245 else {
161569de
JG		 4246 /*
4247 * To make hibernation reliable we need to clear compute ring
4248 * configuration before halting the compute ring.
4249 */
4250 mutex_lock(&rdev->srbm_mutex);
4251 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
4252 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
4253 mutex_unlock(&rdev->srbm_mutex);
4254
841cf442 4255 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
b2b3d8d9
AD		 4256 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4257 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4258 }
841cf442
AD		 4259 udelay(50);
4260}
4261
4262/**
4263 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4264 *
4265 * @rdev: radeon_device pointer
4266 *
4267 * Loads the compute MEC1&2 ucode.
4268 * Returns 0 for success, -EINVAL if the ucode is not available.
4269 */
4270static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4271{
841cf442
AD		 4272 int i;
4273
4274 if (!rdev->mec_fw)
4275 return -EINVAL;
4276
4277 cik_cp_compute_enable(rdev, false);
4278
f2c6b0f4
AD		 4279 if (rdev->new_fw) {
4280 const struct gfx_firmware_header_v1_0 *mec_hdr =
4281 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4282 const __le32 *fw_data;
4283 u32 fw_size;
4284
4285 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4286
4287 /* MEC1 */
4288 fw_data = (const __le32 *)
4289 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4290 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4291 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4292 for (i = 0; i < fw_size; i++)
4293 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4294 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version));
841cf442 4295
841cf442 4296 /* MEC2 */
f2c6b0f4
AD		 4297 if (rdev->family == CHIP_KAVERI) {
4298 const struct gfx_firmware_header_v1_0 *mec2_hdr =
4299 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4300
4301 fw_data = (const __le32 *)
4302 (rdev->mec2_fw->data +
4303 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4304 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4305 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4306 for (i = 0; i < fw_size; i++)
4307 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4308 WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version));
f2c6b0f4
AD		 4309 }
4310 } else {
4311 const __be32 *fw_data;
4312
4313 /* MEC1 */
841cf442 4314 fw_data = (const __be32 *)rdev->mec_fw->data;
f2c6b0f4 4315 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
841cf442 4316 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
f2c6b0f4
AD		 4317 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4318 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4319
4320 if (rdev->family == CHIP_KAVERI) {
4321 /* MEC2 */
4322 fw_data = (const __be32 *)rdev->mec_fw->data;
4323 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4324 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4325 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4326 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4327 }
841cf442
AD		 4328 }
4329
4330 return 0;
4331}
4332
4333/**
4334 * cik_cp_compute_start - start the compute queues
4335 *
4336 * @rdev: radeon_device pointer
4337 *
4338 * Enable the compute queues.
4339 * Returns 0 for success, error for failure.
4340 */
4341static int cik_cp_compute_start(struct radeon_device *rdev)
4342{
963e81f9
AD		 4343 cik_cp_compute_enable(rdev, true);
4344
841cf442
AD		 4345 return 0;
4346}
4347
4348/**
4349 * cik_cp_compute_fini - stop the compute queues
4350 *
4351 * @rdev: radeon_device pointer
4352 *
4353 * Stop the compute queues and tear down the driver queue
4354 * info.
4355 */
4356static void cik_cp_compute_fini(struct radeon_device *rdev)
4357{
963e81f9
AD		 4358 int i, idx, r;
4359
841cf442 4360 cik_cp_compute_enable(rdev, false);
963e81f9
AD		 4361
4362 for (i = 0; i < 2; i++) {
4363 if (i == 0)
4364 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4365 else
4366 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4367
4368 if (rdev->ring[idx].mqd_obj) {
4369 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4370 if (unlikely(r != 0))
4371 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4372
4373 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4374 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4375
4376 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4377 rdev->ring[idx].mqd_obj = NULL;
4378 }
4379 }
841cf442
AD		 4380}
4381
963e81f9
AD		 4382static void cik_mec_fini(struct radeon_device *rdev)
4383{
4384 int r;
4385
4386 if (rdev->mec.hpd_eop_obj) {
4387 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4388 if (unlikely(r != 0))
4389 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4390 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4391 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4392
4393 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4394 rdev->mec.hpd_eop_obj = NULL;
4395 }
4396}
4397
4398#define MEC_HPD_SIZE 2048
4399
4400static int cik_mec_init(struct radeon_device *rdev)
4401{
4402 int r;
4403 u32 *hpd;
4404
4405 /*
4406 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4407 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
62a7b7fb
OG		 4408 * Nonetheless, we assign only 1 pipe because all other pipes will
4409 * be handled by KFD
963e81f9 4410 */
62a7b7fb
OG		 4411 rdev->mec.num_mec = 1;
4412 rdev->mec.num_pipe = 1;
963e81f9
AD		 4413 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4414
4415 if (rdev->mec.hpd_eop_obj == NULL) {
4416 r = radeon_bo_create(rdev,
4417 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4418 PAGE_SIZE, true,
831b6966 4419 RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL,
963e81f9
AD		 4420 &rdev->mec.hpd_eop_obj);
4421 if (r) {
4422 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4423 return r;
4424 }
4425 }
4426
4427 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4428 if (unlikely(r != 0)) {
4429 cik_mec_fini(rdev);
4430 return r;
4431 }
4432 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4433 &rdev->mec.hpd_eop_gpu_addr);
4434 if (r) {
4435 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4436 cik_mec_fini(rdev);
4437 return r;
4438 }
4439 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4440 if (r) {
4441 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4442 cik_mec_fini(rdev);
4443 return r;
4444 }
4445
4446 /* clear memory. Not sure if this is required or not */
4447 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4448
4449 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4450 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4451
4452 return 0;
4453}
4454
4455struct hqd_registers
4456{
4457 u32 cp_mqd_base_addr;
4458 u32 cp_mqd_base_addr_hi;
4459 u32 cp_hqd_active;
4460 u32 cp_hqd_vmid;
4461 u32 cp_hqd_persistent_state;
4462 u32 cp_hqd_pipe_priority;
4463 u32 cp_hqd_queue_priority;
4464 u32 cp_hqd_quantum;
4465 u32 cp_hqd_pq_base;
4466 u32 cp_hqd_pq_base_hi;
4467 u32 cp_hqd_pq_rptr;
4468 u32 cp_hqd_pq_rptr_report_addr;
4469 u32 cp_hqd_pq_rptr_report_addr_hi;
4470 u32 cp_hqd_pq_wptr_poll_addr;
4471 u32 cp_hqd_pq_wptr_poll_addr_hi;
4472 u32 cp_hqd_pq_doorbell_control;
4473 u32 cp_hqd_pq_wptr;
4474 u32 cp_hqd_pq_control;
4475 u32 cp_hqd_ib_base_addr;
4476 u32 cp_hqd_ib_base_addr_hi;
4477 u32 cp_hqd_ib_rptr;
4478 u32 cp_hqd_ib_control;
4479 u32 cp_hqd_iq_timer;
4480 u32 cp_hqd_iq_rptr;
4481 u32 cp_hqd_dequeue_request;
4482 u32 cp_hqd_dma_offload;
4483 u32 cp_hqd_sema_cmd;
4484 u32 cp_hqd_msg_type;
4485 u32 cp_hqd_atomic0_preop_lo;
4486 u32 cp_hqd_atomic0_preop_hi;
4487 u32 cp_hqd_atomic1_preop_lo;
4488 u32 cp_hqd_atomic1_preop_hi;
4489 u32 cp_hqd_hq_scheduler0;
4490 u32 cp_hqd_hq_scheduler1;
4491 u32 cp_mqd_control;
4492};
4493
4494struct bonaire_mqd
4495{
4496 u32 header;
4497 u32 dispatch_initiator;
4498 u32 dimensions[3];
4499 u32 start_idx[3];
4500 u32 num_threads[3];
4501 u32 pipeline_stat_enable;
4502 u32 perf_counter_enable;
4503 u32 pgm[2];
4504 u32 tba[2];
4505 u32 tma[2];
4506 u32 pgm_rsrc[2];
4507 u32 vmid;
4508 u32 resource_limits;
4509 u32 static_thread_mgmt01[2];
4510 u32 tmp_ring_size;
4511 u32 static_thread_mgmt23[2];
4512 u32 restart[3];
4513 u32 thread_trace_enable;
4514 u32 reserved1;
4515 u32 user_data[16];
4516 u32 vgtcs_invoke_count[2];
4517 struct hqd_registers queue_state;
4518 u32 dequeue_cntr;
4519 u32 interrupt_queue[64];
4520};
4521
841cf442
AD		 4522/**
4523 * cik_cp_compute_resume - setup the compute queue registers
4524 *
4525 * @rdev: radeon_device pointer
4526 *
4527 * Program the compute queues and test them to make sure they
4528 * are working.
4529 * Returns 0 for success, error for failure.
4530 */
4531static int cik_cp_compute_resume(struct radeon_device *rdev)
4532{
370ce45b 4533 int r, i, j, idx;
963e81f9
AD		 4534 u32 tmp;
4535 bool use_doorbell = true;
4536 u64 hqd_gpu_addr;
4537 u64 mqd_gpu_addr;
4538 u64 eop_gpu_addr;
4539 u64 wb_gpu_addr;
4540 u32 *buf;
4541 struct bonaire_mqd *mqd;
841cf442 4542
841cf442
AD		 4543 r = cik_cp_compute_start(rdev);
4544 if (r)
4545 return r;
963e81f9
AD		 4546
4547 /* fix up chicken bits */
4548 tmp = RREG32(CP_CPF_DEBUG);
4549 tmp |= (1 << 23);
4550 WREG32(CP_CPF_DEBUG, tmp);
4551
4552 /* init the pipes */
f61d5b46 4553 mutex_lock(&rdev->srbm_mutex);
963e81f9 4554
d59095f7
AR		 4555 for (i = 0; i < rdev->mec.num_pipe; ++i) {
4556 cik_srbm_select(rdev, 0, i, 0, 0);
963e81f9 4557
d59095f7
AR		 4558 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2) ;
4559 /* write the EOP addr */
4560 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4561 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
963e81f9 4562
d59095f7
AR		 4563 /* set the VMID assigned */
4564 WREG32(CP_HPD_EOP_VMID, 0);
62a7b7fb 4565
d59095f7
AR		 4566 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4567 tmp = RREG32(CP_HPD_EOP_CONTROL);
4568 tmp &= ~EOP_SIZE_MASK;
4569 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4570 WREG32(CP_HPD_EOP_CONTROL, tmp);
963e81f9 4571
d59095f7 4572 }
f61d5b46 4573 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4574
4575 /* init the queues. Just two for now. */
4576 for (i = 0; i < 2; i++) {
4577 if (i == 0)
4578 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4579 else
4580 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4581
4582 if (rdev->ring[idx].mqd_obj == NULL) {
4583 r = radeon_bo_create(rdev,
4584 sizeof(struct bonaire_mqd),
4585 PAGE_SIZE, true,
02376d82 4586 RADEON_GEM_DOMAIN_GTT, 0, NULL,
831b6966 4587 NULL, &rdev->ring[idx].mqd_obj);
963e81f9
AD		 4588 if (r) {
4589 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4590 return r;
4591 }
4592 }
4593
4594 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4595 if (unlikely(r != 0)) {
4596 cik_cp_compute_fini(rdev);
4597 return r;
4598 }
4599 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4600 &mqd_gpu_addr);
4601 if (r) {
4602 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4603 cik_cp_compute_fini(rdev);
4604 return r;
4605 }
4606 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4607 if (r) {
4608 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4609 cik_cp_compute_fini(rdev);
4610 return r;
4611 }
4612
963e81f9
AD		 4613 /* init the mqd struct */
4614 memset(buf, 0, sizeof(struct bonaire_mqd));
4615
4616 mqd = (struct bonaire_mqd *)buf;
4617 mqd->header = 0xC0310800;
4618 mqd->static_thread_mgmt01[0] = 0xffffffff;
4619 mqd->static_thread_mgmt01[1] = 0xffffffff;
4620 mqd->static_thread_mgmt23[0] = 0xffffffff;
4621 mqd->static_thread_mgmt23[1] = 0xffffffff;
4622
f61d5b46 4623 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4624 cik_srbm_select(rdev, rdev->ring[idx].me,
4625 rdev->ring[idx].pipe,
4626 rdev->ring[idx].queue, 0);
4627
4628 /* disable wptr polling */
4629 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4630 tmp &= ~WPTR_POLL_EN;
4631 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4632
4633 /* enable doorbell? */
4634 mqd->queue_state.cp_hqd_pq_doorbell_control =
4635 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4636 if (use_doorbell)
4637 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4638 else
4639 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4640 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4641 mqd->queue_state.cp_hqd_pq_doorbell_control);
4642
4643 /* disable the queue if it's active */
4644 mqd->queue_state.cp_hqd_dequeue_request = 0;
4645 mqd->queue_state.cp_hqd_pq_rptr = 0;
4646 mqd->queue_state.cp_hqd_pq_wptr= 0;
4647 if (RREG32(CP_HQD_ACTIVE) & 1) {
4648 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
370ce45b 4649 for (j = 0; j < rdev->usec_timeout; j++) {
963e81f9
AD		 4650 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4651 break;
4652 udelay(1);
4653 }
4654 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4655 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4656 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4657 }
4658
4659 /* set the pointer to the MQD */
4660 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4661 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4662 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4663 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4664 /* set MQD vmid to 0 */
4665 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4666 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4667 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4668
4669 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4670 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4671 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4672 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4673 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4674 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4675
4676 /* set up the HQD, this is similar to CP_RB0_CNTL */
4677 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4678 mqd->queue_state.cp_hqd_pq_control &=
4679 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4680
4681 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4682 order_base_2(rdev->ring[idx].ring_size / 8);
963e81f9 4683 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4684 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
963e81f9
AD		 4685#ifdef __BIG_ENDIAN
4686 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4687#endif
4688 mqd->queue_state.cp_hqd_pq_control &=
4689 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4690 mqd->queue_state.cp_hqd_pq_control |=
4691 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4692 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4693
4694 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4695 if (i == 0)
4696 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4697 else
4698 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4699 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4700 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4701 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4702 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4703 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4704
4705 /* set the wb address wether it's enabled or not */
4706 if (i == 0)
4707 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4708 else
4709 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4710 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4711 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4712 upper_32_bits(wb_gpu_addr) & 0xffff;
4713 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4714 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4715 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4716 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4717
4718 /* enable the doorbell if requested */
4719 if (use_doorbell) {
4720 mqd->queue_state.cp_hqd_pq_doorbell_control =
4721 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4722 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4723 mqd->queue_state.cp_hqd_pq_doorbell_control |=
d5754ab8 4724 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
963e81f9
AD		 4725 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4726 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4727 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4728
4729 } else {
4730 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4731 }
4732 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4733 mqd->queue_state.cp_hqd_pq_doorbell_control);
4734
4735 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4736 rdev->ring[idx].wptr = 0;
4737 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4738 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
ff212f25 4739 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
963e81f9
AD		 4740
4741 /* set the vmid for the queue */
4742 mqd->queue_state.cp_hqd_vmid = 0;
4743 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4744
4745 /* activate the queue */
4746 mqd->queue_state.cp_hqd_active = 1;
4747 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4748
4749 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4750 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4751
4752 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4753 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4754
4755 rdev->ring[idx].ready = true;
4756 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4757 if (r)
4758 rdev->ring[idx].ready = false;
4759 }
4760
841cf442
AD		 4761 return 0;
4762}
4763
841cf442
AD		 4764static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4765{
4766 cik_cp_gfx_enable(rdev, enable);
4767 cik_cp_compute_enable(rdev, enable);
4768}
4769
841cf442
AD		 4770static int cik_cp_load_microcode(struct radeon_device *rdev)
4771{
4772 int r;
4773
4774 r = cik_cp_gfx_load_microcode(rdev);
4775 if (r)
4776 return r;
4777 r = cik_cp_compute_load_microcode(rdev);
4778 if (r)
4779 return r;
4780
4781 return 0;
4782}
4783
841cf442
AD		 4784static void cik_cp_fini(struct radeon_device *rdev)
4785{
4786 cik_cp_gfx_fini(rdev);
4787 cik_cp_compute_fini(rdev);
4788}
4789
841cf442
AD		 4790static int cik_cp_resume(struct radeon_device *rdev)
4791{
4792 int r;
4793
4214faf6
AD		 4794 cik_enable_gui_idle_interrupt(rdev, false);
4795
841cf442
AD		 4796 r = cik_cp_load_microcode(rdev);
4797 if (r)
4798 return r;
4799
4800 r = cik_cp_gfx_resume(rdev);
4801 if (r)
4802 return r;
4803 r = cik_cp_compute_resume(rdev);
4804 if (r)
4805 return r;
4806
4214faf6
AD		 4807 cik_enable_gui_idle_interrupt(rdev, true);
4808
841cf442
AD		 4809 return 0;
4810}
4811
cc066715 4812static void cik_print_gpu_status_regs(struct radeon_device *rdev)
6f2043ce 4813{
6f2043ce
AD		 4814 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4815 RREG32(GRBM_STATUS));
4816 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4817 RREG32(GRBM_STATUS2));
4818 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4819 RREG32(GRBM_STATUS_SE0));
4820 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4821 RREG32(GRBM_STATUS_SE1));
4822 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4823 RREG32(GRBM_STATUS_SE2));
4824 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4825 RREG32(GRBM_STATUS_SE3));
4826 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4827 RREG32(SRBM_STATUS));
4828 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4829 RREG32(SRBM_STATUS2));
cc066715
AD		 4830 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4831 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4832 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4833 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
963e81f9
AD		 4834 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4835 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4836 RREG32(CP_STALLED_STAT1));
4837 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4838 RREG32(CP_STALLED_STAT2));
4839 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4840 RREG32(CP_STALLED_STAT3));
4841 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4842 RREG32(CP_CPF_BUSY_STAT));
4843 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4844 RREG32(CP_CPF_STALLED_STAT1));
4845 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4846 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4847 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4848 RREG32(CP_CPC_STALLED_STAT1));
4849 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
cc066715 4850}
6f2043ce 4851
21a93e13 4852/**
cc066715 4853 * cik_gpu_check_soft_reset - check which blocks are busy
21a93e13
AD		 4854 *
4855 * @rdev: radeon_device pointer
21a93e13 4856 *
cc066715
AD		 4857 * Check which blocks are busy and return the relevant reset
4858 * mask to be used by cik_gpu_soft_reset().
4859 * Returns a mask of the blocks to be reset.
21a93e13 4860 */
2483b4ea 4861u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
21a93e13 4862{
cc066715
AD		 4863 u32 reset_mask = 0;
4864 u32 tmp;
21a93e13 4865
cc066715
AD		 4866 /* GRBM_STATUS */
4867 tmp = RREG32(GRBM_STATUS);
4868 if (tmp & (PA_BUSY | SC_BUSY |
4869 BCI_BUSY | SX_BUSY |
4870 TA_BUSY | VGT_BUSY |
4871 DB_BUSY | CB_BUSY |
4872 GDS_BUSY | SPI_BUSY |
4873 IA_BUSY | IA_BUSY_NO_DMA))
4874 reset_mask |= RADEON_RESET_GFX;
21a93e13 4875
cc066715
AD		 4876 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4877 reset_mask |= RADEON_RESET_CP;
21a93e13 4878
cc066715
AD		 4879 /* GRBM_STATUS2 */
4880 tmp = RREG32(GRBM_STATUS2);
4881 if (tmp & RLC_BUSY)
4882 reset_mask |= RADEON_RESET_RLC;
21a93e13 4883
cc066715
AD		 4884 /* SDMA0_STATUS_REG */
4885 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4886 if (!(tmp & SDMA_IDLE))
4887 reset_mask |= RADEON_RESET_DMA;
21a93e13 4888
cc066715
AD		 4889 /* SDMA1_STATUS_REG */
4890 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4891 if (!(tmp & SDMA_IDLE))
4892 reset_mask |= RADEON_RESET_DMA1;
21a93e13 4893
cc066715
AD		 4894 /* SRBM_STATUS2 */
4895 tmp = RREG32(SRBM_STATUS2);
4896 if (tmp & SDMA_BUSY)
4897 reset_mask |= RADEON_RESET_DMA;
21a93e13 4898
cc066715
AD		 4899 if (tmp & SDMA1_BUSY)
4900 reset_mask |= RADEON_RESET_DMA1;
21a93e13 4901
cc066715
AD		 4902 /* SRBM_STATUS */
4903 tmp = RREG32(SRBM_STATUS);
21a93e13 4904
cc066715
AD		 4905 if (tmp & IH_BUSY)
4906 reset_mask |= RADEON_RESET_IH;
21a93e13 4907
cc066715
AD		 4908 if (tmp & SEM_BUSY)
4909 reset_mask |= RADEON_RESET_SEM;
21a93e13 4910
cc066715
AD		 4911 if (tmp & GRBM_RQ_PENDING)
4912 reset_mask |= RADEON_RESET_GRBM;
21a93e13 4913
cc066715
AD		 4914 if (tmp & VMC_BUSY)
4915 reset_mask |= RADEON_RESET_VMC;
21a93e13 4916
cc066715
AD		 4917 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4918 MCC_BUSY | MCD_BUSY))
4919 reset_mask |= RADEON_RESET_MC;
21a93e13 4920
cc066715
AD		 4921 if (evergreen_is_display_hung(rdev))
4922 reset_mask |= RADEON_RESET_DISPLAY;
4923
4924 /* Skip MC reset as it's mostly likely not hung, just busy */
4925 if (reset_mask & RADEON_RESET_MC) {
4926 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4927 reset_mask &= ~RADEON_RESET_MC;
21a93e13 4928 }
cc066715
AD		 4929
4930 return reset_mask;
21a93e13
AD		 4931}
4932
4933/**
cc066715 4934 * cik_gpu_soft_reset - soft reset GPU
21a93e13
AD		 4935 *
4936 * @rdev: radeon_device pointer
cc066715 4937 * @reset_mask: mask of which blocks to reset
21a93e13 4938 *
cc066715 4939 * Soft reset the blocks specified in @reset_mask.
21a93e13 4940 */
cc066715 4941static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
21a93e13 4942{
6f2043ce 4943 struct evergreen_mc_save save;
cc066715
AD		 4944 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4945 u32 tmp;
21a93e13 4946
cc066715
AD		 4947 if (reset_mask == 0)
4948 return;
21a93e13 4949
cc066715 4950 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
21a93e13 4951
cc066715
AD		 4952 cik_print_gpu_status_regs(rdev);
4953 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4954 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4955 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4956 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
21a93e13 4957
fb2c7f4d
AD		 4958 /* disable CG/PG */
4959 cik_fini_pg(rdev);
4960 cik_fini_cg(rdev);
4961
cc066715
AD		 4962 /* stop the rlc */
4963 cik_rlc_stop(rdev);
21a93e13 4964
cc066715
AD		 4965 /* Disable GFX parsing/prefetching */
4966 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
21a93e13 4967
cc066715
AD		 4968 /* Disable MEC parsing/prefetching */
4969 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
21a93e13 4970
cc066715
AD		 4971 if (reset_mask & RADEON_RESET_DMA) {
4972 /* sdma0 */
4973 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4974 tmp |= SDMA_HALT;
4975 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4976 }
4977 if (reset_mask & RADEON_RESET_DMA1) {
4978 /* sdma1 */
4979 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4980 tmp |= SDMA_HALT;
4981 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4982 }
21a93e13 4983
6f2043ce 4984 evergreen_mc_stop(rdev, &save);
cc066715 4985 if (evergreen_mc_wait_for_idle(rdev)) {
6f2043ce
AD		 4986 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4987 }
21a93e13 4988
cc066715
AD		 4989 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4990 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
21a93e13 4991
cc066715
AD		 4992 if (reset_mask & RADEON_RESET_CP) {
4993 grbm_soft_reset |= SOFT_RESET_CP;
21a93e13 4994
cc066715
AD		 4995 srbm_soft_reset |= SOFT_RESET_GRBM;
4996 }
21a93e13 4997
cc066715
AD		 4998 if (reset_mask & RADEON_RESET_DMA)
4999 srbm_soft_reset |= SOFT_RESET_SDMA;
21a93e13 5000
cc066715
AD		 5001 if (reset_mask & RADEON_RESET_DMA1)
5002 srbm_soft_reset |= SOFT_RESET_SDMA1;
5003
5004 if (reset_mask & RADEON_RESET_DISPLAY)
5005 srbm_soft_reset |= SOFT_RESET_DC;
5006
5007 if (reset_mask & RADEON_RESET_RLC)
5008 grbm_soft_reset |= SOFT_RESET_RLC;
5009
5010 if (reset_mask & RADEON_RESET_SEM)
5011 srbm_soft_reset |= SOFT_RESET_SEM;
5012
5013 if (reset_mask & RADEON_RESET_IH)
5014 srbm_soft_reset |= SOFT_RESET_IH;
5015
5016 if (reset_mask & RADEON_RESET_GRBM)
5017 srbm_soft_reset |= SOFT_RESET_GRBM;
5018
5019 if (reset_mask & RADEON_RESET_VMC)
5020 srbm_soft_reset |= SOFT_RESET_VMC;
5021
5022 if (!(rdev->flags & RADEON_IS_IGP)) {
5023 if (reset_mask & RADEON_RESET_MC)
5024 srbm_soft_reset |= SOFT_RESET_MC;
21a93e13
AD		 5025 }
5026
cc066715
AD		 5027 if (grbm_soft_reset) {
5028 tmp = RREG32(GRBM_SOFT_RESET);
5029 tmp |= grbm_soft_reset;
5030 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5031 WREG32(GRBM_SOFT_RESET, tmp);
5032 tmp = RREG32(GRBM_SOFT_RESET);
21a93e13 5033
cc066715 5034 udelay(50);
21a93e13 5035
cc066715
AD		 5036 tmp &= ~grbm_soft_reset;
5037 WREG32(GRBM_SOFT_RESET, tmp);
5038 tmp = RREG32(GRBM_SOFT_RESET);
5039 }
21a93e13 5040
cc066715
AD		 5041 if (srbm_soft_reset) {
5042 tmp = RREG32(SRBM_SOFT_RESET);
5043 tmp |= srbm_soft_reset;
5044 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5045 WREG32(SRBM_SOFT_RESET, tmp);
5046 tmp = RREG32(SRBM_SOFT_RESET);
21a93e13 5047
cc066715 5048 udelay(50);
21a93e13 5049
cc066715
AD		 5050 tmp &= ~srbm_soft_reset;
5051 WREG32(SRBM_SOFT_RESET, tmp);
5052 tmp = RREG32(SRBM_SOFT_RESET);
5053 }
21a93e13 5054
6f2043ce
AD		 5055 /* Wait a little for things to settle down */
5056 udelay(50);
21a93e13 5057
6f2043ce 5058 evergreen_mc_resume(rdev, &save);
cc066715
AD		 5059 udelay(50);
5060
5061 cik_print_gpu_status_regs(rdev);
21a93e13
AD		 5062}
5063
0279ed19
AD		 5064struct kv_reset_save_regs {
5065 u32 gmcon_reng_execute;
5066 u32 gmcon_misc;
5067 u32 gmcon_misc3;
5068};
5069
5070static void kv_save_regs_for_reset(struct radeon_device *rdev,
5071 struct kv_reset_save_regs *save)
5072{
5073 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5074 save->gmcon_misc = RREG32(GMCON_MISC);
5075 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5076
5077 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5078 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5079 STCTRL_STUTTER_EN));
5080}
5081
5082static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5083 struct kv_reset_save_regs *save)
5084{
5085 int i;
5086
5087 WREG32(GMCON_PGFSM_WRITE, 0);
5088 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5089
5090 for (i = 0; i < 5; i++)
5091 WREG32(GMCON_PGFSM_WRITE, 0);
5092
5093 WREG32(GMCON_PGFSM_WRITE, 0);
5094 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5095
5096 for (i = 0; i < 5; i++)
5097 WREG32(GMCON_PGFSM_WRITE, 0);
5098
5099 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5100 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5101
5102 for (i = 0; i < 5; i++)
5103 WREG32(GMCON_PGFSM_WRITE, 0);
5104
5105 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5106 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5107
5108 for (i = 0; i < 5; i++)
5109 WREG32(GMCON_PGFSM_WRITE, 0);
5110
5111 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5112 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5113
5114 for (i = 0; i < 5; i++)
5115 WREG32(GMCON_PGFSM_WRITE, 0);
5116
5117 WREG32(GMCON_PGFSM_WRITE, 0);
5118 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5119
5120 for (i = 0; i < 5; i++)
5121 WREG32(GMCON_PGFSM_WRITE, 0);
5122
5123 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5124 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5125
5126 for (i = 0; i < 5; i++)
5127 WREG32(GMCON_PGFSM_WRITE, 0);
5128
5129 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5130 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5131
5132 for (i = 0; i < 5; i++)
5133 WREG32(GMCON_PGFSM_WRITE, 0);
5134
5135 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5136 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5137
5138 for (i = 0; i < 5; i++)
5139 WREG32(GMCON_PGFSM_WRITE, 0);
5140
5141 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5142 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5143
5144 for (i = 0; i < 5; i++)
5145 WREG32(GMCON_PGFSM_WRITE, 0);
5146
5147 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5148 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5149
5150 WREG32(GMCON_MISC3, save->gmcon_misc3);
5151 WREG32(GMCON_MISC, save->gmcon_misc);
5152 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5153}
5154
5155static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5156{
5157 struct evergreen_mc_save save;
5158 struct kv_reset_save_regs kv_save = { 0 };
5159 u32 tmp, i;
5160
5161 dev_info(rdev->dev, "GPU pci config reset\n");
5162
5163 /* disable dpm? */
5164
5165 /* disable cg/pg */
5166 cik_fini_pg(rdev);
5167 cik_fini_cg(rdev);
5168
5169 /* Disable GFX parsing/prefetching */
5170 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5171
5172 /* Disable MEC parsing/prefetching */
5173 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5174
5175 /* sdma0 */
5176 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5177 tmp |= SDMA_HALT;
5178 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5179 /* sdma1 */
5180 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5181 tmp |= SDMA_HALT;
5182 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5183 /* XXX other engines? */
5184
5185 /* halt the rlc, disable cp internal ints */
5186 cik_rlc_stop(rdev);
5187
5188 udelay(50);
5189
5190 /* disable mem access */
5191 evergreen_mc_stop(rdev, &save);
5192 if (evergreen_mc_wait_for_idle(rdev)) {
5193 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5194 }
5195
5196 if (rdev->flags & RADEON_IS_IGP)
5197 kv_save_regs_for_reset(rdev, &kv_save);
5198
5199 /* disable BM */
5200 pci_clear_master(rdev->pdev);
5201 /* reset */
5202 radeon_pci_config_reset(rdev);
5203
5204 udelay(100);
5205
5206 /* wait for asic to come out of reset */
5207 for (i = 0; i < rdev->usec_timeout; i++) {
5208 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5209 break;
5210 udelay(1);
5211 }
5212
5213 /* does asic init need to be run first??? */
5214 if (rdev->flags & RADEON_IS_IGP)
5215 kv_restore_regs_for_reset(rdev, &kv_save);
5216}
5217
21a93e13 5218/**
cc066715 5219 * cik_asic_reset - soft reset GPU
21a93e13
AD		 5220 *
5221 * @rdev: radeon_device pointer
71fe2899 5222 * @hard: force hard reset
21a93e13 5223 *
cc066715
AD		 5224 * Look up which blocks are hung and attempt
5225 * to reset them.
6f2043ce 5226 * Returns 0 for success.
21a93e13 5227 */
71fe2899 5228int cik_asic_reset(struct radeon_device *rdev, bool hard)
21a93e13 5229{
cc066715 5230 u32 reset_mask;
21a93e13 5231
71fe2899
JG		 5232 if (hard) {
5233 cik_gpu_pci_config_reset(rdev);
5234 return 0;
5235 }
5236
cc066715 5237 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5238
cc066715
AD		 5239 if (reset_mask)
5240 r600_set_bios_scratch_engine_hung(rdev, true);
21a93e13 5241
0279ed19 5242 /* try soft reset */
cc066715 5243 cik_gpu_soft_reset(rdev, reset_mask);
21a93e13 5244
cc066715
AD		 5245 reset_mask = cik_gpu_check_soft_reset(rdev);
5246
0279ed19
AD		 5247 /* try pci config reset */
5248 if (reset_mask && radeon_hard_reset)
5249 cik_gpu_pci_config_reset(rdev);
5250
5251 reset_mask = cik_gpu_check_soft_reset(rdev);
5252
cc066715
AD		 5253 if (!reset_mask)
5254 r600_set_bios_scratch_engine_hung(rdev, false);
21a93e13
AD		 5255
5256 return 0;
5257}
5258
5259/**
cc066715 5260 * cik_gfx_is_lockup - check if the 3D engine is locked up
21a93e13
AD		 5261 *
5262 * @rdev: radeon_device pointer
cc066715 5263 * @ring: radeon_ring structure holding ring information
21a93e13 5264 *
cc066715
AD		 5265 * Check if the 3D engine is locked up (CIK).
5266 * Returns true if the engine is locked, false if not.
21a93e13 5267 */
cc066715 5268bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
21a93e13 5269{
cc066715 5270 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5271
cc066715
AD		 5272 if (!(reset_mask & (RADEON_RESET_GFX |
5273 RADEON_RESET_COMPUTE |
5274 RADEON_RESET_CP))) {
ff212f25 5275 radeon_ring_lockup_update(rdev, ring);
cc066715 5276 return false;
21a93e13 5277 }
cc066715 5278 return radeon_ring_test_lockup(rdev, ring);
21a93e13
AD		 5279}
5280
1c49165d 5281/* MC */
21a93e13 5282/**
1c49165d 5283 * cik_mc_program - program the GPU memory controller
21a93e13
AD		 5284 *
5285 * @rdev: radeon_device pointer
21a93e13 5286 *
1c49165d
AD		 5287 * Set the location of vram, gart, and AGP in the GPU's
5288 * physical address space (CIK).
21a93e13 5289 */
1c49165d 5290static void cik_mc_program(struct radeon_device *rdev)
21a93e13 5291{
1c49165d 5292 struct evergreen_mc_save save;
21a93e13 5293 u32 tmp;
1c49165d 5294 int i, j;
21a93e13 5295
1c49165d
AD		 5296 /* Initialize HDP */
5297 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5298 WREG32((0x2c14 + j), 0x00000000);
5299 WREG32((0x2c18 + j), 0x00000000);
5300 WREG32((0x2c1c + j), 0x00000000);
5301 WREG32((0x2c20 + j), 0x00000000);
5302 WREG32((0x2c24 + j), 0x00000000);
21a93e13 5303 }
1c49165d 5304 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
21a93e13 5305
1c49165d
AD		 5306 evergreen_mc_stop(rdev, &save);
5307 if (radeon_mc_wait_for_idle(rdev)) {
5308 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5309 }
1c49165d
AD		 5310 /* Lockout access through VGA aperture*/
5311 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5312 /* Update configuration */
5313 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5314 rdev->mc.vram_start >> 12);
5315 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5316 rdev->mc.vram_end >> 12);
5317 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5318 rdev->vram_scratch.gpu_addr >> 12);
5319 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5320 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5321 WREG32(MC_VM_FB_LOCATION, tmp);
5322 /* XXX double check these! */
5323 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5324 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5325 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5326 WREG32(MC_VM_AGP_BASE, 0);
5327 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5328 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5329 if (radeon_mc_wait_for_idle(rdev)) {
5330 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5331 }
1c49165d
AD		 5332 evergreen_mc_resume(rdev, &save);
5333 /* we need to own VRAM, so turn off the VGA renderer here
5334 * to stop it overwriting our objects */
5335 rv515_vga_render_disable(rdev);
21a93e13
AD		 5336}
5337
5338/**
1c49165d 5339 * cik_mc_init - initialize the memory controller driver params
21a93e13
AD		 5340 *
5341 * @rdev: radeon_device pointer
21a93e13 5342 *
1c49165d
AD		 5343 * Look up the amount of vram, vram width, and decide how to place
5344 * vram and gart within the GPU's physical address space (CIK).
5345 * Returns 0 for success.
21a93e13 5346 */
1c49165d 5347static int cik_mc_init(struct radeon_device *rdev)
21a93e13 5348{
1c49165d
AD		 5349 u32 tmp;
5350 int chansize, numchan;
21a93e13 5351
1c49165d
AD		 5352 /* Get VRAM informations */
5353 rdev->mc.vram_is_ddr = true;
5354 tmp = RREG32(MC_ARB_RAMCFG);
5355 if (tmp & CHANSIZE_MASK) {
5356 chansize = 64;
21a93e13 5357 } else {
1c49165d 5358 chansize = 32;
21a93e13 5359 }
1c49165d
AD		 5360 tmp = RREG32(MC_SHARED_CHMAP);
5361 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5362 case 0:
5363 default:
5364 numchan = 1;
5365 break;
5366 case 1:
5367 numchan = 2;
5368 break;
5369 case 2:
5370 numchan = 4;
5371 break;
5372 case 3:
5373 numchan = 8;
5374 break;
5375 case 4:
5376 numchan = 3;
5377 break;
5378 case 5:
5379 numchan = 6;
5380 break;
5381 case 6:
5382 numchan = 10;
5383 break;
5384 case 7:
5385 numchan = 12;
5386 break;
5387 case 8:
5388 numchan = 16;
5389 break;
5390 }
5391 rdev->mc.vram_width = numchan * chansize;
5392 /* Could aper size report 0 ? */
5393 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5394 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5395 /* size in MB on si */
13c5bfda
AD		 5396 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5397 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
1c49165d
AD		 5398 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5399 si_vram_gtt_location(rdev, &rdev->mc);
5400 radeon_update_bandwidth_info(rdev);
5401
5402 return 0;
5403}
5404
5405/*
5406 * GART
5407 * VMID 0 is the physical GPU addresses as used by the kernel.
5408 * VMIDs 1-15 are used for userspace clients and are handled
5409 * by the radeon vm/hsa code.
5410 */
5411/**
5412 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5413 *
5414 * @rdev: radeon_device pointer
5415 *
5416 * Flush the TLB for the VMID 0 page table (CIK).
5417 */
5418void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5419{
5420 /* flush hdp cache */
5421 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5422
5423 /* bits 0-15 are the VM contexts0-15 */
5424 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5425}
5426
5427/**
5428 * cik_pcie_gart_enable - gart enable
5429 *
5430 * @rdev: radeon_device pointer
5431 *
5432 * This sets up the TLBs, programs the page tables for VMID0,
5433 * sets up the hw for VMIDs 1-15 which are allocated on
5434 * demand, and sets up the global locations for the LDS, GDS,
5435 * and GPUVM for FSA64 clients (CIK).
5436 * Returns 0 for success, errors for failure.
5437 */
5438static int cik_pcie_gart_enable(struct radeon_device *rdev)
5439{
5440 int r, i;
5441
5442 if (rdev->gart.robj == NULL) {
5443 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5444 return -EINVAL;
5445 }
5446 r = radeon_gart_table_vram_pin(rdev);
5447 if (r)
5448 return r;
1c49165d
AD		 5449 /* Setup TLB control */
5450 WREG32(MC_VM_MX_L1_TLB_CNTL,
5451 (0xA << 7) |
5452 ENABLE_L1_TLB |
ec3dbbcb 5453 ENABLE_L1_FRAGMENT_PROCESSING |
1c49165d
AD		 5454 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5455 ENABLE_ADVANCED_DRIVER_MODEL |
5456 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5457 /* Setup L2 cache */
5458 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5459 ENABLE_L2_FRAGMENT_PROCESSING |
5460 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5461 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5462 EFFECTIVE_L2_QUEUE_SIZE(7) |
5463 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5464 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5465 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
ec3dbbcb
CK		 5466 BANK_SELECT(4) |
5467 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
1c49165d
AD		 5468 /* setup context0 */
5469 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
7c0411d2 5470 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1c49165d
AD		 5471 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5472 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5473 (u32)(rdev->dummy_page.addr >> 12));
5474 WREG32(VM_CONTEXT0_CNTL2, 0);
5475 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5476 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5477
5478 WREG32(0x15D4, 0);
5479 WREG32(0x15D8, 0);
5480 WREG32(0x15DC, 0);
5481
054e01d6 5482 /* restore context1-15 */
1c49165d
AD		 5483 /* set vm size, must be a multiple of 4 */
5484 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
607d4806 5485 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
1c49165d
AD		 5486 for (i = 1; i < 16; i++) {
5487 if (i < 8)
5488 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
054e01d6 5489 rdev->vm_manager.saved_table_addr[i]);
1c49165d
AD		 5490 else
5491 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
054e01d6 5492 rdev->vm_manager.saved_table_addr[i]);
1c49165d
AD		 5493 }
5494
5495 /* enable context1-15 */
5496 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5497 (u32)(rdev->dummy_page.addr >> 12));
a00024b0 5498 WREG32(VM_CONTEXT1_CNTL2, 4);
1c49165d 5499 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4510fb98 5500 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
a00024b0
AD		 5501 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5502 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5503 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5504 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5505 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5506 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5507 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5508 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5509 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5510 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5511 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5512 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1c49165d 5513
1c49165d
AD		 5514 if (rdev->family == CHIP_KAVERI) {
5515 u32 tmp = RREG32(CHUB_CONTROL);
5516 tmp &= ~BYPASS_VM;
5517 WREG32(CHUB_CONTROL, tmp);
5518 }
5519
5520 /* XXX SH_MEM regs */
5521 /* where to put LDS, scratch, GPUVM in FSA64 space */
f61d5b46 5522 mutex_lock(&rdev->srbm_mutex);
1c49165d 5523 for (i = 0; i < 16; i++) {
b556b12e 5524 cik_srbm_select(rdev, 0, 0, 0, i);
21a93e13 5525 /* CP and shaders */
75cb00dc 5526 WREG32(SH_MEM_CONFIG, SH_MEM_CONFIG_GFX_DEFAULT);
1c49165d
AD		 5527 WREG32(SH_MEM_APE1_BASE, 1);
5528 WREG32(SH_MEM_APE1_LIMIT, 0);
5529 WREG32(SH_MEM_BASES, 0);
21a93e13
AD		 5530 /* SDMA GFX */
5531 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5532 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5533 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5534 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5535 /* XXX SDMA RLC - todo */
1c49165d 5536 }
b556b12e 5537 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 5538 mutex_unlock(&rdev->srbm_mutex);
1c49165d
AD		 5539
5540 cik_pcie_gart_tlb_flush(rdev);
5541 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5542 (unsigned)(rdev->mc.gtt_size >> 20),
5543 (unsigned long long)rdev->gart.table_addr);
5544 rdev->gart.ready = true;
5545 return 0;
5546}
5547
5548/**
5549 * cik_pcie_gart_disable - gart disable
5550 *
5551 * @rdev: radeon_device pointer
5552 *
5553 * This disables all VM page table (CIK).
5554 */
5555static void cik_pcie_gart_disable(struct radeon_device *rdev)
5556{
054e01d6
CK		 5557 unsigned i;
5558
5559 for (i = 1; i < 16; ++i) {
5560 uint32_t reg;
5561 if (i < 8)
5562 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
5563 else
5564 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
5565 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
5566 }
5567
1c49165d
AD		 5568 /* Disable all tables */
5569 WREG32(VM_CONTEXT0_CNTL, 0);
5570 WREG32(VM_CONTEXT1_CNTL, 0);
5571 /* Setup TLB control */
5572 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5573 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5574 /* Setup L2 cache */
5575 WREG32(VM_L2_CNTL,
5576 ENABLE_L2_FRAGMENT_PROCESSING |
5577 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5578 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5579 EFFECTIVE_L2_QUEUE_SIZE(7) |
5580 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5581 WREG32(VM_L2_CNTL2, 0);
5582 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5583 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5584 radeon_gart_table_vram_unpin(rdev);
5585}
5586
5587/**
5588 * cik_pcie_gart_fini - vm fini callback
5589 *
5590 * @rdev: radeon_device pointer
5591 *
5592 * Tears down the driver GART/VM setup (CIK).
5593 */
5594static void cik_pcie_gart_fini(struct radeon_device *rdev)
5595{
5596 cik_pcie_gart_disable(rdev);
5597 radeon_gart_table_vram_free(rdev);
5598 radeon_gart_fini(rdev);
5599}
5600
5601/* vm parser */
5602/**
5603 * cik_ib_parse - vm ib_parse callback
5604 *
5605 * @rdev: radeon_device pointer
5606 * @ib: indirect buffer pointer
5607 *
5608 * CIK uses hw IB checking so this is a nop (CIK).
5609 */
5610int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5611{
5612 return 0;
5613}
5614
5615/*
5616 * vm
5617 * VMID 0 is the physical GPU addresses as used by the kernel.
5618 * VMIDs 1-15 are used for userspace clients and are handled
5619 * by the radeon vm/hsa code.
5620 */
5621/**
5622 * cik_vm_init - cik vm init callback
5623 *
5624 * @rdev: radeon_device pointer
5625 *
5626 * Inits cik specific vm parameters (number of VMs, base of vram for
5627 * VMIDs 1-15) (CIK).
5628 * Returns 0 for success.
5629 */
5630int cik_vm_init(struct radeon_device *rdev)
5631{
62a7b7fb
OG		 5632 /*
5633 * number of VMs
5634 * VMID 0 is reserved for System
f4fa88ab 5635 * radeon graphics/compute will use VMIDs 1-15
62a7b7fb 5636 */
f4fa88ab 5637 rdev->vm_manager.nvm = 16;
1c49165d
AD		 5638 /* base offset of vram pages */
5639 if (rdev->flags & RADEON_IS_IGP) {
5640 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5641 tmp <<= 22;
5642 rdev->vm_manager.vram_base_offset = tmp;
5643 } else
5644 rdev->vm_manager.vram_base_offset = 0;
5645
5646 return 0;
5647}
5648
5649/**
5650 * cik_vm_fini - cik vm fini callback
5651 *
5652 * @rdev: radeon_device pointer
5653 *
5654 * Tear down any asic specific VM setup (CIK).
5655 */
5656void cik_vm_fini(struct radeon_device *rdev)
5657{
5658}
5659
3ec7d11b
AD		 5660/**
5661 * cik_vm_decode_fault - print human readable fault info
5662 *
5663 * @rdev: radeon_device pointer
5664 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5665 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5666 *
5667 * Print human readable fault information (CIK).
5668 */
5669static void cik_vm_decode_fault(struct radeon_device *rdev,
5670 u32 status, u32 addr, u32 mc_client)
5671{
939c0d3c 5672 u32 mc_id;
3ec7d11b
AD		 5673 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5674 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
328a50c7
MD		 5675 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5676 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
3ec7d11b 5677
939c0d3c
AD		 5678 if (rdev->family == CHIP_HAWAII)
5679 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5680 else
5681 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5682
328a50c7 5683 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
3ec7d11b
AD		 5684 protections, vmid, addr,
5685 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
328a50c7 5686 block, mc_client, mc_id);
3ec7d11b
AD		 5687}
5688
f96ab484
AD		 5689/**
5690 * cik_vm_flush - cik vm flush using the CP
5691 *
5692 * @rdev: radeon_device pointer
5693 *
5694 * Update the page table base and flush the VM TLB
5695 * using the CP (CIK).
5696 */
faffaf62
CK		 5697void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
5698 unsigned vm_id, uint64_t pd_addr)
f96ab484 5699{
faffaf62 5700 int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX);
f96ab484
AD		 5701
5702 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
f1d2a26b 5703 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484 5704 WRITE_DATA_DST_SEL(0)));
faffaf62 5705 if (vm_id < 8) {
f96ab484 5706 radeon_ring_write(ring,
faffaf62 5707 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
f96ab484
AD		 5708 } else {
5709 radeon_ring_write(ring,
faffaf62 5710 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
f96ab484
AD		 5711 }
5712 radeon_ring_write(ring, 0);
faffaf62 5713 radeon_ring_write(ring, pd_addr >> 12);
f96ab484
AD		 5714
5715 /* update SH_MEM_* regs */
5716 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 5717 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5718 WRITE_DATA_DST_SEL(0)));
5719 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5720 radeon_ring_write(ring, 0);
faffaf62 5721 radeon_ring_write(ring, VMID(vm_id));
f96ab484
AD		 5722
5723 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
4fb0bbd5 5724 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5725 WRITE_DATA_DST_SEL(0)));
5726 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5727 radeon_ring_write(ring, 0);
5728
5729 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
75cb00dc 5730 radeon_ring_write(ring, SH_MEM_CONFIG_GFX_DEFAULT); /* SH_MEM_CONFIG */
f96ab484
AD		 5731 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5732 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
21a93e13 5733
f96ab484 5734 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 5735 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5736 WRITE_DATA_DST_SEL(0)));
5737 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5738 radeon_ring_write(ring, 0);
5739 radeon_ring_write(ring, VMID(0));
6f2043ce 5740
f96ab484 5741 /* HDP flush */
faffaf62 5742 cik_hdp_flush_cp_ring_emit(rdev, ring->idx);
f96ab484
AD		 5743
5744 /* bits 0-15 are the VM contexts0-15 */
5745 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 5746 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5747 WRITE_DATA_DST_SEL(0)));
5748 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5749 radeon_ring_write(ring, 0);
faffaf62 5750 radeon_ring_write(ring, 1 << vm_id);
f96ab484 5751
3a01fd36
AD		 5752 /* wait for the invalidate to complete */
5753 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5754 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
5755 WAIT_REG_MEM_FUNCTION(0) | /* always */
5756 WAIT_REG_MEM_ENGINE(0))); /* me */
5757 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5758 radeon_ring_write(ring, 0);
5759 radeon_ring_write(ring, 0); /* ref */
5760 radeon_ring_write(ring, 0); /* mask */
5761 radeon_ring_write(ring, 0x20); /* poll interval */
5762
b07fdd38 5763 /* compute doesn't have PFP */
f1d2a26b 5764 if (usepfp) {
b07fdd38
AD		 5765 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5766 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5767 radeon_ring_write(ring, 0x0);
5768 }
cc066715 5769}
6f2043ce 5770
f6796cae
AD		 5771/*
5772 * RLC
5773 * The RLC is a multi-purpose microengine that handles a
5774 * variety of functions, the most important of which is
5775 * the interrupt controller.
5776 */
866d83de
AD		 5777static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5778 bool enable)
f6796cae 5779{
866d83de 5780 u32 tmp = RREG32(CP_INT_CNTL_RING0);
f6796cae 5781
866d83de
AD		 5782 if (enable)
5783 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5784 else
5785 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
f6796cae 5786 WREG32(CP_INT_CNTL_RING0, tmp);
866d83de 5787}
f6796cae 5788
866d83de 5789static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
cc066715 5790{
cc066715 5791 u32 tmp;
6f2043ce 5792
866d83de
AD		 5793 tmp = RREG32(RLC_LB_CNTL);
5794 if (enable)
5795 tmp |= LOAD_BALANCE_ENABLE;
5796 else
5797 tmp &= ~LOAD_BALANCE_ENABLE;
5798 WREG32(RLC_LB_CNTL, tmp);
5799}
cc066715 5800
866d83de
AD		 5801static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5802{
5803 u32 i, j, k;
5804 u32 mask;
cc066715 5805
1c0a4625 5806 mutex_lock(&rdev->grbm_idx_mutex);
f6796cae
AD		 5807 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5808 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5809 cik_select_se_sh(rdev, i, j);
5810 for (k = 0; k < rdev->usec_timeout; k++) {
5811 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5812 break;
5813 udelay(1);
5814 }
5815 }
5816 }
5817 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 5818 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 5819
f6796cae
AD		 5820 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5821 for (k = 0; k < rdev->usec_timeout; k++) {
5822 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5823 break;
5824 udelay(1);
5825 }
5826}
cc066715 5827
22c775ce
AD		 5828static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5829{
5830 u32 tmp;
cc066715 5831
22c775ce
AD		 5832 tmp = RREG32(RLC_CNTL);
5833 if (tmp != rlc)
5834 WREG32(RLC_CNTL, rlc);
5835}
cc066715 5836
22c775ce
AD		 5837static u32 cik_halt_rlc(struct radeon_device *rdev)
5838{
5839 u32 data, orig;
cc066715 5840
22c775ce 5841 orig = data = RREG32(RLC_CNTL);
cc066715 5842
22c775ce
AD		 5843 if (data & RLC_ENABLE) {
5844 u32 i;
cc066715 5845
22c775ce
AD		 5846 data &= ~RLC_ENABLE;
5847 WREG32(RLC_CNTL, data);
cc066715 5848
22c775ce
AD		 5849 for (i = 0; i < rdev->usec_timeout; i++) {
5850 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5851 break;
5852 udelay(1);
5853 }
cc066715 5854
22c775ce
AD		 5855 cik_wait_for_rlc_serdes(rdev);
5856 }
cc066715 5857
22c775ce
AD		 5858 return orig;
5859}
cc066715 5860
a412fce0
AD		 5861void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5862{
5863 u32 tmp, i, mask;
5864
5865 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5866 WREG32(RLC_GPR_REG2, tmp);
5867
5868 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5869 for (i = 0; i < rdev->usec_timeout; i++) {
5870 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5871 break;
5872 udelay(1);
5873 }
5874
5875 for (i = 0; i < rdev->usec_timeout; i++) {
5876 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5877 break;
5878 udelay(1);
5879 }
5880}
5881
5882void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5883{
5884 u32 tmp;
5885
5886 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5887 WREG32(RLC_GPR_REG2, tmp);
5888}
5889
866d83de
AD		 5890/**
5891 * cik_rlc_stop - stop the RLC ME
5892 *
5893 * @rdev: radeon_device pointer
5894 *
5895 * Halt the RLC ME (MicroEngine) (CIK).
5896 */
5897static void cik_rlc_stop(struct radeon_device *rdev)
5898{
22c775ce 5899 WREG32(RLC_CNTL, 0);
866d83de
AD		 5900
5901 cik_enable_gui_idle_interrupt(rdev, false);
5902
866d83de
AD		 5903 cik_wait_for_rlc_serdes(rdev);
5904}
5905
f6796cae
AD		 5906/**
5907 * cik_rlc_start - start the RLC ME
5908 *
5909 * @rdev: radeon_device pointer
5910 *
5911 * Unhalt the RLC ME (MicroEngine) (CIK).
5912 */
5913static void cik_rlc_start(struct radeon_device *rdev)
5914{
f6796cae 5915 WREG32(RLC_CNTL, RLC_ENABLE);
cc066715 5916
866d83de 5917 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 5918
f6796cae 5919 udelay(50);
6f2043ce
AD		 5920}
5921
5922/**
f6796cae 5923 * cik_rlc_resume - setup the RLC hw
6f2043ce
AD		 5924 *
5925 * @rdev: radeon_device pointer
5926 *
f6796cae
AD		 5927 * Initialize the RLC registers, load the ucode,
5928 * and start the RLC (CIK).
5929 * Returns 0 for success, -EINVAL if the ucode is not available.
6f2043ce 5930 */
f6796cae 5931static int cik_rlc_resume(struct radeon_device *rdev)
6f2043ce 5932{
22c775ce 5933 u32 i, size, tmp;
cc066715 5934
f6796cae
AD		 5935 if (!rdev->rlc_fw)
5936 return -EINVAL;
cc066715 5937
cc066715
AD		 5938 cik_rlc_stop(rdev);
5939
22c775ce
AD		 5940 /* disable CG */
5941 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5942 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
cc066715 5943
866d83de 5944 si_rlc_reset(rdev);
6f2043ce 5945
22c775ce 5946 cik_init_pg(rdev);
6f2043ce 5947
22c775ce 5948 cik_init_cg(rdev);
cc066715 5949
f6796cae
AD		 5950 WREG32(RLC_LB_CNTR_INIT, 0);
5951 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
cc066715 5952
1c0a4625 5953 mutex_lock(&rdev->grbm_idx_mutex);
f6796cae
AD		 5954 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5955 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5956 WREG32(RLC_LB_PARAMS, 0x00600408);
5957 WREG32(RLC_LB_CNTL, 0x80000004);
1c0a4625 5958 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 5959
f6796cae
AD		 5960 WREG32(RLC_MC_CNTL, 0);
5961 WREG32(RLC_UCODE_CNTL, 0);
cc066715 5962
f2c6b0f4
AD		 5963 if (rdev->new_fw) {
5964 const struct rlc_firmware_header_v1_0 *hdr =
5965 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5966 const __le32 *fw_data = (const __le32 *)
5967 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5968
5969 radeon_ucode_print_rlc_hdr(&hdr->header);
5970
5971 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5972 WREG32(RLC_GPM_UCODE_ADDR, 0);
5973 for (i = 0; i < size; i++)
5974 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 5975 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version));
f2c6b0f4
AD		 5976 } else {
5977 const __be32 *fw_data;
5978
5979 switch (rdev->family) {
5980 case CHIP_BONAIRE:
5981 case CHIP_HAWAII:
5982 default:
5983 size = BONAIRE_RLC_UCODE_SIZE;
5984 break;
5985 case CHIP_KAVERI:
5986 size = KV_RLC_UCODE_SIZE;
5987 break;
5988 case CHIP_KABINI:
5989 size = KB_RLC_UCODE_SIZE;
5990 break;
5991 case CHIP_MULLINS:
5992 size = ML_RLC_UCODE_SIZE;
5993 break;
5994 }
5995
5996 fw_data = (const __be32 *)rdev->rlc_fw->data;
5997 WREG32(RLC_GPM_UCODE_ADDR, 0);
5998 for (i = 0; i < size; i++)
5999 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6000 WREG32(RLC_GPM_UCODE_ADDR, 0);
6001 }
cc066715 6002
866d83de
AD		 6003 /* XXX - find out what chips support lbpw */
6004 cik_enable_lbpw(rdev, false);
cc066715 6005
22c775ce
AD		 6006 if (rdev->family == CHIP_BONAIRE)
6007 WREG32(RLC_DRIVER_DMA_STATUS, 0);
cc066715 6008
f6796cae 6009 cik_rlc_start(rdev);
cc066715 6010
f6796cae
AD		 6011 return 0;
6012}
cc066715 6013
22c775ce
AD		 6014static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6015{
6016 u32 data, orig, tmp, tmp2;
cc066715 6017
22c775ce 6018 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
cc066715 6019
473359bc 6020 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
ddc76ff6 6021 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 6022
22c775ce 6023 tmp = cik_halt_rlc(rdev);
cc066715 6024
1c0a4625 6025 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6026 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6027 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6028 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6029 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6030 WREG32(RLC_SERDES_WR_CTRL, tmp2);
1c0a4625 6031 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 6032
22c775ce 6033 cik_update_rlc(rdev, tmp);
cc066715 6034
22c775ce
AD		 6035 data |= CGCG_EN | CGLS_EN;
6036 } else {
ddc76ff6 6037 cik_enable_gui_idle_interrupt(rdev, false);
cc066715 6038
22c775ce
AD		 6039 RREG32(CB_CGTT_SCLK_CTRL);
6040 RREG32(CB_CGTT_SCLK_CTRL);
6041 RREG32(CB_CGTT_SCLK_CTRL);
6042 RREG32(CB_CGTT_SCLK_CTRL);
cc066715 6043
22c775ce 6044 data &= ~(CGCG_EN | CGLS_EN);
cc066715 6045 }
6f2043ce 6046
22c775ce
AD		 6047 if (orig != data)
6048 WREG32(RLC_CGCG_CGLS_CTRL, data);
cc066715 6049
6f2043ce
AD		 6050}
6051
22c775ce 6052static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6f2043ce 6053{
22c775ce
AD		 6054 u32 data, orig, tmp = 0;
6055
473359bc
AD		 6056 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6057 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6058 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6059 orig = data = RREG32(CP_MEM_SLP_CNTL);
6060 data |= CP_MEM_LS_EN;
6061 if (orig != data)
6062 WREG32(CP_MEM_SLP_CNTL, data);
6063 }
6064 }
cc066715 6065
22c775ce 6066 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4bb62c95 6067 data |= 0x00000001;
22c775ce
AD		 6068 data &= 0xfffffffd;
6069 if (orig != data)
6070 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6071
6072 tmp = cik_halt_rlc(rdev);
6073
1c0a4625 6074 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6075 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6076 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6077 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6078 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6079 WREG32(RLC_SERDES_WR_CTRL, data);
1c0a4625 6080 mutex_unlock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6081
6082 cik_update_rlc(rdev, tmp);
6083
473359bc
AD		 6084 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6085 orig = data = RREG32(CGTS_SM_CTRL_REG);
6086 data &= ~SM_MODE_MASK;
6087 data |= SM_MODE(0x2);
6088 data |= SM_MODE_ENABLE;
6089 data &= ~CGTS_OVERRIDE;
6090 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6091 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6092 data &= ~CGTS_LS_OVERRIDE;
6093 data &= ~ON_MONITOR_ADD_MASK;
6094 data |= ON_MONITOR_ADD_EN;
6095 data |= ON_MONITOR_ADD(0x96);
6096 if (orig != data)
6097 WREG32(CGTS_SM_CTRL_REG, data);
6098 }
22c775ce
AD		 6099 } else {
6100 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4bb62c95 6101 data |= 0x00000003;
22c775ce
AD		 6102 if (orig != data)
6103 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6104
6105 data = RREG32(RLC_MEM_SLP_CNTL);
6106 if (data & RLC_MEM_LS_EN) {
6107 data &= ~RLC_MEM_LS_EN;
6108 WREG32(RLC_MEM_SLP_CNTL, data);
6109 }
6f2043ce 6110
22c775ce
AD		 6111 data = RREG32(CP_MEM_SLP_CNTL);
6112 if (data & CP_MEM_LS_EN) {
6113 data &= ~CP_MEM_LS_EN;
6114 WREG32(CP_MEM_SLP_CNTL, data);
6115 }
cc066715 6116
22c775ce
AD		 6117 orig = data = RREG32(CGTS_SM_CTRL_REG);
6118 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6119 if (orig != data)
6120 WREG32(CGTS_SM_CTRL_REG, data);
cc066715 6121
22c775ce 6122 tmp = cik_halt_rlc(rdev);
cc066715 6123
1c0a4625 6124 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6125 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6126 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6127 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6128 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6129 WREG32(RLC_SERDES_WR_CTRL, data);
1c0a4625 6130 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 6131
22c775ce 6132 cik_update_rlc(rdev, tmp);
cc066715 6133 }
6f2043ce 6134}
1c49165d 6135
22c775ce 6136static const u32 mc_cg_registers[] =
21a93e13 6137{
22c775ce
AD		 6138 MC_HUB_MISC_HUB_CG,
6139 MC_HUB_MISC_SIP_CG,
6140 MC_HUB_MISC_VM_CG,
6141 MC_XPB_CLK_GAT,
6142 ATC_MISC_CG,
6143 MC_CITF_MISC_WR_CG,
6144 MC_CITF_MISC_RD_CG,
6145 MC_CITF_MISC_VM_CG,
6146 VM_L2_CG,
6147};
21a93e13 6148
22c775ce
AD		 6149static void cik_enable_mc_ls(struct radeon_device *rdev,
6150 bool enable)
1c49165d 6151{
22c775ce
AD		 6152 int i;
6153 u32 orig, data;
1c49165d 6154
22c775ce
AD		 6155 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6156 orig = data = RREG32(mc_cg_registers[i]);
473359bc 6157 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
22c775ce
AD		 6158 data |= MC_LS_ENABLE;
6159 else
6160 data &= ~MC_LS_ENABLE;
6161 if (data != orig)
6162 WREG32(mc_cg_registers[i], data);
1c49165d 6163 }
22c775ce 6164}
1c49165d 6165
22c775ce
AD		 6166static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6167 bool enable)
6168{
6169 int i;
6170 u32 orig, data;
6171
6172 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6173 orig = data = RREG32(mc_cg_registers[i]);
473359bc 6174 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
22c775ce
AD		 6175 data |= MC_CG_ENABLE;
6176 else
6177 data &= ~MC_CG_ENABLE;
6178 if (data != orig)
6179 WREG32(mc_cg_registers[i], data);
1c49165d 6180 }
1c49165d
AD		 6181}
6182
22c775ce
AD		 6183static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6184 bool enable)
1c49165d 6185{
22c775ce 6186 u32 orig, data;
1c49165d 6187
473359bc 6188 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
22c775ce
AD		 6189 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6190 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
1c49165d 6191 } else {
22c775ce
AD		 6192 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6193 data |= 0xff000000;
6194 if (data != orig)
6195 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
1c49165d 6196
22c775ce
AD		 6197 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6198 data |= 0xff000000;
6199 if (data != orig)
6200 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6201 }
1c49165d
AD		 6202}
6203
22c775ce
AD		 6204static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6205 bool enable)
1c49165d 6206{
22c775ce
AD		 6207 u32 orig, data;
6208
473359bc 6209 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
22c775ce
AD		 6210 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6211 data |= 0x100;
6212 if (orig != data)
6213 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6214
6215 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6216 data |= 0x100;
6217 if (orig != data)
6218 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6219 } else {
6220 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6221 data &= ~0x100;
6222 if (orig != data)
6223 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
1c49165d 6224
22c775ce
AD		 6225 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6226 data &= ~0x100;
6227 if (orig != data)
6228 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6229 }
1c49165d
AD		 6230}
6231
22c775ce
AD		 6232static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6233 bool enable)
1c49165d 6234{
22c775ce 6235 u32 orig, data;
1c49165d 6236
473359bc 6237 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
22c775ce
AD		 6238 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6239 data = 0xfff;
6240 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6241
22c775ce
AD		 6242 orig = data = RREG32(UVD_CGC_CTRL);
6243 data |= DCM;
6244 if (orig != data)
6245 WREG32(UVD_CGC_CTRL, data);
6246 } else {
6247 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6248 data &= ~0xfff;
6249 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6250
22c775ce
AD		 6251 orig = data = RREG32(UVD_CGC_CTRL);
6252 data &= ~DCM;
6253 if (orig != data)
6254 WREG32(UVD_CGC_CTRL, data);
1c49165d 6255 }
22c775ce 6256}
1c49165d 6257
473359bc
AD		 6258static void cik_enable_bif_mgls(struct radeon_device *rdev,
6259 bool enable)
6260{
6261 u32 orig, data;
1c49165d 6262
473359bc 6263 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
1c49165d 6264
473359bc
AD		 6265 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6266 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6267 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6268 else
6269 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6270 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
1c49165d 6271
473359bc
AD		 6272 if (orig != data)
6273 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6274}
1c49165d 6275
22c775ce
AD		 6276static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6277 bool enable)
6278{
6279 u32 orig, data;
1c49165d 6280
22c775ce 6281 orig = data = RREG32(HDP_HOST_PATH_CNTL);
1c49165d 6282
473359bc 6283 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
22c775ce
AD		 6284 data &= ~CLOCK_GATING_DIS;
6285 else
6286 data |= CLOCK_GATING_DIS;
6287
6288 if (orig != data)
6289 WREG32(HDP_HOST_PATH_CNTL, data);
1c49165d
AD		 6290}
6291
22c775ce
AD		 6292static void cik_enable_hdp_ls(struct radeon_device *rdev,
6293 bool enable)
1c49165d 6294{
22c775ce
AD		 6295 u32 orig, data;
6296
6297 orig = data = RREG32(HDP_MEM_POWER_LS);
6298
473359bc 6299 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
22c775ce
AD		 6300 data |= HDP_LS_ENABLE;
6301 else
6302 data &= ~HDP_LS_ENABLE;
6303
6304 if (orig != data)
6305 WREG32(HDP_MEM_POWER_LS, data);
6306}
6307
6308void cik_update_cg(struct radeon_device *rdev,
6309 u32 block, bool enable)
6310{
4214faf6 6311
22c775ce 6312 if (block & RADEON_CG_BLOCK_GFX) {
4214faf6 6313 cik_enable_gui_idle_interrupt(rdev, false);
22c775ce
AD		 6314 /* order matters! */
6315 if (enable) {
6316 cik_enable_mgcg(rdev, true);
6317 cik_enable_cgcg(rdev, true);
6318 } else {
6319 cik_enable_cgcg(rdev, false);
6320 cik_enable_mgcg(rdev, false);
6321 }
4214faf6 6322 cik_enable_gui_idle_interrupt(rdev, true);
22c775ce
AD		 6323 }
6324
6325 if (block & RADEON_CG_BLOCK_MC) {
6326 if (!(rdev->flags & RADEON_IS_IGP)) {
6327 cik_enable_mc_mgcg(rdev, enable);
6328 cik_enable_mc_ls(rdev, enable);
6329 }
6330 }
6331
6332 if (block & RADEON_CG_BLOCK_SDMA) {
6333 cik_enable_sdma_mgcg(rdev, enable);
6334 cik_enable_sdma_mgls(rdev, enable);
6335 }
6336
473359bc
AD		 6337 if (block & RADEON_CG_BLOCK_BIF) {
6338 cik_enable_bif_mgls(rdev, enable);
6339 }
6340
22c775ce
AD		 6341 if (block & RADEON_CG_BLOCK_UVD) {
6342 if (rdev->has_uvd)
6343 cik_enable_uvd_mgcg(rdev, enable);
6344 }
6345
6346 if (block & RADEON_CG_BLOCK_HDP) {
6347 cik_enable_hdp_mgcg(rdev, enable);
6348 cik_enable_hdp_ls(rdev, enable);
6349 }
a1d6f97c
AD		 6350
6351 if (block & RADEON_CG_BLOCK_VCE) {
6352 vce_v2_0_enable_mgcg(rdev, enable);
6353 }
1c49165d
AD		 6354}
6355
22c775ce 6356static void cik_init_cg(struct radeon_device *rdev)
1c49165d 6357{
22c775ce 6358
ddc76ff6 6359 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
22c775ce
AD		 6360
6361 if (rdev->has_uvd)
6362 si_init_uvd_internal_cg(rdev);
6363
6364 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6365 RADEON_CG_BLOCK_SDMA |
473359bc 6366 RADEON_CG_BLOCK_BIF |
22c775ce
AD		 6367 RADEON_CG_BLOCK_UVD |
6368 RADEON_CG_BLOCK_HDP), true);
1c49165d
AD		 6369}
6370
473359bc 6371static void cik_fini_cg(struct radeon_device *rdev)
1c49165d 6372{
473359bc
AD		 6373 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6374 RADEON_CG_BLOCK_SDMA |
6375 RADEON_CG_BLOCK_BIF |
6376 RADEON_CG_BLOCK_UVD |
6377 RADEON_CG_BLOCK_HDP), false);
6378
6379 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
1c49165d
AD		 6380}
6381
22c775ce
AD		 6382static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6383 bool enable)
1c49165d 6384{
22c775ce 6385 u32 data, orig;
1c49165d 6386
22c775ce 6387 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6388 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6389 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6390 else
6391 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6392 if (orig != data)
6393 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6394}
6395
22c775ce
AD		 6396static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6397 bool enable)
1c49165d 6398{
22c775ce
AD		 6399 u32 data, orig;
6400
6401 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6402 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6403 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6404 else
6405 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6406 if (orig != data)
6407 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6408}
6409
22c775ce 6410static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
3ec7d11b 6411{
22c775ce 6412 u32 data, orig;
3ec7d11b 6413
22c775ce 6414 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6415 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
22c775ce
AD		 6416 data &= ~DISABLE_CP_PG;
6417 else
6418 data |= DISABLE_CP_PG;
6419 if (orig != data)
6420 WREG32(RLC_PG_CNTL, data);
3ec7d11b
AD		 6421}
6422
22c775ce 6423static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
f96ab484 6424{
22c775ce 6425 u32 data, orig;
f96ab484 6426
22c775ce 6427 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6428 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
22c775ce
AD		 6429 data &= ~DISABLE_GDS_PG;
6430 else
6431 data |= DISABLE_GDS_PG;
6432 if (orig != data)
6433 WREG32(RLC_PG_CNTL, data);
6434}
6435
6436#define CP_ME_TABLE_SIZE 96
6437#define CP_ME_TABLE_OFFSET 2048
6438#define CP_MEC_TABLE_OFFSET 4096
6439
6440void cik_init_cp_pg_table(struct radeon_device *rdev)
6441{
22c775ce
AD		 6442 volatile u32 *dst_ptr;
6443 int me, i, max_me = 4;
6444 u32 bo_offset = 0;
f2c6b0f4 6445 u32 table_offset, table_size;
22c775ce
AD		 6446
6447 if (rdev->family == CHIP_KAVERI)
6448 max_me = 5;
6449
6450 if (rdev->rlc.cp_table_ptr == NULL)
f96ab484
AD		 6451 return;
6452
22c775ce
AD		 6453 /* write the cp table buffer */
6454 dst_ptr = rdev->rlc.cp_table_ptr;
6455 for (me = 0; me < max_me; me++) {
f2c6b0f4
AD		 6456 if (rdev->new_fw) {
6457 const __le32 *fw_data;
6458 const struct gfx_firmware_header_v1_0 *hdr;
6459
6460 if (me == 0) {
6461 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6462 fw_data = (const __le32 *)
6463 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6464 table_offset = le32_to_cpu(hdr->jt_offset);
6465 table_size = le32_to_cpu(hdr->jt_size);
6466 } else if (me == 1) {
6467 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6468 fw_data = (const __le32 *)
6469 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6470 table_offset = le32_to_cpu(hdr->jt_offset);
6471 table_size = le32_to_cpu(hdr->jt_size);
6472 } else if (me == 2) {
6473 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6474 fw_data = (const __le32 *)
6475 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6476 table_offset = le32_to_cpu(hdr->jt_offset);
6477 table_size = le32_to_cpu(hdr->jt_size);
6478 } else if (me == 3) {
6479 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6480 fw_data = (const __le32 *)
6481 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6482 table_offset = le32_to_cpu(hdr->jt_offset);
6483 table_size = le32_to_cpu(hdr->jt_size);
6484 } else {
6485 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6486 fw_data = (const __le32 *)
6487 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6488 table_offset = le32_to_cpu(hdr->jt_offset);
6489 table_size = le32_to_cpu(hdr->jt_size);
6490 }
6491
6492 for (i = 0; i < table_size; i ++) {
6493 dst_ptr[bo_offset + i] =
6494 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6495 }
6496 bo_offset += table_size;
22c775ce 6497 } else {
f2c6b0f4
AD		 6498 const __be32 *fw_data;
6499 table_size = CP_ME_TABLE_SIZE;
6500
6501 if (me == 0) {
6502 fw_data = (const __be32 *)rdev->ce_fw->data;
6503 table_offset = CP_ME_TABLE_OFFSET;
6504 } else if (me == 1) {
6505 fw_data = (const __be32 *)rdev->pfp_fw->data;
6506 table_offset = CP_ME_TABLE_OFFSET;
6507 } else if (me == 2) {
6508 fw_data = (const __be32 *)rdev->me_fw->data;
6509 table_offset = CP_ME_TABLE_OFFSET;
6510 } else {
6511 fw_data = (const __be32 *)rdev->mec_fw->data;
6512 table_offset = CP_MEC_TABLE_OFFSET;
6513 }
22c775ce 6514
f2c6b0f4
AD		 6515 for (i = 0; i < table_size; i ++) {
6516 dst_ptr[bo_offset + i] =
6517 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6518 }
6519 bo_offset += table_size;
22c775ce 6520 }
f96ab484 6521 }
22c775ce 6522}
f96ab484 6523
22c775ce
AD		 6524static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6525 bool enable)
6526{
6527 u32 data, orig;
6528
2b19d17f 6529 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
22c775ce
AD		 6530 orig = data = RREG32(RLC_PG_CNTL);
6531 data |= GFX_PG_ENABLE;
6532 if (orig != data)
6533 WREG32(RLC_PG_CNTL, data);
6534
6535 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6536 data |= AUTO_PG_EN;
6537 if (orig != data)
6538 WREG32(RLC_AUTO_PG_CTRL, data);
6539 } else {
6540 orig = data = RREG32(RLC_PG_CNTL);
6541 data &= ~GFX_PG_ENABLE;
6542 if (orig != data)
6543 WREG32(RLC_PG_CNTL, data);
f96ab484 6544
22c775ce
AD		 6545 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6546 data &= ~AUTO_PG_EN;
6547 if (orig != data)
6548 WREG32(RLC_AUTO_PG_CTRL, data);
f96ab484 6549
22c775ce
AD		 6550 data = RREG32(DB_RENDER_CONTROL);
6551 }
6552}
f96ab484 6553
22c775ce
AD		 6554static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6555{
6556 u32 mask = 0, tmp, tmp1;
6557 int i;
f96ab484 6558
1c0a4625 6559 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6560 cik_select_se_sh(rdev, se, sh);
6561 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6562 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6563 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 6564 mutex_unlock(&rdev->grbm_idx_mutex);
f96ab484 6565
22c775ce 6566 tmp &= 0xffff0000;
f96ab484 6567
22c775ce
AD		 6568 tmp |= tmp1;
6569 tmp >>= 16;
6570
6571 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6572 mask <<= 1;
6573 mask |= 1;
b07fdd38 6574 }
22c775ce
AD		 6575
6576 return (~tmp) & mask;
f96ab484
AD		 6577}
6578
22c775ce 6579static void cik_init_ao_cu_mask(struct radeon_device *rdev)
d0e092d9 6580{
22c775ce
AD		 6581 u32 i, j, k, active_cu_number = 0;
6582 u32 mask, counter, cu_bitmap;
6583 u32 tmp = 0;
d0e092d9 6584
22c775ce
AD		 6585 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6586 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6587 mask = 1;
6588 cu_bitmap = 0;
6589 counter = 0;
6590 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6591 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6592 if (counter < 2)
6593 cu_bitmap |= mask;
6594 counter ++;
d0e092d9 6595 }
22c775ce 6596 mask <<= 1;
d0e092d9 6597 }
d0e092d9 6598
22c775ce
AD		 6599 active_cu_number += counter;
6600 tmp |= (cu_bitmap << (i * 16 + j * 8));
d0e092d9 6601 }
d0e092d9 6602 }
22c775ce
AD		 6603
6604 WREG32(RLC_PG_AO_CU_MASK, tmp);
6605
6606 tmp = RREG32(RLC_MAX_PG_CU);
6607 tmp &= ~MAX_PU_CU_MASK;
6608 tmp |= MAX_PU_CU(active_cu_number);
6609 WREG32(RLC_MAX_PG_CU, tmp);
d0e092d9
AD		 6610}
6611
22c775ce
AD		 6612static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6613 bool enable)
605de6b9 6614{
22c775ce 6615 u32 data, orig;
605de6b9 6616
22c775ce 6617 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6618 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
22c775ce
AD		 6619 data |= STATIC_PER_CU_PG_ENABLE;
6620 else
6621 data &= ~STATIC_PER_CU_PG_ENABLE;
6622 if (orig != data)
6623 WREG32(RLC_PG_CNTL, data);
6624}
6625
6626static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6627 bool enable)
6628{
6629 u32 data, orig;
605de6b9 6630
22c775ce 6631 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6632 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
22c775ce 6633 data |= DYN_PER_CU_PG_ENABLE;
605de6b9 6634 else
22c775ce
AD		 6635 data &= ~DYN_PER_CU_PG_ENABLE;
6636 if (orig != data)
6637 WREG32(RLC_PG_CNTL, data);
6638}
605de6b9 6639
22c775ce
AD		 6640#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6641#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6642
6643static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6644{
6645 u32 data, orig;
6646 u32 i;
6647
6648 if (rdev->rlc.cs_data) {
6649 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6650 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
a0f38609 6651 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
22c775ce 6652 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
605de6b9 6653 } else {
22c775ce
AD		 6654 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6655 for (i = 0; i < 3; i++)
6656 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6657 }
6658 if (rdev->rlc.reg_list) {
6659 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6660 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6661 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
605de6b9 6662 }
605de6b9 6663
22c775ce
AD		 6664 orig = data = RREG32(RLC_PG_CNTL);
6665 data |= GFX_PG_SRC;
6666 if (orig != data)
6667 WREG32(RLC_PG_CNTL, data);
605de6b9 6668
22c775ce
AD		 6669 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6670 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
605de6b9 6671
22c775ce
AD		 6672 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6673 data &= ~IDLE_POLL_COUNT_MASK;
6674 data |= IDLE_POLL_COUNT(0x60);
6675 WREG32(CP_RB_WPTR_POLL_CNTL, data);
605de6b9 6676
22c775ce
AD		 6677 data = 0x10101010;
6678 WREG32(RLC_PG_DELAY, data);
605de6b9 6679
22c775ce
AD		 6680 data = RREG32(RLC_PG_DELAY_2);
6681 data &= ~0xff;
6682 data |= 0x3;
6683 WREG32(RLC_PG_DELAY_2, data);
605de6b9 6684
22c775ce
AD		 6685 data = RREG32(RLC_AUTO_PG_CTRL);
6686 data &= ~GRBM_REG_SGIT_MASK;
6687 data |= GRBM_REG_SGIT(0x700);
6688 WREG32(RLC_AUTO_PG_CTRL, data);
605de6b9 6689
605de6b9
AD		 6690}
6691
22c775ce 6692static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
f6796cae 6693{
473359bc
AD		 6694 cik_enable_gfx_cgpg(rdev, enable);
6695 cik_enable_gfx_static_mgpg(rdev, enable);
6696 cik_enable_gfx_dynamic_mgpg(rdev, enable);
22c775ce 6697}
f6796cae 6698
a0f38609
AD		 6699u32 cik_get_csb_size(struct radeon_device *rdev)
6700{
6701 u32 count = 0;
6702 const struct cs_section_def *sect = NULL;
6703 const struct cs_extent_def *ext = NULL;
f6796cae 6704
a0f38609
AD		 6705 if (rdev->rlc.cs_data == NULL)
6706 return 0;
f6796cae 6707
a0f38609
AD		 6708 /* begin clear state */
6709 count += 2;
6710 /* context control state */
6711 count += 3;
6712
6713 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6714 for (ext = sect->section; ext->extent != NULL; ++ext) {
6715 if (sect->id == SECT_CONTEXT)
6716 count += 2 + ext->reg_count;
6717 else
6718 return 0;
f6796cae
AD		 6719 }
6720 }
a0f38609
AD		 6721 /* pa_sc_raster_config/pa_sc_raster_config1 */
6722 count += 4;
6723 /* end clear state */
6724 count += 2;
6725 /* clear state */
6726 count += 2;
f6796cae 6727
a0f38609 6728 return count;
f6796cae
AD		 6729}
6730
a0f38609 6731void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
f6796cae 6732{
a0f38609
AD		 6733 u32 count = 0, i;
6734 const struct cs_section_def *sect = NULL;
6735 const struct cs_extent_def *ext = NULL;
f6796cae 6736
a0f38609
AD		 6737 if (rdev->rlc.cs_data == NULL)
6738 return;
6739 if (buffer == NULL)
6740 return;
f6796cae 6741
6ba81e53
AD		 6742 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6743 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
a0f38609 6744
6ba81e53
AD		 6745 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6746 buffer[count++] = cpu_to_le32(0x80000000);
6747 buffer[count++] = cpu_to_le32(0x80000000);
a0f38609
AD		 6748
6749 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6750 for (ext = sect->section; ext->extent != NULL; ++ext) {
6751 if (sect->id == SECT_CONTEXT) {
6ba81e53
AD		 6752 buffer[count++] =
6753 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6754 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
a0f38609 6755 for (i = 0; i < ext->reg_count; i++)
6ba81e53 6756 buffer[count++] = cpu_to_le32(ext->extent[i]);
a0f38609
AD		 6757 } else {
6758 return;
6759 }
6760 }
6761 }
f6796cae 6762
6ba81e53
AD		 6763 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6764 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
f6796cae
AD		 6765 switch (rdev->family) {
6766 case CHIP_BONAIRE:
6ba81e53
AD		 6767 buffer[count++] = cpu_to_le32(0x16000012);
6768 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6769 break;
6770 case CHIP_KAVERI:
6ba81e53
AD		 6771 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6772 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6773 break;
6774 case CHIP_KABINI:
f73a9e83 6775 case CHIP_MULLINS:
6ba81e53
AD		 6776 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6777 buffer[count++] = cpu_to_le32(0x00000000);
a0f38609 6778 break;
bbfe90bd 6779 case CHIP_HAWAII:
a8947f57
AD		 6780 buffer[count++] = cpu_to_le32(0x3a00161a);
6781 buffer[count++] = cpu_to_le32(0x0000002e);
bbfe90bd 6782 break;
a0f38609 6783 default:
6ba81e53
AD		 6784 buffer[count++] = cpu_to_le32(0x00000000);
6785 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6786 break;
6787 }
6788
6ba81e53
AD		 6789 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6790 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
f6796cae 6791
6ba81e53
AD		 6792 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6793 buffer[count++] = cpu_to_le32(0);
a0f38609 6794}
f6796cae 6795
473359bc 6796static void cik_init_pg(struct radeon_device *rdev)
22c775ce 6797{
473359bc 6798 if (rdev->pg_flags) {
22c775ce
AD		 6799 cik_enable_sck_slowdown_on_pu(rdev, true);
6800 cik_enable_sck_slowdown_on_pd(rdev, true);
2b19d17f 6801 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 6802 cik_init_gfx_cgpg(rdev);
6803 cik_enable_cp_pg(rdev, true);
6804 cik_enable_gds_pg(rdev, true);
6805 }
22c775ce
AD		 6806 cik_init_ao_cu_mask(rdev);
6807 cik_update_gfx_pg(rdev, true);
6808 }
6809}
f6796cae 6810
473359bc
AD		 6811static void cik_fini_pg(struct radeon_device *rdev)
6812{
6813 if (rdev->pg_flags) {
6814 cik_update_gfx_pg(rdev, false);
2b19d17f 6815 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 6816 cik_enable_cp_pg(rdev, false);
6817 cik_enable_gds_pg(rdev, false);
6818 }
6819 }
f6796cae 6820}
a59781bb
AD		 6821
6822/*
6823 * Interrupts
6824 * Starting with r6xx, interrupts are handled via a ring buffer.
6825 * Ring buffers are areas of GPU accessible memory that the GPU
6826 * writes interrupt vectors into and the host reads vectors out of.
6827 * There is a rptr (read pointer) that determines where the
6828 * host is currently reading, and a wptr (write pointer)
6829 * which determines where the GPU has written. When the
6830 * pointers are equal, the ring is idle. When the GPU
6831 * writes vectors to the ring buffer, it increments the
6832 * wptr. When there is an interrupt, the host then starts
6833 * fetching commands and processing them until the pointers are
6834 * equal again at which point it updates the rptr.
6835 */
6836
6837/**
6838 * cik_enable_interrupts - Enable the interrupt ring buffer
6839 *
6840 * @rdev: radeon_device pointer
6841 *
6842 * Enable the interrupt ring buffer (CIK).
6843 */
6844static void cik_enable_interrupts(struct radeon_device *rdev)
6845{
6846 u32 ih_cntl = RREG32(IH_CNTL);
6847 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6848
6849 ih_cntl |= ENABLE_INTR;
6850 ih_rb_cntl |= IH_RB_ENABLE;
6851 WREG32(IH_CNTL, ih_cntl);
6852 WREG32(IH_RB_CNTL, ih_rb_cntl);
6853 rdev->ih.enabled = true;
6854}
6855
6856/**
6857 * cik_disable_interrupts - Disable the interrupt ring buffer
6858 *
6859 * @rdev: radeon_device pointer
6860 *
6861 * Disable the interrupt ring buffer (CIK).
6862 */
6863static void cik_disable_interrupts(struct radeon_device *rdev)
6864{
6865 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6866 u32 ih_cntl = RREG32(IH_CNTL);
6867
6868 ih_rb_cntl &= ~IH_RB_ENABLE;
6869 ih_cntl &= ~ENABLE_INTR;
6870 WREG32(IH_RB_CNTL, ih_rb_cntl);
6871 WREG32(IH_CNTL, ih_cntl);
6872 /* set rptr, wptr to 0 */
6873 WREG32(IH_RB_RPTR, 0);
6874 WREG32(IH_RB_WPTR, 0);
6875 rdev->ih.enabled = false;
6876 rdev->ih.rptr = 0;
6877}
6878
6879/**
6880 * cik_disable_interrupt_state - Disable all interrupt sources
6881 *
6882 * @rdev: radeon_device pointer
6883 *
6884 * Clear all interrupt enable bits used by the driver (CIK).
6885 */
6886static void cik_disable_interrupt_state(struct radeon_device *rdev)
6887{
6888 u32 tmp;
6889
6890 /* gfx ring */
4214faf6
AD		 6891 tmp = RREG32(CP_INT_CNTL_RING0) &
6892 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6893 WREG32(CP_INT_CNTL_RING0, tmp);
21a93e13
AD		 6894 /* sdma */
6895 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6896 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6897 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6898 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
a59781bb
AD		 6899 /* compute queues */
6900 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6901 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6902 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6903 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6904 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6905 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6906 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6907 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6908 /* grbm */
6909 WREG32(GRBM_INT_CNTL, 0);
dc12a3ec
LL		 6910 /* SRBM */
6911 WREG32(SRBM_INT_CNTL, 0);
a59781bb
AD		 6912 /* vline/vblank, etc. */
6913 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6914 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6915 if (rdev->num_crtc >= 4) {
6916 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6917 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6918 }
6919 if (rdev->num_crtc >= 6) {
6920 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6921 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6922 }
f5d636d2
CK		 6923 /* pflip */
6924 if (rdev->num_crtc >= 2) {
6925 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6926 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6927 }
6928 if (rdev->num_crtc >= 4) {
6929 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6930 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6931 }
6932 if (rdev->num_crtc >= 6) {
6933 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6934 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6935 }
a59781bb
AD		 6936
6937 /* dac hotplug */
6938 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6939
6940 /* digital hotplug */
6941 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6942 WREG32(DC_HPD1_INT_CONTROL, tmp);
6943 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6944 WREG32(DC_HPD2_INT_CONTROL, tmp);
6945 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6946 WREG32(DC_HPD3_INT_CONTROL, tmp);
6947 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6948 WREG32(DC_HPD4_INT_CONTROL, tmp);
6949 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6950 WREG32(DC_HPD5_INT_CONTROL, tmp);
6951 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6952 WREG32(DC_HPD6_INT_CONTROL, tmp);
6953
6954}
6955
6956/**
6957 * cik_irq_init - init and enable the interrupt ring
6958 *
6959 * @rdev: radeon_device pointer
6960 *
6961 * Allocate a ring buffer for the interrupt controller,
6962 * enable the RLC, disable interrupts, enable the IH
6963 * ring buffer and enable it (CIK).
6964 * Called at device load and reume.
6965 * Returns 0 for success, errors for failure.
6966 */
6967static int cik_irq_init(struct radeon_device *rdev)
6968{
6969 int ret = 0;
6970 int rb_bufsz;
6971 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6972
6973 /* allocate ring */
6974 ret = r600_ih_ring_alloc(rdev);
6975 if (ret)
6976 return ret;
6977
6978 /* disable irqs */
6979 cik_disable_interrupts(rdev);
6980
6981 /* init rlc */
6982 ret = cik_rlc_resume(rdev);
6983 if (ret) {
6984 r600_ih_ring_fini(rdev);
6985 return ret;
6986 }
6987
6988 /* setup interrupt control */
2e2b119a
SB		 6989 /* set dummy read address to dummy page address */
6990 WREG32(INTERRUPT_CNTL2, rdev->dummy_page.addr >> 8);
a59781bb
AD		 6991 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6992 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6993 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6994 */
6995 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6996 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6997 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6998 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6999
7000 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 7001 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
a59781bb
AD		 7002
7003 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7004 IH_WPTR_OVERFLOW_CLEAR |
7005 (rb_bufsz << 1));
7006
7007 if (rdev->wb.enabled)
7008 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7009
7010 /* set the writeback address whether it's enabled or not */
7011 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7012 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7013
7014 WREG32(IH_RB_CNTL, ih_rb_cntl);
7015
7016 /* set rptr, wptr to 0 */
7017 WREG32(IH_RB_RPTR, 0);
7018 WREG32(IH_RB_WPTR, 0);
7019
7020 /* Default settings for IH_CNTL (disabled at first) */
7021 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7022 /* RPTR_REARM only works if msi's are enabled */
7023 if (rdev->msi_enabled)
7024 ih_cntl |= RPTR_REARM;
7025 WREG32(IH_CNTL, ih_cntl);
7026
7027 /* force the active interrupt state to all disabled */
7028 cik_disable_interrupt_state(rdev);
7029
7030 pci_set_master(rdev->pdev);
7031
7032 /* enable irqs */
7033 cik_enable_interrupts(rdev);
7034
7035 return ret;
7036}
7037
7038/**
7039 * cik_irq_set - enable/disable interrupt sources
7040 *
7041 * @rdev: radeon_device pointer
7042 *
7043 * Enable interrupt sources on the GPU (vblanks, hpd,
7044 * etc.) (CIK).
7045 * Returns 0 for success, errors for failure.
7046 */
7047int cik_irq_set(struct radeon_device *rdev)
7048{
4214faf6 7049 u32 cp_int_cntl;
28b57b85 7050 u32 cp_m1p0;
a59781bb
AD		 7051 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7052 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7053 u32 grbm_int_cntl = 0;
21a93e13 7054 u32 dma_cntl, dma_cntl1;
a59781bb
AD		 7055
7056 if (!rdev->irq.installed) {
7057 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7058 return -EINVAL;
7059 }
7060 /* don't enable anything if the ih is disabled */
7061 if (!rdev->ih.enabled) {
7062 cik_disable_interrupts(rdev);
7063 /* force the active interrupt state to all disabled */
7064 cik_disable_interrupt_state(rdev);
7065 return 0;
7066 }
7067
4214faf6
AD		 7068 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7069 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7070 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7071
f6b355dd
AD		 7072 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7073 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7074 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7075 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7076 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7077 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
a59781bb 7078
21a93e13
AD		 7079 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7080 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7081
2b0781a6 7082 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
2b0781a6 7083
a59781bb
AD		 7084 /* enable CP interrupts on all rings */
7085 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7086 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7087 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7088 }
2b0781a6
AD		 7089 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7090 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7091 DRM_DEBUG("si_irq_set: sw int cp1\n");
7092 if (ring->me == 1) {
7093 switch (ring->pipe) {
7094 case 0:
7095 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7096 break;
2b0781a6
AD		 7097 default:
7098 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7099 break;
7100 }
7101 } else {
7102 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7103 }
7104 }
7105 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7106 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7107 DRM_DEBUG("si_irq_set: sw int cp2\n");
7108 if (ring->me == 1) {
7109 switch (ring->pipe) {
7110 case 0:
7111 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7112 break;
2b0781a6
AD		 7113 default:
7114 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7115 break;
7116 }
7117 } else {
7118 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7119 }
7120 }
a59781bb 7121
21a93e13
AD		 7122 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7123 DRM_DEBUG("cik_irq_set: sw int dma\n");
7124 dma_cntl |= TRAP_ENABLE;
7125 }
7126
7127 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7128 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7129 dma_cntl1 |= TRAP_ENABLE;
7130 }
7131
a59781bb
AD		 7132 if (rdev->irq.crtc_vblank_int[0] ||
7133 atomic_read(&rdev->irq.pflip[0])) {
7134 DRM_DEBUG("cik_irq_set: vblank 0\n");
7135 crtc1 |= VBLANK_INTERRUPT_MASK;
7136 }
7137 if (rdev->irq.crtc_vblank_int[1] ||
7138 atomic_read(&rdev->irq.pflip[1])) {
7139 DRM_DEBUG("cik_irq_set: vblank 1\n");
7140 crtc2 |= VBLANK_INTERRUPT_MASK;
7141 }
7142 if (rdev->irq.crtc_vblank_int[2] ||
7143 atomic_read(&rdev->irq.pflip[2])) {
7144 DRM_DEBUG("cik_irq_set: vblank 2\n");
7145 crtc3 |= VBLANK_INTERRUPT_MASK;
7146 }
7147 if (rdev->irq.crtc_vblank_int[3] ||
7148 atomic_read(&rdev->irq.pflip[3])) {
7149 DRM_DEBUG("cik_irq_set: vblank 3\n");
7150 crtc4 |= VBLANK_INTERRUPT_MASK;
7151 }
7152 if (rdev->irq.crtc_vblank_int[4] ||
7153 atomic_read(&rdev->irq.pflip[4])) {
7154 DRM_DEBUG("cik_irq_set: vblank 4\n");
7155 crtc5 |= VBLANK_INTERRUPT_MASK;
7156 }
7157 if (rdev->irq.crtc_vblank_int[5] ||
7158 atomic_read(&rdev->irq.pflip[5])) {
7159 DRM_DEBUG("cik_irq_set: vblank 5\n");
7160 crtc6 |= VBLANK_INTERRUPT_MASK;
7161 }
7162 if (rdev->irq.hpd[0]) {
7163 DRM_DEBUG("cik_irq_set: hpd 1\n");
f6b355dd 7164 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7165 }
7166 if (rdev->irq.hpd[1]) {
7167 DRM_DEBUG("cik_irq_set: hpd 2\n");
f6b355dd 7168 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7169 }
7170 if (rdev->irq.hpd[2]) {
7171 DRM_DEBUG("cik_irq_set: hpd 3\n");
f6b355dd 7172 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7173 }
7174 if (rdev->irq.hpd[3]) {
7175 DRM_DEBUG("cik_irq_set: hpd 4\n");
f6b355dd 7176 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7177 }
7178 if (rdev->irq.hpd[4]) {
7179 DRM_DEBUG("cik_irq_set: hpd 5\n");
f6b355dd 7180 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7181 }
7182 if (rdev->irq.hpd[5]) {
7183 DRM_DEBUG("cik_irq_set: hpd 6\n");
f6b355dd 7184 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7185 }
7186
7187 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7188
21a93e13
AD		 7189 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7190 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7191
2b0781a6 7192 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
2b0781a6 7193
a59781bb
AD		 7194 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7195
7196 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7197 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7198 if (rdev->num_crtc >= 4) {
7199 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7200 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7201 }
7202 if (rdev->num_crtc >= 6) {
7203 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7204 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7205 }
7206
f5d636d2
CK		 7207 if (rdev->num_crtc >= 2) {
7208 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7209 GRPH_PFLIP_INT_MASK);
7210 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7211 GRPH_PFLIP_INT_MASK);
7212 }
7213 if (rdev->num_crtc >= 4) {
7214 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7215 GRPH_PFLIP_INT_MASK);
7216 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7217 GRPH_PFLIP_INT_MASK);
7218 }
7219 if (rdev->num_crtc >= 6) {
7220 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7221 GRPH_PFLIP_INT_MASK);
7222 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7223 GRPH_PFLIP_INT_MASK);
7224 }
7225
a59781bb
AD		 7226 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7227 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7228 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7229 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7230 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7231 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7232
cffefd9b
AD		 7233 /* posting read */
7234 RREG32(SRBM_STATUS);
7235
a59781bb
AD		 7236 return 0;
7237}
7238
7239/**
7240 * cik_irq_ack - ack interrupt sources
7241 *
7242 * @rdev: radeon_device pointer
7243 *
7244 * Ack interrupt sources on the GPU (vblanks, hpd,
7245 * etc.) (CIK). Certain interrupts sources are sw
7246 * generated and do not require an explicit ack.
7247 */
7248static inline void cik_irq_ack(struct radeon_device *rdev)
7249{
7250 u32 tmp;
7251
7252 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7253 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7254 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7255 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7256 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7257 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7258 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7259
f5d636d2
CK		 7260 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7261 EVERGREEN_CRTC0_REGISTER_OFFSET);
7262 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7263 EVERGREEN_CRTC1_REGISTER_OFFSET);
7264 if (rdev->num_crtc >= 4) {
7265 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7266 EVERGREEN_CRTC2_REGISTER_OFFSET);
7267 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7268 EVERGREEN_CRTC3_REGISTER_OFFSET);
7269 }
7270 if (rdev->num_crtc >= 6) {
7271 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7272 EVERGREEN_CRTC4_REGISTER_OFFSET);
7273 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7274 EVERGREEN_CRTC5_REGISTER_OFFSET);
7275 }
7276
7277 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7278 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7279 GRPH_PFLIP_INT_CLEAR);
7280 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7281 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7282 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7283 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7284 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7285 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7286 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7287 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7288 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7289 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7290 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7291
7292 if (rdev->num_crtc >= 4) {
f5d636d2
CK		 7293 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7294 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7295 GRPH_PFLIP_INT_CLEAR);
7296 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7297 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7298 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7299 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7300 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7301 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7302 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7303 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7304 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7305 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7306 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7307 }
7308
7309 if (rdev->num_crtc >= 6) {
f5d636d2
CK		 7310 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7311 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7312 GRPH_PFLIP_INT_CLEAR);
7313 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7314 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7315 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7316 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7317 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7318 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7319 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7320 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7321 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7322 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7323 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7324 }
7325
7326 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7327 tmp = RREG32(DC_HPD1_INT_CONTROL);
7328 tmp |= DC_HPDx_INT_ACK;
7329 WREG32(DC_HPD1_INT_CONTROL, tmp);
7330 }
7331 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7332 tmp = RREG32(DC_HPD2_INT_CONTROL);
7333 tmp |= DC_HPDx_INT_ACK;
7334 WREG32(DC_HPD2_INT_CONTROL, tmp);
7335 }
7336 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7337 tmp = RREG32(DC_HPD3_INT_CONTROL);
7338 tmp |= DC_HPDx_INT_ACK;
7339 WREG32(DC_HPD3_INT_CONTROL, tmp);
7340 }
7341 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7342 tmp = RREG32(DC_HPD4_INT_CONTROL);
7343 tmp |= DC_HPDx_INT_ACK;
7344 WREG32(DC_HPD4_INT_CONTROL, tmp);
7345 }
7346 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7347 tmp = RREG32(DC_HPD5_INT_CONTROL);
7348 tmp |= DC_HPDx_INT_ACK;
7349 WREG32(DC_HPD5_INT_CONTROL, tmp);
7350 }
7351 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
3d18e337 7352 tmp = RREG32(DC_HPD6_INT_CONTROL);
a59781bb
AD		 7353 tmp |= DC_HPDx_INT_ACK;
7354 WREG32(DC_HPD6_INT_CONTROL, tmp);
7355 }
f6b355dd
AD		 7356 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT) {
7357 tmp = RREG32(DC_HPD1_INT_CONTROL);
7358 tmp |= DC_HPDx_RX_INT_ACK;
7359 WREG32(DC_HPD1_INT_CONTROL, tmp);
7360 }
7361 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
7362 tmp = RREG32(DC_HPD2_INT_CONTROL);
7363 tmp |= DC_HPDx_RX_INT_ACK;
7364 WREG32(DC_HPD2_INT_CONTROL, tmp);
7365 }
7366 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
7367 tmp = RREG32(DC_HPD3_INT_CONTROL);
7368 tmp |= DC_HPDx_RX_INT_ACK;
7369 WREG32(DC_HPD3_INT_CONTROL, tmp);
7370 }
7371 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
7372 tmp = RREG32(DC_HPD4_INT_CONTROL);
7373 tmp |= DC_HPDx_RX_INT_ACK;
7374 WREG32(DC_HPD4_INT_CONTROL, tmp);
7375 }
7376 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
7377 tmp = RREG32(DC_HPD5_INT_CONTROL);
7378 tmp |= DC_HPDx_RX_INT_ACK;
7379 WREG32(DC_HPD5_INT_CONTROL, tmp);
7380 }
7381 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
3d18e337 7382 tmp = RREG32(DC_HPD6_INT_CONTROL);
f6b355dd
AD		 7383 tmp |= DC_HPDx_RX_INT_ACK;
7384 WREG32(DC_HPD6_INT_CONTROL, tmp);
7385 }
a59781bb
AD		 7386}
7387
7388/**
7389 * cik_irq_disable - disable interrupts
7390 *
7391 * @rdev: radeon_device pointer
7392 *
7393 * Disable interrupts on the hw (CIK).
7394 */
7395static void cik_irq_disable(struct radeon_device *rdev)
7396{
7397 cik_disable_interrupts(rdev);
7398 /* Wait and acknowledge irq */
7399 mdelay(1);
7400 cik_irq_ack(rdev);
7401 cik_disable_interrupt_state(rdev);
7402}
7403
7404/**
7405 * cik_irq_disable - disable interrupts for suspend
7406 *
7407 * @rdev: radeon_device pointer
7408 *
7409 * Disable interrupts and stop the RLC (CIK).
7410 * Used for suspend.
7411 */
7412static void cik_irq_suspend(struct radeon_device *rdev)
7413{
7414 cik_irq_disable(rdev);
7415 cik_rlc_stop(rdev);
7416}
7417
7418/**
7419 * cik_irq_fini - tear down interrupt support
7420 *
7421 * @rdev: radeon_device pointer
7422 *
7423 * Disable interrupts on the hw and free the IH ring
7424 * buffer (CIK).
7425 * Used for driver unload.
7426 */
7427static void cik_irq_fini(struct radeon_device *rdev)
7428{
7429 cik_irq_suspend(rdev);
7430 r600_ih_ring_fini(rdev);
7431}
7432
7433/**
7434 * cik_get_ih_wptr - get the IH ring buffer wptr
7435 *
7436 * @rdev: radeon_device pointer
7437 *
7438 * Get the IH ring buffer wptr from either the register
7439 * or the writeback memory buffer (CIK). Also check for
7440 * ring buffer overflow and deal with it.
7441 * Used by cik_irq_process().
7442 * Returns the value of the wptr.
7443 */
7444static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7445{
7446 u32 wptr, tmp;
7447
7448 if (rdev->wb.enabled)
7449 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7450 else
7451 wptr = RREG32(IH_RB_WPTR);
7452
7453 if (wptr & RB_OVERFLOW) {
11bab0ae 7454 wptr &= ~RB_OVERFLOW;
a59781bb
AD		 7455 /* When a ring buffer overflow happen start parsing interrupt
7456 * from the last not overwritten vector (wptr + 16). Hopefully
7457 * this should allow us to catchup.
7458 */
6cc2fda2
MD		 7459 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
7460 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
a59781bb
AD		 7461 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7462 tmp = RREG32(IH_RB_CNTL);
7463 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7464 WREG32(IH_RB_CNTL, tmp);
7465 }
7466 return (wptr & rdev->ih.ptr_mask);
7467}
7468
7469/* CIK IV Ring
7470 * Each IV ring entry is 128 bits:
7471 * [7:0] - interrupt source id
7472 * [31:8] - reserved
7473 * [59:32] - interrupt source data
7474 * [63:60] - reserved
21a93e13
AD		 7475 * [71:64] - RINGID
7476 * CP:
7477 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
a59781bb
AD		 7478 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7479 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7480 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7481 * PIPE_ID - ME0 0=3D
7482 * - ME1&2 compute dispatcher (4 pipes each)
21a93e13
AD		 7483 * SDMA:
7484 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7485 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7486 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
a59781bb
AD		 7487 * [79:72] - VMID
7488 * [95:80] - PASID
7489 * [127:96] - reserved
7490 */
7491/**
7492 * cik_irq_process - interrupt handler
7493 *
7494 * @rdev: radeon_device pointer
7495 *
7496 * Interrupt hander (CIK). Walk the IH ring,
7497 * ack interrupts and schedule work to handle
7498 * interrupt events.
7499 * Returns irq process return code.
7500 */
7501int cik_irq_process(struct radeon_device *rdev)
7502{
2b0781a6
AD		 7503 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7504 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
a59781bb
AD		 7505 u32 wptr;
7506 u32 rptr;
7507 u32 src_id, src_data, ring_id;
7508 u8 me_id, pipe_id, queue_id;
7509 u32 ring_index;
7510 bool queue_hotplug = false;
f6b355dd 7511 bool queue_dp = false;
a59781bb 7512 bool queue_reset = false;
3ec7d11b 7513 u32 addr, status, mc_client;
41a524ab 7514 bool queue_thermal = false;
a59781bb
AD		 7515
7516 if (!rdev->ih.enabled || rdev->shutdown)
7517 return IRQ_NONE;
7518
7519 wptr = cik_get_ih_wptr(rdev);
7520
7521restart_ih:
7522 /* is somebody else already processing irqs? */
7523 if (atomic_xchg(&rdev->ih.lock, 1))
7524 return IRQ_NONE;
7525
7526 rptr = rdev->ih.rptr;
7527 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7528
7529 /* Order reading of wptr vs. reading of IH ring data */
7530 rmb();
7531
7532 /* display interrupts */
7533 cik_irq_ack(rdev);
7534
7535 while (rptr != wptr) {
7536 /* wptr/rptr are in bytes! */
7537 ring_index = rptr / 4;
e28740ec 7538
a59781bb
AD		 7539 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7540 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7541 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
a59781bb
AD		 7542
7543 switch (src_id) {
7544 case 1: /* D1 vblank/vline */
7545 switch (src_data) {
7546 case 0: /* D1 vblank */
07f18f0b
MK		 7547 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT))
7548 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7549
7550 if (rdev->irq.crtc_vblank_int[0]) {
7551 drm_handle_vblank(rdev->ddev, 0);
7552 rdev->pm.vblank_sync = true;
7553 wake_up(&rdev->irq.vblank_queue);
a59781bb 7554 }
07f18f0b
MK		 7555 if (atomic_read(&rdev->irq.pflip[0]))
7556 radeon_crtc_handle_vblank(rdev, 0);
7557 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7558 DRM_DEBUG("IH: D1 vblank\n");
7559
a59781bb
AD		 7560 break;
7561 case 1: /* D1 vline */
07f18f0b
MK		 7562 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT))
7563 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7564
7565 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7566 DRM_DEBUG("IH: D1 vline\n");
7567
a59781bb
AD		 7568 break;
7569 default:
7570 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7571 break;
7572 }
7573 break;
7574 case 2: /* D2 vblank/vline */
7575 switch (src_data) {
7576 case 0: /* D2 vblank */
07f18f0b
MK		 7577 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
7578 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7579
7580 if (rdev->irq.crtc_vblank_int[1]) {
7581 drm_handle_vblank(rdev->ddev, 1);
7582 rdev->pm.vblank_sync = true;
7583 wake_up(&rdev->irq.vblank_queue);
a59781bb 7584 }
07f18f0b
MK		 7585 if (atomic_read(&rdev->irq.pflip[1]))
7586 radeon_crtc_handle_vblank(rdev, 1);
7587 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7588 DRM_DEBUG("IH: D2 vblank\n");
7589
a59781bb
AD		 7590 break;
7591 case 1: /* D2 vline */
07f18f0b
MK		 7592 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT))
7593 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7594
7595 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7596 DRM_DEBUG("IH: D2 vline\n");
7597
a59781bb
AD		 7598 break;
7599 default:
7600 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7601 break;
7602 }
7603 break;
7604 case 3: /* D3 vblank/vline */
7605 switch (src_data) {
7606 case 0: /* D3 vblank */
07f18f0b
MK		 7607 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
7608 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7609
7610 if (rdev->irq.crtc_vblank_int[2]) {
7611 drm_handle_vblank(rdev->ddev, 2);
7612 rdev->pm.vblank_sync = true;
7613 wake_up(&rdev->irq.vblank_queue);
a59781bb 7614 }
07f18f0b
MK		 7615 if (atomic_read(&rdev->irq.pflip[2]))
7616 radeon_crtc_handle_vblank(rdev, 2);
7617 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7618 DRM_DEBUG("IH: D3 vblank\n");
7619
a59781bb
AD		 7620 break;
7621 case 1: /* D3 vline */
07f18f0b
MK		 7622 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
7623 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7624
7625 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7626 DRM_DEBUG("IH: D3 vline\n");
7627
a59781bb
AD		 7628 break;
7629 default:
7630 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7631 break;
7632 }
7633 break;
7634 case 4: /* D4 vblank/vline */
7635 switch (src_data) {
7636 case 0: /* D4 vblank */
07f18f0b
MK		 7637 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
7638 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7639
7640 if (rdev->irq.crtc_vblank_int[3]) {
7641 drm_handle_vblank(rdev->ddev, 3);
7642 rdev->pm.vblank_sync = true;
7643 wake_up(&rdev->irq.vblank_queue);
a59781bb 7644 }
07f18f0b
MK		 7645 if (atomic_read(&rdev->irq.pflip[3]))
7646 radeon_crtc_handle_vblank(rdev, 3);
7647 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7648 DRM_DEBUG("IH: D4 vblank\n");
7649
a59781bb
AD		 7650 break;
7651 case 1: /* D4 vline */
07f18f0b
MK		 7652 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
7653 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7654
7655 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7656 DRM_DEBUG("IH: D4 vline\n");
7657
a59781bb
AD		 7658 break;
7659 default:
7660 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7661 break;
7662 }
7663 break;
7664 case 5: /* D5 vblank/vline */
7665 switch (src_data) {
7666 case 0: /* D5 vblank */
07f18f0b
MK		 7667 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
7668 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7669
7670 if (rdev->irq.crtc_vblank_int[4]) {
7671 drm_handle_vblank(rdev->ddev, 4);
7672 rdev->pm.vblank_sync = true;
7673 wake_up(&rdev->irq.vblank_queue);
a59781bb 7674 }
07f18f0b
MK		 7675 if (atomic_read(&rdev->irq.pflip[4]))
7676 radeon_crtc_handle_vblank(rdev, 4);
7677 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7678 DRM_DEBUG("IH: D5 vblank\n");
7679
a59781bb
AD		 7680 break;
7681 case 1: /* D5 vline */
07f18f0b
MK		 7682 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
7683 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7684
7685 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7686 DRM_DEBUG("IH: D5 vline\n");
7687
a59781bb
AD		 7688 break;
7689 default:
7690 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7691 break;
7692 }
7693 break;
7694 case 6: /* D6 vblank/vline */
7695 switch (src_data) {
7696 case 0: /* D6 vblank */
07f18f0b
MK		 7697 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
7698 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7699
7700 if (rdev->irq.crtc_vblank_int[5]) {
7701 drm_handle_vblank(rdev->ddev, 5);
7702 rdev->pm.vblank_sync = true;
7703 wake_up(&rdev->irq.vblank_queue);
a59781bb 7704 }
07f18f0b
MK		 7705 if (atomic_read(&rdev->irq.pflip[5]))
7706 radeon_crtc_handle_vblank(rdev, 5);
7707 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7708 DRM_DEBUG("IH: D6 vblank\n");
7709
a59781bb
AD		 7710 break;
7711 case 1: /* D6 vline */
07f18f0b
MK		 7712 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
7713 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7714
7715 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7716 DRM_DEBUG("IH: D6 vline\n");
7717
a59781bb
AD		 7718 break;
7719 default:
7720 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7721 break;
7722 }
7723 break;
f5d636d2
CK		 7724 case 8: /* D1 page flip */
7725 case 10: /* D2 page flip */
7726 case 12: /* D3 page flip */
7727 case 14: /* D4 page flip */
7728 case 16: /* D5 page flip */
7729 case 18: /* D6 page flip */
7730 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
39dc5454
MK		 7731 if (radeon_use_pflipirq > 0)
7732 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
f5d636d2 7733 break;
a59781bb
AD		 7734 case 42: /* HPD hotplug */
7735 switch (src_data) {
7736 case 0:
07f18f0b
MK		 7737 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT))
7738 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7739
7740 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7741 queue_hotplug = true;
7742 DRM_DEBUG("IH: HPD1\n");
7743
a59781bb
AD		 7744 break;
7745 case 1:
07f18f0b
MK		 7746 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT))
7747 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7748
7749 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7750 queue_hotplug = true;
7751 DRM_DEBUG("IH: HPD2\n");
7752
a59781bb
AD		 7753 break;
7754 case 2:
07f18f0b
MK		 7755 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT))
7756 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7757
7758 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7759 queue_hotplug = true;
7760 DRM_DEBUG("IH: HPD3\n");
7761
a59781bb
AD		 7762 break;
7763 case 3:
07f18f0b
MK		 7764 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT))
7765 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7766
7767 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7768 queue_hotplug = true;
7769 DRM_DEBUG("IH: HPD4\n");
7770
a59781bb
AD		 7771 break;
7772 case 4:
07f18f0b
MK		 7773 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT))
7774 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7775
7776 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7777 queue_hotplug = true;
7778 DRM_DEBUG("IH: HPD5\n");
7779
a59781bb
AD		 7780 break;
7781 case 5:
07f18f0b
MK		 7782 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT))
7783 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7784
7785 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7786 queue_hotplug = true;
7787 DRM_DEBUG("IH: HPD6\n");
7788
a59781bb 7789 break;
f6b355dd 7790 case 6:
07f18f0b
MK		 7791 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT))
7792 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7793
7794 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_RX_INTERRUPT;
7795 queue_dp = true;
7796 DRM_DEBUG("IH: HPD_RX 1\n");
7797
f6b355dd
AD		 7798 break;
7799 case 7:
07f18f0b
MK		 7800 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT))
7801 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7802
7803 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
7804 queue_dp = true;
7805 DRM_DEBUG("IH: HPD_RX 2\n");
7806
f6b355dd
AD		 7807 break;
7808 case 8:
07f18f0b
MK		 7809 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
7810 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7811
7812 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
7813 queue_dp = true;
7814 DRM_DEBUG("IH: HPD_RX 3\n");
7815
f6b355dd
AD		 7816 break;
7817 case 9:
07f18f0b
MK		 7818 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
7819 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7820
7821 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
7822 queue_dp = true;
7823 DRM_DEBUG("IH: HPD_RX 4\n");
7824
f6b355dd
AD		 7825 break;
7826 case 10:
07f18f0b
MK		 7827 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
7828 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7829
7830 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
7831 queue_dp = true;
7832 DRM_DEBUG("IH: HPD_RX 5\n");
7833
f6b355dd
AD		 7834 break;
7835 case 11:
07f18f0b
MK		 7836 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
7837 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7838
7839 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
7840 queue_dp = true;
7841 DRM_DEBUG("IH: HPD_RX 6\n");
7842
f6b355dd 7843 break;
a59781bb
AD		 7844 default:
7845 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7846 break;
7847 }
7848 break;
dc12a3ec
LL		 7849 case 96:
7850 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
7851 WREG32(SRBM_INT_ACK, 0x1);
7852 break;
6a3808b8
CK		 7853 case 124: /* UVD */
7854 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7855 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7856 break;
9d97c99b
AD		 7857 case 146:
7858 case 147:
3ec7d11b
AD		 7859 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7860 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7861 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
9b7d786b
CK		 7862 /* reset addr and status */
7863 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7864 if (addr == 0x0 && status == 0x0)
7865 break;
9d97c99b
AD		 7866 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7867 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3ec7d11b 7868 addr);
9d97c99b 7869 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3ec7d11b
AD		 7870 status);
7871 cik_vm_decode_fault(rdev, status, addr, mc_client);
9d97c99b 7872 break;
d93f7937
CK		 7873 case 167: /* VCE */
7874 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
7875 switch (src_data) {
7876 case 0:
7877 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
7878 break;
7879 case 1:
7880 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
7881 break;
7882 default:
7883 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
7884 break;
7885 }
7886 break;
a59781bb
AD		 7887 case 176: /* GFX RB CP_INT */
7888 case 177: /* GFX IB CP_INT */
7889 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7890 break;
7891 case 181: /* CP EOP event */
7892 DRM_DEBUG("IH: CP EOP\n");
21a93e13
AD		 7893 /* XXX check the bitfield order! */
7894 me_id = (ring_id & 0x60) >> 5;
7895 pipe_id = (ring_id & 0x18) >> 3;
7896 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 7897 switch (me_id) {
7898 case 0:
7899 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7900 break;
7901 case 1:
a59781bb 7902 case 2:
2b0781a6
AD		 7903 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7904 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7905 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7906 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
a59781bb
AD		 7907 break;
7908 }
7909 break;
7910 case 184: /* CP Privileged reg access */
7911 DRM_ERROR("Illegal register access in command stream\n");
7912 /* XXX check the bitfield order! */
7913 me_id = (ring_id & 0x60) >> 5;
7914 pipe_id = (ring_id & 0x18) >> 3;
7915 queue_id = (ring_id & 0x7) >> 0;
7916 switch (me_id) {
7917 case 0:
7918 /* This results in a full GPU reset, but all we need to do is soft
7919 * reset the CP for gfx
7920 */
7921 queue_reset = true;
7922 break;
7923 case 1:
7924 /* XXX compute */
2b0781a6 7925 queue_reset = true;
a59781bb
AD		 7926 break;
7927 case 2:
7928 /* XXX compute */
2b0781a6 7929 queue_reset = true;
a59781bb
AD		 7930 break;
7931 }
7932 break;
7933 case 185: /* CP Privileged inst */
7934 DRM_ERROR("Illegal instruction in command stream\n");
21a93e13
AD		 7935 /* XXX check the bitfield order! */
7936 me_id = (ring_id & 0x60) >> 5;
7937 pipe_id = (ring_id & 0x18) >> 3;
7938 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 7939 switch (me_id) {
7940 case 0:
7941 /* This results in a full GPU reset, but all we need to do is soft
7942 * reset the CP for gfx
7943 */
7944 queue_reset = true;
7945 break;
7946 case 1:
7947 /* XXX compute */
2b0781a6 7948 queue_reset = true;
a59781bb
AD		 7949 break;
7950 case 2:
7951 /* XXX compute */
2b0781a6 7952 queue_reset = true;
a59781bb
AD		 7953 break;
7954 }
7955 break;
21a93e13
AD		 7956 case 224: /* SDMA trap event */
7957 /* XXX check the bitfield order! */
7958 me_id = (ring_id & 0x3) >> 0;
7959 queue_id = (ring_id & 0xc) >> 2;
7960 DRM_DEBUG("IH: SDMA trap\n");
7961 switch (me_id) {
7962 case 0:
7963 switch (queue_id) {
7964 case 0:
7965 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7966 break;
7967 case 1:
7968 /* XXX compute */
7969 break;
7970 case 2:
7971 /* XXX compute */
7972 break;
7973 }
7974 break;
7975 case 1:
7976 switch (queue_id) {
7977 case 0:
7978 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7979 break;
7980 case 1:
7981 /* XXX compute */
7982 break;
7983 case 2:
7984 /* XXX compute */
7985 break;
7986 }
7987 break;
7988 }
7989 break;
41a524ab
AD		 7990 case 230: /* thermal low to high */
7991 DRM_DEBUG("IH: thermal low to high\n");
7992 rdev->pm.dpm.thermal.high_to_low = false;
7993 queue_thermal = true;
7994 break;
7995 case 231: /* thermal high to low */
7996 DRM_DEBUG("IH: thermal high to low\n");
7997 rdev->pm.dpm.thermal.high_to_low = true;
7998 queue_thermal = true;
7999 break;
8000 case 233: /* GUI IDLE */
8001 DRM_DEBUG("IH: GUI idle\n");
8002 break;
21a93e13
AD		 8003 case 241: /* SDMA Privileged inst */
8004 case 247: /* SDMA Privileged inst */
8005 DRM_ERROR("Illegal instruction in SDMA command stream\n");
8006 /* XXX check the bitfield order! */
8007 me_id = (ring_id & 0x3) >> 0;
8008 queue_id = (ring_id & 0xc) >> 2;
8009 switch (me_id) {
8010 case 0:
8011 switch (queue_id) {
8012 case 0:
8013 queue_reset = true;
8014 break;
8015 case 1:
8016 /* XXX compute */
8017 queue_reset = true;
8018 break;
8019 case 2:
8020 /* XXX compute */
8021 queue_reset = true;
8022 break;
8023 }
8024 break;
8025 case 1:
8026 switch (queue_id) {
8027 case 0:
8028 queue_reset = true;
8029 break;
8030 case 1:
8031 /* XXX compute */
8032 queue_reset = true;
8033 break;
8034 case 2:
8035 /* XXX compute */
8036 queue_reset = true;
8037 break;
8038 }
8039 break;
8040 }
8041 break;
a59781bb
AD		 8042 default:
8043 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8044 break;
8045 }
8046
8047 /* wptr/rptr are in bytes! */
8048 rptr += 16;
8049 rptr &= rdev->ih.ptr_mask;
f55e03b9 8050 WREG32(IH_RB_RPTR, rptr);
a59781bb 8051 }
f6b355dd
AD		 8052 if (queue_dp)
8053 schedule_work(&rdev->dp_work);
a59781bb 8054 if (queue_hotplug)
cb5d4166 8055 schedule_delayed_work(&rdev->hotplug_work, 0);
3c036389
CK		 8056 if (queue_reset) {
8057 rdev->needs_reset = true;
8058 wake_up_all(&rdev->fence_queue);
8059 }
41a524ab
AD		 8060 if (queue_thermal)
8061 schedule_work(&rdev->pm.dpm.thermal.work);
a59781bb 8062 rdev->ih.rptr = rptr;
a59781bb
AD		 8063 atomic_set(&rdev->ih.lock, 0);
8064
8065 /* make sure wptr hasn't changed while processing */
8066 wptr = cik_get_ih_wptr(rdev);
8067 if (wptr != rptr)
8068 goto restart_ih;
8069
8070 return IRQ_HANDLED;
8071}
7bf94a2c
AD		 8072
8073/*
8074 * startup/shutdown callbacks
8075 */
bc48a15a
JG		 8076static void cik_uvd_init(struct radeon_device *rdev)
8077{
8078 int r;
8079
8080 if (!rdev->has_uvd)
8081 return;
8082
8083 r = radeon_uvd_init(rdev);
8084 if (r) {
8085 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
8086 /*
8087 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
8088 * to early fails cik_uvd_start() and thus nothing happens
8089 * there. So it is pointless to try to go through that code
8090 * hence why we disable uvd here.
8091 */
8092 rdev->has_uvd = 0;
8093 return;
8094 }
8095 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
8096 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
8097}
8098
8099static void cik_uvd_start(struct radeon_device *rdev)
8100{
8101 int r;
8102
8103 if (!rdev->has_uvd)
8104 return;
8105
8106 r = radeon_uvd_resume(rdev);
8107 if (r) {
8108 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
8109 goto error;
8110 }
8111 r = uvd_v4_2_resume(rdev);
8112 if (r) {
8113 dev_err(rdev->dev, "failed UVD 4.2 resume (%d).\n", r);
8114 goto error;
8115 }
8116 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
8117 if (r) {
8118 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
8119 goto error;
8120 }
8121 return;
8122
8123error:
8124 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8125}
8126
8127static void cik_uvd_resume(struct radeon_device *rdev)
8128{
8129 struct radeon_ring *ring;
8130 int r;
8131
8132 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
8133 return;
8134
8135 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
70a033d2 8136 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
bc48a15a
JG		 8137 if (r) {
8138 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
8139 return;
8140 }
8141 r = uvd_v1_0_init(rdev);
8142 if (r) {
8143 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
8144 return;
8145 }
8146}
8147
cb25f7e0
JG		 8148static void cik_vce_init(struct radeon_device *rdev)
8149{
8150 int r;
8151
8152 if (!rdev->has_vce)
8153 return;
8154
8155 r = radeon_vce_init(rdev);
8156 if (r) {
8157 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
8158 /*
8159 * At this point rdev->vce.vcpu_bo is NULL which trickles down
8160 * to early fails cik_vce_start() and thus nothing happens
8161 * there. So it is pointless to try to go through that code
8162 * hence why we disable vce here.
8163 */
8164 rdev->has_vce = 0;
8165 return;
8166 }
8167 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
8168 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
8169 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
8170 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
8171}
8172
8173static void cik_vce_start(struct radeon_device *rdev)
8174{
8175 int r;
8176
8177 if (!rdev->has_vce)
8178 return;
8179
8180 r = radeon_vce_resume(rdev);
8181 if (r) {
8182 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
8183 goto error;
8184 }
8185 r = vce_v2_0_resume(rdev);
8186 if (r) {
8187 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
8188 goto error;
8189 }
8190 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
8191 if (r) {
8192 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
8193 goto error;
8194 }
8195 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
8196 if (r) {
8197 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
8198 goto error;
8199 }
8200 return;
8201
8202error:
8203 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8204 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8205}
8206
8207static void cik_vce_resume(struct radeon_device *rdev)
8208{
8209 struct radeon_ring *ring;
8210 int r;
8211
8212 if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
8213 return;
8214
8215 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8216 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
8217 if (r) {
8218 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
8219 return;
8220 }
8221 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8222 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
8223 if (r) {
8224 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
8225 return;
8226 }
8227 r = vce_v1_0_init(rdev);
8228 if (r) {
8229 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
8230 return;
8231 }
8232}
8233
7bf94a2c
AD		 8234/**
8235 * cik_startup - program the asic to a functional state
8236 *
8237 * @rdev: radeon_device pointer
8238 *
8239 * Programs the asic to a functional state (CIK).
8240 * Called by cik_init() and cik_resume().
8241 * Returns 0 for success, error for failure.
8242 */
8243static int cik_startup(struct radeon_device *rdev)
8244{
8245 struct radeon_ring *ring;
0e16e4cf 8246 u32 nop;
7bf94a2c
AD		 8247 int r;
8248
8a7cd276
AD		 8249 /* enable pcie gen2/3 link */
8250 cik_pcie_gen3_enable(rdev);
7235711a
AD		 8251 /* enable aspm */
8252 cik_program_aspm(rdev);
8a7cd276 8253
e5903d39
AD		 8254 /* scratch needs to be initialized before MC */
8255 r = r600_vram_scratch_init(rdev);
8256 if (r)
8257 return r;
8258
6fab3feb
AD		 8259 cik_mc_program(rdev);
8260
6c7bccea 8261 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7bf94a2c
AD		 8262 r = ci_mc_load_microcode(rdev);
8263 if (r) {
8264 DRM_ERROR("Failed to load MC firmware!\n");
8265 return r;
8266 }
8267 }
8268
7bf94a2c
AD		 8269 r = cik_pcie_gart_enable(rdev);
8270 if (r)
8271 return r;
8272 cik_gpu_init(rdev);
8273
8274 /* allocate rlc buffers */
22c775ce
AD		 8275 if (rdev->flags & RADEON_IS_IGP) {
8276 if (rdev->family == CHIP_KAVERI) {
8277 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8278 rdev->rlc.reg_list_size =
8279 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8280 } else {
8281 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8282 rdev->rlc.reg_list_size =
8283 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8284 }
8285 }
8286 rdev->rlc.cs_data = ci_cs_data;
e70a15f5 8287 rdev->rlc.cp_table_size = ALIGN(CP_ME_TABLE_SIZE * 5 * 4, 2048); /* CP JT */
9406d216 8288 rdev->rlc.cp_table_size += 64 * 1024; /* GDS */
1fd11777 8289 r = sumo_rlc_init(rdev);
7bf94a2c
AD		 8290 if (r) {
8291 DRM_ERROR("Failed to init rlc BOs!\n");
8292 return r;
8293 }
8294
8295 /* allocate wb buffer */
8296 r = radeon_wb_init(rdev);
8297 if (r)
8298 return r;
8299
963e81f9
AD		 8300 /* allocate mec buffers */
8301 r = cik_mec_init(rdev);
8302 if (r) {
8303 DRM_ERROR("Failed to init MEC BOs!\n");
8304 return r;
8305 }
8306
7bf94a2c
AD		 8307 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8308 if (r) {
8309 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8310 return r;
8311 }
8312
963e81f9
AD		 8313 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8314 if (r) {
8315 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8316 return r;
8317 }
8318
8319 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8320 if (r) {
8321 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8322 return r;
8323 }
8324
7bf94a2c
AD		 8325 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8326 if (r) {
8327 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8328 return r;
8329 }
8330
8331 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8332 if (r) {
8333 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8334 return r;
8335 }
8336
bc48a15a 8337 cik_uvd_start(rdev);
cb25f7e0 8338 cik_vce_start(rdev);
d93f7937 8339
7bf94a2c
AD		 8340 /* Enable IRQ */
8341 if (!rdev->irq.installed) {
8342 r = radeon_irq_kms_init(rdev);
8343 if (r)
8344 return r;
8345 }
8346
8347 r = cik_irq_init(rdev);
8348 if (r) {
8349 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8350 radeon_irq_kms_fini(rdev);
8351 return r;
8352 }
8353 cik_irq_set(rdev);
8354
0e16e4cf 8355 if (rdev->family == CHIP_HAWAII) {
78cd3661
AD		 8356 if (rdev->new_fw)
8357 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8358 else
8359 nop = RADEON_CP_PACKET2;
0e16e4cf
AD		 8360 } else {
8361 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8362 }
8363
7bf94a2c
AD		 8364 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8365 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
0e16e4cf 8366 nop);
7bf94a2c
AD		 8367 if (r)
8368 return r;
8369
963e81f9 8370 /* set up the compute queues */
2615b53a 8371 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 8372 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8373 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
0e16e4cf 8374 nop);
963e81f9
AD		 8375 if (r)
8376 return r;
8377 ring->me = 1; /* first MEC */
8378 ring->pipe = 0; /* first pipe */
8379 ring->queue = 0; /* first queue */
8380 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8381
2615b53a 8382 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 8383 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8384 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
0e16e4cf 8385 nop);
963e81f9
AD		 8386 if (r)
8387 return r;
8388 /* dGPU only have 1 MEC */
8389 ring->me = 1; /* first MEC */
8390 ring->pipe = 0; /* first pipe */
8391 ring->queue = 1; /* second queue */
8392 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8393
7bf94a2c
AD		 8394 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8395 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 8396 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 8397 if (r)
8398 return r;
8399
8400 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8401 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2e1e6dad 8402 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 8403 if (r)
8404 return r;
8405
8406 r = cik_cp_resume(rdev);
8407 if (r)
8408 return r;
8409
8410 r = cik_sdma_resume(rdev);
8411 if (r)
8412 return r;
8413
bc48a15a 8414 cik_uvd_resume(rdev);
cb25f7e0 8415 cik_vce_resume(rdev);
d93f7937 8416
7bf94a2c
AD		 8417 r = radeon_ib_pool_init(rdev);
8418 if (r) {
8419 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8420 return r;
8421 }
8422
8423 r = radeon_vm_manager_init(rdev);
8424 if (r) {
8425 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8426 return r;
8427 }
8428
bfc1f97d 8429 r = radeon_audio_init(rdev);
b530602f
AD		 8430 if (r)
8431 return r;
8432
7bf94a2c
AD		 8433 return 0;
8434}
8435
8436/**
8437 * cik_resume - resume the asic to a functional state
8438 *
8439 * @rdev: radeon_device pointer
8440 *
8441 * Programs the asic to a functional state (CIK).
8442 * Called at resume.
8443 * Returns 0 for success, error for failure.
8444 */
8445int cik_resume(struct radeon_device *rdev)
8446{
8447 int r;
8448
8449 /* post card */
8450 atom_asic_init(rdev->mode_info.atom_context);
8451
0aafd313
AD		 8452 /* init golden registers */
8453 cik_init_golden_registers(rdev);
8454
bc6a6295
AD		 8455 if (rdev->pm.pm_method == PM_METHOD_DPM)
8456 radeon_pm_resume(rdev);
6c7bccea 8457
7bf94a2c
AD		 8458 rdev->accel_working = true;
8459 r = cik_startup(rdev);
8460 if (r) {
8461 DRM_ERROR("cik startup failed on resume\n");
8462 rdev->accel_working = false;
8463 return r;
8464 }
8465
8466 return r;
8467
8468}
8469
8470/**
8471 * cik_suspend - suspend the asic
8472 *
8473 * @rdev: radeon_device pointer
8474 *
8475 * Bring the chip into a state suitable for suspend (CIK).
8476 * Called at suspend.
8477 * Returns 0 for success.
8478 */
8479int cik_suspend(struct radeon_device *rdev)
8480{
6c7bccea 8481 radeon_pm_suspend(rdev);
7991d665 8482 radeon_audio_fini(rdev);
7bf94a2c
AD		 8483 radeon_vm_manager_fini(rdev);
8484 cik_cp_enable(rdev, false);
8485 cik_sdma_enable(rdev, false);
bc48a15a
JG		 8486 if (rdev->has_uvd) {
8487 uvd_v1_0_fini(rdev);
8488 radeon_uvd_suspend(rdev);
8489 }
cb25f7e0
JG		 8490 if (rdev->has_vce)
8491 radeon_vce_suspend(rdev);
473359bc
AD		 8492 cik_fini_pg(rdev);
8493 cik_fini_cg(rdev);
7bf94a2c
AD		 8494 cik_irq_suspend(rdev);
8495 radeon_wb_disable(rdev);
8496 cik_pcie_gart_disable(rdev);
8497 return 0;
8498}
8499
8500/* Plan is to move initialization in that function and use
8501 * helper function so that radeon_device_init pretty much
8502 * do nothing more than calling asic specific function. This
8503 * should also allow to remove a bunch of callback function
8504 * like vram_info.
8505 */
8506/**
8507 * cik_init - asic specific driver and hw init
8508 *
8509 * @rdev: radeon_device pointer
8510 *
8511 * Setup asic specific driver variables and program the hw
8512 * to a functional state (CIK).
8513 * Called at driver startup.
8514 * Returns 0 for success, errors for failure.
8515 */
8516int cik_init(struct radeon_device *rdev)
8517{
8518 struct radeon_ring *ring;
8519 int r;
8520
8521 /* Read BIOS */
8522 if (!radeon_get_bios(rdev)) {
8523 if (ASIC_IS_AVIVO(rdev))
8524 return -EINVAL;
8525 }
8526 /* Must be an ATOMBIOS */
8527 if (!rdev->is_atom_bios) {
8528 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8529 return -EINVAL;
8530 }
8531 r = radeon_atombios_init(rdev);
8532 if (r)
8533 return r;
8534
8535 /* Post card if necessary */
8536 if (!radeon_card_posted(rdev)) {
8537 if (!rdev->bios) {
8538 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8539 return -EINVAL;
8540 }
8541 DRM_INFO("GPU not posted. posting now...\n");
8542 atom_asic_init(rdev->mode_info.atom_context);
8543 }
0aafd313
AD		 8544 /* init golden registers */
8545 cik_init_golden_registers(rdev);
7bf94a2c
AD		 8546 /* Initialize scratch registers */
8547 cik_scratch_init(rdev);
8548 /* Initialize surface registers */
8549 radeon_surface_init(rdev);
8550 /* Initialize clocks */
8551 radeon_get_clock_info(rdev->ddev);
8552
8553 /* Fence driver */
8554 r = radeon_fence_driver_init(rdev);
8555 if (r)
8556 return r;
8557
8558 /* initialize memory controller */
8559 r = cik_mc_init(rdev);
8560 if (r)
8561 return r;
8562 /* Memory manager */
8563 r = radeon_bo_init(rdev);
8564 if (r)
8565 return r;
8566
01ac8794
AD		 8567 if (rdev->flags & RADEON_IS_IGP) {
8568 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8569 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8570 r = cik_init_microcode(rdev);
8571 if (r) {
8572 DRM_ERROR("Failed to load firmware!\n");
8573 return r;
8574 }
8575 }
8576 } else {
8577 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8578 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8579 !rdev->mc_fw) {
8580 r = cik_init_microcode(rdev);
8581 if (r) {
8582 DRM_ERROR("Failed to load firmware!\n");
8583 return r;
8584 }
8585 }
8586 }
8587
6c7bccea
AD		 8588 /* Initialize power management */
8589 radeon_pm_init(rdev);
8590
7bf94a2c
AD		 8591 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8592 ring->ring_obj = NULL;
8593 r600_ring_init(rdev, ring, 1024 * 1024);
8594
963e81f9
AD		 8595 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8596 ring->ring_obj = NULL;
8597 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8598 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8599 if (r)
8600 return r;
8601
8602 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8603 ring->ring_obj = NULL;
8604 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8605 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8606 if (r)
8607 return r;
8608
7bf94a2c
AD		 8609 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8610 ring->ring_obj = NULL;
8611 r600_ring_init(rdev, ring, 256 * 1024);
8612
8613 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8614 ring->ring_obj = NULL;
8615 r600_ring_init(rdev, ring, 256 * 1024);
8616
bc48a15a 8617 cik_uvd_init(rdev);
cb25f7e0 8618 cik_vce_init(rdev);
d93f7937 8619
7bf94a2c
AD		 8620 rdev->ih.ring_obj = NULL;
8621 r600_ih_ring_init(rdev, 64 * 1024);
8622
8623 r = r600_pcie_gart_init(rdev);
8624 if (r)
8625 return r;
8626
8627 rdev->accel_working = true;
8628 r = cik_startup(rdev);
8629 if (r) {
8630 dev_err(rdev->dev, "disabling GPU acceleration\n");
8631 cik_cp_fini(rdev);
8632 cik_sdma_fini(rdev);
8633 cik_irq_fini(rdev);
1fd11777 8634 sumo_rlc_fini(rdev);
963e81f9 8635 cik_mec_fini(rdev);
7bf94a2c
AD		 8636 radeon_wb_fini(rdev);
8637 radeon_ib_pool_fini(rdev);
8638 radeon_vm_manager_fini(rdev);
8639 radeon_irq_kms_fini(rdev);
8640 cik_pcie_gart_fini(rdev);
8641 rdev->accel_working = false;
8642 }
8643
8644 /* Don't start up if the MC ucode is missing.
8645 * The default clocks and voltages before the MC ucode
8646 * is loaded are not suffient for advanced operations.
8647 */
8648 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8649 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8650 return -EINVAL;
8651 }
8652
8653 return 0;
8654}
8655
8656/**
8657 * cik_fini - asic specific driver and hw fini
8658 *
8659 * @rdev: radeon_device pointer
8660 *
8661 * Tear down the asic specific driver variables and program the hw
8662 * to an idle state (CIK).
8663 * Called at driver unload.
8664 */
8665void cik_fini(struct radeon_device *rdev)
8666{
6c7bccea 8667 radeon_pm_fini(rdev);
7bf94a2c
AD		 8668 cik_cp_fini(rdev);
8669 cik_sdma_fini(rdev);
473359bc
AD		 8670 cik_fini_pg(rdev);
8671 cik_fini_cg(rdev);
7bf94a2c 8672 cik_irq_fini(rdev);
1fd11777 8673 sumo_rlc_fini(rdev);
963e81f9 8674 cik_mec_fini(rdev);
7bf94a2c
AD		 8675 radeon_wb_fini(rdev);
8676 radeon_vm_manager_fini(rdev);
8677 radeon_ib_pool_fini(rdev);
8678 radeon_irq_kms_fini(rdev);
e409b128 8679 uvd_v1_0_fini(rdev);
87167bb1 8680 radeon_uvd_fini(rdev);
d93f7937 8681 radeon_vce_fini(rdev);
7bf94a2c
AD		 8682 cik_pcie_gart_fini(rdev);
8683 r600_vram_scratch_fini(rdev);
8684 radeon_gem_fini(rdev);
8685 radeon_fence_driver_fini(rdev);
8686 radeon_bo_fini(rdev);
8687 radeon_atombios_fini(rdev);
8688 kfree(rdev->bios);
8689 rdev->bios = NULL;
8690}
cd84a27d 8691
134b480f
AD		 8692void dce8_program_fmt(struct drm_encoder *encoder)
8693{
8694 struct drm_device *dev = encoder->dev;
8695 struct radeon_device *rdev = dev->dev_private;
8696 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8697 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8698 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8699 int bpc = 0;
8700 u32 tmp = 0;
6214bb74 8701 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 8702
6214bb74
AD		 8703 if (connector) {
8704 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 8705 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 8706 dither = radeon_connector->dither;
8707 }
134b480f
AD		 8708
8709 /* LVDS/eDP FMT is set up by atom */
8710 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8711 return;
8712
8713 /* not needed for analog */
8714 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8715 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8716 return;
8717
8718 if (bpc == 0)
8719 return;
8720
8721 switch (bpc) {
8722 case 6:
6214bb74 8723 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8724 /* XXX sort out optimal dither settings */
8725 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8726 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8727 else
8728 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8729 break;
8730 case 8:
6214bb74 8731 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8732 /* XXX sort out optimal dither settings */
8733 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8734 FMT_RGB_RANDOM_ENABLE |
8735 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8736 else
8737 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8738 break;
8739 case 10:
6214bb74 8740 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8741 /* XXX sort out optimal dither settings */
8742 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8743 FMT_RGB_RANDOM_ENABLE |
8744 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8745 else
8746 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8747 break;
8748 default:
8749 /* not needed */
8750 break;
8751 }
8752
8753 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8754}
8755
cd84a27d
AD		 8756/* display watermark setup */
8757/**
8758 * dce8_line_buffer_adjust - Set up the line buffer
8759 *
8760 * @rdev: radeon_device pointer
8761 * @radeon_crtc: the selected display controller
8762 * @mode: the current display mode on the selected display
8763 * controller
8764 *
8765 * Setup up the line buffer allocation for
8766 * the selected display controller (CIK).
8767 * Returns the line buffer size in pixels.
8768 */
8769static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8770 struct radeon_crtc *radeon_crtc,
8771 struct drm_display_mode *mode)
8772{
bc01a8c7
AD		 8773 u32 tmp, buffer_alloc, i;
8774 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
cd84a27d
AD		 8775 /*
8776 * Line Buffer Setup
8777 * There are 6 line buffers, one for each display controllers.
8778 * There are 3 partitions per LB. Select the number of partitions
8779 * to enable based on the display width. For display widths larger
8780 * than 4096, you need use to use 2 display controllers and combine
8781 * them using the stereo blender.
8782 */
8783 if (radeon_crtc->base.enabled && mode) {
bc01a8c7 8784 if (mode->crtc_hdisplay < 1920) {
cd84a27d 8785 tmp = 1;
bc01a8c7
AD		 8786 buffer_alloc = 2;
8787 } else if (mode->crtc_hdisplay < 2560) {
cd84a27d 8788 tmp = 2;
bc01a8c7
AD		 8789 buffer_alloc = 2;
8790 } else if (mode->crtc_hdisplay < 4096) {
cd84a27d 8791 tmp = 0;
bc01a8c7
AD		 8792 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8793 } else {
cd84a27d
AD		 8794 DRM_DEBUG_KMS("Mode too big for LB!\n");
8795 tmp = 0;
bc01a8c7 8796 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
cd84a27d 8797 }
bc01a8c7 8798 } else {
cd84a27d 8799 tmp = 1;
bc01a8c7
AD		 8800 buffer_alloc = 0;
8801 }
cd84a27d
AD		 8802
8803 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8804 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8805
bc01a8c7
AD		 8806 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8807 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8808 for (i = 0; i < rdev->usec_timeout; i++) {
8809 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8810 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8811 break;
8812 udelay(1);
8813 }
8814
cd84a27d
AD		 8815 if (radeon_crtc->base.enabled && mode) {
8816 switch (tmp) {
8817 case 0:
8818 default:
8819 return 4096 * 2;
8820 case 1:
8821 return 1920 * 2;
8822 case 2:
8823 return 2560 * 2;
8824 }
8825 }
8826
8827 /* controller not enabled, so no lb used */
8828 return 0;
8829}
8830
8831/**
8832 * cik_get_number_of_dram_channels - get the number of dram channels
8833 *
8834 * @rdev: radeon_device pointer
8835 *
8836 * Look up the number of video ram channels (CIK).
8837 * Used for display watermark bandwidth calculations
8838 * Returns the number of dram channels
8839 */
8840static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8841{
8842 u32 tmp = RREG32(MC_SHARED_CHMAP);
8843
8844 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8845 case 0:
8846 default:
8847 return 1;
8848 case 1:
8849 return 2;
8850 case 2:
8851 return 4;
8852 case 3:
8853 return 8;
8854 case 4:
8855 return 3;
8856 case 5:
8857 return 6;
8858 case 6:
8859 return 10;
8860 case 7:
8861 return 12;
8862 case 8:
8863 return 16;
8864 }
8865}
8866
8867struct dce8_wm_params {
8868 u32 dram_channels; /* number of dram channels */
8869 u32 yclk; /* bandwidth per dram data pin in kHz */
8870 u32 sclk; /* engine clock in kHz */
8871 u32 disp_clk; /* display clock in kHz */
8872 u32 src_width; /* viewport width */
8873 u32 active_time; /* active display time in ns */
8874 u32 blank_time; /* blank time in ns */
8875 bool interlaced; /* mode is interlaced */
8876 fixed20_12 vsc; /* vertical scale ratio */
8877 u32 num_heads; /* number of active crtcs */
8878 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8879 u32 lb_size; /* line buffer allocated to pipe */
8880 u32 vtaps; /* vertical scaler taps */
8881};
8882
8883/**
8884 * dce8_dram_bandwidth - get the dram bandwidth
8885 *
8886 * @wm: watermark calculation data
8887 *
8888 * Calculate the raw dram bandwidth (CIK).
8889 * Used for display watermark bandwidth calculations
8890 * Returns the dram bandwidth in MBytes/s
8891 */
8892static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8893{
8894 /* Calculate raw DRAM Bandwidth */
8895 fixed20_12 dram_efficiency; /* 0.7 */
8896 fixed20_12 yclk, dram_channels, bandwidth;
8897 fixed20_12 a;
8898
8899 a.full = dfixed_const(1000);
8900 yclk.full = dfixed_const(wm->yclk);
8901 yclk.full = dfixed_div(yclk, a);
8902 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8903 a.full = dfixed_const(10);
8904 dram_efficiency.full = dfixed_const(7);
8905 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8906 bandwidth.full = dfixed_mul(dram_channels, yclk);
8907 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8908
8909 return dfixed_trunc(bandwidth);
8910}
8911
8912/**
8913 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8914 *
8915 * @wm: watermark calculation data
8916 *
8917 * Calculate the dram bandwidth used for display (CIK).
8918 * Used for display watermark bandwidth calculations
8919 * Returns the dram bandwidth for display in MBytes/s
8920 */
8921static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8922{
8923 /* Calculate DRAM Bandwidth and the part allocated to display. */
8924 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8925 fixed20_12 yclk, dram_channels, bandwidth;
8926 fixed20_12 a;
8927
8928 a.full = dfixed_const(1000);
8929 yclk.full = dfixed_const(wm->yclk);
8930 yclk.full = dfixed_div(yclk, a);
8931 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8932 a.full = dfixed_const(10);
8933 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8934 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8935 bandwidth.full = dfixed_mul(dram_channels, yclk);
8936 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8937
8938 return dfixed_trunc(bandwidth);
8939}
8940
8941/**
8942 * dce8_data_return_bandwidth - get the data return bandwidth
8943 *
8944 * @wm: watermark calculation data
8945 *
8946 * Calculate the data return bandwidth used for display (CIK).
8947 * Used for display watermark bandwidth calculations
8948 * Returns the data return bandwidth in MBytes/s
8949 */
8950static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8951{
8952 /* Calculate the display Data return Bandwidth */
8953 fixed20_12 return_efficiency; /* 0.8 */
8954 fixed20_12 sclk, bandwidth;
8955 fixed20_12 a;
8956
8957 a.full = dfixed_const(1000);
8958 sclk.full = dfixed_const(wm->sclk);
8959 sclk.full = dfixed_div(sclk, a);
8960 a.full = dfixed_const(10);
8961 return_efficiency.full = dfixed_const(8);
8962 return_efficiency.full = dfixed_div(return_efficiency, a);
8963 a.full = dfixed_const(32);
8964 bandwidth.full = dfixed_mul(a, sclk);
8965 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8966
8967 return dfixed_trunc(bandwidth);
8968}
8969
8970/**
8971 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8972 *
8973 * @wm: watermark calculation data
8974 *
8975 * Calculate the dmif bandwidth used for display (CIK).
8976 * Used for display watermark bandwidth calculations
8977 * Returns the dmif bandwidth in MBytes/s
8978 */
8979static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8980{
8981 /* Calculate the DMIF Request Bandwidth */
8982 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8983 fixed20_12 disp_clk, bandwidth;
8984 fixed20_12 a, b;
8985
8986 a.full = dfixed_const(1000);
8987 disp_clk.full = dfixed_const(wm->disp_clk);
8988 disp_clk.full = dfixed_div(disp_clk, a);
8989 a.full = dfixed_const(32);
8990 b.full = dfixed_mul(a, disp_clk);
8991
8992 a.full = dfixed_const(10);
8993 disp_clk_request_efficiency.full = dfixed_const(8);
8994 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8995
8996 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8997
8998 return dfixed_trunc(bandwidth);
8999}
9000
9001/**
9002 * dce8_available_bandwidth - get the min available bandwidth
9003 *
9004 * @wm: watermark calculation data
9005 *
9006 * Calculate the min available bandwidth used for display (CIK).
9007 * Used for display watermark bandwidth calculations
9008 * Returns the min available bandwidth in MBytes/s
9009 */
9010static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9011{
9012 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9013 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9014 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9015 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9016
9017 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9018}
9019
9020/**
9021 * dce8_average_bandwidth - get the average available bandwidth
9022 *
9023 * @wm: watermark calculation data
9024 *
9025 * Calculate the average available bandwidth used for display (CIK).
9026 * Used for display watermark bandwidth calculations
9027 * Returns the average available bandwidth in MBytes/s
9028 */
9029static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9030{
9031 /* Calculate the display mode Average Bandwidth
9032 * DisplayMode should contain the source and destination dimensions,
9033 * timing, etc.
9034 */
9035 fixed20_12 bpp;
9036 fixed20_12 line_time;
9037 fixed20_12 src_width;
9038 fixed20_12 bandwidth;
9039 fixed20_12 a;
9040
9041 a.full = dfixed_const(1000);
9042 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9043 line_time.full = dfixed_div(line_time, a);
9044 bpp.full = dfixed_const(wm->bytes_per_pixel);
9045 src_width.full = dfixed_const(wm->src_width);
9046 bandwidth.full = dfixed_mul(src_width, bpp);
9047 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9048 bandwidth.full = dfixed_div(bandwidth, line_time);
9049
9050 return dfixed_trunc(bandwidth);
9051}
9052
9053/**
9054 * dce8_latency_watermark - get the latency watermark
9055 *
9056 * @wm: watermark calculation data
9057 *
9058 * Calculate the latency watermark (CIK).
9059 * Used for display watermark bandwidth calculations
9060 * Returns the latency watermark in ns
9061 */
9062static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9063{
9064 /* First calculate the latency in ns */
9065 u32 mc_latency = 2000; /* 2000 ns. */
9066 u32 available_bandwidth = dce8_available_bandwidth(wm);
9067 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9068 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9069 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9070 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9071 (wm->num_heads * cursor_line_pair_return_time);
9072 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9073 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9074 u32 tmp, dmif_size = 12288;
9075 fixed20_12 a, b, c;
9076
9077 if (wm->num_heads == 0)
9078 return 0;
9079
9080 a.full = dfixed_const(2);
9081 b.full = dfixed_const(1);
9082 if ((wm->vsc.full > a.full) ||
9083 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9084 (wm->vtaps >= 5) ||
9085 ((wm->vsc.full >= a.full) && wm->interlaced))
9086 max_src_lines_per_dst_line = 4;
9087 else
9088 max_src_lines_per_dst_line = 2;
9089
9090 a.full = dfixed_const(available_bandwidth);
9091 b.full = dfixed_const(wm->num_heads);
9092 a.full = dfixed_div(a, b);
ae45bbc2
MK		 9093 tmp = div_u64((u64) dmif_size * (u64) wm->disp_clk, mc_latency + 512);
9094 tmp = min(dfixed_trunc(a), tmp);
cd84a27d 9095
ae45bbc2 9096 lb_fill_bw = min(tmp, wm->disp_clk * wm->bytes_per_pixel / 1000);
cd84a27d
AD		 9097
9098 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9099 b.full = dfixed_const(1000);
9100 c.full = dfixed_const(lb_fill_bw);
9101 b.full = dfixed_div(c, b);
9102 a.full = dfixed_div(a, b);
9103 line_fill_time = dfixed_trunc(a);
9104
9105 if (line_fill_time < wm->active_time)
9106 return latency;
9107 else
9108 return latency + (line_fill_time - wm->active_time);
9109
9110}
9111
9112/**
9113 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9114 * average and available dram bandwidth
9115 *
9116 * @wm: watermark calculation data
9117 *
9118 * Check if the display average bandwidth fits in the display
9119 * dram bandwidth (CIK).
9120 * Used for display watermark bandwidth calculations
9121 * Returns true if the display fits, false if not.
9122 */
9123static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9124{
9125 if (dce8_average_bandwidth(wm) <=
9126 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9127 return true;
9128 else
9129 return false;
9130}
9131
9132/**
9133 * dce8_average_bandwidth_vs_available_bandwidth - check
9134 * average and available bandwidth
9135 *
9136 * @wm: watermark calculation data
9137 *
9138 * Check if the display average bandwidth fits in the display
9139 * available bandwidth (CIK).
9140 * Used for display watermark bandwidth calculations
9141 * Returns true if the display fits, false if not.
9142 */
9143static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9144{
9145 if (dce8_average_bandwidth(wm) <=
9146 (dce8_available_bandwidth(wm) / wm->num_heads))
9147 return true;
9148 else
9149 return false;
9150}
9151
9152/**
9153 * dce8_check_latency_hiding - check latency hiding
9154 *
9155 * @wm: watermark calculation data
9156 *
9157 * Check latency hiding (CIK).
9158 * Used for display watermark bandwidth calculations
9159 * Returns true if the display fits, false if not.
9160 */
9161static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9162{
9163 u32 lb_partitions = wm->lb_size / wm->src_width;
9164 u32 line_time = wm->active_time + wm->blank_time;
9165 u32 latency_tolerant_lines;
9166 u32 latency_hiding;
9167 fixed20_12 a;
9168
9169 a.full = dfixed_const(1);
9170 if (wm->vsc.full > a.full)
9171 latency_tolerant_lines = 1;
9172 else {
9173 if (lb_partitions <= (wm->vtaps + 1))
9174 latency_tolerant_lines = 1;
9175 else
9176 latency_tolerant_lines = 2;
9177 }
9178
9179 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9180
9181 if (dce8_latency_watermark(wm) <= latency_hiding)
9182 return true;
9183 else
9184 return false;
9185}
9186
9187/**
9188 * dce8_program_watermarks - program display watermarks
9189 *
9190 * @rdev: radeon_device pointer
9191 * @radeon_crtc: the selected display controller
9192 * @lb_size: line buffer size
9193 * @num_heads: number of display controllers in use
9194 *
9195 * Calculate and program the display watermarks for the
9196 * selected display controller (CIK).
9197 */
9198static void dce8_program_watermarks(struct radeon_device *rdev,
9199 struct radeon_crtc *radeon_crtc,
9200 u32 lb_size, u32 num_heads)
9201{
9202 struct drm_display_mode *mode = &radeon_crtc->base.mode;
58ea2dea 9203 struct dce8_wm_params wm_low, wm_high;
e6b9a6c8 9204 u32 active_time;
cd84a27d
AD		 9205 u32 line_time = 0;
9206 u32 latency_watermark_a = 0, latency_watermark_b = 0;
9207 u32 tmp, wm_mask;
9208
9209 if (radeon_crtc->base.enabled && num_heads && mode) {
55f61a04
MK		 9210 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
9211 (u32)mode->clock);
9212 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
9213 (u32)mode->clock);
9214 line_time = min(line_time, (u32)65535);
cd84a27d 9215
58ea2dea
AD		 9216 /* watermark for high clocks */
9217 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9218 rdev->pm.dpm_enabled) {
9219 wm_high.yclk =
9220 radeon_dpm_get_mclk(rdev, false) * 10;
9221 wm_high.sclk =
9222 radeon_dpm_get_sclk(rdev, false) * 10;
9223 } else {
9224 wm_high.yclk = rdev->pm.current_mclk * 10;
9225 wm_high.sclk = rdev->pm.current_sclk * 10;
9226 }
9227
9228 wm_high.disp_clk = mode->clock;
9229 wm_high.src_width = mode->crtc_hdisplay;
e6b9a6c8 9230 wm_high.active_time = active_time;
58ea2dea
AD		 9231 wm_high.blank_time = line_time - wm_high.active_time;
9232 wm_high.interlaced = false;
cd84a27d 9233 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
58ea2dea
AD		 9234 wm_high.interlaced = true;
9235 wm_high.vsc = radeon_crtc->vsc;
9236 wm_high.vtaps = 1;
cd84a27d 9237 if (radeon_crtc->rmx_type != RMX_OFF)
58ea2dea
AD		 9238 wm_high.vtaps = 2;
9239 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9240 wm_high.lb_size = lb_size;
9241 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9242 wm_high.num_heads = num_heads;
cd84a27d
AD		 9243
9244 /* set for high clocks */
58ea2dea
AD		 9245 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9246
9247 /* possibly force display priority to high */
9248 /* should really do this at mode validation time... */
9249 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9250 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9251 !dce8_check_latency_hiding(&wm_high) ||
9252 (rdev->disp_priority == 2)) {
9253 DRM_DEBUG_KMS("force priority to high\n");
9254 }
9255
9256 /* watermark for low clocks */
9257 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9258 rdev->pm.dpm_enabled) {
9259 wm_low.yclk =
9260 radeon_dpm_get_mclk(rdev, true) * 10;
9261 wm_low.sclk =
9262 radeon_dpm_get_sclk(rdev, true) * 10;
9263 } else {
9264 wm_low.yclk = rdev->pm.current_mclk * 10;
9265 wm_low.sclk = rdev->pm.current_sclk * 10;
9266 }
9267
9268 wm_low.disp_clk = mode->clock;
9269 wm_low.src_width = mode->crtc_hdisplay;
e6b9a6c8 9270 wm_low.active_time = active_time;
58ea2dea
AD		 9271 wm_low.blank_time = line_time - wm_low.active_time;
9272 wm_low.interlaced = false;
9273 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9274 wm_low.interlaced = true;
9275 wm_low.vsc = radeon_crtc->vsc;
9276 wm_low.vtaps = 1;
9277 if (radeon_crtc->rmx_type != RMX_OFF)
9278 wm_low.vtaps = 2;
9279 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9280 wm_low.lb_size = lb_size;
9281 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9282 wm_low.num_heads = num_heads;
9283
cd84a27d 9284 /* set for low clocks */
58ea2dea 9285 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
cd84a27d
AD		 9286
9287 /* possibly force display priority to high */
9288 /* should really do this at mode validation time... */
58ea2dea
AD		 9289 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9290 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9291 !dce8_check_latency_hiding(&wm_low) ||
cd84a27d
AD		 9292 (rdev->disp_priority == 2)) {
9293 DRM_DEBUG_KMS("force priority to high\n");
9294 }
5b5561b3
MK		 9295
9296 /* Save number of lines the linebuffer leads before the scanout */
9297 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
cd84a27d
AD		 9298 }
9299
9300 /* select wm A */
9301 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9302 tmp = wm_mask;
9303 tmp &= ~LATENCY_WATERMARK_MASK(3);
9304 tmp |= LATENCY_WATERMARK_MASK(1);
9305 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9306 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9307 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9308 LATENCY_HIGH_WATERMARK(line_time)));
9309 /* select wm B */
9310 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9311 tmp &= ~LATENCY_WATERMARK_MASK(3);
9312 tmp |= LATENCY_WATERMARK_MASK(2);
9313 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9314 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9315 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9316 LATENCY_HIGH_WATERMARK(line_time)));
9317 /* restore original selection */
9318 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
58ea2dea
AD		 9319
9320 /* save values for DPM */
9321 radeon_crtc->line_time = line_time;
9322 radeon_crtc->wm_high = latency_watermark_a;
9323 radeon_crtc->wm_low = latency_watermark_b;
cd84a27d
AD		 9324}
9325
9326/**
9327 * dce8_bandwidth_update - program display watermarks
9328 *
9329 * @rdev: radeon_device pointer
9330 *
9331 * Calculate and program the display watermarks and line
9332 * buffer allocation (CIK).
9333 */
9334void dce8_bandwidth_update(struct radeon_device *rdev)
9335{
9336 struct drm_display_mode *mode = NULL;
9337 u32 num_heads = 0, lb_size;
9338 int i;
9339
8efe82ca
AD		 9340 if (!rdev->mode_info.mode_config_initialized)
9341 return;
9342
cd84a27d
AD		 9343 radeon_update_display_priority(rdev);
9344
9345 for (i = 0; i < rdev->num_crtc; i++) {
9346 if (rdev->mode_info.crtcs[i]->base.enabled)
9347 num_heads++;
9348 }
9349 for (i = 0; i < rdev->num_crtc; i++) {
9350 mode = &rdev->mode_info.crtcs[i]->base.mode;
9351 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9352 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9353 }
9354}
44fa346f
AD		 9355
9356/**
9357 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9358 *
9359 * @rdev: radeon_device pointer
9360 *
9361 * Fetches a GPU clock counter snapshot (SI).
9362 * Returns the 64 bit clock counter snapshot.
9363 */
9364uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9365{
9366 uint64_t clock;
9367
9368 mutex_lock(&rdev->gpu_clock_mutex);
9369 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9370 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
3cf8bb1a 9371 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
44fa346f
AD		 9372 mutex_unlock(&rdev->gpu_clock_mutex);
9373 return clock;
9374}
9375
87167bb1 9376static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
3cf8bb1a 9377 u32 cntl_reg, u32 status_reg)
87167bb1
CK		 9378{
9379 int r, i;
9380 struct atom_clock_dividers dividers;
9381 uint32_t tmp;
9382
9383 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9384 clock, false, &dividers);
9385 if (r)
9386 return r;
9387
9388 tmp = RREG32_SMC(cntl_reg);
9389 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9390 tmp |= dividers.post_divider;
9391 WREG32_SMC(cntl_reg, tmp);
9392
9393 for (i = 0; i < 100; i++) {
9394 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9395 break;
9396 mdelay(10);
9397 }
9398 if (i == 100)
9399 return -ETIMEDOUT;
9400
9401 return 0;
9402}
9403
9404int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9405{
9406 int r = 0;
9407
9408 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9409 if (r)
9410 return r;
9411
9412 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9413 return r;
9414}
9415
5ad6bf91
AD		 9416int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9417{
9418 int r, i;
9419 struct atom_clock_dividers dividers;
9420 u32 tmp;
9421
9422 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9423 ecclk, false, &dividers);
9424 if (r)
9425 return r;
9426
9427 for (i = 0; i < 100; i++) {
9428 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9429 break;
9430 mdelay(10);
9431 }
9432 if (i == 100)
9433 return -ETIMEDOUT;
9434
9435 tmp = RREG32_SMC(CG_ECLK_CNTL);
9436 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9437 tmp |= dividers.post_divider;
9438 WREG32_SMC(CG_ECLK_CNTL, tmp);
9439
9440 for (i = 0; i < 100; i++) {
9441 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9442 break;
9443 mdelay(10);
9444 }
9445 if (i == 100)
9446 return -ETIMEDOUT;
9447
9448 return 0;
9449}
9450
8a7cd276 9451static void cik_pcie_gen3_enable(struct radeon_device *rdev)
87167bb1 9452{
8a7cd276
AD		 9453 struct pci_dev *root = rdev->pdev->bus->self;
9454 int bridge_pos, gpu_pos;
9455 u32 speed_cntl, mask, current_data_rate;
9456 int ret, i;
9457 u16 tmp16;
87167bb1 9458
0bd252de
AW		 9459 if (pci_is_root_bus(rdev->pdev->bus))
9460 return;
9461
8a7cd276
AD		 9462 if (radeon_pcie_gen2 == 0)
9463 return;
87167bb1 9464
8a7cd276
AD		 9465 if (rdev->flags & RADEON_IS_IGP)
9466 return;
87167bb1 9467
8a7cd276
AD		 9468 if (!(rdev->flags & RADEON_IS_PCIE))
9469 return;
87167bb1 9470
8a7cd276
AD		 9471 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9472 if (ret != 0)
9473 return;
87167bb1 9474
8a7cd276
AD		 9475 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9476 return;
87167bb1 9477
8a7cd276
AD		 9478 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9479 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9480 LC_CURRENT_DATA_RATE_SHIFT;
9481 if (mask & DRM_PCIE_SPEED_80) {
9482 if (current_data_rate == 2) {
9483 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9484 return;
9485 }
9486 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9487 } else if (mask & DRM_PCIE_SPEED_50) {
9488 if (current_data_rate == 1) {
9489 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9490 return;
9491 }
9492 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9493 }
87167bb1 9494
8a7cd276
AD		 9495 bridge_pos = pci_pcie_cap(root);
9496 if (!bridge_pos)
9497 return;
9498
9499 gpu_pos = pci_pcie_cap(rdev->pdev);
9500 if (!gpu_pos)
9501 return;
9502
9503 if (mask & DRM_PCIE_SPEED_80) {
9504 /* re-try equalization if gen3 is not already enabled */
9505 if (current_data_rate != 2) {
9506 u16 bridge_cfg, gpu_cfg;
9507 u16 bridge_cfg2, gpu_cfg2;
9508 u32 max_lw, current_lw, tmp;
9509
9510 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9511 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9512
9513 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9514 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9515
9516 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9517 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9518
9519 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9520 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9521 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9522
9523 if (current_lw < max_lw) {
9524 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9525 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9526 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9527 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9528 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9529 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9530 }
9531 }
9532
9533 for (i = 0; i < 10; i++) {
9534 /* check status */
9535 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9536 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9537 break;
9538
9539 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9540 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9541
9542 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9543 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9544
9545 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9546 tmp |= LC_SET_QUIESCE;
9547 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9548
9549 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9550 tmp |= LC_REDO_EQ;
9551 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9552
9553 mdelay(100);
9554
9555 /* linkctl */
9556 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9557 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9558 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9559 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9560
9561 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9562 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9563 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9564 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9565
9566 /* linkctl2 */
9567 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9568 tmp16 &= ~((1 << 4) | (7 << 9));
9569 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9570 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9571
9572 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9573 tmp16 &= ~((1 << 4) | (7 << 9));
9574 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9575 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9576
9577 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9578 tmp &= ~LC_SET_QUIESCE;
9579 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9580 }
9581 }
9582 }
9583
9584 /* set the link speed */
9585 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9586 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9587 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9588
9589 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9590 tmp16 &= ~0xf;
9591 if (mask & DRM_PCIE_SPEED_80)
9592 tmp16 |= 3; /* gen3 */
9593 else if (mask & DRM_PCIE_SPEED_50)
9594 tmp16 |= 2; /* gen2 */
9595 else
9596 tmp16 |= 1; /* gen1 */
9597 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9598
9599 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9600 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9601 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9602
9603 for (i = 0; i < rdev->usec_timeout; i++) {
9604 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9605 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9606 break;
9607 udelay(1);
9608 }
9609}
7235711a
AD		 9610
9611static void cik_program_aspm(struct radeon_device *rdev)
9612{
9613 u32 data, orig;
9614 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9615 bool disable_clkreq = false;
9616
9617 if (radeon_aspm == 0)
9618 return;
9619
9620 /* XXX double check IGPs */
9621 if (rdev->flags & RADEON_IS_IGP)
9622 return;
9623
9624 if (!(rdev->flags & RADEON_IS_PCIE))
9625 return;
9626
9627 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9628 data &= ~LC_XMIT_N_FTS_MASK;
9629 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9630 if (orig != data)
9631 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9632
9633 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9634 data |= LC_GO_TO_RECOVERY;
9635 if (orig != data)
9636 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9637
9638 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9639 data |= P_IGNORE_EDB_ERR;
9640 if (orig != data)
9641 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9642
9643 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9644 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9645 data |= LC_PMI_TO_L1_DIS;
9646 if (!disable_l0s)
9647 data |= LC_L0S_INACTIVITY(7);
9648
9649 if (!disable_l1) {
9650 data |= LC_L1_INACTIVITY(7);
9651 data &= ~LC_PMI_TO_L1_DIS;
9652 if (orig != data)
9653 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9654
9655 if (!disable_plloff_in_l1) {
9656 bool clk_req_support;
9657
9658 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9659 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9660 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9661 if (orig != data)
9662 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9663
9664 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9665 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9666 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9667 if (orig != data)
9668 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9669
9670 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9671 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9672 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9673 if (orig != data)
9674 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9675
9676 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9677 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9678 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9679 if (orig != data)
9680 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9681
9682 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9683 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9684 data |= LC_DYN_LANES_PWR_STATE(3);
9685 if (orig != data)
9686 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9687
0bd252de
AW		 9688 if (!disable_clkreq &&
9689 !pci_is_root_bus(rdev->pdev->bus)) {
7235711a
AD		 9690 struct pci_dev *root = rdev->pdev->bus->self;
9691 u32 lnkcap;
9692
9693 clk_req_support = false;
9694 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9695 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9696 clk_req_support = true;
9697 } else {
9698 clk_req_support = false;
9699 }
9700
9701 if (clk_req_support) {
9702 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9703 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9704 if (orig != data)
9705 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9706
9707 orig = data = RREG32_SMC(THM_CLK_CNTL);
9708 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9709 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9710 if (orig != data)
9711 WREG32_SMC(THM_CLK_CNTL, data);
9712
9713 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9714 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9715 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9716 if (orig != data)
9717 WREG32_SMC(MISC_CLK_CTRL, data);
9718
9719 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9720 data &= ~BCLK_AS_XCLK;
9721 if (orig != data)
9722 WREG32_SMC(CG_CLKPIN_CNTL, data);
9723
9724 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9725 data &= ~FORCE_BIF_REFCLK_EN;
9726 if (orig != data)
9727 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9728
9729 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9730 data &= ~MPLL_CLKOUT_SEL_MASK;
9731 data |= MPLL_CLKOUT_SEL(4);
9732 if (orig != data)
9733 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9734 }
9735 }
9736 } else {
9737 if (orig != data)
9738 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9739 }
9740
9741 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9742 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9743 if (orig != data)
9744 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9745
9746 if (!disable_l0s) {
9747 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9748 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9749 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9750 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9751 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9752 data &= ~LC_L0S_INACTIVITY_MASK;
9753 if (orig != data)
9754 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9755 }
9756 }
9757 }
87167bb1 9758}
ubuntu-bionic-kernel mirror