]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/gpu/drm/radeon/cik.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge remote-tracking branches 'regmap/topic/mmio', 'regmap/topic/rbtree' and 'regmap...
[mirror_ubuntu-artful-kernel.git] / drivers / gpu / drm / radeon / cik.c
CommitLineData
8cc1a532
AD		 1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
8cc1a532
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
6f2043ce 29#include "radeon_asic.h"
bfc1f97d 30#include "radeon_audio.h"
8cc1a532
AD		 31#include "cikd.h"
32#include "atom.h"
841cf442 33#include "cik_blit_shaders.h"
8c68e393 34#include "radeon_ucode.h"
22c775ce 35#include "clearstate_ci.h"
e28740ec 36#include "radeon_kfd.h"
02c81327
AD		 37
38MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
39MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
40MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
41MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
42MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
277babc3 43MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
02c81327 44MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
21a93e13 45MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
cc8dbbb4 46MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
f2c6b0f4
AD		 47
48MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
49MODULE_FIRMWARE("radeon/bonaire_me.bin");
50MODULE_FIRMWARE("radeon/bonaire_ce.bin");
51MODULE_FIRMWARE("radeon/bonaire_mec.bin");
52MODULE_FIRMWARE("radeon/bonaire_mc.bin");
53MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
54MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
55MODULE_FIRMWARE("radeon/bonaire_smc.bin");
56
d4775655
AD		 57MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
58MODULE_FIRMWARE("radeon/HAWAII_me.bin");
59MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
60MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
61MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
277babc3 62MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
d4775655
AD		 63MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
64MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
65MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
f2c6b0f4
AD		 66
67MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
68MODULE_FIRMWARE("radeon/hawaii_me.bin");
69MODULE_FIRMWARE("radeon/hawaii_ce.bin");
70MODULE_FIRMWARE("radeon/hawaii_mec.bin");
71MODULE_FIRMWARE("radeon/hawaii_mc.bin");
72MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
73MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
74MODULE_FIRMWARE("radeon/hawaii_smc.bin");
75
02c81327
AD		 76MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
77MODULE_FIRMWARE("radeon/KAVERI_me.bin");
78MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
79MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
80MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
21a93e13 81MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
f2c6b0f4
AD		 82
83MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
84MODULE_FIRMWARE("radeon/kaveri_me.bin");
85MODULE_FIRMWARE("radeon/kaveri_ce.bin");
86MODULE_FIRMWARE("radeon/kaveri_mec.bin");
87MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
88MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
89MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
90
02c81327
AD		 91MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
92MODULE_FIRMWARE("radeon/KABINI_me.bin");
93MODULE_FIRMWARE("radeon/KABINI_ce.bin");
94MODULE_FIRMWARE("radeon/KABINI_mec.bin");
95MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
21a93e13 96MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
f2c6b0f4
AD		 97
98MODULE_FIRMWARE("radeon/kabini_pfp.bin");
99MODULE_FIRMWARE("radeon/kabini_me.bin");
100MODULE_FIRMWARE("radeon/kabini_ce.bin");
101MODULE_FIRMWARE("radeon/kabini_mec.bin");
102MODULE_FIRMWARE("radeon/kabini_rlc.bin");
103MODULE_FIRMWARE("radeon/kabini_sdma.bin");
104
f73a9e83
SL		 105MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
106MODULE_FIRMWARE("radeon/MULLINS_me.bin");
107MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
108MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
109MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
110MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
02c81327 111
f2c6b0f4
AD		 112MODULE_FIRMWARE("radeon/mullins_pfp.bin");
113MODULE_FIRMWARE("radeon/mullins_me.bin");
114MODULE_FIRMWARE("radeon/mullins_ce.bin");
115MODULE_FIRMWARE("radeon/mullins_mec.bin");
116MODULE_FIRMWARE("radeon/mullins_rlc.bin");
117MODULE_FIRMWARE("radeon/mullins_sdma.bin");
118
a59781bb
AD		 119extern int r600_ih_ring_alloc(struct radeon_device *rdev);
120extern void r600_ih_ring_fini(struct radeon_device *rdev);
6f2043ce
AD		 121extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
122extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
cc066715 123extern bool evergreen_is_display_hung(struct radeon_device *rdev);
1fd11777
AD		 124extern void sumo_rlc_fini(struct radeon_device *rdev);
125extern int sumo_rlc_init(struct radeon_device *rdev);
1c49165d 126extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
866d83de 127extern void si_rlc_reset(struct radeon_device *rdev);
22c775ce 128extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
65fcf668 129static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
2483b4ea
CK		 130extern int cik_sdma_resume(struct radeon_device *rdev);
131extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
132extern void cik_sdma_fini(struct radeon_device *rdev);
a1d6f97c 133extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
cc066715 134static void cik_rlc_stop(struct radeon_device *rdev);
8a7cd276 135static void cik_pcie_gen3_enable(struct radeon_device *rdev);
7235711a 136static void cik_program_aspm(struct radeon_device *rdev);
22c775ce
AD		 137static void cik_init_pg(struct radeon_device *rdev);
138static void cik_init_cg(struct radeon_device *rdev);
fb2c7f4d
AD		 139static void cik_fini_pg(struct radeon_device *rdev);
140static void cik_fini_cg(struct radeon_device *rdev);
4214faf6
AD		 141static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
142 bool enable);
6f2043ce 143
353eec2a
AD		 144/**
145 * cik_get_allowed_info_register - fetch the register for the info ioctl
146 *
147 * @rdev: radeon_device pointer
148 * @reg: register offset in bytes
149 * @val: register value
150 *
151 * Returns 0 for success or -EINVAL for an invalid register
152 *
153 */
154int cik_get_allowed_info_register(struct radeon_device *rdev,
155 u32 reg, u32 *val)
156{
157 switch (reg) {
158 case GRBM_STATUS:
159 case GRBM_STATUS2:
160 case GRBM_STATUS_SE0:
161 case GRBM_STATUS_SE1:
162 case GRBM_STATUS_SE2:
163 case GRBM_STATUS_SE3:
164 case SRBM_STATUS:
165 case SRBM_STATUS2:
166 case (SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET):
167 case (SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET):
168 case UVD_STATUS:
169 /* TODO VCE */
170 *val = RREG32(reg);
171 return 0;
172 default:
173 return -EINVAL;
174 }
175}
176
9e5acbc2
DV		 177/*
178 * Indirect registers accessor
179 */
180u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg)
181{
182 unsigned long flags;
183 u32 r;
184
185 spin_lock_irqsave(&rdev->didt_idx_lock, flags);
186 WREG32(CIK_DIDT_IND_INDEX, (reg));
187 r = RREG32(CIK_DIDT_IND_DATA);
188 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
189 return r;
190}
191
192void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v)
193{
194 unsigned long flags;
195
196 spin_lock_irqsave(&rdev->didt_idx_lock, flags);
197 WREG32(CIK_DIDT_IND_INDEX, (reg));
198 WREG32(CIK_DIDT_IND_DATA, (v));
199 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
200}
201
286d9cc6
AD		 202/* get temperature in millidegrees */
203int ci_get_temp(struct radeon_device *rdev)
204{
205 u32 temp;
206 int actual_temp = 0;
207
208 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
209 CTF_TEMP_SHIFT;
210
211 if (temp & 0x200)
212 actual_temp = 255;
213 else
214 actual_temp = temp & 0x1ff;
215
216 actual_temp = actual_temp * 1000;
217
218 return actual_temp;
219}
220
221/* get temperature in millidegrees */
222int kv_get_temp(struct radeon_device *rdev)
223{
224 u32 temp;
225 int actual_temp = 0;
226
227 temp = RREG32_SMC(0xC0300E0C);
228
229 if (temp)
230 actual_temp = (temp / 8) - 49;
231 else
232 actual_temp = 0;
233
234 actual_temp = actual_temp * 1000;
235
236 return actual_temp;
237}
6f2043ce 238
6e2c3c0a
AD		 239/*
240 * Indirect registers accessor
241 */
242u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
243{
0a5b7b0b 244 unsigned long flags;
6e2c3c0a
AD		 245 u32 r;
246
0a5b7b0b 247 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 248 WREG32(PCIE_INDEX, reg);
249 (void)RREG32(PCIE_INDEX);
250 r = RREG32(PCIE_DATA);
0a5b7b0b 251 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 252 return r;
253}
254
255void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
256{
0a5b7b0b
AD		 257 unsigned long flags;
258
259 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 260 WREG32(PCIE_INDEX, reg);
261 (void)RREG32(PCIE_INDEX);
262 WREG32(PCIE_DATA, v);
263 (void)RREG32(PCIE_DATA);
0a5b7b0b 264 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 265}
266
22c775ce
AD		 267static const u32 spectre_rlc_save_restore_register_list[] =
268{
269 (0x0e00 << 16) | (0xc12c >> 2),
270 0x00000000,
271 (0x0e00 << 16) | (0xc140 >> 2),
272 0x00000000,
273 (0x0e00 << 16) | (0xc150 >> 2),
274 0x00000000,
275 (0x0e00 << 16) | (0xc15c >> 2),
276 0x00000000,
277 (0x0e00 << 16) | (0xc168 >> 2),
278 0x00000000,
279 (0x0e00 << 16) | (0xc170 >> 2),
280 0x00000000,
281 (0x0e00 << 16) | (0xc178 >> 2),
282 0x00000000,
283 (0x0e00 << 16) | (0xc204 >> 2),
284 0x00000000,
285 (0x0e00 << 16) | (0xc2b4 >> 2),
286 0x00000000,
287 (0x0e00 << 16) | (0xc2b8 >> 2),
288 0x00000000,
289 (0x0e00 << 16) | (0xc2bc >> 2),
290 0x00000000,
291 (0x0e00 << 16) | (0xc2c0 >> 2),
292 0x00000000,
293 (0x0e00 << 16) | (0x8228 >> 2),
294 0x00000000,
295 (0x0e00 << 16) | (0x829c >> 2),
296 0x00000000,
297 (0x0e00 << 16) | (0x869c >> 2),
298 0x00000000,
299 (0x0600 << 16) | (0x98f4 >> 2),
300 0x00000000,
301 (0x0e00 << 16) | (0x98f8 >> 2),
302 0x00000000,
303 (0x0e00 << 16) | (0x9900 >> 2),
304 0x00000000,
305 (0x0e00 << 16) | (0xc260 >> 2),
306 0x00000000,
307 (0x0e00 << 16) | (0x90e8 >> 2),
308 0x00000000,
309 (0x0e00 << 16) | (0x3c000 >> 2),
310 0x00000000,
311 (0x0e00 << 16) | (0x3c00c >> 2),
312 0x00000000,
313 (0x0e00 << 16) | (0x8c1c >> 2),
314 0x00000000,
315 (0x0e00 << 16) | (0x9700 >> 2),
316 0x00000000,
317 (0x0e00 << 16) | (0xcd20 >> 2),
318 0x00000000,
319 (0x4e00 << 16) | (0xcd20 >> 2),
320 0x00000000,
321 (0x5e00 << 16) | (0xcd20 >> 2),
322 0x00000000,
323 (0x6e00 << 16) | (0xcd20 >> 2),
324 0x00000000,
325 (0x7e00 << 16) | (0xcd20 >> 2),
326 0x00000000,
327 (0x8e00 << 16) | (0xcd20 >> 2),
328 0x00000000,
329 (0x9e00 << 16) | (0xcd20 >> 2),
330 0x00000000,
331 (0xae00 << 16) | (0xcd20 >> 2),
332 0x00000000,
333 (0xbe00 << 16) | (0xcd20 >> 2),
334 0x00000000,
335 (0x0e00 << 16) | (0x89bc >> 2),
336 0x00000000,
337 (0x0e00 << 16) | (0x8900 >> 2),
338 0x00000000,
339 0x3,
340 (0x0e00 << 16) | (0xc130 >> 2),
341 0x00000000,
342 (0x0e00 << 16) | (0xc134 >> 2),
343 0x00000000,
344 (0x0e00 << 16) | (0xc1fc >> 2),
345 0x00000000,
346 (0x0e00 << 16) | (0xc208 >> 2),
347 0x00000000,
348 (0x0e00 << 16) | (0xc264 >> 2),
349 0x00000000,
350 (0x0e00 << 16) | (0xc268 >> 2),
351 0x00000000,
352 (0x0e00 << 16) | (0xc26c >> 2),
353 0x00000000,
354 (0x0e00 << 16) | (0xc270 >> 2),
355 0x00000000,
356 (0x0e00 << 16) | (0xc274 >> 2),
357 0x00000000,
358 (0x0e00 << 16) | (0xc278 >> 2),
359 0x00000000,
360 (0x0e00 << 16) | (0xc27c >> 2),
361 0x00000000,
362 (0x0e00 << 16) | (0xc280 >> 2),
363 0x00000000,
364 (0x0e00 << 16) | (0xc284 >> 2),
365 0x00000000,
366 (0x0e00 << 16) | (0xc288 >> 2),
367 0x00000000,
368 (0x0e00 << 16) | (0xc28c >> 2),
369 0x00000000,
370 (0x0e00 << 16) | (0xc290 >> 2),
371 0x00000000,
372 (0x0e00 << 16) | (0xc294 >> 2),
373 0x00000000,
374 (0x0e00 << 16) | (0xc298 >> 2),
375 0x00000000,
376 (0x0e00 << 16) | (0xc29c >> 2),
377 0x00000000,
378 (0x0e00 << 16) | (0xc2a0 >> 2),
379 0x00000000,
380 (0x0e00 << 16) | (0xc2a4 >> 2),
381 0x00000000,
382 (0x0e00 << 16) | (0xc2a8 >> 2),
383 0x00000000,
384 (0x0e00 << 16) | (0xc2ac >> 2),
385 0x00000000,
386 (0x0e00 << 16) | (0xc2b0 >> 2),
387 0x00000000,
388 (0x0e00 << 16) | (0x301d0 >> 2),
389 0x00000000,
390 (0x0e00 << 16) | (0x30238 >> 2),
391 0x00000000,
392 (0x0e00 << 16) | (0x30250 >> 2),
393 0x00000000,
394 (0x0e00 << 16) | (0x30254 >> 2),
395 0x00000000,
396 (0x0e00 << 16) | (0x30258 >> 2),
397 0x00000000,
398 (0x0e00 << 16) | (0x3025c >> 2),
399 0x00000000,
400 (0x4e00 << 16) | (0xc900 >> 2),
401 0x00000000,
402 (0x5e00 << 16) | (0xc900 >> 2),
403 0x00000000,
404 (0x6e00 << 16) | (0xc900 >> 2),
405 0x00000000,
406 (0x7e00 << 16) | (0xc900 >> 2),
407 0x00000000,
408 (0x8e00 << 16) | (0xc900 >> 2),
409 0x00000000,
410 (0x9e00 << 16) | (0xc900 >> 2),
411 0x00000000,
412 (0xae00 << 16) | (0xc900 >> 2),
413 0x00000000,
414 (0xbe00 << 16) | (0xc900 >> 2),
415 0x00000000,
416 (0x4e00 << 16) | (0xc904 >> 2),
417 0x00000000,
418 (0x5e00 << 16) | (0xc904 >> 2),
419 0x00000000,
420 (0x6e00 << 16) | (0xc904 >> 2),
421 0x00000000,
422 (0x7e00 << 16) | (0xc904 >> 2),
423 0x00000000,
424 (0x8e00 << 16) | (0xc904 >> 2),
425 0x00000000,
426 (0x9e00 << 16) | (0xc904 >> 2),
427 0x00000000,
428 (0xae00 << 16) | (0xc904 >> 2),
429 0x00000000,
430 (0xbe00 << 16) | (0xc904 >> 2),
431 0x00000000,
432 (0x4e00 << 16) | (0xc908 >> 2),
433 0x00000000,
434 (0x5e00 << 16) | (0xc908 >> 2),
435 0x00000000,
436 (0x6e00 << 16) | (0xc908 >> 2),
437 0x00000000,
438 (0x7e00 << 16) | (0xc908 >> 2),
439 0x00000000,
440 (0x8e00 << 16) | (0xc908 >> 2),
441 0x00000000,
442 (0x9e00 << 16) | (0xc908 >> 2),
443 0x00000000,
444 (0xae00 << 16) | (0xc908 >> 2),
445 0x00000000,
446 (0xbe00 << 16) | (0xc908 >> 2),
447 0x00000000,
448 (0x4e00 << 16) | (0xc90c >> 2),
449 0x00000000,
450 (0x5e00 << 16) | (0xc90c >> 2),
451 0x00000000,
452 (0x6e00 << 16) | (0xc90c >> 2),
453 0x00000000,
454 (0x7e00 << 16) | (0xc90c >> 2),
455 0x00000000,
456 (0x8e00 << 16) | (0xc90c >> 2),
457 0x00000000,
458 (0x9e00 << 16) | (0xc90c >> 2),
459 0x00000000,
460 (0xae00 << 16) | (0xc90c >> 2),
461 0x00000000,
462 (0xbe00 << 16) | (0xc90c >> 2),
463 0x00000000,
464 (0x4e00 << 16) | (0xc910 >> 2),
465 0x00000000,
466 (0x5e00 << 16) | (0xc910 >> 2),
467 0x00000000,
468 (0x6e00 << 16) | (0xc910 >> 2),
469 0x00000000,
470 (0x7e00 << 16) | (0xc910 >> 2),
471 0x00000000,
472 (0x8e00 << 16) | (0xc910 >> 2),
473 0x00000000,
474 (0x9e00 << 16) | (0xc910 >> 2),
475 0x00000000,
476 (0xae00 << 16) | (0xc910 >> 2),
477 0x00000000,
478 (0xbe00 << 16) | (0xc910 >> 2),
479 0x00000000,
480 (0x0e00 << 16) | (0xc99c >> 2),
481 0x00000000,
482 (0x0e00 << 16) | (0x9834 >> 2),
483 0x00000000,
484 (0x0000 << 16) | (0x30f00 >> 2),
485 0x00000000,
486 (0x0001 << 16) | (0x30f00 >> 2),
487 0x00000000,
488 (0x0000 << 16) | (0x30f04 >> 2),
489 0x00000000,
490 (0x0001 << 16) | (0x30f04 >> 2),
491 0x00000000,
492 (0x0000 << 16) | (0x30f08 >> 2),
493 0x00000000,
494 (0x0001 << 16) | (0x30f08 >> 2),
495 0x00000000,
496 (0x0000 << 16) | (0x30f0c >> 2),
497 0x00000000,
498 (0x0001 << 16) | (0x30f0c >> 2),
499 0x00000000,
500 (0x0600 << 16) | (0x9b7c >> 2),
501 0x00000000,
502 (0x0e00 << 16) | (0x8a14 >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0x8a18 >> 2),
505 0x00000000,
506 (0x0600 << 16) | (0x30a00 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0x8bf0 >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0x8bcc >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0x8b24 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0x30a04 >> 2),
515 0x00000000,
516 (0x0600 << 16) | (0x30a10 >> 2),
517 0x00000000,
518 (0x0600 << 16) | (0x30a14 >> 2),
519 0x00000000,
520 (0x0600 << 16) | (0x30a18 >> 2),
521 0x00000000,
522 (0x0600 << 16) | (0x30a2c >> 2),
523 0x00000000,
524 (0x0e00 << 16) | (0xc700 >> 2),
525 0x00000000,
526 (0x0e00 << 16) | (0xc704 >> 2),
527 0x00000000,
528 (0x0e00 << 16) | (0xc708 >> 2),
529 0x00000000,
530 (0x0e00 << 16) | (0xc768 >> 2),
531 0x00000000,
532 (0x0400 << 16) | (0xc770 >> 2),
533 0x00000000,
534 (0x0400 << 16) | (0xc774 >> 2),
535 0x00000000,
536 (0x0400 << 16) | (0xc778 >> 2),
537 0x00000000,
538 (0x0400 << 16) | (0xc77c >> 2),
539 0x00000000,
540 (0x0400 << 16) | (0xc780 >> 2),
541 0x00000000,
542 (0x0400 << 16) | (0xc784 >> 2),
543 0x00000000,
544 (0x0400 << 16) | (0xc788 >> 2),
545 0x00000000,
546 (0x0400 << 16) | (0xc78c >> 2),
547 0x00000000,
548 (0x0400 << 16) | (0xc798 >> 2),
549 0x00000000,
550 (0x0400 << 16) | (0xc79c >> 2),
551 0x00000000,
552 (0x0400 << 16) | (0xc7a0 >> 2),
553 0x00000000,
554 (0x0400 << 16) | (0xc7a4 >> 2),
555 0x00000000,
556 (0x0400 << 16) | (0xc7a8 >> 2),
557 0x00000000,
558 (0x0400 << 16) | (0xc7ac >> 2),
559 0x00000000,
560 (0x0400 << 16) | (0xc7b0 >> 2),
561 0x00000000,
562 (0x0400 << 16) | (0xc7b4 >> 2),
563 0x00000000,
564 (0x0e00 << 16) | (0x9100 >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0x3c010 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0x92a8 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0x92ac >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x92b4 >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x92b8 >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x92bc >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x92c0 >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x92c4 >> 2),
581 0x00000000,
582 (0x0e00 << 16) | (0x92c8 >> 2),
583 0x00000000,
584 (0x0e00 << 16) | (0x92cc >> 2),
585 0x00000000,
586 (0x0e00 << 16) | (0x92d0 >> 2),
587 0x00000000,
588 (0x0e00 << 16) | (0x8c00 >> 2),
589 0x00000000,
590 (0x0e00 << 16) | (0x8c04 >> 2),
591 0x00000000,
592 (0x0e00 << 16) | (0x8c20 >> 2),
593 0x00000000,
594 (0x0e00 << 16) | (0x8c38 >> 2),
595 0x00000000,
596 (0x0e00 << 16) | (0x8c3c >> 2),
597 0x00000000,
598 (0x0e00 << 16) | (0xae00 >> 2),
599 0x00000000,
600 (0x0e00 << 16) | (0x9604 >> 2),
601 0x00000000,
602 (0x0e00 << 16) | (0xac08 >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0xac0c >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0xac10 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0xac14 >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0xac58 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0xac68 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0xac6c >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0xac70 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0xac74 >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0xac78 >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0xac7c >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0xac80 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0xac84 >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0xac88 >> 2),
629 0x00000000,
630 (0x0e00 << 16) | (0xac8c >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0x970c >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x9714 >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0x9718 >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x971c >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0x31068 >> 2),
641 0x00000000,
642 (0x4e00 << 16) | (0x31068 >> 2),
643 0x00000000,
644 (0x5e00 << 16) | (0x31068 >> 2),
645 0x00000000,
646 (0x6e00 << 16) | (0x31068 >> 2),
647 0x00000000,
648 (0x7e00 << 16) | (0x31068 >> 2),
649 0x00000000,
650 (0x8e00 << 16) | (0x31068 >> 2),
651 0x00000000,
652 (0x9e00 << 16) | (0x31068 >> 2),
653 0x00000000,
654 (0xae00 << 16) | (0x31068 >> 2),
655 0x00000000,
656 (0xbe00 << 16) | (0x31068 >> 2),
657 0x00000000,
658 (0x0e00 << 16) | (0xcd10 >> 2),
659 0x00000000,
660 (0x0e00 << 16) | (0xcd14 >> 2),
661 0x00000000,
662 (0x0e00 << 16) | (0x88b0 >> 2),
663 0x00000000,
664 (0x0e00 << 16) | (0x88b4 >> 2),
665 0x00000000,
666 (0x0e00 << 16) | (0x88b8 >> 2),
667 0x00000000,
668 (0x0e00 << 16) | (0x88bc >> 2),
669 0x00000000,
670 (0x0400 << 16) | (0x89c0 >> 2),
671 0x00000000,
672 (0x0e00 << 16) | (0x88c4 >> 2),
673 0x00000000,
674 (0x0e00 << 16) | (0x88c8 >> 2),
675 0x00000000,
676 (0x0e00 << 16) | (0x88d0 >> 2),
677 0x00000000,
678 (0x0e00 << 16) | (0x88d4 >> 2),
679 0x00000000,
680 (0x0e00 << 16) | (0x88d8 >> 2),
681 0x00000000,
682 (0x0e00 << 16) | (0x8980 >> 2),
683 0x00000000,
684 (0x0e00 << 16) | (0x30938 >> 2),
685 0x00000000,
686 (0x0e00 << 16) | (0x3093c >> 2),
687 0x00000000,
688 (0x0e00 << 16) | (0x30940 >> 2),
689 0x00000000,
690 (0x0e00 << 16) | (0x89a0 >> 2),
691 0x00000000,
692 (0x0e00 << 16) | (0x30900 >> 2),
693 0x00000000,
694 (0x0e00 << 16) | (0x30904 >> 2),
695 0x00000000,
696 (0x0e00 << 16) | (0x89b4 >> 2),
697 0x00000000,
698 (0x0e00 << 16) | (0x3c210 >> 2),
699 0x00000000,
700 (0x0e00 << 16) | (0x3c214 >> 2),
701 0x00000000,
702 (0x0e00 << 16) | (0x3c218 >> 2),
703 0x00000000,
704 (0x0e00 << 16) | (0x8904 >> 2),
705 0x00000000,
706 0x5,
707 (0x0e00 << 16) | (0x8c28 >> 2),
708 (0x0e00 << 16) | (0x8c2c >> 2),
709 (0x0e00 << 16) | (0x8c30 >> 2),
710 (0x0e00 << 16) | (0x8c34 >> 2),
711 (0x0e00 << 16) | (0x9600 >> 2),
712};
713
714static const u32 kalindi_rlc_save_restore_register_list[] =
715{
716 (0x0e00 << 16) | (0xc12c >> 2),
717 0x00000000,
718 (0x0e00 << 16) | (0xc140 >> 2),
719 0x00000000,
720 (0x0e00 << 16) | (0xc150 >> 2),
721 0x00000000,
722 (0x0e00 << 16) | (0xc15c >> 2),
723 0x00000000,
724 (0x0e00 << 16) | (0xc168 >> 2),
725 0x00000000,
726 (0x0e00 << 16) | (0xc170 >> 2),
727 0x00000000,
728 (0x0e00 << 16) | (0xc204 >> 2),
729 0x00000000,
730 (0x0e00 << 16) | (0xc2b4 >> 2),
731 0x00000000,
732 (0x0e00 << 16) | (0xc2b8 >> 2),
733 0x00000000,
734 (0x0e00 << 16) | (0xc2bc >> 2),
735 0x00000000,
736 (0x0e00 << 16) | (0xc2c0 >> 2),
737 0x00000000,
738 (0x0e00 << 16) | (0x8228 >> 2),
739 0x00000000,
740 (0x0e00 << 16) | (0x829c >> 2),
741 0x00000000,
742 (0x0e00 << 16) | (0x869c >> 2),
743 0x00000000,
744 (0x0600 << 16) | (0x98f4 >> 2),
745 0x00000000,
746 (0x0e00 << 16) | (0x98f8 >> 2),
747 0x00000000,
748 (0x0e00 << 16) | (0x9900 >> 2),
749 0x00000000,
750 (0x0e00 << 16) | (0xc260 >> 2),
751 0x00000000,
752 (0x0e00 << 16) | (0x90e8 >> 2),
753 0x00000000,
754 (0x0e00 << 16) | (0x3c000 >> 2),
755 0x00000000,
756 (0x0e00 << 16) | (0x3c00c >> 2),
757 0x00000000,
758 (0x0e00 << 16) | (0x8c1c >> 2),
759 0x00000000,
760 (0x0e00 << 16) | (0x9700 >> 2),
761 0x00000000,
762 (0x0e00 << 16) | (0xcd20 >> 2),
763 0x00000000,
764 (0x4e00 << 16) | (0xcd20 >> 2),
765 0x00000000,
766 (0x5e00 << 16) | (0xcd20 >> 2),
767 0x00000000,
768 (0x6e00 << 16) | (0xcd20 >> 2),
769 0x00000000,
770 (0x7e00 << 16) | (0xcd20 >> 2),
771 0x00000000,
772 (0x0e00 << 16) | (0x89bc >> 2),
773 0x00000000,
774 (0x0e00 << 16) | (0x8900 >> 2),
775 0x00000000,
776 0x3,
777 (0x0e00 << 16) | (0xc130 >> 2),
778 0x00000000,
779 (0x0e00 << 16) | (0xc134 >> 2),
780 0x00000000,
781 (0x0e00 << 16) | (0xc1fc >> 2),
782 0x00000000,
783 (0x0e00 << 16) | (0xc208 >> 2),
784 0x00000000,
785 (0x0e00 << 16) | (0xc264 >> 2),
786 0x00000000,
787 (0x0e00 << 16) | (0xc268 >> 2),
788 0x00000000,
789 (0x0e00 << 16) | (0xc26c >> 2),
790 0x00000000,
791 (0x0e00 << 16) | (0xc270 >> 2),
792 0x00000000,
793 (0x0e00 << 16) | (0xc274 >> 2),
794 0x00000000,
795 (0x0e00 << 16) | (0xc28c >> 2),
796 0x00000000,
797 (0x0e00 << 16) | (0xc290 >> 2),
798 0x00000000,
799 (0x0e00 << 16) | (0xc294 >> 2),
800 0x00000000,
801 (0x0e00 << 16) | (0xc298 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0xc2a0 >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0xc2a4 >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0xc2a8 >> 2),
808 0x00000000,
809 (0x0e00 << 16) | (0xc2ac >> 2),
810 0x00000000,
811 (0x0e00 << 16) | (0x301d0 >> 2),
812 0x00000000,
813 (0x0e00 << 16) | (0x30238 >> 2),
814 0x00000000,
815 (0x0e00 << 16) | (0x30250 >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0x30254 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0x30258 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0x3025c >> 2),
822 0x00000000,
823 (0x4e00 << 16) | (0xc900 >> 2),
824 0x00000000,
825 (0x5e00 << 16) | (0xc900 >> 2),
826 0x00000000,
827 (0x6e00 << 16) | (0xc900 >> 2),
828 0x00000000,
829 (0x7e00 << 16) | (0xc900 >> 2),
830 0x00000000,
831 (0x4e00 << 16) | (0xc904 >> 2),
832 0x00000000,
833 (0x5e00 << 16) | (0xc904 >> 2),
834 0x00000000,
835 (0x6e00 << 16) | (0xc904 >> 2),
836 0x00000000,
837 (0x7e00 << 16) | (0xc904 >> 2),
838 0x00000000,
839 (0x4e00 << 16) | (0xc908 >> 2),
840 0x00000000,
841 (0x5e00 << 16) | (0xc908 >> 2),
842 0x00000000,
843 (0x6e00 << 16) | (0xc908 >> 2),
844 0x00000000,
845 (0x7e00 << 16) | (0xc908 >> 2),
846 0x00000000,
847 (0x4e00 << 16) | (0xc90c >> 2),
848 0x00000000,
849 (0x5e00 << 16) | (0xc90c >> 2),
850 0x00000000,
851 (0x6e00 << 16) | (0xc90c >> 2),
852 0x00000000,
853 (0x7e00 << 16) | (0xc90c >> 2),
854 0x00000000,
855 (0x4e00 << 16) | (0xc910 >> 2),
856 0x00000000,
857 (0x5e00 << 16) | (0xc910 >> 2),
858 0x00000000,
859 (0x6e00 << 16) | (0xc910 >> 2),
860 0x00000000,
861 (0x7e00 << 16) | (0xc910 >> 2),
862 0x00000000,
863 (0x0e00 << 16) | (0xc99c >> 2),
864 0x00000000,
865 (0x0e00 << 16) | (0x9834 >> 2),
866 0x00000000,
867 (0x0000 << 16) | (0x30f00 >> 2),
868 0x00000000,
869 (0x0000 << 16) | (0x30f04 >> 2),
870 0x00000000,
871 (0x0000 << 16) | (0x30f08 >> 2),
872 0x00000000,
873 (0x0000 << 16) | (0x30f0c >> 2),
874 0x00000000,
875 (0x0600 << 16) | (0x9b7c >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0x8a14 >> 2),
878 0x00000000,
879 (0x0e00 << 16) | (0x8a18 >> 2),
880 0x00000000,
881 (0x0600 << 16) | (0x30a00 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0x8bf0 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0x8bcc >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0x8b24 >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0x30a04 >> 2),
890 0x00000000,
891 (0x0600 << 16) | (0x30a10 >> 2),
892 0x00000000,
893 (0x0600 << 16) | (0x30a14 >> 2),
894 0x00000000,
895 (0x0600 << 16) | (0x30a18 >> 2),
896 0x00000000,
897 (0x0600 << 16) | (0x30a2c >> 2),
898 0x00000000,
899 (0x0e00 << 16) | (0xc700 >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0xc704 >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0xc708 >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0xc768 >> 2),
906 0x00000000,
907 (0x0400 << 16) | (0xc770 >> 2),
908 0x00000000,
909 (0x0400 << 16) | (0xc774 >> 2),
910 0x00000000,
911 (0x0400 << 16) | (0xc798 >> 2),
912 0x00000000,
913 (0x0400 << 16) | (0xc79c >> 2),
914 0x00000000,
915 (0x0e00 << 16) | (0x9100 >> 2),
916 0x00000000,
917 (0x0e00 << 16) | (0x3c010 >> 2),
918 0x00000000,
919 (0x0e00 << 16) | (0x8c00 >> 2),
920 0x00000000,
921 (0x0e00 << 16) | (0x8c04 >> 2),
922 0x00000000,
923 (0x0e00 << 16) | (0x8c20 >> 2),
924 0x00000000,
925 (0x0e00 << 16) | (0x8c38 >> 2),
926 0x00000000,
927 (0x0e00 << 16) | (0x8c3c >> 2),
928 0x00000000,
929 (0x0e00 << 16) | (0xae00 >> 2),
930 0x00000000,
931 (0x0e00 << 16) | (0x9604 >> 2),
932 0x00000000,
933 (0x0e00 << 16) | (0xac08 >> 2),
934 0x00000000,
935 (0x0e00 << 16) | (0xac0c >> 2),
936 0x00000000,
937 (0x0e00 << 16) | (0xac10 >> 2),
938 0x00000000,
939 (0x0e00 << 16) | (0xac14 >> 2),
940 0x00000000,
941 (0x0e00 << 16) | (0xac58 >> 2),
942 0x00000000,
943 (0x0e00 << 16) | (0xac68 >> 2),
944 0x00000000,
945 (0x0e00 << 16) | (0xac6c >> 2),
946 0x00000000,
947 (0x0e00 << 16) | (0xac70 >> 2),
948 0x00000000,
949 (0x0e00 << 16) | (0xac74 >> 2),
950 0x00000000,
951 (0x0e00 << 16) | (0xac78 >> 2),
952 0x00000000,
953 (0x0e00 << 16) | (0xac7c >> 2),
954 0x00000000,
955 (0x0e00 << 16) | (0xac80 >> 2),
956 0x00000000,
957 (0x0e00 << 16) | (0xac84 >> 2),
958 0x00000000,
959 (0x0e00 << 16) | (0xac88 >> 2),
960 0x00000000,
961 (0x0e00 << 16) | (0xac8c >> 2),
962 0x00000000,
963 (0x0e00 << 16) | (0x970c >> 2),
964 0x00000000,
965 (0x0e00 << 16) | (0x9714 >> 2),
966 0x00000000,
967 (0x0e00 << 16) | (0x9718 >> 2),
968 0x00000000,
969 (0x0e00 << 16) | (0x971c >> 2),
970 0x00000000,
971 (0x0e00 << 16) | (0x31068 >> 2),
972 0x00000000,
973 (0x4e00 << 16) | (0x31068 >> 2),
974 0x00000000,
975 (0x5e00 << 16) | (0x31068 >> 2),
976 0x00000000,
977 (0x6e00 << 16) | (0x31068 >> 2),
978 0x00000000,
979 (0x7e00 << 16) | (0x31068 >> 2),
980 0x00000000,
981 (0x0e00 << 16) | (0xcd10 >> 2),
982 0x00000000,
983 (0x0e00 << 16) | (0xcd14 >> 2),
984 0x00000000,
985 (0x0e00 << 16) | (0x88b0 >> 2),
986 0x00000000,
987 (0x0e00 << 16) | (0x88b4 >> 2),
988 0x00000000,
989 (0x0e00 << 16) | (0x88b8 >> 2),
990 0x00000000,
991 (0x0e00 << 16) | (0x88bc >> 2),
992 0x00000000,
993 (0x0400 << 16) | (0x89c0 >> 2),
994 0x00000000,
995 (0x0e00 << 16) | (0x88c4 >> 2),
996 0x00000000,
997 (0x0e00 << 16) | (0x88c8 >> 2),
998 0x00000000,
999 (0x0e00 << 16) | (0x88d0 >> 2),
1000 0x00000000,
1001 (0x0e00 << 16) | (0x88d4 >> 2),
1002 0x00000000,
1003 (0x0e00 << 16) | (0x88d8 >> 2),
1004 0x00000000,
1005 (0x0e00 << 16) | (0x8980 >> 2),
1006 0x00000000,
1007 (0x0e00 << 16) | (0x30938 >> 2),
1008 0x00000000,
1009 (0x0e00 << 16) | (0x3093c >> 2),
1010 0x00000000,
1011 (0x0e00 << 16) | (0x30940 >> 2),
1012 0x00000000,
1013 (0x0e00 << 16) | (0x89a0 >> 2),
1014 0x00000000,
1015 (0x0e00 << 16) | (0x30900 >> 2),
1016 0x00000000,
1017 (0x0e00 << 16) | (0x30904 >> 2),
1018 0x00000000,
1019 (0x0e00 << 16) | (0x89b4 >> 2),
1020 0x00000000,
1021 (0x0e00 << 16) | (0x3e1fc >> 2),
1022 0x00000000,
1023 (0x0e00 << 16) | (0x3c210 >> 2),
1024 0x00000000,
1025 (0x0e00 << 16) | (0x3c214 >> 2),
1026 0x00000000,
1027 (0x0e00 << 16) | (0x3c218 >> 2),
1028 0x00000000,
1029 (0x0e00 << 16) | (0x8904 >> 2),
1030 0x00000000,
1031 0x5,
1032 (0x0e00 << 16) | (0x8c28 >> 2),
1033 (0x0e00 << 16) | (0x8c2c >> 2),
1034 (0x0e00 << 16) | (0x8c30 >> 2),
1035 (0x0e00 << 16) | (0x8c34 >> 2),
1036 (0x0e00 << 16) | (0x9600 >> 2),
1037};
1038
0aafd313
AD		 1039static const u32 bonaire_golden_spm_registers[] =
1040{
1041 0x30800, 0xe0ffffff, 0xe0000000
1042};
1043
1044static const u32 bonaire_golden_common_registers[] =
1045{
1046 0xc770, 0xffffffff, 0x00000800,
1047 0xc774, 0xffffffff, 0x00000800,
1048 0xc798, 0xffffffff, 0x00007fbf,
1049 0xc79c, 0xffffffff, 0x00007faf
1050};
1051
1052static const u32 bonaire_golden_registers[] =
1053{
1054 0x3354, 0x00000333, 0x00000333,
1055 0x3350, 0x000c0fc0, 0x00040200,
1056 0x9a10, 0x00010000, 0x00058208,
1057 0x3c000, 0xffff1fff, 0x00140000,
1058 0x3c200, 0xfdfc0fff, 0x00000100,
1059 0x3c234, 0x40000000, 0x40000200,
1060 0x9830, 0xffffffff, 0x00000000,
1061 0x9834, 0xf00fffff, 0x00000400,
1062 0x9838, 0x0002021c, 0x00020200,
1063 0xc78, 0x00000080, 0x00000000,
1064 0x5bb0, 0x000000f0, 0x00000070,
1065 0x5bc0, 0xf0311fff, 0x80300000,
1066 0x98f8, 0x73773777, 0x12010001,
1067 0x350c, 0x00810000, 0x408af000,
1068 0x7030, 0x31000111, 0x00000011,
1069 0x2f48, 0x73773777, 0x12010001,
1070 0x220c, 0x00007fb6, 0x0021a1b1,
1071 0x2210, 0x00007fb6, 0x002021b1,
1072 0x2180, 0x00007fb6, 0x00002191,
1073 0x2218, 0x00007fb6, 0x002121b1,
1074 0x221c, 0x00007fb6, 0x002021b1,
1075 0x21dc, 0x00007fb6, 0x00002191,
1076 0x21e0, 0x00007fb6, 0x00002191,
1077 0x3628, 0x0000003f, 0x0000000a,
1078 0x362c, 0x0000003f, 0x0000000a,
1079 0x2ae4, 0x00073ffe, 0x000022a2,
1080 0x240c, 0x000007ff, 0x00000000,
1081 0x8a14, 0xf000003f, 0x00000007,
1082 0x8bf0, 0x00002001, 0x00000001,
1083 0x8b24, 0xffffffff, 0x00ffffff,
1084 0x30a04, 0x0000ff0f, 0x00000000,
1085 0x28a4c, 0x07ffffff, 0x06000000,
1086 0x4d8, 0x00000fff, 0x00000100,
1087 0x3e78, 0x00000001, 0x00000002,
1088 0x9100, 0x03000000, 0x0362c688,
1089 0x8c00, 0x000000ff, 0x00000001,
1090 0xe40, 0x00001fff, 0x00001fff,
1091 0x9060, 0x0000007f, 0x00000020,
1092 0x9508, 0x00010000, 0x00010000,
1093 0xac14, 0x000003ff, 0x000000f3,
1094 0xac0c, 0xffffffff, 0x00001032
1095};
1096
1097static const u32 bonaire_mgcg_cgcg_init[] =
1098{
1099 0xc420, 0xffffffff, 0xfffffffc,
1100 0x30800, 0xffffffff, 0xe0000000,
1101 0x3c2a0, 0xffffffff, 0x00000100,
1102 0x3c208, 0xffffffff, 0x00000100,
1103 0x3c2c0, 0xffffffff, 0xc0000100,
1104 0x3c2c8, 0xffffffff, 0xc0000100,
1105 0x3c2c4, 0xffffffff, 0xc0000100,
1106 0x55e4, 0xffffffff, 0x00600100,
1107 0x3c280, 0xffffffff, 0x00000100,
1108 0x3c214, 0xffffffff, 0x06000100,
1109 0x3c220, 0xffffffff, 0x00000100,
1110 0x3c218, 0xffffffff, 0x06000100,
1111 0x3c204, 0xffffffff, 0x00000100,
1112 0x3c2e0, 0xffffffff, 0x00000100,
1113 0x3c224, 0xffffffff, 0x00000100,
1114 0x3c200, 0xffffffff, 0x00000100,
1115 0x3c230, 0xffffffff, 0x00000100,
1116 0x3c234, 0xffffffff, 0x00000100,
1117 0x3c250, 0xffffffff, 0x00000100,
1118 0x3c254, 0xffffffff, 0x00000100,
1119 0x3c258, 0xffffffff, 0x00000100,
1120 0x3c25c, 0xffffffff, 0x00000100,
1121 0x3c260, 0xffffffff, 0x00000100,
1122 0x3c27c, 0xffffffff, 0x00000100,
1123 0x3c278, 0xffffffff, 0x00000100,
1124 0x3c210, 0xffffffff, 0x06000100,
1125 0x3c290, 0xffffffff, 0x00000100,
1126 0x3c274, 0xffffffff, 0x00000100,
1127 0x3c2b4, 0xffffffff, 0x00000100,
1128 0x3c2b0, 0xffffffff, 0x00000100,
1129 0x3c270, 0xffffffff, 0x00000100,
1130 0x30800, 0xffffffff, 0xe0000000,
1131 0x3c020, 0xffffffff, 0x00010000,
1132 0x3c024, 0xffffffff, 0x00030002,
1133 0x3c028, 0xffffffff, 0x00040007,
1134 0x3c02c, 0xffffffff, 0x00060005,
1135 0x3c030, 0xffffffff, 0x00090008,
1136 0x3c034, 0xffffffff, 0x00010000,
1137 0x3c038, 0xffffffff, 0x00030002,
1138 0x3c03c, 0xffffffff, 0x00040007,
1139 0x3c040, 0xffffffff, 0x00060005,
1140 0x3c044, 0xffffffff, 0x00090008,
1141 0x3c048, 0xffffffff, 0x00010000,
1142 0x3c04c, 0xffffffff, 0x00030002,
1143 0x3c050, 0xffffffff, 0x00040007,
1144 0x3c054, 0xffffffff, 0x00060005,
1145 0x3c058, 0xffffffff, 0x00090008,
1146 0x3c05c, 0xffffffff, 0x00010000,
1147 0x3c060, 0xffffffff, 0x00030002,
1148 0x3c064, 0xffffffff, 0x00040007,
1149 0x3c068, 0xffffffff, 0x00060005,
1150 0x3c06c, 0xffffffff, 0x00090008,
1151 0x3c070, 0xffffffff, 0x00010000,
1152 0x3c074, 0xffffffff, 0x00030002,
1153 0x3c078, 0xffffffff, 0x00040007,
1154 0x3c07c, 0xffffffff, 0x00060005,
1155 0x3c080, 0xffffffff, 0x00090008,
1156 0x3c084, 0xffffffff, 0x00010000,
1157 0x3c088, 0xffffffff, 0x00030002,
1158 0x3c08c, 0xffffffff, 0x00040007,
1159 0x3c090, 0xffffffff, 0x00060005,
1160 0x3c094, 0xffffffff, 0x00090008,
1161 0x3c098, 0xffffffff, 0x00010000,
1162 0x3c09c, 0xffffffff, 0x00030002,
1163 0x3c0a0, 0xffffffff, 0x00040007,
1164 0x3c0a4, 0xffffffff, 0x00060005,
1165 0x3c0a8, 0xffffffff, 0x00090008,
1166 0x3c000, 0xffffffff, 0x96e00200,
1167 0x8708, 0xffffffff, 0x00900100,
1168 0xc424, 0xffffffff, 0x0020003f,
1169 0x38, 0xffffffff, 0x0140001c,
1170 0x3c, 0x000f0000, 0x000f0000,
1171 0x220, 0xffffffff, 0xC060000C,
1172 0x224, 0xc0000fff, 0x00000100,
1173 0xf90, 0xffffffff, 0x00000100,
1174 0xf98, 0x00000101, 0x00000000,
1175 0x20a8, 0xffffffff, 0x00000104,
1176 0x55e4, 0xff000fff, 0x00000100,
1177 0x30cc, 0xc0000fff, 0x00000104,
1178 0xc1e4, 0x00000001, 0x00000001,
1179 0xd00c, 0xff000ff0, 0x00000100,
1180 0xd80c, 0xff000ff0, 0x00000100
1181};
1182
1183static const u32 spectre_golden_spm_registers[] =
1184{
1185 0x30800, 0xe0ffffff, 0xe0000000
1186};
1187
1188static const u32 spectre_golden_common_registers[] =
1189{
1190 0xc770, 0xffffffff, 0x00000800,
1191 0xc774, 0xffffffff, 0x00000800,
1192 0xc798, 0xffffffff, 0x00007fbf,
1193 0xc79c, 0xffffffff, 0x00007faf
1194};
1195
1196static const u32 spectre_golden_registers[] =
1197{
1198 0x3c000, 0xffff1fff, 0x96940200,
1199 0x3c00c, 0xffff0001, 0xff000000,
1200 0x3c200, 0xfffc0fff, 0x00000100,
1201 0x6ed8, 0x00010101, 0x00010000,
1202 0x9834, 0xf00fffff, 0x00000400,
1203 0x9838, 0xfffffffc, 0x00020200,
1204 0x5bb0, 0x000000f0, 0x00000070,
1205 0x5bc0, 0xf0311fff, 0x80300000,
1206 0x98f8, 0x73773777, 0x12010001,
1207 0x9b7c, 0x00ff0000, 0x00fc0000,
1208 0x2f48, 0x73773777, 0x12010001,
1209 0x8a14, 0xf000003f, 0x00000007,
1210 0x8b24, 0xffffffff, 0x00ffffff,
1211 0x28350, 0x3f3f3fff, 0x00000082,
f1553174 1212 0x28354, 0x0000003f, 0x00000000,
0aafd313
AD		 1213 0x3e78, 0x00000001, 0x00000002,
1214 0x913c, 0xffff03df, 0x00000004,
1215 0xc768, 0x00000008, 0x00000008,
1216 0x8c00, 0x000008ff, 0x00000800,
1217 0x9508, 0x00010000, 0x00010000,
1218 0xac0c, 0xffffffff, 0x54763210,
1219 0x214f8, 0x01ff01ff, 0x00000002,
1220 0x21498, 0x007ff800, 0x00200000,
1221 0x2015c, 0xffffffff, 0x00000f40,
1222 0x30934, 0xffffffff, 0x00000001
1223};
1224
1225static const u32 spectre_mgcg_cgcg_init[] =
1226{
1227 0xc420, 0xffffffff, 0xfffffffc,
1228 0x30800, 0xffffffff, 0xe0000000,
1229 0x3c2a0, 0xffffffff, 0x00000100,
1230 0x3c208, 0xffffffff, 0x00000100,
1231 0x3c2c0, 0xffffffff, 0x00000100,
1232 0x3c2c8, 0xffffffff, 0x00000100,
1233 0x3c2c4, 0xffffffff, 0x00000100,
1234 0x55e4, 0xffffffff, 0x00600100,
1235 0x3c280, 0xffffffff, 0x00000100,
1236 0x3c214, 0xffffffff, 0x06000100,
1237 0x3c220, 0xffffffff, 0x00000100,
1238 0x3c218, 0xffffffff, 0x06000100,
1239 0x3c204, 0xffffffff, 0x00000100,
1240 0x3c2e0, 0xffffffff, 0x00000100,
1241 0x3c224, 0xffffffff, 0x00000100,
1242 0x3c200, 0xffffffff, 0x00000100,
1243 0x3c230, 0xffffffff, 0x00000100,
1244 0x3c234, 0xffffffff, 0x00000100,
1245 0x3c250, 0xffffffff, 0x00000100,
1246 0x3c254, 0xffffffff, 0x00000100,
1247 0x3c258, 0xffffffff, 0x00000100,
1248 0x3c25c, 0xffffffff, 0x00000100,
1249 0x3c260, 0xffffffff, 0x00000100,
1250 0x3c27c, 0xffffffff, 0x00000100,
1251 0x3c278, 0xffffffff, 0x00000100,
1252 0x3c210, 0xffffffff, 0x06000100,
1253 0x3c290, 0xffffffff, 0x00000100,
1254 0x3c274, 0xffffffff, 0x00000100,
1255 0x3c2b4, 0xffffffff, 0x00000100,
1256 0x3c2b0, 0xffffffff, 0x00000100,
1257 0x3c270, 0xffffffff, 0x00000100,
1258 0x30800, 0xffffffff, 0xe0000000,
1259 0x3c020, 0xffffffff, 0x00010000,
1260 0x3c024, 0xffffffff, 0x00030002,
1261 0x3c028, 0xffffffff, 0x00040007,
1262 0x3c02c, 0xffffffff, 0x00060005,
1263 0x3c030, 0xffffffff, 0x00090008,
1264 0x3c034, 0xffffffff, 0x00010000,
1265 0x3c038, 0xffffffff, 0x00030002,
1266 0x3c03c, 0xffffffff, 0x00040007,
1267 0x3c040, 0xffffffff, 0x00060005,
1268 0x3c044, 0xffffffff, 0x00090008,
1269 0x3c048, 0xffffffff, 0x00010000,
1270 0x3c04c, 0xffffffff, 0x00030002,
1271 0x3c050, 0xffffffff, 0x00040007,
1272 0x3c054, 0xffffffff, 0x00060005,
1273 0x3c058, 0xffffffff, 0x00090008,
1274 0x3c05c, 0xffffffff, 0x00010000,
1275 0x3c060, 0xffffffff, 0x00030002,
1276 0x3c064, 0xffffffff, 0x00040007,
1277 0x3c068, 0xffffffff, 0x00060005,
1278 0x3c06c, 0xffffffff, 0x00090008,
1279 0x3c070, 0xffffffff, 0x00010000,
1280 0x3c074, 0xffffffff, 0x00030002,
1281 0x3c078, 0xffffffff, 0x00040007,
1282 0x3c07c, 0xffffffff, 0x00060005,
1283 0x3c080, 0xffffffff, 0x00090008,
1284 0x3c084, 0xffffffff, 0x00010000,
1285 0x3c088, 0xffffffff, 0x00030002,
1286 0x3c08c, 0xffffffff, 0x00040007,
1287 0x3c090, 0xffffffff, 0x00060005,
1288 0x3c094, 0xffffffff, 0x00090008,
1289 0x3c098, 0xffffffff, 0x00010000,
1290 0x3c09c, 0xffffffff, 0x00030002,
1291 0x3c0a0, 0xffffffff, 0x00040007,
1292 0x3c0a4, 0xffffffff, 0x00060005,
1293 0x3c0a8, 0xffffffff, 0x00090008,
1294 0x3c0ac, 0xffffffff, 0x00010000,
1295 0x3c0b0, 0xffffffff, 0x00030002,
1296 0x3c0b4, 0xffffffff, 0x00040007,
1297 0x3c0b8, 0xffffffff, 0x00060005,
1298 0x3c0bc, 0xffffffff, 0x00090008,
1299 0x3c000, 0xffffffff, 0x96e00200,
1300 0x8708, 0xffffffff, 0x00900100,
1301 0xc424, 0xffffffff, 0x0020003f,
1302 0x38, 0xffffffff, 0x0140001c,
1303 0x3c, 0x000f0000, 0x000f0000,
1304 0x220, 0xffffffff, 0xC060000C,
1305 0x224, 0xc0000fff, 0x00000100,
1306 0xf90, 0xffffffff, 0x00000100,
1307 0xf98, 0x00000101, 0x00000000,
1308 0x20a8, 0xffffffff, 0x00000104,
1309 0x55e4, 0xff000fff, 0x00000100,
1310 0x30cc, 0xc0000fff, 0x00000104,
1311 0xc1e4, 0x00000001, 0x00000001,
1312 0xd00c, 0xff000ff0, 0x00000100,
1313 0xd80c, 0xff000ff0, 0x00000100
1314};
1315
1316static const u32 kalindi_golden_spm_registers[] =
1317{
1318 0x30800, 0xe0ffffff, 0xe0000000
1319};
1320
1321static const u32 kalindi_golden_common_registers[] =
1322{
1323 0xc770, 0xffffffff, 0x00000800,
1324 0xc774, 0xffffffff, 0x00000800,
1325 0xc798, 0xffffffff, 0x00007fbf,
1326 0xc79c, 0xffffffff, 0x00007faf
1327};
1328
1329static const u32 kalindi_golden_registers[] =
1330{
1331 0x3c000, 0xffffdfff, 0x6e944040,
1332 0x55e4, 0xff607fff, 0xfc000100,
1333 0x3c220, 0xff000fff, 0x00000100,
1334 0x3c224, 0xff000fff, 0x00000100,
1335 0x3c200, 0xfffc0fff, 0x00000100,
1336 0x6ed8, 0x00010101, 0x00010000,
1337 0x9830, 0xffffffff, 0x00000000,
1338 0x9834, 0xf00fffff, 0x00000400,
1339 0x5bb0, 0x000000f0, 0x00000070,
1340 0x5bc0, 0xf0311fff, 0x80300000,
1341 0x98f8, 0x73773777, 0x12010001,
1342 0x98fc, 0xffffffff, 0x00000010,
1343 0x9b7c, 0x00ff0000, 0x00fc0000,
1344 0x8030, 0x00001f0f, 0x0000100a,
1345 0x2f48, 0x73773777, 0x12010001,
1346 0x2408, 0x000fffff, 0x000c007f,
1347 0x8a14, 0xf000003f, 0x00000007,
1348 0x8b24, 0x3fff3fff, 0x00ffcfff,
1349 0x30a04, 0x0000ff0f, 0x00000000,
1350 0x28a4c, 0x07ffffff, 0x06000000,
1351 0x4d8, 0x00000fff, 0x00000100,
1352 0x3e78, 0x00000001, 0x00000002,
1353 0xc768, 0x00000008, 0x00000008,
1354 0x8c00, 0x000000ff, 0x00000003,
1355 0x214f8, 0x01ff01ff, 0x00000002,
1356 0x21498, 0x007ff800, 0x00200000,
1357 0x2015c, 0xffffffff, 0x00000f40,
1358 0x88c4, 0x001f3ae3, 0x00000082,
1359 0x88d4, 0x0000001f, 0x00000010,
1360 0x30934, 0xffffffff, 0x00000000
1361};
1362
1363static const u32 kalindi_mgcg_cgcg_init[] =
1364{
1365 0xc420, 0xffffffff, 0xfffffffc,
1366 0x30800, 0xffffffff, 0xe0000000,
1367 0x3c2a0, 0xffffffff, 0x00000100,
1368 0x3c208, 0xffffffff, 0x00000100,
1369 0x3c2c0, 0xffffffff, 0x00000100,
1370 0x3c2c8, 0xffffffff, 0x00000100,
1371 0x3c2c4, 0xffffffff, 0x00000100,
1372 0x55e4, 0xffffffff, 0x00600100,
1373 0x3c280, 0xffffffff, 0x00000100,
1374 0x3c214, 0xffffffff, 0x06000100,
1375 0x3c220, 0xffffffff, 0x00000100,
1376 0x3c218, 0xffffffff, 0x06000100,
1377 0x3c204, 0xffffffff, 0x00000100,
1378 0x3c2e0, 0xffffffff, 0x00000100,
1379 0x3c224, 0xffffffff, 0x00000100,
1380 0x3c200, 0xffffffff, 0x00000100,
1381 0x3c230, 0xffffffff, 0x00000100,
1382 0x3c234, 0xffffffff, 0x00000100,
1383 0x3c250, 0xffffffff, 0x00000100,
1384 0x3c254, 0xffffffff, 0x00000100,
1385 0x3c258, 0xffffffff, 0x00000100,
1386 0x3c25c, 0xffffffff, 0x00000100,
1387 0x3c260, 0xffffffff, 0x00000100,
1388 0x3c27c, 0xffffffff, 0x00000100,
1389 0x3c278, 0xffffffff, 0x00000100,
1390 0x3c210, 0xffffffff, 0x06000100,
1391 0x3c290, 0xffffffff, 0x00000100,
1392 0x3c274, 0xffffffff, 0x00000100,
1393 0x3c2b4, 0xffffffff, 0x00000100,
1394 0x3c2b0, 0xffffffff, 0x00000100,
1395 0x3c270, 0xffffffff, 0x00000100,
1396 0x30800, 0xffffffff, 0xe0000000,
1397 0x3c020, 0xffffffff, 0x00010000,
1398 0x3c024, 0xffffffff, 0x00030002,
1399 0x3c028, 0xffffffff, 0x00040007,
1400 0x3c02c, 0xffffffff, 0x00060005,
1401 0x3c030, 0xffffffff, 0x00090008,
1402 0x3c034, 0xffffffff, 0x00010000,
1403 0x3c038, 0xffffffff, 0x00030002,
1404 0x3c03c, 0xffffffff, 0x00040007,
1405 0x3c040, 0xffffffff, 0x00060005,
1406 0x3c044, 0xffffffff, 0x00090008,
1407 0x3c000, 0xffffffff, 0x96e00200,
1408 0x8708, 0xffffffff, 0x00900100,
1409 0xc424, 0xffffffff, 0x0020003f,
1410 0x38, 0xffffffff, 0x0140001c,
1411 0x3c, 0x000f0000, 0x000f0000,
1412 0x220, 0xffffffff, 0xC060000C,
1413 0x224, 0xc0000fff, 0x00000100,
1414 0x20a8, 0xffffffff, 0x00000104,
1415 0x55e4, 0xff000fff, 0x00000100,
1416 0x30cc, 0xc0000fff, 0x00000104,
1417 0xc1e4, 0x00000001, 0x00000001,
1418 0xd00c, 0xff000ff0, 0x00000100,
1419 0xd80c, 0xff000ff0, 0x00000100
1420};
1421
8efff337
AD		 1422static const u32 hawaii_golden_spm_registers[] =
1423{
1424 0x30800, 0xe0ffffff, 0xe0000000
1425};
1426
1427static const u32 hawaii_golden_common_registers[] =
1428{
1429 0x30800, 0xffffffff, 0xe0000000,
1430 0x28350, 0xffffffff, 0x3a00161a,
1431 0x28354, 0xffffffff, 0x0000002e,
1432 0x9a10, 0xffffffff, 0x00018208,
1433 0x98f8, 0xffffffff, 0x12011003
1434};
1435
1436static const u32 hawaii_golden_registers[] =
1437{
1438 0x3354, 0x00000333, 0x00000333,
1439 0x9a10, 0x00010000, 0x00058208,
1440 0x9830, 0xffffffff, 0x00000000,
1441 0x9834, 0xf00fffff, 0x00000400,
1442 0x9838, 0x0002021c, 0x00020200,
1443 0xc78, 0x00000080, 0x00000000,
1444 0x5bb0, 0x000000f0, 0x00000070,
1445 0x5bc0, 0xf0311fff, 0x80300000,
1446 0x350c, 0x00810000, 0x408af000,
1447 0x7030, 0x31000111, 0x00000011,
1448 0x2f48, 0x73773777, 0x12010001,
1449 0x2120, 0x0000007f, 0x0000001b,
1450 0x21dc, 0x00007fb6, 0x00002191,
1451 0x3628, 0x0000003f, 0x0000000a,
1452 0x362c, 0x0000003f, 0x0000000a,
1453 0x2ae4, 0x00073ffe, 0x000022a2,
1454 0x240c, 0x000007ff, 0x00000000,
1455 0x8bf0, 0x00002001, 0x00000001,
1456 0x8b24, 0xffffffff, 0x00ffffff,
1457 0x30a04, 0x0000ff0f, 0x00000000,
1458 0x28a4c, 0x07ffffff, 0x06000000,
1459 0x3e78, 0x00000001, 0x00000002,
1460 0xc768, 0x00000008, 0x00000008,
1461 0xc770, 0x00000f00, 0x00000800,
1462 0xc774, 0x00000f00, 0x00000800,
1463 0xc798, 0x00ffffff, 0x00ff7fbf,
1464 0xc79c, 0x00ffffff, 0x00ff7faf,
1465 0x8c00, 0x000000ff, 0x00000800,
1466 0xe40, 0x00001fff, 0x00001fff,
1467 0x9060, 0x0000007f, 0x00000020,
1468 0x9508, 0x00010000, 0x00010000,
1469 0xae00, 0x00100000, 0x000ff07c,
1470 0xac14, 0x000003ff, 0x0000000f,
1471 0xac10, 0xffffffff, 0x7564fdec,
1472 0xac0c, 0xffffffff, 0x3120b9a8,
1473 0xac08, 0x20000000, 0x0f9c0000
1474};
1475
1476static const u32 hawaii_mgcg_cgcg_init[] =
1477{
1478 0xc420, 0xffffffff, 0xfffffffd,
1479 0x30800, 0xffffffff, 0xe0000000,
1480 0x3c2a0, 0xffffffff, 0x00000100,
1481 0x3c208, 0xffffffff, 0x00000100,
1482 0x3c2c0, 0xffffffff, 0x00000100,
1483 0x3c2c8, 0xffffffff, 0x00000100,
1484 0x3c2c4, 0xffffffff, 0x00000100,
1485 0x55e4, 0xffffffff, 0x00200100,
1486 0x3c280, 0xffffffff, 0x00000100,
1487 0x3c214, 0xffffffff, 0x06000100,
1488 0x3c220, 0xffffffff, 0x00000100,
1489 0x3c218, 0xffffffff, 0x06000100,
1490 0x3c204, 0xffffffff, 0x00000100,
1491 0x3c2e0, 0xffffffff, 0x00000100,
1492 0x3c224, 0xffffffff, 0x00000100,
1493 0x3c200, 0xffffffff, 0x00000100,
1494 0x3c230, 0xffffffff, 0x00000100,
1495 0x3c234, 0xffffffff, 0x00000100,
1496 0x3c250, 0xffffffff, 0x00000100,
1497 0x3c254, 0xffffffff, 0x00000100,
1498 0x3c258, 0xffffffff, 0x00000100,
1499 0x3c25c, 0xffffffff, 0x00000100,
1500 0x3c260, 0xffffffff, 0x00000100,
1501 0x3c27c, 0xffffffff, 0x00000100,
1502 0x3c278, 0xffffffff, 0x00000100,
1503 0x3c210, 0xffffffff, 0x06000100,
1504 0x3c290, 0xffffffff, 0x00000100,
1505 0x3c274, 0xffffffff, 0x00000100,
1506 0x3c2b4, 0xffffffff, 0x00000100,
1507 0x3c2b0, 0xffffffff, 0x00000100,
1508 0x3c270, 0xffffffff, 0x00000100,
1509 0x30800, 0xffffffff, 0xe0000000,
1510 0x3c020, 0xffffffff, 0x00010000,
1511 0x3c024, 0xffffffff, 0x00030002,
1512 0x3c028, 0xffffffff, 0x00040007,
1513 0x3c02c, 0xffffffff, 0x00060005,
1514 0x3c030, 0xffffffff, 0x00090008,
1515 0x3c034, 0xffffffff, 0x00010000,
1516 0x3c038, 0xffffffff, 0x00030002,
1517 0x3c03c, 0xffffffff, 0x00040007,
1518 0x3c040, 0xffffffff, 0x00060005,
1519 0x3c044, 0xffffffff, 0x00090008,
1520 0x3c048, 0xffffffff, 0x00010000,
1521 0x3c04c, 0xffffffff, 0x00030002,
1522 0x3c050, 0xffffffff, 0x00040007,
1523 0x3c054, 0xffffffff, 0x00060005,
1524 0x3c058, 0xffffffff, 0x00090008,
1525 0x3c05c, 0xffffffff, 0x00010000,
1526 0x3c060, 0xffffffff, 0x00030002,
1527 0x3c064, 0xffffffff, 0x00040007,
1528 0x3c068, 0xffffffff, 0x00060005,
1529 0x3c06c, 0xffffffff, 0x00090008,
1530 0x3c070, 0xffffffff, 0x00010000,
1531 0x3c074, 0xffffffff, 0x00030002,
1532 0x3c078, 0xffffffff, 0x00040007,
1533 0x3c07c, 0xffffffff, 0x00060005,
1534 0x3c080, 0xffffffff, 0x00090008,
1535 0x3c084, 0xffffffff, 0x00010000,
1536 0x3c088, 0xffffffff, 0x00030002,
1537 0x3c08c, 0xffffffff, 0x00040007,
1538 0x3c090, 0xffffffff, 0x00060005,
1539 0x3c094, 0xffffffff, 0x00090008,
1540 0x3c098, 0xffffffff, 0x00010000,
1541 0x3c09c, 0xffffffff, 0x00030002,
1542 0x3c0a0, 0xffffffff, 0x00040007,
1543 0x3c0a4, 0xffffffff, 0x00060005,
1544 0x3c0a8, 0xffffffff, 0x00090008,
1545 0x3c0ac, 0xffffffff, 0x00010000,
1546 0x3c0b0, 0xffffffff, 0x00030002,
1547 0x3c0b4, 0xffffffff, 0x00040007,
1548 0x3c0b8, 0xffffffff, 0x00060005,
1549 0x3c0bc, 0xffffffff, 0x00090008,
1550 0x3c0c0, 0xffffffff, 0x00010000,
1551 0x3c0c4, 0xffffffff, 0x00030002,
1552 0x3c0c8, 0xffffffff, 0x00040007,
1553 0x3c0cc, 0xffffffff, 0x00060005,
1554 0x3c0d0, 0xffffffff, 0x00090008,
1555 0x3c0d4, 0xffffffff, 0x00010000,
1556 0x3c0d8, 0xffffffff, 0x00030002,
1557 0x3c0dc, 0xffffffff, 0x00040007,
1558 0x3c0e0, 0xffffffff, 0x00060005,
1559 0x3c0e4, 0xffffffff, 0x00090008,
1560 0x3c0e8, 0xffffffff, 0x00010000,
1561 0x3c0ec, 0xffffffff, 0x00030002,
1562 0x3c0f0, 0xffffffff, 0x00040007,
1563 0x3c0f4, 0xffffffff, 0x00060005,
1564 0x3c0f8, 0xffffffff, 0x00090008,
1565 0xc318, 0xffffffff, 0x00020200,
1566 0x3350, 0xffffffff, 0x00000200,
1567 0x15c0, 0xffffffff, 0x00000400,
1568 0x55e8, 0xffffffff, 0x00000000,
1569 0x2f50, 0xffffffff, 0x00000902,
1570 0x3c000, 0xffffffff, 0x96940200,
1571 0x8708, 0xffffffff, 0x00900100,
1572 0xc424, 0xffffffff, 0x0020003f,
1573 0x38, 0xffffffff, 0x0140001c,
1574 0x3c, 0x000f0000, 0x000f0000,
1575 0x220, 0xffffffff, 0xc060000c,
1576 0x224, 0xc0000fff, 0x00000100,
1577 0xf90, 0xffffffff, 0x00000100,
1578 0xf98, 0x00000101, 0x00000000,
1579 0x20a8, 0xffffffff, 0x00000104,
1580 0x55e4, 0xff000fff, 0x00000100,
1581 0x30cc, 0xc0000fff, 0x00000104,
1582 0xc1e4, 0x00000001, 0x00000001,
1583 0xd00c, 0xff000ff0, 0x00000100,
1584 0xd80c, 0xff000ff0, 0x00000100
1585};
1586
f73a9e83
SL		 1587static const u32 godavari_golden_registers[] =
1588{
1589 0x55e4, 0xff607fff, 0xfc000100,
1590 0x6ed8, 0x00010101, 0x00010000,
1591 0x9830, 0xffffffff, 0x00000000,
1592 0x98302, 0xf00fffff, 0x00000400,
1593 0x6130, 0xffffffff, 0x00010000,
1594 0x5bb0, 0x000000f0, 0x00000070,
1595 0x5bc0, 0xf0311fff, 0x80300000,
1596 0x98f8, 0x73773777, 0x12010001,
1597 0x98fc, 0xffffffff, 0x00000010,
1598 0x8030, 0x00001f0f, 0x0000100a,
1599 0x2f48, 0x73773777, 0x12010001,
1600 0x2408, 0x000fffff, 0x000c007f,
1601 0x8a14, 0xf000003f, 0x00000007,
1602 0x8b24, 0xffffffff, 0x00ff0fff,
1603 0x30a04, 0x0000ff0f, 0x00000000,
1604 0x28a4c, 0x07ffffff, 0x06000000,
1605 0x4d8, 0x00000fff, 0x00000100,
1606 0xd014, 0x00010000, 0x00810001,
1607 0xd814, 0x00010000, 0x00810001,
1608 0x3e78, 0x00000001, 0x00000002,
1609 0xc768, 0x00000008, 0x00000008,
1610 0xc770, 0x00000f00, 0x00000800,
1611 0xc774, 0x00000f00, 0x00000800,
1612 0xc798, 0x00ffffff, 0x00ff7fbf,
1613 0xc79c, 0x00ffffff, 0x00ff7faf,
1614 0x8c00, 0x000000ff, 0x00000001,
1615 0x214f8, 0x01ff01ff, 0x00000002,
1616 0x21498, 0x007ff800, 0x00200000,
1617 0x2015c, 0xffffffff, 0x00000f40,
1618 0x88c4, 0x001f3ae3, 0x00000082,
1619 0x88d4, 0x0000001f, 0x00000010,
1620 0x30934, 0xffffffff, 0x00000000
1621};
1622
1623
0aafd313
AD		 1624static void cik_init_golden_registers(struct radeon_device *rdev)
1625{
1c0a4625
OG		 1626 /* Some of the registers might be dependent on GRBM_GFX_INDEX */
1627 mutex_lock(&rdev->grbm_idx_mutex);
0aafd313
AD		 1628 switch (rdev->family) {
1629 case CHIP_BONAIRE:
1630 radeon_program_register_sequence(rdev,
1631 bonaire_mgcg_cgcg_init,
1632 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1633 radeon_program_register_sequence(rdev,
1634 bonaire_golden_registers,
1635 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1636 radeon_program_register_sequence(rdev,
1637 bonaire_golden_common_registers,
1638 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1639 radeon_program_register_sequence(rdev,
1640 bonaire_golden_spm_registers,
1641 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1642 break;
1643 case CHIP_KABINI:
1644 radeon_program_register_sequence(rdev,
1645 kalindi_mgcg_cgcg_init,
1646 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1647 radeon_program_register_sequence(rdev,
1648 kalindi_golden_registers,
1649 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1650 radeon_program_register_sequence(rdev,
1651 kalindi_golden_common_registers,
1652 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1653 radeon_program_register_sequence(rdev,
1654 kalindi_golden_spm_registers,
1655 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1656 break;
f73a9e83
SL		 1657 case CHIP_MULLINS:
1658 radeon_program_register_sequence(rdev,
1659 kalindi_mgcg_cgcg_init,
1660 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1661 radeon_program_register_sequence(rdev,
1662 godavari_golden_registers,
1663 (const u32)ARRAY_SIZE(godavari_golden_registers));
1664 radeon_program_register_sequence(rdev,
1665 kalindi_golden_common_registers,
1666 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1667 radeon_program_register_sequence(rdev,
1668 kalindi_golden_spm_registers,
1669 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1670 break;
0aafd313
AD		 1671 case CHIP_KAVERI:
1672 radeon_program_register_sequence(rdev,
1673 spectre_mgcg_cgcg_init,
1674 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1675 radeon_program_register_sequence(rdev,
1676 spectre_golden_registers,
1677 (const u32)ARRAY_SIZE(spectre_golden_registers));
1678 radeon_program_register_sequence(rdev,
1679 spectre_golden_common_registers,
1680 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1681 radeon_program_register_sequence(rdev,
1682 spectre_golden_spm_registers,
1683 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1684 break;
8efff337
AD		 1685 case CHIP_HAWAII:
1686 radeon_program_register_sequence(rdev,
1687 hawaii_mgcg_cgcg_init,
1688 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1689 radeon_program_register_sequence(rdev,
1690 hawaii_golden_registers,
1691 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1692 radeon_program_register_sequence(rdev,
1693 hawaii_golden_common_registers,
1694 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1695 radeon_program_register_sequence(rdev,
1696 hawaii_golden_spm_registers,
1697 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1698 break;
0aafd313
AD		 1699 default:
1700 break;
1701 }
1c0a4625 1702 mutex_unlock(&rdev->grbm_idx_mutex);
0aafd313
AD		 1703}
1704
2c67912c
AD		 1705/**
1706 * cik_get_xclk - get the xclk
1707 *
1708 * @rdev: radeon_device pointer
1709 *
1710 * Returns the reference clock used by the gfx engine
1711 * (CIK).
1712 */
1713u32 cik_get_xclk(struct radeon_device *rdev)
1714{
1715 u32 reference_clock = rdev->clock.spll.reference_freq;
1716
1717 if (rdev->flags & RADEON_IS_IGP) {
1718 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1719 return reference_clock / 2;
1720 } else {
1721 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1722 return reference_clock / 4;
1723 }
1724 return reference_clock;
1725}
1726
75efdee1
AD		 1727/**
1728 * cik_mm_rdoorbell - read a doorbell dword
1729 *
1730 * @rdev: radeon_device pointer
d5754ab8 1731 * @index: doorbell index
75efdee1
AD		 1732 *
1733 * Returns the value in the doorbell aperture at the
d5754ab8 1734 * requested doorbell index (CIK).
75efdee1 1735 */
d5754ab8 1736u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
75efdee1 1737{
d5754ab8
AL		 1738 if (index < rdev->doorbell.num_doorbells) {
1739 return readl(rdev->doorbell.ptr + index);
75efdee1 1740 } else {
d5754ab8 1741 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1742 return 0;
1743 }
1744}
1745
1746/**
1747 * cik_mm_wdoorbell - write a doorbell dword
1748 *
1749 * @rdev: radeon_device pointer
d5754ab8 1750 * @index: doorbell index
75efdee1
AD		 1751 * @v: value to write
1752 *
1753 * Writes @v to the doorbell aperture at the
d5754ab8 1754 * requested doorbell index (CIK).
75efdee1 1755 */
d5754ab8 1756void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
75efdee1 1757{
d5754ab8
AL		 1758 if (index < rdev->doorbell.num_doorbells) {
1759 writel(v, rdev->doorbell.ptr + index);
75efdee1 1760 } else {
d5754ab8 1761 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1762 }
1763}
1764
bc8273fe
AD		 1765#define BONAIRE_IO_MC_REGS_SIZE 36
1766
1767static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1768{
1769 {0x00000070, 0x04400000},
1770 {0x00000071, 0x80c01803},
1771 {0x00000072, 0x00004004},
1772 {0x00000073, 0x00000100},
1773 {0x00000074, 0x00ff0000},
1774 {0x00000075, 0x34000000},
1775 {0x00000076, 0x08000014},
1776 {0x00000077, 0x00cc08ec},
1777 {0x00000078, 0x00000400},
1778 {0x00000079, 0x00000000},
1779 {0x0000007a, 0x04090000},
1780 {0x0000007c, 0x00000000},
1781 {0x0000007e, 0x4408a8e8},
1782 {0x0000007f, 0x00000304},
1783 {0x00000080, 0x00000000},
1784 {0x00000082, 0x00000001},
1785 {0x00000083, 0x00000002},
1786 {0x00000084, 0xf3e4f400},
1787 {0x00000085, 0x052024e3},
1788 {0x00000087, 0x00000000},
1789 {0x00000088, 0x01000000},
1790 {0x0000008a, 0x1c0a0000},
1791 {0x0000008b, 0xff010000},
1792 {0x0000008d, 0xffffefff},
1793 {0x0000008e, 0xfff3efff},
1794 {0x0000008f, 0xfff3efbf},
1795 {0x00000092, 0xf7ffffff},
1796 {0x00000093, 0xffffff7f},
1797 {0x00000095, 0x00101101},
1798 {0x00000096, 0x00000fff},
1799 {0x00000097, 0x00116fff},
1800 {0x00000098, 0x60010000},
1801 {0x00000099, 0x10010000},
1802 {0x0000009a, 0x00006000},
1803 {0x0000009b, 0x00001000},
1804 {0x0000009f, 0x00b48000}
1805};
1806
d4775655
AD		 1807#define HAWAII_IO_MC_REGS_SIZE 22
1808
1809static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1810{
1811 {0x0000007d, 0x40000000},
1812 {0x0000007e, 0x40180304},
1813 {0x0000007f, 0x0000ff00},
1814 {0x00000081, 0x00000000},
1815 {0x00000083, 0x00000800},
1816 {0x00000086, 0x00000000},
1817 {0x00000087, 0x00000100},
1818 {0x00000088, 0x00020100},
1819 {0x00000089, 0x00000000},
1820 {0x0000008b, 0x00040000},
1821 {0x0000008c, 0x00000100},
1822 {0x0000008e, 0xff010000},
1823 {0x00000090, 0xffffefff},
1824 {0x00000091, 0xfff3efff},
1825 {0x00000092, 0xfff3efbf},
1826 {0x00000093, 0xf7ffffff},
1827 {0x00000094, 0xffffff7f},
1828 {0x00000095, 0x00000fff},
1829 {0x00000096, 0x00116fff},
1830 {0x00000097, 0x60010000},
1831 {0x00000098, 0x10010000},
1832 {0x0000009f, 0x00c79000}
1833};
1834
1835
b556b12e
AD		 1836/**
1837 * cik_srbm_select - select specific register instances
1838 *
1839 * @rdev: radeon_device pointer
1840 * @me: selected ME (micro engine)
1841 * @pipe: pipe
1842 * @queue: queue
1843 * @vmid: VMID
1844 *
1845 * Switches the currently active registers instances. Some
1846 * registers are instanced per VMID, others are instanced per
1847 * me/pipe/queue combination.
1848 */
1849static void cik_srbm_select(struct radeon_device *rdev,
1850 u32 me, u32 pipe, u32 queue, u32 vmid)
1851{
1852 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1853 MEID(me & 0x3) |
1854 VMID(vmid & 0xf) |
1855 QUEUEID(queue & 0x7));
1856 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1857}
1858
bc8273fe
AD		 1859/* ucode loading */
1860/**
1861 * ci_mc_load_microcode - load MC ucode into the hw
1862 *
1863 * @rdev: radeon_device pointer
1864 *
1865 * Load the GDDR MC ucode into the hw (CIK).
1866 * Returns 0 on success, error on failure.
1867 */
6c7bccea 1868int ci_mc_load_microcode(struct radeon_device *rdev)
bc8273fe 1869{
f2c6b0f4
AD		 1870 const __be32 *fw_data = NULL;
1871 const __le32 *new_fw_data = NULL;
9feb3dda 1872 u32 running, blackout = 0, tmp;
f2c6b0f4
AD		 1873 u32 *io_mc_regs = NULL;
1874 const __le32 *new_io_mc_regs = NULL;
bcddee29 1875 int i, regs_size, ucode_size;
bc8273fe
AD		 1876
1877 if (!rdev->mc_fw)
1878 return -EINVAL;
1879
f2c6b0f4
AD		 1880 if (rdev->new_fw) {
1881 const struct mc_firmware_header_v1_0 *hdr =
1882 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
bcddee29 1883
f2c6b0f4
AD		 1884 radeon_ucode_print_mc_hdr(&hdr->header);
1885
1886 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1887 new_io_mc_regs = (const __le32 *)
1888 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1889 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1890 new_fw_data = (const __le32 *)
1891 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1892 } else {
1893 ucode_size = rdev->mc_fw->size / 4;
1894
1895 switch (rdev->family) {
1896 case CHIP_BONAIRE:
1897 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1898 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1899 break;
1900 case CHIP_HAWAII:
1901 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1902 regs_size = HAWAII_IO_MC_REGS_SIZE;
1903 break;
1904 default:
1905 return -EINVAL;
1906 }
1907 fw_data = (const __be32 *)rdev->mc_fw->data;
bc8273fe
AD		 1908 }
1909
1910 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1911
1912 if (running == 0) {
1913 if (running) {
1914 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1915 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1916 }
1917
1918 /* reset the engine and set to writable */
1919 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1920 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1921
1922 /* load mc io regs */
1923 for (i = 0; i < regs_size; i++) {
f2c6b0f4
AD		 1924 if (rdev->new_fw) {
1925 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1926 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1927 } else {
1928 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1929 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1930 }
bc8273fe 1931 }
9feb3dda
AD		 1932
1933 tmp = RREG32(MC_SEQ_MISC0);
1934 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {
1935 WREG32(MC_SEQ_IO_DEBUG_INDEX, 5);
1936 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023);
1937 WREG32(MC_SEQ_IO_DEBUG_INDEX, 9);
1938 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0);
1939 }
1940
bc8273fe 1941 /* load the MC ucode */
f2c6b0f4
AD		 1942 for (i = 0; i < ucode_size; i++) {
1943 if (rdev->new_fw)
1944 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1945 else
1946 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1947 }
bc8273fe
AD		 1948
1949 /* put the engine back into the active state */
1950 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1951 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1952 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1953
1954 /* wait for training to complete */
1955 for (i = 0; i < rdev->usec_timeout; i++) {
1956 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1957 break;
1958 udelay(1);
1959 }
1960 for (i = 0; i < rdev->usec_timeout; i++) {
1961 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1962 break;
1963 udelay(1);
1964 }
1965
1966 if (running)
1967 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1968 }
1969
1970 return 0;
1971}
1972
02c81327
AD		 1973/**
1974 * cik_init_microcode - load ucode images from disk
1975 *
1976 * @rdev: radeon_device pointer
1977 *
1978 * Use the firmware interface to load the ucode images into
1979 * the driver (not loaded into hw).
1980 * Returns 0 on success, error on failure.
1981 */
1982static int cik_init_microcode(struct radeon_device *rdev)
1983{
02c81327 1984 const char *chip_name;
f2c6b0f4 1985 const char *new_chip_name;
02c81327 1986 size_t pfp_req_size, me_req_size, ce_req_size,
d4775655 1987 mec_req_size, rlc_req_size, mc_req_size = 0,
277babc3 1988 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
02c81327 1989 char fw_name[30];
f2c6b0f4 1990 int new_fw = 0;
02c81327 1991 int err;
f2c6b0f4 1992 int num_fw;
02c81327
AD		 1993
1994 DRM_DEBUG("\n");
1995
02c81327
AD		 1996 switch (rdev->family) {
1997 case CHIP_BONAIRE:
1998 chip_name = "BONAIRE";
f2c6b0f4 1999 new_chip_name = "bonaire";
02c81327
AD		 2000 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2001 me_req_size = CIK_ME_UCODE_SIZE * 4;
2002 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2003 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2004 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
277babc3
AD		 2005 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
2006 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
21a93e13 2007 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
cc8dbbb4 2008 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
f2c6b0f4 2009 num_fw = 8;
02c81327 2010 break;
d4775655
AD		 2011 case CHIP_HAWAII:
2012 chip_name = "HAWAII";
f2c6b0f4 2013 new_chip_name = "hawaii";
d4775655
AD		 2014 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2015 me_req_size = CIK_ME_UCODE_SIZE * 4;
2016 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2017 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2018 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
2019 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
277babc3 2020 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
d4775655
AD		 2021 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2022 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
f2c6b0f4 2023 num_fw = 8;
d4775655 2024 break;
02c81327
AD		 2025 case CHIP_KAVERI:
2026 chip_name = "KAVERI";
f2c6b0f4 2027 new_chip_name = "kaveri";
02c81327
AD		 2028 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2029 me_req_size = CIK_ME_UCODE_SIZE * 4;
2030 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2031 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2032 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
21a93e13 2033 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 2034 num_fw = 7;
02c81327
AD		 2035 break;
2036 case CHIP_KABINI:
2037 chip_name = "KABINI";
f2c6b0f4 2038 new_chip_name = "kabini";
02c81327
AD		 2039 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2040 me_req_size = CIK_ME_UCODE_SIZE * 4;
2041 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2042 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2043 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
21a93e13 2044 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 2045 num_fw = 6;
02c81327 2046 break;
f73a9e83
SL		 2047 case CHIP_MULLINS:
2048 chip_name = "MULLINS";
f2c6b0f4 2049 new_chip_name = "mullins";
f73a9e83
SL		 2050 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2051 me_req_size = CIK_ME_UCODE_SIZE * 4;
2052 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2053 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2054 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
2055 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 2056 num_fw = 6;
f73a9e83 2057 break;
02c81327
AD		 2058 default: BUG();
2059 }
2060
f2c6b0f4 2061 DRM_INFO("Loading %s Microcode\n", new_chip_name);
02c81327 2062
f2c6b0f4 2063 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
0a168933 2064 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2065 if (err) {
2066 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2067 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2068 if (err)
2069 goto out;
2070 if (rdev->pfp_fw->size != pfp_req_size) {
2071 printk(KERN_ERR
2072 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2073 rdev->pfp_fw->size, fw_name);
2074 err = -EINVAL;
2075 goto out;
2076 }
2077 } else {
2078 err = radeon_ucode_validate(rdev->pfp_fw);
2079 if (err) {
2080 printk(KERN_ERR
2081 "cik_fw: validation failed for firmware \"%s\"\n",
2082 fw_name);
2083 goto out;
2084 } else {
2085 new_fw++;
2086 }
02c81327
AD		 2087 }
2088
f2c6b0f4 2089 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
0a168933 2090 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2091 if (err) {
2092 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2093 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2094 if (err)
2095 goto out;
2096 if (rdev->me_fw->size != me_req_size) {
2097 printk(KERN_ERR
2098 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2099 rdev->me_fw->size, fw_name);
2100 err = -EINVAL;
2101 }
2102 } else {
2103 err = radeon_ucode_validate(rdev->me_fw);
2104 if (err) {
2105 printk(KERN_ERR
2106 "cik_fw: validation failed for firmware \"%s\"\n",
2107 fw_name);
2108 goto out;
2109 } else {
2110 new_fw++;
2111 }
02c81327
AD		 2112 }
2113
f2c6b0f4 2114 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
0a168933 2115 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2116 if (err) {
2117 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2118 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2119 if (err)
2120 goto out;
2121 if (rdev->ce_fw->size != ce_req_size) {
2122 printk(KERN_ERR
2123 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2124 rdev->ce_fw->size, fw_name);
2125 err = -EINVAL;
2126 }
2127 } else {
2128 err = radeon_ucode_validate(rdev->ce_fw);
2129 if (err) {
2130 printk(KERN_ERR
2131 "cik_fw: validation failed for firmware \"%s\"\n",
2132 fw_name);
2133 goto out;
2134 } else {
2135 new_fw++;
2136 }
02c81327
AD		 2137 }
2138
f2c6b0f4 2139 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
0a168933 2140 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2141 if (err) {
2142 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2143 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2144 if (err)
2145 goto out;
2146 if (rdev->mec_fw->size != mec_req_size) {
2147 printk(KERN_ERR
2148 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2149 rdev->mec_fw->size, fw_name);
2150 err = -EINVAL;
2151 }
2152 } else {
2153 err = radeon_ucode_validate(rdev->mec_fw);
2154 if (err) {
2155 printk(KERN_ERR
2156 "cik_fw: validation failed for firmware \"%s\"\n",
2157 fw_name);
2158 goto out;
2159 } else {
2160 new_fw++;
2161 }
2162 }
2163
2164 if (rdev->family == CHIP_KAVERI) {
2165 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2166 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2167 if (err) {
2168 goto out;
2169 } else {
2170 err = radeon_ucode_validate(rdev->mec2_fw);
2171 if (err) {
2172 goto out;
2173 } else {
2174 new_fw++;
2175 }
2176 }
02c81327
AD		 2177 }
2178
f2c6b0f4 2179 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
0a168933 2180 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2181 if (err) {
2182 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2183 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2184 if (err)
2185 goto out;
2186 if (rdev->rlc_fw->size != rlc_req_size) {
2187 printk(KERN_ERR
2188 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2189 rdev->rlc_fw->size, fw_name);
2190 err = -EINVAL;
2191 }
2192 } else {
2193 err = radeon_ucode_validate(rdev->rlc_fw);
2194 if (err) {
2195 printk(KERN_ERR
2196 "cik_fw: validation failed for firmware \"%s\"\n",
2197 fw_name);
2198 goto out;
2199 } else {
2200 new_fw++;
2201 }
02c81327
AD		 2202 }
2203
f2c6b0f4 2204 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
0a168933 2205 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2206 if (err) {
2207 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2208 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2209 if (err)
2210 goto out;
2211 if (rdev->sdma_fw->size != sdma_req_size) {
2212 printk(KERN_ERR
2213 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2214 rdev->sdma_fw->size, fw_name);
2215 err = -EINVAL;
2216 }
2217 } else {
2218 err = radeon_ucode_validate(rdev->sdma_fw);
2219 if (err) {
2220 printk(KERN_ERR
2221 "cik_fw: validation failed for firmware \"%s\"\n",
2222 fw_name);
2223 goto out;
2224 } else {
2225 new_fw++;
2226 }
21a93e13
AD		 2227 }
2228
cc8dbbb4 2229 /* No SMC, MC ucode on APUs */
02c81327 2230 if (!(rdev->flags & RADEON_IS_IGP)) {
f2c6b0f4 2231 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
0a168933 2232 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
277babc3 2233 if (err) {
f2c6b0f4 2234 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
277babc3 2235 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2236 if (err) {
2237 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2238 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2239 if (err)
2240 goto out;
2241 }
2242 if ((rdev->mc_fw->size != mc_req_size) &&
2243 (rdev->mc_fw->size != mc2_req_size)){
2244 printk(KERN_ERR
2245 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
2246 rdev->mc_fw->size, fw_name);
2247 err = -EINVAL;
2248 }
2249 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2250 } else {
2251 err = radeon_ucode_validate(rdev->mc_fw);
2252 if (err) {
2253 printk(KERN_ERR
2254 "cik_fw: validation failed for firmware \"%s\"\n",
2255 fw_name);
277babc3 2256 goto out;
f2c6b0f4
AD		 2257 } else {
2258 new_fw++;
2259 }
277babc3 2260 }
cc8dbbb4 2261
f2c6b0f4 2262 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
cc8dbbb4
AD		 2263 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2264 if (err) {
f2c6b0f4
AD		 2265 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2266 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2267 if (err) {
2268 printk(KERN_ERR
2269 "smc: error loading firmware \"%s\"\n",
2270 fw_name);
2271 release_firmware(rdev->smc_fw);
2272 rdev->smc_fw = NULL;
2273 err = 0;
2274 } else if (rdev->smc_fw->size != smc_req_size) {
2275 printk(KERN_ERR
2276 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2277 rdev->smc_fw->size, fw_name);
2278 err = -EINVAL;
2279 }
2280 } else {
2281 err = radeon_ucode_validate(rdev->smc_fw);
2282 if (err) {
2283 printk(KERN_ERR
2284 "cik_fw: validation failed for firmware \"%s\"\n",
2285 fw_name);
2286 goto out;
2287 } else {
2288 new_fw++;
2289 }
cc8dbbb4 2290 }
02c81327
AD		 2291 }
2292
f2c6b0f4
AD		 2293 if (new_fw == 0) {
2294 rdev->new_fw = false;
2295 } else if (new_fw < num_fw) {
2296 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n");
2297 err = -EINVAL;
2298 } else {
2299 rdev->new_fw = true;
2300 }
2301
02c81327 2302out:
02c81327
AD		 2303 if (err) {
2304 if (err != -EINVAL)
2305 printk(KERN_ERR
2306 "cik_cp: Failed to load firmware \"%s\"\n",
2307 fw_name);
2308 release_firmware(rdev->pfp_fw);
2309 rdev->pfp_fw = NULL;
2310 release_firmware(rdev->me_fw);
2311 rdev->me_fw = NULL;
2312 release_firmware(rdev->ce_fw);
2313 rdev->ce_fw = NULL;
f2c6b0f4
AD		 2314 release_firmware(rdev->mec_fw);
2315 rdev->mec_fw = NULL;
2316 release_firmware(rdev->mec2_fw);
2317 rdev->mec2_fw = NULL;
02c81327
AD		 2318 release_firmware(rdev->rlc_fw);
2319 rdev->rlc_fw = NULL;
f2c6b0f4
AD		 2320 release_firmware(rdev->sdma_fw);
2321 rdev->sdma_fw = NULL;
02c81327
AD		 2322 release_firmware(rdev->mc_fw);
2323 rdev->mc_fw = NULL;
cc8dbbb4
AD		 2324 release_firmware(rdev->smc_fw);
2325 rdev->smc_fw = NULL;
02c81327
AD		 2326 }
2327 return err;
2328}
2329
8cc1a532
AD		 2330/*
2331 * Core functions
2332 */
2333/**
2334 * cik_tiling_mode_table_init - init the hw tiling table
2335 *
2336 * @rdev: radeon_device pointer
2337 *
2338 * Starting with SI, the tiling setup is done globally in a
2339 * set of 32 tiling modes. Rather than selecting each set of
2340 * parameters per surface as on older asics, we just select
2341 * which index in the tiling table we want to use, and the
2342 * surface uses those parameters (CIK).
2343 */
2344static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2345{
2346 const u32 num_tile_mode_states = 32;
2347 const u32 num_secondary_tile_mode_states = 16;
2348 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2349 u32 num_pipe_configs;
2350 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2351 rdev->config.cik.max_shader_engines;
2352
2353 switch (rdev->config.cik.mem_row_size_in_kb) {
2354 case 1:
2355 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2356 break;
2357 case 2:
2358 default:
2359 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2360 break;
2361 case 4:
2362 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2363 break;
2364 }
2365
2366 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2367 if (num_pipe_configs > 8)
21e438af 2368 num_pipe_configs = 16;
8cc1a532 2369
21e438af
AD		 2370 if (num_pipe_configs == 16) {
2371 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2372 switch (reg_offset) {
2373 case 0:
2374 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2375 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2376 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2377 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2378 break;
2379 case 1:
2380 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2381 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2382 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2383 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2384 break;
2385 case 2:
2386 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2387 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2388 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2389 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2390 break;
2391 case 3:
2392 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2393 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2394 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2395 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2396 break;
2397 case 4:
2398 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2399 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2400 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2401 TILE_SPLIT(split_equal_to_row_size));
2402 break;
2403 case 5:
2404 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2405 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2406 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2407 break;
2408 case 6:
2409 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2410 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2411 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2412 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2413 break;
2414 case 7:
2415 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2416 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2417 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2418 TILE_SPLIT(split_equal_to_row_size));
2419 break;
2420 case 8:
2421 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2422 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2423 break;
2424 case 9:
2425 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2426 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2427 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2428 break;
2429 case 10:
2430 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2431 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2432 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2433 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2434 break;
2435 case 11:
2436 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2437 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2438 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2439 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2440 break;
2441 case 12:
2442 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2443 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2444 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2445 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2446 break;
2447 case 13:
2448 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2449 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2450 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2451 break;
2452 case 14:
2453 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2454 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2455 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2456 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2457 break;
2458 case 16:
2459 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2460 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2461 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2462 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2463 break;
2464 case 17:
2465 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2466 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2467 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2468 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2469 break;
2470 case 27:
2471 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2472 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2473 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2474 break;
2475 case 28:
2476 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2477 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2478 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2479 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2480 break;
2481 case 29:
2482 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2483 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2484 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2485 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2486 break;
2487 case 30:
2488 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2489 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2490 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2491 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2492 break;
2493 default:
2494 gb_tile_moden = 0;
2495 break;
2496 }
2497 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2498 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2499 }
2500 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2501 switch (reg_offset) {
2502 case 0:
2503 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2504 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2505 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2506 NUM_BANKS(ADDR_SURF_16_BANK));
2507 break;
2508 case 1:
2509 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2510 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2511 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2512 NUM_BANKS(ADDR_SURF_16_BANK));
2513 break;
2514 case 2:
2515 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2516 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2517 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2518 NUM_BANKS(ADDR_SURF_16_BANK));
2519 break;
2520 case 3:
2521 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2522 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2523 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2524 NUM_BANKS(ADDR_SURF_16_BANK));
2525 break;
2526 case 4:
2527 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2528 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2529 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2530 NUM_BANKS(ADDR_SURF_8_BANK));
2531 break;
2532 case 5:
2533 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2534 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2535 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2536 NUM_BANKS(ADDR_SURF_4_BANK));
2537 break;
2538 case 6:
2539 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2540 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2541 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2542 NUM_BANKS(ADDR_SURF_2_BANK));
2543 break;
2544 case 8:
2545 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2546 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2547 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2548 NUM_BANKS(ADDR_SURF_16_BANK));
2549 break;
2550 case 9:
2551 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2552 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2553 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2554 NUM_BANKS(ADDR_SURF_16_BANK));
2555 break;
2556 case 10:
2557 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2558 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2559 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2560 NUM_BANKS(ADDR_SURF_16_BANK));
2561 break;
2562 case 11:
2563 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2564 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2565 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2566 NUM_BANKS(ADDR_SURF_8_BANK));
2567 break;
2568 case 12:
2569 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2570 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2571 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2572 NUM_BANKS(ADDR_SURF_4_BANK));
2573 break;
2574 case 13:
2575 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2576 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2577 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2578 NUM_BANKS(ADDR_SURF_2_BANK));
2579 break;
2580 case 14:
2581 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2582 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2583 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2584 NUM_BANKS(ADDR_SURF_2_BANK));
2585 break;
2586 default:
2587 gb_tile_moden = 0;
2588 break;
2589 }
1b2c4869 2590 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
21e438af
AD		 2591 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2592 }
2593 } else if (num_pipe_configs == 8) {
8cc1a532
AD		 2594 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2595 switch (reg_offset) {
2596 case 0:
2597 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2598 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2599 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2600 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2601 break;
2602 case 1:
2603 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2604 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2605 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2606 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2607 break;
2608 case 2:
2609 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2610 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2611 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2612 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2613 break;
2614 case 3:
2615 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2616 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2617 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2618 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2619 break;
2620 case 4:
2621 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2622 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2623 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2624 TILE_SPLIT(split_equal_to_row_size));
2625 break;
2626 case 5:
2627 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2628 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2629 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2630 break;
2631 case 6:
2632 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2633 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2634 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2635 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2636 break;
2637 case 7:
2638 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2639 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2640 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2641 TILE_SPLIT(split_equal_to_row_size));
2642 break;
2643 case 8:
2644 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2645 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2646 break;
2647 case 9:
2648 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2649 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2650 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2651 break;
2652 case 10:
2653 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2654 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2655 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2656 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2657 break;
2658 case 11:
2659 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2660 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2661 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2662 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2663 break;
2664 case 12:
2665 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2666 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2667 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2668 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2669 break;
2670 case 13:
2671 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2672 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2673 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2674 break;
2675 case 14:
2676 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2677 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2678 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2679 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2680 break;
2681 case 16:
2682 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2683 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2684 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2685 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2686 break;
2687 case 17:
2688 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2689 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2690 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2691 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2692 break;
2693 case 27:
2694 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2695 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2696 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2697 break;
2698 case 28:
2699 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2700 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2701 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2702 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2703 break;
2704 case 29:
2705 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2706 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2707 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2708 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2709 break;
2710 case 30:
2711 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2712 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2713 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2714 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2715 break;
2716 default:
2717 gb_tile_moden = 0;
2718 break;
2719 }
39aee490 2720 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2721 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2722 }
2723 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2724 switch (reg_offset) {
2725 case 0:
2726 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2728 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2729 NUM_BANKS(ADDR_SURF_16_BANK));
2730 break;
2731 case 1:
2732 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2733 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2734 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2735 NUM_BANKS(ADDR_SURF_16_BANK));
2736 break;
2737 case 2:
2738 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2739 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2740 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2741 NUM_BANKS(ADDR_SURF_16_BANK));
2742 break;
2743 case 3:
2744 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2745 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2746 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2747 NUM_BANKS(ADDR_SURF_16_BANK));
2748 break;
2749 case 4:
2750 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2751 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2752 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2753 NUM_BANKS(ADDR_SURF_8_BANK));
2754 break;
2755 case 5:
2756 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2757 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2758 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2759 NUM_BANKS(ADDR_SURF_4_BANK));
2760 break;
2761 case 6:
2762 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2763 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2764 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2765 NUM_BANKS(ADDR_SURF_2_BANK));
2766 break;
2767 case 8:
2768 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2769 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2770 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2771 NUM_BANKS(ADDR_SURF_16_BANK));
2772 break;
2773 case 9:
2774 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2775 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2776 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2777 NUM_BANKS(ADDR_SURF_16_BANK));
2778 break;
2779 case 10:
2780 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2781 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2782 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2783 NUM_BANKS(ADDR_SURF_16_BANK));
2784 break;
2785 case 11:
2786 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2787 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2788 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2789 NUM_BANKS(ADDR_SURF_16_BANK));
2790 break;
2791 case 12:
2792 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2793 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2794 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2795 NUM_BANKS(ADDR_SURF_8_BANK));
2796 break;
2797 case 13:
2798 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2799 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2800 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2801 NUM_BANKS(ADDR_SURF_4_BANK));
2802 break;
2803 case 14:
2804 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2805 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2806 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2807 NUM_BANKS(ADDR_SURF_2_BANK));
2808 break;
2809 default:
2810 gb_tile_moden = 0;
2811 break;
2812 }
32f79a8a 2813 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2814 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2815 }
2816 } else if (num_pipe_configs == 4) {
2817 if (num_rbs == 4) {
2818 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2819 switch (reg_offset) {
2820 case 0:
2821 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2822 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2823 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2824 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2825 break;
2826 case 1:
2827 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2828 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2829 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2830 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2831 break;
2832 case 2:
2833 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2834 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2835 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2836 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2837 break;
2838 case 3:
2839 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2840 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2841 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2842 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2843 break;
2844 case 4:
2845 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2846 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2847 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2848 TILE_SPLIT(split_equal_to_row_size));
2849 break;
2850 case 5:
2851 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2852 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2853 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2854 break;
2855 case 6:
2856 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2857 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2858 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2859 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2860 break;
2861 case 7:
2862 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2863 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2864 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2865 TILE_SPLIT(split_equal_to_row_size));
2866 break;
2867 case 8:
2868 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2869 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2870 break;
2871 case 9:
2872 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2873 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2874 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2875 break;
2876 case 10:
2877 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2878 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2879 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2880 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2881 break;
2882 case 11:
2883 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2884 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2885 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2886 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2887 break;
2888 case 12:
2889 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2890 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2891 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2892 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2893 break;
2894 case 13:
2895 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2896 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2897 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2898 break;
2899 case 14:
2900 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2901 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2902 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2903 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2904 break;
2905 case 16:
2906 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2907 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2908 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2909 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2910 break;
2911 case 17:
2912 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2913 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2914 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2915 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2916 break;
2917 case 27:
2918 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2919 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2920 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2921 break;
2922 case 28:
2923 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2924 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2925 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2926 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2927 break;
2928 case 29:
2929 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2930 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2931 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2932 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2933 break;
2934 case 30:
2935 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2936 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2937 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2938 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2939 break;
2940 default:
2941 gb_tile_moden = 0;
2942 break;
2943 }
39aee490 2944 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2945 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2946 }
2947 } else if (num_rbs < 4) {
2948 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2949 switch (reg_offset) {
2950 case 0:
2951 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2952 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2953 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2954 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2955 break;
2956 case 1:
2957 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2958 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2959 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2960 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2961 break;
2962 case 2:
2963 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2964 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2965 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2966 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2967 break;
2968 case 3:
2969 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2970 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2971 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2972 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2973 break;
2974 case 4:
2975 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2976 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2977 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2978 TILE_SPLIT(split_equal_to_row_size));
2979 break;
2980 case 5:
2981 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2982 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 2983 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2984 break;
2985 case 6:
2986 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2987 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2988 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2989 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2990 break;
2991 case 7:
2992 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2993 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2994 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2995 TILE_SPLIT(split_equal_to_row_size));
2996 break;
2997 case 8:
2998 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2999 PIPE_CONFIG(ADDR_SURF_P4_8x16));
3000 break;
3001 case 9:
3002 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 3003 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 3004 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
3005 break;
3006 case 10:
3007 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3008 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3009 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3010 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3011 break;
3012 case 11:
3013 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3014 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3015 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3016 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3017 break;
3018 case 12:
3019 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3020 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3021 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3022 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3023 break;
3024 case 13:
3025 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 3026 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 3027 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
3028 break;
3029 case 14:
3030 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3031 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3032 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3033 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3034 break;
3035 case 16:
3036 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3037 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3038 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3039 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3040 break;
3041 case 17:
3042 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3043 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3044 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3045 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3046 break;
3047 case 27:
3048 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 3049 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 3050 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
3051 break;
3052 case 28:
3053 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3054 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3055 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3056 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3057 break;
3058 case 29:
3059 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3060 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3061 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3062 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3063 break;
3064 case 30:
3065 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3066 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3067 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3068 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3069 break;
3070 default:
3071 gb_tile_moden = 0;
3072 break;
3073 }
39aee490 3074 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3075 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3076 }
3077 }
3078 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3079 switch (reg_offset) {
3080 case 0:
3081 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3082 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3083 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3084 NUM_BANKS(ADDR_SURF_16_BANK));
3085 break;
3086 case 1:
3087 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3088 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3089 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3090 NUM_BANKS(ADDR_SURF_16_BANK));
3091 break;
3092 case 2:
3093 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3094 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3095 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3096 NUM_BANKS(ADDR_SURF_16_BANK));
3097 break;
3098 case 3:
3099 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3100 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3101 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3102 NUM_BANKS(ADDR_SURF_16_BANK));
3103 break;
3104 case 4:
3105 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3106 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3107 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3108 NUM_BANKS(ADDR_SURF_16_BANK));
3109 break;
3110 case 5:
3111 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3112 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3113 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3114 NUM_BANKS(ADDR_SURF_8_BANK));
3115 break;
3116 case 6:
3117 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3118 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3119 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3120 NUM_BANKS(ADDR_SURF_4_BANK));
3121 break;
3122 case 8:
3123 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3124 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3125 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3126 NUM_BANKS(ADDR_SURF_16_BANK));
3127 break;
3128 case 9:
3129 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3130 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3131 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3132 NUM_BANKS(ADDR_SURF_16_BANK));
3133 break;
3134 case 10:
3135 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3136 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3137 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3138 NUM_BANKS(ADDR_SURF_16_BANK));
3139 break;
3140 case 11:
3141 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3142 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3143 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3144 NUM_BANKS(ADDR_SURF_16_BANK));
3145 break;
3146 case 12:
3147 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3148 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3149 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3150 NUM_BANKS(ADDR_SURF_16_BANK));
3151 break;
3152 case 13:
3153 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3154 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3155 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3156 NUM_BANKS(ADDR_SURF_8_BANK));
3157 break;
3158 case 14:
3159 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3160 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3161 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3162 NUM_BANKS(ADDR_SURF_4_BANK));
3163 break;
3164 default:
3165 gb_tile_moden = 0;
3166 break;
3167 }
32f79a8a 3168 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3169 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3170 }
3171 } else if (num_pipe_configs == 2) {
3172 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
3173 switch (reg_offset) {
3174 case 0:
3175 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3176 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3177 PIPE_CONFIG(ADDR_SURF_P2) |
3178 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
3179 break;
3180 case 1:
3181 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3182 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3183 PIPE_CONFIG(ADDR_SURF_P2) |
3184 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
3185 break;
3186 case 2:
3187 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3188 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3189 PIPE_CONFIG(ADDR_SURF_P2) |
3190 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3191 break;
3192 case 3:
3193 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3194 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3195 PIPE_CONFIG(ADDR_SURF_P2) |
3196 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
3197 break;
3198 case 4:
3199 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3200 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3201 PIPE_CONFIG(ADDR_SURF_P2) |
3202 TILE_SPLIT(split_equal_to_row_size));
3203 break;
3204 case 5:
3205 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 3206 PIPE_CONFIG(ADDR_SURF_P2) |
8cc1a532
AD		 3207 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3208 break;
3209 case 6:
3210 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3211 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3212 PIPE_CONFIG(ADDR_SURF_P2) |
3213 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3214 break;
3215 case 7:
3216 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3217 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3218 PIPE_CONFIG(ADDR_SURF_P2) |
3219 TILE_SPLIT(split_equal_to_row_size));
3220 break;
3221 case 8:
020ff546
MO		 3222 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3223 PIPE_CONFIG(ADDR_SURF_P2);
8cc1a532
AD		 3224 break;
3225 case 9:
3226 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546
MO		 3227 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3228 PIPE_CONFIG(ADDR_SURF_P2));
8cc1a532
AD		 3229 break;
3230 case 10:
3231 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3232 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3233 PIPE_CONFIG(ADDR_SURF_P2) |
3234 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3235 break;
3236 case 11:
3237 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3238 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3239 PIPE_CONFIG(ADDR_SURF_P2) |
3240 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3241 break;
3242 case 12:
3243 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3244 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3245 PIPE_CONFIG(ADDR_SURF_P2) |
3246 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3247 break;
3248 case 13:
3249 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 3250 PIPE_CONFIG(ADDR_SURF_P2) |
8cc1a532
AD		 3251 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
3252 break;
3253 case 14:
3254 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3255 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3256 PIPE_CONFIG(ADDR_SURF_P2) |
3257 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3258 break;
3259 case 16:
3260 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3261 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3262 PIPE_CONFIG(ADDR_SURF_P2) |
3263 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3264 break;
3265 case 17:
3266 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3267 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3268 PIPE_CONFIG(ADDR_SURF_P2) |
3269 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3270 break;
3271 case 27:
3272 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546
MO		 3273 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3274 PIPE_CONFIG(ADDR_SURF_P2));
8cc1a532
AD		 3275 break;
3276 case 28:
3277 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3278 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3279 PIPE_CONFIG(ADDR_SURF_P2) |
3280 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3281 break;
3282 case 29:
3283 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3284 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3285 PIPE_CONFIG(ADDR_SURF_P2) |
3286 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3287 break;
3288 case 30:
3289 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3290 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3291 PIPE_CONFIG(ADDR_SURF_P2) |
3292 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3293 break;
3294 default:
3295 gb_tile_moden = 0;
3296 break;
3297 }
39aee490 3298 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3299 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3300 }
3301 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3302 switch (reg_offset) {
3303 case 0:
3304 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3305 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3306 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3307 NUM_BANKS(ADDR_SURF_16_BANK));
3308 break;
3309 case 1:
3310 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3311 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3312 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3313 NUM_BANKS(ADDR_SURF_16_BANK));
3314 break;
3315 case 2:
3316 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3317 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3318 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3319 NUM_BANKS(ADDR_SURF_16_BANK));
3320 break;
3321 case 3:
3322 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3323 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3324 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3325 NUM_BANKS(ADDR_SURF_16_BANK));
3326 break;
3327 case 4:
3328 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3329 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3330 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3331 NUM_BANKS(ADDR_SURF_16_BANK));
3332 break;
3333 case 5:
3334 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3335 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3336 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3337 NUM_BANKS(ADDR_SURF_16_BANK));
3338 break;
3339 case 6:
3340 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3341 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3342 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3343 NUM_BANKS(ADDR_SURF_8_BANK));
3344 break;
3345 case 8:
3346 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3347 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3348 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3349 NUM_BANKS(ADDR_SURF_16_BANK));
3350 break;
3351 case 9:
3352 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3353 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3354 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3355 NUM_BANKS(ADDR_SURF_16_BANK));
3356 break;
3357 case 10:
3358 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3359 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3360 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3361 NUM_BANKS(ADDR_SURF_16_BANK));
3362 break;
3363 case 11:
3364 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3365 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3366 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3367 NUM_BANKS(ADDR_SURF_16_BANK));
3368 break;
3369 case 12:
3370 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3371 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3372 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3373 NUM_BANKS(ADDR_SURF_16_BANK));
3374 break;
3375 case 13:
3376 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3377 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3378 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3379 NUM_BANKS(ADDR_SURF_16_BANK));
3380 break;
3381 case 14:
3382 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3383 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3384 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3385 NUM_BANKS(ADDR_SURF_8_BANK));
3386 break;
3387 default:
3388 gb_tile_moden = 0;
3389 break;
3390 }
32f79a8a 3391 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3392 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3393 }
3394 } else
3395 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3396}
3397
3398/**
3399 * cik_select_se_sh - select which SE, SH to address
3400 *
3401 * @rdev: radeon_device pointer
3402 * @se_num: shader engine to address
3403 * @sh_num: sh block to address
3404 *
3405 * Select which SE, SH combinations to address. Certain
3406 * registers are instanced per SE or SH. 0xffffffff means
3407 * broadcast to all SEs or SHs (CIK).
3408 */
3409static void cik_select_se_sh(struct radeon_device *rdev,
3410 u32 se_num, u32 sh_num)
3411{
3412 u32 data = INSTANCE_BROADCAST_WRITES;
3413
3414 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
b0fe3d39 3415 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
8cc1a532
AD		 3416 else if (se_num == 0xffffffff)
3417 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3418 else if (sh_num == 0xffffffff)
3419 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3420 else
3421 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3422 WREG32(GRBM_GFX_INDEX, data);
3423}
3424
3425/**
3426 * cik_create_bitmask - create a bitmask
3427 *
3428 * @bit_width: length of the mask
3429 *
3430 * create a variable length bit mask (CIK).
3431 * Returns the bitmask.
3432 */
3433static u32 cik_create_bitmask(u32 bit_width)
3434{
3435 u32 i, mask = 0;
3436
3437 for (i = 0; i < bit_width; i++) {
3438 mask <<= 1;
3439 mask |= 1;
3440 }
3441 return mask;
3442}
3443
3444/**
972c5ddb 3445 * cik_get_rb_disabled - computes the mask of disabled RBs
8cc1a532
AD		 3446 *
3447 * @rdev: radeon_device pointer
3448 * @max_rb_num: max RBs (render backends) for the asic
3449 * @se_num: number of SEs (shader engines) for the asic
3450 * @sh_per_se: number of SH blocks per SE for the asic
3451 *
3452 * Calculates the bitmask of disabled RBs (CIK).
3453 * Returns the disabled RB bitmask.
3454 */
3455static u32 cik_get_rb_disabled(struct radeon_device *rdev,
9fadb352 3456 u32 max_rb_num_per_se,
8cc1a532
AD		 3457 u32 sh_per_se)
3458{
3459 u32 data, mask;
3460
3461 data = RREG32(CC_RB_BACKEND_DISABLE);
3462 if (data & 1)
3463 data &= BACKEND_DISABLE_MASK;
3464 else
3465 data = 0;
3466 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3467
3468 data >>= BACKEND_DISABLE_SHIFT;
3469
9fadb352 3470 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
8cc1a532
AD		 3471
3472 return data & mask;
3473}
3474
3475/**
3476 * cik_setup_rb - setup the RBs on the asic
3477 *
3478 * @rdev: radeon_device pointer
3479 * @se_num: number of SEs (shader engines) for the asic
3480 * @sh_per_se: number of SH blocks per SE for the asic
3481 * @max_rb_num: max RBs (render backends) for the asic
3482 *
3483 * Configures per-SE/SH RB registers (CIK).
3484 */
3485static void cik_setup_rb(struct radeon_device *rdev,
3486 u32 se_num, u32 sh_per_se,
9fadb352 3487 u32 max_rb_num_per_se)
8cc1a532
AD		 3488{
3489 int i, j;
3490 u32 data, mask;
3491 u32 disabled_rbs = 0;
3492 u32 enabled_rbs = 0;
3493
1c0a4625 3494 mutex_lock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3495 for (i = 0; i < se_num; i++) {
3496 for (j = 0; j < sh_per_se; j++) {
3497 cik_select_se_sh(rdev, i, j);
9fadb352 3498 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
fc821b70
AD		 3499 if (rdev->family == CHIP_HAWAII)
3500 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3501 else
3502 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
8cc1a532
AD		 3503 }
3504 }
3505 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 3506 mutex_unlock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3507
3508 mask = 1;
9fadb352 3509 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
8cc1a532
AD		 3510 if (!(disabled_rbs & mask))
3511 enabled_rbs |= mask;
3512 mask <<= 1;
3513 }
3514
439a1cff
MO		 3515 rdev->config.cik.backend_enable_mask = enabled_rbs;
3516
1c0a4625 3517 mutex_lock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3518 for (i = 0; i < se_num; i++) {
3519 cik_select_se_sh(rdev, i, 0xffffffff);
3520 data = 0;
3521 for (j = 0; j < sh_per_se; j++) {
3522 switch (enabled_rbs & 3) {
fc821b70
AD		 3523 case 0:
3524 if (j == 0)
3525 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3526 else
3527 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3528 break;
8cc1a532
AD		 3529 case 1:
3530 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3531 break;
3532 case 2:
3533 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3534 break;
3535 case 3:
3536 default:
3537 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3538 break;
3539 }
3540 enabled_rbs >>= 2;
3541 }
3542 WREG32(PA_SC_RASTER_CONFIG, data);
3543 }
3544 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 3545 mutex_unlock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3546}
3547
3548/**
3549 * cik_gpu_init - setup the 3D engine
3550 *
3551 * @rdev: radeon_device pointer
3552 *
3553 * Configures the 3D engine and tiling configuration
3554 * registers so that the 3D engine is usable.
3555 */
3556static void cik_gpu_init(struct radeon_device *rdev)
3557{
3558 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3559 u32 mc_shared_chmap, mc_arb_ramcfg;
3560 u32 hdp_host_path_cntl;
3561 u32 tmp;
6101b3ae 3562 int i, j;
8cc1a532
AD		 3563
3564 switch (rdev->family) {
3565 case CHIP_BONAIRE:
3566 rdev->config.cik.max_shader_engines = 2;
3567 rdev->config.cik.max_tile_pipes = 4;
3568 rdev->config.cik.max_cu_per_sh = 7;
3569 rdev->config.cik.max_sh_per_se = 1;
3570 rdev->config.cik.max_backends_per_se = 2;
3571 rdev->config.cik.max_texture_channel_caches = 4;
3572 rdev->config.cik.max_gprs = 256;
3573 rdev->config.cik.max_gs_threads = 32;
3574 rdev->config.cik.max_hw_contexts = 8;
3575
3576 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3577 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3578 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3579 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3580 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3581 break;
b496038b
AD		 3582 case CHIP_HAWAII:
3583 rdev->config.cik.max_shader_engines = 4;
3584 rdev->config.cik.max_tile_pipes = 16;
3585 rdev->config.cik.max_cu_per_sh = 11;
3586 rdev->config.cik.max_sh_per_se = 1;
3587 rdev->config.cik.max_backends_per_se = 4;
3588 rdev->config.cik.max_texture_channel_caches = 16;
3589 rdev->config.cik.max_gprs = 256;
3590 rdev->config.cik.max_gs_threads = 32;
3591 rdev->config.cik.max_hw_contexts = 8;
3592
3593 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3594 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3595 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3596 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3597 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3598 break;
8cc1a532 3599 case CHIP_KAVERI:
b2e4c70a
AD		 3600 rdev->config.cik.max_shader_engines = 1;
3601 rdev->config.cik.max_tile_pipes = 4;
3602 if ((rdev->pdev->device == 0x1304) ||
3603 (rdev->pdev->device == 0x1305) ||
3604 (rdev->pdev->device == 0x130C) ||
3605 (rdev->pdev->device == 0x130F) ||
3606 (rdev->pdev->device == 0x1310) ||
3607 (rdev->pdev->device == 0x1311) ||
3608 (rdev->pdev->device == 0x131C)) {
3609 rdev->config.cik.max_cu_per_sh = 8;
3610 rdev->config.cik.max_backends_per_se = 2;
3611 } else if ((rdev->pdev->device == 0x1309) ||
3612 (rdev->pdev->device == 0x130A) ||
3613 (rdev->pdev->device == 0x130D) ||
7c4622d5
AD		 3614 (rdev->pdev->device == 0x1313) ||
3615 (rdev->pdev->device == 0x131D)) {
b2e4c70a
AD		 3616 rdev->config.cik.max_cu_per_sh = 6;
3617 rdev->config.cik.max_backends_per_se = 2;
3618 } else if ((rdev->pdev->device == 0x1306) ||
3619 (rdev->pdev->device == 0x1307) ||
3620 (rdev->pdev->device == 0x130B) ||
3621 (rdev->pdev->device == 0x130E) ||
3622 (rdev->pdev->device == 0x1315) ||
6dc14baf 3623 (rdev->pdev->device == 0x1318) ||
b2e4c70a
AD		 3624 (rdev->pdev->device == 0x131B)) {
3625 rdev->config.cik.max_cu_per_sh = 4;
3626 rdev->config.cik.max_backends_per_se = 1;
3627 } else {
3628 rdev->config.cik.max_cu_per_sh = 3;
3629 rdev->config.cik.max_backends_per_se = 1;
3630 }
3631 rdev->config.cik.max_sh_per_se = 1;
3632 rdev->config.cik.max_texture_channel_caches = 4;
3633 rdev->config.cik.max_gprs = 256;
3634 rdev->config.cik.max_gs_threads = 16;
3635 rdev->config.cik.max_hw_contexts = 8;
3636
3637 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3638 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3639 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3640 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3641 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
8cc1a532
AD		 3642 break;
3643 case CHIP_KABINI:
f73a9e83 3644 case CHIP_MULLINS:
8cc1a532
AD		 3645 default:
3646 rdev->config.cik.max_shader_engines = 1;
3647 rdev->config.cik.max_tile_pipes = 2;
3648 rdev->config.cik.max_cu_per_sh = 2;
3649 rdev->config.cik.max_sh_per_se = 1;
3650 rdev->config.cik.max_backends_per_se = 1;
3651 rdev->config.cik.max_texture_channel_caches = 2;
3652 rdev->config.cik.max_gprs = 256;
3653 rdev->config.cik.max_gs_threads = 16;
3654 rdev->config.cik.max_hw_contexts = 8;
3655
3656 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3657 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3658 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3659 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3660 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3661 break;
3662 }
3663
3664 /* Initialize HDP */
3665 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3666 WREG32((0x2c14 + j), 0x00000000);
3667 WREG32((0x2c18 + j), 0x00000000);
3668 WREG32((0x2c1c + j), 0x00000000);
3669 WREG32((0x2c20 + j), 0x00000000);
3670 WREG32((0x2c24 + j), 0x00000000);
3671 }
3672
3673 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
dc12a3ec
LL		 3674 WREG32(SRBM_INT_CNTL, 0x1);
3675 WREG32(SRBM_INT_ACK, 0x1);
8cc1a532
AD		 3676
3677 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3678
3679 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3680 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3681
3682 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3683 rdev->config.cik.mem_max_burst_length_bytes = 256;
3684 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3685 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3686 if (rdev->config.cik.mem_row_size_in_kb > 4)
3687 rdev->config.cik.mem_row_size_in_kb = 4;
3688 /* XXX use MC settings? */
3689 rdev->config.cik.shader_engine_tile_size = 32;
3690 rdev->config.cik.num_gpus = 1;
3691 rdev->config.cik.multi_gpu_tile_size = 64;
3692
3693 /* fix up row size */
3694 gb_addr_config &= ~ROW_SIZE_MASK;
3695 switch (rdev->config.cik.mem_row_size_in_kb) {
3696 case 1:
3697 default:
3698 gb_addr_config |= ROW_SIZE(0);
3699 break;
3700 case 2:
3701 gb_addr_config |= ROW_SIZE(1);
3702 break;
3703 case 4:
3704 gb_addr_config |= ROW_SIZE(2);
3705 break;
3706 }
3707
3708 /* setup tiling info dword. gb_addr_config is not adequate since it does
3709 * not have bank info, so create a custom tiling dword.
3710 * bits 3:0 num_pipes
3711 * bits 7:4 num_banks
3712 * bits 11:8 group_size
3713 * bits 15:12 row_size
3714 */
3715 rdev->config.cik.tile_config = 0;
3716 switch (rdev->config.cik.num_tile_pipes) {
3717 case 1:
3718 rdev->config.cik.tile_config |= (0 << 0);
3719 break;
3720 case 2:
3721 rdev->config.cik.tile_config |= (1 << 0);
3722 break;
3723 case 4:
3724 rdev->config.cik.tile_config |= (2 << 0);
3725 break;
3726 case 8:
3727 default:
3728 /* XXX what about 12? */
3729 rdev->config.cik.tile_config |= (3 << 0);
3730 break;
3731 }
a537314e
MD		 3732 rdev->config.cik.tile_config |=
3733 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
8cc1a532
AD		 3734 rdev->config.cik.tile_config |=
3735 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3736 rdev->config.cik.tile_config |=
3737 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3738
3739 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3740 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3741 WREG32(DMIF_ADDR_CALC, gb_addr_config);
21a93e13
AD		 3742 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3743 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
87167bb1
CK		 3744 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3745 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3746 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
8cc1a532
AD		 3747
3748 cik_tiling_mode_table_init(rdev);
3749
3750 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3751 rdev->config.cik.max_sh_per_se,
3752 rdev->config.cik.max_backends_per_se);
3753
52da51f0 3754 rdev->config.cik.active_cus = 0;
65fcf668
AD		 3755 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3756 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6101b3ae
AD		 3757 rdev->config.cik.active_cus +=
3758 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
65fcf668
AD		 3759 }
3760 }
3761
8cc1a532
AD		 3762 /* set HW defaults for 3D engine */
3763 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3764
1c0a4625
OG		 3765 mutex_lock(&rdev->grbm_idx_mutex);
3766 /*
3767 * making sure that the following register writes will be broadcasted
3768 * to all the shaders
3769 */
3770 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
8cc1a532
AD		 3771 WREG32(SX_DEBUG_1, 0x20);
3772
3773 WREG32(TA_CNTL_AUX, 0x00010000);
3774
3775 tmp = RREG32(SPI_CONFIG_CNTL);
3776 tmp |= 0x03000000;
3777 WREG32(SPI_CONFIG_CNTL, tmp);
3778
3779 WREG32(SQ_CONFIG, 1);
3780
3781 WREG32(DB_DEBUG, 0);
3782
3783 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3784 tmp |= 0x00000400;
3785 WREG32(DB_DEBUG2, tmp);
3786
3787 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3788 tmp |= 0x00020200;
3789 WREG32(DB_DEBUG3, tmp);
3790
3791 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3792 tmp |= 0x00018208;
3793 WREG32(CB_HW_CONTROL, tmp);
3794
3795 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3796
3797 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3798 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3799 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3800 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3801
3802 WREG32(VGT_NUM_INSTANCES, 1);
3803
3804 WREG32(CP_PERFMON_CNTL, 0);
3805
3806 WREG32(SQ_CONFIG, 0);
3807
3808 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3809 FORCE_EOV_MAX_REZ_CNT(255)));
3810
3811 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3812 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3813
3814 WREG32(VGT_GS_VERTEX_REUSE, 16);
3815 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3816
3817 tmp = RREG32(HDP_MISC_CNTL);
3818 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3819 WREG32(HDP_MISC_CNTL, tmp);
3820
3821 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3822 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3823
3824 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3825 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
1c0a4625 3826 mutex_unlock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3827
3828 udelay(50);
3829}
3830
2cae3bc3
AD		 3831/*
3832 * GPU scratch registers helpers function.
3833 */
3834/**
3835 * cik_scratch_init - setup driver info for CP scratch regs
3836 *
3837 * @rdev: radeon_device pointer
3838 *
3839 * Set up the number and offset of the CP scratch registers.
3840 * NOTE: use of CP scratch registers is a legacy inferface and
3841 * is not used by default on newer asics (r6xx+). On newer asics,
3842 * memory buffers are used for fences rather than scratch regs.
3843 */
3844static void cik_scratch_init(struct radeon_device *rdev)
3845{
3846 int i;
3847
3848 rdev->scratch.num_reg = 7;
3849 rdev->scratch.reg_base = SCRATCH_REG0;
3850 for (i = 0; i < rdev->scratch.num_reg; i++) {
3851 rdev->scratch.free[i] = true;
3852 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3853 }
3854}
3855
fbc832c7
AD		 3856/**
3857 * cik_ring_test - basic gfx ring test
3858 *
3859 * @rdev: radeon_device pointer
3860 * @ring: radeon_ring structure holding ring information
3861 *
3862 * Allocate a scratch register and write to it using the gfx ring (CIK).
3863 * Provides a basic gfx ring test to verify that the ring is working.
3864 * Used by cik_cp_gfx_resume();
3865 * Returns 0 on success, error on failure.
3866 */
3867int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3868{
3869 uint32_t scratch;
3870 uint32_t tmp = 0;
3871 unsigned i;
3872 int r;
3873
3874 r = radeon_scratch_get(rdev, &scratch);
3875 if (r) {
3876 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3877 return r;
3878 }
3879 WREG32(scratch, 0xCAFEDEAD);
3880 r = radeon_ring_lock(rdev, ring, 3);
3881 if (r) {
3882 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3883 radeon_scratch_free(rdev, scratch);
3884 return r;
3885 }
3886 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3887 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3888 radeon_ring_write(ring, 0xDEADBEEF);
1538a9e0 3889 radeon_ring_unlock_commit(rdev, ring, false);
963e81f9 3890
fbc832c7
AD		 3891 for (i = 0; i < rdev->usec_timeout; i++) {
3892 tmp = RREG32(scratch);
3893 if (tmp == 0xDEADBEEF)
3894 break;
3895 DRM_UDELAY(1);
3896 }
3897 if (i < rdev->usec_timeout) {
3898 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3899 } else {
3900 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3901 ring->idx, scratch, tmp);
3902 r = -EINVAL;
3903 }
3904 radeon_scratch_free(rdev, scratch);
3905 return r;
3906}
3907
780f5ddd
AD		 3908/**
3909 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3910 *
3911 * @rdev: radeon_device pointer
3912 * @ridx: radeon ring index
3913 *
3914 * Emits an hdp flush on the cp.
3915 */
3916static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3917 int ridx)
3918{
3919 struct radeon_ring *ring = &rdev->ring[ridx];
5d259067 3920 u32 ref_and_mask;
780f5ddd 3921
5d259067
AD		 3922 switch (ring->idx) {
3923 case CAYMAN_RING_TYPE_CP1_INDEX:
3924 case CAYMAN_RING_TYPE_CP2_INDEX:
3925 default:
3926 switch (ring->me) {
3927 case 0:
3928 ref_and_mask = CP2 << ring->pipe;
3929 break;
3930 case 1:
3931 ref_and_mask = CP6 << ring->pipe;
3932 break;
3933 default:
3934 return;
3935 }
3936 break;
3937 case RADEON_RING_TYPE_GFX_INDEX:
3938 ref_and_mask = CP0;
3939 break;
3940 }
3941
3942 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3943 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3944 WAIT_REG_MEM_FUNCTION(3) | /* == */
3945 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3946 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3947 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3948 radeon_ring_write(ring, ref_and_mask);
3949 radeon_ring_write(ring, ref_and_mask);
3950 radeon_ring_write(ring, 0x20); /* poll interval */
780f5ddd
AD		 3951}
3952
2cae3bc3 3953/**
b07fdd38 3954 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
2cae3bc3
AD		 3955 *
3956 * @rdev: radeon_device pointer
3957 * @fence: radeon fence object
3958 *
3959 * Emits a fence sequnce number on the gfx ring and flushes
3960 * GPU caches.
3961 */
b07fdd38
AD		 3962void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3963 struct radeon_fence *fence)
2cae3bc3
AD		 3964{
3965 struct radeon_ring *ring = &rdev->ring[fence->ring];
3966 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3967
a9c73a0e
CK		 3968 /* Workaround for cache flush problems. First send a dummy EOP
3969 * event down the pipe with seq one below.
3970 */
3971 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3972 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3973 EOP_TC_ACTION_EN |
3974 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3975 EVENT_INDEX(5)));
3976 radeon_ring_write(ring, addr & 0xfffffffc);
3977 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
3978 DATA_SEL(1) | INT_SEL(0));
3979 radeon_ring_write(ring, fence->seq - 1);
3980 radeon_ring_write(ring, 0);
3981
3982 /* Then send the real EOP event down the pipe. */
2cae3bc3
AD		 3983 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3984 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3985 EOP_TC_ACTION_EN |
3986 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3987 EVENT_INDEX(5)));
3988 radeon_ring_write(ring, addr & 0xfffffffc);
3989 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3990 radeon_ring_write(ring, fence->seq);
3991 radeon_ring_write(ring, 0);
2cae3bc3
AD		 3992}
3993
b07fdd38
AD		 3994/**
3995 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3996 *
3997 * @rdev: radeon_device pointer
3998 * @fence: radeon fence object
3999 *
4000 * Emits a fence sequnce number on the compute ring and flushes
4001 * GPU caches.
4002 */
4003void cik_fence_compute_ring_emit(struct radeon_device *rdev,
4004 struct radeon_fence *fence)
4005{
4006 struct radeon_ring *ring = &rdev->ring[fence->ring];
4007 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
4008
4009 /* RELEASE_MEM - flush caches, send int */
4010 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
4011 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
4012 EOP_TC_ACTION_EN |
4013 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
4014 EVENT_INDEX(5)));
4015 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
4016 radeon_ring_write(ring, addr & 0xfffffffc);
4017 radeon_ring_write(ring, upper_32_bits(addr));
4018 radeon_ring_write(ring, fence->seq);
4019 radeon_ring_write(ring, 0);
b07fdd38
AD		 4020}
4021
86302eea
CK		 4022/**
4023 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
4024 *
4025 * @rdev: radeon_device pointer
4026 * @ring: radeon ring buffer object
4027 * @semaphore: radeon semaphore object
4028 * @emit_wait: Is this a sempahore wait?
4029 *
4030 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
4031 * from running ahead of semaphore waits.
4032 */
1654b817 4033bool cik_semaphore_ring_emit(struct radeon_device *rdev,
2cae3bc3
AD		 4034 struct radeon_ring *ring,
4035 struct radeon_semaphore *semaphore,
4036 bool emit_wait)
4037{
4038 uint64_t addr = semaphore->gpu_addr;
4039 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
4040
4041 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
5e167cdb 4042 radeon_ring_write(ring, lower_32_bits(addr));
2cae3bc3 4043 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
1654b817 4044
86302eea
CK		 4045 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
4046 /* Prevent the PFP from running ahead of the semaphore wait */
4047 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4048 radeon_ring_write(ring, 0x0);
4049 }
4050
1654b817 4051 return true;
2cae3bc3
AD		 4052}
4053
c9dbd705
AD		 4054/**
4055 * cik_copy_cpdma - copy pages using the CP DMA engine
4056 *
4057 * @rdev: radeon_device pointer
4058 * @src_offset: src GPU address
4059 * @dst_offset: dst GPU address
4060 * @num_gpu_pages: number of GPU pages to xfer
57d20a43 4061 * @resv: reservation object to sync to
c9dbd705
AD		 4062 *
4063 * Copy GPU paging using the CP DMA engine (CIK+).
4064 * Used by the radeon ttm implementation to move pages if
4065 * registered as the asic copy callback.
4066 */
57d20a43
CK		 4067struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
4068 uint64_t src_offset, uint64_t dst_offset,
4069 unsigned num_gpu_pages,
4070 struct reservation_object *resv)
c9dbd705 4071{
57d20a43 4072 struct radeon_fence *fence;
975700d2 4073 struct radeon_sync sync;
c9dbd705
AD		 4074 int ring_index = rdev->asic->copy.blit_ring_index;
4075 struct radeon_ring *ring = &rdev->ring[ring_index];
4076 u32 size_in_bytes, cur_size_in_bytes, control;
4077 int i, num_loops;
4078 int r = 0;
4079
975700d2 4080 radeon_sync_create(&sync);
c9dbd705
AD		 4081
4082 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
4083 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
4084 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
4085 if (r) {
4086 DRM_ERROR("radeon: moving bo (%d).\n", r);
975700d2 4087 radeon_sync_free(rdev, &sync, NULL);
57d20a43 4088 return ERR_PTR(r);
c9dbd705
AD		 4089 }
4090
975700d2
CK		 4091 radeon_sync_resv(rdev, &sync, resv, false);
4092 radeon_sync_rings(rdev, &sync, ring->idx);
c9dbd705
AD		 4093
4094 for (i = 0; i < num_loops; i++) {
4095 cur_size_in_bytes = size_in_bytes;
4096 if (cur_size_in_bytes > 0x1fffff)
4097 cur_size_in_bytes = 0x1fffff;
4098 size_in_bytes -= cur_size_in_bytes;
4099 control = 0;
4100 if (size_in_bytes == 0)
4101 control |= PACKET3_DMA_DATA_CP_SYNC;
4102 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4103 radeon_ring_write(ring, control);
4104 radeon_ring_write(ring, lower_32_bits(src_offset));
4105 radeon_ring_write(ring, upper_32_bits(src_offset));
4106 radeon_ring_write(ring, lower_32_bits(dst_offset));
4107 radeon_ring_write(ring, upper_32_bits(dst_offset));
4108 radeon_ring_write(ring, cur_size_in_bytes);
4109 src_offset += cur_size_in_bytes;
4110 dst_offset += cur_size_in_bytes;
4111 }
4112
57d20a43 4113 r = radeon_fence_emit(rdev, &fence, ring->idx);
c9dbd705
AD		 4114 if (r) {
4115 radeon_ring_unlock_undo(rdev, ring);
975700d2 4116 radeon_sync_free(rdev, &sync, NULL);
57d20a43 4117 return ERR_PTR(r);
c9dbd705
AD		 4118 }
4119
1538a9e0 4120 radeon_ring_unlock_commit(rdev, ring, false);
975700d2 4121 radeon_sync_free(rdev, &sync, fence);
c9dbd705 4122
57d20a43 4123 return fence;
c9dbd705
AD		 4124}
4125
2cae3bc3
AD		 4126/*
4127 * IB stuff
4128 */
4129/**
4130 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
4131 *
4132 * @rdev: radeon_device pointer
4133 * @ib: radeon indirect buffer object
4134 *
4135 * Emits an DE (drawing engine) or CE (constant engine) IB
4136 * on the gfx ring. IBs are usually generated by userspace
4137 * acceleration drivers and submitted to the kernel for
4138 * sheduling on the ring. This function schedules the IB
4139 * on the gfx ring for execution by the GPU.
4140 */
4141void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
4142{
4143 struct radeon_ring *ring = &rdev->ring[ib->ring];
7c42bc1a 4144 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
2cae3bc3
AD		 4145 u32 header, control = INDIRECT_BUFFER_VALID;
4146
4147 if (ib->is_const_ib) {
4148 /* set switch buffer packet before const IB */
4149 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
4150 radeon_ring_write(ring, 0);
4151
4152 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
4153 } else {
4154 u32 next_rptr;
4155 if (ring->rptr_save_reg) {
4156 next_rptr = ring->wptr + 3 + 4;
4157 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
4158 radeon_ring_write(ring, ((ring->rptr_save_reg -
4159 PACKET3_SET_UCONFIG_REG_START) >> 2));
4160 radeon_ring_write(ring, next_rptr);
4161 } else if (rdev->wb.enabled) {
4162 next_rptr = ring->wptr + 5 + 4;
4163 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4164 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
4165 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
5e167cdb 4166 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
2cae3bc3
AD		 4167 radeon_ring_write(ring, next_rptr);
4168 }
4169
4170 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
4171 }
4172
7c42bc1a 4173 control |= ib->length_dw | (vm_id << 24);
2cae3bc3
AD		 4174
4175 radeon_ring_write(ring, header);
5f3e226f 4176 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFFC));
2cae3bc3
AD		 4177 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4178 radeon_ring_write(ring, control);
4179}
4180
fbc832c7
AD		 4181/**
4182 * cik_ib_test - basic gfx ring IB test
4183 *
4184 * @rdev: radeon_device pointer
4185 * @ring: radeon_ring structure holding ring information
4186 *
4187 * Allocate an IB and execute it on the gfx ring (CIK).
4188 * Provides a basic gfx ring test to verify that IBs are working.
4189 * Returns 0 on success, error on failure.
4190 */
4191int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
4192{
4193 struct radeon_ib ib;
4194 uint32_t scratch;
4195 uint32_t tmp = 0;
4196 unsigned i;
4197 int r;
4198
4199 r = radeon_scratch_get(rdev, &scratch);
4200 if (r) {
4201 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
4202 return r;
4203 }
4204 WREG32(scratch, 0xCAFEDEAD);
4205 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
4206 if (r) {
4207 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
5510f124 4208 radeon_scratch_free(rdev, scratch);
fbc832c7
AD		 4209 return r;
4210 }
4211 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
4212 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
4213 ib.ptr[2] = 0xDEADBEEF;
4214 ib.length_dw = 3;
1538a9e0 4215 r = radeon_ib_schedule(rdev, &ib, NULL, false);
fbc832c7
AD		 4216 if (r) {
4217 radeon_scratch_free(rdev, scratch);
4218 radeon_ib_free(rdev, &ib);
4219 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
4220 return r;
4221 }
4222 r = radeon_fence_wait(ib.fence, false);
4223 if (r) {
4224 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
5510f124
CK		 4225 radeon_scratch_free(rdev, scratch);
4226 radeon_ib_free(rdev, &ib);
fbc832c7
AD		 4227 return r;
4228 }
4229 for (i = 0; i < rdev->usec_timeout; i++) {
4230 tmp = RREG32(scratch);
4231 if (tmp == 0xDEADBEEF)
4232 break;
4233 DRM_UDELAY(1);
4234 }
4235 if (i < rdev->usec_timeout) {
4236 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
4237 } else {
4238 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
4239 scratch, tmp);
4240 r = -EINVAL;
4241 }
4242 radeon_scratch_free(rdev, scratch);
4243 radeon_ib_free(rdev, &ib);
4244 return r;
4245}
4246
841cf442
AD		 4247/*
4248 * CP.
4249 * On CIK, gfx and compute now have independant command processors.
4250 *
4251 * GFX
4252 * Gfx consists of a single ring and can process both gfx jobs and
4253 * compute jobs. The gfx CP consists of three microengines (ME):
4254 * PFP - Pre-Fetch Parser
4255 * ME - Micro Engine
4256 * CE - Constant Engine
4257 * The PFP and ME make up what is considered the Drawing Engine (DE).
4258 * The CE is an asynchronous engine used for updating buffer desciptors
4259 * used by the DE so that they can be loaded into cache in parallel
4260 * while the DE is processing state update packets.
4261 *
4262 * Compute
4263 * The compute CP consists of two microengines (ME):
4264 * MEC1 - Compute MicroEngine 1
4265 * MEC2 - Compute MicroEngine 2
4266 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
4267 * The queues are exposed to userspace and are programmed directly
4268 * by the compute runtime.
4269 */
4270/**
4271 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
4272 *
4273 * @rdev: radeon_device pointer
4274 * @enable: enable or disable the MEs
4275 *
4276 * Halts or unhalts the gfx MEs.
4277 */
4278static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
4279{
4280 if (enable)
4281 WREG32(CP_ME_CNTL, 0);
4282 else {
50efa51a
AD		 4283 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4284 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
841cf442
AD		 4285 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
4286 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4287 }
4288 udelay(50);
4289}
4290
4291/**
4292 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
4293 *
4294 * @rdev: radeon_device pointer
4295 *
4296 * Loads the gfx PFP, ME, and CE ucode.
4297 * Returns 0 for success, -EINVAL if the ucode is not available.
4298 */
4299static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
4300{
841cf442
AD		 4301 int i;
4302
4303 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
4304 return -EINVAL;
4305
4306 cik_cp_gfx_enable(rdev, false);
4307
f2c6b0f4
AD		 4308 if (rdev->new_fw) {
4309 const struct gfx_firmware_header_v1_0 *pfp_hdr =
4310 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
4311 const struct gfx_firmware_header_v1_0 *ce_hdr =
4312 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
4313 const struct gfx_firmware_header_v1_0 *me_hdr =
4314 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
4315 const __le32 *fw_data;
4316 u32 fw_size;
4317
4318 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
4319 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
4320 radeon_ucode_print_gfx_hdr(&me_hdr->header);
4321
4322 /* PFP */
4323 fw_data = (const __le32 *)
4324 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
4325 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
4326 WREG32(CP_PFP_UCODE_ADDR, 0);
4327 for (i = 0; i < fw_size; i++)
4328 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4329 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version));
f2c6b0f4
AD		 4330
4331 /* CE */
4332 fw_data = (const __le32 *)
4333 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
4334 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
4335 WREG32(CP_CE_UCODE_ADDR, 0);
4336 for (i = 0; i < fw_size; i++)
4337 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4338 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version));
f2c6b0f4
AD		 4339
4340 /* ME */
4341 fw_data = (const __be32 *)
4342 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
4343 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
4344 WREG32(CP_ME_RAM_WADDR, 0);
4345 for (i = 0; i < fw_size; i++)
4346 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
38aea071
AD		 4347 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version));
4348 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version));
f2c6b0f4
AD		 4349 } else {
4350 const __be32 *fw_data;
4351
4352 /* PFP */
4353 fw_data = (const __be32 *)rdev->pfp_fw->data;
4354 WREG32(CP_PFP_UCODE_ADDR, 0);
4355 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
4356 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
4357 WREG32(CP_PFP_UCODE_ADDR, 0);
4358
4359 /* CE */
4360 fw_data = (const __be32 *)rdev->ce_fw->data;
4361 WREG32(CP_CE_UCODE_ADDR, 0);
4362 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
4363 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
4364 WREG32(CP_CE_UCODE_ADDR, 0);
4365
4366 /* ME */
4367 fw_data = (const __be32 *)rdev->me_fw->data;
4368 WREG32(CP_ME_RAM_WADDR, 0);
4369 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
4370 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
4371 WREG32(CP_ME_RAM_WADDR, 0);
4372 }
841cf442 4373
841cf442
AD		 4374 return 0;
4375}
4376
4377/**
4378 * cik_cp_gfx_start - start the gfx ring
4379 *
4380 * @rdev: radeon_device pointer
4381 *
4382 * Enables the ring and loads the clear state context and other
4383 * packets required to init the ring.
4384 * Returns 0 for success, error for failure.
4385 */
4386static int cik_cp_gfx_start(struct radeon_device *rdev)
4387{
4388 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4389 int r, i;
4390
4391 /* init the CP */
4392 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4393 WREG32(CP_ENDIAN_SWAP, 0);
4394 WREG32(CP_DEVICE_ID, 1);
4395
4396 cik_cp_gfx_enable(rdev, true);
4397
4398 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4399 if (r) {
4400 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4401 return r;
4402 }
4403
4404 /* init the CE partitions. CE only used for gfx on CIK */
4405 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4406 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
dc4edad6
JZ		 4407 radeon_ring_write(ring, 0x8000);
4408 radeon_ring_write(ring, 0x8000);
841cf442
AD		 4409
4410 /* setup clear context state */
4411 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4412 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4413
4414 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4415 radeon_ring_write(ring, 0x80000000);
4416 radeon_ring_write(ring, 0x80000000);
4417
4418 for (i = 0; i < cik_default_size; i++)
4419 radeon_ring_write(ring, cik_default_state[i]);
4420
4421 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4422 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4423
4424 /* set clear context state */
4425 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4426 radeon_ring_write(ring, 0);
4427
4428 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4429 radeon_ring_write(ring, 0x00000316);
4430 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4431 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4432
1538a9e0 4433 radeon_ring_unlock_commit(rdev, ring, false);
841cf442
AD		 4434
4435 return 0;
4436}
4437
4438/**
4439 * cik_cp_gfx_fini - stop the gfx ring
4440 *
4441 * @rdev: radeon_device pointer
4442 *
4443 * Stop the gfx ring and tear down the driver ring
4444 * info.
4445 */
4446static void cik_cp_gfx_fini(struct radeon_device *rdev)
4447{
4448 cik_cp_gfx_enable(rdev, false);
4449 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4450}
4451
4452/**
4453 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4454 *
4455 * @rdev: radeon_device pointer
4456 *
4457 * Program the location and size of the gfx ring buffer
4458 * and test it to make sure it's working.
4459 * Returns 0 for success, error for failure.
4460 */
4461static int cik_cp_gfx_resume(struct radeon_device *rdev)
4462{
4463 struct radeon_ring *ring;
4464 u32 tmp;
4465 u32 rb_bufsz;
4466 u64 rb_addr;
4467 int r;
4468
4469 WREG32(CP_SEM_WAIT_TIMER, 0x0);
939c0d3c
AD		 4470 if (rdev->family != CHIP_HAWAII)
4471 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
841cf442
AD		 4472
4473 /* Set the write pointer delay */
4474 WREG32(CP_RB_WPTR_DELAY, 0);
4475
4476 /* set the RB to use vmid 0 */
4477 WREG32(CP_RB_VMID, 0);
4478
4479 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4480
4481 /* ring 0 - compute and gfx */
4482 /* Set ring buffer size */
4483 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 4484 rb_bufsz = order_base_2(ring->ring_size / 8);
4485 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
841cf442
AD		 4486#ifdef __BIG_ENDIAN
4487 tmp |= BUF_SWAP_32BIT;
4488#endif
4489 WREG32(CP_RB0_CNTL, tmp);
4490
4491 /* Initialize the ring buffer's read and write pointers */
4492 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4493 ring->wptr = 0;
4494 WREG32(CP_RB0_WPTR, ring->wptr);
4495
4496 /* set the wb address wether it's enabled or not */
4497 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4498 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4499
4500 /* scratch register shadowing is no longer supported */
4501 WREG32(SCRATCH_UMSK, 0);
4502
4503 if (!rdev->wb.enabled)
4504 tmp |= RB_NO_UPDATE;
4505
4506 mdelay(1);
4507 WREG32(CP_RB0_CNTL, tmp);
4508
4509 rb_addr = ring->gpu_addr >> 8;
4510 WREG32(CP_RB0_BASE, rb_addr);
4511 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4512
841cf442
AD		 4513 /* start the ring */
4514 cik_cp_gfx_start(rdev);
4515 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4516 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4517 if (r) {
4518 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4519 return r;
4520 }
50efa51a
AD		 4521
4522 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4523 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4524
841cf442
AD		 4525 return 0;
4526}
4527
ea31bf69
AD		 4528u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4529 struct radeon_ring *ring)
963e81f9
AD		 4530{
4531 u32 rptr;
4532
ea31bf69
AD		 4533 if (rdev->wb.enabled)
4534 rptr = rdev->wb.wb[ring->rptr_offs/4];
4535 else
4536 rptr = RREG32(CP_RB0_RPTR);
4537
4538 return rptr;
4539}
4540
4541u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4542 struct radeon_ring *ring)
4543{
4544 u32 wptr;
4545
4546 wptr = RREG32(CP_RB0_WPTR);
963e81f9 4547
ea31bf69
AD		 4548 return wptr;
4549}
4550
4551void cik_gfx_set_wptr(struct radeon_device *rdev,
4552 struct radeon_ring *ring)
4553{
4554 WREG32(CP_RB0_WPTR, ring->wptr);
4555 (void)RREG32(CP_RB0_WPTR);
4556}
4557
4558u32 cik_compute_get_rptr(struct radeon_device *rdev,
4559 struct radeon_ring *ring)
4560{
4561 u32 rptr;
963e81f9
AD		 4562
4563 if (rdev->wb.enabled) {
ea31bf69 4564 rptr = rdev->wb.wb[ring->rptr_offs/4];
963e81f9 4565 } else {
f61d5b46 4566 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4567 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4568 rptr = RREG32(CP_HQD_PQ_RPTR);
4569 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4570 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4571 }
963e81f9
AD		 4572
4573 return rptr;
4574}
4575
ea31bf69
AD		 4576u32 cik_compute_get_wptr(struct radeon_device *rdev,
4577 struct radeon_ring *ring)
963e81f9
AD		 4578{
4579 u32 wptr;
4580
4581 if (rdev->wb.enabled) {
ea31bf69
AD		 4582 /* XXX check if swapping is necessary on BE */
4583 wptr = rdev->wb.wb[ring->wptr_offs/4];
963e81f9 4584 } else {
f61d5b46 4585 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4586 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4587 wptr = RREG32(CP_HQD_PQ_WPTR);
4588 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4589 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4590 }
963e81f9
AD		 4591
4592 return wptr;
4593}
4594
ea31bf69
AD		 4595void cik_compute_set_wptr(struct radeon_device *rdev,
4596 struct radeon_ring *ring)
963e81f9 4597{
ea31bf69
AD		 4598 /* XXX check if swapping is necessary on BE */
4599 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
d5754ab8 4600 WDOORBELL32(ring->doorbell_index, ring->wptr);
963e81f9
AD		 4601}
4602
161569de
JG		 4603static void cik_compute_stop(struct radeon_device *rdev,
4604 struct radeon_ring *ring)
4605{
4606 u32 j, tmp;
4607
4608 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4609 /* Disable wptr polling. */
4610 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4611 tmp &= ~WPTR_POLL_EN;
4612 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4613 /* Disable HQD. */
4614 if (RREG32(CP_HQD_ACTIVE) & 1) {
4615 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4616 for (j = 0; j < rdev->usec_timeout; j++) {
4617 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4618 break;
4619 udelay(1);
4620 }
4621 WREG32(CP_HQD_DEQUEUE_REQUEST, 0);
4622 WREG32(CP_HQD_PQ_RPTR, 0);
4623 WREG32(CP_HQD_PQ_WPTR, 0);
4624 }
4625 cik_srbm_select(rdev, 0, 0, 0, 0);
4626}
4627
841cf442
AD		 4628/**
4629 * cik_cp_compute_enable - enable/disable the compute CP MEs
4630 *
4631 * @rdev: radeon_device pointer
4632 * @enable: enable or disable the MEs
4633 *
4634 * Halts or unhalts the compute MEs.
4635 */
4636static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4637{
4638 if (enable)
4639 WREG32(CP_MEC_CNTL, 0);
b2b3d8d9 4640 else {
161569de
JG		 4641 /*
4642 * To make hibernation reliable we need to clear compute ring
4643 * configuration before halting the compute ring.
4644 */
4645 mutex_lock(&rdev->srbm_mutex);
4646 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
4647 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
4648 mutex_unlock(&rdev->srbm_mutex);
4649
841cf442 4650 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
b2b3d8d9
AD		 4651 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4652 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4653 }
841cf442
AD		 4654 udelay(50);
4655}
4656
4657/**
4658 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4659 *
4660 * @rdev: radeon_device pointer
4661 *
4662 * Loads the compute MEC1&2 ucode.
4663 * Returns 0 for success, -EINVAL if the ucode is not available.
4664 */
4665static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4666{
841cf442
AD		 4667 int i;
4668
4669 if (!rdev->mec_fw)
4670 return -EINVAL;
4671
4672 cik_cp_compute_enable(rdev, false);
4673
f2c6b0f4
AD		 4674 if (rdev->new_fw) {
4675 const struct gfx_firmware_header_v1_0 *mec_hdr =
4676 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4677 const __le32 *fw_data;
4678 u32 fw_size;
4679
4680 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4681
4682 /* MEC1 */
4683 fw_data = (const __le32 *)
4684 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4685 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4686 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4687 for (i = 0; i < fw_size; i++)
4688 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4689 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version));
841cf442 4690
841cf442 4691 /* MEC2 */
f2c6b0f4
AD		 4692 if (rdev->family == CHIP_KAVERI) {
4693 const struct gfx_firmware_header_v1_0 *mec2_hdr =
4694 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4695
4696 fw_data = (const __le32 *)
4697 (rdev->mec2_fw->data +
4698 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4699 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4700 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4701 for (i = 0; i < fw_size; i++)
4702 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4703 WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version));
f2c6b0f4
AD		 4704 }
4705 } else {
4706 const __be32 *fw_data;
4707
4708 /* MEC1 */
841cf442 4709 fw_data = (const __be32 *)rdev->mec_fw->data;
f2c6b0f4 4710 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
841cf442 4711 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
f2c6b0f4
AD		 4712 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4713 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4714
4715 if (rdev->family == CHIP_KAVERI) {
4716 /* MEC2 */
4717 fw_data = (const __be32 *)rdev->mec_fw->data;
4718 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4719 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4720 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4721 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4722 }
841cf442
AD		 4723 }
4724
4725 return 0;
4726}
4727
4728/**
4729 * cik_cp_compute_start - start the compute queues
4730 *
4731 * @rdev: radeon_device pointer
4732 *
4733 * Enable the compute queues.
4734 * Returns 0 for success, error for failure.
4735 */
4736static int cik_cp_compute_start(struct radeon_device *rdev)
4737{
963e81f9
AD		 4738 cik_cp_compute_enable(rdev, true);
4739
841cf442
AD		 4740 return 0;
4741}
4742
4743/**
4744 * cik_cp_compute_fini - stop the compute queues
4745 *
4746 * @rdev: radeon_device pointer
4747 *
4748 * Stop the compute queues and tear down the driver queue
4749 * info.
4750 */
4751static void cik_cp_compute_fini(struct radeon_device *rdev)
4752{
963e81f9
AD		 4753 int i, idx, r;
4754
841cf442 4755 cik_cp_compute_enable(rdev, false);
963e81f9
AD		 4756
4757 for (i = 0; i < 2; i++) {
4758 if (i == 0)
4759 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4760 else
4761 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4762
4763 if (rdev->ring[idx].mqd_obj) {
4764 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4765 if (unlikely(r != 0))
4766 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4767
4768 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4769 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4770
4771 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4772 rdev->ring[idx].mqd_obj = NULL;
4773 }
4774 }
841cf442
AD		 4775}
4776
963e81f9
AD		 4777static void cik_mec_fini(struct radeon_device *rdev)
4778{
4779 int r;
4780
4781 if (rdev->mec.hpd_eop_obj) {
4782 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4783 if (unlikely(r != 0))
4784 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4785 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4786 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4787
4788 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4789 rdev->mec.hpd_eop_obj = NULL;
4790 }
4791}
4792
4793#define MEC_HPD_SIZE 2048
4794
4795static int cik_mec_init(struct radeon_device *rdev)
4796{
4797 int r;
4798 u32 *hpd;
4799
4800 /*
4801 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4802 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
62a7b7fb
OG		 4803 * Nonetheless, we assign only 1 pipe because all other pipes will
4804 * be handled by KFD
963e81f9 4805 */
62a7b7fb
OG		 4806 rdev->mec.num_mec = 1;
4807 rdev->mec.num_pipe = 1;
963e81f9
AD		 4808 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4809
4810 if (rdev->mec.hpd_eop_obj == NULL) {
4811 r = radeon_bo_create(rdev,
4812 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4813 PAGE_SIZE, true,
831b6966 4814 RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL,
963e81f9
AD		 4815 &rdev->mec.hpd_eop_obj);
4816 if (r) {
4817 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4818 return r;
4819 }
4820 }
4821
4822 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4823 if (unlikely(r != 0)) {
4824 cik_mec_fini(rdev);
4825 return r;
4826 }
4827 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4828 &rdev->mec.hpd_eop_gpu_addr);
4829 if (r) {
4830 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4831 cik_mec_fini(rdev);
4832 return r;
4833 }
4834 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4835 if (r) {
4836 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4837 cik_mec_fini(rdev);
4838 return r;
4839 }
4840
4841 /* clear memory. Not sure if this is required or not */
4842 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4843
4844 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4845 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4846
4847 return 0;
4848}
4849
4850struct hqd_registers
4851{
4852 u32 cp_mqd_base_addr;
4853 u32 cp_mqd_base_addr_hi;
4854 u32 cp_hqd_active;
4855 u32 cp_hqd_vmid;
4856 u32 cp_hqd_persistent_state;
4857 u32 cp_hqd_pipe_priority;
4858 u32 cp_hqd_queue_priority;
4859 u32 cp_hqd_quantum;
4860 u32 cp_hqd_pq_base;
4861 u32 cp_hqd_pq_base_hi;
4862 u32 cp_hqd_pq_rptr;
4863 u32 cp_hqd_pq_rptr_report_addr;
4864 u32 cp_hqd_pq_rptr_report_addr_hi;
4865 u32 cp_hqd_pq_wptr_poll_addr;
4866 u32 cp_hqd_pq_wptr_poll_addr_hi;
4867 u32 cp_hqd_pq_doorbell_control;
4868 u32 cp_hqd_pq_wptr;
4869 u32 cp_hqd_pq_control;
4870 u32 cp_hqd_ib_base_addr;
4871 u32 cp_hqd_ib_base_addr_hi;
4872 u32 cp_hqd_ib_rptr;
4873 u32 cp_hqd_ib_control;
4874 u32 cp_hqd_iq_timer;
4875 u32 cp_hqd_iq_rptr;
4876 u32 cp_hqd_dequeue_request;
4877 u32 cp_hqd_dma_offload;
4878 u32 cp_hqd_sema_cmd;
4879 u32 cp_hqd_msg_type;
4880 u32 cp_hqd_atomic0_preop_lo;
4881 u32 cp_hqd_atomic0_preop_hi;
4882 u32 cp_hqd_atomic1_preop_lo;
4883 u32 cp_hqd_atomic1_preop_hi;
4884 u32 cp_hqd_hq_scheduler0;
4885 u32 cp_hqd_hq_scheduler1;
4886 u32 cp_mqd_control;
4887};
4888
4889struct bonaire_mqd
4890{
4891 u32 header;
4892 u32 dispatch_initiator;
4893 u32 dimensions[3];
4894 u32 start_idx[3];
4895 u32 num_threads[3];
4896 u32 pipeline_stat_enable;
4897 u32 perf_counter_enable;
4898 u32 pgm[2];
4899 u32 tba[2];
4900 u32 tma[2];
4901 u32 pgm_rsrc[2];
4902 u32 vmid;
4903 u32 resource_limits;
4904 u32 static_thread_mgmt01[2];
4905 u32 tmp_ring_size;
4906 u32 static_thread_mgmt23[2];
4907 u32 restart[3];
4908 u32 thread_trace_enable;
4909 u32 reserved1;
4910 u32 user_data[16];
4911 u32 vgtcs_invoke_count[2];
4912 struct hqd_registers queue_state;
4913 u32 dequeue_cntr;
4914 u32 interrupt_queue[64];
4915};
4916
841cf442
AD		 4917/**
4918 * cik_cp_compute_resume - setup the compute queue registers
4919 *
4920 * @rdev: radeon_device pointer
4921 *
4922 * Program the compute queues and test them to make sure they
4923 * are working.
4924 * Returns 0 for success, error for failure.
4925 */
4926static int cik_cp_compute_resume(struct radeon_device *rdev)
4927{
370ce45b 4928 int r, i, j, idx;
963e81f9
AD		 4929 u32 tmp;
4930 bool use_doorbell = true;
4931 u64 hqd_gpu_addr;
4932 u64 mqd_gpu_addr;
4933 u64 eop_gpu_addr;
4934 u64 wb_gpu_addr;
4935 u32 *buf;
4936 struct bonaire_mqd *mqd;
841cf442 4937
841cf442
AD		 4938 r = cik_cp_compute_start(rdev);
4939 if (r)
4940 return r;
963e81f9
AD		 4941
4942 /* fix up chicken bits */
4943 tmp = RREG32(CP_CPF_DEBUG);
4944 tmp |= (1 << 23);
4945 WREG32(CP_CPF_DEBUG, tmp);
4946
4947 /* init the pipes */
f61d5b46 4948 mutex_lock(&rdev->srbm_mutex);
963e81f9 4949
62a7b7fb 4950 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr;
963e81f9 4951
62a7b7fb 4952 cik_srbm_select(rdev, 0, 0, 0, 0);
963e81f9 4953
62a7b7fb
OG		 4954 /* write the EOP addr */
4955 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4956 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
963e81f9 4957
62a7b7fb
OG		 4958 /* set the VMID assigned */
4959 WREG32(CP_HPD_EOP_VMID, 0);
4960
4961 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4962 tmp = RREG32(CP_HPD_EOP_CONTROL);
4963 tmp &= ~EOP_SIZE_MASK;
4964 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4965 WREG32(CP_HPD_EOP_CONTROL, tmp);
963e81f9 4966
f61d5b46 4967 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4968
4969 /* init the queues. Just two for now. */
4970 for (i = 0; i < 2; i++) {
4971 if (i == 0)
4972 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4973 else
4974 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4975
4976 if (rdev->ring[idx].mqd_obj == NULL) {
4977 r = radeon_bo_create(rdev,
4978 sizeof(struct bonaire_mqd),
4979 PAGE_SIZE, true,
02376d82 4980 RADEON_GEM_DOMAIN_GTT, 0, NULL,
831b6966 4981 NULL, &rdev->ring[idx].mqd_obj);
963e81f9
AD		 4982 if (r) {
4983 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4984 return r;
4985 }
4986 }
4987
4988 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4989 if (unlikely(r != 0)) {
4990 cik_cp_compute_fini(rdev);
4991 return r;
4992 }
4993 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4994 &mqd_gpu_addr);
4995 if (r) {
4996 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4997 cik_cp_compute_fini(rdev);
4998 return r;
4999 }
5000 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
5001 if (r) {
5002 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
5003 cik_cp_compute_fini(rdev);
5004 return r;
5005 }
5006
963e81f9
AD		 5007 /* init the mqd struct */
5008 memset(buf, 0, sizeof(struct bonaire_mqd));
5009
5010 mqd = (struct bonaire_mqd *)buf;
5011 mqd->header = 0xC0310800;
5012 mqd->static_thread_mgmt01[0] = 0xffffffff;
5013 mqd->static_thread_mgmt01[1] = 0xffffffff;
5014 mqd->static_thread_mgmt23[0] = 0xffffffff;
5015 mqd->static_thread_mgmt23[1] = 0xffffffff;
5016
f61d5b46 5017 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 5018 cik_srbm_select(rdev, rdev->ring[idx].me,
5019 rdev->ring[idx].pipe,
5020 rdev->ring[idx].queue, 0);
5021
5022 /* disable wptr polling */
5023 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
5024 tmp &= ~WPTR_POLL_EN;
5025 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
5026
5027 /* enable doorbell? */
5028 mqd->queue_state.cp_hqd_pq_doorbell_control =
5029 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
5030 if (use_doorbell)
5031 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
5032 else
5033 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
5034 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
5035 mqd->queue_state.cp_hqd_pq_doorbell_control);
5036
5037 /* disable the queue if it's active */
5038 mqd->queue_state.cp_hqd_dequeue_request = 0;
5039 mqd->queue_state.cp_hqd_pq_rptr = 0;
5040 mqd->queue_state.cp_hqd_pq_wptr= 0;
5041 if (RREG32(CP_HQD_ACTIVE) & 1) {
5042 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
370ce45b 5043 for (j = 0; j < rdev->usec_timeout; j++) {
963e81f9
AD		 5044 if (!(RREG32(CP_HQD_ACTIVE) & 1))
5045 break;
5046 udelay(1);
5047 }
5048 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
5049 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
5050 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
5051 }
5052
5053 /* set the pointer to the MQD */
5054 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
5055 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
5056 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
5057 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
5058 /* set MQD vmid to 0 */
5059 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
5060 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
5061 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
5062
5063 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
5064 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
5065 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
5066 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
5067 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
5068 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
5069
5070 /* set up the HQD, this is similar to CP_RB0_CNTL */
5071 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
5072 mqd->queue_state.cp_hqd_pq_control &=
5073 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
5074
5075 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 5076 order_base_2(rdev->ring[idx].ring_size / 8);
963e81f9 5077 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 5078 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
963e81f9
AD		 5079#ifdef __BIG_ENDIAN
5080 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
5081#endif
5082 mqd->queue_state.cp_hqd_pq_control &=
5083 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
5084 mqd->queue_state.cp_hqd_pq_control |=
5085 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
5086 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
5087
5088 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
5089 if (i == 0)
5090 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
5091 else
5092 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
5093 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
5094 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
5095 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
5096 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
5097 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
5098
5099 /* set the wb address wether it's enabled or not */
5100 if (i == 0)
5101 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
5102 else
5103 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
5104 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
5105 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
5106 upper_32_bits(wb_gpu_addr) & 0xffff;
5107 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
5108 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
5109 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
5110 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
5111
5112 /* enable the doorbell if requested */
5113 if (use_doorbell) {
5114 mqd->queue_state.cp_hqd_pq_doorbell_control =
5115 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
5116 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
5117 mqd->queue_state.cp_hqd_pq_doorbell_control |=
d5754ab8 5118 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
963e81f9
AD		 5119 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
5120 mqd->queue_state.cp_hqd_pq_doorbell_control &=
5121 ~(DOORBELL_SOURCE | DOORBELL_HIT);
5122
5123 } else {
5124 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
5125 }
5126 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
5127 mqd->queue_state.cp_hqd_pq_doorbell_control);
5128
5129 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
5130 rdev->ring[idx].wptr = 0;
5131 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
5132 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
ff212f25 5133 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
963e81f9
AD		 5134
5135 /* set the vmid for the queue */
5136 mqd->queue_state.cp_hqd_vmid = 0;
5137 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
5138
5139 /* activate the queue */
5140 mqd->queue_state.cp_hqd_active = 1;
5141 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
5142
5143 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 5144 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 5145
5146 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
5147 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
5148
5149 rdev->ring[idx].ready = true;
5150 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
5151 if (r)
5152 rdev->ring[idx].ready = false;
5153 }
5154
841cf442
AD		 5155 return 0;
5156}
5157
841cf442
AD		 5158static void cik_cp_enable(struct radeon_device *rdev, bool enable)
5159{
5160 cik_cp_gfx_enable(rdev, enable);
5161 cik_cp_compute_enable(rdev, enable);
5162}
5163
841cf442
AD		 5164static int cik_cp_load_microcode(struct radeon_device *rdev)
5165{
5166 int r;
5167
5168 r = cik_cp_gfx_load_microcode(rdev);
5169 if (r)
5170 return r;
5171 r = cik_cp_compute_load_microcode(rdev);
5172 if (r)
5173 return r;
5174
5175 return 0;
5176}
5177
841cf442
AD		 5178static void cik_cp_fini(struct radeon_device *rdev)
5179{
5180 cik_cp_gfx_fini(rdev);
5181 cik_cp_compute_fini(rdev);
5182}
5183
841cf442
AD		 5184static int cik_cp_resume(struct radeon_device *rdev)
5185{
5186 int r;
5187
4214faf6
AD		 5188 cik_enable_gui_idle_interrupt(rdev, false);
5189
841cf442
AD		 5190 r = cik_cp_load_microcode(rdev);
5191 if (r)
5192 return r;
5193
5194 r = cik_cp_gfx_resume(rdev);
5195 if (r)
5196 return r;
5197 r = cik_cp_compute_resume(rdev);
5198 if (r)
5199 return r;
5200
4214faf6
AD		 5201 cik_enable_gui_idle_interrupt(rdev, true);
5202
841cf442
AD		 5203 return 0;
5204}
5205
cc066715 5206static void cik_print_gpu_status_regs(struct radeon_device *rdev)
6f2043ce 5207{
6f2043ce
AD		 5208 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
5209 RREG32(GRBM_STATUS));
5210 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
5211 RREG32(GRBM_STATUS2));
5212 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
5213 RREG32(GRBM_STATUS_SE0));
5214 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
5215 RREG32(GRBM_STATUS_SE1));
5216 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
5217 RREG32(GRBM_STATUS_SE2));
5218 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
5219 RREG32(GRBM_STATUS_SE3));
5220 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
5221 RREG32(SRBM_STATUS));
5222 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
5223 RREG32(SRBM_STATUS2));
cc066715
AD		 5224 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
5225 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
5226 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
5227 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
963e81f9
AD		 5228 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
5229 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
5230 RREG32(CP_STALLED_STAT1));
5231 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
5232 RREG32(CP_STALLED_STAT2));
5233 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
5234 RREG32(CP_STALLED_STAT3));
5235 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
5236 RREG32(CP_CPF_BUSY_STAT));
5237 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
5238 RREG32(CP_CPF_STALLED_STAT1));
5239 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
5240 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
5241 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
5242 RREG32(CP_CPC_STALLED_STAT1));
5243 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
cc066715 5244}
6f2043ce 5245
21a93e13 5246/**
cc066715 5247 * cik_gpu_check_soft_reset - check which blocks are busy
21a93e13
AD		 5248 *
5249 * @rdev: radeon_device pointer
21a93e13 5250 *
cc066715
AD		 5251 * Check which blocks are busy and return the relevant reset
5252 * mask to be used by cik_gpu_soft_reset().
5253 * Returns a mask of the blocks to be reset.
21a93e13 5254 */
2483b4ea 5255u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
21a93e13 5256{
cc066715
AD		 5257 u32 reset_mask = 0;
5258 u32 tmp;
21a93e13 5259
cc066715
AD		 5260 /* GRBM_STATUS */
5261 tmp = RREG32(GRBM_STATUS);
5262 if (tmp & (PA_BUSY | SC_BUSY |
5263 BCI_BUSY | SX_BUSY |
5264 TA_BUSY | VGT_BUSY |
5265 DB_BUSY | CB_BUSY |
5266 GDS_BUSY | SPI_BUSY |
5267 IA_BUSY | IA_BUSY_NO_DMA))
5268 reset_mask |= RADEON_RESET_GFX;
21a93e13 5269
cc066715
AD		 5270 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
5271 reset_mask |= RADEON_RESET_CP;
21a93e13 5272
cc066715
AD		 5273 /* GRBM_STATUS2 */
5274 tmp = RREG32(GRBM_STATUS2);
5275 if (tmp & RLC_BUSY)
5276 reset_mask |= RADEON_RESET_RLC;
21a93e13 5277
cc066715
AD		 5278 /* SDMA0_STATUS_REG */
5279 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
5280 if (!(tmp & SDMA_IDLE))
5281 reset_mask |= RADEON_RESET_DMA;
21a93e13 5282
cc066715
AD		 5283 /* SDMA1_STATUS_REG */
5284 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
5285 if (!(tmp & SDMA_IDLE))
5286 reset_mask |= RADEON_RESET_DMA1;
21a93e13 5287
cc066715
AD		 5288 /* SRBM_STATUS2 */
5289 tmp = RREG32(SRBM_STATUS2);
5290 if (tmp & SDMA_BUSY)
5291 reset_mask |= RADEON_RESET_DMA;
21a93e13 5292
cc066715
AD		 5293 if (tmp & SDMA1_BUSY)
5294 reset_mask |= RADEON_RESET_DMA1;
21a93e13 5295
cc066715
AD		 5296 /* SRBM_STATUS */
5297 tmp = RREG32(SRBM_STATUS);
21a93e13 5298
cc066715
AD		 5299 if (tmp & IH_BUSY)
5300 reset_mask |= RADEON_RESET_IH;
21a93e13 5301
cc066715
AD		 5302 if (tmp & SEM_BUSY)
5303 reset_mask |= RADEON_RESET_SEM;
21a93e13 5304
cc066715
AD		 5305 if (tmp & GRBM_RQ_PENDING)
5306 reset_mask |= RADEON_RESET_GRBM;
21a93e13 5307
cc066715
AD		 5308 if (tmp & VMC_BUSY)
5309 reset_mask |= RADEON_RESET_VMC;
21a93e13 5310
cc066715
AD		 5311 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
5312 MCC_BUSY | MCD_BUSY))
5313 reset_mask |= RADEON_RESET_MC;
21a93e13 5314
cc066715
AD		 5315 if (evergreen_is_display_hung(rdev))
5316 reset_mask |= RADEON_RESET_DISPLAY;
5317
5318 /* Skip MC reset as it's mostly likely not hung, just busy */
5319 if (reset_mask & RADEON_RESET_MC) {
5320 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
5321 reset_mask &= ~RADEON_RESET_MC;
21a93e13 5322 }
cc066715
AD		 5323
5324 return reset_mask;
21a93e13
AD		 5325}
5326
5327/**
cc066715 5328 * cik_gpu_soft_reset - soft reset GPU
21a93e13
AD		 5329 *
5330 * @rdev: radeon_device pointer
cc066715 5331 * @reset_mask: mask of which blocks to reset
21a93e13 5332 *
cc066715 5333 * Soft reset the blocks specified in @reset_mask.
21a93e13 5334 */
cc066715 5335static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
21a93e13 5336{
6f2043ce 5337 struct evergreen_mc_save save;
cc066715
AD		 5338 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5339 u32 tmp;
21a93e13 5340
cc066715
AD		 5341 if (reset_mask == 0)
5342 return;
21a93e13 5343
cc066715 5344 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
21a93e13 5345
cc066715
AD		 5346 cik_print_gpu_status_regs(rdev);
5347 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5348 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5349 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5350 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
21a93e13 5351
fb2c7f4d
AD		 5352 /* disable CG/PG */
5353 cik_fini_pg(rdev);
5354 cik_fini_cg(rdev);
5355
cc066715
AD		 5356 /* stop the rlc */
5357 cik_rlc_stop(rdev);
21a93e13 5358
cc066715
AD		 5359 /* Disable GFX parsing/prefetching */
5360 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
21a93e13 5361
cc066715
AD		 5362 /* Disable MEC parsing/prefetching */
5363 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
21a93e13 5364
cc066715
AD		 5365 if (reset_mask & RADEON_RESET_DMA) {
5366 /* sdma0 */
5367 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5368 tmp |= SDMA_HALT;
5369 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5370 }
5371 if (reset_mask & RADEON_RESET_DMA1) {
5372 /* sdma1 */
5373 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5374 tmp |= SDMA_HALT;
5375 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5376 }
21a93e13 5377
6f2043ce 5378 evergreen_mc_stop(rdev, &save);
cc066715 5379 if (evergreen_mc_wait_for_idle(rdev)) {
6f2043ce
AD		 5380 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5381 }
21a93e13 5382
cc066715
AD		 5383 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5384 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
21a93e13 5385
cc066715
AD		 5386 if (reset_mask & RADEON_RESET_CP) {
5387 grbm_soft_reset |= SOFT_RESET_CP;
21a93e13 5388
cc066715
AD		 5389 srbm_soft_reset |= SOFT_RESET_GRBM;
5390 }
21a93e13 5391
cc066715
AD		 5392 if (reset_mask & RADEON_RESET_DMA)
5393 srbm_soft_reset |= SOFT_RESET_SDMA;
21a93e13 5394
cc066715
AD		 5395 if (reset_mask & RADEON_RESET_DMA1)
5396 srbm_soft_reset |= SOFT_RESET_SDMA1;
5397
5398 if (reset_mask & RADEON_RESET_DISPLAY)
5399 srbm_soft_reset |= SOFT_RESET_DC;
5400
5401 if (reset_mask & RADEON_RESET_RLC)
5402 grbm_soft_reset |= SOFT_RESET_RLC;
5403
5404 if (reset_mask & RADEON_RESET_SEM)
5405 srbm_soft_reset |= SOFT_RESET_SEM;
5406
5407 if (reset_mask & RADEON_RESET_IH)
5408 srbm_soft_reset |= SOFT_RESET_IH;
5409
5410 if (reset_mask & RADEON_RESET_GRBM)
5411 srbm_soft_reset |= SOFT_RESET_GRBM;
5412
5413 if (reset_mask & RADEON_RESET_VMC)
5414 srbm_soft_reset |= SOFT_RESET_VMC;
5415
5416 if (!(rdev->flags & RADEON_IS_IGP)) {
5417 if (reset_mask & RADEON_RESET_MC)
5418 srbm_soft_reset |= SOFT_RESET_MC;
21a93e13
AD		 5419 }
5420
cc066715
AD		 5421 if (grbm_soft_reset) {
5422 tmp = RREG32(GRBM_SOFT_RESET);
5423 tmp |= grbm_soft_reset;
5424 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5425 WREG32(GRBM_SOFT_RESET, tmp);
5426 tmp = RREG32(GRBM_SOFT_RESET);
21a93e13 5427
cc066715 5428 udelay(50);
21a93e13 5429
cc066715
AD		 5430 tmp &= ~grbm_soft_reset;
5431 WREG32(GRBM_SOFT_RESET, tmp);
5432 tmp = RREG32(GRBM_SOFT_RESET);
5433 }
21a93e13 5434
cc066715
AD		 5435 if (srbm_soft_reset) {
5436 tmp = RREG32(SRBM_SOFT_RESET);
5437 tmp |= srbm_soft_reset;
5438 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5439 WREG32(SRBM_SOFT_RESET, tmp);
5440 tmp = RREG32(SRBM_SOFT_RESET);
21a93e13 5441
cc066715 5442 udelay(50);
21a93e13 5443
cc066715
AD		 5444 tmp &= ~srbm_soft_reset;
5445 WREG32(SRBM_SOFT_RESET, tmp);
5446 tmp = RREG32(SRBM_SOFT_RESET);
5447 }
21a93e13 5448
6f2043ce
AD		 5449 /* Wait a little for things to settle down */
5450 udelay(50);
21a93e13 5451
6f2043ce 5452 evergreen_mc_resume(rdev, &save);
cc066715
AD		 5453 udelay(50);
5454
5455 cik_print_gpu_status_regs(rdev);
21a93e13
AD		 5456}
5457
0279ed19
AD		 5458struct kv_reset_save_regs {
5459 u32 gmcon_reng_execute;
5460 u32 gmcon_misc;
5461 u32 gmcon_misc3;
5462};
5463
5464static void kv_save_regs_for_reset(struct radeon_device *rdev,
5465 struct kv_reset_save_regs *save)
5466{
5467 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5468 save->gmcon_misc = RREG32(GMCON_MISC);
5469 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5470
5471 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5472 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5473 STCTRL_STUTTER_EN));
5474}
5475
5476static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5477 struct kv_reset_save_regs *save)
5478{
5479 int i;
5480
5481 WREG32(GMCON_PGFSM_WRITE, 0);
5482 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5483
5484 for (i = 0; i < 5; i++)
5485 WREG32(GMCON_PGFSM_WRITE, 0);
5486
5487 WREG32(GMCON_PGFSM_WRITE, 0);
5488 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5489
5490 for (i = 0; i < 5; i++)
5491 WREG32(GMCON_PGFSM_WRITE, 0);
5492
5493 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5494 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5495
5496 for (i = 0; i < 5; i++)
5497 WREG32(GMCON_PGFSM_WRITE, 0);
5498
5499 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5500 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5501
5502 for (i = 0; i < 5; i++)
5503 WREG32(GMCON_PGFSM_WRITE, 0);
5504
5505 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5506 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5507
5508 for (i = 0; i < 5; i++)
5509 WREG32(GMCON_PGFSM_WRITE, 0);
5510
5511 WREG32(GMCON_PGFSM_WRITE, 0);
5512 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5513
5514 for (i = 0; i < 5; i++)
5515 WREG32(GMCON_PGFSM_WRITE, 0);
5516
5517 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5518 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5519
5520 for (i = 0; i < 5; i++)
5521 WREG32(GMCON_PGFSM_WRITE, 0);
5522
5523 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5524 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5525
5526 for (i = 0; i < 5; i++)
5527 WREG32(GMCON_PGFSM_WRITE, 0);
5528
5529 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5530 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5531
5532 for (i = 0; i < 5; i++)
5533 WREG32(GMCON_PGFSM_WRITE, 0);
5534
5535 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5536 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5537
5538 for (i = 0; i < 5; i++)
5539 WREG32(GMCON_PGFSM_WRITE, 0);
5540
5541 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5542 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5543
5544 WREG32(GMCON_MISC3, save->gmcon_misc3);
5545 WREG32(GMCON_MISC, save->gmcon_misc);
5546 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5547}
5548
5549static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5550{
5551 struct evergreen_mc_save save;
5552 struct kv_reset_save_regs kv_save = { 0 };
5553 u32 tmp, i;
5554
5555 dev_info(rdev->dev, "GPU pci config reset\n");
5556
5557 /* disable dpm? */
5558
5559 /* disable cg/pg */
5560 cik_fini_pg(rdev);
5561 cik_fini_cg(rdev);
5562
5563 /* Disable GFX parsing/prefetching */
5564 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5565
5566 /* Disable MEC parsing/prefetching */
5567 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5568
5569 /* sdma0 */
5570 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5571 tmp |= SDMA_HALT;
5572 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5573 /* sdma1 */
5574 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5575 tmp |= SDMA_HALT;
5576 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5577 /* XXX other engines? */
5578
5579 /* halt the rlc, disable cp internal ints */
5580 cik_rlc_stop(rdev);
5581
5582 udelay(50);
5583
5584 /* disable mem access */
5585 evergreen_mc_stop(rdev, &save);
5586 if (evergreen_mc_wait_for_idle(rdev)) {
5587 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5588 }
5589
5590 if (rdev->flags & RADEON_IS_IGP)
5591 kv_save_regs_for_reset(rdev, &kv_save);
5592
5593 /* disable BM */
5594 pci_clear_master(rdev->pdev);
5595 /* reset */
5596 radeon_pci_config_reset(rdev);
5597
5598 udelay(100);
5599
5600 /* wait for asic to come out of reset */
5601 for (i = 0; i < rdev->usec_timeout; i++) {
5602 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5603 break;
5604 udelay(1);
5605 }
5606
5607 /* does asic init need to be run first??? */
5608 if (rdev->flags & RADEON_IS_IGP)
5609 kv_restore_regs_for_reset(rdev, &kv_save);
5610}
5611
21a93e13 5612/**
cc066715 5613 * cik_asic_reset - soft reset GPU
21a93e13
AD		 5614 *
5615 * @rdev: radeon_device pointer
5616 *
cc066715
AD		 5617 * Look up which blocks are hung and attempt
5618 * to reset them.
6f2043ce 5619 * Returns 0 for success.
21a93e13 5620 */
6f2043ce 5621int cik_asic_reset(struct radeon_device *rdev)
21a93e13 5622{
cc066715 5623 u32 reset_mask;
21a93e13 5624
cc066715 5625 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5626
cc066715
AD		 5627 if (reset_mask)
5628 r600_set_bios_scratch_engine_hung(rdev, true);
21a93e13 5629
0279ed19 5630 /* try soft reset */
cc066715 5631 cik_gpu_soft_reset(rdev, reset_mask);
21a93e13 5632
cc066715
AD		 5633 reset_mask = cik_gpu_check_soft_reset(rdev);
5634
0279ed19
AD		 5635 /* try pci config reset */
5636 if (reset_mask && radeon_hard_reset)
5637 cik_gpu_pci_config_reset(rdev);
5638
5639 reset_mask = cik_gpu_check_soft_reset(rdev);
5640
cc066715
AD		 5641 if (!reset_mask)
5642 r600_set_bios_scratch_engine_hung(rdev, false);
21a93e13
AD		 5643
5644 return 0;
5645}
5646
5647/**
cc066715 5648 * cik_gfx_is_lockup - check if the 3D engine is locked up
21a93e13
AD		 5649 *
5650 * @rdev: radeon_device pointer
cc066715 5651 * @ring: radeon_ring structure holding ring information
21a93e13 5652 *
cc066715
AD		 5653 * Check if the 3D engine is locked up (CIK).
5654 * Returns true if the engine is locked, false if not.
21a93e13 5655 */
cc066715 5656bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
21a93e13 5657{
cc066715 5658 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5659
cc066715
AD		 5660 if (!(reset_mask & (RADEON_RESET_GFX |
5661 RADEON_RESET_COMPUTE |
5662 RADEON_RESET_CP))) {
ff212f25 5663 radeon_ring_lockup_update(rdev, ring);
cc066715 5664 return false;
21a93e13 5665 }
cc066715 5666 return radeon_ring_test_lockup(rdev, ring);
21a93e13
AD		 5667}
5668
1c49165d 5669/* MC */
21a93e13 5670/**
1c49165d 5671 * cik_mc_program - program the GPU memory controller
21a93e13
AD		 5672 *
5673 * @rdev: radeon_device pointer
21a93e13 5674 *
1c49165d
AD		 5675 * Set the location of vram, gart, and AGP in the GPU's
5676 * physical address space (CIK).
21a93e13 5677 */
1c49165d 5678static void cik_mc_program(struct radeon_device *rdev)
21a93e13 5679{
1c49165d 5680 struct evergreen_mc_save save;
21a93e13 5681 u32 tmp;
1c49165d 5682 int i, j;
21a93e13 5683
1c49165d
AD		 5684 /* Initialize HDP */
5685 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5686 WREG32((0x2c14 + j), 0x00000000);
5687 WREG32((0x2c18 + j), 0x00000000);
5688 WREG32((0x2c1c + j), 0x00000000);
5689 WREG32((0x2c20 + j), 0x00000000);
5690 WREG32((0x2c24 + j), 0x00000000);
21a93e13 5691 }
1c49165d 5692 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
21a93e13 5693
1c49165d
AD		 5694 evergreen_mc_stop(rdev, &save);
5695 if (radeon_mc_wait_for_idle(rdev)) {
5696 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5697 }
1c49165d
AD		 5698 /* Lockout access through VGA aperture*/
5699 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5700 /* Update configuration */
5701 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5702 rdev->mc.vram_start >> 12);
5703 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5704 rdev->mc.vram_end >> 12);
5705 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5706 rdev->vram_scratch.gpu_addr >> 12);
5707 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5708 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5709 WREG32(MC_VM_FB_LOCATION, tmp);
5710 /* XXX double check these! */
5711 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5712 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5713 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5714 WREG32(MC_VM_AGP_BASE, 0);
5715 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5716 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5717 if (radeon_mc_wait_for_idle(rdev)) {
5718 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5719 }
1c49165d
AD		 5720 evergreen_mc_resume(rdev, &save);
5721 /* we need to own VRAM, so turn off the VGA renderer here
5722 * to stop it overwriting our objects */
5723 rv515_vga_render_disable(rdev);
21a93e13
AD		 5724}
5725
5726/**
1c49165d 5727 * cik_mc_init - initialize the memory controller driver params
21a93e13
AD		 5728 *
5729 * @rdev: radeon_device pointer
21a93e13 5730 *
1c49165d
AD		 5731 * Look up the amount of vram, vram width, and decide how to place
5732 * vram and gart within the GPU's physical address space (CIK).
5733 * Returns 0 for success.
21a93e13 5734 */
1c49165d 5735static int cik_mc_init(struct radeon_device *rdev)
21a93e13 5736{
1c49165d
AD		 5737 u32 tmp;
5738 int chansize, numchan;
21a93e13 5739
1c49165d
AD		 5740 /* Get VRAM informations */
5741 rdev->mc.vram_is_ddr = true;
5742 tmp = RREG32(MC_ARB_RAMCFG);
5743 if (tmp & CHANSIZE_MASK) {
5744 chansize = 64;
21a93e13 5745 } else {
1c49165d 5746 chansize = 32;
21a93e13 5747 }
1c49165d
AD		 5748 tmp = RREG32(MC_SHARED_CHMAP);
5749 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5750 case 0:
5751 default:
5752 numchan = 1;
5753 break;
5754 case 1:
5755 numchan = 2;
5756 break;
5757 case 2:
5758 numchan = 4;
5759 break;
5760 case 3:
5761 numchan = 8;
5762 break;
5763 case 4:
5764 numchan = 3;
5765 break;
5766 case 5:
5767 numchan = 6;
5768 break;
5769 case 6:
5770 numchan = 10;
5771 break;
5772 case 7:
5773 numchan = 12;
5774 break;
5775 case 8:
5776 numchan = 16;
5777 break;
5778 }
5779 rdev->mc.vram_width = numchan * chansize;
5780 /* Could aper size report 0 ? */
5781 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5782 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5783 /* size in MB on si */
13c5bfda
AD		 5784 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5785 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
1c49165d
AD		 5786 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5787 si_vram_gtt_location(rdev, &rdev->mc);
5788 radeon_update_bandwidth_info(rdev);
5789
5790 return 0;
5791}
5792
5793/*
5794 * GART
5795 * VMID 0 is the physical GPU addresses as used by the kernel.
5796 * VMIDs 1-15 are used for userspace clients and are handled
5797 * by the radeon vm/hsa code.
5798 */
5799/**
5800 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5801 *
5802 * @rdev: radeon_device pointer
5803 *
5804 * Flush the TLB for the VMID 0 page table (CIK).
5805 */
5806void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5807{
5808 /* flush hdp cache */
5809 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5810
5811 /* bits 0-15 are the VM contexts0-15 */
5812 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5813}
5814
08dcc57f
BG		 5815static void cik_pcie_init_compute_vmid(struct radeon_device *rdev)
5816{
5817 int i;
5818 uint32_t sh_mem_bases, sh_mem_config;
5819
5820 sh_mem_bases = 0x6000 | 0x6000 << 16;
5821 sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED);
5822 sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED);
5823
5824 mutex_lock(&rdev->srbm_mutex);
5825 for (i = 8; i < 16; i++) {
5826 cik_srbm_select(rdev, 0, 0, 0, i);
5827 /* CP and shaders */
5828 WREG32(SH_MEM_CONFIG, sh_mem_config);
5829 WREG32(SH_MEM_APE1_BASE, 1);
5830 WREG32(SH_MEM_APE1_LIMIT, 0);
5831 WREG32(SH_MEM_BASES, sh_mem_bases);
5832 }
5833 cik_srbm_select(rdev, 0, 0, 0, 0);
5834 mutex_unlock(&rdev->srbm_mutex);
5835}
5836
1c49165d
AD		 5837/**
5838 * cik_pcie_gart_enable - gart enable
5839 *
5840 * @rdev: radeon_device pointer
5841 *
5842 * This sets up the TLBs, programs the page tables for VMID0,
5843 * sets up the hw for VMIDs 1-15 which are allocated on
5844 * demand, and sets up the global locations for the LDS, GDS,
5845 * and GPUVM for FSA64 clients (CIK).
5846 * Returns 0 for success, errors for failure.
5847 */
5848static int cik_pcie_gart_enable(struct radeon_device *rdev)
5849{
5850 int r, i;
5851
5852 if (rdev->gart.robj == NULL) {
5853 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5854 return -EINVAL;
5855 }
5856 r = radeon_gart_table_vram_pin(rdev);
5857 if (r)
5858 return r;
1c49165d
AD		 5859 /* Setup TLB control */
5860 WREG32(MC_VM_MX_L1_TLB_CNTL,
5861 (0xA << 7) |
5862 ENABLE_L1_TLB |
ec3dbbcb 5863 ENABLE_L1_FRAGMENT_PROCESSING |
1c49165d
AD		 5864 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5865 ENABLE_ADVANCED_DRIVER_MODEL |
5866 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5867 /* Setup L2 cache */
5868 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5869 ENABLE_L2_FRAGMENT_PROCESSING |
5870 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5871 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5872 EFFECTIVE_L2_QUEUE_SIZE(7) |
5873 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5874 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5875 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
ec3dbbcb
CK		 5876 BANK_SELECT(4) |
5877 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
1c49165d
AD		 5878 /* setup context0 */
5879 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
7c0411d2 5880 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1c49165d
AD		 5881 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5882 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5883 (u32)(rdev->dummy_page.addr >> 12));
5884 WREG32(VM_CONTEXT0_CNTL2, 0);
5885 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5886 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5887
5888 WREG32(0x15D4, 0);
5889 WREG32(0x15D8, 0);
5890 WREG32(0x15DC, 0);
5891
054e01d6 5892 /* restore context1-15 */
1c49165d
AD		 5893 /* set vm size, must be a multiple of 4 */
5894 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
607d4806 5895 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
1c49165d
AD		 5896 for (i = 1; i < 16; i++) {
5897 if (i < 8)
5898 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
054e01d6 5899 rdev->vm_manager.saved_table_addr[i]);
1c49165d
AD		 5900 else
5901 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
054e01d6 5902 rdev->vm_manager.saved_table_addr[i]);
1c49165d
AD		 5903 }
5904
5905 /* enable context1-15 */
5906 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5907 (u32)(rdev->dummy_page.addr >> 12));
a00024b0 5908 WREG32(VM_CONTEXT1_CNTL2, 4);
1c49165d 5909 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4510fb98 5910 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
a00024b0
AD		 5911 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5912 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5913 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5914 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5915 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5916 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5917 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5918 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5919 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5920 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5921 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5922 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1c49165d 5923
1c49165d
AD		 5924 if (rdev->family == CHIP_KAVERI) {
5925 u32 tmp = RREG32(CHUB_CONTROL);
5926 tmp &= ~BYPASS_VM;
5927 WREG32(CHUB_CONTROL, tmp);
5928 }
5929
5930 /* XXX SH_MEM regs */
5931 /* where to put LDS, scratch, GPUVM in FSA64 space */
f61d5b46 5932 mutex_lock(&rdev->srbm_mutex);
1c49165d 5933 for (i = 0; i < 16; i++) {
b556b12e 5934 cik_srbm_select(rdev, 0, 0, 0, i);
21a93e13 5935 /* CP and shaders */
1c49165d
AD		 5936 WREG32(SH_MEM_CONFIG, 0);
5937 WREG32(SH_MEM_APE1_BASE, 1);
5938 WREG32(SH_MEM_APE1_LIMIT, 0);
5939 WREG32(SH_MEM_BASES, 0);
21a93e13
AD		 5940 /* SDMA GFX */
5941 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5942 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5943 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5944 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5945 /* XXX SDMA RLC - todo */
1c49165d 5946 }
b556b12e 5947 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 5948 mutex_unlock(&rdev->srbm_mutex);
1c49165d 5949
08dcc57f
BG		 5950 cik_pcie_init_compute_vmid(rdev);
5951
1c49165d
AD		 5952 cik_pcie_gart_tlb_flush(rdev);
5953 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5954 (unsigned)(rdev->mc.gtt_size >> 20),
5955 (unsigned long long)rdev->gart.table_addr);
5956 rdev->gart.ready = true;
5957 return 0;
5958}
5959
5960/**
5961 * cik_pcie_gart_disable - gart disable
5962 *
5963 * @rdev: radeon_device pointer
5964 *
5965 * This disables all VM page table (CIK).
5966 */
5967static void cik_pcie_gart_disable(struct radeon_device *rdev)
5968{
054e01d6
CK		 5969 unsigned i;
5970
5971 for (i = 1; i < 16; ++i) {
5972 uint32_t reg;
5973 if (i < 8)
5974 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
5975 else
5976 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
5977 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
5978 }
5979
1c49165d
AD		 5980 /* Disable all tables */
5981 WREG32(VM_CONTEXT0_CNTL, 0);
5982 WREG32(VM_CONTEXT1_CNTL, 0);
5983 /* Setup TLB control */
5984 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5985 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5986 /* Setup L2 cache */
5987 WREG32(VM_L2_CNTL,
5988 ENABLE_L2_FRAGMENT_PROCESSING |
5989 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5990 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5991 EFFECTIVE_L2_QUEUE_SIZE(7) |
5992 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5993 WREG32(VM_L2_CNTL2, 0);
5994 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5995 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5996 radeon_gart_table_vram_unpin(rdev);
5997}
5998
5999/**
6000 * cik_pcie_gart_fini - vm fini callback
6001 *
6002 * @rdev: radeon_device pointer
6003 *
6004 * Tears down the driver GART/VM setup (CIK).
6005 */
6006static void cik_pcie_gart_fini(struct radeon_device *rdev)
6007{
6008 cik_pcie_gart_disable(rdev);
6009 radeon_gart_table_vram_free(rdev);
6010 radeon_gart_fini(rdev);
6011}
6012
6013/* vm parser */
6014/**
6015 * cik_ib_parse - vm ib_parse callback
6016 *
6017 * @rdev: radeon_device pointer
6018 * @ib: indirect buffer pointer
6019 *
6020 * CIK uses hw IB checking so this is a nop (CIK).
6021 */
6022int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
6023{
6024 return 0;
6025}
6026
6027/*
6028 * vm
6029 * VMID 0 is the physical GPU addresses as used by the kernel.
6030 * VMIDs 1-15 are used for userspace clients and are handled
6031 * by the radeon vm/hsa code.
6032 */
6033/**
6034 * cik_vm_init - cik vm init callback
6035 *
6036 * @rdev: radeon_device pointer
6037 *
6038 * Inits cik specific vm parameters (number of VMs, base of vram for
6039 * VMIDs 1-15) (CIK).
6040 * Returns 0 for success.
6041 */
6042int cik_vm_init(struct radeon_device *rdev)
6043{
62a7b7fb
OG		 6044 /*
6045 * number of VMs
6046 * VMID 0 is reserved for System
6047 * radeon graphics/compute will use VMIDs 1-7
6048 * amdkfd will use VMIDs 8-15
6049 */
6050 rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS;
1c49165d
AD		 6051 /* base offset of vram pages */
6052 if (rdev->flags & RADEON_IS_IGP) {
6053 u64 tmp = RREG32(MC_VM_FB_OFFSET);
6054 tmp <<= 22;
6055 rdev->vm_manager.vram_base_offset = tmp;
6056 } else
6057 rdev->vm_manager.vram_base_offset = 0;
6058
6059 return 0;
6060}
6061
6062/**
6063 * cik_vm_fini - cik vm fini callback
6064 *
6065 * @rdev: radeon_device pointer
6066 *
6067 * Tear down any asic specific VM setup (CIK).
6068 */
6069void cik_vm_fini(struct radeon_device *rdev)
6070{
6071}
6072
3ec7d11b
AD		 6073/**
6074 * cik_vm_decode_fault - print human readable fault info
6075 *
6076 * @rdev: radeon_device pointer
6077 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
6078 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
6079 *
6080 * Print human readable fault information (CIK).
6081 */
6082static void cik_vm_decode_fault(struct radeon_device *rdev,
6083 u32 status, u32 addr, u32 mc_client)
6084{
939c0d3c 6085 u32 mc_id;
3ec7d11b
AD		 6086 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
6087 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
328a50c7
MD		 6088 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
6089 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
3ec7d11b 6090
939c0d3c
AD		 6091 if (rdev->family == CHIP_HAWAII)
6092 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
6093 else
6094 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
6095
328a50c7 6096 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
3ec7d11b
AD		 6097 protections, vmid, addr,
6098 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
328a50c7 6099 block, mc_client, mc_id);
3ec7d11b
AD		 6100}
6101
f96ab484
AD		 6102/**
6103 * cik_vm_flush - cik vm flush using the CP
6104 *
6105 * @rdev: radeon_device pointer
6106 *
6107 * Update the page table base and flush the VM TLB
6108 * using the CP (CIK).
6109 */
faffaf62
CK		 6110void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
6111 unsigned vm_id, uint64_t pd_addr)
f96ab484 6112{
faffaf62 6113 int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX);
f96ab484
AD		 6114
6115 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
f1d2a26b 6116 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484 6117 WRITE_DATA_DST_SEL(0)));
faffaf62 6118 if (vm_id < 8) {
f96ab484 6119 radeon_ring_write(ring,
faffaf62 6120 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
f96ab484
AD		 6121 } else {
6122 radeon_ring_write(ring,
faffaf62 6123 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
f96ab484
AD		 6124 }
6125 radeon_ring_write(ring, 0);
faffaf62 6126 radeon_ring_write(ring, pd_addr >> 12);
f96ab484
AD		 6127
6128 /* update SH_MEM_* regs */
6129 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 6130 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 6131 WRITE_DATA_DST_SEL(0)));
6132 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
6133 radeon_ring_write(ring, 0);
faffaf62 6134 radeon_ring_write(ring, VMID(vm_id));
f96ab484
AD		 6135
6136 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
4fb0bbd5 6137 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 6138 WRITE_DATA_DST_SEL(0)));
6139 radeon_ring_write(ring, SH_MEM_BASES >> 2);
6140 radeon_ring_write(ring, 0);
6141
6142 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
6143 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
6144 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
6145 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
21a93e13 6146
f96ab484 6147 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 6148 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 6149 WRITE_DATA_DST_SEL(0)));
6150 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
6151 radeon_ring_write(ring, 0);
6152 radeon_ring_write(ring, VMID(0));
6f2043ce 6153
f96ab484 6154 /* HDP flush */
faffaf62 6155 cik_hdp_flush_cp_ring_emit(rdev, ring->idx);
f96ab484
AD		 6156
6157 /* bits 0-15 are the VM contexts0-15 */
6158 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 6159 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 6160 WRITE_DATA_DST_SEL(0)));
6161 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
6162 radeon_ring_write(ring, 0);
faffaf62 6163 radeon_ring_write(ring, 1 << vm_id);
f96ab484 6164
3a01fd36
AD		 6165 /* wait for the invalidate to complete */
6166 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6167 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
6168 WAIT_REG_MEM_FUNCTION(0) | /* always */
6169 WAIT_REG_MEM_ENGINE(0))); /* me */
6170 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
6171 radeon_ring_write(ring, 0);
6172 radeon_ring_write(ring, 0); /* ref */
6173 radeon_ring_write(ring, 0); /* mask */
6174 radeon_ring_write(ring, 0x20); /* poll interval */
6175
b07fdd38 6176 /* compute doesn't have PFP */
f1d2a26b 6177 if (usepfp) {
b07fdd38
AD		 6178 /* sync PFP to ME, otherwise we might get invalid PFP reads */
6179 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6180 radeon_ring_write(ring, 0x0);
6181 }
cc066715 6182}
6f2043ce 6183
f6796cae
AD		 6184/*
6185 * RLC
6186 * The RLC is a multi-purpose microengine that handles a
6187 * variety of functions, the most important of which is
6188 * the interrupt controller.
6189 */
866d83de
AD		 6190static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
6191 bool enable)
f6796cae 6192{
866d83de 6193 u32 tmp = RREG32(CP_INT_CNTL_RING0);
f6796cae 6194
866d83de
AD		 6195 if (enable)
6196 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6197 else
6198 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
f6796cae 6199 WREG32(CP_INT_CNTL_RING0, tmp);
866d83de 6200}
f6796cae 6201
866d83de 6202static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
cc066715 6203{
cc066715 6204 u32 tmp;
6f2043ce 6205
866d83de
AD		 6206 tmp = RREG32(RLC_LB_CNTL);
6207 if (enable)
6208 tmp |= LOAD_BALANCE_ENABLE;
6209 else
6210 tmp &= ~LOAD_BALANCE_ENABLE;
6211 WREG32(RLC_LB_CNTL, tmp);
6212}
cc066715 6213
866d83de
AD		 6214static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
6215{
6216 u32 i, j, k;
6217 u32 mask;
cc066715 6218
1c0a4625 6219 mutex_lock(&rdev->grbm_idx_mutex);
f6796cae
AD		 6220 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6221 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6222 cik_select_se_sh(rdev, i, j);
6223 for (k = 0; k < rdev->usec_timeout; k++) {
6224 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
6225 break;
6226 udelay(1);
6227 }
6228 }
6229 }
6230 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 6231 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 6232
f6796cae
AD		 6233 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
6234 for (k = 0; k < rdev->usec_timeout; k++) {
6235 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
6236 break;
6237 udelay(1);
6238 }
6239}
cc066715 6240
22c775ce
AD		 6241static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
6242{
6243 u32 tmp;
cc066715 6244
22c775ce
AD		 6245 tmp = RREG32(RLC_CNTL);
6246 if (tmp != rlc)
6247 WREG32(RLC_CNTL, rlc);
6248}
cc066715 6249
22c775ce
AD		 6250static u32 cik_halt_rlc(struct radeon_device *rdev)
6251{
6252 u32 data, orig;
cc066715 6253
22c775ce 6254 orig = data = RREG32(RLC_CNTL);
cc066715 6255
22c775ce
AD		 6256 if (data & RLC_ENABLE) {
6257 u32 i;
cc066715 6258
22c775ce
AD		 6259 data &= ~RLC_ENABLE;
6260 WREG32(RLC_CNTL, data);
cc066715 6261
22c775ce
AD		 6262 for (i = 0; i < rdev->usec_timeout; i++) {
6263 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
6264 break;
6265 udelay(1);
6266 }
cc066715 6267
22c775ce
AD		 6268 cik_wait_for_rlc_serdes(rdev);
6269 }
cc066715 6270
22c775ce
AD		 6271 return orig;
6272}
cc066715 6273
a412fce0
AD		 6274void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
6275{
6276 u32 tmp, i, mask;
6277
6278 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
6279 WREG32(RLC_GPR_REG2, tmp);
6280
6281 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
6282 for (i = 0; i < rdev->usec_timeout; i++) {
6283 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
6284 break;
6285 udelay(1);
6286 }
6287
6288 for (i = 0; i < rdev->usec_timeout; i++) {
6289 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
6290 break;
6291 udelay(1);
6292 }
6293}
6294
6295void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
6296{
6297 u32 tmp;
6298
6299 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
6300 WREG32(RLC_GPR_REG2, tmp);
6301}
6302
866d83de
AD		 6303/**
6304 * cik_rlc_stop - stop the RLC ME
6305 *
6306 * @rdev: radeon_device pointer
6307 *
6308 * Halt the RLC ME (MicroEngine) (CIK).
6309 */
6310static void cik_rlc_stop(struct radeon_device *rdev)
6311{
22c775ce 6312 WREG32(RLC_CNTL, 0);
866d83de
AD		 6313
6314 cik_enable_gui_idle_interrupt(rdev, false);
6315
866d83de
AD		 6316 cik_wait_for_rlc_serdes(rdev);
6317}
6318
f6796cae
AD		 6319/**
6320 * cik_rlc_start - start the RLC ME
6321 *
6322 * @rdev: radeon_device pointer
6323 *
6324 * Unhalt the RLC ME (MicroEngine) (CIK).
6325 */
6326static void cik_rlc_start(struct radeon_device *rdev)
6327{
f6796cae 6328 WREG32(RLC_CNTL, RLC_ENABLE);
cc066715 6329
866d83de 6330 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 6331
f6796cae 6332 udelay(50);
6f2043ce
AD		 6333}
6334
6335/**
f6796cae 6336 * cik_rlc_resume - setup the RLC hw
6f2043ce
AD		 6337 *
6338 * @rdev: radeon_device pointer
6339 *
f6796cae
AD		 6340 * Initialize the RLC registers, load the ucode,
6341 * and start the RLC (CIK).
6342 * Returns 0 for success, -EINVAL if the ucode is not available.
6f2043ce 6343 */
f6796cae 6344static int cik_rlc_resume(struct radeon_device *rdev)
6f2043ce 6345{
22c775ce 6346 u32 i, size, tmp;
cc066715 6347
f6796cae
AD		 6348 if (!rdev->rlc_fw)
6349 return -EINVAL;
cc066715 6350
cc066715
AD		 6351 cik_rlc_stop(rdev);
6352
22c775ce
AD		 6353 /* disable CG */
6354 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
6355 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
cc066715 6356
866d83de 6357 si_rlc_reset(rdev);
6f2043ce 6358
22c775ce 6359 cik_init_pg(rdev);
6f2043ce 6360
22c775ce 6361 cik_init_cg(rdev);
cc066715 6362
f6796cae
AD		 6363 WREG32(RLC_LB_CNTR_INIT, 0);
6364 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
cc066715 6365
1c0a4625 6366 mutex_lock(&rdev->grbm_idx_mutex);
f6796cae
AD		 6367 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6368 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
6369 WREG32(RLC_LB_PARAMS, 0x00600408);
6370 WREG32(RLC_LB_CNTL, 0x80000004);
1c0a4625 6371 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 6372
f6796cae
AD		 6373 WREG32(RLC_MC_CNTL, 0);
6374 WREG32(RLC_UCODE_CNTL, 0);
cc066715 6375
f2c6b0f4
AD		 6376 if (rdev->new_fw) {
6377 const struct rlc_firmware_header_v1_0 *hdr =
6378 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
6379 const __le32 *fw_data = (const __le32 *)
6380 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6381
6382 radeon_ucode_print_rlc_hdr(&hdr->header);
6383
6384 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
6385 WREG32(RLC_GPM_UCODE_ADDR, 0);
6386 for (i = 0; i < size; i++)
6387 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 6388 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version));
f2c6b0f4
AD		 6389 } else {
6390 const __be32 *fw_data;
6391
6392 switch (rdev->family) {
6393 case CHIP_BONAIRE:
6394 case CHIP_HAWAII:
6395 default:
6396 size = BONAIRE_RLC_UCODE_SIZE;
6397 break;
6398 case CHIP_KAVERI:
6399 size = KV_RLC_UCODE_SIZE;
6400 break;
6401 case CHIP_KABINI:
6402 size = KB_RLC_UCODE_SIZE;
6403 break;
6404 case CHIP_MULLINS:
6405 size = ML_RLC_UCODE_SIZE;
6406 break;
6407 }
6408
6409 fw_data = (const __be32 *)rdev->rlc_fw->data;
6410 WREG32(RLC_GPM_UCODE_ADDR, 0);
6411 for (i = 0; i < size; i++)
6412 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6413 WREG32(RLC_GPM_UCODE_ADDR, 0);
6414 }
cc066715 6415
866d83de
AD		 6416 /* XXX - find out what chips support lbpw */
6417 cik_enable_lbpw(rdev, false);
cc066715 6418
22c775ce
AD		 6419 if (rdev->family == CHIP_BONAIRE)
6420 WREG32(RLC_DRIVER_DMA_STATUS, 0);
cc066715 6421
f6796cae 6422 cik_rlc_start(rdev);
cc066715 6423
f6796cae
AD		 6424 return 0;
6425}
cc066715 6426
22c775ce
AD		 6427static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6428{
6429 u32 data, orig, tmp, tmp2;
cc066715 6430
22c775ce 6431 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
cc066715 6432
473359bc 6433 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
ddc76ff6 6434 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 6435
22c775ce 6436 tmp = cik_halt_rlc(rdev);
cc066715 6437
1c0a4625 6438 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6439 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6440 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6441 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6442 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6443 WREG32(RLC_SERDES_WR_CTRL, tmp2);
1c0a4625 6444 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 6445
22c775ce 6446 cik_update_rlc(rdev, tmp);
cc066715 6447
22c775ce
AD		 6448 data |= CGCG_EN | CGLS_EN;
6449 } else {
ddc76ff6 6450 cik_enable_gui_idle_interrupt(rdev, false);
cc066715 6451
22c775ce
AD		 6452 RREG32(CB_CGTT_SCLK_CTRL);
6453 RREG32(CB_CGTT_SCLK_CTRL);
6454 RREG32(CB_CGTT_SCLK_CTRL);
6455 RREG32(CB_CGTT_SCLK_CTRL);
cc066715 6456
22c775ce 6457 data &= ~(CGCG_EN | CGLS_EN);
cc066715 6458 }
6f2043ce 6459
22c775ce
AD		 6460 if (orig != data)
6461 WREG32(RLC_CGCG_CGLS_CTRL, data);
cc066715 6462
6f2043ce
AD		 6463}
6464
22c775ce 6465static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6f2043ce 6466{
22c775ce
AD		 6467 u32 data, orig, tmp = 0;
6468
473359bc
AD		 6469 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6470 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6471 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6472 orig = data = RREG32(CP_MEM_SLP_CNTL);
6473 data |= CP_MEM_LS_EN;
6474 if (orig != data)
6475 WREG32(CP_MEM_SLP_CNTL, data);
6476 }
6477 }
cc066715 6478
22c775ce 6479 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4bb62c95 6480 data |= 0x00000001;
22c775ce
AD		 6481 data &= 0xfffffffd;
6482 if (orig != data)
6483 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6484
6485 tmp = cik_halt_rlc(rdev);
6486
1c0a4625 6487 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6488 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6489 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6490 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6491 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6492 WREG32(RLC_SERDES_WR_CTRL, data);
1c0a4625 6493 mutex_unlock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6494
6495 cik_update_rlc(rdev, tmp);
6496
473359bc
AD		 6497 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6498 orig = data = RREG32(CGTS_SM_CTRL_REG);
6499 data &= ~SM_MODE_MASK;
6500 data |= SM_MODE(0x2);
6501 data |= SM_MODE_ENABLE;
6502 data &= ~CGTS_OVERRIDE;
6503 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6504 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6505 data &= ~CGTS_LS_OVERRIDE;
6506 data &= ~ON_MONITOR_ADD_MASK;
6507 data |= ON_MONITOR_ADD_EN;
6508 data |= ON_MONITOR_ADD(0x96);
6509 if (orig != data)
6510 WREG32(CGTS_SM_CTRL_REG, data);
6511 }
22c775ce
AD		 6512 } else {
6513 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4bb62c95 6514 data |= 0x00000003;
22c775ce
AD		 6515 if (orig != data)
6516 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6517
6518 data = RREG32(RLC_MEM_SLP_CNTL);
6519 if (data & RLC_MEM_LS_EN) {
6520 data &= ~RLC_MEM_LS_EN;
6521 WREG32(RLC_MEM_SLP_CNTL, data);
6522 }
6f2043ce 6523
22c775ce
AD		 6524 data = RREG32(CP_MEM_SLP_CNTL);
6525 if (data & CP_MEM_LS_EN) {
6526 data &= ~CP_MEM_LS_EN;
6527 WREG32(CP_MEM_SLP_CNTL, data);
6528 }
cc066715 6529
22c775ce
AD		 6530 orig = data = RREG32(CGTS_SM_CTRL_REG);
6531 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6532 if (orig != data)
6533 WREG32(CGTS_SM_CTRL_REG, data);
cc066715 6534
22c775ce 6535 tmp = cik_halt_rlc(rdev);
cc066715 6536
1c0a4625 6537 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6538 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6539 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6540 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6541 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6542 WREG32(RLC_SERDES_WR_CTRL, data);
1c0a4625 6543 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 6544
22c775ce 6545 cik_update_rlc(rdev, tmp);
cc066715 6546 }
6f2043ce 6547}
1c49165d 6548
22c775ce 6549static const u32 mc_cg_registers[] =
21a93e13 6550{
22c775ce
AD		 6551 MC_HUB_MISC_HUB_CG,
6552 MC_HUB_MISC_SIP_CG,
6553 MC_HUB_MISC_VM_CG,
6554 MC_XPB_CLK_GAT,
6555 ATC_MISC_CG,
6556 MC_CITF_MISC_WR_CG,
6557 MC_CITF_MISC_RD_CG,
6558 MC_CITF_MISC_VM_CG,
6559 VM_L2_CG,
6560};
21a93e13 6561
22c775ce
AD		 6562static void cik_enable_mc_ls(struct radeon_device *rdev,
6563 bool enable)
1c49165d 6564{
22c775ce
AD		 6565 int i;
6566 u32 orig, data;
1c49165d 6567
22c775ce
AD		 6568 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6569 orig = data = RREG32(mc_cg_registers[i]);
473359bc 6570 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
22c775ce
AD		 6571 data |= MC_LS_ENABLE;
6572 else
6573 data &= ~MC_LS_ENABLE;
6574 if (data != orig)
6575 WREG32(mc_cg_registers[i], data);
1c49165d 6576 }
22c775ce 6577}
1c49165d 6578
22c775ce
AD		 6579static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6580 bool enable)
6581{
6582 int i;
6583 u32 orig, data;
6584
6585 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6586 orig = data = RREG32(mc_cg_registers[i]);
473359bc 6587 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
22c775ce
AD		 6588 data |= MC_CG_ENABLE;
6589 else
6590 data &= ~MC_CG_ENABLE;
6591 if (data != orig)
6592 WREG32(mc_cg_registers[i], data);
1c49165d 6593 }
1c49165d
AD		 6594}
6595
22c775ce
AD		 6596static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6597 bool enable)
1c49165d 6598{
22c775ce 6599 u32 orig, data;
1c49165d 6600
473359bc 6601 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
22c775ce
AD		 6602 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6603 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
1c49165d 6604 } else {
22c775ce
AD		 6605 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6606 data |= 0xff000000;
6607 if (data != orig)
6608 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
1c49165d 6609
22c775ce
AD		 6610 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6611 data |= 0xff000000;
6612 if (data != orig)
6613 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6614 }
1c49165d
AD		 6615}
6616
22c775ce
AD		 6617static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6618 bool enable)
1c49165d 6619{
22c775ce
AD		 6620 u32 orig, data;
6621
473359bc 6622 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
22c775ce
AD		 6623 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6624 data |= 0x100;
6625 if (orig != data)
6626 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6627
6628 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6629 data |= 0x100;
6630 if (orig != data)
6631 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6632 } else {
6633 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6634 data &= ~0x100;
6635 if (orig != data)
6636 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
1c49165d 6637
22c775ce
AD		 6638 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6639 data &= ~0x100;
6640 if (orig != data)
6641 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6642 }
1c49165d
AD		 6643}
6644
22c775ce
AD		 6645static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6646 bool enable)
1c49165d 6647{
22c775ce 6648 u32 orig, data;
1c49165d 6649
473359bc 6650 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
22c775ce
AD		 6651 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6652 data = 0xfff;
6653 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6654
22c775ce
AD		 6655 orig = data = RREG32(UVD_CGC_CTRL);
6656 data |= DCM;
6657 if (orig != data)
6658 WREG32(UVD_CGC_CTRL, data);
6659 } else {
6660 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6661 data &= ~0xfff;
6662 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6663
22c775ce
AD		 6664 orig = data = RREG32(UVD_CGC_CTRL);
6665 data &= ~DCM;
6666 if (orig != data)
6667 WREG32(UVD_CGC_CTRL, data);
1c49165d 6668 }
22c775ce 6669}
1c49165d 6670
473359bc
AD		 6671static void cik_enable_bif_mgls(struct radeon_device *rdev,
6672 bool enable)
6673{
6674 u32 orig, data;
1c49165d 6675
473359bc 6676 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
1c49165d 6677
473359bc
AD		 6678 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6679 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6680 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6681 else
6682 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6683 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
1c49165d 6684
473359bc
AD		 6685 if (orig != data)
6686 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6687}
1c49165d 6688
22c775ce
AD		 6689static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6690 bool enable)
6691{
6692 u32 orig, data;
1c49165d 6693
22c775ce 6694 orig = data = RREG32(HDP_HOST_PATH_CNTL);
1c49165d 6695
473359bc 6696 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
22c775ce
AD		 6697 data &= ~CLOCK_GATING_DIS;
6698 else
6699 data |= CLOCK_GATING_DIS;
6700
6701 if (orig != data)
6702 WREG32(HDP_HOST_PATH_CNTL, data);
1c49165d
AD		 6703}
6704
22c775ce
AD		 6705static void cik_enable_hdp_ls(struct radeon_device *rdev,
6706 bool enable)
1c49165d 6707{
22c775ce
AD		 6708 u32 orig, data;
6709
6710 orig = data = RREG32(HDP_MEM_POWER_LS);
6711
473359bc 6712 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
22c775ce
AD		 6713 data |= HDP_LS_ENABLE;
6714 else
6715 data &= ~HDP_LS_ENABLE;
6716
6717 if (orig != data)
6718 WREG32(HDP_MEM_POWER_LS, data);
6719}
6720
6721void cik_update_cg(struct radeon_device *rdev,
6722 u32 block, bool enable)
6723{
4214faf6 6724
22c775ce 6725 if (block & RADEON_CG_BLOCK_GFX) {
4214faf6 6726 cik_enable_gui_idle_interrupt(rdev, false);
22c775ce
AD		 6727 /* order matters! */
6728 if (enable) {
6729 cik_enable_mgcg(rdev, true);
6730 cik_enable_cgcg(rdev, true);
6731 } else {
6732 cik_enable_cgcg(rdev, false);
6733 cik_enable_mgcg(rdev, false);
6734 }
4214faf6 6735 cik_enable_gui_idle_interrupt(rdev, true);
22c775ce
AD		 6736 }
6737
6738 if (block & RADEON_CG_BLOCK_MC) {
6739 if (!(rdev->flags & RADEON_IS_IGP)) {
6740 cik_enable_mc_mgcg(rdev, enable);
6741 cik_enable_mc_ls(rdev, enable);
6742 }
6743 }
6744
6745 if (block & RADEON_CG_BLOCK_SDMA) {
6746 cik_enable_sdma_mgcg(rdev, enable);
6747 cik_enable_sdma_mgls(rdev, enable);
6748 }
6749
473359bc
AD		 6750 if (block & RADEON_CG_BLOCK_BIF) {
6751 cik_enable_bif_mgls(rdev, enable);
6752 }
6753
22c775ce
AD		 6754 if (block & RADEON_CG_BLOCK_UVD) {
6755 if (rdev->has_uvd)
6756 cik_enable_uvd_mgcg(rdev, enable);
6757 }
6758
6759 if (block & RADEON_CG_BLOCK_HDP) {
6760 cik_enable_hdp_mgcg(rdev, enable);
6761 cik_enable_hdp_ls(rdev, enable);
6762 }
a1d6f97c
AD		 6763
6764 if (block & RADEON_CG_BLOCK_VCE) {
6765 vce_v2_0_enable_mgcg(rdev, enable);
6766 }
1c49165d
AD		 6767}
6768
22c775ce 6769static void cik_init_cg(struct radeon_device *rdev)
1c49165d 6770{
22c775ce 6771
ddc76ff6 6772 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
22c775ce
AD		 6773
6774 if (rdev->has_uvd)
6775 si_init_uvd_internal_cg(rdev);
6776
6777 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6778 RADEON_CG_BLOCK_SDMA |
473359bc 6779 RADEON_CG_BLOCK_BIF |
22c775ce
AD		 6780 RADEON_CG_BLOCK_UVD |
6781 RADEON_CG_BLOCK_HDP), true);
1c49165d
AD		 6782}
6783
473359bc 6784static void cik_fini_cg(struct radeon_device *rdev)
1c49165d 6785{
473359bc
AD		 6786 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6787 RADEON_CG_BLOCK_SDMA |
6788 RADEON_CG_BLOCK_BIF |
6789 RADEON_CG_BLOCK_UVD |
6790 RADEON_CG_BLOCK_HDP), false);
6791
6792 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
1c49165d
AD		 6793}
6794
22c775ce
AD		 6795static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6796 bool enable)
1c49165d 6797{
22c775ce 6798 u32 data, orig;
1c49165d 6799
22c775ce 6800 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6801 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6802 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6803 else
6804 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6805 if (orig != data)
6806 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6807}
6808
22c775ce
AD		 6809static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6810 bool enable)
1c49165d 6811{
22c775ce
AD		 6812 u32 data, orig;
6813
6814 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6815 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6816 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6817 else
6818 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6819 if (orig != data)
6820 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6821}
6822
22c775ce 6823static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
3ec7d11b 6824{
22c775ce 6825 u32 data, orig;
3ec7d11b 6826
22c775ce 6827 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6828 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
22c775ce
AD		 6829 data &= ~DISABLE_CP_PG;
6830 else
6831 data |= DISABLE_CP_PG;
6832 if (orig != data)
6833 WREG32(RLC_PG_CNTL, data);
3ec7d11b
AD		 6834}
6835
22c775ce 6836static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
f96ab484 6837{
22c775ce 6838 u32 data, orig;
f96ab484 6839
22c775ce 6840 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6841 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
22c775ce
AD		 6842 data &= ~DISABLE_GDS_PG;
6843 else
6844 data |= DISABLE_GDS_PG;
6845 if (orig != data)
6846 WREG32(RLC_PG_CNTL, data);
6847}
6848
6849#define CP_ME_TABLE_SIZE 96
6850#define CP_ME_TABLE_OFFSET 2048
6851#define CP_MEC_TABLE_OFFSET 4096
6852
6853void cik_init_cp_pg_table(struct radeon_device *rdev)
6854{
22c775ce
AD		 6855 volatile u32 *dst_ptr;
6856 int me, i, max_me = 4;
6857 u32 bo_offset = 0;
f2c6b0f4 6858 u32 table_offset, table_size;
22c775ce
AD		 6859
6860 if (rdev->family == CHIP_KAVERI)
6861 max_me = 5;
6862
6863 if (rdev->rlc.cp_table_ptr == NULL)
f96ab484
AD		 6864 return;
6865
22c775ce
AD		 6866 /* write the cp table buffer */
6867 dst_ptr = rdev->rlc.cp_table_ptr;
6868 for (me = 0; me < max_me; me++) {
f2c6b0f4
AD		 6869 if (rdev->new_fw) {
6870 const __le32 *fw_data;
6871 const struct gfx_firmware_header_v1_0 *hdr;
6872
6873 if (me == 0) {
6874 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6875 fw_data = (const __le32 *)
6876 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6877 table_offset = le32_to_cpu(hdr->jt_offset);
6878 table_size = le32_to_cpu(hdr->jt_size);
6879 } else if (me == 1) {
6880 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6881 fw_data = (const __le32 *)
6882 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6883 table_offset = le32_to_cpu(hdr->jt_offset);
6884 table_size = le32_to_cpu(hdr->jt_size);
6885 } else if (me == 2) {
6886 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6887 fw_data = (const __le32 *)
6888 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6889 table_offset = le32_to_cpu(hdr->jt_offset);
6890 table_size = le32_to_cpu(hdr->jt_size);
6891 } else if (me == 3) {
6892 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6893 fw_data = (const __le32 *)
6894 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6895 table_offset = le32_to_cpu(hdr->jt_offset);
6896 table_size = le32_to_cpu(hdr->jt_size);
6897 } else {
6898 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6899 fw_data = (const __le32 *)
6900 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6901 table_offset = le32_to_cpu(hdr->jt_offset);
6902 table_size = le32_to_cpu(hdr->jt_size);
6903 }
6904
6905 for (i = 0; i < table_size; i ++) {
6906 dst_ptr[bo_offset + i] =
6907 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6908 }
6909 bo_offset += table_size;
22c775ce 6910 } else {
f2c6b0f4
AD		 6911 const __be32 *fw_data;
6912 table_size = CP_ME_TABLE_SIZE;
6913
6914 if (me == 0) {
6915 fw_data = (const __be32 *)rdev->ce_fw->data;
6916 table_offset = CP_ME_TABLE_OFFSET;
6917 } else if (me == 1) {
6918 fw_data = (const __be32 *)rdev->pfp_fw->data;
6919 table_offset = CP_ME_TABLE_OFFSET;
6920 } else if (me == 2) {
6921 fw_data = (const __be32 *)rdev->me_fw->data;
6922 table_offset = CP_ME_TABLE_OFFSET;
6923 } else {
6924 fw_data = (const __be32 *)rdev->mec_fw->data;
6925 table_offset = CP_MEC_TABLE_OFFSET;
6926 }
22c775ce 6927
f2c6b0f4
AD		 6928 for (i = 0; i < table_size; i ++) {
6929 dst_ptr[bo_offset + i] =
6930 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6931 }
6932 bo_offset += table_size;
22c775ce 6933 }
f96ab484 6934 }
22c775ce 6935}
f96ab484 6936
22c775ce
AD		 6937static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6938 bool enable)
6939{
6940 u32 data, orig;
6941
2b19d17f 6942 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
22c775ce
AD		 6943 orig = data = RREG32(RLC_PG_CNTL);
6944 data |= GFX_PG_ENABLE;
6945 if (orig != data)
6946 WREG32(RLC_PG_CNTL, data);
6947
6948 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6949 data |= AUTO_PG_EN;
6950 if (orig != data)
6951 WREG32(RLC_AUTO_PG_CTRL, data);
6952 } else {
6953 orig = data = RREG32(RLC_PG_CNTL);
6954 data &= ~GFX_PG_ENABLE;
6955 if (orig != data)
6956 WREG32(RLC_PG_CNTL, data);
f96ab484 6957
22c775ce
AD		 6958 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6959 data &= ~AUTO_PG_EN;
6960 if (orig != data)
6961 WREG32(RLC_AUTO_PG_CTRL, data);
f96ab484 6962
22c775ce
AD		 6963 data = RREG32(DB_RENDER_CONTROL);
6964 }
6965}
f96ab484 6966
22c775ce
AD		 6967static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6968{
6969 u32 mask = 0, tmp, tmp1;
6970 int i;
f96ab484 6971
1c0a4625 6972 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6973 cik_select_se_sh(rdev, se, sh);
6974 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6975 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6976 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 6977 mutex_unlock(&rdev->grbm_idx_mutex);
f96ab484 6978
22c775ce 6979 tmp &= 0xffff0000;
f96ab484 6980
22c775ce
AD		 6981 tmp |= tmp1;
6982 tmp >>= 16;
6983
6984 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6985 mask <<= 1;
6986 mask |= 1;
b07fdd38 6987 }
22c775ce
AD		 6988
6989 return (~tmp) & mask;
f96ab484
AD		 6990}
6991
22c775ce 6992static void cik_init_ao_cu_mask(struct radeon_device *rdev)
d0e092d9 6993{
22c775ce
AD		 6994 u32 i, j, k, active_cu_number = 0;
6995 u32 mask, counter, cu_bitmap;
6996 u32 tmp = 0;
d0e092d9 6997
22c775ce
AD		 6998 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6999 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
7000 mask = 1;
7001 cu_bitmap = 0;
7002 counter = 0;
7003 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
7004 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
7005 if (counter < 2)
7006 cu_bitmap |= mask;
7007 counter ++;
d0e092d9 7008 }
22c775ce 7009 mask <<= 1;
d0e092d9 7010 }
d0e092d9 7011
22c775ce
AD		 7012 active_cu_number += counter;
7013 tmp |= (cu_bitmap << (i * 16 + j * 8));
d0e092d9 7014 }
d0e092d9 7015 }
22c775ce
AD		 7016
7017 WREG32(RLC_PG_AO_CU_MASK, tmp);
7018
7019 tmp = RREG32(RLC_MAX_PG_CU);
7020 tmp &= ~MAX_PU_CU_MASK;
7021 tmp |= MAX_PU_CU(active_cu_number);
7022 WREG32(RLC_MAX_PG_CU, tmp);
d0e092d9
AD		 7023}
7024
22c775ce
AD		 7025static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
7026 bool enable)
605de6b9 7027{
22c775ce 7028 u32 data, orig;
605de6b9 7029
22c775ce 7030 orig = data = RREG32(RLC_PG_CNTL);
473359bc 7031 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
22c775ce
AD		 7032 data |= STATIC_PER_CU_PG_ENABLE;
7033 else
7034 data &= ~STATIC_PER_CU_PG_ENABLE;
7035 if (orig != data)
7036 WREG32(RLC_PG_CNTL, data);
7037}
7038
7039static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
7040 bool enable)
7041{
7042 u32 data, orig;
605de6b9 7043
22c775ce 7044 orig = data = RREG32(RLC_PG_CNTL);
473359bc 7045 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
22c775ce 7046 data |= DYN_PER_CU_PG_ENABLE;
605de6b9 7047 else
22c775ce
AD		 7048 data &= ~DYN_PER_CU_PG_ENABLE;
7049 if (orig != data)
7050 WREG32(RLC_PG_CNTL, data);
7051}
605de6b9 7052
22c775ce
AD		 7053#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
7054#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
7055
7056static void cik_init_gfx_cgpg(struct radeon_device *rdev)
7057{
7058 u32 data, orig;
7059 u32 i;
7060
7061 if (rdev->rlc.cs_data) {
7062 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
7063 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
a0f38609 7064 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
22c775ce 7065 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
605de6b9 7066 } else {
22c775ce
AD		 7067 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
7068 for (i = 0; i < 3; i++)
7069 WREG32(RLC_GPM_SCRATCH_DATA, 0);
7070 }
7071 if (rdev->rlc.reg_list) {
7072 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
7073 for (i = 0; i < rdev->rlc.reg_list_size; i++)
7074 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
605de6b9 7075 }
605de6b9 7076
22c775ce
AD		 7077 orig = data = RREG32(RLC_PG_CNTL);
7078 data |= GFX_PG_SRC;
7079 if (orig != data)
7080 WREG32(RLC_PG_CNTL, data);
605de6b9 7081
22c775ce
AD		 7082 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
7083 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
605de6b9 7084
22c775ce
AD		 7085 data = RREG32(CP_RB_WPTR_POLL_CNTL);
7086 data &= ~IDLE_POLL_COUNT_MASK;
7087 data |= IDLE_POLL_COUNT(0x60);
7088 WREG32(CP_RB_WPTR_POLL_CNTL, data);
605de6b9 7089
22c775ce
AD		 7090 data = 0x10101010;
7091 WREG32(RLC_PG_DELAY, data);
605de6b9 7092
22c775ce
AD		 7093 data = RREG32(RLC_PG_DELAY_2);
7094 data &= ~0xff;
7095 data |= 0x3;
7096 WREG32(RLC_PG_DELAY_2, data);
605de6b9 7097
22c775ce
AD		 7098 data = RREG32(RLC_AUTO_PG_CTRL);
7099 data &= ~GRBM_REG_SGIT_MASK;
7100 data |= GRBM_REG_SGIT(0x700);
7101 WREG32(RLC_AUTO_PG_CTRL, data);
605de6b9 7102
605de6b9
AD		 7103}
7104
22c775ce 7105static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
f6796cae 7106{
473359bc
AD		 7107 cik_enable_gfx_cgpg(rdev, enable);
7108 cik_enable_gfx_static_mgpg(rdev, enable);
7109 cik_enable_gfx_dynamic_mgpg(rdev, enable);
22c775ce 7110}
f6796cae 7111
a0f38609
AD		 7112u32 cik_get_csb_size(struct radeon_device *rdev)
7113{
7114 u32 count = 0;
7115 const struct cs_section_def *sect = NULL;
7116 const struct cs_extent_def *ext = NULL;
f6796cae 7117
a0f38609
AD		 7118 if (rdev->rlc.cs_data == NULL)
7119 return 0;
f6796cae 7120
a0f38609
AD		 7121 /* begin clear state */
7122 count += 2;
7123 /* context control state */
7124 count += 3;
7125
7126 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
7127 for (ext = sect->section; ext->extent != NULL; ++ext) {
7128 if (sect->id == SECT_CONTEXT)
7129 count += 2 + ext->reg_count;
7130 else
7131 return 0;
f6796cae
AD		 7132 }
7133 }
a0f38609
AD		 7134 /* pa_sc_raster_config/pa_sc_raster_config1 */
7135 count += 4;
7136 /* end clear state */
7137 count += 2;
7138 /* clear state */
7139 count += 2;
f6796cae 7140
a0f38609 7141 return count;
f6796cae
AD		 7142}
7143
a0f38609 7144void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
f6796cae 7145{
a0f38609
AD		 7146 u32 count = 0, i;
7147 const struct cs_section_def *sect = NULL;
7148 const struct cs_extent_def *ext = NULL;
f6796cae 7149
a0f38609
AD		 7150 if (rdev->rlc.cs_data == NULL)
7151 return;
7152 if (buffer == NULL)
7153 return;
f6796cae 7154
6ba81e53
AD		 7155 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7156 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
a0f38609 7157
6ba81e53
AD		 7158 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
7159 buffer[count++] = cpu_to_le32(0x80000000);
7160 buffer[count++] = cpu_to_le32(0x80000000);
a0f38609
AD		 7161
7162 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
7163 for (ext = sect->section; ext->extent != NULL; ++ext) {
7164 if (sect->id == SECT_CONTEXT) {
6ba81e53
AD		 7165 buffer[count++] =
7166 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
7167 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
a0f38609 7168 for (i = 0; i < ext->reg_count; i++)
6ba81e53 7169 buffer[count++] = cpu_to_le32(ext->extent[i]);
a0f38609
AD		 7170 } else {
7171 return;
7172 }
7173 }
7174 }
f6796cae 7175
6ba81e53
AD		 7176 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
7177 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
f6796cae
AD		 7178 switch (rdev->family) {
7179 case CHIP_BONAIRE:
6ba81e53
AD		 7180 buffer[count++] = cpu_to_le32(0x16000012);
7181 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 7182 break;
7183 case CHIP_KAVERI:
6ba81e53
AD		 7184 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7185 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 7186 break;
7187 case CHIP_KABINI:
f73a9e83 7188 case CHIP_MULLINS:
6ba81e53
AD		 7189 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7190 buffer[count++] = cpu_to_le32(0x00000000);
a0f38609 7191 break;
bbfe90bd 7192 case CHIP_HAWAII:
a8947f57
AD		 7193 buffer[count++] = cpu_to_le32(0x3a00161a);
7194 buffer[count++] = cpu_to_le32(0x0000002e);
bbfe90bd 7195 break;
a0f38609 7196 default:
6ba81e53
AD		 7197 buffer[count++] = cpu_to_le32(0x00000000);
7198 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 7199 break;
7200 }
7201
6ba81e53
AD		 7202 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7203 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
f6796cae 7204
6ba81e53
AD		 7205 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
7206 buffer[count++] = cpu_to_le32(0);
a0f38609 7207}
f6796cae 7208
473359bc 7209static void cik_init_pg(struct radeon_device *rdev)
22c775ce 7210{
473359bc 7211 if (rdev->pg_flags) {
22c775ce
AD		 7212 cik_enable_sck_slowdown_on_pu(rdev, true);
7213 cik_enable_sck_slowdown_on_pd(rdev, true);
2b19d17f 7214 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 7215 cik_init_gfx_cgpg(rdev);
7216 cik_enable_cp_pg(rdev, true);
7217 cik_enable_gds_pg(rdev, true);
7218 }
22c775ce
AD		 7219 cik_init_ao_cu_mask(rdev);
7220 cik_update_gfx_pg(rdev, true);
7221 }
7222}
f6796cae 7223
473359bc
AD		 7224static void cik_fini_pg(struct radeon_device *rdev)
7225{
7226 if (rdev->pg_flags) {
7227 cik_update_gfx_pg(rdev, false);
2b19d17f 7228 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 7229 cik_enable_cp_pg(rdev, false);
7230 cik_enable_gds_pg(rdev, false);
7231 }
7232 }
f6796cae 7233}
a59781bb
AD		 7234
7235/*
7236 * Interrupts
7237 * Starting with r6xx, interrupts are handled via a ring buffer.
7238 * Ring buffers are areas of GPU accessible memory that the GPU
7239 * writes interrupt vectors into and the host reads vectors out of.
7240 * There is a rptr (read pointer) that determines where the
7241 * host is currently reading, and a wptr (write pointer)
7242 * which determines where the GPU has written. When the
7243 * pointers are equal, the ring is idle. When the GPU
7244 * writes vectors to the ring buffer, it increments the
7245 * wptr. When there is an interrupt, the host then starts
7246 * fetching commands and processing them until the pointers are
7247 * equal again at which point it updates the rptr.
7248 */
7249
7250/**
7251 * cik_enable_interrupts - Enable the interrupt ring buffer
7252 *
7253 * @rdev: radeon_device pointer
7254 *
7255 * Enable the interrupt ring buffer (CIK).
7256 */
7257static void cik_enable_interrupts(struct radeon_device *rdev)
7258{
7259 u32 ih_cntl = RREG32(IH_CNTL);
7260 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7261
7262 ih_cntl |= ENABLE_INTR;
7263 ih_rb_cntl |= IH_RB_ENABLE;
7264 WREG32(IH_CNTL, ih_cntl);
7265 WREG32(IH_RB_CNTL, ih_rb_cntl);
7266 rdev->ih.enabled = true;
7267}
7268
7269/**
7270 * cik_disable_interrupts - Disable the interrupt ring buffer
7271 *
7272 * @rdev: radeon_device pointer
7273 *
7274 * Disable the interrupt ring buffer (CIK).
7275 */
7276static void cik_disable_interrupts(struct radeon_device *rdev)
7277{
7278 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7279 u32 ih_cntl = RREG32(IH_CNTL);
7280
7281 ih_rb_cntl &= ~IH_RB_ENABLE;
7282 ih_cntl &= ~ENABLE_INTR;
7283 WREG32(IH_RB_CNTL, ih_rb_cntl);
7284 WREG32(IH_CNTL, ih_cntl);
7285 /* set rptr, wptr to 0 */
7286 WREG32(IH_RB_RPTR, 0);
7287 WREG32(IH_RB_WPTR, 0);
7288 rdev->ih.enabled = false;
7289 rdev->ih.rptr = 0;
7290}
7291
7292/**
7293 * cik_disable_interrupt_state - Disable all interrupt sources
7294 *
7295 * @rdev: radeon_device pointer
7296 *
7297 * Clear all interrupt enable bits used by the driver (CIK).
7298 */
7299static void cik_disable_interrupt_state(struct radeon_device *rdev)
7300{
7301 u32 tmp;
7302
7303 /* gfx ring */
4214faf6
AD		 7304 tmp = RREG32(CP_INT_CNTL_RING0) &
7305 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7306 WREG32(CP_INT_CNTL_RING0, tmp);
21a93e13
AD		 7307 /* sdma */
7308 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7309 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
7310 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7311 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
a59781bb
AD		 7312 /* compute queues */
7313 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
7314 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
7315 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
7316 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
7317 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
7318 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
7319 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
7320 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
7321 /* grbm */
7322 WREG32(GRBM_INT_CNTL, 0);
dc12a3ec
LL		 7323 /* SRBM */
7324 WREG32(SRBM_INT_CNTL, 0);
a59781bb
AD		 7325 /* vline/vblank, etc. */
7326 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7327 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7328 if (rdev->num_crtc >= 4) {
7329 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7330 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7331 }
7332 if (rdev->num_crtc >= 6) {
7333 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7334 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7335 }
f5d636d2
CK		 7336 /* pflip */
7337 if (rdev->num_crtc >= 2) {
7338 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7339 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7340 }
7341 if (rdev->num_crtc >= 4) {
7342 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7343 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7344 }
7345 if (rdev->num_crtc >= 6) {
7346 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7347 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7348 }
a59781bb
AD		 7349
7350 /* dac hotplug */
7351 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
7352
7353 /* digital hotplug */
7354 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7355 WREG32(DC_HPD1_INT_CONTROL, tmp);
7356 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7357 WREG32(DC_HPD2_INT_CONTROL, tmp);
7358 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7359 WREG32(DC_HPD3_INT_CONTROL, tmp);
7360 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7361 WREG32(DC_HPD4_INT_CONTROL, tmp);
7362 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7363 WREG32(DC_HPD5_INT_CONTROL, tmp);
7364 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7365 WREG32(DC_HPD6_INT_CONTROL, tmp);
7366
7367}
7368
7369/**
7370 * cik_irq_init - init and enable the interrupt ring
7371 *
7372 * @rdev: radeon_device pointer
7373 *
7374 * Allocate a ring buffer for the interrupt controller,
7375 * enable the RLC, disable interrupts, enable the IH
7376 * ring buffer and enable it (CIK).
7377 * Called at device load and reume.
7378 * Returns 0 for success, errors for failure.
7379 */
7380static int cik_irq_init(struct radeon_device *rdev)
7381{
7382 int ret = 0;
7383 int rb_bufsz;
7384 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
7385
7386 /* allocate ring */
7387 ret = r600_ih_ring_alloc(rdev);
7388 if (ret)
7389 return ret;
7390
7391 /* disable irqs */
7392 cik_disable_interrupts(rdev);
7393
7394 /* init rlc */
7395 ret = cik_rlc_resume(rdev);
7396 if (ret) {
7397 r600_ih_ring_fini(rdev);
7398 return ret;
7399 }
7400
7401 /* setup interrupt control */
7402 /* XXX this should actually be a bus address, not an MC address. same on older asics */
7403 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7404 interrupt_cntl = RREG32(INTERRUPT_CNTL);
7405 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7406 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7407 */
7408 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7409 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7410 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7411 WREG32(INTERRUPT_CNTL, interrupt_cntl);
7412
7413 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 7414 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
a59781bb
AD		 7415
7416 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7417 IH_WPTR_OVERFLOW_CLEAR |
7418 (rb_bufsz << 1));
7419
7420 if (rdev->wb.enabled)
7421 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7422
7423 /* set the writeback address whether it's enabled or not */
7424 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7425 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7426
7427 WREG32(IH_RB_CNTL, ih_rb_cntl);
7428
7429 /* set rptr, wptr to 0 */
7430 WREG32(IH_RB_RPTR, 0);
7431 WREG32(IH_RB_WPTR, 0);
7432
7433 /* Default settings for IH_CNTL (disabled at first) */
7434 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7435 /* RPTR_REARM only works if msi's are enabled */
7436 if (rdev->msi_enabled)
7437 ih_cntl |= RPTR_REARM;
7438 WREG32(IH_CNTL, ih_cntl);
7439
7440 /* force the active interrupt state to all disabled */
7441 cik_disable_interrupt_state(rdev);
7442
7443 pci_set_master(rdev->pdev);
7444
7445 /* enable irqs */
7446 cik_enable_interrupts(rdev);
7447
7448 return ret;
7449}
7450
7451/**
7452 * cik_irq_set - enable/disable interrupt sources
7453 *
7454 * @rdev: radeon_device pointer
7455 *
7456 * Enable interrupt sources on the GPU (vblanks, hpd,
7457 * etc.) (CIK).
7458 * Returns 0 for success, errors for failure.
7459 */
7460int cik_irq_set(struct radeon_device *rdev)
7461{
4214faf6 7462 u32 cp_int_cntl;
28b57b85 7463 u32 cp_m1p0;
a59781bb
AD		 7464 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7465 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7466 u32 grbm_int_cntl = 0;
21a93e13 7467 u32 dma_cntl, dma_cntl1;
a59781bb
AD		 7468
7469 if (!rdev->irq.installed) {
7470 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7471 return -EINVAL;
7472 }
7473 /* don't enable anything if the ih is disabled */
7474 if (!rdev->ih.enabled) {
7475 cik_disable_interrupts(rdev);
7476 /* force the active interrupt state to all disabled */
7477 cik_disable_interrupt_state(rdev);
7478 return 0;
7479 }
7480
4214faf6
AD		 7481 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7482 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7483 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7484
f6b355dd
AD		 7485 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7486 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7487 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7488 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7489 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7490 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
a59781bb 7491
21a93e13
AD		 7492 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7493 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7494
2b0781a6 7495 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
2b0781a6 7496
a59781bb
AD		 7497 /* enable CP interrupts on all rings */
7498 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7499 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7500 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7501 }
2b0781a6
AD		 7502 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7503 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7504 DRM_DEBUG("si_irq_set: sw int cp1\n");
7505 if (ring->me == 1) {
7506 switch (ring->pipe) {
7507 case 0:
7508 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7509 break;
2b0781a6
AD		 7510 default:
7511 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7512 break;
7513 }
7514 } else {
7515 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7516 }
7517 }
7518 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7519 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7520 DRM_DEBUG("si_irq_set: sw int cp2\n");
7521 if (ring->me == 1) {
7522 switch (ring->pipe) {
7523 case 0:
7524 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7525 break;
2b0781a6
AD		 7526 default:
7527 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7528 break;
7529 }
7530 } else {
7531 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7532 }
7533 }
a59781bb 7534
21a93e13
AD		 7535 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7536 DRM_DEBUG("cik_irq_set: sw int dma\n");
7537 dma_cntl |= TRAP_ENABLE;
7538 }
7539
7540 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7541 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7542 dma_cntl1 |= TRAP_ENABLE;
7543 }
7544
a59781bb
AD		 7545 if (rdev->irq.crtc_vblank_int[0] ||
7546 atomic_read(&rdev->irq.pflip[0])) {
7547 DRM_DEBUG("cik_irq_set: vblank 0\n");
7548 crtc1 |= VBLANK_INTERRUPT_MASK;
7549 }
7550 if (rdev->irq.crtc_vblank_int[1] ||
7551 atomic_read(&rdev->irq.pflip[1])) {
7552 DRM_DEBUG("cik_irq_set: vblank 1\n");
7553 crtc2 |= VBLANK_INTERRUPT_MASK;
7554 }
7555 if (rdev->irq.crtc_vblank_int[2] ||
7556 atomic_read(&rdev->irq.pflip[2])) {
7557 DRM_DEBUG("cik_irq_set: vblank 2\n");
7558 crtc3 |= VBLANK_INTERRUPT_MASK;
7559 }
7560 if (rdev->irq.crtc_vblank_int[3] ||
7561 atomic_read(&rdev->irq.pflip[3])) {
7562 DRM_DEBUG("cik_irq_set: vblank 3\n");
7563 crtc4 |= VBLANK_INTERRUPT_MASK;
7564 }
7565 if (rdev->irq.crtc_vblank_int[4] ||
7566 atomic_read(&rdev->irq.pflip[4])) {
7567 DRM_DEBUG("cik_irq_set: vblank 4\n");
7568 crtc5 |= VBLANK_INTERRUPT_MASK;
7569 }
7570 if (rdev->irq.crtc_vblank_int[5] ||
7571 atomic_read(&rdev->irq.pflip[5])) {
7572 DRM_DEBUG("cik_irq_set: vblank 5\n");
7573 crtc6 |= VBLANK_INTERRUPT_MASK;
7574 }
7575 if (rdev->irq.hpd[0]) {
7576 DRM_DEBUG("cik_irq_set: hpd 1\n");
f6b355dd 7577 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7578 }
7579 if (rdev->irq.hpd[1]) {
7580 DRM_DEBUG("cik_irq_set: hpd 2\n");
f6b355dd 7581 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7582 }
7583 if (rdev->irq.hpd[2]) {
7584 DRM_DEBUG("cik_irq_set: hpd 3\n");
f6b355dd 7585 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7586 }
7587 if (rdev->irq.hpd[3]) {
7588 DRM_DEBUG("cik_irq_set: hpd 4\n");
f6b355dd 7589 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7590 }
7591 if (rdev->irq.hpd[4]) {
7592 DRM_DEBUG("cik_irq_set: hpd 5\n");
f6b355dd 7593 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7594 }
7595 if (rdev->irq.hpd[5]) {
7596 DRM_DEBUG("cik_irq_set: hpd 6\n");
f6b355dd 7597 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7598 }
7599
7600 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7601
21a93e13
AD		 7602 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7603 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7604
2b0781a6 7605 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
2b0781a6 7606
a59781bb
AD		 7607 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7608
7609 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7610 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7611 if (rdev->num_crtc >= 4) {
7612 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7613 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7614 }
7615 if (rdev->num_crtc >= 6) {
7616 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7617 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7618 }
7619
f5d636d2
CK		 7620 if (rdev->num_crtc >= 2) {
7621 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7622 GRPH_PFLIP_INT_MASK);
7623 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7624 GRPH_PFLIP_INT_MASK);
7625 }
7626 if (rdev->num_crtc >= 4) {
7627 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7628 GRPH_PFLIP_INT_MASK);
7629 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7630 GRPH_PFLIP_INT_MASK);
7631 }
7632 if (rdev->num_crtc >= 6) {
7633 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7634 GRPH_PFLIP_INT_MASK);
7635 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7636 GRPH_PFLIP_INT_MASK);
7637 }
7638
a59781bb
AD		 7639 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7640 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7641 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7642 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7643 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7644 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7645
cffefd9b
AD		 7646 /* posting read */
7647 RREG32(SRBM_STATUS);
7648
a59781bb
AD		 7649 return 0;
7650}
7651
7652/**
7653 * cik_irq_ack - ack interrupt sources
7654 *
7655 * @rdev: radeon_device pointer
7656 *
7657 * Ack interrupt sources on the GPU (vblanks, hpd,
7658 * etc.) (CIK). Certain interrupts sources are sw
7659 * generated and do not require an explicit ack.
7660 */
7661static inline void cik_irq_ack(struct radeon_device *rdev)
7662{
7663 u32 tmp;
7664
7665 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7666 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7667 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7668 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7669 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7670 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7671 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7672
f5d636d2
CK		 7673 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7674 EVERGREEN_CRTC0_REGISTER_OFFSET);
7675 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7676 EVERGREEN_CRTC1_REGISTER_OFFSET);
7677 if (rdev->num_crtc >= 4) {
7678 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7679 EVERGREEN_CRTC2_REGISTER_OFFSET);
7680 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7681 EVERGREEN_CRTC3_REGISTER_OFFSET);
7682 }
7683 if (rdev->num_crtc >= 6) {
7684 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7685 EVERGREEN_CRTC4_REGISTER_OFFSET);
7686 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7687 EVERGREEN_CRTC5_REGISTER_OFFSET);
7688 }
7689
7690 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7691 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7692 GRPH_PFLIP_INT_CLEAR);
7693 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7694 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7695 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7696 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7697 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7698 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7699 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7700 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7701 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7702 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7703 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7704
7705 if (rdev->num_crtc >= 4) {
f5d636d2
CK		 7706 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7707 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7708 GRPH_PFLIP_INT_CLEAR);
7709 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7710 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7711 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7712 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7713 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7714 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7715 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7716 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7717 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7718 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7719 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7720 }
7721
7722 if (rdev->num_crtc >= 6) {
f5d636d2
CK		 7723 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7724 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7725 GRPH_PFLIP_INT_CLEAR);
7726 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7727 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7728 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7729 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7730 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7731 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7732 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7733 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7734 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7735 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7736 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7737 }
7738
7739 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7740 tmp = RREG32(DC_HPD1_INT_CONTROL);
7741 tmp |= DC_HPDx_INT_ACK;
7742 WREG32(DC_HPD1_INT_CONTROL, tmp);
7743 }
7744 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7745 tmp = RREG32(DC_HPD2_INT_CONTROL);
7746 tmp |= DC_HPDx_INT_ACK;
7747 WREG32(DC_HPD2_INT_CONTROL, tmp);
7748 }
7749 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7750 tmp = RREG32(DC_HPD3_INT_CONTROL);
7751 tmp |= DC_HPDx_INT_ACK;
7752 WREG32(DC_HPD3_INT_CONTROL, tmp);
7753 }
7754 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7755 tmp = RREG32(DC_HPD4_INT_CONTROL);
7756 tmp |= DC_HPDx_INT_ACK;
7757 WREG32(DC_HPD4_INT_CONTROL, tmp);
7758 }
7759 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7760 tmp = RREG32(DC_HPD5_INT_CONTROL);
7761 tmp |= DC_HPDx_INT_ACK;
7762 WREG32(DC_HPD5_INT_CONTROL, tmp);
7763 }
7764 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7765 tmp = RREG32(DC_HPD5_INT_CONTROL);
7766 tmp |= DC_HPDx_INT_ACK;
7767 WREG32(DC_HPD6_INT_CONTROL, tmp);
7768 }
f6b355dd
AD		 7769 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT) {
7770 tmp = RREG32(DC_HPD1_INT_CONTROL);
7771 tmp |= DC_HPDx_RX_INT_ACK;
7772 WREG32(DC_HPD1_INT_CONTROL, tmp);
7773 }
7774 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
7775 tmp = RREG32(DC_HPD2_INT_CONTROL);
7776 tmp |= DC_HPDx_RX_INT_ACK;
7777 WREG32(DC_HPD2_INT_CONTROL, tmp);
7778 }
7779 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
7780 tmp = RREG32(DC_HPD3_INT_CONTROL);
7781 tmp |= DC_HPDx_RX_INT_ACK;
7782 WREG32(DC_HPD3_INT_CONTROL, tmp);
7783 }
7784 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
7785 tmp = RREG32(DC_HPD4_INT_CONTROL);
7786 tmp |= DC_HPDx_RX_INT_ACK;
7787 WREG32(DC_HPD4_INT_CONTROL, tmp);
7788 }
7789 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
7790 tmp = RREG32(DC_HPD5_INT_CONTROL);
7791 tmp |= DC_HPDx_RX_INT_ACK;
7792 WREG32(DC_HPD5_INT_CONTROL, tmp);
7793 }
7794 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
7795 tmp = RREG32(DC_HPD5_INT_CONTROL);
7796 tmp |= DC_HPDx_RX_INT_ACK;
7797 WREG32(DC_HPD6_INT_CONTROL, tmp);
7798 }
a59781bb
AD		 7799}
7800
7801/**
7802 * cik_irq_disable - disable interrupts
7803 *
7804 * @rdev: radeon_device pointer
7805 *
7806 * Disable interrupts on the hw (CIK).
7807 */
7808static void cik_irq_disable(struct radeon_device *rdev)
7809{
7810 cik_disable_interrupts(rdev);
7811 /* Wait and acknowledge irq */
7812 mdelay(1);
7813 cik_irq_ack(rdev);
7814 cik_disable_interrupt_state(rdev);
7815}
7816
7817/**
7818 * cik_irq_disable - disable interrupts for suspend
7819 *
7820 * @rdev: radeon_device pointer
7821 *
7822 * Disable interrupts and stop the RLC (CIK).
7823 * Used for suspend.
7824 */
7825static void cik_irq_suspend(struct radeon_device *rdev)
7826{
7827 cik_irq_disable(rdev);
7828 cik_rlc_stop(rdev);
7829}
7830
7831/**
7832 * cik_irq_fini - tear down interrupt support
7833 *
7834 * @rdev: radeon_device pointer
7835 *
7836 * Disable interrupts on the hw and free the IH ring
7837 * buffer (CIK).
7838 * Used for driver unload.
7839 */
7840static void cik_irq_fini(struct radeon_device *rdev)
7841{
7842 cik_irq_suspend(rdev);
7843 r600_ih_ring_fini(rdev);
7844}
7845
7846/**
7847 * cik_get_ih_wptr - get the IH ring buffer wptr
7848 *
7849 * @rdev: radeon_device pointer
7850 *
7851 * Get the IH ring buffer wptr from either the register
7852 * or the writeback memory buffer (CIK). Also check for
7853 * ring buffer overflow and deal with it.
7854 * Used by cik_irq_process().
7855 * Returns the value of the wptr.
7856 */
7857static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7858{
7859 u32 wptr, tmp;
7860
7861 if (rdev->wb.enabled)
7862 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7863 else
7864 wptr = RREG32(IH_RB_WPTR);
7865
7866 if (wptr & RB_OVERFLOW) {
11bab0ae 7867 wptr &= ~RB_OVERFLOW;
a59781bb
AD		 7868 /* When a ring buffer overflow happen start parsing interrupt
7869 * from the last not overwritten vector (wptr + 16). Hopefully
7870 * this should allow us to catchup.
7871 */
6cc2fda2
MD		 7872 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
7873 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
a59781bb
AD		 7874 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7875 tmp = RREG32(IH_RB_CNTL);
7876 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7877 WREG32(IH_RB_CNTL, tmp);
7878 }
7879 return (wptr & rdev->ih.ptr_mask);
7880}
7881
7882/* CIK IV Ring
7883 * Each IV ring entry is 128 bits:
7884 * [7:0] - interrupt source id
7885 * [31:8] - reserved
7886 * [59:32] - interrupt source data
7887 * [63:60] - reserved
21a93e13
AD		 7888 * [71:64] - RINGID
7889 * CP:
7890 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
a59781bb
AD		 7891 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7892 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7893 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7894 * PIPE_ID - ME0 0=3D
7895 * - ME1&2 compute dispatcher (4 pipes each)
21a93e13
AD		 7896 * SDMA:
7897 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7898 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7899 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
a59781bb
AD		 7900 * [79:72] - VMID
7901 * [95:80] - PASID
7902 * [127:96] - reserved
7903 */
7904/**
7905 * cik_irq_process - interrupt handler
7906 *
7907 * @rdev: radeon_device pointer
7908 *
7909 * Interrupt hander (CIK). Walk the IH ring,
7910 * ack interrupts and schedule work to handle
7911 * interrupt events.
7912 * Returns irq process return code.
7913 */
7914int cik_irq_process(struct radeon_device *rdev)
7915{
2b0781a6
AD		 7916 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7917 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
a59781bb
AD		 7918 u32 wptr;
7919 u32 rptr;
7920 u32 src_id, src_data, ring_id;
7921 u8 me_id, pipe_id, queue_id;
7922 u32 ring_index;
7923 bool queue_hotplug = false;
f6b355dd 7924 bool queue_dp = false;
a59781bb 7925 bool queue_reset = false;
3ec7d11b 7926 u32 addr, status, mc_client;
41a524ab 7927 bool queue_thermal = false;
a59781bb
AD		 7928
7929 if (!rdev->ih.enabled || rdev->shutdown)
7930 return IRQ_NONE;
7931
7932 wptr = cik_get_ih_wptr(rdev);
7933
7934restart_ih:
7935 /* is somebody else already processing irqs? */
7936 if (atomic_xchg(&rdev->ih.lock, 1))
7937 return IRQ_NONE;
7938
7939 rptr = rdev->ih.rptr;
7940 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7941
7942 /* Order reading of wptr vs. reading of IH ring data */
7943 rmb();
7944
7945 /* display interrupts */
7946 cik_irq_ack(rdev);
7947
7948 while (rptr != wptr) {
7949 /* wptr/rptr are in bytes! */
7950 ring_index = rptr / 4;
e28740ec
OG		 7951
7952 radeon_kfd_interrupt(rdev,
7953 (const void *) &rdev->ih.ring[ring_index]);
7954
a59781bb
AD		 7955 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7956 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7957 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
a59781bb
AD		 7958
7959 switch (src_id) {
7960 case 1: /* D1 vblank/vline */
7961 switch (src_data) {
7962 case 0: /* D1 vblank */
07f18f0b
MK		 7963 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT))
7964 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7965
7966 if (rdev->irq.crtc_vblank_int[0]) {
7967 drm_handle_vblank(rdev->ddev, 0);
7968 rdev->pm.vblank_sync = true;
7969 wake_up(&rdev->irq.vblank_queue);
a59781bb 7970 }
07f18f0b
MK		 7971 if (atomic_read(&rdev->irq.pflip[0]))
7972 radeon_crtc_handle_vblank(rdev, 0);
7973 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7974 DRM_DEBUG("IH: D1 vblank\n");
7975
a59781bb
AD		 7976 break;
7977 case 1: /* D1 vline */
07f18f0b
MK		 7978 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT))
7979 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7980
7981 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7982 DRM_DEBUG("IH: D1 vline\n");
7983
a59781bb
AD		 7984 break;
7985 default:
7986 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7987 break;
7988 }
7989 break;
7990 case 2: /* D2 vblank/vline */
7991 switch (src_data) {
7992 case 0: /* D2 vblank */
07f18f0b
MK		 7993 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
7994 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7995
7996 if (rdev->irq.crtc_vblank_int[1]) {
7997 drm_handle_vblank(rdev->ddev, 1);
7998 rdev->pm.vblank_sync = true;
7999 wake_up(&rdev->irq.vblank_queue);
a59781bb 8000 }
07f18f0b
MK		 8001 if (atomic_read(&rdev->irq.pflip[1]))
8002 radeon_crtc_handle_vblank(rdev, 1);
8003 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
8004 DRM_DEBUG("IH: D2 vblank\n");
8005
a59781bb
AD		 8006 break;
8007 case 1: /* D2 vline */
07f18f0b
MK		 8008 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT))
8009 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8010
8011 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
8012 DRM_DEBUG("IH: D2 vline\n");
8013
a59781bb
AD		 8014 break;
8015 default:
8016 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8017 break;
8018 }
8019 break;
8020 case 3: /* D3 vblank/vline */
8021 switch (src_data) {
8022 case 0: /* D3 vblank */
07f18f0b
MK		 8023 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
8024 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8025
8026 if (rdev->irq.crtc_vblank_int[2]) {
8027 drm_handle_vblank(rdev->ddev, 2);
8028 rdev->pm.vblank_sync = true;
8029 wake_up(&rdev->irq.vblank_queue);
a59781bb 8030 }
07f18f0b
MK		 8031 if (atomic_read(&rdev->irq.pflip[2]))
8032 radeon_crtc_handle_vblank(rdev, 2);
8033 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
8034 DRM_DEBUG("IH: D3 vblank\n");
8035
a59781bb
AD		 8036 break;
8037 case 1: /* D3 vline */
07f18f0b
MK		 8038 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
8039 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8040
8041 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
8042 DRM_DEBUG("IH: D3 vline\n");
8043
a59781bb
AD		 8044 break;
8045 default:
8046 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8047 break;
8048 }
8049 break;
8050 case 4: /* D4 vblank/vline */
8051 switch (src_data) {
8052 case 0: /* D4 vblank */
07f18f0b
MK		 8053 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
8054 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8055
8056 if (rdev->irq.crtc_vblank_int[3]) {
8057 drm_handle_vblank(rdev->ddev, 3);
8058 rdev->pm.vblank_sync = true;
8059 wake_up(&rdev->irq.vblank_queue);
a59781bb 8060 }
07f18f0b
MK		 8061 if (atomic_read(&rdev->irq.pflip[3]))
8062 radeon_crtc_handle_vblank(rdev, 3);
8063 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
8064 DRM_DEBUG("IH: D4 vblank\n");
8065
a59781bb
AD		 8066 break;
8067 case 1: /* D4 vline */
07f18f0b
MK		 8068 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
8069 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8070
8071 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
8072 DRM_DEBUG("IH: D4 vline\n");
8073
a59781bb
AD		 8074 break;
8075 default:
8076 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8077 break;
8078 }
8079 break;
8080 case 5: /* D5 vblank/vline */
8081 switch (src_data) {
8082 case 0: /* D5 vblank */
07f18f0b
MK		 8083 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
8084 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8085
8086 if (rdev->irq.crtc_vblank_int[4]) {
8087 drm_handle_vblank(rdev->ddev, 4);
8088 rdev->pm.vblank_sync = true;
8089 wake_up(&rdev->irq.vblank_queue);
a59781bb 8090 }
07f18f0b
MK		 8091 if (atomic_read(&rdev->irq.pflip[4]))
8092 radeon_crtc_handle_vblank(rdev, 4);
8093 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
8094 DRM_DEBUG("IH: D5 vblank\n");
8095
a59781bb
AD		 8096 break;
8097 case 1: /* D5 vline */
07f18f0b
MK		 8098 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
8099 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8100
8101 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
8102 DRM_DEBUG("IH: D5 vline\n");
8103
a59781bb
AD		 8104 break;
8105 default:
8106 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8107 break;
8108 }
8109 break;
8110 case 6: /* D6 vblank/vline */
8111 switch (src_data) {
8112 case 0: /* D6 vblank */
07f18f0b
MK		 8113 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
8114 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8115
8116 if (rdev->irq.crtc_vblank_int[5]) {
8117 drm_handle_vblank(rdev->ddev, 5);
8118 rdev->pm.vblank_sync = true;
8119 wake_up(&rdev->irq.vblank_queue);
a59781bb 8120 }
07f18f0b
MK		 8121 if (atomic_read(&rdev->irq.pflip[5]))
8122 radeon_crtc_handle_vblank(rdev, 5);
8123 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
8124 DRM_DEBUG("IH: D6 vblank\n");
8125
a59781bb
AD		 8126 break;
8127 case 1: /* D6 vline */
07f18f0b
MK		 8128 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
8129 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8130
8131 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
8132 DRM_DEBUG("IH: D6 vline\n");
8133
a59781bb
AD		 8134 break;
8135 default:
8136 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8137 break;
8138 }
8139 break;
f5d636d2
CK		 8140 case 8: /* D1 page flip */
8141 case 10: /* D2 page flip */
8142 case 12: /* D3 page flip */
8143 case 14: /* D4 page flip */
8144 case 16: /* D5 page flip */
8145 case 18: /* D6 page flip */
8146 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
39dc5454
MK		 8147 if (radeon_use_pflipirq > 0)
8148 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
f5d636d2 8149 break;
a59781bb
AD		 8150 case 42: /* HPD hotplug */
8151 switch (src_data) {
8152 case 0:
07f18f0b
MK		 8153 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT))
8154 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8155
8156 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
8157 queue_hotplug = true;
8158 DRM_DEBUG("IH: HPD1\n");
8159
a59781bb
AD		 8160 break;
8161 case 1:
07f18f0b
MK		 8162 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT))
8163 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8164
8165 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
8166 queue_hotplug = true;
8167 DRM_DEBUG("IH: HPD2\n");
8168
a59781bb
AD		 8169 break;
8170 case 2:
07f18f0b
MK		 8171 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT))
8172 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8173
8174 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
8175 queue_hotplug = true;
8176 DRM_DEBUG("IH: HPD3\n");
8177
a59781bb
AD		 8178 break;
8179 case 3:
07f18f0b
MK		 8180 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT))
8181 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8182
8183 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
8184 queue_hotplug = true;
8185 DRM_DEBUG("IH: HPD4\n");
8186
a59781bb
AD		 8187 break;
8188 case 4:
07f18f0b
MK		 8189 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT))
8190 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8191
8192 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
8193 queue_hotplug = true;
8194 DRM_DEBUG("IH: HPD5\n");
8195
a59781bb
AD		 8196 break;
8197 case 5:
07f18f0b
MK		 8198 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT))
8199 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8200
8201 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
8202 queue_hotplug = true;
8203 DRM_DEBUG("IH: HPD6\n");
8204
a59781bb 8205 break;
f6b355dd 8206 case 6:
07f18f0b
MK		 8207 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT))
8208 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8209
8210 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_RX_INTERRUPT;
8211 queue_dp = true;
8212 DRM_DEBUG("IH: HPD_RX 1\n");
8213
f6b355dd
AD		 8214 break;
8215 case 7:
07f18f0b
MK		 8216 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT))
8217 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8218
8219 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
8220 queue_dp = true;
8221 DRM_DEBUG("IH: HPD_RX 2\n");
8222
f6b355dd
AD		 8223 break;
8224 case 8:
07f18f0b
MK		 8225 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
8226 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8227
8228 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
8229 queue_dp = true;
8230 DRM_DEBUG("IH: HPD_RX 3\n");
8231
f6b355dd
AD		 8232 break;
8233 case 9:
07f18f0b
MK		 8234 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
8235 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8236
8237 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
8238 queue_dp = true;
8239 DRM_DEBUG("IH: HPD_RX 4\n");
8240
f6b355dd
AD		 8241 break;
8242 case 10:
07f18f0b
MK		 8243 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
8244 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8245
8246 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
8247 queue_dp = true;
8248 DRM_DEBUG("IH: HPD_RX 5\n");
8249
f6b355dd
AD		 8250 break;
8251 case 11:
07f18f0b
MK		 8252 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
8253 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8254
8255 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
8256 queue_dp = true;
8257 DRM_DEBUG("IH: HPD_RX 6\n");
8258
f6b355dd 8259 break;
a59781bb
AD		 8260 default:
8261 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8262 break;
8263 }
8264 break;
dc12a3ec
LL		 8265 case 96:
8266 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
8267 WREG32(SRBM_INT_ACK, 0x1);
8268 break;
6a3808b8
CK		 8269 case 124: /* UVD */
8270 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
8271 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
8272 break;
9d97c99b
AD		 8273 case 146:
8274 case 147:
3ec7d11b
AD		 8275 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
8276 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
8277 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
9b7d786b
CK		 8278 /* reset addr and status */
8279 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
8280 if (addr == 0x0 && status == 0x0)
8281 break;
9d97c99b
AD		 8282 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
8283 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3ec7d11b 8284 addr);
9d97c99b 8285 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3ec7d11b
AD		 8286 status);
8287 cik_vm_decode_fault(rdev, status, addr, mc_client);
9d97c99b 8288 break;
d93f7937
CK		 8289 case 167: /* VCE */
8290 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
8291 switch (src_data) {
8292 case 0:
8293 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
8294 break;
8295 case 1:
8296 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
8297 break;
8298 default:
8299 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
8300 break;
8301 }
8302 break;
a59781bb
AD		 8303 case 176: /* GFX RB CP_INT */
8304 case 177: /* GFX IB CP_INT */
8305 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8306 break;
8307 case 181: /* CP EOP event */
8308 DRM_DEBUG("IH: CP EOP\n");
21a93e13
AD		 8309 /* XXX check the bitfield order! */
8310 me_id = (ring_id & 0x60) >> 5;
8311 pipe_id = (ring_id & 0x18) >> 3;
8312 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 8313 switch (me_id) {
8314 case 0:
8315 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8316 break;
8317 case 1:
a59781bb 8318 case 2:
2b0781a6
AD		 8319 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
8320 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8321 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
8322 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
a59781bb
AD		 8323 break;
8324 }
8325 break;
8326 case 184: /* CP Privileged reg access */
8327 DRM_ERROR("Illegal register access in command stream\n");
8328 /* XXX check the bitfield order! */
8329 me_id = (ring_id & 0x60) >> 5;
8330 pipe_id = (ring_id & 0x18) >> 3;
8331 queue_id = (ring_id & 0x7) >> 0;
8332 switch (me_id) {
8333 case 0:
8334 /* This results in a full GPU reset, but all we need to do is soft
8335 * reset the CP for gfx
8336 */
8337 queue_reset = true;
8338 break;
8339 case 1:
8340 /* XXX compute */
2b0781a6 8341 queue_reset = true;
a59781bb
AD		 8342 break;
8343 case 2:
8344 /* XXX compute */
2b0781a6 8345 queue_reset = true;
a59781bb
AD		 8346 break;
8347 }
8348 break;
8349 case 185: /* CP Privileged inst */
8350 DRM_ERROR("Illegal instruction in command stream\n");
21a93e13
AD		 8351 /* XXX check the bitfield order! */
8352 me_id = (ring_id & 0x60) >> 5;
8353 pipe_id = (ring_id & 0x18) >> 3;
8354 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 8355 switch (me_id) {
8356 case 0:
8357 /* This results in a full GPU reset, but all we need to do is soft
8358 * reset the CP for gfx
8359 */
8360 queue_reset = true;
8361 break;
8362 case 1:
8363 /* XXX compute */
2b0781a6 8364 queue_reset = true;
a59781bb
AD		 8365 break;
8366 case 2:
8367 /* XXX compute */
2b0781a6 8368 queue_reset = true;
a59781bb
AD		 8369 break;
8370 }
8371 break;
21a93e13
AD		 8372 case 224: /* SDMA trap event */
8373 /* XXX check the bitfield order! */
8374 me_id = (ring_id & 0x3) >> 0;
8375 queue_id = (ring_id & 0xc) >> 2;
8376 DRM_DEBUG("IH: SDMA trap\n");
8377 switch (me_id) {
8378 case 0:
8379 switch (queue_id) {
8380 case 0:
8381 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8382 break;
8383 case 1:
8384 /* XXX compute */
8385 break;
8386 case 2:
8387 /* XXX compute */
8388 break;
8389 }
8390 break;
8391 case 1:
8392 switch (queue_id) {
8393 case 0:
8394 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8395 break;
8396 case 1:
8397 /* XXX compute */
8398 break;
8399 case 2:
8400 /* XXX compute */
8401 break;
8402 }
8403 break;
8404 }
8405 break;
41a524ab
AD		 8406 case 230: /* thermal low to high */
8407 DRM_DEBUG("IH: thermal low to high\n");
8408 rdev->pm.dpm.thermal.high_to_low = false;
8409 queue_thermal = true;
8410 break;
8411 case 231: /* thermal high to low */
8412 DRM_DEBUG("IH: thermal high to low\n");
8413 rdev->pm.dpm.thermal.high_to_low = true;
8414 queue_thermal = true;
8415 break;
8416 case 233: /* GUI IDLE */
8417 DRM_DEBUG("IH: GUI idle\n");
8418 break;
21a93e13
AD		 8419 case 241: /* SDMA Privileged inst */
8420 case 247: /* SDMA Privileged inst */
8421 DRM_ERROR("Illegal instruction in SDMA command stream\n");
8422 /* XXX check the bitfield order! */
8423 me_id = (ring_id & 0x3) >> 0;
8424 queue_id = (ring_id & 0xc) >> 2;
8425 switch (me_id) {
8426 case 0:
8427 switch (queue_id) {
8428 case 0:
8429 queue_reset = true;
8430 break;
8431 case 1:
8432 /* XXX compute */
8433 queue_reset = true;
8434 break;
8435 case 2:
8436 /* XXX compute */
8437 queue_reset = true;
8438 break;
8439 }
8440 break;
8441 case 1:
8442 switch (queue_id) {
8443 case 0:
8444 queue_reset = true;
8445 break;
8446 case 1:
8447 /* XXX compute */
8448 queue_reset = true;
8449 break;
8450 case 2:
8451 /* XXX compute */
8452 queue_reset = true;
8453 break;
8454 }
8455 break;
8456 }
8457 break;
a59781bb
AD		 8458 default:
8459 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8460 break;
8461 }
8462
8463 /* wptr/rptr are in bytes! */
8464 rptr += 16;
8465 rptr &= rdev->ih.ptr_mask;
f55e03b9 8466 WREG32(IH_RB_RPTR, rptr);
a59781bb 8467 }
f6b355dd
AD		 8468 if (queue_dp)
8469 schedule_work(&rdev->dp_work);
a59781bb 8470 if (queue_hotplug)
cb5d4166 8471 schedule_delayed_work(&rdev->hotplug_work, 0);
3c036389
CK		 8472 if (queue_reset) {
8473 rdev->needs_reset = true;
8474 wake_up_all(&rdev->fence_queue);
8475 }
41a524ab
AD		 8476 if (queue_thermal)
8477 schedule_work(&rdev->pm.dpm.thermal.work);
a59781bb 8478 rdev->ih.rptr = rptr;
a59781bb
AD		 8479 atomic_set(&rdev->ih.lock, 0);
8480
8481 /* make sure wptr hasn't changed while processing */
8482 wptr = cik_get_ih_wptr(rdev);
8483 if (wptr != rptr)
8484 goto restart_ih;
8485
8486 return IRQ_HANDLED;
8487}
7bf94a2c
AD		 8488
8489/*
8490 * startup/shutdown callbacks
8491 */
8492/**
8493 * cik_startup - program the asic to a functional state
8494 *
8495 * @rdev: radeon_device pointer
8496 *
8497 * Programs the asic to a functional state (CIK).
8498 * Called by cik_init() and cik_resume().
8499 * Returns 0 for success, error for failure.
8500 */
8501static int cik_startup(struct radeon_device *rdev)
8502{
8503 struct radeon_ring *ring;
0e16e4cf 8504 u32 nop;
7bf94a2c
AD		 8505 int r;
8506
8a7cd276
AD		 8507 /* enable pcie gen2/3 link */
8508 cik_pcie_gen3_enable(rdev);
7235711a
AD		 8509 /* enable aspm */
8510 cik_program_aspm(rdev);
8a7cd276 8511
e5903d39
AD		 8512 /* scratch needs to be initialized before MC */
8513 r = r600_vram_scratch_init(rdev);
8514 if (r)
8515 return r;
8516
6fab3feb
AD		 8517 cik_mc_program(rdev);
8518
6c7bccea 8519 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7bf94a2c
AD		 8520 r = ci_mc_load_microcode(rdev);
8521 if (r) {
8522 DRM_ERROR("Failed to load MC firmware!\n");
8523 return r;
8524 }
8525 }
8526
7bf94a2c
AD		 8527 r = cik_pcie_gart_enable(rdev);
8528 if (r)
8529 return r;
8530 cik_gpu_init(rdev);
8531
8532 /* allocate rlc buffers */
22c775ce
AD		 8533 if (rdev->flags & RADEON_IS_IGP) {
8534 if (rdev->family == CHIP_KAVERI) {
8535 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8536 rdev->rlc.reg_list_size =
8537 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8538 } else {
8539 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8540 rdev->rlc.reg_list_size =
8541 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8542 }
8543 }
8544 rdev->rlc.cs_data = ci_cs_data;
8545 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
1fd11777 8546 r = sumo_rlc_init(rdev);
7bf94a2c
AD		 8547 if (r) {
8548 DRM_ERROR("Failed to init rlc BOs!\n");
8549 return r;
8550 }
8551
8552 /* allocate wb buffer */
8553 r = radeon_wb_init(rdev);
8554 if (r)
8555 return r;
8556
963e81f9
AD		 8557 /* allocate mec buffers */
8558 r = cik_mec_init(rdev);
8559 if (r) {
8560 DRM_ERROR("Failed to init MEC BOs!\n");
8561 return r;
8562 }
8563
7bf94a2c
AD		 8564 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8565 if (r) {
8566 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8567 return r;
8568 }
8569
963e81f9
AD		 8570 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8571 if (r) {
8572 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8573 return r;
8574 }
8575
8576 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8577 if (r) {
8578 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8579 return r;
8580 }
8581
7bf94a2c
AD		 8582 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8583 if (r) {
8584 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8585 return r;
8586 }
8587
8588 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8589 if (r) {
8590 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8591 return r;
8592 }
8593
2ce529da 8594 r = radeon_uvd_resume(rdev);
87167bb1 8595 if (!r) {
2ce529da
AD		 8596 r = uvd_v4_2_resume(rdev);
8597 if (!r) {
8598 r = radeon_fence_driver_start_ring(rdev,
8599 R600_RING_TYPE_UVD_INDEX);
8600 if (r)
8601 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
8602 }
87167bb1
CK		 8603 }
8604 if (r)
8605 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8606
d93f7937
CK		 8607 r = radeon_vce_resume(rdev);
8608 if (!r) {
8609 r = vce_v2_0_resume(rdev);
8610 if (!r)
8611 r = radeon_fence_driver_start_ring(rdev,
8612 TN_RING_TYPE_VCE1_INDEX);
8613 if (!r)
8614 r = radeon_fence_driver_start_ring(rdev,
8615 TN_RING_TYPE_VCE2_INDEX);
8616 }
8617 if (r) {
8618 dev_err(rdev->dev, "VCE init error (%d).\n", r);
8619 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8620 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8621 }
8622
7bf94a2c
AD		 8623 /* Enable IRQ */
8624 if (!rdev->irq.installed) {
8625 r = radeon_irq_kms_init(rdev);
8626 if (r)
8627 return r;
8628 }
8629
8630 r = cik_irq_init(rdev);
8631 if (r) {
8632 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8633 radeon_irq_kms_fini(rdev);
8634 return r;
8635 }
8636 cik_irq_set(rdev);
8637
0e16e4cf 8638 if (rdev->family == CHIP_HAWAII) {
78cd3661
AD		 8639 if (rdev->new_fw)
8640 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8641 else
8642 nop = RADEON_CP_PACKET2;
0e16e4cf
AD		 8643 } else {
8644 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8645 }
8646
7bf94a2c
AD		 8647 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8648 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
0e16e4cf 8649 nop);
7bf94a2c
AD		 8650 if (r)
8651 return r;
8652
963e81f9 8653 /* set up the compute queues */
2615b53a 8654 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 8655 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8656 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
0e16e4cf 8657 nop);
963e81f9
AD		 8658 if (r)
8659 return r;
8660 ring->me = 1; /* first MEC */
8661 ring->pipe = 0; /* first pipe */
8662 ring->queue = 0; /* first queue */
8663 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8664
2615b53a 8665 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 8666 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8667 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
0e16e4cf 8668 nop);
963e81f9
AD		 8669 if (r)
8670 return r;
8671 /* dGPU only have 1 MEC */
8672 ring->me = 1; /* first MEC */
8673 ring->pipe = 0; /* first pipe */
8674 ring->queue = 1; /* second queue */
8675 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8676
7bf94a2c
AD		 8677 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8678 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 8679 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 8680 if (r)
8681 return r;
8682
8683 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8684 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2e1e6dad 8685 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 8686 if (r)
8687 return r;
8688
8689 r = cik_cp_resume(rdev);
8690 if (r)
8691 return r;
8692
8693 r = cik_sdma_resume(rdev);
8694 if (r)
8695 return r;
8696
87167bb1
CK		 8697 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8698 if (ring->ring_size) {
02c9f7fa 8699 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2e1e6dad 8700 RADEON_CP_PACKET2);
87167bb1 8701 if (!r)
e409b128 8702 r = uvd_v1_0_init(rdev);
87167bb1
CK		 8703 if (r)
8704 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8705 }
8706
d93f7937
CK		 8707 r = -ENOENT;
8708
8709 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8710 if (ring->ring_size)
8711 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8712 VCE_CMD_NO_OP);
8713
8714 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8715 if (ring->ring_size)
8716 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8717 VCE_CMD_NO_OP);
8718
8719 if (!r)
8720 r = vce_v1_0_init(rdev);
8721 else if (r != -ENOENT)
8722 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8723
7bf94a2c
AD		 8724 r = radeon_ib_pool_init(rdev);
8725 if (r) {
8726 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8727 return r;
8728 }
8729
8730 r = radeon_vm_manager_init(rdev);
8731 if (r) {
8732 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8733 return r;
8734 }
8735
bfc1f97d 8736 r = radeon_audio_init(rdev);
b530602f
AD		 8737 if (r)
8738 return r;
8739
e28740ec
OG		 8740 r = radeon_kfd_resume(rdev);
8741 if (r)
8742 return r;
8743
7bf94a2c
AD		 8744 return 0;
8745}
8746
8747/**
8748 * cik_resume - resume the asic to a functional state
8749 *
8750 * @rdev: radeon_device pointer
8751 *
8752 * Programs the asic to a functional state (CIK).
8753 * Called at resume.
8754 * Returns 0 for success, error for failure.
8755 */
8756int cik_resume(struct radeon_device *rdev)
8757{
8758 int r;
8759
8760 /* post card */
8761 atom_asic_init(rdev->mode_info.atom_context);
8762
0aafd313
AD		 8763 /* init golden registers */
8764 cik_init_golden_registers(rdev);
8765
bc6a6295
AD		 8766 if (rdev->pm.pm_method == PM_METHOD_DPM)
8767 radeon_pm_resume(rdev);
6c7bccea 8768
7bf94a2c
AD		 8769 rdev->accel_working = true;
8770 r = cik_startup(rdev);
8771 if (r) {
8772 DRM_ERROR("cik startup failed on resume\n");
8773 rdev->accel_working = false;
8774 return r;
8775 }
8776
8777 return r;
8778
8779}
8780
8781/**
8782 * cik_suspend - suspend the asic
8783 *
8784 * @rdev: radeon_device pointer
8785 *
8786 * Bring the chip into a state suitable for suspend (CIK).
8787 * Called at suspend.
8788 * Returns 0 for success.
8789 */
8790int cik_suspend(struct radeon_device *rdev)
8791{
e28740ec 8792 radeon_kfd_suspend(rdev);
6c7bccea 8793 radeon_pm_suspend(rdev);
7991d665 8794 radeon_audio_fini(rdev);
7bf94a2c
AD		 8795 radeon_vm_manager_fini(rdev);
8796 cik_cp_enable(rdev, false);
8797 cik_sdma_enable(rdev, false);
e409b128 8798 uvd_v1_0_fini(rdev);
87167bb1 8799 radeon_uvd_suspend(rdev);
d93f7937 8800 radeon_vce_suspend(rdev);
473359bc
AD		 8801 cik_fini_pg(rdev);
8802 cik_fini_cg(rdev);
7bf94a2c
AD		 8803 cik_irq_suspend(rdev);
8804 radeon_wb_disable(rdev);
8805 cik_pcie_gart_disable(rdev);
8806 return 0;
8807}
8808
8809/* Plan is to move initialization in that function and use
8810 * helper function so that radeon_device_init pretty much
8811 * do nothing more than calling asic specific function. This
8812 * should also allow to remove a bunch of callback function
8813 * like vram_info.
8814 */
8815/**
8816 * cik_init - asic specific driver and hw init
8817 *
8818 * @rdev: radeon_device pointer
8819 *
8820 * Setup asic specific driver variables and program the hw
8821 * to a functional state (CIK).
8822 * Called at driver startup.
8823 * Returns 0 for success, errors for failure.
8824 */
8825int cik_init(struct radeon_device *rdev)
8826{
8827 struct radeon_ring *ring;
8828 int r;
8829
8830 /* Read BIOS */
8831 if (!radeon_get_bios(rdev)) {
8832 if (ASIC_IS_AVIVO(rdev))
8833 return -EINVAL;
8834 }
8835 /* Must be an ATOMBIOS */
8836 if (!rdev->is_atom_bios) {
8837 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8838 return -EINVAL;
8839 }
8840 r = radeon_atombios_init(rdev);
8841 if (r)
8842 return r;
8843
8844 /* Post card if necessary */
8845 if (!radeon_card_posted(rdev)) {
8846 if (!rdev->bios) {
8847 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8848 return -EINVAL;
8849 }
8850 DRM_INFO("GPU not posted. posting now...\n");
8851 atom_asic_init(rdev->mode_info.atom_context);
8852 }
0aafd313
AD		 8853 /* init golden registers */
8854 cik_init_golden_registers(rdev);
7bf94a2c
AD		 8855 /* Initialize scratch registers */
8856 cik_scratch_init(rdev);
8857 /* Initialize surface registers */
8858 radeon_surface_init(rdev);
8859 /* Initialize clocks */
8860 radeon_get_clock_info(rdev->ddev);
8861
8862 /* Fence driver */
8863 r = radeon_fence_driver_init(rdev);
8864 if (r)
8865 return r;
8866
8867 /* initialize memory controller */
8868 r = cik_mc_init(rdev);
8869 if (r)
8870 return r;
8871 /* Memory manager */
8872 r = radeon_bo_init(rdev);
8873 if (r)
8874 return r;
8875
01ac8794
AD		 8876 if (rdev->flags & RADEON_IS_IGP) {
8877 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8878 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8879 r = cik_init_microcode(rdev);
8880 if (r) {
8881 DRM_ERROR("Failed to load firmware!\n");
8882 return r;
8883 }
8884 }
8885 } else {
8886 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8887 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8888 !rdev->mc_fw) {
8889 r = cik_init_microcode(rdev);
8890 if (r) {
8891 DRM_ERROR("Failed to load firmware!\n");
8892 return r;
8893 }
8894 }
8895 }
8896
6c7bccea
AD		 8897 /* Initialize power management */
8898 radeon_pm_init(rdev);
8899
7bf94a2c
AD		 8900 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8901 ring->ring_obj = NULL;
8902 r600_ring_init(rdev, ring, 1024 * 1024);
8903
963e81f9
AD		 8904 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8905 ring->ring_obj = NULL;
8906 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8907 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8908 if (r)
8909 return r;
8910
8911 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8912 ring->ring_obj = NULL;
8913 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8914 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8915 if (r)
8916 return r;
8917
7bf94a2c
AD		 8918 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8919 ring->ring_obj = NULL;
8920 r600_ring_init(rdev, ring, 256 * 1024);
8921
8922 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8923 ring->ring_obj = NULL;
8924 r600_ring_init(rdev, ring, 256 * 1024);
8925
87167bb1
CK		 8926 r = radeon_uvd_init(rdev);
8927 if (!r) {
8928 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8929 ring->ring_obj = NULL;
8930 r600_ring_init(rdev, ring, 4096);
8931 }
8932
d93f7937
CK		 8933 r = radeon_vce_init(rdev);
8934 if (!r) {
8935 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8936 ring->ring_obj = NULL;
8937 r600_ring_init(rdev, ring, 4096);
8938
8939 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8940 ring->ring_obj = NULL;
8941 r600_ring_init(rdev, ring, 4096);
8942 }
8943
7bf94a2c
AD		 8944 rdev->ih.ring_obj = NULL;
8945 r600_ih_ring_init(rdev, 64 * 1024);
8946
8947 r = r600_pcie_gart_init(rdev);
8948 if (r)
8949 return r;
8950
8951 rdev->accel_working = true;
8952 r = cik_startup(rdev);
8953 if (r) {
8954 dev_err(rdev->dev, "disabling GPU acceleration\n");
8955 cik_cp_fini(rdev);
8956 cik_sdma_fini(rdev);
8957 cik_irq_fini(rdev);
1fd11777 8958 sumo_rlc_fini(rdev);
963e81f9 8959 cik_mec_fini(rdev);
7bf94a2c
AD		 8960 radeon_wb_fini(rdev);
8961 radeon_ib_pool_fini(rdev);
8962 radeon_vm_manager_fini(rdev);
8963 radeon_irq_kms_fini(rdev);
8964 cik_pcie_gart_fini(rdev);
8965 rdev->accel_working = false;
8966 }
8967
8968 /* Don't start up if the MC ucode is missing.
8969 * The default clocks and voltages before the MC ucode
8970 * is loaded are not suffient for advanced operations.
8971 */
8972 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8973 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8974 return -EINVAL;
8975 }
8976
8977 return 0;
8978}
8979
8980/**
8981 * cik_fini - asic specific driver and hw fini
8982 *
8983 * @rdev: radeon_device pointer
8984 *
8985 * Tear down the asic specific driver variables and program the hw
8986 * to an idle state (CIK).
8987 * Called at driver unload.
8988 */
8989void cik_fini(struct radeon_device *rdev)
8990{
6c7bccea 8991 radeon_pm_fini(rdev);
7bf94a2c
AD		 8992 cik_cp_fini(rdev);
8993 cik_sdma_fini(rdev);
473359bc
AD		 8994 cik_fini_pg(rdev);
8995 cik_fini_cg(rdev);
7bf94a2c 8996 cik_irq_fini(rdev);
1fd11777 8997 sumo_rlc_fini(rdev);
963e81f9 8998 cik_mec_fini(rdev);
7bf94a2c
AD		 8999 radeon_wb_fini(rdev);
9000 radeon_vm_manager_fini(rdev);
9001 radeon_ib_pool_fini(rdev);
9002 radeon_irq_kms_fini(rdev);
e409b128 9003 uvd_v1_0_fini(rdev);
87167bb1 9004 radeon_uvd_fini(rdev);
d93f7937 9005 radeon_vce_fini(rdev);
7bf94a2c
AD		 9006 cik_pcie_gart_fini(rdev);
9007 r600_vram_scratch_fini(rdev);
9008 radeon_gem_fini(rdev);
9009 radeon_fence_driver_fini(rdev);
9010 radeon_bo_fini(rdev);
9011 radeon_atombios_fini(rdev);
9012 kfree(rdev->bios);
9013 rdev->bios = NULL;
9014}
cd84a27d 9015
134b480f
AD		 9016void dce8_program_fmt(struct drm_encoder *encoder)
9017{
9018 struct drm_device *dev = encoder->dev;
9019 struct radeon_device *rdev = dev->dev_private;
9020 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
9021 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
9022 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
9023 int bpc = 0;
9024 u32 tmp = 0;
6214bb74 9025 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 9026
6214bb74
AD		 9027 if (connector) {
9028 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 9029 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 9030 dither = radeon_connector->dither;
9031 }
134b480f
AD		 9032
9033 /* LVDS/eDP FMT is set up by atom */
9034 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
9035 return;
9036
9037 /* not needed for analog */
9038 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
9039 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
9040 return;
9041
9042 if (bpc == 0)
9043 return;
9044
9045 switch (bpc) {
9046 case 6:
6214bb74 9047 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 9048 /* XXX sort out optimal dither settings */
9049 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
9050 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
9051 else
9052 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
9053 break;
9054 case 8:
6214bb74 9055 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 9056 /* XXX sort out optimal dither settings */
9057 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
9058 FMT_RGB_RANDOM_ENABLE |
9059 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
9060 else
9061 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
9062 break;
9063 case 10:
6214bb74 9064 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 9065 /* XXX sort out optimal dither settings */
9066 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
9067 FMT_RGB_RANDOM_ENABLE |
9068 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
9069 else
9070 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
9071 break;
9072 default:
9073 /* not needed */
9074 break;
9075 }
9076
9077 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
9078}
9079
cd84a27d
AD		 9080/* display watermark setup */
9081/**
9082 * dce8_line_buffer_adjust - Set up the line buffer
9083 *
9084 * @rdev: radeon_device pointer
9085 * @radeon_crtc: the selected display controller
9086 * @mode: the current display mode on the selected display
9087 * controller
9088 *
9089 * Setup up the line buffer allocation for
9090 * the selected display controller (CIK).
9091 * Returns the line buffer size in pixels.
9092 */
9093static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
9094 struct radeon_crtc *radeon_crtc,
9095 struct drm_display_mode *mode)
9096{
bc01a8c7
AD		 9097 u32 tmp, buffer_alloc, i;
9098 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
cd84a27d
AD		 9099 /*
9100 * Line Buffer Setup
9101 * There are 6 line buffers, one for each display controllers.
9102 * There are 3 partitions per LB. Select the number of partitions
9103 * to enable based on the display width. For display widths larger
9104 * than 4096, you need use to use 2 display controllers and combine
9105 * them using the stereo blender.
9106 */
9107 if (radeon_crtc->base.enabled && mode) {
bc01a8c7 9108 if (mode->crtc_hdisplay < 1920) {
cd84a27d 9109 tmp = 1;
bc01a8c7
AD		 9110 buffer_alloc = 2;
9111 } else if (mode->crtc_hdisplay < 2560) {
cd84a27d 9112 tmp = 2;
bc01a8c7
AD		 9113 buffer_alloc = 2;
9114 } else if (mode->crtc_hdisplay < 4096) {
cd84a27d 9115 tmp = 0;
bc01a8c7
AD		 9116 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
9117 } else {
cd84a27d
AD		 9118 DRM_DEBUG_KMS("Mode too big for LB!\n");
9119 tmp = 0;
bc01a8c7 9120 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
cd84a27d 9121 }
bc01a8c7 9122 } else {
cd84a27d 9123 tmp = 1;
bc01a8c7
AD		 9124 buffer_alloc = 0;
9125 }
cd84a27d
AD		 9126
9127 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
9128 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
9129
bc01a8c7
AD		 9130 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
9131 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
9132 for (i = 0; i < rdev->usec_timeout; i++) {
9133 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
9134 DMIF_BUFFERS_ALLOCATED_COMPLETED)
9135 break;
9136 udelay(1);
9137 }
9138
cd84a27d
AD		 9139 if (radeon_crtc->base.enabled && mode) {
9140 switch (tmp) {
9141 case 0:
9142 default:
9143 return 4096 * 2;
9144 case 1:
9145 return 1920 * 2;
9146 case 2:
9147 return 2560 * 2;
9148 }
9149 }
9150
9151 /* controller not enabled, so no lb used */
9152 return 0;
9153}
9154
9155/**
9156 * cik_get_number_of_dram_channels - get the number of dram channels
9157 *
9158 * @rdev: radeon_device pointer
9159 *
9160 * Look up the number of video ram channels (CIK).
9161 * Used for display watermark bandwidth calculations
9162 * Returns the number of dram channels
9163 */
9164static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
9165{
9166 u32 tmp = RREG32(MC_SHARED_CHMAP);
9167
9168 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
9169 case 0:
9170 default:
9171 return 1;
9172 case 1:
9173 return 2;
9174 case 2:
9175 return 4;
9176 case 3:
9177 return 8;
9178 case 4:
9179 return 3;
9180 case 5:
9181 return 6;
9182 case 6:
9183 return 10;
9184 case 7:
9185 return 12;
9186 case 8:
9187 return 16;
9188 }
9189}
9190
9191struct dce8_wm_params {
9192 u32 dram_channels; /* number of dram channels */
9193 u32 yclk; /* bandwidth per dram data pin in kHz */
9194 u32 sclk; /* engine clock in kHz */
9195 u32 disp_clk; /* display clock in kHz */
9196 u32 src_width; /* viewport width */
9197 u32 active_time; /* active display time in ns */
9198 u32 blank_time; /* blank time in ns */
9199 bool interlaced; /* mode is interlaced */
9200 fixed20_12 vsc; /* vertical scale ratio */
9201 u32 num_heads; /* number of active crtcs */
9202 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
9203 u32 lb_size; /* line buffer allocated to pipe */
9204 u32 vtaps; /* vertical scaler taps */
9205};
9206
9207/**
9208 * dce8_dram_bandwidth - get the dram bandwidth
9209 *
9210 * @wm: watermark calculation data
9211 *
9212 * Calculate the raw dram bandwidth (CIK).
9213 * Used for display watermark bandwidth calculations
9214 * Returns the dram bandwidth in MBytes/s
9215 */
9216static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
9217{
9218 /* Calculate raw DRAM Bandwidth */
9219 fixed20_12 dram_efficiency; /* 0.7 */
9220 fixed20_12 yclk, dram_channels, bandwidth;
9221 fixed20_12 a;
9222
9223 a.full = dfixed_const(1000);
9224 yclk.full = dfixed_const(wm->yclk);
9225 yclk.full = dfixed_div(yclk, a);
9226 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9227 a.full = dfixed_const(10);
9228 dram_efficiency.full = dfixed_const(7);
9229 dram_efficiency.full = dfixed_div(dram_efficiency, a);
9230 bandwidth.full = dfixed_mul(dram_channels, yclk);
9231 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
9232
9233 return dfixed_trunc(bandwidth);
9234}
9235
9236/**
9237 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
9238 *
9239 * @wm: watermark calculation data
9240 *
9241 * Calculate the dram bandwidth used for display (CIK).
9242 * Used for display watermark bandwidth calculations
9243 * Returns the dram bandwidth for display in MBytes/s
9244 */
9245static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9246{
9247 /* Calculate DRAM Bandwidth and the part allocated to display. */
9248 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
9249 fixed20_12 yclk, dram_channels, bandwidth;
9250 fixed20_12 a;
9251
9252 a.full = dfixed_const(1000);
9253 yclk.full = dfixed_const(wm->yclk);
9254 yclk.full = dfixed_div(yclk, a);
9255 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9256 a.full = dfixed_const(10);
9257 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
9258 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
9259 bandwidth.full = dfixed_mul(dram_channels, yclk);
9260 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
9261
9262 return dfixed_trunc(bandwidth);
9263}
9264
9265/**
9266 * dce8_data_return_bandwidth - get the data return bandwidth
9267 *
9268 * @wm: watermark calculation data
9269 *
9270 * Calculate the data return bandwidth used for display (CIK).
9271 * Used for display watermark bandwidth calculations
9272 * Returns the data return bandwidth in MBytes/s
9273 */
9274static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
9275{
9276 /* Calculate the display Data return Bandwidth */
9277 fixed20_12 return_efficiency; /* 0.8 */
9278 fixed20_12 sclk, bandwidth;
9279 fixed20_12 a;
9280
9281 a.full = dfixed_const(1000);
9282 sclk.full = dfixed_const(wm->sclk);
9283 sclk.full = dfixed_div(sclk, a);
9284 a.full = dfixed_const(10);
9285 return_efficiency.full = dfixed_const(8);
9286 return_efficiency.full = dfixed_div(return_efficiency, a);
9287 a.full = dfixed_const(32);
9288 bandwidth.full = dfixed_mul(a, sclk);
9289 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
9290
9291 return dfixed_trunc(bandwidth);
9292}
9293
9294/**
9295 * dce8_dmif_request_bandwidth - get the dmif bandwidth
9296 *
9297 * @wm: watermark calculation data
9298 *
9299 * Calculate the dmif bandwidth used for display (CIK).
9300 * Used for display watermark bandwidth calculations
9301 * Returns the dmif bandwidth in MBytes/s
9302 */
9303static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9304{
9305 /* Calculate the DMIF Request Bandwidth */
9306 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9307 fixed20_12 disp_clk, bandwidth;
9308 fixed20_12 a, b;
9309
9310 a.full = dfixed_const(1000);
9311 disp_clk.full = dfixed_const(wm->disp_clk);
9312 disp_clk.full = dfixed_div(disp_clk, a);
9313 a.full = dfixed_const(32);
9314 b.full = dfixed_mul(a, disp_clk);
9315
9316 a.full = dfixed_const(10);
9317 disp_clk_request_efficiency.full = dfixed_const(8);
9318 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9319
9320 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9321
9322 return dfixed_trunc(bandwidth);
9323}
9324
9325/**
9326 * dce8_available_bandwidth - get the min available bandwidth
9327 *
9328 * @wm: watermark calculation data
9329 *
9330 * Calculate the min available bandwidth used for display (CIK).
9331 * Used for display watermark bandwidth calculations
9332 * Returns the min available bandwidth in MBytes/s
9333 */
9334static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9335{
9336 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9337 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9338 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9339 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9340
9341 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9342}
9343
9344/**
9345 * dce8_average_bandwidth - get the average available bandwidth
9346 *
9347 * @wm: watermark calculation data
9348 *
9349 * Calculate the average available bandwidth used for display (CIK).
9350 * Used for display watermark bandwidth calculations
9351 * Returns the average available bandwidth in MBytes/s
9352 */
9353static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9354{
9355 /* Calculate the display mode Average Bandwidth
9356 * DisplayMode should contain the source and destination dimensions,
9357 * timing, etc.
9358 */
9359 fixed20_12 bpp;
9360 fixed20_12 line_time;
9361 fixed20_12 src_width;
9362 fixed20_12 bandwidth;
9363 fixed20_12 a;
9364
9365 a.full = dfixed_const(1000);
9366 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9367 line_time.full = dfixed_div(line_time, a);
9368 bpp.full = dfixed_const(wm->bytes_per_pixel);
9369 src_width.full = dfixed_const(wm->src_width);
9370 bandwidth.full = dfixed_mul(src_width, bpp);
9371 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9372 bandwidth.full = dfixed_div(bandwidth, line_time);
9373
9374 return dfixed_trunc(bandwidth);
9375}
9376
9377/**
9378 * dce8_latency_watermark - get the latency watermark
9379 *
9380 * @wm: watermark calculation data
9381 *
9382 * Calculate the latency watermark (CIK).
9383 * Used for display watermark bandwidth calculations
9384 * Returns the latency watermark in ns
9385 */
9386static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9387{
9388 /* First calculate the latency in ns */
9389 u32 mc_latency = 2000; /* 2000 ns. */
9390 u32 available_bandwidth = dce8_available_bandwidth(wm);
9391 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9392 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9393 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9394 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9395 (wm->num_heads * cursor_line_pair_return_time);
9396 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9397 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9398 u32 tmp, dmif_size = 12288;
9399 fixed20_12 a, b, c;
9400
9401 if (wm->num_heads == 0)
9402 return 0;
9403
9404 a.full = dfixed_const(2);
9405 b.full = dfixed_const(1);
9406 if ((wm->vsc.full > a.full) ||
9407 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9408 (wm->vtaps >= 5) ||
9409 ((wm->vsc.full >= a.full) && wm->interlaced))
9410 max_src_lines_per_dst_line = 4;
9411 else
9412 max_src_lines_per_dst_line = 2;
9413
9414 a.full = dfixed_const(available_bandwidth);
9415 b.full = dfixed_const(wm->num_heads);
9416 a.full = dfixed_div(a, b);
9417
9418 b.full = dfixed_const(mc_latency + 512);
9419 c.full = dfixed_const(wm->disp_clk);
9420 b.full = dfixed_div(b, c);
9421
9422 c.full = dfixed_const(dmif_size);
9423 b.full = dfixed_div(c, b);
9424
9425 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
9426
9427 b.full = dfixed_const(1000);
9428 c.full = dfixed_const(wm->disp_clk);
9429 b.full = dfixed_div(c, b);
9430 c.full = dfixed_const(wm->bytes_per_pixel);
9431 b.full = dfixed_mul(b, c);
9432
9433 lb_fill_bw = min(tmp, dfixed_trunc(b));
9434
9435 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9436 b.full = dfixed_const(1000);
9437 c.full = dfixed_const(lb_fill_bw);
9438 b.full = dfixed_div(c, b);
9439 a.full = dfixed_div(a, b);
9440 line_fill_time = dfixed_trunc(a);
9441
9442 if (line_fill_time < wm->active_time)
9443 return latency;
9444 else
9445 return latency + (line_fill_time - wm->active_time);
9446
9447}
9448
9449/**
9450 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9451 * average and available dram bandwidth
9452 *
9453 * @wm: watermark calculation data
9454 *
9455 * Check if the display average bandwidth fits in the display
9456 * dram bandwidth (CIK).
9457 * Used for display watermark bandwidth calculations
9458 * Returns true if the display fits, false if not.
9459 */
9460static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9461{
9462 if (dce8_average_bandwidth(wm) <=
9463 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9464 return true;
9465 else
9466 return false;
9467}
9468
9469/**
9470 * dce8_average_bandwidth_vs_available_bandwidth - check
9471 * average and available bandwidth
9472 *
9473 * @wm: watermark calculation data
9474 *
9475 * Check if the display average bandwidth fits in the display
9476 * available bandwidth (CIK).
9477 * Used for display watermark bandwidth calculations
9478 * Returns true if the display fits, false if not.
9479 */
9480static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9481{
9482 if (dce8_average_bandwidth(wm) <=
9483 (dce8_available_bandwidth(wm) / wm->num_heads))
9484 return true;
9485 else
9486 return false;
9487}
9488
9489/**
9490 * dce8_check_latency_hiding - check latency hiding
9491 *
9492 * @wm: watermark calculation data
9493 *
9494 * Check latency hiding (CIK).
9495 * Used for display watermark bandwidth calculations
9496 * Returns true if the display fits, false if not.
9497 */
9498static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9499{
9500 u32 lb_partitions = wm->lb_size / wm->src_width;
9501 u32 line_time = wm->active_time + wm->blank_time;
9502 u32 latency_tolerant_lines;
9503 u32 latency_hiding;
9504 fixed20_12 a;
9505
9506 a.full = dfixed_const(1);
9507 if (wm->vsc.full > a.full)
9508 latency_tolerant_lines = 1;
9509 else {
9510 if (lb_partitions <= (wm->vtaps + 1))
9511 latency_tolerant_lines = 1;
9512 else
9513 latency_tolerant_lines = 2;
9514 }
9515
9516 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9517
9518 if (dce8_latency_watermark(wm) <= latency_hiding)
9519 return true;
9520 else
9521 return false;
9522}
9523
9524/**
9525 * dce8_program_watermarks - program display watermarks
9526 *
9527 * @rdev: radeon_device pointer
9528 * @radeon_crtc: the selected display controller
9529 * @lb_size: line buffer size
9530 * @num_heads: number of display controllers in use
9531 *
9532 * Calculate and program the display watermarks for the
9533 * selected display controller (CIK).
9534 */
9535static void dce8_program_watermarks(struct radeon_device *rdev,
9536 struct radeon_crtc *radeon_crtc,
9537 u32 lb_size, u32 num_heads)
9538{
9539 struct drm_display_mode *mode = &radeon_crtc->base.mode;
58ea2dea 9540 struct dce8_wm_params wm_low, wm_high;
cd84a27d
AD		 9541 u32 pixel_period;
9542 u32 line_time = 0;
9543 u32 latency_watermark_a = 0, latency_watermark_b = 0;
9544 u32 tmp, wm_mask;
9545
9546 if (radeon_crtc->base.enabled && num_heads && mode) {
9547 pixel_period = 1000000 / (u32)mode->clock;
9548 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
9549
58ea2dea
AD		 9550 /* watermark for high clocks */
9551 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9552 rdev->pm.dpm_enabled) {
9553 wm_high.yclk =
9554 radeon_dpm_get_mclk(rdev, false) * 10;
9555 wm_high.sclk =
9556 radeon_dpm_get_sclk(rdev, false) * 10;
9557 } else {
9558 wm_high.yclk = rdev->pm.current_mclk * 10;
9559 wm_high.sclk = rdev->pm.current_sclk * 10;
9560 }
9561
9562 wm_high.disp_clk = mode->clock;
9563 wm_high.src_width = mode->crtc_hdisplay;
9564 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
9565 wm_high.blank_time = line_time - wm_high.active_time;
9566 wm_high.interlaced = false;
cd84a27d 9567 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
58ea2dea
AD		 9568 wm_high.interlaced = true;
9569 wm_high.vsc = radeon_crtc->vsc;
9570 wm_high.vtaps = 1;
cd84a27d 9571 if (radeon_crtc->rmx_type != RMX_OFF)
58ea2dea
AD		 9572 wm_high.vtaps = 2;
9573 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9574 wm_high.lb_size = lb_size;
9575 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9576 wm_high.num_heads = num_heads;
cd84a27d
AD		 9577
9578 /* set for high clocks */
58ea2dea
AD		 9579 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9580
9581 /* possibly force display priority to high */
9582 /* should really do this at mode validation time... */
9583 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9584 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9585 !dce8_check_latency_hiding(&wm_high) ||
9586 (rdev->disp_priority == 2)) {
9587 DRM_DEBUG_KMS("force priority to high\n");
9588 }
9589
9590 /* watermark for low clocks */
9591 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9592 rdev->pm.dpm_enabled) {
9593 wm_low.yclk =
9594 radeon_dpm_get_mclk(rdev, true) * 10;
9595 wm_low.sclk =
9596 radeon_dpm_get_sclk(rdev, true) * 10;
9597 } else {
9598 wm_low.yclk = rdev->pm.current_mclk * 10;
9599 wm_low.sclk = rdev->pm.current_sclk * 10;
9600 }
9601
9602 wm_low.disp_clk = mode->clock;
9603 wm_low.src_width = mode->crtc_hdisplay;
9604 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
9605 wm_low.blank_time = line_time - wm_low.active_time;
9606 wm_low.interlaced = false;
9607 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9608 wm_low.interlaced = true;
9609 wm_low.vsc = radeon_crtc->vsc;
9610 wm_low.vtaps = 1;
9611 if (radeon_crtc->rmx_type != RMX_OFF)
9612 wm_low.vtaps = 2;
9613 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9614 wm_low.lb_size = lb_size;
9615 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9616 wm_low.num_heads = num_heads;
9617
cd84a27d 9618 /* set for low clocks */
58ea2dea 9619 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
cd84a27d
AD		 9620
9621 /* possibly force display priority to high */
9622 /* should really do this at mode validation time... */
58ea2dea
AD		 9623 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9624 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9625 !dce8_check_latency_hiding(&wm_low) ||
cd84a27d
AD		 9626 (rdev->disp_priority == 2)) {
9627 DRM_DEBUG_KMS("force priority to high\n");
9628 }
5b5561b3
MK		 9629
9630 /* Save number of lines the linebuffer leads before the scanout */
9631 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
cd84a27d
AD		 9632 }
9633
9634 /* select wm A */
9635 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9636 tmp = wm_mask;
9637 tmp &= ~LATENCY_WATERMARK_MASK(3);
9638 tmp |= LATENCY_WATERMARK_MASK(1);
9639 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9640 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9641 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9642 LATENCY_HIGH_WATERMARK(line_time)));
9643 /* select wm B */
9644 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9645 tmp &= ~LATENCY_WATERMARK_MASK(3);
9646 tmp |= LATENCY_WATERMARK_MASK(2);
9647 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9648 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9649 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9650 LATENCY_HIGH_WATERMARK(line_time)));
9651 /* restore original selection */
9652 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
58ea2dea
AD		 9653
9654 /* save values for DPM */
9655 radeon_crtc->line_time = line_time;
9656 radeon_crtc->wm_high = latency_watermark_a;
9657 radeon_crtc->wm_low = latency_watermark_b;
cd84a27d
AD		 9658}
9659
9660/**
9661 * dce8_bandwidth_update - program display watermarks
9662 *
9663 * @rdev: radeon_device pointer
9664 *
9665 * Calculate and program the display watermarks and line
9666 * buffer allocation (CIK).
9667 */
9668void dce8_bandwidth_update(struct radeon_device *rdev)
9669{
9670 struct drm_display_mode *mode = NULL;
9671 u32 num_heads = 0, lb_size;
9672 int i;
9673
8efe82ca
AD		 9674 if (!rdev->mode_info.mode_config_initialized)
9675 return;
9676
cd84a27d
AD		 9677 radeon_update_display_priority(rdev);
9678
9679 for (i = 0; i < rdev->num_crtc; i++) {
9680 if (rdev->mode_info.crtcs[i]->base.enabled)
9681 num_heads++;
9682 }
9683 for (i = 0; i < rdev->num_crtc; i++) {
9684 mode = &rdev->mode_info.crtcs[i]->base.mode;
9685 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9686 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9687 }
9688}
44fa346f
AD		 9689
9690/**
9691 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9692 *
9693 * @rdev: radeon_device pointer
9694 *
9695 * Fetches a GPU clock counter snapshot (SI).
9696 * Returns the 64 bit clock counter snapshot.
9697 */
9698uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9699{
9700 uint64_t clock;
9701
9702 mutex_lock(&rdev->gpu_clock_mutex);
9703 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9704 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9705 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9706 mutex_unlock(&rdev->gpu_clock_mutex);
9707 return clock;
9708}
9709
87167bb1
CK		 9710static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9711 u32 cntl_reg, u32 status_reg)
9712{
9713 int r, i;
9714 struct atom_clock_dividers dividers;
9715 uint32_t tmp;
9716
9717 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9718 clock, false, &dividers);
9719 if (r)
9720 return r;
9721
9722 tmp = RREG32_SMC(cntl_reg);
9723 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9724 tmp |= dividers.post_divider;
9725 WREG32_SMC(cntl_reg, tmp);
9726
9727 for (i = 0; i < 100; i++) {
9728 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9729 break;
9730 mdelay(10);
9731 }
9732 if (i == 100)
9733 return -ETIMEDOUT;
9734
9735 return 0;
9736}
9737
9738int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9739{
9740 int r = 0;
9741
9742 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9743 if (r)
9744 return r;
9745
9746 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9747 return r;
9748}
9749
5ad6bf91
AD		 9750int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9751{
9752 int r, i;
9753 struct atom_clock_dividers dividers;
9754 u32 tmp;
9755
9756 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9757 ecclk, false, &dividers);
9758 if (r)
9759 return r;
9760
9761 for (i = 0; i < 100; i++) {
9762 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9763 break;
9764 mdelay(10);
9765 }
9766 if (i == 100)
9767 return -ETIMEDOUT;
9768
9769 tmp = RREG32_SMC(CG_ECLK_CNTL);
9770 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9771 tmp |= dividers.post_divider;
9772 WREG32_SMC(CG_ECLK_CNTL, tmp);
9773
9774 for (i = 0; i < 100; i++) {
9775 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9776 break;
9777 mdelay(10);
9778 }
9779 if (i == 100)
9780 return -ETIMEDOUT;
9781
9782 return 0;
9783}
9784
8a7cd276 9785static void cik_pcie_gen3_enable(struct radeon_device *rdev)
87167bb1 9786{
8a7cd276
AD		 9787 struct pci_dev *root = rdev->pdev->bus->self;
9788 int bridge_pos, gpu_pos;
9789 u32 speed_cntl, mask, current_data_rate;
9790 int ret, i;
9791 u16 tmp16;
87167bb1 9792
0bd252de
AW		 9793 if (pci_is_root_bus(rdev->pdev->bus))
9794 return;
9795
8a7cd276
AD		 9796 if (radeon_pcie_gen2 == 0)
9797 return;
87167bb1 9798
8a7cd276
AD		 9799 if (rdev->flags & RADEON_IS_IGP)
9800 return;
87167bb1 9801
8a7cd276
AD		 9802 if (!(rdev->flags & RADEON_IS_PCIE))
9803 return;
87167bb1 9804
8a7cd276
AD		 9805 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9806 if (ret != 0)
9807 return;
87167bb1 9808
8a7cd276
AD		 9809 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9810 return;
87167bb1 9811
8a7cd276
AD		 9812 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9813 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9814 LC_CURRENT_DATA_RATE_SHIFT;
9815 if (mask & DRM_PCIE_SPEED_80) {
9816 if (current_data_rate == 2) {
9817 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9818 return;
9819 }
9820 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9821 } else if (mask & DRM_PCIE_SPEED_50) {
9822 if (current_data_rate == 1) {
9823 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9824 return;
9825 }
9826 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9827 }
87167bb1 9828
8a7cd276
AD		 9829 bridge_pos = pci_pcie_cap(root);
9830 if (!bridge_pos)
9831 return;
9832
9833 gpu_pos = pci_pcie_cap(rdev->pdev);
9834 if (!gpu_pos)
9835 return;
9836
9837 if (mask & DRM_PCIE_SPEED_80) {
9838 /* re-try equalization if gen3 is not already enabled */
9839 if (current_data_rate != 2) {
9840 u16 bridge_cfg, gpu_cfg;
9841 u16 bridge_cfg2, gpu_cfg2;
9842 u32 max_lw, current_lw, tmp;
9843
9844 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9845 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9846
9847 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9848 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9849
9850 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9851 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9852
9853 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9854 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9855 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9856
9857 if (current_lw < max_lw) {
9858 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9859 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9860 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9861 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9862 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9863 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9864 }
9865 }
9866
9867 for (i = 0; i < 10; i++) {
9868 /* check status */
9869 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9870 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9871 break;
9872
9873 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9874 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9875
9876 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9877 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9878
9879 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9880 tmp |= LC_SET_QUIESCE;
9881 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9882
9883 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9884 tmp |= LC_REDO_EQ;
9885 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9886
9887 mdelay(100);
9888
9889 /* linkctl */
9890 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9891 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9892 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9893 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9894
9895 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9896 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9897 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9898 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9899
9900 /* linkctl2 */
9901 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9902 tmp16 &= ~((1 << 4) | (7 << 9));
9903 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9904 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9905
9906 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9907 tmp16 &= ~((1 << 4) | (7 << 9));
9908 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9909 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9910
9911 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9912 tmp &= ~LC_SET_QUIESCE;
9913 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9914 }
9915 }
9916 }
9917
9918 /* set the link speed */
9919 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9920 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9921 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9922
9923 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9924 tmp16 &= ~0xf;
9925 if (mask & DRM_PCIE_SPEED_80)
9926 tmp16 |= 3; /* gen3 */
9927 else if (mask & DRM_PCIE_SPEED_50)
9928 tmp16 |= 2; /* gen2 */
9929 else
9930 tmp16 |= 1; /* gen1 */
9931 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9932
9933 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9934 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9935 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9936
9937 for (i = 0; i < rdev->usec_timeout; i++) {
9938 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9939 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9940 break;
9941 udelay(1);
9942 }
9943}
7235711a
AD		 9944
9945static void cik_program_aspm(struct radeon_device *rdev)
9946{
9947 u32 data, orig;
9948 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9949 bool disable_clkreq = false;
9950
9951 if (radeon_aspm == 0)
9952 return;
9953
9954 /* XXX double check IGPs */
9955 if (rdev->flags & RADEON_IS_IGP)
9956 return;
9957
9958 if (!(rdev->flags & RADEON_IS_PCIE))
9959 return;
9960
9961 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9962 data &= ~LC_XMIT_N_FTS_MASK;
9963 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9964 if (orig != data)
9965 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9966
9967 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9968 data |= LC_GO_TO_RECOVERY;
9969 if (orig != data)
9970 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9971
9972 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9973 data |= P_IGNORE_EDB_ERR;
9974 if (orig != data)
9975 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9976
9977 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9978 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9979 data |= LC_PMI_TO_L1_DIS;
9980 if (!disable_l0s)
9981 data |= LC_L0S_INACTIVITY(7);
9982
9983 if (!disable_l1) {
9984 data |= LC_L1_INACTIVITY(7);
9985 data &= ~LC_PMI_TO_L1_DIS;
9986 if (orig != data)
9987 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9988
9989 if (!disable_plloff_in_l1) {
9990 bool clk_req_support;
9991
9992 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9993 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9994 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9995 if (orig != data)
9996 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9997
9998 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9999 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
10000 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
10001 if (orig != data)
10002 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
10003
10004 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
10005 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
10006 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
10007 if (orig != data)
10008 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
10009
10010 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
10011 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
10012 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
10013 if (orig != data)
10014 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
10015
10016 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
10017 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
10018 data |= LC_DYN_LANES_PWR_STATE(3);
10019 if (orig != data)
10020 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
10021
0bd252de
AW		 10022 if (!disable_clkreq &&
10023 !pci_is_root_bus(rdev->pdev->bus)) {
7235711a
AD		 10024 struct pci_dev *root = rdev->pdev->bus->self;
10025 u32 lnkcap;
10026
10027 clk_req_support = false;
10028 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
10029 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
10030 clk_req_support = true;
10031 } else {
10032 clk_req_support = false;
10033 }
10034
10035 if (clk_req_support) {
10036 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
10037 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
10038 if (orig != data)
10039 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
10040
10041 orig = data = RREG32_SMC(THM_CLK_CNTL);
10042 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
10043 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
10044 if (orig != data)
10045 WREG32_SMC(THM_CLK_CNTL, data);
10046
10047 orig = data = RREG32_SMC(MISC_CLK_CTRL);
10048 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
10049 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
10050 if (orig != data)
10051 WREG32_SMC(MISC_CLK_CTRL, data);
10052
10053 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
10054 data &= ~BCLK_AS_XCLK;
10055 if (orig != data)
10056 WREG32_SMC(CG_CLKPIN_CNTL, data);
10057
10058 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
10059 data &= ~FORCE_BIF_REFCLK_EN;
10060 if (orig != data)
10061 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
10062
10063 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
10064 data &= ~MPLL_CLKOUT_SEL_MASK;
10065 data |= MPLL_CLKOUT_SEL(4);
10066 if (orig != data)
10067 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
10068 }
10069 }
10070 } else {
10071 if (orig != data)
10072 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
10073 }
10074
10075 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
10076 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
10077 if (orig != data)
10078 WREG32_PCIE_PORT(PCIE_CNTL2, data);
10079
10080 if (!disable_l0s) {
10081 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
10082 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
10083 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
10084 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
10085 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
10086 data &= ~LC_L0S_INACTIVITY_MASK;
10087 if (orig != data)
10088 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
10089 }
10090 }
10091 }
87167bb1 10092}
Ubuntu Artful kernel mirror