]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - drivers/gpu/drm/radeon/si.c
projects / mirror_ubuntu-zesty-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge remote-tracking branches 'asoc/topic/tas2552', 'asoc/topic/tegra', 'asoc/topic...
[mirror_ubuntu-zesty-kernel.git] / drivers / gpu / drm / radeon / si.c
CommitLineData
43b3cd99
AD		 1/*
2 * Copyright 2011 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
0f0de06c 24#include <linux/firmware.h>
0f0de06c
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
760285e7 27#include <drm/drmP.h>
43b3cd99
AD		 28#include "radeon.h"
29#include "radeon_asic.h"
760285e7 30#include <drm/radeon_drm.h>
43b3cd99
AD		 31#include "sid.h"
32#include "atom.h"
48c0c902 33#include "si_blit_shaders.h"
bd8cd539 34#include "clearstate_si.h"
a0ceada6 35#include "radeon_ucode.h"
43b3cd99 36
0f0de06c
AD		 37
38MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
39MODULE_FIRMWARE("radeon/TAHITI_me.bin");
40MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
41MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
1ebe9280 42MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
0f0de06c 43MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
a9e61410 44MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
629bd33c
AD		 45
46MODULE_FIRMWARE("radeon/tahiti_pfp.bin");
47MODULE_FIRMWARE("radeon/tahiti_me.bin");
48MODULE_FIRMWARE("radeon/tahiti_ce.bin");
49MODULE_FIRMWARE("radeon/tahiti_mc.bin");
50MODULE_FIRMWARE("radeon/tahiti_rlc.bin");
51MODULE_FIRMWARE("radeon/tahiti_smc.bin");
52
0f0de06c
AD		 53MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
54MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
55MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
56MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
1ebe9280 57MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
0f0de06c 58MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
a9e61410 59MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
629bd33c
AD		 60
61MODULE_FIRMWARE("radeon/pitcairn_pfp.bin");
62MODULE_FIRMWARE("radeon/pitcairn_me.bin");
63MODULE_FIRMWARE("radeon/pitcairn_ce.bin");
64MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
65MODULE_FIRMWARE("radeon/pitcairn_rlc.bin");
66MODULE_FIRMWARE("radeon/pitcairn_smc.bin");
67
0f0de06c
AD		 68MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
69MODULE_FIRMWARE("radeon/VERDE_me.bin");
70MODULE_FIRMWARE("radeon/VERDE_ce.bin");
71MODULE_FIRMWARE("radeon/VERDE_mc.bin");
1ebe9280 72MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
0f0de06c 73MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
a9e61410 74MODULE_FIRMWARE("radeon/VERDE_smc.bin");
629bd33c
AD		 75
76MODULE_FIRMWARE("radeon/verde_pfp.bin");
77MODULE_FIRMWARE("radeon/verde_me.bin");
78MODULE_FIRMWARE("radeon/verde_ce.bin");
79MODULE_FIRMWARE("radeon/verde_mc.bin");
80MODULE_FIRMWARE("radeon/verde_rlc.bin");
81MODULE_FIRMWARE("radeon/verde_smc.bin");
82
bcc7f5d2
AD		 83MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
84MODULE_FIRMWARE("radeon/OLAND_me.bin");
85MODULE_FIRMWARE("radeon/OLAND_ce.bin");
86MODULE_FIRMWARE("radeon/OLAND_mc.bin");
1ebe9280 87MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
bcc7f5d2 88MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
a9e61410 89MODULE_FIRMWARE("radeon/OLAND_smc.bin");
629bd33c
AD		 90
91MODULE_FIRMWARE("radeon/oland_pfp.bin");
92MODULE_FIRMWARE("radeon/oland_me.bin");
93MODULE_FIRMWARE("radeon/oland_ce.bin");
94MODULE_FIRMWARE("radeon/oland_mc.bin");
95MODULE_FIRMWARE("radeon/oland_rlc.bin");
96MODULE_FIRMWARE("radeon/oland_smc.bin");
97
c04c00b4
AD		 98MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
99MODULE_FIRMWARE("radeon/HAINAN_me.bin");
100MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
101MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
1ebe9280 102MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
c04c00b4 103MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
a9e61410 104MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
0f0de06c 105
629bd33c
AD		 106MODULE_FIRMWARE("radeon/hainan_pfp.bin");
107MODULE_FIRMWARE("radeon/hainan_me.bin");
108MODULE_FIRMWARE("radeon/hainan_ce.bin");
109MODULE_FIRMWARE("radeon/hainan_mc.bin");
110MODULE_FIRMWARE("radeon/hainan_rlc.bin");
111MODULE_FIRMWARE("radeon/hainan_smc.bin");
112
65fcf668 113static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
b9d305df 114static void si_pcie_gen3_enable(struct radeon_device *rdev);
e0bcf165 115static void si_program_aspm(struct radeon_device *rdev);
1fd11777
AD		 116extern void sumo_rlc_fini(struct radeon_device *rdev);
117extern int sumo_rlc_init(struct radeon_device *rdev);
25a857fb
AD		 118extern int r600_ih_ring_alloc(struct radeon_device *rdev);
119extern void r600_ih_ring_fini(struct radeon_device *rdev);
0a96d72b 120extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
c476dde2
AD		 121extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
122extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
ca7db22b 123extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
1c534671 124extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
014bb209 125extern bool evergreen_is_display_hung(struct radeon_device *rdev);
811e4d58
AD		 126static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
127 bool enable);
4a5c8ea5
AD		 128static void si_init_pg(struct radeon_device *rdev);
129static void si_init_cg(struct radeon_device *rdev);
a6f4ae8d
AD		 130static void si_fini_pg(struct radeon_device *rdev);
131static void si_fini_cg(struct radeon_device *rdev);
132static void si_rlc_stop(struct radeon_device *rdev);
0a96d72b 133
6d8cf000
AD		 134static const u32 verde_rlc_save_restore_register_list[] =
135{
136 (0x8000 << 16) | (0x98f4 >> 2),
137 0x00000000,
138 (0x8040 << 16) | (0x98f4 >> 2),
139 0x00000000,
140 (0x8000 << 16) | (0xe80 >> 2),
141 0x00000000,
142 (0x8040 << 16) | (0xe80 >> 2),
143 0x00000000,
144 (0x8000 << 16) | (0x89bc >> 2),
145 0x00000000,
146 (0x8040 << 16) | (0x89bc >> 2),
147 0x00000000,
148 (0x8000 << 16) | (0x8c1c >> 2),
149 0x00000000,
150 (0x8040 << 16) | (0x8c1c >> 2),
151 0x00000000,
152 (0x9c00 << 16) | (0x98f0 >> 2),
153 0x00000000,
154 (0x9c00 << 16) | (0xe7c >> 2),
155 0x00000000,
156 (0x8000 << 16) | (0x9148 >> 2),
157 0x00000000,
158 (0x8040 << 16) | (0x9148 >> 2),
159 0x00000000,
160 (0x9c00 << 16) | (0x9150 >> 2),
161 0x00000000,
162 (0x9c00 << 16) | (0x897c >> 2),
163 0x00000000,
164 (0x9c00 << 16) | (0x8d8c >> 2),
165 0x00000000,
166 (0x9c00 << 16) | (0xac54 >> 2),
167 0X00000000,
168 0x3,
169 (0x9c00 << 16) | (0x98f8 >> 2),
170 0x00000000,
171 (0x9c00 << 16) | (0x9910 >> 2),
172 0x00000000,
173 (0x9c00 << 16) | (0x9914 >> 2),
174 0x00000000,
175 (0x9c00 << 16) | (0x9918 >> 2),
176 0x00000000,
177 (0x9c00 << 16) | (0x991c >> 2),
178 0x00000000,
179 (0x9c00 << 16) | (0x9920 >> 2),
180 0x00000000,
181 (0x9c00 << 16) | (0x9924 >> 2),
182 0x00000000,
183 (0x9c00 << 16) | (0x9928 >> 2),
184 0x00000000,
185 (0x9c00 << 16) | (0x992c >> 2),
186 0x00000000,
187 (0x9c00 << 16) | (0x9930 >> 2),
188 0x00000000,
189 (0x9c00 << 16) | (0x9934 >> 2),
190 0x00000000,
191 (0x9c00 << 16) | (0x9938 >> 2),
192 0x00000000,
193 (0x9c00 << 16) | (0x993c >> 2),
194 0x00000000,
195 (0x9c00 << 16) | (0x9940 >> 2),
196 0x00000000,
197 (0x9c00 << 16) | (0x9944 >> 2),
198 0x00000000,
199 (0x9c00 << 16) | (0x9948 >> 2),
200 0x00000000,
201 (0x9c00 << 16) | (0x994c >> 2),
202 0x00000000,
203 (0x9c00 << 16) | (0x9950 >> 2),
204 0x00000000,
205 (0x9c00 << 16) | (0x9954 >> 2),
206 0x00000000,
207 (0x9c00 << 16) | (0x9958 >> 2),
208 0x00000000,
209 (0x9c00 << 16) | (0x995c >> 2),
210 0x00000000,
211 (0x9c00 << 16) | (0x9960 >> 2),
212 0x00000000,
213 (0x9c00 << 16) | (0x9964 >> 2),
214 0x00000000,
215 (0x9c00 << 16) | (0x9968 >> 2),
216 0x00000000,
217 (0x9c00 << 16) | (0x996c >> 2),
218 0x00000000,
219 (0x9c00 << 16) | (0x9970 >> 2),
220 0x00000000,
221 (0x9c00 << 16) | (0x9974 >> 2),
222 0x00000000,
223 (0x9c00 << 16) | (0x9978 >> 2),
224 0x00000000,
225 (0x9c00 << 16) | (0x997c >> 2),
226 0x00000000,
227 (0x9c00 << 16) | (0x9980 >> 2),
228 0x00000000,
229 (0x9c00 << 16) | (0x9984 >> 2),
230 0x00000000,
231 (0x9c00 << 16) | (0x9988 >> 2),
232 0x00000000,
233 (0x9c00 << 16) | (0x998c >> 2),
234 0x00000000,
235 (0x9c00 << 16) | (0x8c00 >> 2),
236 0x00000000,
237 (0x9c00 << 16) | (0x8c14 >> 2),
238 0x00000000,
239 (0x9c00 << 16) | (0x8c04 >> 2),
240 0x00000000,
241 (0x9c00 << 16) | (0x8c08 >> 2),
242 0x00000000,
243 (0x8000 << 16) | (0x9b7c >> 2),
244 0x00000000,
245 (0x8040 << 16) | (0x9b7c >> 2),
246 0x00000000,
247 (0x8000 << 16) | (0xe84 >> 2),
248 0x00000000,
249 (0x8040 << 16) | (0xe84 >> 2),
250 0x00000000,
251 (0x8000 << 16) | (0x89c0 >> 2),
252 0x00000000,
253 (0x8040 << 16) | (0x89c0 >> 2),
254 0x00000000,
255 (0x8000 << 16) | (0x914c >> 2),
256 0x00000000,
257 (0x8040 << 16) | (0x914c >> 2),
258 0x00000000,
259 (0x8000 << 16) | (0x8c20 >> 2),
260 0x00000000,
261 (0x8040 << 16) | (0x8c20 >> 2),
262 0x00000000,
263 (0x8000 << 16) | (0x9354 >> 2),
264 0x00000000,
265 (0x8040 << 16) | (0x9354 >> 2),
266 0x00000000,
267 (0x9c00 << 16) | (0x9060 >> 2),
268 0x00000000,
269 (0x9c00 << 16) | (0x9364 >> 2),
270 0x00000000,
271 (0x9c00 << 16) | (0x9100 >> 2),
272 0x00000000,
273 (0x9c00 << 16) | (0x913c >> 2),
274 0x00000000,
275 (0x8000 << 16) | (0x90e0 >> 2),
276 0x00000000,
277 (0x8000 << 16) | (0x90e4 >> 2),
278 0x00000000,
279 (0x8000 << 16) | (0x90e8 >> 2),
280 0x00000000,
281 (0x8040 << 16) | (0x90e0 >> 2),
282 0x00000000,
283 (0x8040 << 16) | (0x90e4 >> 2),
284 0x00000000,
285 (0x8040 << 16) | (0x90e8 >> 2),
286 0x00000000,
287 (0x9c00 << 16) | (0x8bcc >> 2),
288 0x00000000,
289 (0x9c00 << 16) | (0x8b24 >> 2),
290 0x00000000,
291 (0x9c00 << 16) | (0x88c4 >> 2),
292 0x00000000,
293 (0x9c00 << 16) | (0x8e50 >> 2),
294 0x00000000,
295 (0x9c00 << 16) | (0x8c0c >> 2),
296 0x00000000,
297 (0x9c00 << 16) | (0x8e58 >> 2),
298 0x00000000,
299 (0x9c00 << 16) | (0x8e5c >> 2),
300 0x00000000,
301 (0x9c00 << 16) | (0x9508 >> 2),
302 0x00000000,
303 (0x9c00 << 16) | (0x950c >> 2),
304 0x00000000,
305 (0x9c00 << 16) | (0x9494 >> 2),
306 0x00000000,
307 (0x9c00 << 16) | (0xac0c >> 2),
308 0x00000000,
309 (0x9c00 << 16) | (0xac10 >> 2),
310 0x00000000,
311 (0x9c00 << 16) | (0xac14 >> 2),
312 0x00000000,
313 (0x9c00 << 16) | (0xae00 >> 2),
314 0x00000000,
315 (0x9c00 << 16) | (0xac08 >> 2),
316 0x00000000,
317 (0x9c00 << 16) | (0x88d4 >> 2),
318 0x00000000,
319 (0x9c00 << 16) | (0x88c8 >> 2),
320 0x00000000,
321 (0x9c00 << 16) | (0x88cc >> 2),
322 0x00000000,
323 (0x9c00 << 16) | (0x89b0 >> 2),
324 0x00000000,
325 (0x9c00 << 16) | (0x8b10 >> 2),
326 0x00000000,
327 (0x9c00 << 16) | (0x8a14 >> 2),
328 0x00000000,
329 (0x9c00 << 16) | (0x9830 >> 2),
330 0x00000000,
331 (0x9c00 << 16) | (0x9834 >> 2),
332 0x00000000,
333 (0x9c00 << 16) | (0x9838 >> 2),
334 0x00000000,
335 (0x9c00 << 16) | (0x9a10 >> 2),
336 0x00000000,
337 (0x8000 << 16) | (0x9870 >> 2),
338 0x00000000,
339 (0x8000 << 16) | (0x9874 >> 2),
340 0x00000000,
341 (0x8001 << 16) | (0x9870 >> 2),
342 0x00000000,
343 (0x8001 << 16) | (0x9874 >> 2),
344 0x00000000,
345 (0x8040 << 16) | (0x9870 >> 2),
346 0x00000000,
347 (0x8040 << 16) | (0x9874 >> 2),
348 0x00000000,
349 (0x8041 << 16) | (0x9870 >> 2),
350 0x00000000,
351 (0x8041 << 16) | (0x9874 >> 2),
352 0x00000000,
353 0x00000000
354};
355
205996c0
AD		 356static const u32 tahiti_golden_rlc_registers[] =
357{
358 0xc424, 0xffffffff, 0x00601005,
359 0xc47c, 0xffffffff, 0x10104040,
360 0xc488, 0xffffffff, 0x0100000a,
361 0xc314, 0xffffffff, 0x00000800,
362 0xc30c, 0xffffffff, 0x800000f4,
363 0xf4a8, 0xffffffff, 0x00000000
364};
365
366static const u32 tahiti_golden_registers[] =
367{
368 0x9a10, 0x00010000, 0x00018208,
369 0x9830, 0xffffffff, 0x00000000,
370 0x9834, 0xf00fffff, 0x00000400,
371 0x9838, 0x0002021c, 0x00020200,
372 0xc78, 0x00000080, 0x00000000,
373 0xd030, 0x000300c0, 0x00800040,
374 0xd830, 0x000300c0, 0x00800040,
375 0x5bb0, 0x000000f0, 0x00000070,
376 0x5bc0, 0x00200000, 0x50100000,
377 0x7030, 0x31000311, 0x00000011,
378 0x277c, 0x00000003, 0x000007ff,
379 0x240c, 0x000007ff, 0x00000000,
380 0x8a14, 0xf000001f, 0x00000007,
381 0x8b24, 0xffffffff, 0x00ffffff,
382 0x8b10, 0x0000ff0f, 0x00000000,
383 0x28a4c, 0x07ffffff, 0x4e000000,
384 0x28350, 0x3f3f3fff, 0x2a00126a,
385 0x30, 0x000000ff, 0x0040,
386 0x34, 0x00000040, 0x00004040,
387 0x9100, 0x07ffffff, 0x03000000,
388 0x8e88, 0x01ff1f3f, 0x00000000,
389 0x8e84, 0x01ff1f3f, 0x00000000,
390 0x9060, 0x0000007f, 0x00000020,
391 0x9508, 0x00010000, 0x00010000,
392 0xac14, 0x00000200, 0x000002fb,
393 0xac10, 0xffffffff, 0x0000543b,
394 0xac0c, 0xffffffff, 0xa9210876,
395 0x88d0, 0xffffffff, 0x000fff40,
396 0x88d4, 0x0000001f, 0x00000010,
397 0x1410, 0x20000000, 0x20fffed8,
398 0x15c0, 0x000c0fc0, 0x000c0400
399};
400
401static const u32 tahiti_golden_registers2[] =
402{
403 0xc64, 0x00000001, 0x00000001
404};
405
406static const u32 pitcairn_golden_rlc_registers[] =
407{
408 0xc424, 0xffffffff, 0x00601004,
409 0xc47c, 0xffffffff, 0x10102020,
410 0xc488, 0xffffffff, 0x01000020,
411 0xc314, 0xffffffff, 0x00000800,
412 0xc30c, 0xffffffff, 0x800000a4
413};
414
415static const u32 pitcairn_golden_registers[] =
416{
417 0x9a10, 0x00010000, 0x00018208,
418 0x9830, 0xffffffff, 0x00000000,
419 0x9834, 0xf00fffff, 0x00000400,
420 0x9838, 0x0002021c, 0x00020200,
421 0xc78, 0x00000080, 0x00000000,
422 0xd030, 0x000300c0, 0x00800040,
423 0xd830, 0x000300c0, 0x00800040,
424 0x5bb0, 0x000000f0, 0x00000070,
425 0x5bc0, 0x00200000, 0x50100000,
426 0x7030, 0x31000311, 0x00000011,
427 0x2ae4, 0x00073ffe, 0x000022a2,
428 0x240c, 0x000007ff, 0x00000000,
429 0x8a14, 0xf000001f, 0x00000007,
430 0x8b24, 0xffffffff, 0x00ffffff,
431 0x8b10, 0x0000ff0f, 0x00000000,
432 0x28a4c, 0x07ffffff, 0x4e000000,
433 0x28350, 0x3f3f3fff, 0x2a00126a,
434 0x30, 0x000000ff, 0x0040,
435 0x34, 0x00000040, 0x00004040,
436 0x9100, 0x07ffffff, 0x03000000,
437 0x9060, 0x0000007f, 0x00000020,
438 0x9508, 0x00010000, 0x00010000,
439 0xac14, 0x000003ff, 0x000000f7,
440 0xac10, 0xffffffff, 0x00000000,
441 0xac0c, 0xffffffff, 0x32761054,
442 0x88d4, 0x0000001f, 0x00000010,
443 0x15c0, 0x000c0fc0, 0x000c0400
444};
445
446static const u32 verde_golden_rlc_registers[] =
447{
448 0xc424, 0xffffffff, 0x033f1005,
449 0xc47c, 0xffffffff, 0x10808020,
450 0xc488, 0xffffffff, 0x00800008,
451 0xc314, 0xffffffff, 0x00001000,
452 0xc30c, 0xffffffff, 0x80010014
453};
454
455static const u32 verde_golden_registers[] =
456{
457 0x9a10, 0x00010000, 0x00018208,
458 0x9830, 0xffffffff, 0x00000000,
459 0x9834, 0xf00fffff, 0x00000400,
460 0x9838, 0x0002021c, 0x00020200,
461 0xc78, 0x00000080, 0x00000000,
462 0xd030, 0x000300c0, 0x00800040,
463 0xd030, 0x000300c0, 0x00800040,
464 0xd830, 0x000300c0, 0x00800040,
465 0xd830, 0x000300c0, 0x00800040,
466 0x5bb0, 0x000000f0, 0x00000070,
467 0x5bc0, 0x00200000, 0x50100000,
468 0x7030, 0x31000311, 0x00000011,
469 0x2ae4, 0x00073ffe, 0x000022a2,
470 0x2ae4, 0x00073ffe, 0x000022a2,
471 0x2ae4, 0x00073ffe, 0x000022a2,
472 0x240c, 0x000007ff, 0x00000000,
473 0x240c, 0x000007ff, 0x00000000,
474 0x240c, 0x000007ff, 0x00000000,
475 0x8a14, 0xf000001f, 0x00000007,
476 0x8a14, 0xf000001f, 0x00000007,
477 0x8a14, 0xf000001f, 0x00000007,
478 0x8b24, 0xffffffff, 0x00ffffff,
479 0x8b10, 0x0000ff0f, 0x00000000,
480 0x28a4c, 0x07ffffff, 0x4e000000,
481 0x28350, 0x3f3f3fff, 0x0000124a,
482 0x28350, 0x3f3f3fff, 0x0000124a,
483 0x28350, 0x3f3f3fff, 0x0000124a,
484 0x30, 0x000000ff, 0x0040,
485 0x34, 0x00000040, 0x00004040,
486 0x9100, 0x07ffffff, 0x03000000,
487 0x9100, 0x07ffffff, 0x03000000,
488 0x8e88, 0x01ff1f3f, 0x00000000,
489 0x8e88, 0x01ff1f3f, 0x00000000,
490 0x8e88, 0x01ff1f3f, 0x00000000,
491 0x8e84, 0x01ff1f3f, 0x00000000,
492 0x8e84, 0x01ff1f3f, 0x00000000,
493 0x8e84, 0x01ff1f3f, 0x00000000,
494 0x9060, 0x0000007f, 0x00000020,
495 0x9508, 0x00010000, 0x00010000,
496 0xac14, 0x000003ff, 0x00000003,
497 0xac14, 0x000003ff, 0x00000003,
498 0xac14, 0x000003ff, 0x00000003,
499 0xac10, 0xffffffff, 0x00000000,
500 0xac10, 0xffffffff, 0x00000000,
501 0xac10, 0xffffffff, 0x00000000,
502 0xac0c, 0xffffffff, 0x00001032,
503 0xac0c, 0xffffffff, 0x00001032,
504 0xac0c, 0xffffffff, 0x00001032,
505 0x88d4, 0x0000001f, 0x00000010,
506 0x88d4, 0x0000001f, 0x00000010,
507 0x88d4, 0x0000001f, 0x00000010,
508 0x15c0, 0x000c0fc0, 0x000c0400
509};
510
511static const u32 oland_golden_rlc_registers[] =
512{
513 0xc424, 0xffffffff, 0x00601005,
514 0xc47c, 0xffffffff, 0x10104040,
515 0xc488, 0xffffffff, 0x0100000a,
516 0xc314, 0xffffffff, 0x00000800,
517 0xc30c, 0xffffffff, 0x800000f4
518};
519
520static const u32 oland_golden_registers[] =
521{
522 0x9a10, 0x00010000, 0x00018208,
523 0x9830, 0xffffffff, 0x00000000,
524 0x9834, 0xf00fffff, 0x00000400,
525 0x9838, 0x0002021c, 0x00020200,
526 0xc78, 0x00000080, 0x00000000,
527 0xd030, 0x000300c0, 0x00800040,
528 0xd830, 0x000300c0, 0x00800040,
529 0x5bb0, 0x000000f0, 0x00000070,
530 0x5bc0, 0x00200000, 0x50100000,
531 0x7030, 0x31000311, 0x00000011,
532 0x2ae4, 0x00073ffe, 0x000022a2,
533 0x240c, 0x000007ff, 0x00000000,
534 0x8a14, 0xf000001f, 0x00000007,
535 0x8b24, 0xffffffff, 0x00ffffff,
536 0x8b10, 0x0000ff0f, 0x00000000,
537 0x28a4c, 0x07ffffff, 0x4e000000,
538 0x28350, 0x3f3f3fff, 0x00000082,
539 0x30, 0x000000ff, 0x0040,
540 0x34, 0x00000040, 0x00004040,
541 0x9100, 0x07ffffff, 0x03000000,
542 0x9060, 0x0000007f, 0x00000020,
543 0x9508, 0x00010000, 0x00010000,
544 0xac14, 0x000003ff, 0x000000f3,
545 0xac10, 0xffffffff, 0x00000000,
546 0xac0c, 0xffffffff, 0x00003210,
547 0x88d4, 0x0000001f, 0x00000010,
548 0x15c0, 0x000c0fc0, 0x000c0400
549};
550
fffbdda4
AD		 551static const u32 hainan_golden_registers[] =
552{
553 0x9a10, 0x00010000, 0x00018208,
554 0x9830, 0xffffffff, 0x00000000,
555 0x9834, 0xf00fffff, 0x00000400,
556 0x9838, 0x0002021c, 0x00020200,
557 0xd0c0, 0xff000fff, 0x00000100,
558 0xd030, 0x000300c0, 0x00800040,
559 0xd8c0, 0xff000fff, 0x00000100,
560 0xd830, 0x000300c0, 0x00800040,
561 0x2ae4, 0x00073ffe, 0x000022a2,
562 0x240c, 0x000007ff, 0x00000000,
563 0x8a14, 0xf000001f, 0x00000007,
564 0x8b24, 0xffffffff, 0x00ffffff,
565 0x8b10, 0x0000ff0f, 0x00000000,
566 0x28a4c, 0x07ffffff, 0x4e000000,
567 0x28350, 0x3f3f3fff, 0x00000000,
568 0x30, 0x000000ff, 0x0040,
569 0x34, 0x00000040, 0x00004040,
570 0x9100, 0x03e00000, 0x03600000,
571 0x9060, 0x0000007f, 0x00000020,
572 0x9508, 0x00010000, 0x00010000,
573 0xac14, 0x000003ff, 0x000000f1,
574 0xac10, 0xffffffff, 0x00000000,
575 0xac0c, 0xffffffff, 0x00003210,
576 0x88d4, 0x0000001f, 0x00000010,
577 0x15c0, 0x000c0fc0, 0x000c0400
578};
579
580static const u32 hainan_golden_registers2[] =
581{
582 0x98f8, 0xffffffff, 0x02010001
583};
584
205996c0
AD		 585static const u32 tahiti_mgcg_cgcg_init[] =
586{
587 0xc400, 0xffffffff, 0xfffffffc,
588 0x802c, 0xffffffff, 0xe0000000,
589 0x9a60, 0xffffffff, 0x00000100,
590 0x92a4, 0xffffffff, 0x00000100,
591 0xc164, 0xffffffff, 0x00000100,
592 0x9774, 0xffffffff, 0x00000100,
593 0x8984, 0xffffffff, 0x06000100,
594 0x8a18, 0xffffffff, 0x00000100,
595 0x92a0, 0xffffffff, 0x00000100,
596 0xc380, 0xffffffff, 0x00000100,
597 0x8b28, 0xffffffff, 0x00000100,
598 0x9144, 0xffffffff, 0x00000100,
599 0x8d88, 0xffffffff, 0x00000100,
600 0x8d8c, 0xffffffff, 0x00000100,
601 0x9030, 0xffffffff, 0x00000100,
602 0x9034, 0xffffffff, 0x00000100,
603 0x9038, 0xffffffff, 0x00000100,
604 0x903c, 0xffffffff, 0x00000100,
605 0xad80, 0xffffffff, 0x00000100,
606 0xac54, 0xffffffff, 0x00000100,
607 0x897c, 0xffffffff, 0x06000100,
608 0x9868, 0xffffffff, 0x00000100,
609 0x9510, 0xffffffff, 0x00000100,
610 0xaf04, 0xffffffff, 0x00000100,
611 0xae04, 0xffffffff, 0x00000100,
612 0x949c, 0xffffffff, 0x00000100,
613 0x802c, 0xffffffff, 0xe0000000,
614 0x9160, 0xffffffff, 0x00010000,
615 0x9164, 0xffffffff, 0x00030002,
616 0x9168, 0xffffffff, 0x00040007,
617 0x916c, 0xffffffff, 0x00060005,
618 0x9170, 0xffffffff, 0x00090008,
619 0x9174, 0xffffffff, 0x00020001,
620 0x9178, 0xffffffff, 0x00040003,
621 0x917c, 0xffffffff, 0x00000007,
622 0x9180, 0xffffffff, 0x00060005,
623 0x9184, 0xffffffff, 0x00090008,
624 0x9188, 0xffffffff, 0x00030002,
625 0x918c, 0xffffffff, 0x00050004,
626 0x9190, 0xffffffff, 0x00000008,
627 0x9194, 0xffffffff, 0x00070006,
628 0x9198, 0xffffffff, 0x000a0009,
629 0x919c, 0xffffffff, 0x00040003,
630 0x91a0, 0xffffffff, 0x00060005,
631 0x91a4, 0xffffffff, 0x00000009,
632 0x91a8, 0xffffffff, 0x00080007,
633 0x91ac, 0xffffffff, 0x000b000a,
634 0x91b0, 0xffffffff, 0x00050004,
635 0x91b4, 0xffffffff, 0x00070006,
636 0x91b8, 0xffffffff, 0x0008000b,
637 0x91bc, 0xffffffff, 0x000a0009,
638 0x91c0, 0xffffffff, 0x000d000c,
639 0x91c4, 0xffffffff, 0x00060005,
640 0x91c8, 0xffffffff, 0x00080007,
641 0x91cc, 0xffffffff, 0x0000000b,
642 0x91d0, 0xffffffff, 0x000a0009,
643 0x91d4, 0xffffffff, 0x000d000c,
644 0x91d8, 0xffffffff, 0x00070006,
645 0x91dc, 0xffffffff, 0x00090008,
646 0x91e0, 0xffffffff, 0x0000000c,
647 0x91e4, 0xffffffff, 0x000b000a,
648 0x91e8, 0xffffffff, 0x000e000d,
649 0x91ec, 0xffffffff, 0x00080007,
650 0x91f0, 0xffffffff, 0x000a0009,
651 0x91f4, 0xffffffff, 0x0000000d,
652 0x91f8, 0xffffffff, 0x000c000b,
653 0x91fc, 0xffffffff, 0x000f000e,
654 0x9200, 0xffffffff, 0x00090008,
655 0x9204, 0xffffffff, 0x000b000a,
656 0x9208, 0xffffffff, 0x000c000f,
657 0x920c, 0xffffffff, 0x000e000d,
658 0x9210, 0xffffffff, 0x00110010,
659 0x9214, 0xffffffff, 0x000a0009,
660 0x9218, 0xffffffff, 0x000c000b,
661 0x921c, 0xffffffff, 0x0000000f,
662 0x9220, 0xffffffff, 0x000e000d,
663 0x9224, 0xffffffff, 0x00110010,
664 0x9228, 0xffffffff, 0x000b000a,
665 0x922c, 0xffffffff, 0x000d000c,
666 0x9230, 0xffffffff, 0x00000010,
667 0x9234, 0xffffffff, 0x000f000e,
668 0x9238, 0xffffffff, 0x00120011,
669 0x923c, 0xffffffff, 0x000c000b,
670 0x9240, 0xffffffff, 0x000e000d,
671 0x9244, 0xffffffff, 0x00000011,
672 0x9248, 0xffffffff, 0x0010000f,
673 0x924c, 0xffffffff, 0x00130012,
674 0x9250, 0xffffffff, 0x000d000c,
675 0x9254, 0xffffffff, 0x000f000e,
676 0x9258, 0xffffffff, 0x00100013,
677 0x925c, 0xffffffff, 0x00120011,
678 0x9260, 0xffffffff, 0x00150014,
679 0x9264, 0xffffffff, 0x000e000d,
680 0x9268, 0xffffffff, 0x0010000f,
681 0x926c, 0xffffffff, 0x00000013,
682 0x9270, 0xffffffff, 0x00120011,
683 0x9274, 0xffffffff, 0x00150014,
684 0x9278, 0xffffffff, 0x000f000e,
685 0x927c, 0xffffffff, 0x00110010,
686 0x9280, 0xffffffff, 0x00000014,
687 0x9284, 0xffffffff, 0x00130012,
688 0x9288, 0xffffffff, 0x00160015,
689 0x928c, 0xffffffff, 0x0010000f,
690 0x9290, 0xffffffff, 0x00120011,
691 0x9294, 0xffffffff, 0x00000015,
692 0x9298, 0xffffffff, 0x00140013,
693 0x929c, 0xffffffff, 0x00170016,
694 0x9150, 0xffffffff, 0x96940200,
695 0x8708, 0xffffffff, 0x00900100,
696 0xc478, 0xffffffff, 0x00000080,
697 0xc404, 0xffffffff, 0x0020003f,
698 0x30, 0xffffffff, 0x0000001c,
699 0x34, 0x000f0000, 0x000f0000,
700 0x160c, 0xffffffff, 0x00000100,
701 0x1024, 0xffffffff, 0x00000100,
702 0x102c, 0x00000101, 0x00000000,
703 0x20a8, 0xffffffff, 0x00000104,
704 0x264c, 0x000c0000, 0x000c0000,
705 0x2648, 0x000c0000, 0x000c0000,
706 0x55e4, 0xff000fff, 0x00000100,
707 0x55e8, 0x00000001, 0x00000001,
708 0x2f50, 0x00000001, 0x00000001,
709 0x30cc, 0xc0000fff, 0x00000104,
710 0xc1e4, 0x00000001, 0x00000001,
711 0xd0c0, 0xfffffff0, 0x00000100,
712 0xd8c0, 0xfffffff0, 0x00000100
713};
714
715static const u32 pitcairn_mgcg_cgcg_init[] =
716{
717 0xc400, 0xffffffff, 0xfffffffc,
718 0x802c, 0xffffffff, 0xe0000000,
719 0x9a60, 0xffffffff, 0x00000100,
720 0x92a4, 0xffffffff, 0x00000100,
721 0xc164, 0xffffffff, 0x00000100,
722 0x9774, 0xffffffff, 0x00000100,
723 0x8984, 0xffffffff, 0x06000100,
724 0x8a18, 0xffffffff, 0x00000100,
725 0x92a0, 0xffffffff, 0x00000100,
726 0xc380, 0xffffffff, 0x00000100,
727 0x8b28, 0xffffffff, 0x00000100,
728 0x9144, 0xffffffff, 0x00000100,
729 0x8d88, 0xffffffff, 0x00000100,
730 0x8d8c, 0xffffffff, 0x00000100,
731 0x9030, 0xffffffff, 0x00000100,
732 0x9034, 0xffffffff, 0x00000100,
733 0x9038, 0xffffffff, 0x00000100,
734 0x903c, 0xffffffff, 0x00000100,
735 0xad80, 0xffffffff, 0x00000100,
736 0xac54, 0xffffffff, 0x00000100,
737 0x897c, 0xffffffff, 0x06000100,
738 0x9868, 0xffffffff, 0x00000100,
739 0x9510, 0xffffffff, 0x00000100,
740 0xaf04, 0xffffffff, 0x00000100,
741 0xae04, 0xffffffff, 0x00000100,
742 0x949c, 0xffffffff, 0x00000100,
743 0x802c, 0xffffffff, 0xe0000000,
744 0x9160, 0xffffffff, 0x00010000,
745 0x9164, 0xffffffff, 0x00030002,
746 0x9168, 0xffffffff, 0x00040007,
747 0x916c, 0xffffffff, 0x00060005,
748 0x9170, 0xffffffff, 0x00090008,
749 0x9174, 0xffffffff, 0x00020001,
750 0x9178, 0xffffffff, 0x00040003,
751 0x917c, 0xffffffff, 0x00000007,
752 0x9180, 0xffffffff, 0x00060005,
753 0x9184, 0xffffffff, 0x00090008,
754 0x9188, 0xffffffff, 0x00030002,
755 0x918c, 0xffffffff, 0x00050004,
756 0x9190, 0xffffffff, 0x00000008,
757 0x9194, 0xffffffff, 0x00070006,
758 0x9198, 0xffffffff, 0x000a0009,
759 0x919c, 0xffffffff, 0x00040003,
760 0x91a0, 0xffffffff, 0x00060005,
761 0x91a4, 0xffffffff, 0x00000009,
762 0x91a8, 0xffffffff, 0x00080007,
763 0x91ac, 0xffffffff, 0x000b000a,
764 0x91b0, 0xffffffff, 0x00050004,
765 0x91b4, 0xffffffff, 0x00070006,
766 0x91b8, 0xffffffff, 0x0008000b,
767 0x91bc, 0xffffffff, 0x000a0009,
768 0x91c0, 0xffffffff, 0x000d000c,
769 0x9200, 0xffffffff, 0x00090008,
770 0x9204, 0xffffffff, 0x000b000a,
771 0x9208, 0xffffffff, 0x000c000f,
772 0x920c, 0xffffffff, 0x000e000d,
773 0x9210, 0xffffffff, 0x00110010,
774 0x9214, 0xffffffff, 0x000a0009,
775 0x9218, 0xffffffff, 0x000c000b,
776 0x921c, 0xffffffff, 0x0000000f,
777 0x9220, 0xffffffff, 0x000e000d,
778 0x9224, 0xffffffff, 0x00110010,
779 0x9228, 0xffffffff, 0x000b000a,
780 0x922c, 0xffffffff, 0x000d000c,
781 0x9230, 0xffffffff, 0x00000010,
782 0x9234, 0xffffffff, 0x000f000e,
783 0x9238, 0xffffffff, 0x00120011,
784 0x923c, 0xffffffff, 0x000c000b,
785 0x9240, 0xffffffff, 0x000e000d,
786 0x9244, 0xffffffff, 0x00000011,
787 0x9248, 0xffffffff, 0x0010000f,
788 0x924c, 0xffffffff, 0x00130012,
789 0x9250, 0xffffffff, 0x000d000c,
790 0x9254, 0xffffffff, 0x000f000e,
791 0x9258, 0xffffffff, 0x00100013,
792 0x925c, 0xffffffff, 0x00120011,
793 0x9260, 0xffffffff, 0x00150014,
794 0x9150, 0xffffffff, 0x96940200,
795 0x8708, 0xffffffff, 0x00900100,
796 0xc478, 0xffffffff, 0x00000080,
797 0xc404, 0xffffffff, 0x0020003f,
798 0x30, 0xffffffff, 0x0000001c,
799 0x34, 0x000f0000, 0x000f0000,
800 0x160c, 0xffffffff, 0x00000100,
801 0x1024, 0xffffffff, 0x00000100,
802 0x102c, 0x00000101, 0x00000000,
803 0x20a8, 0xffffffff, 0x00000104,
804 0x55e4, 0xff000fff, 0x00000100,
805 0x55e8, 0x00000001, 0x00000001,
806 0x2f50, 0x00000001, 0x00000001,
807 0x30cc, 0xc0000fff, 0x00000104,
808 0xc1e4, 0x00000001, 0x00000001,
809 0xd0c0, 0xfffffff0, 0x00000100,
810 0xd8c0, 0xfffffff0, 0x00000100
811};
812
813static const u32 verde_mgcg_cgcg_init[] =
814{
815 0xc400, 0xffffffff, 0xfffffffc,
816 0x802c, 0xffffffff, 0xe0000000,
817 0x9a60, 0xffffffff, 0x00000100,
818 0x92a4, 0xffffffff, 0x00000100,
819 0xc164, 0xffffffff, 0x00000100,
820 0x9774, 0xffffffff, 0x00000100,
821 0x8984, 0xffffffff, 0x06000100,
822 0x8a18, 0xffffffff, 0x00000100,
823 0x92a0, 0xffffffff, 0x00000100,
824 0xc380, 0xffffffff, 0x00000100,
825 0x8b28, 0xffffffff, 0x00000100,
826 0x9144, 0xffffffff, 0x00000100,
827 0x8d88, 0xffffffff, 0x00000100,
828 0x8d8c, 0xffffffff, 0x00000100,
829 0x9030, 0xffffffff, 0x00000100,
830 0x9034, 0xffffffff, 0x00000100,
831 0x9038, 0xffffffff, 0x00000100,
832 0x903c, 0xffffffff, 0x00000100,
833 0xad80, 0xffffffff, 0x00000100,
834 0xac54, 0xffffffff, 0x00000100,
835 0x897c, 0xffffffff, 0x06000100,
836 0x9868, 0xffffffff, 0x00000100,
837 0x9510, 0xffffffff, 0x00000100,
838 0xaf04, 0xffffffff, 0x00000100,
839 0xae04, 0xffffffff, 0x00000100,
840 0x949c, 0xffffffff, 0x00000100,
841 0x802c, 0xffffffff, 0xe0000000,
842 0x9160, 0xffffffff, 0x00010000,
843 0x9164, 0xffffffff, 0x00030002,
844 0x9168, 0xffffffff, 0x00040007,
845 0x916c, 0xffffffff, 0x00060005,
846 0x9170, 0xffffffff, 0x00090008,
847 0x9174, 0xffffffff, 0x00020001,
848 0x9178, 0xffffffff, 0x00040003,
849 0x917c, 0xffffffff, 0x00000007,
850 0x9180, 0xffffffff, 0x00060005,
851 0x9184, 0xffffffff, 0x00090008,
852 0x9188, 0xffffffff, 0x00030002,
853 0x918c, 0xffffffff, 0x00050004,
854 0x9190, 0xffffffff, 0x00000008,
855 0x9194, 0xffffffff, 0x00070006,
856 0x9198, 0xffffffff, 0x000a0009,
857 0x919c, 0xffffffff, 0x00040003,
858 0x91a0, 0xffffffff, 0x00060005,
859 0x91a4, 0xffffffff, 0x00000009,
860 0x91a8, 0xffffffff, 0x00080007,
861 0x91ac, 0xffffffff, 0x000b000a,
862 0x91b0, 0xffffffff, 0x00050004,
863 0x91b4, 0xffffffff, 0x00070006,
864 0x91b8, 0xffffffff, 0x0008000b,
865 0x91bc, 0xffffffff, 0x000a0009,
866 0x91c0, 0xffffffff, 0x000d000c,
867 0x9200, 0xffffffff, 0x00090008,
868 0x9204, 0xffffffff, 0x000b000a,
869 0x9208, 0xffffffff, 0x000c000f,
870 0x920c, 0xffffffff, 0x000e000d,
871 0x9210, 0xffffffff, 0x00110010,
872 0x9214, 0xffffffff, 0x000a0009,
873 0x9218, 0xffffffff, 0x000c000b,
874 0x921c, 0xffffffff, 0x0000000f,
875 0x9220, 0xffffffff, 0x000e000d,
876 0x9224, 0xffffffff, 0x00110010,
877 0x9228, 0xffffffff, 0x000b000a,
878 0x922c, 0xffffffff, 0x000d000c,
879 0x9230, 0xffffffff, 0x00000010,
880 0x9234, 0xffffffff, 0x000f000e,
881 0x9238, 0xffffffff, 0x00120011,
882 0x923c, 0xffffffff, 0x000c000b,
883 0x9240, 0xffffffff, 0x000e000d,
884 0x9244, 0xffffffff, 0x00000011,
885 0x9248, 0xffffffff, 0x0010000f,
886 0x924c, 0xffffffff, 0x00130012,
887 0x9250, 0xffffffff, 0x000d000c,
888 0x9254, 0xffffffff, 0x000f000e,
889 0x9258, 0xffffffff, 0x00100013,
890 0x925c, 0xffffffff, 0x00120011,
891 0x9260, 0xffffffff, 0x00150014,
892 0x9150, 0xffffffff, 0x96940200,
893 0x8708, 0xffffffff, 0x00900100,
894 0xc478, 0xffffffff, 0x00000080,
895 0xc404, 0xffffffff, 0x0020003f,
896 0x30, 0xffffffff, 0x0000001c,
897 0x34, 0x000f0000, 0x000f0000,
898 0x160c, 0xffffffff, 0x00000100,
899 0x1024, 0xffffffff, 0x00000100,
900 0x102c, 0x00000101, 0x00000000,
901 0x20a8, 0xffffffff, 0x00000104,
902 0x264c, 0x000c0000, 0x000c0000,
903 0x2648, 0x000c0000, 0x000c0000,
904 0x55e4, 0xff000fff, 0x00000100,
905 0x55e8, 0x00000001, 0x00000001,
906 0x2f50, 0x00000001, 0x00000001,
907 0x30cc, 0xc0000fff, 0x00000104,
908 0xc1e4, 0x00000001, 0x00000001,
909 0xd0c0, 0xfffffff0, 0x00000100,
910 0xd8c0, 0xfffffff0, 0x00000100
911};
912
913static const u32 oland_mgcg_cgcg_init[] =
914{
915 0xc400, 0xffffffff, 0xfffffffc,
916 0x802c, 0xffffffff, 0xe0000000,
917 0x9a60, 0xffffffff, 0x00000100,
918 0x92a4, 0xffffffff, 0x00000100,
919 0xc164, 0xffffffff, 0x00000100,
920 0x9774, 0xffffffff, 0x00000100,
921 0x8984, 0xffffffff, 0x06000100,
922 0x8a18, 0xffffffff, 0x00000100,
923 0x92a0, 0xffffffff, 0x00000100,
924 0xc380, 0xffffffff, 0x00000100,
925 0x8b28, 0xffffffff, 0x00000100,
926 0x9144, 0xffffffff, 0x00000100,
927 0x8d88, 0xffffffff, 0x00000100,
928 0x8d8c, 0xffffffff, 0x00000100,
929 0x9030, 0xffffffff, 0x00000100,
930 0x9034, 0xffffffff, 0x00000100,
931 0x9038, 0xffffffff, 0x00000100,
932 0x903c, 0xffffffff, 0x00000100,
933 0xad80, 0xffffffff, 0x00000100,
934 0xac54, 0xffffffff, 0x00000100,
935 0x897c, 0xffffffff, 0x06000100,
936 0x9868, 0xffffffff, 0x00000100,
937 0x9510, 0xffffffff, 0x00000100,
938 0xaf04, 0xffffffff, 0x00000100,
939 0xae04, 0xffffffff, 0x00000100,
940 0x949c, 0xffffffff, 0x00000100,
941 0x802c, 0xffffffff, 0xe0000000,
942 0x9160, 0xffffffff, 0x00010000,
943 0x9164, 0xffffffff, 0x00030002,
944 0x9168, 0xffffffff, 0x00040007,
945 0x916c, 0xffffffff, 0x00060005,
946 0x9170, 0xffffffff, 0x00090008,
947 0x9174, 0xffffffff, 0x00020001,
948 0x9178, 0xffffffff, 0x00040003,
949 0x917c, 0xffffffff, 0x00000007,
950 0x9180, 0xffffffff, 0x00060005,
951 0x9184, 0xffffffff, 0x00090008,
952 0x9188, 0xffffffff, 0x00030002,
953 0x918c, 0xffffffff, 0x00050004,
954 0x9190, 0xffffffff, 0x00000008,
955 0x9194, 0xffffffff, 0x00070006,
956 0x9198, 0xffffffff, 0x000a0009,
957 0x919c, 0xffffffff, 0x00040003,
958 0x91a0, 0xffffffff, 0x00060005,
959 0x91a4, 0xffffffff, 0x00000009,
960 0x91a8, 0xffffffff, 0x00080007,
961 0x91ac, 0xffffffff, 0x000b000a,
962 0x91b0, 0xffffffff, 0x00050004,
963 0x91b4, 0xffffffff, 0x00070006,
964 0x91b8, 0xffffffff, 0x0008000b,
965 0x91bc, 0xffffffff, 0x000a0009,
966 0x91c0, 0xffffffff, 0x000d000c,
967 0x91c4, 0xffffffff, 0x00060005,
968 0x91c8, 0xffffffff, 0x00080007,
969 0x91cc, 0xffffffff, 0x0000000b,
970 0x91d0, 0xffffffff, 0x000a0009,
971 0x91d4, 0xffffffff, 0x000d000c,
972 0x9150, 0xffffffff, 0x96940200,
973 0x8708, 0xffffffff, 0x00900100,
974 0xc478, 0xffffffff, 0x00000080,
975 0xc404, 0xffffffff, 0x0020003f,
976 0x30, 0xffffffff, 0x0000001c,
977 0x34, 0x000f0000, 0x000f0000,
978 0x160c, 0xffffffff, 0x00000100,
979 0x1024, 0xffffffff, 0x00000100,
980 0x102c, 0x00000101, 0x00000000,
981 0x20a8, 0xffffffff, 0x00000104,
982 0x264c, 0x000c0000, 0x000c0000,
983 0x2648, 0x000c0000, 0x000c0000,
984 0x55e4, 0xff000fff, 0x00000100,
985 0x55e8, 0x00000001, 0x00000001,
986 0x2f50, 0x00000001, 0x00000001,
987 0x30cc, 0xc0000fff, 0x00000104,
988 0xc1e4, 0x00000001, 0x00000001,
989 0xd0c0, 0xfffffff0, 0x00000100,
990 0xd8c0, 0xfffffff0, 0x00000100
991};
992
fffbdda4
AD		 993static const u32 hainan_mgcg_cgcg_init[] =
994{
995 0xc400, 0xffffffff, 0xfffffffc,
996 0x802c, 0xffffffff, 0xe0000000,
997 0x9a60, 0xffffffff, 0x00000100,
998 0x92a4, 0xffffffff, 0x00000100,
999 0xc164, 0xffffffff, 0x00000100,
1000 0x9774, 0xffffffff, 0x00000100,
1001 0x8984, 0xffffffff, 0x06000100,
1002 0x8a18, 0xffffffff, 0x00000100,
1003 0x92a0, 0xffffffff, 0x00000100,
1004 0xc380, 0xffffffff, 0x00000100,
1005 0x8b28, 0xffffffff, 0x00000100,
1006 0x9144, 0xffffffff, 0x00000100,
1007 0x8d88, 0xffffffff, 0x00000100,
1008 0x8d8c, 0xffffffff, 0x00000100,
1009 0x9030, 0xffffffff, 0x00000100,
1010 0x9034, 0xffffffff, 0x00000100,
1011 0x9038, 0xffffffff, 0x00000100,
1012 0x903c, 0xffffffff, 0x00000100,
1013 0xad80, 0xffffffff, 0x00000100,
1014 0xac54, 0xffffffff, 0x00000100,
1015 0x897c, 0xffffffff, 0x06000100,
1016 0x9868, 0xffffffff, 0x00000100,
1017 0x9510, 0xffffffff, 0x00000100,
1018 0xaf04, 0xffffffff, 0x00000100,
1019 0xae04, 0xffffffff, 0x00000100,
1020 0x949c, 0xffffffff, 0x00000100,
1021 0x802c, 0xffffffff, 0xe0000000,
1022 0x9160, 0xffffffff, 0x00010000,
1023 0x9164, 0xffffffff, 0x00030002,
1024 0x9168, 0xffffffff, 0x00040007,
1025 0x916c, 0xffffffff, 0x00060005,
1026 0x9170, 0xffffffff, 0x00090008,
1027 0x9174, 0xffffffff, 0x00020001,
1028 0x9178, 0xffffffff, 0x00040003,
1029 0x917c, 0xffffffff, 0x00000007,
1030 0x9180, 0xffffffff, 0x00060005,
1031 0x9184, 0xffffffff, 0x00090008,
1032 0x9188, 0xffffffff, 0x00030002,
1033 0x918c, 0xffffffff, 0x00050004,
1034 0x9190, 0xffffffff, 0x00000008,
1035 0x9194, 0xffffffff, 0x00070006,
1036 0x9198, 0xffffffff, 0x000a0009,
1037 0x919c, 0xffffffff, 0x00040003,
1038 0x91a0, 0xffffffff, 0x00060005,
1039 0x91a4, 0xffffffff, 0x00000009,
1040 0x91a8, 0xffffffff, 0x00080007,
1041 0x91ac, 0xffffffff, 0x000b000a,
1042 0x91b0, 0xffffffff, 0x00050004,
1043 0x91b4, 0xffffffff, 0x00070006,
1044 0x91b8, 0xffffffff, 0x0008000b,
1045 0x91bc, 0xffffffff, 0x000a0009,
1046 0x91c0, 0xffffffff, 0x000d000c,
1047 0x91c4, 0xffffffff, 0x00060005,
1048 0x91c8, 0xffffffff, 0x00080007,
1049 0x91cc, 0xffffffff, 0x0000000b,
1050 0x91d0, 0xffffffff, 0x000a0009,
1051 0x91d4, 0xffffffff, 0x000d000c,
1052 0x9150, 0xffffffff, 0x96940200,
1053 0x8708, 0xffffffff, 0x00900100,
1054 0xc478, 0xffffffff, 0x00000080,
1055 0xc404, 0xffffffff, 0x0020003f,
1056 0x30, 0xffffffff, 0x0000001c,
1057 0x34, 0x000f0000, 0x000f0000,
1058 0x160c, 0xffffffff, 0x00000100,
1059 0x1024, 0xffffffff, 0x00000100,
1060 0x20a8, 0xffffffff, 0x00000104,
1061 0x264c, 0x000c0000, 0x000c0000,
1062 0x2648, 0x000c0000, 0x000c0000,
1063 0x2f50, 0x00000001, 0x00000001,
1064 0x30cc, 0xc0000fff, 0x00000104,
1065 0xc1e4, 0x00000001, 0x00000001,
1066 0xd0c0, 0xfffffff0, 0x00000100,
1067 0xd8c0, 0xfffffff0, 0x00000100
1068};
1069
205996c0
AD		 1070static u32 verde_pg_init[] =
1071{
1072 0x353c, 0xffffffff, 0x40000,
1073 0x3538, 0xffffffff, 0x200010ff,
1074 0x353c, 0xffffffff, 0x0,
1075 0x353c, 0xffffffff, 0x0,
1076 0x353c, 0xffffffff, 0x0,
1077 0x353c, 0xffffffff, 0x0,
1078 0x353c, 0xffffffff, 0x0,
1079 0x353c, 0xffffffff, 0x7007,
1080 0x3538, 0xffffffff, 0x300010ff,
1081 0x353c, 0xffffffff, 0x0,
1082 0x353c, 0xffffffff, 0x0,
1083 0x353c, 0xffffffff, 0x0,
1084 0x353c, 0xffffffff, 0x0,
1085 0x353c, 0xffffffff, 0x0,
1086 0x353c, 0xffffffff, 0x400000,
1087 0x3538, 0xffffffff, 0x100010ff,
1088 0x353c, 0xffffffff, 0x0,
1089 0x353c, 0xffffffff, 0x0,
1090 0x353c, 0xffffffff, 0x0,
1091 0x353c, 0xffffffff, 0x0,
1092 0x353c, 0xffffffff, 0x0,
1093 0x353c, 0xffffffff, 0x120200,
1094 0x3538, 0xffffffff, 0x500010ff,
1095 0x353c, 0xffffffff, 0x0,
1096 0x353c, 0xffffffff, 0x0,
1097 0x353c, 0xffffffff, 0x0,
1098 0x353c, 0xffffffff, 0x0,
1099 0x353c, 0xffffffff, 0x0,
1100 0x353c, 0xffffffff, 0x1e1e16,
1101 0x3538, 0xffffffff, 0x600010ff,
1102 0x353c, 0xffffffff, 0x0,
1103 0x353c, 0xffffffff, 0x0,
1104 0x353c, 0xffffffff, 0x0,
1105 0x353c, 0xffffffff, 0x0,
1106 0x353c, 0xffffffff, 0x0,
1107 0x353c, 0xffffffff, 0x171f1e,
1108 0x3538, 0xffffffff, 0x700010ff,
1109 0x353c, 0xffffffff, 0x0,
1110 0x353c, 0xffffffff, 0x0,
1111 0x353c, 0xffffffff, 0x0,
1112 0x353c, 0xffffffff, 0x0,
1113 0x353c, 0xffffffff, 0x0,
1114 0x353c, 0xffffffff, 0x0,
1115 0x3538, 0xffffffff, 0x9ff,
1116 0x3500, 0xffffffff, 0x0,
1117 0x3504, 0xffffffff, 0x10000800,
1118 0x3504, 0xffffffff, 0xf,
1119 0x3504, 0xffffffff, 0xf,
1120 0x3500, 0xffffffff, 0x4,
1121 0x3504, 0xffffffff, 0x1000051e,
1122 0x3504, 0xffffffff, 0xffff,
1123 0x3504, 0xffffffff, 0xffff,
1124 0x3500, 0xffffffff, 0x8,
1125 0x3504, 0xffffffff, 0x80500,
1126 0x3500, 0xffffffff, 0x12,
1127 0x3504, 0xffffffff, 0x9050c,
1128 0x3500, 0xffffffff, 0x1d,
1129 0x3504, 0xffffffff, 0xb052c,
1130 0x3500, 0xffffffff, 0x2a,
1131 0x3504, 0xffffffff, 0x1053e,
1132 0x3500, 0xffffffff, 0x2d,
1133 0x3504, 0xffffffff, 0x10546,
1134 0x3500, 0xffffffff, 0x30,
1135 0x3504, 0xffffffff, 0xa054e,
1136 0x3500, 0xffffffff, 0x3c,
1137 0x3504, 0xffffffff, 0x1055f,
1138 0x3500, 0xffffffff, 0x3f,
1139 0x3504, 0xffffffff, 0x10567,
1140 0x3500, 0xffffffff, 0x42,
1141 0x3504, 0xffffffff, 0x1056f,
1142 0x3500, 0xffffffff, 0x45,
1143 0x3504, 0xffffffff, 0x10572,
1144 0x3500, 0xffffffff, 0x48,
1145 0x3504, 0xffffffff, 0x20575,
1146 0x3500, 0xffffffff, 0x4c,
1147 0x3504, 0xffffffff, 0x190801,
1148 0x3500, 0xffffffff, 0x67,
1149 0x3504, 0xffffffff, 0x1082a,
1150 0x3500, 0xffffffff, 0x6a,
1151 0x3504, 0xffffffff, 0x1b082d,
1152 0x3500, 0xffffffff, 0x87,
1153 0x3504, 0xffffffff, 0x310851,
1154 0x3500, 0xffffffff, 0xba,
1155 0x3504, 0xffffffff, 0x891,
1156 0x3500, 0xffffffff, 0xbc,
1157 0x3504, 0xffffffff, 0x893,
1158 0x3500, 0xffffffff, 0xbe,
1159 0x3504, 0xffffffff, 0x20895,
1160 0x3500, 0xffffffff, 0xc2,
1161 0x3504, 0xffffffff, 0x20899,
1162 0x3500, 0xffffffff, 0xc6,
1163 0x3504, 0xffffffff, 0x2089d,
1164 0x3500, 0xffffffff, 0xca,
1165 0x3504, 0xffffffff, 0x8a1,
1166 0x3500, 0xffffffff, 0xcc,
1167 0x3504, 0xffffffff, 0x8a3,
1168 0x3500, 0xffffffff, 0xce,
1169 0x3504, 0xffffffff, 0x308a5,
1170 0x3500, 0xffffffff, 0xd3,
1171 0x3504, 0xffffffff, 0x6d08cd,
1172 0x3500, 0xffffffff, 0x142,
1173 0x3504, 0xffffffff, 0x2000095a,
1174 0x3504, 0xffffffff, 0x1,
1175 0x3500, 0xffffffff, 0x144,
1176 0x3504, 0xffffffff, 0x301f095b,
1177 0x3500, 0xffffffff, 0x165,
1178 0x3504, 0xffffffff, 0xc094d,
1179 0x3500, 0xffffffff, 0x173,
1180 0x3504, 0xffffffff, 0xf096d,
1181 0x3500, 0xffffffff, 0x184,
1182 0x3504, 0xffffffff, 0x15097f,
1183 0x3500, 0xffffffff, 0x19b,
1184 0x3504, 0xffffffff, 0xc0998,
1185 0x3500, 0xffffffff, 0x1a9,
1186 0x3504, 0xffffffff, 0x409a7,
1187 0x3500, 0xffffffff, 0x1af,
1188 0x3504, 0xffffffff, 0xcdc,
1189 0x3500, 0xffffffff, 0x1b1,
1190 0x3504, 0xffffffff, 0x800,
1191 0x3508, 0xffffffff, 0x6c9b2000,
1192 0x3510, 0xfc00, 0x2000,
1193 0x3544, 0xffffffff, 0xfc0,
1194 0x28d4, 0x00000100, 0x100
1195};
1196
1197static void si_init_golden_registers(struct radeon_device *rdev)
1198{
1199 switch (rdev->family) {
1200 case CHIP_TAHITI:
1201 radeon_program_register_sequence(rdev,
1202 tahiti_golden_registers,
1203 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1204 radeon_program_register_sequence(rdev,
1205 tahiti_golden_rlc_registers,
1206 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1207 radeon_program_register_sequence(rdev,
1208 tahiti_mgcg_cgcg_init,
1209 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1210 radeon_program_register_sequence(rdev,
1211 tahiti_golden_registers2,
1212 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1213 break;
1214 case CHIP_PITCAIRN:
1215 radeon_program_register_sequence(rdev,
1216 pitcairn_golden_registers,
1217 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1218 radeon_program_register_sequence(rdev,
1219 pitcairn_golden_rlc_registers,
1220 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1221 radeon_program_register_sequence(rdev,
1222 pitcairn_mgcg_cgcg_init,
1223 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1224 break;
1225 case CHIP_VERDE:
1226 radeon_program_register_sequence(rdev,
1227 verde_golden_registers,
1228 (const u32)ARRAY_SIZE(verde_golden_registers));
1229 radeon_program_register_sequence(rdev,
1230 verde_golden_rlc_registers,
1231 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1232 radeon_program_register_sequence(rdev,
1233 verde_mgcg_cgcg_init,
1234 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1235 radeon_program_register_sequence(rdev,
1236 verde_pg_init,
1237 (const u32)ARRAY_SIZE(verde_pg_init));
1238 break;
1239 case CHIP_OLAND:
1240 radeon_program_register_sequence(rdev,
1241 oland_golden_registers,
1242 (const u32)ARRAY_SIZE(oland_golden_registers));
1243 radeon_program_register_sequence(rdev,
1244 oland_golden_rlc_registers,
1245 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1246 radeon_program_register_sequence(rdev,
1247 oland_mgcg_cgcg_init,
1248 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1249 break;
fffbdda4
AD		 1250 case CHIP_HAINAN:
1251 radeon_program_register_sequence(rdev,
1252 hainan_golden_registers,
1253 (const u32)ARRAY_SIZE(hainan_golden_registers));
1254 radeon_program_register_sequence(rdev,
1255 hainan_golden_registers2,
1256 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1257 radeon_program_register_sequence(rdev,
1258 hainan_mgcg_cgcg_init,
1259 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1260 break;
205996c0
AD		 1261 default:
1262 break;
1263 }
1264}
1265
454d2e2a
AD		 1266#define PCIE_BUS_CLK 10000
1267#define TCLK (PCIE_BUS_CLK / 10)
1268
1269/**
1270 * si_get_xclk - get the xclk
1271 *
1272 * @rdev: radeon_device pointer
1273 *
1274 * Returns the reference clock used by the gfx engine
1275 * (SI).
1276 */
1277u32 si_get_xclk(struct radeon_device *rdev)
1278{
1279 u32 reference_clock = rdev->clock.spll.reference_freq;
1280 u32 tmp;
1281
1282 tmp = RREG32(CG_CLKPIN_CNTL_2);
1283 if (tmp & MUX_TCLK_TO_XCLK)
1284 return TCLK;
1285
1286 tmp = RREG32(CG_CLKPIN_CNTL);
1287 if (tmp & XTALIN_DIVIDE)
1288 return reference_clock / 4;
1289
1290 return reference_clock;
1291}
1292
1bd47d2e
AD		 1293/* get temperature in millidegrees */
1294int si_get_temp(struct radeon_device *rdev)
1295{
1296 u32 temp;
1297 int actual_temp = 0;
1298
1299 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1300 CTF_TEMP_SHIFT;
1301
1302 if (temp & 0x200)
1303 actual_temp = 255;
1304 else
1305 actual_temp = temp & 0x1ff;
1306
1307 actual_temp = (actual_temp * 1000);
1308
1309 return actual_temp;
1310}
1311
8b074dd6
AD		 1312#define TAHITI_IO_MC_REGS_SIZE 36
1313
1314static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1315 {0x0000006f, 0x03044000},
1316 {0x00000070, 0x0480c018},
1317 {0x00000071, 0x00000040},
1318 {0x00000072, 0x01000000},
1319 {0x00000074, 0x000000ff},
1320 {0x00000075, 0x00143400},
1321 {0x00000076, 0x08ec0800},
1322 {0x00000077, 0x040000cc},
1323 {0x00000079, 0x00000000},
1324 {0x0000007a, 0x21000409},
1325 {0x0000007c, 0x00000000},
1326 {0x0000007d, 0xe8000000},
1327 {0x0000007e, 0x044408a8},
1328 {0x0000007f, 0x00000003},
1329 {0x00000080, 0x00000000},
1330 {0x00000081, 0x01000000},
1331 {0x00000082, 0x02000000},
1332 {0x00000083, 0x00000000},
1333 {0x00000084, 0xe3f3e4f4},
1334 {0x00000085, 0x00052024},
1335 {0x00000087, 0x00000000},
1336 {0x00000088, 0x66036603},
1337 {0x00000089, 0x01000000},
1338 {0x0000008b, 0x1c0a0000},
1339 {0x0000008c, 0xff010000},
1340 {0x0000008e, 0xffffefff},
1341 {0x0000008f, 0xfff3efff},
1342 {0x00000090, 0xfff3efbf},
1343 {0x00000094, 0x00101101},
1344 {0x00000095, 0x00000fff},
1345 {0x00000096, 0x00116fff},
1346 {0x00000097, 0x60010000},
1347 {0x00000098, 0x10010000},
1348 {0x00000099, 0x00006000},
1349 {0x0000009a, 0x00001000},
1350 {0x0000009f, 0x00a77400}
1351};
1352
1353static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1354 {0x0000006f, 0x03044000},
1355 {0x00000070, 0x0480c018},
1356 {0x00000071, 0x00000040},
1357 {0x00000072, 0x01000000},
1358 {0x00000074, 0x000000ff},
1359 {0x00000075, 0x00143400},
1360 {0x00000076, 0x08ec0800},
1361 {0x00000077, 0x040000cc},
1362 {0x00000079, 0x00000000},
1363 {0x0000007a, 0x21000409},
1364 {0x0000007c, 0x00000000},
1365 {0x0000007d, 0xe8000000},
1366 {0x0000007e, 0x044408a8},
1367 {0x0000007f, 0x00000003},
1368 {0x00000080, 0x00000000},
1369 {0x00000081, 0x01000000},
1370 {0x00000082, 0x02000000},
1371 {0x00000083, 0x00000000},
1372 {0x00000084, 0xe3f3e4f4},
1373 {0x00000085, 0x00052024},
1374 {0x00000087, 0x00000000},
1375 {0x00000088, 0x66036603},
1376 {0x00000089, 0x01000000},
1377 {0x0000008b, 0x1c0a0000},
1378 {0x0000008c, 0xff010000},
1379 {0x0000008e, 0xffffefff},
1380 {0x0000008f, 0xfff3efff},
1381 {0x00000090, 0xfff3efbf},
1382 {0x00000094, 0x00101101},
1383 {0x00000095, 0x00000fff},
1384 {0x00000096, 0x00116fff},
1385 {0x00000097, 0x60010000},
1386 {0x00000098, 0x10010000},
1387 {0x00000099, 0x00006000},
1388 {0x0000009a, 0x00001000},
1389 {0x0000009f, 0x00a47400}
1390};
1391
1392static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1393 {0x0000006f, 0x03044000},
1394 {0x00000070, 0x0480c018},
1395 {0x00000071, 0x00000040},
1396 {0x00000072, 0x01000000},
1397 {0x00000074, 0x000000ff},
1398 {0x00000075, 0x00143400},
1399 {0x00000076, 0x08ec0800},
1400 {0x00000077, 0x040000cc},
1401 {0x00000079, 0x00000000},
1402 {0x0000007a, 0x21000409},
1403 {0x0000007c, 0x00000000},
1404 {0x0000007d, 0xe8000000},
1405 {0x0000007e, 0x044408a8},
1406 {0x0000007f, 0x00000003},
1407 {0x00000080, 0x00000000},
1408 {0x00000081, 0x01000000},
1409 {0x00000082, 0x02000000},
1410 {0x00000083, 0x00000000},
1411 {0x00000084, 0xe3f3e4f4},
1412 {0x00000085, 0x00052024},
1413 {0x00000087, 0x00000000},
1414 {0x00000088, 0x66036603},
1415 {0x00000089, 0x01000000},
1416 {0x0000008b, 0x1c0a0000},
1417 {0x0000008c, 0xff010000},
1418 {0x0000008e, 0xffffefff},
1419 {0x0000008f, 0xfff3efff},
1420 {0x00000090, 0xfff3efbf},
1421 {0x00000094, 0x00101101},
1422 {0x00000095, 0x00000fff},
1423 {0x00000096, 0x00116fff},
1424 {0x00000097, 0x60010000},
1425 {0x00000098, 0x10010000},
1426 {0x00000099, 0x00006000},
1427 {0x0000009a, 0x00001000},
1428 {0x0000009f, 0x00a37400}
1429};
1430
bcc7f5d2
AD		 1431static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1432 {0x0000006f, 0x03044000},
1433 {0x00000070, 0x0480c018},
1434 {0x00000071, 0x00000040},
1435 {0x00000072, 0x01000000},
1436 {0x00000074, 0x000000ff},
1437 {0x00000075, 0x00143400},
1438 {0x00000076, 0x08ec0800},
1439 {0x00000077, 0x040000cc},
1440 {0x00000079, 0x00000000},
1441 {0x0000007a, 0x21000409},
1442 {0x0000007c, 0x00000000},
1443 {0x0000007d, 0xe8000000},
1444 {0x0000007e, 0x044408a8},
1445 {0x0000007f, 0x00000003},
1446 {0x00000080, 0x00000000},
1447 {0x00000081, 0x01000000},
1448 {0x00000082, 0x02000000},
1449 {0x00000083, 0x00000000},
1450 {0x00000084, 0xe3f3e4f4},
1451 {0x00000085, 0x00052024},
1452 {0x00000087, 0x00000000},
1453 {0x00000088, 0x66036603},
1454 {0x00000089, 0x01000000},
1455 {0x0000008b, 0x1c0a0000},
1456 {0x0000008c, 0xff010000},
1457 {0x0000008e, 0xffffefff},
1458 {0x0000008f, 0xfff3efff},
1459 {0x00000090, 0xfff3efbf},
1460 {0x00000094, 0x00101101},
1461 {0x00000095, 0x00000fff},
1462 {0x00000096, 0x00116fff},
1463 {0x00000097, 0x60010000},
1464 {0x00000098, 0x10010000},
1465 {0x00000099, 0x00006000},
1466 {0x0000009a, 0x00001000},
1467 {0x0000009f, 0x00a17730}
1468};
1469
c04c00b4
AD		 1470static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1471 {0x0000006f, 0x03044000},
1472 {0x00000070, 0x0480c018},
1473 {0x00000071, 0x00000040},
1474 {0x00000072, 0x01000000},
1475 {0x00000074, 0x000000ff},
1476 {0x00000075, 0x00143400},
1477 {0x00000076, 0x08ec0800},
1478 {0x00000077, 0x040000cc},
1479 {0x00000079, 0x00000000},
1480 {0x0000007a, 0x21000409},
1481 {0x0000007c, 0x00000000},
1482 {0x0000007d, 0xe8000000},
1483 {0x0000007e, 0x044408a8},
1484 {0x0000007f, 0x00000003},
1485 {0x00000080, 0x00000000},
1486 {0x00000081, 0x01000000},
1487 {0x00000082, 0x02000000},
1488 {0x00000083, 0x00000000},
1489 {0x00000084, 0xe3f3e4f4},
1490 {0x00000085, 0x00052024},
1491 {0x00000087, 0x00000000},
1492 {0x00000088, 0x66036603},
1493 {0x00000089, 0x01000000},
1494 {0x0000008b, 0x1c0a0000},
1495 {0x0000008c, 0xff010000},
1496 {0x0000008e, 0xffffefff},
1497 {0x0000008f, 0xfff3efff},
1498 {0x00000090, 0xfff3efbf},
1499 {0x00000094, 0x00101101},
1500 {0x00000095, 0x00000fff},
1501 {0x00000096, 0x00116fff},
1502 {0x00000097, 0x60010000},
1503 {0x00000098, 0x10010000},
1504 {0x00000099, 0x00006000},
1505 {0x0000009a, 0x00001000},
1506 {0x0000009f, 0x00a07730}
1507};
1508
8b074dd6 1509/* ucode loading */
6c7bccea 1510int si_mc_load_microcode(struct radeon_device *rdev)
8b074dd6 1511{
629bd33c
AD		 1512 const __be32 *fw_data = NULL;
1513 const __le32 *new_fw_data = NULL;
8b074dd6 1514 u32 running, blackout = 0;
629bd33c
AD		 1515 u32 *io_mc_regs = NULL;
1516 const __le32 *new_io_mc_regs = NULL;
8c79bae6 1517 int i, regs_size, ucode_size;
8b074dd6
AD		 1518
1519 if (!rdev->mc_fw)
1520 return -EINVAL;
1521
629bd33c
AD		 1522 if (rdev->new_fw) {
1523 const struct mc_firmware_header_v1_0 *hdr =
1524 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1525
1526 radeon_ucode_print_mc_hdr(&hdr->header);
1527 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1528 new_io_mc_regs = (const __le32 *)
1529 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1530 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1531 new_fw_data = (const __le32 *)
1532 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1533 } else {
1534 ucode_size = rdev->mc_fw->size / 4;
8c79bae6 1535
629bd33c
AD		 1536 switch (rdev->family) {
1537 case CHIP_TAHITI:
1538 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1539 regs_size = TAHITI_IO_MC_REGS_SIZE;
1540 break;
1541 case CHIP_PITCAIRN:
1542 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1543 regs_size = TAHITI_IO_MC_REGS_SIZE;
1544 break;
1545 case CHIP_VERDE:
1546 default:
1547 io_mc_regs = (u32 *)&verde_io_mc_regs;
1548 regs_size = TAHITI_IO_MC_REGS_SIZE;
1549 break;
1550 case CHIP_OLAND:
1551 io_mc_regs = (u32 *)&oland_io_mc_regs;
1552 regs_size = TAHITI_IO_MC_REGS_SIZE;
1553 break;
1554 case CHIP_HAINAN:
1555 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1556 regs_size = TAHITI_IO_MC_REGS_SIZE;
1557 break;
1558 }
1559 fw_data = (const __be32 *)rdev->mc_fw->data;
8b074dd6
AD		 1560 }
1561
1562 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1563
1564 if (running == 0) {
1565 if (running) {
1566 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1567 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1568 }
1569
1570 /* reset the engine and set to writable */
1571 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1572 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1573
1574 /* load mc io regs */
1575 for (i = 0; i < regs_size; i++) {
629bd33c
AD		 1576 if (rdev->new_fw) {
1577 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1578 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1579 } else {
1580 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1581 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1582 }
8b074dd6
AD		 1583 }
1584 /* load the MC ucode */
629bd33c
AD		 1585 for (i = 0; i < ucode_size; i++) {
1586 if (rdev->new_fw)
1587 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1588 else
1589 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1590 }
8b074dd6
AD		 1591
1592 /* put the engine back into the active state */
1593 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1594 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1595 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1596
1597 /* wait for training to complete */
1598 for (i = 0; i < rdev->usec_timeout; i++) {
1599 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1600 break;
1601 udelay(1);
1602 }
1603 for (i = 0; i < rdev->usec_timeout; i++) {
1604 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1605 break;
1606 udelay(1);
1607 }
1608
1609 if (running)
1610 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1611 }
1612
1613 return 0;
1614}
1615
0f0de06c
AD		 1616static int si_init_microcode(struct radeon_device *rdev)
1617{
0f0de06c 1618 const char *chip_name;
629bd33c 1619 const char *new_chip_name;
0f0de06c 1620 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1ebe9280 1621 size_t smc_req_size, mc2_req_size;
0f0de06c
AD		 1622 char fw_name[30];
1623 int err;
629bd33c 1624 int new_fw = 0;
0f0de06c
AD		 1625
1626 DRM_DEBUG("\n");
1627
0f0de06c
AD		 1628 switch (rdev->family) {
1629 case CHIP_TAHITI:
1630 chip_name = "TAHITI";
629bd33c 1631 new_chip_name = "tahiti";
0f0de06c
AD		 1632 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1633 me_req_size = SI_PM4_UCODE_SIZE * 4;
1634 ce_req_size = SI_CE_UCODE_SIZE * 4;
1635 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1636 mc_req_size = SI_MC_UCODE_SIZE * 4;
1ebe9280 1637 mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
a9e61410 1638 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
0f0de06c
AD		 1639 break;
1640 case CHIP_PITCAIRN:
1641 chip_name = "PITCAIRN";
629bd33c 1642 new_chip_name = "pitcairn";
0f0de06c
AD		 1643 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1644 me_req_size = SI_PM4_UCODE_SIZE * 4;
1645 ce_req_size = SI_CE_UCODE_SIZE * 4;
1646 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1647 mc_req_size = SI_MC_UCODE_SIZE * 4;
1ebe9280 1648 mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
a9e61410 1649 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
0f0de06c
AD		 1650 break;
1651 case CHIP_VERDE:
1652 chip_name = "VERDE";
629bd33c 1653 new_chip_name = "verde";
0f0de06c
AD		 1654 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1655 me_req_size = SI_PM4_UCODE_SIZE * 4;
1656 ce_req_size = SI_CE_UCODE_SIZE * 4;
1657 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1658 mc_req_size = SI_MC_UCODE_SIZE * 4;
1ebe9280 1659 mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
a9e61410 1660 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
0f0de06c 1661 break;
bcc7f5d2
AD		 1662 case CHIP_OLAND:
1663 chip_name = "OLAND";
629bd33c 1664 new_chip_name = "oland";
bcc7f5d2
AD		 1665 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1666 me_req_size = SI_PM4_UCODE_SIZE * 4;
1667 ce_req_size = SI_CE_UCODE_SIZE * 4;
1668 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1ebe9280 1669 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
a9e61410 1670 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
bcc7f5d2 1671 break;
c04c00b4
AD		 1672 case CHIP_HAINAN:
1673 chip_name = "HAINAN";
629bd33c 1674 new_chip_name = "hainan";
c04c00b4
AD		 1675 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1676 me_req_size = SI_PM4_UCODE_SIZE * 4;
1677 ce_req_size = SI_CE_UCODE_SIZE * 4;
1678 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1ebe9280 1679 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
a9e61410 1680 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
c04c00b4 1681 break;
0f0de06c
AD		 1682 default: BUG();
1683 }
1684
629bd33c 1685 DRM_INFO("Loading %s Microcode\n", new_chip_name);
0f0de06c 1686
629bd33c 1687 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
0a168933 1688 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
629bd33c
AD		 1689 if (err) {
1690 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1691 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1692 if (err)
1693 goto out;
1694 if (rdev->pfp_fw->size != pfp_req_size) {
1695 printk(KERN_ERR
1696 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1697 rdev->pfp_fw->size, fw_name);
1698 err = -EINVAL;
1699 goto out;
1700 }
1701 } else {
1702 err = radeon_ucode_validate(rdev->pfp_fw);
1703 if (err) {
1704 printk(KERN_ERR
1705 "si_cp: validation failed for firmware \"%s\"\n",
1706 fw_name);
1707 goto out;
1708 } else {
1709 new_fw++;
1710 }
0f0de06c
AD		 1711 }
1712
629bd33c 1713 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
0a168933 1714 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
629bd33c
AD		 1715 if (err) {
1716 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1717 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1718 if (err)
1719 goto out;
1720 if (rdev->me_fw->size != me_req_size) {
1721 printk(KERN_ERR
1722 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1723 rdev->me_fw->size, fw_name);
1724 err = -EINVAL;
1725 }
1726 } else {
1727 err = radeon_ucode_validate(rdev->me_fw);
1728 if (err) {
1729 printk(KERN_ERR
1730 "si_cp: validation failed for firmware \"%s\"\n",
1731 fw_name);
1732 goto out;
1733 } else {
1734 new_fw++;
1735 }
0f0de06c
AD		 1736 }
1737
629bd33c 1738 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
0a168933 1739 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
629bd33c
AD		 1740 if (err) {
1741 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1742 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1743 if (err)
1744 goto out;
1745 if (rdev->ce_fw->size != ce_req_size) {
1746 printk(KERN_ERR
1747 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1748 rdev->ce_fw->size, fw_name);
1749 err = -EINVAL;
1750 }
1751 } else {
1752 err = radeon_ucode_validate(rdev->ce_fw);
1753 if (err) {
1754 printk(KERN_ERR
1755 "si_cp: validation failed for firmware \"%s\"\n",
1756 fw_name);
1757 goto out;
1758 } else {
1759 new_fw++;
1760 }
0f0de06c
AD		 1761 }
1762
629bd33c 1763 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
0a168933 1764 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
629bd33c
AD		 1765 if (err) {
1766 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1767 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1768 if (err)
1769 goto out;
1770 if (rdev->rlc_fw->size != rlc_req_size) {
1771 printk(KERN_ERR
1772 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1773 rdev->rlc_fw->size, fw_name);
1774 err = -EINVAL;
1775 }
1776 } else {
1777 err = radeon_ucode_validate(rdev->rlc_fw);
1778 if (err) {
1779 printk(KERN_ERR
1780 "si_cp: validation failed for firmware \"%s\"\n",
1781 fw_name);
1782 goto out;
1783 } else {
1784 new_fw++;
1785 }
0f0de06c
AD		 1786 }
1787
629bd33c 1788 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
0a168933 1789 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1ebe9280 1790 if (err) {
629bd33c 1791 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1ebe9280 1792 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
629bd33c
AD		 1793 if (err) {
1794 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1795 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1796 if (err)
1797 goto out;
1798 }
1799 if ((rdev->mc_fw->size != mc_req_size) &&
1800 (rdev->mc_fw->size != mc2_req_size)) {
1801 printk(KERN_ERR
1802 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1803 rdev->mc_fw->size, fw_name);
1804 err = -EINVAL;
1805 }
1806 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1807 } else {
1808 err = radeon_ucode_validate(rdev->mc_fw);
1809 if (err) {
1810 printk(KERN_ERR
1811 "si_cp: validation failed for firmware \"%s\"\n",
1812 fw_name);
1ebe9280 1813 goto out;
629bd33c
AD		 1814 } else {
1815 new_fw++;
1816 }
1ebe9280 1817 }
0f0de06c 1818
629bd33c 1819 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
0a168933 1820 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
8a53fa23 1821 if (err) {
629bd33c
AD		 1822 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1823 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1824 if (err) {
1825 printk(KERN_ERR
1826 "smc: error loading firmware \"%s\"\n",
1827 fw_name);
1828 release_firmware(rdev->smc_fw);
1829 rdev->smc_fw = NULL;
1830 err = 0;
1831 } else if (rdev->smc_fw->size != smc_req_size) {
1832 printk(KERN_ERR
1833 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1834 rdev->smc_fw->size, fw_name);
1835 err = -EINVAL;
1836 }
1837 } else {
1838 err = radeon_ucode_validate(rdev->smc_fw);
1839 if (err) {
1840 printk(KERN_ERR
1841 "si_cp: validation failed for firmware \"%s\"\n",
1842 fw_name);
1843 goto out;
1844 } else {
1845 new_fw++;
1846 }
a9e61410
AD		 1847 }
1848
629bd33c
AD		 1849 if (new_fw == 0) {
1850 rdev->new_fw = false;
1851 } else if (new_fw < 6) {
1852 printk(KERN_ERR "si_fw: mixing new and old firmware!\n");
1853 err = -EINVAL;
1854 } else {
1855 rdev->new_fw = true;
1856 }
0f0de06c 1857out:
0f0de06c
AD		 1858 if (err) {
1859 if (err != -EINVAL)
1860 printk(KERN_ERR
1861 "si_cp: Failed to load firmware \"%s\"\n",
1862 fw_name);
1863 release_firmware(rdev->pfp_fw);
1864 rdev->pfp_fw = NULL;
1865 release_firmware(rdev->me_fw);
1866 rdev->me_fw = NULL;
1867 release_firmware(rdev->ce_fw);
1868 rdev->ce_fw = NULL;
1869 release_firmware(rdev->rlc_fw);
1870 rdev->rlc_fw = NULL;
1871 release_firmware(rdev->mc_fw);
1872 rdev->mc_fw = NULL;
a9e61410
AD		 1873 release_firmware(rdev->smc_fw);
1874 rdev->smc_fw = NULL;
0f0de06c
AD		 1875 }
1876 return err;
1877}
1878
43b3cd99
AD		 1879/* watermark setup */
1880static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1881 struct radeon_crtc *radeon_crtc,
1882 struct drm_display_mode *mode,
1883 struct drm_display_mode *other_mode)
1884{
290d2457
AD		 1885 u32 tmp, buffer_alloc, i;
1886 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
43b3cd99
AD		 1887 /*
1888 * Line Buffer Setup
1889 * There are 3 line buffers, each one shared by 2 display controllers.
1890 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1891 * the display controllers. The paritioning is done via one of four
1892 * preset allocations specified in bits 21:20:
1893 * 0 - half lb
1894 * 2 - whole lb, other crtc must be disabled
1895 */
1896 /* this can get tricky if we have two large displays on a paired group
1897 * of crtcs. Ideally for multiple large displays we'd assign them to
1898 * non-linked crtcs for maximum line buffer allocation.
1899 */
1900 if (radeon_crtc->base.enabled && mode) {
290d2457 1901 if (other_mode) {
43b3cd99 1902 tmp = 0; /* 1/2 */
290d2457
AD		 1903 buffer_alloc = 1;
1904 } else {
43b3cd99 1905 tmp = 2; /* whole */
290d2457
AD		 1906 buffer_alloc = 2;
1907 }
1908 } else {
43b3cd99 1909 tmp = 0;
290d2457
AD		 1910 buffer_alloc = 0;
1911 }
43b3cd99
AD		 1912
1913 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1914 DC_LB_MEMORY_CONFIG(tmp));
1915
290d2457
AD		 1916 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1917 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1918 for (i = 0; i < rdev->usec_timeout; i++) {
1919 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1920 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1921 break;
1922 udelay(1);
1923 }
1924
43b3cd99
AD		 1925 if (radeon_crtc->base.enabled && mode) {
1926 switch (tmp) {
1927 case 0:
1928 default:
1929 return 4096 * 2;
1930 case 2:
1931 return 8192 * 2;
1932 }
1933 }
1934
1935 /* controller not enabled, so no lb used */
1936 return 0;
1937}
1938
ca7db22b 1939static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
43b3cd99
AD		 1940{
1941 u32 tmp = RREG32(MC_SHARED_CHMAP);
1942
1943 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1944 case 0:
1945 default:
1946 return 1;
1947 case 1:
1948 return 2;
1949 case 2:
1950 return 4;
1951 case 3:
1952 return 8;
1953 case 4:
1954 return 3;
1955 case 5:
1956 return 6;
1957 case 6:
1958 return 10;
1959 case 7:
1960 return 12;
1961 case 8:
1962 return 16;
1963 }
1964}
1965
1966struct dce6_wm_params {
1967 u32 dram_channels; /* number of dram channels */
1968 u32 yclk; /* bandwidth per dram data pin in kHz */
1969 u32 sclk; /* engine clock in kHz */
1970 u32 disp_clk; /* display clock in kHz */
1971 u32 src_width; /* viewport width */
1972 u32 active_time; /* active display time in ns */
1973 u32 blank_time; /* blank time in ns */
1974 bool interlaced; /* mode is interlaced */
1975 fixed20_12 vsc; /* vertical scale ratio */
1976 u32 num_heads; /* number of active crtcs */
1977 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1978 u32 lb_size; /* line buffer allocated to pipe */
1979 u32 vtaps; /* vertical scaler taps */
1980};
1981
1982static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1983{
1984 /* Calculate raw DRAM Bandwidth */
1985 fixed20_12 dram_efficiency; /* 0.7 */
1986 fixed20_12 yclk, dram_channels, bandwidth;
1987 fixed20_12 a;
1988
1989 a.full = dfixed_const(1000);
1990 yclk.full = dfixed_const(wm->yclk);
1991 yclk.full = dfixed_div(yclk, a);
1992 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1993 a.full = dfixed_const(10);
1994 dram_efficiency.full = dfixed_const(7);
1995 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1996 bandwidth.full = dfixed_mul(dram_channels, yclk);
1997 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1998
1999 return dfixed_trunc(bandwidth);
2000}
2001
2002static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2003{
2004 /* Calculate DRAM Bandwidth and the part allocated to display. */
2005 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2006 fixed20_12 yclk, dram_channels, bandwidth;
2007 fixed20_12 a;
2008
2009 a.full = dfixed_const(1000);
2010 yclk.full = dfixed_const(wm->yclk);
2011 yclk.full = dfixed_div(yclk, a);
2012 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2013 a.full = dfixed_const(10);
2014 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2015 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2016 bandwidth.full = dfixed_mul(dram_channels, yclk);
2017 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2018
2019 return dfixed_trunc(bandwidth);
2020}
2021
2022static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
2023{
2024 /* Calculate the display Data return Bandwidth */
2025 fixed20_12 return_efficiency; /* 0.8 */
2026 fixed20_12 sclk, bandwidth;
2027 fixed20_12 a;
2028
2029 a.full = dfixed_const(1000);
2030 sclk.full = dfixed_const(wm->sclk);
2031 sclk.full = dfixed_div(sclk, a);
2032 a.full = dfixed_const(10);
2033 return_efficiency.full = dfixed_const(8);
2034 return_efficiency.full = dfixed_div(return_efficiency, a);
2035 a.full = dfixed_const(32);
2036 bandwidth.full = dfixed_mul(a, sclk);
2037 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2038
2039 return dfixed_trunc(bandwidth);
2040}
2041
2042static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
2043{
2044 return 32;
2045}
2046
2047static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
2048{
2049 /* Calculate the DMIF Request Bandwidth */
2050 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2051 fixed20_12 disp_clk, sclk, bandwidth;
2052 fixed20_12 a, b1, b2;
2053 u32 min_bandwidth;
2054
2055 a.full = dfixed_const(1000);
2056 disp_clk.full = dfixed_const(wm->disp_clk);
2057 disp_clk.full = dfixed_div(disp_clk, a);
2058 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
2059 b1.full = dfixed_mul(a, disp_clk);
2060
2061 a.full = dfixed_const(1000);
2062 sclk.full = dfixed_const(wm->sclk);
2063 sclk.full = dfixed_div(sclk, a);
2064 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
2065 b2.full = dfixed_mul(a, sclk);
2066
2067 a.full = dfixed_const(10);
2068 disp_clk_request_efficiency.full = dfixed_const(8);
2069 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2070
2071 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
2072
2073 a.full = dfixed_const(min_bandwidth);
2074 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
2075
2076 return dfixed_trunc(bandwidth);
2077}
2078
2079static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
2080{
2081 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2082 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
2083 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
2084 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
2085
2086 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2087}
2088
2089static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
2090{
2091 /* Calculate the display mode Average Bandwidth
2092 * DisplayMode should contain the source and destination dimensions,
2093 * timing, etc.
2094 */
2095 fixed20_12 bpp;
2096 fixed20_12 line_time;
2097 fixed20_12 src_width;
2098 fixed20_12 bandwidth;
2099 fixed20_12 a;
2100
2101 a.full = dfixed_const(1000);
2102 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2103 line_time.full = dfixed_div(line_time, a);
2104 bpp.full = dfixed_const(wm->bytes_per_pixel);
2105 src_width.full = dfixed_const(wm->src_width);
2106 bandwidth.full = dfixed_mul(src_width, bpp);
2107 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2108 bandwidth.full = dfixed_div(bandwidth, line_time);
2109
2110 return dfixed_trunc(bandwidth);
2111}
2112
2113static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
2114{
2115 /* First calcualte the latency in ns */
2116 u32 mc_latency = 2000; /* 2000 ns. */
2117 u32 available_bandwidth = dce6_available_bandwidth(wm);
2118 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2119 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2120 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2121 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2122 (wm->num_heads * cursor_line_pair_return_time);
2123 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2124 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2125 u32 tmp, dmif_size = 12288;
2126 fixed20_12 a, b, c;
2127
2128 if (wm->num_heads == 0)
2129 return 0;
2130
2131 a.full = dfixed_const(2);
2132 b.full = dfixed_const(1);
2133 if ((wm->vsc.full > a.full) ||
2134 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2135 (wm->vtaps >= 5) ||
2136 ((wm->vsc.full >= a.full) && wm->interlaced))
2137 max_src_lines_per_dst_line = 4;
2138 else
2139 max_src_lines_per_dst_line = 2;
2140
2141 a.full = dfixed_const(available_bandwidth);
2142 b.full = dfixed_const(wm->num_heads);
2143 a.full = dfixed_div(a, b);
2144
2145 b.full = dfixed_const(mc_latency + 512);
2146 c.full = dfixed_const(wm->disp_clk);
2147 b.full = dfixed_div(b, c);
2148
2149 c.full = dfixed_const(dmif_size);
2150 b.full = dfixed_div(c, b);
2151
2152 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
2153
2154 b.full = dfixed_const(1000);
2155 c.full = dfixed_const(wm->disp_clk);
2156 b.full = dfixed_div(c, b);
2157 c.full = dfixed_const(wm->bytes_per_pixel);
2158 b.full = dfixed_mul(b, c);
2159
2160 lb_fill_bw = min(tmp, dfixed_trunc(b));
2161
2162 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2163 b.full = dfixed_const(1000);
2164 c.full = dfixed_const(lb_fill_bw);
2165 b.full = dfixed_div(c, b);
2166 a.full = dfixed_div(a, b);
2167 line_fill_time = dfixed_trunc(a);
2168
2169 if (line_fill_time < wm->active_time)
2170 return latency;
2171 else
2172 return latency + (line_fill_time - wm->active_time);
2173
2174}
2175
2176static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2177{
2178 if (dce6_average_bandwidth(wm) <=
2179 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2180 return true;
2181 else
2182 return false;
2183};
2184
2185static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2186{
2187 if (dce6_average_bandwidth(wm) <=
2188 (dce6_available_bandwidth(wm) / wm->num_heads))
2189 return true;
2190 else
2191 return false;
2192};
2193
2194static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2195{
2196 u32 lb_partitions = wm->lb_size / wm->src_width;
2197 u32 line_time = wm->active_time + wm->blank_time;
2198 u32 latency_tolerant_lines;
2199 u32 latency_hiding;
2200 fixed20_12 a;
2201
2202 a.full = dfixed_const(1);
2203 if (wm->vsc.full > a.full)
2204 latency_tolerant_lines = 1;
2205 else {
2206 if (lb_partitions <= (wm->vtaps + 1))
2207 latency_tolerant_lines = 1;
2208 else
2209 latency_tolerant_lines = 2;
2210 }
2211
2212 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2213
2214 if (dce6_latency_watermark(wm) <= latency_hiding)
2215 return true;
2216 else
2217 return false;
2218}
2219
2220static void dce6_program_watermarks(struct radeon_device *rdev,
2221 struct radeon_crtc *radeon_crtc,
2222 u32 lb_size, u32 num_heads)
2223{
2224 struct drm_display_mode *mode = &radeon_crtc->base.mode;
c696e53f
AD		 2225 struct dce6_wm_params wm_low, wm_high;
2226 u32 dram_channels;
43b3cd99
AD		 2227 u32 pixel_period;
2228 u32 line_time = 0;
2229 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2230 u32 priority_a_mark = 0, priority_b_mark = 0;
2231 u32 priority_a_cnt = PRIORITY_OFF;
2232 u32 priority_b_cnt = PRIORITY_OFF;
2233 u32 tmp, arb_control3;
2234 fixed20_12 a, b, c;
2235
2236 if (radeon_crtc->base.enabled && num_heads && mode) {
2237 pixel_period = 1000000 / (u32)mode->clock;
2238 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2239 priority_a_cnt = 0;
2240 priority_b_cnt = 0;
2241
ca7db22b 2242 if (rdev->family == CHIP_ARUBA)
c696e53f 2243 dram_channels = evergreen_get_number_of_dram_channels(rdev);
ca7db22b 2244 else
c696e53f
AD		 2245 dram_channels = si_get_number_of_dram_channels(rdev);
2246
2247 /* watermark for high clocks */
2248 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2249 wm_high.yclk =
2250 radeon_dpm_get_mclk(rdev, false) * 10;
2251 wm_high.sclk =
2252 radeon_dpm_get_sclk(rdev, false) * 10;
2253 } else {
2254 wm_high.yclk = rdev->pm.current_mclk * 10;
2255 wm_high.sclk = rdev->pm.current_sclk * 10;
2256 }
2257
2258 wm_high.disp_clk = mode->clock;
2259 wm_high.src_width = mode->crtc_hdisplay;
2260 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2261 wm_high.blank_time = line_time - wm_high.active_time;
2262 wm_high.interlaced = false;
2263 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2264 wm_high.interlaced = true;
2265 wm_high.vsc = radeon_crtc->vsc;
2266 wm_high.vtaps = 1;
2267 if (radeon_crtc->rmx_type != RMX_OFF)
2268 wm_high.vtaps = 2;
2269 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2270 wm_high.lb_size = lb_size;
2271 wm_high.dram_channels = dram_channels;
2272 wm_high.num_heads = num_heads;
2273
2274 /* watermark for low clocks */
2275 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2276 wm_low.yclk =
2277 radeon_dpm_get_mclk(rdev, true) * 10;
2278 wm_low.sclk =
2279 radeon_dpm_get_sclk(rdev, true) * 10;
2280 } else {
2281 wm_low.yclk = rdev->pm.current_mclk * 10;
2282 wm_low.sclk = rdev->pm.current_sclk * 10;
2283 }
2284
2285 wm_low.disp_clk = mode->clock;
2286 wm_low.src_width = mode->crtc_hdisplay;
2287 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2288 wm_low.blank_time = line_time - wm_low.active_time;
2289 wm_low.interlaced = false;
2290 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2291 wm_low.interlaced = true;
2292 wm_low.vsc = radeon_crtc->vsc;
2293 wm_low.vtaps = 1;
2294 if (radeon_crtc->rmx_type != RMX_OFF)
2295 wm_low.vtaps = 2;
2296 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2297 wm_low.lb_size = lb_size;
2298 wm_low.dram_channels = dram_channels;
2299 wm_low.num_heads = num_heads;
43b3cd99
AD		 2300
2301 /* set for high clocks */
c696e53f 2302 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
43b3cd99 2303 /* set for low clocks */
c696e53f 2304 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
43b3cd99
AD		 2305
2306 /* possibly force display priority to high */
2307 /* should really do this at mode validation time... */
c696e53f
AD		 2308 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2309 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2310 !dce6_check_latency_hiding(&wm_high) ||
2311 (rdev->disp_priority == 2)) {
2312 DRM_DEBUG_KMS("force priority to high\n");
2313 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2314 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2315 }
2316 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2317 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2318 !dce6_check_latency_hiding(&wm_low) ||
43b3cd99
AD		 2319 (rdev->disp_priority == 2)) {
2320 DRM_DEBUG_KMS("force priority to high\n");
2321 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2322 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2323 }
2324
2325 a.full = dfixed_const(1000);
2326 b.full = dfixed_const(mode->clock);
2327 b.full = dfixed_div(b, a);
2328 c.full = dfixed_const(latency_watermark_a);
2329 c.full = dfixed_mul(c, b);
2330 c.full = dfixed_mul(c, radeon_crtc->hsc);
2331 c.full = dfixed_div(c, a);
2332 a.full = dfixed_const(16);
2333 c.full = dfixed_div(c, a);
2334 priority_a_mark = dfixed_trunc(c);
2335 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2336
2337 a.full = dfixed_const(1000);
2338 b.full = dfixed_const(mode->clock);
2339 b.full = dfixed_div(b, a);
2340 c.full = dfixed_const(latency_watermark_b);
2341 c.full = dfixed_mul(c, b);
2342 c.full = dfixed_mul(c, radeon_crtc->hsc);
2343 c.full = dfixed_div(c, a);
2344 a.full = dfixed_const(16);
2345 c.full = dfixed_div(c, a);
2346 priority_b_mark = dfixed_trunc(c);
2347 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2348 }
2349
2350 /* select wm A */
2351 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2352 tmp = arb_control3;
2353 tmp &= ~LATENCY_WATERMARK_MASK(3);
2354 tmp |= LATENCY_WATERMARK_MASK(1);
2355 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2356 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2357 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2358 LATENCY_HIGH_WATERMARK(line_time)));
2359 /* select wm B */
2360 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2361 tmp &= ~LATENCY_WATERMARK_MASK(3);
2362 tmp |= LATENCY_WATERMARK_MASK(2);
2363 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2364 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2365 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2366 LATENCY_HIGH_WATERMARK(line_time)));
2367 /* restore original selection */
2368 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2369
2370 /* write the priority marks */
2371 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2372 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2373
7178d2a6
AD		 2374 /* save values for DPM */
2375 radeon_crtc->line_time = line_time;
2376 radeon_crtc->wm_high = latency_watermark_a;
2377 radeon_crtc->wm_low = latency_watermark_b;
43b3cd99
AD		 2378}
2379
2380void dce6_bandwidth_update(struct radeon_device *rdev)
2381{
2382 struct drm_display_mode *mode0 = NULL;
2383 struct drm_display_mode *mode1 = NULL;
2384 u32 num_heads = 0, lb_size;
2385 int i;
2386
8efe82ca
AD		 2387 if (!rdev->mode_info.mode_config_initialized)
2388 return;
2389
43b3cd99
AD		 2390 radeon_update_display_priority(rdev);
2391
2392 for (i = 0; i < rdev->num_crtc; i++) {
2393 if (rdev->mode_info.crtcs[i]->base.enabled)
2394 num_heads++;
2395 }
2396 for (i = 0; i < rdev->num_crtc; i += 2) {
2397 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2398 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2399 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2400 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2401 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2402 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2403 }
2404}
2405
0a96d72b
AD		 2406/*
2407 * Core functions
2408 */
0a96d72b
AD		 2409static void si_tiling_mode_table_init(struct radeon_device *rdev)
2410{
2411 const u32 num_tile_mode_states = 32;
2412 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2413
2414 switch (rdev->config.si.mem_row_size_in_kb) {
2415 case 1:
2416 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2417 break;
2418 case 2:
2419 default:
2420 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2421 break;
2422 case 4:
2423 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2424 break;
2425 }
2426
2427 if ((rdev->family == CHIP_TAHITI) ||
2428 (rdev->family == CHIP_PITCAIRN)) {
2429 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2430 switch (reg_offset) {
2431 case 0: /* non-AA compressed depth or any compressed stencil */
2432 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2433 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2434 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2435 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2436 NUM_BANKS(ADDR_SURF_16_BANK) |
2437 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2438 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2439 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2440 break;
2441 case 1: /* 2xAA/4xAA compressed depth only */
2442 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2443 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2444 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2445 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2446 NUM_BANKS(ADDR_SURF_16_BANK) |
2447 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2448 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2449 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2450 break;
2451 case 2: /* 8xAA compressed depth only */
2452 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2453 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2454 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2455 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2456 NUM_BANKS(ADDR_SURF_16_BANK) |
2457 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2458 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2459 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2460 break;
2461 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2462 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2463 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2464 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2465 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2466 NUM_BANKS(ADDR_SURF_16_BANK) |
2467 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2468 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2469 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2470 break;
2471 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2472 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2473 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2474 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2475 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2476 NUM_BANKS(ADDR_SURF_16_BANK) |
2477 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2478 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2479 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2480 break;
2481 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2482 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2483 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2484 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2485 TILE_SPLIT(split_equal_to_row_size) |
2486 NUM_BANKS(ADDR_SURF_16_BANK) |
2487 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2488 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2489 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2490 break;
2491 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2492 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2493 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2494 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2495 TILE_SPLIT(split_equal_to_row_size) |
2496 NUM_BANKS(ADDR_SURF_16_BANK) |
2497 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2498 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2499 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2500 break;
2501 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2502 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2503 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2504 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2505 TILE_SPLIT(split_equal_to_row_size) |
2506 NUM_BANKS(ADDR_SURF_16_BANK) |
2507 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2508 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2509 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2510 break;
2511 case 8: /* 1D and 1D Array Surfaces */
2512 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2513 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2514 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2515 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2516 NUM_BANKS(ADDR_SURF_16_BANK) |
2517 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2518 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2519 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2520 break;
2521 case 9: /* Displayable maps. */
2522 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2523 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2524 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2525 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2526 NUM_BANKS(ADDR_SURF_16_BANK) |
2527 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2528 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2529 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2530 break;
2531 case 10: /* Display 8bpp. */
2532 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2533 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2534 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2535 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2536 NUM_BANKS(ADDR_SURF_16_BANK) |
2537 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2538 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2539 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2540 break;
2541 case 11: /* Display 16bpp. */
2542 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2543 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2544 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2545 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2546 NUM_BANKS(ADDR_SURF_16_BANK) |
2547 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2548 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2549 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2550 break;
2551 case 12: /* Display 32bpp. */
2552 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2553 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2554 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2555 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2556 NUM_BANKS(ADDR_SURF_16_BANK) |
2557 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2558 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2559 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2560 break;
2561 case 13: /* Thin. */
2562 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2563 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2564 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2565 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2566 NUM_BANKS(ADDR_SURF_16_BANK) |
2567 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2568 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2569 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2570 break;
2571 case 14: /* Thin 8 bpp. */
2572 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2573 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2574 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2575 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2576 NUM_BANKS(ADDR_SURF_16_BANK) |
2577 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2578 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2579 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2580 break;
2581 case 15: /* Thin 16 bpp. */
2582 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2583 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2584 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2585 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2586 NUM_BANKS(ADDR_SURF_16_BANK) |
2587 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2588 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2589 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2590 break;
2591 case 16: /* Thin 32 bpp. */
2592 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2593 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2594 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2595 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2596 NUM_BANKS(ADDR_SURF_16_BANK) |
2597 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2598 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2599 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2600 break;
2601 case 17: /* Thin 64 bpp. */
2602 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2603 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2604 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2605 TILE_SPLIT(split_equal_to_row_size) |
2606 NUM_BANKS(ADDR_SURF_16_BANK) |
2607 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2608 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2609 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2610 break;
2611 case 21: /* 8 bpp PRT. */
2612 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2613 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2614 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2615 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2616 NUM_BANKS(ADDR_SURF_16_BANK) |
2617 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2618 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2619 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2620 break;
2621 case 22: /* 16 bpp PRT */
2622 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2623 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2624 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2625 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2626 NUM_BANKS(ADDR_SURF_16_BANK) |
2627 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2628 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2629 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2630 break;
2631 case 23: /* 32 bpp PRT */
2632 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2633 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2634 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2635 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2636 NUM_BANKS(ADDR_SURF_16_BANK) |
2637 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2638 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2639 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2640 break;
2641 case 24: /* 64 bpp PRT */
2642 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2643 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2644 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2645 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2646 NUM_BANKS(ADDR_SURF_16_BANK) |
2647 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2648 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2649 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2650 break;
2651 case 25: /* 128 bpp PRT */
2652 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2653 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2654 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2655 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2656 NUM_BANKS(ADDR_SURF_8_BANK) |
2657 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2658 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2659 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2660 break;
2661 default:
2662 gb_tile_moden = 0;
2663 break;
2664 }
64d7b8be 2665 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2666 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2667 }
d0ae7fcc 2668 } else if ((rdev->family == CHIP_VERDE) ||
8b02859d
AD		 2669 (rdev->family == CHIP_OLAND) ||
2670 (rdev->family == CHIP_HAINAN)) {
0a96d72b
AD		 2671 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2672 switch (reg_offset) {
2673 case 0: /* non-AA compressed depth or any compressed stencil */
2674 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2675 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2676 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2677 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2678 NUM_BANKS(ADDR_SURF_16_BANK) |
2679 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2680 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2681 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2682 break;
2683 case 1: /* 2xAA/4xAA compressed depth only */
2684 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2685 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2686 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2687 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2688 NUM_BANKS(ADDR_SURF_16_BANK) |
2689 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2690 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2691 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2692 break;
2693 case 2: /* 8xAA compressed depth only */
2694 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2695 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2696 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2697 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2698 NUM_BANKS(ADDR_SURF_16_BANK) |
2699 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2700 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2701 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2702 break;
2703 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2704 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2705 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2706 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2707 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2708 NUM_BANKS(ADDR_SURF_16_BANK) |
2709 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2710 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2711 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2712 break;
2713 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2714 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2715 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2716 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2717 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2718 NUM_BANKS(ADDR_SURF_16_BANK) |
2719 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2720 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2721 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2722 break;
2723 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2724 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2725 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2726 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2727 TILE_SPLIT(split_equal_to_row_size) |
2728 NUM_BANKS(ADDR_SURF_16_BANK) |
2729 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2730 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2731 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2732 break;
2733 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2734 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2735 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2736 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2737 TILE_SPLIT(split_equal_to_row_size) |
2738 NUM_BANKS(ADDR_SURF_16_BANK) |
2739 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2740 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2741 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2742 break;
2743 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2744 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2745 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2746 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2747 TILE_SPLIT(split_equal_to_row_size) |
2748 NUM_BANKS(ADDR_SURF_16_BANK) |
2749 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2750 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2751 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2752 break;
2753 case 8: /* 1D and 1D Array Surfaces */
2754 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2755 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2756 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2757 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2758 NUM_BANKS(ADDR_SURF_16_BANK) |
2759 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2760 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2761 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2762 break;
2763 case 9: /* Displayable maps. */
2764 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2765 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2766 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2767 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2768 NUM_BANKS(ADDR_SURF_16_BANK) |
2769 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2770 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2771 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2772 break;
2773 case 10: /* Display 8bpp. */
2774 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2775 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2776 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2777 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2778 NUM_BANKS(ADDR_SURF_16_BANK) |
2779 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2780 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2781 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2782 break;
2783 case 11: /* Display 16bpp. */
2784 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2785 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2786 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2787 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2788 NUM_BANKS(ADDR_SURF_16_BANK) |
2789 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2790 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2791 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2792 break;
2793 case 12: /* Display 32bpp. */
2794 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2795 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2796 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2797 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2798 NUM_BANKS(ADDR_SURF_16_BANK) |
2799 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2800 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2801 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2802 break;
2803 case 13: /* Thin. */
2804 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2805 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2806 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2807 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2808 NUM_BANKS(ADDR_SURF_16_BANK) |
2809 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2810 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2811 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2812 break;
2813 case 14: /* Thin 8 bpp. */
2814 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2815 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2816 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2817 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2818 NUM_BANKS(ADDR_SURF_16_BANK) |
2819 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2820 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2821 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2822 break;
2823 case 15: /* Thin 16 bpp. */
2824 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2825 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2826 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2827 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2828 NUM_BANKS(ADDR_SURF_16_BANK) |
2829 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2830 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2831 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2832 break;
2833 case 16: /* Thin 32 bpp. */
2834 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2835 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2836 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2837 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2838 NUM_BANKS(ADDR_SURF_16_BANK) |
2839 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2840 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2841 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2842 break;
2843 case 17: /* Thin 64 bpp. */
2844 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2845 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2846 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2847 TILE_SPLIT(split_equal_to_row_size) |
2848 NUM_BANKS(ADDR_SURF_16_BANK) |
2849 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2850 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2851 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2852 break;
2853 case 21: /* 8 bpp PRT. */
2854 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2855 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2856 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2857 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2858 NUM_BANKS(ADDR_SURF_16_BANK) |
2859 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2860 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2861 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2862 break;
2863 case 22: /* 16 bpp PRT */
2864 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2865 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2866 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2867 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2868 NUM_BANKS(ADDR_SURF_16_BANK) |
2869 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2870 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2871 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2872 break;
2873 case 23: /* 32 bpp PRT */
2874 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2875 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2876 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2877 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2878 NUM_BANKS(ADDR_SURF_16_BANK) |
2879 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2880 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2881 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2882 break;
2883 case 24: /* 64 bpp PRT */
2884 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2885 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2886 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2887 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2888 NUM_BANKS(ADDR_SURF_16_BANK) |
2889 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2890 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2891 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2892 break;
2893 case 25: /* 128 bpp PRT */
2894 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2895 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2896 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2897 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2898 NUM_BANKS(ADDR_SURF_8_BANK) |
2899 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2900 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2901 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2902 break;
2903 default:
2904 gb_tile_moden = 0;
2905 break;
2906 }
64d7b8be 2907 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2908 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2909 }
2910 } else
2911 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2912}
2913
1a8ca750
AD		 2914static void si_select_se_sh(struct radeon_device *rdev,
2915 u32 se_num, u32 sh_num)
2916{
2917 u32 data = INSTANCE_BROADCAST_WRITES;
2918
2919 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
79b52d6a 2920 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
1a8ca750
AD		 2921 else if (se_num == 0xffffffff)
2922 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2923 else if (sh_num == 0xffffffff)
2924 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2925 else
2926 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2927 WREG32(GRBM_GFX_INDEX, data);
2928}
2929
2930static u32 si_create_bitmask(u32 bit_width)
2931{
2932 u32 i, mask = 0;
2933
2934 for (i = 0; i < bit_width; i++) {
2935 mask <<= 1;
2936 mask |= 1;
2937 }
2938 return mask;
2939}
2940
2941static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2942{
2943 u32 data, mask;
2944
2945 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2946 if (data & 1)
2947 data &= INACTIVE_CUS_MASK;
2948 else
2949 data = 0;
2950 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2951
2952 data >>= INACTIVE_CUS_SHIFT;
2953
2954 mask = si_create_bitmask(cu_per_sh);
2955
2956 return ~data & mask;
2957}
2958
2959static void si_setup_spi(struct radeon_device *rdev,
2960 u32 se_num, u32 sh_per_se,
2961 u32 cu_per_sh)
2962{
2963 int i, j, k;
2964 u32 data, mask, active_cu;
2965
2966 for (i = 0; i < se_num; i++) {
2967 for (j = 0; j < sh_per_se; j++) {
2968 si_select_se_sh(rdev, i, j);
2969 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2970 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2971
2972 mask = 1;
2973 for (k = 0; k < 16; k++) {
2974 mask <<= k;
2975 if (active_cu & mask) {
2976 data &= ~mask;
2977 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2978 break;
2979 }
2980 }
2981 }
2982 }
2983 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2984}
2985
2986static u32 si_get_rb_disabled(struct radeon_device *rdev,
9fadb352 2987 u32 max_rb_num_per_se,
1a8ca750
AD		 2988 u32 sh_per_se)
2989{
2990 u32 data, mask;
2991
2992 data = RREG32(CC_RB_BACKEND_DISABLE);
2993 if (data & 1)
2994 data &= BACKEND_DISABLE_MASK;
2995 else
2996 data = 0;
2997 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2998
2999 data >>= BACKEND_DISABLE_SHIFT;
3000
9fadb352 3001 mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
1a8ca750
AD		 3002
3003 return data & mask;
3004}
3005
3006static void si_setup_rb(struct radeon_device *rdev,
3007 u32 se_num, u32 sh_per_se,
9fadb352 3008 u32 max_rb_num_per_se)
1a8ca750
AD		 3009{
3010 int i, j;
3011 u32 data, mask;
3012 u32 disabled_rbs = 0;
3013 u32 enabled_rbs = 0;
3014
3015 for (i = 0; i < se_num; i++) {
3016 for (j = 0; j < sh_per_se; j++) {
3017 si_select_se_sh(rdev, i, j);
9fadb352 3018 data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
1a8ca750
AD		 3019 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
3020 }
3021 }
3022 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3023
3024 mask = 1;
9fadb352 3025 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
1a8ca750
AD		 3026 if (!(disabled_rbs & mask))
3027 enabled_rbs |= mask;
3028 mask <<= 1;
3029 }
3030
439a1cff
MO		 3031 rdev->config.si.backend_enable_mask = enabled_rbs;
3032
1a8ca750
AD		 3033 for (i = 0; i < se_num; i++) {
3034 si_select_se_sh(rdev, i, 0xffffffff);
3035 data = 0;
3036 for (j = 0; j < sh_per_se; j++) {
3037 switch (enabled_rbs & 3) {
3038 case 1:
3039 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3040 break;
3041 case 2:
3042 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3043 break;
3044 case 3:
3045 default:
3046 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3047 break;
3048 }
3049 enabled_rbs >>= 2;
3050 }
3051 WREG32(PA_SC_RASTER_CONFIG, data);
3052 }
3053 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3054}
3055
0a96d72b
AD		 3056static void si_gpu_init(struct radeon_device *rdev)
3057{
0a96d72b
AD		 3058 u32 gb_addr_config = 0;
3059 u32 mc_shared_chmap, mc_arb_ramcfg;
0a96d72b 3060 u32 sx_debug_1;
0a96d72b
AD		 3061 u32 hdp_host_path_cntl;
3062 u32 tmp;
6101b3ae 3063 int i, j;
0a96d72b
AD		 3064
3065 switch (rdev->family) {
3066 case CHIP_TAHITI:
3067 rdev->config.si.max_shader_engines = 2;
0a96d72b 3068 rdev->config.si.max_tile_pipes = 12;
1a8ca750
AD		 3069 rdev->config.si.max_cu_per_sh = 8;
3070 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 3071 rdev->config.si.max_backends_per_se = 4;
3072 rdev->config.si.max_texture_channel_caches = 12;
3073 rdev->config.si.max_gprs = 256;
3074 rdev->config.si.max_gs_threads = 32;
3075 rdev->config.si.max_hw_contexts = 8;
3076
3077 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3078 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3079 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3080 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 3081 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 3082 break;
3083 case CHIP_PITCAIRN:
3084 rdev->config.si.max_shader_engines = 2;
0a96d72b 3085 rdev->config.si.max_tile_pipes = 8;
1a8ca750
AD		 3086 rdev->config.si.max_cu_per_sh = 5;
3087 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 3088 rdev->config.si.max_backends_per_se = 4;
3089 rdev->config.si.max_texture_channel_caches = 8;
3090 rdev->config.si.max_gprs = 256;
3091 rdev->config.si.max_gs_threads = 32;
3092 rdev->config.si.max_hw_contexts = 8;
3093
3094 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3095 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3096 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3097 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 3098 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 3099 break;
3100 case CHIP_VERDE:
3101 default:
3102 rdev->config.si.max_shader_engines = 1;
0a96d72b 3103 rdev->config.si.max_tile_pipes = 4;
468ef1a5 3104 rdev->config.si.max_cu_per_sh = 5;
1a8ca750 3105 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 3106 rdev->config.si.max_backends_per_se = 4;
3107 rdev->config.si.max_texture_channel_caches = 4;
3108 rdev->config.si.max_gprs = 256;
3109 rdev->config.si.max_gs_threads = 32;
3110 rdev->config.si.max_hw_contexts = 8;
3111
d0ae7fcc
AD		 3112 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3113 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3114 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3115 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3116 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3117 break;
3118 case CHIP_OLAND:
3119 rdev->config.si.max_shader_engines = 1;
3120 rdev->config.si.max_tile_pipes = 4;
3121 rdev->config.si.max_cu_per_sh = 6;
3122 rdev->config.si.max_sh_per_se = 1;
3123 rdev->config.si.max_backends_per_se = 2;
3124 rdev->config.si.max_texture_channel_caches = 4;
3125 rdev->config.si.max_gprs = 256;
3126 rdev->config.si.max_gs_threads = 16;
3127 rdev->config.si.max_hw_contexts = 8;
3128
0a96d72b
AD		 3129 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3130 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3131 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3132 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 3133 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
0a96d72b 3134 break;
8b02859d
AD		 3135 case CHIP_HAINAN:
3136 rdev->config.si.max_shader_engines = 1;
3137 rdev->config.si.max_tile_pipes = 4;
3138 rdev->config.si.max_cu_per_sh = 5;
3139 rdev->config.si.max_sh_per_se = 1;
3140 rdev->config.si.max_backends_per_se = 1;
3141 rdev->config.si.max_texture_channel_caches = 2;
3142 rdev->config.si.max_gprs = 256;
3143 rdev->config.si.max_gs_threads = 16;
3144 rdev->config.si.max_hw_contexts = 8;
3145
3146 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3147 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3148 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3149 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3150 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
3151 break;
0a96d72b
AD		 3152 }
3153
3154 /* Initialize HDP */
3155 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3156 WREG32((0x2c14 + j), 0x00000000);
3157 WREG32((0x2c18 + j), 0x00000000);
3158 WREG32((0x2c1c + j), 0x00000000);
3159 WREG32((0x2c20 + j), 0x00000000);
3160 WREG32((0x2c24 + j), 0x00000000);
3161 }
3162
3163 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3164
3165 evergreen_fix_pci_max_read_req_size(rdev);
3166
3167 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3168
3169 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3170 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3171
0a96d72b 3172 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
0a96d72b
AD		 3173 rdev->config.si.mem_max_burst_length_bytes = 256;
3174 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3175 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3176 if (rdev->config.si.mem_row_size_in_kb > 4)
3177 rdev->config.si.mem_row_size_in_kb = 4;
3178 /* XXX use MC settings? */
3179 rdev->config.si.shader_engine_tile_size = 32;
3180 rdev->config.si.num_gpus = 1;
3181 rdev->config.si.multi_gpu_tile_size = 64;
3182
1a8ca750
AD		 3183 /* fix up row size */
3184 gb_addr_config &= ~ROW_SIZE_MASK;
0a96d72b
AD		 3185 switch (rdev->config.si.mem_row_size_in_kb) {
3186 case 1:
3187 default:
3188 gb_addr_config |= ROW_SIZE(0);
3189 break;
3190 case 2:
3191 gb_addr_config |= ROW_SIZE(1);
3192 break;
3193 case 4:
3194 gb_addr_config |= ROW_SIZE(2);
3195 break;
3196 }
3197
0a96d72b
AD		 3198 /* setup tiling info dword. gb_addr_config is not adequate since it does
3199 * not have bank info, so create a custom tiling dword.
3200 * bits 3:0 num_pipes
3201 * bits 7:4 num_banks
3202 * bits 11:8 group_size
3203 * bits 15:12 row_size
3204 */
3205 rdev->config.si.tile_config = 0;
3206 switch (rdev->config.si.num_tile_pipes) {
3207 case 1:
3208 rdev->config.si.tile_config |= (0 << 0);
3209 break;
3210 case 2:
3211 rdev->config.si.tile_config |= (1 << 0);
3212 break;
3213 case 4:
3214 rdev->config.si.tile_config |= (2 << 0);
3215 break;
3216 case 8:
3217 default:
3218 /* XXX what about 12? */
3219 rdev->config.si.tile_config |= (3 << 0);
3220 break;
dca571a6
CK		 3221 }
3222 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3223 case 0: /* four banks */
1a8ca750 3224 rdev->config.si.tile_config |= 0 << 4;
dca571a6
CK		 3225 break;
3226 case 1: /* eight banks */
3227 rdev->config.si.tile_config |= 1 << 4;
3228 break;
3229 case 2: /* sixteen banks */
3230 default:
3231 rdev->config.si.tile_config |= 2 << 4;
3232 break;
3233 }
0a96d72b
AD		 3234 rdev->config.si.tile_config |=
3235 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3236 rdev->config.si.tile_config |=
3237 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3238
0a96d72b
AD		 3239 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3240 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
7c1c7c18 3241 WREG32(DMIF_ADDR_CALC, gb_addr_config);
0a96d72b 3242 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
8c5fd7ef
AD		 3243 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3244 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1df0d523
AD		 3245 if (rdev->has_uvd) {
3246 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3247 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3248 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3249 }
0a96d72b 3250
1a8ca750 3251 si_tiling_mode_table_init(rdev);
0a96d72b 3252
1a8ca750
AD		 3253 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3254 rdev->config.si.max_sh_per_se,
3255 rdev->config.si.max_backends_per_se);
0a96d72b 3256
1a8ca750
AD		 3257 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3258 rdev->config.si.max_sh_per_se,
3259 rdev->config.si.max_cu_per_sh);
0a96d72b 3260
52da51f0 3261 rdev->config.si.active_cus = 0;
65fcf668
AD		 3262 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
3263 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
6101b3ae
AD		 3264 rdev->config.si.active_cus +=
3265 hweight32(si_get_cu_active_bitmap(rdev, i, j));
65fcf668
AD		 3266 }
3267 }
0a96d72b
AD		 3268
3269 /* set HW defaults for 3D engine */
3270 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3271 ROQ_IB2_START(0x2b)));
3272 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3273
3274 sx_debug_1 = RREG32(SX_DEBUG_1);
3275 WREG32(SX_DEBUG_1, sx_debug_1);
3276
3277 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3278
3279 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3280 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3281 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3282 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3283
3284 WREG32(VGT_NUM_INSTANCES, 1);
3285
3286 WREG32(CP_PERFMON_CNTL, 0);
3287
3288 WREG32(SQ_CONFIG, 0);
3289
3290 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3291 FORCE_EOV_MAX_REZ_CNT(255)));
3292
3293 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3294 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3295
3296 WREG32(VGT_GS_VERTEX_REUSE, 16);
3297 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3298
3299 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3300 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3301 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3302 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3303 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3304 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3305 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3306 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3307
3308 tmp = RREG32(HDP_MISC_CNTL);
3309 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3310 WREG32(HDP_MISC_CNTL, tmp);
3311
3312 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3313 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3314
3315 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3316
3317 udelay(50);
3318}
c476dde2 3319
2ece2e8b
AD		 3320/*
3321 * GPU scratch registers helpers function.
3322 */
3323static void si_scratch_init(struct radeon_device *rdev)
3324{
3325 int i;
3326
3327 rdev->scratch.num_reg = 7;
3328 rdev->scratch.reg_base = SCRATCH_REG0;
3329 for (i = 0; i < rdev->scratch.num_reg; i++) {
3330 rdev->scratch.free[i] = true;
3331 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3332 }
3333}
3334
3335void si_fence_ring_emit(struct radeon_device *rdev,
3336 struct radeon_fence *fence)
3337{
3338 struct radeon_ring *ring = &rdev->ring[fence->ring];
3339 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3340
3341 /* flush read cache over gart */
3342 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3343 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3344 radeon_ring_write(ring, 0);
3345 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3346 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3347 PACKET3_TC_ACTION_ENA |
3348 PACKET3_SH_KCACHE_ACTION_ENA |
3349 PACKET3_SH_ICACHE_ACTION_ENA);
3350 radeon_ring_write(ring, 0xFFFFFFFF);
3351 radeon_ring_write(ring, 0);
3352 radeon_ring_write(ring, 10); /* poll interval */
3353 /* EVENT_WRITE_EOP - flush caches, send int */
3354 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3355 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
5e167cdb 3356 radeon_ring_write(ring, lower_32_bits(addr));
2ece2e8b
AD		 3357 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3358 radeon_ring_write(ring, fence->seq);
3359 radeon_ring_write(ring, 0);
3360}
3361
3362/*
3363 * IB stuff
3364 */
3365void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3366{
876dc9f3 3367 struct radeon_ring *ring = &rdev->ring[ib->ring];
2ece2e8b
AD		 3368 u32 header;
3369
a85a7da4
AD		 3370 if (ib->is_const_ib) {
3371 /* set switch buffer packet before const IB */
3372 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3373 radeon_ring_write(ring, 0);
45df6803 3374
2ece2e8b 3375 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
a85a7da4 3376 } else {
89d35807 3377 u32 next_rptr;
a85a7da4 3378 if (ring->rptr_save_reg) {
89d35807 3379 next_rptr = ring->wptr + 3 + 4 + 8;
a85a7da4
AD		 3380 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3381 radeon_ring_write(ring, ((ring->rptr_save_reg -
3382 PACKET3_SET_CONFIG_REG_START) >> 2));
3383 radeon_ring_write(ring, next_rptr);
89d35807
AD		 3384 } else if (rdev->wb.enabled) {
3385 next_rptr = ring->wptr + 5 + 4 + 8;
3386 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3387 radeon_ring_write(ring, (1 << 8));
3388 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
5e167cdb 3389 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
89d35807 3390 radeon_ring_write(ring, next_rptr);
a85a7da4
AD		 3391 }
3392
2ece2e8b 3393 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
a85a7da4 3394 }
2ece2e8b
AD		 3395
3396 radeon_ring_write(ring, header);
3397 radeon_ring_write(ring,
3398#ifdef __BIG_ENDIAN
3399 (2 << 0) |
3400#endif
3401 (ib->gpu_addr & 0xFFFFFFFC));
3402 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4bf3dd92
CK		 3403 radeon_ring_write(ring, ib->length_dw |
3404 (ib->vm ? (ib->vm->id << 24) : 0));
2ece2e8b 3405
a85a7da4
AD		 3406 if (!ib->is_const_ib) {
3407 /* flush read cache over gart for this vmid */
3408 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3409 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
4bf3dd92 3410 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
a85a7da4
AD		 3411 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3412 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3413 PACKET3_TC_ACTION_ENA |
3414 PACKET3_SH_KCACHE_ACTION_ENA |
3415 PACKET3_SH_ICACHE_ACTION_ENA);
3416 radeon_ring_write(ring, 0xFFFFFFFF);
3417 radeon_ring_write(ring, 0);
3418 radeon_ring_write(ring, 10); /* poll interval */
3419 }
2ece2e8b
AD		 3420}
3421
48c0c902
AD		 3422/*
3423 * CP.
3424 */
3425static void si_cp_enable(struct radeon_device *rdev, bool enable)
3426{
3427 if (enable)
3428 WREG32(CP_ME_CNTL, 0);
3429 else {
50efa51a
AD		 3430 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3431 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
48c0c902
AD		 3432 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3433 WREG32(SCRATCH_UMSK, 0);
8c5fd7ef
AD		 3434 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3435 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3436 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
48c0c902
AD		 3437 }
3438 udelay(50);
3439}
3440
3441static int si_cp_load_microcode(struct radeon_device *rdev)
3442{
48c0c902
AD		 3443 int i;
3444
629bd33c 3445 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
48c0c902
AD		 3446 return -EINVAL;
3447
3448 si_cp_enable(rdev, false);
3449
629bd33c
AD		 3450 if (rdev->new_fw) {
3451 const struct gfx_firmware_header_v1_0 *pfp_hdr =
3452 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3453 const struct gfx_firmware_header_v1_0 *ce_hdr =
3454 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3455 const struct gfx_firmware_header_v1_0 *me_hdr =
3456 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3457 const __le32 *fw_data;
3458 u32 fw_size;
3459
3460 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3461 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3462 radeon_ucode_print_gfx_hdr(&me_hdr->header);
3463
3464 /* PFP */
3465 fw_data = (const __le32 *)
3466 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3467 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3468 WREG32(CP_PFP_UCODE_ADDR, 0);
3469 for (i = 0; i < fw_size; i++)
3470 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3471 WREG32(CP_PFP_UCODE_ADDR, 0);
3472
3473 /* CE */
3474 fw_data = (const __le32 *)
3475 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3476 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3477 WREG32(CP_CE_UCODE_ADDR, 0);
3478 for (i = 0; i < fw_size; i++)
3479 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3480 WREG32(CP_CE_UCODE_ADDR, 0);
3481
3482 /* ME */
3483 fw_data = (const __be32 *)
3484 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3485 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3486 WREG32(CP_ME_RAM_WADDR, 0);
3487 for (i = 0; i < fw_size; i++)
3488 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3489 WREG32(CP_ME_RAM_WADDR, 0);
3490 } else {
3491 const __be32 *fw_data;
3492
3493 /* PFP */
3494 fw_data = (const __be32 *)rdev->pfp_fw->data;
3495 WREG32(CP_PFP_UCODE_ADDR, 0);
3496 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3497 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3498 WREG32(CP_PFP_UCODE_ADDR, 0);
3499
3500 /* CE */
3501 fw_data = (const __be32 *)rdev->ce_fw->data;
3502 WREG32(CP_CE_UCODE_ADDR, 0);
3503 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3504 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3505 WREG32(CP_CE_UCODE_ADDR, 0);
3506
3507 /* ME */
3508 fw_data = (const __be32 *)rdev->me_fw->data;
3509 WREG32(CP_ME_RAM_WADDR, 0);
3510 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3511 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3512 WREG32(CP_ME_RAM_WADDR, 0);
3513 }
48c0c902
AD		 3514
3515 WREG32(CP_PFP_UCODE_ADDR, 0);
3516 WREG32(CP_CE_UCODE_ADDR, 0);
3517 WREG32(CP_ME_RAM_WADDR, 0);
3518 WREG32(CP_ME_RAM_RADDR, 0);
3519 return 0;
3520}
3521
3522static int si_cp_start(struct radeon_device *rdev)
3523{
3524 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3525 int r, i;
3526
3527 r = radeon_ring_lock(rdev, ring, 7 + 4);
3528 if (r) {
3529 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3530 return r;
3531 }
3532 /* init the CP */
3533 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3534 radeon_ring_write(ring, 0x1);
3535 radeon_ring_write(ring, 0x0);
3536 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3537 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3538 radeon_ring_write(ring, 0);
3539 radeon_ring_write(ring, 0);
3540
3541 /* init the CE partitions */
3542 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3543 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3544 radeon_ring_write(ring, 0xc000);
3545 radeon_ring_write(ring, 0xe000);
1538a9e0 3546 radeon_ring_unlock_commit(rdev, ring, false);
48c0c902
AD		 3547
3548 si_cp_enable(rdev, true);
3549
3550 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3551 if (r) {
3552 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3553 return r;
3554 }
3555
3556 /* setup clear context state */
3557 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3558 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3559
3560 for (i = 0; i < si_default_size; i++)
3561 radeon_ring_write(ring, si_default_state[i]);
3562
3563 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3564 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3565
3566 /* set clear context state */
3567 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3568 radeon_ring_write(ring, 0);
3569
3570 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3571 radeon_ring_write(ring, 0x00000316);
3572 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3573 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3574
1538a9e0 3575 radeon_ring_unlock_commit(rdev, ring, false);
48c0c902
AD		 3576
3577 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3578 ring = &rdev->ring[i];
3579 r = radeon_ring_lock(rdev, ring, 2);
3580
3581 /* clear the compute context state */
3582 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3583 radeon_ring_write(ring, 0);
3584
1538a9e0 3585 radeon_ring_unlock_commit(rdev, ring, false);
48c0c902
AD		 3586 }
3587
3588 return 0;
3589}
3590
3591static void si_cp_fini(struct radeon_device *rdev)
3592{
45df6803 3593 struct radeon_ring *ring;
48c0c902 3594 si_cp_enable(rdev, false);
45df6803
CK		 3595
3596 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3597 radeon_ring_fini(rdev, ring);
3598 radeon_scratch_free(rdev, ring->rptr_save_reg);
3599
3600 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3601 radeon_ring_fini(rdev, ring);
3602 radeon_scratch_free(rdev, ring->rptr_save_reg);
3603
3604 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3605 radeon_ring_fini(rdev, ring);
3606 radeon_scratch_free(rdev, ring->rptr_save_reg);
48c0c902
AD		 3607}
3608
3609static int si_cp_resume(struct radeon_device *rdev)
3610{
3611 struct radeon_ring *ring;
3612 u32 tmp;
3613 u32 rb_bufsz;
3614 int r;
3615
811e4d58
AD		 3616 si_enable_gui_idle_interrupt(rdev, false);
3617
48c0c902
AD		 3618 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3619 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3620
3621 /* Set the write pointer delay */
3622 WREG32(CP_RB_WPTR_DELAY, 0);
3623
3624 WREG32(CP_DEBUG, 0);
3625 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3626
3627 /* ring 0 - compute and gfx */
3628 /* Set ring buffer size */
3629 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 3630 rb_bufsz = order_base_2(ring->ring_size / 8);
3631 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
48c0c902
AD		 3632#ifdef __BIG_ENDIAN
3633 tmp |= BUF_SWAP_32BIT;
3634#endif
3635 WREG32(CP_RB0_CNTL, tmp);
3636
3637 /* Initialize the ring buffer's read and write pointers */
3638 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3639 ring->wptr = 0;
3640 WREG32(CP_RB0_WPTR, ring->wptr);
3641
48fc7f7e 3642 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3643 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3644 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3645
3646 if (rdev->wb.enabled)
3647 WREG32(SCRATCH_UMSK, 0xff);
3648 else {
3649 tmp |= RB_NO_UPDATE;
3650 WREG32(SCRATCH_UMSK, 0);
3651 }
3652
3653 mdelay(1);
3654 WREG32(CP_RB0_CNTL, tmp);
3655
3656 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3657
48c0c902
AD		 3658 /* ring1 - compute only */
3659 /* Set ring buffer size */
3660 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
b72a8925
DV		 3661 rb_bufsz = order_base_2(ring->ring_size / 8);
3662 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
48c0c902
AD		 3663#ifdef __BIG_ENDIAN
3664 tmp |= BUF_SWAP_32BIT;
3665#endif
3666 WREG32(CP_RB1_CNTL, tmp);
3667
3668 /* Initialize the ring buffer's read and write pointers */
3669 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3670 ring->wptr = 0;
3671 WREG32(CP_RB1_WPTR, ring->wptr);
3672
48fc7f7e 3673 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3674 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3675 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3676
3677 mdelay(1);
3678 WREG32(CP_RB1_CNTL, tmp);
3679
3680 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3681
48c0c902
AD		 3682 /* ring2 - compute only */
3683 /* Set ring buffer size */
3684 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
b72a8925
DV		 3685 rb_bufsz = order_base_2(ring->ring_size / 8);
3686 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
48c0c902
AD		 3687#ifdef __BIG_ENDIAN
3688 tmp |= BUF_SWAP_32BIT;
3689#endif
3690 WREG32(CP_RB2_CNTL, tmp);
3691
3692 /* Initialize the ring buffer's read and write pointers */
3693 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3694 ring->wptr = 0;
3695 WREG32(CP_RB2_WPTR, ring->wptr);
3696
48fc7f7e 3697 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3698 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3699 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3700
3701 mdelay(1);
3702 WREG32(CP_RB2_CNTL, tmp);
3703
3704 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3705
48c0c902
AD		 3706 /* start the rings */
3707 si_cp_start(rdev);
3708 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3709 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3710 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3711 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3712 if (r) {
3713 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3714 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3715 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3716 return r;
3717 }
3718 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3719 if (r) {
3720 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3721 }
3722 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3723 if (r) {
3724 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3725 }
3726
811e4d58
AD		 3727 si_enable_gui_idle_interrupt(rdev, true);
3728
50efa51a
AD		 3729 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3730 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3731
48c0c902
AD		 3732 return 0;
3733}
3734
2483b4ea 3735u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
06bc6df0 3736{
014bb209 3737 u32 reset_mask = 0;
1c534671 3738 u32 tmp;
06bc6df0 3739
014bb209
AD		 3740 /* GRBM_STATUS */
3741 tmp = RREG32(GRBM_STATUS);
3742 if (tmp & (PA_BUSY | SC_BUSY |
3743 BCI_BUSY | SX_BUSY |
3744 TA_BUSY | VGT_BUSY |
3745 DB_BUSY | CB_BUSY |
3746 GDS_BUSY | SPI_BUSY |
3747 IA_BUSY | IA_BUSY_NO_DMA))
3748 reset_mask |= RADEON_RESET_GFX;
3749
3750 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3751 CP_BUSY | CP_COHERENCY_BUSY))
3752 reset_mask |= RADEON_RESET_CP;
3753
3754 if (tmp & GRBM_EE_BUSY)
3755 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3756
3757 /* GRBM_STATUS2 */
3758 tmp = RREG32(GRBM_STATUS2);
3759 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3760 reset_mask |= RADEON_RESET_RLC;
3761
3762 /* DMA_STATUS_REG 0 */
3763 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3764 if (!(tmp & DMA_IDLE))
3765 reset_mask |= RADEON_RESET_DMA;
3766
3767 /* DMA_STATUS_REG 1 */
3768 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3769 if (!(tmp & DMA_IDLE))
3770 reset_mask |= RADEON_RESET_DMA1;
3771
3772 /* SRBM_STATUS2 */
3773 tmp = RREG32(SRBM_STATUS2);
3774 if (tmp & DMA_BUSY)
3775 reset_mask |= RADEON_RESET_DMA;
3776
3777 if (tmp & DMA1_BUSY)
3778 reset_mask |= RADEON_RESET_DMA1;
3779
3780 /* SRBM_STATUS */
3781 tmp = RREG32(SRBM_STATUS);
3782
3783 if (tmp & IH_BUSY)
3784 reset_mask |= RADEON_RESET_IH;
3785
3786 if (tmp & SEM_BUSY)
3787 reset_mask |= RADEON_RESET_SEM;
3788
3789 if (tmp & GRBM_RQ_PENDING)
3790 reset_mask |= RADEON_RESET_GRBM;
3791
3792 if (tmp & VMC_BUSY)
3793 reset_mask |= RADEON_RESET_VMC;
19fc42ed 3794
014bb209
AD		 3795 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3796 MCC_BUSY | MCD_BUSY))
3797 reset_mask |= RADEON_RESET_MC;
3798
3799 if (evergreen_is_display_hung(rdev))
3800 reset_mask |= RADEON_RESET_DISPLAY;
3801
3802 /* VM_L2_STATUS */
3803 tmp = RREG32(VM_L2_STATUS);
3804 if (tmp & L2_BUSY)
3805 reset_mask |= RADEON_RESET_VMC;
3806
d808fc88
AD		 3807 /* Skip MC reset as it's mostly likely not hung, just busy */
3808 if (reset_mask & RADEON_RESET_MC) {
3809 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3810 reset_mask &= ~RADEON_RESET_MC;
3811 }
3812
014bb209
AD		 3813 return reset_mask;
3814}
3815
3816static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3817{
3818 struct evergreen_mc_save save;
3819 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3820 u32 tmp;
19fc42ed 3821
06bc6df0 3822 if (reset_mask == 0)
014bb209 3823 return;
06bc6df0
AD		 3824
3825 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3826
1c534671 3827 evergreen_print_gpu_status_regs(rdev);
06bc6df0
AD		 3828 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3829 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3830 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3831 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3832
a6f4ae8d
AD		 3833 /* disable PG/CG */
3834 si_fini_pg(rdev);
3835 si_fini_cg(rdev);
3836
3837 /* stop the rlc */
3838 si_rlc_stop(rdev);
3839
1c534671
AD		 3840 /* Disable CP parsing/prefetching */
3841 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3842
3843 if (reset_mask & RADEON_RESET_DMA) {
3844 /* dma0 */
3845 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3846 tmp &= ~DMA_RB_ENABLE;
3847 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
014bb209
AD		 3848 }
3849 if (reset_mask & RADEON_RESET_DMA1) {
1c534671
AD		 3850 /* dma1 */
3851 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3852 tmp &= ~DMA_RB_ENABLE;
3853 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3854 }
3855
f770d78a
AD		 3856 udelay(50);
3857
3858 evergreen_mc_stop(rdev, &save);
3859 if (evergreen_mc_wait_for_idle(rdev)) {
3860 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3861 }
3862
1c534671
AD		 3863 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3864 grbm_soft_reset = SOFT_RESET_CB |
3865 SOFT_RESET_DB |
3866 SOFT_RESET_GDS |
3867 SOFT_RESET_PA |
3868 SOFT_RESET_SC |
3869 SOFT_RESET_BCI |
3870 SOFT_RESET_SPI |
3871 SOFT_RESET_SX |
3872 SOFT_RESET_TC |
3873 SOFT_RESET_TA |
3874 SOFT_RESET_VGT |
3875 SOFT_RESET_IA;
3876 }
3877
3878 if (reset_mask & RADEON_RESET_CP) {
3879 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3880
3881 srbm_soft_reset |= SOFT_RESET_GRBM;
3882 }
06bc6df0
AD		 3883
3884 if (reset_mask & RADEON_RESET_DMA)
014bb209
AD		 3885 srbm_soft_reset |= SOFT_RESET_DMA;
3886
3887 if (reset_mask & RADEON_RESET_DMA1)
3888 srbm_soft_reset |= SOFT_RESET_DMA1;
3889
3890 if (reset_mask & RADEON_RESET_DISPLAY)
3891 srbm_soft_reset |= SOFT_RESET_DC;
3892
3893 if (reset_mask & RADEON_RESET_RLC)
3894 grbm_soft_reset |= SOFT_RESET_RLC;
3895
3896 if (reset_mask & RADEON_RESET_SEM)
3897 srbm_soft_reset |= SOFT_RESET_SEM;
3898
3899 if (reset_mask & RADEON_RESET_IH)
3900 srbm_soft_reset |= SOFT_RESET_IH;
3901
3902 if (reset_mask & RADEON_RESET_GRBM)
3903 srbm_soft_reset |= SOFT_RESET_GRBM;
3904
3905 if (reset_mask & RADEON_RESET_VMC)
3906 srbm_soft_reset |= SOFT_RESET_VMC;
3907
3908 if (reset_mask & RADEON_RESET_MC)
3909 srbm_soft_reset |= SOFT_RESET_MC;
1c534671
AD		 3910
3911 if (grbm_soft_reset) {
3912 tmp = RREG32(GRBM_SOFT_RESET);
3913 tmp |= grbm_soft_reset;
3914 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3915 WREG32(GRBM_SOFT_RESET, tmp);
3916 tmp = RREG32(GRBM_SOFT_RESET);
3917
3918 udelay(50);
3919
3920 tmp &= ~grbm_soft_reset;
3921 WREG32(GRBM_SOFT_RESET, tmp);
3922 tmp = RREG32(GRBM_SOFT_RESET);
3923 }
3924
3925 if (srbm_soft_reset) {
3926 tmp = RREG32(SRBM_SOFT_RESET);
3927 tmp |= srbm_soft_reset;
3928 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3929 WREG32(SRBM_SOFT_RESET, tmp);
3930 tmp = RREG32(SRBM_SOFT_RESET);
3931
3932 udelay(50);
3933
3934 tmp &= ~srbm_soft_reset;
3935 WREG32(SRBM_SOFT_RESET, tmp);
3936 tmp = RREG32(SRBM_SOFT_RESET);
3937 }
06bc6df0
AD		 3938
3939 /* Wait a little for things to settle down */
3940 udelay(50);
3941
c476dde2 3942 evergreen_mc_resume(rdev, &save);
1c534671
AD		 3943 udelay(50);
3944
1c534671 3945 evergreen_print_gpu_status_regs(rdev);
c476dde2
AD		 3946}
3947
4a5c8ea5
AD		 3948static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3949{
3950 u32 tmp, i;
3951
3952 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3953 tmp |= SPLL_BYPASS_EN;
3954 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3955
3956 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3957 tmp |= SPLL_CTLREQ_CHG;
3958 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3959
3960 for (i = 0; i < rdev->usec_timeout; i++) {
3961 if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3962 break;
3963 udelay(1);
3964 }
3965
3966 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3967 tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
3968 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3969
3970 tmp = RREG32(MPLL_CNTL_MODE);
3971 tmp &= ~MPLL_MCLK_SEL;
3972 WREG32(MPLL_CNTL_MODE, tmp);
3973}
3974
3975static void si_spll_powerdown(struct radeon_device *rdev)
3976{
3977 u32 tmp;
3978
3979 tmp = RREG32(SPLL_CNTL_MODE);
3980 tmp |= SPLL_SW_DIR_CONTROL;
3981 WREG32(SPLL_CNTL_MODE, tmp);
3982
3983 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3984 tmp |= SPLL_RESET;
3985 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3986
3987 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3988 tmp |= SPLL_SLEEP;
3989 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3990
3991 tmp = RREG32(SPLL_CNTL_MODE);
3992 tmp &= ~SPLL_SW_DIR_CONTROL;
3993 WREG32(SPLL_CNTL_MODE, tmp);
3994}
3995
3996static void si_gpu_pci_config_reset(struct radeon_device *rdev)
3997{
3998 struct evergreen_mc_save save;
3999 u32 tmp, i;
4000
4001 dev_info(rdev->dev, "GPU pci config reset\n");
4002
4003 /* disable dpm? */
4004
4005 /* disable cg/pg */
4006 si_fini_pg(rdev);
4007 si_fini_cg(rdev);
4008
4009 /* Disable CP parsing/prefetching */
4010 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4011 /* dma0 */
4012 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
4013 tmp &= ~DMA_RB_ENABLE;
4014 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
4015 /* dma1 */
4016 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
4017 tmp &= ~DMA_RB_ENABLE;
4018 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
4019 /* XXX other engines? */
4020
4021 /* halt the rlc, disable cp internal ints */
4022 si_rlc_stop(rdev);
4023
4024 udelay(50);
4025
4026 /* disable mem access */
4027 evergreen_mc_stop(rdev, &save);
4028 if (evergreen_mc_wait_for_idle(rdev)) {
4029 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4030 }
4031
4032 /* set mclk/sclk to bypass */
4033 si_set_clk_bypass_mode(rdev);
4034 /* powerdown spll */
4035 si_spll_powerdown(rdev);
4036 /* disable BM */
4037 pci_clear_master(rdev->pdev);
4038 /* reset */
4039 radeon_pci_config_reset(rdev);
4040 /* wait for asic to come out of reset */
4041 for (i = 0; i < rdev->usec_timeout; i++) {
4042 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4043 break;
4044 udelay(1);
4045 }
4046}
4047
c476dde2
AD		 4048int si_asic_reset(struct radeon_device *rdev)
4049{
014bb209
AD		 4050 u32 reset_mask;
4051
4052 reset_mask = si_gpu_check_soft_reset(rdev);
4053
4054 if (reset_mask)
4055 r600_set_bios_scratch_engine_hung(rdev, true);
4056
4a5c8ea5 4057 /* try soft reset */
014bb209
AD		 4058 si_gpu_soft_reset(rdev, reset_mask);
4059
4060 reset_mask = si_gpu_check_soft_reset(rdev);
4061
4a5c8ea5
AD		 4062 /* try pci config reset */
4063 if (reset_mask && radeon_hard_reset)
4064 si_gpu_pci_config_reset(rdev);
4065
4066 reset_mask = si_gpu_check_soft_reset(rdev);
4067
014bb209
AD		 4068 if (!reset_mask)
4069 r600_set_bios_scratch_engine_hung(rdev, false);
4070
4071 return 0;
c476dde2
AD		 4072}
4073
123bc183
AD		 4074/**
4075 * si_gfx_is_lockup - Check if the GFX engine is locked up
4076 *
4077 * @rdev: radeon_device pointer
4078 * @ring: radeon_ring structure holding ring information
4079 *
4080 * Check if the GFX engine is locked up.
4081 * Returns true if the engine appears to be locked up, false if not.
4082 */
4083bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4084{
4085 u32 reset_mask = si_gpu_check_soft_reset(rdev);
4086
4087 if (!(reset_mask & (RADEON_RESET_GFX |
4088 RADEON_RESET_COMPUTE |
4089 RADEON_RESET_CP))) {
ff212f25 4090 radeon_ring_lockup_update(rdev, ring);
123bc183
AD		 4091 return false;
4092 }
123bc183
AD		 4093 return radeon_ring_test_lockup(rdev, ring);
4094}
4095
d2800ee5
AD		 4096/* MC */
4097static void si_mc_program(struct radeon_device *rdev)
4098{
4099 struct evergreen_mc_save save;
4100 u32 tmp;
4101 int i, j;
4102
4103 /* Initialize HDP */
4104 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4105 WREG32((0x2c14 + j), 0x00000000);
4106 WREG32((0x2c18 + j), 0x00000000);
4107 WREG32((0x2c1c + j), 0x00000000);
4108 WREG32((0x2c20 + j), 0x00000000);
4109 WREG32((0x2c24 + j), 0x00000000);
4110 }
4111 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4112
4113 evergreen_mc_stop(rdev, &save);
4114 if (radeon_mc_wait_for_idle(rdev)) {
4115 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4116 }
5153550a
AD		 4117 if (!ASIC_IS_NODCE(rdev))
4118 /* Lockout access through VGA aperture*/
4119 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
d2800ee5
AD		 4120 /* Update configuration */
4121 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4122 rdev->mc.vram_start >> 12);
4123 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4124 rdev->mc.vram_end >> 12);
4125 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4126 rdev->vram_scratch.gpu_addr >> 12);
4127 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4128 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4129 WREG32(MC_VM_FB_LOCATION, tmp);
4130 /* XXX double check these! */
4131 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4132 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4133 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4134 WREG32(MC_VM_AGP_BASE, 0);
4135 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4136 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4137 if (radeon_mc_wait_for_idle(rdev)) {
4138 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4139 }
4140 evergreen_mc_resume(rdev, &save);
5153550a
AD		 4141 if (!ASIC_IS_NODCE(rdev)) {
4142 /* we need to own VRAM, so turn off the VGA renderer here
4143 * to stop it overwriting our objects */
4144 rv515_vga_render_disable(rdev);
4145 }
d2800ee5
AD		 4146}
4147
1c49165d
AD		 4148void si_vram_gtt_location(struct radeon_device *rdev,
4149 struct radeon_mc *mc)
d2800ee5
AD		 4150{
4151 if (mc->mc_vram_size > 0xFFC0000000ULL) {
4152 /* leave room for at least 1024M GTT */
4153 dev_warn(rdev->dev, "limiting VRAM\n");
4154 mc->real_vram_size = 0xFFC0000000ULL;
4155 mc->mc_vram_size = 0xFFC0000000ULL;
4156 }
9ed8b1f9 4157 radeon_vram_location(rdev, &rdev->mc, 0);
d2800ee5 4158 rdev->mc.gtt_base_align = 0;
9ed8b1f9 4159 radeon_gtt_location(rdev, mc);
d2800ee5
AD		 4160}
4161
4162static int si_mc_init(struct radeon_device *rdev)
4163{
4164 u32 tmp;
4165 int chansize, numchan;
4166
4167 /* Get VRAM informations */
4168 rdev->mc.vram_is_ddr = true;
4169 tmp = RREG32(MC_ARB_RAMCFG);
4170 if (tmp & CHANSIZE_OVERRIDE) {
4171 chansize = 16;
4172 } else if (tmp & CHANSIZE_MASK) {
4173 chansize = 64;
4174 } else {
4175 chansize = 32;
4176 }
4177 tmp = RREG32(MC_SHARED_CHMAP);
4178 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4179 case 0:
4180 default:
4181 numchan = 1;
4182 break;
4183 case 1:
4184 numchan = 2;
4185 break;
4186 case 2:
4187 numchan = 4;
4188 break;
4189 case 3:
4190 numchan = 8;
4191 break;
4192 case 4:
4193 numchan = 3;
4194 break;
4195 case 5:
4196 numchan = 6;
4197 break;
4198 case 6:
4199 numchan = 10;
4200 break;
4201 case 7:
4202 numchan = 12;
4203 break;
4204 case 8:
4205 numchan = 16;
4206 break;
4207 }
4208 rdev->mc.vram_width = numchan * chansize;
4209 /* Could aper size report 0 ? */
4210 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4211 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4212 /* size in MB on si */
0ca223b0
AD		 4213 tmp = RREG32(CONFIG_MEMSIZE);
4214 /* some boards may have garbage in the upper 16 bits */
4215 if (tmp & 0xffff0000) {
4216 DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4217 if (tmp & 0xffff)
4218 tmp &= 0xffff;
4219 }
4220 rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4221 rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
d2800ee5
AD		 4222 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4223 si_vram_gtt_location(rdev, &rdev->mc);
4224 radeon_update_bandwidth_info(rdev);
4225
4226 return 0;
4227}
4228
4229/*
4230 * GART
4231 */
4232void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4233{
4234 /* flush hdp cache */
4235 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4236
4237 /* bits 0-15 are the VM contexts0-15 */
4238 WREG32(VM_INVALIDATE_REQUEST, 1);
4239}
4240
1109ca09 4241static int si_pcie_gart_enable(struct radeon_device *rdev)
d2800ee5
AD		 4242{
4243 int r, i;
4244
4245 if (rdev->gart.robj == NULL) {
4246 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4247 return -EINVAL;
4248 }
4249 r = radeon_gart_table_vram_pin(rdev);
4250 if (r)
4251 return r;
d2800ee5
AD		 4252 /* Setup TLB control */
4253 WREG32(MC_VM_MX_L1_TLB_CNTL,
4254 (0xA << 7) |
4255 ENABLE_L1_TLB |
ec3dbbcb 4256 ENABLE_L1_FRAGMENT_PROCESSING |
d2800ee5
AD		 4257 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4258 ENABLE_ADVANCED_DRIVER_MODEL |
4259 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4260 /* Setup L2 cache */
4261 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
ec3dbbcb 4262 ENABLE_L2_FRAGMENT_PROCESSING |
d2800ee5
AD		 4263 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4264 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4265 EFFECTIVE_L2_QUEUE_SIZE(7) |
4266 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4267 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4268 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
ec3dbbcb
CK		 4269 BANK_SELECT(4) |
4270 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
d2800ee5
AD		 4271 /* setup context0 */
4272 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4273 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4274 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4275 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4276 (u32)(rdev->dummy_page.addr >> 12));
4277 WREG32(VM_CONTEXT0_CNTL2, 0);
4278 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4279 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4280
4281 WREG32(0x15D4, 0);
4282 WREG32(0x15D8, 0);
4283 WREG32(0x15DC, 0);
4284
4285 /* empty context1-15 */
d2800ee5
AD		 4286 /* set vm size, must be a multiple of 4 */
4287 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
c21b328e 4288 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
23d4f1f2
AD		 4289 /* Assign the pt base to something valid for now; the pts used for
4290 * the VMs are determined by the application and setup and assigned
4291 * on the fly in the vm part of radeon_gart.c
4292 */
d2800ee5
AD		 4293 for (i = 1; i < 16; i++) {
4294 if (i < 8)
4295 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
054e01d6 4296 rdev->vm_manager.saved_table_addr[i]);
d2800ee5
AD		 4297 else
4298 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
054e01d6 4299 rdev->vm_manager.saved_table_addr[i]);
d2800ee5
AD		 4300 }
4301
4302 /* enable context1-15 */
4303 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4304 (u32)(rdev->dummy_page.addr >> 12));
ae133a11 4305 WREG32(VM_CONTEXT1_CNTL2, 4);
fa87e62d 4306 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4510fb98 4307 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
ae133a11
CK		 4308 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4309 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4310 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4311 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4312 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4313 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4314 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4315 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4316 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4317 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4318 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4319 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
d2800ee5
AD		 4320
4321 si_pcie_gart_tlb_flush(rdev);
4322 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4323 (unsigned)(rdev->mc.gtt_size >> 20),
4324 (unsigned long long)rdev->gart.table_addr);
4325 rdev->gart.ready = true;
4326 return 0;
4327}
4328
1109ca09 4329static void si_pcie_gart_disable(struct radeon_device *rdev)
d2800ee5 4330{
054e01d6
CK		 4331 unsigned i;
4332
4333 for (i = 1; i < 16; ++i) {
4334 uint32_t reg;
4335 if (i < 8)
4336 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
4337 else
4338 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
4339 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
4340 }
4341
d2800ee5
AD		 4342 /* Disable all tables */
4343 WREG32(VM_CONTEXT0_CNTL, 0);
4344 WREG32(VM_CONTEXT1_CNTL, 0);
4345 /* Setup TLB control */
4346 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4347 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4348 /* Setup L2 cache */
4349 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4350 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4351 EFFECTIVE_L2_QUEUE_SIZE(7) |
4352 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4353 WREG32(VM_L2_CNTL2, 0);
4354 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4355 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4356 radeon_gart_table_vram_unpin(rdev);
4357}
4358
1109ca09 4359static void si_pcie_gart_fini(struct radeon_device *rdev)
d2800ee5
AD		 4360{
4361 si_pcie_gart_disable(rdev);
4362 radeon_gart_table_vram_free(rdev);
4363 radeon_gart_fini(rdev);
4364}
4365
498dd8b3
AD		 4366/* vm parser */
4367static bool si_vm_reg_valid(u32 reg)
4368{
4369 /* context regs are fine */
4370 if (reg >= 0x28000)
4371 return true;
4372
4373 /* check config regs */
4374 switch (reg) {
4375 case GRBM_GFX_INDEX:
f418b88a 4376 case CP_STRMOUT_CNTL:
498dd8b3
AD		 4377 case VGT_VTX_VECT_EJECT_REG:
4378 case VGT_CACHE_INVALIDATION:
4379 case VGT_ESGS_RING_SIZE:
4380 case VGT_GSVS_RING_SIZE:
4381 case VGT_GS_VERTEX_REUSE:
4382 case VGT_PRIMITIVE_TYPE:
4383 case VGT_INDEX_TYPE:
4384 case VGT_NUM_INDICES:
4385 case VGT_NUM_INSTANCES:
4386 case VGT_TF_RING_SIZE:
4387 case VGT_HS_OFFCHIP_PARAM:
4388 case VGT_TF_MEMORY_BASE:
4389 case PA_CL_ENHANCE:
4390 case PA_SU_LINE_STIPPLE_VALUE:
4391 case PA_SC_LINE_STIPPLE_STATE:
4392 case PA_SC_ENHANCE:
4393 case SQC_CACHES:
4394 case SPI_STATIC_THREAD_MGMT_1:
4395 case SPI_STATIC_THREAD_MGMT_2:
4396 case SPI_STATIC_THREAD_MGMT_3:
4397 case SPI_PS_MAX_WAVE_ID:
4398 case SPI_CONFIG_CNTL:
4399 case SPI_CONFIG_CNTL_1:
4400 case TA_CNTL_AUX:
4401 return true;
4402 default:
4403 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4404 return false;
4405 }
4406}
4407
4408static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4409 u32 *ib, struct radeon_cs_packet *pkt)
4410{
4411 switch (pkt->opcode) {
4412 case PACKET3_NOP:
4413 case PACKET3_SET_BASE:
4414 case PACKET3_SET_CE_DE_COUNTERS:
4415 case PACKET3_LOAD_CONST_RAM:
4416 case PACKET3_WRITE_CONST_RAM:
4417 case PACKET3_WRITE_CONST_RAM_OFFSET:
4418 case PACKET3_DUMP_CONST_RAM:
4419 case PACKET3_INCREMENT_CE_COUNTER:
4420 case PACKET3_WAIT_ON_DE_COUNTER:
4421 case PACKET3_CE_WRITE:
4422 break;
4423 default:
4424 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4425 return -EINVAL;
4426 }
4427 return 0;
4428}
4429
e5b9e750
TS		 4430static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4431{
4432 u32 start_reg, reg, i;
4433 u32 command = ib[idx + 4];
4434 u32 info = ib[idx + 1];
4435 u32 idx_value = ib[idx];
4436 if (command & PACKET3_CP_DMA_CMD_SAS) {
4437 /* src address space is register */
4438 if (((info & 0x60000000) >> 29) == 0) {
4439 start_reg = idx_value << 2;
4440 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4441 reg = start_reg;
4442 if (!si_vm_reg_valid(reg)) {
4443 DRM_ERROR("CP DMA Bad SRC register\n");
4444 return -EINVAL;
4445 }
4446 } else {
4447 for (i = 0; i < (command & 0x1fffff); i++) {
4448 reg = start_reg + (4 * i);
4449 if (!si_vm_reg_valid(reg)) {
4450 DRM_ERROR("CP DMA Bad SRC register\n");
4451 return -EINVAL;
4452 }
4453 }
4454 }
4455 }
4456 }
4457 if (command & PACKET3_CP_DMA_CMD_DAS) {
4458 /* dst address space is register */
4459 if (((info & 0x00300000) >> 20) == 0) {
4460 start_reg = ib[idx + 2];
4461 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4462 reg = start_reg;
4463 if (!si_vm_reg_valid(reg)) {
4464 DRM_ERROR("CP DMA Bad DST register\n");
4465 return -EINVAL;
4466 }
4467 } else {
4468 for (i = 0; i < (command & 0x1fffff); i++) {
4469 reg = start_reg + (4 * i);
4470 if (!si_vm_reg_valid(reg)) {
4471 DRM_ERROR("CP DMA Bad DST register\n");
4472 return -EINVAL;
4473 }
4474 }
4475 }
4476 }
4477 }
4478 return 0;
4479}
4480
498dd8b3
AD		 4481static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4482 u32 *ib, struct radeon_cs_packet *pkt)
4483{
e5b9e750 4484 int r;
498dd8b3
AD		 4485 u32 idx = pkt->idx + 1;
4486 u32 idx_value = ib[idx];
4487 u32 start_reg, end_reg, reg, i;
4488
4489 switch (pkt->opcode) {
4490 case PACKET3_NOP:
4491 case PACKET3_SET_BASE:
4492 case PACKET3_CLEAR_STATE:
4493 case PACKET3_INDEX_BUFFER_SIZE:
4494 case PACKET3_DISPATCH_DIRECT:
4495 case PACKET3_DISPATCH_INDIRECT:
4496 case PACKET3_ALLOC_GDS:
4497 case PACKET3_WRITE_GDS_RAM:
4498 case PACKET3_ATOMIC_GDS:
4499 case PACKET3_ATOMIC:
4500 case PACKET3_OCCLUSION_QUERY:
4501 case PACKET3_SET_PREDICATION:
4502 case PACKET3_COND_EXEC:
4503 case PACKET3_PRED_EXEC:
4504 case PACKET3_DRAW_INDIRECT:
4505 case PACKET3_DRAW_INDEX_INDIRECT:
4506 case PACKET3_INDEX_BASE:
4507 case PACKET3_DRAW_INDEX_2:
4508 case PACKET3_CONTEXT_CONTROL:
4509 case PACKET3_INDEX_TYPE:
4510 case PACKET3_DRAW_INDIRECT_MULTI:
4511 case PACKET3_DRAW_INDEX_AUTO:
4512 case PACKET3_DRAW_INDEX_IMMD:
4513 case PACKET3_NUM_INSTANCES:
4514 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4515 case PACKET3_STRMOUT_BUFFER_UPDATE:
4516 case PACKET3_DRAW_INDEX_OFFSET_2:
4517 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4518 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4519 case PACKET3_MPEG_INDEX:
4520 case PACKET3_WAIT_REG_MEM:
4521 case PACKET3_MEM_WRITE:
4522 case PACKET3_PFP_SYNC_ME:
4523 case PACKET3_SURFACE_SYNC:
4524 case PACKET3_EVENT_WRITE:
4525 case PACKET3_EVENT_WRITE_EOP:
4526 case PACKET3_EVENT_WRITE_EOS:
4527 case PACKET3_SET_CONTEXT_REG:
4528 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4529 case PACKET3_SET_SH_REG:
4530 case PACKET3_SET_SH_REG_OFFSET:
4531 case PACKET3_INCREMENT_DE_COUNTER:
4532 case PACKET3_WAIT_ON_CE_COUNTER:
4533 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4534 case PACKET3_ME_WRITE:
4535 break;
4536 case PACKET3_COPY_DATA:
4537 if ((idx_value & 0xf00) == 0) {
4538 reg = ib[idx + 3] * 4;
4539 if (!si_vm_reg_valid(reg))
4540 return -EINVAL;
4541 }
4542 break;
4543 case PACKET3_WRITE_DATA:
4544 if ((idx_value & 0xf00) == 0) {
4545 start_reg = ib[idx + 1] * 4;
4546 if (idx_value & 0x10000) {
4547 if (!si_vm_reg_valid(start_reg))
4548 return -EINVAL;
4549 } else {
4550 for (i = 0; i < (pkt->count - 2); i++) {
4551 reg = start_reg + (4 * i);
4552 if (!si_vm_reg_valid(reg))
4553 return -EINVAL;
4554 }
4555 }
4556 }
4557 break;
4558 case PACKET3_COND_WRITE:
4559 if (idx_value & 0x100) {
4560 reg = ib[idx + 5] * 4;
4561 if (!si_vm_reg_valid(reg))
4562 return -EINVAL;
4563 }
4564 break;
4565 case PACKET3_COPY_DW:
4566 if (idx_value & 0x2) {
4567 reg = ib[idx + 3] * 4;
4568 if (!si_vm_reg_valid(reg))
4569 return -EINVAL;
4570 }
4571 break;
4572 case PACKET3_SET_CONFIG_REG:
4573 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4574 end_reg = 4 * pkt->count + start_reg - 4;
4575 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4576 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4577 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4578 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4579 return -EINVAL;
4580 }
4581 for (i = 0; i < pkt->count; i++) {
4582 reg = start_reg + (4 * i);
4583 if (!si_vm_reg_valid(reg))
4584 return -EINVAL;
4585 }
4586 break;
5aa709be 4587 case PACKET3_CP_DMA:
e5b9e750
TS		 4588 r = si_vm_packet3_cp_dma_check(ib, idx);
4589 if (r)
4590 return r;
5aa709be 4591 break;
498dd8b3
AD		 4592 default:
4593 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4594 return -EINVAL;
4595 }
4596 return 0;
4597}
4598
4599static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4600 u32 *ib, struct radeon_cs_packet *pkt)
4601{
e5b9e750 4602 int r;
498dd8b3
AD		 4603 u32 idx = pkt->idx + 1;
4604 u32 idx_value = ib[idx];
4605 u32 start_reg, reg, i;
4606
4607 switch (pkt->opcode) {
4608 case PACKET3_NOP:
4609 case PACKET3_SET_BASE:
4610 case PACKET3_CLEAR_STATE:
4611 case PACKET3_DISPATCH_DIRECT:
4612 case PACKET3_DISPATCH_INDIRECT:
4613 case PACKET3_ALLOC_GDS:
4614 case PACKET3_WRITE_GDS_RAM:
4615 case PACKET3_ATOMIC_GDS:
4616 case PACKET3_ATOMIC:
4617 case PACKET3_OCCLUSION_QUERY:
4618 case PACKET3_SET_PREDICATION:
4619 case PACKET3_COND_EXEC:
4620 case PACKET3_PRED_EXEC:
4621 case PACKET3_CONTEXT_CONTROL:
4622 case PACKET3_STRMOUT_BUFFER_UPDATE:
4623 case PACKET3_WAIT_REG_MEM:
4624 case PACKET3_MEM_WRITE:
4625 case PACKET3_PFP_SYNC_ME:
4626 case PACKET3_SURFACE_SYNC:
4627 case PACKET3_EVENT_WRITE:
4628 case PACKET3_EVENT_WRITE_EOP:
4629 case PACKET3_EVENT_WRITE_EOS:
4630 case PACKET3_SET_CONTEXT_REG:
4631 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4632 case PACKET3_SET_SH_REG:
4633 case PACKET3_SET_SH_REG_OFFSET:
4634 case PACKET3_INCREMENT_DE_COUNTER:
4635 case PACKET3_WAIT_ON_CE_COUNTER:
4636 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4637 case PACKET3_ME_WRITE:
4638 break;
4639 case PACKET3_COPY_DATA:
4640 if ((idx_value & 0xf00) == 0) {
4641 reg = ib[idx + 3] * 4;
4642 if (!si_vm_reg_valid(reg))
4643 return -EINVAL;
4644 }
4645 break;
4646 case PACKET3_WRITE_DATA:
4647 if ((idx_value & 0xf00) == 0) {
4648 start_reg = ib[idx + 1] * 4;
4649 if (idx_value & 0x10000) {
4650 if (!si_vm_reg_valid(start_reg))
4651 return -EINVAL;
4652 } else {
4653 for (i = 0; i < (pkt->count - 2); i++) {
4654 reg = start_reg + (4 * i);
4655 if (!si_vm_reg_valid(reg))
4656 return -EINVAL;
4657 }
4658 }
4659 }
4660 break;
4661 case PACKET3_COND_WRITE:
4662 if (idx_value & 0x100) {
4663 reg = ib[idx + 5] * 4;
4664 if (!si_vm_reg_valid(reg))
4665 return -EINVAL;
4666 }
4667 break;
4668 case PACKET3_COPY_DW:
4669 if (idx_value & 0x2) {
4670 reg = ib[idx + 3] * 4;
4671 if (!si_vm_reg_valid(reg))
4672 return -EINVAL;
4673 }
4674 break;
e5b9e750
TS		 4675 case PACKET3_CP_DMA:
4676 r = si_vm_packet3_cp_dma_check(ib, idx);
4677 if (r)
4678 return r;
4679 break;
498dd8b3
AD		 4680 default:
4681 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4682 return -EINVAL;
4683 }
4684 return 0;
4685}
4686
4687int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4688{
4689 int ret = 0;
9d0223d5 4690 u32 idx = 0, i;
498dd8b3
AD		 4691 struct radeon_cs_packet pkt;
4692
4693 do {
4694 pkt.idx = idx;
4e872ae2
IH		 4695 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4696 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
498dd8b3
AD		 4697 pkt.one_reg_wr = 0;
4698 switch (pkt.type) {
4e872ae2 4699 case RADEON_PACKET_TYPE0:
498dd8b3 4700 dev_err(rdev->dev, "Packet0 not allowed!\n");
9d0223d5
AD		 4701 for (i = 0; i < ib->length_dw; i++) {
4702 if (i == idx)
4703 printk("\t0x%08x <---\n", ib->ptr[i]);
4704 else
4705 printk("\t0x%08x\n", ib->ptr[i]);
4706 }
498dd8b3
AD		 4707 ret = -EINVAL;
4708 break;
4e872ae2 4709 case RADEON_PACKET_TYPE2:
498dd8b3
AD		 4710 idx += 1;
4711 break;
4e872ae2
IH		 4712 case RADEON_PACKET_TYPE3:
4713 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
498dd8b3
AD		 4714 if (ib->is_const_ib)
4715 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4716 else {
876dc9f3 4717 switch (ib->ring) {
498dd8b3
AD		 4718 case RADEON_RING_TYPE_GFX_INDEX:
4719 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4720 break;
4721 case CAYMAN_RING_TYPE_CP1_INDEX:
4722 case CAYMAN_RING_TYPE_CP2_INDEX:
4723 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4724 break;
4725 default:
876dc9f3 4726 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
498dd8b3
AD		 4727 ret = -EINVAL;
4728 break;
4729 }
4730 }
4731 idx += pkt.count + 2;
4732 break;
4733 default:
4734 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4735 ret = -EINVAL;
4736 break;
4737 }
4738 if (ret)
4739 break;
4740 } while (idx < ib->length_dw);
4741
4742 return ret;
4743}
4744
d2800ee5
AD		 4745/*
4746 * vm
4747 */
4748int si_vm_init(struct radeon_device *rdev)
4749{
4750 /* number of VMs */
4751 rdev->vm_manager.nvm = 16;
4752 /* base offset of vram pages */
4753 rdev->vm_manager.vram_base_offset = 0;
4754
4755 return 0;
4756}
4757
4758void si_vm_fini(struct radeon_device *rdev)
4759{
4760}
4761
fbf6dc7a
AD		 4762/**
4763 * si_vm_decode_fault - print human readable fault info
4764 *
4765 * @rdev: radeon_device pointer
4766 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4767 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4768 *
4769 * Print human readable fault information (SI).
4770 */
4771static void si_vm_decode_fault(struct radeon_device *rdev,
4772 u32 status, u32 addr)
4773{
4774 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4775 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4776 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4777 char *block;
4778
4779 if (rdev->family == CHIP_TAHITI) {
4780 switch (mc_id) {
4781 case 160:
4782 case 144:
4783 case 96:
4784 case 80:
4785 case 224:
4786 case 208:
4787 case 32:
4788 case 16:
4789 block = "CB";
4790 break;
4791 case 161:
4792 case 145:
4793 case 97:
4794 case 81:
4795 case 225:
4796 case 209:
4797 case 33:
4798 case 17:
4799 block = "CB_FMASK";
4800 break;
4801 case 162:
4802 case 146:
4803 case 98:
4804 case 82:
4805 case 226:
4806 case 210:
4807 case 34:
4808 case 18:
4809 block = "CB_CMASK";
4810 break;
4811 case 163:
4812 case 147:
4813 case 99:
4814 case 83:
4815 case 227:
4816 case 211:
4817 case 35:
4818 case 19:
4819 block = "CB_IMMED";
4820 break;
4821 case 164:
4822 case 148:
4823 case 100:
4824 case 84:
4825 case 228:
4826 case 212:
4827 case 36:
4828 case 20:
4829 block = "DB";
4830 break;
4831 case 165:
4832 case 149:
4833 case 101:
4834 case 85:
4835 case 229:
4836 case 213:
4837 case 37:
4838 case 21:
4839 block = "DB_HTILE";
4840 break;
4841 case 167:
4842 case 151:
4843 case 103:
4844 case 87:
4845 case 231:
4846 case 215:
4847 case 39:
4848 case 23:
4849 block = "DB_STEN";
4850 break;
4851 case 72:
4852 case 68:
4853 case 64:
4854 case 8:
4855 case 4:
4856 case 0:
4857 case 136:
4858 case 132:
4859 case 128:
4860 case 200:
4861 case 196:
4862 case 192:
4863 block = "TC";
4864 break;
4865 case 112:
4866 case 48:
4867 block = "CP";
4868 break;
4869 case 49:
4870 case 177:
4871 case 50:
4872 case 178:
4873 block = "SH";
4874 break;
4875 case 53:
4876 case 190:
4877 block = "VGT";
4878 break;
4879 case 117:
4880 block = "IH";
4881 break;
4882 case 51:
4883 case 115:
4884 block = "RLC";
4885 break;
4886 case 119:
4887 case 183:
4888 block = "DMA0";
4889 break;
4890 case 61:
4891 block = "DMA1";
4892 break;
4893 case 248:
4894 case 120:
4895 block = "HDP";
4896 break;
4897 default:
4898 block = "unknown";
4899 break;
4900 }
4901 } else {
4902 switch (mc_id) {
4903 case 32:
4904 case 16:
4905 case 96:
4906 case 80:
4907 case 160:
4908 case 144:
4909 case 224:
4910 case 208:
4911 block = "CB";
4912 break;
4913 case 33:
4914 case 17:
4915 case 97:
4916 case 81:
4917 case 161:
4918 case 145:
4919 case 225:
4920 case 209:
4921 block = "CB_FMASK";
4922 break;
4923 case 34:
4924 case 18:
4925 case 98:
4926 case 82:
4927 case 162:
4928 case 146:
4929 case 226:
4930 case 210:
4931 block = "CB_CMASK";
4932 break;
4933 case 35:
4934 case 19:
4935 case 99:
4936 case 83:
4937 case 163:
4938 case 147:
4939 case 227:
4940 case 211:
4941 block = "CB_IMMED";
4942 break;
4943 case 36:
4944 case 20:
4945 case 100:
4946 case 84:
4947 case 164:
4948 case 148:
4949 case 228:
4950 case 212:
4951 block = "DB";
4952 break;
4953 case 37:
4954 case 21:
4955 case 101:
4956 case 85:
4957 case 165:
4958 case 149:
4959 case 229:
4960 case 213:
4961 block = "DB_HTILE";
4962 break;
4963 case 39:
4964 case 23:
4965 case 103:
4966 case 87:
4967 case 167:
4968 case 151:
4969 case 231:
4970 case 215:
4971 block = "DB_STEN";
4972 break;
4973 case 72:
4974 case 68:
4975 case 8:
4976 case 4:
4977 case 136:
4978 case 132:
4979 case 200:
4980 case 196:
4981 block = "TC";
4982 break;
4983 case 112:
4984 case 48:
4985 block = "CP";
4986 break;
4987 case 49:
4988 case 177:
4989 case 50:
4990 case 178:
4991 block = "SH";
4992 break;
4993 case 53:
4994 block = "VGT";
4995 break;
4996 case 117:
4997 block = "IH";
4998 break;
4999 case 51:
5000 case 115:
5001 block = "RLC";
5002 break;
5003 case 119:
5004 case 183:
5005 block = "DMA0";
5006 break;
5007 case 61:
5008 block = "DMA1";
5009 break;
5010 case 248:
5011 case 120:
5012 block = "HDP";
5013 break;
5014 default:
5015 block = "unknown";
5016 break;
5017 }
5018 }
5019
5020 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
5021 protections, vmid, addr,
5022 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5023 block, mc_id);
5024}
5025
498522b4 5026void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
d2800ee5 5027{
498522b4 5028 struct radeon_ring *ring = &rdev->ring[ridx];
d2800ee5 5029
ee60e29f 5030 if (vm == NULL)
d2800ee5
AD		 5031 return;
5032
76c44f2c
AD		 5033 /* write new base address */
5034 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
f1d2a26b 5035 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
76c44f2c
AD		 5036 WRITE_DATA_DST_SEL(0)));
5037
ee60e29f 5038 if (vm->id < 8) {
76c44f2c
AD		 5039 radeon_ring_write(ring,
5040 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
ee60e29f 5041 } else {
76c44f2c
AD		 5042 radeon_ring_write(ring,
5043 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
ee60e29f 5044 }
76c44f2c 5045 radeon_ring_write(ring, 0);
fa87e62d 5046 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
ee60e29f 5047
d2800ee5 5048 /* flush hdp cache */
76c44f2c 5049 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 5050 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
76c44f2c
AD		 5051 WRITE_DATA_DST_SEL(0)));
5052 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5053 radeon_ring_write(ring, 0);
ee60e29f
CK		 5054 radeon_ring_write(ring, 0x1);
5055
d2800ee5 5056 /* bits 0-15 are the VM contexts0-15 */
76c44f2c 5057 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 5058 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
76c44f2c
AD		 5059 WRITE_DATA_DST_SEL(0)));
5060 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5061 radeon_ring_write(ring, 0);
498522b4 5062 radeon_ring_write(ring, 1 << vm->id);
58f8cf56
CK		 5063
5064 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5065 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5066 radeon_ring_write(ring, 0x0);
d2800ee5
AD		 5067}
5068
f8f84ac5
AD		 5069/*
5070 * Power and clock gating
5071 */
5072static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
5073{
5074 int i;
5075
5076 for (i = 0; i < rdev->usec_timeout; i++) {
5077 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
5078 break;
5079 udelay(1);
5080 }
5081
5082 for (i = 0; i < rdev->usec_timeout; i++) {
5083 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
5084 break;
5085 udelay(1);
5086 }
5087}
5088
5089static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
5090 bool enable)
5091{
5092 u32 tmp = RREG32(CP_INT_CNTL_RING0);
5093 u32 mask;
5094 int i;
5095
5096 if (enable)
5097 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5098 else
5099 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5100 WREG32(CP_INT_CNTL_RING0, tmp);
5101
5102 if (!enable) {
5103 /* read a gfx register */
5104 tmp = RREG32(DB_DEPTH_INFO);
5105
5106 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
5107 for (i = 0; i < rdev->usec_timeout; i++) {
5108 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
5109 break;
5110 udelay(1);
5111 }
5112 }
5113}
5114
5115static void si_set_uvd_dcm(struct radeon_device *rdev,
5116 bool sw_mode)
5117{
5118 u32 tmp, tmp2;
5119
5120 tmp = RREG32(UVD_CGC_CTRL);
5121 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
5122 tmp |= DCM | CG_DT(1) | CLK_OD(4);
5123
5124 if (sw_mode) {
5125 tmp &= ~0x7ffff800;
5126 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
5127 } else {
5128 tmp |= 0x7ffff800;
5129 tmp2 = 0;
5130 }
5131
5132 WREG32(UVD_CGC_CTRL, tmp);
5133 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
5134}
5135
22c775ce 5136void si_init_uvd_internal_cg(struct radeon_device *rdev)
f8f84ac5
AD		 5137{
5138 bool hw_mode = true;
5139
5140 if (hw_mode) {
5141 si_set_uvd_dcm(rdev, false);
5142 } else {
5143 u32 tmp = RREG32(UVD_CGC_CTRL);
5144 tmp &= ~DCM;
5145 WREG32(UVD_CGC_CTRL, tmp);
5146 }
5147}
5148
5149static u32 si_halt_rlc(struct radeon_device *rdev)
5150{
5151 u32 data, orig;
5152
5153 orig = data = RREG32(RLC_CNTL);
5154
5155 if (data & RLC_ENABLE) {
5156 data &= ~RLC_ENABLE;
5157 WREG32(RLC_CNTL, data);
5158
5159 si_wait_for_rlc_serdes(rdev);
5160 }
5161
5162 return orig;
5163}
5164
5165static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
5166{
5167 u32 tmp;
5168
5169 tmp = RREG32(RLC_CNTL);
5170 if (tmp != rlc)
5171 WREG32(RLC_CNTL, rlc);
5172}
5173
5174static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
5175{
5176 u32 data, orig;
5177
5178 orig = data = RREG32(DMA_PG);
e16866ec 5179 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
f8f84ac5
AD		 5180 data |= PG_CNTL_ENABLE;
5181 else
5182 data &= ~PG_CNTL_ENABLE;
5183 if (orig != data)
5184 WREG32(DMA_PG, data);
5185}
5186
5187static void si_init_dma_pg(struct radeon_device *rdev)
5188{
5189 u32 tmp;
5190
5191 WREG32(DMA_PGFSM_WRITE, 0x00002000);
5192 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
5193
5194 for (tmp = 0; tmp < 5; tmp++)
5195 WREG32(DMA_PGFSM_WRITE, 0);
5196}
5197
5198static void si_enable_gfx_cgpg(struct radeon_device *rdev,
5199 bool enable)
5200{
5201 u32 tmp;
5202
2b19d17f 5203 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
f8f84ac5
AD		 5204 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
5205 WREG32(RLC_TTOP_D, tmp);
5206
5207 tmp = RREG32(RLC_PG_CNTL);
5208 tmp |= GFX_PG_ENABLE;
5209 WREG32(RLC_PG_CNTL, tmp);
5210
5211 tmp = RREG32(RLC_AUTO_PG_CTRL);
5212 tmp |= AUTO_PG_EN;
5213 WREG32(RLC_AUTO_PG_CTRL, tmp);
5214 } else {
5215 tmp = RREG32(RLC_AUTO_PG_CTRL);
5216 tmp &= ~AUTO_PG_EN;
5217 WREG32(RLC_AUTO_PG_CTRL, tmp);
5218
5219 tmp = RREG32(DB_RENDER_CONTROL);
5220 }
5221}
5222
5223static void si_init_gfx_cgpg(struct radeon_device *rdev)
5224{
5225 u32 tmp;
5226
5227 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5228
5229 tmp = RREG32(RLC_PG_CNTL);
5230 tmp |= GFX_PG_SRC;
5231 WREG32(RLC_PG_CNTL, tmp);
5232
5233 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5234
5235 tmp = RREG32(RLC_AUTO_PG_CTRL);
5236
5237 tmp &= ~GRBM_REG_SGIT_MASK;
5238 tmp |= GRBM_REG_SGIT(0x700);
5239 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5240 WREG32(RLC_AUTO_PG_CTRL, tmp);
5241}
5242
ba19031a 5243static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
f8f84ac5
AD		 5244{
5245 u32 mask = 0, tmp, tmp1;
5246 int i;
5247
5248 si_select_se_sh(rdev, se, sh);
5249 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5250 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5251 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5252
5253 tmp &= 0xffff0000;
5254
5255 tmp |= tmp1;
5256 tmp >>= 16;
5257
5258 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5259 mask <<= 1;
5260 mask |= 1;
5261 }
5262
5263 return (~tmp) & mask;
5264}
5265
5266static void si_init_ao_cu_mask(struct radeon_device *rdev)
5267{
5268 u32 i, j, k, active_cu_number = 0;
5269 u32 mask, counter, cu_bitmap;
5270 u32 tmp = 0;
5271
5272 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5273 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5274 mask = 1;
5275 cu_bitmap = 0;
5276 counter = 0;
5277 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
ba19031a 5278 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
f8f84ac5
AD		 5279 if (counter < 2)
5280 cu_bitmap |= mask;
5281 counter++;
5282 }
5283 mask <<= 1;
5284 }
5285
5286 active_cu_number += counter;
5287 tmp |= (cu_bitmap << (i * 16 + j * 8));
5288 }
5289 }
5290
5291 WREG32(RLC_PG_AO_CU_MASK, tmp);
5292
5293 tmp = RREG32(RLC_MAX_PG_CU);
5294 tmp &= ~MAX_PU_CU_MASK;
5295 tmp |= MAX_PU_CU(active_cu_number);
5296 WREG32(RLC_MAX_PG_CU, tmp);
5297}
5298
5299static void si_enable_cgcg(struct radeon_device *rdev,
5300 bool enable)
5301{
5302 u32 data, orig, tmp;
5303
5304 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5305
e16866ec 5306 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5594a558 5307 si_enable_gui_idle_interrupt(rdev, true);
f8f84ac5 5308
f8f84ac5
AD		 5309 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5310
5311 tmp = si_halt_rlc(rdev);
5312
5313 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5314 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5315 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5316
5317 si_wait_for_rlc_serdes(rdev);
5318
5319 si_update_rlc(rdev, tmp);
5320
5321 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5322
5323 data |= CGCG_EN | CGLS_EN;
5324 } else {
5594a558
AD		 5325 si_enable_gui_idle_interrupt(rdev, false);
5326
f8f84ac5
AD		 5327 RREG32(CB_CGTT_SCLK_CTRL);
5328 RREG32(CB_CGTT_SCLK_CTRL);
5329 RREG32(CB_CGTT_SCLK_CTRL);
5330 RREG32(CB_CGTT_SCLK_CTRL);
5331
5332 data &= ~(CGCG_EN | CGLS_EN);
5333 }
5334
5335 if (orig != data)
5336 WREG32(RLC_CGCG_CGLS_CTRL, data);
5337}
5338
5339static void si_enable_mgcg(struct radeon_device *rdev,
5340 bool enable)
5341{
5342 u32 data, orig, tmp = 0;
5343
e16866ec 5344 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
f8f84ac5
AD		 5345 orig = data = RREG32(CGTS_SM_CTRL_REG);
5346 data = 0x96940200;
5347 if (orig != data)
5348 WREG32(CGTS_SM_CTRL_REG, data);
5349
e16866ec
AD		 5350 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5351 orig = data = RREG32(CP_MEM_SLP_CNTL);
5352 data |= CP_MEM_LS_EN;
5353 if (orig != data)
5354 WREG32(CP_MEM_SLP_CNTL, data);
5355 }
f8f84ac5
AD		 5356
5357 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5358 data &= 0xffffffc0;
5359 if (orig != data)
5360 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5361
5362 tmp = si_halt_rlc(rdev);
5363
5364 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5365 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5366 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5367
5368 si_update_rlc(rdev, tmp);
5369 } else {
5370 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5371 data |= 0x00000003;
5372 if (orig != data)
5373 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5374
5375 data = RREG32(CP_MEM_SLP_CNTL);
5376 if (data & CP_MEM_LS_EN) {
5377 data &= ~CP_MEM_LS_EN;
5378 WREG32(CP_MEM_SLP_CNTL, data);
5379 }
5380 orig = data = RREG32(CGTS_SM_CTRL_REG);
5381 data |= LS_OVERRIDE | OVERRIDE;
5382 if (orig != data)
5383 WREG32(CGTS_SM_CTRL_REG, data);
5384
5385 tmp = si_halt_rlc(rdev);
5386
5387 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5388 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5389 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5390
5391 si_update_rlc(rdev, tmp);
5392 }
5393}
5394
5395static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5396 bool enable)
5397{
5398 u32 orig, data, tmp;
5399
e16866ec 5400 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
f8f84ac5
AD		 5401 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5402 tmp |= 0x3fff;
5403 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5404
5405 orig = data = RREG32(UVD_CGC_CTRL);
5406 data |= DCM;
5407 if (orig != data)
5408 WREG32(UVD_CGC_CTRL, data);
5409
5410 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5411 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5412 } else {
5413 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5414 tmp &= ~0x3fff;
5415 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5416
5417 orig = data = RREG32(UVD_CGC_CTRL);
5418 data &= ~DCM;
5419 if (orig != data)
5420 WREG32(UVD_CGC_CTRL, data);
5421
5422 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5423 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5424 }
5425}
5426
5427static const u32 mc_cg_registers[] =
5428{
5429 MC_HUB_MISC_HUB_CG,
5430 MC_HUB_MISC_SIP_CG,
5431 MC_HUB_MISC_VM_CG,
5432 MC_XPB_CLK_GAT,
5433 ATC_MISC_CG,
5434 MC_CITF_MISC_WR_CG,
5435 MC_CITF_MISC_RD_CG,
5436 MC_CITF_MISC_VM_CG,
5437 VM_L2_CG,
5438};
5439
5440static void si_enable_mc_ls(struct radeon_device *rdev,
5441 bool enable)
5442{
5443 int i;
5444 u32 orig, data;
5445
5446 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5447 orig = data = RREG32(mc_cg_registers[i]);
e16866ec 5448 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
f8f84ac5
AD		 5449 data |= MC_LS_ENABLE;
5450 else
5451 data &= ~MC_LS_ENABLE;
5452 if (data != orig)
5453 WREG32(mc_cg_registers[i], data);
5454 }
5455}
5456
e16866ec
AD		 5457static void si_enable_mc_mgcg(struct radeon_device *rdev,
5458 bool enable)
f8f84ac5 5459{
e16866ec
AD		 5460 int i;
5461 u32 orig, data;
5462
5463 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5464 orig = data = RREG32(mc_cg_registers[i]);
5465 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5466 data |= MC_CG_ENABLE;
5467 else
5468 data &= ~MC_CG_ENABLE;
5469 if (data != orig)
5470 WREG32(mc_cg_registers[i], data);
f8f84ac5
AD		 5471 }
5472}
5473
e16866ec
AD		 5474static void si_enable_dma_mgcg(struct radeon_device *rdev,
5475 bool enable)
f8f84ac5 5476{
e16866ec
AD		 5477 u32 orig, data, offset;
5478 int i;
f8f84ac5 5479
e16866ec
AD		 5480 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5481 for (i = 0; i < 2; i++) {
5482 if (i == 0)
5483 offset = DMA0_REGISTER_OFFSET;
5484 else
5485 offset = DMA1_REGISTER_OFFSET;
5486 orig = data = RREG32(DMA_POWER_CNTL + offset);
5487 data &= ~MEM_POWER_OVERRIDE;
5488 if (data != orig)
5489 WREG32(DMA_POWER_CNTL + offset, data);
5490 WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5491 }
5492 } else {
5493 for (i = 0; i < 2; i++) {
5494 if (i == 0)
5495 offset = DMA0_REGISTER_OFFSET;
5496 else
5497 offset = DMA1_REGISTER_OFFSET;
5498 orig = data = RREG32(DMA_POWER_CNTL + offset);
5499 data |= MEM_POWER_OVERRIDE;
5500 if (data != orig)
5501 WREG32(DMA_POWER_CNTL + offset, data);
5502
5503 orig = data = RREG32(DMA_CLK_CTRL + offset);
5504 data = 0xff000000;
5505 if (data != orig)
5506 WREG32(DMA_CLK_CTRL + offset, data);
5507 }
5508 }
f8f84ac5
AD		 5509}
5510
e16866ec
AD		 5511static void si_enable_bif_mgls(struct radeon_device *rdev,
5512 bool enable)
f8f84ac5 5513{
e16866ec 5514 u32 orig, data;
f8f84ac5 5515
e16866ec 5516 orig = data = RREG32_PCIE(PCIE_CNTL2);
f8f84ac5 5517
e16866ec
AD		 5518 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5519 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5520 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5521 else
5522 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5523 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5524
5525 if (orig != data)
5526 WREG32_PCIE(PCIE_CNTL2, data);
f8f84ac5
AD		 5527}
5528
e16866ec
AD		 5529static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5530 bool enable)
f8f84ac5 5531{
e16866ec 5532 u32 orig, data;
f8f84ac5 5533
e16866ec 5534 orig = data = RREG32(HDP_HOST_PATH_CNTL);
f8f84ac5 5535
e16866ec
AD		 5536 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5537 data &= ~CLOCK_GATING_DIS;
5538 else
5539 data |= CLOCK_GATING_DIS;
5540
5541 if (orig != data)
5542 WREG32(HDP_HOST_PATH_CNTL, data);
f8f84ac5
AD		 5543}
5544
e16866ec
AD		 5545static void si_enable_hdp_ls(struct radeon_device *rdev,
5546 bool enable)
347e7592 5547{
e16866ec 5548 u32 orig, data;
347e7592 5549
e16866ec 5550 orig = data = RREG32(HDP_MEM_POWER_LS);
347e7592 5551
e16866ec
AD		 5552 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5553 data |= HDP_LS_ENABLE;
5554 else
5555 data &= ~HDP_LS_ENABLE;
347e7592 5556
e16866ec
AD		 5557 if (orig != data)
5558 WREG32(HDP_MEM_POWER_LS, data);
347e7592
AD		 5559}
5560
68e3a092
AD		 5561static void si_update_cg(struct radeon_device *rdev,
5562 u32 block, bool enable)
347e7592 5563{
e16866ec 5564 if (block & RADEON_CG_BLOCK_GFX) {
811e4d58 5565 si_enable_gui_idle_interrupt(rdev, false);
e16866ec
AD		 5566 /* order matters! */
5567 if (enable) {
5568 si_enable_mgcg(rdev, true);
5569 si_enable_cgcg(rdev, true);
5570 } else {
5571 si_enable_cgcg(rdev, false);
5572 si_enable_mgcg(rdev, false);
347e7592 5573 }
811e4d58 5574 si_enable_gui_idle_interrupt(rdev, true);
347e7592
AD		 5575 }
5576
e16866ec
AD		 5577 if (block & RADEON_CG_BLOCK_MC) {
5578 si_enable_mc_mgcg(rdev, enable);
5579 si_enable_mc_ls(rdev, enable);
347e7592 5580 }
e16866ec
AD		 5581
5582 if (block & RADEON_CG_BLOCK_SDMA) {
5583 si_enable_dma_mgcg(rdev, enable);
347e7592
AD		 5584 }
5585
e16866ec
AD		 5586 if (block & RADEON_CG_BLOCK_BIF) {
5587 si_enable_bif_mgls(rdev, enable);
6d8cf000 5588 }
6d8cf000 5589
e16866ec
AD		 5590 if (block & RADEON_CG_BLOCK_UVD) {
5591 if (rdev->has_uvd) {
5592 si_enable_uvd_mgcg(rdev, enable);
bd8cd539
AD		 5593 }
5594 }
bd8cd539 5595
e16866ec
AD		 5596 if (block & RADEON_CG_BLOCK_HDP) {
5597 si_enable_hdp_mgcg(rdev, enable);
5598 si_enable_hdp_ls(rdev, enable);
347e7592 5599 }
e16866ec 5600}
f8f84ac5
AD		 5601
5602static void si_init_cg(struct radeon_device *rdev)
5603{
e16866ec
AD		 5604 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5605 RADEON_CG_BLOCK_MC |
5606 RADEON_CG_BLOCK_SDMA |
5607 RADEON_CG_BLOCK_BIF |
5608 RADEON_CG_BLOCK_HDP), true);
b2d70917 5609 if (rdev->has_uvd) {
e16866ec 5610 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
f8f84ac5 5611 si_init_uvd_internal_cg(rdev);
347e7592 5612 }
f8f84ac5 5613}
bd8cd539 5614
f8f84ac5
AD		 5615static void si_fini_cg(struct radeon_device *rdev)
5616{
0116e1ef 5617 if (rdev->has_uvd) {
e16866ec 5618 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
347e7592 5619 }
e16866ec
AD		 5620 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5621 RADEON_CG_BLOCK_MC |
5622 RADEON_CG_BLOCK_SDMA |
5623 RADEON_CG_BLOCK_BIF |
5624 RADEON_CG_BLOCK_HDP), false);
5625}
5626
59a82d0e
AD		 5627u32 si_get_csb_size(struct radeon_device *rdev)
5628{
5629 u32 count = 0;
5630 const struct cs_section_def *sect = NULL;
5631 const struct cs_extent_def *ext = NULL;
5632
5633 if (rdev->rlc.cs_data == NULL)
5634 return 0;
5635
5636 /* begin clear state */
5637 count += 2;
5638 /* context control state */
5639 count += 3;
5640
5641 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5642 for (ext = sect->section; ext->extent != NULL; ++ext) {
5643 if (sect->id == SECT_CONTEXT)
5644 count += 2 + ext->reg_count;
5645 else
5646 return 0;
5647 }
bd8cd539 5648 }
59a82d0e
AD		 5649 /* pa_sc_raster_config */
5650 count += 3;
5651 /* end clear state */
5652 count += 2;
5653 /* clear state */
5654 count += 2;
5655
5656 return count;
5657}
5658
5659void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5660{
5661 u32 count = 0, i;
5662 const struct cs_section_def *sect = NULL;
5663 const struct cs_extent_def *ext = NULL;
5664
5665 if (rdev->rlc.cs_data == NULL)
5666 return;
5667 if (buffer == NULL)
5668 return;
5669
6ba81e53
AD		 5670 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5671 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
59a82d0e 5672
6ba81e53
AD		 5673 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5674 buffer[count++] = cpu_to_le32(0x80000000);
5675 buffer[count++] = cpu_to_le32(0x80000000);
59a82d0e
AD		 5676
5677 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5678 for (ext = sect->section; ext->extent != NULL; ++ext) {
5679 if (sect->id == SECT_CONTEXT) {
6ba81e53
AD		 5680 buffer[count++] =
5681 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5682 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
59a82d0e 5683 for (i = 0; i < ext->reg_count; i++)
6ba81e53 5684 buffer[count++] = cpu_to_le32(ext->extent[i]);
59a82d0e
AD		 5685 } else {
5686 return;
bd8cd539 5687 }
bd8cd539
AD		 5688 }
5689 }
bd8cd539 5690
6ba81e53
AD		 5691 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5692 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
59a82d0e
AD		 5693 switch (rdev->family) {
5694 case CHIP_TAHITI:
5695 case CHIP_PITCAIRN:
6ba81e53 5696 buffer[count++] = cpu_to_le32(0x2a00126a);
59a82d0e
AD		 5697 break;
5698 case CHIP_VERDE:
6ba81e53 5699 buffer[count++] = cpu_to_le32(0x0000124a);
59a82d0e
AD		 5700 break;
5701 case CHIP_OLAND:
6ba81e53 5702 buffer[count++] = cpu_to_le32(0x00000082);
59a82d0e
AD		 5703 break;
5704 case CHIP_HAINAN:
6ba81e53 5705 buffer[count++] = cpu_to_le32(0x00000000);
59a82d0e
AD		 5706 break;
5707 default:
6ba81e53 5708 buffer[count++] = cpu_to_le32(0x00000000);
59a82d0e
AD		 5709 break;
5710 }
5711
6ba81e53
AD		 5712 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5713 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
347e7592 5714
6ba81e53
AD		 5715 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5716 buffer[count++] = cpu_to_le32(0);
59a82d0e
AD		 5717}
5718
f8f84ac5
AD		 5719static void si_init_pg(struct radeon_device *rdev)
5720{
0116e1ef
AD		 5721 if (rdev->pg_flags) {
5722 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5723 si_init_dma_pg(rdev);
0116e1ef 5724 }
f8f84ac5 5725 si_init_ao_cu_mask(rdev);
2b19d17f 5726 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
0116e1ef 5727 si_init_gfx_cgpg(rdev);
aa34dba8
AD		 5728 } else {
5729 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5730 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
0116e1ef 5731 }
ca6ebb39
AD		 5732 si_enable_dma_pg(rdev, true);
5733 si_enable_gfx_cgpg(rdev, true);
f8f84ac5
AD		 5734 } else {
5735 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5736 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5737 }
5738}
5739
5740static void si_fini_pg(struct radeon_device *rdev)
5741{
0116e1ef 5742 if (rdev->pg_flags) {
ca6ebb39
AD		 5743 si_enable_dma_pg(rdev, false);
5744 si_enable_gfx_cgpg(rdev, false);
f8f84ac5 5745 }
347e7592
AD		 5746}
5747
347e7592
AD		 5748/*
5749 * RLC
5750 */
866d83de 5751void si_rlc_reset(struct radeon_device *rdev)
d719cef3 5752{
f8f84ac5 5753 u32 tmp = RREG32(GRBM_SOFT_RESET);
d719cef3 5754
f8f84ac5
AD		 5755 tmp |= SOFT_RESET_RLC;
5756 WREG32(GRBM_SOFT_RESET, tmp);
5757 udelay(50);
5758 tmp &= ~SOFT_RESET_RLC;
5759 WREG32(GRBM_SOFT_RESET, tmp);
5760 udelay(50);
d719cef3
AD		 5761}
5762
347e7592
AD		 5763static void si_rlc_stop(struct radeon_device *rdev)
5764{
5765 WREG32(RLC_CNTL, 0);
d719cef3
AD		 5766
5767 si_enable_gui_idle_interrupt(rdev, false);
5768
5769 si_wait_for_rlc_serdes(rdev);
347e7592
AD		 5770}
5771
5772static void si_rlc_start(struct radeon_device *rdev)
5773{
5774 WREG32(RLC_CNTL, RLC_ENABLE);
d719cef3
AD		 5775
5776 si_enable_gui_idle_interrupt(rdev, true);
5777
5778 udelay(50);
5779}
5780
5781static bool si_lbpw_supported(struct radeon_device *rdev)
5782{
5783 u32 tmp;
5784
5785 /* Enable LBPW only for DDR3 */
5786 tmp = RREG32(MC_SEQ_MISC0);
5787 if ((tmp & 0xF0000000) == 0xB0000000)
5788 return true;
5789 return false;
5790}
5791
5792static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5793{
5794 u32 tmp;
5795
5796 tmp = RREG32(RLC_LB_CNTL);
5797 if (enable)
5798 tmp |= LOAD_BALANCE_ENABLE;
5799 else
5800 tmp &= ~LOAD_BALANCE_ENABLE;
5801 WREG32(RLC_LB_CNTL, tmp);
5802
5803 if (!enable) {
5804 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5805 WREG32(SPI_LB_CU_MASK, 0x00ff);
5806 }
347e7592
AD		 5807}
5808
5809static int si_rlc_resume(struct radeon_device *rdev)
5810{
5811 u32 i;
347e7592
AD		 5812
5813 if (!rdev->rlc_fw)
5814 return -EINVAL;
5815
5816 si_rlc_stop(rdev);
5817
f8f84ac5
AD		 5818 si_rlc_reset(rdev);
5819
5820 si_init_pg(rdev);
5821
5822 si_init_cg(rdev);
5823
347e7592
AD		 5824 WREG32(RLC_RL_BASE, 0);
5825 WREG32(RLC_RL_SIZE, 0);
5826 WREG32(RLC_LB_CNTL, 0);
5827 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5828 WREG32(RLC_LB_CNTR_INIT, 0);
d719cef3 5829 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
347e7592 5830
347e7592
AD		 5831 WREG32(RLC_MC_CNTL, 0);
5832 WREG32(RLC_UCODE_CNTL, 0);
5833
629bd33c
AD		 5834 if (rdev->new_fw) {
5835 const struct rlc_firmware_header_v1_0 *hdr =
5836 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5837 u32 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5838 const __le32 *fw_data = (const __le32 *)
5839 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5840
5841 radeon_ucode_print_rlc_hdr(&hdr->header);
5842
5843 for (i = 0; i < fw_size; i++) {
5844 WREG32(RLC_UCODE_ADDR, i);
5845 WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++));
5846 }
5847 } else {
5848 const __be32 *fw_data =
5849 (const __be32 *)rdev->rlc_fw->data;
5850 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5851 WREG32(RLC_UCODE_ADDR, i);
5852 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5853 }
347e7592
AD		 5854 }
5855 WREG32(RLC_UCODE_ADDR, 0);
5856
d719cef3
AD		 5857 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5858
347e7592
AD		 5859 si_rlc_start(rdev);
5860
5861 return 0;
5862}
5863
25a857fb
AD		 5864static void si_enable_interrupts(struct radeon_device *rdev)
5865{
5866 u32 ih_cntl = RREG32(IH_CNTL);
5867 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5868
5869 ih_cntl |= ENABLE_INTR;
5870 ih_rb_cntl |= IH_RB_ENABLE;
5871 WREG32(IH_CNTL, ih_cntl);
5872 WREG32(IH_RB_CNTL, ih_rb_cntl);
5873 rdev->ih.enabled = true;
5874}
5875
5876static void si_disable_interrupts(struct radeon_device *rdev)
5877{
5878 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5879 u32 ih_cntl = RREG32(IH_CNTL);
5880
5881 ih_rb_cntl &= ~IH_RB_ENABLE;
5882 ih_cntl &= ~ENABLE_INTR;
5883 WREG32(IH_RB_CNTL, ih_rb_cntl);
5884 WREG32(IH_CNTL, ih_cntl);
5885 /* set rptr, wptr to 0 */
5886 WREG32(IH_RB_RPTR, 0);
5887 WREG32(IH_RB_WPTR, 0);
5888 rdev->ih.enabled = false;
25a857fb
AD		 5889 rdev->ih.rptr = 0;
5890}
5891
5892static void si_disable_interrupt_state(struct radeon_device *rdev)
5893{
5894 u32 tmp;
5895
811e4d58
AD		 5896 tmp = RREG32(CP_INT_CNTL_RING0) &
5897 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5898 WREG32(CP_INT_CNTL_RING0, tmp);
25a857fb
AD		 5899 WREG32(CP_INT_CNTL_RING1, 0);
5900 WREG32(CP_INT_CNTL_RING2, 0);
8c5fd7ef
AD		 5901 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5902 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5903 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5904 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
25a857fb 5905 WREG32(GRBM_INT_CNTL, 0);
5153550a
AD		 5906 if (rdev->num_crtc >= 2) {
5907 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5908 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5909 }
25a857fb
AD		 5910 if (rdev->num_crtc >= 4) {
5911 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5912 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5913 }
5914 if (rdev->num_crtc >= 6) {
5915 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5916 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5917 }
5918
5153550a
AD		 5919 if (rdev->num_crtc >= 2) {
5920 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5921 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5922 }
25a857fb
AD		 5923 if (rdev->num_crtc >= 4) {
5924 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5925 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5926 }
5927 if (rdev->num_crtc >= 6) {
5928 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5929 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5930 }
5931
5153550a 5932 if (!ASIC_IS_NODCE(rdev)) {
e9a321c6 5933 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5153550a
AD		 5934
5935 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5936 WREG32(DC_HPD1_INT_CONTROL, tmp);
5937 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5938 WREG32(DC_HPD2_INT_CONTROL, tmp);
5939 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5940 WREG32(DC_HPD3_INT_CONTROL, tmp);
5941 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5942 WREG32(DC_HPD4_INT_CONTROL, tmp);
5943 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5944 WREG32(DC_HPD5_INT_CONTROL, tmp);
5945 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5946 WREG32(DC_HPD6_INT_CONTROL, tmp);
5947 }
25a857fb
AD		 5948}
5949
5950static int si_irq_init(struct radeon_device *rdev)
5951{
5952 int ret = 0;
5953 int rb_bufsz;
5954 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5955
5956 /* allocate ring */
5957 ret = r600_ih_ring_alloc(rdev);
5958 if (ret)
5959 return ret;
5960
5961 /* disable irqs */
5962 si_disable_interrupts(rdev);
5963
5964 /* init rlc */
5965 ret = si_rlc_resume(rdev);
5966 if (ret) {
5967 r600_ih_ring_fini(rdev);
5968 return ret;
5969 }
5970
5971 /* setup interrupt control */
5972 /* set dummy read address to ring address */
5973 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5974 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5975 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5976 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5977 */
5978 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5979 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5980 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5981 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5982
5983 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 5984 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
25a857fb
AD		 5985
5986 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5987 IH_WPTR_OVERFLOW_CLEAR |
5988 (rb_bufsz << 1));
5989
5990 if (rdev->wb.enabled)
5991 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5992
5993 /* set the writeback address whether it's enabled or not */
5994 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5995 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5996
5997 WREG32(IH_RB_CNTL, ih_rb_cntl);
5998
5999 /* set rptr, wptr to 0 */
6000 WREG32(IH_RB_RPTR, 0);
6001 WREG32(IH_RB_WPTR, 0);
6002
6003 /* Default settings for IH_CNTL (disabled at first) */
6004 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6005 /* RPTR_REARM only works if msi's are enabled */
6006 if (rdev->msi_enabled)
6007 ih_cntl |= RPTR_REARM;
6008 WREG32(IH_CNTL, ih_cntl);
6009
6010 /* force the active interrupt state to all disabled */
6011 si_disable_interrupt_state(rdev);
6012
2099810f
DA		 6013 pci_set_master(rdev->pdev);
6014
25a857fb
AD		 6015 /* enable irqs */
6016 si_enable_interrupts(rdev);
6017
6018 return ret;
6019}
6020
6021int si_irq_set(struct radeon_device *rdev)
6022{
811e4d58 6023 u32 cp_int_cntl;
25a857fb
AD		 6024 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
6025 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5153550a 6026 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
25a857fb 6027 u32 grbm_int_cntl = 0;
8c5fd7ef 6028 u32 dma_cntl, dma_cntl1;
a9e61410 6029 u32 thermal_int = 0;
25a857fb
AD		 6030
6031 if (!rdev->irq.installed) {
6032 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6033 return -EINVAL;
6034 }
6035 /* don't enable anything if the ih is disabled */
6036 if (!rdev->ih.enabled) {
6037 si_disable_interrupts(rdev);
6038 /* force the active interrupt state to all disabled */
6039 si_disable_interrupt_state(rdev);
6040 return 0;
6041 }
6042
811e4d58
AD		 6043 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6044 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6045
5153550a
AD		 6046 if (!ASIC_IS_NODCE(rdev)) {
6047 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6048 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6049 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6050 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6051 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6052 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6053 }
25a857fb 6054
8c5fd7ef
AD		 6055 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6056 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6057
a9e61410
AD		 6058 thermal_int = RREG32(CG_THERMAL_INT) &
6059 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6060
25a857fb 6061 /* enable CP interrupts on all rings */
736fc37f 6062 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
25a857fb
AD		 6063 DRM_DEBUG("si_irq_set: sw int gfx\n");
6064 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6065 }
736fc37f 6066 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
25a857fb
AD		 6067 DRM_DEBUG("si_irq_set: sw int cp1\n");
6068 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
6069 }
736fc37f 6070 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
25a857fb
AD		 6071 DRM_DEBUG("si_irq_set: sw int cp2\n");
6072 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
6073 }
8c5fd7ef
AD		 6074 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6075 DRM_DEBUG("si_irq_set: sw int dma\n");
6076 dma_cntl |= TRAP_ENABLE;
6077 }
6078
6079 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6080 DRM_DEBUG("si_irq_set: sw int dma1\n");
6081 dma_cntl1 |= TRAP_ENABLE;
6082 }
25a857fb 6083 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 6084 atomic_read(&rdev->irq.pflip[0])) {
25a857fb
AD		 6085 DRM_DEBUG("si_irq_set: vblank 0\n");
6086 crtc1 |= VBLANK_INT_MASK;
6087 }
6088 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 6089 atomic_read(&rdev->irq.pflip[1])) {
25a857fb
AD		 6090 DRM_DEBUG("si_irq_set: vblank 1\n");
6091 crtc2 |= VBLANK_INT_MASK;
6092 }
6093 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 6094 atomic_read(&rdev->irq.pflip[2])) {
25a857fb
AD		 6095 DRM_DEBUG("si_irq_set: vblank 2\n");
6096 crtc3 |= VBLANK_INT_MASK;
6097 }
6098 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 6099 atomic_read(&rdev->irq.pflip[3])) {
25a857fb
AD		 6100 DRM_DEBUG("si_irq_set: vblank 3\n");
6101 crtc4 |= VBLANK_INT_MASK;
6102 }
6103 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 6104 atomic_read(&rdev->irq.pflip[4])) {
25a857fb
AD		 6105 DRM_DEBUG("si_irq_set: vblank 4\n");
6106 crtc5 |= VBLANK_INT_MASK;
6107 }
6108 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 6109 atomic_read(&rdev->irq.pflip[5])) {
25a857fb
AD		 6110 DRM_DEBUG("si_irq_set: vblank 5\n");
6111 crtc6 |= VBLANK_INT_MASK;
6112 }
6113 if (rdev->irq.hpd[0]) {
6114 DRM_DEBUG("si_irq_set: hpd 1\n");
6115 hpd1 |= DC_HPDx_INT_EN;
6116 }
6117 if (rdev->irq.hpd[1]) {
6118 DRM_DEBUG("si_irq_set: hpd 2\n");
6119 hpd2 |= DC_HPDx_INT_EN;
6120 }
6121 if (rdev->irq.hpd[2]) {
6122 DRM_DEBUG("si_irq_set: hpd 3\n");
6123 hpd3 |= DC_HPDx_INT_EN;
6124 }
6125 if (rdev->irq.hpd[3]) {
6126 DRM_DEBUG("si_irq_set: hpd 4\n");
6127 hpd4 |= DC_HPDx_INT_EN;
6128 }
6129 if (rdev->irq.hpd[4]) {
6130 DRM_DEBUG("si_irq_set: hpd 5\n");
6131 hpd5 |= DC_HPDx_INT_EN;
6132 }
6133 if (rdev->irq.hpd[5]) {
6134 DRM_DEBUG("si_irq_set: hpd 6\n");
6135 hpd6 |= DC_HPDx_INT_EN;
6136 }
25a857fb
AD		 6137
6138 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6139 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
6140 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
6141
8c5fd7ef
AD		 6142 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
6143 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
6144
25a857fb
AD		 6145 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6146
a9e61410
AD		 6147 if (rdev->irq.dpm_thermal) {
6148 DRM_DEBUG("dpm thermal\n");
6149 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
6150 }
6151
5153550a
AD		 6152 if (rdev->num_crtc >= 2) {
6153 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6154 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6155 }
25a857fb
AD		 6156 if (rdev->num_crtc >= 4) {
6157 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6158 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6159 }
6160 if (rdev->num_crtc >= 6) {
6161 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6162 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6163 }
6164
5153550a 6165 if (rdev->num_crtc >= 2) {
f5d636d2
CK		 6166 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
6167 GRPH_PFLIP_INT_MASK);
6168 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
6169 GRPH_PFLIP_INT_MASK);
5153550a 6170 }
25a857fb 6171 if (rdev->num_crtc >= 4) {
f5d636d2
CK		 6172 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
6173 GRPH_PFLIP_INT_MASK);
6174 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
6175 GRPH_PFLIP_INT_MASK);
25a857fb
AD		 6176 }
6177 if (rdev->num_crtc >= 6) {
f5d636d2
CK		 6178 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
6179 GRPH_PFLIP_INT_MASK);
6180 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
6181 GRPH_PFLIP_INT_MASK);
25a857fb
AD		 6182 }
6183
5153550a
AD		 6184 if (!ASIC_IS_NODCE(rdev)) {
6185 WREG32(DC_HPD1_INT_CONTROL, hpd1);
6186 WREG32(DC_HPD2_INT_CONTROL, hpd2);
6187 WREG32(DC_HPD3_INT_CONTROL, hpd3);
6188 WREG32(DC_HPD4_INT_CONTROL, hpd4);
6189 WREG32(DC_HPD5_INT_CONTROL, hpd5);
6190 WREG32(DC_HPD6_INT_CONTROL, hpd6);
6191 }
25a857fb 6192
a9e61410
AD		 6193 WREG32(CG_THERMAL_INT, thermal_int);
6194
25a857fb
AD		 6195 return 0;
6196}
6197
6198static inline void si_irq_ack(struct radeon_device *rdev)
6199{
6200 u32 tmp;
6201
5153550a
AD		 6202 if (ASIC_IS_NODCE(rdev))
6203 return;
6204
25a857fb
AD		 6205 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6206 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6207 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6208 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6209 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6210 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6211 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
6212 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
6213 if (rdev->num_crtc >= 4) {
6214 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
6215 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
6216 }
6217 if (rdev->num_crtc >= 6) {
6218 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
6219 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
6220 }
6221
6222 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
6223 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6224 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
6225 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6226 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
6227 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6228 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
6229 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6230 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6231 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6232 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6233 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6234
6235 if (rdev->num_crtc >= 4) {
6236 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
6237 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6238 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
6239 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6240 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6241 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6242 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6243 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6244 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6245 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6246 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6247 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6248 }
6249
6250 if (rdev->num_crtc >= 6) {
6251 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
6252 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6253 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
6254 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6255 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6256 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6257 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6258 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6259 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6260 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6261 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6262 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6263 }
6264
6265 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6266 tmp = RREG32(DC_HPD1_INT_CONTROL);
6267 tmp |= DC_HPDx_INT_ACK;
6268 WREG32(DC_HPD1_INT_CONTROL, tmp);
6269 }
6270 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6271 tmp = RREG32(DC_HPD2_INT_CONTROL);
6272 tmp |= DC_HPDx_INT_ACK;
6273 WREG32(DC_HPD2_INT_CONTROL, tmp);
6274 }
6275 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6276 tmp = RREG32(DC_HPD3_INT_CONTROL);
6277 tmp |= DC_HPDx_INT_ACK;
6278 WREG32(DC_HPD3_INT_CONTROL, tmp);
6279 }
6280 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6281 tmp = RREG32(DC_HPD4_INT_CONTROL);
6282 tmp |= DC_HPDx_INT_ACK;
6283 WREG32(DC_HPD4_INT_CONTROL, tmp);
6284 }
6285 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6286 tmp = RREG32(DC_HPD5_INT_CONTROL);
6287 tmp |= DC_HPDx_INT_ACK;
6288 WREG32(DC_HPD5_INT_CONTROL, tmp);
6289 }
6290 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6291 tmp = RREG32(DC_HPD5_INT_CONTROL);
6292 tmp |= DC_HPDx_INT_ACK;
6293 WREG32(DC_HPD6_INT_CONTROL, tmp);
6294 }
6295}
6296
6297static void si_irq_disable(struct radeon_device *rdev)
6298{
6299 si_disable_interrupts(rdev);
6300 /* Wait and acknowledge irq */
6301 mdelay(1);
6302 si_irq_ack(rdev);
6303 si_disable_interrupt_state(rdev);
6304}
6305
6306static void si_irq_suspend(struct radeon_device *rdev)
6307{
6308 si_irq_disable(rdev);
6309 si_rlc_stop(rdev);
6310}
6311
9b136d51
AD		 6312static void si_irq_fini(struct radeon_device *rdev)
6313{
6314 si_irq_suspend(rdev);
6315 r600_ih_ring_fini(rdev);
6316}
6317
25a857fb
AD		 6318static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6319{
6320 u32 wptr, tmp;
6321
6322 if (rdev->wb.enabled)
6323 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6324 else
6325 wptr = RREG32(IH_RB_WPTR);
6326
6327 if (wptr & RB_OVERFLOW) {
11bab0ae 6328 wptr &= ~RB_OVERFLOW;
25a857fb
AD		 6329 /* When a ring buffer overflow happen start parsing interrupt
6330 * from the last not overwritten vector (wptr + 16). Hopefully
6331 * this should allow us to catchup.
6332 */
6cc2fda2
MD		 6333 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
6334 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
25a857fb
AD		 6335 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6336 tmp = RREG32(IH_RB_CNTL);
6337 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6338 WREG32(IH_RB_CNTL, tmp);
6339 }
6340 return (wptr & rdev->ih.ptr_mask);
6341}
6342
6343/* SI IV Ring
6344 * Each IV ring entry is 128 bits:
6345 * [7:0] - interrupt source id
6346 * [31:8] - reserved
6347 * [59:32] - interrupt source data
6348 * [63:60] - reserved
6349 * [71:64] - RINGID
6350 * [79:72] - VMID
6351 * [127:80] - reserved
6352 */
6353int si_irq_process(struct radeon_device *rdev)
6354{
6355 u32 wptr;
6356 u32 rptr;
6357 u32 src_id, src_data, ring_id;
6358 u32 ring_index;
25a857fb 6359 bool queue_hotplug = false;
a9e61410 6360 bool queue_thermal = false;
fbf6dc7a 6361 u32 status, addr;
25a857fb
AD		 6362
6363 if (!rdev->ih.enabled || rdev->shutdown)
6364 return IRQ_NONE;
6365
6366 wptr = si_get_ih_wptr(rdev);
c20dc369
CK		 6367
6368restart_ih:
6369 /* is somebody else already processing irqs? */
6370 if (atomic_xchg(&rdev->ih.lock, 1))
6371 return IRQ_NONE;
6372
25a857fb
AD		 6373 rptr = rdev->ih.rptr;
6374 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6375
25a857fb
AD		 6376 /* Order reading of wptr vs. reading of IH ring data */
6377 rmb();
6378
6379 /* display interrupts */
6380 si_irq_ack(rdev);
6381
25a857fb
AD		 6382 while (rptr != wptr) {
6383 /* wptr/rptr are in bytes! */
6384 ring_index = rptr / 4;
6385 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6386 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6387 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6388
6389 switch (src_id) {
6390 case 1: /* D1 vblank/vline */
6391 switch (src_data) {
6392 case 0: /* D1 vblank */
6393 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6394 if (rdev->irq.crtc_vblank_int[0]) {
6395 drm_handle_vblank(rdev->ddev, 0);
6396 rdev->pm.vblank_sync = true;
6397 wake_up(&rdev->irq.vblank_queue);
6398 }
736fc37f 6399 if (atomic_read(&rdev->irq.pflip[0]))
1a0e7918 6400 radeon_crtc_handle_vblank(rdev, 0);
25a857fb
AD		 6401 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6402 DRM_DEBUG("IH: D1 vblank\n");
6403 }
6404 break;
6405 case 1: /* D1 vline */
6406 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
6407 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6408 DRM_DEBUG("IH: D1 vline\n");
6409 }
6410 break;
6411 default:
6412 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6413 break;
6414 }
6415 break;
6416 case 2: /* D2 vblank/vline */
6417 switch (src_data) {
6418 case 0: /* D2 vblank */
6419 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6420 if (rdev->irq.crtc_vblank_int[1]) {
6421 drm_handle_vblank(rdev->ddev, 1);
6422 rdev->pm.vblank_sync = true;
6423 wake_up(&rdev->irq.vblank_queue);
6424 }
736fc37f 6425 if (atomic_read(&rdev->irq.pflip[1]))
1a0e7918 6426 radeon_crtc_handle_vblank(rdev, 1);
25a857fb
AD		 6427 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6428 DRM_DEBUG("IH: D2 vblank\n");
6429 }
6430 break;
6431 case 1: /* D2 vline */
6432 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6433 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6434 DRM_DEBUG("IH: D2 vline\n");
6435 }
6436 break;
6437 default:
6438 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6439 break;
6440 }
6441 break;
6442 case 3: /* D3 vblank/vline */
6443 switch (src_data) {
6444 case 0: /* D3 vblank */
6445 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6446 if (rdev->irq.crtc_vblank_int[2]) {
6447 drm_handle_vblank(rdev->ddev, 2);
6448 rdev->pm.vblank_sync = true;
6449 wake_up(&rdev->irq.vblank_queue);
6450 }
736fc37f 6451 if (atomic_read(&rdev->irq.pflip[2]))
1a0e7918 6452 radeon_crtc_handle_vblank(rdev, 2);
25a857fb
AD		 6453 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6454 DRM_DEBUG("IH: D3 vblank\n");
6455 }
6456 break;
6457 case 1: /* D3 vline */
6458 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6459 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6460 DRM_DEBUG("IH: D3 vline\n");
6461 }
6462 break;
6463 default:
6464 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6465 break;
6466 }
6467 break;
6468 case 4: /* D4 vblank/vline */
6469 switch (src_data) {
6470 case 0: /* D4 vblank */
6471 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6472 if (rdev->irq.crtc_vblank_int[3]) {
6473 drm_handle_vblank(rdev->ddev, 3);
6474 rdev->pm.vblank_sync = true;
6475 wake_up(&rdev->irq.vblank_queue);
6476 }
736fc37f 6477 if (atomic_read(&rdev->irq.pflip[3]))
1a0e7918 6478 radeon_crtc_handle_vblank(rdev, 3);
25a857fb
AD		 6479 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6480 DRM_DEBUG("IH: D4 vblank\n");
6481 }
6482 break;
6483 case 1: /* D4 vline */
6484 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6485 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6486 DRM_DEBUG("IH: D4 vline\n");
6487 }
6488 break;
6489 default:
6490 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6491 break;
6492 }
6493 break;
6494 case 5: /* D5 vblank/vline */
6495 switch (src_data) {
6496 case 0: /* D5 vblank */
6497 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6498 if (rdev->irq.crtc_vblank_int[4]) {
6499 drm_handle_vblank(rdev->ddev, 4);
6500 rdev->pm.vblank_sync = true;
6501 wake_up(&rdev->irq.vblank_queue);
6502 }
736fc37f 6503 if (atomic_read(&rdev->irq.pflip[4]))
1a0e7918 6504 radeon_crtc_handle_vblank(rdev, 4);
25a857fb
AD		 6505 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6506 DRM_DEBUG("IH: D5 vblank\n");
6507 }
6508 break;
6509 case 1: /* D5 vline */
6510 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6511 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6512 DRM_DEBUG("IH: D5 vline\n");
6513 }
6514 break;
6515 default:
6516 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6517 break;
6518 }
6519 break;
6520 case 6: /* D6 vblank/vline */
6521 switch (src_data) {
6522 case 0: /* D6 vblank */
6523 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6524 if (rdev->irq.crtc_vblank_int[5]) {
6525 drm_handle_vblank(rdev->ddev, 5);
6526 rdev->pm.vblank_sync = true;
6527 wake_up(&rdev->irq.vblank_queue);
6528 }
736fc37f 6529 if (atomic_read(&rdev->irq.pflip[5]))
1a0e7918 6530 radeon_crtc_handle_vblank(rdev, 5);
25a857fb
AD		 6531 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6532 DRM_DEBUG("IH: D6 vblank\n");
6533 }
6534 break;
6535 case 1: /* D6 vline */
6536 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6537 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6538 DRM_DEBUG("IH: D6 vline\n");
6539 }
6540 break;
6541 default:
6542 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6543 break;
6544 }
6545 break;
f5d636d2
CK		 6546 case 8: /* D1 page flip */
6547 case 10: /* D2 page flip */
6548 case 12: /* D3 page flip */
6549 case 14: /* D4 page flip */
6550 case 16: /* D5 page flip */
6551 case 18: /* D6 page flip */
6552 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
39dc5454
MK		 6553 if (radeon_use_pflipirq > 0)
6554 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
f5d636d2 6555 break;
25a857fb
AD		 6556 case 42: /* HPD hotplug */
6557 switch (src_data) {
6558 case 0:
6559 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6560 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6561 queue_hotplug = true;
6562 DRM_DEBUG("IH: HPD1\n");
6563 }
6564 break;
6565 case 1:
6566 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6567 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6568 queue_hotplug = true;
6569 DRM_DEBUG("IH: HPD2\n");
6570 }
6571 break;
6572 case 2:
6573 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6574 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6575 queue_hotplug = true;
6576 DRM_DEBUG("IH: HPD3\n");
6577 }
6578 break;
6579 case 3:
6580 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6581 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6582 queue_hotplug = true;
6583 DRM_DEBUG("IH: HPD4\n");
6584 }
6585 break;
6586 case 4:
6587 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6588 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6589 queue_hotplug = true;
6590 DRM_DEBUG("IH: HPD5\n");
6591 }
6592 break;
6593 case 5:
6594 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6595 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6596 queue_hotplug = true;
6597 DRM_DEBUG("IH: HPD6\n");
6598 }
6599 break;
6600 default:
6601 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6602 break;
6603 }
6604 break;
b927e1c2
CK		 6605 case 124: /* UVD */
6606 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6607 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6608 break;
ae133a11
CK		 6609 case 146:
6610 case 147:
fbf6dc7a
AD		 6611 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6612 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
9b7d786b
CK		 6613 /* reset addr and status */
6614 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6615 if (addr == 0x0 && status == 0x0)
6616 break;
ae133a11
CK		 6617 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6618 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
fbf6dc7a 6619 addr);
ae133a11 6620 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
fbf6dc7a
AD		 6621 status);
6622 si_vm_decode_fault(rdev, status, addr);
ae133a11 6623 break;
25a857fb
AD		 6624 case 176: /* RINGID0 CP_INT */
6625 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6626 break;
6627 case 177: /* RINGID1 CP_INT */
6628 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6629 break;
6630 case 178: /* RINGID2 CP_INT */
6631 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6632 break;
6633 case 181: /* CP EOP event */
6634 DRM_DEBUG("IH: CP EOP\n");
6635 switch (ring_id) {
6636 case 0:
6637 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6638 break;
6639 case 1:
6640 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6641 break;
6642 case 2:
6643 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6644 break;
6645 }
6646 break;
8c5fd7ef
AD		 6647 case 224: /* DMA trap event */
6648 DRM_DEBUG("IH: DMA trap\n");
6649 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6650 break;
a9e61410
AD		 6651 case 230: /* thermal low to high */
6652 DRM_DEBUG("IH: thermal low to high\n");
6653 rdev->pm.dpm.thermal.high_to_low = false;
6654 queue_thermal = true;
6655 break;
6656 case 231: /* thermal high to low */
6657 DRM_DEBUG("IH: thermal high to low\n");
6658 rdev->pm.dpm.thermal.high_to_low = true;
6659 queue_thermal = true;
6660 break;
25a857fb
AD		 6661 case 233: /* GUI IDLE */
6662 DRM_DEBUG("IH: GUI idle\n");
25a857fb 6663 break;
8c5fd7ef
AD		 6664 case 244: /* DMA trap event */
6665 DRM_DEBUG("IH: DMA1 trap\n");
6666 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6667 break;
25a857fb
AD		 6668 default:
6669 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6670 break;
6671 }
6672
6673 /* wptr/rptr are in bytes! */
6674 rptr += 16;
6675 rptr &= rdev->ih.ptr_mask;
f55e03b9 6676 WREG32(IH_RB_RPTR, rptr);
25a857fb 6677 }
25a857fb
AD		 6678 if (queue_hotplug)
6679 schedule_work(&rdev->hotplug_work);
a9e61410
AD		 6680 if (queue_thermal && rdev->pm.dpm_enabled)
6681 schedule_work(&rdev->pm.dpm.thermal.work);
25a857fb 6682 rdev->ih.rptr = rptr;
c20dc369
CK		 6683 atomic_set(&rdev->ih.lock, 0);
6684
6685 /* make sure wptr hasn't changed while processing */
6686 wptr = si_get_ih_wptr(rdev);
6687 if (wptr != rptr)
6688 goto restart_ih;
6689
25a857fb
AD		 6690 return IRQ_HANDLED;
6691}
6692
9b136d51
AD		 6693/*
6694 * startup/shutdown callbacks
6695 */
6696static int si_startup(struct radeon_device *rdev)
6697{
6698 struct radeon_ring *ring;
6699 int r;
6700
b9d305df
AD		 6701 /* enable pcie gen2/3 link */
6702 si_pcie_gen3_enable(rdev);
e0bcf165
AD		 6703 /* enable aspm */
6704 si_program_aspm(rdev);
b9d305df 6705
e5903d39
AD		 6706 /* scratch needs to be initialized before MC */
6707 r = r600_vram_scratch_init(rdev);
6708 if (r)
6709 return r;
6710
6fab3feb
AD		 6711 si_mc_program(rdev);
6712
6c7bccea
AD		 6713 if (!rdev->pm.dpm_enabled) {
6714 r = si_mc_load_microcode(rdev);
6715 if (r) {
6716 DRM_ERROR("Failed to load MC firmware!\n");
6717 return r;
6718 }
9b136d51
AD		 6719 }
6720
9b136d51
AD		 6721 r = si_pcie_gart_enable(rdev);
6722 if (r)
6723 return r;
6724 si_gpu_init(rdev);
6725
9b136d51 6726 /* allocate rlc buffers */
1fd11777
AD		 6727 if (rdev->family == CHIP_VERDE) {
6728 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6729 rdev->rlc.reg_list_size =
6730 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6731 }
6732 rdev->rlc.cs_data = si_cs_data;
6733 r = sumo_rlc_init(rdev);
9b136d51
AD		 6734 if (r) {
6735 DRM_ERROR("Failed to init rlc BOs!\n");
6736 return r;
6737 }
6738
6739 /* allocate wb buffer */
6740 r = radeon_wb_init(rdev);
6741 if (r)
6742 return r;
6743
6744 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6745 if (r) {
6746 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6747 return r;
6748 }
6749
6750 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6751 if (r) {
6752 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6753 return r;
6754 }
6755
6756 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6757 if (r) {
6758 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6759 return r;
6760 }
6761
8c5fd7ef
AD		 6762 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6763 if (r) {
6764 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6765 return r;
6766 }
6767
6768 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6769 if (r) {
6770 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6771 return r;
6772 }
6773
1df0d523 6774 if (rdev->has_uvd) {
e409b128 6775 r = uvd_v2_2_resume(rdev);
1df0d523
AD		 6776 if (!r) {
6777 r = radeon_fence_driver_start_ring(rdev,
6778 R600_RING_TYPE_UVD_INDEX);
6779 if (r)
6780 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6781 }
f2ba57b5 6782 if (r)
1df0d523 6783 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
f2ba57b5 6784 }
f2ba57b5 6785
9b136d51 6786 /* Enable IRQ */
e49f3959
AH		 6787 if (!rdev->irq.installed) {
6788 r = radeon_irq_kms_init(rdev);
6789 if (r)
6790 return r;
6791 }
6792
9b136d51
AD		 6793 r = si_irq_init(rdev);
6794 if (r) {
6795 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6796 radeon_irq_kms_fini(rdev);
6797 return r;
6798 }
6799 si_irq_set(rdev);
6800
6801 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6802 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2e1e6dad 6803 RADEON_CP_PACKET2);
9b136d51
AD		 6804 if (r)
6805 return r;
6806
6807 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6808 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
2e1e6dad 6809 RADEON_CP_PACKET2);
9b136d51
AD		 6810 if (r)
6811 return r;
6812
6813 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6814 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
2e1e6dad 6815 RADEON_CP_PACKET2);
9b136d51
AD		 6816 if (r)
6817 return r;
6818
8c5fd7ef
AD		 6819 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6820 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 6821 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
8c5fd7ef
AD		 6822 if (r)
6823 return r;
6824
6825 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6826 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2e1e6dad 6827 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
8c5fd7ef
AD		 6828 if (r)
6829 return r;
6830
9b136d51
AD		 6831 r = si_cp_load_microcode(rdev);
6832 if (r)
6833 return r;
6834 r = si_cp_resume(rdev);
6835 if (r)
6836 return r;
6837
8c5fd7ef
AD		 6838 r = cayman_dma_resume(rdev);
6839 if (r)
6840 return r;
6841
1df0d523
AD		 6842 if (rdev->has_uvd) {
6843 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6844 if (ring->ring_size) {
02c9f7fa 6845 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2e1e6dad 6846 RADEON_CP_PACKET2);
1df0d523 6847 if (!r)
e409b128 6848 r = uvd_v1_0_init(rdev);
1df0d523
AD		 6849 if (r)
6850 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6851 }
f2ba57b5
CK		 6852 }
6853
2898c348
CK		 6854 r = radeon_ib_pool_init(rdev);
6855 if (r) {
6856 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
9b136d51 6857 return r;
2898c348 6858 }
9b136d51 6859
c6105f24
CK		 6860 r = radeon_vm_manager_init(rdev);
6861 if (r) {
6862 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
9b136d51 6863 return r;
c6105f24 6864 }
9b136d51 6865
b530602f
AD		 6866 r = dce6_audio_init(rdev);
6867 if (r)
6868 return r;
6869
9b136d51
AD		 6870 return 0;
6871}
6872
6873int si_resume(struct radeon_device *rdev)
6874{
6875 int r;
6876
6877 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6878 * posting will perform necessary task to bring back GPU into good
6879 * shape.
6880 */
6881 /* post card */
6882 atom_asic_init(rdev->mode_info.atom_context);
6883
205996c0
AD		 6884 /* init golden registers */
6885 si_init_golden_registers(rdev);
6886
bc6a6295
AD		 6887 if (rdev->pm.pm_method == PM_METHOD_DPM)
6888 radeon_pm_resume(rdev);
6c7bccea 6889
9b136d51
AD		 6890 rdev->accel_working = true;
6891 r = si_startup(rdev);
6892 if (r) {
6893 DRM_ERROR("si startup failed on resume\n");
6894 rdev->accel_working = false;
6895 return r;
6896 }
6897
6898 return r;
6899
6900}
6901
6902int si_suspend(struct radeon_device *rdev)
6903{
6c7bccea 6904 radeon_pm_suspend(rdev);
b530602f 6905 dce6_audio_fini(rdev);
fa3daf9a 6906 radeon_vm_manager_fini(rdev);
9b136d51 6907 si_cp_enable(rdev, false);
8c5fd7ef 6908 cayman_dma_stop(rdev);
1df0d523 6909 if (rdev->has_uvd) {
e409b128 6910 uvd_v1_0_fini(rdev);
1df0d523
AD		 6911 radeon_uvd_suspend(rdev);
6912 }
e16866ec
AD		 6913 si_fini_pg(rdev);
6914 si_fini_cg(rdev);
9b136d51
AD		 6915 si_irq_suspend(rdev);
6916 radeon_wb_disable(rdev);
6917 si_pcie_gart_disable(rdev);
6918 return 0;
6919}
6920
6921/* Plan is to move initialization in that function and use
6922 * helper function so that radeon_device_init pretty much
6923 * do nothing more than calling asic specific function. This
6924 * should also allow to remove a bunch of callback function
6925 * like vram_info.
6926 */
6927int si_init(struct radeon_device *rdev)
6928{
6929 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6930 int r;
6931
9b136d51
AD		 6932 /* Read BIOS */
6933 if (!radeon_get_bios(rdev)) {
6934 if (ASIC_IS_AVIVO(rdev))
6935 return -EINVAL;
6936 }
6937 /* Must be an ATOMBIOS */
6938 if (!rdev->is_atom_bios) {
6939 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6940 return -EINVAL;
6941 }
6942 r = radeon_atombios_init(rdev);
6943 if (r)
6944 return r;
6945
6946 /* Post card if necessary */
6947 if (!radeon_card_posted(rdev)) {
6948 if (!rdev->bios) {
6949 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6950 return -EINVAL;
6951 }
6952 DRM_INFO("GPU not posted. posting now...\n");
6953 atom_asic_init(rdev->mode_info.atom_context);
6954 }
205996c0
AD		 6955 /* init golden registers */
6956 si_init_golden_registers(rdev);
9b136d51
AD		 6957 /* Initialize scratch registers */
6958 si_scratch_init(rdev);
6959 /* Initialize surface registers */
6960 radeon_surface_init(rdev);
6961 /* Initialize clocks */
6962 radeon_get_clock_info(rdev->ddev);
6963
6964 /* Fence driver */
6965 r = radeon_fence_driver_init(rdev);
6966 if (r)
6967 return r;
6968
6969 /* initialize memory controller */
6970 r = si_mc_init(rdev);
6971 if (r)
6972 return r;
6973 /* Memory manager */
6974 r = radeon_bo_init(rdev);
6975 if (r)
6976 return r;
6977
01ac8794
AD		 6978 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6979 !rdev->rlc_fw || !rdev->mc_fw) {
6980 r = si_init_microcode(rdev);
6981 if (r) {
6982 DRM_ERROR("Failed to load firmware!\n");
6983 return r;
6984 }
6985 }
6986
6c7bccea
AD		 6987 /* Initialize power management */
6988 radeon_pm_init(rdev);
6989
9b136d51
AD		 6990 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6991 ring->ring_obj = NULL;
6992 r600_ring_init(rdev, ring, 1024 * 1024);
6993
6994 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6995 ring->ring_obj = NULL;
6996 r600_ring_init(rdev, ring, 1024 * 1024);
6997
6998 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6999 ring->ring_obj = NULL;
7000 r600_ring_init(rdev, ring, 1024 * 1024);
7001
8c5fd7ef
AD		 7002 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7003 ring->ring_obj = NULL;
7004 r600_ring_init(rdev, ring, 64 * 1024);
7005
7006 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7007 ring->ring_obj = NULL;
7008 r600_ring_init(rdev, ring, 64 * 1024);
7009
1df0d523
AD		 7010 if (rdev->has_uvd) {
7011 r = radeon_uvd_init(rdev);
7012 if (!r) {
7013 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7014 ring->ring_obj = NULL;
7015 r600_ring_init(rdev, ring, 4096);
7016 }
f2ba57b5
CK		 7017 }
7018
9b136d51
AD		 7019 rdev->ih.ring_obj = NULL;
7020 r600_ih_ring_init(rdev, 64 * 1024);
7021
7022 r = r600_pcie_gart_init(rdev);
7023 if (r)
7024 return r;
7025
9b136d51 7026 rdev->accel_working = true;
9b136d51
AD		 7027 r = si_startup(rdev);
7028 if (r) {
7029 dev_err(rdev->dev, "disabling GPU acceleration\n");
7030 si_cp_fini(rdev);
8c5fd7ef 7031 cayman_dma_fini(rdev);
9b136d51 7032 si_irq_fini(rdev);
1fd11777 7033 sumo_rlc_fini(rdev);
9b136d51 7034 radeon_wb_fini(rdev);
2898c348 7035 radeon_ib_pool_fini(rdev);
9b136d51
AD		 7036 radeon_vm_manager_fini(rdev);
7037 radeon_irq_kms_fini(rdev);
7038 si_pcie_gart_fini(rdev);
7039 rdev->accel_working = false;
7040 }
7041
7042 /* Don't start up if the MC ucode is missing.
7043 * The default clocks and voltages before the MC ucode
7044 * is loaded are not suffient for advanced operations.
7045 */
7046 if (!rdev->mc_fw) {
7047 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7048 return -EINVAL;
7049 }
7050
7051 return 0;
7052}
7053
7054void si_fini(struct radeon_device *rdev)
7055{
6c7bccea 7056 radeon_pm_fini(rdev);
9b136d51 7057 si_cp_fini(rdev);
8c5fd7ef 7058 cayman_dma_fini(rdev);
f8f84ac5 7059 si_fini_pg(rdev);
e16866ec 7060 si_fini_cg(rdev);
9b136d51 7061 si_irq_fini(rdev);
1fd11777 7062 sumo_rlc_fini(rdev);
9b136d51
AD		 7063 radeon_wb_fini(rdev);
7064 radeon_vm_manager_fini(rdev);
2898c348 7065 radeon_ib_pool_fini(rdev);
9b136d51 7066 radeon_irq_kms_fini(rdev);
2858c00d 7067 if (rdev->has_uvd) {
e409b128 7068 uvd_v1_0_fini(rdev);
1df0d523 7069 radeon_uvd_fini(rdev);
2858c00d 7070 }
9b136d51
AD		 7071 si_pcie_gart_fini(rdev);
7072 r600_vram_scratch_fini(rdev);
7073 radeon_gem_fini(rdev);
9b136d51
AD		 7074 radeon_fence_driver_fini(rdev);
7075 radeon_bo_fini(rdev);
7076 radeon_atombios_fini(rdev);
7077 kfree(rdev->bios);
7078 rdev->bios = NULL;
7079}
7080
6759a0a7 7081/**
d0418894 7082 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6759a0a7
MO		 7083 *
7084 * @rdev: radeon_device pointer
7085 *
7086 * Fetches a GPU clock counter snapshot (SI).
7087 * Returns the 64 bit clock counter snapshot.
7088 */
d0418894 7089uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6759a0a7
MO		 7090{
7091 uint64_t clock;
7092
7093 mutex_lock(&rdev->gpu_clock_mutex);
7094 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
7095 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
7096 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
7097 mutex_unlock(&rdev->gpu_clock_mutex);
7098 return clock;
7099}
2539eb02 7100
2539eb02
CK		 7101int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
7102{
facd112d 7103 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
2539eb02
CK		 7104 int r;
7105
4ed10835
CK		 7106 /* bypass vclk and dclk with bclk */
7107 WREG32_P(CG_UPLL_FUNC_CNTL_2,
7108 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
7109 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7110
7111 /* put PLL in bypass mode */
7112 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
7113
7114 if (!vclk || !dclk) {
7115 /* keep the Bypass mode, put PLL to sleep */
7116 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
7117 return 0;
7118 }
7119
facd112d
CK		 7120 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
7121 16384, 0x03FFFFFF, 0, 128, 5,
7122 &fb_div, &vclk_div, &dclk_div);
7123 if (r)
7124 return r;
2539eb02
CK		 7125
7126 /* set RESET_ANTI_MUX to 0 */
7127 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7128
7129 /* set VCO_MODE to 1 */
7130 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
7131
7132 /* toggle UPLL_SLEEP to 1 then back to 0 */
7133 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
7134 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
7135
7136 /* deassert UPLL_RESET */
7137 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7138
7139 mdelay(1);
7140
facd112d 7141 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 7142 if (r)
7143 return r;
7144
7145 /* assert UPLL_RESET again */
7146 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
7147
7148 /* disable spread spectrum. */
7149 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7150
7151 /* set feedback divider */
facd112d 7152 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
2539eb02
CK		 7153
7154 /* set ref divider to 0 */
7155 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
7156
facd112d 7157 if (fb_div < 307200)
2539eb02
CK		 7158 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
7159 else
7160 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
7161
7162 /* set PDIV_A and PDIV_B */
7163 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 7164 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
2539eb02
CK		 7165 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
7166
7167 /* give the PLL some time to settle */
7168 mdelay(15);
7169
7170 /* deassert PLL_RESET */
7171 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7172
7173 mdelay(15);
7174
7175 /* switch from bypass mode to normal mode */
7176 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
7177
facd112d 7178 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 7179 if (r)
7180 return r;
7181
7182 /* switch VCLK and DCLK selection */
7183 WREG32_P(CG_UPLL_FUNC_CNTL_2,
7184 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
7185 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7186
7187 mdelay(100);
7188
7189 return 0;
7190}
b9d305df
AD		 7191
7192static void si_pcie_gen3_enable(struct radeon_device *rdev)
7193{
7194 struct pci_dev *root = rdev->pdev->bus->self;
7195 int bridge_pos, gpu_pos;
7196 u32 speed_cntl, mask, current_data_rate;
7197 int ret, i;
7198 u16 tmp16;
7199
0bd252de
AW		 7200 if (pci_is_root_bus(rdev->pdev->bus))
7201 return;
7202
b9d305df
AD		 7203 if (radeon_pcie_gen2 == 0)
7204 return;
7205
7206 if (rdev->flags & RADEON_IS_IGP)
7207 return;
7208
7209 if (!(rdev->flags & RADEON_IS_PCIE))
7210 return;
7211
7212 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
7213 if (ret != 0)
7214 return;
7215
7216 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
7217 return;
7218
7219 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7220 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
7221 LC_CURRENT_DATA_RATE_SHIFT;
7222 if (mask & DRM_PCIE_SPEED_80) {
7223 if (current_data_rate == 2) {
7224 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
7225 return;
7226 }
7227 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
7228 } else if (mask & DRM_PCIE_SPEED_50) {
7229 if (current_data_rate == 1) {
7230 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
7231 return;
7232 }
7233 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
7234 }
7235
7236 bridge_pos = pci_pcie_cap(root);
7237 if (!bridge_pos)
7238 return;
7239
7240 gpu_pos = pci_pcie_cap(rdev->pdev);
7241 if (!gpu_pos)
7242 return;
7243
7244 if (mask & DRM_PCIE_SPEED_80) {
7245 /* re-try equalization if gen3 is not already enabled */
7246 if (current_data_rate != 2) {
7247 u16 bridge_cfg, gpu_cfg;
7248 u16 bridge_cfg2, gpu_cfg2;
7249 u32 max_lw, current_lw, tmp;
7250
7251 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7252 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7253
7254 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
7255 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7256
7257 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
7258 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7259
7260 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
7261 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
7262 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
7263
7264 if (current_lw < max_lw) {
7265 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7266 if (tmp & LC_RENEGOTIATION_SUPPORT) {
7267 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
7268 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
7269 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
7270 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
7271 }
7272 }
7273
7274 for (i = 0; i < 10; i++) {
7275 /* check status */
7276 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
7277 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7278 break;
7279
7280 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7281 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7282
7283 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
7284 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
7285
7286 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7287 tmp |= LC_SET_QUIESCE;
7288 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7289
7290 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7291 tmp |= LC_REDO_EQ;
7292 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7293
7294 mdelay(100);
7295
7296 /* linkctl */
7297 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
7298 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7299 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
7300 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7301
7302 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
7303 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7304 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
7305 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7306
7307 /* linkctl2 */
7308 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
7309 tmp16 &= ~((1 << 4) | (7 << 9));
7310 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
7311 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
7312
7313 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7314 tmp16 &= ~((1 << 4) | (7 << 9));
7315 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
7316 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7317
7318 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7319 tmp &= ~LC_SET_QUIESCE;
7320 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7321 }
7322 }
7323 }
7324
7325 /* set the link speed */
7326 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7327 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7328 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7329
7330 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7331 tmp16 &= ~0xf;
7332 if (mask & DRM_PCIE_SPEED_80)
7333 tmp16 |= 3; /* gen3 */
7334 else if (mask & DRM_PCIE_SPEED_50)
7335 tmp16 |= 2; /* gen2 */
7336 else
7337 tmp16 |= 1; /* gen1 */
7338 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7339
7340 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7341 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7342 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7343
7344 for (i = 0; i < rdev->usec_timeout; i++) {
7345 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7346 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7347 break;
7348 udelay(1);
7349 }
7350}
7351
e0bcf165
AD		 7352static void si_program_aspm(struct radeon_device *rdev)
7353{
7354 u32 data, orig;
7355 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7356 bool disable_clkreq = false;
7357
1294d4a3
AD		 7358 if (radeon_aspm == 0)
7359 return;
7360
e0bcf165
AD		 7361 if (!(rdev->flags & RADEON_IS_PCIE))
7362 return;
7363
7364 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7365 data &= ~LC_XMIT_N_FTS_MASK;
7366 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7367 if (orig != data)
7368 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7369
7370 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7371 data |= LC_GO_TO_RECOVERY;
7372 if (orig != data)
7373 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7374
7375 orig = data = RREG32_PCIE(PCIE_P_CNTL);
7376 data |= P_IGNORE_EDB_ERR;
7377 if (orig != data)
7378 WREG32_PCIE(PCIE_P_CNTL, data);
7379
7380 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7381 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7382 data |= LC_PMI_TO_L1_DIS;
7383 if (!disable_l0s)
7384 data |= LC_L0S_INACTIVITY(7);
7385
7386 if (!disable_l1) {
7387 data |= LC_L1_INACTIVITY(7);
7388 data &= ~LC_PMI_TO_L1_DIS;
7389 if (orig != data)
7390 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7391
7392 if (!disable_plloff_in_l1) {
7393 bool clk_req_support;
7394
7395 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7396 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7397 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7398 if (orig != data)
7399 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7400
7401 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7402 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7403 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7404 if (orig != data)
7405 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7406
7407 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7408 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7409 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7410 if (orig != data)
7411 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7412
7413 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7414 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7415 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7416 if (orig != data)
7417 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7418
7419 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7420 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7421 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7422 if (orig != data)
7423 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7424
7425 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7426 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7427 if (orig != data)
7428 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7429
7430 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7431 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7432 if (orig != data)
7433 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7434
7435 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7436 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7437 if (orig != data)
7438 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7439
7440 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7441 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7442 if (orig != data)
7443 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7444
7445 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7446 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7447 if (orig != data)
7448 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7449
7450 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7451 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7452 if (orig != data)
7453 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7454
7455 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7456 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7457 if (orig != data)
7458 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7459 }
7460 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7461 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7462 data |= LC_DYN_LANES_PWR_STATE(3);
7463 if (orig != data)
7464 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7465
7466 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7467 data &= ~LS2_EXIT_TIME_MASK;
7468 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7469 data |= LS2_EXIT_TIME(5);
7470 if (orig != data)
7471 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7472
7473 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7474 data &= ~LS2_EXIT_TIME_MASK;
7475 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7476 data |= LS2_EXIT_TIME(5);
7477 if (orig != data)
7478 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7479
0bd252de
AW		 7480 if (!disable_clkreq &&
7481 !pci_is_root_bus(rdev->pdev->bus)) {
e0bcf165
AD		 7482 struct pci_dev *root = rdev->pdev->bus->self;
7483 u32 lnkcap;
7484
7485 clk_req_support = false;
7486 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7487 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7488 clk_req_support = true;
7489 } else {
7490 clk_req_support = false;
7491 }
7492
7493 if (clk_req_support) {
7494 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7495 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7496 if (orig != data)
7497 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7498
7499 orig = data = RREG32(THM_CLK_CNTL);
7500 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7501 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7502 if (orig != data)
7503 WREG32(THM_CLK_CNTL, data);
7504
7505 orig = data = RREG32(MISC_CLK_CNTL);
7506 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7507 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7508 if (orig != data)
7509 WREG32(MISC_CLK_CNTL, data);
7510
7511 orig = data = RREG32(CG_CLKPIN_CNTL);
7512 data &= ~BCLK_AS_XCLK;
7513 if (orig != data)
7514 WREG32(CG_CLKPIN_CNTL, data);
7515
7516 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7517 data &= ~FORCE_BIF_REFCLK_EN;
7518 if (orig != data)
7519 WREG32(CG_CLKPIN_CNTL_2, data);
7520
7521 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7522 data &= ~MPLL_CLKOUT_SEL_MASK;
7523 data |= MPLL_CLKOUT_SEL(4);
7524 if (orig != data)
7525 WREG32(MPLL_BYPASSCLK_SEL, data);
7526
7527 orig = data = RREG32(SPLL_CNTL_MODE);
7528 data &= ~SPLL_REFCLK_SEL_MASK;
7529 if (orig != data)
7530 WREG32(SPLL_CNTL_MODE, data);
7531 }
7532 }
7533 } else {
7534 if (orig != data)
7535 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7536 }
7537
7538 orig = data = RREG32_PCIE(PCIE_CNTL2);
7539 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7540 if (orig != data)
7541 WREG32_PCIE(PCIE_CNTL2, data);
7542
7543 if (!disable_l0s) {
7544 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7545 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7546 data = RREG32_PCIE(PCIE_LC_STATUS1);
7547 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7548 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7549 data &= ~LC_L0S_INACTIVITY_MASK;
7550 if (orig != data)
7551 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7552 }
7553 }
7554 }
7555}
ubuntu-zesty-kernel mirror