]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blob - drivers/gpu/drm/radeon/evergreen.c
projects / mirror_ubuntu-zesty-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'clk-fixes-for-linus' of https://git.linaro.org/people/mike.turquette/linux
[mirror_ubuntu-zesty-kernel.git] / drivers / gpu / drm / radeon / evergreen.c
1 /*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <drm/drmP.h>
27 #include "radeon.h"
28 #include "radeon_asic.h"
29 #include <drm/radeon_drm.h>
30 #include "evergreend.h"
31 #include "atom.h"
32 #include "avivod.h"
33 #include "evergreen_reg.h"
34 #include "evergreen_blit_shaders.h"
35 #include "radeon_ucode.h"
36
37 static const u32 crtc_offsets[6] =
38 {
39 EVERGREEN_CRTC0_REGISTER_OFFSET,
40 EVERGREEN_CRTC1_REGISTER_OFFSET,
41 EVERGREEN_CRTC2_REGISTER_OFFSET,
42 EVERGREEN_CRTC3_REGISTER_OFFSET,
43 EVERGREEN_CRTC4_REGISTER_OFFSET,
44 EVERGREEN_CRTC5_REGISTER_OFFSET
45 };
46
47 #include "clearstate_evergreen.h"
48
49 static const u32 sumo_rlc_save_restore_register_list[] =
50 {
51 0x98fc,
52 0x9830,
53 0x9834,
54 0x9838,
55 0x9870,
56 0x9874,
57 0x8a14,
58 0x8b24,
59 0x8bcc,
60 0x8b10,
61 0x8d00,
62 0x8d04,
63 0x8c00,
64 0x8c04,
65 0x8c08,
66 0x8c0c,
67 0x8d8c,
68 0x8c20,
69 0x8c24,
70 0x8c28,
71 0x8c18,
72 0x8c1c,
73 0x8cf0,
74 0x8e2c,
75 0x8e38,
76 0x8c30,
77 0x9508,
78 0x9688,
79 0x9608,
80 0x960c,
81 0x9610,
82 0x9614,
83 0x88c4,
84 0x88d4,
85 0xa008,
86 0x900c,
87 0x9100,
88 0x913c,
89 0x98f8,
90 0x98f4,
91 0x9b7c,
92 0x3f8c,
93 0x8950,
94 0x8954,
95 0x8a18,
96 0x8b28,
97 0x9144,
98 0x9148,
99 0x914c,
100 0x3f90,
101 0x3f94,
102 0x915c,
103 0x9160,
104 0x9178,
105 0x917c,
106 0x9180,
107 0x918c,
108 0x9190,
109 0x9194,
110 0x9198,
111 0x919c,
112 0x91a8,
113 0x91ac,
114 0x91b0,
115 0x91b4,
116 0x91b8,
117 0x91c4,
118 0x91c8,
119 0x91cc,
120 0x91d0,
121 0x91d4,
122 0x91e0,
123 0x91e4,
124 0x91ec,
125 0x91f0,
126 0x91f4,
127 0x9200,
128 0x9204,
129 0x929c,
130 0x9150,
131 0x802c,
132 };
133
134 static void evergreen_gpu_init(struct radeon_device *rdev);
135 void evergreen_fini(struct radeon_device *rdev);
136 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
137 void evergreen_program_aspm(struct radeon_device *rdev);
138 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
139 int ring, u32 cp_int_cntl);
140 extern void cayman_vm_decode_fault(struct radeon_device *rdev,
141 u32 status, u32 addr);
142 void cik_init_cp_pg_table(struct radeon_device *rdev);
143
144 extern u32 si_get_csb_size(struct radeon_device *rdev);
145 extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
146 extern u32 cik_get_csb_size(struct radeon_device *rdev);
147 extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
148 extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
149
150 static const u32 evergreen_golden_registers[] =
151 {
152 0x3f90, 0xffff0000, 0xff000000,
153 0x9148, 0xffff0000, 0xff000000,
154 0x3f94, 0xffff0000, 0xff000000,
155 0x914c, 0xffff0000, 0xff000000,
156 0x9b7c, 0xffffffff, 0x00000000,
157 0x8a14, 0xffffffff, 0x00000007,
158 0x8b10, 0xffffffff, 0x00000000,
159 0x960c, 0xffffffff, 0x54763210,
160 0x88c4, 0xffffffff, 0x000000c2,
161 0x88d4, 0xffffffff, 0x00000010,
162 0x8974, 0xffffffff, 0x00000000,
163 0xc78, 0x00000080, 0x00000080,
164 0x5eb4, 0xffffffff, 0x00000002,
165 0x5e78, 0xffffffff, 0x001000f0,
166 0x6104, 0x01000300, 0x00000000,
167 0x5bc0, 0x00300000, 0x00000000,
168 0x7030, 0xffffffff, 0x00000011,
169 0x7c30, 0xffffffff, 0x00000011,
170 0x10830, 0xffffffff, 0x00000011,
171 0x11430, 0xffffffff, 0x00000011,
172 0x12030, 0xffffffff, 0x00000011,
173 0x12c30, 0xffffffff, 0x00000011,
174 0xd02c, 0xffffffff, 0x08421000,
175 0x240c, 0xffffffff, 0x00000380,
176 0x8b24, 0xffffffff, 0x00ff0fff,
177 0x28a4c, 0x06000000, 0x06000000,
178 0x10c, 0x00000001, 0x00000001,
179 0x8d00, 0xffffffff, 0x100e4848,
180 0x8d04, 0xffffffff, 0x00164745,
181 0x8c00, 0xffffffff, 0xe4000003,
182 0x8c04, 0xffffffff, 0x40600060,
183 0x8c08, 0xffffffff, 0x001c001c,
184 0x8cf0, 0xffffffff, 0x08e00620,
185 0x8c20, 0xffffffff, 0x00800080,
186 0x8c24, 0xffffffff, 0x00800080,
187 0x8c18, 0xffffffff, 0x20202078,
188 0x8c1c, 0xffffffff, 0x00001010,
189 0x28350, 0xffffffff, 0x00000000,
190 0xa008, 0xffffffff, 0x00010000,
191 0x5c4, 0xffffffff, 0x00000001,
192 0x9508, 0xffffffff, 0x00000002,
193 0x913c, 0x0000000f, 0x0000000a
194 };
195
196 static const u32 evergreen_golden_registers2[] =
197 {
198 0x2f4c, 0xffffffff, 0x00000000,
199 0x54f4, 0xffffffff, 0x00000000,
200 0x54f0, 0xffffffff, 0x00000000,
201 0x5498, 0xffffffff, 0x00000000,
202 0x549c, 0xffffffff, 0x00000000,
203 0x5494, 0xffffffff, 0x00000000,
204 0x53cc, 0xffffffff, 0x00000000,
205 0x53c8, 0xffffffff, 0x00000000,
206 0x53c4, 0xffffffff, 0x00000000,
207 0x53c0, 0xffffffff, 0x00000000,
208 0x53bc, 0xffffffff, 0x00000000,
209 0x53b8, 0xffffffff, 0x00000000,
210 0x53b4, 0xffffffff, 0x00000000,
211 0x53b0, 0xffffffff, 0x00000000
212 };
213
214 static const u32 cypress_mgcg_init[] =
215 {
216 0x802c, 0xffffffff, 0xc0000000,
217 0x5448, 0xffffffff, 0x00000100,
218 0x55e4, 0xffffffff, 0x00000100,
219 0x160c, 0xffffffff, 0x00000100,
220 0x5644, 0xffffffff, 0x00000100,
221 0xc164, 0xffffffff, 0x00000100,
222 0x8a18, 0xffffffff, 0x00000100,
223 0x897c, 0xffffffff, 0x06000100,
224 0x8b28, 0xffffffff, 0x00000100,
225 0x9144, 0xffffffff, 0x00000100,
226 0x9a60, 0xffffffff, 0x00000100,
227 0x9868, 0xffffffff, 0x00000100,
228 0x8d58, 0xffffffff, 0x00000100,
229 0x9510, 0xffffffff, 0x00000100,
230 0x949c, 0xffffffff, 0x00000100,
231 0x9654, 0xffffffff, 0x00000100,
232 0x9030, 0xffffffff, 0x00000100,
233 0x9034, 0xffffffff, 0x00000100,
234 0x9038, 0xffffffff, 0x00000100,
235 0x903c, 0xffffffff, 0x00000100,
236 0x9040, 0xffffffff, 0x00000100,
237 0xa200, 0xffffffff, 0x00000100,
238 0xa204, 0xffffffff, 0x00000100,
239 0xa208, 0xffffffff, 0x00000100,
240 0xa20c, 0xffffffff, 0x00000100,
241 0x971c, 0xffffffff, 0x00000100,
242 0x977c, 0xffffffff, 0x00000100,
243 0x3f80, 0xffffffff, 0x00000100,
244 0xa210, 0xffffffff, 0x00000100,
245 0xa214, 0xffffffff, 0x00000100,
246 0x4d8, 0xffffffff, 0x00000100,
247 0x9784, 0xffffffff, 0x00000100,
248 0x9698, 0xffffffff, 0x00000100,
249 0x4d4, 0xffffffff, 0x00000200,
250 0x30cc, 0xffffffff, 0x00000100,
251 0xd0c0, 0xffffffff, 0xff000100,
252 0x802c, 0xffffffff, 0x40000000,
253 0x915c, 0xffffffff, 0x00010000,
254 0x9160, 0xffffffff, 0x00030002,
255 0x9178, 0xffffffff, 0x00070000,
256 0x917c, 0xffffffff, 0x00030002,
257 0x9180, 0xffffffff, 0x00050004,
258 0x918c, 0xffffffff, 0x00010006,
259 0x9190, 0xffffffff, 0x00090008,
260 0x9194, 0xffffffff, 0x00070000,
261 0x9198, 0xffffffff, 0x00030002,
262 0x919c, 0xffffffff, 0x00050004,
263 0x91a8, 0xffffffff, 0x00010006,
264 0x91ac, 0xffffffff, 0x00090008,
265 0x91b0, 0xffffffff, 0x00070000,
266 0x91b4, 0xffffffff, 0x00030002,
267 0x91b8, 0xffffffff, 0x00050004,
268 0x91c4, 0xffffffff, 0x00010006,
269 0x91c8, 0xffffffff, 0x00090008,
270 0x91cc, 0xffffffff, 0x00070000,
271 0x91d0, 0xffffffff, 0x00030002,
272 0x91d4, 0xffffffff, 0x00050004,
273 0x91e0, 0xffffffff, 0x00010006,
274 0x91e4, 0xffffffff, 0x00090008,
275 0x91e8, 0xffffffff, 0x00000000,
276 0x91ec, 0xffffffff, 0x00070000,
277 0x91f0, 0xffffffff, 0x00030002,
278 0x91f4, 0xffffffff, 0x00050004,
279 0x9200, 0xffffffff, 0x00010006,
280 0x9204, 0xffffffff, 0x00090008,
281 0x9208, 0xffffffff, 0x00070000,
282 0x920c, 0xffffffff, 0x00030002,
283 0x9210, 0xffffffff, 0x00050004,
284 0x921c, 0xffffffff, 0x00010006,
285 0x9220, 0xffffffff, 0x00090008,
286 0x9224, 0xffffffff, 0x00070000,
287 0x9228, 0xffffffff, 0x00030002,
288 0x922c, 0xffffffff, 0x00050004,
289 0x9238, 0xffffffff, 0x00010006,
290 0x923c, 0xffffffff, 0x00090008,
291 0x9240, 0xffffffff, 0x00070000,
292 0x9244, 0xffffffff, 0x00030002,
293 0x9248, 0xffffffff, 0x00050004,
294 0x9254, 0xffffffff, 0x00010006,
295 0x9258, 0xffffffff, 0x00090008,
296 0x925c, 0xffffffff, 0x00070000,
297 0x9260, 0xffffffff, 0x00030002,
298 0x9264, 0xffffffff, 0x00050004,
299 0x9270, 0xffffffff, 0x00010006,
300 0x9274, 0xffffffff, 0x00090008,
301 0x9278, 0xffffffff, 0x00070000,
302 0x927c, 0xffffffff, 0x00030002,
303 0x9280, 0xffffffff, 0x00050004,
304 0x928c, 0xffffffff, 0x00010006,
305 0x9290, 0xffffffff, 0x00090008,
306 0x9294, 0xffffffff, 0x00000000,
307 0x929c, 0xffffffff, 0x00000001,
308 0x802c, 0xffffffff, 0x40010000,
309 0x915c, 0xffffffff, 0x00010000,
310 0x9160, 0xffffffff, 0x00030002,
311 0x9178, 0xffffffff, 0x00070000,
312 0x917c, 0xffffffff, 0x00030002,
313 0x9180, 0xffffffff, 0x00050004,
314 0x918c, 0xffffffff, 0x00010006,
315 0x9190, 0xffffffff, 0x00090008,
316 0x9194, 0xffffffff, 0x00070000,
317 0x9198, 0xffffffff, 0x00030002,
318 0x919c, 0xffffffff, 0x00050004,
319 0x91a8, 0xffffffff, 0x00010006,
320 0x91ac, 0xffffffff, 0x00090008,
321 0x91b0, 0xffffffff, 0x00070000,
322 0x91b4, 0xffffffff, 0x00030002,
323 0x91b8, 0xffffffff, 0x00050004,
324 0x91c4, 0xffffffff, 0x00010006,
325 0x91c8, 0xffffffff, 0x00090008,
326 0x91cc, 0xffffffff, 0x00070000,
327 0x91d0, 0xffffffff, 0x00030002,
328 0x91d4, 0xffffffff, 0x00050004,
329 0x91e0, 0xffffffff, 0x00010006,
330 0x91e4, 0xffffffff, 0x00090008,
331 0x91e8, 0xffffffff, 0x00000000,
332 0x91ec, 0xffffffff, 0x00070000,
333 0x91f0, 0xffffffff, 0x00030002,
334 0x91f4, 0xffffffff, 0x00050004,
335 0x9200, 0xffffffff, 0x00010006,
336 0x9204, 0xffffffff, 0x00090008,
337 0x9208, 0xffffffff, 0x00070000,
338 0x920c, 0xffffffff, 0x00030002,
339 0x9210, 0xffffffff, 0x00050004,
340 0x921c, 0xffffffff, 0x00010006,
341 0x9220, 0xffffffff, 0x00090008,
342 0x9224, 0xffffffff, 0x00070000,
343 0x9228, 0xffffffff, 0x00030002,
344 0x922c, 0xffffffff, 0x00050004,
345 0x9238, 0xffffffff, 0x00010006,
346 0x923c, 0xffffffff, 0x00090008,
347 0x9240, 0xffffffff, 0x00070000,
348 0x9244, 0xffffffff, 0x00030002,
349 0x9248, 0xffffffff, 0x00050004,
350 0x9254, 0xffffffff, 0x00010006,
351 0x9258, 0xffffffff, 0x00090008,
352 0x925c, 0xffffffff, 0x00070000,
353 0x9260, 0xffffffff, 0x00030002,
354 0x9264, 0xffffffff, 0x00050004,
355 0x9270, 0xffffffff, 0x00010006,
356 0x9274, 0xffffffff, 0x00090008,
357 0x9278, 0xffffffff, 0x00070000,
358 0x927c, 0xffffffff, 0x00030002,
359 0x9280, 0xffffffff, 0x00050004,
360 0x928c, 0xffffffff, 0x00010006,
361 0x9290, 0xffffffff, 0x00090008,
362 0x9294, 0xffffffff, 0x00000000,
363 0x929c, 0xffffffff, 0x00000001,
364 0x802c, 0xffffffff, 0xc0000000
365 };
366
367 static const u32 redwood_mgcg_init[] =
368 {
369 0x802c, 0xffffffff, 0xc0000000,
370 0x5448, 0xffffffff, 0x00000100,
371 0x55e4, 0xffffffff, 0x00000100,
372 0x160c, 0xffffffff, 0x00000100,
373 0x5644, 0xffffffff, 0x00000100,
374 0xc164, 0xffffffff, 0x00000100,
375 0x8a18, 0xffffffff, 0x00000100,
376 0x897c, 0xffffffff, 0x06000100,
377 0x8b28, 0xffffffff, 0x00000100,
378 0x9144, 0xffffffff, 0x00000100,
379 0x9a60, 0xffffffff, 0x00000100,
380 0x9868, 0xffffffff, 0x00000100,
381 0x8d58, 0xffffffff, 0x00000100,
382 0x9510, 0xffffffff, 0x00000100,
383 0x949c, 0xffffffff, 0x00000100,
384 0x9654, 0xffffffff, 0x00000100,
385 0x9030, 0xffffffff, 0x00000100,
386 0x9034, 0xffffffff, 0x00000100,
387 0x9038, 0xffffffff, 0x00000100,
388 0x903c, 0xffffffff, 0x00000100,
389 0x9040, 0xffffffff, 0x00000100,
390 0xa200, 0xffffffff, 0x00000100,
391 0xa204, 0xffffffff, 0x00000100,
392 0xa208, 0xffffffff, 0x00000100,
393 0xa20c, 0xffffffff, 0x00000100,
394 0x971c, 0xffffffff, 0x00000100,
395 0x977c, 0xffffffff, 0x00000100,
396 0x3f80, 0xffffffff, 0x00000100,
397 0xa210, 0xffffffff, 0x00000100,
398 0xa214, 0xffffffff, 0x00000100,
399 0x4d8, 0xffffffff, 0x00000100,
400 0x9784, 0xffffffff, 0x00000100,
401 0x9698, 0xffffffff, 0x00000100,
402 0x4d4, 0xffffffff, 0x00000200,
403 0x30cc, 0xffffffff, 0x00000100,
404 0xd0c0, 0xffffffff, 0xff000100,
405 0x802c, 0xffffffff, 0x40000000,
406 0x915c, 0xffffffff, 0x00010000,
407 0x9160, 0xffffffff, 0x00030002,
408 0x9178, 0xffffffff, 0x00070000,
409 0x917c, 0xffffffff, 0x00030002,
410 0x9180, 0xffffffff, 0x00050004,
411 0x918c, 0xffffffff, 0x00010006,
412 0x9190, 0xffffffff, 0x00090008,
413 0x9194, 0xffffffff, 0x00070000,
414 0x9198, 0xffffffff, 0x00030002,
415 0x919c, 0xffffffff, 0x00050004,
416 0x91a8, 0xffffffff, 0x00010006,
417 0x91ac, 0xffffffff, 0x00090008,
418 0x91b0, 0xffffffff, 0x00070000,
419 0x91b4, 0xffffffff, 0x00030002,
420 0x91b8, 0xffffffff, 0x00050004,
421 0x91c4, 0xffffffff, 0x00010006,
422 0x91c8, 0xffffffff, 0x00090008,
423 0x91cc, 0xffffffff, 0x00070000,
424 0x91d0, 0xffffffff, 0x00030002,
425 0x91d4, 0xffffffff, 0x00050004,
426 0x91e0, 0xffffffff, 0x00010006,
427 0x91e4, 0xffffffff, 0x00090008,
428 0x91e8, 0xffffffff, 0x00000000,
429 0x91ec, 0xffffffff, 0x00070000,
430 0x91f0, 0xffffffff, 0x00030002,
431 0x91f4, 0xffffffff, 0x00050004,
432 0x9200, 0xffffffff, 0x00010006,
433 0x9204, 0xffffffff, 0x00090008,
434 0x9294, 0xffffffff, 0x00000000,
435 0x929c, 0xffffffff, 0x00000001,
436 0x802c, 0xffffffff, 0xc0000000
437 };
438
439 static const u32 cedar_golden_registers[] =
440 {
441 0x3f90, 0xffff0000, 0xff000000,
442 0x9148, 0xffff0000, 0xff000000,
443 0x3f94, 0xffff0000, 0xff000000,
444 0x914c, 0xffff0000, 0xff000000,
445 0x9b7c, 0xffffffff, 0x00000000,
446 0x8a14, 0xffffffff, 0x00000007,
447 0x8b10, 0xffffffff, 0x00000000,
448 0x960c, 0xffffffff, 0x54763210,
449 0x88c4, 0xffffffff, 0x000000c2,
450 0x88d4, 0xffffffff, 0x00000000,
451 0x8974, 0xffffffff, 0x00000000,
452 0xc78, 0x00000080, 0x00000080,
453 0x5eb4, 0xffffffff, 0x00000002,
454 0x5e78, 0xffffffff, 0x001000f0,
455 0x6104, 0x01000300, 0x00000000,
456 0x5bc0, 0x00300000, 0x00000000,
457 0x7030, 0xffffffff, 0x00000011,
458 0x7c30, 0xffffffff, 0x00000011,
459 0x10830, 0xffffffff, 0x00000011,
460 0x11430, 0xffffffff, 0x00000011,
461 0xd02c, 0xffffffff, 0x08421000,
462 0x240c, 0xffffffff, 0x00000380,
463 0x8b24, 0xffffffff, 0x00ff0fff,
464 0x28a4c, 0x06000000, 0x06000000,
465 0x10c, 0x00000001, 0x00000001,
466 0x8d00, 0xffffffff, 0x100e4848,
467 0x8d04, 0xffffffff, 0x00164745,
468 0x8c00, 0xffffffff, 0xe4000003,
469 0x8c04, 0xffffffff, 0x40600060,
470 0x8c08, 0xffffffff, 0x001c001c,
471 0x8cf0, 0xffffffff, 0x08e00410,
472 0x8c20, 0xffffffff, 0x00800080,
473 0x8c24, 0xffffffff, 0x00800080,
474 0x8c18, 0xffffffff, 0x20202078,
475 0x8c1c, 0xffffffff, 0x00001010,
476 0x28350, 0xffffffff, 0x00000000,
477 0xa008, 0xffffffff, 0x00010000,
478 0x5c4, 0xffffffff, 0x00000001,
479 0x9508, 0xffffffff, 0x00000002
480 };
481
482 static const u32 cedar_mgcg_init[] =
483 {
484 0x802c, 0xffffffff, 0xc0000000,
485 0x5448, 0xffffffff, 0x00000100,
486 0x55e4, 0xffffffff, 0x00000100,
487 0x160c, 0xffffffff, 0x00000100,
488 0x5644, 0xffffffff, 0x00000100,
489 0xc164, 0xffffffff, 0x00000100,
490 0x8a18, 0xffffffff, 0x00000100,
491 0x897c, 0xffffffff, 0x06000100,
492 0x8b28, 0xffffffff, 0x00000100,
493 0x9144, 0xffffffff, 0x00000100,
494 0x9a60, 0xffffffff, 0x00000100,
495 0x9868, 0xffffffff, 0x00000100,
496 0x8d58, 0xffffffff, 0x00000100,
497 0x9510, 0xffffffff, 0x00000100,
498 0x949c, 0xffffffff, 0x00000100,
499 0x9654, 0xffffffff, 0x00000100,
500 0x9030, 0xffffffff, 0x00000100,
501 0x9034, 0xffffffff, 0x00000100,
502 0x9038, 0xffffffff, 0x00000100,
503 0x903c, 0xffffffff, 0x00000100,
504 0x9040, 0xffffffff, 0x00000100,
505 0xa200, 0xffffffff, 0x00000100,
506 0xa204, 0xffffffff, 0x00000100,
507 0xa208, 0xffffffff, 0x00000100,
508 0xa20c, 0xffffffff, 0x00000100,
509 0x971c, 0xffffffff, 0x00000100,
510 0x977c, 0xffffffff, 0x00000100,
511 0x3f80, 0xffffffff, 0x00000100,
512 0xa210, 0xffffffff, 0x00000100,
513 0xa214, 0xffffffff, 0x00000100,
514 0x4d8, 0xffffffff, 0x00000100,
515 0x9784, 0xffffffff, 0x00000100,
516 0x9698, 0xffffffff, 0x00000100,
517 0x4d4, 0xffffffff, 0x00000200,
518 0x30cc, 0xffffffff, 0x00000100,
519 0xd0c0, 0xffffffff, 0xff000100,
520 0x802c, 0xffffffff, 0x40000000,
521 0x915c, 0xffffffff, 0x00010000,
522 0x9178, 0xffffffff, 0x00050000,
523 0x917c, 0xffffffff, 0x00030002,
524 0x918c, 0xffffffff, 0x00010004,
525 0x9190, 0xffffffff, 0x00070006,
526 0x9194, 0xffffffff, 0x00050000,
527 0x9198, 0xffffffff, 0x00030002,
528 0x91a8, 0xffffffff, 0x00010004,
529 0x91ac, 0xffffffff, 0x00070006,
530 0x91e8, 0xffffffff, 0x00000000,
531 0x9294, 0xffffffff, 0x00000000,
532 0x929c, 0xffffffff, 0x00000001,
533 0x802c, 0xffffffff, 0xc0000000
534 };
535
536 static const u32 juniper_mgcg_init[] =
537 {
538 0x802c, 0xffffffff, 0xc0000000,
539 0x5448, 0xffffffff, 0x00000100,
540 0x55e4, 0xffffffff, 0x00000100,
541 0x160c, 0xffffffff, 0x00000100,
542 0x5644, 0xffffffff, 0x00000100,
543 0xc164, 0xffffffff, 0x00000100,
544 0x8a18, 0xffffffff, 0x00000100,
545 0x897c, 0xffffffff, 0x06000100,
546 0x8b28, 0xffffffff, 0x00000100,
547 0x9144, 0xffffffff, 0x00000100,
548 0x9a60, 0xffffffff, 0x00000100,
549 0x9868, 0xffffffff, 0x00000100,
550 0x8d58, 0xffffffff, 0x00000100,
551 0x9510, 0xffffffff, 0x00000100,
552 0x949c, 0xffffffff, 0x00000100,
553 0x9654, 0xffffffff, 0x00000100,
554 0x9030, 0xffffffff, 0x00000100,
555 0x9034, 0xffffffff, 0x00000100,
556 0x9038, 0xffffffff, 0x00000100,
557 0x903c, 0xffffffff, 0x00000100,
558 0x9040, 0xffffffff, 0x00000100,
559 0xa200, 0xffffffff, 0x00000100,
560 0xa204, 0xffffffff, 0x00000100,
561 0xa208, 0xffffffff, 0x00000100,
562 0xa20c, 0xffffffff, 0x00000100,
563 0x971c, 0xffffffff, 0x00000100,
564 0xd0c0, 0xffffffff, 0xff000100,
565 0x802c, 0xffffffff, 0x40000000,
566 0x915c, 0xffffffff, 0x00010000,
567 0x9160, 0xffffffff, 0x00030002,
568 0x9178, 0xffffffff, 0x00070000,
569 0x917c, 0xffffffff, 0x00030002,
570 0x9180, 0xffffffff, 0x00050004,
571 0x918c, 0xffffffff, 0x00010006,
572 0x9190, 0xffffffff, 0x00090008,
573 0x9194, 0xffffffff, 0x00070000,
574 0x9198, 0xffffffff, 0x00030002,
575 0x919c, 0xffffffff, 0x00050004,
576 0x91a8, 0xffffffff, 0x00010006,
577 0x91ac, 0xffffffff, 0x00090008,
578 0x91b0, 0xffffffff, 0x00070000,
579 0x91b4, 0xffffffff, 0x00030002,
580 0x91b8, 0xffffffff, 0x00050004,
581 0x91c4, 0xffffffff, 0x00010006,
582 0x91c8, 0xffffffff, 0x00090008,
583 0x91cc, 0xffffffff, 0x00070000,
584 0x91d0, 0xffffffff, 0x00030002,
585 0x91d4, 0xffffffff, 0x00050004,
586 0x91e0, 0xffffffff, 0x00010006,
587 0x91e4, 0xffffffff, 0x00090008,
588 0x91e8, 0xffffffff, 0x00000000,
589 0x91ec, 0xffffffff, 0x00070000,
590 0x91f0, 0xffffffff, 0x00030002,
591 0x91f4, 0xffffffff, 0x00050004,
592 0x9200, 0xffffffff, 0x00010006,
593 0x9204, 0xffffffff, 0x00090008,
594 0x9208, 0xffffffff, 0x00070000,
595 0x920c, 0xffffffff, 0x00030002,
596 0x9210, 0xffffffff, 0x00050004,
597 0x921c, 0xffffffff, 0x00010006,
598 0x9220, 0xffffffff, 0x00090008,
599 0x9224, 0xffffffff, 0x00070000,
600 0x9228, 0xffffffff, 0x00030002,
601 0x922c, 0xffffffff, 0x00050004,
602 0x9238, 0xffffffff, 0x00010006,
603 0x923c, 0xffffffff, 0x00090008,
604 0x9240, 0xffffffff, 0x00070000,
605 0x9244, 0xffffffff, 0x00030002,
606 0x9248, 0xffffffff, 0x00050004,
607 0x9254, 0xffffffff, 0x00010006,
608 0x9258, 0xffffffff, 0x00090008,
609 0x925c, 0xffffffff, 0x00070000,
610 0x9260, 0xffffffff, 0x00030002,
611 0x9264, 0xffffffff, 0x00050004,
612 0x9270, 0xffffffff, 0x00010006,
613 0x9274, 0xffffffff, 0x00090008,
614 0x9278, 0xffffffff, 0x00070000,
615 0x927c, 0xffffffff, 0x00030002,
616 0x9280, 0xffffffff, 0x00050004,
617 0x928c, 0xffffffff, 0x00010006,
618 0x9290, 0xffffffff, 0x00090008,
619 0x9294, 0xffffffff, 0x00000000,
620 0x929c, 0xffffffff, 0x00000001,
621 0x802c, 0xffffffff, 0xc0000000,
622 0x977c, 0xffffffff, 0x00000100,
623 0x3f80, 0xffffffff, 0x00000100,
624 0xa210, 0xffffffff, 0x00000100,
625 0xa214, 0xffffffff, 0x00000100,
626 0x4d8, 0xffffffff, 0x00000100,
627 0x9784, 0xffffffff, 0x00000100,
628 0x9698, 0xffffffff, 0x00000100,
629 0x4d4, 0xffffffff, 0x00000200,
630 0x30cc, 0xffffffff, 0x00000100,
631 0x802c, 0xffffffff, 0xc0000000
632 };
633
634 static const u32 supersumo_golden_registers[] =
635 {
636 0x5eb4, 0xffffffff, 0x00000002,
637 0x5c4, 0xffffffff, 0x00000001,
638 0x7030, 0xffffffff, 0x00000011,
639 0x7c30, 0xffffffff, 0x00000011,
640 0x6104, 0x01000300, 0x00000000,
641 0x5bc0, 0x00300000, 0x00000000,
642 0x8c04, 0xffffffff, 0x40600060,
643 0x8c08, 0xffffffff, 0x001c001c,
644 0x8c20, 0xffffffff, 0x00800080,
645 0x8c24, 0xffffffff, 0x00800080,
646 0x8c18, 0xffffffff, 0x20202078,
647 0x8c1c, 0xffffffff, 0x00001010,
648 0x918c, 0xffffffff, 0x00010006,
649 0x91a8, 0xffffffff, 0x00010006,
650 0x91c4, 0xffffffff, 0x00010006,
651 0x91e0, 0xffffffff, 0x00010006,
652 0x9200, 0xffffffff, 0x00010006,
653 0x9150, 0xffffffff, 0x6e944040,
654 0x917c, 0xffffffff, 0x00030002,
655 0x9180, 0xffffffff, 0x00050004,
656 0x9198, 0xffffffff, 0x00030002,
657 0x919c, 0xffffffff, 0x00050004,
658 0x91b4, 0xffffffff, 0x00030002,
659 0x91b8, 0xffffffff, 0x00050004,
660 0x91d0, 0xffffffff, 0x00030002,
661 0x91d4, 0xffffffff, 0x00050004,
662 0x91f0, 0xffffffff, 0x00030002,
663 0x91f4, 0xffffffff, 0x00050004,
664 0x915c, 0xffffffff, 0x00010000,
665 0x9160, 0xffffffff, 0x00030002,
666 0x3f90, 0xffff0000, 0xff000000,
667 0x9178, 0xffffffff, 0x00070000,
668 0x9194, 0xffffffff, 0x00070000,
669 0x91b0, 0xffffffff, 0x00070000,
670 0x91cc, 0xffffffff, 0x00070000,
671 0x91ec, 0xffffffff, 0x00070000,
672 0x9148, 0xffff0000, 0xff000000,
673 0x9190, 0xffffffff, 0x00090008,
674 0x91ac, 0xffffffff, 0x00090008,
675 0x91c8, 0xffffffff, 0x00090008,
676 0x91e4, 0xffffffff, 0x00090008,
677 0x9204, 0xffffffff, 0x00090008,
678 0x3f94, 0xffff0000, 0xff000000,
679 0x914c, 0xffff0000, 0xff000000,
680 0x929c, 0xffffffff, 0x00000001,
681 0x8a18, 0xffffffff, 0x00000100,
682 0x8b28, 0xffffffff, 0x00000100,
683 0x9144, 0xffffffff, 0x00000100,
684 0x5644, 0xffffffff, 0x00000100,
685 0x9b7c, 0xffffffff, 0x00000000,
686 0x8030, 0xffffffff, 0x0000100a,
687 0x8a14, 0xffffffff, 0x00000007,
688 0x8b24, 0xffffffff, 0x00ff0fff,
689 0x8b10, 0xffffffff, 0x00000000,
690 0x28a4c, 0x06000000, 0x06000000,
691 0x4d8, 0xffffffff, 0x00000100,
692 0x913c, 0xffff000f, 0x0100000a,
693 0x960c, 0xffffffff, 0x54763210,
694 0x88c4, 0xffffffff, 0x000000c2,
695 0x88d4, 0xffffffff, 0x00000010,
696 0x8974, 0xffffffff, 0x00000000,
697 0xc78, 0x00000080, 0x00000080,
698 0x5e78, 0xffffffff, 0x001000f0,
699 0xd02c, 0xffffffff, 0x08421000,
700 0xa008, 0xffffffff, 0x00010000,
701 0x8d00, 0xffffffff, 0x100e4848,
702 0x8d04, 0xffffffff, 0x00164745,
703 0x8c00, 0xffffffff, 0xe4000003,
704 0x8cf0, 0x1fffffff, 0x08e00620,
705 0x28350, 0xffffffff, 0x00000000,
706 0x9508, 0xffffffff, 0x00000002
707 };
708
709 static const u32 sumo_golden_registers[] =
710 {
711 0x900c, 0x00ffffff, 0x0017071f,
712 0x8c18, 0xffffffff, 0x10101060,
713 0x8c1c, 0xffffffff, 0x00001010,
714 0x8c30, 0x0000000f, 0x00000005,
715 0x9688, 0x0000000f, 0x00000007
716 };
717
718 static const u32 wrestler_golden_registers[] =
719 {
720 0x5eb4, 0xffffffff, 0x00000002,
721 0x5c4, 0xffffffff, 0x00000001,
722 0x7030, 0xffffffff, 0x00000011,
723 0x7c30, 0xffffffff, 0x00000011,
724 0x6104, 0x01000300, 0x00000000,
725 0x5bc0, 0x00300000, 0x00000000,
726 0x918c, 0xffffffff, 0x00010006,
727 0x91a8, 0xffffffff, 0x00010006,
728 0x9150, 0xffffffff, 0x6e944040,
729 0x917c, 0xffffffff, 0x00030002,
730 0x9198, 0xffffffff, 0x00030002,
731 0x915c, 0xffffffff, 0x00010000,
732 0x3f90, 0xffff0000, 0xff000000,
733 0x9178, 0xffffffff, 0x00070000,
734 0x9194, 0xffffffff, 0x00070000,
735 0x9148, 0xffff0000, 0xff000000,
736 0x9190, 0xffffffff, 0x00090008,
737 0x91ac, 0xffffffff, 0x00090008,
738 0x3f94, 0xffff0000, 0xff000000,
739 0x914c, 0xffff0000, 0xff000000,
740 0x929c, 0xffffffff, 0x00000001,
741 0x8a18, 0xffffffff, 0x00000100,
742 0x8b28, 0xffffffff, 0x00000100,
743 0x9144, 0xffffffff, 0x00000100,
744 0x9b7c, 0xffffffff, 0x00000000,
745 0x8030, 0xffffffff, 0x0000100a,
746 0x8a14, 0xffffffff, 0x00000001,
747 0x8b24, 0xffffffff, 0x00ff0fff,
748 0x8b10, 0xffffffff, 0x00000000,
749 0x28a4c, 0x06000000, 0x06000000,
750 0x4d8, 0xffffffff, 0x00000100,
751 0x913c, 0xffff000f, 0x0100000a,
752 0x960c, 0xffffffff, 0x54763210,
753 0x88c4, 0xffffffff, 0x000000c2,
754 0x88d4, 0xffffffff, 0x00000010,
755 0x8974, 0xffffffff, 0x00000000,
756 0xc78, 0x00000080, 0x00000080,
757 0x5e78, 0xffffffff, 0x001000f0,
758 0xd02c, 0xffffffff, 0x08421000,
759 0xa008, 0xffffffff, 0x00010000,
760 0x8d00, 0xffffffff, 0x100e4848,
761 0x8d04, 0xffffffff, 0x00164745,
762 0x8c00, 0xffffffff, 0xe4000003,
763 0x8cf0, 0x1fffffff, 0x08e00410,
764 0x28350, 0xffffffff, 0x00000000,
765 0x9508, 0xffffffff, 0x00000002,
766 0x900c, 0xffffffff, 0x0017071f,
767 0x8c18, 0xffffffff, 0x10101060,
768 0x8c1c, 0xffffffff, 0x00001010
769 };
770
771 static const u32 barts_golden_registers[] =
772 {
773 0x5eb4, 0xffffffff, 0x00000002,
774 0x5e78, 0x8f311ff1, 0x001000f0,
775 0x3f90, 0xffff0000, 0xff000000,
776 0x9148, 0xffff0000, 0xff000000,
777 0x3f94, 0xffff0000, 0xff000000,
778 0x914c, 0xffff0000, 0xff000000,
779 0xc78, 0x00000080, 0x00000080,
780 0xbd4, 0x70073777, 0x00010001,
781 0xd02c, 0xbfffff1f, 0x08421000,
782 0xd0b8, 0x03773777, 0x02011003,
783 0x5bc0, 0x00200000, 0x50100000,
784 0x98f8, 0x33773777, 0x02011003,
785 0x98fc, 0xffffffff, 0x76543210,
786 0x7030, 0x31000311, 0x00000011,
787 0x2f48, 0x00000007, 0x02011003,
788 0x6b28, 0x00000010, 0x00000012,
789 0x7728, 0x00000010, 0x00000012,
790 0x10328, 0x00000010, 0x00000012,
791 0x10f28, 0x00000010, 0x00000012,
792 0x11b28, 0x00000010, 0x00000012,
793 0x12728, 0x00000010, 0x00000012,
794 0x240c, 0x000007ff, 0x00000380,
795 0x8a14, 0xf000001f, 0x00000007,
796 0x8b24, 0x3fff3fff, 0x00ff0fff,
797 0x8b10, 0x0000ff0f, 0x00000000,
798 0x28a4c, 0x07ffffff, 0x06000000,
799 0x10c, 0x00000001, 0x00010003,
800 0xa02c, 0xffffffff, 0x0000009b,
801 0x913c, 0x0000000f, 0x0100000a,
802 0x8d00, 0xffff7f7f, 0x100e4848,
803 0x8d04, 0x00ffffff, 0x00164745,
804 0x8c00, 0xfffc0003, 0xe4000003,
805 0x8c04, 0xf8ff00ff, 0x40600060,
806 0x8c08, 0x00ff00ff, 0x001c001c,
807 0x8cf0, 0x1fff1fff, 0x08e00620,
808 0x8c20, 0x0fff0fff, 0x00800080,
809 0x8c24, 0x0fff0fff, 0x00800080,
810 0x8c18, 0xffffffff, 0x20202078,
811 0x8c1c, 0x0000ffff, 0x00001010,
812 0x28350, 0x00000f01, 0x00000000,
813 0x9508, 0x3700001f, 0x00000002,
814 0x960c, 0xffffffff, 0x54763210,
815 0x88c4, 0x001f3ae3, 0x000000c2,
816 0x88d4, 0x0000001f, 0x00000010,
817 0x8974, 0xffffffff, 0x00000000
818 };
819
820 static const u32 turks_golden_registers[] =
821 {
822 0x5eb4, 0xffffffff, 0x00000002,
823 0x5e78, 0x8f311ff1, 0x001000f0,
824 0x8c8, 0x00003000, 0x00001070,
825 0x8cc, 0x000fffff, 0x00040035,
826 0x3f90, 0xffff0000, 0xfff00000,
827 0x9148, 0xffff0000, 0xfff00000,
828 0x3f94, 0xffff0000, 0xfff00000,
829 0x914c, 0xffff0000, 0xfff00000,
830 0xc78, 0x00000080, 0x00000080,
831 0xbd4, 0x00073007, 0x00010002,
832 0xd02c, 0xbfffff1f, 0x08421000,
833 0xd0b8, 0x03773777, 0x02010002,
834 0x5bc0, 0x00200000, 0x50100000,
835 0x98f8, 0x33773777, 0x00010002,
836 0x98fc, 0xffffffff, 0x33221100,
837 0x7030, 0x31000311, 0x00000011,
838 0x2f48, 0x33773777, 0x00010002,
839 0x6b28, 0x00000010, 0x00000012,
840 0x7728, 0x00000010, 0x00000012,
841 0x10328, 0x00000010, 0x00000012,
842 0x10f28, 0x00000010, 0x00000012,
843 0x11b28, 0x00000010, 0x00000012,
844 0x12728, 0x00000010, 0x00000012,
845 0x240c, 0x000007ff, 0x00000380,
846 0x8a14, 0xf000001f, 0x00000007,
847 0x8b24, 0x3fff3fff, 0x00ff0fff,
848 0x8b10, 0x0000ff0f, 0x00000000,
849 0x28a4c, 0x07ffffff, 0x06000000,
850 0x10c, 0x00000001, 0x00010003,
851 0xa02c, 0xffffffff, 0x0000009b,
852 0x913c, 0x0000000f, 0x0100000a,
853 0x8d00, 0xffff7f7f, 0x100e4848,
854 0x8d04, 0x00ffffff, 0x00164745,
855 0x8c00, 0xfffc0003, 0xe4000003,
856 0x8c04, 0xf8ff00ff, 0x40600060,
857 0x8c08, 0x00ff00ff, 0x001c001c,
858 0x8cf0, 0x1fff1fff, 0x08e00410,
859 0x8c20, 0x0fff0fff, 0x00800080,
860 0x8c24, 0x0fff0fff, 0x00800080,
861 0x8c18, 0xffffffff, 0x20202078,
862 0x8c1c, 0x0000ffff, 0x00001010,
863 0x28350, 0x00000f01, 0x00000000,
864 0x9508, 0x3700001f, 0x00000002,
865 0x960c, 0xffffffff, 0x54763210,
866 0x88c4, 0x001f3ae3, 0x000000c2,
867 0x88d4, 0x0000001f, 0x00000010,
868 0x8974, 0xffffffff, 0x00000000
869 };
870
871 static const u32 caicos_golden_registers[] =
872 {
873 0x5eb4, 0xffffffff, 0x00000002,
874 0x5e78, 0x8f311ff1, 0x001000f0,
875 0x8c8, 0x00003420, 0x00001450,
876 0x8cc, 0x000fffff, 0x00040035,
877 0x3f90, 0xffff0000, 0xfffc0000,
878 0x9148, 0xffff0000, 0xfffc0000,
879 0x3f94, 0xffff0000, 0xfffc0000,
880 0x914c, 0xffff0000, 0xfffc0000,
881 0xc78, 0x00000080, 0x00000080,
882 0xbd4, 0x00073007, 0x00010001,
883 0xd02c, 0xbfffff1f, 0x08421000,
884 0xd0b8, 0x03773777, 0x02010001,
885 0x5bc0, 0x00200000, 0x50100000,
886 0x98f8, 0x33773777, 0x02010001,
887 0x98fc, 0xffffffff, 0x33221100,
888 0x7030, 0x31000311, 0x00000011,
889 0x2f48, 0x33773777, 0x02010001,
890 0x6b28, 0x00000010, 0x00000012,
891 0x7728, 0x00000010, 0x00000012,
892 0x10328, 0x00000010, 0x00000012,
893 0x10f28, 0x00000010, 0x00000012,
894 0x11b28, 0x00000010, 0x00000012,
895 0x12728, 0x00000010, 0x00000012,
896 0x240c, 0x000007ff, 0x00000380,
897 0x8a14, 0xf000001f, 0x00000001,
898 0x8b24, 0x3fff3fff, 0x00ff0fff,
899 0x8b10, 0x0000ff0f, 0x00000000,
900 0x28a4c, 0x07ffffff, 0x06000000,
901 0x10c, 0x00000001, 0x00010003,
902 0xa02c, 0xffffffff, 0x0000009b,
903 0x913c, 0x0000000f, 0x0100000a,
904 0x8d00, 0xffff7f7f, 0x100e4848,
905 0x8d04, 0x00ffffff, 0x00164745,
906 0x8c00, 0xfffc0003, 0xe4000003,
907 0x8c04, 0xf8ff00ff, 0x40600060,
908 0x8c08, 0x00ff00ff, 0x001c001c,
909 0x8cf0, 0x1fff1fff, 0x08e00410,
910 0x8c20, 0x0fff0fff, 0x00800080,
911 0x8c24, 0x0fff0fff, 0x00800080,
912 0x8c18, 0xffffffff, 0x20202078,
913 0x8c1c, 0x0000ffff, 0x00001010,
914 0x28350, 0x00000f01, 0x00000000,
915 0x9508, 0x3700001f, 0x00000002,
916 0x960c, 0xffffffff, 0x54763210,
917 0x88c4, 0x001f3ae3, 0x000000c2,
918 0x88d4, 0x0000001f, 0x00000010,
919 0x8974, 0xffffffff, 0x00000000
920 };
921
922 static void evergreen_init_golden_registers(struct radeon_device *rdev)
923 {
924 switch (rdev->family) {
925 case CHIP_CYPRESS:
926 case CHIP_HEMLOCK:
927 radeon_program_register_sequence(rdev,
928 evergreen_golden_registers,
929 (const u32)ARRAY_SIZE(evergreen_golden_registers));
930 radeon_program_register_sequence(rdev,
931 evergreen_golden_registers2,
932 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
933 radeon_program_register_sequence(rdev,
934 cypress_mgcg_init,
935 (const u32)ARRAY_SIZE(cypress_mgcg_init));
936 break;
937 case CHIP_JUNIPER:
938 radeon_program_register_sequence(rdev,
939 evergreen_golden_registers,
940 (const u32)ARRAY_SIZE(evergreen_golden_registers));
941 radeon_program_register_sequence(rdev,
942 evergreen_golden_registers2,
943 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
944 radeon_program_register_sequence(rdev,
945 juniper_mgcg_init,
946 (const u32)ARRAY_SIZE(juniper_mgcg_init));
947 break;
948 case CHIP_REDWOOD:
949 radeon_program_register_sequence(rdev,
950 evergreen_golden_registers,
951 (const u32)ARRAY_SIZE(evergreen_golden_registers));
952 radeon_program_register_sequence(rdev,
953 evergreen_golden_registers2,
954 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
955 radeon_program_register_sequence(rdev,
956 redwood_mgcg_init,
957 (const u32)ARRAY_SIZE(redwood_mgcg_init));
958 break;
959 case CHIP_CEDAR:
960 radeon_program_register_sequence(rdev,
961 cedar_golden_registers,
962 (const u32)ARRAY_SIZE(cedar_golden_registers));
963 radeon_program_register_sequence(rdev,
964 evergreen_golden_registers2,
965 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
966 radeon_program_register_sequence(rdev,
967 cedar_mgcg_init,
968 (const u32)ARRAY_SIZE(cedar_mgcg_init));
969 break;
970 case CHIP_PALM:
971 radeon_program_register_sequence(rdev,
972 wrestler_golden_registers,
973 (const u32)ARRAY_SIZE(wrestler_golden_registers));
974 break;
975 case CHIP_SUMO:
976 radeon_program_register_sequence(rdev,
977 supersumo_golden_registers,
978 (const u32)ARRAY_SIZE(supersumo_golden_registers));
979 break;
980 case CHIP_SUMO2:
981 radeon_program_register_sequence(rdev,
982 supersumo_golden_registers,
983 (const u32)ARRAY_SIZE(supersumo_golden_registers));
984 radeon_program_register_sequence(rdev,
985 sumo_golden_registers,
986 (const u32)ARRAY_SIZE(sumo_golden_registers));
987 break;
988 case CHIP_BARTS:
989 radeon_program_register_sequence(rdev,
990 barts_golden_registers,
991 (const u32)ARRAY_SIZE(barts_golden_registers));
992 break;
993 case CHIP_TURKS:
994 radeon_program_register_sequence(rdev,
995 turks_golden_registers,
996 (const u32)ARRAY_SIZE(turks_golden_registers));
997 break;
998 case CHIP_CAICOS:
999 radeon_program_register_sequence(rdev,
1000 caicos_golden_registers,
1001 (const u32)ARRAY_SIZE(caicos_golden_registers));
1002 break;
1003 default:
1004 break;
1005 }
1006 }
1007
1008 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1009 unsigned *bankh, unsigned *mtaspect,
1010 unsigned *tile_split)
1011 {
1012 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1013 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1014 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1015 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1016 switch (*bankw) {
1017 default:
1018 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1019 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1020 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1021 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1022 }
1023 switch (*bankh) {
1024 default:
1025 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1026 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1027 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1028 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1029 }
1030 switch (*mtaspect) {
1031 default:
1032 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1033 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1034 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1035 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1036 }
1037 }
1038
1039 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1040 u32 cntl_reg, u32 status_reg)
1041 {
1042 int r, i;
1043 struct atom_clock_dividers dividers;
1044
1045 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1046 clock, false, &dividers);
1047 if (r)
1048 return r;
1049
1050 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1051
1052 for (i = 0; i < 100; i++) {
1053 if (RREG32(status_reg) & DCLK_STATUS)
1054 break;
1055 mdelay(10);
1056 }
1057 if (i == 100)
1058 return -ETIMEDOUT;
1059
1060 return 0;
1061 }
1062
1063 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1064 {
1065 int r = 0;
1066 u32 cg_scratch = RREG32(CG_SCRATCH1);
1067
1068 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1069 if (r)
1070 goto done;
1071 cg_scratch &= 0xffff0000;
1072 cg_scratch |= vclk / 100; /* Mhz */
1073
1074 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1075 if (r)
1076 goto done;
1077 cg_scratch &= 0x0000ffff;
1078 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1079
1080 done:
1081 WREG32(CG_SCRATCH1, cg_scratch);
1082
1083 return r;
1084 }
1085
1086 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1087 {
1088 /* start off with something large */
1089 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1090 int r;
1091
1092 /* bypass vclk and dclk with bclk */
1093 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1094 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1095 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1096
1097 /* put PLL in bypass mode */
1098 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1099
1100 if (!vclk || !dclk) {
1101 /* keep the Bypass mode, put PLL to sleep */
1102 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1103 return 0;
1104 }
1105
1106 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1107 16384, 0x03FFFFFF, 0, 128, 5,
1108 &fb_div, &vclk_div, &dclk_div);
1109 if (r)
1110 return r;
1111
1112 /* set VCO_MODE to 1 */
1113 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1114
1115 /* toggle UPLL_SLEEP to 1 then back to 0 */
1116 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1117 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1118
1119 /* deassert UPLL_RESET */
1120 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1121
1122 mdelay(1);
1123
1124 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1125 if (r)
1126 return r;
1127
1128 /* assert UPLL_RESET again */
1129 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1130
1131 /* disable spread spectrum. */
1132 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1133
1134 /* set feedback divider */
1135 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1136
1137 /* set ref divider to 0 */
1138 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1139
1140 if (fb_div < 307200)
1141 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1142 else
1143 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1144
1145 /* set PDIV_A and PDIV_B */
1146 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1147 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1148 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1149
1150 /* give the PLL some time to settle */
1151 mdelay(15);
1152
1153 /* deassert PLL_RESET */
1154 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1155
1156 mdelay(15);
1157
1158 /* switch from bypass mode to normal mode */
1159 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1160
1161 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1162 if (r)
1163 return r;
1164
1165 /* switch VCLK and DCLK selection */
1166 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1167 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1168 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1169
1170 mdelay(100);
1171
1172 return 0;
1173 }
1174
1175 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1176 {
1177 int readrq;
1178 u16 v;
1179
1180 readrq = pcie_get_readrq(rdev->pdev);
1181 v = ffs(readrq) - 8;
1182 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1183 * to avoid hangs or perfomance issues
1184 */
1185 if ((v == 0) || (v == 6) || (v == 7))
1186 pcie_set_readrq(rdev->pdev, 512);
1187 }
1188
1189 void dce4_program_fmt(struct drm_encoder *encoder)
1190 {
1191 struct drm_device *dev = encoder->dev;
1192 struct radeon_device *rdev = dev->dev_private;
1193 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1194 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1195 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1196 int bpc = 0;
1197 u32 tmp = 0;
1198 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1199
1200 if (connector) {
1201 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1202 bpc = radeon_get_monitor_bpc(connector);
1203 dither = radeon_connector->dither;
1204 }
1205
1206 /* LVDS/eDP FMT is set up by atom */
1207 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1208 return;
1209
1210 /* not needed for analog */
1211 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1212 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1213 return;
1214
1215 if (bpc == 0)
1216 return;
1217
1218 switch (bpc) {
1219 case 6:
1220 if (dither == RADEON_FMT_DITHER_ENABLE)
1221 /* XXX sort out optimal dither settings */
1222 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1223 FMT_SPATIAL_DITHER_EN);
1224 else
1225 tmp |= FMT_TRUNCATE_EN;
1226 break;
1227 case 8:
1228 if (dither == RADEON_FMT_DITHER_ENABLE)
1229 /* XXX sort out optimal dither settings */
1230 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1231 FMT_RGB_RANDOM_ENABLE |
1232 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1233 else
1234 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1235 break;
1236 case 10:
1237 default:
1238 /* not needed */
1239 break;
1240 }
1241
1242 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1243 }
1244
1245 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1246 {
1247 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1248 return true;
1249 else
1250 return false;
1251 }
1252
1253 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1254 {
1255 u32 pos1, pos2;
1256
1257 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1258 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1259
1260 if (pos1 != pos2)
1261 return true;
1262 else
1263 return false;
1264 }
1265
1266 /**
1267 * dce4_wait_for_vblank - vblank wait asic callback.
1268 *
1269 * @rdev: radeon_device pointer
1270 * @crtc: crtc to wait for vblank on
1271 *
1272 * Wait for vblank on the requested crtc (evergreen+).
1273 */
1274 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1275 {
1276 unsigned i = 0;
1277
1278 if (crtc >= rdev->num_crtc)
1279 return;
1280
1281 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1282 return;
1283
1284 /* depending on when we hit vblank, we may be close to active; if so,
1285 * wait for another frame.
1286 */
1287 while (dce4_is_in_vblank(rdev, crtc)) {
1288 if (i++ % 100 == 0) {
1289 if (!dce4_is_counter_moving(rdev, crtc))
1290 break;
1291 }
1292 }
1293
1294 while (!dce4_is_in_vblank(rdev, crtc)) {
1295 if (i++ % 100 == 0) {
1296 if (!dce4_is_counter_moving(rdev, crtc))
1297 break;
1298 }
1299 }
1300 }
1301
1302 /**
1303 * evergreen_page_flip - pageflip callback.
1304 *
1305 * @rdev: radeon_device pointer
1306 * @crtc_id: crtc to cleanup pageflip on
1307 * @crtc_base: new address of the crtc (GPU MC address)
1308 *
1309 * Does the actual pageflip (evergreen+).
1310 * During vblank we take the crtc lock and wait for the update_pending
1311 * bit to go high, when it does, we release the lock, and allow the
1312 * double buffered update to take place.
1313 * Returns the current update pending status.
1314 */
1315 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1316 {
1317 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1318 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
1319 int i;
1320
1321 /* Lock the graphics update lock */
1322 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1323 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1324
1325 /* update the scanout addresses */
1326 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1327 upper_32_bits(crtc_base));
1328 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1329 (u32)crtc_base);
1330
1331 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1332 upper_32_bits(crtc_base));
1333 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1334 (u32)crtc_base);
1335
1336 /* Wait for update_pending to go high. */
1337 for (i = 0; i < rdev->usec_timeout; i++) {
1338 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1339 break;
1340 udelay(1);
1341 }
1342 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1343
1344 /* Unlock the lock, so double-buffering can take place inside vblank */
1345 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1346 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1347 }
1348
1349 /**
1350 * evergreen_page_flip_pending - check if page flip is still pending
1351 *
1352 * @rdev: radeon_device pointer
1353 * @crtc_id: crtc to check
1354 *
1355 * Returns the current update pending status.
1356 */
1357 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1358 {
1359 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1360
1361 /* Return current update_pending status: */
1362 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1363 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1364 }
1365
1366 /* get temperature in millidegrees */
1367 int evergreen_get_temp(struct radeon_device *rdev)
1368 {
1369 u32 temp, toffset;
1370 int actual_temp = 0;
1371
1372 if (rdev->family == CHIP_JUNIPER) {
1373 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1374 TOFFSET_SHIFT;
1375 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1376 TS0_ADC_DOUT_SHIFT;
1377
1378 if (toffset & 0x100)
1379 actual_temp = temp / 2 - (0x200 - toffset);
1380 else
1381 actual_temp = temp / 2 + toffset;
1382
1383 actual_temp = actual_temp * 1000;
1384
1385 } else {
1386 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1387 ASIC_T_SHIFT;
1388
1389 if (temp & 0x400)
1390 actual_temp = -256;
1391 else if (temp & 0x200)
1392 actual_temp = 255;
1393 else if (temp & 0x100) {
1394 actual_temp = temp & 0x1ff;
1395 actual_temp |= ~0x1ff;
1396 } else
1397 actual_temp = temp & 0xff;
1398
1399 actual_temp = (actual_temp * 1000) / 2;
1400 }
1401
1402 return actual_temp;
1403 }
1404
1405 int sumo_get_temp(struct radeon_device *rdev)
1406 {
1407 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1408 int actual_temp = temp - 49;
1409
1410 return actual_temp * 1000;
1411 }
1412
1413 /**
1414 * sumo_pm_init_profile - Initialize power profiles callback.
1415 *
1416 * @rdev: radeon_device pointer
1417 *
1418 * Initialize the power states used in profile mode
1419 * (sumo, trinity, SI).
1420 * Used for profile mode only.
1421 */
1422 void sumo_pm_init_profile(struct radeon_device *rdev)
1423 {
1424 int idx;
1425
1426 /* default */
1427 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1428 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1429 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1430 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1431
1432 /* low,mid sh/mh */
1433 if (rdev->flags & RADEON_IS_MOBILITY)
1434 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1435 else
1436 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1437
1438 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1439 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1440 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1441 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1442
1443 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1444 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1445 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1446 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1447
1448 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1449 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1450 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1451 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1452
1453 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1454 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1455 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1456 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1457
1458 /* high sh/mh */
1459 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1460 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1461 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1462 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1463 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1464 rdev->pm.power_state[idx].num_clock_modes - 1;
1465
1466 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1467 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1468 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1469 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1470 rdev->pm.power_state[idx].num_clock_modes - 1;
1471 }
1472
1473 /**
1474 * btc_pm_init_profile - Initialize power profiles callback.
1475 *
1476 * @rdev: radeon_device pointer
1477 *
1478 * Initialize the power states used in profile mode
1479 * (BTC, cayman).
1480 * Used for profile mode only.
1481 */
1482 void btc_pm_init_profile(struct radeon_device *rdev)
1483 {
1484 int idx;
1485
1486 /* default */
1487 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1488 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1489 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1490 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1491 /* starting with BTC, there is one state that is used for both
1492 * MH and SH. Difference is that we always use the high clock index for
1493 * mclk.
1494 */
1495 if (rdev->flags & RADEON_IS_MOBILITY)
1496 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1497 else
1498 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1499 /* low sh */
1500 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1501 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1502 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1503 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1504 /* mid sh */
1505 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1506 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1507 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1508 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1509 /* high sh */
1510 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1511 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1512 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1513 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1514 /* low mh */
1515 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1516 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1517 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1518 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1519 /* mid mh */
1520 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1521 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1522 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1523 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1524 /* high mh */
1525 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1526 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1527 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1528 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1529 }
1530
1531 /**
1532 * evergreen_pm_misc - set additional pm hw parameters callback.
1533 *
1534 * @rdev: radeon_device pointer
1535 *
1536 * Set non-clock parameters associated with a power state
1537 * (voltage, etc.) (evergreen+).
1538 */
1539 void evergreen_pm_misc(struct radeon_device *rdev)
1540 {
1541 int req_ps_idx = rdev->pm.requested_power_state_index;
1542 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1543 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1544 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1545
1546 if (voltage->type == VOLTAGE_SW) {
1547 /* 0xff0x are flags rather then an actual voltage */
1548 if ((voltage->voltage & 0xff00) == 0xff00)
1549 return;
1550 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1551 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1552 rdev->pm.current_vddc = voltage->voltage;
1553 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1554 }
1555
1556 /* starting with BTC, there is one state that is used for both
1557 * MH and SH. Difference is that we always use the high clock index for
1558 * mclk and vddci.
1559 */
1560 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1561 (rdev->family >= CHIP_BARTS) &&
1562 rdev->pm.active_crtc_count &&
1563 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1564 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1565 voltage = &rdev->pm.power_state[req_ps_idx].
1566 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1567
1568 /* 0xff0x are flags rather then an actual voltage */
1569 if ((voltage->vddci & 0xff00) == 0xff00)
1570 return;
1571 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1572 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1573 rdev->pm.current_vddci = voltage->vddci;
1574 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1575 }
1576 }
1577 }
1578
1579 /**
1580 * evergreen_pm_prepare - pre-power state change callback.
1581 *
1582 * @rdev: radeon_device pointer
1583 *
1584 * Prepare for a power state change (evergreen+).
1585 */
1586 void evergreen_pm_prepare(struct radeon_device *rdev)
1587 {
1588 struct drm_device *ddev = rdev->ddev;
1589 struct drm_crtc *crtc;
1590 struct radeon_crtc *radeon_crtc;
1591 u32 tmp;
1592
1593 /* disable any active CRTCs */
1594 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1595 radeon_crtc = to_radeon_crtc(crtc);
1596 if (radeon_crtc->enabled) {
1597 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1598 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1599 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1600 }
1601 }
1602 }
1603
1604 /**
1605 * evergreen_pm_finish - post-power state change callback.
1606 *
1607 * @rdev: radeon_device pointer
1608 *
1609 * Clean up after a power state change (evergreen+).
1610 */
1611 void evergreen_pm_finish(struct radeon_device *rdev)
1612 {
1613 struct drm_device *ddev = rdev->ddev;
1614 struct drm_crtc *crtc;
1615 struct radeon_crtc *radeon_crtc;
1616 u32 tmp;
1617
1618 /* enable any active CRTCs */
1619 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1620 radeon_crtc = to_radeon_crtc(crtc);
1621 if (radeon_crtc->enabled) {
1622 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1623 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1624 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1625 }
1626 }
1627 }
1628
1629 /**
1630 * evergreen_hpd_sense - hpd sense callback.
1631 *
1632 * @rdev: radeon_device pointer
1633 * @hpd: hpd (hotplug detect) pin
1634 *
1635 * Checks if a digital monitor is connected (evergreen+).
1636 * Returns true if connected, false if not connected.
1637 */
1638 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1639 {
1640 bool connected = false;
1641
1642 switch (hpd) {
1643 case RADEON_HPD_1:
1644 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1645 connected = true;
1646 break;
1647 case RADEON_HPD_2:
1648 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1649 connected = true;
1650 break;
1651 case RADEON_HPD_3:
1652 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1653 connected = true;
1654 break;
1655 case RADEON_HPD_4:
1656 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1657 connected = true;
1658 break;
1659 case RADEON_HPD_5:
1660 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1661 connected = true;
1662 break;
1663 case RADEON_HPD_6:
1664 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1665 connected = true;
1666 break;
1667 default:
1668 break;
1669 }
1670
1671 return connected;
1672 }
1673
1674 /**
1675 * evergreen_hpd_set_polarity - hpd set polarity callback.
1676 *
1677 * @rdev: radeon_device pointer
1678 * @hpd: hpd (hotplug detect) pin
1679 *
1680 * Set the polarity of the hpd pin (evergreen+).
1681 */
1682 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1683 enum radeon_hpd_id hpd)
1684 {
1685 u32 tmp;
1686 bool connected = evergreen_hpd_sense(rdev, hpd);
1687
1688 switch (hpd) {
1689 case RADEON_HPD_1:
1690 tmp = RREG32(DC_HPD1_INT_CONTROL);
1691 if (connected)
1692 tmp &= ~DC_HPDx_INT_POLARITY;
1693 else
1694 tmp |= DC_HPDx_INT_POLARITY;
1695 WREG32(DC_HPD1_INT_CONTROL, tmp);
1696 break;
1697 case RADEON_HPD_2:
1698 tmp = RREG32(DC_HPD2_INT_CONTROL);
1699 if (connected)
1700 tmp &= ~DC_HPDx_INT_POLARITY;
1701 else
1702 tmp |= DC_HPDx_INT_POLARITY;
1703 WREG32(DC_HPD2_INT_CONTROL, tmp);
1704 break;
1705 case RADEON_HPD_3:
1706 tmp = RREG32(DC_HPD3_INT_CONTROL);
1707 if (connected)
1708 tmp &= ~DC_HPDx_INT_POLARITY;
1709 else
1710 tmp |= DC_HPDx_INT_POLARITY;
1711 WREG32(DC_HPD3_INT_CONTROL, tmp);
1712 break;
1713 case RADEON_HPD_4:
1714 tmp = RREG32(DC_HPD4_INT_CONTROL);
1715 if (connected)
1716 tmp &= ~DC_HPDx_INT_POLARITY;
1717 else
1718 tmp |= DC_HPDx_INT_POLARITY;
1719 WREG32(DC_HPD4_INT_CONTROL, tmp);
1720 break;
1721 case RADEON_HPD_5:
1722 tmp = RREG32(DC_HPD5_INT_CONTROL);
1723 if (connected)
1724 tmp &= ~DC_HPDx_INT_POLARITY;
1725 else
1726 tmp |= DC_HPDx_INT_POLARITY;
1727 WREG32(DC_HPD5_INT_CONTROL, tmp);
1728 break;
1729 case RADEON_HPD_6:
1730 tmp = RREG32(DC_HPD6_INT_CONTROL);
1731 if (connected)
1732 tmp &= ~DC_HPDx_INT_POLARITY;
1733 else
1734 tmp |= DC_HPDx_INT_POLARITY;
1735 WREG32(DC_HPD6_INT_CONTROL, tmp);
1736 break;
1737 default:
1738 break;
1739 }
1740 }
1741
1742 /**
1743 * evergreen_hpd_init - hpd setup callback.
1744 *
1745 * @rdev: radeon_device pointer
1746 *
1747 * Setup the hpd pins used by the card (evergreen+).
1748 * Enable the pin, set the polarity, and enable the hpd interrupts.
1749 */
1750 void evergreen_hpd_init(struct radeon_device *rdev)
1751 {
1752 struct drm_device *dev = rdev->ddev;
1753 struct drm_connector *connector;
1754 unsigned enabled = 0;
1755 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1756 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1757
1758 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1759 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1760
1761 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1762 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1763 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1764 * aux dp channel on imac and help (but not completely fix)
1765 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1766 * also avoid interrupt storms during dpms.
1767 */
1768 continue;
1769 }
1770 switch (radeon_connector->hpd.hpd) {
1771 case RADEON_HPD_1:
1772 WREG32(DC_HPD1_CONTROL, tmp);
1773 break;
1774 case RADEON_HPD_2:
1775 WREG32(DC_HPD2_CONTROL, tmp);
1776 break;
1777 case RADEON_HPD_3:
1778 WREG32(DC_HPD3_CONTROL, tmp);
1779 break;
1780 case RADEON_HPD_4:
1781 WREG32(DC_HPD4_CONTROL, tmp);
1782 break;
1783 case RADEON_HPD_5:
1784 WREG32(DC_HPD5_CONTROL, tmp);
1785 break;
1786 case RADEON_HPD_6:
1787 WREG32(DC_HPD6_CONTROL, tmp);
1788 break;
1789 default:
1790 break;
1791 }
1792 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1793 enabled |= 1 << radeon_connector->hpd.hpd;
1794 }
1795 radeon_irq_kms_enable_hpd(rdev, enabled);
1796 }
1797
1798 /**
1799 * evergreen_hpd_fini - hpd tear down callback.
1800 *
1801 * @rdev: radeon_device pointer
1802 *
1803 * Tear down the hpd pins used by the card (evergreen+).
1804 * Disable the hpd interrupts.
1805 */
1806 void evergreen_hpd_fini(struct radeon_device *rdev)
1807 {
1808 struct drm_device *dev = rdev->ddev;
1809 struct drm_connector *connector;
1810 unsigned disabled = 0;
1811
1812 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1813 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1814 switch (radeon_connector->hpd.hpd) {
1815 case RADEON_HPD_1:
1816 WREG32(DC_HPD1_CONTROL, 0);
1817 break;
1818 case RADEON_HPD_2:
1819 WREG32(DC_HPD2_CONTROL, 0);
1820 break;
1821 case RADEON_HPD_3:
1822 WREG32(DC_HPD3_CONTROL, 0);
1823 break;
1824 case RADEON_HPD_4:
1825 WREG32(DC_HPD4_CONTROL, 0);
1826 break;
1827 case RADEON_HPD_5:
1828 WREG32(DC_HPD5_CONTROL, 0);
1829 break;
1830 case RADEON_HPD_6:
1831 WREG32(DC_HPD6_CONTROL, 0);
1832 break;
1833 default:
1834 break;
1835 }
1836 disabled |= 1 << radeon_connector->hpd.hpd;
1837 }
1838 radeon_irq_kms_disable_hpd(rdev, disabled);
1839 }
1840
1841 /* watermark setup */
1842
1843 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1844 struct radeon_crtc *radeon_crtc,
1845 struct drm_display_mode *mode,
1846 struct drm_display_mode *other_mode)
1847 {
1848 u32 tmp, buffer_alloc, i;
1849 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1850 /*
1851 * Line Buffer Setup
1852 * There are 3 line buffers, each one shared by 2 display controllers.
1853 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1854 * the display controllers. The paritioning is done via one of four
1855 * preset allocations specified in bits 2:0:
1856 * first display controller
1857 * 0 - first half of lb (3840 * 2)
1858 * 1 - first 3/4 of lb (5760 * 2)
1859 * 2 - whole lb (7680 * 2), other crtc must be disabled
1860 * 3 - first 1/4 of lb (1920 * 2)
1861 * second display controller
1862 * 4 - second half of lb (3840 * 2)
1863 * 5 - second 3/4 of lb (5760 * 2)
1864 * 6 - whole lb (7680 * 2), other crtc must be disabled
1865 * 7 - last 1/4 of lb (1920 * 2)
1866 */
1867 /* this can get tricky if we have two large displays on a paired group
1868 * of crtcs. Ideally for multiple large displays we'd assign them to
1869 * non-linked crtcs for maximum line buffer allocation.
1870 */
1871 if (radeon_crtc->base.enabled && mode) {
1872 if (other_mode) {
1873 tmp = 0; /* 1/2 */
1874 buffer_alloc = 1;
1875 } else {
1876 tmp = 2; /* whole */
1877 buffer_alloc = 2;
1878 }
1879 } else {
1880 tmp = 0;
1881 buffer_alloc = 0;
1882 }
1883
1884 /* second controller of the pair uses second half of the lb */
1885 if (radeon_crtc->crtc_id % 2)
1886 tmp += 4;
1887 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1888
1889 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1890 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1891 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1892 for (i = 0; i < rdev->usec_timeout; i++) {
1893 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1894 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1895 break;
1896 udelay(1);
1897 }
1898 }
1899
1900 if (radeon_crtc->base.enabled && mode) {
1901 switch (tmp) {
1902 case 0:
1903 case 4:
1904 default:
1905 if (ASIC_IS_DCE5(rdev))
1906 return 4096 * 2;
1907 else
1908 return 3840 * 2;
1909 case 1:
1910 case 5:
1911 if (ASIC_IS_DCE5(rdev))
1912 return 6144 * 2;
1913 else
1914 return 5760 * 2;
1915 case 2:
1916 case 6:
1917 if (ASIC_IS_DCE5(rdev))
1918 return 8192 * 2;
1919 else
1920 return 7680 * 2;
1921 case 3:
1922 case 7:
1923 if (ASIC_IS_DCE5(rdev))
1924 return 2048 * 2;
1925 else
1926 return 1920 * 2;
1927 }
1928 }
1929
1930 /* controller not enabled, so no lb used */
1931 return 0;
1932 }
1933
1934 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1935 {
1936 u32 tmp = RREG32(MC_SHARED_CHMAP);
1937
1938 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1939 case 0:
1940 default:
1941 return 1;
1942 case 1:
1943 return 2;
1944 case 2:
1945 return 4;
1946 case 3:
1947 return 8;
1948 }
1949 }
1950
1951 struct evergreen_wm_params {
1952 u32 dram_channels; /* number of dram channels */
1953 u32 yclk; /* bandwidth per dram data pin in kHz */
1954 u32 sclk; /* engine clock in kHz */
1955 u32 disp_clk; /* display clock in kHz */
1956 u32 src_width; /* viewport width */
1957 u32 active_time; /* active display time in ns */
1958 u32 blank_time; /* blank time in ns */
1959 bool interlaced; /* mode is interlaced */
1960 fixed20_12 vsc; /* vertical scale ratio */
1961 u32 num_heads; /* number of active crtcs */
1962 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1963 u32 lb_size; /* line buffer allocated to pipe */
1964 u32 vtaps; /* vertical scaler taps */
1965 };
1966
1967 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1968 {
1969 /* Calculate DRAM Bandwidth and the part allocated to display. */
1970 fixed20_12 dram_efficiency; /* 0.7 */
1971 fixed20_12 yclk, dram_channels, bandwidth;
1972 fixed20_12 a;
1973
1974 a.full = dfixed_const(1000);
1975 yclk.full = dfixed_const(wm->yclk);
1976 yclk.full = dfixed_div(yclk, a);
1977 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1978 a.full = dfixed_const(10);
1979 dram_efficiency.full = dfixed_const(7);
1980 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1981 bandwidth.full = dfixed_mul(dram_channels, yclk);
1982 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1983
1984 return dfixed_trunc(bandwidth);
1985 }
1986
1987 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1988 {
1989 /* Calculate DRAM Bandwidth and the part allocated to display. */
1990 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1991 fixed20_12 yclk, dram_channels, bandwidth;
1992 fixed20_12 a;
1993
1994 a.full = dfixed_const(1000);
1995 yclk.full = dfixed_const(wm->yclk);
1996 yclk.full = dfixed_div(yclk, a);
1997 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1998 a.full = dfixed_const(10);
1999 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2000 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2001 bandwidth.full = dfixed_mul(dram_channels, yclk);
2002 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2003
2004 return dfixed_trunc(bandwidth);
2005 }
2006
2007 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2008 {
2009 /* Calculate the display Data return Bandwidth */
2010 fixed20_12 return_efficiency; /* 0.8 */
2011 fixed20_12 sclk, bandwidth;
2012 fixed20_12 a;
2013
2014 a.full = dfixed_const(1000);
2015 sclk.full = dfixed_const(wm->sclk);
2016 sclk.full = dfixed_div(sclk, a);
2017 a.full = dfixed_const(10);
2018 return_efficiency.full = dfixed_const(8);
2019 return_efficiency.full = dfixed_div(return_efficiency, a);
2020 a.full = dfixed_const(32);
2021 bandwidth.full = dfixed_mul(a, sclk);
2022 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2023
2024 return dfixed_trunc(bandwidth);
2025 }
2026
2027 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2028 {
2029 /* Calculate the DMIF Request Bandwidth */
2030 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2031 fixed20_12 disp_clk, bandwidth;
2032 fixed20_12 a;
2033
2034 a.full = dfixed_const(1000);
2035 disp_clk.full = dfixed_const(wm->disp_clk);
2036 disp_clk.full = dfixed_div(disp_clk, a);
2037 a.full = dfixed_const(10);
2038 disp_clk_request_efficiency.full = dfixed_const(8);
2039 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2040 a.full = dfixed_const(32);
2041 bandwidth.full = dfixed_mul(a, disp_clk);
2042 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2043
2044 return dfixed_trunc(bandwidth);
2045 }
2046
2047 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2048 {
2049 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2050 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2051 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2052 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2053
2054 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2055 }
2056
2057 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2058 {
2059 /* Calculate the display mode Average Bandwidth
2060 * DisplayMode should contain the source and destination dimensions,
2061 * timing, etc.
2062 */
2063 fixed20_12 bpp;
2064 fixed20_12 line_time;
2065 fixed20_12 src_width;
2066 fixed20_12 bandwidth;
2067 fixed20_12 a;
2068
2069 a.full = dfixed_const(1000);
2070 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2071 line_time.full = dfixed_div(line_time, a);
2072 bpp.full = dfixed_const(wm->bytes_per_pixel);
2073 src_width.full = dfixed_const(wm->src_width);
2074 bandwidth.full = dfixed_mul(src_width, bpp);
2075 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2076 bandwidth.full = dfixed_div(bandwidth, line_time);
2077
2078 return dfixed_trunc(bandwidth);
2079 }
2080
2081 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2082 {
2083 /* First calcualte the latency in ns */
2084 u32 mc_latency = 2000; /* 2000 ns. */
2085 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2086 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2087 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2088 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2089 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2090 (wm->num_heads * cursor_line_pair_return_time);
2091 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2092 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2093 fixed20_12 a, b, c;
2094
2095 if (wm->num_heads == 0)
2096 return 0;
2097
2098 a.full = dfixed_const(2);
2099 b.full = dfixed_const(1);
2100 if ((wm->vsc.full > a.full) ||
2101 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2102 (wm->vtaps >= 5) ||
2103 ((wm->vsc.full >= a.full) && wm->interlaced))
2104 max_src_lines_per_dst_line = 4;
2105 else
2106 max_src_lines_per_dst_line = 2;
2107
2108 a.full = dfixed_const(available_bandwidth);
2109 b.full = dfixed_const(wm->num_heads);
2110 a.full = dfixed_div(a, b);
2111
2112 b.full = dfixed_const(1000);
2113 c.full = dfixed_const(wm->disp_clk);
2114 b.full = dfixed_div(c, b);
2115 c.full = dfixed_const(wm->bytes_per_pixel);
2116 b.full = dfixed_mul(b, c);
2117
2118 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2119
2120 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2121 b.full = dfixed_const(1000);
2122 c.full = dfixed_const(lb_fill_bw);
2123 b.full = dfixed_div(c, b);
2124 a.full = dfixed_div(a, b);
2125 line_fill_time = dfixed_trunc(a);
2126
2127 if (line_fill_time < wm->active_time)
2128 return latency;
2129 else
2130 return latency + (line_fill_time - wm->active_time);
2131
2132 }
2133
2134 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2135 {
2136 if (evergreen_average_bandwidth(wm) <=
2137 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2138 return true;
2139 else
2140 return false;
2141 };
2142
2143 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2144 {
2145 if (evergreen_average_bandwidth(wm) <=
2146 (evergreen_available_bandwidth(wm) / wm->num_heads))
2147 return true;
2148 else
2149 return false;
2150 };
2151
2152 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2153 {
2154 u32 lb_partitions = wm->lb_size / wm->src_width;
2155 u32 line_time = wm->active_time + wm->blank_time;
2156 u32 latency_tolerant_lines;
2157 u32 latency_hiding;
2158 fixed20_12 a;
2159
2160 a.full = dfixed_const(1);
2161 if (wm->vsc.full > a.full)
2162 latency_tolerant_lines = 1;
2163 else {
2164 if (lb_partitions <= (wm->vtaps + 1))
2165 latency_tolerant_lines = 1;
2166 else
2167 latency_tolerant_lines = 2;
2168 }
2169
2170 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2171
2172 if (evergreen_latency_watermark(wm) <= latency_hiding)
2173 return true;
2174 else
2175 return false;
2176 }
2177
2178 static void evergreen_program_watermarks(struct radeon_device *rdev,
2179 struct radeon_crtc *radeon_crtc,
2180 u32 lb_size, u32 num_heads)
2181 {
2182 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2183 struct evergreen_wm_params wm_low, wm_high;
2184 u32 dram_channels;
2185 u32 pixel_period;
2186 u32 line_time = 0;
2187 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2188 u32 priority_a_mark = 0, priority_b_mark = 0;
2189 u32 priority_a_cnt = PRIORITY_OFF;
2190 u32 priority_b_cnt = PRIORITY_OFF;
2191 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2192 u32 tmp, arb_control3;
2193 fixed20_12 a, b, c;
2194
2195 if (radeon_crtc->base.enabled && num_heads && mode) {
2196 pixel_period = 1000000 / (u32)mode->clock;
2197 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2198 priority_a_cnt = 0;
2199 priority_b_cnt = 0;
2200 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2201
2202 /* watermark for high clocks */
2203 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2204 wm_high.yclk =
2205 radeon_dpm_get_mclk(rdev, false) * 10;
2206 wm_high.sclk =
2207 radeon_dpm_get_sclk(rdev, false) * 10;
2208 } else {
2209 wm_high.yclk = rdev->pm.current_mclk * 10;
2210 wm_high.sclk = rdev->pm.current_sclk * 10;
2211 }
2212
2213 wm_high.disp_clk = mode->clock;
2214 wm_high.src_width = mode->crtc_hdisplay;
2215 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2216 wm_high.blank_time = line_time - wm_high.active_time;
2217 wm_high.interlaced = false;
2218 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2219 wm_high.interlaced = true;
2220 wm_high.vsc = radeon_crtc->vsc;
2221 wm_high.vtaps = 1;
2222 if (radeon_crtc->rmx_type != RMX_OFF)
2223 wm_high.vtaps = 2;
2224 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2225 wm_high.lb_size = lb_size;
2226 wm_high.dram_channels = dram_channels;
2227 wm_high.num_heads = num_heads;
2228
2229 /* watermark for low clocks */
2230 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2231 wm_low.yclk =
2232 radeon_dpm_get_mclk(rdev, true) * 10;
2233 wm_low.sclk =
2234 radeon_dpm_get_sclk(rdev, true) * 10;
2235 } else {
2236 wm_low.yclk = rdev->pm.current_mclk * 10;
2237 wm_low.sclk = rdev->pm.current_sclk * 10;
2238 }
2239
2240 wm_low.disp_clk = mode->clock;
2241 wm_low.src_width = mode->crtc_hdisplay;
2242 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2243 wm_low.blank_time = line_time - wm_low.active_time;
2244 wm_low.interlaced = false;
2245 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2246 wm_low.interlaced = true;
2247 wm_low.vsc = radeon_crtc->vsc;
2248 wm_low.vtaps = 1;
2249 if (radeon_crtc->rmx_type != RMX_OFF)
2250 wm_low.vtaps = 2;
2251 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2252 wm_low.lb_size = lb_size;
2253 wm_low.dram_channels = dram_channels;
2254 wm_low.num_heads = num_heads;
2255
2256 /* set for high clocks */
2257 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2258 /* set for low clocks */
2259 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2260
2261 /* possibly force display priority to high */
2262 /* should really do this at mode validation time... */
2263 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2264 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2265 !evergreen_check_latency_hiding(&wm_high) ||
2266 (rdev->disp_priority == 2)) {
2267 DRM_DEBUG_KMS("force priority a to high\n");
2268 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2269 }
2270 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2271 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2272 !evergreen_check_latency_hiding(&wm_low) ||
2273 (rdev->disp_priority == 2)) {
2274 DRM_DEBUG_KMS("force priority b to high\n");
2275 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2276 }
2277
2278 a.full = dfixed_const(1000);
2279 b.full = dfixed_const(mode->clock);
2280 b.full = dfixed_div(b, a);
2281 c.full = dfixed_const(latency_watermark_a);
2282 c.full = dfixed_mul(c, b);
2283 c.full = dfixed_mul(c, radeon_crtc->hsc);
2284 c.full = dfixed_div(c, a);
2285 a.full = dfixed_const(16);
2286 c.full = dfixed_div(c, a);
2287 priority_a_mark = dfixed_trunc(c);
2288 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2289
2290 a.full = dfixed_const(1000);
2291 b.full = dfixed_const(mode->clock);
2292 b.full = dfixed_div(b, a);
2293 c.full = dfixed_const(latency_watermark_b);
2294 c.full = dfixed_mul(c, b);
2295 c.full = dfixed_mul(c, radeon_crtc->hsc);
2296 c.full = dfixed_div(c, a);
2297 a.full = dfixed_const(16);
2298 c.full = dfixed_div(c, a);
2299 priority_b_mark = dfixed_trunc(c);
2300 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2301 }
2302
2303 /* select wm A */
2304 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2305 tmp = arb_control3;
2306 tmp &= ~LATENCY_WATERMARK_MASK(3);
2307 tmp |= LATENCY_WATERMARK_MASK(1);
2308 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2309 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2310 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2311 LATENCY_HIGH_WATERMARK(line_time)));
2312 /* select wm B */
2313 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2314 tmp &= ~LATENCY_WATERMARK_MASK(3);
2315 tmp |= LATENCY_WATERMARK_MASK(2);
2316 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2317 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2318 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2319 LATENCY_HIGH_WATERMARK(line_time)));
2320 /* restore original selection */
2321 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2322
2323 /* write the priority marks */
2324 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2325 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2326
2327 /* save values for DPM */
2328 radeon_crtc->line_time = line_time;
2329 radeon_crtc->wm_high = latency_watermark_a;
2330 radeon_crtc->wm_low = latency_watermark_b;
2331 }
2332
2333 /**
2334 * evergreen_bandwidth_update - update display watermarks callback.
2335 *
2336 * @rdev: radeon_device pointer
2337 *
2338 * Update the display watermarks based on the requested mode(s)
2339 * (evergreen+).
2340 */
2341 void evergreen_bandwidth_update(struct radeon_device *rdev)
2342 {
2343 struct drm_display_mode *mode0 = NULL;
2344 struct drm_display_mode *mode1 = NULL;
2345 u32 num_heads = 0, lb_size;
2346 int i;
2347
2348 if (!rdev->mode_info.mode_config_initialized)
2349 return;
2350
2351 radeon_update_display_priority(rdev);
2352
2353 for (i = 0; i < rdev->num_crtc; i++) {
2354 if (rdev->mode_info.crtcs[i]->base.enabled)
2355 num_heads++;
2356 }
2357 for (i = 0; i < rdev->num_crtc; i += 2) {
2358 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2359 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2360 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2361 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2362 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2363 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2364 }
2365 }
2366
2367 /**
2368 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2369 *
2370 * @rdev: radeon_device pointer
2371 *
2372 * Wait for the MC (memory controller) to be idle.
2373 * (evergreen+).
2374 * Returns 0 if the MC is idle, -1 if not.
2375 */
2376 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2377 {
2378 unsigned i;
2379 u32 tmp;
2380
2381 for (i = 0; i < rdev->usec_timeout; i++) {
2382 /* read MC_STATUS */
2383 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2384 if (!tmp)
2385 return 0;
2386 udelay(1);
2387 }
2388 return -1;
2389 }
2390
2391 /*
2392 * GART
2393 */
2394 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2395 {
2396 unsigned i;
2397 u32 tmp;
2398
2399 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2400
2401 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2402 for (i = 0; i < rdev->usec_timeout; i++) {
2403 /* read MC_STATUS */
2404 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2405 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2406 if (tmp == 2) {
2407 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2408 return;
2409 }
2410 if (tmp) {
2411 return;
2412 }
2413 udelay(1);
2414 }
2415 }
2416
2417 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2418 {
2419 u32 tmp;
2420 int r;
2421
2422 if (rdev->gart.robj == NULL) {
2423 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2424 return -EINVAL;
2425 }
2426 r = radeon_gart_table_vram_pin(rdev);
2427 if (r)
2428 return r;
2429 /* Setup L2 cache */
2430 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2431 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2432 EFFECTIVE_L2_QUEUE_SIZE(7));
2433 WREG32(VM_L2_CNTL2, 0);
2434 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2435 /* Setup TLB control */
2436 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2437 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2438 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2439 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2440 if (rdev->flags & RADEON_IS_IGP) {
2441 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2442 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2443 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2444 } else {
2445 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2446 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2447 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2448 if ((rdev->family == CHIP_JUNIPER) ||
2449 (rdev->family == CHIP_CYPRESS) ||
2450 (rdev->family == CHIP_HEMLOCK) ||
2451 (rdev->family == CHIP_BARTS))
2452 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2453 }
2454 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2455 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2456 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2457 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2458 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2459 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2460 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2461 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2462 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2463 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2464 (u32)(rdev->dummy_page.addr >> 12));
2465 WREG32(VM_CONTEXT1_CNTL, 0);
2466
2467 evergreen_pcie_gart_tlb_flush(rdev);
2468 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2469 (unsigned)(rdev->mc.gtt_size >> 20),
2470 (unsigned long long)rdev->gart.table_addr);
2471 rdev->gart.ready = true;
2472 return 0;
2473 }
2474
2475 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2476 {
2477 u32 tmp;
2478
2479 /* Disable all tables */
2480 WREG32(VM_CONTEXT0_CNTL, 0);
2481 WREG32(VM_CONTEXT1_CNTL, 0);
2482
2483 /* Setup L2 cache */
2484 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2485 EFFECTIVE_L2_QUEUE_SIZE(7));
2486 WREG32(VM_L2_CNTL2, 0);
2487 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2488 /* Setup TLB control */
2489 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2490 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2491 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2492 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2493 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2494 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2495 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2496 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2497 radeon_gart_table_vram_unpin(rdev);
2498 }
2499
2500 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2501 {
2502 evergreen_pcie_gart_disable(rdev);
2503 radeon_gart_table_vram_free(rdev);
2504 radeon_gart_fini(rdev);
2505 }
2506
2507
2508 static void evergreen_agp_enable(struct radeon_device *rdev)
2509 {
2510 u32 tmp;
2511
2512 /* Setup L2 cache */
2513 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2514 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2515 EFFECTIVE_L2_QUEUE_SIZE(7));
2516 WREG32(VM_L2_CNTL2, 0);
2517 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2518 /* Setup TLB control */
2519 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2520 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2521 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2522 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2523 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2524 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2525 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2526 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2527 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2528 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2529 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2530 WREG32(VM_CONTEXT0_CNTL, 0);
2531 WREG32(VM_CONTEXT1_CNTL, 0);
2532 }
2533
2534 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2535 {
2536 u32 crtc_enabled, tmp, frame_count, blackout;
2537 int i, j;
2538
2539 if (!ASIC_IS_NODCE(rdev)) {
2540 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2541 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2542
2543 /* disable VGA render */
2544 WREG32(VGA_RENDER_CONTROL, 0);
2545 }
2546 /* blank the display controllers */
2547 for (i = 0; i < rdev->num_crtc; i++) {
2548 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2549 if (crtc_enabled) {
2550 save->crtc_enabled[i] = true;
2551 if (ASIC_IS_DCE6(rdev)) {
2552 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2553 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2554 radeon_wait_for_vblank(rdev, i);
2555 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2556 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2557 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2558 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2559 }
2560 } else {
2561 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2562 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2563 radeon_wait_for_vblank(rdev, i);
2564 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2565 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2566 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2567 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2568 }
2569 }
2570 /* wait for the next frame */
2571 frame_count = radeon_get_vblank_counter(rdev, i);
2572 for (j = 0; j < rdev->usec_timeout; j++) {
2573 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2574 break;
2575 udelay(1);
2576 }
2577
2578 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2579 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2580 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2581 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2582 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2583 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2584 save->crtc_enabled[i] = false;
2585 /* ***** */
2586 } else {
2587 save->crtc_enabled[i] = false;
2588 }
2589 }
2590
2591 radeon_mc_wait_for_idle(rdev);
2592
2593 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2594 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2595 /* Block CPU access */
2596 WREG32(BIF_FB_EN, 0);
2597 /* blackout the MC */
2598 blackout &= ~BLACKOUT_MODE_MASK;
2599 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2600 }
2601 /* wait for the MC to settle */
2602 udelay(100);
2603
2604 /* lock double buffered regs */
2605 for (i = 0; i < rdev->num_crtc; i++) {
2606 if (save->crtc_enabled[i]) {
2607 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2608 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2609 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2610 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2611 }
2612 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2613 if (!(tmp & 1)) {
2614 tmp |= 1;
2615 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2616 }
2617 }
2618 }
2619 }
2620
2621 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2622 {
2623 u32 tmp, frame_count;
2624 int i, j;
2625
2626 /* update crtc base addresses */
2627 for (i = 0; i < rdev->num_crtc; i++) {
2628 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2629 upper_32_bits(rdev->mc.vram_start));
2630 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2631 upper_32_bits(rdev->mc.vram_start));
2632 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2633 (u32)rdev->mc.vram_start);
2634 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2635 (u32)rdev->mc.vram_start);
2636 }
2637
2638 if (!ASIC_IS_NODCE(rdev)) {
2639 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2640 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2641 }
2642
2643 /* unlock regs and wait for update */
2644 for (i = 0; i < rdev->num_crtc; i++) {
2645 if (save->crtc_enabled[i]) {
2646 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2647 if ((tmp & 0x7) != 3) {
2648 tmp &= ~0x7;
2649 tmp |= 0x3;
2650 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2651 }
2652 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2653 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2654 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2655 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2656 }
2657 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2658 if (tmp & 1) {
2659 tmp &= ~1;
2660 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2661 }
2662 for (j = 0; j < rdev->usec_timeout; j++) {
2663 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2664 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2665 break;
2666 udelay(1);
2667 }
2668 }
2669 }
2670
2671 /* unblackout the MC */
2672 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2673 tmp &= ~BLACKOUT_MODE_MASK;
2674 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2675 /* allow CPU access */
2676 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2677
2678 for (i = 0; i < rdev->num_crtc; i++) {
2679 if (save->crtc_enabled[i]) {
2680 if (ASIC_IS_DCE6(rdev)) {
2681 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2682 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2683 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2684 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2685 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2686 } else {
2687 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2688 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2689 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2690 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2691 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2692 }
2693 /* wait for the next frame */
2694 frame_count = radeon_get_vblank_counter(rdev, i);
2695 for (j = 0; j < rdev->usec_timeout; j++) {
2696 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2697 break;
2698 udelay(1);
2699 }
2700 }
2701 }
2702 if (!ASIC_IS_NODCE(rdev)) {
2703 /* Unlock vga access */
2704 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2705 mdelay(1);
2706 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2707 }
2708 }
2709
2710 void evergreen_mc_program(struct radeon_device *rdev)
2711 {
2712 struct evergreen_mc_save save;
2713 u32 tmp;
2714 int i, j;
2715
2716 /* Initialize HDP */
2717 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2718 WREG32((0x2c14 + j), 0x00000000);
2719 WREG32((0x2c18 + j), 0x00000000);
2720 WREG32((0x2c1c + j), 0x00000000);
2721 WREG32((0x2c20 + j), 0x00000000);
2722 WREG32((0x2c24 + j), 0x00000000);
2723 }
2724 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2725
2726 evergreen_mc_stop(rdev, &save);
2727 if (evergreen_mc_wait_for_idle(rdev)) {
2728 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2729 }
2730 /* Lockout access through VGA aperture*/
2731 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2732 /* Update configuration */
2733 if (rdev->flags & RADEON_IS_AGP) {
2734 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2735 /* VRAM before AGP */
2736 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2737 rdev->mc.vram_start >> 12);
2738 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2739 rdev->mc.gtt_end >> 12);
2740 } else {
2741 /* VRAM after AGP */
2742 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2743 rdev->mc.gtt_start >> 12);
2744 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2745 rdev->mc.vram_end >> 12);
2746 }
2747 } else {
2748 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2749 rdev->mc.vram_start >> 12);
2750 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2751 rdev->mc.vram_end >> 12);
2752 }
2753 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2754 /* llano/ontario only */
2755 if ((rdev->family == CHIP_PALM) ||
2756 (rdev->family == CHIP_SUMO) ||
2757 (rdev->family == CHIP_SUMO2)) {
2758 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2759 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2760 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2761 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2762 }
2763 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2764 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2765 WREG32(MC_VM_FB_LOCATION, tmp);
2766 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2767 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2768 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2769 if (rdev->flags & RADEON_IS_AGP) {
2770 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2771 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2772 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2773 } else {
2774 WREG32(MC_VM_AGP_BASE, 0);
2775 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2776 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2777 }
2778 if (evergreen_mc_wait_for_idle(rdev)) {
2779 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2780 }
2781 evergreen_mc_resume(rdev, &save);
2782 /* we need to own VRAM, so turn off the VGA renderer here
2783 * to stop it overwriting our objects */
2784 rv515_vga_render_disable(rdev);
2785 }
2786
2787 /*
2788 * CP.
2789 */
2790 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2791 {
2792 struct radeon_ring *ring = &rdev->ring[ib->ring];
2793 u32 next_rptr;
2794
2795 /* set to DX10/11 mode */
2796 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2797 radeon_ring_write(ring, 1);
2798
2799 if (ring->rptr_save_reg) {
2800 next_rptr = ring->wptr + 3 + 4;
2801 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2802 radeon_ring_write(ring, ((ring->rptr_save_reg -
2803 PACKET3_SET_CONFIG_REG_START) >> 2));
2804 radeon_ring_write(ring, next_rptr);
2805 } else if (rdev->wb.enabled) {
2806 next_rptr = ring->wptr + 5 + 4;
2807 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2808 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2809 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2810 radeon_ring_write(ring, next_rptr);
2811 radeon_ring_write(ring, 0);
2812 }
2813
2814 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2815 radeon_ring_write(ring,
2816 #ifdef __BIG_ENDIAN
2817 (2 << 0) |
2818 #endif
2819 (ib->gpu_addr & 0xFFFFFFFC));
2820 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2821 radeon_ring_write(ring, ib->length_dw);
2822 }
2823
2824
2825 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2826 {
2827 const __be32 *fw_data;
2828 int i;
2829
2830 if (!rdev->me_fw || !rdev->pfp_fw)
2831 return -EINVAL;
2832
2833 r700_cp_stop(rdev);
2834 WREG32(CP_RB_CNTL,
2835 #ifdef __BIG_ENDIAN
2836 BUF_SWAP_32BIT |
2837 #endif
2838 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2839
2840 fw_data = (const __be32 *)rdev->pfp_fw->data;
2841 WREG32(CP_PFP_UCODE_ADDR, 0);
2842 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2843 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2844 WREG32(CP_PFP_UCODE_ADDR, 0);
2845
2846 fw_data = (const __be32 *)rdev->me_fw->data;
2847 WREG32(CP_ME_RAM_WADDR, 0);
2848 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2849 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2850
2851 WREG32(CP_PFP_UCODE_ADDR, 0);
2852 WREG32(CP_ME_RAM_WADDR, 0);
2853 WREG32(CP_ME_RAM_RADDR, 0);
2854 return 0;
2855 }
2856
2857 static int evergreen_cp_start(struct radeon_device *rdev)
2858 {
2859 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2860 int r, i;
2861 uint32_t cp_me;
2862
2863 r = radeon_ring_lock(rdev, ring, 7);
2864 if (r) {
2865 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2866 return r;
2867 }
2868 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2869 radeon_ring_write(ring, 0x1);
2870 radeon_ring_write(ring, 0x0);
2871 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2872 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2873 radeon_ring_write(ring, 0);
2874 radeon_ring_write(ring, 0);
2875 radeon_ring_unlock_commit(rdev, ring, false);
2876
2877 cp_me = 0xff;
2878 WREG32(CP_ME_CNTL, cp_me);
2879
2880 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
2881 if (r) {
2882 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2883 return r;
2884 }
2885
2886 /* setup clear context state */
2887 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2888 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2889
2890 for (i = 0; i < evergreen_default_size; i++)
2891 radeon_ring_write(ring, evergreen_default_state[i]);
2892
2893 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2894 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2895
2896 /* set clear context state */
2897 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2898 radeon_ring_write(ring, 0);
2899
2900 /* SQ_VTX_BASE_VTX_LOC */
2901 radeon_ring_write(ring, 0xc0026f00);
2902 radeon_ring_write(ring, 0x00000000);
2903 radeon_ring_write(ring, 0x00000000);
2904 radeon_ring_write(ring, 0x00000000);
2905
2906 /* Clear consts */
2907 radeon_ring_write(ring, 0xc0036f00);
2908 radeon_ring_write(ring, 0x00000bc4);
2909 radeon_ring_write(ring, 0xffffffff);
2910 radeon_ring_write(ring, 0xffffffff);
2911 radeon_ring_write(ring, 0xffffffff);
2912
2913 radeon_ring_write(ring, 0xc0026900);
2914 radeon_ring_write(ring, 0x00000316);
2915 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2916 radeon_ring_write(ring, 0x00000010); /* */
2917
2918 radeon_ring_unlock_commit(rdev, ring, false);
2919
2920 return 0;
2921 }
2922
2923 static int evergreen_cp_resume(struct radeon_device *rdev)
2924 {
2925 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2926 u32 tmp;
2927 u32 rb_bufsz;
2928 int r;
2929
2930 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2931 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2932 SOFT_RESET_PA |
2933 SOFT_RESET_SH |
2934 SOFT_RESET_VGT |
2935 SOFT_RESET_SPI |
2936 SOFT_RESET_SX));
2937 RREG32(GRBM_SOFT_RESET);
2938 mdelay(15);
2939 WREG32(GRBM_SOFT_RESET, 0);
2940 RREG32(GRBM_SOFT_RESET);
2941
2942 /* Set ring buffer size */
2943 rb_bufsz = order_base_2(ring->ring_size / 8);
2944 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2945 #ifdef __BIG_ENDIAN
2946 tmp |= BUF_SWAP_32BIT;
2947 #endif
2948 WREG32(CP_RB_CNTL, tmp);
2949 WREG32(CP_SEM_WAIT_TIMER, 0x0);
2950 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2951
2952 /* Set the write pointer delay */
2953 WREG32(CP_RB_WPTR_DELAY, 0);
2954
2955 /* Initialize the ring buffer's read and write pointers */
2956 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2957 WREG32(CP_RB_RPTR_WR, 0);
2958 ring->wptr = 0;
2959 WREG32(CP_RB_WPTR, ring->wptr);
2960
2961 /* set the wb address whether it's enabled or not */
2962 WREG32(CP_RB_RPTR_ADDR,
2963 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2964 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2965 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2966
2967 if (rdev->wb.enabled)
2968 WREG32(SCRATCH_UMSK, 0xff);
2969 else {
2970 tmp |= RB_NO_UPDATE;
2971 WREG32(SCRATCH_UMSK, 0);
2972 }
2973
2974 mdelay(1);
2975 WREG32(CP_RB_CNTL, tmp);
2976
2977 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2978 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2979
2980 evergreen_cp_start(rdev);
2981 ring->ready = true;
2982 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2983 if (r) {
2984 ring->ready = false;
2985 return r;
2986 }
2987 return 0;
2988 }
2989
2990 /*
2991 * Core functions
2992 */
2993 static void evergreen_gpu_init(struct radeon_device *rdev)
2994 {
2995 u32 gb_addr_config;
2996 u32 mc_shared_chmap, mc_arb_ramcfg;
2997 u32 sx_debug_1;
2998 u32 smx_dc_ctl0;
2999 u32 sq_config;
3000 u32 sq_lds_resource_mgmt;
3001 u32 sq_gpr_resource_mgmt_1;
3002 u32 sq_gpr_resource_mgmt_2;
3003 u32 sq_gpr_resource_mgmt_3;
3004 u32 sq_thread_resource_mgmt;
3005 u32 sq_thread_resource_mgmt_2;
3006 u32 sq_stack_resource_mgmt_1;
3007 u32 sq_stack_resource_mgmt_2;
3008 u32 sq_stack_resource_mgmt_3;
3009 u32 vgt_cache_invalidation;
3010 u32 hdp_host_path_cntl, tmp;
3011 u32 disabled_rb_mask;
3012 int i, j, ps_thread_count;
3013
3014 switch (rdev->family) {
3015 case CHIP_CYPRESS:
3016 case CHIP_HEMLOCK:
3017 rdev->config.evergreen.num_ses = 2;
3018 rdev->config.evergreen.max_pipes = 4;
3019 rdev->config.evergreen.max_tile_pipes = 8;
3020 rdev->config.evergreen.max_simds = 10;
3021 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3022 rdev->config.evergreen.max_gprs = 256;
3023 rdev->config.evergreen.max_threads = 248;
3024 rdev->config.evergreen.max_gs_threads = 32;
3025 rdev->config.evergreen.max_stack_entries = 512;
3026 rdev->config.evergreen.sx_num_of_sets = 4;
3027 rdev->config.evergreen.sx_max_export_size = 256;
3028 rdev->config.evergreen.sx_max_export_pos_size = 64;
3029 rdev->config.evergreen.sx_max_export_smx_size = 192;
3030 rdev->config.evergreen.max_hw_contexts = 8;
3031 rdev->config.evergreen.sq_num_cf_insts = 2;
3032
3033 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3034 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3035 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3036 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3037 break;
3038 case CHIP_JUNIPER:
3039 rdev->config.evergreen.num_ses = 1;
3040 rdev->config.evergreen.max_pipes = 4;
3041 rdev->config.evergreen.max_tile_pipes = 4;
3042 rdev->config.evergreen.max_simds = 10;
3043 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3044 rdev->config.evergreen.max_gprs = 256;
3045 rdev->config.evergreen.max_threads = 248;
3046 rdev->config.evergreen.max_gs_threads = 32;
3047 rdev->config.evergreen.max_stack_entries = 512;
3048 rdev->config.evergreen.sx_num_of_sets = 4;
3049 rdev->config.evergreen.sx_max_export_size = 256;
3050 rdev->config.evergreen.sx_max_export_pos_size = 64;
3051 rdev->config.evergreen.sx_max_export_smx_size = 192;
3052 rdev->config.evergreen.max_hw_contexts = 8;
3053 rdev->config.evergreen.sq_num_cf_insts = 2;
3054
3055 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3056 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3057 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3058 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3059 break;
3060 case CHIP_REDWOOD:
3061 rdev->config.evergreen.num_ses = 1;
3062 rdev->config.evergreen.max_pipes = 4;
3063 rdev->config.evergreen.max_tile_pipes = 4;
3064 rdev->config.evergreen.max_simds = 5;
3065 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3066 rdev->config.evergreen.max_gprs = 256;
3067 rdev->config.evergreen.max_threads = 248;
3068 rdev->config.evergreen.max_gs_threads = 32;
3069 rdev->config.evergreen.max_stack_entries = 256;
3070 rdev->config.evergreen.sx_num_of_sets = 4;
3071 rdev->config.evergreen.sx_max_export_size = 256;
3072 rdev->config.evergreen.sx_max_export_pos_size = 64;
3073 rdev->config.evergreen.sx_max_export_smx_size = 192;
3074 rdev->config.evergreen.max_hw_contexts = 8;
3075 rdev->config.evergreen.sq_num_cf_insts = 2;
3076
3077 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3078 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3079 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3080 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3081 break;
3082 case CHIP_CEDAR:
3083 default:
3084 rdev->config.evergreen.num_ses = 1;
3085 rdev->config.evergreen.max_pipes = 2;
3086 rdev->config.evergreen.max_tile_pipes = 2;
3087 rdev->config.evergreen.max_simds = 2;
3088 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3089 rdev->config.evergreen.max_gprs = 256;
3090 rdev->config.evergreen.max_threads = 192;
3091 rdev->config.evergreen.max_gs_threads = 16;
3092 rdev->config.evergreen.max_stack_entries = 256;
3093 rdev->config.evergreen.sx_num_of_sets = 4;
3094 rdev->config.evergreen.sx_max_export_size = 128;
3095 rdev->config.evergreen.sx_max_export_pos_size = 32;
3096 rdev->config.evergreen.sx_max_export_smx_size = 96;
3097 rdev->config.evergreen.max_hw_contexts = 4;
3098 rdev->config.evergreen.sq_num_cf_insts = 1;
3099
3100 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3101 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3102 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3103 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3104 break;
3105 case CHIP_PALM:
3106 rdev->config.evergreen.num_ses = 1;
3107 rdev->config.evergreen.max_pipes = 2;
3108 rdev->config.evergreen.max_tile_pipes = 2;
3109 rdev->config.evergreen.max_simds = 2;
3110 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3111 rdev->config.evergreen.max_gprs = 256;
3112 rdev->config.evergreen.max_threads = 192;
3113 rdev->config.evergreen.max_gs_threads = 16;
3114 rdev->config.evergreen.max_stack_entries = 256;
3115 rdev->config.evergreen.sx_num_of_sets = 4;
3116 rdev->config.evergreen.sx_max_export_size = 128;
3117 rdev->config.evergreen.sx_max_export_pos_size = 32;
3118 rdev->config.evergreen.sx_max_export_smx_size = 96;
3119 rdev->config.evergreen.max_hw_contexts = 4;
3120 rdev->config.evergreen.sq_num_cf_insts = 1;
3121
3122 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3123 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3124 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3125 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3126 break;
3127 case CHIP_SUMO:
3128 rdev->config.evergreen.num_ses = 1;
3129 rdev->config.evergreen.max_pipes = 4;
3130 rdev->config.evergreen.max_tile_pipes = 4;
3131 if (rdev->pdev->device == 0x9648)
3132 rdev->config.evergreen.max_simds = 3;
3133 else if ((rdev->pdev->device == 0x9647) ||
3134 (rdev->pdev->device == 0x964a))
3135 rdev->config.evergreen.max_simds = 4;
3136 else
3137 rdev->config.evergreen.max_simds = 5;
3138 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3139 rdev->config.evergreen.max_gprs = 256;
3140 rdev->config.evergreen.max_threads = 248;
3141 rdev->config.evergreen.max_gs_threads = 32;
3142 rdev->config.evergreen.max_stack_entries = 256;
3143 rdev->config.evergreen.sx_num_of_sets = 4;
3144 rdev->config.evergreen.sx_max_export_size = 256;
3145 rdev->config.evergreen.sx_max_export_pos_size = 64;
3146 rdev->config.evergreen.sx_max_export_smx_size = 192;
3147 rdev->config.evergreen.max_hw_contexts = 8;
3148 rdev->config.evergreen.sq_num_cf_insts = 2;
3149
3150 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3151 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3152 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3153 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3154 break;
3155 case CHIP_SUMO2:
3156 rdev->config.evergreen.num_ses = 1;
3157 rdev->config.evergreen.max_pipes = 4;
3158 rdev->config.evergreen.max_tile_pipes = 4;
3159 rdev->config.evergreen.max_simds = 2;
3160 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3161 rdev->config.evergreen.max_gprs = 256;
3162 rdev->config.evergreen.max_threads = 248;
3163 rdev->config.evergreen.max_gs_threads = 32;
3164 rdev->config.evergreen.max_stack_entries = 512;
3165 rdev->config.evergreen.sx_num_of_sets = 4;
3166 rdev->config.evergreen.sx_max_export_size = 256;
3167 rdev->config.evergreen.sx_max_export_pos_size = 64;
3168 rdev->config.evergreen.sx_max_export_smx_size = 192;
3169 rdev->config.evergreen.max_hw_contexts = 4;
3170 rdev->config.evergreen.sq_num_cf_insts = 2;
3171
3172 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3173 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3174 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3175 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3176 break;
3177 case CHIP_BARTS:
3178 rdev->config.evergreen.num_ses = 2;
3179 rdev->config.evergreen.max_pipes = 4;
3180 rdev->config.evergreen.max_tile_pipes = 8;
3181 rdev->config.evergreen.max_simds = 7;
3182 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3183 rdev->config.evergreen.max_gprs = 256;
3184 rdev->config.evergreen.max_threads = 248;
3185 rdev->config.evergreen.max_gs_threads = 32;
3186 rdev->config.evergreen.max_stack_entries = 512;
3187 rdev->config.evergreen.sx_num_of_sets = 4;
3188 rdev->config.evergreen.sx_max_export_size = 256;
3189 rdev->config.evergreen.sx_max_export_pos_size = 64;
3190 rdev->config.evergreen.sx_max_export_smx_size = 192;
3191 rdev->config.evergreen.max_hw_contexts = 8;
3192 rdev->config.evergreen.sq_num_cf_insts = 2;
3193
3194 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3195 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3196 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3197 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3198 break;
3199 case CHIP_TURKS:
3200 rdev->config.evergreen.num_ses = 1;
3201 rdev->config.evergreen.max_pipes = 4;
3202 rdev->config.evergreen.max_tile_pipes = 4;
3203 rdev->config.evergreen.max_simds = 6;
3204 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3205 rdev->config.evergreen.max_gprs = 256;
3206 rdev->config.evergreen.max_threads = 248;
3207 rdev->config.evergreen.max_gs_threads = 32;
3208 rdev->config.evergreen.max_stack_entries = 256;
3209 rdev->config.evergreen.sx_num_of_sets = 4;
3210 rdev->config.evergreen.sx_max_export_size = 256;
3211 rdev->config.evergreen.sx_max_export_pos_size = 64;
3212 rdev->config.evergreen.sx_max_export_smx_size = 192;
3213 rdev->config.evergreen.max_hw_contexts = 8;
3214 rdev->config.evergreen.sq_num_cf_insts = 2;
3215
3216 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3217 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3218 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3219 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3220 break;
3221 case CHIP_CAICOS:
3222 rdev->config.evergreen.num_ses = 1;
3223 rdev->config.evergreen.max_pipes = 2;
3224 rdev->config.evergreen.max_tile_pipes = 2;
3225 rdev->config.evergreen.max_simds = 2;
3226 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3227 rdev->config.evergreen.max_gprs = 256;
3228 rdev->config.evergreen.max_threads = 192;
3229 rdev->config.evergreen.max_gs_threads = 16;
3230 rdev->config.evergreen.max_stack_entries = 256;
3231 rdev->config.evergreen.sx_num_of_sets = 4;
3232 rdev->config.evergreen.sx_max_export_size = 128;
3233 rdev->config.evergreen.sx_max_export_pos_size = 32;
3234 rdev->config.evergreen.sx_max_export_smx_size = 96;
3235 rdev->config.evergreen.max_hw_contexts = 4;
3236 rdev->config.evergreen.sq_num_cf_insts = 1;
3237
3238 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3239 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3240 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3241 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3242 break;
3243 }
3244
3245 /* Initialize HDP */
3246 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3247 WREG32((0x2c14 + j), 0x00000000);
3248 WREG32((0x2c18 + j), 0x00000000);
3249 WREG32((0x2c1c + j), 0x00000000);
3250 WREG32((0x2c20 + j), 0x00000000);
3251 WREG32((0x2c24 + j), 0x00000000);
3252 }
3253
3254 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3255
3256 evergreen_fix_pci_max_read_req_size(rdev);
3257
3258 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3259 if ((rdev->family == CHIP_PALM) ||
3260 (rdev->family == CHIP_SUMO) ||
3261 (rdev->family == CHIP_SUMO2))
3262 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3263 else
3264 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3265
3266 /* setup tiling info dword. gb_addr_config is not adequate since it does
3267 * not have bank info, so create a custom tiling dword.
3268 * bits 3:0 num_pipes
3269 * bits 7:4 num_banks
3270 * bits 11:8 group_size
3271 * bits 15:12 row_size
3272 */
3273 rdev->config.evergreen.tile_config = 0;
3274 switch (rdev->config.evergreen.max_tile_pipes) {
3275 case 1:
3276 default:
3277 rdev->config.evergreen.tile_config |= (0 << 0);
3278 break;
3279 case 2:
3280 rdev->config.evergreen.tile_config |= (1 << 0);
3281 break;
3282 case 4:
3283 rdev->config.evergreen.tile_config |= (2 << 0);
3284 break;
3285 case 8:
3286 rdev->config.evergreen.tile_config |= (3 << 0);
3287 break;
3288 }
3289 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3290 if (rdev->flags & RADEON_IS_IGP)
3291 rdev->config.evergreen.tile_config |= 1 << 4;
3292 else {
3293 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3294 case 0: /* four banks */
3295 rdev->config.evergreen.tile_config |= 0 << 4;
3296 break;
3297 case 1: /* eight banks */
3298 rdev->config.evergreen.tile_config |= 1 << 4;
3299 break;
3300 case 2: /* sixteen banks */
3301 default:
3302 rdev->config.evergreen.tile_config |= 2 << 4;
3303 break;
3304 }
3305 }
3306 rdev->config.evergreen.tile_config |= 0 << 8;
3307 rdev->config.evergreen.tile_config |=
3308 ((gb_addr_config & 0x30000000) >> 28) << 12;
3309
3310 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3311 u32 efuse_straps_4;
3312 u32 efuse_straps_3;
3313
3314 efuse_straps_4 = RREG32_RCU(0x204);
3315 efuse_straps_3 = RREG32_RCU(0x203);
3316 tmp = (((efuse_straps_4 & 0xf) << 4) |
3317 ((efuse_straps_3 & 0xf0000000) >> 28));
3318 } else {
3319 tmp = 0;
3320 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3321 u32 rb_disable_bitmap;
3322
3323 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3324 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3325 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3326 tmp <<= 4;
3327 tmp |= rb_disable_bitmap;
3328 }
3329 }
3330 /* enabled rb are just the one not disabled :) */
3331 disabled_rb_mask = tmp;
3332 tmp = 0;
3333 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3334 tmp |= (1 << i);
3335 /* if all the backends are disabled, fix it up here */
3336 if ((disabled_rb_mask & tmp) == tmp) {
3337 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3338 disabled_rb_mask &= ~(1 << i);
3339 }
3340
3341 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3342 u32 simd_disable_bitmap;
3343
3344 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3345 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3346 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3347 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3348 tmp <<= 16;
3349 tmp |= simd_disable_bitmap;
3350 }
3351 rdev->config.evergreen.active_simds = hweight32(~tmp);
3352
3353 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3354 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3355
3356 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3357 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3358 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3359 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3360 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3361 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3362 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3363
3364 if ((rdev->config.evergreen.max_backends == 1) &&
3365 (rdev->flags & RADEON_IS_IGP)) {
3366 if ((disabled_rb_mask & 3) == 1) {
3367 /* RB0 disabled, RB1 enabled */
3368 tmp = 0x11111111;
3369 } else {
3370 /* RB1 disabled, RB0 enabled */
3371 tmp = 0x00000000;
3372 }
3373 } else {
3374 tmp = gb_addr_config & NUM_PIPES_MASK;
3375 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3376 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3377 }
3378 WREG32(GB_BACKEND_MAP, tmp);
3379
3380 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3381 WREG32(CGTS_TCC_DISABLE, 0);
3382 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3383 WREG32(CGTS_USER_TCC_DISABLE, 0);
3384
3385 /* set HW defaults for 3D engine */
3386 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3387 ROQ_IB2_START(0x2b)));
3388
3389 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3390
3391 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3392 SYNC_GRADIENT |
3393 SYNC_WALKER |
3394 SYNC_ALIGNER));
3395
3396 sx_debug_1 = RREG32(SX_DEBUG_1);
3397 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3398 WREG32(SX_DEBUG_1, sx_debug_1);
3399
3400
3401 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3402 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3403 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3404 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3405
3406 if (rdev->family <= CHIP_SUMO2)
3407 WREG32(SMX_SAR_CTL0, 0x00010000);
3408
3409 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3410 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3411 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3412
3413 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3414 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3415 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3416
3417 WREG32(VGT_NUM_INSTANCES, 1);
3418 WREG32(SPI_CONFIG_CNTL, 0);
3419 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3420 WREG32(CP_PERFMON_CNTL, 0);
3421
3422 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3423 FETCH_FIFO_HIWATER(0x4) |
3424 DONE_FIFO_HIWATER(0xe0) |
3425 ALU_UPDATE_FIFO_HIWATER(0x8)));
3426
3427 sq_config = RREG32(SQ_CONFIG);
3428 sq_config &= ~(PS_PRIO(3) |
3429 VS_PRIO(3) |
3430 GS_PRIO(3) |
3431 ES_PRIO(3));
3432 sq_config |= (VC_ENABLE |
3433 EXPORT_SRC_C |
3434 PS_PRIO(0) |
3435 VS_PRIO(1) |
3436 GS_PRIO(2) |
3437 ES_PRIO(3));
3438
3439 switch (rdev->family) {
3440 case CHIP_CEDAR:
3441 case CHIP_PALM:
3442 case CHIP_SUMO:
3443 case CHIP_SUMO2:
3444 case CHIP_CAICOS:
3445 /* no vertex cache */
3446 sq_config &= ~VC_ENABLE;
3447 break;
3448 default:
3449 break;
3450 }
3451
3452 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3453
3454 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3455 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3456 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3457 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3458 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3459 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3460 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3461
3462 switch (rdev->family) {
3463 case CHIP_CEDAR:
3464 case CHIP_PALM:
3465 case CHIP_SUMO:
3466 case CHIP_SUMO2:
3467 ps_thread_count = 96;
3468 break;
3469 default:
3470 ps_thread_count = 128;
3471 break;
3472 }
3473
3474 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3475 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3476 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3477 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3478 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3479 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3480
3481 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3482 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3483 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3484 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3485 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3486 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3487
3488 WREG32(SQ_CONFIG, sq_config);
3489 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3490 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3491 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3492 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3493 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3494 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3495 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3496 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3497 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3498 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3499
3500 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3501 FORCE_EOV_MAX_REZ_CNT(255)));
3502
3503 switch (rdev->family) {
3504 case CHIP_CEDAR:
3505 case CHIP_PALM:
3506 case CHIP_SUMO:
3507 case CHIP_SUMO2:
3508 case CHIP_CAICOS:
3509 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3510 break;
3511 default:
3512 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3513 break;
3514 }
3515 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3516 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3517
3518 WREG32(VGT_GS_VERTEX_REUSE, 16);
3519 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3520 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3521
3522 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3523 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3524
3525 WREG32(CB_PERF_CTR0_SEL_0, 0);
3526 WREG32(CB_PERF_CTR0_SEL_1, 0);
3527 WREG32(CB_PERF_CTR1_SEL_0, 0);
3528 WREG32(CB_PERF_CTR1_SEL_1, 0);
3529 WREG32(CB_PERF_CTR2_SEL_0, 0);
3530 WREG32(CB_PERF_CTR2_SEL_1, 0);
3531 WREG32(CB_PERF_CTR3_SEL_0, 0);
3532 WREG32(CB_PERF_CTR3_SEL_1, 0);
3533
3534 /* clear render buffer base addresses */
3535 WREG32(CB_COLOR0_BASE, 0);
3536 WREG32(CB_COLOR1_BASE, 0);
3537 WREG32(CB_COLOR2_BASE, 0);
3538 WREG32(CB_COLOR3_BASE, 0);
3539 WREG32(CB_COLOR4_BASE, 0);
3540 WREG32(CB_COLOR5_BASE, 0);
3541 WREG32(CB_COLOR6_BASE, 0);
3542 WREG32(CB_COLOR7_BASE, 0);
3543 WREG32(CB_COLOR8_BASE, 0);
3544 WREG32(CB_COLOR9_BASE, 0);
3545 WREG32(CB_COLOR10_BASE, 0);
3546 WREG32(CB_COLOR11_BASE, 0);
3547
3548 /* set the shader const cache sizes to 0 */
3549 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3550 WREG32(i, 0);
3551 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3552 WREG32(i, 0);
3553
3554 tmp = RREG32(HDP_MISC_CNTL);
3555 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3556 WREG32(HDP_MISC_CNTL, tmp);
3557
3558 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3559 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3560
3561 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3562
3563 udelay(50);
3564
3565 }
3566
3567 int evergreen_mc_init(struct radeon_device *rdev)
3568 {
3569 u32 tmp;
3570 int chansize, numchan;
3571
3572 /* Get VRAM informations */
3573 rdev->mc.vram_is_ddr = true;
3574 if ((rdev->family == CHIP_PALM) ||
3575 (rdev->family == CHIP_SUMO) ||
3576 (rdev->family == CHIP_SUMO2))
3577 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3578 else
3579 tmp = RREG32(MC_ARB_RAMCFG);
3580 if (tmp & CHANSIZE_OVERRIDE) {
3581 chansize = 16;
3582 } else if (tmp & CHANSIZE_MASK) {
3583 chansize = 64;
3584 } else {
3585 chansize = 32;
3586 }
3587 tmp = RREG32(MC_SHARED_CHMAP);
3588 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3589 case 0:
3590 default:
3591 numchan = 1;
3592 break;
3593 case 1:
3594 numchan = 2;
3595 break;
3596 case 2:
3597 numchan = 4;
3598 break;
3599 case 3:
3600 numchan = 8;
3601 break;
3602 }
3603 rdev->mc.vram_width = numchan * chansize;
3604 /* Could aper size report 0 ? */
3605 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3606 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3607 /* Setup GPU memory space */
3608 if ((rdev->family == CHIP_PALM) ||
3609 (rdev->family == CHIP_SUMO) ||
3610 (rdev->family == CHIP_SUMO2)) {
3611 /* size in bytes on fusion */
3612 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3613 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3614 } else {
3615 /* size in MB on evergreen/cayman/tn */
3616 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3617 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3618 }
3619 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3620 r700_vram_gtt_location(rdev, &rdev->mc);
3621 radeon_update_bandwidth_info(rdev);
3622
3623 return 0;
3624 }
3625
3626 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3627 {
3628 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3629 RREG32(GRBM_STATUS));
3630 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3631 RREG32(GRBM_STATUS_SE0));
3632 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3633 RREG32(GRBM_STATUS_SE1));
3634 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3635 RREG32(SRBM_STATUS));
3636 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3637 RREG32(SRBM_STATUS2));
3638 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3639 RREG32(CP_STALLED_STAT1));
3640 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3641 RREG32(CP_STALLED_STAT2));
3642 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3643 RREG32(CP_BUSY_STAT));
3644 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3645 RREG32(CP_STAT));
3646 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3647 RREG32(DMA_STATUS_REG));
3648 if (rdev->family >= CHIP_CAYMAN) {
3649 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3650 RREG32(DMA_STATUS_REG + 0x800));
3651 }
3652 }
3653
3654 bool evergreen_is_display_hung(struct radeon_device *rdev)
3655 {
3656 u32 crtc_hung = 0;
3657 u32 crtc_status[6];
3658 u32 i, j, tmp;
3659
3660 for (i = 0; i < rdev->num_crtc; i++) {
3661 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3662 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3663 crtc_hung |= (1 << i);
3664 }
3665 }
3666
3667 for (j = 0; j < 10; j++) {
3668 for (i = 0; i < rdev->num_crtc; i++) {
3669 if (crtc_hung & (1 << i)) {
3670 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3671 if (tmp != crtc_status[i])
3672 crtc_hung &= ~(1 << i);
3673 }
3674 }
3675 if (crtc_hung == 0)
3676 return false;
3677 udelay(100);
3678 }
3679
3680 return true;
3681 }
3682
3683 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3684 {
3685 u32 reset_mask = 0;
3686 u32 tmp;
3687
3688 /* GRBM_STATUS */
3689 tmp = RREG32(GRBM_STATUS);
3690 if (tmp & (PA_BUSY | SC_BUSY |
3691 SH_BUSY | SX_BUSY |
3692 TA_BUSY | VGT_BUSY |
3693 DB_BUSY | CB_BUSY |
3694 SPI_BUSY | VGT_BUSY_NO_DMA))
3695 reset_mask |= RADEON_RESET_GFX;
3696
3697 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3698 CP_BUSY | CP_COHERENCY_BUSY))
3699 reset_mask |= RADEON_RESET_CP;
3700
3701 if (tmp & GRBM_EE_BUSY)
3702 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3703
3704 /* DMA_STATUS_REG */
3705 tmp = RREG32(DMA_STATUS_REG);
3706 if (!(tmp & DMA_IDLE))
3707 reset_mask |= RADEON_RESET_DMA;
3708
3709 /* SRBM_STATUS2 */
3710 tmp = RREG32(SRBM_STATUS2);
3711 if (tmp & DMA_BUSY)
3712 reset_mask |= RADEON_RESET_DMA;
3713
3714 /* SRBM_STATUS */
3715 tmp = RREG32(SRBM_STATUS);
3716 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3717 reset_mask |= RADEON_RESET_RLC;
3718
3719 if (tmp & IH_BUSY)
3720 reset_mask |= RADEON_RESET_IH;
3721
3722 if (tmp & SEM_BUSY)
3723 reset_mask |= RADEON_RESET_SEM;
3724
3725 if (tmp & GRBM_RQ_PENDING)
3726 reset_mask |= RADEON_RESET_GRBM;
3727
3728 if (tmp & VMC_BUSY)
3729 reset_mask |= RADEON_RESET_VMC;
3730
3731 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3732 MCC_BUSY | MCD_BUSY))
3733 reset_mask |= RADEON_RESET_MC;
3734
3735 if (evergreen_is_display_hung(rdev))
3736 reset_mask |= RADEON_RESET_DISPLAY;
3737
3738 /* VM_L2_STATUS */
3739 tmp = RREG32(VM_L2_STATUS);
3740 if (tmp & L2_BUSY)
3741 reset_mask |= RADEON_RESET_VMC;
3742
3743 /* Skip MC reset as it's mostly likely not hung, just busy */
3744 if (reset_mask & RADEON_RESET_MC) {
3745 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3746 reset_mask &= ~RADEON_RESET_MC;
3747 }
3748
3749 return reset_mask;
3750 }
3751
3752 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3753 {
3754 struct evergreen_mc_save save;
3755 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3756 u32 tmp;
3757
3758 if (reset_mask == 0)
3759 return;
3760
3761 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3762
3763 evergreen_print_gpu_status_regs(rdev);
3764
3765 /* Disable CP parsing/prefetching */
3766 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3767
3768 if (reset_mask & RADEON_RESET_DMA) {
3769 /* Disable DMA */
3770 tmp = RREG32(DMA_RB_CNTL);
3771 tmp &= ~DMA_RB_ENABLE;
3772 WREG32(DMA_RB_CNTL, tmp);
3773 }
3774
3775 udelay(50);
3776
3777 evergreen_mc_stop(rdev, &save);
3778 if (evergreen_mc_wait_for_idle(rdev)) {
3779 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3780 }
3781
3782 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3783 grbm_soft_reset |= SOFT_RESET_DB |
3784 SOFT_RESET_CB |
3785 SOFT_RESET_PA |
3786 SOFT_RESET_SC |
3787 SOFT_RESET_SPI |
3788 SOFT_RESET_SX |
3789 SOFT_RESET_SH |
3790 SOFT_RESET_TC |
3791 SOFT_RESET_TA |
3792 SOFT_RESET_VC |
3793 SOFT_RESET_VGT;
3794 }
3795
3796 if (reset_mask & RADEON_RESET_CP) {
3797 grbm_soft_reset |= SOFT_RESET_CP |
3798 SOFT_RESET_VGT;
3799
3800 srbm_soft_reset |= SOFT_RESET_GRBM;
3801 }
3802
3803 if (reset_mask & RADEON_RESET_DMA)
3804 srbm_soft_reset |= SOFT_RESET_DMA;
3805
3806 if (reset_mask & RADEON_RESET_DISPLAY)
3807 srbm_soft_reset |= SOFT_RESET_DC;
3808
3809 if (reset_mask & RADEON_RESET_RLC)
3810 srbm_soft_reset |= SOFT_RESET_RLC;
3811
3812 if (reset_mask & RADEON_RESET_SEM)
3813 srbm_soft_reset |= SOFT_RESET_SEM;
3814
3815 if (reset_mask & RADEON_RESET_IH)
3816 srbm_soft_reset |= SOFT_RESET_IH;
3817
3818 if (reset_mask & RADEON_RESET_GRBM)
3819 srbm_soft_reset |= SOFT_RESET_GRBM;
3820
3821 if (reset_mask & RADEON_RESET_VMC)
3822 srbm_soft_reset |= SOFT_RESET_VMC;
3823
3824 if (!(rdev->flags & RADEON_IS_IGP)) {
3825 if (reset_mask & RADEON_RESET_MC)
3826 srbm_soft_reset |= SOFT_RESET_MC;
3827 }
3828
3829 if (grbm_soft_reset) {
3830 tmp = RREG32(GRBM_SOFT_RESET);
3831 tmp |= grbm_soft_reset;
3832 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3833 WREG32(GRBM_SOFT_RESET, tmp);
3834 tmp = RREG32(GRBM_SOFT_RESET);
3835
3836 udelay(50);
3837
3838 tmp &= ~grbm_soft_reset;
3839 WREG32(GRBM_SOFT_RESET, tmp);
3840 tmp = RREG32(GRBM_SOFT_RESET);
3841 }
3842
3843 if (srbm_soft_reset) {
3844 tmp = RREG32(SRBM_SOFT_RESET);
3845 tmp |= srbm_soft_reset;
3846 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3847 WREG32(SRBM_SOFT_RESET, tmp);
3848 tmp = RREG32(SRBM_SOFT_RESET);
3849
3850 udelay(50);
3851
3852 tmp &= ~srbm_soft_reset;
3853 WREG32(SRBM_SOFT_RESET, tmp);
3854 tmp = RREG32(SRBM_SOFT_RESET);
3855 }
3856
3857 /* Wait a little for things to settle down */
3858 udelay(50);
3859
3860 evergreen_mc_resume(rdev, &save);
3861 udelay(50);
3862
3863 evergreen_print_gpu_status_regs(rdev);
3864 }
3865
3866 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
3867 {
3868 struct evergreen_mc_save save;
3869 u32 tmp, i;
3870
3871 dev_info(rdev->dev, "GPU pci config reset\n");
3872
3873 /* disable dpm? */
3874
3875 /* Disable CP parsing/prefetching */
3876 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3877 udelay(50);
3878 /* Disable DMA */
3879 tmp = RREG32(DMA_RB_CNTL);
3880 tmp &= ~DMA_RB_ENABLE;
3881 WREG32(DMA_RB_CNTL, tmp);
3882 /* XXX other engines? */
3883
3884 /* halt the rlc */
3885 r600_rlc_stop(rdev);
3886
3887 udelay(50);
3888
3889 /* set mclk/sclk to bypass */
3890 rv770_set_clk_bypass_mode(rdev);
3891 /* disable BM */
3892 pci_clear_master(rdev->pdev);
3893 /* disable mem access */
3894 evergreen_mc_stop(rdev, &save);
3895 if (evergreen_mc_wait_for_idle(rdev)) {
3896 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3897 }
3898 /* reset */
3899 radeon_pci_config_reset(rdev);
3900 /* wait for asic to come out of reset */
3901 for (i = 0; i < rdev->usec_timeout; i++) {
3902 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3903 break;
3904 udelay(1);
3905 }
3906 }
3907
3908 int evergreen_asic_reset(struct radeon_device *rdev)
3909 {
3910 u32 reset_mask;
3911
3912 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3913
3914 if (reset_mask)
3915 r600_set_bios_scratch_engine_hung(rdev, true);
3916
3917 /* try soft reset */
3918 evergreen_gpu_soft_reset(rdev, reset_mask);
3919
3920 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3921
3922 /* try pci config reset */
3923 if (reset_mask && radeon_hard_reset)
3924 evergreen_gpu_pci_config_reset(rdev);
3925
3926 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3927
3928 if (!reset_mask)
3929 r600_set_bios_scratch_engine_hung(rdev, false);
3930
3931 return 0;
3932 }
3933
3934 /**
3935 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3936 *
3937 * @rdev: radeon_device pointer
3938 * @ring: radeon_ring structure holding ring information
3939 *
3940 * Check if the GFX engine is locked up.
3941 * Returns true if the engine appears to be locked up, false if not.
3942 */
3943 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3944 {
3945 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3946
3947 if (!(reset_mask & (RADEON_RESET_GFX |
3948 RADEON_RESET_COMPUTE |
3949 RADEON_RESET_CP))) {
3950 radeon_ring_lockup_update(rdev, ring);
3951 return false;
3952 }
3953 return radeon_ring_test_lockup(rdev, ring);
3954 }
3955
3956 /*
3957 * RLC
3958 */
3959 #define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3960 #define RLC_CLEAR_STATE_END_MARKER 0x00000001
3961
3962 void sumo_rlc_fini(struct radeon_device *rdev)
3963 {
3964 int r;
3965
3966 /* save restore block */
3967 if (rdev->rlc.save_restore_obj) {
3968 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3969 if (unlikely(r != 0))
3970 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3971 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3972 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3973
3974 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3975 rdev->rlc.save_restore_obj = NULL;
3976 }
3977
3978 /* clear state block */
3979 if (rdev->rlc.clear_state_obj) {
3980 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3981 if (unlikely(r != 0))
3982 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3983 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3984 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3985
3986 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3987 rdev->rlc.clear_state_obj = NULL;
3988 }
3989
3990 /* clear state block */
3991 if (rdev->rlc.cp_table_obj) {
3992 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
3993 if (unlikely(r != 0))
3994 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
3995 radeon_bo_unpin(rdev->rlc.cp_table_obj);
3996 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
3997
3998 radeon_bo_unref(&rdev->rlc.cp_table_obj);
3999 rdev->rlc.cp_table_obj = NULL;
4000 }
4001 }
4002
4003 #define CP_ME_TABLE_SIZE 96
4004
4005 int sumo_rlc_init(struct radeon_device *rdev)
4006 {
4007 const u32 *src_ptr;
4008 volatile u32 *dst_ptr;
4009 u32 dws, data, i, j, k, reg_num;
4010 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4011 u64 reg_list_mc_addr;
4012 const struct cs_section_def *cs_data;
4013 int r;
4014
4015 src_ptr = rdev->rlc.reg_list;
4016 dws = rdev->rlc.reg_list_size;
4017 if (rdev->family >= CHIP_BONAIRE) {
4018 dws += (5 * 16) + 48 + 48 + 64;
4019 }
4020 cs_data = rdev->rlc.cs_data;
4021
4022 if (src_ptr) {
4023 /* save restore block */
4024 if (rdev->rlc.save_restore_obj == NULL) {
4025 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4026 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4027 NULL, &rdev->rlc.save_restore_obj);
4028 if (r) {
4029 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4030 return r;
4031 }
4032 }
4033
4034 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4035 if (unlikely(r != 0)) {
4036 sumo_rlc_fini(rdev);
4037 return r;
4038 }
4039 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4040 &rdev->rlc.save_restore_gpu_addr);
4041 if (r) {
4042 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4043 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4044 sumo_rlc_fini(rdev);
4045 return r;
4046 }
4047
4048 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4049 if (r) {
4050 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4051 sumo_rlc_fini(rdev);
4052 return r;
4053 }
4054 /* write the sr buffer */
4055 dst_ptr = rdev->rlc.sr_ptr;
4056 if (rdev->family >= CHIP_TAHITI) {
4057 /* SI */
4058 for (i = 0; i < rdev->rlc.reg_list_size; i++)
4059 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4060 } else {
4061 /* ON/LN/TN */
4062 /* format:
4063 * dw0: (reg2 << 16) | reg1
4064 * dw1: reg1 save space
4065 * dw2: reg2 save space
4066 */
4067 for (i = 0; i < dws; i++) {
4068 data = src_ptr[i] >> 2;
4069 i++;
4070 if (i < dws)
4071 data |= (src_ptr[i] >> 2) << 16;
4072 j = (((i - 1) * 3) / 2);
4073 dst_ptr[j] = cpu_to_le32(data);
4074 }
4075 j = ((i * 3) / 2);
4076 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4077 }
4078 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4079 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4080 }
4081
4082 if (cs_data) {
4083 /* clear state block */
4084 if (rdev->family >= CHIP_BONAIRE) {
4085 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4086 } else if (rdev->family >= CHIP_TAHITI) {
4087 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4088 dws = rdev->rlc.clear_state_size + (256 / 4);
4089 } else {
4090 reg_list_num = 0;
4091 dws = 0;
4092 for (i = 0; cs_data[i].section != NULL; i++) {
4093 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4094 reg_list_num++;
4095 dws += cs_data[i].section[j].reg_count;
4096 }
4097 }
4098 reg_list_blk_index = (3 * reg_list_num + 2);
4099 dws += reg_list_blk_index;
4100 rdev->rlc.clear_state_size = dws;
4101 }
4102
4103 if (rdev->rlc.clear_state_obj == NULL) {
4104 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4105 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4106 NULL, &rdev->rlc.clear_state_obj);
4107 if (r) {
4108 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4109 sumo_rlc_fini(rdev);
4110 return r;
4111 }
4112 }
4113 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4114 if (unlikely(r != 0)) {
4115 sumo_rlc_fini(rdev);
4116 return r;
4117 }
4118 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4119 &rdev->rlc.clear_state_gpu_addr);
4120 if (r) {
4121 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4122 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4123 sumo_rlc_fini(rdev);
4124 return r;
4125 }
4126
4127 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4128 if (r) {
4129 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4130 sumo_rlc_fini(rdev);
4131 return r;
4132 }
4133 /* set up the cs buffer */
4134 dst_ptr = rdev->rlc.cs_ptr;
4135 if (rdev->family >= CHIP_BONAIRE) {
4136 cik_get_csb_buffer(rdev, dst_ptr);
4137 } else if (rdev->family >= CHIP_TAHITI) {
4138 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4139 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4140 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4141 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4142 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4143 } else {
4144 reg_list_hdr_blk_index = 0;
4145 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4146 data = upper_32_bits(reg_list_mc_addr);
4147 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4148 reg_list_hdr_blk_index++;
4149 for (i = 0; cs_data[i].section != NULL; i++) {
4150 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4151 reg_num = cs_data[i].section[j].reg_count;
4152 data = reg_list_mc_addr & 0xffffffff;
4153 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4154 reg_list_hdr_blk_index++;
4155
4156 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4157 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4158 reg_list_hdr_blk_index++;
4159
4160 data = 0x08000000 | (reg_num * 4);
4161 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4162 reg_list_hdr_blk_index++;
4163
4164 for (k = 0; k < reg_num; k++) {
4165 data = cs_data[i].section[j].extent[k];
4166 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4167 }
4168 reg_list_mc_addr += reg_num * 4;
4169 reg_list_blk_index += reg_num;
4170 }
4171 }
4172 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4173 }
4174 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4175 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4176 }
4177
4178 if (rdev->rlc.cp_table_size) {
4179 if (rdev->rlc.cp_table_obj == NULL) {
4180 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4181 PAGE_SIZE, true,
4182 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4183 NULL, &rdev->rlc.cp_table_obj);
4184 if (r) {
4185 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4186 sumo_rlc_fini(rdev);
4187 return r;
4188 }
4189 }
4190
4191 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4192 if (unlikely(r != 0)) {
4193 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4194 sumo_rlc_fini(rdev);
4195 return r;
4196 }
4197 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4198 &rdev->rlc.cp_table_gpu_addr);
4199 if (r) {
4200 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4201 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4202 sumo_rlc_fini(rdev);
4203 return r;
4204 }
4205 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4206 if (r) {
4207 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4208 sumo_rlc_fini(rdev);
4209 return r;
4210 }
4211
4212 cik_init_cp_pg_table(rdev);
4213
4214 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4215 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4216
4217 }
4218
4219 return 0;
4220 }
4221
4222 static void evergreen_rlc_start(struct radeon_device *rdev)
4223 {
4224 u32 mask = RLC_ENABLE;
4225
4226 if (rdev->flags & RADEON_IS_IGP) {
4227 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4228 }
4229
4230 WREG32(RLC_CNTL, mask);
4231 }
4232
4233 int evergreen_rlc_resume(struct radeon_device *rdev)
4234 {
4235 u32 i;
4236 const __be32 *fw_data;
4237
4238 if (!rdev->rlc_fw)
4239 return -EINVAL;
4240
4241 r600_rlc_stop(rdev);
4242
4243 WREG32(RLC_HB_CNTL, 0);
4244
4245 if (rdev->flags & RADEON_IS_IGP) {
4246 if (rdev->family == CHIP_ARUBA) {
4247 u32 always_on_bitmap =
4248 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4249 /* find out the number of active simds */
4250 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4251 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4252 tmp = hweight32(~tmp);
4253 if (tmp == rdev->config.cayman.max_simds_per_se) {
4254 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4255 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4256 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4257 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4258 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4259 }
4260 } else {
4261 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4262 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4263 }
4264 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4265 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4266 } else {
4267 WREG32(RLC_HB_BASE, 0);
4268 WREG32(RLC_HB_RPTR, 0);
4269 WREG32(RLC_HB_WPTR, 0);
4270 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4271 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4272 }
4273 WREG32(RLC_MC_CNTL, 0);
4274 WREG32(RLC_UCODE_CNTL, 0);
4275
4276 fw_data = (const __be32 *)rdev->rlc_fw->data;
4277 if (rdev->family >= CHIP_ARUBA) {
4278 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4279 WREG32(RLC_UCODE_ADDR, i);
4280 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4281 }
4282 } else if (rdev->family >= CHIP_CAYMAN) {
4283 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4284 WREG32(RLC_UCODE_ADDR, i);
4285 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4286 }
4287 } else {
4288 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4289 WREG32(RLC_UCODE_ADDR, i);
4290 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4291 }
4292 }
4293 WREG32(RLC_UCODE_ADDR, 0);
4294
4295 evergreen_rlc_start(rdev);
4296
4297 return 0;
4298 }
4299
4300 /* Interrupts */
4301
4302 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4303 {
4304 if (crtc >= rdev->num_crtc)
4305 return 0;
4306 else
4307 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4308 }
4309
4310 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4311 {
4312 u32 tmp;
4313
4314 if (rdev->family >= CHIP_CAYMAN) {
4315 cayman_cp_int_cntl_setup(rdev, 0,
4316 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4317 cayman_cp_int_cntl_setup(rdev, 1, 0);
4318 cayman_cp_int_cntl_setup(rdev, 2, 0);
4319 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4320 WREG32(CAYMAN_DMA1_CNTL, tmp);
4321 } else
4322 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4323 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4324 WREG32(DMA_CNTL, tmp);
4325 WREG32(GRBM_INT_CNTL, 0);
4326 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4327 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4328 if (rdev->num_crtc >= 4) {
4329 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4330 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4331 }
4332 if (rdev->num_crtc >= 6) {
4333 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4334 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4335 }
4336
4337 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4338 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4339 if (rdev->num_crtc >= 4) {
4340 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4341 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4342 }
4343 if (rdev->num_crtc >= 6) {
4344 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4345 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4346 }
4347
4348 /* only one DAC on DCE5 */
4349 if (!ASIC_IS_DCE5(rdev))
4350 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4351 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4352
4353 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4354 WREG32(DC_HPD1_INT_CONTROL, tmp);
4355 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4356 WREG32(DC_HPD2_INT_CONTROL, tmp);
4357 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4358 WREG32(DC_HPD3_INT_CONTROL, tmp);
4359 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4360 WREG32(DC_HPD4_INT_CONTROL, tmp);
4361 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4362 WREG32(DC_HPD5_INT_CONTROL, tmp);
4363 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4364 WREG32(DC_HPD6_INT_CONTROL, tmp);
4365
4366 }
4367
4368 int evergreen_irq_set(struct radeon_device *rdev)
4369 {
4370 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4371 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4372 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4373 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4374 u32 grbm_int_cntl = 0;
4375 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
4376 u32 dma_cntl, dma_cntl1 = 0;
4377 u32 thermal_int = 0;
4378
4379 if (!rdev->irq.installed) {
4380 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4381 return -EINVAL;
4382 }
4383 /* don't enable anything if the ih is disabled */
4384 if (!rdev->ih.enabled) {
4385 r600_disable_interrupts(rdev);
4386 /* force the active interrupt state to all disabled */
4387 evergreen_disable_interrupt_state(rdev);
4388 return 0;
4389 }
4390
4391 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4392 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4393 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4394 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4395 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4396 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4397 if (rdev->family == CHIP_ARUBA)
4398 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4399 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4400 else
4401 thermal_int = RREG32(CG_THERMAL_INT) &
4402 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4403
4404 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4405 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4406 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4407 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4408 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4409 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4410
4411 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4412
4413 if (rdev->family >= CHIP_CAYMAN) {
4414 /* enable CP interrupts on all rings */
4415 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4416 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4417 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4418 }
4419 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4420 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4421 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4422 }
4423 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4424 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4425 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4426 }
4427 } else {
4428 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4429 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4430 cp_int_cntl |= RB_INT_ENABLE;
4431 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4432 }
4433 }
4434
4435 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4436 DRM_DEBUG("r600_irq_set: sw int dma\n");
4437 dma_cntl |= TRAP_ENABLE;
4438 }
4439
4440 if (rdev->family >= CHIP_CAYMAN) {
4441 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4442 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4443 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4444 dma_cntl1 |= TRAP_ENABLE;
4445 }
4446 }
4447
4448 if (rdev->irq.dpm_thermal) {
4449 DRM_DEBUG("dpm thermal\n");
4450 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4451 }
4452
4453 if (rdev->irq.crtc_vblank_int[0] ||
4454 atomic_read(&rdev->irq.pflip[0])) {
4455 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4456 crtc1 |= VBLANK_INT_MASK;
4457 }
4458 if (rdev->irq.crtc_vblank_int[1] ||
4459 atomic_read(&rdev->irq.pflip[1])) {
4460 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4461 crtc2 |= VBLANK_INT_MASK;
4462 }
4463 if (rdev->irq.crtc_vblank_int[2] ||
4464 atomic_read(&rdev->irq.pflip[2])) {
4465 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4466 crtc3 |= VBLANK_INT_MASK;
4467 }
4468 if (rdev->irq.crtc_vblank_int[3] ||
4469 atomic_read(&rdev->irq.pflip[3])) {
4470 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4471 crtc4 |= VBLANK_INT_MASK;
4472 }
4473 if (rdev->irq.crtc_vblank_int[4] ||
4474 atomic_read(&rdev->irq.pflip[4])) {
4475 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4476 crtc5 |= VBLANK_INT_MASK;
4477 }
4478 if (rdev->irq.crtc_vblank_int[5] ||
4479 atomic_read(&rdev->irq.pflip[5])) {
4480 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4481 crtc6 |= VBLANK_INT_MASK;
4482 }
4483 if (rdev->irq.hpd[0]) {
4484 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4485 hpd1 |= DC_HPDx_INT_EN;
4486 }
4487 if (rdev->irq.hpd[1]) {
4488 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4489 hpd2 |= DC_HPDx_INT_EN;
4490 }
4491 if (rdev->irq.hpd[2]) {
4492 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4493 hpd3 |= DC_HPDx_INT_EN;
4494 }
4495 if (rdev->irq.hpd[3]) {
4496 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4497 hpd4 |= DC_HPDx_INT_EN;
4498 }
4499 if (rdev->irq.hpd[4]) {
4500 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4501 hpd5 |= DC_HPDx_INT_EN;
4502 }
4503 if (rdev->irq.hpd[5]) {
4504 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4505 hpd6 |= DC_HPDx_INT_EN;
4506 }
4507 if (rdev->irq.afmt[0]) {
4508 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4509 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4510 }
4511 if (rdev->irq.afmt[1]) {
4512 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4513 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4514 }
4515 if (rdev->irq.afmt[2]) {
4516 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4517 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4518 }
4519 if (rdev->irq.afmt[3]) {
4520 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4521 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4522 }
4523 if (rdev->irq.afmt[4]) {
4524 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4525 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4526 }
4527 if (rdev->irq.afmt[5]) {
4528 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4529 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4530 }
4531
4532 if (rdev->family >= CHIP_CAYMAN) {
4533 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4534 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4535 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4536 } else
4537 WREG32(CP_INT_CNTL, cp_int_cntl);
4538
4539 WREG32(DMA_CNTL, dma_cntl);
4540
4541 if (rdev->family >= CHIP_CAYMAN)
4542 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4543
4544 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4545
4546 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4547 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4548 if (rdev->num_crtc >= 4) {
4549 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4550 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4551 }
4552 if (rdev->num_crtc >= 6) {
4553 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4554 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4555 }
4556
4557 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
4558 GRPH_PFLIP_INT_MASK);
4559 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
4560 GRPH_PFLIP_INT_MASK);
4561 if (rdev->num_crtc >= 4) {
4562 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
4563 GRPH_PFLIP_INT_MASK);
4564 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
4565 GRPH_PFLIP_INT_MASK);
4566 }
4567 if (rdev->num_crtc >= 6) {
4568 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
4569 GRPH_PFLIP_INT_MASK);
4570 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
4571 GRPH_PFLIP_INT_MASK);
4572 }
4573
4574 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4575 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4576 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4577 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4578 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4579 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4580 if (rdev->family == CHIP_ARUBA)
4581 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4582 else
4583 WREG32(CG_THERMAL_INT, thermal_int);
4584
4585 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4586 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4587 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4588 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4589 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4590 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4591
4592 return 0;
4593 }
4594
4595 static void evergreen_irq_ack(struct radeon_device *rdev)
4596 {
4597 u32 tmp;
4598
4599 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4600 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4601 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4602 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4603 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4604 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4605 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4606 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4607 if (rdev->num_crtc >= 4) {
4608 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4609 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4610 }
4611 if (rdev->num_crtc >= 6) {
4612 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4613 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4614 }
4615
4616 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4617 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4618 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4619 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4620 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4621 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4622
4623 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4624 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4625 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4626 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4627 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4628 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4629 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4630 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4631 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4632 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4633 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4634 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4635
4636 if (rdev->num_crtc >= 4) {
4637 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4638 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4639 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4640 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4641 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4642 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4643 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4644 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4645 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4646 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4647 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4648 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4649 }
4650
4651 if (rdev->num_crtc >= 6) {
4652 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4653 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4654 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4655 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4656 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4657 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4658 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4659 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4660 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4661 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4662 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4663 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4664 }
4665
4666 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4667 tmp = RREG32(DC_HPD1_INT_CONTROL);
4668 tmp |= DC_HPDx_INT_ACK;
4669 WREG32(DC_HPD1_INT_CONTROL, tmp);
4670 }
4671 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4672 tmp = RREG32(DC_HPD2_INT_CONTROL);
4673 tmp |= DC_HPDx_INT_ACK;
4674 WREG32(DC_HPD2_INT_CONTROL, tmp);
4675 }
4676 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4677 tmp = RREG32(DC_HPD3_INT_CONTROL);
4678 tmp |= DC_HPDx_INT_ACK;
4679 WREG32(DC_HPD3_INT_CONTROL, tmp);
4680 }
4681 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4682 tmp = RREG32(DC_HPD4_INT_CONTROL);
4683 tmp |= DC_HPDx_INT_ACK;
4684 WREG32(DC_HPD4_INT_CONTROL, tmp);
4685 }
4686 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4687 tmp = RREG32(DC_HPD5_INT_CONTROL);
4688 tmp |= DC_HPDx_INT_ACK;
4689 WREG32(DC_HPD5_INT_CONTROL, tmp);
4690 }
4691 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4692 tmp = RREG32(DC_HPD5_INT_CONTROL);
4693 tmp |= DC_HPDx_INT_ACK;
4694 WREG32(DC_HPD6_INT_CONTROL, tmp);
4695 }
4696 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4697 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4698 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4699 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4700 }
4701 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4702 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4703 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4704 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4705 }
4706 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4707 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4708 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4709 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4710 }
4711 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4712 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4713 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4714 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4715 }
4716 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4717 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4718 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4719 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4720 }
4721 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4722 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4723 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4724 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4725 }
4726 }
4727
4728 static void evergreen_irq_disable(struct radeon_device *rdev)
4729 {
4730 r600_disable_interrupts(rdev);
4731 /* Wait and acknowledge irq */
4732 mdelay(1);
4733 evergreen_irq_ack(rdev);
4734 evergreen_disable_interrupt_state(rdev);
4735 }
4736
4737 void evergreen_irq_suspend(struct radeon_device *rdev)
4738 {
4739 evergreen_irq_disable(rdev);
4740 r600_rlc_stop(rdev);
4741 }
4742
4743 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4744 {
4745 u32 wptr, tmp;
4746
4747 if (rdev->wb.enabled)
4748 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4749 else
4750 wptr = RREG32(IH_RB_WPTR);
4751
4752 if (wptr & RB_OVERFLOW) {
4753 wptr &= ~RB_OVERFLOW;
4754 /* When a ring buffer overflow happen start parsing interrupt
4755 * from the last not overwritten vector (wptr + 16). Hopefully
4756 * this should allow us to catchup.
4757 */
4758 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4759 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4760 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4761 tmp = RREG32(IH_RB_CNTL);
4762 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4763 WREG32(IH_RB_CNTL, tmp);
4764 }
4765 return (wptr & rdev->ih.ptr_mask);
4766 }
4767
4768 int evergreen_irq_process(struct radeon_device *rdev)
4769 {
4770 u32 wptr;
4771 u32 rptr;
4772 u32 src_id, src_data;
4773 u32 ring_index;
4774 bool queue_hotplug = false;
4775 bool queue_hdmi = false;
4776 bool queue_thermal = false;
4777 u32 status, addr;
4778
4779 if (!rdev->ih.enabled || rdev->shutdown)
4780 return IRQ_NONE;
4781
4782 wptr = evergreen_get_ih_wptr(rdev);
4783
4784 restart_ih:
4785 /* is somebody else already processing irqs? */
4786 if (atomic_xchg(&rdev->ih.lock, 1))
4787 return IRQ_NONE;
4788
4789 rptr = rdev->ih.rptr;
4790 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4791
4792 /* Order reading of wptr vs. reading of IH ring data */
4793 rmb();
4794
4795 /* display interrupts */
4796 evergreen_irq_ack(rdev);
4797
4798 while (rptr != wptr) {
4799 /* wptr/rptr are in bytes! */
4800 ring_index = rptr / 4;
4801 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4802 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4803
4804 switch (src_id) {
4805 case 1: /* D1 vblank/vline */
4806 switch (src_data) {
4807 case 0: /* D1 vblank */
4808 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4809 if (rdev->irq.crtc_vblank_int[0]) {
4810 drm_handle_vblank(rdev->ddev, 0);
4811 rdev->pm.vblank_sync = true;
4812 wake_up(&rdev->irq.vblank_queue);
4813 }
4814 if (atomic_read(&rdev->irq.pflip[0]))
4815 radeon_crtc_handle_vblank(rdev, 0);
4816 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4817 DRM_DEBUG("IH: D1 vblank\n");
4818 }
4819 break;
4820 case 1: /* D1 vline */
4821 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4822 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4823 DRM_DEBUG("IH: D1 vline\n");
4824 }
4825 break;
4826 default:
4827 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4828 break;
4829 }
4830 break;
4831 case 2: /* D2 vblank/vline */
4832 switch (src_data) {
4833 case 0: /* D2 vblank */
4834 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4835 if (rdev->irq.crtc_vblank_int[1]) {
4836 drm_handle_vblank(rdev->ddev, 1);
4837 rdev->pm.vblank_sync = true;
4838 wake_up(&rdev->irq.vblank_queue);
4839 }
4840 if (atomic_read(&rdev->irq.pflip[1]))
4841 radeon_crtc_handle_vblank(rdev, 1);
4842 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4843 DRM_DEBUG("IH: D2 vblank\n");
4844 }
4845 break;
4846 case 1: /* D2 vline */
4847 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4848 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4849 DRM_DEBUG("IH: D2 vline\n");
4850 }
4851 break;
4852 default:
4853 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4854 break;
4855 }
4856 break;
4857 case 3: /* D3 vblank/vline */
4858 switch (src_data) {
4859 case 0: /* D3 vblank */
4860 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4861 if (rdev->irq.crtc_vblank_int[2]) {
4862 drm_handle_vblank(rdev->ddev, 2);
4863 rdev->pm.vblank_sync = true;
4864 wake_up(&rdev->irq.vblank_queue);
4865 }
4866 if (atomic_read(&rdev->irq.pflip[2]))
4867 radeon_crtc_handle_vblank(rdev, 2);
4868 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4869 DRM_DEBUG("IH: D3 vblank\n");
4870 }
4871 break;
4872 case 1: /* D3 vline */
4873 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4874 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
4875 DRM_DEBUG("IH: D3 vline\n");
4876 }
4877 break;
4878 default:
4879 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4880 break;
4881 }
4882 break;
4883 case 4: /* D4 vblank/vline */
4884 switch (src_data) {
4885 case 0: /* D4 vblank */
4886 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4887 if (rdev->irq.crtc_vblank_int[3]) {
4888 drm_handle_vblank(rdev->ddev, 3);
4889 rdev->pm.vblank_sync = true;
4890 wake_up(&rdev->irq.vblank_queue);
4891 }
4892 if (atomic_read(&rdev->irq.pflip[3]))
4893 radeon_crtc_handle_vblank(rdev, 3);
4894 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
4895 DRM_DEBUG("IH: D4 vblank\n");
4896 }
4897 break;
4898 case 1: /* D4 vline */
4899 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4900 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
4901 DRM_DEBUG("IH: D4 vline\n");
4902 }
4903 break;
4904 default:
4905 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4906 break;
4907 }
4908 break;
4909 case 5: /* D5 vblank/vline */
4910 switch (src_data) {
4911 case 0: /* D5 vblank */
4912 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4913 if (rdev->irq.crtc_vblank_int[4]) {
4914 drm_handle_vblank(rdev->ddev, 4);
4915 rdev->pm.vblank_sync = true;
4916 wake_up(&rdev->irq.vblank_queue);
4917 }
4918 if (atomic_read(&rdev->irq.pflip[4]))
4919 radeon_crtc_handle_vblank(rdev, 4);
4920 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
4921 DRM_DEBUG("IH: D5 vblank\n");
4922 }
4923 break;
4924 case 1: /* D5 vline */
4925 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4926 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
4927 DRM_DEBUG("IH: D5 vline\n");
4928 }
4929 break;
4930 default:
4931 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4932 break;
4933 }
4934 break;
4935 case 6: /* D6 vblank/vline */
4936 switch (src_data) {
4937 case 0: /* D6 vblank */
4938 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4939 if (rdev->irq.crtc_vblank_int[5]) {
4940 drm_handle_vblank(rdev->ddev, 5);
4941 rdev->pm.vblank_sync = true;
4942 wake_up(&rdev->irq.vblank_queue);
4943 }
4944 if (atomic_read(&rdev->irq.pflip[5]))
4945 radeon_crtc_handle_vblank(rdev, 5);
4946 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
4947 DRM_DEBUG("IH: D6 vblank\n");
4948 }
4949 break;
4950 case 1: /* D6 vline */
4951 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4952 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
4953 DRM_DEBUG("IH: D6 vline\n");
4954 }
4955 break;
4956 default:
4957 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4958 break;
4959 }
4960 break;
4961 case 8: /* D1 page flip */
4962 case 10: /* D2 page flip */
4963 case 12: /* D3 page flip */
4964 case 14: /* D4 page flip */
4965 case 16: /* D5 page flip */
4966 case 18: /* D6 page flip */
4967 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4968 if (radeon_use_pflipirq > 0)
4969 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4970 break;
4971 case 42: /* HPD hotplug */
4972 switch (src_data) {
4973 case 0:
4974 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4975 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
4976 queue_hotplug = true;
4977 DRM_DEBUG("IH: HPD1\n");
4978 }
4979 break;
4980 case 1:
4981 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4982 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
4983 queue_hotplug = true;
4984 DRM_DEBUG("IH: HPD2\n");
4985 }
4986 break;
4987 case 2:
4988 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4989 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
4990 queue_hotplug = true;
4991 DRM_DEBUG("IH: HPD3\n");
4992 }
4993 break;
4994 case 3:
4995 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4996 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
4997 queue_hotplug = true;
4998 DRM_DEBUG("IH: HPD4\n");
4999 }
5000 break;
5001 case 4:
5002 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5003 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5004 queue_hotplug = true;
5005 DRM_DEBUG("IH: HPD5\n");
5006 }
5007 break;
5008 case 5:
5009 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5010 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5011 queue_hotplug = true;
5012 DRM_DEBUG("IH: HPD6\n");
5013 }
5014 break;
5015 default:
5016 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5017 break;
5018 }
5019 break;
5020 case 44: /* hdmi */
5021 switch (src_data) {
5022 case 0:
5023 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
5024 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
5025 queue_hdmi = true;
5026 DRM_DEBUG("IH: HDMI0\n");
5027 }
5028 break;
5029 case 1:
5030 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
5031 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
5032 queue_hdmi = true;
5033 DRM_DEBUG("IH: HDMI1\n");
5034 }
5035 break;
5036 case 2:
5037 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
5038 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
5039 queue_hdmi = true;
5040 DRM_DEBUG("IH: HDMI2\n");
5041 }
5042 break;
5043 case 3:
5044 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
5045 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
5046 queue_hdmi = true;
5047 DRM_DEBUG("IH: HDMI3\n");
5048 }
5049 break;
5050 case 4:
5051 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
5052 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
5053 queue_hdmi = true;
5054 DRM_DEBUG("IH: HDMI4\n");
5055 }
5056 break;
5057 case 5:
5058 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
5059 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5060 queue_hdmi = true;
5061 DRM_DEBUG("IH: HDMI5\n");
5062 }
5063 break;
5064 default:
5065 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5066 break;
5067 }
5068 case 124: /* UVD */
5069 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5070 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
5071 break;
5072 case 146:
5073 case 147:
5074 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5075 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
5076 /* reset addr and status */
5077 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5078 if (addr == 0x0 && status == 0x0)
5079 break;
5080 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5081 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5082 addr);
5083 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5084 status);
5085 cayman_vm_decode_fault(rdev, status, addr);
5086 break;
5087 case 176: /* CP_INT in ring buffer */
5088 case 177: /* CP_INT in IB1 */
5089 case 178: /* CP_INT in IB2 */
5090 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
5091 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5092 break;
5093 case 181: /* CP EOP event */
5094 DRM_DEBUG("IH: CP EOP\n");
5095 if (rdev->family >= CHIP_CAYMAN) {
5096 switch (src_data) {
5097 case 0:
5098 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5099 break;
5100 case 1:
5101 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5102 break;
5103 case 2:
5104 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5105 break;
5106 }
5107 } else
5108 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5109 break;
5110 case 224: /* DMA trap event */
5111 DRM_DEBUG("IH: DMA trap\n");
5112 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5113 break;
5114 case 230: /* thermal low to high */
5115 DRM_DEBUG("IH: thermal low to high\n");
5116 rdev->pm.dpm.thermal.high_to_low = false;
5117 queue_thermal = true;
5118 break;
5119 case 231: /* thermal high to low */
5120 DRM_DEBUG("IH: thermal high to low\n");
5121 rdev->pm.dpm.thermal.high_to_low = true;
5122 queue_thermal = true;
5123 break;
5124 case 233: /* GUI IDLE */
5125 DRM_DEBUG("IH: GUI idle\n");
5126 break;
5127 case 244: /* DMA trap event */
5128 if (rdev->family >= CHIP_CAYMAN) {
5129 DRM_DEBUG("IH: DMA1 trap\n");
5130 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5131 }
5132 break;
5133 default:
5134 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5135 break;
5136 }
5137
5138 /* wptr/rptr are in bytes! */
5139 rptr += 16;
5140 rptr &= rdev->ih.ptr_mask;
5141 WREG32(IH_RB_RPTR, rptr);
5142 }
5143 if (queue_hotplug)
5144 schedule_work(&rdev->hotplug_work);
5145 if (queue_hdmi)
5146 schedule_work(&rdev->audio_work);
5147 if (queue_thermal && rdev->pm.dpm_enabled)
5148 schedule_work(&rdev->pm.dpm.thermal.work);
5149 rdev->ih.rptr = rptr;
5150 atomic_set(&rdev->ih.lock, 0);
5151
5152 /* make sure wptr hasn't changed while processing */
5153 wptr = evergreen_get_ih_wptr(rdev);
5154 if (wptr != rptr)
5155 goto restart_ih;
5156
5157 return IRQ_HANDLED;
5158 }
5159
5160 static int evergreen_startup(struct radeon_device *rdev)
5161 {
5162 struct radeon_ring *ring;
5163 int r;
5164
5165 /* enable pcie gen2 link */
5166 evergreen_pcie_gen2_enable(rdev);
5167 /* enable aspm */
5168 evergreen_program_aspm(rdev);
5169
5170 /* scratch needs to be initialized before MC */
5171 r = r600_vram_scratch_init(rdev);
5172 if (r)
5173 return r;
5174
5175 evergreen_mc_program(rdev);
5176
5177 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5178 r = ni_mc_load_microcode(rdev);
5179 if (r) {
5180 DRM_ERROR("Failed to load MC firmware!\n");
5181 return r;
5182 }
5183 }
5184
5185 if (rdev->flags & RADEON_IS_AGP) {
5186 evergreen_agp_enable(rdev);
5187 } else {
5188 r = evergreen_pcie_gart_enable(rdev);
5189 if (r)
5190 return r;
5191 }
5192 evergreen_gpu_init(rdev);
5193
5194 /* allocate rlc buffers */
5195 if (rdev->flags & RADEON_IS_IGP) {
5196 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5197 rdev->rlc.reg_list_size =
5198 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5199 rdev->rlc.cs_data = evergreen_cs_data;
5200 r = sumo_rlc_init(rdev);
5201 if (r) {
5202 DRM_ERROR("Failed to init rlc BOs!\n");
5203 return r;
5204 }
5205 }
5206
5207 /* allocate wb buffer */
5208 r = radeon_wb_init(rdev);
5209 if (r)
5210 return r;
5211
5212 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5213 if (r) {
5214 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5215 return r;
5216 }
5217
5218 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5219 if (r) {
5220 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5221 return r;
5222 }
5223
5224 r = uvd_v2_2_resume(rdev);
5225 if (!r) {
5226 r = radeon_fence_driver_start_ring(rdev,
5227 R600_RING_TYPE_UVD_INDEX);
5228 if (r)
5229 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5230 }
5231
5232 if (r)
5233 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5234
5235 /* Enable IRQ */
5236 if (!rdev->irq.installed) {
5237 r = radeon_irq_kms_init(rdev);
5238 if (r)
5239 return r;
5240 }
5241
5242 r = r600_irq_init(rdev);
5243 if (r) {
5244 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5245 radeon_irq_kms_fini(rdev);
5246 return r;
5247 }
5248 evergreen_irq_set(rdev);
5249
5250 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5251 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5252 RADEON_CP_PACKET2);
5253 if (r)
5254 return r;
5255
5256 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5257 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5258 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5259 if (r)
5260 return r;
5261
5262 r = evergreen_cp_load_microcode(rdev);
5263 if (r)
5264 return r;
5265 r = evergreen_cp_resume(rdev);
5266 if (r)
5267 return r;
5268 r = r600_dma_resume(rdev);
5269 if (r)
5270 return r;
5271
5272 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5273 if (ring->ring_size) {
5274 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
5275 RADEON_CP_PACKET2);
5276 if (!r)
5277 r = uvd_v1_0_init(rdev);
5278
5279 if (r)
5280 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5281 }
5282
5283 r = radeon_ib_pool_init(rdev);
5284 if (r) {
5285 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5286 return r;
5287 }
5288
5289 r = r600_audio_init(rdev);
5290 if (r) {
5291 DRM_ERROR("radeon: audio init failed\n");
5292 return r;
5293 }
5294
5295 return 0;
5296 }
5297
5298 int evergreen_resume(struct radeon_device *rdev)
5299 {
5300 int r;
5301
5302 /* reset the asic, the gfx blocks are often in a bad state
5303 * after the driver is unloaded or after a resume
5304 */
5305 if (radeon_asic_reset(rdev))
5306 dev_warn(rdev->dev, "GPU reset failed !\n");
5307 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5308 * posting will perform necessary task to bring back GPU into good
5309 * shape.
5310 */
5311 /* post card */
5312 atom_asic_init(rdev->mode_info.atom_context);
5313
5314 /* init golden registers */
5315 evergreen_init_golden_registers(rdev);
5316
5317 if (rdev->pm.pm_method == PM_METHOD_DPM)
5318 radeon_pm_resume(rdev);
5319
5320 rdev->accel_working = true;
5321 r = evergreen_startup(rdev);
5322 if (r) {
5323 DRM_ERROR("evergreen startup failed on resume\n");
5324 rdev->accel_working = false;
5325 return r;
5326 }
5327
5328 return r;
5329
5330 }
5331
5332 int evergreen_suspend(struct radeon_device *rdev)
5333 {
5334 radeon_pm_suspend(rdev);
5335 r600_audio_fini(rdev);
5336 uvd_v1_0_fini(rdev);
5337 radeon_uvd_suspend(rdev);
5338 r700_cp_stop(rdev);
5339 r600_dma_stop(rdev);
5340 evergreen_irq_suspend(rdev);
5341 radeon_wb_disable(rdev);
5342 evergreen_pcie_gart_disable(rdev);
5343
5344 return 0;
5345 }
5346
5347 /* Plan is to move initialization in that function and use
5348 * helper function so that radeon_device_init pretty much
5349 * do nothing more than calling asic specific function. This
5350 * should also allow to remove a bunch of callback function
5351 * like vram_info.
5352 */
5353 int evergreen_init(struct radeon_device *rdev)
5354 {
5355 int r;
5356
5357 /* Read BIOS */
5358 if (!radeon_get_bios(rdev)) {
5359 if (ASIC_IS_AVIVO(rdev))
5360 return -EINVAL;
5361 }
5362 /* Must be an ATOMBIOS */
5363 if (!rdev->is_atom_bios) {
5364 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5365 return -EINVAL;
5366 }
5367 r = radeon_atombios_init(rdev);
5368 if (r)
5369 return r;
5370 /* reset the asic, the gfx blocks are often in a bad state
5371 * after the driver is unloaded or after a resume
5372 */
5373 if (radeon_asic_reset(rdev))
5374 dev_warn(rdev->dev, "GPU reset failed !\n");
5375 /* Post card if necessary */
5376 if (!radeon_card_posted(rdev)) {
5377 if (!rdev->bios) {
5378 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5379 return -EINVAL;
5380 }
5381 DRM_INFO("GPU not posted. posting now...\n");
5382 atom_asic_init(rdev->mode_info.atom_context);
5383 }
5384 /* init golden registers */
5385 evergreen_init_golden_registers(rdev);
5386 /* Initialize scratch registers */
5387 r600_scratch_init(rdev);
5388 /* Initialize surface registers */
5389 radeon_surface_init(rdev);
5390 /* Initialize clocks */
5391 radeon_get_clock_info(rdev->ddev);
5392 /* Fence driver */
5393 r = radeon_fence_driver_init(rdev);
5394 if (r)
5395 return r;
5396 /* initialize AGP */
5397 if (rdev->flags & RADEON_IS_AGP) {
5398 r = radeon_agp_init(rdev);
5399 if (r)
5400 radeon_agp_disable(rdev);
5401 }
5402 /* initialize memory controller */
5403 r = evergreen_mc_init(rdev);
5404 if (r)
5405 return r;
5406 /* Memory manager */
5407 r = radeon_bo_init(rdev);
5408 if (r)
5409 return r;
5410
5411 if (ASIC_IS_DCE5(rdev)) {
5412 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5413 r = ni_init_microcode(rdev);
5414 if (r) {
5415 DRM_ERROR("Failed to load firmware!\n");
5416 return r;
5417 }
5418 }
5419 } else {
5420 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5421 r = r600_init_microcode(rdev);
5422 if (r) {
5423 DRM_ERROR("Failed to load firmware!\n");
5424 return r;
5425 }
5426 }
5427 }
5428
5429 /* Initialize power management */
5430 radeon_pm_init(rdev);
5431
5432 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5433 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5434
5435 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5436 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5437
5438 r = radeon_uvd_init(rdev);
5439 if (!r) {
5440 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5441 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5442 4096);
5443 }
5444
5445 rdev->ih.ring_obj = NULL;
5446 r600_ih_ring_init(rdev, 64 * 1024);
5447
5448 r = r600_pcie_gart_init(rdev);
5449 if (r)
5450 return r;
5451
5452 rdev->accel_working = true;
5453 r = evergreen_startup(rdev);
5454 if (r) {
5455 dev_err(rdev->dev, "disabling GPU acceleration\n");
5456 r700_cp_fini(rdev);
5457 r600_dma_fini(rdev);
5458 r600_irq_fini(rdev);
5459 if (rdev->flags & RADEON_IS_IGP)
5460 sumo_rlc_fini(rdev);
5461 radeon_wb_fini(rdev);
5462 radeon_ib_pool_fini(rdev);
5463 radeon_irq_kms_fini(rdev);
5464 evergreen_pcie_gart_fini(rdev);
5465 rdev->accel_working = false;
5466 }
5467
5468 /* Don't start up if the MC ucode is missing on BTC parts.
5469 * The default clocks and voltages before the MC ucode
5470 * is loaded are not suffient for advanced operations.
5471 */
5472 if (ASIC_IS_DCE5(rdev)) {
5473 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5474 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5475 return -EINVAL;
5476 }
5477 }
5478
5479 return 0;
5480 }
5481
5482 void evergreen_fini(struct radeon_device *rdev)
5483 {
5484 radeon_pm_fini(rdev);
5485 r600_audio_fini(rdev);
5486 r700_cp_fini(rdev);
5487 r600_dma_fini(rdev);
5488 r600_irq_fini(rdev);
5489 if (rdev->flags & RADEON_IS_IGP)
5490 sumo_rlc_fini(rdev);
5491 radeon_wb_fini(rdev);
5492 radeon_ib_pool_fini(rdev);
5493 radeon_irq_kms_fini(rdev);
5494 uvd_v1_0_fini(rdev);
5495 radeon_uvd_fini(rdev);
5496 evergreen_pcie_gart_fini(rdev);
5497 r600_vram_scratch_fini(rdev);
5498 radeon_gem_fini(rdev);
5499 radeon_fence_driver_fini(rdev);
5500 radeon_agp_fini(rdev);
5501 radeon_bo_fini(rdev);
5502 radeon_atombios_fini(rdev);
5503 kfree(rdev->bios);
5504 rdev->bios = NULL;
5505 }
5506
5507 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5508 {
5509 u32 link_width_cntl, speed_cntl;
5510
5511 if (radeon_pcie_gen2 == 0)
5512 return;
5513
5514 if (rdev->flags & RADEON_IS_IGP)
5515 return;
5516
5517 if (!(rdev->flags & RADEON_IS_PCIE))
5518 return;
5519
5520 /* x2 cards have a special sequence */
5521 if (ASIC_IS_X2(rdev))
5522 return;
5523
5524 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5525 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5526 return;
5527
5528 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5529 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5530 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5531 return;
5532 }
5533
5534 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5535
5536 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5537 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5538
5539 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5540 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5541 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5542
5543 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5544 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5545 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5546
5547 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5548 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5549 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5550
5551 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5552 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5553 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5554
5555 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5556 speed_cntl |= LC_GEN2_EN_STRAP;
5557 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5558
5559 } else {
5560 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5561 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5562 if (1)
5563 link_width_cntl |= LC_UPCONFIGURE_DIS;
5564 else
5565 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5566 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5567 }
5568 }
5569
5570 void evergreen_program_aspm(struct radeon_device *rdev)
5571 {
5572 u32 data, orig;
5573 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5574 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5575 /* fusion_platform = true
5576 * if the system is a fusion system
5577 * (APU or DGPU in a fusion system).
5578 * todo: check if the system is a fusion platform.
5579 */
5580 bool fusion_platform = false;
5581
5582 if (radeon_aspm == 0)
5583 return;
5584
5585 if (!(rdev->flags & RADEON_IS_PCIE))
5586 return;
5587
5588 switch (rdev->family) {
5589 case CHIP_CYPRESS:
5590 case CHIP_HEMLOCK:
5591 case CHIP_JUNIPER:
5592 case CHIP_REDWOOD:
5593 case CHIP_CEDAR:
5594 case CHIP_SUMO:
5595 case CHIP_SUMO2:
5596 case CHIP_PALM:
5597 case CHIP_ARUBA:
5598 disable_l0s = true;
5599 break;
5600 default:
5601 disable_l0s = false;
5602 break;
5603 }
5604
5605 if (rdev->flags & RADEON_IS_IGP)
5606 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5607
5608 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5609 if (fusion_platform)
5610 data &= ~MULTI_PIF;
5611 else
5612 data |= MULTI_PIF;
5613 if (data != orig)
5614 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5615
5616 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5617 if (fusion_platform)
5618 data &= ~MULTI_PIF;
5619 else
5620 data |= MULTI_PIF;
5621 if (data != orig)
5622 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5623
5624 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5625 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5626 if (!disable_l0s) {
5627 if (rdev->family >= CHIP_BARTS)
5628 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5629 else
5630 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5631 }
5632
5633 if (!disable_l1) {
5634 if (rdev->family >= CHIP_BARTS)
5635 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5636 else
5637 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5638
5639 if (!disable_plloff_in_l1) {
5640 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5641 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5642 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5643 if (data != orig)
5644 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5645
5646 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5647 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5648 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5649 if (data != orig)
5650 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5651
5652 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5653 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5654 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5655 if (data != orig)
5656 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5657
5658 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5659 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5660 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5661 if (data != orig)
5662 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5663
5664 if (rdev->family >= CHIP_BARTS) {
5665 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5666 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5667 data |= PLL_RAMP_UP_TIME_0(4);
5668 if (data != orig)
5669 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5670
5671 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5672 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5673 data |= PLL_RAMP_UP_TIME_1(4);
5674 if (data != orig)
5675 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5676
5677 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5678 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5679 data |= PLL_RAMP_UP_TIME_0(4);
5680 if (data != orig)
5681 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5682
5683 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5684 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5685 data |= PLL_RAMP_UP_TIME_1(4);
5686 if (data != orig)
5687 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5688 }
5689
5690 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5691 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5692 data |= LC_DYN_LANES_PWR_STATE(3);
5693 if (data != orig)
5694 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5695
5696 if (rdev->family >= CHIP_BARTS) {
5697 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5698 data &= ~LS2_EXIT_TIME_MASK;
5699 data |= LS2_EXIT_TIME(1);
5700 if (data != orig)
5701 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5702
5703 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5704 data &= ~LS2_EXIT_TIME_MASK;
5705 data |= LS2_EXIT_TIME(1);
5706 if (data != orig)
5707 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5708 }
5709 }
5710 }
5711
5712 /* evergreen parts only */
5713 if (rdev->family < CHIP_BARTS)
5714 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5715
5716 if (pcie_lc_cntl != pcie_lc_cntl_old)
5717 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5718 }
ubuntu-zesty-kernel mirror